download on demand (#2631)

* draft a download-api

* basic implementation

* allow partial downloads for protected chats

* use a separate column for download_state

* force a minimal timeout for delete_server_after in combination with partial messages

* add a warning if a possible download may expire by delete_server_after

* test load_imap_deletion_msgid()

* add a test for a partial download

* improve documentation and visibility

* let get_download_limit() return Result<Option>

* rusty getters

* apply MIN_DELETE_SERVER_AFTER to shown availability time

* move stub-creation to download.rs, use stock-strings, nicer logging

* make clippy happy (cargo clippy --tests)

* refine tests and comments

* fix typo

* remove superfluous closure in ffi

* respect partial_download for immediately scheduled DeleteMsgOnImap jobs

src/ephemeral.rs

@@ -71,11 +71,13 @@ use crate::constants::{
 };
 use crate::context::Context;
 use crate::dc_tools::time;
+use crate::download::MIN_DELETE_SERVER_AFTER;
 use crate::events::EventType;
 use crate::job;
 use crate::message::{Message, MessageState, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::stock_str;
+use std::cmp::max;
 
 #[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize)]
 pub enum Timer {
@@ -439,23 +441,32 @@ pub async fn schedule_ephemeral_task(context: &Context) {
 pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> anyhow::Result<Option<MsgId>> {
     let now = time();
 
-    let threshold_timestamp = match context.get_config_delete_server_after().await? {
-        None => 0,
-        Some(delete_server_after) => now - delete_server_after,
-    };
+    let (threshold_timestamp, threshold_timestamp_extended) =
+        match context.get_config_delete_server_after().await? {
+            None => (0, 0),
+            Some(delete_server_after) => (
+                now - delete_server_after,
+                now - max(delete_server_after, MIN_DELETE_SERVER_AFTER),
+            ),
+        };
 
     context
         .sql
         .query_row_optional(
             "SELECT id FROM msgs \
          WHERE ( \
-         timestamp < ? \
+         ((download_state = 0 AND timestamp < ?) OR (download_state != 0 AND timestamp < ?)) \
          OR (ephemeral_timestamp != 0 AND ephemeral_timestamp <= ?) \
          ) \
          AND server_uid != 0 \
          AND NOT id IN (SELECT foreign_id FROM jobs WHERE action = ?)
          LIMIT 1",
-            paramsv![threshold_timestamp, now, job::Action::DeleteMsgOnImap],
+            paramsv![
+                threshold_timestamp,
+                threshold_timestamp_extended,
+                now,
+                job::Action::DeleteMsgOnImap
+            ],
             |row| {
                 let msg_id: MsgId = row.get(0)?;
                 Ok(msg_id)
@@ -500,6 +511,8 @@ mod tests {
     use async_std::task::sleep;
 
     use super::*;
+    use crate::config::Config;
+    use crate::download::DownloadState;
     use crate::test_utils::TestContext;
     use crate::{
         chat::{self, Chat, ChatItem},
@@ -771,4 +784,58 @@ mod tests {
             assert!(rawtxt.is_none_or_empty(), "{:?}", rawtxt);
         }
     }
+
+    #[async_std::test]
+    async fn test_load_imap_deletion_msgid() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        const HOUR: i64 = 60 * 60;
+        let now = time();
+        for (id, timestamp, ephemeral_timestamp) in &[
+            (900, now - 2 * HOUR, 0),
+            (1000, now - 23 * HOUR - MIN_DELETE_SERVER_AFTER, 0),
+            (1010, now - 23 * HOUR, 0),
+            (1020, now - 21 * HOUR, 0),
+            (1030, now - 19 * HOUR, 0),
+            (2000, now - 18 * HOUR, now - HOUR),
+            (2020, now - 17 * HOUR, now + HOUR),
+        ] {
+            t.sql
+                .execute(
+                    "INSERT INTO msgs (id, server_uid, timestamp, ephemeral_timestamp) VALUES (?,?,?,?);",
+                    paramsv![id, id, timestamp, ephemeral_timestamp],
+                )
+                .await?;
+        }
+
+        assert_eq!(load_imap_deletion_msgid(&t).await?, Some(MsgId::new(2000)));
+
+        MsgId::new(2000).delete_from_db(&t).await?;
+        assert_eq!(load_imap_deletion_msgid(&t).await?, None);
+
+        t.set_config(Config::DeleteServerAfter, Some(&*(25 * HOUR).to_string()))
+            .await?;
+        assert_eq!(load_imap_deletion_msgid(&t).await?, Some(MsgId::new(1000)));
+
+        MsgId::new(1000)
+            .update_download_state(&t, DownloadState::Available)
+            .await?;
+        assert_eq!(load_imap_deletion_msgid(&t).await?, Some(MsgId::new(1000))); // delete downloadable anyway
+
+        MsgId::new(1000).delete_from_db(&t).await?;
+        assert_eq!(load_imap_deletion_msgid(&t).await?, None);
+
+        t.set_config(Config::DeleteServerAfter, Some(&*(22 * HOUR).to_string()))
+            .await?;
+        assert_eq!(load_imap_deletion_msgid(&t).await?, Some(MsgId::new(1010)));
+
+        MsgId::new(1010)
+            .update_download_state(&t, DownloadState::Available)
+            .await?;
+        assert_eq!(load_imap_deletion_msgid(&t).await?, None); // keep downloadable for now
+
+        MsgId::new(1010).delete_from_db(&t).await?;
+        assert_eq!(load_imap_deletion_msgid(&t).await?, None);
+
+        Ok(())
+    }
 }