Fix imex race condition, (#2255)

fix #2254: if the DB was closed without calling stop_io() and then an interrupt arrives (e.g. incoming message), the db was corrupted.

* Add result.log() for logging with less boilerplate code

* Bugfix: Resultify housekeeping() to make it abort if the db is closed instead of just deleting everything

* Require the UI to call dc_stop_io() before backup export

* Prepare a bit better for closed-db: Resultify get_uidvalidity and get_uid_next and let job::load_next() wait until the db is open

About the bug (before this PR):
if the DB was closed without calling stop_io() and then an interrupt arrives (e.g. incoming message):
- I don't know if it downloads the message, but of course at some point the process of receiving the message will fail
- In my test, DC is just in the process of moving a message when the imex starts, but then can't delete the job or update the msg server_uid
- Then, when job::load_next() is called, no job can be loaded. That's why it calls `load_housekeeping_job()`. As `load_housekeeping_job()` can't load the time of the last housekeeping, it assumes we never ran housekeeping and returns a new Housekeeping job, which is immediately executed.
- housekeeping can't find any blobs referenced in the db and therefore deletes almost all blobs.

src/sql.rs

@@ -8,6 +8,7 @@ use std::path::Path;
 use std::time::Duration;
 
 use anyhow::format_err;
+use anyhow::Context as _;
 use rusqlite::{Connection, Error as SqlError, OpenFlags};
 
 use crate::chat::{add_device_msg, update_device_icon, update_saved_messages_icon};
@@ -495,7 +496,7 @@ pub fn get_rowid2(
     )
 }
 
-pub async fn housekeeping(context: &Context) {
+pub async fn housekeeping(context: &Context) -> anyhow::Result<()> {
     if let Err(err) = crate::ephemeral::delete_expired_messages(context).await {
         warn!(context, "Failed to delete expired messages: {}", err);
     }
@@ -510,28 +511,28 @@ pub async fn housekeeping(context: &Context) {
         "SELECT param FROM msgs  WHERE chat_id!=3   AND type!=10;",
         Param::File,
     )
-    .await;
+    .await?;
     maybe_add_from_param(
         context,
         &mut files_in_use,
         "SELECT param FROM jobs;",
         Param::File,
     )
-    .await;
+    .await?;
     maybe_add_from_param(
         context,
         &mut files_in_use,
         "SELECT param FROM chats;",
         Param::ProfileImage,
     )
-    .await;
+    .await?;
     maybe_add_from_param(
         context,
         &mut files_in_use,
         "SELECT param FROM contacts;",
         Param::ProfileImage,
     )
-    .await;
+    .await?;
 
     context
         .sql
@@ -547,9 +548,7 @@ pub async fn housekeeping(context: &Context) {
             },
         )
         .await
-        .unwrap_or_else(|err| {
-            warn!(context, "sql: failed query: {}", err);
-        });
+        .context("housekeeping: failed to SELECT value FROM config")?;
 
     info!(context, "{} files in use.", files_in_use.len(),);
     /* go through directory and delete unused files */
@@ -637,6 +636,7 @@ pub async fn housekeeping(context: &Context) {
         warn!(context, "Can't set config: {}", e);
     }
     info!(context, "Housekeeping done.");
+    Ok(())
 }
 
 #[allow(clippy::indexing_slicing)]
@@ -665,7 +665,7 @@ async fn maybe_add_from_param(
     files_in_use: &mut HashSet<String>,
     query: &str,
     param_id: Param,
-) {
+) -> anyhow::Result<()> {
     context
         .sql
         .query_map(
@@ -683,9 +683,7 @@ async fn maybe_add_from_param(
             },
         )
         .await
-        .unwrap_or_else(|err| {
-            warn!(context, "sql: failed to add_from_param: {}", err);
-        });
+        .context(format!("housekeeping: failed to add_from_param {}", query))
 }
 
 #[allow(clippy::cognitive_complexity)]
@@ -1570,8 +1568,10 @@ async fn prune_tombstones(context: &Context) -> Result<()> {
 
 #[cfg(test)]
 mod test {
+    use async_std::fs::File;
+
     use super::*;
-    use crate::test_utils::TestContext;
+    use crate::{test_utils::TestContext, Event, EventType};
 
     #[test]
     fn test_maybe_add_file() {
@@ -1613,4 +1613,44 @@ mod test {
         assert!(!t.ctx.sql.col_exists("msgs", "foobar").await.unwrap());
         assert!(!t.ctx.sql.col_exists("foobar", "foobar").await.unwrap());
     }
+
+    #[async_std::test]
+    async fn test_housekeeping_db_closed() {
+        let t = TestContext::new().await;
+
+        let avatar_src = t.dir.path().join("avatar.png");
+        let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
+        File::create(&avatar_src)
+            .await
+            .unwrap()
+            .write_all(avatar_bytes)
+            .await
+            .unwrap();
+        t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
+            .await
+            .unwrap();
+
+        t.add_event_sink(move |event: Event| async move {
+            match event.typ {
+                EventType::Info(s) => assert!(
+                    !s.contains("Keeping new unreferenced file"),
+                    "File {} was almost deleted, only reason it was kept is that it was created recently (as the tests don't run for a long time)",
+                    s
+                ),
+                EventType::Error(s) => panic!(s),
+                _ => {}
+            }
+        })
+        .await;
+
+        let a = t.get_config(Config::Selfavatar).await.unwrap();
+        assert_eq!(avatar_bytes, &async_std::fs::read(&a).await.unwrap()[..]);
+
+        t.sql.close().await;
+        housekeeping(&t).await.unwrap_err(); // housekeeping should fail as the db is closed
+        t.sql.open(&t, &t.get_dbfile(), false).await.unwrap();
+
+        let a = t.get_config(Config::Selfavatar).await.unwrap();
+        assert_eq!(avatar_bytes, &async_std::fs::read(&a).await.unwrap()[..]);
+    }
 }