Tar backup (#1749)

Fix #1729
Co-authored-by: holger krekel  <holger@merlinux.eu>
Co-authored-by: Alexander Krotov <ilabdsf@gmail.com>

Cargo.lock

@@ -257,6 +257,20 @@ dependencies = [
  "trust-dns-resolver",
 ]
 
+[[package]]
+name = "async-tar"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb619eae01ab289095debb1ff7c02710d5124c20edde1b2eca926572a34c3998"
+dependencies = [
+ "async-std",
+ "filetime",
+ "libc",
+ "pin-project",
+ "redox_syscall",
+ "xattr",
+]
+
 [[package]]
 name = "async-task"
 version = "3.0.0"
@@ -799,6 +813,7 @@ dependencies = [
  "async-smtp",
  "async-std",
  "async-std-resolver",
+ "async-tar",
  "async-trait",
  "backtrace",
  "base64 0.12.3",
@@ -807,6 +822,7 @@ dependencies = [
  "charset",
  "chrono",
  "deltachat_derive",
+ "dirs 3.0.1",
  "email",
  "encoded-words",
  "escaper",
@@ -947,6 +963,26 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "dirs"
+version = "3.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "142995ed02755914747cc6ca76fc7e4583cd18578746716d0508ea6ed558b9ff"
+dependencies = [
+ "dirs-sys",
+]
+
+[[package]]
+name = "dirs-sys"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e93d7f5705de3e49895a2b5e0b8855a1c27f080192ae9c32a6432d50741a57a"
+dependencies = [
+ "libc",
+ "redox_users",
+ "winapi",
+]
+
 [[package]]
 name = "discard"
 version = "1.0.4"
@@ -1179,6 +1215,18 @@ version = "1.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4bd3bdaaf0a72155260a1c098989b60db1cbb22d6a628e64f16237aa4da93cc7"
 
+[[package]]
+name = "filetime"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ed85775dcc68644b5c950ac06a2b23768d3bc9390464151aaf27136998dcf9e"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "redox_syscall",
+ "winapi",
+]
+
 [[package]]
 name = "flate2"
 version = "1.0.16"
@@ -2621,7 +2669,7 @@ version = "4.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0f47ea1ceb347d2deae482d655dc8eef4bd82363d3329baffa3818bd76fea48b"
 dependencies = [
- "dirs",
+ "dirs 1.0.5",
  "libc",
  "log",
  "memchr",
@@ -3539,6 +3587,15 @@ dependencies = [
  "zeroize",
 ]
 
+[[package]]
+name = "xattr"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "zeroize"
 version = "1.1.0"

Cargo.toml

@@ -59,12 +59,14 @@ anyhow = "1.0.28"
 async-trait = "0.1.31"
 url = "2.1.1"
 async-std-resolver = "0.19.5"
+async-tar = "0.3.0"
 uuid = { version = "0.8", features = ["serde", "v4"] }
 
 pretty_env_logger = { version = "0.4.0", optional = true }
 log = {version = "0.4.8", optional = true }
 rustyline = { version = "4.1.0", optional = true }
 ansi_term = { version = "0.12.1", optional = true }
+dirs = { version = "3.0.1", optional=true }
 toml = "0.5.6"
 
 
@@ -96,7 +98,7 @@ required-features = ["repl"]
 [features]
 default = []
 internals = []
-repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term"]
+repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs"]
 vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored"]
 nightly = ["pgp/nightly"]
 

deltachat-ffi/deltachat.h

@@ -1773,8 +1773,8 @@ dc_contact_t*   dc_get_contact               (dc_context_t* context, uint32_t co
  * - **DC_IMEX_EXPORT_BACKUP** (11) - Export a backup to the directory given as `param1`.
  *   The backup contains all contacts, chats, images and other data and device independent settings.
  *   The backup does not contain device dependent settings as ringtones or LED notification settings.
- *   The name of the backup is typically `delta-chat.<day>.bak`, if more than one backup is create on a day,
+ *   The name of the backup is typically `delta-chat-<day>.tar`, if more than one backup is create on a day,
- *   the format is `delta-chat.<day>-<number>.bak`
+ *   the format is `delta-chat-<day>-<number>.tar`
  *
  * - **DC_IMEX_IMPORT_BACKUP** (12) - `param1` is the file (not: directory) to import. The file is normally
  *   created by DC_IMEX_EXPORT_BACKUP and detected by dc_imex_has_backup(). Importing a backup

examples/repl/cmdline.rs

@@ -1,3 +1,5 @@
+extern crate dirs;
+
 use std::str::FromStr;
 
 use anyhow::{bail, ensure};
@@ -442,17 +444,21 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             has_backup(&context, blobdir).await?;
         }
         "export-backup" => {
-            imex(&context, ImexMode::ExportBackup, Some(blobdir)).await?;
+            let dir = dirs::home_dir().unwrap_or_default();
+            imex(&context, ImexMode::ExportBackup, Some(&dir)).await?;
+            println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-backup" => {
             ensure!(!arg1.is_empty(), "Argument <backup-file> missing.");
             imex(&context, ImexMode::ImportBackup, Some(arg1)).await?;
         }
         "export-keys" => {
-            imex(&context, ImexMode::ExportSelfKeys, Some(blobdir)).await?;
+            let dir = dirs::home_dir().unwrap_or_default();
+            imex(&context, ImexMode::ExportSelfKeys, Some(&dir)).await?;
+            println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-keys" => {
-            imex(&context, ImexMode::ImportSelfKeys, Some(blobdir)).await?;
+            imex(&context, ImexMode::ImportSelfKeys, Some(arg1)).await?;
         }
         "export-setup" => {
             let setup_code = create_setup_code(&context);

src/dc_tools.rs

@@ -450,7 +450,7 @@ pub fn dc_open_file_std<P: AsRef<std::path::Path>>(
     }
 }
 
-pub(crate) async fn dc_get_next_backup_path(
+pub(crate) async fn get_next_backup_path_old(
     folder: impl AsRef<Path>,
     backup_time: i64,
 ) -> Result<PathBuf, Error> {
@@ -470,6 +470,32 @@ pub(crate) async fn dc_get_next_backup_path(
     bail!("could not create backup file, disk full?");
 }
 
+/// Returns Ok((temp_path, dest_path)) on success. The backup can then be written to temp_path. If the backup succeeded,
+/// it can be renamed to dest_path. This guarantees that the backup is complete.
+pub(crate) async fn get_next_backup_path_new(
+    folder: impl AsRef<Path>,
+    backup_time: i64,
+) -> Result<(PathBuf, PathBuf), Error> {
+    let folder = PathBuf::from(folder.as_ref());
+    let stem = chrono::NaiveDateTime::from_timestamp(backup_time, 0)
+        .format("delta-chat-backup-%Y-%m-%d")
+        .to_string();
+
+    // 64 backup files per day should be enough for everyone
+    for i in 0..64 {
+        let mut tempfile = folder.clone();
+        tempfile.push(format!("{}-{:02}.tar.part", stem, i));
+
+        let mut destfile = folder.clone();
+        destfile.push(format!("{}-{:02}.tar", stem, i));
+
+        if !tempfile.exists().await && !destfile.exists().await {
+            return Ok((tempfile, destfile));
+        }
+    }
+    bail!("could not create backup file, disk full?");
+}
+
 pub(crate) fn time() -> i64 {
     SystemTime::now()
         .duration_since(SystemTime::UNIX_EPOCH)

src/events.rs

@@ -127,7 +127,7 @@ pub enum EventType {
     #[strum(props(id = "150"))]
     NewBlobFile(String),
 
-    /// Emitted when an new file in the $BLOBDIR was created
+    /// Emitted when an file in the $BLOBDIR was deleted
     #[strum(props(id = "151"))]
     DeletedBlobFile(String),
 

src/imex.rs

@@ -1,10 +1,16 @@
 //! # Import/export module
 
 use std::any::Any;
-use std::cmp::{max, min};
+use std::{
+    cmp::{max, min},
+    ffi::OsStr,
+};
 
 use async_std::path::{Path, PathBuf};
-use async_std::prelude::*;
+use async_std::{
+    fs::{self, File},
+    prelude::*,
+};
 use rand::{thread_rng, Rng};
 
 use crate::blob::BlobObject;
@@ -24,6 +30,11 @@ use crate::param::*;
 use crate::pgp;
 use crate::sql::{self, Sql};
 use crate::stock::StockMessage;
+use async_tar::Archive;
+
+// Name of the database file in the backup.
+const DBFILE_BACKUP_NAME: &str = "dc_database_backup.sqlite";
+const BLOBS_BACKUP_NAME: &str = "blobs_backup";
 
 #[derive(Debug, Display, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive)]
 #[repr(i32)]
@@ -42,8 +53,8 @@ pub enum ImexMode {
     /// Export a backup to the directory given as `param1`.
     /// The backup contains all contacts, chats, images and other data and device independent settings.
     /// The backup does not contain device dependent settings as ringtones or LED notification settings.
-    /// The name of the backup is typically `delta-chat.<day>.bak`, if more than one backup is create on a day,
+    /// The name of the backup is typically `delta-chat-<day>.tar`, if more than one backup is create on a day,
-    /// the format is `delta-chat.<day>-<number>.bak`
+    /// the format is `delta-chat-<day>-<number>.tar`
     ExportBackup = 11,
 
     /// `param1` is the file (not: directory) to import. The file is normally
@@ -85,6 +96,37 @@ pub async fn imex(
 
 /// Returns the filename of the backup found (otherwise an error)
 pub async fn has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Result<String> {
+    let dir_name = dir_name.as_ref();
+    let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
+    let mut newest_backup_name = "".to_string();
+    let mut newest_backup_path: Option<PathBuf> = None;
+
+    while let Some(dirent) = dir_iter.next().await {
+        if let Ok(dirent) = dirent {
+            let path = dirent.path();
+            let name = dirent.file_name();
+            let name: String = name.to_string_lossy().into();
+            if name.starts_with("delta-chat")
+                && name.ends_with(".tar")
+                && (newest_backup_name.is_empty() || name > newest_backup_name)
+            {
+                // We just use string comparison to determine which backup is newer.
+                // This works fine because the filenames have the form ...delta-chat-backup-2020-07-24-00.tar
+                newest_backup_path = Some(path);
+                newest_backup_name = name;
+            }
+        }
+    }
+
+    match newest_backup_path {
+        Some(path) => Ok(path.to_string_lossy().into_owned()),
+        None => has_backup_old(context, dir_name).await,
+        // When we decide to remove support for .bak backups, we can replace this with `None => bail!("no backup found in {}", dir_name.display()),`.
+    }
+}
+
+/// Returns the filename of the backup found (otherwise an error)
+pub async fn has_backup_old(context: &Context, dir_name: impl AsRef<Path>) -> Result<String> {
     let dir_name = dir_name.as_ref();
     let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
     let mut newest_backup_time = 0;
@@ -373,7 +415,7 @@ async fn imex_inner(
 
     ensure!(context.sql.is_open().await, "Database not opened.");
 
-    let path = param.unwrap();
+    let path = param.ok_or_else(|| format_err!("Imex: Param was None"))?;
     if what == ImexMode::ExportBackup || what == ImexMode::ExportSelfKeys {
         // before we export anything, make sure the private key exists
         if e2ee::ensure_secret_key_exists(context).await.is_err() {
@@ -386,7 +428,11 @@ async fn imex_inner(
     let success = match what {
         ImexMode::ExportSelfKeys => export_self_keys(context, path).await,
         ImexMode::ImportSelfKeys => import_self_keys(context, path).await,
-        ImexMode::ExportBackup => export_backup(context, path).await,
+
+        // TODO In some months we can change the export_backup_old() call to export_backup() and delete export_backup_old().
+        // (now is 07/2020)
+        ImexMode::ExportBackup => export_backup_old(context, path).await,
+        // import_backup() will call import_backup_old() if this is an old backup.
         ImexMode::ImportBackup => import_backup(context, path).await,
     };
 
@@ -405,6 +451,75 @@ async fn imex_inner(
 
 /// Import Backup
 async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Result<()> {
+    if backup_to_import
+        .as_ref()
+        .to_string_lossy()
+        .ends_with(".bak")
+    {
+        // Backwards compability
+        return import_backup_old(context, backup_to_import).await;
+    }
+
+    info!(
+        context,
+        "Import \"{}\" to \"{}\".",
+        backup_to_import.as_ref().display(),
+        context.get_dbfile().display()
+    );
+
+    ensure!(
+        !context.is_configured().await,
+        "Cannot import backups to accounts in use."
+    );
+    context.sql.close().await;
+    dc_delete_file(context, context.get_dbfile()).await;
+    ensure!(
+        !context.get_dbfile().exists().await,
+        "Cannot delete old database."
+    );
+
+    let backup_file = File::open(backup_to_import).await?;
+    let archive = Archive::new(backup_file);
+    let mut entries = archive.entries()?;
+    while let Some(file) = entries.next().await {
+        let f = &mut file?;
+        if f.path()?.file_name() == Some(OsStr::new(DBFILE_BACKUP_NAME)) {
+            // async_tar can't unpack to a specified file name, so we just unpack to the blobdir and then move the unpacked file.
+            f.unpack_in(context.get_blobdir()).await?;
+            fs::rename(
+                context.get_blobdir().join(DBFILE_BACKUP_NAME),
+                context.get_dbfile(),
+            )
+            .await?;
+            context.emit_event(EventType::ImexProgress(400)); // Just guess the progress, we at least have the dbfile by now
+        } else {
+            // async_tar will unpack to blobdir/BLOBS_BACKUP_NAME, so we move the file afterwards.
+            f.unpack_in(context.get_blobdir()).await?;
+            let from_path = context.get_blobdir().join(f.path()?);
+            if from_path.is_file().await {
+                if let Some(name) = from_path.file_name() {
+                    fs::rename(&from_path, context.get_blobdir().join(name)).await?;
+                } else {
+                    warn!(context, "No file name");
+                }
+            }
+        }
+    }
+
+    ensure!(
+        context
+            .sql
+            .open(&context, &context.get_dbfile(), false)
+            .await,
+        "could not re-open db"
+    );
+
+    delete_and_reset_all_device_msgs(&context).await?;
+
+    Ok(())
+}
+
+async fn import_backup_old(context: &Context, backup_to_import: impl AsRef<Path>) -> Result<()> {
     info!(
         context,
         "Import \"{}\" to \"{}\".",
@@ -512,14 +627,83 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
 /*******************************************************************************
  * Export backup
  ******************************************************************************/
-/* the FILE_PROGRESS macro calls the callback with the permille of files processed.
+#[allow(unused)]
-The macro avoids weird values of 0% or 100% while still working. */
 async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
+    // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
+    let now = time();
+    let (temp_path, dest_path) = get_next_backup_path_new(dir, now).await?;
+
+    context
+        .sql
+        .set_raw_config_int(context, "backup_time", now as i32)
+        .await?;
+    sql::housekeeping(context).await;
+
+    context
+        .sql
+        .execute("VACUUM;", paramsv![])
+        .await
+        .map_err(|e| warn!(context, "Vacuum failed, exporting anyway {}", e));
+
+    // we close the database during the export
+    context.sql.close().await;
+
+    info!(
+        context,
+        "Backup '{}' to '{}'.",
+        context.get_dbfile().display(),
+        dest_path.display(),
+    );
+
+    let res = export_backup_inner(context, &temp_path).await;
+
+    // we re-open the database after export is finished
+    context
+        .sql
+        .open(&context, &context.get_dbfile(), false)
+        .await;
+
+    match &res {
+        Ok(_) => {
+            fs::rename(temp_path, &dest_path).await?;
+            context.emit_event(EventType::ImexFileWritten(dest_path));
+        }
+        Err(e) => {
+            error!(context, "backup failed: {}", e);
+            // Not using dc_delete_file() here because it would send a DeletedBlobFile event
+            fs::remove_file(temp_path).await?;
+        }
+    }
+
+    res
+}
+
+async fn export_backup_inner(context: &Context, temp_path: &PathBuf) -> Result<()> {
+    let file = File::create(temp_path).await?;
+
+    let mut builder = async_tar::Builder::new(file);
+
+    // append_path_with_name() wants the source path as the first argument, append_dir_all() wants it as the second argument.
+    builder
+        .append_path_with_name(context.get_dbfile(), DBFILE_BACKUP_NAME)
+        .await?;
+
+    context.emit_event(EventType::ImexProgress(500));
+
+    builder
+        .append_dir_all(BLOBS_BACKUP_NAME, context.get_blobdir())
+        .await?;
+
+    builder.finish().await?;
+    Ok(())
+}
+
+async fn export_backup_old(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
     // FIXME: we should write to a temporary file first and rename it on success. this would guarantee the backup is complete.
     // let dest_path_filename = dc_get_next_backup_file(context, dir, res);
     let now = time();
-    let dest_path_filename = dc_get_next_backup_path(dir, now).await?;
+    let dest_path_filename = get_next_backup_path_old(dir, now).await?;
     let dest_path_string = dest_path_filename.to_string_lossy().to_string();
 
     sql::housekeeping(context).await;