diff --git a/Cargo.lock b/Cargo.lock index a82eb8ca5..59c9036d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -257,6 +257,20 @@ dependencies = [ "trust-dns-resolver", ] +[[package]] +name = "async-tar" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb619eae01ab289095debb1ff7c02710d5124c20edde1b2eca926572a34c3998" +dependencies = [ + "async-std", + "filetime", + "libc", + "pin-project", + "redox_syscall", + "xattr", +] + [[package]] name = "async-task" version = "3.0.0" @@ -799,6 +813,7 @@ dependencies = [ "async-smtp", "async-std", "async-std-resolver", + "async-tar", "async-trait", "backtrace", "base64 0.12.3", @@ -807,6 +822,7 @@ dependencies = [ "charset", "chrono", "deltachat_derive", + "dirs 3.0.1", "email", "encoded-words", "escaper", @@ -947,6 +963,26 @@ dependencies = [ "winapi", ] +[[package]] +name = "dirs" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142995ed02755914747cc6ca76fc7e4583cd18578746716d0508ea6ed558b9ff" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e93d7f5705de3e49895a2b5e0b8855a1c27f080192ae9c32a6432d50741a57a" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "discard" version = "1.0.4" @@ -1179,6 +1215,18 @@ version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4bd3bdaaf0a72155260a1c098989b60db1cbb22d6a628e64f16237aa4da93cc7" +[[package]] +name = "filetime" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed85775dcc68644b5c950ac06a2b23768d3bc9390464151aaf27136998dcf9e" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "winapi", +] + [[package]] name = "flate2" version = "1.0.16" @@ -2621,7 +2669,7 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f47ea1ceb347d2deae482d655dc8eef4bd82363d3329baffa3818bd76fea48b" dependencies = [ - "dirs", + "dirs 1.0.5", "libc", "log", "memchr", @@ -3539,6 +3587,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "xattr" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" +dependencies = [ + "libc", +] + [[package]] name = "zeroize" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index fdce41407..50617bf30 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,12 +59,14 @@ anyhow = "1.0.28" async-trait = "0.1.31" url = "2.1.1" async-std-resolver = "0.19.5" +async-tar = "0.3.0" uuid = { version = "0.8", features = ["serde", "v4"] } pretty_env_logger = { version = "0.4.0", optional = true } log = {version = "0.4.8", optional = true } rustyline = { version = "4.1.0", optional = true } ansi_term = { version = "0.12.1", optional = true } +dirs = { version = "3.0.1", optional=true } toml = "0.5.6" @@ -96,7 +98,7 @@ required-features = ["repl"] [features] default = [] internals = [] -repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term"] +repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs"] vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored"] nightly = ["pgp/nightly"] diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 9db239080..86e8ea4b4 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -1773,8 +1773,8 @@ dc_contact_t* dc_get_contact (dc_context_t* context, uint32_t co * - **DC_IMEX_EXPORT_BACKUP** (11) - Export a backup to the directory given as `param1`. * The backup contains all contacts, chats, images and other data and device independent settings. * The backup does not contain device dependent settings as ringtones or LED notification settings. - * The name of the backup is typically `delta-chat..bak`, if more than one backup is create on a day, - * the format is `delta-chat.-.bak` + * The name of the backup is typically `delta-chat-.tar`, if more than one backup is create on a day, + * the format is `delta-chat--.tar` * * - **DC_IMEX_IMPORT_BACKUP** (12) - `param1` is the file (not: directory) to import. The file is normally * created by DC_IMEX_EXPORT_BACKUP and detected by dc_imex_has_backup(). Importing a backup diff --git a/examples/repl/cmdline.rs b/examples/repl/cmdline.rs index 58cf4582f..90cf2bf79 100644 --- a/examples/repl/cmdline.rs +++ b/examples/repl/cmdline.rs @@ -1,3 +1,5 @@ +extern crate dirs; + use std::str::FromStr; use anyhow::{bail, ensure}; @@ -442,17 +444,21 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu has_backup(&context, blobdir).await?; } "export-backup" => { - imex(&context, ImexMode::ExportBackup, Some(blobdir)).await?; + let dir = dirs::home_dir().unwrap_or_default(); + imex(&context, ImexMode::ExportBackup, Some(&dir)).await?; + println!("Exported to {}.", dir.to_string_lossy()); } "import-backup" => { ensure!(!arg1.is_empty(), "Argument missing."); imex(&context, ImexMode::ImportBackup, Some(arg1)).await?; } "export-keys" => { - imex(&context, ImexMode::ExportSelfKeys, Some(blobdir)).await?; + let dir = dirs::home_dir().unwrap_or_default(); + imex(&context, ImexMode::ExportSelfKeys, Some(&dir)).await?; + println!("Exported to {}.", dir.to_string_lossy()); } "import-keys" => { - imex(&context, ImexMode::ImportSelfKeys, Some(blobdir)).await?; + imex(&context, ImexMode::ImportSelfKeys, Some(arg1)).await?; } "export-setup" => { let setup_code = create_setup_code(&context); diff --git a/src/dc_tools.rs b/src/dc_tools.rs index f937a0ef9..fa57d2c29 100644 --- a/src/dc_tools.rs +++ b/src/dc_tools.rs @@ -450,7 +450,7 @@ pub fn dc_open_file_std>( } } -pub(crate) async fn dc_get_next_backup_path( +pub(crate) async fn get_next_backup_path_old( folder: impl AsRef, backup_time: i64, ) -> Result { @@ -470,6 +470,32 @@ pub(crate) async fn dc_get_next_backup_path( bail!("could not create backup file, disk full?"); } +/// Returns Ok((temp_path, dest_path)) on success. The backup can then be written to temp_path. If the backup succeeded, +/// it can be renamed to dest_path. This guarantees that the backup is complete. +pub(crate) async fn get_next_backup_path_new( + folder: impl AsRef, + backup_time: i64, +) -> Result<(PathBuf, PathBuf), Error> { + let folder = PathBuf::from(folder.as_ref()); + let stem = chrono::NaiveDateTime::from_timestamp(backup_time, 0) + .format("delta-chat-backup-%Y-%m-%d") + .to_string(); + + // 64 backup files per day should be enough for everyone + for i in 0..64 { + let mut tempfile = folder.clone(); + tempfile.push(format!("{}-{:02}.tar.part", stem, i)); + + let mut destfile = folder.clone(); + destfile.push(format!("{}-{:02}.tar", stem, i)); + + if !tempfile.exists().await && !destfile.exists().await { + return Ok((tempfile, destfile)); + } + } + bail!("could not create backup file, disk full?"); +} + pub(crate) fn time() -> i64 { SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) diff --git a/src/events.rs b/src/events.rs index f51929790..45ed034b0 100644 --- a/src/events.rs +++ b/src/events.rs @@ -127,7 +127,7 @@ pub enum EventType { #[strum(props(id = "150"))] NewBlobFile(String), - /// Emitted when an new file in the $BLOBDIR was created + /// Emitted when an file in the $BLOBDIR was deleted #[strum(props(id = "151"))] DeletedBlobFile(String), diff --git a/src/imex.rs b/src/imex.rs index 4b2f7c67e..f448f3fe2 100644 --- a/src/imex.rs +++ b/src/imex.rs @@ -1,10 +1,16 @@ //! # Import/export module use std::any::Any; -use std::cmp::{max, min}; +use std::{ + cmp::{max, min}, + ffi::OsStr, +}; use async_std::path::{Path, PathBuf}; -use async_std::prelude::*; +use async_std::{ + fs::{self, File}, + prelude::*, +}; use rand::{thread_rng, Rng}; use crate::blob::BlobObject; @@ -24,6 +30,11 @@ use crate::param::*; use crate::pgp; use crate::sql::{self, Sql}; use crate::stock::StockMessage; +use async_tar::Archive; + +// Name of the database file in the backup. +const DBFILE_BACKUP_NAME: &str = "dc_database_backup.sqlite"; +const BLOBS_BACKUP_NAME: &str = "blobs_backup"; #[derive(Debug, Display, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive)] #[repr(i32)] @@ -42,8 +53,8 @@ pub enum ImexMode { /// Export a backup to the directory given as `param1`. /// The backup contains all contacts, chats, images and other data and device independent settings. /// The backup does not contain device dependent settings as ringtones or LED notification settings. - /// The name of the backup is typically `delta-chat..bak`, if more than one backup is create on a day, - /// the format is `delta-chat.-.bak` + /// The name of the backup is typically `delta-chat-.tar`, if more than one backup is create on a day, + /// the format is `delta-chat--.tar` ExportBackup = 11, /// `param1` is the file (not: directory) to import. The file is normally @@ -85,6 +96,37 @@ pub async fn imex( /// Returns the filename of the backup found (otherwise an error) pub async fn has_backup(context: &Context, dir_name: impl AsRef) -> Result { + let dir_name = dir_name.as_ref(); + let mut dir_iter = async_std::fs::read_dir(dir_name).await?; + let mut newest_backup_name = "".to_string(); + let mut newest_backup_path: Option = None; + + while let Some(dirent) = dir_iter.next().await { + if let Ok(dirent) = dirent { + let path = dirent.path(); + let name = dirent.file_name(); + let name: String = name.to_string_lossy().into(); + if name.starts_with("delta-chat") + && name.ends_with(".tar") + && (newest_backup_name.is_empty() || name > newest_backup_name) + { + // We just use string comparison to determine which backup is newer. + // This works fine because the filenames have the form ...delta-chat-backup-2020-07-24-00.tar + newest_backup_path = Some(path); + newest_backup_name = name; + } + } + } + + match newest_backup_path { + Some(path) => Ok(path.to_string_lossy().into_owned()), + None => has_backup_old(context, dir_name).await, + // When we decide to remove support for .bak backups, we can replace this with `None => bail!("no backup found in {}", dir_name.display()),`. + } +} + +/// Returns the filename of the backup found (otherwise an error) +pub async fn has_backup_old(context: &Context, dir_name: impl AsRef) -> Result { let dir_name = dir_name.as_ref(); let mut dir_iter = async_std::fs::read_dir(dir_name).await?; let mut newest_backup_time = 0; @@ -373,7 +415,7 @@ async fn imex_inner( ensure!(context.sql.is_open().await, "Database not opened."); - let path = param.unwrap(); + let path = param.ok_or_else(|| format_err!("Imex: Param was None"))?; if what == ImexMode::ExportBackup || what == ImexMode::ExportSelfKeys { // before we export anything, make sure the private key exists if e2ee::ensure_secret_key_exists(context).await.is_err() { @@ -386,7 +428,11 @@ async fn imex_inner( let success = match what { ImexMode::ExportSelfKeys => export_self_keys(context, path).await, ImexMode::ImportSelfKeys => import_self_keys(context, path).await, - ImexMode::ExportBackup => export_backup(context, path).await, + + // TODO In some months we can change the export_backup_old() call to export_backup() and delete export_backup_old(). + // (now is 07/2020) + ImexMode::ExportBackup => export_backup_old(context, path).await, + // import_backup() will call import_backup_old() if this is an old backup. ImexMode::ImportBackup => import_backup(context, path).await, }; @@ -405,6 +451,75 @@ async fn imex_inner( /// Import Backup async fn import_backup(context: &Context, backup_to_import: impl AsRef) -> Result<()> { + if backup_to_import + .as_ref() + .to_string_lossy() + .ends_with(".bak") + { + // Backwards compability + return import_backup_old(context, backup_to_import).await; + } + + info!( + context, + "Import \"{}\" to \"{}\".", + backup_to_import.as_ref().display(), + context.get_dbfile().display() + ); + + ensure!( + !context.is_configured().await, + "Cannot import backups to accounts in use." + ); + context.sql.close().await; + dc_delete_file(context, context.get_dbfile()).await; + ensure!( + !context.get_dbfile().exists().await, + "Cannot delete old database." + ); + + let backup_file = File::open(backup_to_import).await?; + let archive = Archive::new(backup_file); + let mut entries = archive.entries()?; + while let Some(file) = entries.next().await { + let f = &mut file?; + if f.path()?.file_name() == Some(OsStr::new(DBFILE_BACKUP_NAME)) { + // async_tar can't unpack to a specified file name, so we just unpack to the blobdir and then move the unpacked file. + f.unpack_in(context.get_blobdir()).await?; + fs::rename( + context.get_blobdir().join(DBFILE_BACKUP_NAME), + context.get_dbfile(), + ) + .await?; + context.emit_event(EventType::ImexProgress(400)); // Just guess the progress, we at least have the dbfile by now + } else { + // async_tar will unpack to blobdir/BLOBS_BACKUP_NAME, so we move the file afterwards. + f.unpack_in(context.get_blobdir()).await?; + let from_path = context.get_blobdir().join(f.path()?); + if from_path.is_file().await { + if let Some(name) = from_path.file_name() { + fs::rename(&from_path, context.get_blobdir().join(name)).await?; + } else { + warn!(context, "No file name"); + } + } + } + } + + ensure!( + context + .sql + .open(&context, &context.get_dbfile(), false) + .await, + "could not re-open db" + ); + + delete_and_reset_all_device_msgs(&context).await?; + + Ok(()) +} + +async fn import_backup_old(context: &Context, backup_to_import: impl AsRef) -> Result<()> { info!( context, "Import \"{}\" to \"{}\".", @@ -512,14 +627,83 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef) -> /******************************************************************************* * Export backup ******************************************************************************/ -/* the FILE_PROGRESS macro calls the callback with the permille of files processed. -The macro avoids weird values of 0% or 100% while still working. */ +#[allow(unused)] async fn export_backup(context: &Context, dir: impl AsRef) -> Result<()> { + // get a fine backup file name (the name includes the date so that multiple backup instances are possible) + let now = time(); + let (temp_path, dest_path) = get_next_backup_path_new(dir, now).await?; + + context + .sql + .set_raw_config_int(context, "backup_time", now as i32) + .await?; + sql::housekeeping(context).await; + + context + .sql + .execute("VACUUM;", paramsv![]) + .await + .map_err(|e| warn!(context, "Vacuum failed, exporting anyway {}", e)); + + // we close the database during the export + context.sql.close().await; + + info!( + context, + "Backup '{}' to '{}'.", + context.get_dbfile().display(), + dest_path.display(), + ); + + let res = export_backup_inner(context, &temp_path).await; + + // we re-open the database after export is finished + context + .sql + .open(&context, &context.get_dbfile(), false) + .await; + + match &res { + Ok(_) => { + fs::rename(temp_path, &dest_path).await?; + context.emit_event(EventType::ImexFileWritten(dest_path)); + } + Err(e) => { + error!(context, "backup failed: {}", e); + // Not using dc_delete_file() here because it would send a DeletedBlobFile event + fs::remove_file(temp_path).await?; + } + } + + res +} + +async fn export_backup_inner(context: &Context, temp_path: &PathBuf) -> Result<()> { + let file = File::create(temp_path).await?; + + let mut builder = async_tar::Builder::new(file); + + // append_path_with_name() wants the source path as the first argument, append_dir_all() wants it as the second argument. + builder + .append_path_with_name(context.get_dbfile(), DBFILE_BACKUP_NAME) + .await?; + + context.emit_event(EventType::ImexProgress(500)); + + builder + .append_dir_all(BLOBS_BACKUP_NAME, context.get_blobdir()) + .await?; + + builder.finish().await?; + Ok(()) +} + +async fn export_backup_old(context: &Context, dir: impl AsRef) -> Result<()> { // get a fine backup file name (the name includes the date so that multiple backup instances are possible) // FIXME: we should write to a temporary file first and rename it on success. this would guarantee the backup is complete. // let dest_path_filename = dc_get_next_backup_file(context, dir, res); let now = time(); - let dest_path_filename = dc_get_next_backup_path(dir, now).await?; + let dest_path_filename = get_next_backup_path_old(dir, now).await?; let dest_path_string = dest_path_filename.to_string_lossy().to_string(); sql::housekeeping(context).await;