Mirrors/chatmail-core
1
0
Fork 0
mirror of https://github.com/chatmail/core.git synced 2026-04-27 02:16:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Tar backup (#1749)

Browse Source 
Fix #1729
Co-authored-by: holger krekel  <holger@merlinux.eu>
Co-authored-by: Alexander Krotov <ilabdsf@gmail.com>
This commit is contained in:
Hocuri
2020-08-18 11:54:46 +02:00
committed by GitHub
parent 77c61ab25b
commit cf72d9a41e
7 changed files with 293 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
Cargo.lock generated
View File

@@ -257,6 +257,20 @@ dependencies = [
"trust-dns-resolver",
]
[[package]]
name = "async-tar"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb619eae01ab289095debb1ff7c02710d5124c20edde1b2eca926572a34c3998"
dependencies = [
"async-std",
"filetime",
"libc",
"pin-project",
"redox_syscall",
"xattr",
]
[[package]]
name = "async-task"
version = "3.0.0"
@@ -799,6 +813,7 @@ dependencies = [
"async-smtp",
"async-std",
"async-std-resolver",
"async-tar",
"async-trait",
"backtrace",
"base64 0.12.3",
@@ -807,6 +822,7 @@ dependencies = [
"charset",
"chrono",
"deltachat_derive",
"dirs 3.0.1",
"email",
"encoded-words",
"escaper",
@@ -947,6 +963,26 @@ dependencies = [
"winapi",
]
[[package]]
name = "dirs"
version = "3.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "142995ed02755914747cc6ca76fc7e4583cd18578746716d0508ea6ed558b9ff"
dependencies = [
"dirs-sys",
]
[[package]]
name = "dirs-sys"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e93d7f5705de3e49895a2b5e0b8855a1c27f080192ae9c32a6432d50741a57a"
dependencies = [
"libc",
"redox_users",
"winapi",
]
[[package]]
name = "discard"
version = "1.0.4"
@@ -1179,6 +1215,18 @@ version = "1.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bd3bdaaf0a72155260a1c098989b60db1cbb22d6a628e64f16237aa4da93cc7"
[[package]]
name = "filetime"
version = "0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ed85775dcc68644b5c950ac06a2b23768d3bc9390464151aaf27136998dcf9e"
dependencies = [
"cfg-if",
"libc",
"redox_syscall",
"winapi",
]
[[package]]
name = "flate2"
version = "1.0.16"
@@ -2621,7 +2669,7 @@ version = "4.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f47ea1ceb347d2deae482d655dc8eef4bd82363d3329baffa3818bd76fea48b"
dependencies = [
"dirs",
"dirs 1.0.5",
"libc",
"log",
"memchr",
@@ -3539,6 +3587,15 @@ dependencies = [
"zeroize",
]
[[package]]
name = "xattr"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c"
dependencies = [
"libc",
]
[[package]]
name = "zeroize"
version = "1.1.0"

4
Cargo.toml
View File

@@ -59,12 +59,14 @@ anyhow = "1.0.28"
async-trait = "0.1.31"
url = "2.1.1"
async-std-resolver = "0.19.5"
async-tar = "0.3.0"
uuid = { version = "0.8", features = ["serde", "v4"] }
pretty_env_logger = { version = "0.4.0", optional = true }
log = {version = "0.4.8", optional = true }
rustyline = { version = "4.1.0", optional = true }
ansi_term = { version = "0.12.1", optional = true }
dirs = { version = "3.0.1", optional=true }
toml = "0.5.6"
@@ -96,7 +98,7 @@ required-features = ["repl"]
[features]
default = []
internals = []
repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term"]
repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs"]
vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored"]
nightly = ["pgp/nightly"]

4
deltachat-ffi/deltachat.h
View File

@@ -1773,8 +1773,8 @@ dc_contact_t* dc_get_contact (dc_context_t* context, uint32_t co
* - **DC_IMEX_EXPORT_BACKUP** (11) - Export a backup to the directory given as `param1`.
* The backup contains all contacts, chats, images and other data and device independent settings.
* The backup does not contain device dependent settings as ringtones or LED notification settings.
* The name of the backup is typically `delta-chat.<day>.bak`, if more than one backup is create on a day,
* the format is `delta-chat.<day>-<number>.bak`
* The name of the backup is typically `delta-chat-<day>.tar`, if more than one backup is create on a day,
* the format is `delta-chat-<day>-<number>.tar`
*
* - **DC_IMEX_IMPORT_BACKUP** (12) - `param1` is the file (not: directory) to import. The file is normally
* created by DC_IMEX_EXPORT_BACKUP and detected by dc_imex_has_backup(). Importing a backup

12
examples/repl/cmdline.rs
View File

@@ -1,3 +1,5 @@
extern crate dirs;
use std::str::FromStr;
use anyhow::{bail, ensure};
@@ -442,17 +444,21 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
has_backup(&context, blobdir).await?;
}
"export-backup" => {
imex(&context, ImexMode::ExportBackup, Some(blobdir)).await?;
let dir = dirs::home_dir().unwrap_or_default();
imex(&context, ImexMode::ExportBackup, Some(&dir)).await?;
println!("Exported to {}.", dir.to_string_lossy());
}
"import-backup" => {
ensure!(!arg1.is_empty(), "Argument <backup-file> missing.");
imex(&context, ImexMode::ImportBackup, Some(arg1)).await?;
}
"export-keys" => {
imex(&context, ImexMode::ExportSelfKeys, Some(blobdir)).await?;
let dir = dirs::home_dir().unwrap_or_default();
imex(&context, ImexMode::ExportSelfKeys, Some(&dir)).await?;
println!("Exported to {}.", dir.to_string_lossy());
}
"import-keys" => {
imex(&context, ImexMode::ImportSelfKeys, Some(blobdir)).await?;
imex(&context, ImexMode::ImportSelfKeys, Some(arg1)).await?;
}
"export-setup" => {
let setup_code = create_setup_code(&context);

28
src/dc_tools.rs
View File

@@ -450,7 +450,7 @@ pub fn dc_open_file_std<P: AsRef<std::path::Path>>(
}
}
pub(crate) async fn dc_get_next_backup_path(
pub(crate) async fn get_next_backup_path_old(
folder: impl AsRef<Path>,
backup_time: i64,
) -> Result<PathBuf, Error> {
@@ -470,6 +470,32 @@ pub(crate) async fn dc_get_next_backup_path(
bail!("could not create backup file, disk full?");
}
/// Returns Ok((temp_path, dest_path)) on success. The backup can then be written to temp_path. If the backup succeeded,
/// it can be renamed to dest_path. This guarantees that the backup is complete.
pub(crate) async fn get_next_backup_path_new(
folder: impl AsRef<Path>,
backup_time: i64,
) -> Result<(PathBuf, PathBuf), Error> {
let folder = PathBuf::from(folder.as_ref());
let stem = chrono::NaiveDateTime::from_timestamp(backup_time, 0)
.format("delta-chat-backup-%Y-%m-%d")
.to_string();
// 64 backup files per day should be enough for everyone
for i in 0..64 {
let mut tempfile = folder.clone();
tempfile.push(format!("{}-{:02}.tar.part", stem, i));
let mut destfile = folder.clone();
destfile.push(format!("{}-{:02}.tar", stem, i));
if !tempfile.exists().await && !destfile.exists().await {
return Ok((tempfile, destfile));
}
}
bail!("could not create backup file, disk full?");
}
pub(crate) fn time() -> i64 {
SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)

2
src/events.rs
View File

@@ -127,7 +127,7 @@ pub enum EventType {
#[strum(props(id = "150"))]
NewBlobFile(String),
/// Emitted when an new file in the $BLOBDIR was created
/// Emitted when an file in the $BLOBDIR was deleted
#[strum(props(id = "151"))]
DeletedBlobFile(String),

202
src/imex.rs
View File

@@ -1,10 +1,16 @@
//! # Import/export module
use std::any::Any;
use std::cmp::{max, min};
use std::{
cmp::{max, min},
ffi::OsStr,
};
use async_std::path::{Path, PathBuf};
use async_std::prelude::*;
use async_std::{
fs::{self, File},
prelude::*,
};
use rand::{thread_rng, Rng};
use crate::blob::BlobObject;
@@ -24,6 +30,11 @@ use crate::param::*;
use crate::pgp;
use crate::sql::{self, Sql};
use crate::stock::StockMessage;
use async_tar::Archive;
// Name of the database file in the backup.
const DBFILE_BACKUP_NAME: &str = "dc_database_backup.sqlite";
const BLOBS_BACKUP_NAME: &str = "blobs_backup";
#[derive(Debug, Display, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive)]
#[repr(i32)]
@@ -42,8 +53,8 @@ pub enum ImexMode {
/// Export a backup to the directory given as `param1`.
/// The backup contains all contacts, chats, images and other data and device independent settings.
/// The backup does not contain device dependent settings as ringtones or LED notification settings.
/// The name of the backup is typically `delta-chat.<day>.bak`, if more than one backup is create on a day,
/// the format is `delta-chat.<day>-<number>.bak`
/// The name of the backup is typically `delta-chat-<day>.tar`, if more than one backup is create on a day,
/// the format is `delta-chat-<day>-<number>.tar`
ExportBackup = 11,
/// `param1` is the file (not: directory) to import. The file is normally
@@ -85,6 +96,37 @@ pub async fn imex(
/// Returns the filename of the backup found (otherwise an error)
pub async fn has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Result<String> {
let dir_name = dir_name.as_ref();
let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
let mut newest_backup_name = "".to_string();
let mut newest_backup_path: Option<PathBuf> = None;
while let Some(dirent) = dir_iter.next().await {
if let Ok(dirent) = dirent {
let path = dirent.path();
let name = dirent.file_name();
let name: String = name.to_string_lossy().into();
if name.starts_with("delta-chat")
&& name.ends_with(".tar")
&& (newest_backup_name.is_empty() || name > newest_backup_name)
{
// We just use string comparison to determine which backup is newer.
// This works fine because the filenames have the form ...delta-chat-backup-2020-07-24-00.tar
newest_backup_path = Some(path);
newest_backup_name = name;
}
}
}
match newest_backup_path {
Some(path) => Ok(path.to_string_lossy().into_owned()),
None => has_backup_old(context, dir_name).await,
// When we decide to remove support for .bak backups, we can replace this with `None => bail!("no backup found in {}", dir_name.display()),`.
}
}
/// Returns the filename of the backup found (otherwise an error)
pub async fn has_backup_old(context: &Context, dir_name: impl AsRef<Path>) -> Result<String> {
let dir_name = dir_name.as_ref();
let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
let mut newest_backup_time = 0;
@@ -373,7 +415,7 @@ async fn imex_inner(
ensure!(context.sql.is_open().await, "Database not opened.");
let path = param.unwrap();
let path = param.ok_or_else(|| format_err!("Imex: Param was None"))?;
if what == ImexMode::ExportBackup || what == ImexMode::ExportSelfKeys {
// before we export anything, make sure the private key exists
if e2ee::ensure_secret_key_exists(context).await.is_err() {
@@ -386,7 +428,11 @@ async fn imex_inner(
let success = match what {
ImexMode::ExportSelfKeys => export_self_keys(context, path).await,
ImexMode::ImportSelfKeys => import_self_keys(context, path).await,
ImexMode::ExportBackup => export_backup(context, path).await,
// TODO In some months we can change the export_backup_old() call to export_backup() and delete export_backup_old().
// (now is 07/2020)
ImexMode::ExportBackup => export_backup_old(context, path).await,
// import_backup() will call import_backup_old() if this is an old backup.
ImexMode::ImportBackup => import_backup(context, path).await,
};
@@ -405,6 +451,75 @@ async fn imex_inner(
/// Import Backup
async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Result<()> {
if backup_to_import
.as_ref()
.to_string_lossy()
.ends_with(".bak")
{
// Backwards compability
return import_backup_old(context, backup_to_import).await;
}
info!(
context,
"Import \"{}\" to \"{}\".",
backup_to_import.as_ref().display(),
context.get_dbfile().display()
);
ensure!(
!context.is_configured().await,
"Cannot import backups to accounts in use."
);
context.sql.close().await;
dc_delete_file(context, context.get_dbfile()).await;
ensure!(
!context.get_dbfile().exists().await,
"Cannot delete old database."
);
let backup_file = File::open(backup_to_import).await?;
let archive = Archive::new(backup_file);
let mut entries = archive.entries()?;
while let Some(file) = entries.next().await {
let f = &mut file?;
if f.path()?.file_name() == Some(OsStr::new(DBFILE_BACKUP_NAME)) {
// async_tar can't unpack to a specified file name, so we just unpack to the blobdir and then move the unpacked file.
f.unpack_in(context.get_blobdir()).await?;
fs::rename(
context.get_blobdir().join(DBFILE_BACKUP_NAME),
context.get_dbfile(),
)
.await?;
context.emit_event(EventType::ImexProgress(400)); // Just guess the progress, we at least have the dbfile by now
} else {
// async_tar will unpack to blobdir/BLOBS_BACKUP_NAME, so we move the file afterwards.
f.unpack_in(context.get_blobdir()).await?;
let from_path = context.get_blobdir().join(f.path()?);
if from_path.is_file().await {
if let Some(name) = from_path.file_name() {
fs::rename(&from_path, context.get_blobdir().join(name)).await?;
} else {
warn!(context, "No file name");
}
}
}
}
ensure!(
context
.sql
.open(&context, &context.get_dbfile(), false)
.await,
"could not re-open db"
);
delete_and_reset_all_device_msgs(&context).await?;
Ok(())
}
async fn import_backup_old(context: &Context, backup_to_import: impl AsRef<Path>) -> Result<()> {
info!(
context,
"Import \"{}\" to \"{}\".",
@@ -512,14 +627,83 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
/*******************************************************************************
* Export backup
******************************************************************************/
/* the FILE_PROGRESS macro calls the callback with the permille of files processed.
The macro avoids weird values of 0% or 100% while still working. */
#[allow(unused)]
async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
// get a fine backup file name (the name includes the date so that multiple backup instances are possible)
let now = time();
let (temp_path, dest_path) = get_next_backup_path_new(dir, now).await?;
context
.sql
.set_raw_config_int(context, "backup_time", now as i32)
.await?;
sql::housekeeping(context).await;
context
.sql
.execute("VACUUM;", paramsv![])
.await
.map_err(|e| warn!(context, "Vacuum failed, exporting anyway {}", e));
// we close the database during the export
context.sql.close().await;
info!(
context,
"Backup '{}' to '{}'.",
context.get_dbfile().display(),
dest_path.display(),
);
let res = export_backup_inner(context, &temp_path).await;
// we re-open the database after export is finished
context
.sql
.open(&context, &context.get_dbfile(), false)
.await;
match &res {
Ok(_) => {
fs::rename(temp_path, &dest_path).await?;
context.emit_event(EventType::ImexFileWritten(dest_path));
}
Err(e) => {
error!(context, "backup failed: {}", e);
// Not using dc_delete_file() here because it would send a DeletedBlobFile event
fs::remove_file(temp_path).await?;
}
}
res
}
async fn export_backup_inner(context: &Context, temp_path: &PathBuf) -> Result<()> {
let file = File::create(temp_path).await?;
let mut builder = async_tar::Builder::new(file);
// append_path_with_name() wants the source path as the first argument, append_dir_all() wants it as the second argument.
builder
.append_path_with_name(context.get_dbfile(), DBFILE_BACKUP_NAME)
.await?;
context.emit_event(EventType::ImexProgress(500));
builder
.append_dir_all(BLOBS_BACKUP_NAME, context.get_blobdir())
.await?;
builder.finish().await?;
Ok(())
}
async fn export_backup_old(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
// get a fine backup file name (the name includes the date so that multiple backup instances are possible)
// FIXME: we should write to a temporary file first and rename it on success. this would guarantee the backup is complete.
// let dest_path_filename = dc_get_next_backup_file(context, dir, res);
let now = time();
let dest_path_filename = dc_get_next_backup_path(dir, now).await?;
let dest_path_filename = get_next_backup_path_old(dir, now).await?;
let dest_path_string = dest_path_filename.to_string_lossy().to_string();
sql::housekeeping(context).await;