From 3c38fa6b70ecbe2657a69bb35fe5dd1b0ccecd91 Mon Sep 17 00:00:00 2001 From: link2xt Date: Thu, 6 Jan 2022 08:54:58 +0000 Subject: [PATCH] Add API for passphrase-protected accounts To create encrypted account with account manager, call dc_accounts_add_closed_account(). Open this account with dc_context_open() using the passphrase you want to use for encryption. When application is loaded next time and account manager is created, it will open all accounts that have no passphrase set. For encrypted accounts dc_context_is_open() will return 0. To open them, call dc_context_open() with the correct passphrase. After opening, call dc_context_start_io() on this account or just dc_accounts_start_io() to start all accounts that are not started yet. Support for legacy SQLite-based backup format is removed in this commit. --- deltachat-ffi/deltachat.h | 71 ++++++- deltachat-ffi/src/lib.rs | 79 +++++++- python/src/deltachat/account.py | 6 + python/tests/test_account.py | 4 +- python/tests/test_lowlevel.py | 3 +- src/accounts.rs | 90 ++++++++- src/context.rs | 96 ++++++++-- src/dc_tools.rs | 70 ------- src/imex.rs | 181 +----------------- src/sql.rs | 320 +++++++++++++++++++------------- 10 files changed, 515 insertions(+), 405 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 122f33f4d..e8bdb1c4e 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -179,24 +179,66 @@ typedef struct _dc_accounts_event_emitter dc_accounts_event_emitter_t; // create/open/config/information /** - * Create a new context object. After creation it is usually - * opened, connected and mails are fetched. + * Create a new context object and try to open it without passphrase. If + * database is encrypted, the result is the same as using + * dc_context_new_closed() and the database should be opened with + * dc_context_open() before using. * * @memberof dc_context_t - * @param os_name is only for decorative use. - * You can give the name of the app, the operating system, - * the used environment and/or the version here. + * @param os_name Deprecated, pass NULL or empty string here. * @param dbfile The file to use to store the database, * something like `~/file` won't work, use absolute paths. * @param blobdir Deprecated, pass NULL or an empty string here. * @return A context object with some public members. * The object must be passed to the other context functions * and must be freed using dc_context_unref() after usage. + */ +dc_context_t* dc_context_new (const char* os_name, const char* dbfile, const char* blobdir); + + +/** + * Create a new context object. After creation it is usually opened with + * dc_context_open() and started with dc_start_io() so it is connected and + * mails are fetched. + * + * @memberof dc_context_t + * @param dbfile The file to use to store the database, + * something like `~/file` won't work, use absolute paths. + * @return A context object with some public members. + * The object must be passed to the other context functions + * and must be freed using dc_context_unref() after usage. * * If you want to use multiple context objects at the same time, * this can be managed using dc_accounts_t. */ -dc_context_t* dc_context_new (const char* os_name, const char* dbfile, const char* blobdir); +dc_context_t* dc_context_new_closed (const char* dbfile); + + +/** + * Opens the database with the given passphrase. This can only be used on + * closed context, such as created by dc_context_new_closed(). If the database + * is new, this operation sets the database passphrase. For existing databases + * the passphrase should be the one used to encrypt the database the first + * time. + * + * @memberof dc_context_t + * @param context The context object. + * @param passphrase The passphrase to use with the database. Pass NULL or + * empty string to use no passphrase and no encryption. + * @return 1 if the database is opened with this passphrase, 0 if the + * passphrase is incorrect and on error. + */ +int dc_context_open (dc_context_t *context, const char* passphrase); + + +/** + * Returns 1 if database is open. + * + * @memberof dc_context_t + * @param context The context object. + * @return 1 if database is open, 0 if database is closed + */ +int dc_context_is_open (dc_context_t *context); /** @@ -2470,6 +2512,7 @@ void dc_str_unref (char* str); * To make this possible, some dc_context_t functions must not be called * when using the account manager: * - use dc_accounts_add_account() and dc_accounts_get_account() instead of dc_context_new() + * - use dc_accounts_add_closed_account() instead of dc_context_new_closed() * - use dc_accounts_start_io() and dc_accounts_stop_io() instead of dc_start_io() and dc_stop_io() * - use dc_accounts_maybe_network() instead of dc_maybe_network() * - use dc_accounts_get_event_emitter() instead of dc_get_event_emitter() @@ -2527,6 +2570,22 @@ void dc_accounts_unref (dc_accounts_t* accounts); */ uint32_t dc_accounts_add_account (dc_accounts_t* accounts); +/** + * Add a new closed account to the account manager. + * Internally, dc_context_new_closed() is called using a unique database-name + * in the directory specified at dc_accounts_new(). + * + * If the function succeeds, + * dc_accounts_get_all() will return one more account + * and you can access the newly created account using dc_accounts_get_account(). + * Moreover, the newly created account will be the selected one. + * + * @memberof dc_accounts_t + * @param accounts Account manager as created by dc_accounts_new(). + * @return Account-id, use dc_accounts_get_account() to get the context object. + * On errors, 0 is returned. + */ +uint32_t dc_accounts_add_closed_account (dc_accounts_t* accounts); /** * Migrate independent accounts into accounts managed by the account manager. diff --git a/deltachat-ffi/src/lib.rs b/deltachat-ffi/src/lib.rs index a02a13f7d..8f1fdf5f4 100644 --- a/deltachat-ffi/src/lib.rs +++ b/deltachat-ffi/src/lib.rs @@ -27,6 +27,7 @@ use async_std::sync::RwLock; use async_std::task::{block_on, spawn}; use deltachat::qr_code_generator::get_securejoin_qr_svg; use num_traits::{FromPrimitive, ToPrimitive}; +use rand::Rng; use deltachat::chat::{ChatId, ChatVisibility, MuteDuration, ProtectionStatus}; use deltachat::constants::DC_MSG_ID_LAST_SPECIAL; @@ -75,7 +76,6 @@ pub unsafe extern "C" fn dc_context_new( } let ctx = if blobdir.is_null() || *blobdir == 0 { - use rand::Rng; // generate random ID as this functionality is not yet available on the C-api. let id = rand::thread_rng().gen(); block_on(Context::new(as_path(dbfile).to_path_buf().into(), id)) @@ -86,12 +86,63 @@ pub unsafe extern "C" fn dc_context_new( match ctx { Ok(ctx) => Box::into_raw(Box::new(ctx)), Err(err) => { - eprintln!("failed to create context: {}", err); + eprintln!("failed to create context: {:#}", err); ptr::null_mut() } } } +#[no_mangle] +pub unsafe extern "C" fn dc_context_new_closed(dbfile: *const libc::c_char) -> *mut dc_context_t { + setup_panic!(); + + if dbfile.is_null() { + eprintln!("ignoring careless call to dc_context_new_closed()"); + return ptr::null_mut(); + } + + let id = rand::thread_rng().gen(); + match block_on(Context::new_closed( + as_path(dbfile).to_path_buf().into(), + id, + )) { + Ok(context) => Box::into_raw(Box::new(context)), + Err(err) => { + eprintln!("failed to create context: {:#}", err); + ptr::null_mut() + } + } +} + +#[no_mangle] +pub unsafe extern "C" fn dc_context_open( + context: *mut dc_context_t, + passphrase: *const libc::c_char, +) -> libc::c_int { + if context.is_null() { + eprintln!("ignoring careless call to dc_context_open()"); + return 0; + } + + let ctx = &*context; + let passphrase = to_string_lossy(passphrase); + block_on(ctx.open(passphrase)) + .log_err(ctx, "dc_context_open() failed") + .map(|b| b as libc::c_int) + .unwrap_or(0) +} + +#[no_mangle] +pub unsafe extern "C" fn dc_context_is_open(context: *mut dc_context_t) -> libc::c_int { + if context.is_null() { + eprintln!("ignoring careless call to dc_context_is_open()"); + return 0; + } + + let ctx = &*context; + block_on(ctx.is_open()) as libc::c_int +} + /// Release the context structure. /// /// This function releases the memory of the `dc_context_t` structure. @@ -3965,6 +4016,30 @@ pub unsafe extern "C" fn dc_accounts_add_account(accounts: *mut dc_accounts_t) - }) } +#[no_mangle] +pub unsafe extern "C" fn dc_accounts_add_closed_account(accounts: *mut dc_accounts_t) -> u32 { + if accounts.is_null() { + eprintln!("ignoring careless call to dc_accounts_add_closed_account()"); + return 0; + } + + let accounts = &mut *accounts; + + block_on(async move { + let mut accounts = accounts.write().await; + match accounts.add_closed_account().await { + Ok(id) => id, + Err(err) => { + accounts.emit_event(EventType::Error(format!( + "Failed to add account: {:#}", + err + ))); + 0 + } + } + }) +} + #[no_mangle] pub unsafe extern "C" fn dc_accounts_remove_account( accounts: *mut dc_accounts_t, diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index 5f8e39bf9..a9444eb32 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -179,6 +179,12 @@ class Account(object): """ return True if lib.dc_is_configured(self._dc_context) else False + def is_open(self) -> bool: + """Determine if account is open + + :returns True if account is open.""" + return True if lib.dc_context_is_open(self._dc_context) else False + def set_avatar(self, img_path: Optional[str]) -> None: """Set self avatar. diff --git a/python/tests/test_account.py b/python/tests/test_account.py index f3dd4abbb..6b1b52a41 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -41,8 +41,8 @@ class TestOfflineAccountBasic: def test_wrong_db(self, tmpdir): p = tmpdir.join("hello.db") p.write("123") - with pytest.raises(ValueError): - Account(p.strpath) + account = Account(p.strpath) + assert not account.is_open() def test_os_name(self, tmpdir): p = tmpdir.join("hello.db") diff --git a/python/tests/test_lowlevel.py b/python/tests/test_lowlevel.py index 7477eb588..10e27ae9d 100644 --- a/python/tests/test_lowlevel.py +++ b/python/tests/test_lowlevel.py @@ -36,7 +36,8 @@ def test_wrong_db(tmpdir): # write an invalid database file p.write("x123" * 10) - assert ffi.NULL == lib.dc_context_new(ffi.NULL, p.strpath.encode("ascii"), ffi.NULL) + context = lib.dc_context_new(ffi.NULL, p.strpath.encode("ascii"), ffi.NULL) + assert not lib.dc_context_is_open(context) def test_empty_blobdir(tmpdir): diff --git a/src/accounts.rs b/src/accounts.rs index 9275549ce..9340b0c18 100644 --- a/src/accounts.rs +++ b/src/accounts.rs @@ -59,7 +59,10 @@ impl Accounts { let config = Config::from_file(config_file) .await .context("failed to load accounts config")?; - let accounts = config.load_accounts().await?; + let accounts = config + .load_accounts() + .await + .context("failed to load accounts")?; let emitter = EventEmitter::new(); @@ -68,7 +71,9 @@ impl Accounts { emitter.sender.send(events.get_emitter()).await?; for account in accounts.values() { - emitter.add_account(account).await?; + emitter.add_account(account).await.with_context(|| { + format!("failed to add account {} to event emitter", account.id) + })?; } Ok(Self { @@ -106,7 +111,9 @@ impl Accounts { Ok(()) } - /// Add a new account. + /// Add a new account and opens it. + /// + /// Returns account ID. pub async fn add_account(&mut self) -> Result { let account_config = self.config.new_account(&self.dir).await?; @@ -117,6 +124,17 @@ impl Accounts { Ok(account_config.id) } + /// Adds a new closed account. + pub async fn add_closed_account(&mut self) -> Result { + let account_config = self.config.new_account(&self.dir).await?; + + let ctx = Context::new_closed(account_config.dbfile().into(), account_config.id).await?; + self.emitter.add_account(&ctx).await?; + self.accounts.insert(account_config.id, ctx); + + Ok(account_config.id) + } + /// Remove an account. pub async fn remove_account(&mut self, id: u32) -> Result<()> { let ctx = self.accounts.remove(&id); @@ -184,7 +202,7 @@ impl Accounts { match res { Ok(_) => { - let ctx = Context::with_blobdir(new_dbfile, new_blobdir, account_config.id).await?; + let ctx = Context::new(new_dbfile, account_config.id).await?; self.emitter.add_account(&ctx).await?; self.accounts.insert(account_config.id, ctx); Ok(account_config.id) @@ -385,7 +403,15 @@ impl Config { pub async fn load_accounts(&self) -> Result> { let mut accounts = BTreeMap::new(); for account_config in &self.inner.accounts { - let ctx = Context::new(account_config.dbfile().into(), account_config.id).await?; + let ctx = Context::new(account_config.dbfile().into(), account_config.id) + .await + .with_context(|| { + format!( + "failed to create context from file {:?}", + account_config.dbfile() + ) + })?; + accounts.insert(account_config.id, ctx); } @@ -410,8 +436,13 @@ impl Config { self.sync().await?; - self.select_account(id).await.expect("just added"); - let cfg = self.get_account(id).await.expect("just added"); + self.select_account(id) + .await + .context("failed to select just added account")?; + let cfg = self + .get_account(id) + .await + .context("failed to get just added account")?; Ok(cfg) } @@ -703,4 +734,49 @@ mod tests { Ok(()) } + + #[async_std::test] + async fn test_encrypted_account() -> Result<()> { + let dir = tempfile::tempdir().context("failed to create tempdir")?; + let p: PathBuf = dir.path().join("accounts").into(); + + let mut accounts = Accounts::new(p.clone()) + .await + .context("failed to create accounts manager")?; + + assert_eq!(accounts.accounts.len(), 0); + let account_id = accounts + .add_closed_account() + .await + .context("failed to add closed account")?; + let account = accounts + .get_selected_account() + .await + .context("failed to get account")?; + assert_eq!(account.id, account_id); + let passphrase_set_success = account + .open("foobar".to_string()) + .await + .context("failed to set passphrase")?; + assert!(passphrase_set_success); + drop(accounts); + + let accounts = Accounts::new(p.clone()) + .await + .context("failed to create second accounts manager")?; + let account = accounts + .get_selected_account() + .await + .context("failed to get account")?; + assert_eq!(account.is_open().await, false); + + // Try wrong passphrase. + assert_eq!(account.open("barfoo".to_string()).await?, false); + assert_eq!(account.open("".to_string()).await?, false); + + assert_eq!(account.open("foobar".to_string()).await?, true); + assert_eq!(account.is_open().await, true); + + Ok(()) + } } diff --git a/src/context.rs b/src/context.rs index 515f432bf..880bc7b8c 100644 --- a/src/context.rs +++ b/src/context.rs @@ -5,7 +5,7 @@ use std::ffi::OsString; use std::ops::Deref; use std::time::{Instant, SystemTime}; -use anyhow::{bail, ensure, Context as _, Result}; +use anyhow::{bail, ensure, Result}; use async_std::{ channel::{self, Receiver, Sender}, path::{Path, PathBuf}, @@ -42,8 +42,6 @@ impl Deref for Context { #[derive(Debug)] pub struct InnerContext { - /// Database file path - pub(crate) dbfile: PathBuf, /// Blob directory path pub(crate) blobdir: PathBuf, pub(crate) sql: Sql, @@ -106,10 +104,19 @@ pub fn get_info() -> BTreeMap<&'static str, String> { } impl Context { - /// Creates new context. + /// Creates new context and opens the database. pub async fn new(dbfile: PathBuf, id: u32) -> Result { - // pretty_env_logger::try_init_timed().ok(); + let context = Self::new_closed(dbfile, id).await?; + // Open the database if is not encrypted. + if context.set_passphrase("".to_string()).await? { + context.sql.open(&context).await?; + } + Ok(context) + } + + /// Creates new context without opening the database. + pub async fn new_closed(dbfile: PathBuf, id: u32) -> Result { let mut blob_fname = OsString::new(); blob_fname.push(dbfile.file_name().unwrap_or_default()); blob_fname.push("-blobs"); @@ -117,7 +124,35 @@ impl Context { if !blobdir.exists().await { async_std::fs::create_dir_all(&blobdir).await?; } - Context::with_blobdir(dbfile, blobdir, id).await + let context = Context::with_blobdir(dbfile, blobdir, id).await?; + Ok(context) + } + + /// Opens the database with the given passphrase. + /// + /// Returns true if passphrase is correct, false is passphrase is not correct. Fails on other + /// errors. + pub async fn open(&self, passphrase: String) -> Result { + if self.sql.set_passphrase(passphrase).await? { + self.sql.open(self).await?; + Ok(true) + } else { + Ok(false) + } + } + + /// Returns true if database is open. + pub async fn is_open(&self) -> bool { + self.sql.is_open().await + } + + /// Sets the database passphrase. + /// + /// Returns true if passphrase is correct. + /// + /// Fails if database is already open. + pub async fn set_passphrase(&self, passphrase: String) -> Result { + self.sql.set_passphrase(passphrase).await } pub(crate) async fn with_blobdir( @@ -134,9 +169,8 @@ impl Context { let inner = InnerContext { id, blobdir, - dbfile, running_state: RwLock::new(Default::default()), - sql: Sql::new(), + sql: Sql::new(dbfile), bob: Default::default(), last_smeared_timestamp: RwLock::new(0), generating_key_mutex: Mutex::new(()), @@ -155,10 +189,6 @@ impl Context { let ctx = Context { inner: Arc::new(inner), }; - ctx.sql - .open(&ctx, &ctx.dbfile, false) - .await - .context("failed to open SQL database")?; Ok(ctx) } @@ -196,7 +226,7 @@ impl Context { /// Returns database file path. pub fn get_dbfile(&self) -> &Path { - self.dbfile.as_path() + self.sql.dbfile.as_path() } /// Returns blob directory path. @@ -641,16 +671,21 @@ mod tests { use crate::dc_tools::dc_create_outgoing_rfc724_mid; use crate::message::Message; use crate::test_utils::TestContext; + use anyhow::Context as _; use std::time::Duration; use strum::IntoEnumIterator; + use tempfile::tempdir; #[async_std::test] - async fn test_wrong_db() { - let tmp = tempfile::tempdir().unwrap(); + async fn test_wrong_db() -> Result<()> { + let tmp = tempfile::tempdir()?; let dbfile = tmp.path().join("db.sqlite"); - std::fs::write(&dbfile, b"123").unwrap(); - let res = Context::new(dbfile.into(), 1).await; - assert!(res.is_err()); + std::fs::write(&dbfile, b"123")?; + let res = Context::new(dbfile.into(), 1).await?; + + // Broken database is indistinguishable from encrypted one. + assert_eq!(res.is_open().await, false); + Ok(()) } #[async_std::test] @@ -1002,4 +1037,29 @@ mod tests { Ok(()) } + + #[async_std::test] + async fn test_set_passphrase() -> Result<()> { + let dir = tempdir()?; + let dbfile = dir.path().join("db.sqlite"); + + let id = 1; + let context = Context::new_closed(dbfile.clone().into(), id) + .await + .context("failed to create context")?; + assert_eq!(context.open("foo".to_string()).await?, true); + assert_eq!(context.is_open().await, true); + drop(context); + + let id = 2; + let context = Context::new(dbfile.into(), id) + .await + .context("failed to create context")?; + assert_eq!(context.is_open().await, false); + assert_eq!(context.set_passphrase("bar".to_string()).await?, false); + assert_eq!(context.open("false".to_string()).await?, false); + assert_eq!(context.open("foo".to_string()).await?, true); + + Ok(()) + } } diff --git a/src/dc_tools.rs b/src/dc_tools.rs index a1ecd0e96..3c3d8980c 100644 --- a/src/dc_tools.rs +++ b/src/dc_tools.rs @@ -354,63 +354,6 @@ pub async fn dc_delete_files_in_dir(context: &Context, path: impl AsRef) { } } -pub(crate) async fn dc_copy_file( - context: &Context, - src_path: impl AsRef, - dest_path: impl AsRef, -) -> bool { - let src_abs = dc_get_abs_path(context, &src_path); - let mut src_file = match fs::File::open(&src_abs).await { - Ok(file) => file, - Err(err) => { - warn!( - context, - "failed to open for read '{}': {}", - src_abs.display(), - err - ); - return false; - } - }; - - let dest_abs = dc_get_abs_path(context, &dest_path); - let mut dest_file = match fs::OpenOptions::new() - .create_new(true) - .write(true) - .open(&dest_abs) - .await - { - Ok(file) => file, - Err(err) => { - warn!( - context, - "failed to open for write '{}': {}", - dest_abs.display(), - err - ); - return false; - } - }; - - match io::copy(&mut src_file, &mut dest_file).await { - Ok(_) => true, - Err(err) => { - error!( - context, - "Cannot copy \"{}\" to \"{}\": {}", - src_abs.display(), - dest_abs.display(), - err - ); - { - // Attempt to remove the failed file, swallow errors resulting from that. - fs::remove_file(dest_abs).await.ok(); - } - false - } - } -} - pub(crate) async fn dc_create_folder( context: &Context, path: impl AsRef, @@ -1025,20 +968,7 @@ Hop: From: hq5.example.org; By: hq5.example.org; Date: Mon, 27 Dec 2021 11:21:22 assert!(dc_file_exist!(context, &abs_path).await); - assert!(dc_copy_file(context, "$BLOBDIR/foobar", "$BLOBDIR/dada").await); - - // attempting to copy a second time should fail - assert!(!dc_copy_file(context, "$BLOBDIR/foobar", "$BLOBDIR/dada").await); - - assert_eq!(dc_get_filebytes(context, "$BLOBDIR/dada").await, 7); - - let buf = dc_read_file(context, "$BLOBDIR/dada").await.unwrap(); - - assert_eq!(buf.len(), 7); - assert_eq!(&buf, b"content"); - assert!(dc_delete_file(context, "$BLOBDIR/foobar").await); - assert!(dc_delete_file(context, "$BLOBDIR/dada").await); assert!(dc_create_folder(context, "$BLOBDIR/foobar-folder") .await .is_ok()); diff --git a/src/imex.rs b/src/imex.rs index 2c221f67c..70259f40f 100644 --- a/src/imex.rs +++ b/src/imex.rs @@ -19,7 +19,7 @@ use crate::config::Config; use crate::constants::{Viewtype, DC_CONTACT_ID_SELF}; use crate::context::Context; use crate::dc_tools::{ - dc_copy_file, dc_create_folder, dc_delete_file, dc_delete_files_in_dir, dc_get_filesuffix_lc, + dc_create_folder, dc_delete_file, dc_delete_files_in_dir, dc_get_filesuffix_lc, dc_open_file_std, dc_read_file, dc_write_file, get_next_backup_path, time, EmailAddress, }; use crate::e2ee; @@ -30,7 +30,7 @@ use crate::message::{Message, MsgId}; use crate::mimeparser::SystemMessage; use crate::param::Param; use crate::pgp; -use crate::sql::{self, Sql}; +use crate::sql; use crate::stock_str; // Name of the database file in the backup. @@ -116,14 +116,14 @@ async fn cleanup_aborted_imex(context: &Context, what: ImexMode) { dc_delete_files_in_dir(context, context.get_blobdir()).await; } if what == ImexMode::ExportBackup || what == ImexMode::ImportBackup { - if let Err(e) = context.sql.open(context, context.get_dbfile(), false).await { + if let Err(e) = context.sql.open(context).await { warn!(context, "Re-opening db after imex failed: {}", e); } } } /// Returns the filename of the backup found (otherwise an error) -pub async fn has_backup(context: &Context, dir_name: &Path) -> Result { +pub async fn has_backup(_context: &Context, dir_name: &Path) -> Result { let mut dir_iter = async_std::fs::read_dir(dir_name).await?; let mut newest_backup_name = "".to_string(); let mut newest_backup_path: Option = None; @@ -145,59 +145,6 @@ pub async fn has_backup(context: &Context, dir_name: &Path) -> Result { } } - match newest_backup_path { - Some(path) => Ok(path.to_string_lossy().into_owned()), - None => has_backup_old(context, dir_name).await, - // When we decide to remove support for .bak backups, we can replace this with `None => bail!("no backup found in {}", dir_name.display()),`. - } -} - -/// Returns the filename of the backup found (otherwise an error) -pub async fn has_backup_old(context: &Context, dir_name: &Path) -> Result { - let mut dir_iter = async_std::fs::read_dir(dir_name).await?; - let mut newest_backup_time = 0; - let mut newest_backup_name = "".to_string(); - let mut newest_backup_path: Option = None; - while let Some(dirent) = dir_iter.next().await { - if let Ok(dirent) = dirent { - let path = dirent.path(); - let name = dirent.file_name(); - let name = name.to_string_lossy(); - if name.starts_with("delta-chat") && name.ends_with(".bak") { - let sql = Sql::new(); - match sql.open(context, &path, true).await { - Ok(_) => { - let curr_backup_time = sql - .get_raw_config_int("backup_time") - .await? - .unwrap_or_default(); - if curr_backup_time > newest_backup_time { - newest_backup_path = Some(path); - newest_backup_time = curr_backup_time; - } - info!(context, "backup_time of {} is {}", name, curr_backup_time); - sql.close().await; - } - Err(e) => { - warn!( - context, - "Found backup file {} which could not be opened: {}", name, e - ); - // On some Android devices we can't open sql files that are not in our private directory - // (see ). So, compare names - // to still find the newest backup. - let name: String = name.into(); - if newest_backup_time == 0 - && (newest_backup_name.is_empty() || name > newest_backup_name) - { - newest_backup_path = Some(path); - newest_backup_name = name; - } - } - } - } - } - } match newest_backup_path { Some(path) => Ok(path.to_string_lossy().into_owned()), None => bail!("no backup found in {}", dir_name.display()), @@ -468,18 +415,12 @@ async fn imex_inner(context: &Context, what: ImexMode, path: &Path) -> Result<() ImexMode::ImportSelfKeys => import_self_keys(context, path).await, ImexMode::ExportBackup => export_backup(context, path).await, - // import_backup() will call import_backup_old() if this is an old backup. ImexMode::ImportBackup => import_backup(context, path).await, } } /// Import Backup async fn import_backup(context: &Context, backup_to_import: &Path) -> Result<()> { - if backup_to_import.to_string_lossy().ends_with(".bak") { - // Backwards compability - return import_backup_old(context, backup_to_import).await; - } - info!( context, "Import \"{}\" to \"{}\".", @@ -543,7 +484,7 @@ async fn import_backup(context: &Context, backup_to_import: &Path) -> Result<()> context .sql - .open(context, context.get_dbfile(), false) + .open(context) .await .context("Could not re-open db")?; @@ -552,116 +493,6 @@ async fn import_backup(context: &Context, backup_to_import: &Path) -> Result<()> Ok(()) } -async fn import_backup_old(context: &Context, backup_to_import: &Path) -> Result<()> { - info!( - context, - "Import \"{}\" to \"{}\".", - backup_to_import.display(), - context.get_dbfile().display() - ); - - ensure!( - !context.is_configured().await?, - "Cannot import backups to accounts in use." - ); - ensure!( - !context.scheduler.read().await.is_running(), - "cannot import backup, IO already running" - ); - context.sql.close().await; - dc_delete_file(context, context.get_dbfile()).await; - ensure!( - !context.get_dbfile().exists().await, - "Cannot delete old database." - ); - - ensure!( - dc_copy_file(context, backup_to_import, context.get_dbfile()).await, - "could not copy file" - ); - /* error already logged */ - /* re-open copied database file */ - context - .sql - .open(context, context.get_dbfile(), false) - .await - .context("Could not re-open db")?; - - delete_and_reset_all_device_msgs(context).await?; - - let total_files_cnt = context - .sql - .count("SELECT COUNT(*) FROM backup_blobs;", paramsv![]) - .await?; - info!( - context, - "***IMPORT-in-progress: total_files_cnt={:?}", total_files_cnt, - ); - - // Load IDs only for now, without the file contents, to avoid - // consuming too much memory. - let file_ids = context - .sql - .query_map( - "SELECT id FROM backup_blobs ORDER BY id", - paramsv![], - |row| row.get(0), - |ids| { - ids.collect::, _>>() - .map_err(Into::into) - }, - ) - .await?; - - let mut all_files_extracted = true; - for (processed_files_cnt, file_id) in file_ids.into_iter().enumerate() { - // Load a single blob into memory - let (file_name, file_blob) = context - .sql - .query_row( - "SELECT file_name, file_content FROM backup_blobs WHERE id = ?", - paramsv![file_id], - |row| { - let file_name: String = row.get(0)?; - let file_blob: Vec = row.get(1)?; - Ok((file_name, file_blob)) - }, - ) - .await?; - - if context.shall_stop_ongoing().await { - all_files_extracted = false; - break; - } - let mut permille = processed_files_cnt * 1000 / total_files_cnt; - if permille < 10 { - permille = 10 - } - if permille > 990 { - permille = 990 - } - context.emit_event(EventType::ImexProgress(permille)); - if file_blob.is_empty() { - continue; - } - - let path_filename = context.get_blobdir().join(file_name); - dc_write_file(context, &path_filename, &file_blob).await?; - } - - if all_files_extracted { - // only delete backup_blobs if all files were successfully extracted - context - .sql - .execute("DROP TABLE backup_blobs;", paramsv![]) - .await?; - context.sql.execute("VACUUM;", paramsv![]).await.ok(); - Ok(()) - } else { - bail!("received stop signal"); - } -} - /******************************************************************************* * Export backup ******************************************************************************/ @@ -702,7 +533,7 @@ async fn export_backup(context: &Context, dir: &Path) -> Result<()> { let res = export_backup_inner(context, &temp_path).await; // we re-open the database after export is finished - context.sql.open(context, context.get_dbfile(), false).await; + context.sql.open(context).await; match &res { Ok(_) => { diff --git a/src/sql.rs b/src/sql.rs index 7e7510cae..acc20f2c9 100644 --- a/src/sql.rs +++ b/src/sql.rs @@ -8,8 +8,9 @@ use std::convert::TryFrom; use std::time::Duration; use anyhow::{bail, Context as _, Result}; +use async_std::path::PathBuf; use async_std::prelude::*; -use rusqlite::OpenFlags; +use rusqlite::{Connection, OpenFlags}; use crate::blob::BlobObject; use crate::chat::{add_device_msg, update_device_icon, update_saved_messages_icon}; @@ -38,20 +39,55 @@ mod migrations; /// A wrapper around the underlying Sqlite3 object. #[derive(Debug)] pub struct Sql { - pool: RwLock>>, -} + /// Database file path + pub(crate) dbfile: PathBuf, -impl Default for Sql { - fn default() -> Self { - Self { - pool: RwLock::new(None), - } - } + pool: RwLock>>, + + /// SQLCipher passphrase. + /// + /// Empty string if database is not encrypted. + passphrase: RwLock, } impl Sql { - pub fn new() -> Sql { - Self::default() + pub fn new(dbfile: PathBuf) -> Sql { + Self { + dbfile, + pool: Default::default(), + passphrase: Default::default(), + } + } + + /// Sets SQLCipher passphrase for key derivation. + /// + /// Returns true if passphrase is correct, i.e. the database is new or can be unlocked with + /// this passphrase, and false if the database is already encrypted with another passphrase or + /// corrupted. + /// + /// Fails if database is already open. + pub async fn set_passphrase(&self, passphrase: String) -> Result { + if self.is_open().await { + bail!("Database is already opened."); + } + + // Hold the lock to prevent other thread from opening the database. + let _lock = self.pool.write().await; + + // Test that the key is correct using a single connection. + let connection = Connection::open(&self.dbfile)?; + connection + .pragma_update(None, "key", &passphrase) + .context("failed to set PRAGMA key")?; + let key_is_correct = connection + .query_row("SELECT count(*) FROM sqlite_master", [], |_row| Ok(())) + .is_ok(); + + if key_is_correct { + *self.passphrase.write().await = passphrase; + } + + Ok(key_is_correct) } /// Checks if there is currently a connection to the underlying Sqlite database. @@ -65,24 +101,20 @@ impl Sql { // drop closes the connection } - pub fn new_pool( + fn new_pool( dbfile: &Path, - readonly: bool, - ) -> anyhow::Result> { + passphrase: String, + ) -> Result> { let mut open_flags = OpenFlags::SQLITE_OPEN_NO_MUTEX; - if readonly { - open_flags.insert(OpenFlags::SQLITE_OPEN_READ_ONLY); - } else { - open_flags.insert(OpenFlags::SQLITE_OPEN_READ_WRITE); - open_flags.insert(OpenFlags::SQLITE_OPEN_CREATE); - } + open_flags.insert(OpenFlags::SQLITE_OPEN_READ_WRITE); + open_flags.insert(OpenFlags::SQLITE_OPEN_CREATE); // this actually creates min_idle database handles just now. // therefore, with_init() must not try to modify the database as otherwise // we easily get busy-errors (eg. table-creation, journal_mode etc. should be done on only one handle) let mgr = r2d2_sqlite::SqliteConnectionManager::file(dbfile) .with_flags(open_flags) - .with_init(|c| { + .with_init(move |c| { c.execute_batch(&format!( "PRAGMA cipher_memory_security = OFF; -- Too slow on Android PRAGMA secure_delete=on; @@ -91,6 +123,7 @@ impl Sql { ", Duration::from_secs(10).as_millis() ))?; + c.pragma_update(None, "key", passphrase.clone())?; Ok(()) }); @@ -103,116 +136,123 @@ impl Sql { Ok(pool) } + async fn try_open(&self, context: &Context, dbfile: &Path, passphrase: String) -> Result<()> { + *self.pool.write().await = Some(Self::new_pool(dbfile, passphrase.to_string())?); + + { + let conn = self.get_conn().await?; + + // Try to enable auto_vacuum. This will only be + // applied if the database is new or after successful + // VACUUM, which usually happens before backup export. + // When auto_vacuum is INCREMENTAL, it is possible to + // use PRAGMA incremental_vacuum to return unused + // database pages to the filesystem. + conn.pragma_update(None, "auto_vacuum", &"INCREMENTAL".to_string())?; + + // journal_mode is persisted, it is sufficient to change it only for one handle. + conn.pragma_update(None, "journal_mode", &"WAL".to_string())?; + + // Default synchronous=FULL is much slower. NORMAL is sufficient for WAL mode. + conn.pragma_update(None, "synchronous", &"NORMAL".to_string())?; + } + + // (1) update low-level database structure. + // this should be done before updates that use high-level objects that + // rely themselves on the low-level structure. + + let (recalc_fingerprints, update_icons, disable_server_delete, recode_avatar) = + migrations::run(context, self) + .await + .context("failed to run migrations")?; + + // (2) updates that require high-level objects + // the structure is complete now and all objects are usable + + if recalc_fingerprints { + info!(context, "[migration] recalc fingerprints"); + let addrs = self + .query_map( + "SELECT addr FROM acpeerstates;", + paramsv![], + |row| row.get::<_, String>(0), + |addrs| { + addrs + .collect::, _>>() + .map_err(Into::into) + }, + ) + .await?; + for addr in &addrs { + if let Some(ref mut peerstate) = Peerstate::from_addr(context, addr).await? { + peerstate.recalc_fingerprint(); + peerstate.save_to_db(self, false).await?; + } + } + } + + if update_icons { + update_saved_messages_icon(context).await?; + update_device_icon(context).await?; + } + + if disable_server_delete { + // We now always watch all folders and delete messages there if delete_server is enabled. + // So, for people who have delete_server enabled, disable it and add a hint to the devicechat: + if context.get_config_delete_server_after().await?.is_some() { + let mut msg = Message::new(Viewtype::Text); + msg.text = Some(stock_str::delete_server_turned_off(context).await); + add_device_msg(context, None, Some(&mut msg)).await?; + context + .set_config(Config::DeleteServerAfter, Some("0")) + .await?; + } + } + + if recode_avatar { + if let Some(avatar) = context.get_config(Config::Selfavatar).await? { + let mut blob = BlobObject::new_from_path(context, avatar.as_ref()).await?; + match blob.recode_to_avatar_size(context).await { + Ok(()) => { + context + .set_config(Config::Selfavatar, Some(&avatar)) + .await? + } + Err(e) => { + warn!(context, "Migrations can't recode avatar, removing. {:#}", e); + context.set_config(Config::Selfavatar, None).await? + } + } + } + } + + Ok(()) + } + /// Opens the provided database and runs any necessary migrations. /// If a database is already open, this will return an error. - pub async fn open( - &self, - context: &Context, - dbfile: &Path, - readonly: bool, - ) -> anyhow::Result<()> { + pub async fn open(&self, context: &Context) -> Result<()> { if self.is_open().await { error!( context, - "Cannot open, database \"{:?}\" already opened.", dbfile, + "Cannot open, database \"{:?}\" already opened.", self.dbfile, ); bail!("SQL database is already opened."); } - *self.pool.write().await = Some(Self::new_pool(dbfile, readonly)?); + let passphrase_lock = self.passphrase.read().await; + let passphrase: &str = passphrase_lock.as_ref(); - if !readonly { - { - let conn = self.get_conn().await?; - - // Try to enable auto_vacuum. This will only be - // applied if the database is new or after successful - // VACUUM, which usually happens before backup export. - // When auto_vacuum is INCREMENTAL, it is possible to - // use PRAGMA incremental_vacuum to return unused - // database pages to the filesystem. - conn.pragma_update(None, "auto_vacuum", &"INCREMENTAL".to_string())?; - - // journal_mode is persisted, it is sufficient to change it only for one handle. - conn.pragma_update(None, "journal_mode", &"WAL".to_string())?; - - // Default synchronous=FULL is much slower. NORMAL is sufficient for WAL mode. - conn.pragma_update(None, "synchronous", &"NORMAL".to_string())?; - } - - // (1) update low-level database structure. - // this should be done before updates that use high-level objects that - // rely themselves on the low-level structure. - - let (recalc_fingerprints, update_icons, disable_server_delete, recode_avatar) = - migrations::run(context, self) - .await - .context("failed to run migrations")?; - - // (2) updates that require high-level objects - // the structure is complete now and all objects are usable - - if recalc_fingerprints { - info!(context, "[migration] recalc fingerprints"); - let addrs = self - .query_map( - "SELECT addr FROM acpeerstates;", - paramsv![], - |row| row.get::<_, String>(0), - |addrs| { - addrs - .collect::, _>>() - .map_err(Into::into) - }, - ) - .await?; - for addr in &addrs { - if let Some(ref mut peerstate) = Peerstate::from_addr(context, addr).await? { - peerstate.recalc_fingerprint(); - peerstate.save_to_db(self, false).await?; - } - } - } - - if update_icons { - update_saved_messages_icon(context).await?; - update_device_icon(context).await?; - } - - if disable_server_delete { - // We now always watch all folders and delete messages there if delete_server is enabled. - // So, for people who have delete_server enabled, disable it and add a hint to the devicechat: - if context.get_config_delete_server_after().await?.is_some() { - let mut msg = Message::new(Viewtype::Text); - msg.text = Some(stock_str::delete_server_turned_off(context).await); - add_device_msg(context, None, Some(&mut msg)).await?; - context - .set_config(Config::DeleteServerAfter, Some("0")) - .await?; - } - } - - if recode_avatar { - if let Some(avatar) = context.get_config(Config::Selfavatar).await? { - let mut blob = BlobObject::new_from_path(context, avatar.as_ref()).await?; - match blob.recode_to_avatar_size(context).await { - Ok(()) => { - context - .set_config(Config::Selfavatar, Some(&avatar)) - .await? - } - Err(e) => { - warn!(context, "Migrations can't recode avatar, removing. {:#}", e); - context.set_config(Config::Selfavatar, None).await? - } - } - } - } + if let Err(err) = self + .try_open(context, &self.dbfile, passphrase.to_string()) + .await + { + self.close().await; + Err(err) + } else { + info!(context, "Opened database {:?}.", self.dbfile); + Ok(()) } - - info!(context, "Opened database {:?}.", dbfile); - - Ok(()) } /// Execute the given query, returning the number of affected rows. @@ -788,7 +828,7 @@ mod tests { t.sql.close().await; housekeeping(&t).await.unwrap_err(); // housekeeping should fail as the db is closed - t.sql.open(&t, t.get_dbfile(), false).await.unwrap(); + t.sql.open(&t).await.unwrap(); let a = t.get_config(Config::Selfavatar).await.unwrap().unwrap(); assert_eq!(avatar_bytes, &async_std::fs::read(&a).await.unwrap()[..]); @@ -828,14 +868,14 @@ mod tests { // Create a separate empty database for testing. let dir = tempdir()?; let dbfile = dir.path().join("testdb.sqlite"); - let sql = Sql::new(); + let sql = Sql::new(dbfile.into()); // Create database with all the tables. - sql.open(&t, dbfile.as_ref(), false).await.unwrap(); + sql.open(&t).await.unwrap(); sql.close().await; // Reopen the database - sql.open(&t, dbfile.as_ref(), false).await?; + sql.open(&t).await?; sql.execute( "INSERT INTO config (keyname, value) VALUES (?, ?);", paramsv!("foo", "bar"), @@ -888,4 +928,36 @@ mod tests { Ok(()) } + + #[async_std::test] + async fn test_set_passphrase() -> Result<()> { + use tempfile::tempdir; + + // The context is used only for logging. + let t = TestContext::new().await; + + // Create a separate empty database for testing. + let dir = tempdir()?; + let dbfile = dir.path().join("testdb.sqlite"); + let sql = Sql::new(dbfile.clone().into()); + + sql.set_passphrase("foo".to_string()).await?; + sql.open(&t) + .await + .context("failed to open the database first time")?; + sql.close().await; + + // Reopen the database + let sql = Sql::new(dbfile.into()); + + // Test that we can't open encrypted database without a passphrase. + assert!(sql.open(&t).await.is_err()); + + // Now set the passphrase and open the database, it should succeed. + sql.set_passphrase("foo".to_string()).await?; + sql.open(&t) + .await + .context("failed to open the database second time")?; + Ok(()) + } }