From 51b9e86d71fe1e80bb0935c1c3bb711555ec36d5 Mon Sep 17 00:00:00 2001 From: Hocuri Date: Tue, 21 Oct 2025 15:29:21 +0200 Subject: [PATCH] Opt-in weekly sending of statistics (#6851) This way, the statistics / self-reporting bot will be made into an opt-in regular sending of statistics, where you enable the setting once and then they will be sent automatically. The statistics will be sent to a bot, so that the user can see exactly which data is being sent, and how often. The chat will be archived and muted by default, so that it doesn't disturb the user. The collected statistics will focus on the public-key-verification that is performed while scanning a QR code. Later on, we can add more statistics to collect. **Context:** _This is just to give a rough idea; I would need to write a lot more than a few paragraphs in order to fully explain all the context here_. End-to-end encrypted messengers are generally susceptible to MitM attacks. In order to mitigate against this, messengers offer some way of verifying the chat partner's public key. However, numerous studies found that most popular messengers implement this public-key-verification in a way that is not understood by users, and therefore ineffective - [a 2021 "State of Knowledge" paper concludes:](https://dl.acm.org/doi/pdf/10.1145/3558482.3581773) > Based on our evaluation, we have determined that all current E2EE apps, particularly when operating in opportunistic E2EE mode, are incapable of repelling active man-in-the-middle (MitM) attacks. In addition, we find that none of the current E2EE apps provide better and more usable [public key verification] ceremonies, resulting in insecure E2EE communications against active MitM attacks. This is why Delta Chat tries to go a different route: When the user scans a QR code (regardless of whether the QR code creates a 1:1 chat, invites to a group, or subscribes to a broadcast channel), a public-key-verification is performed in the background, without the user even having to know about this. The statistics collected here are supposed to tell us whether Delta Chat succeeds to nudge the users into using QR codes in a way that is secure against MitM attacks. **Plan for statistics-sending:** - [x] Get this PR reviewed and merged (but don't make it available in the UI yet; if Android wants to make a release in the meantime, I will create a PR that removes the option there) - [x] Set the interval to 1 week again (right now, it's 1 minute for testing) - [ ] Write something for people who are interested in what exactly we count, and link to it (see `TODO[blog post]` in the code) - [ ] Prepare a short survey for participants - [ ] Fine-tune the texts at https://github.com/deltachat/deltachat-android/pull/3794, and get it reviewed and merged - [ ] After the next release, ask people to enable the statistics-sending --- ...f-reporting-bot.vcf => statistics-bot.vcf} | 0 deltachat-jsonrpc/src/api.rs | 39 +- deltachat-jsonrpc/src/api/types/qr.rs | 51 + deltachat-time/src/lib.rs | 5 + src/config.rs | 35 +- src/constants.rs | 2 + src/context.rs | 165 +--- src/context/context_tests.rs | 24 +- src/ephemeral.rs | 10 +- src/lib.rs | 3 + src/receive_imf.rs | 3 + src/scheduler.rs | 9 +- src/securejoin.rs | 41 +- src/sql/migrations.rs | 39 + src/stats.rs | 896 ++++++++++++++++++ src/stats/stats_tests.rs | 595 ++++++++++++ src/stock_str.rs | 10 + 17 files changed, 1747 insertions(+), 180 deletions(-) rename assets/{self-reporting-bot.vcf => statistics-bot.vcf} (100%) create mode 100644 src/stats.rs create mode 100644 src/stats/stats_tests.rs diff --git a/assets/self-reporting-bot.vcf b/assets/statistics-bot.vcf similarity index 100% rename from assets/self-reporting-bot.vcf rename to assets/statistics-bot.vcf diff --git a/deltachat-jsonrpc/src/api.rs b/deltachat-jsonrpc/src/api.rs index 595f6e989..5171e2a14 100644 --- a/deltachat-jsonrpc/src/api.rs +++ b/deltachat-jsonrpc/src/api.rs @@ -66,7 +66,7 @@ use self::types::{ }, }; use crate::api::types::chat_list::{get_chat_list_item_by_id, ChatListItemFetchResult}; -use crate::api::types::qr::QrObject; +use crate::api::types::qr::{QrObject, SecurejoinSource, SecurejoinUiPath}; #[derive(Debug)] struct AccountState { @@ -381,11 +381,6 @@ impl CommandApi { Ok(BlobObject::create_and_deduplicate(&ctx, file, file)?.to_abs_path()) } - async fn draft_self_report(&self, account_id: u32) -> Result { - let ctx = self.get_context(account_id).await?; - Ok(ctx.draft_self_report().await?.to_u32()) - } - /// Sets the given configuration key. async fn set_config(&self, account_id: u32, key: String, value: Option) -> Result<()> { let ctx = self.get_context(account_id).await?; @@ -884,6 +879,38 @@ impl CommandApi { Ok(chat_id.to_u32()) } + /// Like `secure_join()`, but allows to pass a source and a UI-path. + /// You only need this if your UI has an option to send statistics + /// to Delta Chat's developers. + /// + /// **source**: The source where the QR code came from. + /// E.g. a link that was clicked inside or outside Delta Chat, + /// the "Paste from Clipboard" action, + /// the "Load QR code as image" action, + /// or a QR code scan. + /// + /// **uipath**: Which UI path did the user use to arrive at the QR code screen. + /// If the SecurejoinSource was ExternalLink or InternalLink, + /// pass `None` here, because the QR code screen wasn't even opened. + /// ``` + async fn secure_join_with_ux_info( + &self, + account_id: u32, + qr: String, + source: Option, + uipath: Option, + ) -> Result { + let ctx = self.get_context(account_id).await?; + let chat_id = securejoin::join_securejoin_with_ux_info( + &ctx, + &qr, + source.map(Into::into), + uipath.map(Into::into), + ) + .await?; + Ok(chat_id.to_u32()) + } + async fn leave_group(&self, account_id: u32, chat_id: u32) -> Result<()> { let ctx = self.get_context(account_id).await?; remove_contact_from_chat(&ctx, ChatId::new(chat_id), ContactId::SELF).await diff --git a/deltachat-jsonrpc/src/api/types/qr.rs b/deltachat-jsonrpc/src/api/types/qr.rs index 0414ec9e5..8270e0da6 100644 --- a/deltachat-jsonrpc/src/api/types/qr.rs +++ b/deltachat-jsonrpc/src/api/types/qr.rs @@ -1,4 +1,5 @@ use deltachat::qr::Qr; +use serde::Deserialize; use serde::Serialize; use typescript_type_def::TypeDef; @@ -304,3 +305,53 @@ impl From for QrObject { } } } + +#[derive(Deserialize, TypeDef, schemars::JsonSchema)] +pub enum SecurejoinSource { + /// Because of some problem, it is unknown where the QR code came from. + Unknown, + /// The user opened a link somewhere outside Delta Chat + ExternalLink, + /// The user clicked on a link in a message inside Delta Chat + InternalLink, + /// The user clicked "Paste from Clipboard" in the QR scan activity + Clipboard, + /// The user clicked "Load QR code as image" in the QR scan activity + ImageLoaded, + /// The user scanned a QR code + Scan, +} + +#[derive(Deserialize, TypeDef, schemars::JsonSchema)] +pub enum SecurejoinUiPath { + /// The UI path is unknown, or the user didn't open the QR code screen at all. + Unknown, + /// The user directly clicked on the QR icon in the main screen + QrIcon, + /// The user first clicked on the `+` button in the main screen, + /// and then on "New Contact" + NewContact, +} + +impl From for deltachat::SecurejoinSource { + fn from(value: SecurejoinSource) -> Self { + match value { + SecurejoinSource::Unknown => deltachat::SecurejoinSource::Unknown, + SecurejoinSource::ExternalLink => deltachat::SecurejoinSource::ExternalLink, + SecurejoinSource::InternalLink => deltachat::SecurejoinSource::InternalLink, + SecurejoinSource::Clipboard => deltachat::SecurejoinSource::Clipboard, + SecurejoinSource::ImageLoaded => deltachat::SecurejoinSource::ImageLoaded, + SecurejoinSource::Scan => deltachat::SecurejoinSource::Scan, + } + } +} + +impl From for deltachat::SecurejoinUiPath { + fn from(value: SecurejoinUiPath) -> Self { + match value { + SecurejoinUiPath::Unknown => deltachat::SecurejoinUiPath::Unknown, + SecurejoinUiPath::QrIcon => deltachat::SecurejoinUiPath::QrIcon, + SecurejoinUiPath::NewContact => deltachat::SecurejoinUiPath::NewContact, + } + } +} diff --git a/deltachat-time/src/lib.rs b/deltachat-time/src/lib.rs index c8d7d6f6c..b3b8b2a21 100644 --- a/deltachat-time/src/lib.rs +++ b/deltachat-time/src/lib.rs @@ -20,6 +20,11 @@ impl SystemTimeTools { pub fn shift(duration: Duration) { *SYSTEM_TIME_SHIFT.write().unwrap() += duration; } + + /// Simulates the system clock being rewound by `duration`. + pub fn shift_back(duration: Duration) { + *SYSTEM_TIME_SHIFT.write().unwrap() -= duration; + } } #[cfg(test)] diff --git a/src/config.rs b/src/config.rs index 1f73262b2..aabec2455 100644 --- a/src/config.rs +++ b/src/config.rs @@ -14,7 +14,6 @@ use tokio::fs; use crate::blob::BlobObject; use crate::configure::EnteredLoginParam; -use crate::constants; use crate::context::Context; use crate::events::EventType; use crate::log::{LogExt, info}; @@ -23,6 +22,7 @@ use crate::mimefactory::RECOMMENDED_FILE_SIZE; use crate::provider::{Provider, get_provider_by_id}; use crate::sync::{self, Sync::*, SyncData}; use crate::tools::get_abs_path; +use crate::{constants, stats}; /// The available configuration keys. #[derive( @@ -408,9 +408,22 @@ pub enum Config { /// used for signatures, encryption to self and included in `Autocrypt` header. KeyId, - /// This key is sent to the self_reporting bot so that the bot can recognize the user + /// Send statistics to Delta Chat's developers. + /// Can be exposed to the user as a setting. + StatsSending, + + /// Last time statistics were sent to Delta Chat's developers + StatsLastSent, + + /// Last time `update_message_stats()` was called + StatsLastUpdate, + + /// This key is sent to the statistics bot so that the bot can recognize the user /// without storing the email address - SelfReportingId, + StatsId, + + /// The last contact id that already existed when statistics-sending was enabled for the first time. + StatsLastOldContactId, /// MsgId of webxdc map integration. WebxdcIntegration, @@ -576,8 +589,9 @@ impl Context { /// Returns boolean configuration value for the given key. pub async fn get_config_bool(&self, key: Config) -> Result { Ok(self - .get_config_parsed::(key) + .get_config(key) .await? + .and_then(|s| s.parse::().ok()) .map(|x| x != 0) .unwrap_or_default()) } @@ -716,10 +730,19 @@ impl Context { true => self.scheduler.pause(self).await?, _ => Default::default(), }; + if key == Config::StatsSending { + let old_value = self.get_config(key).await?; + let old_value = bool_from_config(old_value.as_deref()); + let new_value = bool_from_config(value); + stats::pre_sending_config_change(self, old_value, new_value).await?; + } self.set_config_internal(key, value).await?; if key == Config::SentboxWatch { self.last_full_folder_scan.lock().await.take(); } + if key == Config::StatsSending { + stats::maybe_send_stats(self).await?; + } Ok(()) } @@ -871,6 +894,10 @@ pub(crate) fn from_bool(val: bool) -> Option<&'static str> { Some(if val { "1" } else { "0" }) } +pub(crate) fn bool_from_config(config: Option<&str>) -> bool { + config.is_some_and(|v| v.parse::().unwrap_or_default() != 0) +} + // Separate impl block for self address handling impl Context { /// Determine whether the specified addr maps to the/a self addr. diff --git a/src/constants.rs b/src/constants.rs index b1275fd21..b536d040c 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -101,6 +101,8 @@ pub const DC_CHAT_ID_LAST_SPECIAL: ChatId = ChatId::new(9); Copy, PartialEq, Eq, + PartialOrd, + Ord, FromPrimitive, ToPrimitive, FromSql, diff --git a/src/context.rs b/src/context.rs index 8d160ae9f..9dd664836 100644 --- a/src/context.rs +++ b/src/context.rs @@ -10,28 +10,22 @@ use std::time::Duration; use anyhow::{Context as _, Result, bail, ensure}; use async_channel::{self as channel, Receiver, Sender}; -use pgp::types::PublicKeyTrait; use ratelimit::Ratelimit; use tokio::sync::{Mutex, Notify, RwLock}; use crate::chat::{ChatId, get_chat_cnt}; -use crate::chatlist_events; use crate::config::Config; -use crate::constants::{ - self, DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT, DC_CHAT_ID_TRASH, DC_VERSION_STR, -}; -use crate::contact::{Contact, ContactId, import_vcard, mark_contact_id_as_verified}; +use crate::constants::{self, DC_BACKGROUND_FETCH_QUOTA_CHECK_RATELIMIT, DC_VERSION_STR}; +use crate::contact::{Contact, ContactId}; use crate::debug_logging::DebugLogging; -use crate::download::DownloadState; use crate::events::{Event, EventEmitter, EventType, Events}; use crate::imap::{FolderMeaning, Imap, ServerMetadata}; -use crate::key::{load_self_secret_key, self_fingerprint}; +use crate::key::self_fingerprint; use crate::log::{info, warn}; use crate::logged_debug_assert; use crate::login_param::{ConfiguredLoginParam, EnteredLoginParam}; -use crate::message::{self, Message, MessageState, MsgId}; +use crate::message::{self, MessageState, MsgId}; use crate::net::tls::TlsSessionStore; -use crate::param::{Param, Params}; use crate::peer_channels::Iroh; use crate::push::PushSubscriber; use crate::quota::QuotaInfo; @@ -39,7 +33,8 @@ use crate::scheduler::{ConnectivityStore, SchedulerState, convert_folder_meaning use crate::sql::Sql; use crate::stock_str::StockStrings; use crate::timesmearing::SmearedTimestamp; -use crate::tools::{self, create_id, duration_to_str, time, time_elapsed}; +use crate::tools::{self, duration_to_str, time, time_elapsed}; +use crate::{chatlist_events, stats}; /// Builder for the [`Context`]. /// @@ -1066,6 +1061,22 @@ impl Context { .await? .unwrap_or_default(), ); + res.insert( + "stats_id", + self.get_config(Config::StatsId) + .await? + .unwrap_or_else(|| "".to_string()), + ); + res.insert( + "stats_sending", + stats::should_send_stats(self).await?.to_string(), + ); + res.insert( + "stats_last_sent", + self.get_config_i64(Config::StatsLastSent) + .await? + .to_string(), + ); res.insert( "fail_on_receiving_full_msg", self.sql @@ -1080,138 +1091,6 @@ impl Context { Ok(res) } - async fn get_self_report(&self) -> Result { - #[derive(Default)] - struct ChatNumbers { - opportunistic_dc: u32, - opportunistic_mua: u32, - unencrypted_dc: u32, - unencrypted_mua: u32, - } - - let mut res = String::new(); - res += &format!("core_version {}\n", get_version_str()); - - let num_msgs: u32 = self - .sql - .query_get_value( - "SELECT COUNT(*) FROM msgs WHERE hidden=0 AND chat_id!=?", - (DC_CHAT_ID_TRASH,), - ) - .await? - .unwrap_or_default(); - res += &format!("num_msgs {num_msgs}\n"); - - let num_chats: u32 = self - .sql - .query_get_value("SELECT COUNT(*) FROM chats WHERE id>9 AND blocked!=1", ()) - .await? - .unwrap_or_default(); - res += &format!("num_chats {num_chats}\n"); - - let db_size = tokio::fs::metadata(&self.sql.dbfile).await?.len(); - res += &format!("db_size_bytes {db_size}\n"); - - let secret_key = &load_self_secret_key(self).await?.primary_key; - let key_created = secret_key.public_key().created_at().timestamp(); - res += &format!("key_created {key_created}\n"); - - // how many of the chats active in the last months are: - // - opportunistic-encrypted and the contact uses Delta Chat - // - opportunistic-encrypted and the contact uses a classical MUA - // - unencrypted and the contact uses Delta Chat - // - unencrypted and the contact uses a classical MUA - let three_months_ago = time().saturating_sub(3600 * 24 * 30 * 3); - let chats = self - .sql - .query_map( - "SELECT m.param, m.msgrmsg - FROM chats c - JOIN msgs m - ON c.id=m.chat_id - AND m.id=( - SELECT id - FROM msgs - WHERE chat_id=c.id - AND hidden=0 - AND download_state=? - AND to_id!=? - ORDER BY timestamp DESC, id DESC LIMIT 1) - WHERE c.id>9 - AND (c.blocked=0 OR c.blocked=2) - AND IFNULL(m.timestamp,c.created_timestamp) > ? - GROUP BY c.id", - (DownloadState::Done, ContactId::INFO, three_months_ago), - |row| { - let message_param: Params = - row.get::<_, String>(1)?.parse().unwrap_or_default(); - let is_dc_message: bool = row.get(2)?; - Ok((message_param, is_dc_message)) - }, - |rows| { - let mut chats = ChatNumbers::default(); - for row in rows { - let (message_param, is_dc_message) = row?; - let encrypted = message_param - .get_bool(Param::GuaranteeE2ee) - .unwrap_or(false); - - if encrypted { - if is_dc_message { - chats.opportunistic_dc += 1; - } else { - chats.opportunistic_mua += 1; - } - } else if is_dc_message { - chats.unencrypted_dc += 1; - } else { - chats.unencrypted_mua += 1; - } - } - Ok(chats) - }, - ) - .await?; - res += &format!("chats_opportunistic_dc {}\n", chats.opportunistic_dc); - res += &format!("chats_opportunistic_mua {}\n", chats.opportunistic_mua); - res += &format!("chats_unencrypted_dc {}\n", chats.unencrypted_dc); - res += &format!("chats_unencrypted_mua {}\n", chats.unencrypted_mua); - - let self_reporting_id = match self.get_config(Config::SelfReportingId).await? { - Some(id) => id, - None => { - let id = create_id(); - self.set_config(Config::SelfReportingId, Some(&id)).await?; - id - } - }; - res += &format!("self_reporting_id {self_reporting_id}"); - - Ok(res) - } - - /// Drafts a message with statistics about the usage of Delta Chat. - /// The user can inspect the message if they want, and then hit "Send". - /// - /// On the other end, a bot will receive the message and make it available - /// to Delta Chat's developers. - pub async fn draft_self_report(&self) -> Result { - const SELF_REPORTING_BOT_VCARD: &str = include_str!("../assets/self-reporting-bot.vcf"); - let contact_id: ContactId = *import_vcard(self, SELF_REPORTING_BOT_VCARD) - .await? - .first() - .context("Self reporting bot vCard does not contain a contact")?; - mark_contact_id_as_verified(self, contact_id, Some(ContactId::SELF)).await?; - - let chat_id = ChatId::create_for_contact(self, contact_id).await?; - - let mut msg = Message::new_text(self.get_self_report().await?); - - chat_id.set_draft(self, Some(&mut msg)).await?; - - Ok(chat_id) - } - /// Get a list of fresh, unmuted messages in unblocked chats. /// /// The list starts with the most recent message diff --git a/src/context/context_tests.rs b/src/context/context_tests.rs index 03dcaf3bb..4a20c3af3 100644 --- a/src/context/context_tests.rs +++ b/src/context/context_tests.rs @@ -6,9 +6,9 @@ use super::*; use crate::chat::{Chat, MuteDuration, get_chat_contacts, get_chat_msgs, send_msg, set_muted}; use crate::chatlist::Chatlist; use crate::constants::Chattype; -use crate::mimeparser::SystemMessage; +use crate::message::Message; use crate::receive_imf::receive_imf; -use crate::test_utils::{E2EE_INFO_MSGS, TestContext, get_chat_msg}; +use crate::test_utils::{E2EE_INFO_MSGS, TestContext}; use crate::tools::{SystemTime, create_outgoing_rfc724_mid}; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -276,7 +276,6 @@ async fn test_get_info_completeness() { "mail_port", "mail_security", "notify_about_wrong_pw", - "self_reporting_id", "selfstatus", "send_server", "send_user", @@ -296,6 +295,8 @@ async fn test_get_info_completeness() { "webxdc_integration", "device_token", "encrypted_device_token", + "stats_last_update", + "stats_last_old_contact_id", ]; let t = TestContext::new().await; let info = t.get_info().await.unwrap(); @@ -598,23 +599,6 @@ async fn test_get_next_msgs() -> Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn test_draft_self_report() -> Result<()> { - let alice = TestContext::new_alice().await; - - let chat_id = alice.draft_self_report().await?; - let msg = get_chat_msg(&alice, chat_id, 0, 1).await; - assert_eq!(msg.get_info_type(), SystemMessage::ChatE2ee); - - let mut draft = chat_id.get_draft(&alice).await?.unwrap(); - assert!(draft.text.starts_with("core_version")); - - // Test that sending into the chat works: - let _sent = alice.send_msg(chat_id, &mut draft).await; - - Ok(()) -} - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_cache_is_cleared_when_io_is_started() -> Result<()> { let alice = TestContext::new_alice().await; diff --git a/src/ephemeral.rs b/src/ephemeral.rs index 84a580517..c7a56774a 100644 --- a/src/ephemeral.rs +++ b/src/ephemeral.rs @@ -80,12 +80,12 @@ use crate::contact::ContactId; use crate::context::Context; use crate::download::MIN_DELETE_SERVER_AFTER; use crate::events::EventType; -use crate::location; use crate::log::{LogExt, error, info, warn}; use crate::message::{Message, MessageState, MsgId, Viewtype}; use crate::mimeparser::SystemMessage; use crate::stock_str; use crate::tools::{SystemTime, duration_to_str, time}; +use crate::{location, stats}; /// Ephemeral timer value. #[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize)] @@ -610,7 +610,7 @@ pub(crate) async fn ephemeral_loop(context: &Context, interrupt_receiver: Receiv + Duration::from_secs(1) } else { // no messages to be deleted for now, wait long for one to occur - now + Duration::from_secs(86400) + now + Duration::from_secs(86400) // 1 day }; if let Ok(duration) = until.duration_since(now) { @@ -637,6 +637,12 @@ pub(crate) async fn ephemeral_loop(context: &Context, interrupt_receiver: Receiv } } + // Make sure that the statistics stay correct by updating them _before_ deleting messages: + stats::maybe_update_message_stats(context) + .await + .log_err(context) + .ok(); + delete_expired_messages(context, time()) .await .log_err(context) diff --git a/src/lib.rs b/src/lib.rs index 6a33b23c9..d3d30e386 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -99,6 +99,9 @@ pub mod html; pub mod net; pub mod plaintext; mod push; +mod stats; +pub use stats::SecurejoinSource; +pub use stats::SecurejoinUiPath; pub mod summary; mod debug_logging; diff --git a/src/receive_imf.rs b/src/receive_imf.rs index e64a74a7e..11e74ab63 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -40,6 +40,7 @@ use crate::reaction::{Reaction, set_msg_reaction}; use crate::rusqlite::OptionalExtension; use crate::securejoin::{self, handle_securejoin_handshake, observe_securejoin_on_other_device}; use crate::simplify; +use crate::stats::STATISTICS_BOT_EMAIL; use crate::stock_str; use crate::sync::Sync::*; use crate::tools::{self, buf_compress, remove_subject_prefix}; @@ -1700,6 +1701,8 @@ async fn add_parts( // No check for `hidden` because only reactions are such and they should be `InFresh`. { MessageState::InSeen + } else if mime_parser.from.addr == STATISTICS_BOT_EMAIL { + MessageState::InNoticed } else { MessageState::InFresh }; diff --git a/src/scheduler.rs b/src/scheduler.rs index bbf1ef016..0c1134e43 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -15,7 +15,6 @@ use tokio_util::task::TaskTracker; pub(crate) use self::connectivity::ConnectivityStore; use crate::config::{self, Config}; -use crate::constants; use crate::contact::{ContactId, RecentlySeenLoop}; use crate::context::Context; use crate::download::{DownloadState, download_msg}; @@ -27,7 +26,9 @@ use crate::log::{LogExt, error, info, warn}; use crate::message::MsgId; use crate::smtp::{Smtp, send_smtp_messages}; use crate::sql; +use crate::stats::maybe_send_stats; use crate::tools::{self, duration_to_str, maybe_add_time_based_warnings, time, time_elapsed}; +use crate::{constants, stats}; pub(crate) mod connectivity; @@ -513,6 +514,7 @@ async fn inbox_fetch_idle(ctx: &Context, imap: &mut Imap, mut session: Session) } }; + maybe_send_stats(ctx).await.log_err(ctx).ok(); match ctx.get_config_bool(Config::FetchedExistingMsgs).await { Ok(fetched_existing_msgs) => { if !fetched_existing_msgs { @@ -807,6 +809,11 @@ async fn smtp_loop( } } + stats::maybe_update_message_stats(&ctx) + .await + .log_err(&ctx) + .ok(); + // Fake Idle info!(ctx, "SMTP fake idle started."); match &connection.last_send_error { diff --git a/src/securejoin.rs b/src/securejoin.rs index 4e3bea5bb..92cfd8998 100644 --- a/src/securejoin.rs +++ b/src/securejoin.rs @@ -14,6 +14,7 @@ use crate::e2ee::ensure_secret_key_exists; use crate::events::EventType; use crate::headerdef::HeaderDef; use crate::key::{DcKey, Fingerprint, load_self_public_key}; +use crate::log::LogExt as _; use crate::log::{error, info, warn}; use crate::message::{Message, Viewtype}; use crate::mimeparser::{MimeMessage, SystemMessage}; @@ -21,13 +22,14 @@ use crate::param::Param; use crate::qr::check_qr; use crate::securejoin::bob::JoinerProgress; use crate::sync::Sync::*; -use crate::token; use crate::tools::{create_id, time}; +use crate::{SecurejoinSource, stats}; +use crate::{SecurejoinUiPath, token}; mod bob; mod qrinvite; -use qrinvite::QrInvite; +pub(crate) use qrinvite::QrInvite; use crate::token::Namespace; @@ -168,12 +170,38 @@ async fn get_self_fingerprint(context: &Context) -> Result { /// /// The function returns immediately and the handshake will run in background. pub async fn join_securejoin(context: &Context, qr: &str) -> Result { - securejoin(context, qr).await.map_err(|err| { + join_securejoin_with_ux_info(context, qr, None, None).await +} + +/// Take a scanned QR-code and do the setup-contact/join-group/invite handshake. +/// +/// This is the start of the process for the joiner. See the module and ffi documentation +/// for more details. +/// +/// The function returns immediately and the handshake will run in background. +/// +/// **source** and **uipath** are for statistics-sending, +/// if the user enabled it in the settings; +/// if you don't have statistics-sending implemented, just pass `None` here. +pub async fn join_securejoin_with_ux_info( + context: &Context, + qr: &str, + source: Option, + uipath: Option, +) -> Result { + let res = securejoin(context, qr).await.map_err(|err| { warn!(context, "Fatal joiner error: {:#}", err); // The user just scanned this QR code so has context on what failed. error!(context, "QR process failed"); err - }) + })?; + + stats::count_securejoin_ux_info(context, source, uipath) + .await + .log_err(context) + .ok(); + + Ok(res) } async fn securejoin(context: &Context, qr: &str) -> Result { @@ -187,6 +215,11 @@ async fn securejoin(context: &Context, qr: &str) -> Result { let invite = QrInvite::try_from(qr_scan)?; + stats::count_securejoin_invite(context, &invite) + .await + .log_err(context) + .ok(); + bob::start_protocol(context, invite).await } diff --git a/src/sql/migrations.rs b/src/sql/migrations.rs index 54aac6506..851fec670 100644 --- a/src/sql/migrations.rs +++ b/src/sql/migrations.rs @@ -1271,6 +1271,45 @@ CREATE INDEX gossip_timestamp_index ON gossip_timestamp (chat_id, fingerprint); .await?; } + inc_and_check(&mut migration_version, 135)?; + if dbversion < migration_version { + sql.execute_migration( + "CREATE TABLE stats_securejoin_sources( + source INTEGER PRIMARY KEY, + count INTEGER NOT NULL DEFAULT 0 + ) STRICT; + CREATE TABLE stats_securejoin_uipaths( + uipath INTEGER PRIMARY KEY, + count INTEGER NOT NULL DEFAULT 0 + ) STRICT; + CREATE TABLE stats_securejoin_invites( + already_existed INTEGER NOT NULL, + already_verified INTEGER NOT NULL, + type TEXT NOT NULL + ) STRICT; + CREATE TABLE stats_msgs( + chattype INTEGER PRIMARY KEY, + verified INTEGER NOT NULL DEFAULT 0, + unverified_encrypted INTEGER NOT NULL DEFAULT 0, + unencrypted INTEGER NOT NULL DEFAULT 0, + only_to_self INTEGER NOT NULL DEFAULT 0, + last_counted_msg_id INTEGER NOT NULL DEFAULT 0 + ) STRICT;", + migration_version, + ) + .await?; + } + + inc_and_check(&mut migration_version, 136)?; + if dbversion < migration_version { + sql.execute_migration( + "CREATE TABLE stats_sending_enabled_events(timestamp INTEGER NOT NULL) STRICT; + CREATE TABLE stats_sending_disabled_events(timestamp INTEGER NOT NULL) STRICT;", + migration_version, + ) + .await?; + } + let new_version = sql .get_raw_config_int(VERSION_CFG) .await? diff --git a/src/stats.rs b/src/stats.rs new file mode 100644 index 000000000..d360f9681 --- /dev/null +++ b/src/stats.rs @@ -0,0 +1,896 @@ +//! Delta Chat has an advanced option +//! "Send statistics to the developers of Delta Chat". +//! If this is enabled, a JSON file with some anonymous statistics +//! will be sent to a bot once a week. + +use std::collections::{BTreeMap, BTreeSet}; + +use anyhow::{Context as _, Result}; +use deltachat_derive::FromSql; +use num_traits::ToPrimitive; +use pgp::types::PublicKeyTrait; +use rusqlite::OptionalExtension; +use serde::Serialize; + +use crate::chat::{self, ChatId, MuteDuration}; +use crate::config::Config; +use crate::constants::Chattype; +use crate::contact::{Contact, ContactId, Origin, import_vcard, mark_contact_id_as_verified}; +use crate::context::{Context, get_version_str}; +use crate::key::load_self_public_keyring; +use crate::log::LogExt; +use crate::message::{Message, Viewtype}; +use crate::securejoin::QrInvite; +use crate::tools::{create_id, time}; + +pub(crate) const STATISTICS_BOT_EMAIL: &str = "self_reporting@testrun.org"; +const STATISTICS_BOT_VCARD: &str = include_str!("../assets/statistics-bot.vcf"); +const SENDING_INTERVAL_SECONDS: i64 = 3600 * 24 * 7; // 1 week +// const SENDING_INTERVAL_SECONDS: i64 = 60; // 1 minute (for testing) +const MESSAGE_STATS_UPDATE_INTERVAL_SECONDS: i64 = 4 * 60; // 4 minutes (less than the lowest ephemeral messages timeout) + +#[derive(Serialize)] +struct Statistics { + core_version: String, + key_create_timestamps: Vec, + stats_id: String, + is_chatmail: bool, + contact_stats: Vec, + message_stats: BTreeMap, + securejoin_sources: SecurejoinSources, + securejoin_uipaths: SecurejoinUiPaths, + securejoin_invites: Vec, + sending_enabled_timestamps: Vec, + sending_disabled_timestamps: Vec, +} + +#[derive(Serialize, PartialEq)] +enum VerifiedStatus { + Direct, + Transitive, + TransitiveViaBot, + Opportunistic, + Unencrypted, +} + +#[derive(Serialize)] +struct ContactStat { + #[serde(skip_serializing)] + id: ContactId, + + verified: VerifiedStatus, + + // If one of the boolean properties is false, + // we leave them away. + // This way, the Json file becomes a lot smaller. + #[serde(skip_serializing_if = "is_false")] + bot: bool, + + #[serde(skip_serializing_if = "is_false")] + direct_chat: bool, + + last_seen: u64, + + #[serde(skip_serializing_if = "Option::is_none")] + transitive_chain: Option, + + /// Whether the contact was established after stats-sending was enabled + #[serde(skip_serializing_if = "is_false")] + new: bool, +} + +fn is_false(b: &bool) -> bool { + !b +} + +#[derive(Serialize, Default)] +struct MessageStats { + verified: u32, + unverified_encrypted: u32, + unencrypted: u32, + only_to_self: u32, +} + +/// Where a securejoin invite link or QR code came from. +/// This is only used if the user enabled StatsSending. +#[repr(u32)] +#[derive( + Debug, Clone, Copy, ToPrimitive, FromPrimitive, FromSql, PartialEq, Eq, PartialOrd, Ord, +)] +pub enum SecurejoinSource { + /// Because of some problem, it is unknown where the QR code came from. + Unknown = 0, + /// The user opened a link somewhere outside Delta Chat + ExternalLink = 1, + /// The user clicked on a link in a message inside Delta Chat + InternalLink = 2, + /// The user clicked "Paste from Clipboard" in the QR scan activity + Clipboard = 3, + /// The user clicked "Load QR code as image" in the QR scan activity + ImageLoaded = 4, + /// The user scanned a QR code + Scan = 5, +} + +#[derive(Serialize)] +struct SecurejoinSources { + unknown: u32, + external_link: u32, + internal_link: u32, + clipboard: u32, + image_loaded: u32, + scan: u32, +} + +/// How the user opened the QR activity in order scan a QR code on Android. +/// This is only used if the user enabled StatsSending. +#[derive( + Debug, Clone, Copy, ToPrimitive, FromPrimitive, FromSql, PartialEq, Eq, PartialOrd, Ord, +)] +pub enum SecurejoinUiPath { + /// The UI path is unknown, or the user didn't open the QR code screen at all. + Unknown = 0, + /// The user directly clicked on the QR icon in the main screen + QrIcon = 1, + /// The user first clicked on the `+` button in the main screen, + /// and then on "New Contact" + NewContact = 2, +} + +#[derive(Serialize)] +struct SecurejoinUiPaths { + other: u32, + qr_icon: u32, + new_contact: u32, +} + +/// Some information on an invite-joining event +/// (i.e. a qr scan or a clicked link). +#[derive(Serialize)] +struct JoinedInvite { + /// Whether the contact already existed before. + /// If this is false, then a contact was newly created. + already_existed: bool, + /// If a contact already existed, + /// this tells us whether the contact was verified already. + already_verified: bool, + /// The type of the invite: + /// "contact" for 1:1 invites that setup a verified contact, + /// "group" for invites that invite to a group + /// and also perform the contact verification 'along the way'. + typ: String, +} + +pub(crate) async fn pre_sending_config_change( + context: &Context, + old_value: bool, + new_value: bool, +) -> Result<()> { + // These functions are no-ops if they were called in the past already; + // just call them opportunistically: + ensure_last_old_contact_id(context).await?; + // Make sure that StatsId is available for the UI, + // in order to open the survey with the StatsId as a parameter: + stats_id(context).await?; + + if old_value != new_value { + if new_value { + // Only count messages sent from now on: + set_last_counted_msg_id(context).await?; + } else { + // Update message stats one last time in case it's enabled again in the future: + update_message_stats(context).await?; + } + + let sql_table = if new_value { + "stats_sending_enabled_events" + } else { + "stats_sending_disabled_events" + }; + + context + .sql + .execute(&format!("INSERT INTO {sql_table} VALUES(?)"), (time(),)) + .await?; + } + + Ok(()) +} + +/// Sends a message with statistics about the usage of Delta Chat, +/// if the last time such a message was sent +/// was more than a week ago. +/// +/// On the other end, a bot will receive the message and make it available +/// to Delta Chat's developers. +pub async fn maybe_send_stats(context: &Context) -> Result> { + if should_send_stats(context).await? + && time_has_passed(context, Config::StatsLastSent, SENDING_INTERVAL_SECONDS).await? + { + let chat_id = send_stats(context).await?; + + return Ok(Some(chat_id)); + } + Ok(None) +} + +pub(crate) async fn maybe_update_message_stats(context: &Context) -> Result<()> { + if should_send_stats(context).await? + && time_has_passed( + context, + Config::StatsLastUpdate, + MESSAGE_STATS_UPDATE_INTERVAL_SECONDS, + ) + .await? + { + update_message_stats(context).await?; + } + + Ok(()) +} + +async fn time_has_passed(context: &Context, config: Config, seconds: i64) -> Result { + let last_time = context.get_config_i64(config).await?; + let next_time = last_time.saturating_add(seconds); + + let res = if next_time <= time() { + // Already set the config to the current time. + // This prevents infinite loops in the (unlikely) case of an error: + context + .set_config_internal(config, Some(&time().to_string())) + .await?; + true + } else { + if time() < last_time { + // The clock was rewound. + // Reset the config, so that the statistics will be sent normally in a week, + // or be normally updated in a few minutes. + context + .set_config_internal(config, Some(&time().to_string())) + .await?; + } + false + }; + + Ok(res) +} + +#[allow(clippy::unused_async, unused)] +pub(crate) async fn should_send_stats(context: &Context) -> Result { + #[cfg(any(target_os = "android", test))] + { + context.get_config_bool(Config::StatsSending).await + } + + // If the user enables statistics-sending on Android, + // and then transfers the account to e.g. Desktop, + // we should not send any statistics: + #[cfg(not(any(target_os = "android", test)))] + { + Ok(false) + } +} + +async fn send_stats(context: &Context) -> Result { + info!(context, "Sending statistics."); + + update_message_stats(context).await?; + + let chat_id = get_stats_chat_id(context).await?; + + let mut msg = Message::new(Viewtype::File); + msg.set_text(crate::stock_str::stats_msg_body(context).await); + + let stats = get_stats(context).await?; + + msg.set_file_from_bytes( + context, + "statistics.txt", + stats.as_bytes(), + Some("text/plain"), + )?; + + chat::send_msg(context, chat_id, &mut msg) + .await + .context("Failed to send statistics message") + .log_err(context) + .ok(); + + Ok(chat_id) +} + +async fn set_last_counted_msg_id(context: &Context) -> Result<()> { + context + .sql + .execute( + "UPDATE stats_msgs + SET last_counted_msg_id=(SELECT MAX(id) FROM msgs)", + (), + ) + .await?; + + Ok(()) +} + +async fn ensure_last_old_contact_id(context: &Context) -> Result<()> { + if context.config_exists(Config::StatsLastOldContactId).await? { + // The user had statistics-sending enabled already in the past, + // keep the 'last old contact id' as-is + return Ok(()); + } + + let last_contact_id: u64 = context + .sql + .query_get_value("SELECT MAX(id) FROM contacts", ()) + .await? + .unwrap_or(0); + + context + .sql + .set_raw_config( + Config::StatsLastOldContactId.as_ref(), + Some(&last_contact_id.to_string()), + ) + .await?; + + Ok(()) +} + +async fn get_stats(context: &Context) -> Result { + // The Id of the last contact that already existed when the user enabled the setting. + // Newer contacts will get the `new` flag set. + let last_old_contact = context + .get_config_u32(Config::StatsLastOldContactId) + .await?; + + let key_create_timestamps: Vec = load_self_public_keyring(context) + .await? + .iter() + .map(|k| k.created_at().timestamp()) + .collect(); + + let sending_enabled_timestamps = + get_timestamps(context, "stats_sending_enabled_events").await?; + let sending_disabled_timestamps = + get_timestamps(context, "stats_sending_disabled_events").await?; + + let stats = Statistics { + core_version: get_version_str().to_string(), + key_create_timestamps, + stats_id: stats_id(context).await?, + is_chatmail: context.is_chatmail().await?, + contact_stats: get_contact_stats(context, last_old_contact).await?, + message_stats: get_message_stats(context).await?, + securejoin_sources: get_securejoin_source_stats(context).await?, + securejoin_uipaths: get_securejoin_uipath_stats(context).await?, + securejoin_invites: get_securejoin_invite_stats(context).await?, + sending_enabled_timestamps, + sending_disabled_timestamps, + }; + + Ok(serde_json::to_string_pretty(&stats)?) +} + +async fn get_timestamps(context: &Context, sql_table: &str) -> Result> { + let res = context + .sql + .query_map( + &format!("SELECT timestamp FROM {sql_table} LIMIT 1000"), + (), + |row| row.get(0), + |rows| { + rows.collect::>>() + .map_err(Into::into) + }, + ) + .await?; + + Ok(res) +} + +pub(crate) async fn stats_id(context: &Context) -> Result { + Ok(match context.get_config(Config::StatsId).await? { + Some(id) => id, + None => { + let id = create_id(); + context + .set_config_internal(Config::StatsId, Some(&id)) + .await?; + id + } + }) +} + +async fn get_stats_chat_id(context: &Context) -> Result { + let contact_id: ContactId = *import_vcard(context, STATISTICS_BOT_VCARD) + .await? + .first() + .context("Statistics bot vCard does not contain a contact")?; + mark_contact_id_as_verified(context, contact_id, Some(ContactId::SELF)).await?; + + let chat_id = if let Some(res) = ChatId::lookup_by_contact(context, contact_id).await? { + // Already exists, no need to create. + res + } else { + let chat_id = ChatId::get_for_contact(context, contact_id).await?; + chat::set_muted(context, chat_id, MuteDuration::Forever).await?; + chat_id + }; + + Ok(chat_id) +} + +async fn get_contact_stats(context: &Context, last_old_contact: u32) -> Result> { + let mut verified_by_map: BTreeMap = BTreeMap::new(); + let mut bot_ids: BTreeSet = BTreeSet::new(); + + let mut contacts: Vec = context + .sql + .query_map( + "SELECT id, fingerprint<>'', verifier, last_seen, is_bot FROM contacts c + WHERE id>9 AND origin>? AND addr<>?", + (Origin::Hidden, STATISTICS_BOT_EMAIL), + |row| { + let id = row.get(0)?; + let is_encrypted: bool = row.get(1)?; + let verifier: ContactId = row.get(2)?; + let last_seen: u64 = row.get(3)?; + let bot: bool = row.get(4)?; + + let verified = match (is_encrypted, verifier) { + (true, ContactId::SELF) => VerifiedStatus::Direct, + (true, ContactId::UNDEFINED) => VerifiedStatus::Opportunistic, + (true, _) => VerifiedStatus::Transitive, // TransitiveViaBot will be filled later + (false, _) => VerifiedStatus::Unencrypted, + }; + + if verifier != ContactId::UNDEFINED { + verified_by_map.insert(id, verifier); + } + + if bot { + bot_ids.insert(id); + } + + Ok(ContactStat { + id, + verified, + bot, + direct_chat: false, // will be filled later + last_seen, + transitive_chain: None, // will be filled later + new: id.to_u32() > last_old_contact, + }) + }, + |rows| { + rows.collect::, _>>() + .map_err(Into::into) + }, + ) + .await?; + + // Fill TransitiveViaBot and transitive_chain + for contact in &mut contacts { + if contact.verified == VerifiedStatus::Transitive { + let mut transitive_chain: u32 = 0; + let mut has_bot = false; + let mut current_verifier_id = contact.id; + + while current_verifier_id != ContactId::SELF && transitive_chain < 100 { + current_verifier_id = match verified_by_map.get(¤t_verifier_id) { + Some(id) => *id, + None => { + // The chain ends here, probably because some verification was done + // before we started recording verifiers. + // It's unclear how long the chain really is. + transitive_chain = 0; + break; + } + }; + if bot_ids.contains(¤t_verifier_id) { + has_bot = true; + } + transitive_chain = transitive_chain.saturating_add(1); + } + + if transitive_chain > 0 { + contact.transitive_chain = Some(transitive_chain); + } + + if has_bot { + contact.verified = VerifiedStatus::TransitiveViaBot; + } + } + } + + // Fill direct_chat + for contact in &mut contacts { + let direct_chat = context + .sql + .exists( + "SELECT COUNT(*) + FROM chats_contacts cc INNER JOIN chats + WHERE cc.contact_id=? AND chats.type=?", + (contact.id, Chattype::Single), + ) + .await?; + contact.direct_chat = direct_chat; + } + + Ok(contacts) +} + +/// - `last_msg_id`: The last msg_id that was already counted in the previous stats. +/// Only messages newer than that will be counted. +/// - `one_one_chats`: If true, only messages in 1:1 chats are counted. +/// If false, only messages in other chats (groups and broadcast channels) are counted. +async fn get_message_stats(context: &Context) -> Result> { + let mut map: BTreeMap = context + .sql + .query_map( + "SELECT chattype, verified, unverified_encrypted, unencrypted, only_to_self + FROM stats_msgs", + (), + |row| { + let chattype: Chattype = row.get(0)?; + let verified: u32 = row.get(1)?; + let unverified_encrypted: u32 = row.get(2)?; + let unencrypted: u32 = row.get(3)?; + let only_to_self: u32 = row.get(4)?; + let message_stats = MessageStats { + verified, + unverified_encrypted, + unencrypted, + only_to_self, + }; + Ok((chattype, message_stats)) + }, + |rows| Ok(rows.collect::>>()?), + ) + .await?; + + // Fill zeroes if a chattype wasn't present: + for chattype in [Chattype::Group, Chattype::Single, Chattype::OutBroadcast] { + map.entry(chattype).or_default(); + } + + Ok(map) +} + +pub(crate) async fn update_message_stats(context: &Context) -> Result<()> { + for chattype in [Chattype::Single, Chattype::Group, Chattype::OutBroadcast] { + update_message_stats_inner(context, chattype).await?; + } + context + .set_config_internal(Config::StatsLastUpdate, Some(&time().to_string())) + .await?; + Ok(()) +} + +async fn update_message_stats_inner(context: &Context, chattype: Chattype) -> Result<()> { + let stats_bot_chat_id = get_stats_chat_id(context).await?; + + let trans_fn = |t: &mut rusqlite::Transaction| { + // The ID of the last msg that was already counted in the previously sent stats. + // Only newer messages will be counted in the current statistics. + let last_counted_msg_id: u32 = t + .query_row( + "SELECT last_counted_msg_id FROM stats_msgs WHERE chattype=?", + (chattype,), + |row| row.get(0), + ) + .optional()? + .unwrap_or(0); + t.execute( + "UPDATE stats_msgs + SET last_counted_msg_id=(SELECT MAX(id) FROM msgs) + WHERE chattype=?", + (chattype,), + )?; + + // This table will hold all empty chats, + // i.e. all chats that do not contain any members except for self. + // Messages in these chats are not actually sent out. + t.execute( + "CREATE TEMP TABLE temp.empty_chats ( + id INTEGER PRIMARY KEY + ) STRICT", + (), + )?; + + // id>9 because chat ids 0..9 are "special" chats like the trash chat, + // and contact ids 0..9 are "special" contact ids like the 'device'. + t.execute( + "INSERT INTO temp.empty_chats + SELECT id FROM chats + WHERE id>9 AND NOT EXISTS( + SELECT * + FROM contacts, chats_contacts + WHERE chats_contacts.contact_id=contacts.id AND chats_contacts.chat_id=chats.id + AND contacts.id>9 + )", + (), + )?; + + // This table will hold all verified chats, + // i.e. all chats that only contain verified contacts. + t.execute( + "CREATE TEMP TABLE temp.verified_chats ( + id INTEGER PRIMARY KEY + ) STRICT", + (), + )?; + + // Verified chats are chats that are not empty, + // and do not contain any unverified contacts + t.execute( + "INSERT INTO temp.verified_chats + SELECT id FROM chats + WHERE id>9 + AND id NOT IN (SELECT id FROM temp.empty_chats) + AND NOT EXISTS( + SELECT * + FROM contacts, chats_contacts + WHERE chats_contacts.contact_id=contacts.id AND chats_contacts.chat_id=chats.id + AND contacts.id>9 + AND contacts.verifier=0 + )", + (), + )?; + + // This table will hold all 1:1 chats. + t.execute( + "CREATE TEMP TABLE temp.chat_with_correct_type ( + id INTEGER PRIMARY KEY + ) STRICT", + (), + )?; + + t.execute( + "INSERT INTO temp.chat_with_correct_type + SELECT id FROM chats + WHERE type=?;", + (chattype,), + )?; + + // - `from_id=?` is to count only outgoing messages. + // - `chat_id<>?` excludes the chat with the statistics bot itself, + // - `id>?` excludes messages that were already counted in the previously sent statistics, or messages sent before the config was enabled + // - `hidden=0` excludes hidden system messages, which are not actually shown to the user. + // Note that reactions are also not counted as a message. + // - `chat_id>9` excludes messages in the 'Trash' chat, which is an internal chat assigned to messages that are not shown to the user + let general_requirements = "id>? AND from_id=? AND chat_id<>? + AND hidden=0 AND chat_id>9 AND chat_id IN temp.chat_with_correct_type" + .to_string(); + let params = (last_counted_msg_id, ContactId::SELF, stats_bot_chat_id); + + let verified: u32 = t.query_row( + &format!( + "SELECT COUNT(*) FROM msgs + WHERE chat_id IN temp.verified_chats + AND {general_requirements}" + ), + params, + |row| row.get(0), + )?; + + let unverified_encrypted: u32 = t.query_row( + &format!( + // (param GLOB '*\nc=1*' OR param GLOB 'c=1*') matches all messages that are end-to-end encrypted + "SELECT COUNT(*) FROM msgs + WHERE chat_id NOT IN temp.verified_chats AND chat_id NOT IN temp.empty_chats + AND (param GLOB '*\nc=1*' OR param GLOB 'c=1*') + AND {general_requirements}" + ), + params, + |row| row.get(0), + )?; + + let unencrypted: u32 = t.query_row( + &format!( + "SELECT COUNT(*) FROM msgs + WHERE chat_id NOT IN temp.verified_chats AND chat_id NOT IN temp.empty_chats + AND NOT (param GLOB '*\nc=1*' OR param GLOB 'c=1*') + AND {general_requirements}" + ), + params, + |row| row.get(0), + )?; + + let only_to_self: u32 = t.query_row( + &format!( + "SELECT COUNT(*) FROM msgs + WHERE chat_id IN temp.empty_chats + AND {general_requirements}" + ), + params, + |row| row.get(0), + )?; + + t.execute("DROP TABLE temp.verified_chats", ())?; + t.execute("DROP TABLE temp.empty_chats", ())?; + t.execute("DROP TABLE temp.chat_with_correct_type", ())?; + + t.execute( + "INSERT INTO stats_msgs(chattype) VALUES (?) + ON CONFLICT(chattype) DO NOTHING", + (chattype,), + )?; + t.execute( + "UPDATE stats_msgs SET + verified=verified+?, + unverified_encrypted=unverified_encrypted+?, + unencrypted=unencrypted+?, + only_to_self=only_to_self+? + WHERE chattype=?", + ( + verified, + unverified_encrypted, + unencrypted, + only_to_self, + chattype, + ), + )?; + + Ok(()) + }; + + context.sql.transaction(trans_fn).await?; + + Ok(()) +} + +pub(crate) async fn count_securejoin_ux_info( + context: &Context, + source: Option, + uipath: Option, +) -> Result<()> { + if !should_send_stats(context).await? { + return Ok(()); + } + + let source = source + .context("Missing securejoin source") + .log_err(context) + .unwrap_or(SecurejoinSource::Unknown); + + // We only get a UI path if the source is a QR code scan, + // a loaded image, or a link pasted from the QR code, + // so, no need to log an error if `uipath` is None: + let uipath = uipath.unwrap_or(SecurejoinUiPath::Unknown); + + context + .sql + .transaction(|conn| { + conn.execute( + "INSERT INTO stats_securejoin_sources VALUES (?, 1) + ON CONFLICT (source) DO UPDATE SET count=count+1;", + (source.to_u32(),), + )?; + + conn.execute( + "INSERT INTO stats_securejoin_uipaths VALUES (?, 1) + ON CONFLICT (uipath) DO UPDATE SET count=count+1;", + (uipath.to_u32(),), + )?; + Ok(()) + }) + .await?; + + Ok(()) +} + +async fn get_securejoin_source_stats(context: &Context) -> Result { + let map = context + .sql + .query_map( + "SELECT source, count FROM stats_securejoin_sources", + (), + |row| { + let source: SecurejoinSource = row.get(0)?; + let count: u32 = row.get(1)?; + Ok((source, count)) + }, + |rows| Ok(rows.collect::>>()?), + ) + .await?; + + let stats = SecurejoinSources { + unknown: *map.get(&SecurejoinSource::Unknown).unwrap_or(&0), + external_link: *map.get(&SecurejoinSource::ExternalLink).unwrap_or(&0), + internal_link: *map.get(&SecurejoinSource::InternalLink).unwrap_or(&0), + clipboard: *map.get(&SecurejoinSource::Clipboard).unwrap_or(&0), + image_loaded: *map.get(&SecurejoinSource::ImageLoaded).unwrap_or(&0), + scan: *map.get(&SecurejoinSource::Scan).unwrap_or(&0), + }; + + Ok(stats) +} + +async fn get_securejoin_uipath_stats(context: &Context) -> Result { + let map = context + .sql + .query_map( + "SELECT uipath, count FROM stats_securejoin_uipaths", + (), + |row| { + let uipath: SecurejoinUiPath = row.get(0)?; + let count: u32 = row.get(1)?; + Ok((uipath, count)) + }, + |rows| Ok(rows.collect::>>()?), + ) + .await?; + + let stats = SecurejoinUiPaths { + other: *map.get(&SecurejoinUiPath::Unknown).unwrap_or(&0), + qr_icon: *map.get(&SecurejoinUiPath::QrIcon).unwrap_or(&0), + new_contact: *map.get(&SecurejoinUiPath::NewContact).unwrap_or(&0), + }; + + Ok(stats) +} + +pub(crate) async fn count_securejoin_invite(context: &Context, invite: &QrInvite) -> Result<()> { + if !should_send_stats(context).await? { + return Ok(()); + } + + let contact = Contact::get_by_id(context, invite.contact_id()).await?; + + // If the contact was created just now by the QR code scan, + // (or if a contact existed in the database + // but it was not visible in the contacts list in the UI + // e.g. because it's a past contact of a group we're in), + // then its origin is UnhandledSecurejoinQrScan. + let already_existed = contact.origin > Origin::UnhandledSecurejoinQrScan; + + // Check whether the contact was verified already before the QR scan. + let already_verified = contact.is_verified(context).await?; + + let typ = match invite { + QrInvite::Contact { .. } => "contact", + QrInvite::Group { .. } => "group", + }; + + context + .sql + .execute( + "INSERT INTO stats_securejoin_invites (already_existed, already_verified, type) + VALUES (?, ?, ?)", + (already_existed, already_verified, typ), + ) + .await?; + + Ok(()) +} + +async fn get_securejoin_invite_stats(context: &Context) -> Result> { + let qr_scans: Vec = context + .sql + .query_map( + "SELECT already_existed, already_verified, type FROM stats_securejoin_invites", + (), + |row| { + let already_existed: bool = row.get(0)?; + let already_verified: bool = row.get(1)?; + let typ: String = row.get(2)?; + + Ok(JoinedInvite { + already_existed, + already_verified, + typ, + }) + }, + |rows| { + rows.collect::, _>>() + .map_err(Into::into) + }, + ) + .await?; + + Ok(qr_scans) +} + +#[cfg(test)] +mod stats_tests; diff --git a/src/stats/stats_tests.rs b/src/stats/stats_tests.rs new file mode 100644 index 000000000..ab1ab9c29 --- /dev/null +++ b/src/stats/stats_tests.rs @@ -0,0 +1,595 @@ +use std::time::Duration; + +use super::*; +use crate::chat::{ + Chat, create_broadcast, create_group, create_group_unencrypted, get_chat_contacts, +}; +use crate::mimeparser::SystemMessage; +use crate::qr::check_qr; +use crate::securejoin::{get_securejoin_qr, join_securejoin, join_securejoin_with_ux_info}; +use crate::test_utils::{TestContext, TestContextManager, get_chat_msg}; +use crate::tools::SystemTime; +use pretty_assertions::assert_eq; +use serde_json::{Number, Value}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_maybe_send_stats() -> Result<()> { + let alice = &TestContext::new_alice().await; + + // Can't use `set_config()` here, because this would directly send the statistics, + // and we wouldn't know the chat id + alice + .set_config_internal(Config::StatsSending, Some("1")) + .await?; + + let chat_id = maybe_send_stats(alice).await?.unwrap(); + let msg = get_chat_msg(alice, chat_id, 0, 2).await; + assert_eq!(msg.get_info_type(), SystemMessage::ChatE2ee); + + let chat = Chat::load_from_db(alice, chat_id).await?; + assert!(chat.is_encrypted(alice).await?); + let contacts = get_chat_contacts(alice, chat_id).await?; + assert_eq!(contacts.len(), 1); + let contact = Contact::get_by_id(alice, contacts[0]).await?; + assert!(contact.is_verified(alice).await?); + + let msg = get_chat_msg(alice, chat_id, 1, 2).await; + assert_eq!(msg.get_filename().unwrap(), "statistics.txt"); + + let stats = tokio::fs::read(msg.get_file(alice).unwrap()).await?; + let stats = std::str::from_utf8(&stats)?; + println!("\nEmpty account:\n{stats}\n"); + assert!(stats.contains(r#""contact_stats": []"#)); + + let r: serde_json::Value = serde_json::from_str(stats)?; + assert_eq!( + r.get("contact_stats").unwrap(), + &serde_json::Value::Array(vec![]) + ); + assert_eq!(r.get("core_version").unwrap(), get_version_str()); + + assert_eq!(maybe_send_stats(alice).await?, None); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_rewound_time() -> Result<()> { + let alice = &TestContext::new_alice().await; + alice.set_config_bool(Config::StatsSending, true).await?; + + // Enabling StatsSending directly sends the first statistics, + // so that the user immediately sees the result of enabling it: + assert!(maybe_send_stats(alice).await?.is_none()); + let sent = alice.pop_sent_msg().await; + assert_eq!( + sent.load_from_db().await.get_filename().unwrap(), + "statistics.txt" + ); + + const EIGHT_DAYS: Duration = Duration::from_secs(3600 * 24 * 14); + SystemTime::shift(EIGHT_DAYS); + + maybe_send_stats(alice).await?.unwrap(); + + // The system's time is rewound + SystemTime::shift_back(EIGHT_DAYS); + + assert!(maybe_send_stats(alice).await?.is_none()); + + // After eight days pass again, stats are sent again + SystemTime::shift(EIGHT_DAYS); + maybe_send_stats(alice).await?.unwrap(); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_stats_one_contact() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + alice.set_config_bool(Config::StatsSending, true).await?; + + let stats = get_stats(alice).await?; + let r: serde_json::Value = serde_json::from_str(&stats)?; + + tcm.send_recv_accept(bob, alice, "Hi!").await; + + let stats = get_stats(alice).await?; + println!("\nWith Bob:\n{stats}\n"); + let r2: serde_json::Value = serde_json::from_str(&stats)?; + + assert_eq!( + r.get("key_create_timestamps").unwrap(), + r2.get("key_create_timestamps").unwrap() + ); + assert_eq!(r.get("stats_id").unwrap(), r2.get("stats_id").unwrap()); + let contact_stats = r2.get("contact_stats").unwrap().as_array().unwrap(); + assert_eq!(contact_stats.len(), 1); + let contact_info = &contact_stats[0]; + assert!(contact_info.get("bot").is_none()); + assert_eq!( + contact_info.get("direct_chat").unwrap(), + &serde_json::Value::Bool(true) + ); + assert!(contact_info.get("transitive_chain").is_none(),); + assert_eq!( + contact_info.get("verified").unwrap(), + &serde_json::Value::String("Opportunistic".to_string()) + ); + assert_eq!( + contact_info.get("new").unwrap(), + &serde_json::Value::Bool(true) + ); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_message_stats() -> Result<()> { + #[track_caller] + fn check_stats(stats: &str, expected: &BTreeMap) { + let actual: serde_json::Value = serde_json::from_str(stats).unwrap(); + let actual = &actual["message_stats"]; + + let expected = serde_json::to_string_pretty(&expected).unwrap(); + let expected: serde_json::Value = serde_json::from_str(&expected).unwrap(); + + assert_eq!(actual, &expected); + } + + async fn update_get_stats(context: &Context) -> String { + update_message_stats(context).await.unwrap(); + get_stats(context).await.unwrap() + } + + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + // Can't use `set_config()` here, because this would directly send the statistics + alice + .set_config_internal(Config::StatsSending, Some("1")) + .await?; + let email_chat = alice.create_email_chat(bob).await; + let encrypted_chat = alice.create_chat(bob).await; + + let mut expected: BTreeMap = BTreeMap::from_iter([ + (Chattype::Single, MessageStats::default()), + (Chattype::Group, MessageStats::default()), + (Chattype::OutBroadcast, MessageStats::default()), + ]); + + check_stats(&update_get_stats(alice).await, &expected); + + alice.send_text(email_chat.id, "foo").await; + expected.get_mut(&Chattype::Single).unwrap().unencrypted += 1; + check_stats(&update_get_stats(alice).await, &expected); + + alice.send_text(encrypted_chat.id, "foo").await; + expected + .get_mut(&Chattype::Single) + .unwrap() + .unverified_encrypted += 1; + check_stats(&update_get_stats(alice).await, &expected); + + alice.send_text(encrypted_chat.id, "foo").await; + expected + .get_mut(&Chattype::Single) + .unwrap() + .unverified_encrypted += 1; + check_stats(&update_get_stats(alice).await, &expected); + + let group = alice.create_group_with_members("Pizza", &[bob]).await; + alice.send_text(group, "foo").await; + expected + .get_mut(&Chattype::Group) + .unwrap() + .unverified_encrypted += 1; + check_stats(&update_get_stats(alice).await, &expected); + + tcm.execute_securejoin(alice, bob).await; + check_stats(&update_get_stats(alice).await, &expected); + + alice.send_text(alice.get_self_chat().await.id, "foo").await; + expected.get_mut(&Chattype::Single).unwrap().only_to_self += 1; + check_stats(&update_get_stats(alice).await, &expected); + + let empty_group = create_group(alice, "Notes").await?; + alice.send_text(empty_group, "foo").await; + expected.get_mut(&Chattype::Group).unwrap().only_to_self += 1; + check_stats(&update_get_stats(alice).await, &expected); + + let empty_unencrypted = create_group_unencrypted(alice, "Email thread").await?; + alice.send_text(empty_unencrypted, "foo").await; + expected.get_mut(&Chattype::Group).unwrap().only_to_self += 1; + check_stats(&update_get_stats(alice).await, &expected); + + let group = alice.create_group_with_members("Pizza 2", &[bob]).await; + alice.send_text(group, "foo").await; + expected.get_mut(&Chattype::Group).unwrap().verified += 1; + check_stats(&update_get_stats(alice).await, &expected); + + let empty_broadcast = create_broadcast(alice, "Channel".to_string()).await?; + alice.send_text(empty_broadcast, "foo").await; + expected + .get_mut(&Chattype::OutBroadcast) + .unwrap() + .only_to_self += 1; + check_stats(&update_get_stats(alice).await, &expected); + + // Incoming messages are not counted: + let rcvd = tcm.send_recv(bob, alice, "bar").await; + check_stats(&update_get_stats(alice).await, &expected); + + // Reactions are not counted: + crate::reaction::send_reaction(alice, rcvd.id, "👍") + .await + .unwrap(); + check_stats(&update_get_stats(alice).await, &expected); + + let before_sending = get_stats(alice).await.unwrap(); + + let stats = send_and_read_stats(alice).await; + // The stats are supposed not to have changed yet + assert_eq!(before_sending, stats); + + // Shift by 8 days so that the next stats-sending is due: + SystemTime::shift(Duration::from_secs(8 * 24 * 3600)); + + let stats = send_and_read_stats(alice).await; + assert_eq!(before_sending, stats); + + check_stats(&stats, &expected); + + SystemTime::shift(Duration::from_secs(8 * 24 * 3600)); + tcm.send_recv(alice, bob, "Hi").await; + expected.get_mut(&Chattype::Single).unwrap().verified += 1; + update_message_stats(alice).await?; + update_message_stats(alice).await?; + tcm.send_recv(alice, bob, "Hi").await; + expected.get_mut(&Chattype::Single).unwrap().verified += 1; + tcm.send_recv(alice, bob, "Hi").await; + expected.get_mut(&Chattype::Single).unwrap().verified += 1; + + check_stats(&send_and_read_stats(alice).await, &expected); + + Ok(()) +} + +async fn send_and_read_stats(context: &TestContext) -> String { + let chat_id = maybe_send_stats(context).await.unwrap().unwrap(); + let msg = context.get_last_msg_in(chat_id).await; + assert_eq!(msg.get_filename().unwrap(), "statistics.txt"); + + let stats = tokio::fs::read(msg.get_file(context).unwrap()) + .await + .unwrap(); + String::from_utf8(stats).unwrap() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_stats_securejoin_sources() -> Result<()> { + async fn check_stats(context: &TestContext, expected: &SecurejoinSources) { + let stats = get_stats(context).await.unwrap(); + let actual: serde_json::Value = serde_json::from_str(&stats).unwrap(); + let actual = &actual["securejoin_sources"]; + + let expected = serde_json::to_string_pretty(&expected).unwrap(); + let expected: serde_json::Value = serde_json::from_str(&expected).unwrap(); + + assert_eq!(actual, &expected); + } + + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + alice.set_config_bool(Config::StatsSending, true).await?; + + let mut expected = SecurejoinSources { + unknown: 0, + external_link: 0, + internal_link: 0, + clipboard: 0, + image_loaded: 0, + scan: 0, + }; + + check_stats(alice, &expected).await; + + let qr = get_securejoin_qr(bob, None).await?; + + join_securejoin(alice, &qr).await?; + expected.unknown += 1; + check_stats(alice, &expected).await; + + join_securejoin(alice, &qr).await?; + expected.unknown += 1; + check_stats(alice, &expected).await; + + join_securejoin_with_ux_info(alice, &qr, Some(SecurejoinSource::Clipboard), None).await?; + expected.clipboard += 1; + check_stats(alice, &expected).await; + + join_securejoin_with_ux_info(alice, &qr, Some(SecurejoinSource::ExternalLink), None).await?; + expected.external_link += 1; + check_stats(alice, &expected).await; + + join_securejoin_with_ux_info(alice, &qr, Some(SecurejoinSource::InternalLink), None).await?; + expected.internal_link += 1; + check_stats(alice, &expected).await; + + join_securejoin_with_ux_info(alice, &qr, Some(SecurejoinSource::ImageLoaded), None).await?; + expected.image_loaded += 1; + check_stats(alice, &expected).await; + + join_securejoin_with_ux_info(alice, &qr, Some(SecurejoinSource::Scan), None).await?; + expected.scan += 1; + check_stats(alice, &expected).await; + + join_securejoin_with_ux_info(alice, &qr, Some(SecurejoinSource::Clipboard), None).await?; + expected.clipboard += 1; + check_stats(alice, &expected).await; + + join_securejoin_with_ux_info(alice, &qr, Some(SecurejoinSource::Clipboard), None).await?; + expected.clipboard += 1; + check_stats(alice, &expected).await; + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_stats_securejoin_uipaths() -> Result<()> { + async fn check_stats(context: &TestContext, expected: &SecurejoinUiPaths) { + let stats = get_stats(context).await.unwrap(); + let actual: serde_json::Value = serde_json::from_str(&stats).unwrap(); + let actual = &actual["securejoin_uipaths"]; + + let expected = serde_json::to_string_pretty(&expected).unwrap(); + let expected: serde_json::Value = serde_json::from_str(&expected).unwrap(); + + assert_eq!(actual, &expected); + } + + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + alice.set_config_bool(Config::StatsSending, true).await?; + + let mut expected = SecurejoinUiPaths { + other: 0, + qr_icon: 0, + new_contact: 0, + }; + + check_stats(alice, &expected).await; + + let qr = get_securejoin_qr(bob, None).await?; + + join_securejoin(alice, &qr).await?; + expected.other += 1; + check_stats(alice, &expected).await; + + join_securejoin(alice, &qr).await?; + expected.other += 1; + check_stats(alice, &expected).await; + + join_securejoin_with_ux_info(alice, &qr, None, Some(SecurejoinUiPath::NewContact)).await?; + expected.new_contact += 1; + check_stats(alice, &expected).await; + + join_securejoin_with_ux_info(alice, &qr, None, Some(SecurejoinUiPath::NewContact)).await?; + expected.new_contact += 1; + check_stats(alice, &expected).await; + + join_securejoin_with_ux_info(alice, &qr, None, Some(SecurejoinUiPath::QrIcon)).await?; + expected.qr_icon += 1; + check_stats(alice, &expected).await; + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_stats_securejoin_invites() -> Result<()> { + async fn check_stats(context: &TestContext, expected: &[JoinedInvite]) { + let stats = get_stats(context).await.unwrap(); + let actual: serde_json::Value = serde_json::from_str(&stats).unwrap(); + let actual = &actual["securejoin_invites"]; + + let expected = serde_json::to_string_pretty(&expected).unwrap(); + let expected: serde_json::Value = serde_json::from_str(&expected).unwrap(); + + assert_eq!(actual, &expected); + } + + let mut tcm = TestContextManager::new(); + let alice = &tcm.alice().await; + let bob = &tcm.bob().await; + let charlie = &tcm.charlie().await; + alice.set_config_bool(Config::StatsSending, true).await?; + + let mut expected = vec![]; + + check_stats(alice, &expected).await; + + let qr = get_securejoin_qr(bob, None).await?; + + // The UI will call `check_qr()` first, which must not make the stats wrong: + check_qr(alice, &qr).await?; + tcm.exec_securejoin_qr(alice, bob, &qr).await; + expected.push(JoinedInvite { + already_existed: false, + already_verified: false, + typ: "contact".to_string(), + }); + check_stats(alice, &expected).await; + + check_qr(alice, &qr).await?; + tcm.exec_securejoin_qr(alice, bob, &qr).await; + expected.push(JoinedInvite { + already_existed: true, + already_verified: true, + typ: "contact".to_string(), + }); + check_stats(alice, &expected).await; + + let group_id = create_group(bob, "Group chat").await?; + let qr = get_securejoin_qr(bob, Some(group_id)).await?; + + check_qr(alice, &qr).await?; + tcm.exec_securejoin_qr(alice, bob, &qr).await; + expected.push(JoinedInvite { + already_existed: true, + already_verified: true, + typ: "group".to_string(), + }); + check_stats(alice, &expected).await; + + // A contact with Charlie exists already: + alice.add_or_lookup_contact(charlie).await; + let group_id = create_group(charlie, "Group chat 2").await?; + let qr = get_securejoin_qr(charlie, Some(group_id)).await?; + + check_qr(alice, &qr).await?; + tcm.exec_securejoin_qr(alice, bob, &qr).await; + expected.push(JoinedInvite { + already_existed: true, + already_verified: false, + typ: "group".to_string(), + }); + check_stats(alice, &expected).await; + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_stats_is_chatmail() -> Result<()> { + let alice = &TestContext::new_alice().await; + alice.set_config_bool(Config::StatsSending, true).await?; + + let r = get_stats(alice).await?; + let r: serde_json::Value = serde_json::from_str(&r)?; + assert_eq!(r.get("is_chatmail").unwrap().as_bool().unwrap(), false); + + alice.set_config_bool(Config::IsChatmail, true).await?; + + let r = get_stats(alice).await?; + let r: serde_json::Value = serde_json::from_str(&r)?; + assert_eq!(r.get("is_chatmail").unwrap().as_bool().unwrap(), true); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_stats_key_creation_timestamp() -> Result<()> { + // Alice uses a pregenerated key. It was created at this timestamp: + const ALICE_KEY_CREATION_TIME: u128 = 1582855645; + + let alice = &TestContext::new_alice().await; + alice.set_config_bool(Config::StatsSending, true).await?; + + let r = get_stats(alice).await?; + let r: serde_json::Value = serde_json::from_str(&r)?; + let key_create_timestamps = r.get("key_create_timestamps").unwrap().as_array().unwrap(); + assert_eq!( + key_create_timestamps, + &vec![Value::Number( + Number::from_u128(ALICE_KEY_CREATION_TIME).unwrap() + )] + ); + + Ok(()) +} + +/// We record the timestamp when StatsSending is enabled. +/// If it's disabled and then enabled again, we also record these timestamps. +/// This test enables, disables, and reenables StatsSending, +/// and checks that the timestamps are recorded correctly. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_stats_enable_disable_timestamps() -> Result<()> { + async fn get_timestamps(context: &TestContext) -> (Vec, Vec) { + let stats = get_stats(context).await.unwrap(); + let stats: serde_json::Value = serde_json::from_str(&stats).unwrap(); + let enabled_ts = &stats["sending_enabled_timestamps"]; + let disabled_ts = &stats["sending_disabled_timestamps"]; + + let enabled_ts = enabled_ts + .as_array() + .unwrap() + .iter() + .map(|v| v.as_i64().unwrap()) + .collect(); + let disabled_ts = disabled_ts + .as_array() + .unwrap() + .iter() + .map(|v| v.as_i64().unwrap()) + .collect(); + + (enabled_ts, disabled_ts) + } + + let alice = &TestContext::new_alice().await; + + // ============================== Enable the setting, and check corresponding timestamp ============================== + let enabled_min = time(); + alice.set_config_bool(Config::StatsSending, true).await?; + let enabled_max = time(); + + let (enabled_ts, disabled_ts) = get_timestamps(alice).await; + + // The enabling timestamp was inbetween `enabled_min` and `enabled_max`: + assert_eq!(enabled_ts.len(), 1); + assert!(enabled_ts[0] >= enabled_min); + assert!(enabled_ts[0] <= enabled_max); + + assert!(disabled_ts.is_empty()); + + // Enabling again should not make a difference + alice.set_config_bool(Config::StatsSending, true).await?; + SystemTime::shift(Duration::from_secs(10)); + alice.set_config_bool(Config::StatsSending, true).await?; + assert_eq!( + get_timestamps(alice).await, + (enabled_ts.clone(), disabled_ts.clone()) + ); + + // ============================== Disable the setting, and check corresponding timestamp ============================== + let disabled_min = time(); + alice.set_config_bool(Config::StatsSending, false).await?; + let disabled_max = time(); + + let (new_enabled_ts, new_disabled_ts) = get_timestamps(alice).await; + + assert_eq!(new_enabled_ts, enabled_ts); // The timestamp of enabling didn't change + + // The disabling timestamp was inbetween `disabled_min` and `disabled_max`: + assert_eq!(new_disabled_ts.len(), 1); + assert!(new_disabled_ts[0] >= disabled_min); + assert!(new_disabled_ts[0] <= disabled_max); + + // The time should have advanced in the meantime (because of SystemTime::shift()): + assert_ne!(new_disabled_ts[0], enabled_ts[0]); + + // ============================== Enable the setting again ============================== + SystemTime::shift(Duration::from_secs(10)); + let enabled_min = time(); + alice.set_config_bool(Config::StatsSending, true).await?; + let enabled_max = time(); + + let (newer_enabled_ts, newer_disabled_ts) = get_timestamps(alice).await; + + // The timestamp of disabling didn't change: + assert_eq!(newer_disabled_ts, new_disabled_ts); + + // The enabling timestamp was inbetween `enabled_min` and `enabled_max`: + assert_eq!(newer_enabled_ts.len(), 2); + assert!(newer_enabled_ts[1] >= enabled_min); + assert!(newer_enabled_ts[1] <= enabled_max); + assert_eq!(newer_enabled_ts[0], new_enabled_ts[0]); + + // The time should have advanced in the meantime (because of SystemTime::shift()): + assert_ne!(newer_disabled_ts[0], newer_enabled_ts[1]); + + Ok(()) +} diff --git a/src/stock_str.rs b/src/stock_str.rs index 381e9a81f..89266471a 100644 --- a/src/stock_str.rs +++ b/src/stock_str.rs @@ -431,6 +431,11 @@ https://delta.chat/donate"))] #[strum(props(fallback = "Scan to join channel %1$s"))] SecureJoinBrodcastQRDescription = 201, + + #[strum(props( + fallback = "The attachment contains anonymous usage statistics, which helps us improve Delta Chat. Thank you!" + ))] + StatsMsgBody = 210, } impl StockMessage { @@ -1262,6 +1267,11 @@ pub(crate) async fn unencrypted_email(context: &Context, provider: &str) -> Stri .replace1(provider) } +/// Stock string: `The attachment contains anonymous usage statistics, which helps us improve Delta Chat. Thank you!` +pub(crate) async fn stats_msg_body(context: &Context) -> String { + translated(context, StockMessage::StatsMsgBody).await +} + pub(crate) async fn aeap_explanation_and_link( context: &Context, old_addr: &str,