Files
chatmail-core/src/debug_logging.rs
Hocuri 65a9c4b79b File deduplication (#6332)
When receiving messages, blobs will be deduplicated with the new
function `create_and_deduplicate_from_bytes()`. For sending files, this
adds a new function `set_file_and_deduplicate()` instead of
deduplicating by default.

This is for
https://github.com/deltachat/deltachat-core-rust/issues/6265; read the
issue description there for more details.

TODO:
- [x] Set files as read-only
- [x] Don't do a write when the file is already identical
- [x] The first 32 chars or so of the 64-character hash are enough. I
calculated that if 10b people (i.e. all of humanity) use DC, and each of
them has 200k distinct blob files (I have 4k in my day-to-day account),
and we used 20 chars, then the expected value for the number of name
collisions would be ~0.0002 (and the probability that there is a least
one name collision is lower than that) [^1]. I added 12 more characters
to be on the super safe side, but this wouldn't be necessary and I could
also make it 20 instead of 32.
- Not 100% sure whether that's necessary at all - it would mainly be
necessary if we might hit a length limit on some file systems (the
blobdir is usually sth like
`accounts/2ff9fc096d2f46b6832b24a1ed99c0d6/dc.db-blobs` (53 chars), plus
64 chars for the filename would be 117).
- [x] "touch" the files to prevent them from being deleted
- [x] TODOs in the code

For later PRs:
- Replace `BlobObject::create(…)` with
`BlobObject::create_and_deduplicate(…)` in order to deduplicate
everytime core creates a file
- Modify JsonRPC to deduplicate blob files
- Possibly rename BlobObject.name to BlobObject.file in order to prevent
confusion (because `name` usually means "user-visible-name", not "name
of the file on disk").

[^1]: Calculated with both https://printfn.github.io/fend/ and
https://www.geogebra.org/calculator, both of which came to the same
result
([1](https://github.com/user-attachments/assets/bbb62550-3781-48b5-88b1-ba0e29c28c0d),

[2](https://github.com/user-attachments/assets/82171212-b797-4117-a39f-0e132eac7252))

---------

Co-authored-by: l <link2xt@testrun.org>
2025-01-21 19:42:19 +01:00

182 lines
5.7 KiB
Rust

//! Forward log messages to logging webxdc
use crate::chat::ChatId;
use crate::config::Config;
use crate::context::Context;
use crate::events::EventType;
use crate::message::{Message, MsgId, Viewtype};
use crate::param::Param;
use crate::tools::time;
use crate::webxdc::StatusUpdateItem;
use async_channel::{self as channel, Receiver, Sender};
use serde_json::json;
use std::path::PathBuf;
use tokio::task;
#[derive(Debug)]
pub(crate) struct DebugLogging {
/// The message containing the logging xdc
pub(crate) msg_id: MsgId,
/// Handle to the background task responsible for sending
pub(crate) loop_handle: task::JoinHandle<()>,
/// Channel that log events should be sent to.
/// A background loop will receive and handle them.
pub(crate) sender: Sender<DebugEventLogData>,
}
impl DebugLogging {
pub(crate) fn log_event(&self, event: EventType) {
let event_data = DebugEventLogData {
time: time(),
msg_id: self.msg_id,
event,
};
self.sender.try_send(event_data).ok();
}
}
/// Store all information needed to log an event to a webxdc.
pub struct DebugEventLogData {
pub time: i64,
pub msg_id: MsgId,
pub event: EventType,
}
/// Creates a loop which forwards all log messages send into the channel to the associated
/// logging xdc.
pub async fn debug_logging_loop(context: &Context, events: Receiver<DebugEventLogData>) {
while let Ok(DebugEventLogData {
time,
msg_id,
event,
}) = events.recv().await
{
match context
.write_status_update_inner(
&msg_id,
&StatusUpdateItem {
payload: json!({
"event": event,
"time": time,
}),
info: None,
href: None,
summary: None,
document: None,
uid: None,
notify: None,
},
time,
)
.await
{
Err(err) => {
eprintln!("Can't log event to webxdc status update: {err:#}");
}
Ok(serial) => {
if let Some(serial) = serial {
if !matches!(event, EventType::WebxdcStatusUpdate { .. }) {
context.emit_event(EventType::WebxdcStatusUpdate {
msg_id,
status_update_serial: serial,
});
}
} else {
// This should not happen as the update has no `uid`.
error!(context, "Debug logging update is not created.");
};
}
}
}
}
/// Set message as new logging webxdc if filename and chat_id fit
pub async fn maybe_set_logging_xdc(
context: &Context,
msg: &Message,
chat_id: ChatId,
) -> anyhow::Result<()> {
maybe_set_logging_xdc_inner(
context,
msg.get_viewtype(),
chat_id,
msg.param
.get_path(Param::Filename, context)
.unwrap_or_default(),
msg.get_id(),
)
.await?;
Ok(())
}
/// Set message as new logging webxdc if filename and chat_id fit
pub async fn maybe_set_logging_xdc_inner(
context: &Context,
viewtype: Viewtype,
chat_id: ChatId,
filename: Option<PathBuf>,
msg_id: MsgId,
) -> anyhow::Result<()> {
if viewtype == Viewtype::Webxdc {
if let Some(file) = filename {
if let Some(file_name) = file.file_name().and_then(|name| name.to_str()) {
if file_name.starts_with("debug_logging")
&& file_name.ends_with(".xdc")
&& chat_id.is_self_talk(context).await?
{
set_debug_logging_xdc(context, Some(msg_id)).await?;
}
}
}
}
Ok(())
}
/// Set the webxdc contained in the msg as the current logging xdc on the context and save it to db
/// If id is a `None` value, disable debug logging
pub(crate) async fn set_debug_logging_xdc(ctx: &Context, id: Option<MsgId>) -> anyhow::Result<()> {
match id {
Some(msg_id) => {
ctx.sql
.set_raw_config(
Config::DebugLogging.as_ref(),
Some(msg_id.to_string().as_ref()),
)
.await?;
{
let debug_logging = &mut *ctx.debug_logging.write().expect("RwLock is poisoned");
match debug_logging {
// Switch logging xdc
Some(debug_logging) => debug_logging.msg_id = msg_id,
// Bootstrap background loop for message forwarding
None => {
let (sender, debug_logging_recv) = channel::bounded(1000);
let loop_handle = {
let ctx = ctx.clone();
task::spawn(async move {
debug_logging_loop(&ctx, debug_logging_recv).await
})
};
*debug_logging = Some(DebugLogging {
msg_id,
loop_handle,
sender,
});
}
}
}
info!(ctx, "replacing logging webxdc");
}
// Delete current debug logging
None => {
ctx.sql
.set_raw_config(Config::DebugLogging.as_ref(), None)
.await?;
*ctx.debug_logging.write().expect("RwLock is poisoned") = None;
info!(ctx, "removing logging webxdc");
}
}
Ok(())
}