mirror of
https://github.com/chatmail/core.git
synced 2026-04-24 08:56:29 +03:00
When receiving messages, blobs will be deduplicated with the new function `create_and_deduplicate_from_bytes()`. For sending files, this adds a new function `set_file_and_deduplicate()` instead of deduplicating by default. This is for https://github.com/deltachat/deltachat-core-rust/issues/6265; read the issue description there for more details. TODO: - [x] Set files as read-only - [x] Don't do a write when the file is already identical - [x] The first 32 chars or so of the 64-character hash are enough. I calculated that if 10b people (i.e. all of humanity) use DC, and each of them has 200k distinct blob files (I have 4k in my day-to-day account), and we used 20 chars, then the expected value for the number of name collisions would be ~0.0002 (and the probability that there is a least one name collision is lower than that) [^1]. I added 12 more characters to be on the super safe side, but this wouldn't be necessary and I could also make it 20 instead of 32. - Not 100% sure whether that's necessary at all - it would mainly be necessary if we might hit a length limit on some file systems (the blobdir is usually sth like `accounts/2ff9fc096d2f46b6832b24a1ed99c0d6/dc.db-blobs` (53 chars), plus 64 chars for the filename would be 117). - [x] "touch" the files to prevent them from being deleted - [x] TODOs in the code For later PRs: - Replace `BlobObject::create(…)` with `BlobObject::create_and_deduplicate(…)` in order to deduplicate everytime core creates a file - Modify JsonRPC to deduplicate blob files - Possibly rename BlobObject.name to BlobObject.file in order to prevent confusion (because `name` usually means "user-visible-name", not "name of the file on disk"). [^1]: Calculated with both https://printfn.github.io/fend/ and https://www.geogebra.org/calculator, both of which came to the same result ([1](https://github.com/user-attachments/assets/bbb62550-3781-48b5-88b1-ba0e29c28c0d), [2](https://github.com/user-attachments/assets/82171212-b797-4117-a39f-0e132eac7252)) --------- Co-authored-by: l <link2xt@testrun.org>
182 lines
5.7 KiB
Rust
182 lines
5.7 KiB
Rust
//! Forward log messages to logging webxdc
|
|
use crate::chat::ChatId;
|
|
use crate::config::Config;
|
|
use crate::context::Context;
|
|
use crate::events::EventType;
|
|
use crate::message::{Message, MsgId, Viewtype};
|
|
use crate::param::Param;
|
|
use crate::tools::time;
|
|
use crate::webxdc::StatusUpdateItem;
|
|
use async_channel::{self as channel, Receiver, Sender};
|
|
use serde_json::json;
|
|
use std::path::PathBuf;
|
|
use tokio::task;
|
|
|
|
#[derive(Debug)]
|
|
pub(crate) struct DebugLogging {
|
|
/// The message containing the logging xdc
|
|
pub(crate) msg_id: MsgId,
|
|
/// Handle to the background task responsible for sending
|
|
pub(crate) loop_handle: task::JoinHandle<()>,
|
|
/// Channel that log events should be sent to.
|
|
/// A background loop will receive and handle them.
|
|
pub(crate) sender: Sender<DebugEventLogData>,
|
|
}
|
|
|
|
impl DebugLogging {
|
|
pub(crate) fn log_event(&self, event: EventType) {
|
|
let event_data = DebugEventLogData {
|
|
time: time(),
|
|
msg_id: self.msg_id,
|
|
event,
|
|
};
|
|
|
|
self.sender.try_send(event_data).ok();
|
|
}
|
|
}
|
|
|
|
/// Store all information needed to log an event to a webxdc.
|
|
pub struct DebugEventLogData {
|
|
pub time: i64,
|
|
pub msg_id: MsgId,
|
|
pub event: EventType,
|
|
}
|
|
|
|
/// Creates a loop which forwards all log messages send into the channel to the associated
|
|
/// logging xdc.
|
|
pub async fn debug_logging_loop(context: &Context, events: Receiver<DebugEventLogData>) {
|
|
while let Ok(DebugEventLogData {
|
|
time,
|
|
msg_id,
|
|
event,
|
|
}) = events.recv().await
|
|
{
|
|
match context
|
|
.write_status_update_inner(
|
|
&msg_id,
|
|
&StatusUpdateItem {
|
|
payload: json!({
|
|
"event": event,
|
|
"time": time,
|
|
}),
|
|
info: None,
|
|
href: None,
|
|
summary: None,
|
|
document: None,
|
|
uid: None,
|
|
notify: None,
|
|
},
|
|
time,
|
|
)
|
|
.await
|
|
{
|
|
Err(err) => {
|
|
eprintln!("Can't log event to webxdc status update: {err:#}");
|
|
}
|
|
Ok(serial) => {
|
|
if let Some(serial) = serial {
|
|
if !matches!(event, EventType::WebxdcStatusUpdate { .. }) {
|
|
context.emit_event(EventType::WebxdcStatusUpdate {
|
|
msg_id,
|
|
status_update_serial: serial,
|
|
});
|
|
}
|
|
} else {
|
|
// This should not happen as the update has no `uid`.
|
|
error!(context, "Debug logging update is not created.");
|
|
};
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Set message as new logging webxdc if filename and chat_id fit
|
|
pub async fn maybe_set_logging_xdc(
|
|
context: &Context,
|
|
msg: &Message,
|
|
chat_id: ChatId,
|
|
) -> anyhow::Result<()> {
|
|
maybe_set_logging_xdc_inner(
|
|
context,
|
|
msg.get_viewtype(),
|
|
chat_id,
|
|
msg.param
|
|
.get_path(Param::Filename, context)
|
|
.unwrap_or_default(),
|
|
msg.get_id(),
|
|
)
|
|
.await?;
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Set message as new logging webxdc if filename and chat_id fit
|
|
pub async fn maybe_set_logging_xdc_inner(
|
|
context: &Context,
|
|
viewtype: Viewtype,
|
|
chat_id: ChatId,
|
|
filename: Option<PathBuf>,
|
|
msg_id: MsgId,
|
|
) -> anyhow::Result<()> {
|
|
if viewtype == Viewtype::Webxdc {
|
|
if let Some(file) = filename {
|
|
if let Some(file_name) = file.file_name().and_then(|name| name.to_str()) {
|
|
if file_name.starts_with("debug_logging")
|
|
&& file_name.ends_with(".xdc")
|
|
&& chat_id.is_self_talk(context).await?
|
|
{
|
|
set_debug_logging_xdc(context, Some(msg_id)).await?;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Set the webxdc contained in the msg as the current logging xdc on the context and save it to db
|
|
/// If id is a `None` value, disable debug logging
|
|
pub(crate) async fn set_debug_logging_xdc(ctx: &Context, id: Option<MsgId>) -> anyhow::Result<()> {
|
|
match id {
|
|
Some(msg_id) => {
|
|
ctx.sql
|
|
.set_raw_config(
|
|
Config::DebugLogging.as_ref(),
|
|
Some(msg_id.to_string().as_ref()),
|
|
)
|
|
.await?;
|
|
{
|
|
let debug_logging = &mut *ctx.debug_logging.write().expect("RwLock is poisoned");
|
|
match debug_logging {
|
|
// Switch logging xdc
|
|
Some(debug_logging) => debug_logging.msg_id = msg_id,
|
|
// Bootstrap background loop for message forwarding
|
|
None => {
|
|
let (sender, debug_logging_recv) = channel::bounded(1000);
|
|
let loop_handle = {
|
|
let ctx = ctx.clone();
|
|
task::spawn(async move {
|
|
debug_logging_loop(&ctx, debug_logging_recv).await
|
|
})
|
|
};
|
|
*debug_logging = Some(DebugLogging {
|
|
msg_id,
|
|
loop_handle,
|
|
sender,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
info!(ctx, "replacing logging webxdc");
|
|
}
|
|
// Delete current debug logging
|
|
None => {
|
|
ctx.sql
|
|
.set_raw_config(Config::DebugLogging.as_ref(), None)
|
|
.await?;
|
|
*ctx.debug_logging.write().expect("RwLock is poisoned") = None;
|
|
info!(ctx, "removing logging webxdc");
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|