Compare commits

..

2 Commits

Author SHA1 Message Date
Simon Laux
eb89acb70a fix exports variable
(nodejs/npm said it must start with `./`)
2024-04-30 01:36:22 +02:00
link2xt
b607f12b0e build(node): migrate from CommonJS to ESM modules 2024-04-27 06:36:10 +00:00
38 changed files with 284 additions and 244 deletions

1
Cargo.lock generated
View File

@@ -1214,6 +1214,7 @@ dependencies = [
"reqwest",
"rusqlite",
"rust-hsluv",
"sanitize-filename",
"serde",
"serde_json",
"sha-1",

View File

@@ -82,6 +82,7 @@ regex = { workspace = true }
reqwest = { version = "0.12.2", features = ["json"] }
rusqlite = { workspace = true, features = ["sqlcipher"] }
rust-hsluv = "0.1"
sanitize-filename = "0.5"
serde_json = "1"
serde = { version = "1.0", features = ["derive"] }
sha-1 = "0.10"

View File

@@ -4094,8 +4094,7 @@ char* dc_msg_get_subject (const dc_msg_t* msg);
*
* Typically files are associated with images, videos, audios, documents.
* Plain text messages do not have a file.
* The filename isn't meaningful, only the extension is preserved. To obtain the original attachment
* filename use dc_msg_get_filename().
* File name may be mangled. To obtain the original attachment filename use dc_msg_get_filename().
*
* @memberof dc_msg_t
* @param msg The message object.

View File

@@ -1,9 +1,13 @@
import { join } from 'path'
import * as url from 'url'
/**
* bindings are not typed yet.
* if the available function names are required they can be found inside of `../src/module.c`
*/
export const bindings: any = require('node-gyp-build')(join(__dirname, '../'))
import build from 'node-gyp-build'
export const bindings: any = build(
join(url.fileURLToPath(new URL('.', import.meta.url)), '../')
)
export default bindings

View File

@@ -1,11 +1,11 @@
/* eslint-disable camelcase */
import binding from './binding'
import binding from './binding.js'
import rawDebug from 'debug'
const debug = rawDebug('deltachat:node:chat')
import { C } from './constants'
import { integerToHexColor } from './util'
import { ChatJSON } from './types'
import { C } from './constants.js'
import { integerToHexColor } from './util.js'
import { ChatJSON } from './types.js'
interface NativeChat {}
/**

View File

@@ -1,9 +1,10 @@
/* eslint-disable camelcase */
import binding from './binding'
import { Lot } from './lot'
import { Chat } from './chat'
const debug = require('debug')('deltachat:node:chatlist')
import binding from './binding.js'
import { Lot } from './lot.js'
import { Chat } from './chat.js'
import rawDebug from 'debug'
const debug = rawDebug('deltachat:node:chatlist')
interface NativeChatList {}
/**

View File

@@ -1,9 +1,10 @@
import { integerToHexColor } from './util'
import { integerToHexColor } from './util.js'
/* eslint-disable camelcase */
import binding from './binding'
const debug = require('debug')('deltachat:node:contact')
import binding from './binding.js'
import rawDebug from 'debug'
const debug = rawDebug('deltachat:node:contact')
interface NativeContact {}
/**

View File

@@ -1,17 +1,17 @@
/* eslint-disable camelcase */
import binding from './binding'
import { C, EventId2EventName } from './constants'
import { Chat } from './chat'
import { ChatList } from './chatlist'
import { Contact } from './contact'
import { Message } from './message'
import { Lot } from './lot'
import { Locations } from './locations'
import binding from './binding.js'
import { C, EventId2EventName } from './constants.js'
import { Chat } from './chat.js'
import { ChatList } from './chatlist.js'
import { Contact } from './contact.js'
import { Message } from './message.js'
import { Lot } from './lot.js'
import { Locations } from './locations.js'
import rawDebug from 'debug'
import { AccountManager } from './deltachat'
import { AccountManager } from './deltachat.js'
import { join } from 'path'
import { EventEmitter } from 'stream'
import { EventEmitter } from 'events'
const debug = rawDebug('deltachat:node:index')
const noop = function () {}

View File

@@ -1,13 +1,13 @@
/* eslint-disable camelcase */
import binding from './binding'
import { EventId2EventName } from './constants'
import binding from './binding.js'
import { EventId2EventName } from './constants.js'
import { EventEmitter } from 'events'
import { existsSync } from 'fs'
import rawDebug from 'debug'
import { tmpdir } from 'os'
import { join } from 'path'
import { Context } from './context'
import { Context } from './context.js'
const debug = rawDebug('deltachat:node:index')
const noop = function () {}

View File

@@ -1,20 +1,20 @@
import { AccountManager } from './deltachat'
import { AccountManager } from './deltachat.js'
export default AccountManager
export { Context } from './context'
export { Chat } from './chat'
export { ChatList } from './chatlist'
export { C } from './constants'
export { Contact } from './contact'
export { Context } from './context.js'
export { Chat } from './chat.js'
export { ChatList } from './chatlist.js'
export { C } from './constants.js'
export { Contact } from './contact.js'
export { AccountManager as DeltaChat }
export { Locations } from './locations'
export { Lot } from './lot'
export { Locations } from './locations.js'
export { Lot } from './lot.js'
export {
Message,
MessageState,
MessageViewType,
MessageDownloadState,
} from './message'
} from './message.js'
export * from './types'
export * from './types.js'

View File

@@ -1,7 +1,8 @@
/* eslint-disable camelcase */
const binding = require('../binding')
const debug = require('debug')('deltachat:node:locations')
import binding from './binding.js'
import rawDebug from 'debug'
const debug = rawDebug('deltachat:node:locations')
interface NativeLocations {}
/**

View File

@@ -1,7 +1,8 @@
/* eslint-disable camelcase */
const binding = require('../binding')
const debug = require('debug')('deltachat:node:lot')
import binding from './binding.js'
import rawDebug from 'debug'
const debug = rawDebug('deltachat:node:lot')
interface NativeLot {}
/**

View File

@@ -1,11 +1,12 @@
/* eslint-disable camelcase */
import binding from './binding'
import { C } from './constants'
import { Lot } from './lot'
import { Chat } from './chat'
import { WebxdcInfo } from './context'
const debug = require('debug')('deltachat:node:message')
import binding from './binding.js'
import { C } from './constants.js'
import { Lot } from './lot.js'
import { Chat } from './chat.js'
import { WebxdcInfo } from './context.js'
import rawDebug from 'debug'
const debug = rawDebug('deltachat:node:message')
export enum MessageDownloadState {
Available = C.DC_DOWNLOAD_AVAILABLE,

1
node/lib/node-gyp-build.d.ts vendored Normal file
View File

@@ -0,0 +1 @@
declare module 'node-gyp-build'

View File

@@ -1,4 +1,4 @@
import { C } from './constants'
import { C } from './constants.js'
export type ChatTypes =
| C.DC_CHAT_TYPE_GROUP

View File

@@ -1,4 +1,4 @@
const spawnSync = require('child_process').spawnSync
import { spawnSync } from 'child_process'
const verbose = isVerbose()
@@ -23,4 +23,4 @@ function isVerbose () {
return loglevel === 'verbose' || process.env.CI === 'true'
}
module.exports = { spawn, log, isVerbose, verbose }
export { spawn, log, isVerbose, verbose }

View File

@@ -1,14 +1,16 @@
#!/usr/bin/env node
const fs = require('fs')
const path = require('path')
import fs from 'fs'
import path from 'path'
import * as url from 'url'
const data = []
const header = path.resolve(__dirname, '../../deltachat-ffi/deltachat.h')
const header = path.resolve(url.fileURLToPath(new URL('.', import.meta.url)), '../../deltachat-ffi/deltachat.h')
console.log('Generating constants...')
const header_data = fs.readFileSync(header, 'UTF-8')
const regex = /^#define\s+(\w+)\s+(\w+)/gm
var match
while (null != (match = regex.exec(header_data))) {
const key = match[1]
const value = parseInt(match[2])
@@ -17,8 +19,6 @@ while (null != (match = regex.exec(header_data))) {
}
}
delete header_data
const constants = data
.filter(
({ key }) => key.toUpperCase()[0] === key[0] // check if define name is uppercase
@@ -49,17 +49,17 @@ const events = data
// backwards compat
fs.writeFileSync(
path.resolve(__dirname, '../constants.js'),
path.resolve(url.fileURLToPath(new URL('.', import.meta.url)), '../constants.js'),
`// Generated!\n\nmodule.exports = {\n${constants}\n}\n`
)
// backwards compat
fs.writeFileSync(
path.resolve(__dirname, '../events.js'),
path.resolve(url.fileURLToPath(new URL('.', import.meta.url)), '../events.js'),
`/* eslint-disable quotes */\n// Generated!\n\nmodule.exports = {\n${events}\n}\n`
)
fs.writeFileSync(
path.resolve(__dirname, '../lib/constants.ts'),
path.resolve(url.fileURLToPath(new URL('.', import.meta.url)), '../lib/constants.js'),
`// Generated!\n\nexport enum C {\n${constants.replace(/:/g, ' =')},\n}\n
// Generated!\n\nexport const EventId2EventName: { [key: number]: string } = {\n${events},\n}\n`
)

View File

@@ -1,6 +1,7 @@
const {execSync} = require('child_process')
const {existsSync} = require('fs')
const {join} = require('path')
import {execSync} from 'child_process'
import {existsSync} from 'fs'
import {join} from 'path'
import * as url from 'url'
const run = (cmd) => {
console.log('[i] running `' + cmd + '`')
@@ -16,7 +17,7 @@ if (process.env.USE_SYSTEM_LIBDELTACHAT === 'true') {
run('npm run install:prebuilds')
}
if (!existsSync(join(__dirname, '..', 'dist'))) {
if (!existsSync(join(url.fileURLToPath(new URL('.', import.meta.url)), '..', 'dist'))) {
console.log('[i] Didn\'t find already built typescript bindings. Trying to transpile them. If this fail, make sure typescript is installed ;)')
run('npm run build:bindings:ts')
}

View File

@@ -1,4 +1,5 @@
const { readFileSync } = require('fs')
import { readFileSync } from 'fs'
import { request } from 'https'
const sha = JSON.parse(
readFileSync(process.env['GITHUB_EVENT_PATH'], 'utf8')
@@ -21,8 +22,6 @@ const STATUS_DATA = {
target_url: base_url + file_url,
}
const http = require('https')
const options = {
method: 'POST',
headers: {
@@ -32,7 +31,7 @@ const options = {
},
}
const req = http.request(GITHUB_API_URL, options, function(res) {
const req = request(GITHUB_API_URL, options, function (res) {
var chunks = []
res.on('data', function(chunk) {
chunks.push(chunk)

View File

@@ -1,5 +1,6 @@
const fs = require('fs')
const path = require('path')
import fs from 'fs'
import path from 'path'
import * as url from 'url'
if (process.platform !== 'win32') {
console.log('postinstall: not windows, so skipping!')
@@ -7,7 +8,7 @@ if (process.platform !== 'win32') {
}
const from = path.resolve(
__dirname,
url.fileURLToPath(new URL('.', import.meta.url)),
'..',
'..',
'target',
@@ -19,7 +20,7 @@ const getDestination = () => {
const argv = process.argv
if (argv.length === 3 && argv[2] === '--prebuild') {
return path.resolve(
__dirname,
url.fileURLToPath(new URL('.', import.meta.url)),
'..',
'prebuilds',
'win32-x64',
@@ -27,7 +28,7 @@ const getDestination = () => {
)
} else {
return path.resolve(
__dirname,
url.fileURLToPath(new URL('.', import.meta.url)),
'..',
'build',
'Release',

View File

@@ -1,7 +1,8 @@
const path = require('path')
const { spawn } = require('./common')
import path from 'path'
import { spawn } from './common.js'
import * as url from 'url'
const opts = {
cwd: path.resolve(__dirname, '../..'),
cwd: path.resolve(url.fileURLToPath(new URL('.', import.meta.url)), '../..'),
stdio: 'inherit'
}

View File

@@ -444,6 +444,7 @@ describe('Offline Tests with unconfigured account', function () {
context.setChatProfileImage(chatId, imagePath)
const blobPath = context.getChat(chatId).getProfileImage()
expect(blobPath.startsWith(blobs)).to.be.true
expect(blobPath.includes('image')).to.be.true
expect(blobPath.endsWith('.jpeg')).to.be.true
context.setChatProfileImage(chatId, null)

View File

@@ -3,7 +3,8 @@
"outDir": "dist",
"rootDir": "./lib",
"sourceMap": true,
"module": "commonjs",
"module": "nodenext",
"moduleResolution": "nodenext",
"target": "es5",
"esModuleInterop": true,
"declaration": true,

View File

@@ -1,4 +1,5 @@
{
"type": "module",
"dependencies": {
"debug": "^4.1.1",
"napi-macros": "^2.0.0",
@@ -10,7 +11,7 @@
"@types/node": "^20.8.10",
"chai": "~4.3.10",
"chai-as-promised": "^7.1.1",
"mocha": "^8.2.1",
"mocha": "^10.2.0",
"node-gyp": "^10.0.0",
"prebuildify": "^5.0.1",
"prebuildify-ci": "^1.0.5",
@@ -27,7 +28,7 @@
],
"homepage": "https://github.com/deltachat/deltachat-core-rust/tree/master/node",
"license": "GPL-3.0-or-later",
"main": "node/dist/index.js",
"exports": "./node/dist/index.js",
"name": "deltachat-node",
"repository": {
"type": "git",

View File

@@ -108,7 +108,7 @@ class Message:
@props.with_doc
def filename(self):
"""file path if there was an attachment, otherwise empty string."""
"""filename if there was an attachment, otherwise empty string."""
return from_dc_charpointer(lib.dc_msg_get_file(self._dc_msg))
def set_file(self, path, mime_type=None):
@@ -121,6 +121,7 @@ class Message:
@props.with_doc
def basename(self) -> str:
"""basename of the attachment if it exists, otherwise empty string."""
# FIXME, it does not return basename
return from_dc_charpointer(lib.dc_msg_get_filename(self._dc_msg))
@props.with_doc

View File

@@ -181,11 +181,13 @@ def test_send_file_twice_unicode_filename_mangling(tmp_path, acfactory, lp):
msg = send_and_receive_message()
assert msg.text == "withfile"
assert open(msg.filename).read() == "some data"
msg.filename.index(basename)
assert msg.filename.endswith(ext)
msg2 = send_and_receive_message()
assert msg2.text == "withfile"
assert open(msg2.filename).read() == "some data"
msg2.filename.index(basename)
assert msg2.filename.endswith(ext)
assert msg.filename != msg2.filename
@@ -212,6 +214,7 @@ def test_send_file_html_attachment(tmp_path, acfactory, lp):
msg = ac2.get_message_by_id(ev.data2)
assert open(msg.filename).read() == content
msg.filename.index(basename)
assert msg.filename.endswith(ext)

View File

@@ -50,6 +50,7 @@ class TestOnlineInCreation:
src = tmp_path / "file.txt"
src.write_text("hello there\n")
msg = chat.send_file(str(src))
assert msg.filename.startswith(os.path.join(ac1.get_blobdir(), "file"))
assert msg.filename.endswith(".txt")
def test_forward_increation(self, acfactory, data, lp):

View File

@@ -444,30 +444,27 @@ class TestOfflineChat:
assert msg.filemime == "image/png"
@pytest.mark.parametrize(
("stem", "ext", "typein", "typeout"),
("fn", "typein", "typeout"),
[
("r", "", None, "application/octet-stream"),
("r", ".txt", None, "text/plain"),
("r", ".txt", "text/plain", "text/plain"),
("r", ".txt", "image/png", "image/png"),
("r", None, "application/octet-stream"),
("r.txt", None, "text/plain"),
("r.txt", "text/plain", "text/plain"),
("r.txt", "image/png", "image/png"),
],
)
def test_message_file(self, chat1, data, lp, stem, ext, typein, typeout):
def test_message_file(self, chat1, data, lp, fn, typein, typeout):
lp.sec("sending file")
fn = stem + ext
fp = data.get_path(fn)
msg = chat1.send_file(fp, typein)
assert msg
assert msg.id > 0
assert msg.is_file()
assert os.path.exists(msg.filename)
assert msg.filename.endswith(ext)
assert msg.basename == fn
assert msg.filename.endswith(msg.basename)
assert msg.filemime == typeout
msg2 = chat1.send_file(fp, typein)
assert msg2 != msg
assert msg2.filename != msg.filename
assert msg2.basename == fn
def test_create_contact(self, acfactory):
ac1 = acfactory.get_pseudo_configured_account()

View File

@@ -19,7 +19,7 @@ use tokio::{fs, io};
use tokio_stream::wrappers::ReadDirStream;
use crate::config::Config;
use crate::constants::{self, MediaQuality, BLOB_CREATE_ATTEMPTS};
use crate::constants::{self, MediaQuality};
use crate::context::Context;
use crate::events::EventType;
use crate::log::LogExt;
@@ -56,8 +56,8 @@ impl<'a> BlobObject<'a> {
data: &[u8],
) -> Result<BlobObject<'a>> {
let blobdir = context.get_blobdir();
let ext = BlobObject::get_extension(suggested_name);
let (name, mut file) = BlobObject::create_new_file(context, blobdir, &ext).await?;
let (stem, ext) = BlobObject::sanitise_name(suggested_name);
let (name, mut file) = BlobObject::create_new_file(context, blobdir, &stem, &ext).await?;
file.write_all(data).await.context("file write failure")?;
// workaround a bug in async-std
@@ -77,11 +77,13 @@ impl<'a> BlobObject<'a> {
async fn create_new_file(
context: &Context,
dir: &Path,
stem: &str,
ext: &str,
) -> Result<(String, fs::File)> {
const MAX_ATTEMPT: u32 = 16;
let mut attempt = 0;
let mut name = format!("{stem}{ext}");
loop {
let name = format!("{:016x}{}", rand::random::<u64>(), ext);
attempt += 1;
let path = dir.join(&name);
match fs::OpenOptions::new()
@@ -92,10 +94,12 @@ impl<'a> BlobObject<'a> {
{
Ok(file) => return Ok((name, file)),
Err(err) => {
if attempt >= BLOB_CREATE_ATTEMPTS {
if attempt >= MAX_ATTEMPT {
return Err(err).context("failed to create file");
} else if attempt == 1 && !dir.exists() {
fs::create_dir_all(dir).await.log_err(context).ok();
} else {
name = format!("{}-{}{}", stem, rand::random::<u32>(), ext);
}
}
}
@@ -112,9 +116,9 @@ impl<'a> BlobObject<'a> {
let mut src_file = fs::File::open(src)
.await
.with_context(|| format!("failed to open file {}", src.display()))?;
let ext = BlobObject::get_extension(&src.to_string_lossy());
let (stem, ext) = BlobObject::sanitise_name(&src.to_string_lossy());
let (name, mut dst_file) =
BlobObject::create_new_file(context, context.get_blobdir(), &ext).await?;
BlobObject::create_new_file(context, context.get_blobdir(), &stem, &ext).await?;
let name_for_err = name.clone();
if let Err(err) = io::copy(&mut src_file, &mut dst_file).await {
// Attempt to remove the failed file, swallow errors resulting from that.
@@ -138,8 +142,10 @@ impl<'a> BlobObject<'a> {
///
/// If the source file is not a path to into the blob directory
/// the file will be copied into the blob directory first. If the
/// source file is already in the blobdir (but not in a subdirectory)
/// it will not be copied and only be created if it is a valid blobname.
/// source file is already in the blobdir it will not be copied
/// and only be created if it is a valid blobname, that is no
/// subdirectory is used and [BlobObject::sanitise_name] does not
/// modify the filename.
///
/// Paths into the blob directory may be either defined by an absolute path
/// or by the relative prefix `$BLOBDIR`.
@@ -156,7 +162,8 @@ impl<'a> BlobObject<'a> {
/// Returns a [BlobObject] for an existing blob from a path.
///
/// The path must designate a file directly in the blobdir and
/// must use a valid blob name. That means it must be valid UTF-8
/// must use a valid blob name. That is after sanitisation the
/// name must still be the same, that means it must be valid UTF-8
/// and not have any special characters in it.
pub fn from_path(context: &'a Context, path: &Path) -> Result<BlobObject<'a>> {
let rel_path = path
@@ -230,46 +237,65 @@ impl<'a> BlobObject<'a> {
}
}
/// Get a file extension if any, including the dot, in lower case, otherwise an empty string.
fn get_extension(name: &str) -> String {
let mut name = name;
/// Create a safe name based on a messy input string.
///
/// The safe name will be a valid filename on Unix and Windows and
/// not contain any path separators. The input can contain path
/// segments separated by either Unix or Windows path separators,
/// the rightmost non-empty segment will be used as name,
/// sanitised for special characters.
///
/// The resulting name is returned as a tuple, the first part
/// being the stem or basename and the second being an extension,
/// including the dot. E.g. "foo.txt" is returned as `("foo",
/// ".txt")` while "bar" is returned as `("bar", "")`.
///
/// The extension part will always be lowercased.
fn sanitise_name(name: &str) -> (String, String) {
let mut name = name.to_string();
for part in name.rsplit('/') {
if !part.is_empty() {
name = part;
name = part.to_string();
break;
}
}
for part in name.rsplit('\\') {
if !part.is_empty() {
name = part;
name = part.to_string();
break;
}
}
let opts = sanitize_filename::Options {
truncate: true,
windows: true,
replacement: "",
};
let clean = sanitize_filename::sanitize_with_options(name, opts);
// Let's take the tricky filename
// "file.with_lots_of_characters_behind_point_and_double_ending.tar.gz" as an example.
// Assume that the extension is 32 chars maximum.
let ext: String = name
.chars()
// Split it into "file" and "with_lots_of_characters_behind_point_and_double_ending.tar.gz":
let mut iter = clean.splitn(2, '.');
let stem: String = iter.next().unwrap_or_default().chars().take(64).collect();
// stem == "file"
let ext_chars = iter.next().unwrap_or_default().chars();
let ext: String = ext_chars
.rev()
.take_while(|c| !c.is_whitespace())
.take(33)
.take(32)
.collect::<Vec<_>>()
.iter()
.rev()
.collect();
// ext == "nd_point_and_double_ending.tar.gz"
// ext == "d_point_and_double_ending.tar.gz"
// Split it into "nd_point_and_double_ending" and "tar.gz":
let mut iter = ext.splitn(2, '.');
iter.next();
let ext = iter.next().unwrap_or_default();
if ext.is_empty() {
String::new()
(stem, "".to_string())
} else {
format!(".{ext}").to_lowercase()
// Return ".tar.gz".
(stem, format!(".{ext}").to_lowercase())
// Return ("file", ".d_point_and_double_ending.tar.gz")
// which is not perfect but acceptable.
}
}
@@ -717,7 +743,6 @@ fn add_white_bg(img: &mut DynamicImage) {
#[cfg(test)]
mod tests {
use fs::File;
use regex::Regex;
use super::*;
use crate::chat::{self, create_group_chat, ProtectionStatus};
@@ -737,43 +762,32 @@ mod tests {
async fn test_create() {
let t = TestContext::new().await;
let blob = BlobObject::create(&t, "foo", b"hello").await.unwrap();
let re = Regex::new("^[[:xdigit:]]{16}$").unwrap();
assert!(re.is_match(blob.as_file_name()));
let fname = t.get_blobdir().join(blob.as_file_name());
let fname = t.get_blobdir().join("foo");
let data = fs::read(fname).await.unwrap();
assert_eq!(data, b"hello");
assert_eq!(
blob.as_name(),
"$BLOBDIR/".to_string() + blob.as_file_name()
);
assert_eq!(
blob.to_abs_path(),
t.get_blobdir().join(blob.as_file_name())
);
assert_eq!(blob.as_name(), "$BLOBDIR/foo");
assert_eq!(blob.to_abs_path(), t.get_blobdir().join("foo"));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_lowercase_ext() {
let t = TestContext::new().await;
let blob = BlobObject::create(&t, "foo.TXT", b"hello").await.unwrap();
let re = Regex::new("^\\$BLOBDIR/[[:xdigit:]]{16}.txt$").unwrap();
assert!(re.is_match(blob.as_name()));
assert_eq!(blob.as_name(), "$BLOBDIR/foo.txt");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_as_file_name() {
let t = TestContext::new().await;
let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
let re = Regex::new("^[[:xdigit:]]{16}.txt$").unwrap();
assert!(re.is_match(blob.as_file_name()));
assert_eq!(blob.as_file_name(), "foo.txt");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_as_rel_path() {
let t = TestContext::new().await;
let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
let re = Regex::new("^[[:xdigit:]]{16}.txt$").unwrap();
assert!(re.is_match(blob.as_rel_path().to_str().unwrap()));
assert_eq!(blob.as_rel_path(), Path::new("foo.txt"));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -788,30 +802,30 @@ mod tests {
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_create_dup() {
let t = TestContext::new().await;
let re = Regex::new("^[[:xdigit:]]{16}.txt$").unwrap();
let blob = BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
assert!(re.is_match(blob.as_rel_path().to_str().unwrap()));
let foo_path = t.get_blobdir().join(blob.as_file_name());
BlobObject::create(&t, "foo.txt", b"hello").await.unwrap();
let foo_path = t.get_blobdir().join("foo.txt");
assert!(foo_path.exists());
let blob = BlobObject::create(&t, "foo.txt", b"world").await.unwrap();
assert!(re.is_match(blob.as_rel_path().to_str().unwrap()));
let foo_path2 = t.get_blobdir().join(blob.as_file_name());
assert!(foo_path2.exists());
assert!(foo_path != foo_path2);
BlobObject::create(&t, "foo.txt", b"world").await.unwrap();
let mut dir = fs::read_dir(t.get_blobdir()).await.unwrap();
while let Ok(Some(dirent)) = dir.next_entry().await {
let fname = dirent.file_name();
if fname == foo_path.file_name().unwrap() {
assert_eq!(fs::read(&foo_path).await.unwrap(), b"hello");
} else {
let name = fname.to_str().unwrap();
assert!(name.starts_with("foo"));
assert!(name.ends_with(".txt"));
}
}
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_double_ext_preserved() {
let t = TestContext::new().await;
let blob = BlobObject::create(&t, "foo.tar.gz", b"hello")
BlobObject::create(&t, "foo.tar.gz", b"hello")
.await
.unwrap();
let re = Regex::new("^[[:xdigit:]]{16}.tar.gz$").unwrap();
assert!(re.is_match(blob.as_file_name()));
let foo_path = t.get_blobdir().join(blob.as_file_name());
let foo_path = t.get_blobdir().join("foo.tar.gz");
assert!(foo_path.exists());
BlobObject::create(&t, "foo.tar.gz", b"world")
.await
@@ -824,6 +838,7 @@ mod tests {
} else {
let name = fname.to_str().unwrap();
println!("{name}");
assert!(name.starts_with("foo"));
assert!(name.ends_with(".tar.gz"));
}
}
@@ -844,8 +859,7 @@ mod tests {
let src = t.dir.path().join("src");
fs::write(&src, b"boo").await.unwrap();
let blob = BlobObject::create_and_copy(&t, src.as_ref()).await.unwrap();
let re = Regex::new("^\\$BLOBDIR/[[:xdigit:]]{16}$").unwrap();
assert!(re.is_match(blob.as_name()));
assert_eq!(blob.as_name(), "$BLOBDIR/src");
let data = fs::read(blob.to_abs_path()).await.unwrap();
assert_eq!(data, b"boo");
@@ -866,8 +880,7 @@ mod tests {
let blob = BlobObject::new_from_path(&t, src_ext.as_ref())
.await
.unwrap();
let re = Regex::new("^\\$BLOBDIR/[[:xdigit:]]{16}$").unwrap();
assert!(re.is_match(blob.as_name()));
assert_eq!(blob.as_name(), "$BLOBDIR/external");
let data = fs::read(blob.to_abs_path()).await.unwrap();
assert_eq!(data, b"boo");
@@ -878,6 +891,19 @@ mod tests {
let data = fs::read(blob.to_abs_path()).await.unwrap();
assert_eq!(data, b"boo");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_create_from_name_long() {
let t = TestContext::new().await;
let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
fs::write(&src_ext, b"boo").await.unwrap();
let blob = BlobObject::new_from_path(&t, src_ext.as_ref())
.await
.unwrap();
assert_eq!(
blob.as_name(),
"$BLOBDIR/autocrypt-setup-message-4137848473.html"
);
}
#[test]
fn test_is_blob_name() {
@@ -890,24 +916,42 @@ mod tests {
}
#[test]
fn test_get_extension() {
let ext = BlobObject::get_extension("Я ЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯ.txt");
fn test_sanitise_name() {
let (stem, ext) =
BlobObject::sanitise_name("Я ЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯ.txt");
assert_eq!(ext, ".txt");
assert!(!stem.is_empty());
let ext = BlobObject::get_extension("wot.tar.gz");
// the extensions are kept together as between stem and extension a number may be added -
// and `foo.tar.gz` should become `foo-1234.tar.gz` and not `foo.tar-1234.gz`
let (stem, ext) = BlobObject::sanitise_name("wot.tar.gz");
assert_eq!(stem, "wot");
assert_eq!(ext, ".tar.gz");
let ext = BlobObject::get_extension(".foo.bar");
let (stem, ext) = BlobObject::sanitise_name(".foo.bar");
assert_eq!(stem, "");
assert_eq!(ext, ".foo.bar");
let ext = BlobObject::get_extension("foo?.bar");
let (stem, ext) = BlobObject::sanitise_name("foo?.bar");
assert!(stem.contains("foo"));
assert!(!stem.contains('?'));
assert_eq!(ext, ".bar");
let ext = BlobObject::get_extension("no-extension");
let (stem, ext) = BlobObject::sanitise_name("no-extension");
assert_eq!(stem, "no-extension");
assert_eq!(ext, "");
let ext = BlobObject::get_extension("path/ignored\\this: is* forbidden?.c");
let (stem, ext) = BlobObject::sanitise_name("path/ignored\\this: is* forbidden?.c");
assert_eq!(ext, ".c");
assert!(!stem.contains("path"));
assert!(!stem.contains("ignored"));
assert!(stem.contains("this"));
assert!(stem.contains("forbidden"));
assert!(!stem.contains('/'));
assert!(!stem.contains('\\'));
assert!(!stem.contains(':'));
assert!(!stem.contains('*'));
assert!(!stem.contains('?'));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -950,21 +994,19 @@ mod tests {
let avatar_src = t.dir.path().join("avatar.jpg");
let avatar_bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
fs::write(&avatar_src, avatar_bytes).await.unwrap();
let avatar_blob = t.get_blobdir().join("avatar.jpg");
assert!(!avatar_blob.exists());
t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
.await
.unwrap();
let avatar_blob = t.get_config(Config::Selfavatar).await.unwrap().unwrap();
let blobdir = t.get_blobdir().to_str().unwrap();
assert!(avatar_blob.starts_with(blobdir));
let re = Regex::new("[[:xdigit:]]{16}.jpg$").unwrap();
assert!(re.is_match(&avatar_blob));
let avatar_blob = Path::new(&avatar_blob);
assert!(avatar_blob.exists());
assert!(fs::metadata(&avatar_blob).await.unwrap().len() < avatar_bytes.len() as u64);
let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap();
assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
check_image_size(avatar_src, 1000, 1000);
check_image_size(
avatar_blob,
&avatar_blob,
constants::BALANCED_AVATAR_SIZE,
constants::BALANCED_AVATAR_SIZE,
);
@@ -974,7 +1016,7 @@ mod tests {
file.metadata().await.unwrap().len()
}
let mut blob = BlobObject::new_from_path(&t, avatar_blob).await.unwrap();
let mut blob = BlobObject::new_from_path(&t, &avatar_blob).await.unwrap();
let maybe_sticker = &mut false;
let strict_limits = true;
blob.recode_to_size(
@@ -986,8 +1028,8 @@ mod tests {
strict_limits,
)
.unwrap();
assert!(file_size(avatar_blob).await <= 3000);
assert!(file_size(avatar_blob).await > 2000);
assert!(file_size(&avatar_blob).await <= 3000);
assert!(file_size(&avatar_blob).await > 2000);
tokio::task::block_in_place(move || {
let img = image::open(avatar_blob).unwrap();
assert!(img.width() > 130);
@@ -1027,19 +1069,18 @@ mod tests {
let avatar_src = t.dir.path().join("avatar.png");
let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
fs::write(&avatar_src, avatar_bytes).await.unwrap();
let avatar_blob = t.get_blobdir().join("avatar.png");
assert!(!avatar_blob.exists());
t.set_config(Config::Selfavatar, Some(avatar_src.to_str().unwrap()))
.await
.unwrap();
let avatar_blob = t.get_config(Config::Selfavatar).await.unwrap().unwrap();
let blobdir = t.get_blobdir().to_str().unwrap();
assert!(avatar_blob.starts_with(blobdir));
let re = Regex::new("[[:xdigit:]]{16}.png$").unwrap();
assert!(re.is_match(&avatar_blob));
assert!(Path::new(&avatar_blob).exists());
assert!(avatar_blob.exists());
assert_eq!(
fs::metadata(&avatar_blob).await.unwrap().len(),
avatar_bytes.len() as u64
);
let avatar_cfg = t.get_config(Config::Selfavatar).await.unwrap();
assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]

View File

@@ -4520,7 +4520,6 @@ mod tests {
use crate::message::delete_msgs;
use crate::receive_imf::receive_imf;
use crate::test_utils::{sync, TestContext, TestContextManager};
use regex::Regex;
use strum::IntoEnumIterator;
use tokio::fs;
@@ -7079,11 +7078,9 @@ mod tests {
// the file bob receives should not contain BIDI-control characters
assert_eq!(
msg.param.get(Param::Filename).unwrap(),
"harmless_file.txt.exe"
Some("$BLOBDIR/harmless_file.txt.exe"),
msg.param.get(Param::File),
);
let re = Regex::new("^\\$BLOBDIR/[[:xdigit:]]{16}.txt.exe$").unwrap();
assert!(re.is_match(msg.param.get(Param::File).unwrap()));
Ok(())
}

View File

@@ -211,9 +211,6 @@ pub(crate) const DC_FOLDERS_CONFIGURED_KEY: &str = "folders_configured";
// this value can be increased if the folder configuration is changed and must be redone on next program start
pub(crate) const DC_FOLDERS_CONFIGURED_VERSION: i32 = 4;
// Maximum attemps to create a blob file.
pub(crate) const BLOB_CREATE_ATTEMPTS: u32 = 2;
// If more recipients are needed in SMTP's `RCPT TO:` header, the recipient list is split into
// chunks. This does not affect MIME's `To:` header. Can be overwritten by setting
// `max_smtp_rcpt_to` in the provider db.

View File

@@ -9,6 +9,7 @@ use crate::tools::time;
use crate::webxdc::StatusUpdateItem;
use async_channel::{self as channel, Receiver, Sender};
use serde_json::json;
use std::path::PathBuf;
use tokio::task;
#[derive(Debug)]
@@ -97,7 +98,7 @@ pub async fn maybe_set_logging_xdc(
context,
msg.get_viewtype(),
chat_id,
msg.param.get(Param::Filename),
msg.param.get_path(Param::File, context).unwrap_or_default(),
msg.get_id(),
)
.await?;
@@ -110,16 +111,18 @@ pub async fn maybe_set_logging_xdc_inner(
context: &Context,
viewtype: Viewtype,
chat_id: ChatId,
file_name: Option<&str>,
file: Option<PathBuf>,
msg_id: MsgId,
) -> anyhow::Result<()> {
if viewtype == Viewtype::Webxdc {
if let Some(file_name) = file_name {
if file_name.starts_with("debug_logging")
&& file_name.ends_with(".xdc")
&& chat_id.is_self_talk(context).await?
{
set_debug_logging_xdc(context, Some(msg_id)).await?;
if let Some(file) = file {
if let Some(file_name) = file.file_name().and_then(|name| name.to_str()) {
if file_name.starts_with("debug_logging")
&& file_name.ends_with(".xdc")
&& chat_id.is_self_talk(context).await?
{
set_debug_logging_xdc(context, Some(msg_id)).await?;
}
}
}
}

View File

@@ -651,9 +651,8 @@ mod tests {
};
let msg = Message::load_from_db(&ctx1, *msgid).await.unwrap();
assert_eq!(&msg.get_filename().unwrap(), "hello.txt");
let path = msg.get_file(&ctx1).unwrap();
assert_eq!(path.with_extension("txt"), path);
assert_eq!(path.with_file_name("hello.txt"), path);
let text = fs::read_to_string(&path).await.unwrap();
assert_eq!(text, "i am attachment");

View File

@@ -304,12 +304,7 @@ WHERE id=?;
if let Some(path) = msg.get_file(context) {
let bytes = get_filebytes(context, &path).await?;
ret += &format!(
"\nFile: {}, name: {}, {} bytes\n",
path.display(),
msg.get_filename().unwrap_or_default(),
bytes
);
ret += &format!("\nFile: {}, {} bytes\n", path.display(), bytes);
}
if msg.viewtype != Viewtype::Text {
@@ -606,8 +601,7 @@ impl Message {
None
}
/// Returns the full path to the file associated with a message. The filename isn't meaningful,
/// only the extension is preserved.
/// Returns the full path to the file associated with a message.
pub fn get_file(&self, context: &Context) -> Option<PathBuf> {
self.param.get_path(Param::File, context).unwrap_or(None)
}
@@ -760,6 +754,8 @@ impl Message {
}
/// Returns original filename (as shown in chat).
///
/// To get the full path, use [`Self::get_file()`].
pub fn get_filename(&self) -> Option<String> {
if let Some(name) = self.param.get(Param::Filename) {
return Some(name.to_string());

View File

@@ -2254,7 +2254,6 @@ mod tests {
#![allow(clippy::indexing_slicing)]
use mailparse::ParsedMail;
use regex::Regex;
use super::*;
use crate::{
@@ -3455,7 +3454,7 @@ On 2020-10-25, Bob wrote:
assert_eq!(msg.chat_blocked, Blocked::Request);
assert_eq!(msg.state, MessageState::InFresh);
assert_eq!(msg.get_filebytes(&t).await.unwrap().unwrap(), 2115);
assert_eq!(msg.get_file(&t).unwrap().extension().unwrap(), "png");
assert!(msg.get_file(&t).is_some());
assert_eq!(msg.get_filename().unwrap(), "avatar64x64.png");
assert_eq!(msg.get_width(), 64);
assert_eq!(msg.get_height(), 64);
@@ -3801,8 +3800,10 @@ Message.
mime_message.parts[0].msg,
"this is a classic email I attached the .EML file".to_string()
);
let re = Regex::new("^\\$BLOBDIR/[[:xdigit:]]{16}.eml$").unwrap();
assert!(re.is_match(mime_message.parts[0].param.get(Param::File).unwrap()));
assert_eq!(
mime_message.parts[0].param.get(Param::File),
Some("$BLOBDIR/.eml")
);
assert_eq!(mime_message.parts[0].org_filename, Some(".eml".to_string()));

View File

@@ -547,6 +547,8 @@ mod tests {
assert!(p.get_blob(Param::File, &t, false).await.is_err());
fs::write(fname, b"boo").await.unwrap();
let blob = p.get_blob(Param::File, &t, true).await.unwrap().unwrap();
assert!(blob.as_file_name().starts_with("foo"));
// Blob in blobdir, expect blob.
let bar_path = t.get_blobdir().join("bar");

View File

@@ -1577,7 +1577,9 @@ RETURNING id
context,
part.typ,
chat_id,
part.param.get(Param::Filename),
part.param
.get_path(Param::File, context)
.unwrap_or_default(),
*msg_id,
)
.await?;

View File

@@ -1,6 +1,5 @@
use std::time::Duration;
use regex::Regex;
use tokio::fs;
use super::*;
@@ -1594,6 +1593,7 @@ async fn test_pdf_filename_simple() {
assert_eq!(msg.viewtype, Viewtype::File);
assert_eq!(msg.text, "mail body");
let file_path = msg.param.get(Param::File).unwrap();
assert!(file_path.starts_with("$BLOBDIR/simple"));
assert!(file_path.ends_with(".pdf"));
}
@@ -1609,6 +1609,7 @@ async fn test_pdf_filename_continuation() {
assert_eq!(msg.viewtype, Viewtype::File);
assert_eq!(msg.text, "mail body");
let file_path = msg.param.get(Param::File).unwrap();
assert!(file_path.starts_with("$BLOBDIR/test pdf äöüß"));
assert!(file_path.ends_with(".pdf"));
}
@@ -2961,27 +2962,20 @@ Reply from different address
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_weird_and_duplicated_filenames() -> Result<()> {
async fn test_long_and_duplicated_filenames() -> Result<()> {
let mut tcm = TestContextManager::new();
let alice = tcm.alice().await;
let bob = tcm.bob().await;
for (filename_sent, expected_ext) in &[
("foo.bar very long file name test baz.tar.gz", "tar.gz"),
(
"foo.barabababababababbababababverylongfilenametestbaz.tar.gz",
"tar.gz",
),
("fooo...tar.gz", "..tar.gz"),
("foo. .tar.gz", "tar.gz"),
(
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.tar.gz",
"tar.gz",
),
("a.tar.gz", "tar.gz"),
("a.tar.gz", "tar.gz"),
("a.a..a.a.a.a.tar.gz", "a..a.a.a.a.tar.gz"),
("a. tar.tar.gz", "tar.gz"),
for filename_sent in &[
"foo.bar very long file name test baz.tar.gz",
"foobarabababababababbababababverylongfilenametestbaz.tar.gz",
"fooo...tar.gz",
"foo. .tar.gz",
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.tar.gz",
"a.tar.gz",
"a.tar.gz",
"a.a..a.a.a.a.tar.gz",
] {
let attachment = alice.blobdir.join(filename_sent);
let content = format!("File content of {filename_sent}");
@@ -2995,33 +2989,23 @@ async fn test_weird_and_duplicated_filenames() -> Result<()> {
let msg_bob = bob.recv_msg(&sent).await;
async fn check_message(
msg: &Message,
t: &TestContext,
filename: &str,
expected_ext: &str,
content: &str,
) {
async fn check_message(msg: &Message, t: &TestContext, filename: &str, content: &str) {
assert_eq!(msg.get_viewtype(), Viewtype::File);
let resulting_filename = msg.get_filename().unwrap();
assert_eq!(resulting_filename, filename);
let path = msg.get_file(t).unwrap();
if !msg.get_state().is_outgoing() {
let re =
Regex::new(&("^[[:xdigit:]]{16}.".to_string() + expected_ext + "$")).unwrap();
assert!(
re.is_match(path.file_name().unwrap().to_str().unwrap()),
"invalid path {path:?}"
);
}
let path2 = path.with_file_name("saved.txt");
msg.save_file(t, &path2).await.unwrap();
assert!(
path.to_str().unwrap().ends_with(".tar.gz"),
"path {path:?} doesn't end with .tar.gz"
);
assert_eq!(fs::read_to_string(&path).await.unwrap(), content);
assert_eq!(fs::read_to_string(&path2).await.unwrap(), content);
fs::remove_file(path2).await.unwrap();
}
check_message(&msg_alice, &alice, filename_sent, expected_ext, &content).await;
check_message(&msg_bob, &bob, filename_sent, expected_ext, &content).await;
check_message(&msg_alice, &alice, filename_sent, &content).await;
check_message(&msg_bob, &bob, filename_sent, &content).await;
}
Ok(())