mirror of
https://github.com/chatmail/core.git
synced 2026-04-02 05:22:14 +03:00
fix(tools): make sure dc_truncate can handle arbitrary utf8 valu… (#460)
fix(tools): make sure dc_truncate can handle arbitrary utf8 values
This commit is contained in:
Friedel Ziegelmayer
GitHub
57
Cargo.lock
generated
57
Cargo.lock
generated
@@ -110,6 +110,19 @@ dependencies = [
|
||||
"byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bit-set"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"bit-vec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bit-vec"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "bitfield"
|
||||
version = "0.13.2"
|
||||
@@ -489,6 +502,7 @@ dependencies = [
|
||||
"pkg-config 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"pretty_assertions 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"pretty_env_logger 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"proptest 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"quick-xml 0.15.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"r2d2 0.8.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"r2d2_sqlite 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
@@ -1647,6 +1661,25 @@ dependencies = [
|
||||
"unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proptest"
|
||||
version = "0.9.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"bit-set 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"num-traits 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex-syntax 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rusty-fork 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "publicsuffix"
|
||||
version = "1.5.2"
|
||||
@@ -2020,6 +2053,17 @@ dependencies = [
|
||||
"semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rusty-fork"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"wait-timeout 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustyline"
|
||||
version = "4.1.0"
|
||||
@@ -2664,6 +2708,14 @@ name = "void"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "wait-timeout"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "walkdir"
|
||||
version = "2.2.9"
|
||||
@@ -2811,6 +2863,8 @@ dependencies = [
|
||||
"checksum backtrace 0.3.34 (registry+https://github.com/rust-lang/crates.io-index)" = "b5164d292487f037ece34ec0de2fcede2faa162f085dd96d2385ab81b12765ba"
|
||||
"checksum backtrace-sys 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)" = "82a830b4ef2d1124a711c71d263c5abdc710ef8e907bd508c88be475cebc422b"
|
||||
"checksum base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e"
|
||||
"checksum bit-set 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e84c238982c4b1e1ee668d136c510c67a13465279c0cb367ea6baf6310620a80"
|
||||
"checksum bit-vec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f59bbe95d4e52a6398ec21238d31577f2b28a9d86807f06ca59d191d8440d0bb"
|
||||
"checksum bitfield 0.13.2 (registry+https://github.com/rust-lang/crates.io-index)" = "46afbd2983a5d5a7bd740ccb198caf5b82f45c40c09c0eed36052d91cb92e719"
|
||||
"checksum bitflags 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3d155346769a6855b86399e9bc3814ab343cd3d62c7e985113d46a0ec3c281fd"
|
||||
"checksum blake2b_simd 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "461f4b879a8eb70c1debf7d0788a9a5ff15f1ea9d25925fea264ef4258bed6b2"
|
||||
@@ -2973,6 +3027,7 @@ dependencies = [
|
||||
"checksum proc-macro-hack 0.5.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e688f31d92ffd7c1ddc57a1b4e6d773c0f2a14ee437a4b0a4f5a69c80eb221c8"
|
||||
"checksum proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759"
|
||||
"checksum proc-macro2 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4c5c2380ae88876faae57698be9e9775e3544decad214599c3a6266cca6ac802"
|
||||
"checksum proptest 0.9.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cf147e022eacf0c8a054ab864914a7602618adba841d800a9a9868a5237a529f"
|
||||
"checksum publicsuffix 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5afecba86dcf1e4fd610246f89899d1924fe12e1e89f555eb7c7f710f3c5ad1d"
|
||||
"checksum pulldown-cmark 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "eef52fac62d0ea7b9b4dc7da092aa64ea7ec3d90af6679422d3d7e0e14b6ee15"
|
||||
"checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0"
|
||||
@@ -3010,6 +3065,7 @@ dependencies = [
|
||||
"checksum rust-argon2 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4ca4eaef519b494d1f2848fc602d18816fed808a981aedf4f1f00ceb7c9d32cf"
|
||||
"checksum rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783"
|
||||
"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
|
||||
"checksum rusty-fork 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3dd93264e10c577503e926bd1430193eeb5d21b059148910082245309b424fae"
|
||||
"checksum rustyline 4.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0f47ea1ceb347d2deae482d655dc8eef4bd82363d3329baffa3818bd76fea48b"
|
||||
"checksum ryu 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c92464b447c0ee8c4fb3824ecc8383b81717b9f1e74ba2e72540aef7b9f82997"
|
||||
"checksum safemem 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d2b08423011dae9a5ca23f07cf57dac3857f5c885d352b76f6d95f4aea9434d0"
|
||||
@@ -3086,6 +3142,7 @@ dependencies = [
|
||||
"checksum vcpkg 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "33dd455d0f96e90a75803cfeb7f948768c08d70a6de9a8d2362461935698bf95"
|
||||
"checksum version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd"
|
||||
"checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
|
||||
"checksum wait-timeout 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6"
|
||||
"checksum walkdir 2.2.9 (registry+https://github.com/rust-lang/crates.io-index)" = "9658c94fa8b940eab2250bd5a457f9c48b748420d71293b165c8cdbe2f55f71e"
|
||||
"checksum want 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b6395efa4784b027708f7451087e647ec73cc74f5d9bc2e418404248d679a230"
|
||||
"checksum wasi 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fd5442abcac6525a045cc8c795aedb60da7a2e5e89c7bf18a0d5357849bb23c7"
|
||||
|
||||
@@ -55,6 +55,7 @@ bitflags = "1.1.0"
|
||||
tempfile = "3.0"
|
||||
pretty_assertions = "0.6.1"
|
||||
pretty_env_logger = "0.3.0"
|
||||
proptest = "0.9.4"
|
||||
|
||||
[workspace]
|
||||
members = [
|
||||
|
||||
9
proptest-regressions/dc_tools.txt
Normal file
9
proptest-regressions/dc_tools.txt
Normal file
@@ -0,0 +1,9 @@
|
||||
# Seeds for failure cases proptest has generated in the past. It is
|
||||
# automatically read and these particular cases re-run before any
|
||||
# novel cases are generated.
|
||||
#
|
||||
# It is recommended to check this file in to source control so that
|
||||
# everyone who runs the test benefits from these saved cases.
|
||||
cc c310754465ee0261807b96fa9bcc4861ff9aa286e94667524b5960c69f9b6620 # shrinks to buf = "", approx_chars = 0, do_unwrap = false
|
||||
cc 5fd8d730b0a9cdf7308ce58818ca9aefc0255c9ba2a0878944fc48d43a67315b # shrinks to buf = "𑒀ὐ￠🜀\u{1e01b}A a🟠", approx_chars = 0, do_unwrap = false
|
||||
cc c6a0029a54137a4b9efc9ef2ea6d9a7dd1d60d1c937bb472b66a174618ba8013 # shrinks to buf = "𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A 丽ⷐએ ", approx_chars = 0, do_unwrap = false
|
||||
@@ -261,11 +261,18 @@ pub unsafe fn dc_replace_bad_utf8_chars(buf: *mut libc::c_char) {
|
||||
pub fn dc_truncate(buf: &str, approx_chars: usize, do_unwrap: bool) -> Cow<str> {
|
||||
let ellipse = if do_unwrap { "..." } else { "[...]" };
|
||||
|
||||
if approx_chars > 0 && buf.len() > approx_chars + ellipse.len() {
|
||||
if let Some(index) = buf[..approx_chars].rfind(|c| c == ' ' || c == '\n') {
|
||||
let count = buf.chars().count();
|
||||
if approx_chars > 0 && count > approx_chars + ellipse.len() {
|
||||
let end_pos = buf
|
||||
.char_indices()
|
||||
.nth(approx_chars)
|
||||
.map(|(n, _)| n)
|
||||
.unwrap_or_default();
|
||||
|
||||
if let Some(index) = buf[..end_pos].rfind(|c| c == ' ' || c == '\n') {
|
||||
Cow::Owned(format!("{}{}", &buf[..index + 1], ellipse))
|
||||
} else {
|
||||
Cow::Owned(format!("{}{}", &buf[..approx_chars], ellipse))
|
||||
Cow::Owned(format!("{}{}", &buf[..end_pos], ellipse))
|
||||
}
|
||||
} else {
|
||||
Cow::Borrowed(buf)
|
||||
@@ -1533,6 +1540,36 @@ mod tests {
|
||||
assert_eq!(dc_truncate("123456", 4, true), "123456");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_dc_truncate_edge() {
|
||||
assert_eq!(dc_truncate("", 4, false), "");
|
||||
assert_eq!(dc_truncate("", 4, true), "");
|
||||
|
||||
assert_eq!(dc_truncate("\n hello \n world", 4, false), "\n [...]");
|
||||
assert_eq!(dc_truncate("\n hello \n world", 4, true), "\n ...");
|
||||
|
||||
assert_eq!(
|
||||
dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A 丽ⷐએ", 1, false),
|
||||
"𐠈[...]"
|
||||
);
|
||||
assert_eq!(
|
||||
dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A 丽ⷐએ", 0, false),
|
||||
"𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A 丽ⷐએ"
|
||||
);
|
||||
|
||||
// 9 characters, so no truncation
|
||||
assert_eq!(
|
||||
dc_truncate("𑒀ὐ￠🜀\u{1e01b}A a🟠", 6, false),
|
||||
"𑒀ὐ￠🜀\u{1e01b}A a🟠",
|
||||
);
|
||||
|
||||
// 12 characters, truncation
|
||||
assert_eq!(
|
||||
dc_truncate("𑒀ὐ￠🜀\u{1e01b}A a🟠bcd", 6, false),
|
||||
"𑒀ὐ￠🜀\u{1e01b}A[...]",
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_dc_null_terminate_1() {
|
||||
unsafe {
|
||||
@@ -1870,4 +1907,37 @@ mod tests {
|
||||
assert_eq!(EmailAddress::new("u@.tt").is_ok(), false);
|
||||
assert_eq!(EmailAddress::new("@d.tt").is_ok(), false);
|
||||
}
|
||||
|
||||
use proptest::prelude::*;
|
||||
|
||||
proptest! {
|
||||
#[test]
|
||||
fn test_dc_truncate(
|
||||
buf: String,
|
||||
approx_chars in 0..10000usize,
|
||||
do_unwrap: bool,
|
||||
) {
|
||||
let res = dc_truncate(&buf, approx_chars, do_unwrap);
|
||||
let el_len = if do_unwrap { 3 } else { 5 };
|
||||
let l = res.chars().count();
|
||||
if approx_chars > 0 {
|
||||
assert!(
|
||||
l <= approx_chars + el_len,
|
||||
"buf: '{}' - res: '{}' - len {}, approx {}",
|
||||
&buf, &res, res.len(), approx_chars
|
||||
);
|
||||
} else {
|
||||
assert_eq!(&res, &buf);
|
||||
}
|
||||
|
||||
if approx_chars > 0 && buf.chars().count() > approx_chars + el_len {
|
||||
let l = res.len();
|
||||
if do_unwrap {
|
||||
assert_eq!(&res[l-3..l], "...", "missing ellipsis in {}", &res);
|
||||
} else {
|
||||
assert_eq!(&res[l-5..l], "[...]", "missing ellipsis in {}", &res);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user