Compare commits

..

19 Commits

Author SHA1 Message Date
link2xt
f2ab9456cd python: use pyright 2023-02-20 10:33:51 +00:00
link2xt
cdaef3bd0d python: do not shadow variables to fix pyright warnings 2023-02-20 10:33:16 +00:00
link2xt
e2151e26ee ci: pin rustfmt version 2023-02-19 23:40:04 +00:00
link2xt
446214fd7b sql: use transaction in Contact::add_or_lookup() 2023-02-19 23:16:44 +00:00
link2xt
9389e11007 ffi: log create_contact() errors 2023-02-19 23:16:44 +00:00
link2xt
5bdd49b451 Add Unreleased section to changelog 2023-02-19 21:57:36 +00:00
link2xt
42a7e91f05 python: stop using pytest-rerunfailures 2023-02-19 21:55:28 +00:00
link2xt
44953d6bcc Release 1.109.0 2023-02-19 21:40:33 +00:00
link2xt
10066b2bc7 sql: use semaphore to limit access to the connection pool
This ensures that if multiple connections are returned
to the pool at the same time, waiters get them in the order
they were placed in the queue.
2023-02-19 17:06:25 +00:00
Simon Laux
609fc67f0d fix websocket server & add ci test for building it (#4047)
the axum update broke the websocket server, because yerpc still uses a the old 5.9 version.
So I needed to downgrade the dependency until https://github.com/deltachat/yerpc/pull/35 is merged.

I want to retry the kaiOS client experiment, for this I need a working websocket server binary.
2023-02-19 14:57:00 +01:00
Hocuri
641d102aba Add dc_msg_set_subject() C FFI (#4057) 2023-02-19 13:42:39 +00:00
link2xt
f65e1c1587 Sort dependencies in Cargo.toml 2023-02-19 13:18:58 +00:00
link2xt
626ec5e793 Document the meaning of empty Message.subject 2023-02-19 13:05:39 +00:00
link2xt
85517abf58 Derive Debug for Pool 2023-02-19 02:26:26 +00:00
link2xt
75f65b06e8 Run VACUUM on the same connection as sqlcipher_export
Otherwise export_and_import_backup test fails due to SQLITE_SCHEMA error.
2023-02-19 02:26:26 +00:00
link2xt
f2b05ccc29 Reimplement connection pool on top of crossbeam 2023-02-19 02:26:26 +00:00
link2xt
e88f21c010 Remove try_many_times r2d2 bug workaround 2023-02-19 02:26:26 +00:00
link2xt
ed8e2c4818 Replace r2d2 with an own connection pool
New connection pool does not use threads
and does not remove idle connections
or create new connections at runtime.
2023-02-19 02:26:26 +00:00
link2xt
48fee4fc92 python: replace "while 1" with "while True"
This makes pyright recognize that the function never returns implicitly.
2023-02-18 11:31:07 +00:00
30 changed files with 340 additions and 416 deletions

View File

@@ -10,26 +10,20 @@ env:
RUSTFLAGS: -Dwarnings
jobs:
fmt:
name: Rustfmt
lint:
name: Rustfmt and Clippy
runs-on: ubuntu-latest
env:
RUSTUP_TOOLCHAIN: 1.67.1
steps:
- uses: actions/checkout@v3
- run: cargo fmt --all -- --check
run_clippy:
name: Clippy
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Install clippy
run: rustup toolchain install 1.67.1 --component clippy
- name: Install rustfmt and clippy
run: rustup toolchain install $RUSTUP_TOOLCHAIN --component rustfmt --component clippy
- name: Cache rust cargo artifacts
uses: swatinem/rust-cache@v2
- name: Run rustfmt
run: cargo fmt --all -- --check
- name: Run clippy
env:
RUSTUP_TOOLCHAIN: 1.67.1
run: scripts/clippy.sh
docs:

View File

@@ -35,6 +35,10 @@ jobs:
npm run test
env:
DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
- name: make sure websocket server version still builds
run: |
cd deltachat-jsonrpc
cargo build --bin deltachat-jsonrpc-server --features webserver
- name: Run linter
run: |
cd deltachat-jsonrpc/typescript

View File

@@ -2,21 +2,33 @@
## Unreleased
### Changes
- use transaction in `Contact::add_or_lookup()` #4059
### Fixes
### API-Changes
## 1.109.0
### Changes
- deltachat-rpc-client: use `dataclass` for `Account`, `Chat`, `Contact` and `Message` #4042
- python: mark bindings as supporting typing according to PEP 561 #4045
- retry filesystem operations during account migration #4043
- replace `r2d2` and `r2d2_sqlite` dependencies with an own connection pool #4050 #4053
### Fixes
- deltachat-rpc-server: do not block stdin while processing the request. #4041
deltachat-rpc-server now reads the next request as soon as previous request handler is spawned.
- enable `auto_vacuum` on all SQL connections #2955
- Enable `auto_vacuum` on all SQL connections. #2955
- Replace `r2d2` connection pool with an own implementation. #4050 #4053 #4043 #4061
This change improves reliability
by closing all database connections immediately when the context is closed.
### API-Changes
- Remove `MimeMessage::from_bytes()` public interface. #4033
- BREAKING Types: jsonrpc: `get_messages` now returns a map with `MessageLoadResult` instead of failing completely if one of the requested messages could not be loaded. #4038
- Add `dc_msg_set_subject()`. C-FFI #4057
- Mark python bindings as supporting typing according to PEP 561 #4045
## 1.108.0

129
Cargo.lock generated
View File

@@ -239,7 +239,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acee9fd5073ab6b045a275b3e709c163dd36c90685219cb21804a147b58dba43"
dependencies = [
"async-trait",
"axum-core 0.2.9",
"axum-core",
"base64 0.13.1",
"bitflags",
"bytes",
@@ -248,7 +248,7 @@ dependencies = [
"http-body",
"hyper",
"itoa",
"matchit 0.5.0",
"matchit",
"memchr",
"mime",
"percent-encoding",
@@ -259,43 +259,7 @@ dependencies = [
"sha-1",
"sync_wrapper",
"tokio",
"tokio-tungstenite 0.17.2",
"tower",
"tower-http",
"tower-layer",
"tower-service",
]
[[package]]
name = "axum"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5694b64066a2459918d8074c2ce0d5a88f409431994c2356617c8ae0c4721fc"
dependencies = [
"async-trait",
"axum-core 0.3.2",
"base64 0.20.0",
"bitflags",
"bytes",
"futures-util",
"http",
"http-body",
"hyper",
"itoa",
"matchit 0.7.0",
"memchr",
"mime",
"percent-encoding",
"pin-project-lite",
"rustversion",
"serde",
"serde_json",
"serde_path_to_error",
"serde_urlencoded",
"sha1",
"sync_wrapper",
"tokio",
"tokio-tungstenite 0.18.0",
"tokio-tungstenite",
"tower",
"tower-http",
"tower-layer",
@@ -318,23 +282,6 @@ dependencies = [
"tower-service",
]
[[package]]
name = "axum-core"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1cae3e661676ffbacb30f1a824089a8c9150e71017f7e1e38f2aa32009188d34"
dependencies = [
"async-trait",
"bytes",
"futures-util",
"http",
"http-body",
"mime",
"rustversion",
"tower-layer",
"tower-service",
]
[[package]]
name = "backtrace"
version = "0.3.67"
@@ -368,12 +315,6 @@ version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
[[package]]
name = "base64"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5"
[[package]]
name = "base64"
version = "0.21.0"
@@ -893,7 +834,7 @@ checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb"
[[package]]
name = "deltachat"
version = "1.108.0"
version = "1.109.0"
dependencies = [
"ansi_term",
"anyhow",
@@ -907,6 +848,7 @@ dependencies = [
"bitflags",
"chrono",
"criterion",
"crossbeam-queue",
"deltachat_derive",
"email",
"encoded-words",
@@ -928,7 +870,6 @@ dependencies = [
"num-traits",
"num_cpus",
"once_cell",
"parking_lot",
"percent-encoding",
"pgp",
"pretty_env_logger",
@@ -965,11 +906,11 @@ dependencies = [
[[package]]
name = "deltachat-jsonrpc"
version = "1.108.0"
version = "1.109.0"
dependencies = [
"anyhow",
"async-channel",
"axum 0.6.4",
"axum",
"deltachat",
"env_logger 0.10.0",
"futures",
@@ -987,7 +928,7 @@ dependencies = [
[[package]]
name = "deltachat-repl"
version = "1.108.0"
version = "1.109.0"
dependencies = [
"ansi_term",
"anyhow",
@@ -1002,7 +943,7 @@ dependencies = [
[[package]]
name = "deltachat-rpc-server"
version = "1.108.0"
version = "1.109.0"
dependencies = [
"anyhow",
"deltachat-jsonrpc",
@@ -1025,7 +966,7 @@ dependencies = [
[[package]]
name = "deltachat_ffi"
version = "1.108.0"
version = "1.109.0"
dependencies = [
"anyhow",
"deltachat",
@@ -2158,12 +2099,6 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb"
[[package]]
name = "matchit"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40"
[[package]]
name = "md-5"
version = "0.10.5"
@@ -3220,15 +3155,6 @@ dependencies = [
"serde",
]
[[package]]
name = "serde_path_to_error"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "184c643044780f7ceb59104cef98a5a6f12cb2288a7bc701ab93a362b49fd47d"
dependencies = [
"serde",
]
[[package]]
name = "serde_spanned"
version = "0.6.1"
@@ -3638,19 +3564,7 @@ dependencies = [
"futures-util",
"log",
"tokio",
"tungstenite 0.17.3",
]
[[package]]
name = "tokio-tungstenite"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54319c93411147bced34cb5609a80e0a8e44c5999c93903a81cd866630ec0bfd"
dependencies = [
"futures-util",
"log",
"tokio",
"tungstenite 0.18.0",
"tungstenite",
]
[[package]]
@@ -3860,25 +3774,6 @@ dependencies = [
"utf-8",
]
[[package]]
name = "tungstenite"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "30ee6ab729cd4cf0fd55218530c4522ed30b7b6081752839b68fcec8d0960788"
dependencies = [
"base64 0.13.1",
"byteorder",
"bytes",
"http",
"httparse",
"log",
"rand 0.8.5",
"sha1",
"thiserror",
"url",
"utf-8",
]
[[package]]
name = "twofish"
version = "0.7.1"
@@ -4339,7 +4234,7 @@ dependencies = [
"async-channel",
"async-mutex",
"async-trait",
"axum 0.5.17",
"axum",
"futures",
"futures-util",
"log",

View File

@@ -1,6 +1,6 @@
[package]
name = "deltachat"
version = "1.108.0"
version = "1.109.0"
edition = "2021"
license = "MPL-2.0"
rust-version = "1.63"
@@ -29,21 +29,24 @@ format-flowed = { path = "./format-flowed" }
ratelimit = { path = "./deltachat-ratelimit" }
anyhow = "1"
async-channel = "1.8.0"
async-imap = { git = "https://github.com/async-email/async-imap", branch = "master", default-features = false, features = ["runtime-tokio"] }
async-native-tls = { version = "0.4", default-features = false, features = ["runtime-tokio"] }
async-smtp = { version = "0.8", default-features = false, features = ["runtime-tokio"] }
trust-dns-resolver = "0.22"
tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar
async_zip = { version = "0.0.9", default-features = false, features = ["deflate"] }
backtrace = "0.3"
base64 = "0.21"
bitflags = "1.3"
chrono = { version = "0.4", default-features=false, features = ["clock", "std"] }
crossbeam-queue = "0.3"
email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
encoded-words = { git = "https://github.com/async-email/encoded-words", branch = "master" }
escaper = "0.1"
fast-socks5 = "0.8"
futures = "0.3"
futures-lite = "1.12.0"
hex = "0.4.0"
humansize = "2"
image = { version = "0.24.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
kamadak-exif = "0.5"
lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
@@ -54,14 +57,15 @@ num_cpus = "1.15"
num-derive = "0.3"
num-traits = "0.2"
once_cell = "1.17.0"
parking_lot = "0.12"
percent-encoding = "2.2"
pgp = { version = "0.9", default-features = false }
pretty_env_logger = { version = "0.4", optional = true }
qrcodegen = "1.7.0"
quick-xml = "0.27"
rand = "0.8"
regex = "1.7"
rusqlite = { version = "0.28", features = ["sqlcipher", "release_memory"] }
reqwest = { version = "0.11.14", features = ["json"] }
rusqlite = { version = "0.28", features = ["sqlcipher"] }
rust-hsluv = "0.1"
sanitize-filename = "0.4"
serde_json = "1.0"
@@ -71,21 +75,17 @@ sha2 = "0.10"
smallvec = "1"
strum = "0.24"
strum_macros = "0.24"
thiserror = "1"
toml = "0.7"
url = "2"
uuid = { version = "1", features = ["serde", "v4"] }
fast-socks5 = "0.8"
humansize = "2"
qrcodegen = "1.7.0"
tagger = "4.3.4"
textwrap = "0.16.0"
async-channel = "1.8.0"
futures-lite = "1.12.0"
tokio-stream = { version = "0.1.11", features = ["fs"] }
thiserror = "1"
tokio-io-timeout = "1.2.0"
reqwest = { version = "0.11.14", features = ["json"] }
async_zip = { version = "0.0.9", default-features = false, features = ["deflate"] }
tokio-stream = { version = "0.1.11", features = ["fs"] }
tokio-tar = { version = "0.3" } # TODO: integrate tokio into async-tar
tokio = { version = "1", features = ["fs", "rt-multi-thread", "macros"] }
toml = "0.7"
trust-dns-resolver = "0.22"
url = "2"
uuid = { version = "1", features = ["serde", "v4"] }
[dev-dependencies]
ansi_term = "0.12.0"

View File

@@ -1,6 +1,6 @@
[package]
name = "deltachat_ffi"
version = "1.108.0"
version = "1.109.0"
description = "Deltachat FFI"
edition = "2018"
readme = "README.md"

View File

@@ -4327,6 +4327,18 @@ void dc_msg_set_text (dc_msg_t* msg, const char* text);
void dc_msg_set_html (dc_msg_t* msg, const char* html);
/**
* Sets the email's subject. If it's empty, a default subject
* will be used (e.g. `Message from Alice` or `Re: <last subject>`).
* This does not alter any information in the database.
*
* @memberof dc_msg_t
* @param msg The message object.
* @param subject The new subject.
*/
void dc_msg_set_subject (dc_msg_t* msg, const char* subject);
/**
* Set different sender name for a message.
* This overrides the name set by the dc_set_config()-option `displayname`.

View File

@@ -2003,12 +2003,10 @@ pub unsafe extern "C" fn dc_create_contact(
let ctx = &*context;
let name = to_string_lossy(name);
block_on(async move {
Contact::create(ctx, &name, &to_string_lossy(addr))
.await
.map(|id| id.to_u32())
.unwrap_or(0)
})
block_on(Contact::create(ctx, &name, &to_string_lossy(addr)))
.log_err(ctx, "Cannot create contact")
.map(|id| id.to_u32())
.unwrap_or(0)
}
#[no_mangle]
@@ -3598,6 +3596,16 @@ pub unsafe extern "C" fn dc_msg_set_html(msg: *mut dc_msg_t, html: *const libc::
ffi_msg.message.set_html(to_opt_string_lossy(html))
}
#[no_mangle]
pub unsafe extern "C" fn dc_msg_set_subject(msg: *mut dc_msg_t, subject: *const libc::c_char) {
if msg.is_null() {
eprintln!("ignoring careless call to dc_msg_get_subject()");
return;
}
let ffi_msg = &mut *msg;
ffi_msg.message.set_subject(to_string_lossy(subject));
}
#[no_mangle]
pub unsafe extern "C" fn dc_msg_set_override_sender_name(
msg: *mut dc_msg_t,

View File

@@ -1,6 +1,6 @@
[package]
name = "deltachat-jsonrpc"
version = "1.108.0"
version = "1.109.0"
description = "DeltaChat JSON-RPC API"
edition = "2021"
default-run = "deltachat-jsonrpc-server"
@@ -28,7 +28,7 @@ sanitize-filename = "0.4"
walkdir = "2.3.2"
# optional dependencies
axum = { version = "0.6.4", optional = true, features = ["ws"] }
axum = { version = "0.5.9", optional = true, features = ["ws"] }
env_logger = { version = "0.10.0", optional = true }
[dev-dependencies]

View File

@@ -48,5 +48,5 @@
},
"type": "module",
"types": "dist/deltachat.d.ts",
"version": "1.108.0"
"version": "1.109.0"
}

View File

@@ -1,6 +1,6 @@
[package]
name = "deltachat-repl"
version = "1.108.0"
version = "1.109.0"
edition = "2021"
[dependencies]

View File

@@ -1,6 +1,6 @@
[package]
name = "deltachat-rpc-server"
version = "1.108.0"
version = "1.109.0"
description = "DeltaChat JSON-RPC server"
edition = "2021"
readme = "README.md"

View File

@@ -60,5 +60,5 @@
"test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit"
},
"types": "node/dist/index.d.ts",
"version": "1.108.0"
"version": "1.109.0"
}

View File

@@ -55,3 +55,5 @@ line-length = 120
[tool.isort]
profile = "black"
[tool.pyright]

View File

@@ -284,9 +284,9 @@ class Account:
:returns: :class:`deltachat.contact.Contact` instance.
"""
(name, addr) = self.get_contact_addr_and_name(obj, name)
name = as_dc_charpointer(name)
addr = as_dc_charpointer(addr)
contact_id = lib.dc_create_contact(self._dc_context, name, addr)
name_c = as_dc_charpointer(name)
addr_c = as_dc_charpointer(addr)
contact_id = lib.dc_create_contact(self._dc_context, name_c, addr_c)
return Contact(self, contact_id)
def get_contact(self, obj) -> Optional[Contact]:
@@ -363,12 +363,12 @@ class Account:
:returns: list of :class:`deltachat.contact.Contact` objects.
"""
flags = 0
query = as_dc_charpointer(query)
query_c = as_dc_charpointer(query)
if only_verified:
flags |= const.DC_GCL_VERIFIED_ONLY
if with_self:
flags |= const.DC_GCL_ADD_SELF
dc_array = ffi.gc(lib.dc_get_contacts(self._dc_context, flags, query), lib.dc_array_unref)
dc_array = ffi.gc(lib.dc_get_contacts(self._dc_context, flags, query_c), lib.dc_array_unref)
return list(iter_array(dc_array, lambda x: Contact(self, x)))
def get_fresh_messages(self) -> Generator[Message, None, None]:

View File

@@ -162,8 +162,8 @@ class Chat:
:param name: as a unicode string.
:returns: True on success, False otherwise
"""
name = as_dc_charpointer(name)
return bool(lib.dc_set_chat_name(self.account._dc_context, self.id, name))
name_c = as_dc_charpointer(name)
return bool(lib.dc_set_chat_name(self.account._dc_context, self.id, name_c))
def get_color(self):
"""return the color of the chat.
@@ -463,9 +463,10 @@ class Chat:
:returns: None
"""
contact = self.account.get_contact(obj)
ret = lib.dc_remove_contact_from_chat(self.account._dc_context, self.id, contact.id)
if ret != 1:
raise ValueError(f"could not remove contact {contact!r} from chat")
if contact:
ret = lib.dc_remove_contact_from_chat(self.account._dc_context, self.id, contact.id)
if ret != 1:
raise ValueError("could not remove contact {contact!r} from chat")
def get_contacts(self):
"""get all contacts for this chat.

View File

@@ -147,7 +147,7 @@ class DirectImap:
log("Message", msg.uid, "has empty body")
continue
path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
path = pathlib.Path(str(dir)).joinpath("IMAP", str(self.logid), imapfolder)
path.mkdir(parents=True, exist_ok=True)
fn = path.joinpath(str(msg.uid))
fn.write_bytes(body)
@@ -207,14 +207,14 @@ class IdleManager:
return res
def wait_for_new_message(self, timeout=None) -> bytes:
while 1:
while True:
for item in self.check(timeout=timeout):
if b"EXISTS" in item or b"RECENT" in item:
return item
def wait_for_seen(self, timeout=None) -> int:
"""Return first message with SEEN flag from a running idle-stream."""
while 1:
while True:
for item in self.check(timeout=timeout):
if FETCH in item:
self.log(str(item))

View File

@@ -108,25 +108,27 @@ class FFIEventTracker:
return ev
def iter_events(self, timeout=None, check_error=True):
while 1:
while True:
yield self.get(timeout=timeout, check_error=check_error)
def get_matching(self, event_name_regex, check_error=True, timeout=None):
def get_matching(self, event_name_regex, check_error=True, timeout=None) -> FFIEvent:
rex = re.compile(f"^(?:{event_name_regex})$")
for ev in self.iter_events(timeout=timeout, check_error=check_error):
if rex.match(ev.name):
return ev
assert False
def get_info_contains(self, regex: str) -> FFIEvent:
rex = re.compile(regex)
while 1:
while True:
ev = self.get_matching("DC_EVENT_INFO")
if rex.search(ev.data2):
return ev
assert False
def get_info_regex_groups(self, regex, check_error=True):
rex = re.compile(regex)
while 1:
while True:
ev = self.get_matching("DC_EVENT_INFO", check_error=check_error)
m = rex.match(ev.data2)
if m is not None:
@@ -137,7 +139,7 @@ class FFIEventTracker:
This only works reliably if the connectivity doesn't change
again too quickly, otherwise we might miss it.
"""
while 1:
while True:
if self.account.get_connectivity() == connectivity:
return
self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
@@ -146,7 +148,7 @@ class FFIEventTracker:
"""Wait until the connectivity changes to `expected_next`.
Fails the test if it changes to something else.
"""
while 1:
while True:
current = self.account.get_connectivity()
if current == expected_next:
return
@@ -156,7 +158,7 @@ class FFIEventTracker:
self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
def wait_for_all_work_done(self):
while 1:
while True:
if self.account.all_work_done():
return
self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
@@ -164,7 +166,7 @@ class FFIEventTracker:
def ensure_event_not_queued(self, event_name_regex):
__tracebackhide__ = True
rex = re.compile(f"(?:{event_name_regex}).*")
while 1:
while True:
try:
ev = self._event_queue.get(False)
except Empty:
@@ -173,7 +175,7 @@ class FFIEventTracker:
assert not rex.match(ev.name), f"event found {ev}"
def wait_securejoin_inviter_progress(self, target):
while 1:
while True:
event = self.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS")
if event.data2 >= target:
print(f"** SECUREJOINT-INVITER PROGRESS {target}", self.account)

View File

@@ -250,7 +250,7 @@ def data(request):
fn = os.path.join(path, *bn.split("/"))
if os.path.exists(fn):
return fn
print(f"WARNING: path does not exist: {fn!r}")
print(f"WARNING: path does not exist: {request!r}")
return None
def read_path(self, bn, mode="r"):
@@ -309,7 +309,7 @@ class ACSetup:
def wait_one_configured(self, account):
"""wait until this account has successfully configured."""
if self._account2state[account] == self.CONFIGURING:
while 1:
while True:
acc = self._pop_config_success()
if acc == account:
break
@@ -638,7 +638,7 @@ class BotProcess:
def _run_stdout_thread(self) -> None:
try:
while 1:
while True:
line = self.popen.stdout.readline()
if not line:
break
@@ -659,7 +659,7 @@ class BotProcess:
for next_pattern in patterns:
print("+++FNMATCH:", next_pattern)
ignored = []
while 1:
while True:
line = self.stdout_queue.get()
if line is None:
if ignored:

View File

@@ -85,7 +85,7 @@ class ConfigureTracker:
self._imap_finished.wait()
def wait_progress(self, data1=None):
while 1:
while True:
evdata = self._progress.get()
if data1 is None or evdata == data1:
break

View File

@@ -1,6 +1,7 @@
import os.path
import shutil
from filecmp import cmp
from typing import Optional
import pytest
@@ -13,7 +14,7 @@ def wait_msg_delivered(account, msg_list):
msg_list.remove((ev.data1, ev.data2))
def wait_msgs_changed(account, msgs_list):
def wait_msgs_changed(account, msgs_list: list[tuple[int, Optional[int]]]):
"""wait for one or more MSGS_CHANGED events to match msgs_list contents."""
account.log(f"waiting for msgs_list={msgs_list}")
msgs_list = list(msgs_list)

View File

@@ -5,10 +5,11 @@ envlist =
lint
mypy
auditwheels
pyright
[testenv]
commands =
pytest -n6 --extra-info --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
pytest -n6 --extra-info -v -rsXx --ignored --strict-tls {posargs: tests examples}
pip wheel . -w {toxworkdir}/wheelhouse --no-deps
setenv =
# Avoid stack overflow when Rust core is built without optimizations.
@@ -21,11 +22,11 @@ passenv =
RUSTC_WRAPPER
deps =
pytest
pytest-rerunfailures
pytest-timeout
pytest-xdist
pdbpp
requests
pyright: pyright
[testenv:.pkg]
passenv =
@@ -69,6 +70,10 @@ deps =
commands =
mypy --no-incremental src/
[testenv:pyright]
commands =
pyright
[testenv:doc]
changedir=doc
deps =

View File

@@ -1,7 +1,6 @@
//! # Account manager module.
use std::collections::BTreeMap;
use std::future::Future;
use std::path::{Path, PathBuf};
use anyhow::{ensure, Context as _, Result};
@@ -151,7 +150,7 @@ impl Accounts {
if let Some(cfg) = self.config.get_account(id) {
let account_path = self.dir.join(cfg.dir);
try_many_times(|| fs::remove_dir_all(&account_path))
fs::remove_dir_all(&account_path)
.await
.context("failed to remove account data")?;
}
@@ -187,10 +186,10 @@ impl Accounts {
fs::create_dir_all(self.dir.join(&account_config.dir))
.await
.context("failed to create dir")?;
try_many_times(|| fs::rename(&dbfile, &new_dbfile))
fs::rename(&dbfile, &new_dbfile)
.await
.context("failed to rename dbfile")?;
try_many_times(|| fs::rename(&blobdir, &new_blobdir))
fs::rename(&blobdir, &new_blobdir)
.await
.context("failed to rename blobdir")?;
if walfile.exists() {
@@ -215,7 +214,7 @@ impl Accounts {
}
Err(err) => {
let account_path = std::path::PathBuf::from(&account_config.dir);
try_many_times(|| fs::remove_dir_all(&account_path))
fs::remove_dir_all(&account_path)
.await
.context("failed to remove account data")?;
self.config.remove_account(account_config.id).await?;
@@ -472,33 +471,6 @@ impl Config {
}
}
/// Spend up to 1 minute trying to do the operation.
///
/// Files may remain locked up to 30 seconds due to r2d2 bug:
/// <https://github.com/sfackler/r2d2/issues/99>
async fn try_many_times<F, Fut, T>(f: F) -> std::result::Result<(), T>
where
F: Fn() -> Fut,
Fut: Future<Output = std::result::Result<(), T>>,
{
let mut counter = 0;
loop {
counter += 1;
if let Err(err) = f().await {
if counter > 60 {
return Err(err);
}
// Wait 1 second and try again.
tokio::time::sleep(std::time::Duration::from_millis(1000)).await;
} else {
break;
}
}
Ok(())
}
/// Configuration of a single account.
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
struct AccountConfig {

View File

@@ -3397,7 +3397,7 @@ pub async fn forward_msgs(context: &Context, msg_ids: &[MsgId], chat_id: ChatId)
msg.param.remove(Param::WebxdcSummaryTimestamp);
msg.in_reply_to = None;
// do not leak data as group names; a default subject is generated by mimfactory
// do not leak data as group names; a default subject is generated by mimefactory
msg.subject = "".to_string();
let new_msg_id: MsgId;

View File

@@ -13,6 +13,7 @@ use async_channel::{self as channel, Receiver, Sender};
use deltachat_derive::{FromSql, ToSql};
use once_cell::sync::Lazy;
use regex::Regex;
use rusqlite::OptionalExtension;
use serde::{Deserialize, Serialize};
use tokio::task;
use tokio::time::{timeout, Duration};
@@ -520,8 +521,6 @@ impl Contact {
/// Depending on the origin, both, "row_name" and "row_authname" are updated from "name".
///
/// Returns the contact_id and a `Modifier` value indicating if a modification occurred.
///
/// Returns None if the contact with such address cannot exist.
pub(crate) async fn add_or_lookup(
context: &Context,
name: &str,
@@ -566,14 +565,12 @@ impl Contact {
);
let mut update_addr = false;
let mut row_id = 0;
if let Some((id, row_name, row_addr, row_origin, row_authname)) = context
.sql
.query_row_optional(
"SELECT id, name, addr, origin, authname \
FROM contacts WHERE addr=? COLLATE NOCASE;",
paramsv![addr.to_string()],
let row_id = context.sql.transaction(|transaction| {
let row = transaction.query_row(
"SELECT id, name, addr, origin, authname
FROM contacts WHERE addr=? COLLATE NOCASE",
[addr.to_string()],
|row| {
let row_id: isize = row.get(0)?;
let row_name: String = row.get(1)?;
@@ -582,120 +579,130 @@ impl Contact {
let row_authname: String = row.get(4)?;
Ok((row_id, row_name, row_addr, row_origin, row_authname))
},
)
.await?
{
let update_name = manual && name != row_name;
let update_authname = !manual
&& name != row_authname
&& !name.is_empty()
&& (origin >= row_origin
|| origin == Origin::IncomingUnknownFrom
|| row_authname.is_empty());
}).optional()?;
row_id = u32::try_from(id)?;
if origin >= row_origin && addr.as_ref() != row_addr {
update_addr = true;
}
if update_name || update_authname || update_addr || origin > row_origin {
let new_name = if update_name {
name.to_string()
} else {
row_name
};
let row_id;
if let Some((id, row_name, row_addr, row_origin, row_authname)) = row {
let update_name = manual && name != row_name;
let update_authname = !manual
&& name != row_authname
&& !name.is_empty()
&& (origin >= row_origin
|| origin == Origin::IncomingUnknownFrom
|| row_authname.is_empty());
context
.sql
.execute(
"UPDATE contacts SET name=?, addr=?, origin=?, authname=? WHERE id=?;",
paramsv![
new_name,
if update_addr {
addr.to_string()
} else {
row_addr
},
if origin > row_origin {
origin
} else {
row_origin
},
if update_authname {
name.to_string()
} else {
row_authname
},
row_id
],
)
.await
.ok();
row_id = u32::try_from(id)?;
if origin >= row_origin && addr.as_ref() != row_addr {
update_addr = true;
}
if update_name || update_authname || update_addr || origin > row_origin {
let new_name = if update_name {
name.to_string()
} else {
row_name
};
if update_name || update_authname {
// Update the contact name also if it is used as a group name.
// This is one of the few duplicated data, however, getting the chat list is easier this way.
let chat_id: Option<i32> = context.sql.query_get_value(
"SELECT id FROM chats WHERE type=? AND id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)",
paramsv![Chattype::Single, isize::try_from(row_id)?]
).await?;
if let Some(chat_id) = chat_id {
let contact = Contact::get_by_id(context, ContactId::new(row_id)).await?;
let chat_name = contact.get_display_name();
match context
.sql
.execute(
"UPDATE chats SET name=?1 WHERE id=?2 AND name!=?3",
paramsv![chat_name, chat_id, chat_name],
)
.await
{
Err(err) => warn!(context, "Can't update chat name: {}", err),
Ok(count) => {
if count > 0 {
// Chat name updated
context.emit_event(EventType::ChatModified(ChatId::new(
chat_id.try_into()?,
)));
}
transaction
.execute(
"UPDATE contacts SET name=?, addr=?, origin=?, authname=? WHERE id=?;",
paramsv![
new_name,
if update_addr {
addr.to_string()
} else {
row_addr
},
if origin > row_origin {
origin
} else {
row_origin
},
if update_authname {
name.to_string()
} else {
row_authname
},
row_id
],
)?;
if update_name || update_authname {
// Update the contact name also if it is used as a group name.
// This is one of the few duplicated data, however, getting the chat list is easier this way.
let chat_id: Option<ChatId> = transaction.query_row(
"SELECT id FROM chats WHERE type=? AND id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)",
params![Chattype::Single, isize::try_from(row_id)?],
|row| {
let chat_id: ChatId = row.get(0)?;
Ok(chat_id)
}
).optional()?;
if let Some(chat_id) = chat_id {
let contact_id = ContactId::new(row_id);
let (addr, name, authname) =
transaction.query_row(
"SELECT addr, name, authname
FROM contacts
WHERE id=?",
params![contact_id],
|row| {
let addr: String = row.get(0)?;
let name: String = row.get(1)?;
let authname: String = row.get(2)?;
Ok((addr, name, authname))
})?;
let chat_name = if !name.is_empty() {
name
} else if !authname.is_empty() {
authname
} else {
addr
};
let count = transaction.execute(
"UPDATE chats SET name=?1 WHERE id=?2 AND name!=?1",
params![chat_name, chat_id])?;
if count > 0 {
// Chat name updated
context.emit_event(EventType::ChatModified(chat_id));
}
}
}
sth_modified = Modifier::Modified;
}
sth_modified = Modifier::Modified;
}
} else {
let update_name = manual;
let update_authname = !manual;
if let Ok(new_row_id) = context
.sql
.insert(
"INSERT INTO contacts (name, addr, origin, authname) VALUES(?, ?, ?, ?);",
paramsv![
if update_name {
name.to_string()
} else {
"".to_string()
},
addr,
origin,
if update_authname {
name.to_string()
} else {
"".to_string()
}
],
)
.await
{
row_id = u32::try_from(new_row_id)?;
sth_modified = Modifier::Created;
info!(context, "added contact id={} addr={}", row_id, &addr);
} else {
error!(context, "Cannot add contact.");
let update_name = manual;
let update_authname = !manual;
transaction
.execute(
"INSERT INTO contacts (name, addr, origin, authname)
VALUES (?, ?, ?, ?);",
params![
if update_name {
name.to_string()
} else {
"".to_string()
},
addr,
origin,
if update_authname {
name.to_string()
} else {
"".to_string()
}
],
)?;
sth_modified = Modifier::Created;
row_id = u32::try_from(transaction.last_insert_rowid())?;
info!(context, "added contact id={} addr={}", row_id, &addr);
}
}
Ok(row_id)
}).await?;
Ok((ContactId::new(row_id), sth_modified))
}

View File

@@ -5,7 +5,7 @@ use std::ffi::OsStr;
use std::path::{Path, PathBuf};
use ::pgp::types::KeyTrait;
use anyhow::{bail, ensure, format_err, Context as _, Result};
use anyhow::{bail, ensure, format_err, Context as _, Error, Result};
use futures::StreamExt;
use futures_lite::FutureExt;
use rand::{thread_rng, Rng};
@@ -508,6 +508,9 @@ fn get_next_backup_path(folder: &Path, backup_time: i64) -> Result<(PathBuf, Pat
bail!("could not create backup file, disk full?");
}
/// Exports the database to a separate file with the given passphrase.
///
/// Set passphrase to empty string to export the database unencrypted.
async fn export_backup(context: &Context, dir: &Path, passphrase: String) -> Result<()> {
// get a fine backup file name (the name includes the date so that multiple backup instances are possible)
let now = time();
@@ -521,13 +524,6 @@ async fn export_backup(context: &Context, dir: &Path, passphrase: String) -> Res
.await?;
sql::housekeeping(context).await.ok_or_log(context);
context
.sql
.execute("VACUUM;", paramsv![])
.await
.map_err(|e| warn!(context, "Vacuum failed, exporting anyway {}", e))
.ok();
ensure!(
context.scheduler.read().await.is_none(),
"cannot export backup, IO is running"
@@ -540,11 +536,29 @@ async fn export_backup(context: &Context, dir: &Path, passphrase: String) -> Res
dest_path.display(),
);
context
.sql
.export(&temp_db_path, passphrase)
.await
.with_context(|| format!("failed to backup plaintext database to {temp_db_path:?}"))?;
let path_str = temp_db_path
.to_str()
.with_context(|| format!("path {temp_db_path:?} is not valid unicode"))?;
let conn = context.sql.get_conn().await?;
tokio::task::block_in_place(move || {
if let Err(err) = conn.execute("VACUUM", params![]) {
info!(context, "Vacuum failed, exporting anyway: {:#}.", err);
}
conn.execute(
"ATTACH DATABASE ? AS backup KEY ?",
paramsv![path_str, passphrase],
)
.context("failed to attach backup database")?;
let res = conn
.query_row("SELECT sqlcipher_export('backup')", [], |_row| Ok(()))
.context("failed to export to attached backup database");
conn.execute("DETACH DATABASE backup", [])
.context("failed to detach backup database")?;
res?;
Ok::<_, Error>(())
})?;
let res = export_backup_inner(context, &temp_db_path, &temp_path).await;

View File

@@ -265,6 +265,8 @@ pub struct Message {
pub(crate) text: Option<String>,
/// Message subject.
///
/// If empty, a default subject will be generated when sending.
pub(crate) subject: String,
/// `Message-ID` header value.
@@ -795,6 +797,12 @@ impl Message {
self.text = text;
}
/// Sets the email's subject. If it's empty, a default subject
/// will be used (e.g. `Message from Alice` or `Re: <last subject>`).
pub fn set_subject(&mut self, subject: String) {
self.subject = subject;
}
/// Sets the file associated with a message.
///
/// This function does not use the file or check if it exists,

View File

@@ -1612,6 +1612,22 @@ mod tests {
assert_eq!(maybe_encode_words("äöü"), "=?utf-8?b?w6TDtsO8?=");
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_manually_set_subject() -> Result<()> {
let t = TestContext::new_alice().await;
let chat = t.create_chat_with_contact("bob", "bob@example.org").await;
let mut msg = Message::new(Viewtype::Text);
msg.set_subject("Subjeeeeect".to_string());
let sent_msg = t.send_msg(chat.id, &mut msg).await;
let payload = sent_msg.payload();
assert_eq!(payload.match_indices("Subject: Subjeeeeect").count(), 1);
Ok(())
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_subject_from_mua() {
// 1.: Receive a mail from an MUA

View File

@@ -124,31 +124,6 @@ impl Sql {
// drop closes the connection
}
/// Exports the database to a separate file with the given passphrase.
///
/// Set passphrase to empty string to export the database unencrypted.
pub(crate) async fn export(&self, path: &Path, passphrase: String) -> Result<()> {
let path_str = path
.to_str()
.with_context(|| format!("path {path:?} is not valid unicode"))?;
let conn = self.get_conn().await?;
tokio::task::block_in_place(move || {
conn.execute(
"ATTACH DATABASE ? AS backup KEY ?",
paramsv![path_str, passphrase],
)
.context("failed to attach backup database")?;
let res = conn
.query_row("SELECT sqlcipher_export('backup')", [], |_row| Ok(()))
.context("failed to export to attached backup database");
conn.execute("DETACH DATABASE backup", [])
.context("failed to detach backup database")?;
res?;
Ok(())
})
}
/// Imports the database from a separate file with the given passphrase.
pub(crate) async fn import(&self, path: &Path, passphrase: String) -> Result<()> {
let path_str = path
@@ -363,7 +338,7 @@ impl Sql {
pub(crate) async fn get_conn(&self) -> Result<PooledConnection> {
let lock = self.pool.read().await;
let pool = lock.as_ref().context("no SQL connection")?;
let conn = pool.get();
let conn = pool.get().await?;
Ok(conn)
}

View File

@@ -1,21 +1,21 @@
//! Connection pool.
use std::fmt;
use std::ops::{Deref, DerefMut};
use std::sync::{Arc, Weak};
use parking_lot::{Condvar, Mutex};
use anyhow::{Context, Result};
use crossbeam_queue::ArrayQueue;
use rusqlite::Connection;
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
/// Inner connection pool.
#[derive(Debug)]
struct InnerPool {
/// Available connections.
connections: Mutex<Vec<Connection>>,
connections: ArrayQueue<Connection>,
/// Conditional variable to notify about added connections.
///
/// Used to wait for available connection when the pool is empty.
cond: Condvar,
/// Counts the number of available connections.
semaphore: Arc<Semaphore>,
}
impl InnerPool {
@@ -23,10 +23,7 @@ impl InnerPool {
///
/// The connection could be new or returned back.
fn put(&self, connection: Connection) {
let mut connections = self.connections.lock();
connections.push(connection);
drop(connections);
self.cond.notify_one();
self.connections.force_push(connection);
}
}
@@ -37,6 +34,9 @@ pub struct PooledConnection {
/// Only `None` right after moving the connection back to the pool.
conn: Option<Connection>,
/// Semaphore permit, dropped after returning the connection to the pool.
_permit: OwnedSemaphorePermit,
}
impl Drop for PooledConnection {
@@ -44,7 +44,6 @@ impl Drop for PooledConnection {
// Put the connection back unless the pool is already dropped.
if let Some(pool) = self.pool.upgrade() {
if let Some(conn) = self.conn.take() {
conn.release_memory().ok();
pool.put(conn);
}
}
@@ -66,41 +65,38 @@ impl DerefMut for PooledConnection {
}
/// Connection pool.
#[derive(Clone)]
#[derive(Clone, Debug)]
pub struct Pool {
/// Reference to the actual connection pool.
inner: Arc<InnerPool>,
}
impl fmt::Debug for Pool {
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
write!(fmt, "Pool")
}
}
impl Pool {
/// Creates a new connection pool.
pub fn new(connections: Vec<Connection>) -> Self {
let inner = Arc::new(InnerPool {
connections: Mutex::new(connections),
cond: Condvar::new(),
connections: ArrayQueue::new(connections.len()),
semaphore: Arc::new(Semaphore::new(connections.len())),
});
for connection in connections {
inner.connections.force_push(connection);
}
Pool { inner }
}
/// Retrieves a connection from the pool.
pub fn get(&self) -> PooledConnection {
let mut connections = self.inner.connections.lock();
loop {
if let Some(conn) = connections.pop() {
return PooledConnection {
pool: Arc::downgrade(&self.inner),
conn: Some(conn),
};
}
self.inner.cond.wait(&mut connections);
}
pub async fn get(&self) -> Result<PooledConnection> {
let permit = self.inner.semaphore.clone().acquire_owned().await?;
let conn = self
.inner
.connections
.pop()
.context("got a permit when there are no connections in the pool")?;
let conn = PooledConnection {
pool: Arc::downgrade(&self.inner),
conn: Some(conn),
_permit: permit,
};
Ok(conn)
}
}