diff --git a/.gitignore b/.gitignore index ff914ab6b..1f574be4a 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,9 @@ python/.eggs __pycache__ python/src/deltachat/capi*.so python/.venv/ +python/venv/ +venv/ +env/ python/liveconfig* diff --git a/CHANGELOG.md b/CHANGELOG.md index 34866af2a..2b88db8a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,40 @@ # Changelog +## [1.125.0] - 2023-10-14 + +### API-Changes + +- [**breaking**] deltachat-rpc-client: Replace `asyncio` with threads. +- Validate boolean values passed to `set_config`. Attempts to set values other than `0` and `1` will result in an error. + +### CI + +- Reduce required Python version for deltachat-rpc-client from 3.8 to 3.7. + +### Features / Changes + +- Add developer option to disable IDLE. + +### Fixes + +- `deltachat-rpc-client`: Run `deltachat-rpc-server` in its own process group. This prevents reception of `SIGINT` by the server when the bot is terminated with `^C`. +- python: Don't automatically set the displayname to "bot" when setting log level. +- Don't update `timestamp`, `timestamp_rcvd`, `state` when replacing partially downloaded message ([#4700](https://github.com/deltachat/deltachat-core-rust/pull/4700)). +- Assign encrypted partially downloaded group messages to 1:1 chat ([#4757](https://github.com/deltachat/deltachat-core-rust/pull/4757)). +- Return all contacts from `Contact::get_all` for bots ([#4811](https://github.com/deltachat/deltachat-core-rust/pull/4811)). +- Set connectivity status to "connected" during fake idle. +- Return verifier contacts regardless of their origin. +- Don't try to send more MDNs if there's a temporary SMTP error ([#4534](https://github.com/deltachat/deltachat-core-rust/pull/4534)). + +### Refactor + +- deltachat-rpc-client: Close stdin instead of sending `SIGTERM`. +- deltachat-rpc-client: Remove print() calls. Standard `logging` package is for logging instead. + +### Tests + +- deltachat-rpc-client: Enable logs in pytest. + ## [1.124.1] - 2023-10-05 ### Fixes @@ -2879,3 +2914,4 @@ https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed [1.123.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.122.0...v1.123.0 [1.124.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.123.0...v1.124.0 [1.124.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.124.0...v1.124.1 +[1.125.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.124.1...v1.125.0 diff --git a/Cargo.lock b/Cargo.lock index 353e8609b..5e910fb96 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1085,7 +1085,7 @@ dependencies = [ [[package]] name = "deltachat" -version = "1.124.1" +version = "1.125.0" dependencies = [ "ansi_term", "anyhow", @@ -1162,7 +1162,7 @@ dependencies = [ [[package]] name = "deltachat-jsonrpc" -version = "1.124.1" +version = "1.125.0" dependencies = [ "anyhow", "async-channel", @@ -1186,7 +1186,7 @@ dependencies = [ [[package]] name = "deltachat-repl" -version = "1.124.1" +version = "1.125.0" dependencies = [ "ansi_term", "anyhow", @@ -1201,7 +1201,7 @@ dependencies = [ [[package]] name = "deltachat-rpc-server" -version = "1.124.1" +version = "1.125.0" dependencies = [ "anyhow", "deltachat", @@ -1226,7 +1226,7 @@ dependencies = [ [[package]] name = "deltachat_ffi" -version = "1.124.1" +version = "1.125.0" dependencies = [ "anyhow", "deltachat", diff --git a/Cargo.toml b/Cargo.toml index 8cb2c3b1b..341ae158a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat" -version = "1.124.1" +version = "1.125.0" edition = "2021" license = "MPL-2.0" rust-version = "1.67" diff --git a/deltachat-ffi/Cargo.toml b/deltachat-ffi/Cargo.toml index 9558bfeb1..e53f1d96f 100644 --- a/deltachat-ffi/Cargo.toml +++ b/deltachat-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat_ffi" -version = "1.124.1" +version = "1.125.0" description = "Deltachat FFI" edition = "2018" readme = "README.md" diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 4988f1e97..fb6051117 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -492,6 +492,9 @@ char* dc_get_blobdir (const dc_context_t* context); * - `fetch_existing_msgs` = 1=fetch most recent existing messages on configure (default), * 0=do not fetch existing messages on configure. * In both cases, existing recipients are added to the contact database. + * - `disable_idle` = 1=disable IMAP IDLE even if the server supports it, + * 0=use IMAP IDLE if the server supports it. + * This is a developer option used for testing polling used as an IDLE fallback. * - `download_limit` = Messages up to this number of bytes are downloaded automatically. * For larger messages, only the header is downloaded and a placeholder is shown. * These messages can be downloaded fully using dc_download_full_msg() later. @@ -500,6 +503,9 @@ char* dc_get_blobdir (const dc_context_t* context); * to not mess up with non-delivery-reports or read-receipts. * 0=no limit (default). * Changes affect future messages only. + * - `gossip_period` = How often to gossip Autocrypt keys in chats with multiple recipients, in + * seconds. 2 days by default. + * This is not supposed to be changed by UIs and only used for testing. * - `verified_one_on_one_chats` = Feature flag for verified 1:1 chats; the UI should set it * to 1 if it supports verified 1:1 chats. * Regardless of this setting, `dc_chat_is_protected()` returns true while the key is verified, diff --git a/deltachat-jsonrpc/Cargo.toml b/deltachat-jsonrpc/Cargo.toml index 50a0462fa..49744eb46 100644 --- a/deltachat-jsonrpc/Cargo.toml +++ b/deltachat-jsonrpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-jsonrpc" -version = "1.124.1" +version = "1.125.0" description = "DeltaChat JSON-RPC API" edition = "2021" default-run = "deltachat-jsonrpc-server" diff --git a/deltachat-jsonrpc/typescript/package.json b/deltachat-jsonrpc/typescript/package.json index 7c098b98a..4f3dd7b4d 100644 --- a/deltachat-jsonrpc/typescript/package.json +++ b/deltachat-jsonrpc/typescript/package.json @@ -55,5 +55,5 @@ }, "type": "module", "types": "dist/deltachat.d.ts", - "version": "1.124.1" + "version": "1.125.0" } diff --git a/deltachat-repl/Cargo.toml b/deltachat-repl/Cargo.toml index 7c165c1e0..c916b6a2c 100644 --- a/deltachat-repl/Cargo.toml +++ b/deltachat-repl/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-repl" -version = "1.124.1" +version = "1.125.0" license = "MPL-2.0" edition = "2021" diff --git a/deltachat-rpc-client/pyproject.toml b/deltachat-rpc-client/pyproject.toml index 9d7324ca6..d0253741b 100644 --- a/deltachat-rpc-client/pyproject.toml +++ b/deltachat-rpc-client/pyproject.toml @@ -7,7 +7,6 @@ name = "deltachat-rpc-client" description = "Python client for Delta Chat core JSON-RPC interface" classifiers = [ "Development Status :: 5 - Production/Stable", - "Framework :: AsyncIO", "Intended Audience :: Developers", "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", "Operating System :: POSIX :: Linux", @@ -72,3 +71,6 @@ line-length = 120 [tool.isort] profile = "black" + +[tool.pytest.ini_options] +log_cli = true diff --git a/deltachat-rpc-server/Cargo.toml b/deltachat-rpc-server/Cargo.toml index 4f30164cd..a609ecc9c 100644 --- a/deltachat-rpc-server/Cargo.toml +++ b/deltachat-rpc-server/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deltachat-rpc-server" -version = "1.124.1" +version = "1.125.0" description = "DeltaChat JSON-RPC server" edition = "2021" readme = "README.md" diff --git a/package.json b/package.json index 0682f85b5..05486b8fe 100644 --- a/package.json +++ b/package.json @@ -60,5 +60,5 @@ "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit" }, "types": "node/dist/index.d.ts", - "version": "1.124.1" + "version": "1.125.0" } diff --git a/python/src/deltachat/account.py b/python/src/deltachat/account.py index b53bbe7a0..69268e18c 100644 --- a/python/src/deltachat/account.py +++ b/python/src/deltachat/account.py @@ -617,18 +617,18 @@ class Account: # meta API for start/stop and event based processing # - def run_account(self, addr=None, password=None, account_plugins=None, show_ffi=False): - from .events import FFIEventLogger - + def run_account(self, addr=None, password=None, account_plugins=None, show_ffi=False, displayname=None): """get the account running, configure it if necessary. add plugins if provided. :param addr: the email address of the account :param password: the password of the account :param account_plugins: a list of plugins to add :param show_ffi: show low level ffi events + :param displayname: the display name of the account """ + from .events import FFIEventLogger + if show_ffi: - self.set_config("displayname", "bot") log = FFIEventLogger(self) self.add_account_plugin(log) @@ -644,6 +644,8 @@ class Account: configtracker = self.configure() configtracker.wait_finish() + if displayname: + self.set_config("displayname", displayname) # start IO threads and configure if necessary self.start_io() diff --git a/python/tests/test_0_complex_or_slow.py b/python/tests/test_0_complex_or_slow.py index 50ae55549..0c9256d98 100644 --- a/python/tests/test_0_complex_or_slow.py +++ b/python/tests/test_0_complex_or_slow.py @@ -542,8 +542,6 @@ def test_see_new_verified_member_after_going_online(acfactory, tmp_path, lp): assert msg_in.text == msg_out.text assert msg_in.get_sender_contact().addr == ac2_addr - ac1.set_config("bcc_self", "0") - def test_use_new_verified_group_after_going_online(acfactory, tmp_path, lp): """Another test for the bug #3836: @@ -592,4 +590,67 @@ def test_use_new_verified_group_after_going_online(acfactory, tmp_path, lp): assert msg_in.get_sender_contact().addr == ac2.get_config("addr") assert msg_in.text == msg_out.text - ac2.set_config("bcc_self", "0") + +def test_verified_group_vs_delete_server_after(acfactory, tmp_path, lp): + """Test for the issue #4346: + - User is added to a verified group. + - First device of the user downloads "member added" from the group. + - First device removes "member added" from the server. + - Some new messages are sent to the group. + - Second device comes online, receives these new messages. The result is a verified group with unverified members. + - First device re-gossips Autocrypt keys to the group. + - Now the seconds device has all members verified. + """ + ac1, ac2 = acfactory.get_online_accounts(2) + ac2_offl = acfactory.new_online_configuring_account(cloned_from=ac2) + for ac in [ac2, ac2_offl]: + ac.set_config("bcc_self", "1") + ac2.set_config("delete_server_after", "1") + ac2.set_config("gossip_period", "0") # Re-gossip in every message + acfactory.bring_accounts_online() + dir = tmp_path / "exportdir" + dir.mkdir() + ac2.export_self_keys(str(dir)) + ac2_offl.import_self_keys(str(dir)) + ac2_offl.stop_io() + + lp.sec("ac1: create verified-group QR, ac2 scans and joins") + chat1 = ac1.create_group_chat("hello", verified=True) + assert chat1.is_protected() + qr = chat1.get_join_qr() + lp.sec("ac2: start QR-code based join-group protocol") + chat2 = ac2.qr_join_chat(qr) + ac1._evtracker.wait_securejoin_inviter_progress(1000) + # Wait for "Member Me () added by ." message. + msg_in = ac2._evtracker.wait_next_incoming_message() + assert msg_in.is_system_message() + + lp.sec("ac2: waiting for 'member added' to be deleted on the server") + ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED") + + lp.sec("ac1: sending 'hi' to the group") + ac2.set_config("delete_server_after", "0") + chat1.send_text("hi") + + lp.sec("ac2_offl: going online, checking the 'hi' message") + ac2_offl.start_io() + msg_in = ac2_offl._evtracker.wait_next_incoming_message() + assert not msg_in.is_system_message() + assert msg_in.text.startswith("[Sender of this message is not verified:") + ac2_offl_ac1_contact = msg_in.get_sender_contact() + assert ac2_offl_ac1_contact.addr == ac1.get_config("addr") + assert not ac2_offl_ac1_contact.is_verified() + chat2_offl = msg_in.chat + assert chat2_offl.is_protected() + + lp.sec("ac2: sending message re-gossiping Autocrypt keys") + chat2.send_text("hi2") + + lp.sec("ac2_offl: receiving message") + ev = ac2_offl._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED") + msg_in = ac2_offl.get_message_by_id(ev.data2) + assert not msg_in.is_system_message() + assert msg_in.text == "hi2" + assert msg_in.chat == chat2_offl + assert msg_in.get_sender_contact().addr == ac2.get_config("addr") + assert ac2_offl_ac1_contact.is_verified() diff --git a/python/tests/test_1_online.py b/python/tests/test_1_online.py index 75bfe3a23..431463952 100644 --- a/python/tests/test_1_online.py +++ b/python/tests/test_1_online.py @@ -1718,12 +1718,10 @@ def test_qr_new_group_unblocked(acfactory, lp): ac1_new_chat = ac1.create_group_chat("Another group") ac1_new_chat.add_contact(ac2) - ac1_new_chat.send_text("Hello!") - # Receive "Member added" message. ac2._evtracker.wait_next_incoming_message() - # Receive "Hello!" message. + ac1_new_chat.send_text("Hello!") ac2_msg = ac2._evtracker.wait_next_incoming_message() assert ac2_msg.text == "Hello!" assert ac2_msg.chat.is_contact_request() @@ -1946,13 +1944,15 @@ def test_set_get_group_image(acfactory, data, lp): lp.sec("ac1: add ac2 to promoted group chat") chat.add_contact(ac2) # sends one message + lp.sec("ac2: wait for receiving member added message from ac1") + msg1 = ac2._evtracker.wait_next_incoming_message() + assert msg1.is_system_message() # Member added + lp.sec("ac1: send a first message to ac2") chat.send_text("hi") # sends another message assert chat.is_promoted() lp.sec("ac2: wait for receiving message from ac1") - msg1 = ac2._evtracker.wait_next_incoming_message() - assert msg1.is_system_message() # Member added msg2 = ac2._evtracker.wait_next_incoming_message() assert msg2.text == "hi" assert msg1.chat.id == msg2.chat.id diff --git a/release-date.in b/release-date.in index 741e6cba9..ac787018e 100644 --- a/release-date.in +++ b/release-date.in @@ -1 +1 @@ -2023-10-05 \ No newline at end of file +2023-10-14 \ No newline at end of file diff --git a/scripts/README.md b/scripts/README.md index 4ce3ddc32..18dc3bd0c 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -18,7 +18,7 @@ and an own build machine. - `remote_tests_rust.sh` rsyncs to the build machine and runs `run-rust-test.sh` remotely on the build machine. -- `make-python-testenv.sh` creates or updates local python test development environment. +- `make-python-testenv.sh` creates local python test development environment. Reusing the same environment is faster than running `run-python-test.sh` which always recreates environment from scratch and runs additional lints. diff --git a/scripts/make-python-testenv.sh b/scripts/make-python-testenv.sh index baff40d4c..64f959a68 100755 --- a/scripts/make-python-testenv.sh +++ b/scripts/make-python-testenv.sh @@ -4,8 +4,8 @@ # It rebuilds the core and bindings as needed. # # After running the script, you can either -# run `pytest` directly with `env/bin/pytest python/` -# or activate the environment with `. env/bin/activacte` +# run `pytest` directly with `venv/bin/pytest python/` +# or activate the environment with `. venv/bin/activate` # and run `pytest` from there. set -euo pipefail @@ -13,9 +13,5 @@ export DCC_RS_TARGET=debug export DCC_RS_DEV="$PWD" cargo build -p deltachat_ffi --features jsonrpc -if test -d env; then - env/bin/pip install -e python --force-reinstall -else - tox -e py --devenv env - env/bin/pip install --upgrade pip -fi +tox -c python -e py --devenv venv +env/bin/pip install --upgrade pip diff --git a/src/config.rs b/src/config.rs index 65c1a20c0..a5bf6fa70 100644 --- a/src/config.rs +++ b/src/config.rs @@ -286,6 +286,12 @@ pub enum Config { #[strum(props(default = "60"))] ScanAllFoldersDebounceSecs, + /// Whether to avoid using IMAP IDLE even if the server supports it. + /// + /// This is a developer option for testing "fake idle". + #[strum(props(default = "0"))] + DisableIdle, + /// Defines the max. size (in bytes) of messages downloaded automatically. /// 0 = no limit. #[strum(props(default = "0"))] @@ -313,6 +319,13 @@ pub enum Config { /// Last message processed by the bot. LastMsgId, + /// How often to gossip Autocrypt keys in chats with multiple recipients, in seconds. 2 days by + /// default. + /// + /// This is not supposed to be changed by UIs and only used for testing. + #[strum(props(default = "172800"))] + GossipPeriod, + /// Feature flag for verified 1:1 chats; the UI should set it /// to 1 if it supports verified 1:1 chats. /// Regardless of this setting, `chat.is_protected()` returns true while the key is verified, @@ -475,6 +488,28 @@ impl Context { .set_raw_config(key.as_ref(), value.as_deref()) .await?; } + Config::Socks5Enabled + | Config::BccSelf + | Config::E2eeEnabled + | Config::MdnsEnabled + | Config::SentboxWatch + | Config::MvboxMove + | Config::OnlyFetchMvbox + | Config::FetchExistingMsgs + | Config::DeleteToTrash + | Config::SaveMimeHeaders + | Config::Configured + | Config::Bot + | Config::NotifyAboutWrongPw + | Config::SendSyncMsgs + | Config::SignUnencrypted + | Config::DisableIdle => { + ensure!( + matches!(value, None | Some("0") | Some("1")), + "Boolean value must be either 0 or 1" + ); + self.sql.set_raw_config(key.as_ref(), value).await?; + } _ => { self.sql.set_raw_config(key.as_ref(), value).await?; } @@ -624,6 +659,18 @@ mod tests { ); } + /// Tests that "bot" config can only be set to "0" or "1". + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn test_set_config_bot() { + let t = TestContext::new().await; + + assert!(t.set_config(Config::Bot, None).await.is_ok()); + assert!(t.set_config(Config::Bot, Some("0")).await.is_ok()); + assert!(t.set_config(Config::Bot, Some("1")).await.is_ok()); + assert!(t.set_config(Config::Bot, Some("2")).await.is_err()); + assert!(t.set_config(Config::Bot, Some("Foobar")).await.is_err()); + } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn test_media_quality_config_option() { let t = TestContext::new().await; diff --git a/src/contact.rs b/src/contact.rs index 161fb571e..ccf6b29c2 100644 --- a/src/contact.rs +++ b/src/contact.rs @@ -812,7 +812,11 @@ impl Contact { let mut ret = Vec::new(); let flag_verified_only = (listflags & DC_GCL_VERIFIED_ONLY) != 0; let flag_add_self = (listflags & DC_GCL_ADD_SELF) != 0; - + let minimal_origin = if context.get_config_bool(Config::Bot).await? { + Origin::Unknown + } else { + Origin::IncomingReplyTo + }; if flag_verified_only || query.is_some() { let s3str_like_cmd = format!("%{}%", query.unwrap_or("")); context @@ -832,7 +836,7 @@ impl Contact { ), rusqlite::params_from_iter(params_iter(&self_addrs).chain(params_slice![ ContactId::LAST_SPECIAL, - Origin::IncomingReplyTo, + minimal_origin, s3str_like_cmd, s3str_like_cmd, if flag_verified_only { 0i32 } else { 1i32 } @@ -882,10 +886,10 @@ impl Contact { ORDER BY last_seen DESC, id DESC;", sql::repeat_vars(self_addrs.len()) ), - rusqlite::params_from_iter(params_iter(&self_addrs).chain(params_slice![ - ContactId::LAST_SPECIAL, - Origin::IncomingReplyTo - ])), + rusqlite::params_from_iter( + params_iter(&self_addrs) + .chain(params_slice![ContactId::LAST_SPECIAL, minimal_origin]), + ), |row| row.get::<_, ContactId>(0), |ids| { for id in ids { @@ -1245,7 +1249,7 @@ impl Contact { return Ok(Some(ContactId::SELF)); } - match Contact::lookup_id_by_addr(context, &verifier_addr, Origin::AddressBook).await? { + match Contact::lookup_id_by_addr(context, &verifier_addr, Origin::Unknown).await? { Some(contact_id) => Ok(Some(contact_id)), None => { let addr = &self.addr; diff --git a/src/context.rs b/src/context.rs index ef273f6a7..f4d6e9fb6 100644 --- a/src/context.rs +++ b/src/context.rs @@ -579,6 +579,7 @@ impl Context { let mdns_enabled = self.get_config_int(Config::MdnsEnabled).await?; let bcc_self = self.get_config_int(Config::BccSelf).await?; let send_sync_msgs = self.get_config_int(Config::SendSyncMsgs).await?; + let disable_idle = self.get_config_bool(Config::DisableIdle).await?; let prv_key_cnt = self.sql.count("SELECT COUNT(*) FROM keypairs;", ()).await?; @@ -691,6 +692,7 @@ impl Context { ); res.insert("bcc_self", bcc_self.to_string()); res.insert("send_sync_msgs", send_sync_msgs.to_string()); + res.insert("disable_idle", disable_idle.to_string()); res.insert("private_key_count", prv_key_cnt.to_string()); res.insert("public_key_count", pub_key_cnt.to_string()); res.insert("fingerprint", fingerprint_str); @@ -752,7 +754,6 @@ impl Context { .await? .to_string(), ); - res.insert( "debug_logging", self.get_config_int(Config::DebugLogging).await?.to_string(), @@ -761,6 +762,10 @@ impl Context { "last_msg_id", self.get_config_int(Config::LastMsgId).await?.to_string(), ); + res.insert( + "gossip_period", + self.get_config_int(Config::GossipPeriod).await?.to_string(), + ); res.insert( "verified_one_on_one_chats", self.get_config_bool(Config::VerifiedOneOnOneChats) diff --git a/src/download.rs b/src/download.rs index 4146cd966..285dce2e8 100644 --- a/src/download.rs +++ b/src/download.rs @@ -23,7 +23,7 @@ use crate::{job_try, stock_str, EventType}; /// eg. to assign them to the correct chat. /// As these messages are typically small, /// they're caught by `MIN_DOWNLOAD_LIMIT`. -const MIN_DOWNLOAD_LIMIT: u32 = 32768; +pub(crate) const MIN_DOWNLOAD_LIMIT: u32 = 32768; /// If a message is downloaded only partially /// and `delete_server_after` is set to small timeouts (eg. "at once"), diff --git a/src/imap.rs b/src/imap.rs index 96740a070..8f009f8ba 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -391,6 +391,7 @@ impl Imap { "IMAP-LOGIN as {}", self.config.lp.user ))); + self.connectivity.set_connected(context).await; info!(context, "Successfully logged into IMAP server"); Ok(()) } diff --git a/src/imap/idle.rs b/src/imap/idle.rs index fbce499b7..1cdc60bc9 100644 --- a/src/imap/idle.rs +++ b/src/imap/idle.rs @@ -7,7 +7,9 @@ use futures_lite::FutureExt; use super::session::Session; use super::Imap; +use crate::config::Config; use crate::imap::{client::IMAP_TIMEOUT, FolderMeaning}; +use crate::log::LogExt; use crate::{context::Context, scheduler::InterruptInfo}; const IDLE_TIMEOUT: Duration = Duration::from_secs(23 * 60); @@ -21,6 +23,10 @@ impl Session { ) -> Result<(Self, InterruptInfo)> { use futures::future::FutureExt; + if context.get_config_bool(Config::DisableIdle).await? { + bail!("IMAP IDLE is disabled"); + } + if !self.can_idle() { bail!("IMAP server does not have IDLE capability"); } @@ -163,7 +169,14 @@ impl Imap { continue; } if let Some(session) = &self.session { - if session.can_idle() { + if session.can_idle() + && !context + .get_config_bool(Config::DisableIdle) + .await + .context("Failed to get disable_idle config") + .log_err(context) + .unwrap_or_default() + { // we only fake-idled because network was gone during IDLE, probably break InterruptInfo::new(false); } diff --git a/src/mimefactory.rs b/src/mimefactory.rs index 46d4b1ed0..2ac272afc 100644 --- a/src/mimefactory.rs +++ b/src/mimefactory.rs @@ -359,9 +359,10 @@ impl<'a> MimeFactory<'a> { async fn should_do_gossip(&self, context: &Context) -> Result { match &self.loaded { Loaded::Message { chat } => { - // beside key- and member-changes, force re-gossip every 48 hours + // beside key- and member-changes, force a periodic re-gossip. let gossiped_timestamp = chat.id.get_gossiped_timestamp(context).await?; - if time() > gossiped_timestamp + (2 * 24 * 60 * 60) { + let gossip_period = context.get_config_i64(Config::GossipPeriod).await?; + if time() >= gossiped_timestamp + gossip_period { Ok(true) } else { let cmd = self.msg.param.get_cmd(); diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 502a4fa10..b8b6b0a3a 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -109,7 +109,7 @@ pub(crate) struct MimeMessage { /// The decrypted, raw mime structure. /// - /// This is non-empty only if the message was actually encrypted. It is used + /// This is non-empty iff `is_mime_modified` and the message was actually encrypted. It is used /// for e.g. late-parsing HTML. pub decoded_data: Vec, diff --git a/src/receive_imf.rs b/src/receive_imf.rs index 3d4e96ebf..14fa6f061 100644 --- a/src/receive_imf.rs +++ b/src/receive_imf.rs @@ -77,6 +77,24 @@ pub async fn receive_imf( let mail = parse_mail(imf_raw).context("can't parse mail")?; let rfc724_mid = imap::prefetch_get_message_id(&mail.headers).unwrap_or_else(imap::create_message_id); + if let Some(download_limit) = context.download_limit().await? { + let download_limit: usize = download_limit.try_into()?; + if imf_raw.len() > download_limit { + let head = std::str::from_utf8(imf_raw)? + .split("\r\n\r\n") + .next() + .context("No empty line in the message")?; + return receive_imf_inner( + context, + &rfc724_mid, + head.as_bytes(), + seen, + Some(imf_raw.len().try_into()?), + false, + ) + .await; + } + } receive_imf_inner(context, &rfc724_mid, imf_raw, seen, None, false).await } @@ -582,6 +600,7 @@ async fn add_parts( if let Some((new_chat_id, new_chat_id_blocked)) = create_or_lookup_group( context, mime_parser, + is_partial_download.is_some(), if test_normal_chat.is_none() { allow_creation } else { @@ -858,6 +877,7 @@ async fn add_parts( if let Some((new_chat_id, new_chat_id_blocked)) = create_or_lookup_group( context, mime_parser, + is_partial_download.is_some(), allow_creation, Blocked::Not, from_id, @@ -1213,8 +1233,8 @@ INSERT INTO msgs ) ON CONFLICT (id) DO UPDATE SET rfc724_mid=excluded.rfc724_mid, chat_id=excluded.chat_id, - from_id=excluded.from_id, to_id=excluded.to_id, timestamp=excluded.timestamp, timestamp_sent=excluded.timestamp_sent, - timestamp_rcvd=excluded.timestamp_rcvd, type=excluded.type, state=excluded.state, msgrmsg=excluded.msgrmsg, + from_id=excluded.from_id, to_id=excluded.to_id, timestamp_sent=excluded.timestamp_sent, + type=excluded.type, msgrmsg=excluded.msgrmsg, txt=excluded.txt, subject=excluded.subject, txt_raw=excluded.txt_raw, param=excluded.param, bytes=excluded.bytes, mime_headers=excluded.mime_headers, mime_compressed=excluded.mime_compressed, mime_in_reply_to=excluded.mime_in_reply_to, @@ -1545,6 +1565,7 @@ async fn is_probably_private_reply( async fn create_or_lookup_group( context: &Context, mime_parser: &mut MimeMessage, + is_partial_download: bool, allow_creation: bool, create_blocked: Blocked, from_id: ContactId, @@ -1677,7 +1698,7 @@ async fn create_or_lookup_group( if let Some(chat_id) = chat_id { Ok(Some((chat_id, chat_id_blocked))) - } else if mime_parser.decrypting_failed { + } else if is_partial_download || mime_parser.decrypting_failed { // It is possible that the message was sent to a valid, // yet unknown group, which was rejected because // Chat-Group-Name, which is in the encrypted part, was diff --git a/src/receive_imf/tests.rs b/src/receive_imf/tests.rs index 89c21c2c8..1b1476313 100644 --- a/src/receive_imf/tests.rs +++ b/src/receive_imf/tests.rs @@ -10,8 +10,9 @@ use crate::chat::{get_chat_msgs, ChatItem, ChatVisibility}; use crate::chatlist::Chatlist; use crate::config::Config; use crate::constants::{DC_GCL_FOR_FORWARDING, DC_GCL_NO_SPECIALS}; +use crate::download::{DownloadState, MIN_DOWNLOAD_LIMIT}; use crate::imap::prefetch_should_download; -use crate::message::Message; +use crate::message::{self, Message}; use crate::test_utils::{get_chat_msg, TestContext, TestContextManager}; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -2973,6 +2974,7 @@ async fn test_auto_accept_for_bots() -> Result<()> { let msg = t.get_last_msg().await; let chat = chat::Chat::load_from_db(&t, msg.chat_id).await?; assert!(!chat.is_contact_request()); + assert!(Contact::get_all(&t, 0, None).await?.len() == 1); Ok(()) } @@ -3697,3 +3699,114 @@ async fn test_keep_member_list_if_possibly_nomember() -> Result<()> { assert!(is_contact_in_chat(&bob, bob_chat_id, bob_alice_contact).await?); Ok(()) } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_download_later() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = tcm.alice().await; + alice.set_config(Config::DownloadLimit, Some("1")).await?; + assert_eq!(alice.download_limit().await?, Some(MIN_DOWNLOAD_LIMIT)); + + let bob = tcm.bob().await; + let bob_chat = bob.create_chat(&alice).await; + let text = String::from_utf8(vec![b'a'; MIN_DOWNLOAD_LIMIT as usize])?; + let sent_msg = bob.send_text(bob_chat.id, &text).await; + let msg = alice.recv_msg(&sent_msg).await; + assert_eq!(msg.download_state, DownloadState::Available); + assert_eq!(msg.state, MessageState::InFresh); + + let hi_msg = tcm.send_recv(&bob, &alice, "hi").await; + + alice.set_config(Config::DownloadLimit, None).await?; + let msg = alice.recv_msg(&sent_msg).await; + assert_eq!(msg.download_state, DownloadState::Done); + assert_eq!(msg.state, MessageState::InFresh); + assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, hi_msg.id); + assert!(msg.timestamp_sort <= hi_msg.timestamp_sort); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_create_group_with_big_msg() -> Result<()> { + let mut tcm = TestContextManager::new(); + let alice = tcm.alice().await; + let bob = tcm.bob().await; + let ba_contact = Contact::create( + &bob, + "alice", + &alice.get_config(Config::Addr).await?.unwrap(), + ) + .await?; + let file_bytes = include_bytes!("../../test-data/image/screenshot.png"); + + let bob_grp_id = create_group_chat(&bob, ProtectionStatus::Unprotected, "Group").await?; + add_contact_to_chat(&bob, bob_grp_id, ba_contact).await?; + let mut msg = Message::new(Viewtype::Image); + msg.set_file_from_bytes(&bob, "a.jpg", file_bytes, None) + .await?; + let sent_msg = bob.send_msg(bob_grp_id, &mut msg).await; + assert!(!msg.get_showpadlock()); + + alice.set_config(Config::DownloadLimit, Some("1")).await?; + assert_eq!(alice.download_limit().await?, Some(MIN_DOWNLOAD_LIMIT)); + let msg = alice.recv_msg(&sent_msg).await; + assert_eq!(msg.download_state, DownloadState::Available); + let alice_grp = Chat::load_from_db(&alice, msg.chat_id).await?; + assert_eq!(alice_grp.typ, Chattype::Group); + assert_eq!(alice_grp.name, "Group"); + assert_eq!( + chat::get_chat_contacts(&alice, alice_grp.id).await?.len(), + 2 + ); + + alice.set_config(Config::DownloadLimit, None).await?; + let msg = alice.recv_msg(&sent_msg).await; + assert_eq!(msg.download_state, DownloadState::Done); + assert_eq!(msg.state, MessageState::InFresh); + assert_eq!(msg.viewtype, Viewtype::Image); + assert_eq!(msg.chat_id, alice_grp.id); + let alice_grp = Chat::load_from_db(&alice, msg.chat_id).await?; + assert_eq!(alice_grp.typ, Chattype::Group); + assert_eq!(alice_grp.name, "Group"); + assert_eq!( + chat::get_chat_contacts(&alice, alice_grp.id).await?.len(), + 2 + ); + + let ab_chat_id = tcm.send_recv_accept(&alice, &bob, "hi").await.chat_id; + // Now Bob can send encrypted messages to Alice. + + let bob_grp_id = create_group_chat(&bob, ProtectionStatus::Unprotected, "Group1").await?; + add_contact_to_chat(&bob, bob_grp_id, ba_contact).await?; + let mut msg = Message::new(Viewtype::Image); + msg.set_file_from_bytes(&bob, "a.jpg", file_bytes, None) + .await?; + let sent_msg = bob.send_msg(bob_grp_id, &mut msg).await; + assert!(msg.get_showpadlock()); + + alice.set_config(Config::DownloadLimit, Some("1")).await?; + let msg = alice.recv_msg(&sent_msg).await; + assert_eq!(msg.download_state, DownloadState::Available); + // Until fully downloaded, an encrypted message must sit in the 1:1 chat. + assert_eq!(msg.chat_id, ab_chat_id); + + alice.set_config(Config::DownloadLimit, None).await?; + let msg = alice.recv_msg(&sent_msg).await; + assert_eq!(msg.download_state, DownloadState::Done); + assert_eq!(msg.state, MessageState::InFresh); + assert_eq!(msg.viewtype, Viewtype::Image); + assert_ne!(msg.chat_id, ab_chat_id); + let alice_grp = Chat::load_from_db(&alice, msg.chat_id).await?; + assert_eq!(alice_grp.typ, Chattype::Group); + assert_eq!(alice_grp.name, "Group1"); + assert_eq!( + chat::get_chat_contacts(&alice, alice_grp.id).await?.len(), + 2 + ); + + // The big message must go away from the 1:1 chat. + assert_eq!(alice.get_last_msg_in(ab_chat_id).await.text, "hi"); + + Ok(()) +} diff --git a/src/scheduler.rs b/src/scheduler.rs index 7ca697335..180f856dd 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -574,6 +574,19 @@ async fn fetch_idle( .await; } + if ctx + .get_config_bool(Config::DisableIdle) + .await + .context("Failed to get disable_idle config") + .log_err(ctx) + .unwrap_or_default() + { + info!(ctx, "IMAP IDLE is disabled, going to fake idle."); + return connection + .fake_idle(ctx, Some(watch_folder), folder_meaning) + .await; + } + info!(ctx, "IMAP session supports IDLE, using it."); match session .idle( diff --git a/src/smtp.rs b/src/smtp.rs index 457023623..c30f4626e 100644 --- a/src/smtp.rs +++ b/src/smtp.rs @@ -674,12 +674,14 @@ pub(crate) async fn send_smtp_messages(context: &Context, connection: &mut Smtp) /// On failure returns an error without removing any `smtp_mdns` entries, the caller is responsible /// for removing the corresponding entry to prevent endless loop in case the entry is invalid, e.g. /// points to non-existent message or contact. +/// +/// Returns true on success, false on temporary error. async fn send_mdn_msg_id( context: &Context, msg_id: MsgId, contact_id: ContactId, smtp: &mut Smtp, -) -> Result<()> { +) -> Result { let contact = Contact::get_by_id(context, contact_id).await?; if contact.is_blocked() { return Err(format_err!("Contact is blocked")); @@ -731,14 +733,14 @@ async fn send_mdn_msg_id( .execute(&q, rusqlite::params_from_iter(additional_msg_ids)) .await?; } - Ok(()) + Ok(true) } SendResult::Retry => { info!( context, "Temporary SMTP failure while sending an MDN for {}", msg_id ); - Ok(()) + Ok(false) } SendResult::Failure(err) => Err(err), } @@ -785,15 +787,20 @@ async fn send_mdn(context: &Context, smtp: &mut Smtp) -> Result { .await .context("failed to update MDN retries count")?; - if let Err(err) = send_mdn_msg_id(context, msg_id, contact_id, smtp).await { + let res = send_mdn_msg_id(context, msg_id, contact_id, smtp).await; + if let Err(ref err) = res { // If there is an error, for example there is no message corresponding to the msg_id in the // database, do not try to send this MDN again. + warn!( + context, + "Error sending MDN for {msg_id}, removing it: {err:#}." + ); context .sql .execute("DELETE FROM smtp_mdns WHERE msg_id = ?", (msg_id,)) .await?; - Err(err) - } else { - Ok(true) } + // If there's a temporary error, pretend there are no more MDNs to send. It's unlikely that + // other MDNs could be sent successfully in case of connectivity problems. + res }