Merge tag 'v1.125.0'

Release 1.125.0

.gitignore

@@ -18,6 +18,9 @@ python/.eggs
 __pycache__
 python/src/deltachat/capi*.so
 python/.venv/
+python/venv/
+venv/
+env/
 
 python/liveconfig*
 

CHANGELOG.md

@@ -1,5 +1,40 @@
 # Changelog
 
+## [1.125.0] - 2023-10-14
+
+### API-Changes
+
+- [**breaking**] deltachat-rpc-client: Replace `asyncio` with threads.
+- Validate boolean values passed to `set_config`. Attempts to set values other than `0` and `1` will result in an error.
+
+### CI
+
+- Reduce required Python version for deltachat-rpc-client from 3.8 to 3.7.
+
+### Features / Changes
+
+- Add developer option to disable IDLE.
+
+### Fixes
+
+- `deltachat-rpc-client`: Run `deltachat-rpc-server` in its own process group. This prevents reception of `SIGINT` by the server when the bot is terminated with `^C`.
+- python: Don't automatically set the displayname to "bot" when setting log level.
+- Don't update `timestamp`, `timestamp_rcvd`, `state` when replacing partially downloaded message ([#4700](https://github.com/deltachat/deltachat-core-rust/pull/4700)).
+- Assign encrypted partially downloaded group messages to 1:1 chat ([#4757](https://github.com/deltachat/deltachat-core-rust/pull/4757)).
+- Return all contacts from `Contact::get_all` for bots ([#4811](https://github.com/deltachat/deltachat-core-rust/pull/4811)).
+- Set connectivity status to "connected" during fake idle.
+- Return verifier contacts regardless of their origin.
+- Don't try to send more MDNs if there's a temporary SMTP error ([#4534](https://github.com/deltachat/deltachat-core-rust/pull/4534)).
+
+### Refactor
+
+- deltachat-rpc-client: Close stdin instead of sending `SIGTERM`.
+- deltachat-rpc-client: Remove print() calls. Standard `logging` package is for logging instead.
+
+### Tests
+
+- deltachat-rpc-client: Enable logs in pytest.
+
 ## [1.124.1] - 2023-10-05
 
 ### Fixes
@@ -2879,3 +2914,4 @@ https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
 [1.123.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.122.0...v1.123.0
 [1.124.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.123.0...v1.124.0
 [1.124.1]: https://github.com/deltachat/deltachat-core-rust/compare/v1.124.0...v1.124.1
+[1.125.0]: https://github.com/deltachat/deltachat-core-rust/compare/v1.124.1...v1.125.0

Cargo.lock

@@ -1085,7 +1085,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat"
-version = "1.124.1"
+version = "1.125.0"
 dependencies = [
  "ansi_term",
  "anyhow",
@@ -1162,7 +1162,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat-jsonrpc"
-version = "1.124.1"
+version = "1.125.0"
 dependencies = [
  "anyhow",
  "async-channel",
@@ -1186,7 +1186,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat-repl"
-version = "1.124.1"
+version = "1.125.0"
 dependencies = [
  "ansi_term",
  "anyhow",
@@ -1201,7 +1201,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat-rpc-server"
-version = "1.124.1"
+version = "1.125.0"
 dependencies = [
  "anyhow",
  "deltachat",
@@ -1226,7 +1226,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat_ffi"
-version = "1.124.1"
+version = "1.125.0"
 dependencies = [
  "anyhow",
  "deltachat",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.124.1"
+version = "1.125.0"
 edition = "2021"
 license = "MPL-2.0"
 rust-version = "1.67"

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.124.1"
+version = "1.125.0"
 description = "Deltachat FFI"
 edition = "2018"
 readme = "README.md"

deltachat-ffi/deltachat.h

@@ -492,6 +492,9 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `fetch_existing_msgs` = 1=fetch most recent existing messages on configure (default),
  *                    0=do not fetch existing messages on configure.
  *                    In both cases, existing recipients are added to the contact database.
+ * - `disable_idle` = 1=disable IMAP IDLE even if the server supports it,
+ *                    0=use IMAP IDLE if the server supports it.
+ *                    This is a developer option used for testing polling used as an IDLE fallback.
  * - `download_limit` = Messages up to this number of bytes are downloaded automatically.
  *                    For larger messages, only the header is downloaded and a placeholder is shown.
  *                    These messages can be downloaded fully using dc_download_full_msg() later.
@@ -500,6 +503,9 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    to not mess up with non-delivery-reports or read-receipts.
  *                    0=no limit (default).
  *                    Changes affect future messages only.
+ * - `gossip_period` = How often to gossip Autocrypt keys in chats with multiple recipients, in
+ *                    seconds. 2 days by default.
+ *                    This is not supposed to be changed by UIs and only used for testing.
  * - `verified_one_on_one_chats` = Feature flag for verified 1:1 chats; the UI should set it
  *                    to 1 if it supports verified 1:1 chats.
  *                    Regardless of this setting, `dc_chat_is_protected()` returns true while the key is verified,

deltachat-jsonrpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-jsonrpc"
-version = "1.124.1"
+version = "1.125.0"
 description = "DeltaChat JSON-RPC API"
 edition = "2021"
 default-run = "deltachat-jsonrpc-server"

deltachat-jsonrpc/typescript/package.json

@@ -55,5 +55,5 @@
   },
   "type": "module",
   "types": "dist/deltachat.d.ts",
-  "version": "1.124.1"
+  "version": "1.125.0"
 }

deltachat-repl/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-repl"
-version = "1.124.1"
+version = "1.125.0"
 license = "MPL-2.0"
 edition = "2021"
 

deltachat-rpc-client/pyproject.toml

@@ -7,7 +7,6 @@ name = "deltachat-rpc-client"
 description = "Python client for Delta Chat core JSON-RPC interface"
 classifiers = [
     "Development Status :: 5 - Production/Stable",
-    "Framework :: AsyncIO",
     "Intended Audience :: Developers",
     "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
     "Operating System :: POSIX :: Linux",
@@ -72,3 +71,6 @@ line-length = 120
 
 [tool.isort]
 profile = "black"
+
+[tool.pytest.ini_options]
+log_cli = true

deltachat-rpc-server/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat-rpc-server"
-version = "1.124.1"
+version = "1.125.0"
 description = "DeltaChat JSON-RPC server"
 edition = "2021"
 readme = "README.md"

package.json

@@ -60,5 +60,5 @@
     "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail --exit"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.124.1"
+  "version": "1.125.0"
 }

python/src/deltachat/account.py

@@ -617,18 +617,18 @@ class Account:
     # meta API for start/stop and event based processing
     #
 
-    def run_account(self, addr=None, password=None, account_plugins=None, show_ffi=False):
+    def run_account(self, addr=None, password=None, account_plugins=None, show_ffi=False, displayname=None):
-        from .events import FFIEventLogger
-
         """get the account running, configure it if necessary. add plugins if provided.
 
         :param addr: the email address of the account
         :param password: the password of the account
         :param account_plugins: a list of plugins to add
         :param show_ffi: show low level ffi events
+        :param displayname: the display name of the account
         """
+        from .events import FFIEventLogger
+
         if show_ffi:
-            self.set_config("displayname", "bot")
             log = FFIEventLogger(self)
             self.add_account_plugin(log)
 
@@ -644,6 +644,8 @@ class Account:
             configtracker = self.configure()
             configtracker.wait_finish()
 
+        if displayname:
+            self.set_config("displayname", displayname)
         # start IO threads and configure if necessary
         self.start_io()
 

python/tests/test_0_complex_or_slow.py

@@ -542,8 +542,6 @@ def test_see_new_verified_member_after_going_online(acfactory, tmp_path, lp):
     assert msg_in.text == msg_out.text
     assert msg_in.get_sender_contact().addr == ac2_addr
 
-    ac1.set_config("bcc_self", "0")
-
 
 def test_use_new_verified_group_after_going_online(acfactory, tmp_path, lp):
     """Another test for the bug #3836:
@@ -592,4 +590,67 @@ def test_use_new_verified_group_after_going_online(acfactory, tmp_path, lp):
     assert msg_in.get_sender_contact().addr == ac2.get_config("addr")
     assert msg_in.text == msg_out.text
 
-    ac2.set_config("bcc_self", "0")
+
+def test_verified_group_vs_delete_server_after(acfactory, tmp_path, lp):
+    """Test for the issue #4346:
+    - User is added to a verified group.
+    - First device of the user downloads "member added" from the group.
+    - First device removes "member added" from the server.
+    - Some new messages are sent to the group.
+    - Second device comes online, receives these new messages. The result is a verified group with unverified members.
+    - First device re-gossips Autocrypt keys to the group.
+    - Now the seconds device has all members verified.
+    """
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac2_offl = acfactory.new_online_configuring_account(cloned_from=ac2)
+    for ac in [ac2, ac2_offl]:
+        ac.set_config("bcc_self", "1")
+    ac2.set_config("delete_server_after", "1")
+    ac2.set_config("gossip_period", "0")  # Re-gossip in every message
+    acfactory.bring_accounts_online()
+    dir = tmp_path / "exportdir"
+    dir.mkdir()
+    ac2.export_self_keys(str(dir))
+    ac2_offl.import_self_keys(str(dir))
+    ac2_offl.stop_io()
+
+    lp.sec("ac1: create verified-group QR, ac2 scans and joins")
+    chat1 = ac1.create_group_chat("hello", verified=True)
+    assert chat1.is_protected()
+    qr = chat1.get_join_qr()
+    lp.sec("ac2: start QR-code based join-group protocol")
+    chat2 = ac2.qr_join_chat(qr)
+    ac1._evtracker.wait_securejoin_inviter_progress(1000)
+    # Wait for "Member Me (<addr>) added by <addr>." message.
+    msg_in = ac2._evtracker.wait_next_incoming_message()
+    assert msg_in.is_system_message()
+
+    lp.sec("ac2: waiting for 'member added' to be deleted on the server")
+    ac2._evtracker.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
+
+    lp.sec("ac1: sending 'hi' to the group")
+    ac2.set_config("delete_server_after", "0")
+    chat1.send_text("hi")
+
+    lp.sec("ac2_offl: going online, checking the 'hi' message")
+    ac2_offl.start_io()
+    msg_in = ac2_offl._evtracker.wait_next_incoming_message()
+    assert not msg_in.is_system_message()
+    assert msg_in.text.startswith("[Sender of this message is not verified:")
+    ac2_offl_ac1_contact = msg_in.get_sender_contact()
+    assert ac2_offl_ac1_contact.addr == ac1.get_config("addr")
+    assert not ac2_offl_ac1_contact.is_verified()
+    chat2_offl = msg_in.chat
+    assert chat2_offl.is_protected()
+
+    lp.sec("ac2: sending message re-gossiping Autocrypt keys")
+    chat2.send_text("hi2")
+
+    lp.sec("ac2_offl: receiving message")
+    ev = ac2_offl._evtracker.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
+    msg_in = ac2_offl.get_message_by_id(ev.data2)
+    assert not msg_in.is_system_message()
+    assert msg_in.text == "hi2"
+    assert msg_in.chat == chat2_offl
+    assert msg_in.get_sender_contact().addr == ac2.get_config("addr")
+    assert ac2_offl_ac1_contact.is_verified()

python/tests/test_1_online.py

@@ -1718,12 +1718,10 @@ def test_qr_new_group_unblocked(acfactory, lp):
 
     ac1_new_chat = ac1.create_group_chat("Another group")
     ac1_new_chat.add_contact(ac2)
-    ac1_new_chat.send_text("Hello!")
-
     # Receive "Member added" message.
     ac2._evtracker.wait_next_incoming_message()
 
-    # Receive "Hello!" message.
+    ac1_new_chat.send_text("Hello!")
     ac2_msg = ac2._evtracker.wait_next_incoming_message()
     assert ac2_msg.text == "Hello!"
     assert ac2_msg.chat.is_contact_request()
@@ -1946,13 +1944,15 @@ def test_set_get_group_image(acfactory, data, lp):
     lp.sec("ac1: add ac2 to promoted group chat")
     chat.add_contact(ac2)  # sends one message
 
+    lp.sec("ac2: wait for receiving member added message from ac1")
+    msg1 = ac2._evtracker.wait_next_incoming_message()
+    assert msg1.is_system_message()  # Member added
+
     lp.sec("ac1: send a first message to ac2")
     chat.send_text("hi")  # sends another message
     assert chat.is_promoted()
 
     lp.sec("ac2: wait for receiving message from ac1")
-    msg1 = ac2._evtracker.wait_next_incoming_message()
-    assert msg1.is_system_message()  # Member added
     msg2 = ac2._evtracker.wait_next_incoming_message()
     assert msg2.text == "hi"
     assert msg1.chat.id == msg2.chat.id

release-date.in

@@ -1 +1 @@
-2023-10-05
+2023-10-14

scripts/README.md

@@ -18,7 +18,7 @@ and an own build machine.
 - `remote_tests_rust.sh` rsyncs to the build machine and runs
   `run-rust-test.sh` remotely on the build machine. 
 
-- `make-python-testenv.sh` creates or updates local python test development environment.
+- `make-python-testenv.sh` creates local python test development environment.
   Reusing the same environment is faster than running `run-python-test.sh` which always
   recreates environment from scratch and runs additional lints.
 

scripts/make-python-testenv.sh

@@ -4,8 +4,8 @@
 # It rebuilds the core and bindings as needed.
 #
 # After running the script, you can either
-# run `pytest` directly with `env/bin/pytest python/`
+# run `pytest` directly with `venv/bin/pytest python/`
-# or activate the environment with `. env/bin/activacte`
+# or activate the environment with `. venv/bin/activate`
 # and run `pytest` from there.
 set -euo pipefail
 
@@ -13,9 +13,5 @@ export DCC_RS_TARGET=debug
 export DCC_RS_DEV="$PWD"
 cargo build -p deltachat_ffi --features jsonrpc
 
-if test -d env; then
+tox -c python -e py --devenv venv
-	env/bin/pip install -e python --force-reinstall
+env/bin/pip install --upgrade pip
-else
-	tox -e py --devenv env
-	env/bin/pip install --upgrade pip
-fi

src/config.rs

@@ -286,6 +286,12 @@ pub enum Config {
     #[strum(props(default = "60"))]
     ScanAllFoldersDebounceSecs,
 
+    /// Whether to avoid using IMAP IDLE even if the server supports it.
+    ///
+    /// This is a developer option for testing "fake idle".
+    #[strum(props(default = "0"))]
+    DisableIdle,
+
     /// Defines the max. size (in bytes) of messages downloaded automatically.
     /// 0 = no limit.
     #[strum(props(default = "0"))]
@@ -313,6 +319,13 @@ pub enum Config {
     /// Last message processed by the bot.
     LastMsgId,
 
+    /// How often to gossip Autocrypt keys in chats with multiple recipients, in seconds. 2 days by
+    /// default.
+    ///
+    /// This is not supposed to be changed by UIs and only used for testing.
+    #[strum(props(default = "172800"))]
+    GossipPeriod,
+
     /// Feature flag for verified 1:1 chats; the UI should set it
     /// to 1 if it supports verified 1:1 chats.
     /// Regardless of this setting, `chat.is_protected()` returns true while the key is verified,
@@ -475,6 +488,28 @@ impl Context {
                     .set_raw_config(key.as_ref(), value.as_deref())
                     .await?;
             }
+            Config::Socks5Enabled
+            | Config::BccSelf
+            | Config::E2eeEnabled
+            | Config::MdnsEnabled
+            | Config::SentboxWatch
+            | Config::MvboxMove
+            | Config::OnlyFetchMvbox
+            | Config::FetchExistingMsgs
+            | Config::DeleteToTrash
+            | Config::SaveMimeHeaders
+            | Config::Configured
+            | Config::Bot
+            | Config::NotifyAboutWrongPw
+            | Config::SendSyncMsgs
+            | Config::SignUnencrypted
+            | Config::DisableIdle => {
+                ensure!(
+                    matches!(value, None | Some("0") | Some("1")),
+                    "Boolean value must be either 0 or 1"
+                );
+                self.sql.set_raw_config(key.as_ref(), value).await?;
+            }
             _ => {
                 self.sql.set_raw_config(key.as_ref(), value).await?;
             }
@@ -624,6 +659,18 @@ mod tests {
         );
     }
 
+    /// Tests that "bot" config can only be set to "0" or "1".
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_set_config_bot() {
+        let t = TestContext::new().await;
+
+        assert!(t.set_config(Config::Bot, None).await.is_ok());
+        assert!(t.set_config(Config::Bot, Some("0")).await.is_ok());
+        assert!(t.set_config(Config::Bot, Some("1")).await.is_ok());
+        assert!(t.set_config(Config::Bot, Some("2")).await.is_err());
+        assert!(t.set_config(Config::Bot, Some("Foobar")).await.is_err());
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_media_quality_config_option() {
         let t = TestContext::new().await;

src/contact.rs

@@ -812,7 +812,11 @@ impl Contact {
         let mut ret = Vec::new();
         let flag_verified_only = (listflags & DC_GCL_VERIFIED_ONLY) != 0;
         let flag_add_self = (listflags & DC_GCL_ADD_SELF) != 0;
-
+        let minimal_origin = if context.get_config_bool(Config::Bot).await? {
+            Origin::Unknown
+        } else {
+            Origin::IncomingReplyTo
+        };
         if flag_verified_only || query.is_some() {
             let s3str_like_cmd = format!("%{}%", query.unwrap_or(""));
             context
@@ -832,7 +836,7 @@ impl Contact {
                     ),
                     rusqlite::params_from_iter(params_iter(&self_addrs).chain(params_slice![
                         ContactId::LAST_SPECIAL,
-                        Origin::IncomingReplyTo,
+                        minimal_origin,
                         s3str_like_cmd,
                         s3str_like_cmd,
                         if flag_verified_only { 0i32 } else { 1i32 }
@@ -882,10 +886,10 @@ impl Contact {
                  ORDER BY last_seen DESC, id DESC;",
                         sql::repeat_vars(self_addrs.len())
                     ),
-                    rusqlite::params_from_iter(params_iter(&self_addrs).chain(params_slice![
+                    rusqlite::params_from_iter(
-                        ContactId::LAST_SPECIAL,
+                        params_iter(&self_addrs)
-                        Origin::IncomingReplyTo
+                            .chain(params_slice![ContactId::LAST_SPECIAL, minimal_origin]),
-                    ])),
+                    ),
                     |row| row.get::<_, ContactId>(0),
                     |ids| {
                         for id in ids {
@@ -1245,7 +1249,7 @@ impl Contact {
             return Ok(Some(ContactId::SELF));
         }
 
-        match Contact::lookup_id_by_addr(context, &verifier_addr, Origin::AddressBook).await? {
+        match Contact::lookup_id_by_addr(context, &verifier_addr, Origin::Unknown).await? {
             Some(contact_id) => Ok(Some(contact_id)),
             None => {
                 let addr = &self.addr;

src/context.rs

@@ -579,6 +579,7 @@ impl Context {
         let mdns_enabled = self.get_config_int(Config::MdnsEnabled).await?;
         let bcc_self = self.get_config_int(Config::BccSelf).await?;
         let send_sync_msgs = self.get_config_int(Config::SendSyncMsgs).await?;
+        let disable_idle = self.get_config_bool(Config::DisableIdle).await?;
 
         let prv_key_cnt = self.sql.count("SELECT COUNT(*) FROM keypairs;", ()).await?;
 
@@ -691,6 +692,7 @@ impl Context {
         );
         res.insert("bcc_self", bcc_self.to_string());
         res.insert("send_sync_msgs", send_sync_msgs.to_string());
+        res.insert("disable_idle", disable_idle.to_string());
         res.insert("private_key_count", prv_key_cnt.to_string());
         res.insert("public_key_count", pub_key_cnt.to_string());
         res.insert("fingerprint", fingerprint_str);
@@ -752,7 +754,6 @@ impl Context {
                 .await?
                 .to_string(),
         );
-
         res.insert(
             "debug_logging",
             self.get_config_int(Config::DebugLogging).await?.to_string(),
@@ -761,6 +762,10 @@ impl Context {
             "last_msg_id",
             self.get_config_int(Config::LastMsgId).await?.to_string(),
         );
+        res.insert(
+            "gossip_period",
+            self.get_config_int(Config::GossipPeriod).await?.to_string(),
+        );
         res.insert(
             "verified_one_on_one_chats",
             self.get_config_bool(Config::VerifiedOneOnOneChats)

src/download.rs

@@ -23,7 +23,7 @@ use crate::{job_try, stock_str, EventType};
 /// eg. to assign them to the correct chat.
 /// As these messages are typically small,
 /// they're caught by `MIN_DOWNLOAD_LIMIT`.
-const MIN_DOWNLOAD_LIMIT: u32 = 32768;
+pub(crate) const MIN_DOWNLOAD_LIMIT: u32 = 32768;
 
 /// If a message is downloaded only partially
 /// and `delete_server_after` is set to small timeouts (eg. "at once"),

src/imap.rs

@@ -391,6 +391,7 @@ impl Imap {
                     "IMAP-LOGIN as {}",
                     self.config.lp.user
                 )));
+                self.connectivity.set_connected(context).await;
                 info!(context, "Successfully logged into IMAP server");
                 Ok(())
             }

src/imap/idle.rs

@@ -7,7 +7,9 @@ use futures_lite::FutureExt;
 
 use super::session::Session;
 use super::Imap;
+use crate::config::Config;
 use crate::imap::{client::IMAP_TIMEOUT, FolderMeaning};
+use crate::log::LogExt;
 use crate::{context::Context, scheduler::InterruptInfo};
 
 const IDLE_TIMEOUT: Duration = Duration::from_secs(23 * 60);
@@ -21,6 +23,10 @@ impl Session {
     ) -> Result<(Self, InterruptInfo)> {
         use futures::future::FutureExt;
 
+        if context.get_config_bool(Config::DisableIdle).await? {
+            bail!("IMAP IDLE is disabled");
+        }
+
         if !self.can_idle() {
             bail!("IMAP server does not have IDLE capability");
         }
@@ -163,7 +169,14 @@ impl Imap {
                         continue;
                     }
                     if let Some(session) = &self.session {
-                        if session.can_idle() {
+                        if session.can_idle()
+                            && !context
+                                .get_config_bool(Config::DisableIdle)
+                                .await
+                                .context("Failed to get disable_idle config")
+                                .log_err(context)
+                                .unwrap_or_default()
+                        {
                             // we only fake-idled because network was gone during IDLE, probably
                             break InterruptInfo::new(false);
                         }

src/mimefactory.rs

@@ -359,9 +359,10 @@ impl<'a> MimeFactory<'a> {
     async fn should_do_gossip(&self, context: &Context) -> Result<bool> {
         match &self.loaded {
             Loaded::Message { chat } => {
-                // beside key- and member-changes, force re-gossip every 48 hours
+                // beside key- and member-changes, force a periodic re-gossip.
                 let gossiped_timestamp = chat.id.get_gossiped_timestamp(context).await?;
-                if time() > gossiped_timestamp + (2 * 24 * 60 * 60) {
+                let gossip_period = context.get_config_i64(Config::GossipPeriod).await?;
+                if time() >= gossiped_timestamp + gossip_period {
                     Ok(true)
                 } else {
                     let cmd = self.msg.param.get_cmd();

src/mimeparser.rs

@@ -109,7 +109,7 @@ pub(crate) struct MimeMessage {
 
     /// The decrypted, raw mime structure.
     ///
-    /// This is non-empty only if the message was actually encrypted.  It is used
+    /// This is non-empty iff `is_mime_modified` and the message was actually encrypted. It is used
     /// for e.g. late-parsing HTML.
     pub decoded_data: Vec<u8>,
 

src/receive_imf.rs

@@ -77,6 +77,24 @@ pub async fn receive_imf(
     let mail = parse_mail(imf_raw).context("can't parse mail")?;
     let rfc724_mid =
         imap::prefetch_get_message_id(&mail.headers).unwrap_or_else(imap::create_message_id);
+    if let Some(download_limit) = context.download_limit().await? {
+        let download_limit: usize = download_limit.try_into()?;
+        if imf_raw.len() > download_limit {
+            let head = std::str::from_utf8(imf_raw)?
+                .split("\r\n\r\n")
+                .next()
+                .context("No empty line in the message")?;
+            return receive_imf_inner(
+                context,
+                &rfc724_mid,
+                head.as_bytes(),
+                seen,
+                Some(imf_raw.len().try_into()?),
+                false,
+            )
+            .await;
+        }
+    }
     receive_imf_inner(context, &rfc724_mid, imf_raw, seen, None, false).await
 }
 
@@ -582,6 +600,7 @@ async fn add_parts(
             if let Some((new_chat_id, new_chat_id_blocked)) = create_or_lookup_group(
                 context,
                 mime_parser,
+                is_partial_download.is_some(),
                 if test_normal_chat.is_none() {
                     allow_creation
                 } else {
@@ -858,6 +877,7 @@ async fn add_parts(
                 if let Some((new_chat_id, new_chat_id_blocked)) = create_or_lookup_group(
                     context,
                     mime_parser,
+                    is_partial_download.is_some(),
                     allow_creation,
                     Blocked::Not,
                     from_id,
@@ -1213,8 +1233,8 @@ INSERT INTO msgs
   )
 ON CONFLICT (id) DO UPDATE
 SET rfc724_mid=excluded.rfc724_mid, chat_id=excluded.chat_id,
-    from_id=excluded.from_id, to_id=excluded.to_id, timestamp=excluded.timestamp, timestamp_sent=excluded.timestamp_sent,
+    from_id=excluded.from_id, to_id=excluded.to_id, timestamp_sent=excluded.timestamp_sent,
-    timestamp_rcvd=excluded.timestamp_rcvd, type=excluded.type, state=excluded.state, msgrmsg=excluded.msgrmsg,
+    type=excluded.type, msgrmsg=excluded.msgrmsg,
     txt=excluded.txt, subject=excluded.subject, txt_raw=excluded.txt_raw, param=excluded.param,
     bytes=excluded.bytes, mime_headers=excluded.mime_headers,
     mime_compressed=excluded.mime_compressed, mime_in_reply_to=excluded.mime_in_reply_to,
@@ -1545,6 +1565,7 @@ async fn is_probably_private_reply(
 async fn create_or_lookup_group(
     context: &Context,
     mime_parser: &mut MimeMessage,
+    is_partial_download: bool,
     allow_creation: bool,
     create_blocked: Blocked,
     from_id: ContactId,
@@ -1677,7 +1698,7 @@ async fn create_or_lookup_group(
 
     if let Some(chat_id) = chat_id {
         Ok(Some((chat_id, chat_id_blocked)))
-    } else if mime_parser.decrypting_failed {
+    } else if is_partial_download || mime_parser.decrypting_failed {
         // It is possible that the message was sent to a valid,
         // yet unknown group, which was rejected because
         // Chat-Group-Name, which is in the encrypted part, was

src/receive_imf/tests.rs

@@ -10,8 +10,9 @@ use crate::chat::{get_chat_msgs, ChatItem, ChatVisibility};
 use crate::chatlist::Chatlist;
 use crate::config::Config;
 use crate::constants::{DC_GCL_FOR_FORWARDING, DC_GCL_NO_SPECIALS};
+use crate::download::{DownloadState, MIN_DOWNLOAD_LIMIT};
 use crate::imap::prefetch_should_download;
-use crate::message::Message;
+use crate::message::{self, Message};
 use crate::test_utils::{get_chat_msg, TestContext, TestContextManager};
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -2973,6 +2974,7 @@ async fn test_auto_accept_for_bots() -> Result<()> {
     let msg = t.get_last_msg().await;
     let chat = chat::Chat::load_from_db(&t, msg.chat_id).await?;
     assert!(!chat.is_contact_request());
+    assert!(Contact::get_all(&t, 0, None).await?.len() == 1);
     Ok(())
 }
 
@@ -3697,3 +3699,114 @@ async fn test_keep_member_list_if_possibly_nomember() -> Result<()> {
     assert!(is_contact_in_chat(&bob, bob_chat_id, bob_alice_contact).await?);
     Ok(())
 }
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_download_later() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = tcm.alice().await;
+    alice.set_config(Config::DownloadLimit, Some("1")).await?;
+    assert_eq!(alice.download_limit().await?, Some(MIN_DOWNLOAD_LIMIT));
+
+    let bob = tcm.bob().await;
+    let bob_chat = bob.create_chat(&alice).await;
+    let text = String::from_utf8(vec![b'a'; MIN_DOWNLOAD_LIMIT as usize])?;
+    let sent_msg = bob.send_text(bob_chat.id, &text).await;
+    let msg = alice.recv_msg(&sent_msg).await;
+    assert_eq!(msg.download_state, DownloadState::Available);
+    assert_eq!(msg.state, MessageState::InFresh);
+
+    let hi_msg = tcm.send_recv(&bob, &alice, "hi").await;
+
+    alice.set_config(Config::DownloadLimit, None).await?;
+    let msg = alice.recv_msg(&sent_msg).await;
+    assert_eq!(msg.download_state, DownloadState::Done);
+    assert_eq!(msg.state, MessageState::InFresh);
+    assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, hi_msg.id);
+    assert!(msg.timestamp_sort <= hi_msg.timestamp_sort);
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_create_group_with_big_msg() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = tcm.alice().await;
+    let bob = tcm.bob().await;
+    let ba_contact = Contact::create(
+        &bob,
+        "alice",
+        &alice.get_config(Config::Addr).await?.unwrap(),
+    )
+    .await?;
+    let file_bytes = include_bytes!("../../test-data/image/screenshot.png");
+
+    let bob_grp_id = create_group_chat(&bob, ProtectionStatus::Unprotected, "Group").await?;
+    add_contact_to_chat(&bob, bob_grp_id, ba_contact).await?;
+    let mut msg = Message::new(Viewtype::Image);
+    msg.set_file_from_bytes(&bob, "a.jpg", file_bytes, None)
+        .await?;
+    let sent_msg = bob.send_msg(bob_grp_id, &mut msg).await;
+    assert!(!msg.get_showpadlock());
+
+    alice.set_config(Config::DownloadLimit, Some("1")).await?;
+    assert_eq!(alice.download_limit().await?, Some(MIN_DOWNLOAD_LIMIT));
+    let msg = alice.recv_msg(&sent_msg).await;
+    assert_eq!(msg.download_state, DownloadState::Available);
+    let alice_grp = Chat::load_from_db(&alice, msg.chat_id).await?;
+    assert_eq!(alice_grp.typ, Chattype::Group);
+    assert_eq!(alice_grp.name, "Group");
+    assert_eq!(
+        chat::get_chat_contacts(&alice, alice_grp.id).await?.len(),
+        2
+    );
+
+    alice.set_config(Config::DownloadLimit, None).await?;
+    let msg = alice.recv_msg(&sent_msg).await;
+    assert_eq!(msg.download_state, DownloadState::Done);
+    assert_eq!(msg.state, MessageState::InFresh);
+    assert_eq!(msg.viewtype, Viewtype::Image);
+    assert_eq!(msg.chat_id, alice_grp.id);
+    let alice_grp = Chat::load_from_db(&alice, msg.chat_id).await?;
+    assert_eq!(alice_grp.typ, Chattype::Group);
+    assert_eq!(alice_grp.name, "Group");
+    assert_eq!(
+        chat::get_chat_contacts(&alice, alice_grp.id).await?.len(),
+        2
+    );
+
+    let ab_chat_id = tcm.send_recv_accept(&alice, &bob, "hi").await.chat_id;
+    // Now Bob can send encrypted messages to Alice.
+
+    let bob_grp_id = create_group_chat(&bob, ProtectionStatus::Unprotected, "Group1").await?;
+    add_contact_to_chat(&bob, bob_grp_id, ba_contact).await?;
+    let mut msg = Message::new(Viewtype::Image);
+    msg.set_file_from_bytes(&bob, "a.jpg", file_bytes, None)
+        .await?;
+    let sent_msg = bob.send_msg(bob_grp_id, &mut msg).await;
+    assert!(msg.get_showpadlock());
+
+    alice.set_config(Config::DownloadLimit, Some("1")).await?;
+    let msg = alice.recv_msg(&sent_msg).await;
+    assert_eq!(msg.download_state, DownloadState::Available);
+    // Until fully downloaded, an encrypted message must sit in the 1:1 chat.
+    assert_eq!(msg.chat_id, ab_chat_id);
+
+    alice.set_config(Config::DownloadLimit, None).await?;
+    let msg = alice.recv_msg(&sent_msg).await;
+    assert_eq!(msg.download_state, DownloadState::Done);
+    assert_eq!(msg.state, MessageState::InFresh);
+    assert_eq!(msg.viewtype, Viewtype::Image);
+    assert_ne!(msg.chat_id, ab_chat_id);
+    let alice_grp = Chat::load_from_db(&alice, msg.chat_id).await?;
+    assert_eq!(alice_grp.typ, Chattype::Group);
+    assert_eq!(alice_grp.name, "Group1");
+    assert_eq!(
+        chat::get_chat_contacts(&alice, alice_grp.id).await?.len(),
+        2
+    );
+
+    // The big message must go away from the 1:1 chat.
+    assert_eq!(alice.get_last_msg_in(ab_chat_id).await.text, "hi");
+
+    Ok(())
+}

src/scheduler.rs

@@ -574,6 +574,19 @@ async fn fetch_idle(
                 .await;
         }
 
+        if ctx
+            .get_config_bool(Config::DisableIdle)
+            .await
+            .context("Failed to get disable_idle config")
+            .log_err(ctx)
+            .unwrap_or_default()
+        {
+            info!(ctx, "IMAP IDLE is disabled, going to fake idle.");
+            return connection
+                .fake_idle(ctx, Some(watch_folder), folder_meaning)
+                .await;
+        }
+
         info!(ctx, "IMAP session supports IDLE, using it.");
         match session
             .idle(

src/smtp.rs

@@ -674,12 +674,14 @@ pub(crate) async fn send_smtp_messages(context: &Context, connection: &mut Smtp)
 /// On failure returns an error without removing any `smtp_mdns` entries, the caller is responsible
 /// for removing the corresponding entry to prevent endless loop in case the entry is invalid, e.g.
 /// points to non-existent message or contact.
+///
+/// Returns true on success, false on temporary error.
 async fn send_mdn_msg_id(
     context: &Context,
     msg_id: MsgId,
     contact_id: ContactId,
     smtp: &mut Smtp,
-) -> Result<()> {
+) -> Result<bool> {
     let contact = Contact::get_by_id(context, contact_id).await?;
     if contact.is_blocked() {
         return Err(format_err!("Contact is blocked"));
@@ -731,14 +733,14 @@ async fn send_mdn_msg_id(
                     .execute(&q, rusqlite::params_from_iter(additional_msg_ids))
                     .await?;
             }
-            Ok(())
+            Ok(true)
         }
         SendResult::Retry => {
             info!(
                 context,
                 "Temporary SMTP failure while sending an MDN for {}", msg_id
             );
-            Ok(())
+            Ok(false)
         }
         SendResult::Failure(err) => Err(err),
     }
@@ -785,15 +787,20 @@ async fn send_mdn(context: &Context, smtp: &mut Smtp) -> Result<bool> {
         .await
         .context("failed to update MDN retries count")?;
 
-    if let Err(err) = send_mdn_msg_id(context, msg_id, contact_id, smtp).await {
+    let res = send_mdn_msg_id(context, msg_id, contact_id, smtp).await;
+    if let Err(ref err) = res {
         // If there is an error, for example there is no message corresponding to the msg_id in the
         // database, do not try to send this MDN again.
+        warn!(
+            context,
+            "Error sending MDN for {msg_id}, removing it: {err:#}."
+        );
         context
             .sql
             .execute("DELETE FROM smtp_mdns WHERE msg_id = ?", (msg_id,))
             .await?;
-        Err(err)
-    } else {
-        Ok(true)
     }
+    // If there's a temporary error, pretend there are no more MDNs to send. It's unlikely that
+    // other MDNs could be sent successfully in case of connectivity problems.
+    res
 }