feat: add message parser api to jsonrpc and cffi

api: cffi add `dc_parse_message_text_to_ast_json` and `dc_msg_get_parsed_text_as_json

api: jsonrpc: add `get_parsed_message_text_ast_json` and `parse_text_to_ast_json`

.gitignore

@@ -18,6 +18,9 @@ python/.eggs
 __pycache__
 python/src/deltachat/capi*.so
 python/.venv/
+python/venv/
+venv/
+env/
 
 python/liveconfig*
 

Cargo.lock

@@ -1118,6 +1118,7 @@ dependencies = [
  "chrono",
  "criterion",
  "deltachat_derive",
+ "deltachat_message_parser",
  "email",
  "encoded-words",
  "escaper",
@@ -1259,6 +1260,18 @@ dependencies = [
  "yerpc",
 ]
 
+[[package]]
+name = "deltachat_message_parser"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "826f80f4b32f51457773351b2d821dc1f45273a38235e8fd3bdf662b67b70bcd"
+dependencies = [
+ "nom",
+ "serde",
+ "serde_derive",
+ "unic-idna-punycode",
+]
+
 [[package]]
 name = "der"
 version = "0.6.1"
@@ -5221,6 +5234,12 @@ version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94"
 
+[[package]]
+name = "unic-idna-punycode"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06feaedcbf9f1fc259144d833c0d630b8b15207b0486ab817d29258bc89f2f8a"
+
 [[package]]
 name = "unicode-bidi"
 version = "0.3.13"

Cargo.toml

@@ -94,6 +94,7 @@ toml = "0.7"
 trust-dns-resolver = "0.22"
 url = "2"
 uuid = { version = "1", features = ["serde", "v4"] }
+deltachat_message_parser = "0.8.0"
 
 [dev-dependencies]
 ansi_term = "0.12.0"
@@ -154,8 +155,4 @@ harness = false
 [features]
 default = ["vendored"]
 internals = []
-vendored = [
-  "async-native-tls/vendored",
-  "rusqlite/bundled-sqlcipher-vendored-openssl",
-  "reqwest/native-tls-vendored"
-]
+vendored = ["async-native-tls/vendored", "rusqlite/bundled-sqlcipher-vendored-openssl", "reqwest/native-tls-vendored"]

deltachat-ffi/deltachat.h

@@ -492,6 +492,9 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `fetch_existing_msgs` = 1=fetch most recent existing messages on configure (default),
  *                    0=do not fetch existing messages on configure.
  *                    In both cases, existing recipients are added to the contact database.
+ * - `disable_idle` = 1=disable IMAP IDLE even if the server supports it,
+ *                    0=use IMAP IDLE if the server supports it.
+ *                    This is a developer option used for testing polling used as an IDLE fallback.
  * - `download_limit` = Messages up to this number of bytes are downloaded automatically.
  *                    For larger messages, only the header is downloaded and a placeholder is shown.
  *                    These messages can be downloaded fully using dc_download_full_msg() later.
@@ -3990,6 +3993,36 @@ int64_t          dc_msg_get_sort_timestamp     (const dc_msg_t* msg);
 char*           dc_msg_get_text               (const dc_msg_t* msg);
 
 
+#define         MESSAGE_PARSER_MODE_ONLY_TEXT        0x00
+#define         MESSAGE_PARSER_MODE_DESKTOP_SET      0x01
+#define         MESSAGE_PARSER_MODE_MARKDOWN         0x02
+
+/**
+ * Parse text with the message parser.
+ *
+ * @memberof dc_context_t
+ * @param input The text to parse.
+ * @param mode Sets the parsing mode, you can choose between MESSAGE_PARSER_MODE_ONLY_TEXT, MESSAGE_PARSER_MODE_DESKTOP_SET and MESSAGE_PARSER_MODE_MARKDOWN.
+ *        Look at https://github.com/deltachat/message-parser/blob/master/spec.md#modes-of-the-parser to learn more about the parser modes.
+ * @return Abstract Syntax Tree for your message that you can use to display parts of a message specially like links.
+ *         This ast is returned in json (look at the sourcecode for reference for the format: https://github.com/deltachat/message-parser/blob/master/src/parser/mod.rs#L11)
+ */
+char*           dc_parse_message_text_to_ast_json (const char* input, int mode);
+
+
+/**
+ * Parse the text of a message with the message parser.
+ *
+ * @memberof dc_msg_t
+ * @param msg The message object.
+ * @param mode Sets the parsing mode, you can choose between MESSAGE_PARSER_MODE_ONLY_TEXT, MESSAGE_PARSER_MODE_DESKTOP_SET and MESSAGE_PARSER_MODE_MARKDOWN.
+ *        Look at https://github.com/deltachat/message-parser/blob/master/spec.md#modes-of-the-parser to learn more about the parser modes.
+ * @return Abstract Syntax Tree for your message that you can use to display parts of a message specially like links.
+ *         This ast is returned in json (look at the sourcecode for reference for the format: https://github.com/deltachat/message-parser/blob/master/src/parser/mod.rs#L11)
+ */
+char*           dc_msg_get_parsed_text_as_json (const dc_msg_t* msg, int mode);
+
+
 /**
  * Get the subject of the e-mail.
  * If there is no subject associated with the message, an empty string is returned.

deltachat-ffi/src/lib.rs

@@ -31,6 +31,7 @@ use deltachat::ephemeral::Timer as EphemeralTimer;
 use deltachat::imex::BackupProvider;
 use deltachat::key::{DcKey, DcSecretKey};
 use deltachat::message::MsgId;
+use deltachat::message_parser::parser;
 use deltachat::net::read_url_blob;
 use deltachat::qr_code_generator::{generate_backup_qr, get_securejoin_qr_svg};
 use deltachat::reaction::{get_msg_reactions, send_reaction, Reactions};
@@ -3356,6 +3357,58 @@ pub unsafe extern "C" fn dc_msg_get_text(msg: *mut dc_msg_t) -> *mut libc::c_cha
     ffi_msg.message.get_text().strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_parse_message_text_to_ast_json(
+    text: *const libc::c_char,
+    mode: u32,
+) -> *mut libc::c_char {
+    if text.is_null() {
+        eprintln!("ignoring careless call to dc_parse_message_text_to_ast_json()");
+    }
+    let text = to_string_lossy(text);
+    let result = match mode {
+        0 /* OnlyText */ => parser::parse_only_text(&text),
+        1 /* DesktopSet */ => parser::parse_desktop_set(&text),
+        2 /* Markdown */ => parser::parse_markdown_text(&text),
+        _ => {
+            eprintln!("ignoring careless call to dc_parse_message_text_to_ast_json() - invalid mode");
+            return "".strdup();
+        }
+    };
+    if let Ok(result) = serde_json::to_string(&result) {
+        result.strdup()
+    } else {
+        "".strdup()
+    }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_msg_get_parsed_text_as_json(
+    msg: *mut dc_msg_t,
+    mode: u32,
+) -> *mut libc::c_char {
+    if msg.is_null() {
+        eprintln!("ignoring careless call to dc_msg_get_parsed_text_as_json()");
+        return "".strdup();
+    }
+    let ffi_msg = &*msg;
+    let text = ffi_msg.message.get_text();
+    let result = match mode {
+        0 /* OnlyText */ => parser::parse_only_text(&text),
+        1 /* DesktopSet */ => parser::parse_desktop_set(&text),
+        2 /* Markdown */ => parser::parse_markdown_text(&text),
+        _ => {
+            eprintln!("ignoring careless call to dc_msg_get_parsed_text_as_json() - invalid mode");
+            return "".strdup();
+        }
+    };
+    if let Ok(result) = serde_json::to_string(&result) {
+        result.strdup()
+    } else {
+        "".strdup()
+    }
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_msg_get_subject(msg: *mut dc_msg_t) -> *mut libc::c_char {
     if msg.is_null() {

deltachat-jsonrpc/src/api/mod.rs

@@ -43,11 +43,13 @@ use types::contact::ContactObject;
 use types::events::Event;
 use types::http::HttpResponse;
 use types::message::{MessageData, MessageObject, MessageReadReceipt};
+use types::message_parser;
 use types::provider_info::ProviderInfo;
 use types::reactions::JSONRPCReactions;
 use types::webxdc::WebxdcMessageInfo;
 
 use self::types::message::MessageLoadResult;
+use self::types::message_parser::MessageParserMode;
 use self::types::{
     chat::{BasicChat, JSONRPCChatVisibility, MuteDuration},
     location::JsonrpcLocation,
@@ -1081,6 +1083,31 @@ impl CommandApi {
         MsgId::new(message_id).get_html(&ctx).await
     }
 
+    // no specific typings because of https://github.com/dbeckwith/rust-typescript-type-def/issues/18
+    async fn get_parsed_message_text_ast_json(
+        &self,
+        account_id: u32,
+        message_id: u32,
+        mode: MessageParserMode,
+    ) -> Result<serde_json::Value> {
+        let ctx = self.get_context(account_id).await?;
+        let msg_text = Message::load_from_db(&ctx, MsgId::new(message_id))
+            .await?
+            .get_text();
+        let result = message_parser::parse_text(&msg_text, mode);
+        Ok(serde_json::to_value(result)?)
+    }
+
+    // no specific typings because of https://github.com/dbeckwith/rust-typescript-type-def/issues/18
+    async fn parse_text_to_ast_json(
+        &self,
+        text: String,
+        mode: MessageParserMode,
+    ) -> Result<serde_json::Value> {
+        let result = message_parser::parse_text(&text, mode);
+        Ok(serde_json::to_value(result)?)
+    }
+
     /// get multiple messages in one call,
     /// if loading one message fails the error is stored in the result object in it's place.
     ///

deltachat-jsonrpc/src/api/types/message_parser.rs

@@ -0,0 +1,19 @@
+use deltachat::message_parser::parser::{self, Element};
+use serde::{Deserialize, Serialize};
+use typescript_type_def::TypeDef;
+
+#[repr(u8)]
+#[derive(Serialize, Deserialize, TypeDef, schemars::JsonSchema)]
+pub enum MessageParserMode {
+    OnlyText,
+    DesktopSet,
+    Markdown,
+}
+
+pub fn parse_text(input: &str, mode: MessageParserMode) -> std::vec::Vec<Element> {
+    match mode {
+        MessageParserMode::OnlyText => parser::parse_only_text(input),
+        MessageParserMode::DesktopSet => parser::parse_desktop_set(input),
+        MessageParserMode::Markdown => parser::parse_markdown_text(input),
+    }
+}

deltachat-jsonrpc/src/api/types/mod.rs

@@ -6,6 +6,7 @@ pub mod events;
 pub mod http;
 pub mod location;
 pub mod message;
+pub mod message_parser;
 pub mod provider_info;
 pub mod qr;
 pub mod reactions;

deltachat-rpc-client/pyproject.toml

@@ -7,7 +7,6 @@ name = "deltachat-rpc-client"
 description = "Python client for Delta Chat core JSON-RPC interface"
 classifiers = [
     "Development Status :: 5 - Production/Stable",
-    "Framework :: AsyncIO",
     "Intended Audience :: Developers",
     "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
     "Operating System :: POSIX :: Linux",

python/src/deltachat/account.py

@@ -617,18 +617,18 @@ class Account:
     # meta API for start/stop and event based processing
     #
 
-    def run_account(self, addr=None, password=None, account_plugins=None, show_ffi=False):
-        from .events import FFIEventLogger
-
+    def run_account(self, addr=None, password=None, account_plugins=None, show_ffi=False, displayname=None):
         """get the account running, configure it if necessary. add plugins if provided.
 
         :param addr: the email address of the account
         :param password: the password of the account
         :param account_plugins: a list of plugins to add
         :param show_ffi: show low level ffi events
+        :param displayname: the display name of the account
         """
+        from .events import FFIEventLogger
+
         if show_ffi:
-            self.set_config("displayname", "bot")
             log = FFIEventLogger(self)
             self.add_account_plugin(log)
 
@@ -644,6 +644,8 @@ class Account:
             configtracker = self.configure()
             configtracker.wait_finish()
 
+        if displayname:
+            self.set_config("displayname", displayname)
         # start IO threads and configure if necessary
         self.start_io()
 

python/tests/test_1_online.py

@@ -1926,13 +1926,15 @@ def test_set_get_group_image(acfactory, data, lp):
     lp.sec("ac1: add ac2 to promoted group chat")
     chat.add_contact(ac2)  # sends one message
 
+    lp.sec("ac2: wait for receiving member added message from ac1")
+    msg1 = ac2._evtracker.wait_next_incoming_message()
+    assert msg1.is_system_message()  # Member added
+
     lp.sec("ac1: send a first message to ac2")
     chat.send_text("hi")  # sends another message
     assert chat.is_promoted()
 
     lp.sec("ac2: wait for receiving message from ac1")
-    msg1 = ac2._evtracker.wait_next_incoming_message()
-    assert msg1.is_system_message()  # Member added
     msg2 = ac2._evtracker.wait_next_incoming_message()
     assert msg2.text == "hi"
     assert msg1.chat.id == msg2.chat.id

scripts/README.md

@@ -18,7 +18,7 @@ and an own build machine.
 - `remote_tests_rust.sh` rsyncs to the build machine and runs
   `run-rust-test.sh` remotely on the build machine. 
 
-- `make-python-testenv.sh` creates or updates local python test development environment.
+- `make-python-testenv.sh` creates local python test development environment.
   Reusing the same environment is faster than running `run-python-test.sh` which always
   recreates environment from scratch and runs additional lints.
 

scripts/make-python-testenv.sh

@@ -4,8 +4,8 @@
 # It rebuilds the core and bindings as needed.
 #
 # After running the script, you can either
-# run `pytest` directly with `env/bin/pytest python/`
-# or activate the environment with `. env/bin/activacte`
+# run `pytest` directly with `venv/bin/pytest python/`
+# or activate the environment with `. venv/bin/activate`
 # and run `pytest` from there.
 set -euo pipefail
 
@@ -13,9 +13,5 @@ export DCC_RS_TARGET=debug
 export DCC_RS_DEV="$PWD"
 cargo build -p deltachat_ffi --features jsonrpc
 
-if test -d env; then
-	env/bin/pip install -e python --force-reinstall
-else
-	tox -e py --devenv env
-	env/bin/pip install --upgrade pip
-fi
+tox -c python -e py --devenv venv
+env/bin/pip install --upgrade pip

src/config.rs

@@ -286,6 +286,12 @@ pub enum Config {
     #[strum(props(default = "60"))]
     ScanAllFoldersDebounceSecs,
 
+    /// Whether to avoid using IMAP IDLE even if the server supports it.
+    ///
+    /// This is a developer option for testing "fake idle".
+    #[strum(props(default = "0"))]
+    DisableIdle,
+
     /// Defines the max. size (in bytes) of messages downloaded automatically.
     /// 0 = no limit.
     #[strum(props(default = "0"))]
@@ -465,6 +471,28 @@ impl Context {
                     .set_raw_config(key.as_ref(), value.as_deref())
                     .await?;
             }
+            Config::Socks5Enabled
+            | Config::BccSelf
+            | Config::E2eeEnabled
+            | Config::MdnsEnabled
+            | Config::SentboxWatch
+            | Config::MvboxMove
+            | Config::OnlyFetchMvbox
+            | Config::FetchExistingMsgs
+            | Config::DeleteToTrash
+            | Config::SaveMimeHeaders
+            | Config::Configured
+            | Config::Bot
+            | Config::NotifyAboutWrongPw
+            | Config::SendSyncMsgs
+            | Config::SignUnencrypted
+            | Config::DisableIdle => {
+                ensure!(
+                    matches!(value, None | Some("0") | Some("1")),
+                    "Boolean value must be either 0 or 1"
+                );
+                self.sql.set_raw_config(key.as_ref(), value).await?;
+            }
             _ => {
                 self.sql.set_raw_config(key.as_ref(), value).await?;
             }
@@ -614,6 +642,18 @@ mod tests {
         );
     }
 
+    /// Tests that "bot" config can only be set to "0" or "1".
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_set_config_bot() {
+        let t = TestContext::new().await;
+
+        assert!(t.set_config(Config::Bot, None).await.is_ok());
+        assert!(t.set_config(Config::Bot, Some("0")).await.is_ok());
+        assert!(t.set_config(Config::Bot, Some("1")).await.is_ok());
+        assert!(t.set_config(Config::Bot, Some("2")).await.is_err());
+        assert!(t.set_config(Config::Bot, Some("Foobar")).await.is_err());
+    }
+
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_media_quality_config_option() {
         let t = TestContext::new().await;

src/contact.rs

@@ -812,7 +812,11 @@ impl Contact {
         let mut ret = Vec::new();
         let flag_verified_only = (listflags & DC_GCL_VERIFIED_ONLY) != 0;
         let flag_add_self = (listflags & DC_GCL_ADD_SELF) != 0;
-
+        let minimal_origin = if context.get_config_bool(Config::Bot).await? {
+            Origin::Unknown
+        } else {
+            Origin::IncomingReplyTo
+        };
         if flag_verified_only || query.is_some() {
             let s3str_like_cmd = format!("%{}%", query.unwrap_or(""));
             context
@@ -832,7 +836,7 @@ impl Contact {
                     ),
                     rusqlite::params_from_iter(params_iter(&self_addrs).chain(params_slice![
                         ContactId::LAST_SPECIAL,
-                        Origin::IncomingReplyTo,
+                        minimal_origin,
                         s3str_like_cmd,
                         s3str_like_cmd,
                         if flag_verified_only { 0i32 } else { 1i32 }
@@ -882,10 +886,10 @@ impl Contact {
                  ORDER BY last_seen DESC, id DESC;",
                         sql::repeat_vars(self_addrs.len())
                     ),
-                    rusqlite::params_from_iter(params_iter(&self_addrs).chain(params_slice![
-                        ContactId::LAST_SPECIAL,
-                        Origin::IncomingReplyTo
-                    ])),
+                    rusqlite::params_from_iter(
+                        params_iter(&self_addrs)
+                            .chain(params_slice![ContactId::LAST_SPECIAL, minimal_origin]),
+                    ),
                     |row| row.get::<_, ContactId>(0),
                     |ids| {
                         for id in ids {

src/context.rs

@@ -579,6 +579,7 @@ impl Context {
         let mdns_enabled = self.get_config_int(Config::MdnsEnabled).await?;
         let bcc_self = self.get_config_int(Config::BccSelf).await?;
         let send_sync_msgs = self.get_config_int(Config::SendSyncMsgs).await?;
+        let disable_idle = self.get_config_bool(Config::DisableIdle).await?;
 
         let prv_key_cnt = self.sql.count("SELECT COUNT(*) FROM keypairs;", ()).await?;
 
@@ -691,6 +692,7 @@ impl Context {
         );
         res.insert("bcc_self", bcc_self.to_string());
         res.insert("send_sync_msgs", send_sync_msgs.to_string());
+        res.insert("disable_idle", disable_idle.to_string());
         res.insert("private_key_count", prv_key_cnt.to_string());
         res.insert("public_key_count", pub_key_cnt.to_string());
         res.insert("fingerprint", fingerprint_str);

src/download.rs

@@ -23,7 +23,7 @@ use crate::{job_try, stock_str, EventType};
 /// eg. to assign them to the correct chat.
 /// As these messages are typically small,
 /// they're caught by `MIN_DOWNLOAD_LIMIT`.
-const MIN_DOWNLOAD_LIMIT: u32 = 32768;
+pub(crate) const MIN_DOWNLOAD_LIMIT: u32 = 32768;
 
 /// If a message is downloaded only partially
 /// and `delete_server_after` is set to small timeouts (eg. "at once"),

src/imap/idle.rs

@@ -7,7 +7,9 @@ use futures_lite::FutureExt;
 
 use super::session::Session;
 use super::Imap;
+use crate::config::Config;
 use crate::imap::{client::IMAP_TIMEOUT, FolderMeaning};
+use crate::log::LogExt;
 use crate::{context::Context, scheduler::InterruptInfo};
 
 const IDLE_TIMEOUT: Duration = Duration::from_secs(23 * 60);
@@ -21,6 +23,10 @@ impl Session {
     ) -> Result<(Self, InterruptInfo)> {
         use futures::future::FutureExt;
 
+        if context.get_config_bool(Config::DisableIdle).await? {
+            bail!("IMAP IDLE is disabled");
+        }
+
         if !self.can_idle() {
             bail!("IMAP server does not have IDLE capability");
         }
@@ -163,7 +169,14 @@ impl Imap {
                         continue;
                     }
                     if let Some(session) = &self.session {
-                        if session.can_idle() {
+                        if session.can_idle()
+                            && !context
+                                .get_config_bool(Config::DisableIdle)
+                                .await
+                                .context("Failed to get disable_idle config")
+                                .log_err(context)
+                                .unwrap_or_default()
+                        {
                             // we only fake-idled because network was gone during IDLE, probably
                             break InterruptInfo::new(false);
                         }

src/lib.rs

@@ -110,6 +110,8 @@ pub mod tools;
 pub mod accounts;
 pub mod reaction;
 
+pub use deltachat_message_parser as message_parser;
+
 /// If set IMAP/incoming and SMTP/outgoing MIME messages will be printed.
 pub const DCC_MIME_DEBUG: &str = "DCC_MIME_DEBUG";
 

src/mimeparser.rs

@@ -110,7 +110,7 @@ pub(crate) struct MimeMessage {
 
     /// The decrypted, raw mime structure.
     ///
-    /// This is non-empty only if the message was actually encrypted.  It is used
+    /// This is non-empty iff `is_mime_modified` and the message was actually encrypted. It is used
     /// for e.g. late-parsing HTML.
     pub decoded_data: Vec<u8>,
 

src/receive_imf.rs

@@ -77,6 +77,24 @@ pub async fn receive_imf(
     let mail = parse_mail(imf_raw).context("can't parse mail")?;
     let rfc724_mid =
         imap::prefetch_get_message_id(&mail.headers).unwrap_or_else(imap::create_message_id);
+    if let Some(download_limit) = context.download_limit().await? {
+        let download_limit: usize = download_limit.try_into()?;
+        if imf_raw.len() > download_limit {
+            let head = std::str::from_utf8(imf_raw)?
+                .split("\r\n\r\n")
+                .next()
+                .context("No empty line in the message")?;
+            return receive_imf_inner(
+                context,
+                &rfc724_mid,
+                head.as_bytes(),
+                seen,
+                Some(imf_raw.len().try_into()?),
+                false,
+            )
+            .await;
+        }
+    }
     receive_imf_inner(context, &rfc724_mid, imf_raw, seen, None, false).await
 }
 
@@ -578,6 +596,7 @@ async fn add_parts(
             if let Some((new_chat_id, new_chat_id_blocked)) = create_or_lookup_group(
                 context,
                 mime_parser,
+                is_partial_download.is_some(),
                 if test_normal_chat.is_none() {
                     allow_creation
                 } else {
@@ -804,6 +823,7 @@ async fn add_parts(
                 if let Some((new_chat_id, new_chat_id_blocked)) = create_or_lookup_group(
                     context,
                     mime_parser,
+                    is_partial_download.is_some(),
                     allow_creation,
                     Blocked::Not,
                     from_id,
@@ -1186,8 +1206,8 @@ INSERT INTO msgs
   )
 ON CONFLICT (id) DO UPDATE
 SET rfc724_mid=excluded.rfc724_mid, chat_id=excluded.chat_id,
-    from_id=excluded.from_id, to_id=excluded.to_id, timestamp=excluded.timestamp, timestamp_sent=excluded.timestamp_sent,
-    timestamp_rcvd=excluded.timestamp_rcvd, type=excluded.type, state=excluded.state, msgrmsg=excluded.msgrmsg,
+    from_id=excluded.from_id, to_id=excluded.to_id, timestamp_sent=excluded.timestamp_sent,
+    type=excluded.type, msgrmsg=excluded.msgrmsg,
     txt=excluded.txt, subject=excluded.subject, txt_raw=excluded.txt_raw, param=excluded.param,
     bytes=excluded.bytes, mime_headers=excluded.mime_headers,
     mime_compressed=excluded.mime_compressed, mime_in_reply_to=excluded.mime_in_reply_to,
@@ -1500,6 +1520,7 @@ async fn is_probably_private_reply(
 async fn create_or_lookup_group(
     context: &Context,
     mime_parser: &mut MimeMessage,
+    is_partial_download: bool,
     allow_creation: bool,
     create_blocked: Blocked,
     from_id: ContactId,
@@ -1630,7 +1651,7 @@ async fn create_or_lookup_group(
 
     if let Some(chat_id) = chat_id {
         Ok(Some((chat_id, chat_id_blocked)))
-    } else if mime_parser.decrypting_failed {
+    } else if is_partial_download || mime_parser.decrypting_failed {
         // It is possible that the message was sent to a valid,
         // yet unknown group, which was rejected because
         // Chat-Group-Name, which is in the encrypted part, was

src/receive_imf/tests.rs

@@ -10,8 +10,9 @@ use crate::chat::{get_chat_msgs, ChatItem, ChatVisibility};
 use crate::chatlist::Chatlist;
 use crate::config::Config;
 use crate::constants::{DC_GCL_FOR_FORWARDING, DC_GCL_NO_SPECIALS};
+use crate::download::{DownloadState, MIN_DOWNLOAD_LIMIT};
 use crate::imap::prefetch_should_download;
-use crate::message::Message;
+use crate::message::{self, Message};
 use crate::test_utils::{get_chat_msg, TestContext, TestContextManager};
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
@@ -2973,6 +2974,7 @@ async fn test_auto_accept_for_bots() -> Result<()> {
     let msg = t.get_last_msg().await;
     let chat = chat::Chat::load_from_db(&t, msg.chat_id).await?;
     assert!(!chat.is_contact_request());
+    assert!(Contact::get_all(&t, 0, None).await?.len() == 1);
     Ok(())
 }
 
@@ -3697,3 +3699,114 @@ async fn test_keep_member_list_if_possibly_nomember() -> Result<()> {
     assert!(is_contact_in_chat(&bob, bob_chat_id, bob_alice_contact).await?);
     Ok(())
 }
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_download_later() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = tcm.alice().await;
+    alice.set_config(Config::DownloadLimit, Some("1")).await?;
+    assert_eq!(alice.download_limit().await?, Some(MIN_DOWNLOAD_LIMIT));
+
+    let bob = tcm.bob().await;
+    let bob_chat = bob.create_chat(&alice).await;
+    let text = String::from_utf8(vec![b'a'; MIN_DOWNLOAD_LIMIT as usize])?;
+    let sent_msg = bob.send_text(bob_chat.id, &text).await;
+    let msg = alice.recv_msg(&sent_msg).await;
+    assert_eq!(msg.download_state, DownloadState::Available);
+    assert_eq!(msg.state, MessageState::InFresh);
+
+    let hi_msg = tcm.send_recv(&bob, &alice, "hi").await;
+
+    alice.set_config(Config::DownloadLimit, None).await?;
+    let msg = alice.recv_msg(&sent_msg).await;
+    assert_eq!(msg.download_state, DownloadState::Done);
+    assert_eq!(msg.state, MessageState::InFresh);
+    assert_eq!(alice.get_last_msg_in(msg.chat_id).await.id, hi_msg.id);
+    assert!(msg.timestamp_sort <= hi_msg.timestamp_sort);
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_create_group_with_big_msg() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = tcm.alice().await;
+    let bob = tcm.bob().await;
+    let ba_contact = Contact::create(
+        &bob,
+        "alice",
+        &alice.get_config(Config::Addr).await?.unwrap(),
+    )
+    .await?;
+    let file_bytes = include_bytes!("../../test-data/image/screenshot.png");
+
+    let bob_grp_id = create_group_chat(&bob, ProtectionStatus::Unprotected, "Group").await?;
+    add_contact_to_chat(&bob, bob_grp_id, ba_contact).await?;
+    let mut msg = Message::new(Viewtype::Image);
+    msg.set_file_from_bytes(&bob, "a.jpg", file_bytes, None)
+        .await?;
+    let sent_msg = bob.send_msg(bob_grp_id, &mut msg).await;
+    assert!(!msg.get_showpadlock());
+
+    alice.set_config(Config::DownloadLimit, Some("1")).await?;
+    assert_eq!(alice.download_limit().await?, Some(MIN_DOWNLOAD_LIMIT));
+    let msg = alice.recv_msg(&sent_msg).await;
+    assert_eq!(msg.download_state, DownloadState::Available);
+    let alice_grp = Chat::load_from_db(&alice, msg.chat_id).await?;
+    assert_eq!(alice_grp.typ, Chattype::Group);
+    assert_eq!(alice_grp.name, "Group");
+    assert_eq!(
+        chat::get_chat_contacts(&alice, alice_grp.id).await?.len(),
+        2
+    );
+
+    alice.set_config(Config::DownloadLimit, None).await?;
+    let msg = alice.recv_msg(&sent_msg).await;
+    assert_eq!(msg.download_state, DownloadState::Done);
+    assert_eq!(msg.state, MessageState::InFresh);
+    assert_eq!(msg.viewtype, Viewtype::Image);
+    assert_eq!(msg.chat_id, alice_grp.id);
+    let alice_grp = Chat::load_from_db(&alice, msg.chat_id).await?;
+    assert_eq!(alice_grp.typ, Chattype::Group);
+    assert_eq!(alice_grp.name, "Group");
+    assert_eq!(
+        chat::get_chat_contacts(&alice, alice_grp.id).await?.len(),
+        2
+    );
+
+    let ab_chat_id = tcm.send_recv_accept(&alice, &bob, "hi").await.chat_id;
+    // Now Bob can send encrypted messages to Alice.
+
+    let bob_grp_id = create_group_chat(&bob, ProtectionStatus::Unprotected, "Group1").await?;
+    add_contact_to_chat(&bob, bob_grp_id, ba_contact).await?;
+    let mut msg = Message::new(Viewtype::Image);
+    msg.set_file_from_bytes(&bob, "a.jpg", file_bytes, None)
+        .await?;
+    let sent_msg = bob.send_msg(bob_grp_id, &mut msg).await;
+    assert!(msg.get_showpadlock());
+
+    alice.set_config(Config::DownloadLimit, Some("1")).await?;
+    let msg = alice.recv_msg(&sent_msg).await;
+    assert_eq!(msg.download_state, DownloadState::Available);
+    // Until fully downloaded, an encrypted message must sit in the 1:1 chat.
+    assert_eq!(msg.chat_id, ab_chat_id);
+
+    alice.set_config(Config::DownloadLimit, None).await?;
+    let msg = alice.recv_msg(&sent_msg).await;
+    assert_eq!(msg.download_state, DownloadState::Done);
+    assert_eq!(msg.state, MessageState::InFresh);
+    assert_eq!(msg.viewtype, Viewtype::Image);
+    assert_ne!(msg.chat_id, ab_chat_id);
+    let alice_grp = Chat::load_from_db(&alice, msg.chat_id).await?;
+    assert_eq!(alice_grp.typ, Chattype::Group);
+    assert_eq!(alice_grp.name, "Group1");
+    assert_eq!(
+        chat::get_chat_contacts(&alice, alice_grp.id).await?.len(),
+        2
+    );
+
+    // The big message must go away from the 1:1 chat.
+    assert_eq!(alice.get_last_msg_in(ab_chat_id).await.text, "hi");
+
+    Ok(())
+}

src/scheduler.rs

@@ -574,6 +574,19 @@ async fn fetch_idle(
                 .await;
         }
 
+        if ctx
+            .get_config_bool(Config::DisableIdle)
+            .await
+            .context("Failed to get disable_idle config")
+            .log_err(ctx)
+            .unwrap_or_default()
+        {
+            info!(ctx, "IMAP IDLE is disabled, going to fake idle.");
+            return connection
+                .fake_idle(ctx, Some(watch_folder), folder_meaning)
+                .await;
+        }
+
         info!(ctx, "IMAP session supports IDLE, using it.");
         match session
             .idle(