New APIs for message processing loops

This patch adds new C APIs
dc_get_next_msgs() and dc_wait_next_msgs(),
and their JSON-RPC counterparts
get_next_msgs() and wait_next_msgs().

New configuration "last_msg_id"
tracks the last message ID processed by the bot.
get_next_msgs() returns message IDs above
the "last_msg_id".
wait_next_msgs() waits for new message notification
and calls get_next_msgs().
wait_next_msgs() can be used to build
a separate message processing loop
independent of the event loop.

Async Python API get_fresh_messages_in_arrival_order()
is deprecated in favor of get_next_messages().

Introduced Python APIs:
- Account.wait_next_incoming_message()
- Message.is_from_self()
- Message.is_from_device()

Introduced Rust APIs:
- Context.set_config_u32()
- Context.get_config_u32()

src/context.rs

@@ -11,7 +11,7 @@ use std::time::{Duration, Instant, SystemTime};
 use anyhow::{bail, ensure, Context as _, Result};
 use async_channel::{self as channel, Receiver, Sender};
 use ratelimit::Ratelimit;
-use tokio::sync::{Mutex, RwLock};
+use tokio::sync::{Mutex, Notify, RwLock};
 use tokio::task;
 
 use crate::chat::{get_chat_cnt, ChatId};
@@ -218,6 +218,11 @@ pub struct InnerContext {
     /// IMAP UID resync request.
     pub(crate) resync_request: AtomicBool,
 
+    /// Notify about new messages.
+    ///
+    /// This causes [`Context::wait_next_msgs`] to wake up.
+    pub(crate) new_msgs_notify: Notify,
+
     /// Server ID response if ID capability is supported
     /// and the server returned non-NIL on the inbox connection.
     /// <https://datatracker.ietf.org/doc/html/rfc2971>
@@ -363,6 +368,11 @@ impl Context {
             blobdir.display()
         );
 
+        let new_msgs_notify = Notify::new();
+        // Notify once immediately to allow processing old messages
+        // without starting I/O.
+        new_msgs_notify.notify_one();
+
         let inner = InnerContext {
             id,
             blobdir,
@@ -379,6 +389,7 @@ impl Context {
             quota: RwLock::new(None),
             quota_update_request: AtomicBool::new(false),
             resync_request: AtomicBool::new(false),
+            new_msgs_notify,
             server_id: RwLock::new(None),
             creation_time: std::time::SystemTime::now(),
             last_full_folder_scan: Mutex::new(None),
@@ -767,6 +778,10 @@ impl Context {
             "debug_logging",
             self.get_config_int(Config::DebugLogging).await?.to_string(),
         );
+        res.insert(
+            "last_msg_id",
+            self.get_config_int(Config::LastMsgId).await?.to_string(),
+        );
 
         let elapsed = self.creation_time.elapsed();
         res.insert("uptime", duration_to_str(elapsed.unwrap_or_default()));
@@ -813,6 +828,66 @@ impl Context {
         Ok(list)
     }
 
+    /// Returns a list of messages with database ID higher than requested.
+    ///
+    /// Blocked contacts and chats are excluded,
+    /// but self-sent messages and contact requests are included in the results.
+    pub async fn get_next_msgs(&self) -> Result<Vec<MsgId>> {
+        let last_msg_id = match self.get_config(Config::LastMsgId).await? {
+            Some(s) => MsgId::new(s.parse()?),
+            None => MsgId::new_unset(),
+        };
+
+        let list = self
+            .sql
+            .query_map(
+                "SELECT m.id
+                     FROM msgs m
+                     LEFT JOIN contacts ct
+                            ON m.from_id=ct.id
+                     LEFT JOIN chats c
+                            ON m.chat_id=c.id
+                     WHERE m.id>?
+                       AND m.hidden=0
+                       AND m.chat_id>9
+                       AND ct.blocked=0
+                       AND c.blocked!=1
+                     ORDER BY m.id ASC",
+                (
+                    last_msg_id.to_u32(), // Explicitly convert to u32 because 0 is allowed.
+                ),
+                |row| {
+                    let msg_id: MsgId = row.get(0)?;
+                    Ok(msg_id)
+                },
+                |rows| {
+                    let mut list = Vec::new();
+                    for row in rows {
+                        list.push(row?);
+                    }
+                    Ok(list)
+                },
+            )
+            .await?;
+        Ok(list)
+    }
+
+    /// Returns a list of messages with database ID higher than last marked as seen.
+    ///
+    /// This function is supposed to be used by bot to request messages
+    /// that are not processed yet.
+    ///
+    /// Waits for notification and returns a result.
+    /// Note that the result may be empty if the message is deleted
+    /// shortly after notification or notification is manually triggered
+    /// to interrupt waiting.
+    /// Notification may be manually triggered by calling [`Self::stop_io`].
+    pub async fn wait_next_msgs(&self) -> Result<Vec<MsgId>> {
+        self.new_msgs_notify.notified().await;
+        let list = self.get_next_msgs().await?;
+        Ok(list)
+    }
+
     /// Searches for messages containing the query string.
     ///
     /// If `chat_id` is provided this searches only for messages in this chat, if `chat_id`
@@ -1444,4 +1519,38 @@ mod tests {
 
         Ok(())
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_get_next_msgs() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        let alice_chat = alice.create_chat(&bob).await;
+
+        assert!(alice.get_next_msgs().await?.is_empty());
+        assert!(bob.get_next_msgs().await?.is_empty());
+
+        let sent_msg = alice.send_text(alice_chat.id, "Hi Bob").await;
+        let received_msg = bob.recv_msg(&sent_msg).await;
+
+        let bob_next_msg_ids = bob.get_next_msgs().await?;
+        assert_eq!(bob_next_msg_ids.len(), 1);
+        assert_eq!(bob_next_msg_ids.get(0), Some(&received_msg.id));
+
+        bob.set_config_u32(Config::LastMsgId, received_msg.id.to_u32())
+            .await?;
+        assert!(bob.get_next_msgs().await?.is_empty());
+
+        // Next messages include self-sent messages.
+        let alice_next_msg_ids = alice.get_next_msgs().await?;
+        assert_eq!(alice_next_msg_ids.len(), 1);
+        assert_eq!(alice_next_msg_ids.get(0), Some(&sent_msg.sender_msg_id));
+
+        alice
+            .set_config_u32(Config::LastMsgId, sent_msg.sender_msg_id.to_u32())
+            .await?;
+        assert!(alice.get_next_msgs().await?.is_empty());
+
+        Ok(())
+    }
 }