New APIs for message processing loops

This patch adds new C APIs
dc_get_next_msgs() and dc_wait_next_msgs(),
and their JSON-RPC counterparts
get_next_msgs() and wait_next_msgs().

New configuration "last_msg_id"
tracks the last message ID processed by the bot.
get_next_msgs() returns message IDs above
the "last_msg_id".
wait_next_msgs() waits for new message notification
and calls get_next_msgs().
wait_next_msgs() can be used to build
a separate message processing loop
independent of the event loop.

Async Python API get_fresh_messages_in_arrival_order()
is deprecated in favor of get_next_messages().

Introduced Python APIs:
- Account.wait_next_incoming_message()
- Message.is_from_self()
- Message.is_from_device()

Introduced Rust APIs:
- Context.set_config_u32()
- Context.get_config_u32()

deltachat-ffi/deltachat.h

@@ -181,12 +181,17 @@ typedef struct _dc_event_emitter dc_accounts_event_emitter_t;
  * and check it in the event loop thread
  * every time before calling dc_get_next_event().
  * To terminate the event loop, main thread should:
- * 1. Notify event loop that it should terminate by atomically setting the
- *    boolean flag in the memory shared between the main thread and event loop.
+ * 1. Notify background threads,
+ *    such as event loop (blocking in dc_get_next_event())
+ *    and message processing loop (blocking in dc_wait_next_msgs()),
+ *    that they should terminate by atomically setting the
+ *    boolean flag in the memory
+ *    shared between the main thread and background loop threads.
  * 2. Call dc_stop_io() or dc_accounts_stop_io(), depending
  *    on whether a single account or account manager is used.
  *    Stopping I/O is guaranteed to emit at least one event
  *    and interrupt the event loop even if it was blocked on dc_get_next_event().
+ *    Stopping I/O is guaranteed to interrupt a single dc_wait_next_msgs().
  * 3. Wait until the event loop thread notices the flag,
  *    exits the event loop and terminates.
  * 4. Call dc_context_unref() or dc_accounts_unref().
@@ -457,6 +462,16 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    Prevents adding the "Device messages" and "Saved messages" chats,
  *                    adds Auto-Submitted header to outgoing messages
  *                    and accepts contact requests automatically (calling dc_accept_chat() is not needed for bots).
+ * - `last_msg_id` = database ID of the last message processed by the bot.
+ *                   This ID and IDs below it are guaranteed not to be returned
+ *                   by dc_get_next_msgs() and dc_wait_next_msgs().
+ *                   The value is updated automatically
+ *                   when dc_markseen_msgs() is called,
+ *                   but the bot can also set it manually if it processed
+ *                   the message but does not want to mark it as seen.
+ *                   For most bots calling `dc_markseen_msgs()` is the
+ *                   recommended way to update this value
+ *                   even for self-sent messages.
  * - `fetch_existing_msgs` = 1=fetch most recent existing messages on configure (default),
  *                    0=do not fetch existing messages on configure.
  *                    In both cases, existing recipients are added to the contact database.
@@ -1343,6 +1358,56 @@ int             dc_estimate_deletion_cnt    (dc_context_t* context, int from_ser
 dc_array_t*     dc_get_fresh_msgs            (dc_context_t* context);
 
 
+/**
+ * Returns the message IDs of all messages of any chat
+ * with a database ID higher than `last_msg_id` config value.
+ *
+ * This function is intended for use by bots.
+ * Self-sent messages, device messages,
+ * messages from contact requests
+ * and muted chats are included,
+ * but messages from explicitly blocked contacts
+ * and chats are ignored.
+ *
+ * This function may be called as a part of event loop
+ * triggered by DC_EVENT_INCOMING_MSG if you are only interested
+ * in the incoming messages.
+ * Otherwise use a separate message processing loop
+ * calling dc_wait_next_msgs() in a separate thread.
+ *
+ * @memberof dc_context_t
+ * @param context The context object as returned from dc_context_new().
+ * @return An array of message IDs, must be dc_array_unref()'d when no longer used.
+ *     On errors, the list is empty. NULL is never returned.
+ */
+dc_array_t*     dc_get_next_msgs             (dc_context_t* context);
+
+
+/**
+ * Waits for notification of new messages
+ * and returns an array of new message IDs.
+ * See the documentation for dc_get_next_msgs()
+ * for the details of return value.
+ *
+ * This function waits for internal notification of
+ * a new message in the database and returns afterwards.
+ * Notification is also sent when I/O is started
+ * to allow processing new messages
+ * and when I/O is stopped using dc_stop_io() or dc_accounts_stop_io()
+ * to allow for manual interruption of the message processing loop.
+ * The function may return an empty array if there are
+ * no messages after notification,
+ * which may happen on start or if the message is quickly deleted
+ * after adding it to the database.
+ *
+ * @memberof dc_context_t
+ * @param context The context object as returned from dc_context_new().
+ * @return An array of message IDs, must be dc_array_unref()'d when no longer used.
+ *     On errors, the list is empty. NULL is never returned.
+ */
+dc_array_t*     dc_wait_next_msgs            (dc_context_t* context);
+
+
 /**
  * Mark all messages in a chat as _noticed_.
  * _Noticed_ messages are no longer _fresh_ and do not count as being unseen
@@ -1942,6 +2007,11 @@ int             dc_resend_msgs               (dc_context_t* context, const uint3
  * Moreover, timer is started for incoming ephemeral messages.
  * This also happens for contact requests chats.
  *
+ * This function updates last_msg_id configuration value
+ * to the maximum of the current value and IDs passed to this function.
+ * Bots which mark messages as seen can rely on this side effect
+ * to avoid updating last_msg_id value manually.
+ *
  * One #DC_EVENT_MSGS_NOTICED event is emitted per modified chat.
  *
  * @memberof dc_context_t

deltachat-ffi/src/lib.rs

@@ -1280,6 +1280,50 @@ pub unsafe extern "C" fn dc_get_fresh_msgs(
     })
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_get_next_msgs(context: *mut dc_context_t) -> *mut dc_array::dc_array_t {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_get_next_msgs()");
+        return ptr::null_mut();
+    }
+    let ctx = &*context;
+
+    let msg_ids = block_on(ctx.get_next_msgs())
+        .context("failed to get next messages")
+        .log_err(ctx)
+        .unwrap_or_default();
+    let arr = dc_array_t::from(
+        msg_ids
+            .iter()
+            .map(|msg_id| msg_id.to_u32())
+            .collect::<Vec<u32>>(),
+    );
+    Box::into_raw(Box::new(arr))
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_wait_next_msgs(
+    context: *mut dc_context_t,
+) -> *mut dc_array::dc_array_t {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_wait_next_msgs()");
+        return ptr::null_mut();
+    }
+    let ctx = &*context;
+
+    let msg_ids = block_on(ctx.wait_next_msgs())
+        .context("failed to wait for next messages")
+        .log_err(ctx)
+        .unwrap_or_default();
+    let arr = dc_array_t::from(
+        msg_ids
+            .iter()
+            .map(|msg_id| msg_id.to_u32())
+            .collect::<Vec<u32>>(),
+    );
+    Box::into_raw(Box::new(arr))
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_marknoticed_chat(context: *mut dc_context_t, chat_id: u32) {
     if context.is_null() {