fix: Ensure that message being sent is added to the bottom (#8027)

Currently this fails, so if the user fixes their clock incorrectly set to the future, they need to
delete previously sent messages or wait until this future comes again.

Cargo.lock

@@ -827,9 +827,9 @@ dependencies = [
 
 [[package]]
 name = "chrono"
-version = "0.4.43"
+version = "0.4.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118"
+checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0"
 dependencies = [
  "iana-time-zone",
  "num-traits",
@@ -2654,7 +2654,7 @@ dependencies = [
  "hyper",
  "libc",
  "pin-project-lite",
- "socket2 0.6.0",
+ "socket2 0.5.9",
  "tokio",
  "tower-service",
  "tracing",
@@ -3260,9 +3260,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.182"
+version = "0.2.184"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112"
+checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af"
 
 [[package]]
 name = "libm"
@@ -4235,18 +4235,18 @@ dependencies = [
 
 [[package]]
 name = "pin-project"
-version = "1.1.10"
+version = "1.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a"
+checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517"
 dependencies = [
  "pin-project-internal",
 ]
 
 [[package]]
 name = "pin-project-internal"
-version = "1.1.10"
+version = "1.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861"
+checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -4729,9 +4729,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.44"
+version = "1.0.45"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4"
+checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924"
 dependencies = [
  "proc-macro2",
 ]
@@ -5988,9 +5988,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
 
 [[package]]
 name = "tempfile"
-version = "3.26.0"
+version = "3.27.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0"
+checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
  "getrandom 0.3.3",
@@ -6144,9 +6144,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
 [[package]]
 name = "tokio"
-version = "1.49.0"
+version = "1.50.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86"
+checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d"
 dependencies = [
  "bytes",
  "libc",
@@ -6403,9 +6403,9 @@ dependencies = [
 
 [[package]]
 name = "tracing-subscriber"
-version = "0.3.22"
+version = "0.3.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e"
+checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319"
 dependencies = [
  "matchers",
  "nu-ansi-term",

Cargo.toml

@@ -181,7 +181,7 @@ harness = false
 anyhow = "1"
 async-channel = "2.5.0"
 base64 = "0.22"
-chrono = { version = "0.4.43", default-features = false }
+chrono = { version = "0.4.44", default-features = false }
 deltachat-contact-tools = { path = "deltachat-contact-tools" }
 deltachat-jsonrpc = { path = "deltachat-jsonrpc", default-features = false }
 deltachat = { path = ".", default-features = false }
@@ -198,7 +198,7 @@ rusqlite = "0.37"
 sanitize-filename = "0.6"
 serde = "1.0"
 serde_json = "1"
-tempfile = "3.25.0"
+tempfile = "3.27.0"
 thiserror = "2"
 tokio = "1"
 tokio-util = "0.7.18"

deltachat-ffi/deltachat.h

@@ -364,18 +364,14 @@ uint32_t        dc_get_id                    (dc_context_t* context);
  * To get these events, you have to create an event emitter using this function
  * and call dc_get_next_event() on the emitter.
  *
+ * Events are broadcasted to all existing event emitters.
+ * Events emitted before creation of event emitter
+ * are not available to event emitter.
+ *
  * @memberof dc_context_t
  * @param context The context object as created by dc_context_new().
  * @return Returns the event emitter, NULL on errors.
  *     Must be freed using dc_event_emitter_unref() after usage.
- *
- * Note: Use only one event emitter per context.
- * The result of having multiple event emitters is unspecified.
- * Currently events are broadcasted to all existing event emitters,
- * but previous versions delivered events to only one event emitter
- * and this behavior may change again in the future.
- * Events emitted before creation of event emitter
- * may or may not be available to event emitter.
  */
 dc_event_emitter_t* dc_get_event_emitter(dc_context_t* context);
 
@@ -3323,18 +3319,14 @@ void           dc_accounts_set_push_device_token (dc_accounts_t* accounts, const
  * This is similar to dc_get_event_emitter(), which, however,
  * must not be called for accounts handled by the account manager.
  *
+ * Events are broadcasted to all existing event emitters.
+ * Events emitted before creation of event emitter
+ * are not available to event emitter.
+ *
  * @memberof dc_accounts_t
  * @param accounts The account manager as created by dc_accounts_new().
  * @return Returns the event emitter, NULL on errors.
  *     Must be freed using dc_event_emitter_unref() after usage.
- *
- * Note: Use only one event emitter per account manager.
- * The result of having multiple event emitters is unspecified.
- * Currently events are broadcasted to all existing event emitters,
- * but previous versions delivered events to only one event emitter
- * and this behavior may change again in the future.
- * Events emitted before creation of event emitter
- * are not available to event emitter.
  */
 dc_event_emitter_t* dc_accounts_get_event_emitter (dc_accounts_t* accounts);
 
@@ -5979,21 +5971,14 @@ void dc_event_channel_unref(dc_event_channel_t* event_channel);
  * To get these events, you have to create an event emitter using this function
  * and call dc_get_next_event() on the emitter.
  *
- * This is similar to dc_get_event_emitter(), which, however,
- * must not be called for accounts handled by the account manager.
+ * Events are broadcasted to all existing event emitters.
+ * Events emitted before creation of event emitter
+ * are not available to event emitter.
  * 
  * @memberof dc_event_channel_t
  * @param The event channel.
  * @return Returns the event emitter, NULL on errors.
  *     Must be freed using dc_event_emitter_unref() after usage.
- * 
- * Note: Use only one event emitter per account manager / event channel.
- * The result of having multiple event emitters is unspecified.
- * Currently events are broadcasted to all existing event emitters,
- * but previous versions delivered events to only one event emitter
- * and this behavior may change again in the future.
- * Events emitted before creation of event emitter
- * are not available to event emitter.
  */
 dc_event_emitter_t* dc_event_channel_get_event_emitter(dc_event_channel_t* event_channel);
 

src/chat.rs

@@ -2910,11 +2910,23 @@ pub(crate) async fn create_send_msg_jobs(context: &Context, msg: &mut Message) -
         msg.param.remove(Param::GuaranteeE2ee);
     }
     msg.subject.clone_from(&rendered_msg.subject);
+    // Sort the message to the bottom. Employ `msgs_index7` to compute `timestamp`.
     context
         .sql
         .execute(
-            "UPDATE msgs SET pre_rfc724_mid=?, subject=?, param=? WHERE id=?",
+            "
+UPDATE msgs SET
+    timestamp=(
+        SELECT MAX(timestamp) FROM msgs WHERE
+            state IN(10,13,16,18,20,24,26,28) AND
+            hidden IN(0,1) AND
+            chat_id=?
+    ),
+    pre_rfc724_mid=?, subject=?, param=?
+WHERE id=?
+            ",
             (
+                msg.chat_id,
                 &msg.pre_rfc724_mid,
                 &msg.subject,
                 msg.param.to_string(),

src/chat/chat_tests.rs

@@ -5034,6 +5034,33 @@ async fn test_do_not_overwrite_draft() -> Result<()> {
     Ok(())
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_outgoing_msg_after_another_from_future() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let t = &tcm.alice().await;
+    let chat_id = t.get_self_chat().await.id;
+    // Simulate sending a message with clock set to the future.
+    let msg_id = send_text_msg(t, chat_id, "test".to_string()).await?;
+    t.sql
+        .execute(
+            "UPDATE msgs SET timestamp=timestamp+3600 WHERE id=?",
+            (msg_id,),
+        )
+        .await?;
+    let timestamp_sent: i64 = t
+        .sql
+        .query_get_value("SELECT timestamp_sent FROM msgs WHERE id=?", (msg_id,))
+        .await?
+        .unwrap();
+    // Let's have a check here that locally sent messages have zero `timestamp_sent`, it can be a
+    // useful invariant.
+    assert_eq!(timestamp_sent, 0);
+
+    let msg_id = send_text_msg(t, chat_id, "Fixed my clock".to_string()).await?;
+    assert_eq!(t.get_last_msg_in(chat_id).await.id, msg_id);
+    Ok(())
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn test_info_contact_id() -> Result<()> {
     let mut tcm = TestContextManager::new();

src/mimeparser.rs

@@ -373,7 +373,7 @@ impl MimeMessage {
         hop_info += "\n\n";
         hop_info += &dkim_results.to_string();
 
-        let incoming = !context.is_self_addr(&from.addr).await?;
+        let from_is_not_self_addr = !context.is_self_addr(&from.addr).await?;
 
         let mut aheader_values = mail.headers.get_all_values(HeaderDef::Autocrypt.into());
 
@@ -438,7 +438,7 @@ impl MimeMessage {
         };
 
         let mut autocrypt_header = None;
-        if incoming {
+        if from_is_not_self_addr {
             // See `get_all_addresses_from_header()` for why we take the last valid header.
             for val in aheader_values.iter().rev() {
                 autocrypt_header = match Aheader::from_str(val) {
@@ -469,7 +469,7 @@ impl MimeMessage {
             None
         };
 
-        let mut public_keyring = if incoming {
+        let mut public_keyring = if from_is_not_self_addr {
             if let Some(autocrypt_header) = autocrypt_header {
                 vec![autocrypt_header.public_key]
             } else {
@@ -654,6 +654,15 @@ impl MimeMessage {
             .into_iter()
             .last()
             .map(|(fp, recipient_fps)| (fp, recipient_fps.into_iter().collect::<HashSet<_>>()));
+
+        let incoming = if let Some((ref sig_fp, _)) = signature {
+            sig_fp.hex() != key::self_fingerprint(context).await?
+        } else {
+            // rare case of getting a cleartext message
+            // so we determine 'incoming' flag by From-address
+            from_is_not_self_addr
+        };
+
         let mut parser = MimeMessage {
             parts: Vec::new(),
             headers,

src/receive_imf.rs

@@ -45,6 +45,7 @@ use crate::securejoin::{
     self, get_secure_join_step, handle_securejoin_handshake, observe_securejoin_on_other_device,
 };
 use crate::simplify;
+use crate::smtp::msg_has_pending_smtp_job;
 use crate::stats::STATISTICS_BOT_EMAIL;
 use crate::stock_str;
 use crate::sync::Sync::*;
@@ -582,14 +583,7 @@ pub(crate) async fn receive_imf_inner(
                 (rfc724_mid_orig, &self_addr),
             )
             .await?;
-        if !context
-            .sql
-            .exists(
-                "SELECT COUNT(*) FROM smtp WHERE rfc724_mid=?",
-                (rfc724_mid_orig,),
-            )
-            .await?
-        {
+        if !msg_has_pending_smtp_job(context, msg_id).await? {
             msg_id.set_delivered(context).await?;
         }
         return Ok(None);

src/receive_imf/receive_imf_tests.rs

@@ -13,9 +13,10 @@ use crate::constants::DC_GCL_FOR_FORWARDING;
 use crate::contact;
 use crate::imap::prefetch_should_download;
 use crate::imex::{ImexMode, imex};
+use crate::key;
 use crate::securejoin::get_securejoin_qr;
 use crate::test_utils::{
-    E2EE_INFO_MSGS, TestContext, TestContextManager, get_chat_msg, mark_as_verified,
+    E2EE_INFO_MSGS, TestContext, TestContextManager, alice_keypair, get_chat_msg, mark_as_verified,
 };
 use crate::tools::{SystemTime, time};
 
@@ -5561,3 +5562,31 @@ async fn test_calendar_alternative() -> Result<()> {
 
     Ok(())
 }
+
+/// Tests that outgoing encrypted messages are detected
+/// by verifying own signature, completely ignoring From address.
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn test_outgoing_determined_by_signature() -> Result<()> {
+    let mut tcm = TestContextManager::new();
+    let alice = &tcm.alice().await;
+    let bob = &tcm.bob().await;
+
+    // alice_dev2: same key, different address.
+    let different_from = "very@different.from";
+    assert!(!alice.is_self_addr(different_from).await?);
+    let alice_dev2 = &tcm.unconfigured().await;
+    alice_dev2.configure_addr(different_from).await;
+    key::store_self_keypair(alice_dev2, &alice_keypair()).await?;
+    assert_ne!(
+        alice.get_config(Config::Addr).await?.unwrap(),
+        different_from
+    );
+
+    // Send message from alice_dev2 and check alice sees it as outgoing
+    let chat_id = alice_dev2.create_chat_id(bob).await;
+    let sent_msg = alice_dev2.send_text(chat_id, "hello from new device").await;
+    let msg = alice.recv_msg(&sent_msg).await;
+    assert_eq!(msg.state, MessageState::OutDelivered);
+
+    Ok(())
+}

src/smtp.rs

@@ -465,11 +465,7 @@ pub(crate) async fn send_msg_to_smtp(
     match status {
         SendResult::Retry => Err(format_err!("Retry")),
         SendResult::Success => {
-            if !context
-                .sql
-                .exists("SELECT COUNT(*) FROM smtp WHERE msg_id=?", (msg_id,))
-                .await?
-            {
+            if !msg_has_pending_smtp_job(context, msg_id).await? {
                 msg_id.set_delivered(context).await?;
             }
             Ok(())
@@ -478,6 +474,16 @@ pub(crate) async fn send_msg_to_smtp(
     }
 }
 
+pub(crate) async fn msg_has_pending_smtp_job(
+    context: &Context,
+    msg_id: MsgId,
+) -> Result<bool, Error> {
+    context
+        .sql
+        .exists("SELECT COUNT(*) FROM smtp WHERE msg_id=?", (msg_id,))
+        .await
+}
+
 /// Attempts to send queued MDNs.
 async fn send_mdns(context: &Context, connection: &mut Smtp) -> Result<()> {
     loop {

src/test_utils.rs

@@ -40,6 +40,7 @@ use crate::message::{Message, MessageState, MsgId, update_msg_state};
 use crate::mimeparser::{MimeMessage, SystemMessage};
 use crate::receive_imf::receive_imf;
 use crate::securejoin::{get_securejoin_qr, join_securejoin};
+use crate::smtp::msg_has_pending_smtp_job;
 use crate::stock_str::StockStrings;
 use crate::tools::time;
 
@@ -658,10 +659,7 @@ impl TestContext {
             .execute("DELETE FROM smtp WHERE id=?;", (rowid,))
             .await
             .expect("failed to remove job");
-        if !self
-            .ctx
-            .sql
-            .exists("SELECT COUNT(*) FROM smtp WHERE msg_id=?", (msg_id,))
+        if !msg_has_pending_smtp_job(self, msg_id)
             .await
             .expect("Failed to check for more jobs")
         {