From 1d62448903a5145dc33af9ec46131a7f9ca57bb8 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Thu, 7 Jan 2021 23:30:51 +0100 Subject: [PATCH 1/8] add flag to get only info/system messages of chat useful for creating an "audit log" view of a chat that helps to find actions like who added/removed who quickly and without endless scrolling --- deltachat-ffi/deltachat.h | 2 ++ src/chat.rs | 58 ++++++++++++++++++++++++++++++++++++--- src/constants.rs | 1 + 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 2cdb05fad..189fde176 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -1000,6 +1000,7 @@ dc_msg_t* dc_get_draft (dc_context_t* context, uint32_t ch #define DC_GCM_ADDDAYMARKER 0x01 +#define DC_GCM_SYSTEM_ONLY 0x02 /** @@ -1018,6 +1019,7 @@ dc_msg_t* dc_get_draft (dc_context_t* context, uint32_t ch * @param flags If set to DC_GCM_ADDDAYMARKER, the marker DC_MSG_ID_DAYMARKER will * be added before each day (regarding the local timezone). Set this to 0 if you do not want this behaviour. * To get the concrete time of the marker, use dc_array_get_timestamp(). + * If set to DC_GCM_SYSTEM_ONLY, only system messages will be returned, can be combined with DC_GCM_ADDDAYMARKER. * @param marker1before An optional message ID. If set, the id DC_MSG_ID_MARKER1 will be added just * before the given ID in the returned array. Set this to 0 if you do not want this behaviour. * @return Array of message IDs, must be dc_array_unref()'d when no longer used. diff --git a/src/chat.rs b/src/chat.rs index a4a3102a1..8abf959ef 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -2,6 +2,7 @@ use deltachat_derive::{FromSql, ToSql}; use std::convert::TryFrom; +use std::str::FromStr; use std::time::{Duration, SystemTime}; use anyhow::Context as _; @@ -1764,14 +1765,42 @@ pub async fn get_chat_msgs( } } - let process_row = - |row: &rusqlite::Row| Ok((row.get::<_, MsgId>("id")?, row.get::<_, i64>("timestamp")?)); + let process_row = if (flags & DC_GCM_SYSTEM_ONLY) != 0 { + |row: &rusqlite::Row| { + // is_info logic taken from Message.is_info() + let params = row.get::<_, String>("param")?; + let (from_id, to_id) = (row.get::<_, u32>("from_id")?, row.get::<_, u32>("to_id")?); + let is_info_msg: bool = from_id == DC_CONTACT_ID_INFO as u32 + || to_id == DC_CONTACT_ID_INFO as u32 + || match Params::from_str(¶ms) { + Ok(p) => { + let cmd = p.get_cmd(); + cmd != SystemMessage::Unknown && cmd != SystemMessage::AutocryptSetupMessage + } + _ => false, + }; + + Ok(( + row.get::<_, MsgId>("id")?, + row.get::<_, i64>("timestamp")?, + !is_info_msg, + )) + } + } else { + |row: &rusqlite::Row| { + Ok(( + row.get::<_, MsgId>("id")?, + row.get::<_, i64>("timestamp")?, + false, + )) + } + }; let process_rows = |rows: rusqlite::MappedRows<_>| { let mut ret = Vec::new(); let mut last_day = 0; let cnv_to_local = dc_gm2local_offset(); for row in rows { - let (curr_id, ts) = row?; + let (curr_id, ts, exclude_message): (MsgId, i64, bool) = row?; if let Some(marker_id) = marker1before { if curr_id == marker_id { ret.push(ChatItem::Marker1); @@ -1787,7 +1816,9 @@ pub async fn get_chat_msgs( last_day = curr_day; } } - ret.push(ChatItem::Message { msg_id: curr_id }); + if !exclude_message { + ret.push(ChatItem::Message { msg_id: curr_id }); + } } Ok(ret) }; @@ -1815,6 +1846,25 @@ pub async fn get_chat_msgs( process_rows, ) .await + } else if (flags & DC_GCM_SYSTEM_ONLY) != 0 { + context + .sql + .query_map( + "SELECT m.id AS id, m.timestamp AS timestamp, m.param AS param, m.from_id AS from_id, m.to_id AS to_id + FROM msgs m + WHERE m.chat_id=? + AND m.hidden=0 + AND ( + m.param GLOB \"*S=*\" + OR m.from_id == 2 + OR m.to_id == 2 + ) + ORDER BY m.timestamp, m.id;", + paramsv![chat_id], + process_row, + process_rows, + ) + .await } else { context .sql diff --git a/src/constants.rs b/src/constants.rs index 6ce68abd1..1f03e04f8 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -99,6 +99,7 @@ pub const DC_GCL_ADD_ALLDONE_HINT: usize = 0x04; pub const DC_GCL_FOR_FORWARDING: usize = 0x08; pub const DC_GCM_ADDDAYMARKER: u32 = 0x01; +pub const DC_GCM_SYSTEM_ONLY: u32 = 0x02; pub const DC_GCL_VERIFIED_ONLY: usize = 0x01; pub const DC_GCL_ADD_SELF: usize = 0x02; From 32cbdc630df32d9728f3b003738320b499f98332 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 10 Jan 2021 15:25:11 +0100 Subject: [PATCH 2/8] add test for chat audit log --- python/tests/test_account.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/python/tests/test_account.py b/python/tests/test_account.py index 557b10a8d..ec5133e88 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -8,6 +8,8 @@ from deltachat import const, Account from deltachat.message import Message from deltachat.tracker import ImexTracker from deltachat.hookspec import account_hookimpl +from deltachat.capi import ffi, lib +from deltachat.cutil import iter_array from datetime import datetime, timedelta @@ -595,6 +597,28 @@ class TestOfflineChat: assert in_list[1][1] == chat assert in_list[1][2] == contacts[3] + def test_audit_log_view(self, ac1, lp): + lp.sec("ac1: test audit log (show only system messages)") + chat = ac1.create_group_chat(name="audit log sample data") + # promote chat + chat.send_text("hello") + assert chat.is_promoted() + + lp.sec("create test data") + chat.add_contact(ac1.create_contact("some-1@example.org")) + chat.set_name("audit log test group") + chat.send_text("a message in between") + + lp.sec("check message count of all messages") + assert len(chat.get_messages()) == 4 + + lp.sec("check message count of only system messages (without daymarkers)") + dc_array = ffi.gc( + lib.dc_get_chat_msgs(ac1._dc_context, chat.id, 2, 0), # todo replace `2` with const.DC_GCM_SYSTEM_ONLY + lib.dc_array_unref + ) + assert len(list(iter_array(dc_array, lambda x: x))) == 2 + def test_basic_imap_api(acfactory, tmpdir): ac1, ac2 = acfactory.get_two_online_accounts() From 7b80801cb7839f22c9c3fba58de04c17ee48f3d9 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 10 Jan 2021 15:53:28 +0100 Subject: [PATCH 3/8] clarifiy test name (that it test without daymarker) --- python/tests/test_account.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/tests/test_account.py b/python/tests/test_account.py index ec5133e88..6540b3e85 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -597,7 +597,7 @@ class TestOfflineChat: assert in_list[1][1] == chat assert in_list[1][2] == contacts[3] - def test_audit_log_view(self, ac1, lp): + def test_audit_log_view_without_daymarker(self, ac1, lp): lp.sec("ac1: test audit log (show only system messages)") chat = ac1.create_group_chat(name="audit log sample data") # promote chat From 93845c2a18e1014f9ed6e3b00e97ee17d26c6355 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Sun, 10 Jan 2021 19:20:30 +0100 Subject: [PATCH 4/8] include constant and use it --- python/src/deltachat/_build.py | 1 + python/tests/test_account.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/python/src/deltachat/_build.py b/python/src/deltachat/_build.py index 687be4ac3..2af24eee8 100644 --- a/python/src/deltachat/_build.py +++ b/python/src/deltachat/_build.py @@ -145,6 +145,7 @@ def extract_defines(flags): | DC_STR | DC_CONTACT_ID | DC_GCL + | DC_GCM | DC_SOCKET | DC_CHAT | DC_PROVIDER diff --git a/python/tests/test_account.py b/python/tests/test_account.py index 6540b3e85..321215d8d 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -614,7 +614,7 @@ class TestOfflineChat: lp.sec("check message count of only system messages (without daymarkers)") dc_array = ffi.gc( - lib.dc_get_chat_msgs(ac1._dc_context, chat.id, 2, 0), # todo replace `2` with const.DC_GCM_SYSTEM_ONLY + lib.dc_get_chat_msgs(ac1._dc_context, chat.id, const.DC_GCM_SYSTEM_ONLY, 0), lib.dc_array_unref ) assert len(list(iter_array(dc_array, lambda x: x))) == 2 From 9957bad83d2a6163788a649f494067e5d8acd4dd Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Thu, 14 Jan 2021 16:19:59 +0100 Subject: [PATCH 5/8] rename DC_GCM_SYSTEM_ONLY to DC_GCM_INFO_ONLY apply suggestions of r10s --- deltachat-ffi/deltachat.h | 4 ++-- python/tests/test_account.py | 2 +- src/chat.rs | 11 ++++++----- src/constants.rs | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 189fde176..1004f7971 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -1000,7 +1000,7 @@ dc_msg_t* dc_get_draft (dc_context_t* context, uint32_t ch #define DC_GCM_ADDDAYMARKER 0x01 -#define DC_GCM_SYSTEM_ONLY 0x02 +#define DC_GCM_INFO_ONLY 0x02 /** @@ -1019,7 +1019,7 @@ dc_msg_t* dc_get_draft (dc_context_t* context, uint32_t ch * @param flags If set to DC_GCM_ADDDAYMARKER, the marker DC_MSG_ID_DAYMARKER will * be added before each day (regarding the local timezone). Set this to 0 if you do not want this behaviour. * To get the concrete time of the marker, use dc_array_get_timestamp(). - * If set to DC_GCM_SYSTEM_ONLY, only system messages will be returned, can be combined with DC_GCM_ADDDAYMARKER. + * If set to DC_GCM_INFO_ONLY, only system messages will be returned, can be combined with DC_GCM_ADDDAYMARKER. * @param marker1before An optional message ID. If set, the id DC_MSG_ID_MARKER1 will be added just * before the given ID in the returned array. Set this to 0 if you do not want this behaviour. * @return Array of message IDs, must be dc_array_unref()'d when no longer used. diff --git a/python/tests/test_account.py b/python/tests/test_account.py index 321215d8d..35a985fdd 100644 --- a/python/tests/test_account.py +++ b/python/tests/test_account.py @@ -614,7 +614,7 @@ class TestOfflineChat: lp.sec("check message count of only system messages (without daymarkers)") dc_array = ffi.gc( - lib.dc_get_chat_msgs(ac1._dc_context, chat.id, const.DC_GCM_SYSTEM_ONLY, 0), + lib.dc_get_chat_msgs(ac1._dc_context, chat.id, const.DC_GCM_INFO_ONLY, 0), lib.dc_array_unref ) assert len(list(iter_array(dc_array, lambda x: x))) == 2 diff --git a/src/chat.rs b/src/chat.rs index 8abf959ef..8aae7cced 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -1765,7 +1765,7 @@ pub async fn get_chat_msgs( } } - let process_row = if (flags & DC_GCM_SYSTEM_ONLY) != 0 { + let process_row = if (flags & DC_GCM_INFO_ONLY) != 0 { |row: &rusqlite::Row| { // is_info logic taken from Message.is_info() let params = row.get::<_, String>("param")?; @@ -1846,21 +1846,22 @@ pub async fn get_chat_msgs( process_rows, ) .await - } else if (flags & DC_GCM_SYSTEM_ONLY) != 0 { + } else if (flags & DC_GCM_INFO_ONLY) != 0 { context .sql .query_map( + // GLOB is used here instead of LIKE becase it is case-sensitive "SELECT m.id AS id, m.timestamp AS timestamp, m.param AS param, m.from_id AS from_id, m.to_id AS to_id FROM msgs m WHERE m.chat_id=? AND m.hidden=0 AND ( m.param GLOB \"*S=*\" - OR m.from_id == 2 - OR m.to_id == 2 + OR m.from_id == ? + OR m.to_id == ? ) ORDER BY m.timestamp, m.id;", - paramsv![chat_id], + paramsv![chat_id, DC_CONTACT_ID_INFO, DC_CONTACT_ID_INFO], process_row, process_rows, ) diff --git a/src/constants.rs b/src/constants.rs index 1f03e04f8..3fd7ae8ac 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -99,7 +99,7 @@ pub const DC_GCL_ADD_ALLDONE_HINT: usize = 0x04; pub const DC_GCL_FOR_FORWARDING: usize = 0x08; pub const DC_GCM_ADDDAYMARKER: u32 = 0x01; -pub const DC_GCM_SYSTEM_ONLY: u32 = 0x02; +pub const DC_GCM_INFO_ONLY: u32 = 0x02; pub const DC_GCL_VERIFIED_ONLY: usize = 0x01; pub const DC_GCL_ADD_SELF: usize = 0x02; From 7154a47f857fb905ab61f2e53291f23159b49212 Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Thu, 14 Jan 2021 18:33:12 +0100 Subject: [PATCH 6/8] Update deltachat-ffi/deltachat.h Co-authored-by: bjoern --- deltachat-ffi/deltachat.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deltachat-ffi/deltachat.h b/deltachat-ffi/deltachat.h index 1004f7971..283a7601d 100644 --- a/deltachat-ffi/deltachat.h +++ b/deltachat-ffi/deltachat.h @@ -1000,7 +1000,7 @@ dc_msg_t* dc_get_draft (dc_context_t* context, uint32_t ch #define DC_GCM_ADDDAYMARKER 0x01 -#define DC_GCM_INFO_ONLY 0x02 +#define DC_GCM_INFO_ONLY 0x02 /** From dae20d90ed535c31abebf161f3a7dc1e5b21eee4 Mon Sep 17 00:00:00 2001 From: "B. Petersen" Date: Mon, 18 Jan 2021 17:45:57 +0100 Subject: [PATCH 7/8] add missing import --- src/chat.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/chat.rs b/src/chat.rs index 8aae7cced..8d8f8221f 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -18,7 +18,7 @@ use crate::constants::{ Blocked, Chattype, ShowEmails, Viewtype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK, DC_CHAT_ID_DEADDROP, DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH, DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_INFO, DC_CONTACT_ID_LAST_SPECIAL, DC_CONTACT_ID_SELF, DC_GCM_ADDDAYMARKER, - DC_RESEND_USER_AVATAR_DAYS, + DC_GCM_INFO_ONLY, DC_RESEND_USER_AVATAR_DAYS, }; use crate::contact::{addr_cmp, Contact, Origin, VerifiedStatus}; use crate::context::Context; From 43e95ce68be59bf1970828491751c35892054f9d Mon Sep 17 00:00:00 2001 From: Simon Laux Date: Mon, 18 Jan 2021 18:56:52 +0100 Subject: [PATCH 8/8] fix formatting of python code --- python/src/deltachat/_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/src/deltachat/_build.py b/python/src/deltachat/_build.py index 2af24eee8..252c5c212 100644 --- a/python/src/deltachat/_build.py +++ b/python/src/deltachat/_build.py @@ -145,7 +145,7 @@ def extract_defines(flags): | DC_STR | DC_CONTACT_ID | DC_GCL - | DC_GCM + | DC_GCM | DC_SOCKET | DC_CHAT | DC_PROVIDER