more sql fixes

Cargo.lock

@@ -90,7 +90,7 @@ dependencies = [
 [[package]]
 name = "async-imap"
 version = "0.1.1"
-source = "git+https://github.com/async-email/async-imap#377d40837028b454c6365ff13e3f35cc341a908e"
+source = "git+https://github.com/async-email/async-imap#1327f678cf5515842fc309636459372e6ab40db2"
 dependencies = [
  "async-attributes 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "async-std 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)",

deltachat-ffi/deltachat.h

@@ -4387,16 +4387,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_EVENT_SECUREJOIN_JOINER_PROGRESS       2061
 
 
-/**
- * This event is sent out to the inviter when a joiner successfully joined a group.
- *
- * @param data1 (int) chat_id
- * @param data2 (int) contact_id
- * @return 0
- */
-#define DC_EVENT_SECUREJOIN_MEMBER_ADDED 2062
-
-
 /**
  * @}
  */

deltachat-ffi/src/lib.rs

@@ -175,15 +175,6 @@ impl ContextWrapper {
                         contact_id as uintptr_t,
                         progress as uintptr_t,
                     ),
-                    Event::SecurejoinMemberAdded {
-                        chat_id,
-                        contact_id,
-                    } => ffi_cb(
-                        self,
-                        event_id,
-                        chat_id as uintptr_t,
-                        contact_id as uintptr_t,
-                    ),
                 }
             }
             None => 0,

examples/repl/cmdline.rs

@@ -132,8 +132,10 @@ fn poke_spec(context: &Context, spec: *const libc::c_char) -> libc::c_int {
         real_spec = rs.unwrap();
     }
     if let Some(suffix) = dc_get_filesuffix_lc(&real_spec) {
-        if suffix == "eml" && dc_poke_eml_file(context, &real_spec).is_ok() {
-            read_cnt += 1
+        if suffix == "eml" {
+            if dc_poke_eml_file(context, &real_spec).is_ok() {
+                read_cnt += 1
+            }
         }
     } else {
         /* import a directory */
@@ -586,7 +588,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
             let members = chat::get_chat_contacts(context, sel_chat.id);
             let subtitle = if sel_chat.is_device_talk() {
                 "device-talk".to_string()
-            } else if sel_chat.get_type() == Chattype::Single && !members.is_empty() {
+            } else if sel_chat.get_type() == Chattype::Single && members.len() >= 1 {
                 let contact = Contact::get_by_id(context, members[0])?;
                 contact.get_addr().to_string()
             } else {
@@ -860,7 +862,11 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
         "archive" | "unarchive" => {
             ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
             let chat_id = arg1.parse()?;
-            chat::archive(context, chat_id, arg0 == "archive")?;
+            chat::archive(
+                context,
+                chat_id,
+                if arg0 == "archive" { true } else { false },
+            )?;
         }
         "delchat" => {
             ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");

examples/repl/main.rs

@@ -235,7 +235,7 @@ impl Completer for DcHelper {
     }
 }
 
-const IMEX_COMMANDS: [&str; 12] = [
+const IMEX_COMMANDS: [&'static str; 12] = [
     "initiate-key-transfer",
     "get-setupcodebegin",
     "continue-key-transfer",
@@ -250,7 +250,7 @@ const IMEX_COMMANDS: [&str; 12] = [
     "stop",
 ];
 
-const DB_COMMANDS: [&str; 11] = [
+const DB_COMMANDS: [&'static str; 11] = [
     "info",
     "open",
     "close",
@@ -264,7 +264,7 @@ const DB_COMMANDS: [&str; 11] = [
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 24] = [
+const CHAT_COMMANDS: [&'static str; 24] = [
     "listchats",
     "listarchived",
     "chat",
@@ -290,7 +290,7 @@ const CHAT_COMMANDS: [&str; 24] = [
     "unarchive",
     "delchat",
 ];
-const MESSAGE_COMMANDS: [&str; 8] = [
+const MESSAGE_COMMANDS: [&'static str; 8] = [
     "listmsgs",
     "msginfo",
     "listfresh",
@@ -300,7 +300,7 @@ const MESSAGE_COMMANDS: [&str; 8] = [
     "unstar",
     "delmsg",
 ];
-const CONTACT_COMMANDS: [&str; 6] = [
+const CONTACT_COMMANDS: [&'static str; 6] = [
     "listcontacts",
     "listverified",
     "addcontact",
@@ -308,7 +308,7 @@ const CONTACT_COMMANDS: [&str; 6] = [
     "delcontact",
     "cleanupcontacts",
 ];
-const MISC_COMMANDS: [&str; 9] = [
+const MISC_COMMANDS: [&'static str; 9] = [
     "getqr", "getbadqr", "checkqr", "event", "fileinfo", "clear", "exit", "quit", "help",
 ];
 
@@ -334,8 +334,8 @@ impl Hinter for DcHelper {
     }
 }
 
-static COLORED_PROMPT: &str = "\x1b[1;32m> \x1b[0m";
-static PROMPT: &str = "> ";
+static COLORED_PROMPT: &'static str = "\x1b[1;32m> \x1b[0m";
+static PROMPT: &'static str = "> ";
 
 impl Highlighter for DcHelper {
     fn highlight_prompt<'p>(&self, prompt: &'p str) -> Cow<'p, str> {

examples/simple.rs

@@ -113,7 +113,7 @@ fn main() {
 
     println!("stopping threads");
 
-    *running.write().unwrap() = false;
+    *running.clone().write().unwrap() = false;
     deltachat::job::interrupt_inbox_idle(&ctx, true);
     deltachat::job::interrupt_smtp_idle(&ctx);
 

python/src/deltachat/const.py

@@ -98,7 +98,6 @@ DC_EVENT_IMEX_PROGRESS = 2051
 DC_EVENT_IMEX_FILE_WRITTEN = 2052
 DC_EVENT_SECUREJOIN_INVITER_PROGRESS = 2060
 DC_EVENT_SECUREJOIN_JOINER_PROGRESS = 2061
-DC_EVENT_SECUREJOIN_MEMBER_ADDED = 2062
 DC_EVENT_FILE_COPIED = 2055
 DC_EVENT_IS_OFFLINE = 2081
 DC_EVENT_GET_STRING = 2091
@@ -151,8 +150,7 @@ DC_STR_MSGLOCATIONENABLED = 64
 DC_STR_MSGLOCATIONDISABLED = 65
 DC_STR_LOCATION = 66
 DC_STR_STICKER = 67
-DC_STR_DEVICE_MESSAGES = 68
-DC_STR_COUNT = 68
+DC_STR_COUNT = 67
 # end const generated
 
 

python/tests/test_account.py

@@ -461,20 +461,6 @@ class TestOnlineAccount:
         assert ev[2] > const.DC_CHAT_ID_LAST_SPECIAL
         ev = ac2._evlogger.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
 
-    def test_move_works_on_self_sent(self, acfactory):
-        ac1 = acfactory.get_online_configuring_account(mvbox=True)
-        ac1.set_config("bcc_self", "1")
-        ac2 = acfactory.get_online_configuring_account()
-        wait_configuration_progress(ac2, 1000)
-        wait_configuration_progress(ac1, 1000)
-        chat = self.get_chat(ac1, ac2)
-        chat.send_text("message1")
-        chat.send_text("message2")
-        chat.send_text("message3")
-        ac1._evlogger.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
-        ac1._evlogger.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
-        ac1._evlogger.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
-
     def test_forward_messages(self, acfactory):
         ac1, ac2 = acfactory.get_two_online_accounts()
         chat = self.get_chat(ac1, ac2)
@@ -782,7 +768,6 @@ class TestOnlineAccount:
         ac1._evlogger.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
         ac2._evlogger.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
         wait_securejoin_inviter_progress(ac1, 1000)
-        ac1._evlogger.get_matching("DC_EVENT_SECUREJOIN_MEMBER_ADDED")
 
     def test_qr_verified_group_and_chatting(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()

python/tox.ini

@@ -53,7 +53,7 @@ deps =
     sphinx==2.2.0
     breathe
 commands =
-    sphinx-build -Q -w toxdoc-warnings.log -b html . _build/html
+    sphinx-build -w toxdoc-warnings.log -b html . _build/html
 
 
 [testenv:lintdoc]

src/configure/mod.rs

@@ -45,8 +45,9 @@ pub fn dc_is_configured(context: &Context) -> bool {
 /*******************************************************************************
  * Configure JOB
  ******************************************************************************/
+// the other dc_job_do_DC_JOB_*() functions are declared static in the c-file
 #[allow(non_snake_case, unused_must_use)]
-pub fn JobConfigureImap(context: &Context) {
+pub fn dc_job_do_DC_JOB_CONFIGURE_IMAP(context: &Context) {
     if !context.sql.is_open() {
         error!(context, "Cannot configure, database not opened.",);
         progress!(context, 0);
@@ -353,14 +354,19 @@ pub fn JobConfigureImap(context: &Context) {
             }
             16 => {
                 progress!(context, 900);
-                let create_mvbox = context.get_config_bool(Config::MvboxWatch)
-                    || context.get_config_bool(Config::MvboxMove);
+                let flags: libc::c_int = if context.get_config_bool(Config::MvboxWatch)
+                    || context.get_config_bool(Config::MvboxMove)
+                {
+                    DC_CREATE_MVBOX as i32
+                } else {
+                    0
+                };
                 context
                     .inbox_thread
                     .read()
                     .unwrap()
                     .imap
-                    .configure_folders(context, create_mvbox);
+                    .configure_folders(context, flags);
                 true
             }
             17 => {
@@ -600,7 +606,7 @@ pub fn read_autoconf_file(context: &Context, url: &str) -> Option<String> {
 mod tests {
 
     use crate::config::*;
-    use crate::configure::JobConfigureImap;
+    use crate::configure::dc_job_do_DC_JOB_CONFIGURE_IMAP;
     use crate::test_utils::*;
 
     #[test]
@@ -610,6 +616,6 @@ mod tests {
             .set_config(Config::Addr, Some("probably@unexistant.addr"))
             .unwrap();
         t.ctx.set_config(Config::MailPw, Some("123456")).unwrap();
-        JobConfigureImap(&t.ctx);
+        dc_job_do_DC_JOB_CONFIGURE_IMAP(&t.ctx);
     }
 }

src/constants.rs

@@ -8,6 +8,21 @@ lazy_static! {
     pub static ref DC_VERSION_STR: String = env!("CARGO_PKG_VERSION").to_string();
 }
 
+#[repr(u8)]
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql)]
+pub enum MoveState {
+    Undefined = 0,
+    Pending = 1,
+    Stay = 2,
+    Moving = 3,
+}
+
+impl Default for MoveState {
+    fn default() -> Self {
+        MoveState::Undefined
+    }
+}
+
 // some defaults
 const DC_E2EE_DEFAULT_ENABLED: i32 = 1;
 const DC_INBOX_WATCH_DEFAULT: i32 = 1;
@@ -119,6 +134,8 @@ pub const DC_CONTACT_ID_INFO: u32 = 2;
 pub const DC_CONTACT_ID_DEVICE: u32 = 5;
 pub const DC_CONTACT_ID_LAST_SPECIAL: u32 = 9;
 
+pub const DC_CREATE_MVBOX: usize = 1;
+
 // Flags for empty server job
 
 pub const DC_EMPTY_MVBOX: u32 = 0x01;

src/contact.rs

@@ -585,12 +585,7 @@ impl Contact {
 
             let mut self_key = Key::from_self_public(context, &loginparam.addr, &context.sql);
 
-            if peerstate.is_some()
-                && peerstate
-                    .as_ref()
-                    .and_then(|p| p.peek_key(PeerstateVerifiedStatus::Unverified))
-                    .is_some()
-            {
+            if peerstate.is_some() && peerstate.as_ref().and_then(|p| p.peek_key(0)).is_some() {
                 let peerstate = peerstate.as_ref().unwrap();
                 let p =
                     context.stock_str(if peerstate.prefer_encrypt == EncryptPreference::Mutual {
@@ -610,11 +605,11 @@ impl Contact {
                     .map(|k| k.formatted_fingerprint())
                     .unwrap_or_default();
                 let fingerprint_other_verified = peerstate
-                    .peek_key(PeerstateVerifiedStatus::BidirectVerified)
+                    .peek_key(2)
                     .map(|k| k.formatted_fingerprint())
                     .unwrap_or_default();
                 let fingerprint_other_unverified = peerstate
-                    .peek_key(PeerstateVerifiedStatus::Unverified)
+                    .peek_key(0)
                     .map(|k| k.formatted_fingerprint())
                     .unwrap_or_default();
                 if peerstate.addr.is_some() && &loginparam.addr < peerstate.addr.as_ref().unwrap() {

src/context.rs

@@ -435,9 +435,10 @@ impl Context {
             return;
         }
 
-        if self.is_mvbox(folder) {
+        if !self.is_inbox(folder) && !self.is_sentbox(folder) {
             return;
         }
+
         if let Ok(msg) = Message::load_from_db(self, msg_id) {
             if msg.is_setupmessage() {
                 // do not move setup messages;
@@ -445,6 +446,10 @@ impl Context {
                 return;
             }
 
+            if self.is_mvbox(folder) {
+                message::update_msg_move_state(self, &msg.rfc724_mid, MoveState::Stay);
+            }
+
             // 1 = dc message, 2 = reply to dc message
             if 0 != msg.is_dc_message {
                 job_add(
@@ -454,6 +459,7 @@ impl Context {
                     Params::new(),
                     0,
                 );
+                message::update_msg_move_state(self, &msg.rfc724_mid, MoveState::Moving);
             }
         }
     }
@@ -489,25 +495,12 @@ pub struct BobStatus {
     pub qr_scan: Option<Lot>,
 }
 
-#[derive(Debug, PartialEq)]
-pub enum PerformJobsNeeded {
-    Not,
-    AtOnce,
-    AvoidDos,
-}
-
-impl Default for PerformJobsNeeded {
-    fn default() -> Self {
-        Self::Not
-    }
-}
-
 #[derive(Default, Debug)]
 pub struct SmtpState {
     pub idle: bool,
     pub suspended: bool,
     pub doing_jobs: bool,
-    pub perform_jobs_needed: PerformJobsNeeded,
+    pub perform_jobs_needed: i32,
     pub probe_network: bool,
 }
 

src/dc_receive_imf.rs

@@ -1640,11 +1640,7 @@ fn check_verified_properties(
                 info!(context, "{} has verified {}.", contact.get_addr(), to_addr,);
                 let fp = peerstate.gossip_key_fingerprint.clone();
                 if let Some(fp) = fp {
-                    peerstate.set_verified(
-                        DC_PS_GOSSIP_KEY,
-                        &fp,
-                        PeerstateVerifiedStatus::BidirectVerified,
-                    );
+                    peerstate.set_verified(0, &fp, 2);
                     peerstate.save_to_db(&context.sql, false)?;
                     is_verified = true;
                 }

src/e2ee.rs

@@ -75,7 +75,7 @@ impl EncryptHelper {
         &mut self,
         factory: &mut MimeFactory,
         e2ee_guaranteed: bool,
-        min_verified: PeerstateVerifiedStatus,
+        min_verified: libc::c_int,
         do_gossip: bool,
         mut in_out_message: *mut Mailmime,
         imffields_unprotected: *mut mailimf_fields,
@@ -118,10 +118,10 @@ impl EncryptHelper {
                 return Ok(false);
             }
 
-            if let Some(key) = peerstate.peek_key(min_verified) {
+            if let Some(key) = peerstate.peek_key(min_verified as usize) {
                 keyring.add_owned(key.clone());
                 if do_gossip {
-                    if let Some(header) = peerstate.render_gossip_header(min_verified) {
+                    if let Some(header) = peerstate.render_gossip_header(min_verified as usize) {
                         gossip_headers.push(header.to_string());
                     }
                 }

src/events.rs

@@ -243,11 +243,4 @@ pub enum Event {
     /// @return 0
     #[strum(props(id = "2061"))]
     SecurejoinJoinerProgress { contact_id: u32, progress: usize },
-
-    /// This event is sent out to the inviter when a joiner successfully joined a group.
-    /// @param data1 (int) chat_id
-    /// @param data2 (int) contact_id
-    /// @return 0
-    #[strum(props(id = "2062"))]
-    SecurejoinMemberAdded { chat_id: u32, contact_id: u32 },
 }

src/imap.rs

@@ -12,7 +12,7 @@ use async_imap::{
     types::{Fetch, Flag, Mailbox, Name, NameAttribute},
 };
 use async_std::prelude::*;
-use async_std::sync::{Mutex, RwLock};
+use async_std::sync::{Arc, Mutex, RwLock};
 use async_std::task;
 
 use crate::constants::*;
@@ -23,7 +23,7 @@ use crate::events::Event;
 use crate::imap_client::*;
 use crate::job::{job_add, Action};
 use crate::login_param::{CertificateChecks, LoginParam};
-use crate::message::{self, update_server_uid};
+use crate::message::{self, update_msg_move_state, update_server_uid};
 use crate::oauth2::dc_get_oauth2_access_token;
 use crate::param::Params;
 use crate::stock::StockMessage;
@@ -45,10 +45,11 @@ const SELECT_ALL: &str = "1:*";
 
 #[derive(Debug)]
 pub struct Imap {
-    config: RwLock<ImapConfig>,
-    session: Mutex<Option<Session>>,
-    connected: Mutex<bool>,
-    interrupt: Mutex<Option<stop_token::StopSource>>,
+    config: Arc<RwLock<ImapConfig>>,
+
+    session: Arc<Mutex<Option<Session>>>,
+    connected: Arc<Mutex<bool>>,
+    interrupt: Arc<Mutex<Option<stop_token::StopSource>>>,
     skip_next_idle_wait: AtomicBool,
     should_reconnect: AtomicBool,
 }
@@ -117,10 +118,10 @@ impl Default for ImapConfig {
 impl Imap {
     pub fn new() -> Self {
         Imap {
-            session: Mutex::new(None),
-            config: RwLock::new(ImapConfig::default()),
-            interrupt: Mutex::new(None),
-            connected: Mutex::new(false),
+            session: Arc::new(Mutex::new(None)),
+            config: Arc::new(RwLock::new(ImapConfig::default())),
+            interrupt: Arc::new(Mutex::new(None)),
+            connected: Arc::new(Mutex::new(false)),
             skip_next_idle_wait: AtomicBool::new(false),
             should_reconnect: AtomicBool::new(false),
         }
@@ -305,7 +306,7 @@ impl Imap {
                 .unwrap_or_default()
                 < 3
             {
-                self.configure_folders(context, true);
+                self.configure_folders(context, 0x1);
             }
             return Ok(());
         }
@@ -886,44 +887,38 @@ impl Imap {
             let interval = async_std::stream::interval(Duration::from_secs(60));
             let mut interrupt_interval = interrupt.stop_token().stop_stream(interval);
             *self.interrupt.lock().await = Some(interrupt);
-            if self.skip_next_idle_wait.load(Ordering::SeqCst) {
-                // interrupt_idle has happened before we
-                // provided self.interrupt
-                self.skip_next_idle_wait.store(false, Ordering::SeqCst);
-                info!(context, "fake-idle wait was skipped");
-            } else {
-                // loop until we are interrupted or if we fetched something
-                while let Some(_) = interrupt_interval.next().await {
-                    // try to connect with proper login params
-                    // (setup_handle_if_needed might not know about them if we
-                    // never successfully connected)
-                    if let Err(err) = self.connect_configured(context) {
-                        warn!(context, "fake_idle: could not connect: {}", err);
-                        continue;
-                    }
-                    if self.config.read().await.can_idle {
-                        // we only fake-idled because network was gone during IDLE, probably
-                        break;
-                    }
-                    info!(context, "fake_idle is connected");
-                    // we are connected, let's see if fetching messages results
-                    // in anything.  If so, we behave as if IDLE had data but
-                    // will have already fetched the messages so perform_*_fetch
-                    // will not find any new.
 
-                    if let Some(ref watch_folder) = watch_folder {
-                        match self.fetch_from_single_folder(context, watch_folder).await {
-                            Ok(res) => {
-                                info!(context, "fetch_from_single_folder returned {:?}", res);
-                                if res {
-                                    break;
-                                }
-                            }
-                            Err(err) => {
-                                error!(context, "could not fetch from folder: {}", err);
-                                self.trigger_reconnect()
+            // loop until we are interrupted or if we fetched something
+            while let Some(_) = interrupt_interval.next().await {
+                // try to connect with proper login params
+                // (setup_handle_if_needed might not know about them if we
+                // never successfully connected)
+                if let Err(err) = self.connect_configured(context) {
+                    warn!(context, "fake_idle: could not connect: {}", err);
+                    continue;
+                }
+                if self.config.read().await.can_idle {
+                    // we only fake-idled because network was gone during IDLE, probably
+                    break;
+                }
+                info!(context, "fake_idle is connected");
+                // we are connected, let's see if fetching messages results
+                // in anything.  If so, we behave as if IDLE had data but
+                // will have already fetched the messages so perform_*_fetch
+                // will not find any new.
+
+                if let Some(ref watch_folder) = watch_folder {
+                    match self.fetch_from_single_folder(context, watch_folder).await {
+                        Ok(res) => {
+                            info!(context, "fetch_from_single_folder returned {:?}", res);
+                            if res {
+                                break;
                             }
                         }
+                        Err(err) => {
+                            error!(context, "could not fetch from folder: {}", err);
+                            self.trigger_reconnect()
+                        }
                     }
                 }
             }
@@ -1206,7 +1201,7 @@ impl Imap {
         })
     }
 
-    pub fn configure_folders(&self, context: &Context, create_mvbox: bool) {
+    pub fn configure_folders(&self, context: &Context, flags: libc::c_int) {
         task::block_on(async move {
             if !self.is_connected().await {
                 return;
@@ -1237,7 +1232,7 @@ impl Imap {
                     .find(|folder| folder.name() == "DeltaChat" || folder.name() == fallback_folder)
                     .map(|n| n.name().to_string());
 
-                if mvbox_folder.is_none() && create_mvbox {
+                if mvbox_folder.is_none() && 0 != (flags as usize & DC_CREATE_MVBOX) {
                     info!(context, "Creating MVBOX-folder \"DeltaChat\"...",);
 
                     match session.create("DeltaChat").await {
@@ -1413,7 +1408,6 @@ fn precheck_imf(context: &Context, rfc724_mid: &str, server_folder: &str, server
     {
         if old_server_folder.is_empty() && old_server_uid == 0 {
             info!(context, "[move] detected bbc-self {}", rfc724_mid,);
-            context.do_heuristics_moves(server_folder.as_ref(), msg_id);
             job_add(
                 context,
                 Action::MarkseenMsgOnImap,
@@ -1423,6 +1417,7 @@ fn precheck_imf(context: &Context, rfc724_mid: &str, server_folder: &str, server
             );
         } else if old_server_folder != server_folder {
             info!(context, "[move] detected moved message {}", rfc724_mid,);
+            update_msg_move_state(context, &rfc724_mid, MoveState::Stay);
         }
 
         if old_server_folder != server_folder || old_server_uid != server_uid {

src/imex.rs

@@ -367,7 +367,7 @@ pub fn normalize_setup_code(s: &str) -> String {
 }
 
 #[allow(non_snake_case)]
-pub fn JobImexImap(context: &Context, job: &Job) -> Result<()> {
+pub fn job_do_DC_JOB_IMEX_IMAP(context: &Context, job: &Job) -> Result<()> {
     ensure!(context.alloc_ongoing(), "could not allocate ongoing");
     let what: Option<ImexMode> = job.param.get_int(Param::Cmd).and_then(ImexMode::from_i32);
     let param = job.param.get(Param::Arg).unwrap_or_default();

src/job.rs

@@ -1,3 +1,4 @@
+use std::sync::{Arc, RwLock};
 use std::time::Duration;
 
 use deltachat_derive::{FromSql, ToSql};
@@ -8,12 +9,13 @@ use crate::chat;
 use crate::config::Config;
 use crate::configure::*;
 use crate::constants::*;
-use crate::context::{Context, PerformJobsNeeded};
+use crate::context::Context;
 use crate::dc_tools::*;
 use crate::error::Error;
 use crate::events::Event;
 use crate::imap::*;
 use crate::imex::*;
+use crate::job_thread::JobThread;
 use crate::location;
 use crate::login_param::LoginParam;
 use crate::message::MsgId;
@@ -22,9 +24,6 @@ use crate::mimefactory::{vec_contains_lowercase, Loaded, MimeFactory};
 use crate::param::*;
 use crate::sql;
 
-// results in ~3 weeks for the last backoff timespan
-const JOB_RETRIES: u32 = 17;
-
 /// Thread IDs
 #[derive(Debug, Display, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
 #[repr(i32)]
@@ -34,6 +33,14 @@ enum Thread {
     Smtp = 5000,
 }
 
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+enum ThreadExecutor {
+    Inbox,
+    Mvbox,
+    Sentbox,
+    Smtp,
+}
+
 #[derive(Debug, Display, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive)]
 enum TryAgain {
     Dont,
@@ -110,7 +117,7 @@ pub struct Job {
     pub foreign_id: u32,
     pub desired_timestamp: i64,
     pub added_timestamp: i64,
-    pub tries: u32,
+    pub tries: i32,
     pub param: Params,
     try_again: TryAgain,
     pub pending_error: Option<String>,
@@ -140,7 +147,7 @@ impl Job {
     }
 
     #[allow(non_snake_case)]
-    fn SendMsgToSmtp(&mut self, context: &Context) {
+    fn do_DC_JOB_SEND(&mut self, context: &Context) {
         /* connect to SMTP server, if not yet done */
         if !context.smtp.lock().unwrap().is_connected() {
             let loginparam = LoginParam::from_database(context, "configured_");
@@ -212,8 +219,9 @@ impl Job {
     }
 
     #[allow(non_snake_case)]
-    fn MoveMsg(&mut self, context: &Context) {
-        let imap_inbox = &context.inbox_thread.read().unwrap().imap;
+    fn do_DC_JOB_MOVE_MSG(&mut self, context: &Context, thread: ThreadExecutor) {
+        let t = get_mailbox_for_executor(context, thread);
+        let imap_inbox = &t.read().unwrap().imap;
 
         if let Ok(msg) = Message::load_from_db(context, MsgId::new(self.foreign_id)) {
             if context
@@ -222,7 +230,7 @@ impl Job {
                 .unwrap_or_default()
                 < 3
             {
-                imap_inbox.configure_folders(context, true);
+                imap_inbox.configure_folders(context, 0x1i32);
             }
             let dest_folder = context
                 .sql
@@ -257,8 +265,9 @@ impl Job {
     }
 
     #[allow(non_snake_case)]
-    fn DeleteMsgOnImap(&mut self, context: &Context) {
-        let imap_inbox = &context.inbox_thread.read().unwrap().imap;
+    fn do_DC_JOB_DELETE_MSG_ON_IMAP(&mut self, context: &Context, thread: ThreadExecutor) {
+        let t = get_mailbox_for_executor(context, thread);
+        let imap_inbox = &t.read().unwrap().imap;
 
         if let Ok(mut msg) = Message::load_from_db(context, MsgId::new(self.foreign_id)) {
             if !msg.rfc724_mid.is_empty() {
@@ -286,24 +295,26 @@ impl Job {
     }
 
     #[allow(non_snake_case)]
-    fn EmptyServer(&mut self, context: &Context) {
-        let imap_inbox = &context.inbox_thread.read().unwrap().imap;
-        if self.foreign_id & DC_EMPTY_MVBOX > 0 {
+    fn do_DC_JOB_EMPTY_SERVER(&mut self, context: &Context, thread: ThreadExecutor) {
+        let t = get_mailbox_for_executor(context, thread);
+        let imap_inbox = &t.read().unwrap().imap;
+
+        if thread == ThreadExecutor::Mvbox && DC_EMPTY_MVBOX > 0 {
             if let Some(mvbox_folder) = context
                 .sql
                 .get_raw_config(context, "configured_mvbox_folder")
             {
                 imap_inbox.empty_folder(context, &mvbox_folder);
             }
-        }
-        if self.foreign_id & DC_EMPTY_INBOX > 0 {
+        } else if thread == ThreadExecutor::Inbox && DC_EMPTY_INBOX > 0 {
             imap_inbox.empty_folder(context, "INBOX");
         }
     }
 
     #[allow(non_snake_case)]
-    fn MarkseenMsgOnImap(&mut self, context: &Context) {
-        let imap_inbox = &context.inbox_thread.read().unwrap().imap;
+    fn do_DC_JOB_MARKSEEN_MSG_ON_IMAP(&mut self, context: &Context, thread: ThreadExecutor) {
+        let t = get_mailbox_for_executor(context, thread);
+        let imap_inbox = &t.read().unwrap().imap;
 
         if let Ok(msg) = Message::load_from_db(context, MsgId::new(self.foreign_id)) {
             let folder = msg.server_folder.as_ref().unwrap();
@@ -330,14 +341,17 @@ impl Job {
     }
 
     #[allow(non_snake_case)]
-    fn MarkseenMdnOnImap(&mut self, context: &Context) {
+    fn do_DC_JOB_MARKSEEN_MDN_ON_IMAP(&mut self, context: &Context, thread: ThreadExecutor) {
         let folder = self
             .param
             .get(Param::ServerFolder)
             .unwrap_or_default()
             .to_string();
         let uid = self.param.get_int(Param::ServerUid).unwrap_or_default() as u32;
-        let imap_inbox = &context.inbox_thread.read().unwrap().imap;
+
+        let t = get_mailbox_for_executor(context, thread);
+        let imap_inbox = &t.read().unwrap().imap;
+
         if imap_inbox.set_seen(context, &folder, uid) == ImapActionResult::RetryLater {
             self.try_again_later(TryAgain::StandardDelay, None);
             return;
@@ -349,7 +363,7 @@ impl Job {
                 .unwrap_or_default()
                 < 3
             {
-                imap_inbox.configure_folders(context, true);
+                imap_inbox.configure_folders(context, 0x1i32);
             }
             let dest_folder = context
                 .sql
@@ -480,7 +494,7 @@ pub fn perform_smtp_jobs(context: &Context) {
 
         let probe_smtp_network = state.probe_network;
         state.probe_network = false;
-        state.perform_jobs_needed = PerformJobsNeeded::Not;
+        state.perform_jobs_needed = 0;
 
         if state.suspended {
             info!(context, "SMTP-jobs suspended.",);
@@ -491,7 +505,7 @@ pub fn perform_smtp_jobs(context: &Context) {
     };
 
     info!(context, "SMTP-jobs started...",);
-    job_perform(context, Thread::Smtp, probe_smtp_network);
+    job_perform(context, ThreadExecutor::Smtp, probe_smtp_network);
     info!(context, "SMTP-jobs ended.");
 
     {
@@ -508,27 +522,24 @@ pub fn perform_smtp_idle(context: &Context) {
         let &(ref lock, ref cvar) = &*context.smtp_state.clone();
         let mut state = lock.lock().unwrap();
 
-        match state.perform_jobs_needed {
-            PerformJobsNeeded::AtOnce => {
-                info!(
-                    context,
-                    "SMTP-idle will not be started because of waiting jobs.",
-                );
-            }
-            PerformJobsNeeded::Not | PerformJobsNeeded::AvoidDos => {
-                let dur = get_next_wakeup_time(context, Thread::Smtp);
+        if state.perform_jobs_needed == 1 {
+            info!(
+                context,
+                "SMTP-idle will not be started because of waiting jobs.",
+            );
+        } else {
+            let dur = get_next_wakeup_time(context, Thread::Smtp);
 
-                loop {
-                    let res = cvar.wait_timeout(state, dur).unwrap();
-                    state = res.0;
+            loop {
+                let res = cvar.wait_timeout(state, dur).unwrap();
+                state = res.0;
 
-                    if state.idle || res.1.timed_out() {
-                        // We received the notification and the value has been updated, we can leave.
-                        break;
-                    }
+                if state.idle || res.1.timed_out() {
+                    // We received the notification and the value has been updated, we can leave.
+                    break;
                 }
-                state.idle = false;
             }
+            state.idle = false;
         }
     }
 
@@ -707,32 +718,111 @@ pub fn perform_inbox_jobs(context: &Context) {
     *context.probe_imap_network.write().unwrap() = false;
     *context.perform_inbox_jobs_needed.write().unwrap() = false;
 
-    job_perform(context, Thread::Imap, probe_imap_network);
+    job_perform(context, ThreadExecutor::Inbox, probe_imap_network);
     info!(context, "dc_perform_inbox_jobs ended.",);
 }
 
 pub fn perform_mvbox_jobs(context: &Context) {
-    info!(context, "dc_perform_mbox_jobs EMPTY (for now).",);
+    info!(context, "dc_perform_mbox_jobs starting.",);
+
+    let probe_imap_network = *context.probe_imap_network.clone().read().unwrap();
+    *context.probe_imap_network.write().unwrap() = false;
+    *context.perform_inbox_jobs_needed.write().unwrap() = false;
+
+    job_perform(context, ThreadExecutor::Mvbox, probe_imap_network);
+    info!(context, "dc_perform_mbox_jobs ended.",);
 }
 
 pub fn perform_sentbox_jobs(context: &Context) {
-    info!(context, "dc_perform_sentbox_jobs EMPTY (for now).",);
+    info!(context, "dc_perform_sentbox_jobs starting.",);
+
+    let probe_imap_network = *context.probe_imap_network.clone().read().unwrap();
+    *context.probe_imap_network.write().unwrap() = false;
+    *context.perform_inbox_jobs_needed.write().unwrap() = false;
+
+    job_perform(context, ThreadExecutor::Sentbox, probe_imap_network);
+    info!(context, "dc_perform_sentbox_jobs ended.",);
 }
 
-fn job_perform(context: &Context, thread: Thread, probe_network: bool) {
-    let jobs: Vec<Job> = load_jobs(context, thread, probe_network);
+fn job_perform(context: &Context, thread: ThreadExecutor, probe_network: bool) {
+    let folder = get_folder_for_executor(context, thread);
 
-    for mut job in jobs {
+    let fallback = if thread == ThreadExecutor::Inbox || thread == ThreadExecutor::Smtp {
+        // only select non empty ones on the inbox thread or smtp
+        "OR (j.foreign_id IS NULL)) "
+    } else {
+        ") "
+    };
+
+    let query_start = concat!(
+        "SELECT ",
+        "j.id AS id, ",
+        "j.action AS action, ",
+        "j.foreign_id AS foreign_id, ",
+        "j.param AS param, ",
+        "j.added_timestamp AS added_timestamp, ",
+        "j.desired_timestamp AS desired_timestamp, ",
+        "j.tries AS tries ",
+        "FROM jobs j ",
+        "LEFT JOIN msgs m ON j.foreign_id=m.id ",
+    )
+    .to_string();
+    let query = if !probe_network {
+        // processing for first-try and after backoff-timeouts:
+        // process jobs in the order they were added.
+        query_start
+            + "WHERE j.thread=? AND j.desired_timestamp<=? AND "
+            + "((j.foreign_id IS NOT NULL AND m.server_folder=?) "
+            + fallback
+            + "ORDER BY j.action DESC, j.added_timestamp;"
+    } else {
+        // processing after call to dc_maybe_network():
+        // process _all_ pending jobs that failed before
+        // in the order of their backoff-times.
+        query_start
+            + "WHERE j.thread=? AND j.tries>0 AND "
+            + "((j.foreign_id IS NOT NULL AND m.server_folder=?) "
+            + fallback
+            + "ORDER BY j.desired_timestamp, j.action DESC;"
+    };
+
+    let params_no_probe = params![thread as i64, time(), folder];
+    let params_probe = params![thread as i64, folder];
+    let params: &[&dyn rusqlite::ToSql] = if !probe_network {
+        params_no_probe
+    } else {
+        params_probe
+    };
+    let jobs: Result<Vec<Job>, _> = context
+        .sql
+        .query_map(
+            query,
+            params,
+            |row| {
+                let job = Job {
+                    job_id: row.get(0)?,
+                    action: row.get(1)?,
+                    foreign_id: row.get(2)?,
+                    desired_timestamp: row.get(5)?,
+                    added_timestamp: row.get(4)?,
+                    tries: row.get(6)?,
+                    param: row.get::<_, String>(3)?.parse().unwrap_or_default(),
+                    try_again: TryAgain::Dont,
+                    pending_error: None,
+                };
+
+                Ok(job)
+            },
+            |jobs| jobs.collect::<Result<Vec<Job>, _>>().map_err(Into::into),
+        )
+        .map_err(|err| {
+            warn!(context, "query failed: {:?}", err);
+        });
+
+    for mut job in jobs.unwrap_or_default() {
         info!(
             context,
-            "{}-job #{}, action {} started...",
-            if thread == Thread::Imap {
-                "INBOX"
-            } else {
-                "SMTP"
-            },
-            job.job_id,
-            job.action,
+            "{:?}-job #{}, action {} started...", thread, job.job_id, job.action,
         );
 
         // some configuration jobs are "exclusive":
@@ -756,31 +846,34 @@ fn job_perform(context: &Context, thread: Thread, probe_network: bool) {
             suspend_smtp_thread(context, true);
         }
 
-        for _tries in 0..2 {
+        let mut tries = 0;
+        while tries <= 1 {
             // this can be modified by a job using dc_job_try_again_later()
             job.try_again = TryAgain::Dont;
 
             match job.action {
                 Action::Unknown => {
-                    info!(context, "Unknown job id found");
+                    warn!(context, "Unknown job id found");
                 }
-                Action::SendMsgToSmtp => job.SendMsgToSmtp(context),
-                Action::EmptyServer => job.EmptyServer(context),
-                Action::DeleteMsgOnImap => job.DeleteMsgOnImap(context),
-                Action::MarkseenMsgOnImap => job.MarkseenMsgOnImap(context),
-                Action::MarkseenMdnOnImap => job.MarkseenMdnOnImap(context),
-                Action::MoveMsg => job.MoveMsg(context),
-                Action::SendMdn => job.SendMsgToSmtp(context),
-                Action::ConfigureImap => JobConfigureImap(context),
-                Action::ImexImap => match JobImexImap(context, &job) {
+                Action::SendMsgToSmtp => job.do_DC_JOB_SEND(context),
+                Action::EmptyServer => job.do_DC_JOB_EMPTY_SERVER(context, thread),
+                Action::DeleteMsgOnImap => job.do_DC_JOB_DELETE_MSG_ON_IMAP(context, thread),
+                Action::MarkseenMsgOnImap => job.do_DC_JOB_MARKSEEN_MSG_ON_IMAP(context, thread),
+                Action::MarkseenMdnOnImap => job.do_DC_JOB_MARKSEEN_MDN_ON_IMAP(context, thread),
+                Action::MoveMsg => job.do_DC_JOB_MOVE_MSG(context, thread),
+                Action::SendMdn => job.do_DC_JOB_SEND(context),
+                Action::ConfigureImap => dc_job_do_DC_JOB_CONFIGURE_IMAP(context),
+                Action::ImexImap => match job_do_DC_JOB_IMEX_IMAP(context, &job) {
                     Ok(()) => {}
                     Err(err) => {
                         error!(context, "{}", err);
                     }
                 },
-                Action::MaybeSendLocations => location::JobMaybeSendLocations(context, &job),
+                Action::MaybeSendLocations => {
+                    location::job_do_DC_JOB_MAYBE_SEND_LOCATIONS(context, &job)
+                }
                 Action::MaybeSendLocationsEnded => {
-                    location::JobMaybeSendLocationsEnded(context, &mut job)
+                    location::job_do_DC_JOB_MAYBE_SEND_LOC_ENDED(context, &mut job)
                 }
                 Action::Housekeeping => sql::housekeeping(context),
                 Action::SendMdnOld => {}
@@ -789,6 +882,7 @@ fn job_perform(context: &Context, thread: Thread, probe_network: bool) {
             if job.try_again != TryAgain::AtOnce {
                 break;
             }
+            tries += 1
         }
         if Action::ConfigureImap == job.action || Action::ImexImap == job.action {
             context
@@ -807,32 +901,28 @@ fn job_perform(context: &Context, thread: Thread, probe_network: bool) {
             break;
         } else if job.try_again == TryAgain::AtOnce || job.try_again == TryAgain::StandardDelay {
             let tries = job.tries + 1;
-            if tries < JOB_RETRIES {
+            if tries < 17 {
                 job.tries = tries;
                 let time_offset = get_backoff_time_offset(tries);
                 job.desired_timestamp = job.added_timestamp + time_offset;
                 job.update(context);
                 info!(
                     context,
-                    "{}-job #{} not succeeded on try #{}, retry in ADD_TIME+{} (in {} seconds).",
-                    if thread == Thread::Imap {
-                        "INBOX"
-                    } else {
-                        "SMTP"
-                    },
+                    "{:?}-job #{} not succeeded on try #{}, retry in ADD_TIME+{} (in {} seconds).",
+                    thread,
                     job.job_id as u32,
                     tries,
                     time_offset,
                     job.added_timestamp + time_offset - time()
                 );
-                if thread == Thread::Smtp && tries < JOB_RETRIES - 1 {
+                if thread == ThreadExecutor::Smtp && tries < 17 - 1 {
                     context
                         .smtp_state
                         .clone()
                         .0
                         .lock()
                         .unwrap()
-                        .perform_jobs_needed = PerformJobsNeeded::AvoidDos;
+                        .perform_jobs_needed = 2;
                 }
             } else {
                 if job.action == Action::SendMsgToSmtp {
@@ -858,11 +948,13 @@ fn job_perform(context: &Context, thread: Thread, probe_network: bool) {
     }
 }
 
-fn get_backoff_time_offset(tries: u32) -> i64 {
-    let n = 2_i32.pow(tries - 1) * 60;
+#[allow(non_snake_case)]
+fn get_backoff_time_offset(c_tries: libc::c_int) -> i64 {
+    // results in ~3 weeks for the last backoff timespan
+    let N = 2_i32.pow((c_tries - 1) as u32) * 60;
     let mut rng = thread_rng();
-    let r: i32 = rng.gen();
-    let mut seconds = r % (n + 1);
+    let n: i32 = rng.gen();
+    let mut seconds = n % (N + 1);
     if seconds < 1 {
         seconds = 1;
     }
@@ -951,124 +1043,76 @@ pub fn job_add(
     ).ok();
 
     match thread {
-        Thread::Imap => interrupt_inbox_idle(context, false),
+        Thread::Imap => {
+            let folder: String = context
+                .sql
+                .query_get_value(
+                    context,
+                    "SELECT server_folder FROM msgs WHERE id=?",
+                    params![foreign_id],
+                )
+                .unwrap_or_default();
+
+            let mbox_thread = get_matching_mailbox(context, &folder);
+            let res = mbox_thread.try_read();
+            match res {
+                Ok(mbox_thread) => {
+                    mbox_thread.interrupt_idle(context);
+                }
+                Err(err) => {
+                    *context.perform_inbox_jobs_needed.write().unwrap() = true;
+                    warn!(context, "could not interrupt idle: {}", err);
+                }
+            }
+        }
         Thread::Smtp => interrupt_smtp_idle(context),
         Thread::Unknown => {}
     }
 }
 
+fn get_folder_for_executor(context: &Context, thread: ThreadExecutor) -> String {
+    let t = get_mailbox_for_executor(context, thread);
+    let b = t.read().unwrap();
+    b.get_watch_folder(context).unwrap_or_default().to_string()
+}
+
+fn get_mailbox_for_executor(context: &Context, thread: ThreadExecutor) -> Arc<RwLock<JobThread>> {
+    match thread {
+        ThreadExecutor::Inbox => context.inbox_thread.clone(),
+        ThreadExecutor::Mvbox => context.mvbox_thread.clone(),
+        ThreadExecutor::Sentbox => context.sentbox_thread.clone(),
+        ThreadExecutor::Smtp => panic!("do not use for smtp"),
+    }
+}
+
+fn get_matching_mailbox(context: &Context, folder: &String) -> Arc<RwLock<JobThread>> {
+    let mvbox_folder = context
+        .mvbox_thread
+        .read()
+        .unwrap()
+        .get_watch_folder(context);
+    let sentbox_folder = context
+        .sentbox_thread
+        .read()
+        .unwrap()
+        .get_watch_folder(context);
+
+    if mvbox_folder.is_some() && folder == mvbox_folder.as_ref().unwrap() {
+        context.mvbox_thread.clone()
+    } else if sentbox_folder.is_some() && folder == sentbox_folder.as_ref().unwrap() {
+        context.sentbox_thread.clone()
+    } else {
+        context.inbox_thread.clone()
+    }
+}
+
 pub fn interrupt_smtp_idle(context: &Context) {
     info!(context, "Interrupting SMTP-idle...",);
 
     let &(ref lock, ref cvar) = &*context.smtp_state.clone();
     let mut state = lock.lock().unwrap();
 
-    state.perform_jobs_needed = PerformJobsNeeded::AtOnce;
+    state.perform_jobs_needed = 1;
     state.idle = true;
     cvar.notify_one();
-    info!(context, "Interrupting SMTP-idle... ended",);
-}
-
-/// Load jobs from the database.
-///
-/// Load jobs for this "[Thread]", i.e. either load SMTP jobs or load
-/// IMAP jobs.  The `probe_network` parameter decides how to query
-/// jobs, this is tricky and probably wrong currently.  Look at the
-/// SQL queries for details.
-fn load_jobs(context: &Context, thread: Thread, probe_network: bool) -> Vec<Job> {
-    let query = if probe_network {
-        // Processing after call to dc_maybe_network():
-        // process all pending jobs in the order of their priority.
-        // If jobs have the same priority, process the
-        // one that was added earlier.
-        "SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries
-         FROM jobs WHERE thread=? ORDER BY action DESC, added_timestamp;"
-    } else {
-        // Processing for the first try and after backoff-timeouts:
-        // process jobs that have their backoff expired
-        // in the order of their backoff times.
-        "SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries
-         FROM jobs WHERE thread=? AND desired_timestamp<=? ORDER BY desired_timestamp;"
-    };
-
-    let params_no_probe = params![thread as i64, time()];
-    let params_probe = params![thread as i64];
-    let params: &[&dyn rusqlite::ToSql] = if !probe_network {
-        params_no_probe
-    } else {
-        params_probe
-    };
-
-    context
-        .sql
-        .query_map(
-            query,
-            params,
-            |row| {
-                let job = Job {
-                    job_id: row.get(0)?,
-                    action: row.get(1)?,
-                    foreign_id: row.get(2)?,
-                    desired_timestamp: row.get(5)?,
-                    added_timestamp: row.get(4)?,
-                    tries: row.get(6)?,
-                    param: row.get::<_, String>(3)?.parse().unwrap_or_default(),
-                    try_again: TryAgain::Dont,
-                    pending_error: None,
-                };
-
-                Ok(job)
-            },
-            |jobs| {
-                let mut ret: Vec<Job> = Vec::new();
-                for job in jobs {
-                    match job {
-                        Ok(j) => ret.push(j),
-                        Err(e) => warn!(context, "Bad job from the database: {}", e),
-                    }
-                }
-                Ok(ret)
-            },
-        )
-        .unwrap_or_default()
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    use crate::test_utils::*;
-
-    fn insert_job(context: &Context, foreign_id: i64) {
-        let now = time();
-        context
-            .sql
-            .execute(
-                "INSERT INTO jobs
-                   (added_timestamp, thread, action, foreign_id, param, desired_timestamp)
-                 VALUES (?, ?, ?, ?, ?, ?);",
-                params![
-                    now,
-                    Thread::from(Action::MoveMsg),
-                    Action::MoveMsg,
-                    foreign_id,
-                    Params::new().to_string(),
-                    now
-                ],
-            )
-            .unwrap();
-    }
-
-    #[test]
-    fn test_load_jobs() {
-        // We want to ensure that loading jobs skips over jobs which
-        // fails to load from the database instead of failing to load
-        // all jobs.
-        let t = dummy_context();
-        insert_job(&t.ctx, 0);
-        insert_job(&t.ctx, -1); // This can not be loaded into Job struct.
-        insert_job(&t.ctx, 1);
-        let jobs = load_jobs(&t.ctx, Thread::from(Action::MoveMsg), false);
-        assert_eq!(jobs.len(), 2);
-    }
 }

src/job_thread.rs

@@ -70,7 +70,6 @@ impl JobThread {
 
         state.idle = true;
         cvar.notify_one();
-        info!(context, "Interrupting {}-IDLE... finished", self.name);
     }
 
     pub fn fetch(&mut self, context: &Context, use_network: bool) {
@@ -117,7 +116,7 @@ impl JobThread {
         }
     }
 
-    fn get_watch_folder(&self, context: &Context) -> Option<String> {
+    pub(crate) fn get_watch_folder(&self, context: &Context) -> Option<String> {
         match context.sql.get_raw_config(context, self.folder_config_name) {
             Some(name) => Some(name),
             None => {

src/location.rs

@@ -538,7 +538,7 @@ pub fn save(
 }
 
 #[allow(non_snake_case)]
-pub fn JobMaybeSendLocations(context: &Context, _job: &Job) {
+pub fn job_do_DC_JOB_MAYBE_SEND_LOCATIONS(context: &Context, _job: &Job) {
     let now = time();
     let mut continue_streaming = false;
     info!(
@@ -628,7 +628,7 @@ pub fn JobMaybeSendLocations(context: &Context, _job: &Job) {
 }
 
 #[allow(non_snake_case)]
-pub fn JobMaybeSendLocationsEnded(context: &Context, job: &mut Job) {
+pub fn job_do_DC_JOB_MAYBE_SEND_LOC_ENDED(context: &Context, job: &mut Job) {
     // this function is called when location-streaming _might_ have ended for a chat.
     // the function checks, if location-streaming is really ended;
     // if so, a device-message is added if not yet done.

src/log.rs

@@ -5,10 +5,10 @@ macro_rules! info {
     ($ctx:expr,  $msg:expr) => {
         info!($ctx, $msg,)
     };
-    ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
+    ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {
         let formatted = format!($msg, $($args),*);
         emit_event!($ctx, $crate::Event::Info(formatted));
-    }};
+    };
 }
 
 #[macro_export]
@@ -16,10 +16,10 @@ macro_rules! warn {
     ($ctx:expr, $msg:expr) => {
         warn!($ctx, $msg,)
     };
-    ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
+    ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {
         let formatted = format!($msg, $($args),*);
         emit_event!($ctx, $crate::Event::Warning(formatted));
-    }};
+    };
 }
 
 #[macro_export]
@@ -27,10 +27,10 @@ macro_rules! error {
     ($ctx:expr, $msg:expr) => {
         error!($ctx, $msg,)
     };
-    ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
+    ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {
         let formatted = format!($msg, $($args),*);
         emit_event!($ctx, $crate::Event::Error(formatted));
-    }};
+    };
 }
 
 #[macro_export]

src/message.rs

@@ -159,6 +159,7 @@ pub struct Message {
     pub(crate) from_id: u32,
     pub(crate) to_id: u32,
     pub(crate) chat_id: u32,
+    pub(crate) move_state: MoveState,
     pub(crate) type_0: Viewtype,
     pub(crate) state: MessageState,
     pub(crate) hidden: bool,
@@ -199,6 +200,7 @@ impl Message {
                 "    m.mime_in_reply_to AS mime_in_reply_to,",
                 "    m.server_folder AS server_folder,",
                 "    m.server_uid AS server_uid,",
+                "    m.move_state as move_state,",
                 "    m.chat_id AS chat_id,",
                 "    m.from_id AS from_id,",
                 "    m.to_id AS to_id,",
@@ -226,6 +228,7 @@ impl Message {
                 msg.in_reply_to = row.get::<_, Option<String>>("mime_in_reply_to")?;
                 msg.server_folder = row.get::<_, Option<String>>("server_folder")?;
                 msg.server_uid = row.get("server_uid")?;
+                msg.move_state = row.get("move_state")?;
                 msg.chat_id = row.get("chat_id")?;
                 msg.from_id = row.get("from_id")?;
                 msg.to_id = row.get("to_id")?;
@@ -1071,6 +1074,18 @@ pub fn exists(context: &Context, msg_id: MsgId) -> bool {
     }
 }
 
+pub fn update_msg_move_state(context: &Context, rfc724_mid: &str, state: MoveState) -> bool {
+    // we update the move_state for all messages belonging to a given Message-ID
+    // so that the state stay intact when parts are deleted
+    sql::execute(
+        context,
+        &context.sql,
+        "UPDATE msgs SET move_state=? WHERE rfc724_mid=?;",
+        params![state as i32, rfc724_mid],
+    )
+    .is_ok()
+}
+
 pub fn set_msg_failed(context: &Context, msg_id: MsgId, error: Option<impl AsRef<str>>) {
     if let Ok(mut msg) = Message::load_from_db(context, msg_id) {
         if msg.state.can_fail() {

src/mimefactory.rs

@@ -230,7 +230,7 @@ impl<'a> MimeFactory<'a> {
 
         // 1=add Autocrypt-header (needed eg. for handshaking), 2=no Autocrypte-header (used for MDN)
         let mut e2ee_guaranteed = false;
-        let mut min_verified = crate::peerstate::PeerstateVerifiedStatus::Unverified;
+        let mut min_verified: libc::c_int = 0;
         let mut do_gossip = false;
         let mut grpimage = None;
         let force_plaintext: libc::c_int;
@@ -246,7 +246,7 @@ impl<'a> MimeFactory<'a> {
                     wrapmime::new_custom_field(imf_fields, "Chat-Verified", "1");
                     force_plaintext = 0;
                     e2ee_guaranteed = true;
-                    min_verified = crate::peerstate::PeerstateVerifiedStatus::BidirectVerified;
+                    min_verified = 2
                 } else {
                     force_plaintext = self
                         .msg

src/peerstate.rs

@@ -12,17 +12,6 @@ use crate::error::*;
 use crate::key::*;
 use crate::sql::{self, Sql};
 
-pub const DC_PS_GOSSIP_KEY: u32 = 0;
-pub const DC_PS_PUBLIC_KEY: u32 = 1;
-
-#[derive(Debug, PartialEq, Eq, Clone, Copy, FromPrimitive)]
-#[repr(u8)]
-pub enum PeerstateVerifiedStatus {
-    Unverified = 0,
-    //Verified = 1, // not used
-    BidirectVerified = 2,
-}
-
 /// Peerstate represents the state of an Autocrypt peer.
 pub struct Peerstate<'a> {
     pub context: &'a Context,
@@ -322,7 +311,7 @@ impl<'a> Peerstate<'a> {
         };
     }
 
-    pub fn render_gossip_header(&self, min_verified: PeerstateVerifiedStatus) -> Option<String> {
+    pub fn render_gossip_header(&self, min_verified: usize) -> Option<String> {
         if let Some(ref addr) = self.addr {
             if let Some(key) = self.peek_key(min_verified) {
                 // TODO: avoid cloning
@@ -338,12 +327,12 @@ impl<'a> Peerstate<'a> {
         None
     }
 
-    pub fn peek_key(&self, min_verified: PeerstateVerifiedStatus) -> Option<&Key> {
+    pub fn peek_key(&self, min_verified: usize) -> Option<&Key> {
         if self.public_key.is_none() && self.gossip_key.is_none() && self.verified_key.is_none() {
             return None;
         }
 
-        if min_verified != PeerstateVerifiedStatus::Unverified {
+        if 0 != min_verified {
             return self.verified_key.as_ref();
         }
         if self.public_key.is_some() {
@@ -353,17 +342,10 @@ impl<'a> Peerstate<'a> {
         self.gossip_key.as_ref()
     }
 
-    pub fn set_verified(
-        &mut self,
-        which_key: u32,
-        fingerprint: &str,
-        verified: PeerstateVerifiedStatus,
-    ) -> bool {
+    pub fn set_verified(&mut self, which_key: usize, fingerprint: &str, verified: usize) -> bool {
         let mut success = false;
-        if !(which_key != DC_PS_GOSSIP_KEY && which_key != DC_PS_PUBLIC_KEY
-            || verified != PeerstateVerifiedStatus::BidirectVerified)
-        {
-            if which_key == DC_PS_PUBLIC_KEY
+        if !(which_key != 0 && which_key != 1 || verified != 2) {
+            if which_key == 1
                 && self.public_key_fingerprint.is_some()
                 && self.public_key_fingerprint.as_ref().unwrap() == fingerprint
             {
@@ -372,7 +354,7 @@ impl<'a> Peerstate<'a> {
                 self.verified_key_fingerprint = self.public_key_fingerprint.clone();
                 success = true;
             }
-            if which_key == DC_PS_GOSSIP_KEY
+            if which_key == 0
                 && self.gossip_key_fingerprint.is_some()
                 && self.gossip_key_fingerprint.as_ref().unwrap() == fingerprint
             {

src/securejoin.rs

@@ -611,14 +611,6 @@ pub fn handle_securejoin_handshake(
                 }
                 inviter_progress!(context, contact_id, 800);
                 inviter_progress!(context, contact_id, 1000);
-                let field_grpid = mimeparser
-                    .lookup_optional_field("Secure-Join-Group")
-                    .unwrap_or_default();
-                let (group_chat_id, _, _) = chat::get_chat_id_by_grpid(context, &field_grpid);
-                context.call_cb(Event::SecurejoinMemberAdded {
-                    chat_id: group_chat_id,
-                    contact_id: contact_id,
-                });
             } else {
                 warn!(context, "vg-member-added-received invalid.",);
                 return ret;
@@ -672,11 +664,7 @@ fn mark_peer_as_verified(context: &Context, fingerprint: impl AsRef<str>) -> Res
     if let Some(ref mut peerstate) =
         Peerstate::from_fingerprint(context, &context.sql, fingerprint.as_ref())
     {
-        if peerstate.set_verified(
-            DC_PS_PUBLIC_KEY,
-            fingerprint.as_ref(),
-            PeerstateVerifiedStatus::BidirectVerified,
-        ) {
+        if peerstate.set_verified(1, fingerprint.as_ref(), 2) {
             peerstate.prefer_encrypt = EncryptPreference::Mutual;
             peerstate.to_save = Some(ToSave::All);
             peerstate

src/sql.rs

@@ -729,7 +729,6 @@ fn open(
         }
         if dbversion < 48 {
             info!(context, "[migration] v48");
-            // NOTE: move_state is not used anymore
             sql.execute(
                 "ALTER TABLE msgs ADD COLUMN move_state INTEGER DEFAULT 1;",
                 params![],

tests/stress.rs

@@ -128,7 +128,7 @@ fn test_encryption_decryption() {
     public_keyring.add_ref(&public_key);
 
     let mut public_keyring2 = Keyring::default();
-    public_keyring2.add_owned(public_key2);
+    public_keyring2.add_owned(public_key2.clone());
 
     let mut valid_signatures: HashSet<String> = Default::default();
 
@@ -220,7 +220,7 @@ fn create_test_context() -> TestContext {
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     let ctx = Context::new(Box::new(cb), "FakeOs".into(), dbfile).unwrap();
-    TestContext { ctx, dir }
+    TestContext { ctx: ctx, dir: dir }
 }
 
 #[test]