Change the order of job processing

.gitignore

@@ -23,5 +23,3 @@ python/liveconfig*
 # ignore doxgen generated files
 deltachat-ffi/html
 deltachat-ffi/xml
-
-.rsynclist

python/tests/test_account.py

@@ -794,7 +794,6 @@ class TestOnlineAccount:
         chat2 = ac2.qr_join_chat(qr)
         assert chat2.id >= 10
         wait_securejoin_inviter_progress(ac1, 1000)
-        ac1._evlogger.get_matching("DC_EVENT_SECUREJOIN_MEMBER_ADDED")
 
         lp.sec("ac2: read member added message")
         msg = ac2.wait_next_incoming_message()

src/aheader.rs

@@ -44,8 +44,8 @@ impl str::FromStr for EncryptPreference {
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         match s {
             "mutual" => Ok(EncryptPreference::Mutual),
-            "nopreference" => Ok(EncryptPreference::NoPreference),
-            _ => Err(()),
+            "reset" => Ok(EncryptPreference::Reset),
+            _ => Ok(EncryptPreference::NoPreference),
         }
     }
 }

src/chat.rs

@@ -637,7 +637,7 @@ pub fn create_or_lookup_by_contact_id(
         context,
         &context.sql,
         format!(
-            "INSERT INTO chats (type, name, param, blocked, grpid, created_timestamp) VALUES({}, '{}', '{}', {}, '{}', {})",
+            "INSERT INTO chats (type, name, param, blocked, grpid) VALUES({}, '{}', '{}', {}, '{}')",
             100,
             chat_name,
             match contact_id {
@@ -650,7 +650,6 @@ pub fn create_or_lookup_by_contact_id(
             },
             create_blocked as u8,
             contact.get_addr(),
-            time(),
         ),
         params![],
     )?;
@@ -1389,7 +1388,7 @@ pub fn create_group_chat(
     sql::execute(
         context,
         &context.sql,
-        "INSERT INTO chats (type, name, grpid, param, created_timestamp) VALUES(?, ?, ?, \'U=1\', ?);",
+        "INSERT INTO chats (type, name, grpid, param) VALUES(?, ?, ?, \'U=1\');",
         params![
             if verified != VerifiedStatus::Unverified {
                 Chattype::VerifiedGroup
@@ -1397,8 +1396,7 @@ pub fn create_group_chat(
                 Chattype::Group
             },
             chat_name.as_ref(),
-            grpid,
-            time(),
+            grpid
         ],
     )?;
 
@@ -1488,7 +1486,7 @@ pub(crate) fn add_contact_to_chat_ex(
     let self_addr = context
         .get_config(Config::ConfiguredAddr)
         .unwrap_or_default();
-    if addr_cmp(contact.get_addr(), &self_addr) {
+    if contact.get_addr() == &self_addr {
         // ourself is added using DC_CONTACT_ID_SELF, do not add this address explicitly.
         // if SELF is not in the group, members cannot be added at all.
         warn!(

src/chatlist.rs

@@ -6,7 +6,7 @@ use crate::contact::*;
 use crate::context::*;
 use crate::error::Result;
 use crate::lot::Lot;
-use crate::message::{Message, MessageState, MsgId};
+use crate::message::{Message, MsgId};
 use crate::stock::StockMessage;
 
 /// An object representing a single chatlist in memory.
@@ -119,42 +119,46 @@ impl Chatlist {
         let mut ids = if let Some(query_contact_id) = query_contact_id {
             // show chats shared with a given contact
             context.sql.query_map(
-                "SELECT c.id, m.id
-                 FROM chats c
-                 LEFT JOIN msgs m
-                        ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
-                                 FROM msgs
-                                WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?))
-                 WHERE c.id>9
-                   AND c.blocked=0
-                   AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)
-                 GROUP BY c.id
-                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                params![MessageState::OutDraft, query_contact_id as i32],
+                concat!(
+                    "SELECT c.id, m.id",
+                    " FROM chats c",
+                    " LEFT JOIN msgs m",
+                    "        ON c.id=m.chat_id",
+                    "       AND m.timestamp=(",
+                    "               SELECT MAX(timestamp)",
+                    "                 FROM msgs",
+                    "                WHERE chat_id=c.id",
+                    "                  AND hidden=0)",
+                    " WHERE c.id>9",
+                    "   AND c.blocked=0",
+                    "   AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)",
+                    " GROUP BY c.id",
+                    " ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;"
+                ),
+                params![query_contact_id as i32],
                 process_row,
                 process_rows,
             )?
         } else if 0 != listflags & DC_GCL_ARCHIVED_ONLY {
             // show archived chats
             context.sql.query_map(
-                "SELECT c.id, m.id
-                 FROM chats c
-                 LEFT JOIN msgs m
-                        ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
-                                 FROM msgs
-                                WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?))
-                 WHERE c.id>9
-                   AND c.blocked=0
-                   AND c.archived=1
-                 GROUP BY c.id
-                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                params![MessageState::OutDraft],
+                concat!(
+                    "SELECT c.id, m.id",
+                    " FROM chats c",
+                    " LEFT JOIN msgs m",
+                    "        ON c.id=m.chat_id",
+                    "       AND m.timestamp=(",
+                    "               SELECT MAX(timestamp)",
+                    "                 FROM msgs",
+                    "                WHERE chat_id=c.id",
+                    "                  AND hidden=0)",
+                    " WHERE c.id>9",
+                    "   AND c.blocked=0",
+                    "   AND c.archived=1",
+                    " GROUP BY c.id",
+                    " ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;"
+                ),
+                params![],
                 process_row,
                 process_rows,
             )?
@@ -164,42 +168,46 @@ impl Chatlist {
 
             let str_like_cmd = format!("%{}%", query);
             context.sql.query_map(
-                "SELECT c.id, m.id
-                 FROM chats c
-                 LEFT JOIN msgs m
-                        ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
-                                 FROM msgs
-                                WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?))
-                 WHERE c.id>9
-                   AND c.blocked=0
-                   AND c.name LIKE ?
-                 GROUP BY c.id
-                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                params![MessageState::OutDraft, str_like_cmd],
+                concat!(
+                    "SELECT c.id, m.id",
+                    " FROM chats c",
+                    " LEFT JOIN msgs m",
+                    "        ON c.id=m.chat_id",
+                    "       AND m.timestamp=(",
+                    "               SELECT MAX(timestamp)",
+                    "                 FROM msgs",
+                    "                WHERE chat_id=c.id",
+                    "                  AND hidden=0)",
+                    " WHERE c.id>9",
+                    "   AND c.blocked=0",
+                    "   AND c.name LIKE ?",
+                    " GROUP BY c.id",
+                    " ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;"
+                ),
+                params![str_like_cmd],
                 process_row,
                 process_rows,
             )?
         } else {
             //  show normal chatlist
             let mut ids = context.sql.query_map(
-                "SELECT c.id, m.id
-                 FROM chats c
-                 LEFT JOIN msgs m
-                        ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
-                                 FROM msgs
-                                WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?))
-                 WHERE c.id>9
-                   AND c.blocked=0
-                   AND c.archived=0
-                 GROUP BY c.id
-                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                params![MessageState::OutDraft],
+                concat!(
+                    "SELECT c.id, m.id",
+                    " FROM chats c",
+                    " LEFT JOIN msgs m",
+                    "        ON c.id=m.chat_id",
+                    "       AND m.timestamp=(",
+                    "               SELECT MAX(timestamp)",
+                    "                 FROM msgs",
+                    "                WHERE chat_id=c.id",
+                    "                  AND hidden=0)",
+                    " WHERE c.id>9",
+                    "   AND c.blocked=0",
+                    "   AND c.archived=0",
+                    " GROUP BY c.id",
+                    " ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;"
+                ),
+                params![],
                 process_row,
                 process_rows,
             )?;
@@ -289,7 +297,7 @@ impl Chatlist {
         let lastmsg_id = self.ids[index].1;
         let mut lastcontact = None;
 
-        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id) {
+        let mut lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id) {
             if lastmsg.from_id != DC_CONTACT_ID_SELF
                 && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup)
             {
@@ -301,6 +309,16 @@ impl Chatlist {
             None
         };
 
+        if let Ok(draft) = get_draft(context, chat.id) {
+            if draft.is_some()
+                && (lastmsg.is_none()
+                    || draft.as_ref().unwrap().timestamp_sort
+                        > lastmsg.as_ref().unwrap().timestamp_sort)
+            {
+                lastmsg = draft;
+            }
+        }
+
         if chat.id == DC_CHAT_ID_ARCHIVED_LINK {
             ret.text2 = None;
         } else if lastmsg.is_none() || lastmsg.as_ref().unwrap().from_id == DC_CONTACT_ID_UNDEFINED
@@ -344,44 +362,3 @@ fn get_last_deaddrop_fresh_msg(context: &Context) -> Option<MsgId> {
         params![],
     )
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    use crate::chat;
-    use crate::test_utils::*;
-
-    #[test]
-    fn test_try_load() {
-        let t = dummy_context();
-        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat").unwrap();
-        let chat_id2 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "b chat").unwrap();
-        let chat_id3 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "c chat").unwrap();
-
-        // check that the chatlist starts with the most recent message
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
-        assert_eq!(chats.len(), 3);
-        assert_eq!(chats.get_chat_id(0), chat_id3);
-        assert_eq!(chats.get_chat_id(1), chat_id2);
-        assert_eq!(chats.get_chat_id(2), chat_id1);
-
-        // drafts are sorted to the top
-        let mut msg = Message::new(Viewtype::Text);
-        msg.set_text(Some("hello".to_string()));
-        set_draft(&t.ctx, chat_id2, Some(&mut msg));
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
-        assert_eq!(chats.get_chat_id(0), chat_id2);
-
-        // check chatlist query and archive functionality
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("b"), None).unwrap();
-        assert_eq!(chats.len(), 1);
-
-        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None).unwrap();
-        assert_eq!(chats.len(), 0);
-
-        chat::archive(&t.ctx, chat_id1, true).ok();
-        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None).unwrap();
-        assert_eq!(chats.len(), 1);
-    }
-}

src/configure/mod.rs

@@ -355,19 +355,13 @@ pub fn JobConfigureImap(context: &Context) {
                 progress!(context, 900);
                 let create_mvbox = context.get_config_bool(Config::MvboxWatch)
                     || context.get_config_bool(Config::MvboxMove);
-                let imap = &context.inbox_thread.read().unwrap().imap;
-                if let Err(err) = imap.ensure_configured_folders(context, create_mvbox) {
-                    error!(context, "configuring folders failed: {:?}", err);
-                    false
-                } else {
-                    let res = imap.select_with_uidvalidity(context, "INBOX");
-                    if let Err(err) = res {
-                        error!(context, "could not read INBOX status: {:?}", err);
-                        false
-                    } else {
-                        true
-                    }
-                }
+                context
+                    .inbox_thread
+                    .read()
+                    .unwrap()
+                    .imap
+                    .configure_folders(context, create_mvbox);
+                true
             }
             17 => {
                 progress!(context, 910);
@@ -389,10 +383,11 @@ pub fn JobConfigureImap(context: &Context) {
                 // (~30 seconds on a Moto G4 play) and might looks as if message sending is always that slow.
                 e2ee::ensure_secret_key_exists(context);
                 success = true;
-                info!(context, "key generation completed");
+                info!(context, "Configure completed.");
                 progress!(context, 940);
                 break; // We are done here
             }
+
             _ => {
                 error!(context, "Internal error: step counter out of bound",);
                 break;
@@ -415,6 +410,24 @@ pub fn JobConfigureImap(context: &Context) {
         context.smtp.clone().lock().unwrap().disconnect();
     }
 
+    /*
+    if !success {
+        // disconnect if configure did not succeed
+        if imap_connected_here {
+            // context.inbox.read().unwrap().disconnect(context);
+        }
+        if smtp_connected_here {
+            // context.smtp.clone().lock().unwrap().disconnect();
+        }
+    } else {
+        assert!(imap_connected_here && smtp_connected_here);
+        info!(
+            context,
+            0, "Keeping IMAP/SMTP connections open after successful configuration"
+        );
+    }
+    */
+
     // remember the entered parameters on success
     // and restore to last-entered on failure.
     // this way, the parameters visible to the ui are always in-sync with the current configuration.

src/contact.rs

@@ -272,7 +272,7 @@ impl Contact {
             .get_config(Config::ConfiguredAddr)
             .unwrap_or_default();
 
-        if addr_cmp(addr_normalized, addr_self) {
+        if addr_normalized == addr_self {
             return DC_CONTACT_ID_SELF;
         }
 
@@ -309,7 +309,7 @@ impl Contact {
             .get_config(Config::ConfiguredAddr)
             .unwrap_or_default();
 
-        if addr_cmp(addr, addr_self) {
+        if addr == addr_self {
             return Ok((DC_CONTACT_ID_SELF, sth_modified));
         }
 
@@ -1026,16 +1026,17 @@ fn cat_fingerprint(
 }
 
 pub fn addr_cmp(addr1: impl AsRef<str>, addr2: impl AsRef<str>) -> bool {
-    let norm1 = addr_normalize(addr1.as_ref()).to_lowercase();
-    let norm2 = addr_normalize(addr2.as_ref()).to_lowercase();
+    let norm1 = addr_normalize(addr1.as_ref());
+    let norm2 = addr_normalize(addr2.as_ref());
 
     norm1 == norm2
 }
 
 pub fn addr_equals_self(context: &Context, addr: impl AsRef<str>) -> bool {
     if !addr.as_ref().is_empty() {
+        let normalized_addr = addr_normalize(addr.as_ref());
         if let Some(self_addr) = context.get_config(Config::ConfiguredAddr) {
-            return addr_cmp(addr, self_addr);
+            return normalized_addr == self_addr;
         }
     }
     false
@@ -1087,10 +1088,6 @@ mod tests {
     fn test_normalize_addr() {
         assert_eq!(addr_normalize("mailto:john@doe.com"), "john@doe.com");
         assert_eq!(addr_normalize("  hello@world.com   "), "hello@world.com");
-
-        // normalisation preserves case to allow user-defined spelling.
-        // however, case is ignored on addr_cmp()
-        assert_ne!(addr_normalize("John@Doe.com"), "john@doe.com");
     }
 
     #[test]
@@ -1212,11 +1209,4 @@ mod tests {
         assert_eq!(contact.get_addr(), ""); // we're not configured
         assert!(!contact.is_blocked());
     }
-
-    #[test]
-    fn test_addr_cmp() {
-        assert!(addr_cmp("AA@AA.ORG", "aa@aa.ORG"));
-        assert!(addr_cmp(" aa@aa.ORG ", "AA@AA.ORG"));
-        assert!(addr_cmp(" mailto:AA@AA.ORG", "Aa@Aa.orG"));
-    }
 }

src/dc_receive_imf.rs

@@ -392,8 +392,6 @@ unsafe fn add_parts(
             MessageState::InFresh
         };
         *to_id = DC_CONTACT_ID_SELF;
-        let mut needs_stop_ongoing_process = false;
-
         // handshake messages must be processed _before_ chats are created
         // (eg. contacs may be marked as verified)
         if mime_parser.lookup_field("Secure-Join").is_some() {
@@ -401,24 +399,11 @@ unsafe fn add_parts(
             msgrmsg = 1;
             *chat_id = 0;
             allow_creation = 1;
-            match handle_securejoin_handshake(context, mime_parser, *from_id) {
-                Ok(ret) => {
-                    if ret.hide_this_msg {
-                        *hidden = 1;
-                        *needs_delete_job = ret.delete_this_msg;
-                        state = MessageState::InSeen;
-                    }
-                    if let Some(status) = ret.bob_securejoin_success {
-                        context.bob.write().unwrap().status = status as i32;
-                    }
-                    needs_stop_ongoing_process = ret.stop_ongoing_process;
-                }
-                Err(err) => {
-                    warn!(
-                        context,
-                        "Unexpected messaged passed to Secure-Join handshake protocol: {}", err
-                    );
-                }
+            let handshake = handle_securejoin_handshake(context, mime_parser, *from_id);
+            if 0 != handshake & DC_HANDSHAKE_STOP_NORMAL_PROCESSING {
+                *hidden = 1;
+                *needs_delete_job = 0 != handshake & DC_HANDSHAKE_ADD_DELETE_JOB;
+                state = MessageState::InSeen;
             }
         }
 
@@ -516,13 +501,6 @@ unsafe fn add_parts(
         {
             state = MessageState::InNoticed;
         }
-
-        if needs_stop_ongoing_process {
-            // The Secure-Join protocol finished and the group
-            // creation handling is done.  Stopping the ongoing
-            // process will let dc_join_securejoin() return.
-            context.stop_ongoing();
-        }
     } else {
         // Outgoing
 
@@ -1461,7 +1439,7 @@ fn create_group_record(
     if sql::execute(
         context,
         &context.sql,
-        "INSERT INTO chats (type, name, grpid, blocked, created_timestamp) VALUES(?, ?, ?, ?, ?);",
+        "INSERT INTO chats (type, name, grpid, blocked) VALUES(?, ?, ?, ?);",
         params![
             if VerifiedStatus::Unverified != create_verified {
                 Chattype::VerifiedGroup
@@ -1471,7 +1449,6 @@ fn create_group_record(
             grpname.as_ref(),
             grpid.as_ref(),
             create_blocked,
-            time(),
         ],
     )
     .is_err()

src/dc_tools.rs

@@ -523,13 +523,9 @@ pub(crate) fn dc_create_folder(context: &Context, path: impl AsRef<std::path::Pa
 }
 
 /// Write a the given content to provied file path.
-pub(crate) fn dc_write_file(
-    context: &Context,
-    path: impl AsRef<Path>,
-    buf: &[u8],
-) -> Result<(), std::io::Error> {
+pub(crate) fn dc_write_file(context: &Context, path: impl AsRef<Path>, buf: &[u8]) -> bool {
     let path_abs = dc_get_abs_path(context, &path);
-    fs::write(&path_abs, buf).map_err(|err| {
+    if let Err(err) = fs::write(&path_abs, buf) {
         warn!(
             context,
             "Cannot write {} bytes to \"{}\": {}",
@@ -537,8 +533,10 @@ pub(crate) fn dc_write_file(
             path.as_ref().display(),
             err
         );
-        err
-    })
+        false
+    } else {
+        true
+    }
 }
 
 pub fn dc_read_file<P: AsRef<std::path::Path>>(
@@ -1325,7 +1323,7 @@ mod tests {
             dc_delete_file(context, "$BLOBDIR/foobar.dadada");
             dc_delete_file(context, "$BLOBDIR/foobar-folder");
         }
-        assert!(dc_write_file(context, "$BLOBDIR/foobar", b"content").is_ok());
+        assert!(dc_write_file(context, "$BLOBDIR/foobar", b"content"));
         assert!(dc_file_exist(context, "$BLOBDIR/foobar",));
         assert!(!dc_file_exist(context, "$BLOBDIR/foobarx"));
         assert_eq!(dc_get_filebytes(context, "$BLOBDIR/foobar"), 7);
@@ -1357,7 +1355,7 @@ mod tests {
         assert!(!dc_delete_file(context, "$BLOBDIR/foobar-folder"));
 
         let fn0 = "$BLOBDIR/data.data";
-        assert!(dc_write_file(context, &fn0, b"content").is_ok());
+        assert!(dc_write_file(context, &fn0, b"content"));
 
         assert!(dc_delete_file(context, &fn0));
         assert!(!dc_file_exist(context, &fn0));

src/imap.rs

@@ -18,7 +18,7 @@ use async_std::task;
 use crate::constants::*;
 use crate::context::Context;
 use crate::dc_receive_imf::dc_receive_imf;
-use crate::error::Error;
+use crate::error::{Error, Result};
 use crate::events::Event;
 use crate::imap_client::*;
 use crate::job::{job_add, Action};
@@ -40,7 +40,6 @@ pub enum ImapActionResult {
 }
 
 const PREFETCH_FLAGS: &str = "(UID ENVELOPE)";
-const JUST_UID: &str = "(UID)";
 const BODY_FLAGS: &str = "(FLAGS BODY.PEEK[])";
 const SELECT_ALL: &str = "1:*";
 
@@ -115,24 +114,6 @@ impl Default for ImapConfig {
     }
 }
 
-#[derive(Debug, Fail)]
-enum SelectError {
-    #[fail(display = "Could not obtain imap-session object.")]
-    NoSession,
-
-    #[fail(display = "Connection Lost or no connection established")]
-    ConnectionLost,
-
-    #[fail(display = "imap-close (to expunge messages) failed: {}", _0)]
-    CloseExpungeFailed(#[cause] async_imap::error::Error),
-
-    #[fail(display = "Folder name invalid: {:?}", _0)]
-    BadFolderName(String),
-
-    #[fail(display = "async-imap select error: {:?}", _0)]
-    Other(String),
-}
-
 impl Imap {
     pub fn new() -> Self {
         Imap {
@@ -157,7 +138,7 @@ impl Imap {
         self.should_reconnect.store(true, Ordering::Relaxed)
     }
 
-    fn setup_handle_if_needed(&self, context: &Context) -> Result<(), Error> {
+    fn setup_handle_if_needed(&self, context: &Context) -> Result<()> {
         task::block_on(async move {
             if self.config.read().await.imap_server.is_empty() {
                 return Err(Error::ImapInTeardown);
@@ -304,7 +285,7 @@ impl Imap {
     }
 
     /// Connects to imap account using already-configured parameters.
-    pub fn connect_configured(&self, context: &Context) -> Result<(), Error> {
+    pub fn connect_configured(&self, context: &Context) -> Result<()> {
         if async_std::task::block_on(async move {
             self.is_connected().await && !self.should_reconnect()
         }) {
@@ -318,12 +299,19 @@ impl Imap {
         // the trailing underscore is correct
 
         if self.connect(context, &param) {
-            self.ensure_configured_folders(context, true)
-        } else {
-            Err(Error::ImapConnectionFailed(
-                format!("{}", param).to_string(),
-            ))
+            if context
+                .sql
+                .get_raw_config_int(context, "folders_configured")
+                .unwrap_or_default()
+                < 3
+            {
+                self.configure_folders(context, true);
+            }
+            return Ok(());
         }
+        return Err(Error::ImapConnectionFailed(
+            format!("{}", param).to_string(),
+        ));
     }
 
     /// tries connecting to imap account using the specific login
@@ -408,7 +396,7 @@ impl Imap {
         });
     }
 
-    pub fn fetch(&self, context: &Context, watch_folder: &str) -> Result<(), Error> {
+    pub fn fetch(&self, context: &Context, watch_folder: &str) -> Result<()> {
         task::block_on(async move {
             if !context.sql.is_open() {
                 // probably shutdown
@@ -418,24 +406,24 @@ impl Imap {
                 .fetch_from_single_folder(context, &watch_folder)
                 .await?
             {
-                // We fetch until no more new messages are there.
+                // During the fetch commands new messages may arrive.  So we fetch until we do not
+                // get any more. If IDLE is called directly after, there is only a small chance that
+                // messages are missed and delayed until the next IDLE call
             }
             Ok(())
         })
     }
 
-    /// select a folder, possibly update uid_validity and, if needed,
-    /// expunge the folder to remove delete-marked messages.
     async fn select_folder<S: AsRef<str>>(
         &self,
         context: &Context,
         folder: Option<S>,
-    ) -> Result<(), SelectError> {
+    ) -> ImapActionResult {
         if self.session.lock().await.is_none() {
             let mut cfg = self.config.write().await;
             cfg.selected_folder = None;
             cfg.selected_folder_needs_expunge = false;
-            return Err(SelectError::NoSession);
+            return ImapActionResult::Failed;
         }
 
         // if there is a new folder and the new folder is equal to the selected one, there's nothing to do.
@@ -443,7 +431,7 @@ impl Imap {
         if let Some(ref folder) = folder {
             if let Some(ref selected_folder) = self.config.read().await.selected_folder {
                 if folder.as_ref() == selected_folder {
-                    return Ok(());
+                    return ImapActionResult::AlreadyDone;
                 }
             }
         }
@@ -462,11 +450,12 @@ impl Imap {
                             info!(context, "close/expunge succeeded");
                         }
                         Err(err) => {
-                            return Err(SelectError::CloseExpungeFailed(err));
+                            warn!(context, "failed to close session: {:?}", err);
+                            return ImapActionResult::Failed;
                         }
                     }
                 } else {
-                    return Err(SelectError::NoSession);
+                    return ImapActionResult::Failed;
                 }
             }
             self.config.write().await.selected_folder_needs_expunge = false;
@@ -475,39 +464,31 @@ impl Imap {
         // select new folder
         if let Some(ref folder) = folder {
             if let Some(ref mut session) = &mut *self.session.lock().await {
-                let res = session.select(folder).await;
-
-                // https://tools.ietf.org/html/rfc3501#section-6.3.1
-                // says that if the server reports select failure we are in
-                // authenticated (not-select) state.
-
-                match res {
+                match session.select(folder).await {
                     Ok(mailbox) => {
                         let mut config = self.config.write().await;
                         config.selected_folder = Some(folder.as_ref().to_string());
                         config.selected_mailbox = Some(mailbox);
-                        Ok(())
-                    }
-                    Err(async_imap::error::Error::ConnectionLost) => {
-                        self.trigger_reconnect();
-                        self.config.write().await.selected_folder = None;
-                        Err(SelectError::ConnectionLost)
-                    }
-                    Err(async_imap::error::Error::Validate(_)) => {
-                        Err(SelectError::BadFolderName(folder.as_ref().to_string()))
                     }
                     Err(err) => {
+                        warn!(
+                            context,
+                            "Cannot select folder: {}; {:?}.",
+                            folder.as_ref(),
+                            err
+                        );
+
                         self.config.write().await.selected_folder = None;
                         self.trigger_reconnect();
-                        Err(SelectError::Other(err.to_string()))
+                        return ImapActionResult::Failed;
                     }
                 }
             } else {
-                Err(SelectError::NoSession)
+                unreachable!();
             }
-        } else {
-            Ok(())
         }
+
+        ImapActionResult::Success
     }
 
     fn get_config_last_seen_uid<S: AsRef<str>>(&self, context: &Context, folder: S) -> (u32, u32) {
@@ -532,95 +513,89 @@ impl Imap {
         }
     }
 
-    /// return Result with (uid_validity, last_seen_uid) tuple.
-    pub(crate) fn select_with_uidvalidity(
+    async fn fetch_from_single_folder<S: AsRef<str>>(
         &self,
         context: &Context,
-        folder: &str,
-    ) -> Result<(u32, u32), Error> {
-        task::block_on(async move {
-            if let Err(err) = self.select_folder(context, Some(folder)).await {
-                bail!("could not select folder {:?}: {:?}", folder, err);
+        folder: S,
+    ) -> Result<bool> {
+        match self.select_folder(context, Some(&folder)).await {
+            ImapActionResult::Failed | ImapActionResult::RetryLater => {
+                bail!("Cannot select folder {:?} for fetching.", folder.as_ref());
             }
+            ImapActionResult::Success | ImapActionResult::AlreadyDone => {}
+        }
 
-            // compare last seen UIDVALIDITY against the current one
-            let (uid_validity, last_seen_uid) = self.get_config_last_seen_uid(context, &folder);
+        // compare last seen UIDVALIDITY against the current one
+        let (mut uid_validity, mut last_seen_uid) = self.get_config_last_seen_uid(context, &folder);
 
-            let config = self.config.read().await;
-            let mailbox = config.selected_mailbox.as_ref().expect("just selected");
+        let config = self.config.read().await;
+        let mailbox = config.selected_mailbox.as_ref().expect("just selected");
 
-            let new_uid_validity = match mailbox.uid_validity {
-                Some(v) => v,
-                None => bail!("Cannot get UIDVALIDITY for folder {:?}", folder),
-            };
+        ensure!(
+            mailbox.uid_validity.is_some(),
+            "Cannot get UIDVALIDITY for folder {:?}",
+            folder.as_ref()
+        );
 
-            if new_uid_validity == uid_validity {
-                return Ok((uid_validity, last_seen_uid));
-            }
+        let new_uid_validity = mailbox.uid_validity.unwrap();
+        if new_uid_validity != uid_validity {
+            // First time this folder is selected or UIDVALIDITY has changed.
+            // Init lastseenuid and save it to config.
+            info!(
+                context,
+                "new_uid_validity={} current local uid_validity={} lastseenuid={}",
+                new_uid_validity,
+                uid_validity,
+                last_seen_uid
+            );
 
             if mailbox.exists == 0 {
-                info!(context, "Folder \"{}\" is empty.", folder);
+                info!(context, "Folder \"{}\" is empty.", folder.as_ref());
 
                 // set lastseenuid=0 for empty folders.
                 // id we do not do this here, we'll miss the first message
                 // as we will get in here again and fetch from lastseenuid+1 then
 
                 self.set_config_last_seen_uid(context, &folder, new_uid_validity, 0);
-                return Ok((new_uid_validity, 0));
+                return Ok(false);
             }
 
-            // uid_validity has changed or is being set the first time.
-            // find the last seen uid within the new uid_validity scope.
-            let new_last_seen_uid = match mailbox.uid_next {
-                Some(uid_next) => {
-                    uid_next - 1 // XXX could uid_next be 0?
-                }
-                None => {
-                    warn!(
-                        context,
-                        "IMAP folder has no uid_next, fall back to fetching"
-                    );
-                    if let Some(ref mut session) = &mut *self.session.lock().await {
-                        // note that we use fetch by sequence number
-                        // and thus we only need to get exactly the
-                        // last-index message.
-                        let set = format!("{}", mailbox.exists);
-                        match session.fetch(set, JUST_UID).await {
-                            Ok(list) => list[0].uid.unwrap_or_default(),
-                            Err(err) => {
-                                bail!("fetch failed: {:?}", err);
-                            }
-                        }
-                    } else {
-                        return Err(Error::ImapNoConnection);
+            let list = if let Some(ref mut session) = &mut *self.session.lock().await {
+                // `FETCH <message sequence number> (UID)`
+                let set = format!("{}", mailbox.exists);
+                match session.fetch(set, PREFETCH_FLAGS).await {
+                    Ok(list) => list,
+                    Err(err) => {
+                        bail!("fetch failed: {}", err);
                     }
                 }
+            } else {
+                return Err(Error::ImapNoConnection);
             };
 
-            self.set_config_last_seen_uid(context, &folder, new_uid_validity, new_last_seen_uid);
+            last_seen_uid = list[0].uid.unwrap_or_else(|| 0);
+
+            // if the UIDVALIDITY has _changed_, decrease lastseenuid by one to avoid gaps (well add 1 below
+            if uid_validity > 0 && last_seen_uid > 1 {
+                last_seen_uid -= 1;
+            }
+
+            uid_validity = new_uid_validity;
+            self.set_config_last_seen_uid(context, &folder, uid_validity, last_seen_uid);
             info!(
                 context,
-                "uid/validity change: new {}/{} current {}/{}",
-                new_last_seen_uid,
-                new_uid_validity,
+                "lastseenuid initialized to {} for {}@{}",
+                last_seen_uid,
+                folder.as_ref(),
                 uid_validity,
-                last_seen_uid
             );
-            Ok((new_uid_validity, new_last_seen_uid))
-        })
-    }
-
-    async fn fetch_from_single_folder<S: AsRef<str>>(
-        &self,
-        context: &Context,
-        folder: S,
-    ) -> Result<bool, Error> {
-        let (uid_validity, last_seen_uid) =
-            self.select_with_uidvalidity(context, folder.as_ref())?;
+        }
 
         let mut read_cnt = 0;
+        let mut read_errors = 0;
+        let mut new_last_seen_uid = 0;
 
-        let mut list = if let Some(ref mut session) = &mut *self.session.lock().await {
+        let list = if let Some(ref mut session) = &mut *self.session.lock().await {
             // fetch messages with larger UID than the last one seen
             // (`UID FETCH lastseenuid+1:*)`, see RFC 4549
             let set = format!("{}:*", last_seen_uid + 1);
@@ -634,51 +609,44 @@ impl Imap {
             return Err(Error::ImapNoConnection);
         };
 
-        // prefetch info from all unfetched mails
-        let mut new_last_seen_uid = last_seen_uid;
-        let mut read_errors = 0;
-
-        list.sort_unstable_by_key(|msg| msg.uid.unwrap_or_default());
-
+        // go through all mails in folder (this is typically _fast_ as we already have the whole list)
         for msg in &list {
-            let cur_uid = msg.uid.unwrap_or_default();
-            if cur_uid <= last_seen_uid {
-                warn!(
-                    context,
-                    "wrong uid {}, last seen was {}", cur_uid, last_seen_uid
-                );
-                continue;
-            }
-            read_cnt += 1;
+            let cur_uid = msg.uid.unwrap_or_else(|| 0);
+            if cur_uid > last_seen_uid {
+                read_cnt += 1;
 
-            let message_id = prefetch_get_message_id(msg).unwrap_or_default();
+                let message_id = prefetch_get_message_id(msg).unwrap_or_default();
 
-            if !precheck_imf(context, &message_id, folder.as_ref(), cur_uid) {
-                // check passed, go fetch the rest
-                if self.fetch_single_msg(context, &folder, cur_uid).await == 0 {
+                if !precheck_imf(context, &message_id, folder.as_ref(), cur_uid) {
+                    // check passed, go fetch the rest
+                    if self.fetch_single_msg(context, &folder, cur_uid).await == 0 {
+                        info!(
+                            context,
+                            "Read error for message {} from \"{}\", trying over later.",
+                            message_id,
+                            folder.as_ref()
+                        );
+
+                        read_errors += 1;
+                    }
+                } else {
+                    // check failed
                     info!(
                         context,
-                        "Read error for message {} from \"{}\", trying over later.",
+                        "Skipping message {} from \"{}\" by precheck.",
                         message_id,
-                        folder.as_ref()
+                        folder.as_ref(),
                     );
-                    read_errors += 1;
                 }
-            } else {
-                // we know the message-id already or don't want the message otherwise.
-                info!(
-                    context,
-                    "Skipping message {} from \"{}\" by precheck.",
-                    message_id,
-                    folder.as_ref(),
-                );
-            }
-            if read_errors == 0 {
-                new_last_seen_uid = cur_uid;
+                if cur_uid > new_last_seen_uid {
+                    new_last_seen_uid = cur_uid
+                }
             }
         }
 
-        if new_last_seen_uid > last_seen_uid {
+        if 0 == read_errors && new_last_seen_uid > 0 {
+            // TODO: it might be better to increase the lastseenuid also on partial errors.
+            // however, this requires to sort the list before going through it above.
             self.set_config_last_seen_uid(context, &folder, uid_validity, new_last_seen_uid);
         }
 
@@ -786,7 +754,7 @@ impl Imap {
         1
     }
 
-    pub fn idle(&self, context: &Context, watch_folder: Option<String>) -> Result<(), Error> {
+    pub fn idle(&self, context: &Context, watch_folder: Option<String>) -> Result<()> {
         task::block_on(async move {
             if !self.config.read().await.can_idle {
                 return Err(Error::ImapMissesIdle);
@@ -794,11 +762,12 @@ impl Imap {
 
             self.setup_handle_if_needed(context)?;
 
-            if let Err(err) = self.select_folder(context, watch_folder.clone()).await {
-                return Err(Error::ImapSelectFailed(format!(
-                    "{:?}: {:?}",
-                    watch_folder, err
-                )));
+            match self.select_folder(context, watch_folder.clone()).await {
+                ImapActionResult::Success | ImapActionResult::AlreadyDone => {}
+
+                ImapActionResult::Failed | ImapActionResult::RetryLater => {
+                    return Err(Error::ImapSelectFailed(format!("{:?}", watch_folder)));
+                }
             }
 
             let session = self.session.lock().await.take();
@@ -1128,22 +1097,14 @@ impl Imap {
                 }
             }
             match self.select_folder(context, Some(&folder)).await {
-                Ok(()) => None,
-                Err(SelectError::ConnectionLost) => {
-                    warn!(context, "Lost imap connection");
-                    Some(ImapActionResult::RetryLater)
-                }
-                Err(SelectError::NoSession) => {
-                    warn!(context, "no imap session");
-                    Some(ImapActionResult::Failed)
-                }
-                Err(SelectError::BadFolderName(folder_name)) => {
-                    warn!(context, "invalid folder name: {:?}", folder_name);
-                    Some(ImapActionResult::Failed)
-                }
-                Err(err) => {
-                    warn!(context, "failed to select folder: {:?}: {:?}", folder, err);
-                    Some(ImapActionResult::RetryLater)
+                ImapActionResult::Success | ImapActionResult::AlreadyDone => None,
+                res => {
+                    warn!(
+                        context,
+                        "Cannot select folder {} for preparing IMAP operation", folder
+                    );
+
+                    Some(res)
                 }
             }
         })
@@ -1245,35 +1206,19 @@ impl Imap {
         })
     }
 
-    pub fn ensure_configured_folders(
-        &self,
-        context: &Context,
-        create_mvbox: bool,
-    ) -> Result<(), Error> {
-        let folders_configured = context
-            .sql
-            .get_raw_config_int(context, "folders_configured");
-        if folders_configured.unwrap_or_default() >= 3 {
-            // the "3" here we increase if we have future updates to
-            // to folder configuration
-            return Ok(());
-        }
-
+    pub fn configure_folders(&self, context: &Context, create_mvbox: bool) {
         task::block_on(async move {
-            ensure!(
-                self.is_connected().await,
-                "cannot configure folders: not connected"
-            );
+            if !self.is_connected().await {
+                return;
+            }
 
             info!(context, "Configuring IMAP-folders.");
 
             if let Some(ref mut session) = &mut *self.session.lock().await {
-                let folders = match self.list_folders(session, context).await {
-                    Some(f) => f,
-                    None => {
-                        bail!("could not obtain list of imap folders");
-                    }
-                };
+                let folders = self
+                    .list_folders(session, context)
+                    .await
+                    .expect("no folders found");
 
                 let sentbox_folder =
                     folders
@@ -1304,7 +1249,7 @@ impl Imap {
                         Err(err) => {
                             warn!(
                                 context,
-                                "Cannot create MVBOX-folder, trying to create INBOX subfolder. ({})",
+                                "Cannot create MVBOX-folder, using trying INBOX subfolder. ({})",
                                 err
                             );
 
@@ -1326,34 +1271,35 @@ impl Imap {
                     // that may be used by other MUAs to list folders.
                     // for the LIST command, the folder is always visible.
                     if let Some(ref mvbox) = mvbox_folder {
-                        if let Err(err) = session.subscribe(mvbox).await {
-                            warn!(context, "could not subscribe to {:?}: {:?}", mvbox, err);
-                        }
+                        // TODO: better error handling
+                        session.subscribe(mvbox).await.expect("failed to subscribe");
                     }
                 }
                 context
                     .sql
-                    .set_raw_config(context, "configured_inbox_folder", Some("INBOX"))?;
+                    .set_raw_config(context, "configured_inbox_folder", Some("INBOX"))
+                    .ok();
                 if let Some(ref mvbox_folder) = mvbox_folder {
-                    context.sql.set_raw_config(
-                        context,
-                        "configured_mvbox_folder",
-                        Some(mvbox_folder),
-                    )?;
+                    context
+                        .sql
+                        .set_raw_config(context, "configured_mvbox_folder", Some(mvbox_folder))
+                        .ok();
                 }
                 if let Some(ref sentbox_folder) = sentbox_folder {
-                    context.sql.set_raw_config(
-                        context,
-                        "configured_sentbox_folder",
-                        Some(sentbox_folder.name()),
-                    )?;
+                    context
+                        .sql
+                        .set_raw_config(
+                            context,
+                            "configured_sentbox_folder",
+                            Some(sentbox_folder.name()),
+                        )
+                        .ok();
                 }
                 context
                     .sql
-                    .set_raw_config_int(context, "folders_configured", 3)?;
+                    .set_raw_config_int(context, "folders_configured", 3)
+                    .ok();
             }
-            info!(context, "FINISHED configuring IMAP-folders.");
-            Ok(())
         })
     }
 
@@ -1384,35 +1330,33 @@ impl Imap {
             info!(context, "emptying folder {}", folder);
 
             if folder.is_empty() {
-                error!(context, "cannot perform empty, folder not set");
+                warn!(context, "cannot perform empty, folder not set");
                 return;
             }
-            if let Err(err) = self.select_folder(context, Some(&folder)).await {
-                // we want to report all error to the user
-                // (no retry should be attempted)
-                error!(
-                    context,
-                    "Could not select {} for expunging: {:?}", folder, err
-                );
-                return;
-            }
-
-            if !self
-                .add_flag_finalized_with_set(context, SELECT_ALL, "\\Deleted")
-                .await
-            {
-                error!(context, "Cannot mark messages for deletion {}", folder);
-                return;
-            }
-
-            // we now trigger expunge to actually delete messages
-            self.config.write().await.selected_folder_needs_expunge = true;
-            match self.select_folder::<String>(context, None).await {
-                Ok(()) => {
-                    emit_event!(context, Event::ImapFolderEmptied(folder.to_string()));
+            match self.select_folder(context, Some(&folder)).await {
+                ImapActionResult::Success | ImapActionResult::AlreadyDone => {
+                    if !self
+                        .add_flag_finalized_with_set(context, SELECT_ALL, "\\Deleted")
+                        .await
+                    {
+                        warn!(context, "Cannot empty folder {}", folder);
+                    } else {
+                        // we now trigger expunge to actually delete messages
+                        self.config.write().await.selected_folder_needs_expunge = true;
+                        if self.select_folder::<String>(context, None).await
+                            == ImapActionResult::Success
+                        {
+                            emit_event!(context, Event::ImapFolderEmptied(folder.to_string()));
+                        } else {
+                            warn!(
+                                context,
+                                "could not perform expunge on empty-marked folder {}", folder
+                            );
+                        }
+                    }
                 }
-                Err(err) => {
-                    error!(context, "expunge failed {}: {:?}", folder, err);
+                ImapActionResult::Failed | ImapActionResult::RetryLater => {
+                    warn!(context, "could not select folder {}", folder);
                 }
             }
         });
@@ -1490,7 +1434,7 @@ fn precheck_imf(context: &Context, rfc724_mid: &str, server_folder: &str, server
     }
 }
 
-fn prefetch_get_message_id(prefetch_msg: &Fetch) -> Result<String, Error> {
+fn prefetch_get_message_id(prefetch_msg: &Fetch) -> Result<String> {
     ensure!(
         prefetch_msg.envelope().is_some(),
         "Fetched message has no envelope"

src/imap_client.rs

@@ -72,12 +72,11 @@ impl Client {
     pub async fn secure<S: AsRef<str>>(
         self,
         domain: S,
-        certificate_checks: CertificateChecks,
+        _certificate_checks: CertificateChecks,
     ) -> ImapResult<Client> {
         match self {
             Client::Insecure(client) => {
-                let tls_config = dc_build_tls_config(certificate_checks);
-                let tls: async_tls::TlsConnector = Arc::new(tls_config).into();
+                let tls = async_tls::TlsConnector::new();
 
                 let client_sec = client.secure(domain, &tls).await?;
 

src/imex.rs

@@ -476,15 +476,14 @@ fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Resul
                 }
 
                 let path_filename = context.get_blobdir().join(file_name);
-                if dc_write_file(context, &path_filename, &file_blob).is_err() {
-                    bail!(
-                        "Storage full? Cannot write file {} with {} bytes.",
-                        path_filename.display(),
-                        file_blob.len(),
-                    );
-                } else {
+                if dc_write_file(context, &path_filename, &file_blob) {
                     continue;
                 }
+                bail!(
+                    "Storage full? Cannot write file {} with {} bytes.",
+                    path_filename.display(),
+                    file_blob.len(),
+                );
             }
             Ok(())
         },
@@ -687,14 +686,14 @@ fn export_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
                 let (id, public_key, private_key, is_default) = key_pair?;
                 let id = Some(id).filter(|_| is_default != 0);
                 if let Some(key) = public_key {
-                    if export_key_to_asc_file(context, &dir, id, &key).is_err() {
+                    if !export_key_to_asc_file(context, &dir, id, &key) {
                         export_errors += 1;
                     }
                 } else {
                     export_errors += 1;
                 }
                 if let Some(key) = private_key {
-                    if export_key_to_asc_file(context, &dir, id, &key).is_err() {
+                    if !export_key_to_asc_file(context, &dir, id, &key) {
                         export_errors += 1;
                     }
                 } else {
@@ -718,7 +717,8 @@ fn export_key_to_asc_file(
     dir: impl AsRef<Path>,
     id: Option<i64>,
     key: &Key,
-) -> std::io::Result<()> {
+) -> bool {
+    let mut success = false;
     let file_name = {
         let kind = if key.is_public() { "public" } else { "private" };
         let id = id.map_or("default".into(), |i| i.to_string());
@@ -728,13 +728,14 @@ fn export_key_to_asc_file(
     info!(context, "Exporting key {}", file_name.display());
     dc_delete_file(context, &file_name);
 
-    let res = key.write_asc_to_file(&file_name, context);
-    if res.is_err() {
+    if !key.write_asc_to_file(&file_name, context) {
         error!(context, "Cannot write key to {}", file_name.display());
     } else {
         context.call_cb(Event::ImexFileWritten(file_name));
+        success = true;
     }
-    res
+
+    success
 }
 
 #[cfg(test)]
@@ -798,7 +799,7 @@ mod tests {
         let base64 = include_str!("../test-data/key/public.asc");
         let key = Key::from_base64(base64, KeyType::Public).unwrap();
         let blobdir = "$BLOBDIR";
-        assert!(export_key_to_asc_file(&context.ctx, blobdir, None, &key).is_ok());
+        assert!(export_key_to_asc_file(&context.ctx, blobdir, None, &key));
         let blobdir = context.ctx.get_blobdir().to_str().unwrap();
         let filename = format!("{}/public-key-default.asc", blobdir);
         let bytes = std::fs::read(&filename).unwrap();

src/job.rs

@@ -20,7 +20,6 @@ use crate::message::MsgId;
 use crate::message::{self, Message, MessageState};
 use crate::mimefactory::{vec_contains_lowercase, Loaded, MimeFactory};
 use crate::param::*;
-use crate::smtp::SmtpError;
 use crate::sql;
 
 // results in ~3 weeks for the last backoff timespan
@@ -185,22 +184,11 @@ impl Job {
                     // otherwise might send it twice.
                     let mut smtp = context.smtp.lock().unwrap();
                     match smtp.send(context, recipients_list, body, self.job_id) {
-                        Err(SmtpError::SendError(err)) => {
-                            // Remote error, retry later.
+                        Err(err) => {
                             smtp.disconnect();
-                            info!(context, "SMTP failed to send: {}", err);
+                            warn!(context, "smtp failed: {}", err);
                             self.try_again_later(TryAgain::AtOnce, Some(err.to_string()));
                         }
-                        Err(SmtpError::EnvelopeError(err)) => {
-                            // Local error, job is invalid, do not retry.
-                            smtp.disconnect();
-                            warn!(context, "SMTP job is invalid: {}", err);
-                        }
-                        Err(SmtpError::NoTransport) => {
-                            // Should never happen.
-                            // It does not even make sense to disconnect here.
-                            error!(context, "SMTP job failed because SMTP has no transport");
-                        }
                         Ok(()) => {
                             // smtp success, update db ASAP, then delete smtp file
                             if 0 != self.foreign_id {
@@ -228,10 +216,13 @@ impl Job {
         let imap_inbox = &context.inbox_thread.read().unwrap().imap;
 
         if let Ok(msg) = Message::load_from_db(context, MsgId::new(self.foreign_id)) {
-            if let Err(err) = imap_inbox.ensure_configured_folders(context, true) {
-                self.try_again_later(TryAgain::StandardDelay, None);
-                warn!(context, "could not configure folders: {:?}", err);
-                return;
+            if context
+                .sql
+                .get_raw_config_int(context, "folders_configured")
+                .unwrap_or_default()
+                < 3
+            {
+                imap_inbox.configure_folders(context, true);
             }
             let dest_folder = context
                 .sql
@@ -352,10 +343,13 @@ impl Job {
             return;
         }
         if 0 != self.param.get_int(Param::AlsoMove).unwrap_or_default() {
-            if let Err(err) = imap_inbox.ensure_configured_folders(context, true) {
-                self.try_again_later(TryAgain::StandardDelay, None);
-                warn!(context, "configuring folders failed: {:?}", err);
-                return;
+            if context
+                .sql
+                .get_raw_config_int(context, "folders_configured")
+                .unwrap_or_default()
+                < 3
+            {
+                imap_inbox.configure_folders(context, true);
             }
             let dest_folder = context
                 .sql
@@ -816,11 +810,11 @@ fn job_perform(context: &Context, thread: Thread, probe_network: bool) {
             if tries < JOB_RETRIES {
                 job.tries = tries;
                 let time_offset = get_backoff_time_offset(tries);
-                job.desired_timestamp = time() + time_offset;
+                job.desired_timestamp = job.added_timestamp + time_offset;
                 job.update(context);
                 info!(
                     context,
-                    "{}-job #{} not succeeded on try #{}, retry in {} seconds.",
+                    "{}-job #{} not succeeded on try #{}, retry in ADD_TIME+{} (in {} seconds).",
                     if thread == Thread::Imap {
                         "INBOX"
                     } else {
@@ -828,7 +822,8 @@ fn job_perform(context: &Context, thread: Thread, probe_network: bool) {
                     },
                     job.job_id as u32,
                     tries,
-                    time_offset
+                    time_offset,
+                    job.added_timestamp + time_offset - time()
                 );
                 if thread == Thread::Smtp && tries < JOB_RETRIES - 1 {
                     context
@@ -981,17 +976,19 @@ pub fn interrupt_smtp_idle(context: &Context) {
 /// jobs, this is tricky and probably wrong currently.  Look at the
 /// SQL queries for details.
 fn load_jobs(context: &Context, thread: Thread, probe_network: bool) -> Vec<Job> {
-    let query = if !probe_network {
-        // processing for first-try and after backoff-timeouts:
-        // process jobs in the order they were added.
-        "SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries \
-         FROM jobs WHERE thread=? AND desired_timestamp<=? ORDER BY action DESC, added_timestamp;"
+    let query = if probe_network {
+        // Processing after call to dc_maybe_network():
+        // process all pending jobs in the order of their priority.
+        // If jobs have the same priority, process the
+        // one that was added earlier.
+        "SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries
+         FROM jobs WHERE thread=? ORDER BY action DESC, added_timestamp;"
     } else {
-        // processing after call to dc_maybe_network():
-        // process _all_ pending jobs that failed before
-        // in the order of their backoff-times.
-        "SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries \
-         FROM jobs WHERE thread=? AND tries>0 ORDER BY desired_timestamp, action DESC;"
+        // Processing for the first try and after backoff-timeouts:
+        // process jobs that have their backoff expired
+        // in the order of their backoff times.
+        "SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries
+         FROM jobs WHERE thread=? AND desired_timestamp<=? ORDER BY desired_timestamp;"
     };
 
     let params_no_probe = params![thread as i64, time()];

src/key.rs

@@ -217,18 +217,15 @@ impl Key {
             .expect("failed to serialize key")
     }
 
-    pub fn write_asc_to_file(
-        &self,
-        file: impl AsRef<Path>,
-        context: &Context,
-    ) -> std::io::Result<()> {
+    pub fn write_asc_to_file(&self, file: impl AsRef<Path>, context: &Context) -> bool {
         let file_content = self.to_asc(None).into_bytes();
 
-        let res = dc_write_file(context, &file, &file_content);
-        if res.is_err() {
+        if dc_write_file(context, &file, &file_content) {
+            true
+        } else {
             error!(context, "Cannot write key to {}", file.as_ref().display());
+            false
         }
-        res
     }
 
     pub fn fingerprint(&self) -> String {

src/mimefactory.rs

@@ -707,7 +707,7 @@ impl<'a> MimeFactory<'a> {
                     .get_config(Config::ConfiguredAddr)
                     .unwrap_or_default();
 
-                if !email_to_remove.is_empty() && !addr_cmp(email_to_remove, self_addr) {
+                if !email_to_remove.is_empty() && email_to_remove != self_addr {
                     if !vec_contains_lowercase(&factory.recipients_addr, &email_to_remove) {
                         factory.recipients_names.push("".to_string());
                         factory.recipients_addr.push(email_to_remove.to_string());

src/oauth2.rs

@@ -355,8 +355,6 @@ fn normalize_addr(addr: &str) -> &str {
 mod tests {
     use super::*;
 
-    use crate::test_utils::*;
-
     #[test]
     fn test_normalize_addr() {
         assert_eq!(normalize_addr(" hello@mail.de  "), "hello@mail.de");
@@ -386,24 +384,4 @@ mod tests {
 
         assert_eq!(Oauth2::from_address("hello@web.de"), None);
     }
-
-    #[test]
-    fn test_dc_get_oauth2_addr() {
-        let ctx = dummy_context();
-        let addr = "dignifiedquire@gmail.com";
-        let code = "fail";
-        let res = dc_get_oauth2_addr(&ctx.ctx, addr, code);
-        // this should fail as it is an invalid password
-        assert_eq!(res, None);
-    }
-
-    #[test]
-    fn test_dc_get_oauth2_token() {
-        let ctx = dummy_context();
-        let addr = "dignifiedquire@gmail.com";
-        let code = "fail";
-        let res = dc_get_oauth2_access_token(&ctx.ctx, addr, code, false);
-        // this should fail as it is an invalid password
-        assert_eq!(res, None);
-    }
 }

src/securejoin.rs

@@ -242,8 +242,7 @@ pub fn dc_join_securejoin(context: &Context, qr: &str) -> u32 {
 
     // Bob -> Alice
     while !context.shall_stop_ongoing() {
-        // Don't sleep too long, the user is waiting.
-        std::thread::sleep(std::time::Duration::from_millis(200));
+        std::thread::sleep(std::time::Duration::new(0, 3_000_000));
     }
     cleanup(&context, contact_chat_id, true, join_vg)
 }
@@ -319,40 +318,21 @@ fn fingerprint_equals_sender(
     false
 }
 
-pub(crate) struct HandshakeMessageStatus {
-    pub(crate) hide_this_msg: bool,
-    pub(crate) delete_this_msg: bool,
-    pub(crate) stop_ongoing_process: bool,
-    pub(crate) bob_securejoin_success: Option<bool>,
-}
-
-impl Default for HandshakeMessageStatus {
-    fn default() -> Self {
-        Self {
-            hide_this_msg: true,
-            delete_this_msg: false,
-            stop_ongoing_process: false,
-            bob_securejoin_success: None,
-        }
-    }
-}
-
 /* library private: secure-join */
-pub(crate) fn handle_securejoin_handshake(
+pub fn handle_securejoin_handshake(
     context: &Context,
     mimeparser: &MimeParser,
     contact_id: u32,
-) -> Result<HandshakeMessageStatus, Error> {
+) -> libc::c_int {
     let own_fingerprint: String;
 
-    ensure!(
-        contact_id > DC_CONTACT_ID_LAST_SPECIAL,
-        "handle_securejoin_handshake(): called with special contact id"
-    );
+    if contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
+        return 0;
+    }
     let step = match mimeparser.lookup_optional_field("Secure-Join") {
         Some(s) => s,
         None => {
-            bail!("This message is not a Secure-Join message");
+            return 0;
         }
     };
     info!(
@@ -365,8 +345,8 @@ pub(crate) fn handle_securejoin_handshake(
     if contact_chat_id_blocked != Blocked::Not {
         chat::unblock(context, contact_chat_id);
     }
+    let mut ret: libc::c_int = DC_HANDSHAKE_STOP_NORMAL_PROCESSING;
     let join_vg = step.starts_with("vg-");
-    let mut ret = HandshakeMessageStatus::default();
 
     match step.as_str() {
         "vg-request" | "vc-request" => {
@@ -382,12 +362,12 @@ pub(crate) fn handle_securejoin_handshake(
                 Some(n) => n,
                 None => {
                     warn!(context, "Secure-join denied (invitenumber missing).",);
-                    return Ok(ret);
+                    return ret;
                 }
             };
             if !token::exists(context, token::Namespace::InviteNumber, &invitenumber) {
                 warn!(context, "Secure-join denied (bad invitenumber).",);
-                return Ok(ret);
+                return ret;
             }
             info!(context, "Secure-join requested.",);
 
@@ -413,7 +393,7 @@ pub(crate) fn handle_securejoin_handshake(
             if cond {
                 warn!(context, "auth-required message out of sync.",);
                 // no error, just aborted somehow or a mail from another handshake
-                return Ok(ret);
+                return ret;
             }
             let scanned_fingerprint_of_alice = get_qr_attr!(context, fingerprint).to_string();
             let auth = get_qr_attr!(context, auth).to_string();
@@ -428,9 +408,8 @@ pub(crate) fn handle_securejoin_handshake(
                         "Not encrypted."
                     },
                 );
-                ret.stop_ongoing_process = true;
-                ret.bob_securejoin_success = Some(false);
-                return Ok(ret);
+                end_bobs_joining(context, DC_BOB_ERROR);
+                return ret;
             }
             if !fingerprint_equals_sender(context, &scanned_fingerprint_of_alice, contact_chat_id) {
                 could_not_establish_secure_connection(
@@ -438,9 +417,8 @@ pub(crate) fn handle_securejoin_handshake(
                     contact_chat_id,
                     "Fingerprint mismatch on joiner-side.",
                 );
-                ret.stop_ongoing_process = true;
-                ret.bob_securejoin_success = Some(false);
-                return Ok(ret);
+                end_bobs_joining(context, DC_BOB_ERROR);
+                return ret;
             }
             info!(context, "Fingerprint verified.",);
             own_fingerprint = get_self_fingerprint(context).unwrap();
@@ -475,7 +453,7 @@ pub(crate) fn handle_securejoin_handshake(
                         contact_chat_id,
                         "Fingerprint not provided.",
                     );
-                    return Ok(ret);
+                    return ret;
                 }
             };
             if !encrypted_and_signed(mimeparser, &fingerprint) {
@@ -484,7 +462,7 @@ pub(crate) fn handle_securejoin_handshake(
                     contact_chat_id,
                     "Auth not encrypted.",
                 );
-                return Ok(ret);
+                return ret;
             }
             if !fingerprint_equals_sender(context, &fingerprint, contact_chat_id) {
                 could_not_establish_secure_connection(
@@ -492,7 +470,7 @@ pub(crate) fn handle_securejoin_handshake(
                     contact_chat_id,
                     "Fingerprint mismatch on inviter-side.",
                 );
-                return Ok(ret);
+                return ret;
             }
             info!(context, "Fingerprint verified.",);
             // verify that the `Secure-Join-Auth:`-header matches the secret written to the QR code
@@ -504,12 +482,12 @@ pub(crate) fn handle_securejoin_handshake(
                         contact_chat_id,
                         "Auth not provided.",
                     );
-                    return Ok(ret);
+                    return ret;
                 }
             };
             if !token::exists(context, token::Namespace::Auth, &auth_0) {
                 could_not_establish_secure_connection(context, contact_chat_id, "Auth invalid.");
-                return Ok(ret);
+                return ret;
             }
             if mark_peer_as_verified(context, fingerprint).is_err() {
                 could_not_establish_secure_connection(
@@ -517,7 +495,7 @@ pub(crate) fn handle_securejoin_handshake(
                     contact_chat_id,
                     "Fingerprint mismatch on inviter-side.",
                 );
-                return Ok(ret);
+                return ret;
             }
             Contact::scaleup_origin_by_id(context, contact_id, Origin::SecurejoinInvited);
             info!(context, "Auth verified.",);
@@ -531,7 +509,7 @@ pub(crate) fn handle_securejoin_handshake(
                 let (group_chat_id, _, _) = chat::get_chat_id_by_grpid(context, &field_grpid);
                 if group_chat_id == 0 {
                     error!(context, "Chat {} not found.", &field_grpid);
-                    return Ok(ret);
+                    return ret;
                 } else {
                     if let Err(err) =
                         chat::add_contact_to_chat_ex(context, group_chat_id, contact_id, true)
@@ -546,11 +524,11 @@ pub(crate) fn handle_securejoin_handshake(
         }
         "vg-member-added" | "vc-contact-confirm" => {
             if join_vg {
-                ret.hide_this_msg = false;
+                ret = DC_HANDSHAKE_CONTINUE_NORMAL_PROCESSING;
             }
             if context.bob.read().unwrap().expects != DC_VC_CONTACT_CONFIRM {
                 info!(context, "Message belongs to a different handshake.",);
-                return Ok(ret);
+                return ret;
             }
             let cond = {
                 let bob = context.bob.read().unwrap();
@@ -562,7 +540,7 @@ pub(crate) fn handle_securejoin_handshake(
                     context,
                     "Message out of sync or belongs to a different handshake.",
                 );
-                return Ok(ret);
+                return ret;
             }
             let scanned_fingerprint_of_alice = get_qr_attr!(context, fingerprint).to_string();
 
@@ -586,8 +564,8 @@ pub(crate) fn handle_securejoin_handshake(
                     contact_chat_id,
                     "Contact confirm message not encrypted.",
                 );
-                ret.bob_securejoin_success = Some(false);
-                return Ok(ret);
+                end_bobs_joining(context, DC_BOB_ERROR);
+                return ret;
             }
 
             if mark_peer_as_verified(context, &scanned_fingerprint_of_alice).is_err() {
@@ -596,7 +574,7 @@ pub(crate) fn handle_securejoin_handshake(
                     contact_chat_id,
                     "Fingerprint mismatch on joiner-side.",
                 );
-                return Ok(ret);
+                return ret;
             }
             Contact::scaleup_origin_by_id(context, contact_id, Origin::SecurejoinJoined);
             emit_event!(context, Event::ContactsChanged(None));
@@ -605,7 +583,7 @@ pub(crate) fn handle_securejoin_handshake(
                 .unwrap_or_default();
             if join_vg && !addr_equals_self(context, cg_member_added) {
                 info!(context, "Message belongs to a different handshake (scaled up contact anyway to allow creation of group).");
-                return Ok(ret);
+                return ret;
             }
             secure_connection_established(context, contact_chat_id);
             context.bob.write().unwrap().expects = 0;
@@ -619,8 +597,7 @@ pub(crate) fn handle_securejoin_handshake(
                     "",
                 );
             }
-            ret.stop_ongoing_process = true;
-            ret.bob_securejoin_success = Some(true);
+            end_bobs_joining(context, DC_BOB_SUCCESS);
         }
         "vg-member-added-received" => {
             /* ============================================================
@@ -630,7 +607,7 @@ pub(crate) fn handle_securejoin_handshake(
             if let Ok(contact) = Contact::get_by_id(context, contact_id) {
                 if contact.is_verified(context) == VerifiedStatus::Unverified {
                     warn!(context, "vg-member-added-received invalid.",);
-                    return Ok(ret);
+                    return ret;
                 }
                 inviter_progress!(context, contact_id, 800);
                 inviter_progress!(context, contact_id, 1000);
@@ -644,17 +621,22 @@ pub(crate) fn handle_securejoin_handshake(
                 });
             } else {
                 warn!(context, "vg-member-added-received invalid.",);
-                return Ok(ret);
+                return ret;
             }
         }
         _ => {
             warn!(context, "invalid step: {}", step);
         }
     }
-    if ret.hide_this_msg {
-        ret.delete_this_msg = true;
+    if ret == DC_HANDSHAKE_STOP_NORMAL_PROCESSING {
+        ret |= DC_HANDSHAKE_ADD_DELETE_JOB;
     }
-    Ok(ret)
+    ret
+}
+
+fn end_bobs_joining(context: &Context, status: libc::c_int) {
+    context.bob.write().unwrap().status = status;
+    context.stop_ongoing();
 }
 
 fn secure_connection_established(context: &Context, contact_chat_id: u32) {

src/smtp.rs

@@ -1,8 +1,6 @@
 use lettre::smtp::client::net::*;
 use lettre::*;
 
-use failure::Fail;
-
 use crate::constants::*;
 use crate::context::Context;
 use crate::error::Error;
@@ -19,16 +17,6 @@ pub struct Smtp {
     from: Option<EmailAddress>,
 }
 
-#[derive(Debug, Fail)]
-pub enum SmtpError {
-    #[fail(display = "Envelope error: {}", _0)]
-    EnvelopeError(#[cause] lettre::error::Error),
-    #[fail(display = "Send error: {}", _0)]
-    SendError(#[cause] lettre::smtp::error::Error),
-    #[fail(display = "SMTP has no transport")]
-    NoTransport,
-}
-
 impl Smtp {
     /// Create a new Smtp instances.
     pub fn new() -> Self {
@@ -156,7 +144,7 @@ impl Smtp {
         recipients: Vec<EmailAddress>,
         message: Vec<u8>,
         job_id: u32,
-    ) -> Result<(), SmtpError> {
+    ) -> Result<(), Error> {
         let message_len = message.len();
 
         let recipients_display = recipients
@@ -166,28 +154,36 @@ impl Smtp {
             .join(",");
 
         if let Some(ref mut transport) = self.transport {
-            let envelope =
-                Envelope::new(self.from.clone(), recipients).map_err(SmtpError::EnvelopeError)?;
+            let envelope = match Envelope::new(self.from.clone(), recipients) {
+                Ok(env) => env,
+                Err(err) => {
+                    bail!("{}", err);
+                }
+            };
             let mail = SendableEmail::new(
                 envelope,
                 format!("{}", job_id), // only used for internal logging
                 message,
             );
 
-            transport.send(mail).map_err(SmtpError::SendError)?;
-
-            context.call_cb(Event::SmtpMessageSent(format!(
-                "Message len={} was smtp-sent to {}",
-                message_len, recipients_display
-            )));
-            self.transport_connected = true;
-            Ok(())
+            match transport.send(mail) {
+                Ok(_) => {
+                    context.call_cb(Event::SmtpMessageSent(format!(
+                        "Message len={} was smtp-sent to {}",
+                        message_len, recipients_display
+                    )));
+                    self.transport_connected = true;
+                    Ok(())
+                }
+                Err(err) => {
+                    bail!("SMTP failed len={}: error: {}", message_len, err);
+                }
+            }
         } else {
-            warn!(
-                context,
-                "uh? SMTP has no transport, failed to send to {}", recipients_display
+            bail!(
+                "uh? SMTP has no transport,  failed to send to {:?}",
+                recipients_display
             );
-            return Err(SmtpError::NoTransport);
         }
     }
 }

src/sql.rs

@@ -829,14 +829,6 @@ fn open(
             update_icons = true;
             sql.set_raw_config_int(context, "dbversion", 59)?;
         }
-        if dbversion < 60 {
-            info!(context, "[migration] v60");
-            sql.execute(
-                "ALTER TABLE chats ADD COLUMN created_timestamp INTEGER DEFAULT 0;",
-                NO_PARAMS,
-            )?;
-            sql.set_raw_config_int(context, "dbversion", 60)?;
-        }
 
         // (2) updates that require high-level objects
         // (the structure is complete now and all objects are usable)

tests/stress.rs

@@ -233,6 +233,26 @@ fn test_dc_get_oauth2_url() {
     assert_eq!(res, Some("https://accounts.google.com/o/oauth2/auth?client_id=959970109878%2D4mvtgf6feshskf7695nfln6002mom908%2Eapps%2Egoogleusercontent%2Ecom&redirect_uri=chat%2Edelta%3A%2Fcom%2Eb44t%2Emessenger&response_type=code&scope=https%3A%2F%2Fmail.google.com%2F%20email&access_type=offline".into()));
 }
 
+#[test]
+fn test_dc_get_oauth2_addr() {
+    let ctx = create_test_context();
+    let addr = "dignifiedquire@gmail.com";
+    let code = "fail";
+    let res = dc_get_oauth2_addr(&ctx.ctx, addr, code);
+    // this should fail as it is an invalid password
+    assert_eq!(res, None);
+}
+
+#[test]
+fn test_dc_get_oauth2_token() {
+    let ctx = create_test_context();
+    let addr = "dignifiedquire@gmail.com";
+    let code = "fail";
+    let res = dc_get_oauth2_access_token(&ctx.ctx, addr, code, false);
+    // this should fail as it is an invalid password
+    assert_eq!(res, None);
+}
+
 #[test]
 fn test_stress_tests() {
     unsafe {