add changelog and bump to beta.4

fix plus-space-decoding in qr-code

CHANGELOG.md

@@ -1,4 +1,46 @@
-# API changes
+# Changelog 
+
+## 1.0.0-beta4
+
+- fix more than one sending of autocrypt setup message
+
+- fix recognition of mailto-address-qr-codes, add tests
+
+- tune down error to warning when adding self to chat
+
+## 1.0.0-beta3
+
+- add back `dc_empty_server()` #682
+
+- if `show_emails` is set to `DC_SHOW_EMAILS_ALL`,
+  email-based contact requests are added to the chatlist directly
+
+- fix IMAP hangs #717 and cleanups
+
+- several rPGP fixes
+
+- code streamlining and rustifications
+
+
+## 1.0.0-beta2
+
+- https://c.delta.chat docs are now regenerated again through our CI 
+
+- several rPGP cleanups, security fixes and better multi-platform support 
+
+- reconnect on io errors and broken pipes (imap)
+
+- probe SMTP with real connection not just setup
+
+- various imap/smtp related fixes
+
+- use to_string_lossy in most places instead of relying on valid utf-8
+  encodings
+ 
+- rework, rustify and test autoconfig-reading and parsing 
+
+- some rustifications/boolifications of c-ints 
+
 
 ## 1.0.0-beta1 
 

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.0.0-beta.1"
+version = "1.0.0-beta.4"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL"
@@ -19,8 +19,8 @@ reqwest = "0.9.15"
 num-derive = "0.2.5"
 num-traits = "0.2.6"
 native-tls = "0.2.3"
-lettre = { git = "https://github.com/deltachat/lettre" }
-imap = { git = "https://github.com/jonhoo/rust-imap", rev = "281d2eb8ab50dc656ceff2ae749ca5045f334e15" }
+lettre = { git = "https://github.com/deltachat/lettre", branch = "master" }
+imap = { git = "https://github.com/deltachat/rust-imap", branch = "master" }
 base64 = "0.10"
 charset = "0.1"
 percent-encoding = "2.0"
@@ -48,6 +48,7 @@ escaper = "0.1.0"
 bitflags = "1.1.0"
 jetscii = "0.4.4"
 debug_stub_derive = "0.3.0"
+sanitize-filename = "0.2.1"
 
 [dev-dependencies]
 tempfile = "3.0"

Xargo.toml

@@ -1,6 +0,0 @@
-[dependencies.std]
-features = ["panic-unwind"]
-
-# if using `cargo test`
-[dependencies.test]
-stage = 1

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.0.0-beta.1"
+version = "1.0.0-beta.4"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/deltachat.h

@@ -1211,7 +1211,7 @@ void            dc_marknoticed_all_chats     (dc_context_t* context);
  * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
  * @param chat_id The chat ID to get all messages with media from.
- * @param msg_type Specify a message type to query here, one of the DC_MSG_* constats.
+ * @param msg_type Specify a message type to query here, one of the @ref DC_MSG constants.
  * @param msg_type2 Alternative message type to search for. 0 to skip.
  * @param msg_type3 Alternative message type to search for. 0 to skip.
  * @return An array with messages from the given chat ID that have the wanted message types.
@@ -1513,6 +1513,16 @@ char*           dc_get_mime_headers          (dc_context_t* context, uint32_t ms
  */
 void            dc_delete_msgs               (dc_context_t* context, const uint32_t* msg_ids, int msg_cnt);
 
+/**
+ * Empty IMAP server folder: delete all messages. 
+ *
+ * @memberof dc_context_t
+ * @param context The context object as created by dc_context_new()
+ * @param flags What to delete, a combination of the @ref DC_EMPTY flags
+ * @return None.
+ */
+void            dc_empty_server              (dc_context_t* context, const uint32_t flags);
+
 
 /**
  * Forward messages to another chat.
@@ -3879,6 +3889,67 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_LP_IMAP_SOCKET_FLAGS (DC_LP_IMAP_SOCKET_STARTTLS|DC_LP_IMAP_SOCKET_SSL|DC_LP_IMAP_SOCKET_PLAIN) // if none of these flags are set, the default is chosen
 #define DC_LP_SMTP_SOCKET_FLAGS (DC_LP_SMTP_SOCKET_STARTTLS|DC_LP_SMTP_SOCKET_SSL|DC_LP_SMTP_SOCKET_PLAIN) // if none of these flags are set, the default is chosen
 
+/**
+ * @defgroup DC_CERTCK DC_CERTCK
+ *
+ * These constants configure TLS certificate checks for IMAP and SMTP connections.
+ *
+ * These constants are set via dc_set_config
+ * using keys "imap_certificate_checks" and "smtp_certificate_checks".
+ *
+ * @addtogroup DC_CERTCK
+ * @{
+ */
+
+/**
+ * Configure certificate checks automatically.
+ */
+#define DC_CERTCK_AUTO 0
+
+/**
+ * Strictly check TLS certificates.
+ * Require that both the certificate and hostname are valid.
+ */
+#define DC_CERTCK_STRICT 1
+
+/**
+ * Accept invalid hostnames, but not invalid certificates.
+ */
+#define DC_CERTCK_ACCEPT_INVALID_HOSTNAMES 2
+
+/**
+ * Accept invalid certificates, including self-signed ones
+ * or having incorrect hostname.
+ */
+#define DC_CERTCK_ACCEPT_INVALID_CERTIFICATES 3
+
+/**
+ * @}
+ */
+
+
+/**
+ * @defgroup DC_EMPTY DC_EMPTY
+ *
+ * These constants configure emptying imap folders with dc_empty_server()
+ *
+ * @addtogroup DC_EMPTY
+ * @{
+ */
+
+/**
+ * Clear all mvbox messages.
+ */
+#define DC_EMPTY_MVBOX 0x01
+
+/**
+ * Clear all INBOX messages.
+ */
+#define DC_EMPTY_INBOX 0x02
+
+/**
+ * @}
+ */
 
 
 /**
@@ -3893,7 +3964,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @{
  */
 
-
 /**
  * The library-user may write an informational string to the log.
  * Passed to the callback given to dc_context_new().
@@ -3959,6 +4029,16 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  */
 #define DC_EVENT_IMAP_MESSAGE_MOVED   105
 
+/**
+ * Emitted when an IMAP folder was emptied. 
+ *
+ * @param data1 0
+ * @param data2 (const char*) folder name. 
+ *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
+ */
+#define DC_EVENT_IMAP_FOLDER_EMPTIED  106
+
 /**
  * Emitted when a new blob file was successfully written 
  *

deltachat-ffi/src/lib.rs

@@ -22,9 +22,13 @@ use std::sync::RwLock;
 use libc::uintptr_t;
 use num_traits::{FromPrimitive, ToPrimitive};
 
+use deltachat::constants::DC_MSG_ID_LAST_SPECIAL;
 use deltachat::contact::Contact;
 use deltachat::context::Context;
-use deltachat::dc_tools::{as_path, as_str, dc_strdup, to_string_lossy, OsStrExt, StrExt};
+use deltachat::dc_tools::{
+    as_path, dc_strdup, to_opt_string_lossy, to_string_lossy, OsStrExt, StrExt,
+};
+use deltachat::message::MsgId;
 use deltachat::stock::StockMessage;
 use deltachat::*;
 
@@ -126,6 +130,7 @@ impl ContextWrapper {
                     | Event::SmtpMessageSent(msg)
                     | Event::ImapMessageDeleted(msg)
                     | Event::ImapMessageMoved(msg)
+                    | Event::ImapFolderEmptied(msg)
                     | Event::NewBlobFile(msg)
                     | Event::DeletedBlobFile(msg)
                     | Event::Warning(msg)
@@ -139,9 +144,12 @@ impl ContextWrapper {
                     | Event::IncomingMsg { chat_id, msg_id }
                     | Event::MsgDelivered { chat_id, msg_id }
                     | Event::MsgFailed { chat_id, msg_id }
-                    | Event::MsgRead { chat_id, msg_id } => {
-                        ffi_cb(self, event_id, chat_id as uintptr_t, msg_id as uintptr_t)
-                    }
+                    | Event::MsgRead { chat_id, msg_id } => ffi_cb(
+                        self,
+                        event_id,
+                        chat_id as uintptr_t,
+                        msg_id.to_u32() as uintptr_t,
+                    ),
                     Event::ChatModified(chat_id) => ffi_cb(self, event_id, chat_id as uintptr_t, 0),
                     Event::ContactsChanged(id) | Event::LocationChanged(id) => {
                         let id = id.unwrap_or_default();
@@ -303,11 +311,14 @@ pub unsafe extern "C" fn dc_set_config(
         return 0;
     }
     let ffi_context = &*context;
-    match config::Config::from_str(as_str(key)) {
+    match config::Config::from_str(&to_string_lossy(key)) {
         // When ctx.set_config() fails it already logged the error.
         // TODO: Context::set_config() should not log this
         Ok(key) => ffi_context
-            .with_inner(|ctx| ctx.set_config(key, as_opt_str(value)).is_ok() as libc::c_int)
+            .with_inner(|ctx| {
+                ctx.set_config(key, to_opt_string_lossy(value).as_ref().map(|x| x.as_str()))
+                    .is_ok() as libc::c_int
+            })
             .unwrap_or(0),
         Err(_) => {
             ffi_context.error("dc_set_config(): invalid key");
@@ -326,7 +337,7 @@ pub unsafe extern "C" fn dc_get_config(
         return "".strdup();
     }
     let ffi_context = &*context;
-    match config::Config::from_str(as_str(key)) {
+    match config::Config::from_str(&to_string_lossy(key)) {
         Ok(key) => ffi_context
             .with_inner(|ctx| ctx.get_config(key).unwrap_or_default().strdup())
             .unwrap_or_else(|_| "".strdup()),
@@ -347,7 +358,7 @@ pub unsafe extern "C" fn dc_set_stock_translation(
         eprintln!("ignoring careless call to dc_set_stock_string");
         return 0;
     }
-    let msg = as_str(stock_msg).to_string();
+    let msg = to_string_lossy(stock_msg);
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| match StockMessage::from_u32(stock_id) {
@@ -646,22 +657,24 @@ pub unsafe extern "C" fn dc_get_chatlist(
         return ptr::null_mut();
     }
     let ffi_context = &*context;
-    let qs = if query_str.is_null() {
-        None
-    } else {
-        Some(as_str(query_str))
-    };
+    let qs = to_opt_string_lossy(query_str);
+
     let qi = if query_id == 0 { None } else { Some(query_id) };
     ffi_context
-        .with_inner(
-            |ctx| match chatlist::Chatlist::try_load(ctx, flags as usize, qs, qi) {
+        .with_inner(|ctx| {
+            match chatlist::Chatlist::try_load(
+                ctx,
+                flags as usize,
+                qs.as_ref().map(|x| x.as_str()),
+                qi,
+            ) {
                 Ok(list) => {
                     let ffi_list = ChatlistWrapper { context, list };
                     Box::into_raw(Box::new(ffi_list))
                 }
                 Err(_) => ptr::null_mut(),
-            },
-        )
+            }
+        })
         .unwrap_or_else(|_| ptr::null_mut())
 }
 
@@ -674,7 +687,8 @@ pub unsafe extern "C" fn dc_create_chat_by_msg_id(context: *mut dc_context_t, ms
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            chat::create_by_msg_id(ctx, msg_id).unwrap_or_log_default(ctx, "Failed to create chat")
+            chat::create_by_msg_id(ctx, MsgId::new(msg_id))
+                .unwrap_or_log_default(ctx, "Failed to create chat")
         })
         .unwrap_or(0)
 }
@@ -708,10 +722,7 @@ pub unsafe extern "C" fn dc_get_chat_id_by_contact_id(
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| {
-            chat::get_by_contact_id(ctx, contact_id)
-                .unwrap_or_log_default(ctx, "Failed to get chat")
-        })
+        .with_inner(|ctx| chat::get_by_contact_id(ctx, contact_id).unwrap_or(0))
         .unwrap_or(0)
 }
 
@@ -732,6 +743,7 @@ pub unsafe extern "C" fn dc_prepare_msg(
             chat::prepare_msg(ctx, chat_id, &mut ffi_msg.message)
                 .unwrap_or_log_default(ctx, "Failed to prepare message")
         })
+        .map(|msg_id| msg_id.to_u32())
         .unwrap_or(0)
 }
 
@@ -752,6 +764,7 @@ pub unsafe extern "C" fn dc_send_msg(
             chat::send_msg(ctx, chat_id, &mut ffi_msg.message)
                 .unwrap_or_log_default(ctx, "Failed to send message")
         })
+        .map(|msg_id| msg_id.to_u32())
         .unwrap_or(0)
 }
 
@@ -770,6 +783,7 @@ pub unsafe extern "C" fn dc_send_text_msg(
     ffi_context
         .with_inner(|ctx| {
             chat::send_text_msg(ctx, chat_id, text_to_send)
+                .map(|msg_id| msg_id.to_u32())
                 .unwrap_or_log_default(ctx, "Failed to send text message")
         })
         .unwrap_or(0)
@@ -834,9 +848,19 @@ pub unsafe extern "C" fn dc_get_chat_msgs(
         return ptr::null_mut();
     }
     let ffi_context = &*context;
+    let marker_flag = if marker1before <= DC_MSG_ID_LAST_SPECIAL {
+        None
+    } else {
+        Some(MsgId::new(marker1before))
+    };
     ffi_context
         .with_inner(|ctx| {
-            let arr = dc_array_t::from(chat::get_chat_msgs(ctx, chat_id, flags, marker1before));
+            let arr = dc_array_t::from(
+                chat::get_chat_msgs(ctx, chat_id, flags, marker_flag)
+                    .iter()
+                    .map(|msg_id| msg_id.to_u32())
+                    .collect::<Vec<u32>>(),
+            );
             Box::into_raw(Box::new(arr))
         })
         .unwrap_or_else(|_| ptr::null_mut())
@@ -880,7 +904,12 @@ pub unsafe extern "C" fn dc_get_fresh_msgs(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            let arr = dc_array_t::from(ctx.get_fresh_msgs());
+            let arr = dc_array_t::from(
+                ctx.get_fresh_msgs()
+                    .iter()
+                    .map(|msg_id| msg_id.to_u32())
+                    .collect::<Vec<u32>>(),
+            );
             Box::into_raw(Box::new(arr))
         })
         .unwrap_or_else(|_| ptr::null_mut())
@@ -942,13 +971,12 @@ pub unsafe extern "C" fn dc_get_chat_media(
         from_prim(or_msg_type3).expect(&format!("incorrect or_msg_type3 = {}", or_msg_type3));
     ffi_context
         .with_inner(|ctx| {
-            let arr = dc_array_t::from(chat::get_chat_media(
-                ctx,
-                chat_id,
-                msg_type,
-                or_msg_type2,
-                or_msg_type3,
-            ));
+            let arr = dc_array_t::from(
+                chat::get_chat_media(ctx, chat_id, msg_type, or_msg_type2, or_msg_type3)
+                    .iter()
+                    .map(|msg_id| msg_id.to_u32())
+                    .collect::<Vec<u32>>(),
+            );
             Box::into_raw(Box::new(arr))
         })
         .unwrap_or_else(|_| ptr::null_mut())
@@ -981,7 +1009,16 @@ pub unsafe extern "C" fn dc_get_next_media(
         from_prim(or_msg_type3).expect(&format!("incorrect or_msg_type3 = {}", or_msg_type3));
     ffi_context
         .with_inner(|ctx| {
-            chat::get_next_media(ctx, msg_id, direction, msg_type, or_msg_type2, or_msg_type3)
+            chat::get_next_media(
+                ctx,
+                MsgId::new(msg_id),
+                direction,
+                msg_type,
+                or_msg_type2,
+                or_msg_type3,
+            )
+            .map(|msg_id| msg_id.to_u32())
+            .unwrap_or(0)
         })
         .unwrap_or(0)
 }
@@ -1052,7 +1089,12 @@ pub unsafe extern "C" fn dc_search_msgs(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            let arr = dc_array_t::from(ctx.search_msgs(chat_id, as_str(query)));
+            let arr = dc_array_t::from(
+                ctx.search_msgs(chat_id, to_string_lossy(query))
+                    .iter()
+                    .map(|msg_id| msg_id.to_u32())
+                    .collect::<Vec<u32>>(),
+            );
             Box::into_raw(Box::new(arr))
         })
         .unwrap_or_else(|_| ptr::null_mut())
@@ -1094,7 +1136,7 @@ pub unsafe extern "C" fn dc_create_group_chat(
     };
     ffi_context
         .with_inner(|ctx| {
-            chat::create_group_chat(ctx, verified, as_str(name))
+            chat::create_group_chat(ctx, verified, to_string_lossy(name))
                 .unwrap_or_log_default(ctx, "Failed to create group chat")
         })
         .unwrap_or(0)
@@ -1166,7 +1208,7 @@ pub unsafe extern "C" fn dc_set_chat_name(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            chat::set_chat_name(ctx, chat_id, as_str(name))
+            chat::set_chat_name(ctx, chat_id, to_string_lossy(name))
                 .map(|_| 1)
                 .unwrap_or_log_default(ctx, "Failed to set chat name")
         })
@@ -1186,15 +1228,9 @@ pub unsafe extern "C" fn dc_set_chat_profile_image(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            chat::set_chat_profile_image(ctx, chat_id, {
-                if image.is_null() {
-                    ""
-                } else {
-                    as_str(image)
-                }
-            })
-            .map(|_| 1)
-            .unwrap_or_log_default(ctx, "Failed to set profile image")
+            chat::set_chat_profile_image(ctx, chat_id, to_string_lossy(image))
+                .map(|_| 1)
+                .unwrap_or_log_default(ctx, "Failed to set profile image")
         })
         .unwrap_or(0)
 }
@@ -1210,7 +1246,7 @@ pub unsafe extern "C" fn dc_get_msg_info(
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| message::get_msg_info(ctx, msg_id).strdup())
+        .with_inner(|ctx| message::get_msg_info(ctx, MsgId::new(msg_id)).strdup())
         .unwrap_or_else(|_| ptr::null_mut())
 }
 
@@ -1226,7 +1262,7 @@ pub unsafe extern "C" fn dc_get_mime_headers(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            message::get_mime_headers(ctx, msg_id)
+            message::get_mime_headers(ctx, MsgId::new(msg_id))
                 .map(|s| s.strdup())
                 .unwrap_or_else(|| ptr::null_mut())
         })
@@ -1244,11 +1280,22 @@ pub unsafe extern "C" fn dc_delete_msgs(
         return;
     }
     let ffi_context = &*context;
-
     let ids = std::slice::from_raw_parts(msg_ids, msg_cnt as usize);
-
+    let msg_ids: Vec<MsgId> = ids.iter().map(|id| MsgId::new(*id)).collect();
     ffi_context
-        .with_inner(|ctx| message::delete_msgs(ctx, ids))
+        .with_inner(|ctx| message::delete_msgs(ctx, &msg_ids[..]))
+        .unwrap_or(())
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_empty_server(context: *mut dc_context_t, flags: u32) {
+    if context.is_null() || flags == 0 {
+        eprintln!("ignoring careless call to dc_empty_server()");
+        return;
+    }
+    let ffi_context = &*context;
+    ffi_context
+        .with_inner(|ctx| message::dc_empty_server(ctx, flags))
         .unwrap_or(())
 }
 
@@ -1268,11 +1315,11 @@ pub unsafe extern "C" fn dc_forward_msgs(
         return;
     }
     let ids = std::slice::from_raw_parts(msg_ids, msg_cnt as usize);
-
+    let msg_ids: Vec<MsgId> = ids.iter().map(|id| MsgId::new(*id)).collect();
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            chat::forward_msgs(ctx, ids, chat_id)
+            chat::forward_msgs(ctx, &msg_ids[..], chat_id)
                 .unwrap_or_log_default(ctx, "Failed to forward message")
         })
         .unwrap_or_default()
@@ -1301,10 +1348,14 @@ pub unsafe extern "C" fn dc_markseen_msgs(
         return;
     }
     let ids = std::slice::from_raw_parts(msg_ids, msg_cnt as usize);
-
+    let msg_ids: Vec<MsgId> = ids
+        .iter()
+        .filter(|id| **id > DC_MSG_ID_LAST_SPECIAL)
+        .map(|id| MsgId::new(*id))
+        .collect();
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| message::markseen_msgs(ctx, ids))
+        .with_inner(|ctx| message::markseen_msgs(ctx, &msg_ids[..]))
         .ok();
 }
 
@@ -1319,12 +1370,11 @@ pub unsafe extern "C" fn dc_star_msgs(
         eprintln!("ignoring careless call to dc_star_msgs()");
         return;
     }
-
     let ids = std::slice::from_raw_parts(msg_ids, msg_cnt as usize);
-
+    let msg_ids: Vec<MsgId> = ids.iter().map(|id| MsgId::new(*id)).collect();
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| message::star_msgs(ctx, ids, star == 1))
+        .with_inner(|ctx| message::star_msgs(ctx, &msg_ids[..], star == 1))
         .ok();
 }
 
@@ -1337,7 +1387,7 @@ pub unsafe extern "C" fn dc_get_msg(context: *mut dc_context_t, msg_id: u32) ->
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            let message = match message::Message::load_from_db(ctx, msg_id) {
+            let message = match message::Message::load_from_db(ctx, MsgId::new(msg_id)) {
                 Ok(msg) => msg,
                 Err(e) => {
                     error!(ctx, "Error getting msg #{}: {}", msg_id, e);
@@ -1357,7 +1407,7 @@ pub unsafe extern "C" fn dc_may_be_valid_addr(addr: *const libc::c_char) -> libc
         return 0;
     }
 
-    contact::may_be_valid_addr(as_str(addr)) as libc::c_int
+    contact::may_be_valid_addr(&to_string_lossy(addr)) as libc::c_int
 }
 
 #[no_mangle]
@@ -1371,7 +1421,7 @@ pub unsafe extern "C" fn dc_lookup_contact_id_by_addr(
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| Contact::lookup_id_by_addr(ctx, as_str(addr)))
+        .with_inner(|ctx| Contact::lookup_id_by_addr(ctx, to_string_lossy(addr)))
         .unwrap_or(0)
 }
 
@@ -1386,12 +1436,14 @@ pub unsafe extern "C" fn dc_create_contact(
         return 0;
     }
     let ffi_context = &*context;
-    let name = if name.is_null() { "" } else { as_str(name) };
+    let name = to_string_lossy(name);
     ffi_context
-        .with_inner(|ctx| match Contact::create(ctx, name, as_str(addr)) {
-            Ok(id) => id,
-            Err(_) => 0,
-        })
+        .with_inner(
+            |ctx| match Contact::create(ctx, name, to_string_lossy(addr)) {
+                Ok(id) => id,
+                Err(_) => 0,
+            },
+        )
         .unwrap_or(0)
 }
 
@@ -1407,7 +1459,7 @@ pub unsafe extern "C" fn dc_add_address_book(
     let ffi_context = &*context;
     ffi_context
         .with_inner(
-            |ctx| match Contact::add_address_book(ctx, as_str(addr_book)) {
+            |ctx| match Contact::add_address_book(ctx, to_string_lossy(addr_book)) {
                 Ok(cnt) => cnt as libc::c_int,
                 Err(_) => 0,
             },
@@ -1426,11 +1478,7 @@ pub unsafe extern "C" fn dc_get_contacts(
         return ptr::null_mut();
     }
     let ffi_context = &*context;
-    let query = if query.is_null() {
-        None
-    } else {
-        Some(as_str(query))
-    };
+    let query = to_opt_string_lossy(query);
     ffi_context
         .with_inner(|ctx| match Contact::get_all(ctx, flags, query) {
             Ok(contacts) => Box::into_raw(Box::new(dc_array_t::from(contacts))),
@@ -1567,7 +1615,7 @@ pub unsafe extern "C" fn dc_imex(
 
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| imex::imex(ctx, what, as_opt_str(param1)))
+        .with_inner(|ctx| imex::imex(ctx, what, to_opt_string_lossy(param1)))
         .ok();
 }
 
@@ -1582,7 +1630,7 @@ pub unsafe extern "C" fn dc_imex_has_backup(
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| match imex::has_backup(ctx, as_str(dir)) {
+        .with_inner(|ctx| match imex::has_backup(ctx, to_string_lossy(dir)) {
             Ok(res) => res.strdup(),
             Err(err) => {
                 error!(ctx, "dc_imex_has_backup: {}", err);
@@ -1625,15 +1673,16 @@ pub unsafe extern "C" fn dc_continue_key_transfer(
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(
-            |ctx| match imex::continue_key_transfer(ctx, msg_id, as_str(setup_code)) {
+        .with_inner(|ctx| {
+            match imex::continue_key_transfer(ctx, MsgId::new(msg_id), &to_string_lossy(setup_code))
+            {
                 Ok(()) => 1,
                 Err(err) => {
                     error!(ctx, "dc_continue_key_transfer: {}", err);
                     0
                 }
-            },
-        )
+            }
+        })
         .unwrap_or(0)
 }
 
@@ -1659,7 +1708,7 @@ pub unsafe extern "C" fn dc_check_qr(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            let lot = qr::check_qr(ctx, as_str(qr));
+            let lot = qr::check_qr(ctx, to_string_lossy(qr));
             Box::into_raw(Box::new(lot))
         })
         .unwrap_or_else(|_| ptr::null_mut())
@@ -1695,7 +1744,7 @@ pub unsafe extern "C" fn dc_join_securejoin(
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| securejoin::dc_join_securejoin(ctx, as_str(qr)))
+        .with_inner(|ctx| securejoin::dc_join_securejoin(ctx, &to_string_lossy(qr)))
         .unwrap_or(0)
 }
 
@@ -1744,7 +1793,7 @@ pub unsafe extern "C" fn dc_set_location(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| location::set(ctx, latitude, longitude, accuracy))
-        .unwrap_or(0)
+        .unwrap_or(false) as _
 }
 
 #[no_mangle]
@@ -2037,7 +2086,11 @@ pub unsafe extern "C" fn dc_chatlist_get_msg_id(
         return 0;
     }
     let ffi_list = &*chatlist;
-    ffi_list.list.get_msg_id(index as usize)
+    ffi_list
+        .list
+        .get_msg_id(index as usize)
+        .map(|msg_id| msg_id.to_u32())
+        .unwrap_or(0)
 }
 
 #[no_mangle]
@@ -2279,7 +2332,7 @@ pub unsafe extern "C" fn dc_msg_get_id(msg: *mut dc_msg_t) -> u32 {
         return 0;
     }
     let ffi_msg = &*msg;
-    ffi_msg.message.get_id()
+    ffi_msg.message.get_id().to_u32()
 }
 
 #[no_mangle]
@@ -2610,8 +2663,7 @@ pub unsafe extern "C" fn dc_msg_set_text(msg: *mut dc_msg_t, text: *const libc::
         return;
     }
     let ffi_msg = &mut *msg;
-    // TODO: {text} equal to NULL is treated as "", which is strange. Does anyone rely on it?
-    ffi_msg.message.set_text(as_opt_str(text).map(Into::into))
+    ffi_msg.message.set_text(to_opt_string_lossy(text))
 }
 
 #[no_mangle]
@@ -2625,7 +2677,10 @@ pub unsafe extern "C" fn dc_msg_set_file(
         return;
     }
     let ffi_msg = &mut *msg;
-    ffi_msg.message.set_file(as_str(file), as_opt_str(filemime))
+    ffi_msg.message.set_file(
+        to_string_lossy(file),
+        to_opt_string_lossy(filemime).as_ref().map(|x| x.as_str()),
+    )
 }
 
 #[no_mangle]
@@ -2920,14 +2975,6 @@ pub unsafe extern "C" fn dc_str_unref(s: *mut libc::c_char) {
     libc::free(s as *mut _)
 }
 
-fn as_opt_str<'a>(s: *const libc::c_char) -> Option<&'a str> {
-    if s.is_null() {
-        return None;
-    }
-
-    Some(as_str(s))
-}
-
 pub mod providers;
 
 pub trait ResultExt<T> {

examples/repl/cmdline.rs

@@ -14,7 +14,7 @@ use deltachat::imex::*;
 use deltachat::job::*;
 use deltachat::location;
 use deltachat::lot::LotState;
-use deltachat::message::{self, Message, MessageState};
+use deltachat::message::{self, Message, MessageState, MsgId};
 use deltachat::peerstate::*;
 use deltachat::qr::*;
 use deltachat::sql;
@@ -86,7 +86,7 @@ pub unsafe fn dc_reset_tables(context: &Context, bits: i32) -> i32 {
 
     context.call_cb(Event::MsgsChanged {
         chat_id: 0,
-        msg_id: 0,
+        msg_id: MsgId::new(0),
     });
 
     1
@@ -170,7 +170,7 @@ fn poke_spec(context: &Context, spec: *const libc::c_char) -> libc::c_int {
     if read_cnt > 0 {
         context.call_cb(Event::MsgsChanged {
             chat_id: 0,
-            msg_id: 0,
+            msg_id: MsgId::new(0),
         });
     }
     1
@@ -192,9 +192,9 @@ unsafe fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
     let msgtext = msg.get_text();
     info!(
         context,
-        "{}#{}{}{}: {} (Contact#{}): {} {}{}{}{} [{}]",
+        "{}#{}{}{}: {} (Contact#{}): {} {}{}{}{}{} [{}]",
         prefix.as_ref(),
-        msg.get_id() as libc::c_int,
+        msg.get_id(),
         if msg.get_showpadlock() { "🔒" } else { "" },
         if msg.has_location() { "📍" } else { "" },
         &contact_name,
@@ -211,22 +211,27 @@ unsafe fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
             "[FRESH]"
         },
         if msg.is_info() { "[INFO]" } else { "" },
+        if msg.is_forwarded() {
+            "[FORWARDED]"
+        } else {
+            ""
+        },
         statestr,
         &temp2,
     );
 }
 
-unsafe fn log_msglist(context: &Context, msglist: &Vec<u32>) -> Result<(), Error> {
+unsafe fn log_msglist(context: &Context, msglist: &Vec<MsgId>) -> Result<(), Error> {
     let mut lines_out = 0;
     for &msg_id in msglist {
-        if msg_id == 9 as libc::c_uint {
+        if msg_id.is_daymarker() {
             info!(
                 context,
                 "--------------------------------------------------------------------------------"
             );
 
             lines_out += 1
-        } else if msg_id > 0 {
+        } else if !msg_id.is_special() {
             if lines_out == 0 {
                 info!(
                     context,
@@ -399,6 +404,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
                  checkqr <qr-content>\n\
                  event <event-id to test>\n\
                  fileinfo <file>\n\
+                 emptyserver <flags> (1=MVBOX 2=INBOX)\n\
                  clear -- clear screen\n\
                  exit or quit\n\
                  ============================================="
@@ -413,7 +419,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
         },
         "get-setupcodebegin" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let msg_id: u32 = arg1.parse()?;
+            let msg_id: MsgId = MsgId::new(arg1.parse()?);
             let msg = Message::load_from_db(context, msg_id)?;
             if msg.is_setupmessage() {
                 let setupcodebegin = msg.get_setupcodebegin(context);
@@ -431,7 +437,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
                 !arg1.is_empty() && !arg2.is_empty(),
                 "Arguments <msg-id> <setup-code> expected"
             );
-            continue_key_transfer(context, arg1.parse()?, &arg2)?;
+            continue_key_transfer(context, MsgId::new(arg1.parse()?), &arg2)?;
         }
         "has-backup" => {
             has_backup(context, blobdir)?;
@@ -576,7 +582,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
             ensure!(sel_chat.is_some(), "Failed to select chat");
             let sel_chat = sel_chat.as_ref().unwrap();
 
-            let msglist = chat::get_chat_msgs(context, sel_chat.get_id(), 0x1, 0);
+            let msglist = chat::get_chat_msgs(context, sel_chat.get_id(), 0x1, None);
             let temp2 = sel_chat.get_subtitle(context);
             let temp_name = sel_chat.get_name();
             info!(
@@ -612,7 +618,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
         }
         "createchatbymsg" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing");
-            let msg_id: u32 = arg1.parse()?;
+            let msg_id = MsgId::new(arg1.parse()?);
             let chat_id = chat::create_by_msg_id(context, msg_id)?;
             let chat = Chat::load_from_db(context, chat_id)?;
 
@@ -741,7 +747,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
             let longitude = arg2.parse()?;
 
             let continue_streaming = location::set(context, latitude, longitude, 0.);
-            if 0 != continue_streaming {
+            if continue_streaming {
                 println!("Success, streaming should be continued.");
             } else {
                 println!("Success, streaming can be stoppped.");
@@ -844,7 +850,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
         }
         "msginfo" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let id = arg1.parse()?;
+            let id = MsgId::new(arg1.parse()?);
             let res = message::get_msg_info(context, id);
             println!("{}", res);
         }
@@ -860,27 +866,27 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
                 "Arguments <msg-id> <chat-id> expected"
             );
 
-            let mut msg_ids = [0; 1];
+            let mut msg_ids = [MsgId::new(0); 1];
             let chat_id = arg2.parse()?;
-            msg_ids[0] = arg1.parse()?;
+            msg_ids[0] = MsgId::new(arg1.parse()?);
             chat::forward_msgs(context, &msg_ids, chat_id)?;
         }
         "markseen" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let mut msg_ids = [0; 1];
-            msg_ids[0] = arg1.parse()?;
+            let mut msg_ids = [MsgId::new(0); 1];
+            msg_ids[0] = MsgId::new(arg1.parse()?);
             message::markseen_msgs(context, &msg_ids);
         }
         "star" | "unstar" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let mut msg_ids = [0; 1];
-            msg_ids[0] = arg1.parse()?;
+            let mut msg_ids = [MsgId::new(0); 1];
+            msg_ids[0] = MsgId::new(arg1.parse()?);
             message::star_msgs(context, &msg_ids, arg0 == "star");
         }
         "delmsg" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let mut ids = [0; 1];
-            ids[0] = arg1.parse()?;
+            let mut ids = [MsgId::new(0); 1];
+            ids[0] = MsgId::new(arg1.parse()?);
             message::delete_msgs(context, &ids);
         }
         "listcontacts" | "contacts" | "listverified" => {
@@ -971,6 +977,11 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
                 bail!("Command failed.");
             }
         }
+        "emptyserver" => {
+            ensure!(!arg1.is_empty(), "Argument <flags> missing");
+
+            message::dc_empty_server(context, arg1.parse()?);
+        }
         "" => (),
         _ => bail!("Unknown command: \"{}\" type ? for help.", arg0),
     }

python/src/deltachat/account.py

@@ -135,6 +135,17 @@ class Account(object):
         if not self.is_configured():
             raise ValueError("need to configure first")
 
+    def empty_server_folders(self, inbox=False, mvbox=False):
+        """ empty server folders. """
+        flags = 0
+        if inbox:
+            flags |= const.DC_EMPTY_INBOX
+        if mvbox:
+            flags |= const.DC_EMPTY_MVBOX
+        if not flags:
+            raise ValueError("no flags set")
+        lib.dc_empty_server(self._dc_context, flags)
+
     def get_infostring(self):
         """ return info of the configured account. """
         self.check_is_configured()

python/src/deltachat/const.py

@@ -57,12 +57,27 @@ DC_MSG_AUDIO = 40
 DC_MSG_VOICE = 41
 DC_MSG_VIDEO = 50
 DC_MSG_FILE = 60
+DC_LP_AUTH_OAUTH2 = 0x2
+DC_LP_AUTH_NORMAL = 0x4
+DC_LP_IMAP_SOCKET_STARTTLS = 0x100
+DC_LP_IMAP_SOCKET_SSL = 0x200
+DC_LP_IMAP_SOCKET_PLAIN = 0x400
+DC_LP_SMTP_SOCKET_STARTTLS = 0x10000
+DC_LP_SMTP_SOCKET_SSL = 0x20000
+DC_LP_SMTP_SOCKET_PLAIN = 0x40000
+DC_CERTCK_AUTO = 0
+DC_CERTCK_STRICT = 1
+DC_CERTCK_ACCEPT_INVALID_HOSTNAMES = 2
+DC_CERTCK_ACCEPT_INVALID_CERTIFICATES = 3
+DC_EMPTY_MVBOX = 0x01
+DC_EMPTY_INBOX = 0x02
 DC_EVENT_INFO = 100
 DC_EVENT_SMTP_CONNECTED = 101
 DC_EVENT_IMAP_CONNECTED = 102
 DC_EVENT_SMTP_MESSAGE_SENT = 103
 DC_EVENT_IMAP_MESSAGE_DELETED = 104
 DC_EVENT_IMAP_MESSAGE_MOVED = 105
+DC_EVENT_IMAP_FOLDER_EMPTIED = 106
 DC_EVENT_NEW_BLOB_FILE = 150
 DC_EVENT_DELETED_BLOB_FILE = 151
 DC_EVENT_WARNING = 300
@@ -82,9 +97,9 @@ DC_EVENT_IMEX_PROGRESS = 2051
 DC_EVENT_IMEX_FILE_WRITTEN = 2052
 DC_EVENT_SECUREJOIN_INVITER_PROGRESS = 2060
 DC_EVENT_SECUREJOIN_JOINER_PROGRESS = 2061
-DC_EVENT_GET_STRING = 2091
 DC_EVENT_FILE_COPIED = 2055
 DC_EVENT_IS_OFFLINE = 2081
+DC_EVENT_GET_STRING = 2091
 DC_STR_SELFNOTINGRP = 21
 DC_PROVIDER_STATUS_OK = 1
 DC_PROVIDER_STATUS_PREPARATION = 2
@@ -139,8 +154,8 @@ DC_STR_COUNT = 67
 
 
 def read_event_defines(f):
-    rex = re.compile(r'#define\s+((?:DC_EVENT_|DC_QR|DC_MSG|DC_STATE_|DC_STR|'
-                     r'DC_CONTACT_ID_|DC_GCL|DC_CHAT|DC_PROVIDER)\S+)\s+([x\d]+).*')
+    rex = re.compile(r'#define\s+((?:DC_EVENT|DC_QR|DC_MSG|DC_LP|DC_EMPTY|DC_CERTCK|DC_STATE|DC_STR|'
+                     r'DC_CONTACT_ID|DC_GCL|DC_CHAT|DC_PROVIDER)_\S+)\s+([x\d]+).*')
     for line in f:
         m = rex.match(line)
         if m:

python/src/deltachat/message.py

@@ -109,6 +109,10 @@ class Message(object):
         """ return True if this message was encrypted. """
         return bool(lib.dc_msg_get_showpadlock(self._dc_msg))
 
+    def is_forwarded(self):
+        """ return True if this message was forwarded. """
+        return bool(lib.dc_msg_is_forwarded(self._dc_msg))
+
     def get_message_info(self):
         """ Return informational text for a single message.
 

python/tests/conftest.py

@@ -4,6 +4,7 @@ import pytest
 import requests
 import time
 from deltachat import Account
+from deltachat import const
 from deltachat.capi import lib
 import tempfile
 
@@ -150,6 +151,11 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig):
             lib.dc_set_config(ac._dc_context, b"configured", b"1")
             return ac
 
+        def peek_online_config(self):
+            if not session_liveconfig:
+                pytest.skip("specify DCC_PY_LIVECONFIG or --liveconfig")
+            return session_liveconfig.get(self.live_count)
+
         def get_online_config(self):
             if not session_liveconfig:
                 pytest.skip("specify DCC_PY_LIVECONFIG or --liveconfig")
@@ -159,8 +165,8 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig):
                 configdict["e2ee_enabled"] = "1"
 
             # Enable strict certificate checks for online accounts
-            configdict["imap_certificate_checks"] = "1"
-            configdict["smtp_certificate_checks"] = "1"
+            configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+            configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
 
             tmpdb = tmpdir.join("livedb%d" % self.live_count)
             ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
@@ -174,6 +180,12 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig):
             ac.start_threads(mvbox=mvbox, sentbox=sentbox)
             return ac
 
+        def get_one_online_account(self):
+            ac1 = self.get_online_configuring_account()
+            wait_successful_IMAP_SMTP_connection(ac1)
+            wait_configuration_progress(ac1, 1000)
+            return ac1
+
         def get_two_online_accounts(self):
             ac1 = self.get_online_configuring_account()
             ac2 = self.get_online_configuring_account()

python/tests/test_account.py

@@ -365,10 +365,12 @@ class TestOfflineChat:
 
 
 class TestOnlineAccount:
-    def get_chat(self, ac1, ac2):
+    def get_chat(self, ac1, ac2, both_created=False):
         c2 = ac1.create_contact(email=ac2.get_config("addr"))
         chat = ac1.create_chat_by_contact(c2)
         assert chat.id > const.DC_CHAT_ID_LAST_SPECIAL
+        if both_created:
+            ac2.create_chat_by_contact(ac2.create_contact(email=ac1.get_config("addr")))
         return chat
 
     def test_configure_canceled(self, acfactory):
@@ -389,7 +391,8 @@ class TestOnlineAccount:
 
     def test_one_account_send_bcc_setting(self, acfactory, lp):
         ac1 = acfactory.get_online_configuring_account()
-        c2 = ac1.create_contact(email="notexists@testrun.org")
+        ac2_config = acfactory.peek_online_config()
+        c2 = ac1.create_contact(email=ac2_config["addr"])
         chat = ac1.create_chat_by_contact(c2)
         assert chat.id > const.DC_CHAT_ID_LAST_SPECIAL
         wait_successful_IMAP_SMTP_connection(ac1)
@@ -410,6 +413,7 @@ class TestOnlineAccount:
         lp.sec("send out message without bcc")
         ac1.set_config("bcc_self", "0")
         msg_out = chat.send_text("message3")
+        assert not msg_out.is_forwarded()
         ev = ac1._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
         assert ev[2] == msg_out.id
         ev = ac1._evlogger.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
@@ -452,6 +456,7 @@ class TestOnlineAccount:
         # check the message arrived in contact-requests/deaddrop
         chat2 = msg_in.chat
         assert msg_in in chat2.get_messages()
+        assert not msg_in.is_forwarded()
         assert chat2.is_deaddrop()
         assert chat2 == ac2.get_deaddrop_chat()
         chat3 = ac2.create_group_chat("newgroup")
@@ -459,9 +464,47 @@ class TestOnlineAccount:
         ac2.forward_messages([msg_in], chat3)
         assert chat3.is_promoted()
         messages = chat3.get_messages()
+        msg = messages[-1]
+        assert msg.is_forwarded()
         ac2.delete_messages(messages)
         assert not chat3.get_messages()
 
+    def test_forward_own_message(self, acfactory, lp):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        chat = self.get_chat(ac1, ac2, both_created=True)
+
+        lp.sec("sending message")
+        msg_out = chat.send_text("message2")
+
+        lp.sec("receiving message")
+        ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG")
+        msg_in = ac2.get_message_by_id(ev[2])
+        assert msg_in.text == "message2"
+        assert not msg_in.is_forwarded()
+
+        lp.sec("ac1: creating group chat, and forward own message")
+        group = ac1.create_group_chat("newgroup2")
+        group.add_contact(ac1.create_contact(ac2.get_config("addr")))
+        ac1.forward_messages([msg_out], group)
+
+        # wait for other account to receive
+        ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG")
+        msg_in = ac2.get_message_by_id(ev[2])
+        assert msg_in.text == "message2"
+        assert msg_in.is_forwarded()
+
+    def test_send_self_message_and_empty_folder(self, acfactory, lp):
+        ac1 = acfactory.get_one_online_account()
+        lp.sec("ac1: create self chat")
+        chat = ac1.create_chat_by_contact(ac1.get_self_contact())
+        chat.send_text("hello")
+        ac1._evlogger.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
+        ac1.empty_server_folders(inbox=True, mvbox=True)
+        ev = ac1._evlogger.get_matching("DC_EVENT_IMAP_FOLDER_EMPTIED")
+        assert ev[2] == "DeltaChat"
+        ev = ac1._evlogger.get_matching("DC_EVENT_IMAP_FOLDER_EMPTIED")
+        assert ev[2] == "INBOX"
+
     def test_send_and_receive_message_markseen(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
@@ -481,6 +524,7 @@ class TestOnlineAccount:
         assert ev[2] == msg_out.id
         msg_in = ac2.get_message_by_id(msg_out.id)
         assert msg_in.text == "message1"
+        assert not msg_in.is_forwarded()
 
         lp.sec("check the message arrived in contact-requets/deaddrop")
         chat2 = msg_in.chat
@@ -638,6 +682,27 @@ class TestOnlineAccount:
         msg.continue_key_transfer(setup_code)
         assert ac1.get_info()["fingerprint"] == ac2.get_info()["fingerprint"]
 
+    def test_ac_setup_message_twice(self, acfactory, lp):
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.clone_online_account(ac1)
+        ac2._evlogger.set_timeout(30)
+        wait_configuration_progress(ac2, 1000)
+        wait_configuration_progress(ac1, 1000)
+        lp.sec("trigger ac setup message but ignore")
+        assert ac1.get_info()["fingerprint"] != ac2.get_info()["fingerprint"]
+        ac1.initiate_key_transfer()
+        ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
+
+        lp.sec("trigger second ac setup message, wait for receive ")
+        setup_code2 = ac1.initiate_key_transfer()
+        ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
+        msg = ac2.get_message_by_id(ev[2])
+        assert msg.is_setup_message()
+        assert msg.get_setupcodebegin() == setup_code2[:2]
+        lp.sec("process second setup message")
+        msg.continue_key_transfer(setup_code2)
+        assert ac1.get_info()["fingerprint"] == ac2.get_info()["fingerprint"]
+
     def test_qr_setup_contact(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
         lp.sec("ac1: create QR code and let ac2 scan it, starting the securejoin")

run-integration-tests.sh

@@ -23,7 +23,7 @@ if [ $? != 0 ]; then
 fi
 
 pushd python
-if [ -e "./liveconfig" && -z "$DCC_PY_LIVECONFIG" ]; then
+if [ -e "./liveconfig" -a -z "$DCC_PY_LIVECONFIG" ]; then
     export DCC_PY_LIVECONFIG=liveconfig
 fi
 tox "$@"

spec.md

@@ -334,9 +334,7 @@ only on image changes.
 
 # Miscellaneous
 
-Messengers SHOULD use the header `Chat-Predecessor`
-instead of `In-Reply-To` as the latter one results
-in infinite threads on typical MUAs.
+Messengers SHOULD use the header `In-Reply-To` as usual.
 
 Messengers SHOULD add a `Chat-Voice-message: 1` header
 if an attached audio file is a voice message.
@@ -346,7 +344,7 @@ to specify the duration of attached audio or video files.
 The value MUST be the duration in milliseconds.
 This allows the receiver to show the time without knowing the file format.
 
-    Chat-Predecessor: foo123@domain
+    In-Reply-To: Gr.12345uvwxyZ.0005@domain
     Chat-Voice-Message: 1
     Chat-Duration: 10000
 

src/blob.rs

@@ -0,0 +1,618 @@
+use std::fmt;
+use std::fs;
+use std::io::Write;
+use std::path::{Path, PathBuf};
+
+use crate::context::Context;
+use crate::events::Event;
+
+/// Represents a file in the blob directory.
+///
+/// The object has a name, which will always be valid UTF-8.  Having a
+/// blob object does not imply the respective file exists, however
+/// when using one of the `create*()` methods a unique file is
+/// created.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct BlobObject<'a> {
+    blobdir: &'a Path,
+    name: String,
+}
+
+impl<'a> BlobObject<'a> {
+    /// Creates a new blob object with a unique name.
+    ///
+    /// Creates a new file in the blob directory.  The name will be
+    /// derived from the platform-agnostic basename of the suggested
+    /// name, followed by a random number and followed by a possible
+    /// extension.  The `data` will be written into the file.
+    ///
+    /// # Errors
+    ///
+    /// [BlobErrorKind::CreateFailure] is used when the file could not
+    /// be created.  You can expect [BlobError.cause] to contain an
+    /// underlying error.
+    ///
+    /// [BlobErrorKind::WriteFailure] is used when the file could not
+    /// be written to.  You can expect [BlobError.cause] to contain an
+    /// underlying error.
+    pub fn create(
+        context: &'a Context,
+        suggested_name: impl AsRef<str>,
+        data: &[u8],
+    ) -> std::result::Result<BlobObject<'a>, BlobError> {
+        let blobdir = context.get_blobdir();
+        let (stem, ext) = BlobObject::sanitise_name(suggested_name.as_ref().to_string());
+        let mut name = format!("{}{}", stem, ext);
+        let max_attempt = 15;
+        for attempt in 0..max_attempt {
+            let path = blobdir.join(&name);
+            match fs::OpenOptions::new()
+                .create_new(true)
+                .write(true)
+                .open(&path)
+            {
+                Ok(mut file) => {
+                    file.write_all(data)
+                        .map_err(|err| BlobError::new_write_failure(blobdir, &name, err))?;
+                    let blob = BlobObject {
+                        blobdir,
+                        name: format!("$BLOBDIR/{}", name),
+                    };
+                    context.call_cb(Event::NewBlobFile(blob.as_name().to_string()));
+                    return Ok(blob);
+                }
+                Err(err) => {
+                    if attempt == max_attempt {
+                        return Err(BlobError::new_create_failure(blobdir, &name, err));
+                    } else {
+                        name = format!("{}-{}{}", stem, rand::random::<u32>(), ext);
+                    }
+                }
+            }
+        }
+        Err(BlobError::new_create_failure(
+            blobdir,
+            &name,
+            format_err!("Unreachable code - supposedly"),
+        ))
+    }
+
+    /// Creates a new blob object with unique name by copying an existing file.
+    ///
+    /// This creates a new blob as described in [BlobObject::create]
+    /// but also copies an existing file into it.
+    ///
+    /// # Errors
+    ///
+    /// In addition to the errors in [BlobObject::create] the
+    /// [BlobErrorKind::CopyFailure] is used when the data can not be
+    /// copied.
+    pub fn create_and_copy(
+        context: &'a Context,
+        src: impl AsRef<Path>,
+    ) -> std::result::Result<BlobObject<'a>, BlobError> {
+        let blob = BlobObject::create(context, src.as_ref().to_string_lossy(), b"")?;
+        fs::copy(src.as_ref(), blob.to_abs_path()).map_err(|err| {
+            fs::remove_file(blob.to_abs_path()).ok();
+            BlobError::new_copy_failure(blob.blobdir, &blob.name, src.as_ref(), err)
+        })?;
+        Ok(blob)
+    }
+
+    /// Creates a blob from a file, possibly copying it to the blobdir.
+    ///
+    /// If the source file is not a path to into the blob directory
+    /// the file will be copied into the blob directory first.  If the
+    /// source file is already in the blobdir it will not be copied
+    /// and only be created if it is a valid blobname, that is no
+    /// subdirectory is used and [BlobObject::sanitise_name] does not
+    /// modify the filename.
+    ///
+    /// # Errors
+    ///
+    /// This merely delegates to the [BlobObject::create_and_copy] and
+    /// the [BlobObject::from_path] methods.  See those for possible
+    /// errors.
+    pub fn create_from_path(
+        context: &Context,
+        src: impl AsRef<Path>,
+    ) -> std::result::Result<BlobObject, BlobError> {
+        match src.as_ref().starts_with(context.get_blobdir()) {
+            true => BlobObject::from_path(context, src),
+            false => BlobObject::create_and_copy(context, src),
+        }
+    }
+
+    /// Returns a [BlobObject] for an existing blob from a path.
+    ///
+    /// The path must designate a file directly in the blobdir and
+    /// must use a valid blob name.  That is after sanitisation the
+    /// name must still be the same, that means it must be valid UTF-8
+    /// and not have any special characters in it.
+    ///
+    /// # Errors
+    ///
+    /// [BlobErrorKind::WrongBlobdir] is used if the path is not in
+    /// the blob directory.
+    ///
+    /// [BlobErrorKind::WrongName] is used if the file name does not
+    /// remain identical after sanitisation.
+    pub fn from_path(
+        context: &Context,
+        path: impl AsRef<Path>,
+    ) -> std::result::Result<BlobObject, BlobError> {
+        let rel_path = path
+            .as_ref()
+            .strip_prefix(context.get_blobdir())
+            .map_err(|_| BlobError::new_wrong_blobdir(context.get_blobdir(), path.as_ref()))?;
+        let name = rel_path
+            .to_str()
+            .ok_or_else(|| BlobError::new_wrong_name(path.as_ref()))?;
+        BlobObject::from_name(context, name.to_string())
+    }
+
+    /// Returns a [BlobObject] for an existing blob.
+    ///
+    /// The `name` may optionally be prefixed with the `$BLOBDIR/`
+    /// prefixed, as returned by [BlobObject::as_name].  This is how
+    /// you want to create a [BlobObject] for a filename read from the
+    /// database.
+    ///
+    /// # Errors
+    ///
+    /// [BlobErrorKind::WrongName] is used if the name is not a valid
+    /// blobname, i.e. if [BlobObject::sanitise_name] does modify the
+    /// provided name.
+    pub fn from_name(
+        context: &'a Context,
+        name: String,
+    ) -> std::result::Result<BlobObject<'a>, BlobError> {
+        let name: String = match name.starts_with("$BLOBDIR/") {
+            true => name.splitn(2, '/').last().unwrap().to_string(),
+            false => name,
+        };
+        let (stem, ext) = BlobObject::sanitise_name(name.clone());
+        if format!("{}{}", stem, ext) != name.as_ref() {
+            return Err(BlobError::new_wrong_name(name));
+        }
+        Ok(BlobObject {
+            blobdir: context.get_blobdir(),
+            name: format!("$BLOBDIR/{}", name),
+        })
+    }
+
+    /// Returns the absolute path to the blob in the filesystem.
+    pub fn to_abs_path(&self) -> PathBuf {
+        let fname = Path::new(&self.name).strip_prefix("$BLOBDIR/").unwrap();
+        self.blobdir.join(fname)
+    }
+
+    /// Returns the blob name, as stored in the database.
+    ///
+    /// This returns the blob in the `$BLOBDIR/<name>` format used in
+    /// the database.  Do not use this unless you're about to store
+    /// this string in the database or [Params].  Eventually even
+    /// those conversions should be handled by the type system.
+    ///
+    /// [Params]: crate::param::Params
+    pub fn as_name(&self) -> &str {
+        &self.name
+    }
+
+    /// Returns the filename of the blob.
+    pub fn as_file_name(&self) -> &str {
+        self.name.rsplitn(2, '/').next().unwrap()
+    }
+
+    /// The path relative in the blob directory.
+    pub fn as_rel_path(&self) -> &Path {
+        Path::new(self.as_file_name())
+    }
+
+    /// Returns the extension of the blob.
+    ///
+    /// If a blob's filename has an extension, it is always guaranteed
+    /// to be lowercase.
+    pub fn suffix(&self) -> Option<&str> {
+        let ext = self.name.rsplitn(2, '.').next();
+        if ext == Some(&self.name) {
+            None
+        } else {
+            ext
+        }
+    }
+
+    /// Create a safe name based on a messy input string.
+    ///
+    /// The safe name will be a valid filename on Unix and Windows and
+    /// not contain any path separators.  The input can contain path
+    /// segments separated by either Unix or Windows path separators,
+    /// the rightmost non-empty segment will be used as name,
+    /// sanitised for special characters.
+    ///
+    /// The resulting name is returned as a tuple, the first part
+    /// being the stem or basename and the second being an extension,
+    /// including the dot.  E.g. "foo.txt" is returned as `("foo",
+    /// ".txt")` while "bar" is returned as `("bar", "")`.
+    ///
+    /// The extension part will always be lowercased.
+    fn sanitise_name(mut name: String) -> (String, String) {
+        for part in name.rsplit('/') {
+            if part.len() > 0 {
+                name = part.to_string();
+                break;
+            }
+        }
+        for part in name.rsplit('\\') {
+            if part.len() > 0 {
+                name = part.to_string();
+                break;
+            }
+        }
+        let opts = sanitize_filename::Options {
+            truncate: true,
+            windows: true,
+            replacement: "",
+        };
+
+        let clean = sanitize_filename::sanitize_with_options(name, opts);
+        let mut iter = clean.rsplitn(2, '.');
+        let mut ext = iter.next().unwrap_or_default().to_string();
+        let mut stem = iter.next().unwrap_or_default().to_string();
+        ext.truncate(32);
+        stem.truncate(64);
+        match stem.len() {
+            0 => (ext, "".to_string()),
+            _ => (stem, format!(".{}", ext).to_lowercase()),
+        }
+    }
+}
+
+impl<'a> fmt::Display for BlobObject<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "$BLOBDIR/{}", self.name)
+    }
+}
+
+/// Errors for the [BlobObject].
+///
+/// To keep the return type small and thus the happy path fast this
+/// stores everything on the heap.
+#[derive(Debug)]
+pub struct BlobError {
+    inner: Box<BlobErrorInner>,
+}
+
+#[derive(Debug)]
+struct BlobErrorInner {
+    kind: BlobErrorKind,
+    data: BlobErrorData,
+    backtrace: failure::Backtrace,
+}
+
+/// Error kind for [BlobError].
+///
+/// Each error kind has associated data in the [BlobErrorData].
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum BlobErrorKind {
+    /// Failed to create the blob.
+    CreateFailure,
+    /// Failed to write data to blob.
+    WriteFailure,
+    /// Failed to copy data to blob.
+    CopyFailure,
+    /// Blob is not in the blobdir.
+    WrongBlobdir,
+    /// Blob has a bad name.
+    ///
+    /// E.g. the name is not sanitised correctly or contains a
+    /// sub-directory.
+    WrongName,
+}
+
+/// Associated data for each [BlobError] error kind.
+///
+/// This is not stored directly on the [BlobErrorKind] so that the
+/// kind can stay trivially Copy and Eq.  It is however possible to
+/// create a [BlobError] with mismatching [BlobErrorKind] and
+/// [BlobErrorData], don't do that.
+///
+/// Any blobname stored here is the bare name, without the `$BLOBDIR`
+/// prefix.  All data is owned so that errors do not need to be tied
+/// to any lifetimes.
+#[derive(Debug)]
+enum BlobErrorData {
+    CreateFailure {
+        blobdir: PathBuf,
+        blobname: String,
+        cause: failure::Error,
+    },
+    WriteFailure {
+        blobdir: PathBuf,
+        blobname: String,
+        cause: failure::Error,
+    },
+    CopyFailure {
+        blobdir: PathBuf,
+        blobname: String,
+        src: PathBuf,
+        cause: failure::Error,
+    },
+    WrongBlobdir {
+        blobdir: PathBuf,
+        src: PathBuf,
+    },
+    WrongName {
+        blobname: PathBuf,
+    },
+}
+
+impl BlobError {
+    pub fn kind(&self) -> BlobErrorKind {
+        self.inner.kind
+    }
+
+    fn new_create_failure(
+        blobdir: impl Into<PathBuf>,
+        blobname: impl Into<String>,
+        cause: impl Into<failure::Error>,
+    ) -> BlobError {
+        BlobError {
+            inner: Box::new(BlobErrorInner {
+                kind: BlobErrorKind::CreateFailure,
+                data: BlobErrorData::CreateFailure {
+                    blobdir: blobdir.into(),
+                    blobname: blobname.into(),
+                    cause: cause.into(),
+                },
+                backtrace: failure::Backtrace::new(),
+            }),
+        }
+    }
+
+    fn new_write_failure(
+        blobdir: impl Into<PathBuf>,
+        blobname: impl Into<String>,
+        cause: impl Into<failure::Error>,
+    ) -> BlobError {
+        BlobError {
+            inner: Box::new(BlobErrorInner {
+                kind: BlobErrorKind::WriteFailure,
+                data: BlobErrorData::WriteFailure {
+                    blobdir: blobdir.into(),
+                    blobname: blobname.into(),
+                    cause: cause.into(),
+                },
+                backtrace: failure::Backtrace::new(),
+            }),
+        }
+    }
+
+    fn new_copy_failure(
+        blobdir: impl Into<PathBuf>,
+        blobname: impl Into<String>,
+        src: impl Into<PathBuf>,
+        cause: impl Into<failure::Error>,
+    ) -> BlobError {
+        BlobError {
+            inner: Box::new(BlobErrorInner {
+                kind: BlobErrorKind::CopyFailure,
+                data: BlobErrorData::CopyFailure {
+                    blobdir: blobdir.into(),
+                    blobname: blobname.into(),
+                    src: src.into(),
+                    cause: cause.into(),
+                },
+                backtrace: failure::Backtrace::new(),
+            }),
+        }
+    }
+
+    fn new_wrong_blobdir(blobdir: impl Into<PathBuf>, src: impl Into<PathBuf>) -> BlobError {
+        BlobError {
+            inner: Box::new(BlobErrorInner {
+                kind: BlobErrorKind::WrongBlobdir,
+                data: BlobErrorData::WrongBlobdir {
+                    blobdir: blobdir.into(),
+                    src: src.into(),
+                },
+                backtrace: failure::Backtrace::new(),
+            }),
+        }
+    }
+
+    fn new_wrong_name(blobname: impl Into<PathBuf>) -> BlobError {
+        BlobError {
+            inner: Box::new(BlobErrorInner {
+                kind: BlobErrorKind::WrongName,
+                data: BlobErrorData::WrongName {
+                    blobname: blobname.into(),
+                },
+                backtrace: failure::Backtrace::new(),
+            }),
+        }
+    }
+}
+
+impl fmt::Display for BlobError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        // Match on the data rather than kind, they are equivalent for
+        // identifying purposes but contain the actual data we need.
+        match &self.inner.data {
+            BlobErrorData::CreateFailure {
+                blobdir, blobname, ..
+            } => write!(
+                f,
+                "Failed to create blob {} in {}",
+                blobname,
+                blobdir.display()
+            ),
+            BlobErrorData::WriteFailure {
+                blobdir, blobname, ..
+            } => write!(
+                f,
+                "Failed to write data to blob {} in {}",
+                blobname,
+                blobdir.display()
+            ),
+            BlobErrorData::CopyFailure {
+                blobdir,
+                blobname,
+                src,
+                ..
+            } => write!(
+                f,
+                "Failed to copy data from {} to blob {} in {}",
+                src.display(),
+                blobname,
+                blobdir.display(),
+            ),
+            BlobErrorData::WrongBlobdir { blobdir, src } => write!(
+                f,
+                "File path {} is not in blobdir {}",
+                src.display(),
+                blobdir.display(),
+            ),
+            BlobErrorData::WrongName { blobname } => {
+                write!(f, "Blob has a bad name: {}", blobname.display(),)
+            }
+        }
+    }
+}
+
+impl failure::Fail for BlobError {
+    fn cause(&self) -> Option<&dyn failure::Fail> {
+        match &self.inner.data {
+            BlobErrorData::CreateFailure { cause, .. }
+            | BlobErrorData::WriteFailure { cause, .. }
+            | BlobErrorData::CopyFailure { cause, .. } => Some(cause.as_fail()),
+            _ => None,
+        }
+    }
+
+    fn backtrace(&self) -> Option<&failure::Backtrace> {
+        Some(&self.inner.backtrace)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use crate::test_utils::*;
+
+    #[test]
+    fn test_create() {
+        let t = dummy_context();
+        let blob = BlobObject::create(&t.ctx, "foo", b"hello").unwrap();
+        let fname = t.ctx.get_blobdir().join("foo");
+        let data = fs::read(fname).unwrap();
+        assert_eq!(data, b"hello");
+        assert_eq!(blob.as_name(), "$BLOBDIR/foo");
+        assert_eq!(blob.to_abs_path(), t.ctx.get_blobdir().join("foo"));
+    }
+
+    #[test]
+    fn test_lowercase_ext() {
+        let t = dummy_context();
+        let blob = BlobObject::create(&t.ctx, "foo.TXT", b"hello").unwrap();
+        assert_eq!(blob.as_name(), "$BLOBDIR/foo.txt");
+    }
+
+    #[test]
+    fn test_as_file_name() {
+        let t = dummy_context();
+        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
+        assert_eq!(blob.as_file_name(), "foo.txt");
+    }
+
+    #[test]
+    fn test_as_rel_path() {
+        let t = dummy_context();
+        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
+        assert_eq!(blob.as_rel_path(), Path::new("foo.txt"));
+    }
+
+    #[test]
+    fn test_suffix() {
+        let t = dummy_context();
+        let foo = BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
+        assert_eq!(foo.suffix(), Some("txt"));
+        let bar = BlobObject::create(&t.ctx, "bar", b"world").unwrap();
+        assert_eq!(bar.suffix(), None);
+    }
+
+    #[test]
+    fn test_create_dup() {
+        let t = dummy_context();
+        BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
+        let foo = t.ctx.get_blobdir().join("foo.txt");
+        assert!(foo.exists());
+        BlobObject::create(&t.ctx, "foo.txt", b"world").unwrap();
+        for dirent in fs::read_dir(t.ctx.get_blobdir()).unwrap() {
+            let fname = dirent.unwrap().file_name();
+            if fname == foo.file_name().unwrap() {
+                assert_eq!(fs::read(&foo).unwrap(), b"hello");
+            } else {
+                let name = fname.to_str().unwrap();
+                assert!(name.starts_with("foo"));
+                assert!(name.ends_with(".txt"));
+            }
+        }
+    }
+
+    #[test]
+    fn test_create_long_names() {
+        let t = dummy_context();
+        let s = "1".repeat(150);
+        let blob = BlobObject::create(&t.ctx, &s, b"data").unwrap();
+        let blobname = blob.as_name().split('/').last().unwrap();
+        assert!(blobname.len() < 128);
+    }
+
+    #[test]
+    fn test_create_and_copy() {
+        let t = dummy_context();
+        let src = t.dir.path().join("src");
+        fs::write(&src, b"boo").unwrap();
+        let blob = BlobObject::create_and_copy(&t.ctx, &src).unwrap();
+        assert_eq!(blob.as_name(), "$BLOBDIR/src");
+        let data = fs::read(blob.to_abs_path()).unwrap();
+        assert_eq!(data, b"boo");
+
+        let whoops = t.dir.path().join("whoops");
+        assert!(BlobObject::create_and_copy(&t.ctx, &whoops).is_err());
+        let whoops = t.ctx.get_blobdir().join("whoops");
+        assert!(!whoops.exists());
+    }
+
+    #[test]
+    fn test_create_from_path() {
+        let t = dummy_context();
+
+        let src_ext = t.dir.path().join("external");
+        fs::write(&src_ext, b"boo").unwrap();
+        let blob = BlobObject::create_from_path(&t.ctx, &src_ext).unwrap();
+        assert_eq!(blob.as_name(), "$BLOBDIR/external");
+        let data = fs::read(blob.to_abs_path()).unwrap();
+        assert_eq!(data, b"boo");
+
+        let src_int = t.ctx.get_blobdir().join("internal");
+        fs::write(&src_int, b"boo").unwrap();
+        let blob = BlobObject::create_from_path(&t.ctx, &src_int).unwrap();
+        assert_eq!(blob.as_name(), "$BLOBDIR/internal");
+        let data = fs::read(blob.to_abs_path()).unwrap();
+        assert_eq!(data, b"boo");
+    }
+    #[test]
+    fn test_create_from_name_long() {
+        let t = dummy_context();
+        let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
+        fs::write(&src_ext, b"boo").unwrap();
+        let blob = BlobObject::create_from_path(&t.ctx, &src_ext).unwrap();
+        assert_eq!(
+            blob.as_name(),
+            "$BLOBDIR/autocrypt-setup-message-4137848473.html"
+        );
+    }
+}

src/chat.rs

@@ -1,5 +1,8 @@
 use std::path::{Path, PathBuf};
 
+use itertools::Itertools;
+
+use crate::blob::{BlobErrorKind, BlobObject};
 use crate::chatlist::*;
 use crate::config::*;
 use crate::constants::*;
@@ -10,7 +13,7 @@ use crate::dc_tools::*;
 use crate::error::Error;
 use crate::events::Event;
 use crate::job::*;
-use crate::message::{self, Message, MessageState};
+use crate::message::{self, InvalidMsgId, Message, MessageState, MsgId};
 use crate::param::*;
 use crate::sql::{self, Sql};
 use crate::stock::StockMessage;
@@ -237,7 +240,7 @@ impl Chat {
         context: &Context,
         msg: &mut Message,
         timestamp: i64,
-    ) -> Result<u32, Error> {
+    ) -> Result<MsgId, Error> {
         let mut do_guarantee_e2ee: bool;
         let e2ee_enabled: bool;
         let mut new_references = "".into();
@@ -251,7 +254,7 @@ impl Chat {
             || self.typ == Chattype::VerifiedGroup)
         {
             error!(context, "Cannot send to chat type #{}.", self.typ,);
-            return Ok(0);
+            bail!("Cannot set to chat type #{}", self.typ);
         }
 
         if (self.typ == Chattype::Group || self.typ == Chattype::VerifiedGroup)
@@ -261,7 +264,7 @@ impl Chat {
                 context,
                 Event::ErrorSelfNotInGroup("Cannot send message; self not in group.".into())
             );
-            return Ok(0);
+            bail!("Cannot set message; self not in group.");
         }
 
         if let Some(from) = context.get_config(Config::ConfiguredAddr) {
@@ -285,7 +288,10 @@ impl Chat {
                         context,
                         "Cannot send message, contact for chat #{} not found.", self.id,
                     );
-                    return Ok(0);
+                    bail!(
+                        "Cannot set message, contact for chat #{} not found.",
+                        self.id
+                    );
                 }
             } else {
                 if self.typ == Chattype::Group || self.typ == Chattype::VerifiedGroup {
@@ -360,7 +366,7 @@ impl Chat {
                 }
             }
             if do_guarantee_e2ee {
-                msg.param.set_int(Param::GuranteeE2ee, 1);
+                msg.param.set_int(Param::GuaranteeE2ee, 1);
             }
             // reset eg. for forwarding
             msg.param.remove(Param::ErroneousE2ee);
@@ -471,7 +477,7 @@ impl Chat {
             error!(context, "Cannot send message, not configured.",);
         }
 
-        Ok(msg_id)
+        Ok(MsgId::new(msg_id))
     }
 }
 
@@ -493,7 +499,7 @@ impl Chat {
 /// to the message and, depending on the contacts origin, messages from the
 /// same group may be shown or not - so, all in all, it is fine to show the
 /// contact name only.
-pub fn create_by_msg_id(context: &Context, msg_id: u32) -> Result<u32, Error> {
+pub fn create_by_msg_id(context: &Context, msg_id: MsgId) -> Result<u32, Error> {
     let mut chat_id = 0;
     let mut send_event = false;
 
@@ -513,7 +519,7 @@ pub fn create_by_msg_id(context: &Context, msg_id: u32) -> Result<u32, Error> {
     if send_event {
         context.call_cb(Event::MsgsChanged {
             chat_id: 0,
-            msg_id: 0,
+            msg_id: MsgId::new(0),
         });
     }
 
@@ -556,7 +562,7 @@ pub fn create_by_contact_id(context: &Context, contact_id: u32) -> Result<u32, E
 
     context.call_cb(Event::MsgsChanged {
         chat_id: 0,
-        msg_id: 0,
+        msg_id: MsgId::new(0),
     });
 
     Ok(chat_id)
@@ -642,7 +648,7 @@ pub fn prepare_msg<'a>(
     context: &'a Context,
     chat_id: u32,
     msg: &mut Message,
-) -> Result<u32, Error> {
+) -> Result<MsgId, Error> {
     ensure!(
         chat_id > DC_CHAT_ID_LAST_SPECIAL,
         "Cannot prepare message for special chat"
@@ -671,32 +677,16 @@ pub fn msgtype_has_file(msgtype: Viewtype) -> bool {
     }
 }
 
-fn prepare_msg_common(context: &Context, chat_id: u32, msg: &mut Message) -> Result<u32, Error> {
-    msg.id = 0;
-
+fn prepare_msg_common(context: &Context, chat_id: u32, msg: &mut Message) -> Result<MsgId, Error> {
+    msg.id = MsgId::new_unset();
     if msg.type_0 == Viewtype::Text {
         // the caller should check if the message text is empty
     } else if msgtype_has_file(msg.type_0) {
-        let path_filename = msg.param.get(Param::File);
-
-        ensure!(
-            path_filename.is_some(),
-            "Attachment missing for message of type #{}.",
-            msg.type_0
-        );
-
-        let mut path_filename = path_filename.unwrap().to_string();
-
-        if msg.state == MessageState::OutPreparing && !dc_is_blobdir_path(context, &path_filename) {
-            bail!("Files must be created in the blob-directory.");
-        }
-
-        ensure!(
-            dc_make_rel_and_copy(context, &mut path_filename),
-            "Failed to copy"
-        );
-
-        msg.param.set(Param::File, &path_filename);
+        let blob = msg
+            .param
+            .get_blob(Param::File, context, !msg.is_increation())?
+            .ok_or_else(|| format_err!("Attachment missing for message of type #{}", msg.type_0))?;
+        msg.param.set(Param::File, blob.as_name());
         if msg.type_0 == Viewtype::File || msg.type_0 == Viewtype::Image {
             // Correct the type, take care not to correct already very special
             // formats as GIF or VOICE.
@@ -705,19 +695,21 @@ fn prepare_msg_common(context: &Context, chat_id: u32, msg: &mut Message) -> Res
             // - from FILE to AUDIO/VIDEO/IMAGE
             // - from FILE/IMAGE to GIF */
             if let Some((better_type, better_mime)) =
-                message::guess_msgtype_from_suffix(Path::new(&path_filename))
+                message::guess_msgtype_from_suffix(&blob.to_abs_path())
             {
                 msg.type_0 = better_type;
                 msg.param.set(Param::MimeType, better_mime);
             }
         } else if !msg.param.exists(Param::MimeType) {
-            if let Some((_, mime)) = message::guess_msgtype_from_suffix(Path::new(&path_filename)) {
+            if let Some((_, mime)) = message::guess_msgtype_from_suffix(&blob.to_abs_path()) {
                 msg.param.set(Param::MimeType, mime);
             }
         }
         info!(
             context,
-            "Attaching \"{}\" for message type #{}.", &path_filename, msg.type_0
+            "Attaching \"{}\" for message type #{}.",
+            blob.to_abs_path().display(),
+            msg.type_0
         );
     } else {
         bail!("Cannot send messages of type #{}.", msg.type_0);
@@ -726,6 +718,11 @@ fn prepare_msg_common(context: &Context, chat_id: u32, msg: &mut Message) -> Res
     unarchive(context, chat_id)?;
 
     let mut chat = Chat::load_from_db(context, chat_id)?;
+
+    // The OutPreparing state is set by dc_prepare_msg() before it
+    // calls this function and the message is left in the OutPreparing
+    // state.  Otherwise we got called by send_msg() and we change the
+    // state to OutPending.
     if msg.state != MessageState::OutPreparing {
         msg.state = MessageState::OutPending;
     }
@@ -747,7 +744,7 @@ fn last_msg_in_chat_encrypted(context: &Context, sql: &Sql, chat_id: u32) -> boo
 
     if let Some(ref packed) = packed {
         match packed.parse::<Params>() {
-            Ok(param) => param.exists(Param::GuranteeE2ee),
+            Ok(param) => param.exists(Param::GuaranteeE2ee),
             Err(err) => {
                 error!(context, "invalid params stored: '{}', {:?}", packed, err);
                 false
@@ -787,7 +784,11 @@ pub fn unarchive(context: &Context, chat_id: u32) -> Result<(), Error> {
 /// However, this does not imply, the message really reached the recipient -
 /// sending may be delayed eg. due to network problems. However, from your
 /// view, you're done with the message. Sooner or later it will find its way.
-pub fn send_msg(context: &Context, chat_id: u32, msg: &mut Message) -> Result<u32, Error> {
+pub fn send_msg(context: &Context, chat_id: u32, msg: &mut Message) -> Result<MsgId, Error> {
+    // dc_prepare_msg() leaves the message state to OutPreparing, we
+    // only have to change the state to OutPending in this case.
+    // Otherwise we still have to prepare the message, which will set
+    // the state to OutPending.
     if msg.state != MessageState::OutPreparing {
         // automatically prepare normal messages
         prepare_msg_common(context, chat_id, msg)?;
@@ -815,12 +816,17 @@ pub fn send_msg(context: &Context, chat_id: u32, msg: &mut Message) -> Result<u3
         let forwards = msg.param.get(Param::PrepForwards);
         if let Some(forwards) = forwards {
             for forward in forwards.split(' ') {
-                let id: i32 = forward.parse().unwrap_or_default();
-                if 0 == id {
-                    // avoid hanging if user tampers with db
-                    break;
-                } else if let Ok(mut copy) = Message::load_from_db(context, id as u32) {
-                    send_msg(context, 0, &mut copy)?;
+                match forward
+                    .parse::<u32>()
+                    .map_err(|_| InvalidMsgId)
+                    .map(|id| MsgId::new(id))
+                {
+                    Ok(msg_id) => {
+                        if let Ok(mut msg) = Message::load_from_db(context, msg_id) {
+                            send_msg(context, 0, &mut msg)?;
+                        };
+                    }
+                    Err(_) => (),
                 }
             }
             msg.param.remove(Param::PrepForwards);
@@ -831,7 +837,11 @@ pub fn send_msg(context: &Context, chat_id: u32, msg: &mut Message) -> Result<u3
     Ok(msg.id)
 }
 
-pub fn send_text_msg(context: &Context, chat_id: u32, text_to_send: String) -> Result<u32, Error> {
+pub fn send_text_msg(
+    context: &Context,
+    chat_id: u32,
+    text_to_send: String,
+) -> Result<MsgId, Error> {
     ensure!(
         chat_id > DC_CHAT_ID_LAST_SPECIAL,
         "bad chat_id = {} <= 9",
@@ -855,7 +865,10 @@ pub fn set_draft(context: &Context, chat_id: u32, msg: Option<&mut Message>) {
     };
 
     if changed {
-        context.call_cb(Event::MsgsChanged { chat_id, msg_id: 0 });
+        context.call_cb(Event::MsgsChanged {
+            chat_id,
+            msg_id: MsgId::new(0),
+        });
     }
 }
 
@@ -863,39 +876,37 @@ pub fn set_draft(context: &Context, chat_id: u32, msg: Option<&mut Message>) {
 ///
 /// Return {true}, if message was deleted, {false} otherwise.
 fn maybe_delete_draft(context: &Context, chat_id: u32) -> bool {
-    let draft = get_draft_msg_id(context, chat_id);
-    if draft != 0 {
-        Message::delete_from_db(context, draft);
-        return true;
+    match get_draft_msg_id(context, chat_id) {
+        Some(msg_id) => {
+            Message::delete_from_db(context, msg_id);
+            true
+        }
+        None => false,
     }
-    false
 }
 
 /// Set provided message as draft message for specified chat.
 ///
 /// Return true on success, false on database error.
-fn do_set_draft(context: &Context, chat_id: u32, msg: &mut Message) -> bool {
+fn do_set_draft(context: &Context, chat_id: u32, msg: &mut Message) -> Result<(), Error> {
     match msg.type_0 {
-        Viewtype::Unknown => return false,
-        Viewtype::Text => {
-            if msg.text.as_ref().map_or(false, |s| s.is_empty()) {
-                return false;
+        Viewtype::Unknown => bail!("Can not set draft of unknown type."),
+        Viewtype::Text => match msg.text.as_ref() {
+            Some(text) => {
+                if text.is_empty() {
+                    bail!("No text in draft");
+                }
             }
-        }
+            None => bail!("No text in draft"),
+        },
         _ => {
-            if let Some(path_filename) = msg.param.get(Param::File) {
-                let mut path_filename = path_filename.to_string();
-                if msg.is_increation() && !dc_is_blobdir_path(context, &path_filename) {
-                    return false;
-                }
-                if !dc_make_rel_and_copy(context, &mut path_filename) {
-                    return false;
-                }
-                msg.param.set(Param::File, path_filename);
-            }
+            let blob = msg
+                .param
+                .get_blob(Param::File, context, !msg.is_increation())?
+                .ok_or_else(|| format_err!("No file stored in params"))?;
+            msg.param.set(Param::File, blob.as_name());
         }
     }
-
     sql::execute(
         context,
         &context.sql,
@@ -912,109 +923,124 @@ fn do_set_draft(context: &Context, chat_id: u32, msg: &mut Message) -> bool {
             1,
         ],
     )
-    .is_ok()
 }
 
 // similar to as dc_set_draft() but does not emit an event
 fn set_draft_raw(context: &Context, chat_id: u32, msg: &mut Message) -> bool {
     let deleted = maybe_delete_draft(context, chat_id);
-    let set = do_set_draft(context, chat_id, msg);
+    let set = do_set_draft(context, chat_id, msg).is_ok();
 
     // Can't inline. Both functions above must be called, no shortcut!
     deleted || set
 }
 
-fn get_draft_msg_id(context: &Context, chat_id: u32) -> u32 {
-    context
-        .sql
-        .query_get_value::<_, i32>(
-            context,
-            "SELECT id FROM msgs WHERE chat_id=? AND state=?;",
-            params![chat_id as i32, MessageState::OutDraft],
-        )
-        .unwrap_or_default() as u32
+fn get_draft_msg_id(context: &Context, chat_id: u32) -> Option<MsgId> {
+    context.sql.query_get_value::<_, MsgId>(
+        context,
+        "SELECT id FROM msgs WHERE chat_id=? AND state=?;",
+        params![chat_id as i32, MessageState::OutDraft],
+    )
 }
 
 pub fn get_draft(context: &Context, chat_id: u32) -> Result<Option<Message>, Error> {
     if chat_id <= DC_CHAT_ID_LAST_SPECIAL {
         return Ok(None);
     }
-    let draft_msg_id = get_draft_msg_id(context, chat_id);
-    if draft_msg_id == 0 {
-        return Ok(None);
+    match get_draft_msg_id(context, chat_id) {
+        Some(draft_msg_id) => Ok(Some(Message::load_from_db(context, draft_msg_id)?)),
+        None => Ok(None),
     }
-    Ok(Some(Message::load_from_db(context, draft_msg_id)?))
 }
 
-pub fn get_chat_msgs(context: &Context, chat_id: u32, flags: u32, marker1before: u32) -> Vec<u32> {
-    let mut ret = Vec::new();
-
-    let mut last_day = 0;
-    let cnv_to_local = dc_gm2local_offset();
-
-    let process_row = |row: &rusqlite::Row| Ok((row.get::<_, i32>(0)?, row.get::<_, i64>(1)?));
+pub fn get_chat_msgs(
+    context: &Context,
+    chat_id: u32,
+    flags: u32,
+    marker1before: Option<MsgId>,
+) -> Vec<MsgId> {
+    let process_row =
+        |row: &rusqlite::Row| Ok((row.get::<_, MsgId>("id")?, row.get::<_, i64>("timestamp")?));
     let process_rows = |rows: rusqlite::MappedRows<_>| {
+        let mut ret = Vec::new();
+        let mut last_day = 0;
+        let cnv_to_local = dc_gm2local_offset();
         for row in rows {
             let (curr_id, ts) = row?;
-            if curr_id as u32 == marker1before {
-                ret.push(DC_MSG_ID_MARKER1);
+            if let Some(marker_id) = marker1before {
+                if curr_id == marker_id {
+                    ret.push(MsgId::new(DC_MSG_ID_MARKER1));
+                }
             }
-            if 0 != flags & 0x1 {
+            if (flags & DC_GCM_ADDDAYMARKER) != 0 {
                 let curr_local_timestamp = ts + cnv_to_local;
                 let curr_day = curr_local_timestamp / 86400;
                 if curr_day != last_day {
-                    ret.push(DC_MSG_ID_LAST_SPECIAL);
+                    ret.push(MsgId::new(DC_MSG_ID_DAYMARKER));
                     last_day = curr_day;
                 }
             }
-            ret.push(curr_id as u32);
+            ret.push(curr_id);
         }
-        Ok(())
+        Ok(ret)
     };
-
-    let success = if chat_id == 1 {
+    let success = if chat_id == DC_CHAT_ID_DEADDROP {
         let show_emails = context.get_config_int(Config::ShowEmails);
         context.sql.query_map(
-            "SELECT m.id, m.timestamp FROM msgs m \
-             LEFT JOIN chats ON m.chat_id=chats.id \
-             LEFT JOIN contacts ON m.from_id=contacts.id WHERE m.from_id!=1   \
-             AND m.from_id!=2   \
-             AND m.hidden=0    \
-             AND chats.blocked=2   \
-             AND contacts.blocked=0   \
-             AND m.msgrmsg>=?  \
-             ORDER BY m.timestamp,m.id;",
+            concat!(
+                "SELECT m.id AS id, m.timestamp AS timestamp",
+                " FROM msgs m",
+                " LEFT JOIN chats",
+                "        ON m.chat_id=chats.id",
+                " LEFT JOIN contacts",
+                "        ON m.from_id=contacts.id",
+                " WHERE m.from_id!=1",
+                "   AND m.from_id!=2",
+                "   AND m.hidden=0",
+                "   AND chats.blocked=2",
+                "   AND contacts.blocked=0",
+                "   AND m.msgrmsg>=?",
+                " ORDER BY m.timestamp,m.id;"
+            ),
             params![if show_emails == 2 { 0 } else { 1 }],
             process_row,
             process_rows,
         )
-    } else if chat_id == 5 {
+    } else if chat_id == DC_CHAT_ID_STARRED {
         context.sql.query_map(
-            "SELECT m.id, m.timestamp FROM msgs m \
-             LEFT JOIN contacts ct ON m.from_id=ct.id WHERE m.starred=1    \
-             AND m.hidden=0    \
-             AND ct.blocked=0 \
-             ORDER BY m.timestamp,m.id;",
+            concat!(
+                "SELECT m.id AS id, m.timestamp AS timestamp",
+                " FROM msgs m",
+                " LEFT JOIN contacts ct",
+                "        ON m.from_id=ct.id",
+                " WHERE m.starred=1",
+                "   AND m.hidden=0",
+                "   AND ct.blocked=0",
+                " ORDER BY m.timestamp,m.id;"
+            ),
             params![],
             process_row,
             process_rows,
         )
     } else {
         context.sql.query_map(
-            "SELECT m.id, m.timestamp FROM msgs m \
-             WHERE m.chat_id=?    \
-             AND m.hidden=0  \
-             ORDER BY m.timestamp,m.id;",
+            concat!(
+                "SELECT m.id AS id, m.timestamp AS timestamp",
+                " FROM msgs m",
+                " WHERE m.chat_id=?",
+                "   AND m.hidden=0",
+                " ORDER BY m.timestamp, m.id;"
+            ),
             params![chat_id as i32],
             process_row,
             process_rows,
         )
     };
-
-    if success.is_ok() {
-        ret
-    } else {
-        Vec::new()
+    match success {
+        Ok(ret) => ret,
+        Err(e) => {
+            error!(context, "Failed to get chat messages: {}", e);
+            Vec::new()
+        }
     }
 }
 
@@ -1061,7 +1087,7 @@ pub fn marknoticed_chat(context: &Context, chat_id: u32) -> Result<(), Error> {
 
     context.call_cb(Event::MsgsChanged {
         chat_id: 0,
-        msg_id: 0,
+        msg_id: MsgId::new(0),
     });
 
     Ok(())
@@ -1085,7 +1111,7 @@ pub fn marknoticed_all_chats(context: &Context) -> Result<(), Error> {
     )?;
 
     context.call_cb(Event::MsgsChanged {
-        msg_id: 0,
+        msg_id: MsgId::new(0),
         chat_id: 0,
     });
 
@@ -1098,31 +1124,44 @@ pub fn get_chat_media(
     msg_type: Viewtype,
     msg_type2: Viewtype,
     msg_type3: Viewtype,
-) -> Vec<u32> {
-    context.sql.query_map(
-        "SELECT id FROM msgs WHERE chat_id=? AND (type=? OR type=? OR type=?) ORDER BY timestamp, id;",
-        params![
-            chat_id as i32,
-            msg_type,
-            if msg_type2 != Viewtype::Unknown {
-                msg_type2
-            } else {
-                msg_type
-            }, if msg_type3 != Viewtype::Unknown {
-                msg_type3
-            } else {
-                msg_type
+) -> Vec<MsgId> {
+    context
+        .sql
+        .query_map(
+            concat!(
+                "SELECT",
+                "    id",
+                " FROM msgs",
+                " WHERE chat_id=? AND (type=? OR type=? OR type=?)",
+                " ORDER BY timestamp, id;"
+            ),
+            params![
+                chat_id as i32,
+                msg_type,
+                if msg_type2 != Viewtype::Unknown {
+                    msg_type2
+                } else {
+                    msg_type
+                },
+                if msg_type3 != Viewtype::Unknown {
+                    msg_type3
+                } else {
+                    msg_type
+                },
+            ],
+            |row| row.get::<_, MsgId>(0),
+            |ids| {
+                let mut ret = Vec::new();
+                for id in ids {
+                    match id {
+                        Ok(msg_id) => ret.push(msg_id),
+                        Err(_) => (),
+                    }
+                }
+                Ok(ret)
             },
-        ],
-        |row| row.get::<_, i32>(0),
-        |ids| {
-            let mut ret = Vec::new();
-            for id in ids {
-                ret.push(id? as u32);
-            }
-            Ok(ret)
-        }
-    ).unwrap_or_default()
+        )
+        .unwrap_or_default()
 }
 
 /// Indicates the direction over which to iterate.
@@ -1135,16 +1174,16 @@ pub enum Direction {
 
 pub fn get_next_media(
     context: &Context,
-    curr_msg_id: u32,
+    curr_msg_id: MsgId,
     direction: Direction,
     msg_type: Viewtype,
     msg_type2: Viewtype,
     msg_type3: Viewtype,
-) -> u32 {
-    let mut ret = 0;
+) -> Option<MsgId> {
+    let mut ret: Option<MsgId> = None;
 
     if let Ok(msg) = Message::load_from_db(context, curr_msg_id) {
-        let list = get_chat_media(
+        let list: Vec<MsgId> = get_chat_media(
             context,
             msg.chat_id,
             if msg_type != Viewtype::Unknown {
@@ -1160,12 +1199,12 @@ pub fn get_next_media(
                 match direction {
                     Direction::Forward => {
                         if i + 1 < list.len() {
-                            ret = list[i + 1]
+                            ret = Some(list[i + 1]);
                         }
                     }
                     Direction::Backward => {
                         if i >= 1 {
-                            ret = list[i - 1];
+                            ret = Some(list[i - 1]);
                         }
                     }
                 }
@@ -1204,7 +1243,7 @@ pub fn archive(context: &Context, chat_id: u32, archive: bool) -> Result<(), Err
     )?;
 
     context.call_cb(Event::MsgsChanged {
-        msg_id: 0,
+        msg_id: MsgId::new(0),
         chat_id: 0,
     });
 
@@ -1249,7 +1288,7 @@ pub fn delete(context: &Context, chat_id: u32) -> Result<(), Error> {
     )?;
 
     context.call_cb(Event::MsgsChanged {
-        msg_id: 0,
+        msg_id: MsgId::new(0),
         chat_id: 0,
     });
 
@@ -1318,7 +1357,7 @@ pub fn create_group_chat(
         }
 
         context.call_cb(Event::MsgsChanged {
-            msg_id: 0,
+            msg_id: MsgId::new(0),
             chat_id: 0,
         });
     }
@@ -1374,8 +1413,8 @@ pub(crate) fn add_contact_to_chat_ex(
         chat_id
     );
     ensure!(
-        Contact::real_exists_by_id(context, contact_id) && contact_id != DC_CONTACT_ID_SELF,
-        "invalid contact_id {} for removal in group",
+        Contact::real_exists_by_id(context, contact_id) || contact_id == DC_CONTACT_ID_SELF,
+        "invalid contact_id {} for adding to group",
         contact_id
     );
 
@@ -1395,10 +1434,14 @@ pub(crate) fn add_contact_to_chat_ex(
         .get_config(Config::ConfiguredAddr)
         .unwrap_or_default();
     if contact.get_addr() == &self_addr {
-        bail!("invalid attempt to add self e-mail address to group");
+        // ourself is added using DC_CONTACT_ID_SELF, do not add this address explicitly.
+        // if SELF is not in the group, members cannot be added at all.
+        warn!(
+            context,
+            "invalid attempt to add self e-mail address to group"
+        );
+        return Ok(false);
     }
-    // ourself is added using DC_CONTACT_ID_SELF, do not add it explicitly.
-    // if SELF is not in the group, members cannot be added at all.
 
     if is_contact_in_chat(context, chat_id, contact_id) {
         if !from_handshake {
@@ -1427,16 +1470,19 @@ pub(crate) fn add_contact_to_chat_ex(
             "",
             DC_CONTACT_ID_SELF as u32,
         ));
-        msg.param.set_int(Param::Cmd, 4);
+        msg.param.set_cmd(SystemMessage::MemberAddedToGroup);
         msg.param.set(Param::Arg, contact.get_addr());
         msg.param.set_int(Param::Arg2, from_handshake.into());
         msg.id = send_msg(context, chat_id, &mut msg)?;
         context.call_cb(Event::MsgsChanged {
             chat_id,
-            msg_id: msg.id,
+            msg_id: MsgId::from(msg.id),
         });
     }
-    context.call_cb(Event::MsgsChanged { chat_id, msg_id: 0 });
+    context.call_cb(Event::MsgsChanged {
+        chat_id,
+        msg_id: MsgId::new(0),
+    });
     Ok(true)
 }
 
@@ -1539,7 +1585,7 @@ pub fn remove_contact_from_chat(
                                 DC_CONTACT_ID_SELF,
                             ));
                         }
-                        msg.param.set_int(Param::Cmd, 5);
+                        msg.param.set_cmd(SystemMessage::MemberRemovedFromGroup);
                         msg.param.set(Param::Arg, contact.get_addr());
                         msg.id = send_msg(context, chat_id, &mut msg)?;
                         context.call_cb(Event::MsgsChanged {
@@ -1634,7 +1680,7 @@ pub fn set_chat_name(
                         new_name.as_ref(),
                         DC_CONTACT_ID_SELF,
                     ));
-                    msg.param.set_int(Param::Cmd, 2);
+                    msg.param.set_cmd(SystemMessage::GroupNameChanged);
                     if !chat.name.is_empty() {
                         msg.param.set(Param::Arg, &chat.name);
                     }
@@ -1657,6 +1703,11 @@ pub fn set_chat_name(
     Ok(())
 }
 
+/// Set a new profile image for the chat.
+///
+/// The profile image can only be set when you are a member of the
+/// chat.  To remove the profile image pass an empty string for the
+/// `new_image` parameter.
 #[allow(non_snake_case)]
 pub fn set_chat_profile_image(
     context: &Context,
@@ -1664,126 +1715,121 @@ pub fn set_chat_profile_image(
     new_image: impl AsRef<str>, // XXX use PathBuf
 ) -> Result<(), Error> {
     ensure!(chat_id > DC_CHAT_ID_LAST_SPECIAL, "Invalid chat ID");
-
     let mut chat = Chat::load_from_db(context, chat_id)?;
-
-    if real_group_exists(context, chat_id) {
-        /* we should respect this - whatever we send to the group, it gets discarded anyway! */
-        if !is_contact_in_chat(context, chat_id, DC_CONTACT_ID_SELF) {
-            emit_event!(
-                context,
-                Event::ErrorSelfNotInGroup(
-                    "Cannot set chat profile image; self not in group.".into()
-                )
-            );
-            bail!("Failed to set profile image");
-        }
-        let mut new_image_rel: String;
-        if !new_image.as_ref().is_empty() {
-            new_image_rel = new_image.as_ref().to_string();
-            if !dc_make_rel_and_copy(context, &mut new_image_rel) {
-                bail!("Failed to get relative path for profile image");
-            }
-        } else {
-            new_image_rel = "".to_string();
-        }
-
-        chat.param.set(Param::ProfileImage, &new_image_rel);
-        if chat.update_param(context).is_ok() {
-            if chat.is_promoted() {
-                let mut msg = Message::default();
-                msg.param
-                    .set_int(Param::Cmd, SystemMessage::GroupImageChanged as i32);
-                msg.type_0 = Viewtype::Text;
-                msg.text = Some(context.stock_system_msg(
-                    if new_image_rel == "" {
-                        msg.param.remove(Param::Arg);
-                        StockMessage::MsgGrpImgDeleted
-                    } else {
-                        msg.param.set(Param::Arg, &new_image_rel);
-                        StockMessage::MsgGrpImgChanged
-                    },
-                    "",
-                    "",
-                    DC_CONTACT_ID_SELF,
-                ));
-                msg.id = send_msg(context, chat_id, &mut msg)?;
-                emit_event!(
-                    context,
-                    Event::MsgsChanged {
-                        chat_id,
-                        msg_id: msg.id
-                    }
-                );
-            }
-            emit_event!(context, Event::ChatModified(chat_id));
-            return Ok(());
-        }
+    ensure!(
+        real_group_exists(context, chat_id),
+        "Failed to set profile image; group does not exist"
+    );
+    /* we should respect this - whatever we send to the group, it gets discarded anyway! */
+    if !is_contact_in_chat(context, chat_id, DC_CONTACT_ID_SELF) {
+        emit_event!(
+            context,
+            Event::ErrorSelfNotInGroup("Cannot set chat profile image; self not in group.".into())
+        );
+        bail!("Failed to set profile image");
     }
-    bail!("Failed to set profile image");
+    let mut msg = Message::new(Viewtype::Text);
+    msg.param
+        .set_int(Param::Cmd, SystemMessage::GroupImageChanged as i32);
+    if new_image.as_ref().is_empty() {
+        chat.param.remove(Param::ProfileImage);
+        msg.param.remove(Param::Arg);
+        msg.text = Some(context.stock_system_msg(
+            StockMessage::MsgGrpImgDeleted,
+            "",
+            "",
+            DC_CONTACT_ID_SELF,
+        ));
+    } else {
+        let image_blob = BlobObject::from_path(context, Path::new(new_image.as_ref())).or_else(
+            |err| match err.kind() {
+                BlobErrorKind::WrongBlobdir => {
+                    BlobObject::create_and_copy(context, Path::new(new_image.as_ref()))
+                }
+                _ => Err(err),
+            },
+        )?;
+        chat.param.set(Param::ProfileImage, image_blob.as_name());
+        msg.param.set(Param::Arg, image_blob.as_name());
+        msg.text = Some(context.stock_system_msg(
+            StockMessage::MsgGrpImgChanged,
+            "",
+            "",
+            DC_CONTACT_ID_SELF,
+        ));
+    }
+    chat.update_param(context)?;
+    if chat.is_promoted() {
+        msg.id = send_msg(context, chat_id, &mut msg)?;
+        emit_event!(
+            context,
+            Event::MsgsChanged {
+                chat_id,
+                msg_id: msg.id
+            }
+        );
+    }
+    emit_event!(context, Event::ChatModified(chat_id));
+    Ok(())
 }
 
-pub fn forward_msgs(context: &Context, msg_ids: &[u32], chat_id: u32) -> Result<(), Error> {
-    ensure!(!msg_ids.is_empty(), "empty msgs_ids: no one to forward to");
+pub fn forward_msgs(context: &Context, msg_ids: &[MsgId], chat_id: u32) -> Result<(), Error> {
+    ensure!(!msg_ids.is_empty(), "empty msgs_ids: nothing to forward");
     ensure!(
         chat_id > DC_CHAT_ID_LAST_SPECIAL,
         "can not forward to special chat"
     );
 
-    let mut created_db_entries = Vec::new();
+    let mut created_chats: Vec<u32> = Vec::new();
+    let mut created_msgs: Vec<MsgId> = Vec::new();
     let mut curr_timestamp: i64;
 
     unarchive(context, chat_id)?;
     if let Ok(mut chat) = Chat::load_from_db(context, chat_id) {
         curr_timestamp = dc_create_smeared_timestamps(context, msg_ids.len());
-        let idsstr = msg_ids
-            .iter()
-            .enumerate()
-            .fold(String::with_capacity(2 * msg_ids.len()), |acc, (i, n)| {
-                (if i == 0 { acc } else { acc + "," }) + &n.to_string()
-            });
-
         let ids = context.sql.query_map(
             format!(
                 "SELECT id FROM msgs WHERE id IN({}) ORDER BY timestamp,id",
-                idsstr
+                msg_ids.iter().map(|_| "?").join(",")
             ),
-            params![],
-            |row| row.get::<_, i32>(0),
+            msg_ids,
+            |row| row.get::<_, MsgId>(0),
             |ids| ids.collect::<Result<Vec<_>, _>>().map_err(Into::into),
         )?;
 
         for id in ids {
-            let src_msg_id = id;
-            let msg = Message::load_from_db(context, src_msg_id as u32);
+            let src_msg_id: MsgId = id;
+            let msg = Message::load_from_db(context, src_msg_id);
             if msg.is_err() {
                 break;
             }
             let mut msg = msg.unwrap();
             let original_param = msg.param.clone();
-            if msg.from_id != DC_CONTACT_ID_SELF {
-                msg.param.set_int(Param::Forwarded, 1);
-            }
-            msg.param.remove(Param::GuranteeE2ee);
+
+            // we tested a sort of broadcast
+            // by not marking own forwarded messages as such,
+            // however, this turned out to be to confusing and unclear.
+            msg.param.set_int(Param::Forwarded, 1);
+
+            msg.param.remove(Param::GuaranteeE2ee);
             msg.param.remove(Param::ForcePlaintext);
             msg.param.remove(Param::Cmd);
 
-            let new_msg_id: u32;
+            let new_msg_id: MsgId;
             if msg.state == MessageState::OutPreparing {
                 let fresh9 = curr_timestamp;
                 curr_timestamp += 1;
-                new_msg_id = chat
-                    .prepare_msg_raw(context, &mut msg, fresh9)
-                    .unwrap_or_default();
+                new_msg_id = chat.prepare_msg_raw(context, &mut msg, fresh9)?;
                 let save_param = msg.param.clone();
                 msg.param = original_param;
-                msg.id = src_msg_id as u32;
+                msg.id = src_msg_id;
 
                 if let Some(old_fwd) = msg.param.get(Param::PrepForwards) {
-                    let new_fwd = format!("{} {}", old_fwd, new_msg_id);
+                    let new_fwd = format!("{} {}", old_fwd, new_msg_id.to_u32());
                     msg.param.set(Param::PrepForwards, new_fwd);
                 } else {
-                    msg.param.set(Param::PrepForwards, new_msg_id.to_string());
+                    msg.param
+                        .set(Param::PrepForwards, new_msg_id.to_u32().to_string());
                 }
 
                 msg.save_param_to_disk(context);
@@ -1792,23 +1838,19 @@ pub fn forward_msgs(context: &Context, msg_ids: &[u32], chat_id: u32) -> Result<
                 msg.state = MessageState::OutPending;
                 let fresh10 = curr_timestamp;
                 curr_timestamp += 1;
-                new_msg_id = chat
-                    .prepare_msg_raw(context, &mut msg, fresh10)
-                    .unwrap_or_default();
+                new_msg_id = chat.prepare_msg_raw(context, &mut msg, fresh10)?;
                 job_send_msg(context, new_msg_id)?;
             }
-            created_db_entries.push(chat_id);
-            created_db_entries.push(new_msg_id);
+            created_chats.push(chat_id);
+            created_msgs.push(new_msg_id);
         }
     }
-
-    for i in (0..created_db_entries.len()).step_by(2) {
+    for (chat_id, msg_id) in created_chats.iter().zip(created_msgs.iter()) {
         context.call_cb(Event::MsgsChanged {
-            chat_id: created_db_entries[i],
-            msg_id: created_db_entries[i + 1],
+            chat_id: *chat_id,
+            msg_id: *msg_id,
         });
     }
-
     Ok(())
 }
 
@@ -1875,8 +1917,11 @@ pub fn add_device_msg(context: &Context, chat_id: u32, text: impl AsRef<str>) {
         return;
     }
 
-    let msg_id = sql::get_rowid(context, &context.sql, "msgs", "rfc724_mid", &rfc724_mid);
-    context.call_cb(Event::MsgsChanged { chat_id, msg_id });
+    let row_id = sql::get_rowid(context, &context.sql, "msgs", "rfc724_mid", &rfc724_mid);
+    context.call_cb(Event::MsgsChanged {
+        chat_id,
+        msg_id: MsgId::new(row_id),
+    });
 }
 
 #[cfg(test)]

src/chatlist.rs

@@ -4,7 +4,7 @@ use crate::contact::*;
 use crate::context::*;
 use crate::error::Result;
 use crate::lot::Lot;
-use crate::message::Message;
+use crate::message::{Message, MsgId};
 use crate::stock::StockMessage;
 
 /// An object representing a single chatlist in memory.
@@ -34,7 +34,7 @@ use crate::stock::StockMessage;
 #[derive(Debug)]
 pub struct Chatlist {
     /// Stores pairs of `chat_id, message_id`
-    ids: Vec<(u32, u32)>,
+    ids: Vec<(u32, MsgId)>,
 }
 
 impl Chatlist {
@@ -86,25 +86,12 @@ impl Chatlist {
         query: Option<&str>,
         query_contact_id: Option<u32>,
     ) -> Result<Self> {
-        let mut add_archived_link_item = 0;
-
-        // select with left join and minimum:
-        // - the inner select must use `hidden` and _not_ `m.hidden`
-        //   which would refer the outer select and take a lot of time
-        // - `GROUP BY` is needed several messages may have the same timestamp
-        // - the list starts with the newest chats
-        // nb: the query currently shows messages from blocked contacts in groups.
-        // however, for normal-groups, this is okay as the message is also returned by dc_get_chat_msgs()
-        // (otherwise it would be hard to follow conversations, wa and tg do the same)
-        // for the deaddrop, however, they should really be hidden, however, _currently_ the deaddrop is not
-        // shown at all permanent in the chatlist.
+        let mut add_archived_link_item = false;
 
         let process_row = |row: &rusqlite::Row| {
-            let chat_id: i32 = row.get(0)?;
-            // TODO: verify that it is okay for this to be Null
-            let msg_id: i32 = row.get(1).unwrap_or_default();
-
-            Ok((chat_id as u32, msg_id as u32))
+            let chat_id: u32 = row.get(0)?;
+            let msg_id: MsgId = row.get(1).unwrap_or_default();
+            Ok((chat_id, msg_id))
         };
 
         let process_rows = |rows: rusqlite::MappedRows<_>| {
@@ -112,36 +99,63 @@ impl Chatlist {
                 .map_err(Into::into)
         };
 
-        // nb: the query currently shows messages from blocked contacts in groups.
-        // however, for normal-groups, this is okay as the message is also returned by dc_get_chat_msgs()
-        // (otherwise it would be hard to follow conversations, wa and tg do the same)
-        // for the deaddrop, however, they should really be hidden, however, _currently_ the deaddrop is not
+        // select with left join and minimum:
+        //
+        // - the inner select must use `hidden` and _not_ `m.hidden`
+        //   which would refer the outer select and take a lot of time
+        // - `GROUP BY` is needed several messages may have the same
+        //   timestamp
+        // - the list starts with the newest chats
+        //
+        // nb: the query currently shows messages from blocked
+        // contacts in groups.  however, for normal-groups, this is
+        // okay as the message is also returned by dc_get_chat_msgs()
+        // (otherwise it would be hard to follow conversations, wa and
+        // tg do the same) for the deaddrop, however, they should
+        // really be hidden, however, _currently_ the deaddrop is not
         // shown at all permanent in the chatlist.
-
         let mut ids = if let Some(query_contact_id) = query_contact_id {
             // show chats shared with a given contact
             context.sql.query_map(
-            "SELECT c.id, m.id FROM chats c  LEFT JOIN msgs m         \
-             ON c.id=m.chat_id        \
-             AND m.timestamp=( SELECT MAX(timestamp)   \
-             FROM msgs  WHERE chat_id=c.id    \
-             AND (hidden=0 OR (hidden=1 AND state=19))) WHERE c.id>9   \
-             AND c.blocked=0 AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)  \
-             GROUP BY c.id  ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;",
-            params![query_contact_id as i32],
-            process_row,
-            process_rows,
-        )?
+                concat!(
+                    "SELECT c.id, m.id",
+                    " FROM chats c",
+                    " LEFT JOIN msgs m",
+                    "        ON c.id=m.chat_id",
+                    "       AND m.timestamp=(",
+                    "               SELECT MAX(timestamp)",
+                    "                 FROM msgs",
+                    "                WHERE chat_id=c.id",
+                    "                  AND (hidden=0 OR (hidden=1 AND state=19)))",
+                    " WHERE c.id>9",
+                    "   AND c.blocked=0",
+                    "   AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)",
+                    " GROUP BY c.id",
+                    " ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;"
+                ),
+                params![query_contact_id as i32],
+                process_row,
+                process_rows,
+            )?
         } else if 0 != listflags & DC_GCL_ARCHIVED_ONLY {
             // show archived chats
             context.sql.query_map(
-                "SELECT c.id, m.id FROM chats c  LEFT JOIN msgs m         \
-                 ON c.id=m.chat_id        \
-                 AND m.timestamp=( SELECT MAX(timestamp)   \
-                 FROM msgs  WHERE chat_id=c.id    \
-                 AND (hidden=0 OR (hidden=1 AND state=19))) WHERE c.id>9   \
-                 AND c.blocked=0 AND c.archived=1  GROUP BY c.id  \
-                 ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;",
+                concat!(
+                    "SELECT c.id, m.id",
+                    " FROM chats c",
+                    " LEFT JOIN msgs m",
+                    "        ON c.id=m.chat_id",
+                    "       AND m.timestamp=(",
+                    "               SELECT MAX(timestamp)",
+                    "                 FROM msgs",
+                    "                WHERE chat_id=c.id",
+                    "                  AND (hidden=0 OR (hidden=1 AND state=19)))",
+                    " WHERE c.id>9",
+                    "   AND c.blocked=0",
+                    "   AND c.archived=1",
+                    " GROUP BY c.id",
+                    " ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;"
+                ),
                 params![],
                 process_row,
                 process_rows,
@@ -152,13 +166,22 @@ impl Chatlist {
 
             let str_like_cmd = format!("%{}%", query);
             context.sql.query_map(
-                "SELECT c.id, m.id FROM chats c  LEFT JOIN msgs m         \
-                 ON c.id=m.chat_id        \
-                 AND m.timestamp=( SELECT MAX(timestamp)   \
-                 FROM msgs  WHERE chat_id=c.id    \
-                 AND (hidden=0 OR (hidden=1 AND state=19))) WHERE c.id>9   \
-                 AND c.blocked=0 AND c.name LIKE ?  \
-                 GROUP BY c.id  ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;",
+                concat!(
+                    "SELECT c.id, m.id",
+                    " FROM chats c",
+                    " LEFT JOIN msgs m",
+                    "        ON c.id=m.chat_id",
+                    "       AND m.timestamp=(",
+                    "               SELECT MAX(timestamp)",
+                    "                 FROM msgs",
+                    "                WHERE chat_id=c.id",
+                    "                  AND (hidden=0 OR (hidden=1 AND state=19)))",
+                    " WHERE c.id>9",
+                    "   AND c.blocked=0",
+                    "   AND c.name LIKE ?",
+                    " GROUP BY c.id",
+                    " ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;"
+                ),
                 params![str_like_cmd],
                 process_row,
                 process_rows,
@@ -166,34 +189,40 @@ impl Chatlist {
         } else {
             //  show normal chatlist
             let mut ids = context.sql.query_map(
-                "SELECT c.id, m.id FROM chats c  \
-                 LEFT JOIN msgs m         \
-                 ON c.id=m.chat_id        \
-                 AND m.timestamp=( SELECT MAX(timestamp)   \
-                 FROM msgs  WHERE chat_id=c.id    \
-                 AND (hidden=0 OR (hidden=1 AND state=19))) WHERE c.id>9   \
-                 AND c.blocked=0 AND c.archived=0  \
-                 GROUP BY c.id  \
-                 ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;",
+                concat!(
+                    "SELECT c.id, m.id",
+                    " FROM chats c",
+                    " LEFT JOIN msgs m",
+                    "        ON c.id=m.chat_id",
+                    "       AND m.timestamp=(",
+                    "               SELECT MAX(timestamp)",
+                    "                 FROM msgs",
+                    "                WHERE chat_id=c.id",
+                    "                  AND (hidden=0 OR (hidden=1 AND state=19)))",
+                    " WHERE c.id>9",
+                    "   AND c.blocked=0",
+                    "   AND c.archived=0",
+                    " GROUP BY c.id",
+                    " ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;"
+                ),
                 params![],
                 process_row,
                 process_rows,
             )?;
             if 0 == listflags & DC_GCL_NO_SPECIALS {
-                let last_deaddrop_fresh_msg_id = get_last_deaddrop_fresh_msg(context);
-                if last_deaddrop_fresh_msg_id > 0 {
+                if let Some(last_deaddrop_fresh_msg_id) = get_last_deaddrop_fresh_msg(context) {
                     ids.insert(0, (DC_CHAT_ID_DEADDROP, last_deaddrop_fresh_msg_id));
                 }
-                add_archived_link_item = 1;
+                add_archived_link_item = true;
             }
             ids
         };
 
-        if 0 != add_archived_link_item && dc_get_archived_cnt(context) > 0 {
+        if add_archived_link_item && dc_get_archived_cnt(context) > 0 {
             if ids.is_empty() && 0 != listflags & DC_GCL_ADD_ALLDONE_HINT {
-                ids.push((DC_CHAT_ID_ALLDONE_HINT, 0));
+                ids.push((DC_CHAT_ID_ALLDONE_HINT, MsgId::new(0)));
             }
-            ids.push((DC_CHAT_ID_ARCHIVED_LINK, 0));
+            ids.push((DC_CHAT_ID_ARCHIVED_LINK, MsgId::new(0)));
         }
 
         Ok(Chatlist { ids })
@@ -221,12 +250,9 @@ impl Chatlist {
     /// Get a single message ID of a chatlist.
     ///
     /// To get the message object from the message ID, use dc_get_msg().
-    pub fn get_msg_id(&self, index: usize) -> u32 {
-        if index >= self.ids.len() {
-            return 0;
-        }
-
-        self.ids[index].1
+    pub fn get_msg_id(&self, index: usize) -> Result<MsgId> {
+        ensure!(index >= self.ids.len(), "Chatlist index out of range");
+        Ok(self.ids[index].1)
     }
 
     /// Get a summary for a chatlist index.
@@ -268,9 +294,9 @@ impl Chatlist {
         let lastmsg_id = self.ids[index].1;
         let mut lastcontact = None;
 
-        let lastmsg = if 0 != lastmsg_id {
+        let lastmsg = if !lastmsg_id.is_special() {
             if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id) {
-                if lastmsg.from_id != 1 as libc::c_uint
+                if lastmsg.from_id != 1
                     && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup)
                 {
                     lastcontact = Contact::load_from_db(context, lastmsg.from_id).ok();
@@ -308,19 +334,21 @@ pub fn dc_get_archived_cnt(context: &Context) -> u32 {
         .unwrap_or_default()
 }
 
-fn get_last_deaddrop_fresh_msg(context: &Context) -> u32 {
-    // We have an index over the state-column, this should be sufficient as there are typically
-    // only few fresh messages.
-    context
-        .sql
-        .query_get_value(
-            context,
-            "SELECT m.id  FROM msgs m  LEFT JOIN chats c ON c.id=m.chat_id  \
-             WHERE m.state=10   \
-             AND m.hidden=0    \
-             AND c.blocked=2 \
-             ORDER BY m.timestamp DESC, m.id DESC;",
-            params![],
-        )
-        .unwrap_or_default()
+fn get_last_deaddrop_fresh_msg(context: &Context) -> Option<MsgId> {
+    // We have an index over the state-column, this should be
+    // sufficient as there are typically only few fresh messages.
+    context.sql.query_get_value(
+        context,
+        concat!(
+            "SELECT m.id",
+            " FROM msgs m",
+            " LEFT JOIN chats c",
+            "        ON c.id=m.chat_id",
+            " WHERE m.state=10",
+            "   AND m.hidden=0",
+            "   AND c.blocked=2",
+            " ORDER BY m.timestamp DESC, m.id DESC;"
+        ),
+        params![],
+    )
 }

src/config.rs

@@ -45,7 +45,7 @@ pub enum Config {
     MvboxWatch,
     #[strum(props(default = "1"))]
     MvboxMove,
-    #[strum(props(default = "0"))]
+    #[strum(props(default = "0"))] // also change ShowEmails.default() on changes
     ShowEmails,
     SaveMimeHeaders,
     ConfiguredAddr,

src/configure/auto_mozilla.rs

@@ -3,6 +3,7 @@ use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
 use crate::constants::*;
 use crate::context::Context;
+use crate::error::Error;
 use crate::login_param::LoginParam;
 
 use super::read_autoconf_file;
@@ -11,7 +12,7 @@ use super::read_autoconf_file;
  ******************************************************************************/
 /* documentation: https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration */
 struct MozAutoconfigure<'a> {
-    pub in_0: &'a LoginParam,
+    pub in_emailaddr: &'a str,
     pub in_emaildomain: &'a str,
     pub in_emaillocalpart: &'a str,
     pub out: LoginParam,
@@ -21,6 +22,7 @@ struct MozAutoconfigure<'a> {
     pub tag_config: MozConfigTag,
 }
 
+#[derive(PartialEq)]
 enum MozServer {
     Undefined,
     Imap,
@@ -35,25 +37,20 @@ enum MozConfigTag {
     Username,
 }
 
-pub fn moz_autoconfigure(
-    context: &Context,
-    url: &str,
-    param_in: &LoginParam,
-) -> Option<LoginParam> {
-    let xml_raw = read_autoconf_file(context, url)?;
-
-    // Split address into local part and domain part.
-    let p = param_in.addr.find('@')?;
-    let (in_emaillocalpart, in_emaildomain) = param_in.addr.split_at(p);
-    let in_emaildomain = &in_emaildomain[1..];
-
-    let mut reader = quick_xml::Reader::from_str(&xml_raw);
+pub fn moz_parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result<LoginParam, Error> {
+    let mut reader = quick_xml::Reader::from_str(xml_raw);
     reader.trim_text(true);
 
-    let mut buf = Vec::new();
+    // Split address into local part and domain part.
+    let p = match in_emailaddr.find('@') {
+        Some(i) => i,
+        None => bail!("Email address {} does not contain @", in_emailaddr),
+    };
+    let (in_emaillocalpart, in_emaildomain) = in_emailaddr.split_at(p);
+    let in_emaildomain = &in_emaildomain[1..];
 
     let mut moz_ac = MozAutoconfigure {
-        in_0: param_in,
+        in_emailaddr,
         in_emaildomain,
         in_emaillocalpart,
         out: LoginParam::new(),
@@ -62,6 +59,8 @@ pub fn moz_autoconfigure(
         tag_server: MozServer::Undefined,
         tag_config: MozConfigTag::Undefined,
     };
+
+    let mut buf = Vec::new();
     loop {
         match reader.read_event(&mut buf) {
             Ok(quick_xml::events::Event::Start(ref e)) => {
@@ -72,8 +71,7 @@ pub fn moz_autoconfigure(
                 moz_autoconfigure_text_cb(e, &mut moz_ac, &reader)
             }
             Err(e) => {
-                warn!(
-                    context,
+                bail!(
                     "Configure xml: Error at position {}: {:?}",
                     reader.buffer_position(),
                     e
@@ -91,11 +89,26 @@ pub fn moz_autoconfigure(
         || moz_ac.out.send_port == 0
     {
         let r = moz_ac.out.to_string();
-        warn!(context, "Bad or incomplete autoconfig: {}", r,);
-        return None;
+        bail!("Bad or incomplete autoconfig: {}", r,);
     }
 
-    Some(moz_ac.out)
+    Ok(moz_ac.out)
+}
+
+pub fn moz_autoconfigure(
+    context: &Context,
+    url: &str,
+    param_in: &LoginParam,
+) -> Option<LoginParam> {
+    let xml_raw = read_autoconf_file(context, url)?;
+
+    match moz_parse_xml(&param_in.addr, &xml_raw) {
+        Err(err) => {
+            warn!(context, "{}", err);
+            None
+        }
+        Ok(lp) => Some(lp),
+    }
 }
 
 fn moz_autoconfigure_text_cb<B: std::io::BufRead>(
@@ -105,7 +118,7 @@ fn moz_autoconfigure_text_cb<B: std::io::BufRead>(
 ) {
     let val = event.unescape_and_decode(reader).unwrap_or_default();
 
-    let addr = &moz_ac.in_0.addr;
+    let addr = moz_ac.in_emailaddr;
     let email_local = moz_ac.in_emaillocalpart;
     let email_domain = moz_ac.in_emaildomain;
 
@@ -160,13 +173,17 @@ fn moz_autoconfigure_endtag_cb(event: &BytesEnd, moz_ac: &mut MozAutoconfigure)
     let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
 
     if tag == "incomingserver" {
+        if moz_ac.tag_server == MozServer::Imap {
+            moz_ac.out_imap_set = true;
+        }
         moz_ac.tag_server = MozServer::Undefined;
         moz_ac.tag_config = MozConfigTag::Undefined;
-        moz_ac.out_imap_set = true;
     } else if tag == "outgoingserver" {
+        if moz_ac.tag_server == MozServer::Smtp {
+            moz_ac.out_smtp_set = true;
+        }
         moz_ac.tag_server = MozServer::Undefined;
         moz_ac.tag_config = MozConfigTag::Undefined;
-        moz_ac.out_smtp_set = true;
     } else {
         moz_ac.tag_config = MozConfigTag::Undefined;
     }
@@ -217,3 +234,89 @@ fn moz_autoconfigure_starttag_cb<B: std::io::BufRead>(
         moz_ac.tag_config = MozConfigTag::Username;
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_outlook_autoconfig() {
+        // Copied from https://autoconfig.thunderbird.net/v1.1/outlook.com on 2019-10-11
+        let xml_raw =
+"<clientConfig version=\"1.1\">
+  <emailProvider id=\"outlook.com\">
+    <domain>hotmail.com</domain>
+    <domain>hotmail.co.uk</domain>
+    <domain>hotmail.co.jp</domain>
+    <domain>hotmail.com.br</domain>
+    <domain>hotmail.de</domain>
+    <domain>hotmail.fr</domain>
+    <domain>hotmail.it</domain>
+    <domain>hotmail.es</domain>
+    <domain>live.com</domain>
+    <domain>live.co.uk</domain>
+    <domain>live.co.jp</domain>
+    <domain>live.de</domain>
+    <domain>live.fr</domain>
+    <domain>live.it</domain>
+    <domain>live.jp</domain>
+    <domain>msn.com</domain>
+    <domain>outlook.com</domain>
+    <displayName>Outlook.com (Microsoft)</displayName>
+    <displayShortName>Outlook</displayShortName>
+    <incomingServer type=\"exchange\">
+      <hostname>outlook.office365.com</hostname>
+      <port>443</port>
+      <username>%EMAILADDRESS%</username>
+      <socketType>SSL</socketType>
+      <authentication>OAuth2</authentication>
+      <owaURL>https://outlook.office365.com/owa/</owaURL>
+      <ewsURL>https://outlook.office365.com/ews/exchange.asmx</ewsURL>
+      <useGlobalPreferredServer>true</useGlobalPreferredServer>
+    </incomingServer>
+    <incomingServer type=\"imap\">
+      <hostname>outlook.office365.com</hostname>
+      <port>993</port>
+      <socketType>SSL</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </incomingServer>
+    <incomingServer type=\"pop3\">
+      <hostname>outlook.office365.com</hostname>
+      <port>995</port>
+      <socketType>SSL</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+      <pop3>
+        <leaveMessagesOnServer>true</leaveMessagesOnServer>
+        <!-- Outlook.com docs specifically mention that POP3 deletes have effect on the main inbox on webmail and IMAP -->
+      </pop3>
+    </incomingServer>
+    <outgoingServer type=\"smtp\">
+      <hostname>smtp.office365.com</hostname>
+      <port>587</port>
+      <socketType>STARTTLS</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </outgoingServer>
+    <documentation url=\"http://windows.microsoft.com/en-US/windows/outlook/send-receive-from-app\">
+      <descr lang=\"en\">Set up an email app with Outlook.com</descr>
+    </documentation>
+  </emailProvider>
+  <webMail>
+    <loginPage url=\"https://www.outlook.com/\"/>
+    <loginPageInfo url=\"https://www.outlook.com/\">
+      <username>%EMAILADDRESS%</username>
+      <usernameField id=\"i0116\" name=\"login\"/>
+      <passwordField id=\"i0118\" name=\"passwd\"/>
+      <loginButton id=\"idSIButton9\" name=\"SI\"/>
+    </loginPageInfo>
+  </webMail>
+</clientConfig>";
+        let res = moz_parse_xml("example@outlook.com", xml_raw).expect("XML parsing failed");
+        assert_eq!(res.mail_server, "outlook.office365.com");
+        assert_eq!(res.mail_port, 993);
+        assert_eq!(res.send_server, "smtp.office365.com");
+        assert_eq!(res.send_port, 587);
+    }
+}

src/configure/auto_outlook.rs

@@ -3,6 +3,7 @@ use quick_xml::events::BytesEnd;
 
 use crate::constants::*;
 use crate::context::Context;
+use crate::error::Error;
 use crate::login_param::LoginParam;
 
 use super::read_autoconf_file;
@@ -19,6 +20,100 @@ struct OutlookAutodiscover {
     pub config_redirecturl: Option<String>,
 }
 
+enum ParsingResult {
+    LoginParam(LoginParam),
+    RedirectUrl(String),
+}
+
+fn outlk_parse_xml(xml_raw: &str) -> Result<ParsingResult, Error> {
+    let mut outlk_ad = OutlookAutodiscover {
+        out: LoginParam::new(),
+        out_imap_set: false,
+        out_smtp_set: false,
+        config_type: None,
+        config_server: String::new(),
+        config_port: 0,
+        config_ssl: String::new(),
+        config_redirecturl: None,
+    };
+
+    let mut reader = quick_xml::Reader::from_str(&xml_raw);
+    reader.trim_text(true);
+
+    let mut buf = Vec::new();
+
+    let mut current_tag: Option<String> = None;
+
+    loop {
+        match reader.read_event(&mut buf) {
+            Ok(quick_xml::events::Event::Start(ref e)) => {
+                let tag = String::from_utf8_lossy(e.name()).trim().to_lowercase();
+
+                if tag == "protocol" {
+                    outlk_ad.config_type = None;
+                    outlk_ad.config_server = String::new();
+                    outlk_ad.config_port = 0;
+                    outlk_ad.config_ssl = String::new();
+                    outlk_ad.config_redirecturl = None;
+
+                    current_tag = None;
+                } else {
+                    current_tag = Some(tag);
+                }
+            }
+            Ok(quick_xml::events::Event::End(ref e)) => {
+                outlk_autodiscover_endtag_cb(e, &mut outlk_ad);
+                current_tag = None;
+            }
+            Ok(quick_xml::events::Event::Text(ref e)) => {
+                let val = e.unescape_and_decode(&reader).unwrap_or_default();
+
+                if let Some(ref tag) = current_tag {
+                    match tag.as_str() {
+                        "type" => {
+                            outlk_ad.config_type = Some(val.trim().to_lowercase().to_string())
+                        }
+                        "server" => outlk_ad.config_server = val.trim().to_string(),
+                        "port" => outlk_ad.config_port = val.trim().parse().unwrap_or_default(),
+                        "ssl" => outlk_ad.config_ssl = val.trim().to_string(),
+                        "redirecturl" => outlk_ad.config_redirecturl = Some(val.trim().to_string()),
+                        _ => {}
+                    };
+                }
+            }
+            Err(e) => {
+                bail!(
+                    "Configure xml: Error at position {}: {:?}",
+                    reader.buffer_position(),
+                    e
+                );
+            }
+            Ok(quick_xml::events::Event::Eof) => break,
+            _ => (),
+        }
+        buf.clear();
+    }
+
+    // XML redirect via redirecturl
+    if outlk_ad.config_redirecturl.is_none()
+        || outlk_ad.config_redirecturl.as_ref().unwrap().is_empty()
+    {
+        if outlk_ad.out.mail_server.is_empty()
+            || outlk_ad.out.mail_port == 0
+            || outlk_ad.out.send_server.is_empty()
+            || outlk_ad.out.send_port == 0
+        {
+            let r = outlk_ad.out.to_string();
+            bail!("Bad or incomplete autoconfig: {}", r,);
+        }
+        Ok(ParsingResult::LoginParam(outlk_ad.out))
+    } else {
+        Ok(ParsingResult::RedirectUrl(
+            outlk_ad.config_redirecturl.unwrap(),
+        ))
+    }
+}
+
 pub fn outlk_autodiscover(
     context: &Context,
     url: &str,
@@ -27,94 +122,16 @@ pub fn outlk_autodiscover(
     let mut url = url.to_string();
     /* Follow up to 10 xml-redirects (http-redirects are followed in read_autoconf_file() */
     for _i in 0..10 {
-        let mut outlk_ad = OutlookAutodiscover {
-            out: LoginParam::new(),
-            out_imap_set: false,
-            out_smtp_set: false,
-            config_type: None,
-            config_server: String::new(),
-            config_port: 0,
-            config_ssl: String::new(),
-            config_redirecturl: None,
-        };
-
         if let Some(xml_raw) = read_autoconf_file(context, &url) {
-            let mut reader = quick_xml::Reader::from_str(&xml_raw);
-            reader.trim_text(true);
-
-            let mut buf = Vec::new();
-
-            let mut current_tag: Option<String> = None;
-
-            loop {
-                match reader.read_event(&mut buf) {
-                    Ok(quick_xml::events::Event::Start(ref e)) => {
-                        let tag = String::from_utf8_lossy(e.name()).trim().to_lowercase();
-
-                        if tag == "protocol" {
-                            outlk_ad.config_type = None;
-                            outlk_ad.config_server = String::new();
-                            outlk_ad.config_port = 0;
-                            outlk_ad.config_ssl = String::new();
-                            outlk_ad.config_redirecturl = None;
-
-                            current_tag = None;
-                        } else {
-                            current_tag = Some(tag);
-                        }
-                    }
-                    Ok(quick_xml::events::Event::End(ref e)) => {
-                        outlk_autodiscover_endtag_cb(e, &mut outlk_ad);
-                        current_tag = None;
-                    }
-                    Ok(quick_xml::events::Event::Text(ref e)) => {
-                        let val = e.unescape_and_decode(&reader).unwrap_or_default();
-
-                        if let Some(ref tag) = current_tag {
-                            match tag.as_str() {
-                                "type" => outlk_ad.config_type = Some(val.trim().to_string()),
-                                "server" => outlk_ad.config_server = val.trim().to_string(),
-                                "port" => {
-                                    outlk_ad.config_port = val.trim().parse().unwrap_or_default()
-                                }
-                                "ssl" => outlk_ad.config_ssl = val.trim().to_string(),
-                                "redirecturl" => {
-                                    outlk_ad.config_redirecturl = Some(val.trim().to_string())
-                                }
-                                _ => {}
-                            };
-                        }
-                    }
-                    Err(e) => {
-                        error!(
-                            context,
-                            "Configure xml: Error at position {}: {:?}",
-                            reader.buffer_position(),
-                            e
-                        );
-                    }
-                    Ok(quick_xml::events::Event::Eof) => break,
-                    _ => (),
-                }
-                buf.clear();
-            }
-
-            // XML redirect via redirecturl
-            if outlk_ad.config_redirecturl.is_none()
-                || outlk_ad.config_redirecturl.as_ref().unwrap().is_empty()
-            {
-                if outlk_ad.out.mail_server.is_empty()
-                    || outlk_ad.out.mail_port == 0
-                    || outlk_ad.out.send_server.is_empty()
-                    || outlk_ad.out.send_port == 0
-                {
-                    let r = outlk_ad.out.to_string();
-                    warn!(context, "Bad or incomplete autoconfig: {}", r,);
+            match outlk_parse_xml(&xml_raw) {
+                Err(err) => {
+                    warn!(context, "{}", err);
                     return None;
                 }
-                return Some(outlk_ad.out);
-            } else {
-                url = outlk_ad.config_redirecturl.unwrap();
+                Ok(res) => match res {
+                    ParsingResult::RedirectUrl(redirect_url) => url = redirect_url,
+                    ParsingResult::LoginParam(login_param) => return Some(login_param),
+                },
             }
         } else {
             return None;
@@ -155,3 +172,78 @@ fn outlk_autodiscover_endtag_cb(event: &BytesEnd, outlk_ad: &mut OutlookAutodisc
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_redirect() {
+        let res = outlk_parse_xml("
+<?xml version=\"1.0\" encoding=\"utf-8\"?>
+  <Autodiscover xmlns=\"http://schemas.microsoft.com/exchange/autodiscover/responseschema/2006\">
+    <Response xmlns=\"http://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a\">
+      <Account>
+        <AccountType>email</AccountType>
+        <Action>redirectUrl</Action>
+        <RedirectUrl>https://mail.example.com/autodiscover/autodiscover.xml</RedirectUrl>
+      </Account>
+    </Response>
+  </Autodiscover>
+ ").expect("XML is not parsed successfully");
+        match res {
+            ParsingResult::LoginParam(_lp) => {
+                panic!("redirecturl is not found");
+            }
+            ParsingResult::RedirectUrl(url) => {
+                assert_eq!(
+                    url,
+                    "https://mail.example.com/autodiscover/autodiscover.xml"
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn test_parse_loginparam() {
+        let res = outlk_parse_xml(
+            "\
+<?xml version=\"1.0\" encoding=\"utf-8\"?>
+<Autodiscover xmlns=\"http://schemas.microsoft.com/exchange/autodiscover/responseschema/2006\">
+  <Response xmlns=\"http://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a\">
+    <Account>
+      <AccountType>email</AccountType>
+      <Action>settings</Action>
+      <Protocol>
+        <Type>IMAP</Type>
+        <Server>example.com</Server>
+        <Port>993</Port>
+        <SSL>on</SSL>
+        <AuthRequired>on</AuthRequired>
+      </Protocol>
+      <Protocol>
+        <Type>SMTP</Type>
+        <Server>smtp.example.com</Server>
+        <Port>25</Port>
+        <SSL>off</SSL>
+        <AuthRequired>on</AuthRequired>
+      </Protocol>
+    </Account>
+  </Response>
+</Autodiscover>",
+        )
+        .expect("XML is not parsed successfully");
+
+        match res {
+            ParsingResult::LoginParam(lp) => {
+                assert_eq!(lp.mail_server, "example.com");
+                assert_eq!(lp.mail_port, 993);
+                assert_eq!(lp.send_server, "smtp.example.com");
+                assert_eq!(lp.send_port, 25);
+            }
+            ParsingResult::RedirectUrl(_) => {
+                panic!("RedirectUrl is not expected");
+            }
+        }
+    }
+}

src/configure/mod.rs

@@ -112,6 +112,7 @@ pub fn dc_job_do_DC_JOB_CONFIGURE_IMAP(context: &Context) {
                         dc_get_oauth2_addr(context, &param.addr, &param.mail_pw)
                             .and_then(|e| e.parse().ok())
                     {
+                        info!(context, "Authorized address is {}", oauth2_addr);
                         param.addr = oauth2_addr;
                         context
                             .sql

src/constants.rs

@@ -44,6 +44,20 @@ impl Default for Blocked {
     }
 }
 
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[repr(u8)]
+pub enum ShowEmails {
+    Off = 0,
+    AcceptedContacts = 1,
+    All = 2,
+}
+
+impl Default for ShowEmails {
+    fn default() -> Self {
+        ShowEmails::Off // also change Config.ShowEmails props(default) on changes
+    }
+}
+
 pub const DC_IMAP_SEEN: u32 = 0x1;
 
 pub const DC_HANDSHAKE_CONTINUE_NORMAL_PROCESSING: i32 = 0x01;
@@ -54,7 +68,7 @@ pub const DC_GCL_ARCHIVED_ONLY: usize = 0x01;
 pub const DC_GCL_NO_SPECIALS: usize = 0x02;
 pub const DC_GCL_ADD_ALLDONE_HINT: usize = 0x04;
 
-const DC_GCM_ADDDAYMARKER: usize = 0x01;
+pub const DC_GCM_ADDDAYMARKER: u32 = 0x01;
 
 pub const DC_GCL_VERIFIED_ONLY: usize = 0x01;
 pub const DC_GCL_ADD_SELF: usize = 0x02;
@@ -106,7 +120,7 @@ impl Default for Chattype {
 }
 
 pub const DC_MSG_ID_MARKER1: u32 = 1;
-const DC_MSG_ID_DAYMARKER: u32 = 9;
+pub const DC_MSG_ID_DAYMARKER: u32 = 9;
 pub const DC_MSG_ID_LAST_SPECIAL: u32 = 9;
 
 /// approx. max. length returned by dc_msg_get_text()
@@ -121,6 +135,11 @@ pub const DC_CONTACT_ID_LAST_SPECIAL: u32 = 9;
 
 pub const DC_CREATE_MVBOX: usize = 1;
 
+// Flags for empty server job
+
+pub const DC_EMPTY_MVBOX: u32 = 0x01;
+pub const DC_EMPTY_INBOX: u32 = 0x02;
+
 // Flags for configuring IMAP and SMTP servers.
 // These flags are optional
 // and may be set together with the username, password etc.
@@ -251,11 +270,6 @@ const DC_ERROR_SEE_STRING: usize = 0; // deprecated;
 const DC_ERROR_SELF_NOT_IN_GROUP: usize = 1; // deprecated;
 const DC_STR_SELFNOTINGRP: usize = 21; // deprecated;
 
-/// Values for dc_get|set_config("show_emails")
-const DC_SHOW_EMAILS_OFF: usize = 0;
-const DC_SHOW_EMAILS_ACCEPTED_CONTACTS: usize = 1;
-const DC_SHOW_EMAILS_ALL: usize = 2;
-
 // TODO: Strings need some doumentation about used placeholders.
 // These constants are used to set stock translation strings
 

src/contact.rs

@@ -14,7 +14,7 @@ use crate::error::Result;
 use crate::events::Event;
 use crate::key::*;
 use crate::login_param::LoginParam;
-use crate::message::MessageState;
+use crate::message::{MessageState, MsgId};
 use crate::peerstate::*;
 use crate::sql;
 use crate::stock::StockMessage;
@@ -243,7 +243,7 @@ impl Contact {
         {
             context.call_cb(Event::MsgsChanged {
                 chat_id: 0,
-                msg_id: 0,
+                msg_id: MsgId::new(0),
             });
         }
     }

src/context.rs

@@ -1,7 +1,5 @@
 use std::collections::HashMap;
 use std::ffi::OsString;
-use std::fs;
-use std::io::Write;
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Condvar, Mutex, RwLock};
 
@@ -11,7 +9,6 @@ use crate::chat::*;
 use crate::config::Config;
 use crate::constants::*;
 use crate::contact::*;
-use crate::dc_tools::{dc_copy_file, dc_derive_safe_stem_ext};
 use crate::error::*;
 use crate::events::Event;
 use crate::imap::*;
@@ -20,11 +17,10 @@ use crate::job_thread::JobThread;
 use crate::key::*;
 use crate::login_param::LoginParam;
 use crate::lot::Lot;
-use crate::message::{self, Message};
+use crate::message::{self, Message, MsgId};
 use crate::param::Params;
 use crate::smtp::*;
 use crate::sql::Sql;
-use rand::{thread_rng, Rng};
 
 /// Callback function type for [Context]
 ///
@@ -150,7 +146,7 @@ impl Context {
         };
 
         ensure!(
-            ctx.sql.open(&ctx, &ctx.dbfile, 0),
+            ctx.sql.open(&ctx, &ctx.dbfile, false),
             "Failed opening sqlite database"
         );
 
@@ -165,59 +161,6 @@ impl Context {
         self.blobdir.as_path()
     }
 
-    pub fn copy_to_blobdir(&self, orig_filename: impl AsRef<str>) -> Result<String> {
-        // return a $BLOBDIR/<filename> with the content of orig_filename
-        // copied into it. The <filename> will be safely derived from
-        // orig_filename, and will not clash with existing filenames.
-        let dest = self.new_blob_file(&orig_filename, b"")?;
-        if dc_copy_file(
-            &self,
-            PathBuf::from(orig_filename.as_ref()),
-            PathBuf::from(&dest),
-        ) {
-            Ok(dest)
-        } else {
-            bail!("could not copy {} to {}", orig_filename.as_ref(), dest);
-        }
-    }
-
-    pub fn new_blob_file(&self, orig_filename: impl AsRef<str>, data: &[u8]) -> Result<String> {
-        // return a $BLOBDIR/<FILENAME> string which corresponds to the
-        // respective file in the blobdir, and which contains the data.
-        // FILENAME is computed by looking and possibly mangling the
-        // basename of orig_filename. The resulting filenames are meant
-        // to be human-readable.
-        let (stem, ext) = dc_derive_safe_stem_ext(orig_filename.as_ref());
-
-        // ext starts with "." or is empty string, so we can always resconstruct
-
-        for i in 0..3 {
-            let candidate_basename = match i {
-                // first a try to just use the (possibly mangled) original basename
-                0 => format!("{}{}", stem, ext),
-
-                // otherwise extend stem with random numbers
-                _ => {
-                    let mut rng = thread_rng();
-                    let random_id: u32 = rng.gen();
-                    format!("{}-{}{}", stem, random_id, ext)
-                }
-            };
-            let path = self.get_blobdir().join(&candidate_basename);
-            if let Ok(mut file) = fs::OpenOptions::new()
-                .create_new(true)
-                .write(true)
-                .open(&path)
-            {
-                file.write_all(data)?;
-                let db_entry = format!("$BLOBDIR/{}", candidate_basename);
-                self.call_cb(Event::NewBlobFile(db_entry.clone()));
-                return Ok(db_entry);
-            }
-        }
-        bail!("out of luck to create new blob file");
-    }
-
     pub fn call_cb(&self, event: Event) -> uintptr_t {
         (*self.cb)(self, event)
     }
@@ -370,24 +313,30 @@ impl Context {
         res
     }
 
-    pub fn get_fresh_msgs(&self) -> Vec<u32> {
+    pub fn get_fresh_msgs(&self) -> Vec<MsgId> {
         let show_deaddrop = 0;
-
         self.sql
             .query_map(
-                "SELECT m.id FROM msgs m LEFT JOIN contacts ct \
-                 ON m.from_id=ct.id LEFT JOIN chats c ON m.chat_id=c.id WHERE m.state=?   \
-                 AND m.hidden=0   \
-                 AND m.chat_id>?   \
-                 AND ct.blocked=0   \
-                 AND (c.blocked=0 OR c.blocked=?) ORDER BY m.timestamp DESC,m.id DESC;",
+                concat!(
+                    "SELECT m.id",
+                    " FROM msgs m",
+                    " LEFT JOIN contacts ct",
+                    "        ON m.from_id=ct.id",
+                    " LEFT JOIN chats c",
+                    "        ON m.chat_id=c.id",
+                    " WHERE m.state=?",
+                    "   AND m.hidden=0",
+                    "   AND m.chat_id>?",
+                    "   AND ct.blocked=0",
+                    "   AND (c.blocked=0 OR c.blocked=?)",
+                    " ORDER BY m.timestamp DESC,m.id DESC;"
+                ),
                 &[10, 9, if 0 != show_deaddrop { 2 } else { 0 }],
-                |row| row.get(0),
+                |row| row.get::<_, MsgId>(0),
                 |rows| {
                     let mut ret = Vec::new();
                     for row in rows {
-                        let id: u32 = row?;
-                        ret.push(id);
+                        ret.push(row?);
                     }
                     Ok(ret)
                 },
@@ -396,7 +345,7 @@ impl Context {
     }
 
     #[allow(non_snake_case)]
-    pub fn search_msgs(&self, chat_id: u32, query: impl AsRef<str>) -> Vec<u32> {
+    pub fn search_msgs(&self, chat_id: u32, query: impl AsRef<str>) -> Vec<MsgId> {
         let real_query = query.as_ref().trim();
         if real_query.is_empty() {
             return Vec::new();
@@ -405,25 +354,43 @@ impl Context {
         let strLikeBeg = format!("{}%", real_query);
 
         let query = if 0 != chat_id {
-            "SELECT m.id, m.timestamp FROM msgs m LEFT JOIN contacts ct ON m.from_id=ct.id WHERE m.chat_id=?  \
-         AND m.hidden=0  \
-         AND ct.blocked=0 AND (txt LIKE ? OR ct.name LIKE ?) ORDER BY m.timestamp,m.id;"
+            concat!(
+                "SELECT m.id AS id, m.timestamp AS timestamp",
+                " FROM msgs m",
+                " LEFT JOIN contacts ct",
+                "        ON m.from_id=ct.id",
+                " WHERE m.chat_id=?",
+                "   AND m.hidden=0",
+                "   AND ct.blocked=0",
+                "   AND (txt LIKE ? OR ct.name LIKE ?)",
+                " ORDER BY m.timestamp,m.id;"
+            )
         } else {
-            "SELECT m.id, m.timestamp FROM msgs m LEFT JOIN contacts ct ON m.from_id=ct.id \
-         LEFT JOIN chats c ON m.chat_id=c.id WHERE m.chat_id>9 AND m.hidden=0  \
-         AND (c.blocked=0 OR c.blocked=?) \
-         AND ct.blocked=0 AND (m.txt LIKE ? OR ct.name LIKE ?) ORDER BY m.timestamp DESC,m.id DESC;"
+            concat!(
+                "SELECT m.id AS id, m.timestamp AS timestamp",
+                " FROM msgs m",
+                " LEFT JOIN contacts ct",
+                "        ON m.from_id=ct.id",
+                " LEFT JOIN chats c",
+                "        ON m.chat_id=c.id",
+                " WHERE m.chat_id>9",
+                "   AND m.hidden=0",
+                "   AND (c.blocked=0 OR c.blocked=?)",
+                "   AND ct.blocked=0",
+                "   AND (m.txt LIKE ? OR ct.name LIKE ?)",
+                " ORDER BY m.timestamp DESC,m.id DESC;"
+            )
         };
 
         self.sql
             .query_map(
                 query,
                 params![chat_id as i32, &strLikeInText, &strLikeBeg],
-                |row| row.get::<_, i32>(0),
+                |row| row.get::<_, MsgId>("id"),
                 |rows| {
                     let mut ret = Vec::new();
                     for id in rows {
-                        ret.push(id? as u32);
+                        ret.push(id?);
                     }
                     Ok(ret)
                 },
@@ -454,7 +421,7 @@ impl Context {
         }
     }
 
-    pub fn do_heuristics_moves(&self, folder: &str, msg_id: u32) {
+    pub fn do_heuristics_moves(&self, folder: &str, msg_id: MsgId) {
         if !self.get_config_bool(Config::MvboxMove) {
             return;
         }
@@ -479,7 +446,7 @@ impl Context {
                 job_add(
                     self,
                     Action::MoveMsg,
-                    msg.id as libc::c_int,
+                    msg.id.to_u32() as i32,
                     Params::new(),
                     0,
                 );
@@ -536,7 +503,6 @@ pub fn get_version_str() -> &'static str {
 mod tests {
     use super::*;
 
-    use crate::dc_tools::*;
     use crate::test_utils::*;
 
     #[test]
@@ -574,51 +540,6 @@ mod tests {
         assert!(res.is_err());
     }
 
-    #[test]
-    fn test_new_blob_file() {
-        let t = dummy_context();
-        let context = t.ctx;
-        let x = &context.new_blob_file("hello", b"data").unwrap();
-        assert!(dc_file_exist(&context, x));
-        assert!(x.starts_with("$BLOBDIR"));
-        assert!(dc_read_file(&context, x).unwrap() == b"data");
-
-        let y = &context.new_blob_file("hello", b"data").unwrap();
-        assert!(dc_file_exist(&context, y));
-        assert!(y.starts_with("$BLOBDIR/hello-"));
-
-        let x = &context.new_blob_file("xyz/hello.png", b"data").unwrap();
-        assert!(dc_file_exist(&context, x));
-        assert_eq!(x, "$BLOBDIR/hello.png");
-
-        let y = &context.new_blob_file("hello\\world.png", b"data").unwrap();
-        assert!(dc_file_exist(&context, y));
-        assert_eq!(y, "$BLOBDIR/world.png");
-    }
-
-    #[test]
-    fn test_new_blob_file_long_names() {
-        let t = dummy_context();
-        let context = t.ctx;
-        let s = "12312312039182039182039812039810293810293810293810293801293801293123123";
-        let x = &context.new_blob_file(s, b"data").unwrap();
-        println!("blobfilename '{}'", x);
-        println!("xxxxfilename '{}'", s);
-        assert!(x.len() < s.len());
-        assert!(dc_file_exist(&context, x));
-        assert!(x.starts_with("$BLOBDIR"));
-    }
-
-    #[test]
-    fn test_new_blob_file_unicode() {
-        let t = dummy_context();
-        let context = t.ctx;
-        let s = "helloäworld.qwe";
-        let x = &context.new_blob_file(s, b"data").unwrap();
-        assert_eq!(x, "$BLOBDIR/hello-world.qwe");
-        assert_eq!(dc_read_file(&context, x).unwrap(), b"data");
-    }
-
     #[test]
     fn test_sqlite_parent_not_exists() {
         let tmp = tempfile::tempdir().unwrap();

src/dc_mimeparser.rs

@@ -13,6 +13,7 @@ use mmime::mailmime::types::*;
 use mmime::mailmime::*;
 use mmime::other::*;
 
+use crate::blob::BlobObject;
 use crate::constants::Viewtype;
 use crate::contact::*;
 use crate::context::Context;
@@ -607,12 +608,8 @@ impl<'a> MimeParser<'a> {
 
         /* regard `Content-Transfer-Encoding:` */
         let mut desired_filename = String::default();
-        let mut simplifier: Option<Simplify> = None;
         match mime_type {
             DC_MIMETYPE_TEXT_PLAIN | DC_MIMETYPE_TEXT_HTML => {
-                if simplifier.is_none() {
-                    simplifier = Some(Simplify::new());
-                }
                 /* get from `Content-Type: text/...; charset=utf-8`; must not be free()'d */
                 let charset = mailmime_content_charset_get((*mime).mm_content_type);
                 if !charset.is_null()
@@ -640,13 +637,14 @@ impl<'a> MimeParser<'a> {
                 /* check header directly as is_send_by_messenger is not yet set up */
                 let is_msgrmsg = self.lookup_optional_field("Chat-Version").is_some();
 
+                let mut simplifier = Simplify::new();
                 let simplified_txt = if decoded_data.is_empty() {
                     "".into()
                 } else {
                     let input = std::string::String::from_utf8_lossy(&decoded_data);
                     let is_html = mime_type == 70;
 
-                    simplifier.unwrap().simplify(&input, is_html, is_msgrmsg)
+                    simplifier.simplify(&input, is_html, is_msgrmsg)
                 };
                 if !simplified_txt.is_empty() {
                     let mut part = Part::default();
@@ -658,7 +656,7 @@ impl<'a> MimeParser<'a> {
                     self.do_add_single_part(part);
                 }
 
-                if simplifier.unwrap().is_forwarded {
+                if simplifier.is_forwarded {
                     self.is_forwarded = true;
                 }
             }
@@ -774,8 +772,8 @@ impl<'a> MimeParser<'a> {
         desired_filename: &str,
     ) {
         /* write decoded data to new blob file */
-        let bpath = match self.context.new_blob_file(desired_filename, decoded_data) {
-            Ok(path) => path,
+        let blob = match BlobObject::create(self.context, desired_filename, decoded_data) {
+            Ok(blob) => blob,
             Err(err) => {
                 error!(
                     self.context,
@@ -789,7 +787,7 @@ impl<'a> MimeParser<'a> {
         part.typ = msg_type;
         part.mimetype = mime_type;
         part.bytes = decoded_data.len() as libc::c_int;
-        part.param.set(Param::File, bpath);
+        part.param.set(Param::File, blob.as_name());
         if let Some(raw_mime) = raw_mime {
             part.param.set(Param::MimeType, raw_mime);
         }
@@ -805,7 +803,7 @@ impl<'a> MimeParser<'a> {
 
     fn do_add_single_part(&mut self, mut part: Part) {
         if self.encrypted && self.signatures.len() > 0 {
-            part.param.set_int(Param::GuranteeE2ee, 1);
+            part.param.set_int(Param::GuaranteeE2ee, 1);
         } else if self.encrypted {
             part.param.set_int(Param::ErroneousE2ee, 0x2);
         }
@@ -1223,6 +1221,7 @@ mod tests {
     }
 
     proptest! {
+        #[ignore]
         #[test]
         fn test_dc_mailmime_parse_crash_fuzzy(data in "[!-~\t ]{2000,}") {
             // this test doesn't exercise much of dc_mimeparser anymore

src/dc_receive_imf.rs

@@ -10,6 +10,9 @@ use mmime::mailmime::*;
 use mmime::other::*;
 use sha2::{Digest, Sha256};
 
+use num_traits::FromPrimitive;
+
+use crate::blob::BlobObject;
 use crate::chat::{self, Chat};
 use crate::config::Config;
 use crate::constants::*;
@@ -22,7 +25,7 @@ use crate::error::Result;
 use crate::events::Event;
 use crate::job::*;
 use crate::location;
-use crate::message::{self, MessageState};
+use crate::message::{self, MessageState, MsgId};
 use crate::param::*;
 use crate::peerstate::*;
 use crate::securejoin::handle_securejoin_handshake;
@@ -82,7 +85,7 @@ pub unsafe fn dc_receive_imf(
     let mut hidden = 0;
 
     let mut needs_delete_job = false;
-    let mut insert_msg_id = 0;
+    let mut insert_msg_id = MsgId::new_unset();
 
     let mut sent_timestamp = 0;
     let mut created_db_entries = Vec::new();
@@ -94,17 +97,17 @@ pub unsafe fn dc_receive_imf(
     // helper method to handle early exit and memory cleanup
     let cleanup = |context: &Context,
                    create_event_to_send: &Option<CreateEvent>,
-                   created_db_entries: &Vec<(usize, usize)>,
-                   rr_event_to_send: &Vec<(u32, u32)>| {
+                   created_db_entries: &Vec<(usize, MsgId)>,
+                   rr_event_to_send: &Vec<(u32, MsgId)>| {
         if let Some(create_event_to_send) = create_event_to_send {
             for (chat_id, msg_id) in created_db_entries {
                 let event = match create_event_to_send {
                     CreateEvent::MsgsChanged => Event::MsgsChanged {
-                        msg_id: *msg_id as u32,
+                        msg_id: *msg_id,
                         chat_id: *chat_id as u32,
                     },
                     CreateEvent::IncomingMsg => Event::IncomingMsg {
-                        msg_id: *msg_id as u32,
+                        msg_id: *msg_id,
                         chat_id: *chat_id as u32,
                     },
                 };
@@ -270,7 +273,7 @@ pub unsafe fn dc_receive_imf(
         job_add(
             context,
             Action::DeleteMsgOnImap,
-            created_db_entries[0].1 as i32,
+            created_db_entries[0].1.to_u32() as i32,
             Params::new(),
             0,
         );
@@ -310,8 +313,8 @@ unsafe fn add_parts(
     flags: u32,
     needs_delete_job: &mut bool,
     to_self: i32,
-    insert_msg_id: &mut u32,
-    created_db_entries: &mut Vec<(usize, usize)>,
+    insert_msg_id: &mut MsgId,
+    created_db_entries: &mut Vec<(usize, MsgId)>,
     create_event_to_send: &mut Option<CreateEvent>,
 ) -> Result<()> {
     let mut state: MessageState;
@@ -366,12 +369,14 @@ unsafe fn add_parts(
     // incoming non-chat messages may be discarded;
     // maybe this can be optimized later, by checking the state before the message body is downloaded
     let mut allow_creation = 1;
+    let show_emails =
+        ShowEmails::from_i32(context.get_config_int(Config::ShowEmails)).unwrap_or_default();
     if mime_parser.is_system_message != SystemMessage::AutocryptSetupMessage && msgrmsg == 0 {
-        let show_emails = context.get_config_int(Config::ShowEmails);
-        if show_emails == 0 {
-            *chat_id = 3;
+        // this message is a classic email not a chat-message nor a reply to one
+        if show_emails == ShowEmails::Off {
+            *chat_id = DC_CHAT_ID_TRASH;
             allow_creation = 0
-        } else if show_emails == 1 {
+        } else if show_emails == ShowEmails::AcceptedContacts {
             allow_creation = 0
         }
     }
@@ -439,7 +444,7 @@ unsafe fn add_parts(
         if *chat_id == 0 {
             // check if the message belongs to a mailing list
             if mime_parser.is_mailinglist_message() {
-                *chat_id = 3;
+                *chat_id = DC_CHAT_ID_TRASH;
                 info!(context, "Message belongs to a mailing list and is ignored.",);
             }
         }
@@ -486,12 +491,13 @@ unsafe fn add_parts(
         }
 
         // if the chat_id is blocked,
-        // for unknown senders and non-delta messages set the state to NOTICED
-        // to not result in a contact request (this would require the state FRESH)
+        // for unknown senders and non-delta-messages set the state to NOTICED
+        // to not result in a chatlist-contact-request (this would require the state FRESH)
         if Blocked::Not != chat_id_blocked
             && state == MessageState::InFresh
             && !incoming_origin.is_verified()
             && msgrmsg == 0
+            && show_emails != ShowEmails::All
         {
             state = MessageState::InNoticed;
         }
@@ -678,9 +684,10 @@ unsafe fn add_parts(
                 ])?;
 
                 txt_raw = None;
-                *insert_msg_id =
+                let row_id =
                     sql::get_rowid_with_conn(context, conn, "msgs", "rfc724_mid", &rfc724_mid);
-                created_db_entries.push((*chat_id as usize, *insert_msg_id as usize));
+                *insert_msg_id = MsgId::new(row_id);
+                created_db_entries.push((*chat_id as usize, *insert_msg_id));
             }
             Ok(())
         },
@@ -713,7 +720,7 @@ unsafe fn handle_reports(
     mime_parser: &MimeParser,
     from_id: u32,
     sent_timestamp: i64,
-    rr_event_to_send: &mut Vec<(u32, u32)>,
+    rr_event_to_send: &mut Vec<(u32, MsgId)>,
     server_folder: impl AsRef<str>,
     server_uid: u32,
 ) {
@@ -804,20 +811,15 @@ unsafe fn handle_reports(
                                     if let Ok(rfc724_mid) = wrapmime::parse_message_id(as_str(
                                         (*of_org_msgid).fld_value,
                                     )) {
-                                        let mut chat_id_0 = 0;
-                                        let mut msg_id = 0;
-
-                                        if message::mdn_from_ext(
+                                        if let Some((chat_id, msg_id)) = message::mdn_from_ext(
                                             context,
                                             from_id,
                                             &rfc724_mid,
                                             sent_timestamp,
-                                            &mut chat_id_0,
-                                            &mut msg_id,
                                         ) {
-                                            rr_event_to_send.push((chat_id_0, msg_id));
+                                            rr_event_to_send.push((chat_id, msg_id));
+                                            mdn_consumed = 1;
                                         }
-                                        mdn_consumed = (msg_id != 0) as libc::c_int;
                                     }
                                 }
                             }
@@ -845,7 +847,7 @@ fn save_locations(
     mime_parser: &MimeParser,
     chat_id: u32,
     from_id: u32,
-    insert_msg_id: u32,
+    insert_msg_id: MsgId,
     hidden: i32,
 ) {
     let mut location_id_written = false;
@@ -1219,7 +1221,7 @@ unsafe fn create_or_lookup_group(
             "grp-image-change {} chat {}", X_MrGrpImageChanged, chat_id
         );
         let mut changed = false;
-        let mut grpimage = "".to_string();
+        let mut grpimage: Option<BlobObject> = None;
         if X_MrGrpImageChanged == "0" {
             changed = true;
         } else {
@@ -1227,22 +1229,27 @@ unsafe fn create_or_lookup_group(
                 if part.typ == Viewtype::Image {
                     grpimage = part
                         .param
-                        .get(Param::File)
-                        .map(|s| s.to_string())
-                        .unwrap_or_else(|| "".to_string());
+                        .get_blob(Param::File, context, true)
+                        .unwrap_or(None);
                     info!(context, "found image {:?}", grpimage);
                     changed = true;
                 }
             }
         }
         if changed {
-            info!(context, "New group image set to '{}'.", grpimage);
+            info!(
+                context,
+                "New group image set to '{}'.",
+                grpimage
+                    .as_ref()
+                    .map(|blob| blob.as_name().to_string())
+                    .unwrap_or_default()
+            );
             if let Ok(mut chat) = Chat::load_from_db(context, chat_id) {
-                if grpimage.is_empty() {
-                    chat.param.remove(Param::ProfileImage);
-                } else {
-                    chat.param.set(Param::ProfileImage, grpimage);
-                }
+                match grpimage {
+                    Some(blob) => chat.param.set(Param::ProfileImage, blob.as_name()),
+                    None => chat.param.remove(Param::ProfileImage),
+                };
                 chat.update_param(context)?;
                 send_EVENT_CHAT_MODIFIED = 1;
             }

src/dc_simplify.rs

@@ -142,8 +142,8 @@ impl Simplify {
             ret += "[...]";
         }
         /* we write empty lines only in case and non-empty line follows */
-        let mut pending_linebreaks: libc::c_int = 0i32;
-        let mut content_lines_added: libc::c_int = 0i32;
+        let mut pending_linebreaks = 0;
+        let mut content_lines_added = 0;
         for l in l_first..l_last {
             let line = lines[l];
             if is_empty_line(line) {

src/dc_tools.rs

@@ -392,10 +392,6 @@ pub(crate) fn dc_get_abs_path<P: AsRef<std::path::Path>>(
     }
 }
 
-pub(crate) fn dc_file_exist(context: &Context, path: impl AsRef<std::path::Path>) -> bool {
-    dc_get_abs_path(context, &path).exists()
-}
-
 pub(crate) fn dc_get_filebytes(context: &Context, path: impl AsRef<std::path::Path>) -> u64 {
     let path_abs = dc_get_abs_path(context, &path);
     match fs::metadata(&path_abs) {
@@ -545,41 +541,6 @@ pub(crate) fn dc_get_next_backup_path(
     bail!("could not create backup file, disk full?");
 }
 
-pub(crate) fn dc_is_blobdir_path(context: &Context, path: impl AsRef<str>) -> bool {
-    context
-        .get_blobdir()
-        .to_str()
-        .map(|s| path.as_ref().starts_with(s))
-        .unwrap_or_default()
-        || path.as_ref().starts_with("$BLOBDIR")
-}
-
-fn dc_make_rel_path(context: &Context, path: &mut String) {
-    if context
-        .get_blobdir()
-        .to_str()
-        .map(|s| path.starts_with(s))
-        .unwrap_or_default()
-    {
-        *path = path.replace(
-            context.get_blobdir().to_str().unwrap_or_default(),
-            "$BLOBDIR",
-        );
-    }
-}
-
-pub(crate) fn dc_make_rel_and_copy(context: &Context, path: &mut String) -> bool {
-    if dc_is_blobdir_path(context, &path) {
-        dc_make_rel_path(context, path);
-        return true;
-    }
-    if let Ok(blobdir_path) = context.copy_to_blobdir(&path) {
-        *path = blobdir_path;
-        return true;
-    }
-    false
-}
-
 /// Error type for the [OsStrExt] trait
 #[derive(Debug, Fail, PartialEq)]
 pub enum CStringError {
@@ -735,6 +696,14 @@ pub fn to_string_lossy(s: *const libc::c_char) -> String {
     cstr.to_string_lossy().to_string()
 }
 
+pub fn to_opt_string_lossy(s: *const libc::c_char) -> Option<String> {
+    if s.is_null() {
+        return None;
+    }
+
+    Some(to_string_lossy(s))
+}
+
 pub fn as_str<'a>(s: *const libc::c_char) -> &'a str {
     as_str_safe(s).unwrap_or_else(|err| panic!("{}", err))
 }
@@ -1012,21 +981,6 @@ mod tests {
         }
     }
 
-    fn strndup(s: *const libc::c_char, n: libc::c_ulong) -> *mut libc::c_char {
-        if s.is_null() {
-            return std::ptr::null_mut();
-        }
-
-        let end = std::cmp::min(n as usize, unsafe { strlen(s) });
-        unsafe {
-            let result = libc::malloc(end + 1);
-            memcpy(result, s as *const _, end);
-            std::ptr::write_bytes(result.offset(end as isize), b'\x00', 1);
-
-            result as *mut _
-        }
-    }
-
     #[test]
     fn test_dc_str_to_clist_1() {
         unsafe {
@@ -1292,32 +1246,6 @@ mod tests {
         assert_eq!(res, Some("123-45-7@stub".into()));
     }
 
-    #[test]
-    fn test_dc_make_rel_path() {
-        let t = dummy_context();
-        let mut foo: String = t
-            .ctx
-            .get_blobdir()
-            .join("foo")
-            .to_string_lossy()
-            .into_owned();
-        dc_make_rel_path(&t.ctx, &mut foo);
-        assert_eq!(foo, format!("$BLOBDIR{}foo", std::path::MAIN_SEPARATOR));
-    }
-
-    #[test]
-    fn test_strndup() {
-        unsafe {
-            let res = strndup(b"helloworld\x00" as *const u8 as *const libc::c_char, 4);
-            assert_eq!(
-                to_string_lossy(res),
-                to_string_lossy(b"hell\x00" as *const u8 as *const libc::c_char)
-            );
-            assert_eq!(strlen(res), 4);
-            free(res as *mut _);
-        }
-    }
-
     #[test]
     fn test_file_get_safe_basename() {
         assert_eq!(get_safe_basename("12312/hello"), "hello");
@@ -1335,6 +1263,11 @@ mod tests {
     fn test_file_handling() {
         let t = dummy_context();
         let context = &t.ctx;
+        let dc_file_exist = |ctx: &Context, fname: &str| {
+            ctx.get_blobdir()
+                .join(Path::new(fname).file_name().unwrap())
+                .exists()
+        };
 
         assert!(!dc_delete_file(context, "$BLOBDIR/lkqwjelqkwlje"));
         if dc_file_exist(context, "$BLOBDIR/foobar")
@@ -1358,10 +1291,6 @@ mod tests {
             .to_string_lossy()
             .to_string();
 
-        assert!(dc_is_blobdir_path(context, &abs_path));
-
-        assert!(dc_is_blobdir_path(context, "$BLOBDIR/fofo",));
-        assert!(!dc_is_blobdir_path(context, "/BLOBDIR/fofo",));
         assert!(dc_file_exist(context, &abs_path));
 
         assert!(dc_copy_file(context, "$BLOBDIR/foobar", "$BLOBDIR/dada",));

src/error.rs

@@ -30,6 +30,10 @@ pub enum Error {
     Base64Decode(base64::DecodeError),
     #[fail(display = "{:?}", _0)]
     FromUtf8(std::string::FromUtf8Error),
+    #[fail(display = "{}", _0)]
+    BlobError(#[cause] crate::blob::BlobError),
+    #[fail(display = "Invalid Message ID.")]
+    InvalidMsgId,
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -94,6 +98,18 @@ impl From<std::string::FromUtf8Error> for Error {
     }
 }
 
+impl From<crate::blob::BlobError> for Error {
+    fn from(err: crate::blob::BlobError) -> Error {
+        Error::BlobError(err)
+    }
+}
+
+impl From<crate::message::InvalidMsgId> for Error {
+    fn from(_err: crate::message::InvalidMsgId) -> Error {
+        Error::InvalidMsgId
+    }
+}
+
 #[macro_export]
 macro_rules! bail {
     ($e:expr) => {

src/events.rs

@@ -2,6 +2,8 @@ use std::path::PathBuf;
 
 use strum::EnumProperty;
 
+use crate::message::MsgId;
+
 impl Event {
     /// Returns the corresponding Event id.
     pub fn as_id(&self) -> i32 {
@@ -52,6 +54,12 @@ pub enum Event {
     #[strum(props(id = "105"))]
     ImapMessageMoved(String),
 
+    /// Emitted when an IMAP folder was emptied
+    ///
+    /// @return 0
+    #[strum(props(id = "106"))]
+    ImapFolderEmptied(String),
+
     /// Emitted when an new file in the $BLOBDIR was created
     ///
     /// @return 0
@@ -125,7 +133,7 @@ pub enum Event {
     ///
     /// @return 0
     #[strum(props(id = "2000"))]
-    MsgsChanged { chat_id: u32, msg_id: u32 },
+    MsgsChanged { chat_id: u32, msg_id: MsgId },
 
     /// There is a fresh message. Typically, the user will show an notification
     /// when receiving this message.
@@ -134,28 +142,28 @@ pub enum Event {
     ///
     /// @return 0
     #[strum(props(id = "2005"))]
-    IncomingMsg { chat_id: u32, msg_id: u32 },
+    IncomingMsg { chat_id: u32, msg_id: MsgId },
 
     /// A single message is sent successfully. State changed from  DC_STATE_OUT_PENDING to
     /// DC_STATE_OUT_DELIVERED, see dc_msg_get_state().
     ///
     /// @return 0
     #[strum(props(id = "2010"))]
-    MsgDelivered { chat_id: u32, msg_id: u32 },
+    MsgDelivered { chat_id: u32, msg_id: MsgId },
 
     /// A single message could not be sent. State changed from DC_STATE_OUT_PENDING or DC_STATE_OUT_DELIVERED to
     /// DC_STATE_OUT_FAILED, see dc_msg_get_state().
     ///
     /// @return 0
     #[strum(props(id = "2012"))]
-    MsgFailed { chat_id: u32, msg_id: u32 },
+    MsgFailed { chat_id: u32, msg_id: MsgId },
 
     /// A single message is read by the receiver. State changed from DC_STATE_OUT_DELIVERED to
     /// DC_STATE_OUT_MDN_RCVD, see dc_msg_get_state().
     ///
     /// @return 0
     #[strum(props(id = "2015"))]
-    MsgRead { chat_id: u32, msg_id: u32 },
+    MsgRead { chat_id: u32, msg_id: MsgId },
 
     /// Chat changed.  The name or the image of a chat group was changed or members were added or removed.
     /// Or the verify state of a chat has changed.

src/imap.rs

@@ -5,6 +5,7 @@ use std::sync::{
 };
 use std::time::{Duration, SystemTime};
 
+use crate::configure::dc_connect_to_configured_imap;
 use crate::constants::*;
 use crate::context::Context;
 use crate::dc_receive_imf::dc_receive_imf;
@@ -15,9 +16,11 @@ use crate::login_param::{dc_build_tls, CertificateChecks, LoginParam};
 use crate::message::{self, update_msg_move_state, update_server_uid};
 use crate::oauth2::dc_get_oauth2_access_token;
 use crate::param::Params;
+use crate::stock::StockMessage;
 use crate::wrapmime;
 
 const DC_IMAP_SEEN: usize = 0x0001;
+const DCC_IMAP_DEBUG: &str = "DCC_IMAP_DEBUG";
 
 #[derive(Debug, Display, Clone, Copy, PartialEq, Eq)]
 pub enum ImapResult {
@@ -29,6 +32,7 @@ pub enum ImapResult {
 
 const PREFETCH_FLAGS: &str = "(UID ENVELOPE)";
 const BODY_FLAGS: &str = "(FLAGS BODY.PEEK[])";
+const SELECT_ALL: &str = "1:*";
 
 #[derive(Debug)]
 pub struct Imap {
@@ -96,7 +100,7 @@ impl<'a> IdleHandle<'a> {
         }
     }
 
-    pub fn wait_keepalive(self) -> imap::error::Result<()> {
+    pub fn wait_keepalive(self) -> imap::error::Result<bool> {
         match self {
             IdleHandle::Secure(i) => i.wait_keepalive(),
             IdleHandle::Insecure(i) => i.wait_keepalive(),
@@ -116,8 +120,11 @@ impl Client {
         let s = stream.try_clone().expect("cloning the stream failed");
         let tls_stream = native_tls::TlsConnector::connect(&tls, domain.as_ref(), s)?;
 
-        let client = imap::Client::new(tls_stream);
-        // TODO: Read greeting
+        let mut client = imap::Client::new(tls_stream);
+        if std::env::var(DCC_IMAP_DEBUG).is_ok() {
+            client.debug = true;
+        }
+        client.read_greeting()?;
 
         Ok(Client::Secure(client, stream))
     }
@@ -125,8 +132,11 @@ impl Client {
     pub fn connect_insecure<A: net::ToSocketAddrs>(addr: A) -> imap::error::Result<Self> {
         let stream = net::TcpStream::connect(addr)?;
 
-        let client = imap::Client::new(stream.try_clone().unwrap());
-        // TODO: Read greeting
+        let mut client = imap::Client::new(stream.try_clone().unwrap());
+        if std::env::var(DCC_IMAP_DEBUG).is_ok() {
+            client.debug = true;
+        }
+        client.read_greeting()?;
 
         Ok(Client::Insecure(client, stream))
     }
@@ -437,15 +447,14 @@ impl Imap {
                 let config = self.config.read().unwrap();
                 let imap_server: &str = config.imap_server.as_ref();
                 let imap_port = config.imap_port;
-
-                emit_event!(
-                    context,
-                    Event::ErrorNetwork(format!(
-                        "Could not connect to IMAP-server {}:{}. ({})",
-                        imap_server, imap_port, err
-                    ))
+                let message = context.stock_string_repl_str2(
+                    StockMessage::ServerResponse,
+                    format!("{}:{}", imap_server, imap_port),
+                    format!("{}", err),
                 );
 
+                emit_event!(context, Event::ErrorNetwork(message));
+
                 return false;
             }
         };
@@ -459,9 +468,12 @@ impl Imap {
                 true
             }
             Err((err, _)) => {
+                let imap_user = self.config.read().unwrap().imap_user.to_owned();
+                let message = context.stock_string_repl_str(StockMessage::CannotLogin, &imap_user);
+
                 emit_event!(
                     context,
-                    Event::ErrorNetwork(format!("Cannot login ({})", err))
+                    Event::ErrorNetwork(format!("{} ({})", message, err))
                 );
                 self.unsetup_handle(context);
 
@@ -471,13 +483,11 @@ impl Imap {
     }
 
     fn unsetup_handle(&self, context: &Context) {
-        info!(context, "IMAP unsetup_handle starts");
-
         info!(context, "IMAP unsetup_handle step 1 (closing down stream).");
         let stream = self.stream.write().unwrap().take();
         if let Some(stream) = stream {
             if let Err(err) = stream.shutdown(net::Shutdown::Both) {
-                eprintln!("failed to shutdown connection: {:?}", err);
+                warn!(context, "failed to shutdown connection: {:?}", err);
             }
         }
 
@@ -487,7 +497,7 @@ impl Imap {
         );
         if let Some(mut session) = self.session.lock().unwrap().take() {
             if let Err(err) = session.close() {
-                eprintln!("failed to close connection: {:?}", err);
+                warn!(context, "failed to close connection: {:?}", err);
             }
         }
 
@@ -598,9 +608,9 @@ impl Imap {
         self.config.write().unwrap().watch_folder = Some(watch_folder);
     }
 
-    pub fn fetch(&self, context: &Context) -> libc::c_int {
+    pub fn fetch(&self, context: &Context) -> bool {
         if !self.is_connected() || !context.sql.is_open() {
-            return 0;
+            return false;
         }
 
         self.setup_handle_if_needed(context);
@@ -616,9 +626,9 @@ impl Imap {
                     break;
                 }
             }
-            1
+            true
         } else {
-            0
+            false
         }
     }
 
@@ -641,7 +651,8 @@ impl Imap {
         }
 
         // deselect existing folder, if needed (it's also done implicitly by SELECT, however, without EXPUNGE then)
-        if self.config.read().unwrap().selected_folder_needs_expunge {
+        let needs_expunge = { self.config.read().unwrap().selected_folder_needs_expunge };
+        if needs_expunge {
             if let Some(ref folder) = self.config.read().unwrap().selected_folder {
                 info!(context, "Expunge messages in \"{}\".", folder);
 
@@ -649,16 +660,19 @@ impl Imap {
                 // https://tools.ietf.org/html/rfc3501#section-6.4.2
                 if let Some(ref mut session) = &mut *self.session.lock().unwrap() {
                     match session.close() {
-                        Ok(_) => {}
+                        Ok(_) => {
+                            info!(context, "close/expunge succeeded");
+                        }
                         Err(err) => {
-                            eprintln!("failed to close session: {:?}", err);
+                            warn!(context, "failed to close session: {:?}", err);
+                            return 0;
                         }
                     }
                 } else {
                     return 0;
                 }
-                self.config.write().unwrap().selected_folder_needs_expunge = true;
             }
+            self.config.write().unwrap().selected_folder_needs_expunge = false;
         }
 
         // select new folder
@@ -991,23 +1005,36 @@ impl Imap {
             std::thread::spawn(move || {
                 let &(ref lock, ref cvar) = &*v;
                 if let Some(ref mut session) = &mut *session.lock().unwrap() {
-                    let mut idle = match session.idle() {
-                        Ok(idle) => idle,
-                        Err(err) => {
-                            eprintln!("failed to setup idle: {:?}", err);
-                            return;
+                    loop {
+                        let res = match session.idle() {
+                            Ok(mut idle) => {
+                                // most servers do not allow more than ~28 minutes; stay clearly below that.
+                                // a good value that is also used by other MUAs is 23 minutes.
+                                // if needed, the ui can call dc_imap_interrupt_idle() to trigger a reconnect.
+                                // idle.set_keepalive(Duration::from_secs(23 * 60));
+                                idle.set_keepalive(Duration::from_secs(23 * 60));
+                                idle.wait_keepalive()
+                            }
+                            Err(err) => {
+                                let _ = sender.send(Err(imap::error::Error::Bad(err.to_string())));
+                                break;
+                            }
+                        };
+                        match res {
+                            Ok(true) => {
+                                let _ = sender.send(Ok(()));
+                                break;
+                            }
+                            Ok(false) => {} // continue loop
+                            Err(err) => {
+                                let _ = sender.send(Err(imap::error::Error::Bad(format!(
+                                    "wait_keepalive failed {}",
+                                    err
+                                ))));
+                                break;
+                            }
                         }
-                    };
-
-                    // most servers do not allow more than ~28 minutes; stay clearly below that.
-                    // a good value that is also used by other MUAs is 23 minutes.
-                    // if needed, the ui can call dc_imap_interrupt_idle() to trigger a reconnect.
-                    idle.set_keepalive(Duration::from_secs(23 * 60));
-                    let res = idle.wait_keepalive();
-
-                    // Ignoring the error, as this happens when we try sending after the drop
-                    let _send_res = sender.send(res);
-
+                    }
                     // Trigger condvar
                     let mut watch = lock.lock().unwrap();
                     *watch = true;
@@ -1025,8 +1052,12 @@ impl Imap {
                 info!(context, "IMAP-IDLE has data.");
             }
             Err(err) => match err {
-                imap::error::Error::ConnectionLost => {
+                imap::error::Error::Io(_)
+                | imap::error::Error::ConnectionLost
+                | imap::error::Error::Bad(_) => {
                     info!(context, "IMAP-IDLE wait cancelled, we will reconnect soon.");
+                    self.unsetup_handle(context);
+                    info!(context, "IMAP-IDLE has SHUTDOWN");
                     self.should_reconnect.store(true, Ordering::Relaxed);
                 }
                 _ => {
@@ -1098,20 +1129,28 @@ impl Imap {
                 return;
             }
 
-            // check for new messages. fetch_from_single_folder() has the side-effect that messages
-            // are also downloaded, however, typically this would take place in the FETCH command
-            // following IDLE otherwise, so this seems okay here.
-            if self.setup_handle_if_needed(context) {
-                if let Some(ref watch_folder) = self.config.read().unwrap().watch_folder {
-                    if 0 != self.fetch_from_single_folder(context, watch_folder) {
-                        do_fake_idle = false;
-                    }
+            // check if we want to finish fake-idling.
+            if !self.is_connected() {
+                // try to connect with proper login params
+                // (setup_handle_if_needed might not know about them if we
+                // never successfully connected)
+                if dc_connect_to_configured_imap(context, &self) != 0 {
+                    return;
                 }
-            } else {
-                // if we cannot connect, set the starting time to a small value which will
-                // result in larger timeouts (60 instead of 5 seconds) for re-checking the availablility of network.
-                // to get the _exact_ moment of re-available network, the ui should call interrupt_idle()
+                // we cannot connect, wait long next time (currently 60 secs, see above)
                 wait_long = true;
+                continue;
+            }
+            // we are connected, let's see if fetching messages results
+            // in anything.  If so, we behave as if IDLE had data but
+            // will have already fetched the messages so perform_*_fetch
+            // will not find any new.
+
+            let watch_folder = self.config.read().unwrap().watch_folder.clone();
+            if let Some(watch_folder) = watch_folder {
+                if 0 != self.fetch_from_single_folder(context, watch_folder) {
+                    do_fake_idle = false;
+                }
             }
         }
     }
@@ -1207,18 +1246,22 @@ impl Imap {
         if server_uid == 0 {
             return true; // might be moved but we don't want to have a stuck job
         }
+        let s = server_uid.to_string();
+        self.add_flag_finalized_with_set(context, &s, flag)
+    }
+
+    fn add_flag_finalized_with_set(&self, context: &Context, uid_set: &str, flag: &str) -> bool {
         if self.should_reconnect() {
             return false;
         }
         if let Some(ref mut session) = &mut *self.session.lock().unwrap() {
-            let set = format!("{}", server_uid);
             let query = format!("+FLAGS ({})", flag);
-            match session.uid_store(&set, &query) {
+            match session.uid_store(uid_set, &query) {
                 Ok(_) => {}
                 Err(err) => {
                     warn!(
                         context,
-                        "IMAP failed to store: ({}, {}) {:?}", set, query, err
+                        "IMAP failed to store: ({}, {}) {:?}", uid_set, query, err
                     );
                 }
             }
@@ -1454,6 +1497,30 @@ impl Imap {
             None
         }
     }
+
+    pub fn empty_folder(&self, context: &Context, folder: &str) {
+        info!(context, "emptying folder {}", folder);
+
+        if folder.is_empty() || self.select_folder(context, Some(&folder)) == 0 {
+            warn!(context, "Cannot select folder '{}' for emptying", folder);
+            return;
+        }
+
+        if !self.add_flag_finalized_with_set(context, SELECT_ALL, "\\Deleted") {
+            warn!(context, "Cannot empty folder {}", folder);
+        } else {
+            // we now trigger expunge to actually delete messages
+            self.config.write().unwrap().selected_folder_needs_expunge = true;
+            if self.select_folder::<String>(context, None) == 0 {
+                warn!(
+                    context,
+                    "could not perform expunge on empty-marked folder {}", folder
+                );
+            } else {
+                emit_event!(context, Event::ImapFolderEmptied(folder.to_string()));
+            }
+        }
+    }
 }
 
 /// Try to get the folder meaning by the name of the folder only used if the server does not support XLIST.
@@ -1509,7 +1576,7 @@ fn precheck_imf(context: &Context, rfc724_mid: &str, server_folder: &str, server
             job_add(
                 context,
                 Action::MarkseenMsgOnImap,
-                msg_id as libc::c_int,
+                msg_id.to_u32() as i32,
                 Params::new(),
                 0,
             );

src/imex.rs

@@ -4,18 +4,20 @@ use std::path::{Path, PathBuf};
 use num_traits::FromPrimitive;
 use rand::{thread_rng, Rng};
 
+use crate::blob::BlobObject;
 use crate::chat;
 use crate::config::Config;
 use crate::configure::*;
 use crate::constants::*;
 use crate::context::Context;
+use crate::dc_mimeparser::SystemMessage;
 use crate::dc_tools::*;
 use crate::e2ee;
 use crate::error::*;
 use crate::events::Event;
 use crate::job::*;
 use crate::key::*;
-use crate::message::Message;
+use crate::message::{Message, MsgId};
 use crate::param::*;
 use crate::pgp;
 use crate::sql::{self, Sql};
@@ -87,7 +89,7 @@ pub fn has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Result<Strin
                 let name = name.to_string_lossy();
                 if name.starts_with("delta-chat") && name.ends_with(".bak") {
                     let sql = Sql::new();
-                    if sql.open(context, &path, 0x1) {
+                    if sql.open(context, &path, true) {
                         let curr_backup_time =
                             sql.get_raw_config_int(context, "backup_time")
                                 .unwrap_or_default() as u64;
@@ -122,19 +124,20 @@ fn do_initiate_key_transfer(context: &Context) -> Result<String> {
     let setup_file_content = render_setup_file(context, &setup_code)?;
     /* encrypting may also take a while ... */
     ensure!(!context.shall_stop_ongoing(), "canceled");
-    let setup_file_name = context.new_blob_file(
+    let setup_file_blob = BlobObject::create(
+        context,
         "autocrypt-setup-message.html",
         setup_file_content.as_bytes(),
     )?;
 
-    let chat_id = chat::create_by_contact_id(context, 1)?;
+    let chat_id = chat::create_by_contact_id(context, DC_CONTACT_ID_SELF)?;
     msg = Message::default();
     msg.type_0 = Viewtype::File;
-    msg.param.set(Param::File, setup_file_name);
+    msg.param.set(Param::File, setup_file_blob.as_name());
 
     msg.param
         .set(Param::MimeType, "application/autocrypt-setup");
-    msg.param.set_int(Param::Cmd, 6);
+    msg.param.set_cmd(SystemMessage::AutocryptSetupMessage);
     msg.param
         .set_int(Param::ForcePlaintext, DC_FP_NO_AUTOCRYPT_HEADER);
 
@@ -225,8 +228,8 @@ pub fn create_setup_code(_context: &Context) -> String {
     ret
 }
 
-pub fn continue_key_transfer(context: &Context, msg_id: u32, setup_code: &str) -> Result<()> {
-    ensure!(msg_id > DC_MSG_ID_LAST_SPECIAL, "wrong id");
+pub fn continue_key_transfer(context: &Context, msg_id: MsgId, setup_code: &str) -> Result<()> {
+    ensure!(!msg_id.is_special(), "wrong id");
 
     let msg = Message::load_from_db(context, msg_id);
     if msg.is_err() {
@@ -402,7 +405,7 @@ fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Resul
     context.sql.close(&context);
     dc_delete_file(context, context.get_dbfile());
     ensure!(
-        !dc_file_exist(context, context.get_dbfile()),
+        !context.get_dbfile().exists(),
         "Cannot delete old database."
     );
 
@@ -413,7 +416,7 @@ fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Resul
     /* error already logged */
     /* re-open copied database file */
     ensure!(
-        context.sql.open(&context, &context.get_dbfile(), 0),
+        context.sql.open(&context, &context.get_dbfile(), false),
         "could not re-open db"
     );
 
@@ -496,7 +499,7 @@ fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
         dest_path_filename.display(),
     );
     let copied = dc_copy_file(context, context.get_dbfile(), &dest_path_filename);
-    context.sql.open(&context, &context.get_dbfile(), 0);
+    context.sql.open(&context, &context.get_dbfile(), false);
     if !copied {
         let s = dest_path_filename.to_string_lossy().to_string();
         bail!(
@@ -526,7 +529,7 @@ fn add_files_to_export(context: &Context, dest_path_filename: &PathBuf) -> Resul
     // the source to be locked, neigher the destination as it is used only here)
     let sql = Sql::new();
     ensure!(
-        sql.open(context, &dest_path_filename, 0),
+        sql.open(context, &dest_path_filename, false),
         "could not open db"
     );
     if !sql.table_exists("backup_blobs") {

src/job.rs

@@ -3,6 +3,7 @@ use std::time::Duration;
 use deltachat_derive::{FromSql, ToSql};
 use rand::{thread_rng, Rng};
 
+use crate::blob::BlobObject;
 use crate::chat;
 use crate::config::Config;
 use crate::configure::*;
@@ -15,6 +16,7 @@ use crate::imap::*;
 use crate::imex::*;
 use crate::location;
 use crate::login_param::LoginParam;
+use crate::message::MsgId;
 use crate::message::{self, Message, MessageState};
 use crate::mimefactory::{vec_contains_lowercase, Loaded, MimeFactory};
 use crate::param::*;
@@ -42,6 +44,7 @@ pub enum Action {
 
     // Jobs in the INBOX-thread, range from DC_IMAP_THREAD..DC_IMAP_THREAD+999
     Housekeeping = 105, // low priority ...
+    EmptyServer = 107,
     DeleteMsgOnImap = 110,
     MarkseenMdnOnImap = 120,
     MarkseenMsgOnImap = 130,
@@ -73,6 +76,7 @@ impl From<Action> for Thread {
 
             Housekeeping => Thread::Imap,
             DeleteMsgOnImap => Thread::Imap,
+            EmptyServer => Thread::Imap,
             MarkseenMdnOnImap => Thread::Imap,
             MarkseenMsgOnImap => Thread::Imap,
             MoveMsg => Thread::Imap,
@@ -133,13 +137,13 @@ impl Job {
             let connected = context.smtp.lock().unwrap().connect(context, &loginparam);
 
             if !connected {
-                self.try_again_later(3i32, None);
+                self.try_again_later(3, None);
                 return;
             }
         }
 
-        if let Some(filename) = self.param.get(Param::File) {
-            if let Ok(body) = dc_read_file(context, filename) {
+        if let Some(filename) = self.param.get_path(Param::File, context).unwrap_or(None) {
+            if let Ok(body) = dc_read_file(context, &filename) {
                 if let Some(recipients) = self.param.get(Param::Recipients) {
                     let recipients_list = recipients
                         .split('\x1e')
@@ -155,7 +159,9 @@ impl Job {
                     /* if there is a msg-id and it does not exist in the db, cancel sending.
                     this happends if dc_delete_msgs() was called
                     before the generated mime was sent out */
-                    if 0 != self.foreign_id && !message::exists(context, self.foreign_id) {
+                    if 0 != self.foreign_id
+                        && !message::exists(context, MsgId::new(self.foreign_id))
+                    {
                         warn!(
                             context,
                             "Not sending Message {} as it was deleted", self.foreign_id
@@ -172,14 +178,14 @@ impl Job {
                         Err(err) => {
                             sock.disconnect();
                             warn!(context, "smtp failed: {}", err);
-                            self.try_again_later(-1i32, Some(err.to_string()));
+                            self.try_again_later(-1, Some(err.to_string()));
                         }
                         Ok(()) => {
                             // smtp success, update db ASAP, then delete smtp file
                             if 0 != self.foreign_id {
                                 message::update_msg_state(
                                     context,
-                                    self.foreign_id,
+                                    MsgId::new(self.foreign_id),
                                     MessageState::OutDelivered,
                                 );
                                 let chat_id: i32 = context
@@ -192,7 +198,7 @@ impl Job {
                                     .unwrap_or_default();
                                 context.call_cb(Event::MsgDelivered {
                                     chat_id: chat_id as u32,
-                                    msg_id: self.foreign_id,
+                                    msg_id: MsgId::new(self.foreign_id),
                                 });
                             }
                             // now also delete the generated file
@@ -207,7 +213,7 @@ impl Job {
     }
 
     // this value does not increase the number of tries
-    fn try_again_later(&mut self, try_again: libc::c_int, pending_error: Option<String>) {
+    fn try_again_later(&mut self, try_again: i32, pending_error: Option<String>) {
         self.try_again = try_again;
         self.pending_error = pending_error;
     }
@@ -216,7 +222,7 @@ impl Job {
     fn do_DC_JOB_MOVE_MSG(&mut self, context: &Context) {
         let inbox = context.inbox.read().unwrap();
 
-        if let Ok(msg) = Message::load_from_db(context, self.foreign_id) {
+        if let Ok(msg) = Message::load_from_db(context, MsgId::new(self.foreign_id)) {
             if context
                 .sql
                 .get_raw_config_int(context, "folders_configured")
@@ -261,7 +267,7 @@ impl Job {
     fn do_DC_JOB_DELETE_MSG_ON_IMAP(&mut self, context: &Context) {
         let inbox = context.inbox.read().unwrap();
 
-        if let Ok(mut msg) = Message::load_from_db(context, self.foreign_id) {
+        if let Ok(mut msg) = Message::load_from_db(context, MsgId::new(self.foreign_id)) {
             if !msg.rfc724_mid.is_empty() {
                 /* eg. device messages have no Message-ID */
                 if message::rfc724_mid_cnt(context, &msg.rfc724_mid) > 1 {
@@ -285,11 +291,27 @@ impl Job {
         }
     }
 
+    #[allow(non_snake_case)]
+    fn do_DC_JOB_EMPTY_SERVER(&mut self, context: &Context) {
+        let inbox = context.inbox.read().unwrap();
+        if self.foreign_id & DC_EMPTY_MVBOX > 0 {
+            if let Some(mvbox_folder) = context
+                .sql
+                .get_raw_config(context, "configured_mvbox_folder")
+            {
+                inbox.empty_folder(context, &mvbox_folder);
+            }
+        }
+        if self.foreign_id & DC_EMPTY_INBOX > 0 {
+            inbox.empty_folder(context, "INBOX");
+        }
+    }
+
     #[allow(non_snake_case)]
     fn do_DC_JOB_MARKSEEN_MSG_ON_IMAP(&mut self, context: &Context) {
         let inbox = context.inbox.read().unwrap();
 
-        if let Ok(msg) = Message::load_from_db(context, self.foreign_id) {
+        if let Ok(msg) = Message::load_from_db(context, MsgId::new(self.foreign_id)) {
             let folder = msg.server_folder.as_ref().unwrap();
             match inbox.set_seen(context, folder, msg.server_uid) {
                 ImapResult::RetryLater => {
@@ -560,15 +582,11 @@ pub fn job_action_exists(context: &Context, action: Action) -> bool {
 
 /* special case for DC_JOB_SEND_MSG_TO_SMTP */
 #[allow(non_snake_case)]
-pub fn job_send_msg(context: &Context, msg_id: u32) -> Result<(), Error> {
+pub fn job_send_msg(context: &Context, msg_id: MsgId) -> Result<(), Error> {
     let mut mimefactory = MimeFactory::load_msg(context, msg_id)?;
 
     if chat::msgtype_has_file(mimefactory.msg.type_0) {
-        let file_param = mimefactory
-            .msg
-            .param
-            .get(Param::File)
-            .map(|s| s.to_string());
+        let file_param = mimefactory.msg.param.get_path(Param::File, context)?;
         if let Some(pathNfilename) = file_param {
             if (mimefactory.msg.type_0 == Viewtype::Image
                 || mimefactory.msg.type_0 == Viewtype::Gif)
@@ -597,7 +615,7 @@ pub fn job_send_msg(context: &Context, msg_id: u32) -> Result<(), Error> {
     if 0 != mimefactory
         .msg
         .param
-        .get_int(Param::GuranteeE2ee)
+        .get_int(Param::GuaranteeE2ee)
         .unwrap_or_default()
         && !mimefactory.out_encrypted
     {
@@ -610,7 +628,7 @@ pub fn job_send_msg(context: &Context, msg_id: u32) -> Result<(), Error> {
         bail!(
             "e2e encryption unavailable {} - {:?}",
             msg_id,
-            mimefactory.msg.param.get_int(Param::GuranteeE2ee),
+            mimefactory.msg.param.get_int(Param::GuaranteeE2ee),
         );
     }
     if context.get_config_bool(Config::BccSelf)
@@ -652,11 +670,11 @@ pub fn job_send_msg(context: &Context, msg_id: u32) -> Result<(), Error> {
         && mimefactory
             .msg
             .param
-            .get_int(Param::GuranteeE2ee)
+            .get_int(Param::GuaranteeE2ee)
             .unwrap_or_default()
             == 0
     {
-        mimefactory.msg.param.set_int(Param::GuranteeE2ee, 1);
+        mimefactory.msg.param.set_int(Param::GuaranteeE2ee, 1);
         mimefactory.msg.save_param_to_disk(context);
     }
     add_smtp_job(context, Action::SendMsgToSmtp, &mut mimefactory)?;
@@ -776,6 +794,7 @@ fn job_perform(context: &Context, thread: Thread, probe_network: bool) {
                     warn!(context, "Unknown job id found");
                 }
                 Action::SendMsgToSmtp => job.do_DC_JOB_SEND(context),
+                Action::EmptyServer => job.do_DC_JOB_EMPTY_SERVER(context),
                 Action::DeleteMsgOnImap => job.do_DC_JOB_DELETE_MSG_ON_IMAP(context),
                 Action::MarkseenMsgOnImap => job.do_DC_JOB_MARKSEEN_MSG_ON_IMAP(context),
                 Action::MarkseenMdnOnImap => job.do_DC_JOB_MARKSEEN_MDN_ON_IMAP(context),
@@ -861,7 +880,11 @@ fn job_perform(context: &Context, thread: Thread, probe_network: bool) {
                 }
             } else {
                 if job.action == Action::SendMsgToSmtp {
-                    message::set_msg_failed(context, job.foreign_id, job.pending_error.as_ref());
+                    message::set_msg_failed(
+                        context,
+                        MsgId::new(job.foreign_id),
+                        job.pending_error.as_ref(),
+                    );
                 }
                 job.delete(context);
             }
@@ -912,7 +935,7 @@ pub fn connect_to_inbox(context: &Context, inbox: &Imap) -> libc::c_int {
     ret_connected
 }
 
-fn send_mdn(context: &Context, msg_id: u32) -> Result<(), Error> {
+fn send_mdn(context: &Context, msg_id: MsgId) -> Result<(), Error> {
     let mut mimefactory = MimeFactory::load_mdn(context, msg_id)?;
     unsafe { mimefactory.render()? };
     add_smtp_job(context, Action::SendMdn, &mut mimefactory)?;
@@ -933,15 +956,15 @@ fn add_smtp_job(context: &Context, action: Action, mimefactory: &MimeFactory) ->
             (*mimefactory.out).len,
         )
     };
-    let bpath = context.new_blob_file(&mimefactory.rfc724_mid, bytes)?;
+    let blob = BlobObject::create(context, &mimefactory.rfc724_mid, bytes)?;
     let recipients = mimefactory.recipients_addr.join("\x1e");
-    param.set(Param::File, &bpath);
+    param.set(Param::File, blob.as_name());
     param.set(Param::Recipients, &recipients);
     job_add(
         context,
         action,
         (if mimefactory.loaded == Loaded::Message {
-            mimefactory.msg.id
+            mimefactory.msg.id.to_u32() as i32
         } else {
             0
         }) as libc::c_int,

src/key.rs

@@ -180,15 +180,15 @@ impl Key {
 
         let encoded = base64::encode(&buf);
         encoded
-            .as_bytes()
-            .chunks(break_every)
-            .fold(String::new(), |mut res, buf| {
-                // safe because we are using a base64 encoded string
-                res += unsafe { std::str::from_utf8_unchecked(buf) };
-                res += " ";
+            .chars()
+            .enumerate()
+            .fold(String::new(), |mut res, (i, c)| {
+                if i > 0 && i % break_every == 0 {
+                    res.push(' ')
+                }
+                res.push(c);
                 res
             })
-            .trim()
             .to_string()
     }
 

src/lib.rs

@@ -30,6 +30,7 @@ pub(crate) mod events;
 pub use events::*;
 
 mod aheader;
+pub mod blob;
 pub mod chat;
 pub mod chatlist;
 pub mod config;

src/location.rs

@@ -6,11 +6,12 @@ use crate::chat;
 use crate::config::Config;
 use crate::constants::*;
 use crate::context::*;
+use crate::dc_mimeparser::SystemMessage;
 use crate::dc_tools::*;
 use crate::error::Error;
 use crate::events::Event;
 use crate::job::*;
-use crate::message::Message;
+use crate::message::{Message, MsgId};
 use crate::param::*;
 use crate::sql;
 use crate::stock::StockMessage;
@@ -195,10 +196,8 @@ impl Kml {
 // location streaming
 pub fn send_locations_to_chat(context: &Context, chat_id: u32, seconds: i64) {
     let now = time();
-    let mut msg: Message;
-    let is_sending_locations_before: bool;
-    if !(seconds < 0 || chat_id <= 9i32 as libc::c_uint) {
-        is_sending_locations_before = is_sending_locations_to_chat(context, chat_id);
+    if !(seconds < 0 || chat_id <= DC_CHAT_ID_LAST_SPECIAL) {
+        let is_sending_locations_before = is_sending_locations_to_chat(context, chat_id);
         if sql::execute(
             context,
             &context.sql,
@@ -215,10 +214,10 @@ pub fn send_locations_to_chat(context: &Context, chat_id: u32, seconds: i64) {
         .is_ok()
         {
             if 0 != seconds && !is_sending_locations_before {
-                msg = Message::new(Viewtype::Text);
+                let mut msg = Message::new(Viewtype::Text);
                 msg.text =
                     Some(context.stock_system_msg(StockMessage::MsgLocationEnabled, "", "", 0));
-                msg.param.set_int(Param::Cmd, 8);
+                msg.param.set_cmd(SystemMessage::LocationStreamingEnabled);
                 chat::send_msg(context, chat_id, &mut msg).unwrap_or_default();
             } else if 0 == seconds && is_sending_locations_before {
                 let stock_str =
@@ -227,7 +226,7 @@ pub fn send_locations_to_chat(context: &Context, chat_id: u32, seconds: i64) {
             }
             context.call_cb(Event::ChatModified(chat_id));
             if 0 != seconds {
-                schedule_MAYBE_SEND_LOCATIONS(context, 0i32);
+                schedule_MAYBE_SEND_LOCATIONS(context, false);
                 job_add(
                     context,
                     Action::MaybeSendLocationsEnded,
@@ -241,8 +240,8 @@ pub fn send_locations_to_chat(context: &Context, chat_id: u32, seconds: i64) {
 }
 
 #[allow(non_snake_case)]
-fn schedule_MAYBE_SEND_LOCATIONS(context: &Context, flags: i32) {
-    if 0 != flags & 0x1 || !job_action_exists(context, Action::MaybeSendLocations) {
+fn schedule_MAYBE_SEND_LOCATIONS(context: &Context, force_schedule: bool) {
+    if force_schedule || !job_action_exists(context, Action::MaybeSendLocations) {
         job_add(context, Action::MaybeSendLocations, 0, Params::new(), 60);
     };
 }
@@ -257,9 +256,9 @@ pub fn is_sending_locations_to_chat(context: &Context, chat_id: u32) -> bool {
         .unwrap_or_default()
 }
 
-pub fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64) -> libc::c_int {
+pub fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64) -> bool {
     if latitude == 0.0 && longitude == 0.0 {
-        return 1;
+        return true;
     }
     let mut continue_streaming = false;
 
@@ -288,12 +287,12 @@ pub fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64) -> l
             }
         }
         if continue_streaming {
-            context.call_cb(Event::LocationChanged(Some(1)));
+            context.call_cb(Event::LocationChanged(Some(DC_CONTACT_ID_SELF)));
         };
-        schedule_MAYBE_SEND_LOCATIONS(context, 0);
+        schedule_MAYBE_SEND_LOCATIONS(context, false);
     }
 
-    continue_streaming as libc::c_int
+    continue_streaming
 }
 
 pub fn get_range(
@@ -403,7 +402,7 @@ pub fn get_kml(context: &Context, chat_id: u32) -> Result<(String, u32), Error>
              AND independent=0 \
              GROUP BY timestamp \
              ORDER BY timestamp;",
-            params![1, locations_send_begin, locations_last_sent, 1],
+            params![DC_CONTACT_ID_SELF, locations_send_begin, locations_last_sent, DC_CONTACT_ID_SELF],
             |row| {
                 let location_id: i32 = row.get(0)?;
                 let latitude: f64 = row.get(1)?;
@@ -476,12 +475,16 @@ pub fn set_kml_sent_timestamp(
     Ok(())
 }
 
-pub fn set_msg_location_id(context: &Context, msg_id: u32, location_id: u32) -> Result<(), Error> {
+pub fn set_msg_location_id(
+    context: &Context,
+    msg_id: MsgId,
+    location_id: u32,
+) -> Result<(), Error> {
     sql::execute(
         context,
         &context.sql,
         "UPDATE msgs SET location_id=? WHERE id=?;",
-        params![location_id, msg_id as i32],
+        params![location_id, msg_id],
     )?;
 
     Ok(())
@@ -494,7 +497,7 @@ pub fn save(
     locations: &[Location],
     independent: i32,
 ) -> Result<u32, Error> {
-    ensure!(chat_id > 9, "Invalid chat id");
+    ensure!(chat_id > DC_CHAT_ID_LAST_SPECIAL, "Invalid chat id");
     context.sql.prepare2(
         "SELECT id FROM locations WHERE timestamp=? AND from_id=?",
         "INSERT INTO locations\
@@ -540,7 +543,7 @@ pub fn save(
 #[allow(non_snake_case)]
 pub fn job_do_DC_JOB_MAYBE_SEND_LOCATIONS(context: &Context, _job: &Job) {
     let now = time();
-    let mut continue_streaming: libc::c_int = 1;
+    let mut continue_streaming = false;
     info!(
         context,
         " ----------------- MAYBE_SEND_LOCATIONS -------------- ",
@@ -555,7 +558,7 @@ pub fn job_do_DC_JOB_MAYBE_SEND_LOCATIONS(context: &Context, _job: &Job) {
             let chat_id: i32 = row.get(0)?;
             let locations_send_begin: i64 = row.get(1)?;
             let locations_last_sent: i64 = row.get(2)?;
-            continue_streaming = 1;
+            continue_streaming = true;
 
             // be a bit tolerant as the timer may not align exactly with time(NULL)
             if now - locations_last_sent < (60 - 3) {
@@ -585,7 +588,11 @@ pub fn job_do_DC_JOB_MAYBE_SEND_LOCATIONS(context: &Context, _job: &Job) {
                         .into_iter()
                         .filter_map(|(chat_id, locations_send_begin, locations_last_sent)| {
                             if !stmt_locations
-                                .exists(params![1, locations_send_begin, locations_last_sent,])
+                                .exists(params![
+                                    DC_CONTACT_ID_SELF,
+                                    locations_send_begin,
+                                    locations_last_sent,
+                                ])
                                 .unwrap_or_default()
                             {
                                 // if there is no new location, there's nothing to send.
@@ -603,7 +610,7 @@ pub fn job_do_DC_JOB_MAYBE_SEND_LOCATIONS(context: &Context, _job: &Job) {
                                 // and dc_set_location() is typically called periodically, this is ok)
                                 let mut msg = Message::new(Viewtype::Text);
                                 msg.hidden = true;
-                                msg.param.set_int(Param::Cmd, 9);
+                                msg.param.set_cmd(SystemMessage::LocationOnly);
                                 Some((chat_id, msg))
                             }
                         })
@@ -618,8 +625,8 @@ pub fn job_do_DC_JOB_MAYBE_SEND_LOCATIONS(context: &Context, _job: &Job) {
             chat::send_msg(context, chat_id as u32, &mut msg).unwrap_or_default();
         }
     }
-    if 0 != continue_streaming {
-        schedule_MAYBE_SEND_LOCATIONS(context, 0x1);
+    if continue_streaming {
+        schedule_MAYBE_SEND_LOCATIONS(context, true);
     }
 }
 

src/message.rs

@@ -1,6 +1,7 @@
 use std::path::{Path, PathBuf};
 
 use deltachat_derive::{FromSql, ToSql};
+use failure::Fail;
 
 use crate::chat::{self, Chat};
 use crate::constants::*;
@@ -17,9 +18,134 @@ use crate::pgp::*;
 use crate::sql;
 use crate::stock::StockMessage;
 
-/// In practice, the user additionally cuts the string himself pixel-accurate.
+// In practice, the user additionally cuts the string themselves
+// pixel-accurate.
 const SUMMARY_CHARACTERS: usize = 160;
 
+/// Message ID, including reserved IDs.
+///
+/// Some message IDs are reserved to identify special message types.
+/// This type can represent both the special as well as normal
+/// messages.
+#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)]
+pub struct MsgId(u32);
+
+impl MsgId {
+    /// Create a new [MsgId].
+    pub fn new(id: u32) -> MsgId {
+        MsgId(id)
+    }
+
+    /// Create a new unset [MsgId].
+    pub fn new_unset() -> MsgId {
+        MsgId(0)
+    }
+
+    /// Whether the message ID signifies a special message.
+    ///
+    /// This kind of message ID can not be used for real messages.
+    pub fn is_special(&self) -> bool {
+        match self.0 {
+            0..=DC_MSG_ID_LAST_SPECIAL => true,
+            _ => false,
+        }
+    }
+
+    /// Whether the message ID is unset.
+    ///
+    /// When a message is created it initially has a ID of `0`, which
+    /// is filled in by a real message ID once the message is saved in
+    /// the database.  This returns true while the message has not
+    /// been saved and thus not yet been given an actual message ID.
+    ///
+    /// When this is `true`, [MsgId::is_special] will also always be
+    /// `true`.
+    pub fn is_unset(&self) -> bool {
+        self.0 == 0
+    }
+
+    /// Whether the message ID is the special marker1 marker.
+    ///
+    /// See the docs of the `dc_get_chat_msgs` C API for details.
+    pub fn is_marker1(&self) -> bool {
+        self.0 == DC_MSG_ID_MARKER1
+    }
+
+    /// Whether the message ID is the special day marker.
+    ///
+    /// See the docs of the `dc_get_chat_msgs` C API for details.
+    pub fn is_daymarker(&self) -> bool {
+        self.0 == DC_MSG_ID_DAYMARKER
+    }
+
+    /// Bad evil escape hatch.
+    ///
+    /// Avoid using this, eventually types should be cleaned up enough
+    /// that it is no longer necessary.
+    pub fn to_u32(&self) -> u32 {
+        self.0
+    }
+}
+
+impl std::fmt::Display for MsgId {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        // Would be nice if we could use match here, but no computed values in ranges.
+        if self.0 == DC_MSG_ID_MARKER1 {
+            write!(f, "Msg#Marker1")
+        } else if self.0 == DC_MSG_ID_DAYMARKER {
+            write!(f, "Msg#DayMarker")
+        } else if self.0 <= DC_MSG_ID_LAST_SPECIAL {
+            write!(f, "Msg#UnknownSpecial")
+        } else {
+            write!(f, "Msg#{}", self.0)
+        }
+    }
+}
+
+/// Allow converting [MsgId] to an SQLite type.
+///
+/// This allows you to directly store [MsgId] into the database.
+///
+/// # Errors
+///
+/// This **does** ensure that no special message IDs are written into
+/// the database and the conversion will fail if this is not the case.
+impl rusqlite::types::ToSql for MsgId {
+    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
+        if self.0 <= DC_MSG_ID_LAST_SPECIAL {
+            return Err(rusqlite::Error::ToSqlConversionFailure(Box::new(
+                InvalidMsgId.compat(),
+            )));
+        }
+        let val = rusqlite::types::Value::Integer(self.0 as i64);
+        let out = rusqlite::types::ToSqlOutput::Owned(val);
+        Ok(out)
+    }
+}
+
+/// Allow converting an SQLite integer directly into [MsgId].
+impl rusqlite::types::FromSql for MsgId {
+    fn column_result(value: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
+        // Would be nice if we could use match here, but alas.
+        i64::column_result(value).and_then(|val| {
+            if 0 <= val && val <= std::u32::MAX as i64 {
+                Ok(MsgId::new(val as u32))
+            } else {
+                Err(rusqlite::types::FromSqlError::OutOfRange(val))
+            }
+        })
+    }
+}
+
+/// Message ID was invalid.
+///
+/// This usually occurs when trying to use a message ID of
+/// [DC_MSG_ID_LAST_SPECIAL] or below in a situation where this is not
+/// possible.
+#[derive(Debug, Fail)]
+#[fail(display = "Invalid Message ID.")]
+pub struct InvalidMsgId;
+
 /// An object representing a single message in memory.
 /// The message object is not updated.
 /// If you want an update, you have to recreate the object.
@@ -29,7 +155,7 @@ const SUMMARY_CHARACTERS: usize = 160;
 /// approx. max. length returned by dc_get_msg_info()
 #[derive(Debug, Clone, Default)]
 pub struct Message {
-    pub(crate) id: u32,
+    pub(crate) id: MsgId,
     pub(crate) from_id: u32,
     pub(crate) to_id: u32,
     pub(crate) chat_id: u32,
@@ -61,70 +187,105 @@ impl Message {
         msg
     }
 
-    pub fn load_from_db(context: &Context, id: u32) -> Result<Message, Error> {
+    pub fn load_from_db(context: &Context, id: MsgId) -> Result<Message, Error> {
+        ensure!(
+            !id.is_special(),
+            "Can not load special message IDs from DB."
+        );
         context.sql.query_row(
-        "SELECT  \
-         m.id,rfc724_mid,m.mime_in_reply_to,m.server_folder,m.server_uid,m.move_state,m.chat_id,  \
-         m.from_id,m.to_id,m.timestamp,m.timestamp_sent,m.timestamp_rcvd, m.type,m.state,m.msgrmsg,m.txt,  \
-         m.param,m.starred,m.hidden,m.location_id, c.blocked  \
-         FROM msgs m \
-         LEFT JOIN chats c ON c.id=m.chat_id WHERE m.id=?;",
-        params![id as i32],
-        |row| {
-            let mut msg = Message::default();
-            msg.id = row.get::<_, i32>(0)? as u32;
-            msg.rfc724_mid = row.get::<_, String>(1)?;
-            msg.in_reply_to = row.get::<_, Option<String>>(2)?;
-            msg.server_folder = row.get::<_, Option<String>>(3)?;
-            msg.server_uid = row.get(4)?;
-            msg.move_state = row.get(5)?;
-            msg.chat_id = row.get(6)?;
-            msg.from_id = row.get(7)?;
-            msg.to_id = row.get(8)?;
-            msg.timestamp_sort = row.get(9)?;
-            msg.timestamp_sent = row.get(10)?;
-            msg.timestamp_rcvd = row.get(11)?;
-            msg.type_0 = row.get(12)?;
-            msg.state = row.get(13)?;
-            msg.is_dc_message = row.get(14)?;
+            concat!(
+                "SELECT",
+                "    m.id AS id,",
+                "    rfc724_mid AS rfc724mid,",
+                "    m.mime_in_reply_to AS mime_in_reply_to,",
+                "    m.server_folder AS server_folder,",
+                "    m.server_uid AS server_uid,",
+                "    m.move_state as move_state,",
+                "    m.chat_id AS chat_id,",
+                "    m.from_id AS from_id,",
+                "    m.to_id AS to_id,",
+                "    m.timestamp AS timestamp,",
+                "    m.timestamp_sent AS timestamp_sent,",
+                "    m.timestamp_rcvd AS timestamp_rcvd,",
+                "    m.type AS type,",
+                "    m.state AS state,",
+                "    m.msgrmsg AS msgrmsg,",
+                "    m.txt AS txt,",
+                "    m.param AS param,",
+                "    m.starred AS starred,",
+                "    m.hidden AS hidden,",
+                "    m.location_id AS location,",
+                "    c.blocked AS blocked",
+                " FROM msgs m LEFT JOIN chats c ON c.id=m.chat_id",
+                " WHERE m.id=?;"
+            ),
+            params![id],
+            |row| {
+                let mut msg = Message::default();
+                // msg.id = row.get::<_, AnyMsgId>("id")?;
+                msg.id = row.get("id")?;
+                msg.rfc724_mid = row.get::<_, String>("rfc724mid")?;
+                msg.in_reply_to = row.get::<_, Option<String>>("mime_in_reply_to")?;
+                msg.server_folder = row.get::<_, Option<String>>("server_folder")?;
+                msg.server_uid = row.get("server_uid")?;
+                msg.move_state = row.get("move_state")?;
+                msg.chat_id = row.get("chat_id")?;
+                msg.from_id = row.get("from_id")?;
+                msg.to_id = row.get("to_id")?;
+                msg.timestamp_sort = row.get("timestamp")?;
+                msg.timestamp_sent = row.get("timestamp_sent")?;
+                msg.timestamp_rcvd = row.get("timestamp_rcvd")?;
+                msg.type_0 = row.get("type")?;
+                msg.state = row.get("state")?;
+                msg.is_dc_message = row.get("msgrmsg")?;
 
-            let text;
-            if let rusqlite::types::ValueRef::Text(buf) = row.get_raw(15) {
-                if let Ok(t) = String::from_utf8(buf.to_vec()) {
-                    text = t;
+                let text;
+                if let rusqlite::types::ValueRef::Text(buf) = row.get_raw("txt") {
+                    if let Ok(t) = String::from_utf8(buf.to_vec()) {
+                        text = t;
+                    } else {
+                        warn!(
+                            context,
+                            concat!(
+                                "dc_msg_load_from_db: could not get ",
+                                "text column as non-lossy utf8 id {}"
+                            ),
+                            id
+                        );
+                        text = String::from_utf8_lossy(buf).into_owned();
+                    }
                 } else {
-                    warn!(context, "dc_msg_load_from_db: could not get text column as non-lossy utf8 id {}", id);
-                    text = String::from_utf8_lossy(buf).into_owned();
+                    text = "".to_string();
                 }
-            } else {
-                text = "".to_string();
-            }
-            msg.text = Some(text);
+                msg.text = Some(text);
 
-            msg.param = row.get::<_, String>(16)?.parse().unwrap_or_default();
-            msg.starred = row.get(17)?;
-            msg.hidden = row.get(18)?;
-            msg.location_id = row.get(19)?;
-            msg.chat_blocked = row.get::<_, Option<Blocked>>(20)?.unwrap_or_default();
+                msg.param = row.get::<_, String>("param")?.parse().unwrap_or_default();
+                msg.starred = row.get("starred")?;
+                msg.hidden = row.get("hidden")?;
+                msg.location_id = row.get("location")?;
+                msg.chat_blocked = row
+                    .get::<_, Option<Blocked>>("blocked")?
+                    .unwrap_or_default();
 
-            Ok(msg)
-        })
+                Ok(msg)
+            },
+        )
     }
 
-    pub fn delete_from_db(context: &Context, msg_id: u32) {
+    pub fn delete_from_db(context: &Context, msg_id: MsgId) {
         if let Ok(msg) = Message::load_from_db(context, msg_id) {
             sql::execute(
                 context,
                 &context.sql,
                 "DELETE FROM msgs WHERE id=?;",
-                params![msg.id as i32],
+                params![msg.id],
             )
             .ok();
             sql::execute(
                 context,
                 &context.sql,
                 "DELETE FROM msgs_mdns WHERE msg_id=?;",
-                params![msg.id as i32],
+                params![msg.id],
             )
             .ok();
         }
@@ -145,9 +306,7 @@ impl Message {
     }
 
     pub fn get_file(&self, context: &Context) -> Option<PathBuf> {
-        self.param
-            .get(Param::File)
-            .map(|f| dc_get_abs_path(context, f))
+        self.param.get_path(Param::File, context).unwrap_or(None)
     }
 
     /// Check if a message has a location bound to it.
@@ -190,7 +349,7 @@ impl Message {
         }
     }
 
-    pub fn get_id(&self) -> u32 {
+    pub fn get_id(&self) -> MsgId {
         self.id
     }
 
@@ -237,8 +396,9 @@ impl Message {
 
     pub fn get_filebytes(&self, context: &Context) -> u64 {
         self.param
-            .get(Param::File)
-            .map(|file| dc_get_filebytes(context, &file))
+            .get_path(Param::File, context)
+            .unwrap_or(None)
+            .map(|path| dc_get_filebytes(context, &path))
             .unwrap_or_default()
     }
 
@@ -255,7 +415,7 @@ impl Message {
     }
 
     pub fn get_showpadlock(&self) -> bool {
-        self.param.get_int(Param::GuranteeE2ee).unwrap_or_default() != 0
+        self.param.get_int(Param::GuaranteeE2ee).unwrap_or_default() != 0
     }
 
     pub fn get_summary(&mut self, context: &Context, chat: Option<&Chat>) -> Lot {
@@ -321,6 +481,14 @@ impl Message {
             || cmd != SystemMessage::Unknown && cmd != SystemMessage::AutocryptSetupMessage
     }
 
+    /// Whether the message is still being created.
+    ///
+    /// Messages with attachments might be created before the
+    /// attachment is ready.  In this case some more restrictions on
+    /// the attachment apply, e.g. if the file to be attached is still
+    /// being written to or otherwise will still change it can not be
+    /// copied to the blobdir.  Thus those attachments need to be
+    /// created immediately in the blobdir with a valid filename.
     pub fn is_increation(&self) -> bool {
         chat::msgtype_has_file(self.type_0) && self.state == MessageState::OutPreparing
     }
@@ -393,7 +561,7 @@ impl Message {
             context,
             &context.sql,
             "UPDATE msgs SET param=? WHERE id=?;",
-            params![self.param.to_string(), self.id as i32],
+            params![self.param.to_string(), self.id],
         )
         .is_ok()
     }
@@ -503,7 +671,7 @@ impl Lot {
     }
 }
 
-pub fn get_msg_info(context: &Context, msg_id: u32) -> String {
+pub fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
     let mut ret = String::new();
 
     let msg = Message::load_from_db(context, msg_id);
@@ -516,11 +684,11 @@ pub fn get_msg_info(context: &Context, msg_id: u32) -> String {
     let rawtxt: Option<String> = context.sql.query_get_value(
         context,
         "SELECT txt_raw FROM msgs WHERE id=?;",
-        params![msg_id as i32],
+        params![msg_id],
     );
 
     if rawtxt.is_none() {
-        ret += &format!("Cannot load message #{}.", msg_id as usize);
+        ret += &format!("Cannot load message {}.", msg_id);
         return ret;
     }
     let rawtxt = rawtxt.unwrap_or_default();
@@ -553,7 +721,7 @@ pub fn get_msg_info(context: &Context, msg_id: u32) -> String {
 
     if let Ok(rows) = context.sql.query_map(
         "SELECT contact_id, timestamp_sent FROM msgs_mdns WHERE msg_id=?;",
-        params![msg_id as i32],
+        params![msg_id],
         |row| {
             let contact_id: i32 = row.get(0)?;
             let ts: i64 = row.get(1)?;
@@ -598,7 +766,7 @@ pub fn get_msg_info(context: &Context, msg_id: u32) -> String {
         if 0 != e2ee_errors & 0x2 {
             ret += ", Encrypted, no valid signature";
         }
-    } else if 0 != msg.param.get_int(Param::GuranteeE2ee).unwrap_or_default() {
+    } else if 0 != msg.param.get_int(Param::GuaranteeE2ee).unwrap_or_default() {
         ret += ", Encrypted";
     }
 
@@ -663,21 +831,21 @@ pub fn guess_msgtype_from_suffix(path: &Path) -> Option<(Viewtype, &str)> {
     Some(info)
 }
 
-pub fn get_mime_headers(context: &Context, msg_id: u32) -> Option<String> {
+pub fn get_mime_headers(context: &Context, msg_id: MsgId) -> Option<String> {
     context.sql.query_get_value(
         context,
         "SELECT mime_headers FROM msgs WHERE id=?;",
-        params![msg_id as i32],
+        params![msg_id],
     )
 }
 
-pub fn delete_msgs(context: &Context, msg_ids: &[u32]) {
+pub fn delete_msgs(context: &Context, msg_ids: &[MsgId]) {
     for msg_id in msg_ids.iter() {
         update_msg_chat_id(context, *msg_id, DC_CHAT_ID_TRASH);
         job_add(
             context,
             Action::DeleteMsgOnImap,
-            *msg_id as libc::c_int,
+            msg_id.to_u32() as i32,
             Params::new(),
             0,
         );
@@ -686,35 +854,45 @@ pub fn delete_msgs(context: &Context, msg_ids: &[u32]) {
     if !msg_ids.is_empty() {
         context.call_cb(Event::MsgsChanged {
             chat_id: 0,
-            msg_id: 0,
+            msg_id: MsgId::new(0),
         });
         job_kill_action(context, Action::Housekeeping);
         job_add(context, Action::Housekeeping, 0, Params::new(), 10);
     };
 }
 
-fn update_msg_chat_id(context: &Context, msg_id: u32, chat_id: u32) -> bool {
+fn update_msg_chat_id(context: &Context, msg_id: MsgId, chat_id: u32) -> bool {
     sql::execute(
         context,
         &context.sql,
         "UPDATE msgs SET chat_id=? WHERE id=?;",
-        params![chat_id as i32, msg_id as i32],
+        params![chat_id as i32, msg_id],
     )
     .is_ok()
 }
 
-pub fn markseen_msgs(context: &Context, msg_ids: &[u32]) -> bool {
+pub fn markseen_msgs(context: &Context, msg_ids: &[MsgId]) -> bool {
     if msg_ids.is_empty() {
         return false;
     }
 
     let msgs = context.sql.prepare(
-        "SELECT m.state, c.blocked  FROM msgs m  LEFT JOIN chats c ON c.id=m.chat_id  WHERE m.id=? AND m.chat_id>9",
+        concat!(
+            "SELECT",
+            "    m.state AS state,",
+            "    c.blocked AS blocked",
+            " FROM msgs m LEFT JOIN chats c ON c.id=m.chat_id",
+            " WHERE m.id=? AND m.chat_id>9"
+        ),
         |mut stmt, _| {
             let mut res = Vec::with_capacity(msg_ids.len());
             for id in msg_ids.iter() {
-                let query_res = stmt.query_row(params![*id as i32], |row| {
-                    Ok((row.get::<_, MessageState>(0)?, row.get::<_, Option<Blocked>>(1)?.unwrap_or_default()))
+                let query_res = stmt.query_row(params![*id], |row| {
+                    Ok((
+                        row.get::<_, MessageState>("state")?,
+                        row.get::<_, Option<Blocked>>("blocked")?
+                            .unwrap_or_default(),
+                    ))
                 });
                 if let Err(rusqlite::Error::QueryReturnedNoRows) = query_res {
                     continue;
@@ -724,7 +902,7 @@ pub fn markseen_msgs(context: &Context, msg_ids: &[u32]) -> bool {
             }
 
             Ok(res)
-        }
+        },
     );
 
     if msgs.is_err() {
@@ -738,12 +916,12 @@ pub fn markseen_msgs(context: &Context, msg_ids: &[u32]) -> bool {
         if curr_blocked == Blocked::Not {
             if curr_state == MessageState::InFresh || curr_state == MessageState::InNoticed {
                 update_msg_state(context, *id, MessageState::InSeen);
-                info!(context, "Seen message #{}.", id);
+                info!(context, "Seen message {}.", id);
 
                 job_add(
                     context,
                     Action::MarkseenMsgOnImap,
-                    *id as i32,
+                    id.to_u32() as i32,
                     Params::new(),
                     0,
                 );
@@ -758,24 +936,24 @@ pub fn markseen_msgs(context: &Context, msg_ids: &[u32]) -> bool {
     if send_event {
         context.call_cb(Event::MsgsChanged {
             chat_id: 0,
-            msg_id: 0,
+            msg_id: MsgId::new(0),
         });
     }
 
     true
 }
 
-pub fn update_msg_state(context: &Context, msg_id: u32, state: MessageState) -> bool {
+pub fn update_msg_state(context: &Context, msg_id: MsgId, state: MessageState) -> bool {
     sql::execute(
         context,
         &context.sql,
         "UPDATE msgs SET state=? WHERE id=?;",
-        params![state, msg_id as i32],
+        params![state, msg_id],
     )
     .is_ok()
 }
 
-pub fn star_msgs(context: &Context, msg_ids: &[u32], star: bool) -> bool {
+pub fn star_msgs(context: &Context, msg_ids: &[MsgId], star: bool) -> bool {
     if msg_ids.is_empty() {
         return false;
     }
@@ -783,7 +961,7 @@ pub fn star_msgs(context: &Context, msg_ids: &[u32], star: bool) -> bool {
         .sql
         .prepare("UPDATE msgs SET starred=? WHERE id=?;", |mut stmt, _| {
             for msg_id in msg_ids.iter() {
-                stmt.execute(params![star as i32, *msg_id as i32])?;
+                stmt.execute(params![star as i32, *msg_id])?;
             }
             Ok(())
         })
@@ -812,17 +990,14 @@ pub fn get_summarytext_by_raw(
                     .stock_str(StockMessage::AcSetupMsgSubject)
                     .to_string()
             } else {
-                let file_name: String = if let Some(file_path) = param.get(Param::File) {
-                    if let Some(file_name) = Path::new(file_path).file_name() {
-                        Some(file_name.to_string_lossy().into_owned())
-                    } else {
-                        None
-                    }
-                } else {
-                    None
-                }
-                .unwrap_or_else(|| "ErrFileName".to_string());
-
+                let file_name: String = param
+                    .get_path(Param::File, context)
+                    .unwrap_or(None)
+                    .and_then(|path| {
+                        path.file_name()
+                            .map(|fname| fname.to_string_lossy().into_owned())
+                    })
+                    .unwrap_or_else(|| String::from("ErrFileName"));
                 let label = context.stock_str(if viewtype == Viewtype::Audio {
                     StockMessage::Audio
                 } else {
@@ -846,7 +1021,9 @@ pub fn get_summarytext_by_raw(
     }
 
     if let Some(text) = text {
-        if prefix.is_empty() {
+        if text.as_ref().is_empty() {
+            prefix
+        } else if prefix.is_empty() {
             dc_truncate(text.as_ref(), approx_characters, true).to_string()
         } else {
             let tmp = format!("{} – {}", prefix, text.as_ref());
@@ -864,8 +1041,8 @@ pub fn get_summarytext_by_raw(
 
 // Context functions to work with messages
 
-pub fn exists(context: &Context, msg_id: u32) -> bool {
-    if msg_id <= DC_CHAT_ID_LAST_SPECIAL {
+pub fn exists(context: &Context, msg_id: MsgId) -> bool {
+    if msg_id.is_special() {
         return false;
     }
 
@@ -894,7 +1071,7 @@ pub fn update_msg_move_state(context: &Context, rfc724_mid: &str, state: MoveSta
     .is_ok()
 }
 
-pub fn set_msg_failed(context: &Context, msg_id: u32, error: Option<impl AsRef<str>>) {
+pub fn set_msg_failed(context: &Context, msg_id: MsgId, error: Option<impl AsRef<str>>) {
     if let Ok(mut msg) = Message::load_from_db(context, msg_id) {
         if msg.state.can_fail() {
             msg.state = MessageState::OutFailed;
@@ -908,7 +1085,7 @@ pub fn set_msg_failed(context: &Context, msg_id: u32, error: Option<impl AsRef<s
             context,
             &context.sql,
             "UPDATE msgs SET state=?, param=? WHERE id=?;",
-            params![msg.state, msg.param.to_string(), msg_id as i32],
+            params![msg.state, msg.param.to_string(), msg_id],
         )
         .is_ok()
         {
@@ -920,38 +1097,39 @@ pub fn set_msg_failed(context: &Context, msg_id: u32, error: Option<impl AsRef<s
     }
 }
 
-/// returns true if an event should be send
+/// returns Some if an event should be send
 pub fn mdn_from_ext(
     context: &Context,
     from_id: u32,
     rfc724_mid: &str,
     timestamp_sent: i64,
-    ret_chat_id: &mut u32,
-    ret_msg_id: &mut u32,
-) -> bool {
-    if from_id <= 9 || rfc724_mid.is_empty() || *ret_chat_id != 0 || *ret_msg_id != 0 {
-        return false;
+) -> Option<(u32, MsgId)> {
+    if from_id <= 9 || rfc724_mid.is_empty() {
+        return None;
     }
 
-    let mut read_by_all = false;
-
     if let Ok((msg_id, chat_id, chat_type, msg_state)) = context.sql.query_row(
-        "SELECT m.id, c.id, c.type, m.state FROM msgs m  \
-         LEFT JOIN chats c ON m.chat_id=c.id  \
-         WHERE rfc724_mid=? AND from_id=1  \
-         ORDER BY m.id;",
+        concat!(
+            "SELECT",
+            "    m.id AS msg_id,",
+            "    c.id AS chat_id,",
+            "    c.type AS type,",
+            "    m.state AS state",
+            " FROM msgs m LEFT JOIN chats c ON m.chat_id=c.id",
+            " WHERE rfc724_mid=? AND from_id=1",
+            " ORDER BY m.id;"
+        ),
         params![rfc724_mid],
         |row| {
             Ok((
-                row.get::<_, i32>(0)?,
-                row.get::<_, i32>(1)?,
-                row.get::<_, Chattype>(2)?,
-                row.get::<_, MessageState>(3)?,
+                row.get::<_, MsgId>("msg_id")?,
+                row.get::<_, u32>("chat_id")?,
+                row.get::<_, Chattype>("type")?,
+                row.get::<_, MessageState>("state")?,
             ))
         },
     ) {
-        *ret_msg_id = msg_id as u32;
-        *ret_chat_id = chat_id as u32;
+        let mut read_by_all = false;
 
         // if already marked as MDNS_RCVD msgstate_can_fail() returns false.
         // however, it is important, that ret_msg_id is set above as this
@@ -961,20 +1139,20 @@ pub fn mdn_from_ext(
                 .sql
                 .exists(
                     "SELECT contact_id FROM msgs_mdns WHERE msg_id=? AND contact_id=?;",
-                    params![*ret_msg_id as i32, from_id as i32,],
+                    params![msg_id, from_id as i32,],
                 )
                 .unwrap_or_default();
 
             if !mdn_already_in_table {
                 context.sql.execute(
                     "INSERT INTO msgs_mdns (msg_id, contact_id, timestamp_sent) VALUES (?, ?, ?);",
-                    params![*ret_msg_id as i32, from_id as i32, timestamp_sent],
+                    params![msg_id, from_id as i32, timestamp_sent],
                 ).unwrap_or_default(); // TODO: better error handling
             }
 
             // Normal chat? that's quite easy.
             if chat_type == Chattype::Single {
-                update_msg_state(context, *ret_msg_id, MessageState::OutMdnRcvd);
+                update_msg_state(context, msg_id, MessageState::OutMdnRcvd);
                 read_by_all = true;
             } else {
                 // send event about new state
@@ -983,7 +1161,7 @@ pub fn mdn_from_ext(
                     .query_get_value::<_, isize>(
                         context,
                         "SELECT COUNT(*) FROM msgs_mdns WHERE msg_id=?;",
-                        params![*ret_msg_id as i32],
+                        params![msg_id],
                     )
                     .unwrap_or_default() as usize;
                 /*
@@ -999,16 +1177,19 @@ pub fn mdn_from_ext(
                 (S=Sender, R=Recipient)
                  */
                 // for rounding, SELF is already included!
-                let soll_cnt = (chat::get_chat_contact_cnt(context, *ret_chat_id) + 1) / 2;
+                let soll_cnt = (chat::get_chat_contact_cnt(context, chat_id) + 1) / 2;
                 if ist_cnt >= soll_cnt {
-                    update_msg_state(context, *ret_msg_id, MessageState::OutMdnRcvd);
+                    update_msg_state(context, msg_id, MessageState::OutMdnRcvd);
                     read_by_all = true;
                 } // else wait for more receipts
             }
         }
+        return match read_by_all {
+            true => Some((chat_id, msg_id)),
+            false => None,
+        };
     }
-
-    read_by_all
+    None
 }
 
 /// The number of messages assigned to real chat (!=deaddrop, !=trash)
@@ -1062,7 +1243,7 @@ pub fn rfc724_mid_cnt(context: &Context, rfc724_mid: &str) -> libc::c_int {
 pub(crate) fn rfc724_mid_exists(
     context: &Context,
     rfc724_mid: &str,
-) -> Result<(String, u32, u32), Error> {
+) -> Result<(String, u32, MsgId), Error> {
     ensure!(!rfc724_mid.is_empty(), "empty rfc724_mid");
 
     context.sql.query_row(
@@ -1071,7 +1252,7 @@ pub(crate) fn rfc724_mid_exists(
         |row| {
             let server_folder = row.get::<_, Option<String>>(0)?.unwrap_or_default();
             let server_uid = row.get(1)?;
-            let msg_id = row.get(2)?;
+            let msg_id: MsgId = row.get(2)?;
 
             Ok((server_folder, server_uid, msg_id))
         },
@@ -1095,6 +1276,12 @@ pub fn update_server_uid(
     }
 }
 
+#[allow(dead_code)]
+pub fn dc_empty_server(context: &Context, flags: u32) {
+    job_kill_action(context, Action::EmptyServer);
+    job_add(context, Action::EmptyServer, flags as i32, Params::new(), 0);
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -1130,4 +1317,121 @@ mod tests {
         let _msg2 = Message::load_from_db(ctx, msg_id).unwrap();
         assert_eq!(_msg2.get_filemime(), None);
     }
+
+    #[test]
+    pub fn test_get_summarytext_by_raw() {
+        let d = test::dummy_context();
+        let ctx = &d.ctx;
+
+        let some_text = Some("bla bla".to_string());
+        let empty_text = Some("".to_string());
+        let no_text: Option<String> = None;
+
+        let mut some_file = Params::new();
+        some_file.set(Param::File, "foo.bar");
+
+        assert_eq!(
+            get_summarytext_by_raw(
+                Viewtype::Text,
+                some_text.as_ref(),
+                &mut Params::new(),
+                50,
+                &ctx
+            ),
+            "bla bla" // for simple text, the type is not added to the summary
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(Viewtype::Image, no_text.as_ref(), &mut some_file, 50, &ctx,),
+            "Image" // file names are not added for images
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(Viewtype::Video, no_text.as_ref(), &mut some_file, 50, &ctx,),
+            "Video" // file names are not added for videos
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(Viewtype::Gif, no_text.as_ref(), &mut some_file, 50, &ctx,),
+            "GIF" // file names are not added for GIFs
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(
+                Viewtype::Sticker,
+                no_text.as_ref(),
+                &mut some_file,
+                50,
+                &ctx,
+            ),
+            "Sticker" // file names are not added for stickers
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(
+                Viewtype::Voice,
+                empty_text.as_ref(),
+                &mut some_file,
+                50,
+                &ctx,
+            ),
+            "Voice message" // file names are not added for voice messages, empty text is skipped
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(Viewtype::Voice, no_text.as_ref(), &mut some_file, 50, &ctx),
+            "Voice message" // file names are not added for voice messages
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(
+                Viewtype::Voice,
+                some_text.as_ref(),
+                &mut some_file,
+                50,
+                &ctx
+            ),
+            "Voice message \u{2013} bla bla" // `\u{2013}` explicitly checks for "EN DASH"
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(Viewtype::Audio, no_text.as_ref(), &mut some_file, 50, &ctx),
+            "Audio \u{2013} foo.bar" // file name is added for audio
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(
+                Viewtype::Audio,
+                empty_text.as_ref(),
+                &mut some_file,
+                50,
+                &ctx,
+            ),
+            "Audio \u{2013} foo.bar" // file name is added for audio, empty text is not added
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(
+                Viewtype::Audio,
+                some_text.as_ref(),
+                &mut some_file,
+                50,
+                &ctx
+            ),
+            "Audio \u{2013} foo.bar \u{2013} bla bla" // file name and text added for audio
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(Viewtype::File, some_text.as_ref(), &mut some_file, 50, &ctx),
+            "File \u{2013} foo.bar \u{2013} bla bla" // file name is added for files
+        );
+
+        let mut asm_file = Params::new();
+        asm_file.set(Param::File, "foo.bar");
+        asm_file.set_cmd(SystemMessage::AutocryptSetupMessage);
+        assert_eq!(
+            get_summarytext_by_raw(Viewtype::File, no_text.as_ref(), &mut asm_file, 50, &ctx),
+            "Autocrypt Setup Message" // file name is not added for autocrypt setup messages
+        );
+    }
 }

src/mimefactory.rs

@@ -1,4 +1,3 @@
-use std::path::Path;
 use std::ptr;
 
 use chrono::TimeZone;
@@ -23,6 +22,7 @@ use crate::dc_tools::*;
 use crate::e2ee::*;
 use crate::error::Error;
 use crate::location;
+use crate::message::MsgId;
 use crate::message::{self, Message};
 use crate::param::*;
 use crate::stock::StockMessage;
@@ -108,7 +108,7 @@ impl<'a> MimeFactory<'a> {
         Ok(())
     }
 
-    pub fn load_mdn(context: &'a Context, msg_id: u32) -> Result<MimeFactory, Error> {
+    pub fn load_mdn(context: &'a Context, msg_id: MsgId) -> Result<MimeFactory, Error> {
         if !context.get_config_bool(Config::MdnsEnabled) {
             // MDNs not enabled - check this is late, in the job. the
             // user may have changed its choice while offline ...
@@ -257,7 +257,7 @@ impl<'a> MimeFactory<'a> {
                         e2ee_guaranteed = self
                             .msg
                             .param
-                            .get_int(Param::GuranteeE2ee)
+                            .get_int(Param::GuaranteeE2ee)
                             .unwrap_or_default()
                             != 0;
                     }
@@ -538,7 +538,7 @@ impl<'a> MimeFactory<'a> {
                     != self
                         .msg
                         .param
-                        .get_int(Param::GuranteeE2ee)
+                        .get_int(Param::GuaranteeE2ee)
                         .unwrap_or_default()
                 {
                     self.context
@@ -654,8 +654,8 @@ impl<'a> MimeFactory<'a> {
         Ok(())
     }
 
-    pub fn load_msg(context: &Context, msg_id: u32) -> Result<MimeFactory, Error> {
-        ensure!(msg_id > DC_CHAT_ID_LAST_SPECIAL, "Invalid chat id");
+    pub fn load_msg(context: &Context, msg_id: MsgId) -> Result<MimeFactory, Error> {
+        ensure!(!msg_id.is_special(), "Invalid chat id");
 
         let msg = Message::load_from_db(context, msg_id)?;
         let chat = Chat::load_from_db(context, msg.chat_id)?;
@@ -721,7 +721,7 @@ impl<'a> MimeFactory<'a> {
         }
         let row = context.sql.query_row(
             "SELECT mime_in_reply_to, mime_references FROM msgs WHERE id=?",
-            params![factory.msg.id as i32],
+            params![factory.msg.id],
             |row| {
                 let in_reply_to: String = row.get(0)?;
                 let references: String = row.get(1)?;
@@ -794,26 +794,21 @@ fn build_body_file(
     msg: &Message,
     base_name: &str,
 ) -> Result<(*mut Mailmime, String), Error> {
-    let path_filename = match msg.param.get(Param::File) {
-        None => {
-            bail!("msg has no filename");
-        }
-        Some(path) => path,
-    };
-    let suffix = dc_get_filesuffix_lc(path_filename).unwrap_or_else(|| "dat".into());
+    let blob = msg
+        .param
+        .get_blob(Param::File, context, true)?
+        .ok_or_else(|| format_err!("msg has no filename"))?;
+    let suffix = blob.suffix().unwrap_or("dat");
 
-    /* get file name to use for sending
-    (for privacy purposes, we do not transfer the original filenames eg. for images;
-    these names are normally not needed and contain timestamps, running numbers etc.) */
-    let filename_to_send = match msg.type_0 {
+    // Get file name to use for sending.  For privacy purposes, we do
+    // not transfer the original filenames eg. for images; these names
+    // are normally not needed and contain timestamps, running numbers
+    // etc.
+    let filename_to_send: String = match msg.type_0 {
         Viewtype::Voice => chrono::Utc
             .timestamp(msg.timestamp_sort as i64, 0)
-            .format(&format!("voice-message_%Y-%m-%d_%H-%M-%S.{}", suffix))
+            .format(&format!("voice-message_%Y-%m-%d_%H-%M-%S.{}", &suffix))
             .to_string(),
-        Viewtype::Audio => Path::new(path_filename)
-            .file_name()
-            .map(|c| c.to_string_lossy().to_string())
-            .unwrap_or_default(),
         Viewtype::Image | Viewtype::Gif => format!(
             "{}.{}",
             if base_name.is_empty() {
@@ -824,18 +819,14 @@ fn build_body_file(
             &suffix,
         ),
         Viewtype::Video => format!("video.{}", &suffix),
-        _ => Path::new(path_filename)
-            .file_name()
-            .map(|c| c.to_string_lossy().to_string())
-            .unwrap_or_default(),
+        _ => blob.as_file_name().to_string(),
     };
 
     /* check mimetype */
     let mimetype = match msg.param.get(Param::MimeType) {
         Some(mtype) => mtype,
         None => {
-            let path = Path::new(path_filename);
-            if let Some(res) = message::guess_msgtype_from_suffix(&path) {
+            if let Some(res) = message::guess_msgtype_from_suffix(blob.as_rel_path()) {
                 res.1
             } else {
                 "application/octet-stream"
@@ -895,7 +886,7 @@ fn build_body_file(
         wrapmime::append_ct_param(content, "name", &filename_encoded)?;
 
         let mime_sub = mailmime_new_empty(content, mime_fields);
-        let abs_path = dc_get_abs_path(context, path_filename).to_c_string()?;
+        let abs_path = blob.to_abs_path().to_c_string()?;
         mailmime_set_body_file(mime_sub, dc_strdup(abs_path.as_ptr()));
         Ok((mime_sub, filename_to_send))
     }
@@ -912,13 +903,11 @@ pub(crate) fn vec_contains_lowercase(vec: &[String], part: &str) -> bool {
 }
 
 fn is_file_size_okay(context: &Context, msg: &Message) -> bool {
-    let mut file_size_okay = true;
-    let path = msg.param.get(Param::File).unwrap_or_default();
-    let bytes = dc_get_filebytes(context, &path);
-
-    if bytes > (49 * 1024 * 1024 / 4 * 3) {
-        file_size_okay = false;
+    match msg.param.get_path(Param::File, context).unwrap_or(None) {
+        Some(path) => {
+            let bytes = dc_get_filebytes(context, &path);
+            bytes <= (49 * 1024 * 1024 / 4 * 3)
+        }
+        None => false,
     }
-
-    file_size_okay
 }

src/oauth2.rs

@@ -7,6 +7,7 @@ use crate::context::Context;
 use crate::dc_tools::*;
 
 const OAUTH2_GMAIL: Oauth2 = Oauth2 {
+    // see https://developers.google.com/identity/protocols/OAuth2InstalledApp
     client_id: "959970109878-4mvtgf6feshskf7695nfln6002mom908.apps.googleusercontent.com",
     get_code: "https://accounts.google.com/o/oauth2/auth?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&response_type=code&scope=https%3A%2F%2Fmail.google.com%2F%20email&access_type=offline",
     init_token: "https://accounts.google.com/o/oauth2/token?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&code=$CODE&grant_type=authorization_code",
@@ -15,6 +16,7 @@ const OAUTH2_GMAIL: Oauth2 = Oauth2 {
 };
 
 const OAUTH2_YANDEX: Oauth2 = Oauth2 {
+    // see https://tech.yandex.com/oauth/doc/dg/reference/auto-code-client-docpage/
     client_id: "c4d0b6735fc8420a816d7e1303469341",
     get_code: "https://oauth.yandex.com/authorize?client_id=$CLIENT_ID&response_type=code&scope=mail%3Aimap_full%20mail%3Asmtp&force_confirm=true",
     init_token: "https://oauth.yandex.com/token?grant_type=authorization_code&code=$CODE&client_id=$CLIENT_ID&client_secret=58b8c6e94cf44fbe952da8511955dacf",
@@ -89,6 +91,7 @@ pub fn dc_get_oauth2_access_token(
             }
         }
 
+        // generate new token: build & call auth url
         let refresh_token = context.sql.get_raw_config(context, "oauth2_refresh_token");
         let refresh_token_for = context
             .sql
@@ -120,14 +123,37 @@ pub fn dc_get_oauth2_access_token(
                     false,
                 )
             };
-        let mut token_url = replace_in_uri(&token_url, "$CLIENT_ID", oauth2.client_id);
-        token_url = replace_in_uri(&token_url, "$REDIRECT_URI", &redirect_uri);
-        token_url = replace_in_uri(&token_url, "$CODE", code.as_ref());
-        if let Some(ref token) = refresh_token {
-            token_url = replace_in_uri(&token_url, "$REFRESH_TOKEN", token);
+
+        // to allow easier specification of different configurations,
+        // token_url is in GET-method-format, sth. as https://domain?param1=val1&param2=val2 -
+        // convert this to POST-format ...
+        let mut parts = token_url.splitn(2, '?');
+        let post_url = parts.next().unwrap_or_default();
+        let post_args = parts.next().unwrap_or_default();
+        let mut post_param = HashMap::new();
+        for key_value_pair in post_args.split('&') {
+            let mut parts = key_value_pair.splitn(2, '=');
+            let key = parts.next().unwrap_or_default();
+            let mut value = parts.next().unwrap_or_default();
+
+            if value == "$CLIENT_ID" {
+                value = oauth2.client_id;
+            } else if value == "$REDIRECT_URI" {
+                value = &redirect_uri;
+            } else if value == "$CODE" {
+                value = code.as_ref();
+            } else if value == "$REFRESH_TOKEN" && refresh_token.is_some() {
+                value = refresh_token.as_ref().unwrap();
+            }
+
+            post_param.insert(key, value);
         }
 
-        let response = reqwest::Client::new().post(&token_url).send();
+        // ... and POST
+        let response = reqwest::Client::new()
+            .post(post_url)
+            .form(&post_param)
+            .send();
         if response.is_err() {
             warn!(
                 context,
@@ -139,13 +165,14 @@ pub fn dc_get_oauth2_access_token(
         if !response.status().is_success() {
             warn!(
                 context,
-                "Error calling OAuth2 at {}: {:?}",
+                "Unsuccessful response when calling OAuth2 at {}: {:?}",
                 token_url,
                 response.status()
             );
             return None;
         }
 
+        // generate new token: parse returned json
         let parsed: reqwest::Result<Response> = response.json();
         if parsed.is_err() {
             warn!(
@@ -155,6 +182,8 @@ pub fn dc_get_oauth2_access_token(
             return None;
         }
         println!("response: {:?}", &parsed);
+
+        // update refresh_token if given, typically on the first round, but we update it later as well.
         let response = parsed.unwrap();
         if let Some(ref token) = response.refresh_token {
             context
@@ -268,7 +297,7 @@ impl Oauth2 {
             return None;
         }
 
-        let parsed: reqwest::Result<HashMap<String, String>> = response.json();
+        let parsed: reqwest::Result<HashMap<String, serde_json::Value>> = response.json();
         if parsed.is_err() {
             warn!(
                 context,
@@ -277,11 +306,13 @@ impl Oauth2 {
             return None;
         }
         if let Ok(response) = parsed {
+            // serde_json::Value.as_str() removes the quotes of json-strings
             let addr = response.get("email");
             if addr.is_none() {
                 warn!(context, "E-mail missing in userinfo.");
+                return None;
             }
-
+            let addr = addr.unwrap().as_str();
             addr.map(|addr| addr.to_string())
         } else {
             warn!(context, "Failed to parse userinfo.");

src/param.rs

@@ -1,9 +1,12 @@
 use std::collections::BTreeMap;
 use std::fmt;
+use std::path::PathBuf;
 use std::str;
 
 use num_traits::FromPrimitive;
 
+use crate::blob::{BlobError, BlobObject};
+use crate::context::Context;
 use crate::dc_mimeparser::SystemMessage;
 use crate::error;
 
@@ -22,7 +25,7 @@ pub enum Param {
     /// For Messages
     MimeType = b'm',
     /// For Messages: message is encryoted, outgoing: guarantee E2EE or the message is not send
-    GuranteeE2ee = b'c',
+    GuaranteeE2ee = b'c',
     /// For Messages: decrypted with validation errors or without mutual set, if neither
     /// 'c' nor 'e' are preset, the messages is only transport encrypted.
     ErroneousE2ee = b'e',
@@ -46,6 +49,14 @@ pub enum Param {
     /// For Messages
     Error = b'L',
     /// For Messages: space-separated list of messaged IDs of forwarded copies.
+    ///
+    /// This is used when a [Message] is in the
+    /// [MessageState::OutPending] state but is already forwarded.
+    /// In this case the forwarded messages are written to the
+    /// database and their message IDs are added to this parameter of
+    /// the original message, which is also saved in the database.
+    /// When the original message is then finally sent this parameter
+    /// is used to also send all the forwarded messages.
     PrepForwards = b'P',
     /// For Jobs
     SetLatitude = b'l',
@@ -186,11 +197,82 @@ impl Params {
             .unwrap_or_default()
     }
 
+    /// Set the parameter behind `Param::Cmd`.
+    pub fn set_cmd(&mut self, value: SystemMessage) {
+        self.set_int(Param::Cmd, value as i32);
+    }
+
     /// Get the given parameter and parse as `f64`.
     pub fn get_float(&self, key: Param) -> Option<f64> {
         self.get(key).and_then(|s| s.parse().ok())
     }
 
+    /// Gets the given parameter and parse as [ParamsFile].
+    ///
+    /// See also [Params::get_blob] and [Params::get_path] which may
+    /// be more convenient.
+    pub fn get_file<'a>(
+        &self,
+        key: Param,
+        context: &'a Context,
+    ) -> Result<Option<ParamsFile<'a>>, BlobError> {
+        let val = match self.get(key) {
+            Some(val) => val,
+            None => return Ok(None),
+        };
+        ParamsFile::from_param(context, val).map(|file| Some(file))
+    }
+
+    /// Gets the parameter and returns a [BlobObject] for it.
+    ///
+    /// This parses the parameter value as a [ParamsFile] and than
+    /// tries to return a [BlobObject] for that file.  If the file is
+    /// not yet a valid blob, one will be created by copying the file
+    /// only if `create` is set to `true`, otherwise the a [BlobError]
+    /// will result.
+    ///
+    /// Note that in the [ParamsFile::FsPath] case the blob can be
+    /// created without copying if the path already referes to a valid
+    /// blob.  If so a [BlobObject] will be returned regardless of the
+    /// `create` argument.
+    pub fn get_blob<'a>(
+        &self,
+        key: Param,
+        context: &'a Context,
+        create: bool,
+    ) -> Result<Option<BlobObject<'a>>, BlobError> {
+        let val = match self.get(key) {
+            Some(val) => val,
+            None => return Ok(None),
+        };
+        let file = ParamsFile::from_param(context, val)?;
+        let blob = match file {
+            ParamsFile::FsPath(path) => match create {
+                true => BlobObject::create_from_path(context, path)?,
+                false => BlobObject::from_path(context, path)?,
+            },
+            ParamsFile::Blob(blob) => blob,
+        };
+        Ok(Some(blob))
+    }
+
+    /// Gets the parameter and returns a [PathBuf] for it.
+    ///
+    /// This parses the parameter value as a [ParamsFile] and returns
+    /// a [PathBuf] to the file.
+    pub fn get_path(&self, key: Param, context: &Context) -> Result<Option<PathBuf>, BlobError> {
+        let val = match self.get(key) {
+            Some(val) => val,
+            None => return Ok(None),
+        };
+        let file = ParamsFile::from_param(context, val)?;
+        let path = match file {
+            ParamsFile::FsPath(path) => path,
+            ParamsFile::Blob(blob) => blob.to_abs_path(),
+        };
+        Ok(Some(path))
+    }
+
     /// Set the given paramter to the passed in `i32`.
     pub fn set_int(&mut self, key: Param, value: i32) -> &mut Self {
         self.set(key, format!("{}", value));
@@ -204,10 +286,42 @@ impl Params {
     }
 }
 
+/// The value contained in [Param::File].
+///
+/// Because the only way to construct this object is from a valid
+/// UTF-8 string it is always safe to convert the value contained
+/// within the [ParamsFile::FsPath] back to a [String] or [&str].
+/// Despite the type itself does not guarantee this.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum ParamsFile<'a> {
+    FsPath(PathBuf),
+    Blob(BlobObject<'a>),
+}
+
+impl<'a> ParamsFile<'a> {
+    /// Parse the [Param::File] value into an object.
+    ///
+    /// If the value was stored into the [Params] correctly this
+    /// should not fail.
+    pub fn from_param(context: &'a Context, src: &str) -> Result<ParamsFile<'a>, BlobError> {
+        let param = match src.starts_with("$BLOBDIR/") {
+            true => ParamsFile::Blob(BlobObject::from_name(context, src.to_string())?),
+            false => ParamsFile::FsPath(PathBuf::from(src)),
+        };
+        Ok(param)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
 
+    use std::fs;
+    use std::path::Path;
+
+    use crate::blob::BlobErrorKind;
+    use crate::test_utils::*;
+
     #[test]
     fn test_dc_param() {
         let mut p1: Params = "\r\n\r\na=1\nf=2\n\nc = 3 ".parse().unwrap();
@@ -225,7 +339,7 @@ mod tests {
 
         p1.set(Param::Forwarded, "foo")
             .set_int(Param::File, 2)
-            .remove(Param::GuranteeE2ee)
+            .remove(Param::GuaranteeE2ee)
             .set_int(Param::Duration, 4);
 
         assert_eq!(p1.to_string(), "a=foo\nd=4\nf=2");
@@ -251,4 +365,64 @@ mod tests {
             .unwrap();
         assert_eq!(p1.get(Param::Forwarded).unwrap(), "cli%40deltachat.de");
     }
+
+    #[test]
+    fn test_params_file_fs_path() {
+        let t = dummy_context();
+        if let ParamsFile::FsPath(p) = ParamsFile::from_param(&t.ctx, "/foo/bar/baz").unwrap() {
+            assert_eq!(p, Path::new("/foo/bar/baz"));
+        } else {
+            assert!(false, "Wrong enum variant");
+        }
+    }
+
+    #[test]
+    fn test_params_file_blob() {
+        let t = dummy_context();
+        if let ParamsFile::Blob(b) = ParamsFile::from_param(&t.ctx, "$BLOBDIR/foo").unwrap() {
+            assert_eq!(b.as_name(), "$BLOBDIR/foo");
+        } else {
+            assert!(false, "Wrong enum variant");
+        }
+    }
+
+    // Tests for Params::get_file(), Params::get_path() and Params::get_blob().
+    #[test]
+    fn test_params_get_fileparam() {
+        let t = dummy_context();
+        let fname = t.dir.path().join("foo");
+        let mut p = Params::new();
+        p.set(Param::File, fname.to_str().unwrap());
+
+        let file = p.get_file(Param::File, &t.ctx).unwrap().unwrap();
+        assert_eq!(file, ParamsFile::FsPath(fname.clone()));
+
+        let path = p.get_path(Param::File, &t.ctx).unwrap().unwrap();
+        assert_eq!(path, fname);
+
+        // Blob does not exist yet, expect BlobError.
+        let err = p.get_blob(Param::File, &t.ctx, false).unwrap_err();
+        assert_eq!(err.kind(), BlobErrorKind::WrongBlobdir);
+
+        fs::write(fname, b"boo").unwrap();
+        let blob = p.get_blob(Param::File, &t.ctx, true).unwrap().unwrap();
+        assert_eq!(
+            blob,
+            BlobObject::from_name(&t.ctx, "foo".to_string()).unwrap()
+        );
+
+        // Blob in blobdir, expect blob.
+        let bar = t.ctx.get_blobdir().join("bar");
+        p.set(Param::File, bar.to_str().unwrap());
+        let blob = p.get_blob(Param::File, &t.ctx, false).unwrap().unwrap();
+        assert_eq!(
+            blob,
+            BlobObject::from_name(&t.ctx, "bar".to_string()).unwrap()
+        );
+
+        p.remove(Param::File);
+        assert!(p.get_file(Param::File, &t.ctx).unwrap().is_none());
+        assert!(p.get_path(Param::File, &t.ctx).unwrap().is_none());
+        assert!(p.get_blob(Param::File, &t.ctx, false).unwrap().is_none());
+    }
 }

src/qr.rs

@@ -90,7 +90,8 @@ fn decode_openpgp(context: &Context, qr: &str) -> Lot {
 
     // what is up with that param name?
     let name = if let Some(encoded_name) = param.get(Param::SetLongitude) {
-        match percent_decode_str(encoded_name).decode_utf8() {
+        let encoded_name = encoded_name.replace("+", "%20"); // sometimes spaces are encoded as `+`
+        match percent_decode_str(&encoded_name).decode_utf8() {
             Ok(name) => name.to_string(),
             Err(err) => return format_err!("Invalid name: {}", err).into(),
         }
@@ -104,7 +105,8 @@ fn decode_openpgp(context: &Context, qr: &str) -> Lot {
 
     let grpname = if grpid.is_some() {
         if let Some(encoded_name) = param.get(Param::GroupName) {
-            match percent_decode_str(encoded_name).decode_utf8() {
+            let encoded_name = encoded_name.replace("+", "%20"); // sometimes spaces are encoded as `+`
+            match percent_decode_str(&encoded_name).decode_utf8() {
                 Ok(name) => Some(name.to_string()),
                 Err(err) => return format_err!("Invalid group name: {}", err).into(),
             }
@@ -186,7 +188,7 @@ fn decode_mailto(context: &Context, qr: &str) -> Lot {
     let addr = if let Some(query_index) = payload.find('?') {
         &payload[..query_index]
     } else {
-        return format_err!("Invalid mailto found").into();
+        payload
     };
 
     let addr = match normalize_address(addr) {
@@ -405,13 +407,21 @@ mod tests {
             &ctx.ctx,
             "mailto:stress@test.local?subject=hello&body=world",
         );
-
         println!("{:?}", res);
         assert_eq!(res.get_state(), LotState::QrAddr);
         assert_ne!(res.get_id(), 0);
-
         let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
         assert_eq!(contact.get_addr(), "stress@test.local");
+
+        let res = check_qr(&ctx.ctx, "mailto:no-questionmark@example.org");
+        assert_eq!(res.get_state(), LotState::QrAddr);
+        assert_ne!(res.get_id(), 0);
+        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
+        assert_eq!(contact.get_addr(), "no-questionmark@example.org");
+
+        let res = check_qr(&ctx.ctx, "mailto:no-addr");
+        assert_eq!(res.get_state(), LotState::QrError);
+        assert!(res.get_text1().is_some());
     }
 
     #[test]
@@ -434,12 +444,13 @@ mod tests {
 
         let res = check_qr(
             &ctx.ctx,
-            "OPENPGP4FPR:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&g=testtesttest&x=h-0oKQf2CDK&i=9JEXlxAqGM0&s=0V7LzL9cxRL"
+            "OPENPGP4FPR:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&g=test%20%3F+test%20%21&x=h-0oKQf2CDK&i=9JEXlxAqGM0&s=0V7LzL9cxRL"
         );
 
         println!("{:?}", res);
         assert_eq!(res.get_state(), LotState::QrAskVerifyGroup);
         assert_ne!(res.get_id(), 0);
+        assert_eq!(res.get_text1().unwrap(), "test ? test !");
 
         let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
         assert_eq!(contact.get_addr(), "cli@deltachat.de");
@@ -451,7 +462,7 @@ mod tests {
 
         let res = check_qr(
             &ctx.ctx,
-            "OPENPGP4FPR:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&n=&i=TbnwJ6lSvD5&s=0ejvbdFSQxB"
+            "OPENPGP4FPR:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&n=J%C3%B6rn%20P.+P.&i=TbnwJ6lSvD5&s=0ejvbdFSQxB"
         );
 
         println!("{:?}", res);
@@ -460,5 +471,6 @@ mod tests {
 
         let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
         assert_eq!(contact.get_addr(), "cli@deltachat.de");
+        assert_eq!(contact.get_name(), "Jörn P. P.");
     }
 }

src/securejoin.rs

@@ -257,7 +257,7 @@ fn send_handshake_msg(
     msg.type_0 = Viewtype::Text;
     msg.text = Some(format!("Secure-Join: {}", step));
     msg.hidden = true;
-    msg.param.set_int(Param::Cmd, 7);
+    msg.param.set_cmd(SystemMessage::SecurejoinMessage);
     if step.is_empty() {
         msg.param.remove(Param::Arg);
     } else {
@@ -278,7 +278,7 @@ fn send_handshake_msg(
             ForcePlaintext::AddAutocryptHeader as i32,
         );
     } else {
-        msg.param.set_int(Param::GuranteeE2ee, 1);
+        msg.param.set_int(Param::GuaranteeE2ee, 1);
     }
     // TODO. handle cleanup on error
     chat::send_msg(context, contact_chat_id, &mut msg).unwrap_or_default();

src/smtp.rs

@@ -71,7 +71,7 @@ impl Smtp {
         let tls = dc_build_tls(lp.smtp_certificate_checks).unwrap();
         let tls_parameters = ClientTlsParameters::new(domain.to_string(), tls);
 
-        let creds = if 0 != lp.server_flags & (DC_LP_AUTH_OAUTH2 as i32) {
+        let (creds, mechanism) = if 0 != lp.server_flags & (DC_LP_AUTH_OAUTH2 as i32) {
             // oauth2
             let addr = &lp.addr;
             let send_pw = &lp.send_pw;
@@ -81,15 +81,24 @@ impl Smtp {
             }
             let user = &lp.send_user;
 
-            lettre::smtp::authentication::Credentials::new(
-                user.to_string(),
-                access_token.unwrap_or_default(),
+            (
+                lettre::smtp::authentication::Credentials::new(
+                    user.to_string(),
+                    access_token.unwrap_or_default(),
+                ),
+                vec![lettre::smtp::authentication::Mechanism::Xoauth2],
             )
         } else {
             // plain
             let user = lp.send_user.clone();
             let pw = lp.send_pw.clone();
-            lettre::smtp::authentication::Credentials::new(user, pw)
+            (
+                lettre::smtp::authentication::Credentials::new(user, pw),
+                vec![
+                    lettre::smtp::authentication::Mechanism::Plain,
+                    lettre::smtp::authentication::Mechanism::Login,
+                ],
+            )
         };
 
         let security = if 0
@@ -105,19 +114,29 @@ impl Smtp {
                 let client = client
                     .smtp_utf8(true)
                     .credentials(creds)
+                    .authentication_mechanism(mechanism)
                     .connection_reuse(lettre::smtp::ConnectionReuseParameters::ReuseUnlimited);
-                self.transport = Some(client.transport());
-                context.call_cb(Event::SmtpConnected(format!(
-                    "SMTP-LOGIN as {} ok",
-                    lp.send_user,
-                )));
-                true
+                let mut trans = client.transport();
+                match trans.connect() {
+                    Ok(()) => {
+                        self.transport = Some(trans);
+                        self.transport_connected = true;
+                        context.call_cb(Event::SmtpConnected(format!(
+                            "SMTP-LOGIN as {} ok",
+                            lp.send_user,
+                        )));
+                        return true;
+                    }
+                    Err(err) => {
+                        warn!(context, "SMTP: failed to connect {:?}", err);
+                    }
+                }
             }
             Err(err) => {
-                warn!(context, "SMTP: failed to establish connection {:?}", err);
-                false
+                warn!(context, "SMTP: failed to setup connection {:?}", err);
             }
         }
+        false
     }
 
     /// SMTP-Send a prepared mail to recipients.

src/sql.rs

@@ -11,8 +11,6 @@ use crate::error::{Error, Result};
 use crate::param::*;
 use crate::peerstate::*;
 
-const DC_OPEN_READONLY: usize = 0x01;
-
 /// A wrapper around the underlying Sqlite3 object.
 #[derive(DebugStub)]
 pub struct Sql {
@@ -42,8 +40,8 @@ impl Sql {
     }
 
     // return true on success, false on failure
-    pub fn open(&self, context: &Context, dbfile: &std::path::Path, flags: libc::c_int) -> bool {
-        match open(context, self, dbfile, flags) {
+    pub fn open(&self, context: &Context, dbfile: &std::path::Path, readonly: bool) -> bool {
+        match open(context, self, dbfile, readonly) {
             Ok(_) => true,
             Err(Error::SqlAlreadyOpen) => false,
             Err(_) => {
@@ -320,7 +318,7 @@ fn open(
     context: &Context,
     sql: &Sql,
     dbfile: impl AsRef<std::path::Path>,
-    flags: libc::c_int,
+    readonly: bool,
 ) -> Result<()> {
     if sql.is_open() {
         error!(
@@ -332,7 +330,7 @@ fn open(
     }
 
     let mut open_flags = OpenFlags::SQLITE_OPEN_NO_MUTEX;
-    if 0 != (flags & DC_OPEN_READONLY as i32) {
+    if readonly {
         open_flags.insert(OpenFlags::SQLITE_OPEN_READ_ONLY);
     } else {
         open_flags.insert(OpenFlags::SQLITE_OPEN_READ_WRITE);
@@ -351,7 +349,7 @@ fn open(
         *sql.pool.write().unwrap() = Some(pool);
     }
 
-    if 0 == flags & DC_OPEN_READONLY as i32 {
+    if !readonly {
         let mut exists_before_update = 0;
         let mut dbversion_before_update = 0;
         /* Init tables to dbversion=0 */

src/stock.rs

@@ -96,7 +96,7 @@ pub enum StockMessage {
     SelfTalkSubTitle = 50,
     #[strum(props(fallback = "Cannot login as %1$s."))]
     CannotLogin = 60,
-    #[strum(props(fallback = "Response from %1$s: %2$s"))]
+    #[strum(props(fallback = "Could not connect to %1$s: %2$s"))]
     ServerResponse = 61,
     #[strum(props(fallback = "%1$s by %2$s."))]
     MsgActionByUser = 62,
@@ -192,7 +192,7 @@ impl Context {
     /// placeholders with the string in `insert` and does the same for
     /// `%2$s`, `%2$d` and `%2$@` for `insert2`.
     /// (the `%1$@` variant is used on iOS, the other are used on Android and Desktop)
-    fn stock_string_repl_str2(
+    pub fn stock_string_repl_str2(
         &self,
         id: StockMessage,
         insert: impl AsRef<str>,
@@ -334,7 +334,7 @@ mod tests {
         assert_eq!(
             t.ctx
                 .stock_string_repl_str2(StockMessage::ServerResponse, "foo", "bar"),
-            "Response from foo: bar"
+            "Could not connect to foo: bar"
         );
     }
 

src/top_evil_rs.py

@@ -16,18 +16,20 @@ if __name__ == "__main__":
         free = s.count("free(")
         unsafe_fn = s.count("unsafe fn")
         chars = s.count("c_char") + s.count("CStr")
-        filestats.append((fn, unsafe, free, unsafe_fn, chars))
+        libc = s.count("libc")
+        filestats.append((fn, unsafe, free, unsafe_fn, chars, libc))
 
-    sum_unsafe, sum_free, sum_unsafe_fn, sum_chars = 0, 0, 0, 0
+    sum_unsafe, sum_free, sum_unsafe_fn, sum_chars, sum_libc = 0, 0, 0, 0, 0
 
-    for fn, unsafe, free, unsafe_fn, chars in reversed(sorted(filestats, key=lambda x: sum(x[1:]))):
-        if unsafe + free + unsafe_fn + chars == 0:
+    for fn, unsafe, free, unsafe_fn, chars, libc in reversed(sorted(filestats, key=lambda x: sum(x[1:]))):
+        if unsafe + free + unsafe_fn + chars + libc == 0:
             continue
-        print("{0: <25} unsafe: {1: >3} free: {2: >3} unsafe-fn: {3: >3} chars: {4: >3}".format(str(fn), unsafe, free, unsafe_fn, chars))
+        print("{0: <25} unsafe: {1: >3} free: {2: >3} unsafe-fn: {3: >3} chars: {4: >3} libc: {5: >3}".format(str(fn), unsafe, free, unsafe_fn, chars, libc))
         sum_unsafe += unsafe
         sum_free += free
         sum_unsafe_fn += unsafe_fn
         sum_chars += chars
+        sum_libc += libc
 
 
     print()
@@ -35,3 +37,4 @@ if __name__ == "__main__":
     print("total free:", sum_free)
     print("total unsafe-fn:", sum_unsafe_fn)
     print("total c_chars:", sum_chars)
+    print("total libc:", sum_libc)