adapt tests for new avatar recoding

CHANGELOG.md

@@ -1,42 +1,5 @@
 # Changelog 
 
-## 1.0.0-beta.19
-
-- #1058 timeout smtp-send if it doesn't complete in 15 minutes 
-
-- #1059 trim down logging
-
-## 1.0.0-beta.18
-
-- #1056 avoid panicking when we couldn't read imap-server's greeting
-  message 
-
-- #1055 avoid panicking when we don't have a selected folder
-
-- #1052 #1049 #1051 improve logging to add thread-id/name and
-  file/lineno to each info/warn message.
-
-- #1050 allow python bindings to initialize Account with "os_name".
-
-
-## 1.0.0-beta.17
-
-- #1044 implement avatar recoding to 192x192 in core to keep file sizes small. 
-
-- #1024 fix #1021 SQL/injection malformed Chat-Group-Name breakage
-
-- #1036 fix smtp crash by pulling in a fixed async-smtp 
-
-- #1039 fix read-receipts appearing as normal messages when you change
-  MDN settings 
-
-- #1040 do not panic on SystemTimeDifference
-
-- #1043 avoid potential crashes in malformed From/Chat-Disposition... headers  
-
-- #1045 #1041 #1038 #1035 #1034 #1029 #1025 various cleanups and doc
-  improvments 
-
 ## 1.0.0-beta.16
 
 - alleviate login problems with providers which only

Cargo.lock

@@ -85,7 +85,7 @@ dependencies = [
 [[package]]
 name = "async-imap"
 version = "0.1.1"
-source = "git+https://github.com/async-email/async-imap?branch=dcc-stable#4f0852971c3f397c2b8a343ef1378992c02f61de"
+source = "git+https://github.com/async-email/async-imap#d7836416766b55d8d03587ea5326eecf501c2030"
 dependencies = [
  "async-attributes 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "async-native-tls 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -124,7 +124,7 @@ dependencies = [
 [[package]]
 name = "async-smtp"
 version = "0.1.0"
-source = "git+https://github.com/async-email/async-smtp#6a4830032953f06020edc09db8daa06193e2b93f"
+source = "git+https://github.com/async-email/async-smtp#4f8416a0b8e0f8369459bb2fd342e79a17eb836b"
 dependencies = [
  "async-native-tls 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "async-std 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -657,9 +657,9 @@ dependencies = [
 
 [[package]]
 name = "deltachat"
-version = "1.0.0-beta.19"
+version = "1.0.0-beta.16"
 dependencies = [
- "async-imap 0.1.1 (git+https://github.com/async-email/async-imap?branch=dcc-stable)",
+ "async-imap 0.1.1 (git+https://github.com/async-email/async-imap)",
  "async-native-tls 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "async-smtp 0.1.0 (git+https://github.com/async-email/async-smtp)",
  "async-std 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -734,9 +734,9 @@ dependencies = [
 
 [[package]]
 name = "deltachat_ffi"
-version = "1.0.0-beta.19"
+version = "1.0.0-beta.16"
 dependencies = [
- "deltachat 1.0.0-beta.19",
+ "deltachat 1.0.0-beta.16",
  "deltachat-provider-database 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "human-panic 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3469,7 +3469,7 @@ dependencies = [
 "checksum arrayvec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8"
 "checksum ascii_utils 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "71938f30533e4d95a6d17aa530939da3842c2ab6f4f84b9dae68447e4129f74a"
 "checksum async-attributes 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "efd3d156917d94862e779f356c5acae312b08fd3121e792c857d7928c8088423"
-"checksum async-imap 0.1.1 (git+https://github.com/async-email/async-imap?branch=dcc-stable)" = "<none>"
+"checksum async-imap 0.1.1 (git+https://github.com/async-email/async-imap)" = "<none>"
 "checksum async-macros 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "644a5a8de80f2085a1e7e57cd1544a2a7438f6e003c0790999bd43b92a77cdb2"
 "checksum async-native-tls 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a29e9e4ed87f4070dd6099d0bd5edfc7692c94442c80d656b50a802c6fde23b7"
 "checksum async-smtp 0.1.0 (git+https://github.com/async-email/async-smtp)" = "<none>"

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.0.0-beta.19"
+version = "1.0.0-beta.16"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL"
@@ -17,13 +17,10 @@ smallvec = "1.0.0"
 reqwest = { version = "0.9.15" }
 num-derive = "0.3.0"
 num-traits = "0.2.6"
-async-smtp = { git = "https://github.com/async-email/async-smtp" }
+async-smtp = { git = "https://github.com/async-email/async-smtp", branch = "master" }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "native_tls" }
-
-# XXX newer commits of async-imap lead to import-export tests hanging
-async-imap = { git = "https://github.com/async-email/async-imap", branch = "dcc-stable" }
-
+async-imap = { git = "https://github.com/async-email/async-imap" }
 async-native-tls = "0.1.1"
 async-std = { version = "1.0", features = ["unstable"] }
 base64 = "0.11"

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.0.0-beta.19"
+version = "1.0.0-beta.16"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/src/lib.rs

@@ -509,7 +509,7 @@ pub unsafe extern "C" fn dc_interrupt_imap_idle(context: *mut dc_context_t) {
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| job::interrupt_inbox_idle(ctx))
+        .with_inner(|ctx| job::interrupt_inbox_idle(ctx, true))
         .unwrap_or(())
 }
 

examples/repl/cmdline.rs

@@ -491,7 +491,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
             println!("{:#?}", context.get_info());
         }
         "interrupt" => {
-            interrupt_inbox_idle(context);
+            interrupt_inbox_idle(context, true);
         }
         "maybenetwork" => {
             maybe_network(context);

examples/repl/main.rs

@@ -202,7 +202,7 @@ fn stop_threads(context: &Context) {
         println!("Stopping threads");
         IS_RUNNING.store(false, Ordering::Relaxed);
 
-        interrupt_inbox_idle(context);
+        interrupt_inbox_idle(context, true);
         interrupt_mvbox_idle(context);
         interrupt_sentbox_idle(context);
         interrupt_smtp_idle(context);

examples/simple.rs

@@ -104,7 +104,7 @@ fn main() {
     println!("stopping threads");
 
     *running.write().unwrap() = false;
-    deltachat::job::interrupt_inbox_idle(&ctx);
+    deltachat::job::interrupt_inbox_idle(&ctx, true);
     deltachat::job::interrupt_smtp_idle(&ctx);
 
     println!("joining");

python/src/deltachat/account.py

@@ -26,7 +26,7 @@ class Account(object):
     by the underlying deltachat core library.  All public Account methods are
     meant to be memory-safe and return memory-safe objects.
     """
-    def __init__(self, db_path, logid=None, eventlogging=True, os_name=None, debug=True):
+    def __init__(self, db_path, logid=None, eventlogging=True, debug=True):
         """ initialize account object.
 
         :param db_path: a path to the account database. The database
@@ -34,11 +34,10 @@ class Account(object):
         :param logid: an optional logging prefix that should be used with
                       the default internal logging.
         :param eventlogging: if False no eventlogging and no context callback will be configured
-        :param os_name: this will be put to the X-Mailer header in outgoing messages
         :param debug: turn on debug logging for events.
         """
         self._dc_context = ffi.gc(
-            lib.dc_context_new(lib.py_dc_callback, ffi.NULL, as_dc_charpointer(os_name)),
+            lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
             _destroy_dc_context,
         )
         if eventlogging:

python/tests/test_account.py

@@ -16,14 +16,6 @@ class TestOfflineAccountBasic:
         with pytest.raises(ValueError):
             Account(p.strpath)
 
-    def test_os_name(self, tmpdir):
-        p = tmpdir.join("hello.db")
-        # we can't easily test if os_name is used in X-Mailer
-        # outgoing messages without a full Online test
-        # but we at least check Account accepts the arg
-        ac1 = Account(p.strpath, os_name="solarpunk")
-        ac1.get_info()
-
     def test_getinfo(self, acfactory):
         ac1 = acfactory.get_unconfigured_account()
         d = ac1.get_info()
@@ -695,39 +687,6 @@ class TestOnlineAccount:
         except queue.Empty:
             pass  # mark_seen_messages() has generated events before it returns
 
-    def test_mdn_asymetric(self, acfactory, lp):
-        ac1, ac2 = acfactory.get_two_online_accounts()
-
-        lp.sec("ac1: create chat with ac2")
-        chat = self.get_chat(ac1, ac2, both_created=True)
-
-        # make sure mdns are enabled (usually enabled by default already)
-        ac1.set_config("mdns_enabled", "1")
-        ac2.set_config("mdns_enabled", "1")
-
-        lp.sec("sending text message from ac1 to ac2")
-        msg_out = chat.send_text("message1")
-
-        assert len(chat.get_messages()) == 1
-
-        lp.sec("disable ac1 MDNs")
-        ac1.set_config("mdns_enabled", "0")
-
-        lp.sec("wait for ac2 to receive message")
-        msg = ac2.wait_next_incoming_message()
-
-        assert len(msg.chat.get_messages()) == 1
-
-        lp.sec("ac2: mark incoming message as seen")
-        ac2.mark_seen_messages([msg])
-
-        lp.sec("ac1: waiting for incoming activity")
-        # wait for MOVED event because even ignored read-receipts should be moved
-        ac1._evlogger.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
-
-        assert len(chat.get_messages()) == 1
-        assert not msg_out.is_out_mdn_received()
-
     def test_send_and_receive_will_encrypt_decrypt(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 

set_core_version.py

@@ -55,7 +55,6 @@ if __name__ == "__main__":
     replace_toml_version("Cargo.toml", newversion)
     replace_toml_version("deltachat-ffi/Cargo.toml", newversion)
 
-    subprocess.call(["cargo", "check"])
     subprocess.call(["git", "add", "-u"])
     # subprocess.call(["cargo", "update", "-p", "deltachat"])
 

src/config.rs

@@ -144,7 +144,7 @@ impl Context {
             }
             Config::InboxWatch => {
                 let ret = self.sql.set_raw_config(self, key, value);
-                interrupt_inbox_idle(self);
+                interrupt_inbox_idle(self, true);
                 ret
             }
             Config::SentboxWatch => {

src/constants.rs

@@ -114,7 +114,7 @@ pub const DC_MSG_ID_LAST_SPECIAL: u32 = 9;
 /// approx. max. length returned by dc_msg_get_text()
 const DC_MAX_GET_TEXT_LEN: usize = 30000;
 /// approx. max. length returned by dc_get_msg_info()
-const DC_MAX_GET_INFO_LEN: usize = 100_000;
+const DC_MAX_GET_INFO_LEN: usize = 100000;
 
 pub const DC_CONTACT_ID_UNDEFINED: u32 = 0;
 pub const DC_CONTACT_ID_SELF: u32 = 1;

src/contact.rs

@@ -100,11 +100,11 @@ pub enum Origin {
     /// address is in our address book
     AdressBook = 0x80000,
     /// set on Alice's side for contacts like Bob that have scanned the QR code offered by her. Only means the contact has once been established using the "securejoin" procedure in the past, getting the current key verification status requires calling dc_contact_is_verified() !
-    SecurejoinInvited = 0x0100_0000,
+    SecurejoinInvited = 0x1000000,
     /// set on Bob's side for contacts scanned and verified from a QR code. Only means the contact has once been established using the "securejoin" procedure in the past, getting the current key verification status requires calling dc_contact_is_verified() !
-    SecurejoinJoined = 0x0200_0000,
+    SecurejoinJoined = 0x2000000,
     /// contact added mannually by dc_create_contact(), this should be the largets origin as otherwise the user cannot modify the names
-    ManuallyCreated = 0x0400_0000,
+    ManuallyCreated = 0x4000000,
 }
 
 impl Default for Origin {
@@ -971,7 +971,7 @@ fn set_block_contact(context: &Context, contact_id: u32, new_blocking: bool) {
 pub fn set_profile_image(
     context: &Context,
     contact_id: u32,
-    profile_image: &AvatarAction,
+    profile_image: AvatarAction,
 ) -> Result<()> {
     // the given profile image is expected to be already in the blob directory
     // as profile images can be set only by receiving messages, this should be always the case, however.

src/dc_receive_imf.rs

@@ -73,13 +73,15 @@ pub fn dc_receive_imf(
     let mut sent_timestamp = 0;
     let mut created_db_entries = Vec::new();
     let mut create_event_to_send = Some(CreateEvent::MsgsChanged);
+    let mut rr_event_to_send = Vec::new();
 
     let mut to_ids = ContactIds::new();
 
     // helper method to handle early exit and memory cleanup
     let cleanup = |context: &Context,
                    create_event_to_send: &Option<CreateEvent>,
-                   created_db_entries: &Vec<(usize, MsgId)>| {
+                   created_db_entries: &Vec<(usize, MsgId)>,
+                   rr_event_to_send: &Vec<(u32, MsgId)>| {
         if let Some(create_event_to_send) = create_event_to_send {
             for (chat_id, msg_id) in created_db_entries {
                 let event = match create_event_to_send {
@@ -95,6 +97,12 @@ pub fn dc_receive_imf(
                 context.call_cb(event);
             }
         }
+        for (chat_id, msg_id) in rr_event_to_send {
+            context.call_cb(Event::MsgRead {
+                chat_id: *chat_id,
+                msg_id: *msg_id,
+            });
+        }
     };
 
     if let Some(value) = mime_parser.get(HeaderDef::Date) {
@@ -171,7 +179,7 @@ pub fn dc_receive_imf(
             }
         }
     };
-    if mime_parser.parts.last().is_some() {
+    if mime_parser.get_last_nonmeta().is_some() {
         if let Err(err) = add_parts(
             context,
             &mut mime_parser,
@@ -194,17 +202,30 @@ pub fn dc_receive_imf(
             &mut created_db_entries,
             &mut create_event_to_send,
         ) {
-            cleanup(context, &create_event_to_send, &created_db_entries);
+            cleanup(
+                context,
+                &create_event_to_send,
+                &created_db_entries,
+                &rr_event_to_send,
+            );
             bail!("add_parts error: {:?}", err);
         }
     } else {
-        // there are parts in this message, do some basic calculations so that the variables
+        // there are no non-meta data in message, do some basic calculations so that the varaiables
         // are correct in the further processing
         if sent_timestamp > time() {
             sent_timestamp = time()
         }
     }
 
+    mime_parser.handle_reports(
+        from_id,
+        sent_timestamp,
+        &mut rr_event_to_send,
+        &server_folder,
+        server_uid,
+    );
+
     if mime_parser.location_kml.is_some() || mime_parser.message_kml.is_some() {
         save_locations(
             context,
@@ -217,7 +238,7 @@ pub fn dc_receive_imf(
     }
 
     if mime_parser.user_avatar != AvatarAction::None {
-        match contact::set_profile_image(&context, from_id, &mime_parser.user_avatar) {
+        match contact::set_profile_image(&context, from_id, mime_parser.user_avatar) {
             Ok(()) => {
                 context.call_cb(Event::ChatModified(chat_id));
             }
@@ -245,9 +266,12 @@ pub fn dc_receive_imf(
         "received message {} has Message-Id: {}", server_uid, rfc724_mid
     );
 
-    cleanup(context, &create_event_to_send, &created_db_entries);
-
-    mime_parser.handle_reports(from_id, sent_timestamp, &server_folder, server_uid);
+    cleanup(
+        context,
+        &create_event_to_send,
+        &created_db_entries,
+        &rr_event_to_send,
+    );
 
     Ok(())
 }
@@ -588,6 +612,10 @@ fn add_parts(
             let subject = mime_parser.get_subject().unwrap_or_default();
 
             for part in mime_parser.parts.iter_mut() {
+                if part.is_meta {
+                    continue;
+                }
+
                 if mime_parser.location_kml.is_some()
                     && icnt == 1
                     && (part.msg == "-location-" || part.msg.is_empty())
@@ -1546,6 +1574,10 @@ fn dc_add_or_lookup_contacts_by_address_list(
         }
     };
 
+    info!(
+        context,
+        "dc_add_or_lookup_contacts_by_address raw={:?} addrs={:?}", addr_list_raw, addrs
+    );
     for addr in addrs.iter() {
         match addr {
             mailparse::MailAddr::Single(info) => {
@@ -1587,6 +1619,10 @@ fn add_or_lookup_contact_by_addr(
         .map(normalize_name)
         .unwrap_or_default();
 
+    info!(
+        context,
+        "looking up addr={:?} display_name={:?}", addr, display_name_normalized
+    );
     let (row_id, _modified) =
         Contact::add_or_lookup(context, display_name_normalized, addr, origin)?;
     ensure!(row_id > 0, "could not add contact: {:?}", addr);

src/dc_simplify.rs

@@ -1,169 +1,175 @@
 use crate::dehtml::*;
 
-/// Remove standard (RFC 3676, §4.3) footer if it is found.
-fn remove_message_footer<'a>(lines: &'a [&str]) -> &'a [&'a str] {
+#[derive(Copy, Clone)]
+pub struct Simplify {
+    pub is_forwarded: bool,
+}
+
+/// Return index of footer line in vector of message lines, or vector length if
+/// no footer is found.
+///
+/// Also return whether not-standard (rfc3676, §4.3) footer is found.
+fn find_message_footer(lines: &[&str]) -> (usize, bool) {
     for (ix, &line) in lines.iter().enumerate() {
         // quoted-printable may encode `-- ` to `-- =20` which is converted
         // back to `--  `
         match line {
-            "-- " | "--  " => return &lines[..ix],
+            "-- " | "--  " => return (ix, false),
+            "--" | "---" | "----" => return (ix, true),
             _ => (),
         }
     }
-    lines
+    (lines.len(), false)
 }
 
-/// Remove nonstandard footer and a boolean indicating whether such
-/// footer was removed.
-fn remove_nonstandard_footer<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
-    for (ix, &line) in lines.iter().enumerate() {
-        if line == "--"
-            || line == "---"
-            || line == "----"
-            || line.starts_with("-----")
-            || line.starts_with("_____")
-            || line.starts_with("=====")
-            || line.starts_with("*****")
-            || line.starts_with("~~~~~")
-        {
-            return (&lines[..ix], true);
+impl Simplify {
+    pub fn new() -> Self {
+        Simplify {
+            is_forwarded: false,
         }
     }
-    (lines, false)
-}
 
-fn split_lines(buf: &str) -> Vec<&str> {
-    buf.split('\n').collect()
-}
+    /// Simplify and normalise text: Remove quotes, signatures, unnecessary
+    /// lineends etc.
+    /// The data returned from simplify() must be free()'d when no longer used.
+    pub fn simplify(&mut self, input: &str, is_html: bool, is_msgrmsg: bool) -> String {
+        let mut out = if is_html {
+            dehtml(input)
+        } else {
+            input.to_string()
+        };
 
-/// Simplify message text for chat display.
-/// Remove quotes, signatures, trailing empty lines etc.
-pub fn simplify(input: &str, is_html: bool, is_chat_message: bool) -> (String, bool) {
-    let mut out = if is_html {
-        dehtml(input)
-    } else {
-        input.to_string()
-    };
+        out.retain(|c| c != '\r');
+        out = self.simplify_plain_text(&out, is_msgrmsg);
+        out.retain(|c| c != '\r');
 
-    out.retain(|c| c != '\r');
-    let lines = split_lines(&out);
-    let (lines, is_forwarded) = skip_forward_header(&lines);
-
-    let lines = remove_message_footer(lines);
-    let (lines, has_nonstandard_footer) = remove_nonstandard_footer(lines);
-    let (lines, has_bottom_quote) = if !is_chat_message {
-        remove_bottom_quote(lines)
-    } else {
-        (lines, false)
-    };
-    let (lines, has_top_quote) = if !is_chat_message {
-        remove_top_quote(lines)
-    } else {
-        (lines, false)
-    };
-
-    // re-create buffer from the remaining lines
-    let text = render_message(
-        lines,
-        has_top_quote,
-        has_nonstandard_footer || has_bottom_quote,
-    );
-    (text, is_forwarded)
-}
-
-/// Skips "forwarded message" header.
-/// Returns message body lines and a boolean indicating whether
-/// a message is forwarded or not.
-fn skip_forward_header<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
-    if lines.len() >= 3
-        && lines[0] == "---------- Forwarded message ----------"
-        && lines[1].starts_with("From: ")
-        && lines[2].is_empty()
-    {
-        (&lines[3..], true)
-    } else {
-        (lines, false)
+        out
     }
-}
 
-fn remove_bottom_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
-    let mut last_quoted_line = None;
-    for (l, line) in lines.iter().enumerate().rev() {
-        if is_plain_quote(line) {
-            last_quoted_line = Some(l)
-        } else if !is_empty_line(line) {
-            break;
-        }
-    }
-    if let Some(mut l_last) = last_quoted_line {
-        if l_last > 1 && is_empty_line(lines[l_last - 1]) {
-            l_last -= 1
-        }
-        if l_last > 1 {
-            let line = lines[l_last - 1];
-            if is_quoted_headline(line) {
-                l_last -= 1
+    /**
+     * Simplify Plain Text
+     */
+    #[allow(non_snake_case, clippy::mut_range_bound, clippy::needless_range_loop)]
+    fn simplify_plain_text(&mut self, buf_terminated: &str, is_msgrmsg: bool) -> String {
+        /* This function ...
+        ... removes all text after the line `-- ` (footer mark)
+        ... removes full quotes at the beginning and at the end of the text -
+            these are all lines starting with the character `>`
+        ... remove a non-empty line before the removed quote (contains sth. like "On 2.9.2016, Bjoern wrote:" in different formats and lanugages) */
+        /* split the given buffer into lines */
+        let lines: Vec<_> = buf_terminated.split('\n').collect();
+        let mut l_first: usize = 0;
+        let mut is_cut_at_begin = false;
+        let (mut l_last, mut is_cut_at_end) = find_message_footer(&lines);
+
+        if l_last > l_first + 2 {
+            let line0 = lines[l_first];
+            let line1 = lines[l_first + 1];
+            let line2 = lines[l_first + 2];
+            if line0 == "---------- Forwarded message ----------"
+                && line1.starts_with("From: ")
+                && line2.is_empty()
+            {
+                self.is_forwarded = true;
+                l_first += 3
             }
         }
-        (&lines[..l_last], true)
-    } else {
-        (lines, false)
-    }
-}
-
-fn remove_top_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
-    let mut last_quoted_line = None;
-    let mut has_quoted_headline = false;
-    for (l, line) in lines.iter().enumerate() {
-        if is_plain_quote(line) {
-            last_quoted_line = Some(l)
-        } else if !is_empty_line(line) {
-            if is_quoted_headline(line) && !has_quoted_headline && last_quoted_line.is_none() {
-                has_quoted_headline = true
-            } else {
-                /* non-quoting line found */
+        for l in l_first..l_last {
+            let line = lines[l];
+            if line == "-----"
+                || line == "_____"
+                || line == "====="
+                || line == "*****"
+                || line == "~~~~~"
+            {
+                l_last = l;
+                is_cut_at_end = true;
+                /* done */
                 break;
             }
         }
-    }
-    if let Some(last_quoted_line) = last_quoted_line {
-        (&lines[last_quoted_line + 1..], true)
-    } else {
-        (lines, false)
-    }
-}
-
-fn render_message(lines: &[&str], is_cut_at_begin: bool, is_cut_at_end: bool) -> String {
-    let mut ret = String::new();
-    if is_cut_at_begin {
-        ret += "[...]";
-    }
-    /* we write empty lines only in case and non-empty line follows */
-    let mut pending_linebreaks = 0;
-    let mut empty_body = true;
-    for line in lines {
-        if is_empty_line(line) {
-            pending_linebreaks += 1
-        } else {
-            if !empty_body {
-                if pending_linebreaks > 2 {
-                    pending_linebreaks = 2
-                }
-                while 0 != pending_linebreaks {
-                    ret += "\n";
-                    pending_linebreaks -= 1
+        if !is_msgrmsg {
+            let mut l_lastQuotedLine = None;
+            for l in (l_first..l_last).rev() {
+                let line = lines[l];
+                if is_plain_quote(line) {
+                    l_lastQuotedLine = Some(l)
+                } else if !is_empty_line(line) {
+                    break;
+                }
+            }
+            if let Some(last_quoted_line) = l_lastQuotedLine {
+                l_last = last_quoted_line;
+                is_cut_at_end = true;
+                if l_last > 1 && is_empty_line(lines[l_last - 1]) {
+                    l_last -= 1
+                }
+                if l_last > 1 {
+                    let line = lines[l_last - 1];
+                    if is_quoted_headline(line) {
+                        l_last -= 1
+                    }
                 }
             }
-            // the incoming message might contain invalid UTF8
-            ret += line;
-            empty_body = false;
-            pending_linebreaks = 1
         }
+        if !is_msgrmsg {
+            let mut l_lastQuotedLine_0 = None;
+            let mut hasQuotedHeadline = 0;
+            for l in l_first..l_last {
+                let line = lines[l];
+                if is_plain_quote(line) {
+                    l_lastQuotedLine_0 = Some(l)
+                } else if !is_empty_line(line) {
+                    if is_quoted_headline(line)
+                        && 0 == hasQuotedHeadline
+                        && l_lastQuotedLine_0.is_none()
+                    {
+                        hasQuotedHeadline = 1i32
+                    } else {
+                        /* non-quoting line found */
+                        break;
+                    }
+                }
+            }
+            if let Some(last_quoted_line) = l_lastQuotedLine_0 {
+                l_first = last_quoted_line + 1;
+                is_cut_at_begin = true
+            }
+        }
+        /* re-create buffer from the remaining lines */
+        let mut ret = String::new();
+        if is_cut_at_begin {
+            ret += "[...]";
+        }
+        /* we write empty lines only in case and non-empty line follows */
+        let mut pending_linebreaks = 0;
+        let mut content_lines_added = 0;
+        for l in l_first..l_last {
+            let line = lines[l];
+            if is_empty_line(line) {
+                pending_linebreaks += 1
+            } else {
+                if 0 != content_lines_added {
+                    if pending_linebreaks > 2i32 {
+                        pending_linebreaks = 2i32
+                    }
+                    while 0 != pending_linebreaks {
+                        ret += "\n";
+                        pending_linebreaks -= 1
+                    }
+                }
+                // the incoming message might contain invalid UTF8
+                ret += line;
+                content_lines_added += 1;
+                pending_linebreaks = 1i32
+            }
+        }
+        if is_cut_at_end && (!is_cut_at_begin || 0 != content_lines_added) {
+            ret += " [...]";
+        }
+
+        ret
     }
-    if is_cut_at_end && (!is_cut_at_begin || !empty_body) {
-        ret += " [...]";
-    }
-    ret
 }
 
 /**
@@ -207,59 +213,50 @@ mod tests {
         #[test]
         // proptest does not support [[:graphical:][:space:]] regex.
         fn test_simplify_plain_text_fuzzy(input in "[!-~\t \n]+") {
-            let (output, _is_forwarded) = simplify(&input, false, true);
+            let output = Simplify::new().simplify_plain_text(&input, true);
             assert!(output.split('\n').all(|s| s != "-- "));
         }
     }
 
     #[test]
     fn test_simplify_trim() {
+        let mut simplify = Simplify::new();
         let html = "\r\r\nline1<br>\r\n\r\n\r\rline2\n\r";
-        let (plain, is_forwarded) = simplify(html, true, false);
+        let plain = simplify.simplify(html, true, false);
 
         assert_eq!(plain, "line1\nline2");
-        assert!(!is_forwarded);
     }
 
     #[test]
     fn test_simplify_parse_href() {
+        let mut simplify = Simplify::new();
         let html = "<a href=url>text</a";
-        let (plain, is_forwarded) = simplify(html, true, false);
+        let plain = simplify.simplify(html, true, false);
 
         assert_eq!(plain, "[text](url)");
-        assert!(!is_forwarded);
     }
 
     #[test]
     fn test_simplify_bold_text() {
+        let mut simplify = Simplify::new();
         let html = "<!DOCTYPE name [<!DOCTYPE ...>]><!-- comment -->text <b><?php echo ... ?>bold</b><![CDATA[<>]]>";
-        let (plain, is_forwarded) = simplify(html, true, false);
+        let plain = simplify.simplify(html, true, false);
 
         assert_eq!(plain, "text *bold*<>");
-        assert!(!is_forwarded);
-    }
-
-    #[test]
-    fn test_simplify_forwarded_message() {
-        let text = "---------- Forwarded message ----------\r\nFrom: test@example.com\r\n\r\nForwarded message\r\n-- \r\nSignature goes here";
-        let (plain, is_forwarded) = simplify(text, false, false);
-
-        assert_eq!(plain, "Forwarded message");
-        assert!(is_forwarded);
     }
 
     #[test]
     fn test_simplify_html_encoded() {
+        let mut simplify = Simplify::new();
         let html =
                 "&lt;&gt;&quot;&apos;&amp; &auml;&Auml;&ouml;&Ouml;&uuml;&Uuml;&szlig; foo&AElig;&ccedil;&Ccedil; &diams;&lrm;&rlm;&zwnj;&noent;&zwj;";
 
-        let (plain, is_forwarded) = simplify(html, true, false);
+        let plain = simplify.simplify(html, true, false);
 
         assert_eq!(
             plain,
             "<>\"\'& äÄöÖüÜß fooÆçÇ \u{2666}\u{200e}\u{200f}\u{200c}&noent;\u{200d}"
         );
-        assert!(!is_forwarded);
     }
 
     #[test]
@@ -273,19 +270,4 @@ mod tests {
         assert!(!is_plain_quote("Life is pain"));
         assert!(!is_plain_quote(""));
     }
-
-    #[test]
-    fn test_remove_top_quote() {
-        let (lines, has_top_quote) = remove_top_quote(&["> first", "> second"]);
-        assert!(lines.is_empty());
-        assert!(has_top_quote);
-
-        let (lines, has_top_quote) = remove_top_quote(&["> first", "> second", "not a quote"]);
-        assert_eq!(lines, &["not a quote"]);
-        assert!(has_top_quote);
-
-        let (lines, has_top_quote) = remove_top_quote(&["not a quote", "> first", "> second"]);
-        assert_eq!(lines, &["not a quote", "> first", "> second"]);
-        assert!(!has_top_quote);
-    }
 }

src/dc_tools.rs

@@ -49,8 +49,8 @@ pub(crate) fn dc_truncate(buf: &str, approx_chars: usize, do_unwrap: bool) -> Co
 /// - harmonize together while being different enough
 /// (therefore, we cannot just use random rgb colors :)
 const COLORS: [u32; 16] = [
-    0xe5_65_55, 0xf2_8c_48, 0x8e_85_ee, 0x76_c8_4d, 0x5b_b6_cc, 0x54_9c_dd, 0xd2_5c_99, 0xb3_78_00,
-    0xf2_30_30, 0x39_b2_49, 0xbb_24_3b, 0x96_40_78, 0x66_87_4f, 0x30_8a_b9, 0x12_7e_d0, 0xbe_45_0c,
+    0xe56555, 0xf28c48, 0x8e85ee, 0x76c84d, 0x5bb6cc, 0x549cdd, 0xd25c99, 0xb37800, 0xf23030,
+    0x39b249, 0xbb243b, 0x964078, 0x66874f, 0x308ab9, 0x127ed0, 0xbe450c,
 ];
 
 pub(crate) fn dc_str_to_color(s: impl AsRef<str>) -> u32 {
@@ -59,7 +59,7 @@ pub(crate) fn dc_str_to_color(s: impl AsRef<str>) -> u32 {
     let bytes = str_lower.as_bytes();
     for (i, byte) in bytes.iter().enumerate() {
         checksum += (i + 1) * *byte as usize;
-        checksum %= 0x00ff_ffff;
+        checksum %= 0xffffff;
     }
     let color_index = checksum % COLORS.len();
 
@@ -381,14 +381,11 @@ pub(crate) fn dc_copy_file(
     }
 }
 
-pub(crate) fn dc_create_folder(
-    context: &Context,
-    path: impl AsRef<std::path::Path>,
-) -> Result<(), std::io::Error> {
+pub(crate) fn dc_create_folder(context: &Context, path: impl AsRef<std::path::Path>) -> bool {
     let path_abs = dc_get_abs_path(context, &path);
     if !path_abs.exists() {
         match fs::create_dir_all(path_abs) {
-            Ok(_) => Ok(()),
+            Ok(_) => true,
             Err(err) => {
                 warn!(
                     context,
@@ -396,11 +393,11 @@ pub(crate) fn dc_create_folder(
                     path.as_ref().display(),
                     err
                 );
-                Err(err)
+                false
             }
         }
     } else {
-        Ok(())
+        true
     }
 }
 
@@ -828,7 +825,7 @@ mod tests {
 
         assert!(dc_delete_file(context, "$BLOBDIR/foobar"));
         assert!(dc_delete_file(context, "$BLOBDIR/dada"));
-        assert!(dc_create_folder(context, "$BLOBDIR/foobar-folder").is_ok());
+        assert!(dc_create_folder(context, "$BLOBDIR/foobar-folder"));
         assert!(dc_file_exist(context, "$BLOBDIR/foobar-folder",));
         assert!(!dc_delete_file(context, "$BLOBDIR/foobar-folder"));
 

src/dehtml.rs

@@ -139,12 +139,13 @@ fn dehtml_starttag_cb<B: std::io::BufRead>(
             dehtml.add_text = AddText::YesPreserveLineEnds;
         }
         "a" => {
-            if let Some(href) = event
-                .html_attributes()
-                .filter_map(|attr| attr.ok())
-                .find(|attr| String::from_utf8_lossy(attr.key).trim().to_lowercase() == "href")
-            {
+            if let Some(href) = event.html_attributes().find(|attr| {
+                attr.as_ref()
+                    .map(|a| String::from_utf8_lossy(a.key).trim().to_lowercase() == "href")
+                    .unwrap_or_default()
+            }) {
                 let href = href
+                    .unwrap()
                     .unescape_and_decode_value(reader)
                     .unwrap_or_default()
                     .to_lowercase();

src/e2ee.rs

@@ -2,7 +2,7 @@
 
 use std::collections::HashSet;
 
-use mailparse::{MailHeaderMap, ParsedMail};
+use mailparse::MailHeaderMap;
 use num_traits::FromPrimitive;
 
 use crate::aheader::*;
@@ -14,6 +14,7 @@ use crate::keyring::*;
 use crate::peerstate::*;
 use crate::pgp;
 use crate::securejoin::handle_degrade_event;
+use crate::wrapmime;
 
 #[derive(Debug)]
 pub struct EncryptHelper {
@@ -117,7 +118,7 @@ impl EncryptHelper {
 
 pub fn try_decrypt(
     context: &Context,
-    mail: &ParsedMail<'_>,
+    mail: &mailparse::ParsedMail<'_>,
     message_time: i64,
 ) -> Result<(Option<Vec<u8>>, HashSet<String>)> {
     let from = mail
@@ -231,36 +232,9 @@ fn load_or_generate_self_public_key(context: &Context, self_addr: impl AsRef<str
     }
 }
 
-/// Returns a reference to the encrypted payload and validates the autocrypt structure.
-fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Result<&'a ParsedMail<'b>> {
-    ensure!(
-        mail.ctype.mimetype == "multipart/encrypted",
-        "Not a multipart/encrypted message: {}",
-        mail.ctype.mimetype
-    );
-    ensure!(
-        mail.subparts.len() == 2,
-        "Invalid Autocrypt Level 1 Mime Parts"
-    );
-
-    ensure!(
-        mail.subparts[0].ctype.mimetype == "application/pgp-encrypted",
-        "Invalid Autocrypt Level 1 version part: {:?}",
-        mail.subparts[0].ctype,
-    );
-
-    ensure!(
-        mail.subparts[1].ctype.mimetype == "application/octet-stream",
-        "Invalid Autocrypt Level 1 encrypted part: {:?}",
-        mail.subparts[1].ctype
-    );
-
-    Ok(&mail.subparts[1])
-}
-
 fn decrypt_if_autocrypt_message<'a>(
     context: &Context,
-    mail: &ParsedMail<'a>,
+    mail: &mailparse::ParsedMail<'a>,
     private_keyring: &Keyring,
     public_keyring_for_validate: &Keyring,
     ret_valid_signatures: &mut HashSet<String>,
@@ -272,7 +246,7 @@ fn decrypt_if_autocrypt_message<'a>(
     //
     // Errors are returned for failures related to decryption of AC-messages.
 
-    let encrypted_data_part = match get_autocrypt_mime(mail) {
+    let encrypted_data_part = match wrapmime::get_autocrypt_mime(mail) {
         Err(_) => {
             // not an autocrypt mime message, abort and ignore
             return Ok(None);
@@ -293,7 +267,7 @@ fn decrypt_if_autocrypt_message<'a>(
 /// Returns Ok(None) if nothing encrypted was found.
 fn decrypt_part(
     _context: &Context,
-    mail: &ParsedMail<'_>,
+    mail: &mailparse::ParsedMail<'_>,
     private_keyring: &Keyring,
     public_keyring_for_validate: &Keyring,
     ret_valid_signatures: &mut HashSet<String>,
@@ -339,7 +313,7 @@ fn has_decrypted_pgp_armor(input: &[u8]) -> bool {
 /// However, Delta Chat itself has no problem with encrypted multipart/report
 /// parts and MUAs should be encouraged to encrpyt multipart/reports as well so
 /// that we could use the normal Autocrypt processing.
-fn contains_report(mail: &ParsedMail<'_>) -> bool {
+fn contains_report(mail: &mailparse::ParsedMail<'_>) -> bool {
     mail.ctype.mimetype == "multipart/report"
 }
 

src/imap/idle.rs

@@ -241,7 +241,7 @@ impl Imap {
                 "IMAP-fake-IDLE done after {:.4}s",
                 SystemTime::now()
                     .duration_since(fake_idle_start_time)
-                    .unwrap_or_default()
+                    .unwrap()
                     .as_millis() as f64
                     / 1000.,
             );

src/imap/mod.rs

@@ -23,6 +23,7 @@ use crate::message::{self, update_server_uid};
 use crate::oauth2::dc_get_oauth2_access_token;
 use crate::param::Params;
 use crate::stock::StockMessage;
+use crate::wrapmime;
 
 mod idle;
 pub mod select_folder;
@@ -63,9 +64,6 @@ pub enum Error {
     #[fail(display = "IMAP select folder error")]
     SelectFolderError(#[cause] select_folder::Error),
 
-    #[fail(display = "No mailbox selected, folder: {:?}", _0)]
-    NoMailbox(String),
-
     #[fail(display = "IMAP other error: {:?}", _0)]
     Other(String),
 }
@@ -481,10 +479,7 @@ impl Imap {
             let (uid_validity, last_seen_uid) = self.get_config_last_seen_uid(context, &folder);
 
             let config = self.config.read().await;
-            let mailbox = config
-                .selected_mailbox
-                .as_ref()
-                .ok_or_else(|| Error::NoMailbox(folder.to_string()))?;
+            let mailbox = config.selected_mailbox.as_ref().expect("just selected");
 
             let new_uid_validity = match mailbox.uid_validity {
                 Some(v) => v,
@@ -1228,18 +1223,6 @@ fn precheck_imf(context: &Context, rfc724_mid: &str, server_folder: &str, server
     }
 }
 
-fn parse_message_id(message_id: &[u8]) -> crate::error::Result<String> {
-    let value = std::str::from_utf8(message_id)?;
-    let addrs = mailparse::addrparse(value)
-        .map_err(|err| format_err!("failed to parse message id {:?}", err))?;
-
-    if let Some(info) = addrs.extract_single_info() {
-        return Ok(info.addr);
-    }
-
-    bail!("could not parse message_id: {}", value);
-}
-
 fn prefetch_get_message_id(prefetch_msg: &Fetch) -> Result<String> {
     if prefetch_msg.envelope().is_none() {
         return Err(Error::Other(
@@ -1252,22 +1235,5 @@ fn prefetch_get_message_id(prefetch_msg: &Fetch) -> Result<String> {
         return Err(Error::Other("prefetch: No message ID found".to_string()));
     }
 
-    parse_message_id(&message_id.unwrap()).map_err(Into::into)
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_parse_message_id() {
-        assert_eq!(
-            parse_message_id(b"Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org").unwrap(),
-            "Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org"
-        );
-        assert_eq!(
-            parse_message_id(b"<Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org>").unwrap(),
-            "Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org"
-        );
-    }
+    wrapmime::parse_message_id(&message_id.unwrap()).map_err(Into::into)
 }

src/imap_client.rs

@@ -46,7 +46,7 @@ impl Client {
         let _greeting = client
             .read_response()
             .await
-            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
+            .expect("failed to read greeting");
 
         Ok(Client::Secure(client))
     }
@@ -61,7 +61,7 @@ impl Client {
         let _greeting = client
             .read_response()
             .await
-            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
+            .expect("failed to read greeting");
 
         Ok(Client::Insecure(client))
     }

src/imex.rs

@@ -380,7 +380,7 @@ pub fn JobImexImap(context: &Context, job: &Job) -> Result<()> {
             context.free_ongoing();
             bail!("Cannot create private key or private key not available.");
         } else {
-            dc_create_folder(context, &param)?;
+            dc_create_folder(context, &param);
         }
     }
     let path = Path::new(param);

src/job.rs

@@ -201,17 +201,9 @@ impl Job {
                         println!("{}", String::from_utf8_lossy(&body));
                     }
                     match task::block_on(smtp.send(context, recipients_list, body, self.job_id)) {
-                        Err(crate::smtp::send::Error::SendTimeout(err)) => {
-                            warn!(context, "SMTP send timed out {:?}", err);
-                            smtp.disconnect();
-                            self.try_again_later(
-                                TryAgain::AtOnce,
-                                Some("send-timeout".to_string()),
-                            );
-                        }
                         Err(crate::smtp::send::Error::SendError(err)) => {
                             // Remote error, retry later.
-                            warn!(context, "SMTP failed to send: {}", err);
+                            info!(context, "SMTP failed to send: {}", err);
                             smtp.disconnect();
                             self.try_again_later(TryAgain::AtOnce, Some(err.to_string()));
                         }
@@ -480,19 +472,19 @@ pub fn perform_sentbox_idle(context: &Context) {
         .idle(context, use_network);
 }
 
-pub fn interrupt_inbox_idle(context: &Context) {
-    info!(context, "interrupt_inbox_idle called");
-    // we do not block on trying to obtain the thread lock
-    // because we don't know in which state the thread is.
-    // If it's currently fetching then we can not get the lock
-    // but we flag it for checking jobs so that idle will be skipped.
-    match context.inbox_thread.try_read() {
-        Ok(inbox_thread) => {
-            inbox_thread.interrupt_idle(context);
-        }
-        Err(err) => {
-            *context.perform_inbox_jobs_needed.write().unwrap() = true;
-            warn!(context, "could not interrupt idle: {}", err);
+pub fn interrupt_inbox_idle(context: &Context, block: bool) {
+    info!(context, "interrupt_inbox_idle called blocking={}", block);
+    if block {
+        context.inbox_thread.read().unwrap().interrupt_idle(context);
+    } else {
+        match context.inbox_thread.try_read() {
+            Ok(inbox_thread) => {
+                inbox_thread.interrupt_idle(context);
+            }
+            Err(err) => {
+                *context.perform_inbox_jobs_needed.write().unwrap() = true;
+                warn!(context, "could not interrupt idle: {}", err);
+            }
         }
     }
 }
@@ -604,7 +596,7 @@ pub fn maybe_network(context: &Context) {
     }
 
     interrupt_smtp_idle(context);
-    interrupt_inbox_idle(context);
+    interrupt_inbox_idle(context, true);
     interrupt_mvbox_idle(context);
     interrupt_sentbox_idle(context);
 }
@@ -639,7 +631,7 @@ pub fn job_send_msg(context: &Context, msg_id: MsgId) -> Result<(), Error> {
     msg.try_calc_and_set_dimensions(context).ok();
 
     /* create message */
-    let needs_encryption = msg.param.get_bool(Param::GuaranteeE2ee).unwrap_or_default();
+    let needs_encryption = msg.param.get_int(Param::GuaranteeE2ee).unwrap_or_default();
 
     let attach_selfavatar = match chat::shall_attach_selfavatar(context, msg.chat_id) {
         Ok(attach_selfavatar) => attach_selfavatar,
@@ -655,7 +647,7 @@ pub fn job_send_msg(context: &Context, msg_id: MsgId) -> Result<(), Error> {
         err
     })?;
 
-    if needs_encryption && !rendered_msg.is_encrypted {
+    if 0 != needs_encryption && !rendered_msg.is_encrypted {
         /* unrecoverable */
         message::set_msg_failed(
             context,
@@ -708,7 +700,7 @@ pub fn job_send_msg(context: &Context, msg_id: MsgId) -> Result<(), Error> {
         }
     }
 
-    if rendered_msg.is_encrypted && !needs_encryption {
+    if rendered_msg.is_encrypted && needs_encryption == 0 {
         msg.param.set_int(Param::GuaranteeE2ee, 1);
         msg.save_param_to_disk(context);
     }
@@ -972,7 +964,7 @@ pub fn job_add(
     ).ok();
 
     match thread {
-        Thread::Imap => interrupt_inbox_idle(context),
+        Thread::Imap => interrupt_inbox_idle(context, false),
         Thread::Smtp => interrupt_smtp_idle(context),
         Thread::Unknown => {}
     }

src/lib.rs

@@ -1,7 +1,7 @@
 #![deny(clippy::correctness, missing_debug_implementations, clippy::all)]
 // for now we hide warnings to not clutter/hide errors during "cargo clippy"
 #![allow(clippy::cognitive_complexity, clippy::too_many_arguments)]
-#![allow(clippy::match_bool)]
+#![allow(clippy::unreadable_literal, clippy::match_bool)]
 #![feature(ptr_wrapping_offset_from)]
 #![feature(drain_filter)]
 
@@ -20,7 +20,7 @@ extern crate strum_macros;
 extern crate debug_stub_derive;
 
 #[macro_use]
-pub mod log;
+mod log;
 #[macro_use]
 pub mod error;
 
@@ -63,6 +63,7 @@ pub mod sql;
 pub mod stock;
 mod token;
 #[macro_use]
+mod wrapmime;
 mod dehtml;
 
 pub mod dc_receive_imf;

src/log.rs

@@ -7,13 +7,7 @@ macro_rules! info {
     };
     ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
         let formatted = format!($msg, $($args),*);
-        let thread = ::std::thread::current();
-        let full = format!("{thid:?} {file}:{line}: {msg}",
-                           thid = thread.id(),
-                           file = file!(),
-                           line = line!(),
-                           msg = &formatted);
-        emit_event!($ctx, $crate::Event::Info(full));
+        emit_event!($ctx, $crate::Event::Info(formatted));
     }};
 }
 
@@ -24,13 +18,7 @@ macro_rules! warn {
     };
     ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
         let formatted = format!($msg, $($args),*);
-        let thread = ::std::thread::current();
-        let full = format!("{thid:?} {file}:{line}: {msg}",
-                           thid = thread.id(),
-                           file = file!(),
-                           line = line!(),
-                           msg = &formatted);
-        emit_event!($ctx, $crate::Event::Warning(full));
+        emit_event!($ctx, $crate::Event::Warning(formatted));
     }};
 }
 

src/message.rs

@@ -720,7 +720,7 @@ pub fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
         return ret;
     }
     let rawtxt = rawtxt.unwrap_or_default();
-    let rawtxt = dc_truncate(rawtxt.trim(), 100_000, false);
+    let rawtxt = dc_truncate(rawtxt.trim(), 100000, false);
 
     let fts = dc_timestamp_to_str(msg.get_timestamp());
     ret += &format!("Sent: {}", fts);

src/mimeparser.rs

@@ -2,7 +2,7 @@ use std::collections::{HashMap, HashSet};
 
 use deltachat_derive::{FromSql, ToSql};
 use lettre_email::mime::{self, Mime};
-use mailparse::{DispositionType, MailAddr, MailHeaderMap};
+use mailparse::{DispositionType, MailHeaderMap};
 
 use crate::aheader::Aheader;
 use crate::blob::BlobObject;
@@ -14,11 +14,11 @@ use crate::dc_simplify::*;
 use crate::dc_tools::*;
 use crate::e2ee;
 use crate::error::Result;
-use crate::events::Event;
 use crate::headerdef::HeaderDef;
 use crate::job::{job_add, Action};
 use crate::location;
 use crate::message;
+use crate::message::MsgId;
 use crate::param::*;
 use crate::peerstate::Peerstate;
 use crate::securejoin::handle_degrade_event;
@@ -40,6 +40,7 @@ pub struct MimeParser<'a> {
     pub user_avatar: AvatarAction,
     pub group_avatar: AvatarAction,
     reports: Vec<Report>,
+    mdns_enabled: bool,
     parsed_protected_headers: bool,
 }
 
@@ -85,6 +86,7 @@ const MIME_AC_SETUP_FILE: &str = "application/autocrypt-setup";
 impl<'a> MimeParser<'a> {
     pub fn from_bytes(context: &'a Context, body: &[u8]) -> Result<Self> {
         let mail = mailparse::parse_mail(body)?;
+        let mdns_enabled = context.get_config_bool(Config::MdnsEnabled);
 
         let mut parser = MimeParser {
             parts: Vec::new(),
@@ -102,6 +104,7 @@ impl<'a> MimeParser<'a> {
             message_kml: None,
             user_avatar: AvatarAction::None,
             group_avatar: AvatarAction::None,
+            mdns_enabled,
             parsed_protected_headers: false,
         };
 
@@ -208,6 +211,7 @@ impl<'a> MimeParser<'a> {
                         || filepart.typ == Viewtype::Voice
                         || filepart.typ == Viewtype::Video
                         || filepart.typ == Viewtype::File)
+                    && !filepart.is_meta
             };
 
             if need_drop {
@@ -224,7 +228,7 @@ impl<'a> MimeParser<'a> {
             }
         }
         if let Some(ref subject) = self.get_subject() {
-            let mut prepend_subject = true;
+            let mut prepend_subject = 1i32;
             if !self.decrypting_failed {
                 let colon = subject.find(':');
                 if colon == Some(2)
@@ -232,10 +236,10 @@ impl<'a> MimeParser<'a> {
                     || self.has_chat_version()
                     || subject.contains("Chat:")
                 {
-                    prepend_subject = false
+                    prepend_subject = 0i32
                 }
             }
-            if prepend_subject {
+            if 0 != prepend_subject {
                 let subj = if let Some(n) = subject.find('[') {
                     &subject[0..n]
                 } else {
@@ -287,27 +291,30 @@ impl<'a> MimeParser<'a> {
                 }
             }
         }
+        if !self.decrypting_failed {
+            if let Some(dn_field) = self.get(HeaderDef::ChatDispositionNotificationTo) {
+                if self.get_last_nonmeta().is_some() {
+                    let addrs = mailparse::addrparse(&dn_field).unwrap();
 
-        // See if an MDN is requested from the other side
-        if !self.decrypting_failed && !self.parts.is_empty() {
-            if let Some(ref dn_to_addr) =
-                self.parse_first_addr(HeaderDef::ChatDispositionNotificationTo)
-            {
-                if let Some(ref from_addr) = self.parse_first_addr(HeaderDef::From_) {
-                    if compare_addrs(from_addr, dn_to_addr) {
-                        if let Some(part) = self.parts.last_mut() {
-                            part.param.set_int(Param::WantsMdn, 1);
+                    if let Some(dn_to_addr) = addrs.first() {
+                        if let Some(from_field) = self.get(HeaderDef::From_) {
+                            let from_addrs = mailparse::addrparse(&from_field).unwrap();
+
+                            if let Some(from_addr) = from_addrs.first() {
+                                if compare_addrs(from_addr, dn_to_addr) {
+                                    if let Some(part_4) = self.get_last_nonmeta_mut() {
+                                        part_4.param.set_int(Param::WantsMdn, 1);
+                                    }
+                                }
+                            }
                         }
                     }
                 }
             }
         }
 
-        // If there were no parts, especially a non-DC mail user may
-        // just have send a message in the subject with an empty body.
-        // Besides, we want to show something in case our incoming-processing
-        // failed to properly handle an incoming message.
-        if self.parts.is_empty() && self.reports.is_empty() {
+        // Cleanup - and try to create at least an empty part if there are no parts yet
+        if self.get_last_nonmeta().is_none() && self.reports.is_empty() {
             let mut part = Part::default();
             part.typ = Viewtype::Text;
 
@@ -346,6 +353,14 @@ impl<'a> MimeParser<'a> {
         AvatarAction::None
     }
 
+    pub fn get_last_nonmeta(&self) -> Option<&Part> {
+        self.parts.iter().rev().find(|part| !part.is_meta)
+    }
+
+    pub fn get_last_nonmeta_mut(&mut self) -> Option<&mut Part> {
+        self.parts.iter_mut().rev().find(|part| !part.is_meta)
+    }
+
     pub fn was_encrypted(&self) -> bool {
         !self.signatures.is_empty()
     }
@@ -373,20 +388,6 @@ impl<'a> MimeParser<'a> {
         self.header.get(&headerdef.get_headername())
     }
 
-    fn parse_first_addr(&self, headerdef: HeaderDef) -> Option<MailAddr> {
-        if let Some(value) = self.get(headerdef.clone()) {
-            match mailparse::addrparse(&value) {
-                Ok(ref addrs) => {
-                    return addrs.first().cloned();
-                }
-                Err(err) => {
-                    warn!(self.context, "header {} parse error: {:?}", headerdef, err);
-                }
-            }
-        }
-        None
-    }
-
     fn parse_mime_recursive(&mut self, mail: &mailparse::ParsedMail<'_>) -> Result<bool> {
         if mail.ctype.params.get("protected-headers").is_some() {
             if mail.ctype.mimetype == "text/rfc822-headers" {
@@ -576,11 +577,12 @@ impl<'a> MimeParser<'a> {
                         }
                     };
 
-                    let (simplified_txt, is_forwarded) = if decoded_data.is_empty() {
-                        ("".into(), false)
+                    let mut simplifier = Simplify::new();
+                    let simplified_txt = if decoded_data.is_empty() {
+                        "".into()
                     } else {
                         let is_html = mime_type == mime::TEXT_HTML;
-                        simplify(&decoded_data, is_html, self.has_chat_version())
+                        simplifier.simplify(&decoded_data, is_html, self.has_chat_version())
                     };
 
                     if !simplified_txt.is_empty() {
@@ -592,7 +594,7 @@ impl<'a> MimeParser<'a> {
                         self.do_add_single_part(part);
                     }
 
-                    if is_forwarded {
+                    if simplifier.is_forwarded {
                         self.is_forwarded = true;
                     }
                 }
@@ -724,6 +726,11 @@ impl<'a> MimeParser<'a> {
     }
 
     fn process_report(&self, report: &mailparse::ParsedMail<'_>) -> Result<Option<Report>> {
+        // to get a clear functionality, do not show incoming MDNs if the options is disabled
+        if !self.mdns_enabled {
+            return Ok(None);
+        }
+
         // parse as mailheaders
         let report_body = report.subparts[1].get_body_raw()?;
         let (report_fields, _) = mailparse::parse_headers(&report_body)?;
@@ -742,46 +749,33 @@ impl<'a> MimeParser<'a> {
                 }));
             }
         }
-        warn!(
-            self.context,
-            "ignoring unknown disposition-notification, Message-Id: {:?}",
-            report_fields.get_first_value("Message-ID").ok()
-        );
 
         Ok(None)
     }
 
-    // Handle reports (only MDNs for now)
+    // Handle reports (mainly MDNs)
     pub fn handle_reports(
         &self,
         from_id: u32,
         sent_timestamp: i64,
+        rr_event_to_send: &mut Vec<(u32, MsgId)>,
         server_folder: impl AsRef<str>,
         server_uid: u32,
     ) {
-        if self.reports.is_empty() {
-            return;
-        }
-        // If a user disabled MDNs we do not show pending incoming ones anymore
-        // but we do want them to potentially get moved from the INBOX still.
-        let mdns_enabled = self.context.get_config_bool(Config::MdnsEnabled);
-
         for report in &self.reports {
-            let mut mdn_recognized = false;
+            let mut mdn_consumed = false;
 
-            if mdns_enabled {
-                if let Some((chat_id, msg_id)) = message::mdn_from_ext(
-                    self.context,
-                    from_id,
-                    &report.original_message_id,
-                    sent_timestamp,
-                ) {
-                    self.context.call_cb(Event::MsgRead { chat_id, msg_id });
-                    mdn_recognized = true;
-                }
+            if let Some((chat_id, msg_id)) = message::mdn_from_ext(
+                self.context,
+                from_id,
+                &report.original_message_id,
+                sent_timestamp,
+            ) {
+                rr_event_to_send.push((chat_id, msg_id));
+                mdn_consumed = true;
             }
 
-            if self.has_chat_version() || mdn_recognized || !mdns_enabled {
+            if self.has_chat_version() || mdn_consumed {
                 let mut param = Params::new();
                 param.set(Param::ServerFolder, server_folder.as_ref());
                 param.set_int(Param::ServerUid, server_uid as i32);
@@ -874,6 +868,7 @@ fn is_known(key: &str) -> bool {
 #[derive(Debug, Default, Clone)]
 pub struct Part {
     pub typ: Viewtype,
+    pub is_meta: bool,
     pub mimetype: Option<Mime>,
     pub msg: String,
     pub msg_raw: Option<String>,
@@ -1115,26 +1110,6 @@ mod tests {
         );
     }
 
-    #[test]
-    fn test_parse_first_addr() {
-        let context = dummy_context();
-        let raw = b"From: hello@one.org, world@two.org\n\
-                    Chat-Disposition-Notification-To: wrong
-                    Content-Type: text/plain;
-                    Chat-Version: 1.0\n\
-                    \n\
-                    test1\n\
-                    \x00";
-
-        let mimeparser = MimeParser::from_bytes(&context.ctx, &raw[..]).unwrap();
-
-        let of = mimeparser.parse_first_addr(HeaderDef::From_).unwrap();
-        assert_eq!(of, mailparse::addrparse("hello@one.org").unwrap()[0]);
-
-        let of = mimeparser.parse_first_addr(HeaderDef::ChatDispositionNotificationTo);
-        assert!(of.is_none());
-    }
-
     #[test]
     fn test_mimeparser_with_context() {
         let context = dummy_context();

src/smtp/mod.rs

@@ -2,20 +2,17 @@
 
 pub mod send;
 
-use std::time::Duration;
-
 use async_smtp::smtp::client::net::*;
 use async_smtp::*;
 
+use async_std::task;
+
 use crate::constants::*;
 use crate::context::Context;
 use crate::events::Event;
 use crate::login_param::{dc_build_tls, LoginParam};
 use crate::oauth2::*;
 
-/// SMTP write and read timeout in seconds.
-const SMTP_TIMEOUT: u64 = 30;
-
 #[derive(Debug, Fail)]
 pub enum Error {
     #[fail(display = "Bad parameters")]
@@ -24,12 +21,12 @@ pub enum Error {
     InvalidLoginAddress {
         address: String,
         #[cause]
-        error: error::Error,
+        error: async_smtp::error::Error,
     },
     #[fail(display = "SMTP failed to connect: {:?}", _0)]
-    ConnectionFailure(#[cause] smtp::error::Error),
+    ConnectionFailure(#[cause] async_smtp::smtp::error::Error),
     #[fail(display = "SMTP: failed to setup connection {:?}", _0)]
-    ConnectionSetupFailure(#[cause] smtp::error::Error),
+    ConnectionSetupFailure(#[cause] async_smtp::smtp::error::Error),
     #[fail(display = "SMTP: oauth2 error {:?}", _0)]
     Oauth2Error { address: String },
     #[fail(display = "TLS error")]
@@ -47,7 +44,7 @@ pub type Result<T> = std::result::Result<T, Error>;
 #[derive(Default, DebugStub)]
 pub struct Smtp {
     #[debug_stub(some = "SmtpTransport")]
-    transport: Option<smtp::SmtpTransport>,
+    transport: Option<async_smtp::smtp::SmtpTransport>,
     /// Email address we are sending from.
     from: Option<EmailAddress>,
 }
@@ -60,25 +57,18 @@ impl Smtp {
 
     /// Disconnect the SMTP transport and drop it entirely.
     pub fn disconnect(&mut self) {
-        if let Some(mut transport) = self.transport.take() {
-            async_std::task::block_on(transport.close()).ok();
+        if let Some(ref mut transport) = self.transport.take() {
+            transport.close();
         }
     }
 
-    /// Check whether we are connected.
+    /// check whether we are connected
     pub fn is_connected(&self) -> bool {
-        self.transport
-            .as_ref()
-            .map(|t| t.is_connected())
-            .unwrap_or_default()
+        self.transport.is_some()
     }
 
-    /// Connect using the provided login params.
+    /// Connect using the provided login params
     pub fn connect(&mut self, context: &Context, lp: &LoginParam) -> Result<()> {
-        async_std::task::block_on(self.inner_connect(context, lp))
-    }
-
-    async fn inner_connect(&mut self, context: &Context, lp: &LoginParam) -> Result<()> {
         if self.is_connected() {
             warn!(context, "SMTP already connected.");
             return Ok(());
@@ -114,21 +104,21 @@ impl Smtp {
             }
             let user = &lp.send_user;
             (
-                smtp::authentication::Credentials::new(
+                async_smtp::smtp::authentication::Credentials::new(
                     user.to_string(),
                     access_token.unwrap_or_default(),
                 ),
-                vec![smtp::authentication::Mechanism::Xoauth2],
+                vec![async_smtp::smtp::authentication::Mechanism::Xoauth2],
             )
         } else {
             // plain
             let user = lp.send_user.clone();
             let pw = lp.send_pw.clone();
             (
-                smtp::authentication::Credentials::new(user, pw),
+                async_smtp::smtp::authentication::Credentials::new(user, pw),
                 vec![
-                    smtp::authentication::Mechanism::Plain,
-                    smtp::authentication::Mechanism::Login,
+                    async_smtp::smtp::authentication::Mechanism::Plain,
+                    async_smtp::smtp::authentication::Mechanism::Login,
                 ],
             )
         };
@@ -136,31 +126,30 @@ impl Smtp {
         let security = if 0
             != lp.server_flags & (DC_LP_SMTP_SOCKET_STARTTLS | DC_LP_SMTP_SOCKET_PLAIN) as i32
         {
-            smtp::ClientSecurity::Opportunistic(tls_parameters)
+            async_smtp::smtp::ClientSecurity::Opportunistic(tls_parameters)
         } else {
-            smtp::ClientSecurity::Wrapper(tls_parameters)
+            async_smtp::smtp::ClientSecurity::Wrapper(tls_parameters)
         };
 
-        let client = smtp::SmtpClient::with_security((domain.as_str(), port), security)
-            .await
-            .map_err(Error::ConnectionSetupFailure)?;
+        let client = task::block_on(async_smtp::smtp::SmtpClient::with_security(
+            (domain.as_str(), port),
+            security,
+        ))
+        .map_err(Error::ConnectionSetupFailure)?;
 
         let client = client
             .smtp_utf8(true)
             .credentials(creds)
             .authentication_mechanism(mechanism)
-            .connection_reuse(smtp::ConnectionReuseParameters::ReuseUnlimited)
-            .timeout(Some(Duration::from_secs(SMTP_TIMEOUT)));
-
+            .connection_reuse(async_smtp::smtp::ConnectionReuseParameters::ReuseUnlimited);
         let mut trans = client.into_transport();
-        trans.connect().await.map_err(Error::ConnectionFailure)?;
+        task::block_on(trans.connect()).map_err(Error::ConnectionFailure)?;
 
         self.transport = Some(trans);
         context.call_cb(Event::SmtpConnected(format!(
             "SMTP-LOGIN as {} ok",
             lp.send_user,
         )));
-
         Ok(())
     }
 }

src/smtp/send.rs

@@ -16,14 +16,6 @@ pub enum Error {
     SendError(#[cause] async_smtp::smtp::error::Error),
     #[fail(display = "SMTP has no transport")]
     NoTransport,
-    #[fail(display = "SMTP send timed out")]
-    SendTimeout(#[cause] async_std::future::TimeoutError),
-}
-
-impl From<async_std::future::TimeoutError> for Error {
-    fn from(err: async_std::future::TimeoutError) -> Error {
-        Error::SendTimeout(err)
-    }
 }
 
 impl Smtp {
@@ -51,7 +43,6 @@ impl Smtp {
             format!("{}", job_id), // only used for internal logging
             message,
         );
-
         if let Some(ref mut transport) = self.transport {
             transport.send(mail).await.map_err(Error::SendError)?;
 
@@ -59,6 +50,7 @@ impl Smtp {
                 "Message len={} was smtp-sent to {}",
                 message_len, recipients_display
             )));
+
             Ok(())
         } else {
             warn!(

src/sql.rs

@@ -544,7 +544,7 @@ fn open(
         // --------------------------------------------------------------------
 
         let mut dbversion = dbversion_before_update;
-        let mut recalc_fingerprints = false;
+        let mut recalc_fingerprints = 0;
         let mut update_icons = false;
 
         if dbversion < 1 {
@@ -687,7 +687,7 @@ fn open(
                 "CREATE INDEX acpeerstates_index4 ON acpeerstates (gossip_key_fingerprint);",
                 params![],
             )?;
-            recalc_fingerprints = true;
+            recalc_fingerprints = 1;
             dbversion = 34;
             sql.set_raw_config_int(context, "dbversion", 34)?;
         }
@@ -872,7 +872,7 @@ fn open(
         // (the structure is complete now and all objects are usable)
         // --------------------------------------------------------------------
 
-        if recalc_fingerprints {
+        if 0 != recalc_fingerprints {
             info!(context, "[migration] recalc fingerprints");
             sql.query_map(
                 "SELECT addr FROM acpeerstates;",

src/wrapmime.rs

@@ -0,0 +1,59 @@
+use mailparse::ParsedMail;
+
+use crate::error::Error;
+
+pub fn parse_message_id(message_id: &[u8]) -> Result<String, Error> {
+    let value = std::str::from_utf8(message_id)?;
+    let addrs = mailparse::addrparse(value)
+        .map_err(|err| format_err!("failed to parse message id {:?}", err))?;
+
+    if let Some(info) = addrs.extract_single_info() {
+        return Ok(info.addr);
+    }
+
+    bail!("could not parse message_id: {}", value);
+}
+
+/// Returns a reference to the encrypted payload and validates the autocrypt structure.
+pub fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Result<&'a ParsedMail<'b>, Error> {
+    ensure!(
+        mail.ctype.mimetype == "multipart/encrypted",
+        "Not a multipart/encrypted message: {}",
+        mail.ctype.mimetype
+    );
+    ensure!(
+        mail.subparts.len() == 2,
+        "Invalid Autocrypt Level 1 Mime Parts"
+    );
+
+    ensure!(
+        mail.subparts[0].ctype.mimetype == "application/pgp-encrypted",
+        "Invalid Autocrypt Level 1 version part: {:?}",
+        mail.subparts[0].ctype,
+    );
+
+    ensure!(
+        mail.subparts[1].ctype.mimetype == "application/octet-stream",
+        "Invalid Autocrypt Level 1 encrypted part: {:?}",
+        mail.subparts[1].ctype
+    );
+
+    Ok(&mail.subparts[1])
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_message_id() {
+        assert_eq!(
+            parse_message_id(b"Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org").unwrap(),
+            "Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org"
+        );
+        assert_eq!(
+            parse_message_id(b"<Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org>").unwrap(),
+            "Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org"
+        );
+    }
+}