use rfc2047 crate from @valodim and remove dc_strencode.rs completely

Cargo.lock

@@ -644,6 +644,7 @@ dependencies = [
  "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "reqwest 0.9.22 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rfc2047 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "rusqlite 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustls 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustyline 4.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2286,6 +2287,11 @@ dependencies = [
  "winreg 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "rfc2047"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "ring"
 version = "0.16.9"
@@ -3591,6 +3597,7 @@ dependencies = [
 "checksum rental 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "8545debe98b2b139fb04cad8618b530e9b07c152d99a5de83c860b877d67847f"
 "checksum rental-impl 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "475e68978dc5b743f2f40d8e0a8fdc83f1c5e78cbf4b8fa5e74e73beebc340de"
 "checksum reqwest 0.9.22 (registry+https://github.com/rust-lang/crates.io-index)" = "2c2064233e442ce85c77231ebd67d9eca395207dec2127fe0bbedde4bd29a650"
+"checksum rfc2047 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6986b5de4fa30c92b50020fdedf45bf9296d9842df4355b62f92dcc18bcdee15"
 "checksum ring 0.16.9 (registry+https://github.com/rust-lang/crates.io-index)" = "6747f8da1f2b1fabbee1aaa4eb8a11abf9adef0bf58a41cee45db5d59cecdfac"
 "checksum ripemd160 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ad5112e0dbbb87577bfbc56c42450235e3012ce336e29c5befd7807bd626da4a"
 "checksum rsa 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5108a8bbfb84fe77d829d77d5a89255dcd189dfe5c4de5a33d0a47f12808bb15"

Cargo.toml

@@ -19,6 +19,7 @@ num-derive = "0.2.5"
 num-traits = "0.2.6"
 lettre = { git = "https://github.com/deltachat/lettre", branch = "feat/mail" }
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "feat/mail" }
+rfc2047 = "0.1.1"
 async-imap = { git = "https://github.com/async-email/async-imap", branch="master" }
 async-tls = "0.6"
 async-std = { version = "1.0", features = ["unstable"] }

python/tests/test_account.py

@@ -430,29 +430,31 @@ class TestOnlineAccount:
         assert self_addr not in ev[2]
         ev = ac1._evlogger.get_matching("DC_EVENT_DELETED_BLOB_FILE")
 
-    def test_prepare_file(self, acfactory, lp):
+    def test_prepare_file_with_unicode(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
         chat = self.get_chat(ac1, ac2)
 
         lp.sec("ac1: prepare and send attachment + text to ac2")
         blobdir = ac1.get_blobdir()
-        p = os.path.join(blobdir, "somedata.txt")
+        basename = "somedata.txt"  # XXX try unicode
+        p = os.path.join(blobdir, basename)
         with open(p, "w") as f:
             f.write("some data")
         msg = Message.new_empty(ac1, "file")
-        msg.set_text("hello world")
+        msg.set_text("hello ä world")
         msg.set_file(p)
         message = chat.prepare_message(msg)
         assert message.is_out_preparing()
-        assert message.text == "hello world"
+        assert message.text == "hello ä world"
         chat.send_prepared(message)
 
         lp.sec("ac2: receive message")
         ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
         assert ev[2] > const.DC_CHAT_ID_LAST_SPECIAL
         msg = ac2.get_message_by_id(ev[1])
-        assert msg.text == "hello world"
+        assert msg.text == "hello ä world"
         assert open(msg.filename).read() == "some data"
+        assert msg.filename.endswith(basename)
 
     def test_mvbox_sentbox_threads(self, acfactory, lp):
         lp.sec("ac1: start with mvbox thread")

src/dc_strencode.rs

@@ -1,69 +0,0 @@
-use itertools::Itertools;
-
-/// Encode non-ascii-strings as `=?UTF-8?Q?Bj=c3=b6rn_Petersen?=`.
-/// Belongs to RFC 2047: https://tools.ietf.org/html/rfc2047
-///
-/// We do not fold at position 72; this would result in empty words as `=?utf-8?Q??=` which are correct,
-/// but cannot be displayed by some mail programs (eg. Android Stock Mail).
-/// however, this is not needed, as long as _one_ word is not longer than 72 characters.
-/// _if_ it is, the display may get weird.  This affects the subject only.
-/// the best solution wor all this would be if libetpan encodes the line as only libetpan knowns when a header line is full.
-///
-/// @param to_encode Null-terminated UTF-8-string to encode.
-/// @return Returns the encoded string which must be free()'d when no longed needed.
-///     On errors, NULL is returned.
-pub fn dc_encode_header_words(input: impl AsRef<str>) -> String {
-    let mut result = String::default();
-    for (_, group) in &input.as_ref().chars().group_by(|c| c.is_whitespace()) {
-        let word: String = group.collect();
-        result.push_str(&quote_word(&word.as_bytes()));
-    }
-
-    result
-}
-
-fn must_encode(byte: u8) -> bool {
-    // XXX do we need to put ":" in here?
-    static SPECIALS: &[u8] = b",!\"#$@[\\]^`{|}~=?_";
-
-    SPECIALS.iter().any(|b| *b == byte)
-}
-
-fn quote_word(word: &[u8]) -> String {
-    let mut result = String::default();
-    let mut encoded = false;
-
-    for byte in word {
-        let byte = *byte;
-        if byte >= 128 || must_encode(byte) {
-            result.push_str(&format!("={:2X}", byte));
-            encoded = true;
-        } else if byte == b' ' {
-            result.push('_');
-            encoded = true;
-        } else {
-            result.push(byte as _);
-        }
-    }
-
-    if encoded {
-        result = format!("=?utf-8?Q?{}?=", &result);
-    }
-    result
-}
-
-/* ******************************************************************************
- * Encode/decode header words, RFC 2047
- ******************************************************************************/
-
-pub fn dc_needs_ext_header(to_check: impl AsRef<str>) -> bool {
-    let to_check = to_check.as_ref();
-
-    if to_check.is_empty() {
-        return false;
-    }
-
-    to_check.chars().any(|c| {
-        !c.is_ascii_alphanumeric() && c != '-' && c != '_' && c != '.' && c != '~' && c != '%'
-    })
-}

src/lib.rs

@@ -1,5 +1,6 @@
 #![deny(clippy::correctness, missing_debug_implementations, clippy::all)]
-#![warn(
+// for now we hide warnings to not clutter/hide errors during "cargo check"
+#![allow(
     clippy::type_complexity,
     clippy::cognitive_complexity,
     clippy::too_many_arguments,
@@ -75,7 +76,6 @@ mod dehtml;
 pub mod dc_array;
 pub mod dc_receive_imf;
 mod dc_simplify;
-mod dc_strencode;
 pub mod dc_tools;
 
 /// if set imap/incoming and smtp/outgoing MIME messages will be printed

src/mimefactory.rs

@@ -1,12 +1,12 @@
 use chrono::TimeZone;
 use lettre_email::{mime, Address, Header, MimeMultipartType, PartBuilder};
+use rfc2047::rfc2047_encode;
 
 use crate::chat::{self, Chat};
 use crate::config::Config;
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::{get_version_str, Context};
-use crate::dc_strencode::*;
 use crate::dc_tools::*;
 use crate::e2ee::*;
 use crate::error::Error;
@@ -383,7 +383,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let mut unprotected_headers: Vec<Header> = Vec::new();
 
         let from = Address::new_mailbox_with_name(
-            dc_encode_header_words(&self.from_displayname),
+            rfc2047_encode(&self.from_displayname).into_owned(),
             self.from_addr.clone(),
         );
 
@@ -395,7 +395,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                 to.push(Address::new_mailbox(addr.clone()));
             } else {
                 to.push(Address::new_mailbox_with_name(
-                    dc_encode_header_words(name),
+                    rfc2047_encode(name).into_owned(),
                     addr.clone(),
                 ));
             }
@@ -451,7 +451,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let e2ee_guranteed = self.is_e2ee_guranteed();
         let mut encrypt_helper = EncryptHelper::new(self.context)?;
 
-        let subject = dc_encode_header_words(subject_str);
+        let subject = rfc2047_encode(&subject_str).into_owned();
 
         let mut message = match self.loaded {
             Loaded::Message => {
@@ -611,7 +611,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         if chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup {
             protected_headers.push(Header::new("Chat-Group-ID".into(), chat.grpid.clone()));
 
-            let encoded = dc_encode_header_words(&chat.name);
+            let encoded = rfc2047_encode(&chat.name).into_owned();
             protected_headers.push(Header::new("Chat-Group-Name".into(), encoded));
 
             match command {
@@ -991,9 +991,10 @@ fn build_body_file(
     let cd_value = if needs_ext {
         format!("attachment; filename=\"{}\"", &filename_to_send)
     } else {
+        // XXX do we need to encode filenames?
         format!(
             "attachment; filename*=\"{}\"",
-            dc_encode_header_words(&filename_to_send)
+            rfc2047_encode(&filename_to_send)
         )
     };
 
@@ -1028,3 +1029,19 @@ fn is_file_size_okay(context: &Context, msg: &Message) -> bool {
         None => false,
     }
 }
+
+/* ******************************************************************************
+ * Encode/decode header words, RFC 2047
+ ******************************************************************************/
+
+pub fn dc_needs_ext_header(to_check: impl AsRef<str>) -> bool {
+    let to_check = to_check.as_ref();
+
+    if to_check.is_empty() {
+        return false;
+    }
+
+    to_check.chars().any(|c| {
+        !c.is_ascii_alphanumeric() && c != '-' && c != '_' && c != '.' && c != '~' && c != '%'
+    })
+}