use encoded-words crate, which friedel ported from python

Cargo.lock

@@ -620,6 +620,7 @@ dependencies = [
  "chrono 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "debug_stub_derive 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "deltachat_derive 0.1.0",
+ "encoded-words 0.1.0 (git+https://github.com/async-email/encoded-words)",
  "escaper 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -796,6 +797,20 @@ dependencies = [
  "version_check 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "encoded-words"
+version = "0.1.0"
+source = "git+https://github.com/async-email/encoded-words#019e833f0c9ea7d4b0b693aab44e66d78d18f1d0"
+dependencies = [
+ "base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "charset 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "encoding_rs 0.8.20 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "thiserror 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "encoding"
 version = "0.2.33"
@@ -2760,6 +2775,24 @@ dependencies = [
  "wincolor 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "thiserror"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "thiserror-impl 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "thread-local-object"
 version = "0.1.0"
@@ -3428,6 +3461,7 @@ dependencies = [
 "checksum ed25519-dalek 1.0.0-pre.2 (registry+https://github.com/rust-lang/crates.io-index)" = "845aaacc16f01178f33349e7c992ecd0cee095aa5e577f0f4dee35971bd36455"
 "checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3"
 "checksum email 0.0.21 (git+https://github.com/deltachat/rust-email)" = "<none>"
+"checksum encoded-words 0.1.0 (git+https://github.com/async-email/encoded-words)" = "<none>"
 "checksum encoding 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "6b0d943856b990d12d3b55b359144ff341533e516d94098b1d3fc1ac666d36ec"
 "checksum encoding-index-japanese 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)" = "04e8b2ff42e9a05335dbf8b5c6f7567e5591d0d916ccef4e0b1710d32a0d0c91"
 "checksum encoding-index-korean 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)" = "4dc33fb8e6bcba213fe2f14275f0963fd16f0a02c878e3095ecfdf5bee529d81"
@@ -3644,6 +3678,8 @@ dependencies = [
 "checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9"
 "checksum termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "adc4587ead41bf016f11af03e55a624c06568b5a19db4e90fde573d805074f83"
 "checksum termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "96d6098003bde162e4277c70665bd87c326f5a0c3f3fbfb285787fa482d54e6e"
+"checksum thiserror 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)" = "6f357d1814b33bc2dc221243f8424104bfe72dbe911d5b71b3816a2dff1c977e"
+"checksum thiserror-impl 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)" = "eb2e25d25307eb8436894f727aba8f65d07adf02e5b35a13cebed48bd282bfef"
 "checksum thread-local-object 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7da3caa820d0308c84c8654f6cafd81cc3195d45433311cbe22fcf44fc8be071"
 "checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b"
 "checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f"

Cargo.toml

@@ -54,6 +54,7 @@ rustls = "0.16.0"
 webpki-roots = "0.18.0"
 webpki = "0.21.0"
 mailparse = "0.10.1"
+encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
 
 [dev-dependencies]
 tempfile = "3.0"

README.md

@@ -87,6 +87,15 @@ $ cargo test --all
 $ cargo build -p deltachat_ffi --release
 ```
 
+## Debugging environment variables 
+
+- `DCC_IMAP_DEBUG`: if set IMAP protocol commands and responses will be
+  printed
+
+- `DCC_MIME_DEBUG`: if set outgoing and incoming message will be printed 
+
+
+
 ### Expensive tests
 
 Some tests are expensive and marked with `#[ignore]`, to run these

python/tests/test_account.py

@@ -430,29 +430,31 @@ class TestOnlineAccount:
         assert self_addr not in ev[2]
         ev = ac1._evlogger.get_matching("DC_EVENT_DELETED_BLOB_FILE")
 
-    def test_prepare_file(self, acfactory, lp):
+    def test_prepare_file_with_unicode(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
         chat = self.get_chat(ac1, ac2)
 
         lp.sec("ac1: prepare and send attachment + text to ac2")
         blobdir = ac1.get_blobdir()
-        p = os.path.join(blobdir, "somedata.txt")
+        basename = "somedäüta.txt"
+        p = os.path.join(blobdir, basename)
         with open(p, "w") as f:
             f.write("some data")
         msg = Message.new_empty(ac1, "file")
-        msg.set_text("hello world")
+        msg.set_text("hello ä world")
         msg.set_file(p)
         message = chat.prepare_message(msg)
         assert message.is_out_preparing()
-        assert message.text == "hello world"
+        assert message.text == "hello ä world"
         chat.send_prepared(message)
 
         lp.sec("ac2: receive message")
         ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
         assert ev[2] > const.DC_CHAT_ID_LAST_SPECIAL
         msg = ac2.get_message_by_id(ev[1])
-        assert msg.text == "hello world"
+        assert msg.text == "hello ä world"
         assert open(msg.filename).read() == "some data"
+        assert msg.filename.endswith(basename)
 
     def test_mvbox_sentbox_threads(self, acfactory, lp):
         lp.sec("ac1: start with mvbox thread")
@@ -658,6 +660,32 @@ class TestOnlineAccount:
         ev = ac1._evlogger.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
         assert not msg.is_encrypted()
 
+    def test_send_first_message_as_long_unicode_with_cr(self, acfactory, lp):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        ac2.set_config("save_mime_headers", "1")
+
+        lp.sec("ac1: create chat with ac2")
+        chat = self.get_chat(ac1, ac2, both_created=True)
+
+        lp.sec("sending multi-line non-unicode message from ac1 to ac2")
+        text1 = "hello\nworld"
+        msg_out = chat.send_text(text1)
+        assert not msg_out.is_encrypted()
+
+        lp.sec("sending multi-line unicode text message from ac1 to ac2")
+        text2 = "äalis\nthis is ßßÄ"
+        msg_out = chat.send_text(text2)
+        assert not msg_out.is_encrypted()
+
+        lp.sec("wait for ac2 to receive multi-line non-unicode message")
+        msg_in = ac2.wait_next_incoming_message()
+        assert msg_in.text == text1
+
+        lp.sec("wait for ac2 to receive multi-line unicode message")
+        msg_in = ac2.wait_next_incoming_message()
+        assert msg_in.text == text2
+        assert ac1.get_config("addr") in msg_in.chat.get_name()
+
     def test_reply_encrypted(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 

src/dc_receive_imf.rs

@@ -47,6 +47,11 @@ pub fn dc_receive_imf(
         server_uid,
     );
 
+    if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
+        info!(context, "dc_receive_imf: incoming message mime-body:");
+        println!("{}", String::from_utf8_lossy(imf_raw));
+    }
+
     let mime_parser = MimeParser::from_bytes(context, imf_raw);
     let mut mime_parser = if let Err(err) = mime_parser {
         warn!(context, "dc_receive_imf parse error: {}", err);

src/dc_strencode.rs

@@ -1,68 +0,0 @@
-use itertools::Itertools;
-
-/// Encode non-ascii-strings as `=?UTF-8?Q?Bj=c3=b6rn_Petersen?=`.
-/// Belongs to RFC 2047: https://tools.ietf.org/html/rfc2047
-///
-/// We do not fold at position 72; this would result in empty words as `=?utf-8?Q??=` which are correct,
-/// but cannot be displayed by some mail programs (eg. Android Stock Mail).
-/// however, this is not needed, as long as _one_ word is not longer than 72 characters.
-/// _if_ it is, the display may get weird.  This affects the subject only.
-/// the best solution wor all this would be if libetpan encodes the line as only libetpan knowns when a header line is full.
-///
-/// @param to_encode Null-terminated UTF-8-string to encode.
-/// @return Returns the encoded string which must be free()'d when no longed needed.
-///     On errors, NULL is returned.
-pub fn dc_encode_header_words(input: impl AsRef<str>) -> String {
-    let mut result = String::default();
-    for (_, group) in &input.as_ref().chars().group_by(|c| c.is_whitespace()) {
-        let word: String = group.collect();
-        result.push_str(&quote_word(&word.as_bytes()));
-    }
-
-    result
-}
-
-fn must_encode(byte: u8) -> bool {
-    static SPECIALS: &[u8] = b",:!\"#$@[\\]^`{|}~=?_";
-
-    SPECIALS.iter().any(|b| *b == byte)
-}
-
-fn quote_word(word: &[u8]) -> String {
-    let mut result = String::default();
-    let mut encoded = false;
-
-    for byte in word {
-        let byte = *byte;
-        if byte >= 128 || must_encode(byte) {
-            result.push_str(&format!("={:2X}", byte));
-            encoded = true;
-        } else if byte == b' ' {
-            result.push('_');
-            encoded = true;
-        } else {
-            result.push(byte as _);
-        }
-    }
-
-    if encoded {
-        result = format!("=?utf-8?Q?{}?=", &result);
-    }
-    result
-}
-
-/* ******************************************************************************
- * Encode/decode header words, RFC 2047
- ******************************************************************************/
-
-pub fn dc_needs_ext_header(to_check: impl AsRef<str>) -> bool {
-    let to_check = to_check.as_ref();
-
-    if to_check.is_empty() {
-        return false;
-    }
-
-    to_check.chars().any(|c| {
-        !c.is_ascii_alphanumeric() && c != '-' && c != '_' && c != '.' && c != '~' && c != '%'
-    })
-}

src/imap_client.rs

@@ -11,8 +11,6 @@ use async_tls::client::TlsStream;
 
 use crate::login_param::{dc_build_tls_config, CertificateChecks};
 
-const DCC_IMAP_DEBUG: &str = "DCC_IMAP_DEBUG";
-
 #[derive(Debug)]
 pub(crate) enum Client {
     Secure(ImapClient<TlsStream<TcpStream>>),
@@ -42,7 +40,7 @@ impl Client {
         let tls_connector: async_tls::TlsConnector = Arc::new(tls_config).into();
         let tls_stream = tls_connector.connect(domain.as_ref(), stream)?.await?;
         let mut client = ImapClient::new(tls_stream);
-        if std::env::var(DCC_IMAP_DEBUG).is_ok() {
+        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
             client.debug = true;
         }
 
@@ -58,7 +56,7 @@ impl Client {
         let stream = TcpStream::connect(addr).await?;
 
         let mut client = ImapClient::new(stream);
-        if std::env::var(DCC_IMAP_DEBUG).is_ok() {
+        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
             client.debug = true;
         }
         let _greeting = client

src/job.rs

@@ -192,6 +192,10 @@ impl Job {
                     // was sent we need to mark it in the database ASAP as we
                     // otherwise might send it twice.
                     let mut smtp = context.smtp.lock().unwrap();
+                    if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
+                        info!(context, "smtp-sending out mime message:");
+                        println!("{}", String::from_utf8_lossy(&body));
+                    }
                     match smtp.send(context, recipients_list, body, self.job_id) {
                         Err(crate::smtp::send::Error::SendError(err)) => {
                             // Remote error, retry later.

src/lib.rs

@@ -1,5 +1,6 @@
 #![deny(clippy::correctness, missing_debug_implementations, clippy::all)]
-#![warn(
+// for now we hide warnings to not clutter/hide errors during "cargo clippy"
+#![allow(
     clippy::type_complexity,
     clippy::cognitive_complexity,
     clippy::too_many_arguments,
@@ -75,8 +76,13 @@ mod dehtml;
 pub mod dc_array;
 pub mod dc_receive_imf;
 mod dc_simplify;
-mod dc_strencode;
 pub mod dc_tools;
 
+/// if set imap/incoming and smtp/outgoing MIME messages will be printed
+pub const DCC_MIME_DEBUG: &str = "DCC_MIME_DEBUG";
+
+/// if set IMAP protocol commands and responses will be printed
+pub const DCC_IMAP_DEBUG: &str = "DCC_IMAP_DEBUG";
+
 #[cfg(test)]
 mod test_utils;

src/mimefactory.rs

@@ -6,7 +6,6 @@ use crate::config::Config;
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::{get_version_str, Context};
-use crate::dc_strencode::*;
 use crate::dc_tools::*;
 use crate::e2ee::*;
 use crate::error::Error;
@@ -333,13 +332,19 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             Loaded::Message => {
                 match self.chat {
                     Some(ref chat) => {
-                        let raw_subject = message::get_summarytext_by_raw(
+                        let raw = message::get_summarytext_by_raw(
                             self.msg.type_0,
                             self.msg.text.as_ref(),
                             &self.msg.param,
                             32,
                             self.context,
                         );
+                        let mut lines = raw.lines();
+                        let raw_subject = if let Some(line) = lines.next() {
+                            line
+                        } else {
+                            ""
+                        };
 
                         let afwd_email = self.msg.param.exists(Param::Forwarded);
                         let fwd = if afwd_email { "Fwd: " } else { "" };
@@ -377,7 +382,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let mut unprotected_headers: Vec<Header> = Vec::new();
 
         let from = Address::new_mailbox_with_name(
-            dc_encode_header_words(&self.from_displayname),
+            encode_words(&self.from_displayname),
             self.from_addr.clone(),
         );
 
@@ -389,7 +394,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                 to.push(Address::new_mailbox(addr.clone()));
             } else {
                 to.push(Address::new_mailbox_with_name(
-                    dc_encode_header_words(name),
+                    encode_words(name),
                     addr.clone(),
                 ));
             }
@@ -445,7 +450,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let e2ee_guranteed = self.is_e2ee_guranteed();
         let mut encrypt_helper = EncryptHelper::new(self.context)?;
 
-        let subject = dc_encode_header_words(subject_str);
+        let subject = encode_words(&subject_str);
 
         let mut message = match self.loaded {
             Loaded::Message => {
@@ -605,7 +610,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         if chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup {
             protected_headers.push(Header::new("Chat-Group-ID".into(), chat.grpid.clone()));
 
-            let encoded = dc_encode_header_words(&chat.name);
+            let encoded = encode_words(&chat.name);
             protected_headers.push(Header::new("Chat-Group-Name".into(), encoded));
 
             match command {
@@ -975,20 +980,18 @@ fn build_body_file(
         }
     };
 
-    let needs_ext = dc_needs_ext_header(&filename_to_send);
-
     // create mime part, for Content-Disposition, see RFC 2183.
     // `Content-Disposition: attachment` seems not to make a difference to `Content-Disposition: inline`
     // at least on tested Thunderbird and Gma'l in 2017.
     // But I've heard about problems with inline and outl'k, so we just use the attachment-type until we
     // run into other problems ...
-    let cd_value = if needs_ext {
-        format!("attachment; filename=\"{}\"", &filename_to_send)
-    } else {
+    let cd_value = if needs_encoding(&filename_to_send) {
         format!(
             "attachment; filename*=\"{}\"",
-            dc_encode_header_words(&filename_to_send)
+            encode_words(&filename_to_send)
         )
+    } else {
+        format!("attachment; filename=\"{}\"", &filename_to_send)
     };
 
     let body = std::fs::read(blob.to_abs_path())?;
@@ -1022,3 +1025,23 @@ fn is_file_size_okay(context: &Context, msg: &Message) -> bool {
         None => false,
     }
 }
+
+/* ******************************************************************************
+ * Encode/decode header words, RFC 2047
+ ******************************************************************************/
+
+fn encode_words(word: &str) -> String {
+    encoded_words::encode(word, None, encoded_words::EncodingFlag::Shortest, None)
+}
+
+pub fn needs_encoding(to_check: impl AsRef<str>) -> bool {
+    let to_check = to_check.as_ref();
+
+    if to_check.is_empty() {
+        return false;
+    }
+
+    to_check.chars().any(|c| {
+        !c.is_ascii_alphanumeric() && c != '-' && c != '_' && c != '.' && c != '~' && c != '%'
+    })
+}