make setup_handle_if_needed async, call it ahead of select_folder and fetch_new_messages (renamed from fetch_from_single_folder), and be more eager triggering reconnect on error conditions

fix doc

CHANGELOG.md

@@ -1,5 +1,21 @@
 # Changelog 
 
+## 1.0.0-beta.10 (pending)
+
+- fix grpid-determination from in-reply-to and references headers. @hpk42
+
+- only send Autocrypt-gossip headers on encrypted messages. @dignifiedquire
+
+- fix reply-to-encrypted message to also be encrypted. @hpk42
+
+- remove last unsafe code from dc_receive_imf :) @hpk42
+
+- add experimental new dc_chat_get_info_json FFI/API so that desktop devs
+  can play with using it. @jikstra
+
+- fix encoding of subjects and attachment-filenames @hpk42
+  @dignifiedquire . 
+
 ## 1.0.0-beta.9
 
 - historic: we now use the mailparse crate and lettre-email to generate mime

Cargo.lock

@@ -607,7 +607,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat"
-version = "1.0.0-beta.9"
+version = "1.0.0-beta.10"
 dependencies = [
  "async-imap 0.1.1 (git+https://github.com/async-email/async-imap)",
  "async-std 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -620,6 +620,7 @@ dependencies = [
  "chrono 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)",
  "debug_stub_derive 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "deltachat_derive 0.1.0",
+ "encoded-words 0.1.0 (git+https://github.com/async-email/encoded-words)",
  "escaper 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -682,9 +683,9 @@ dependencies = [
 
 [[package]]
 name = "deltachat_ffi"
-version = "1.0.0-beta.9"
+version = "1.0.0-beta.10"
 dependencies = [
- "deltachat 1.0.0-beta.9",
+ "deltachat 1.0.0-beta.10",
  "deltachat-provider-database 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "human-panic 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -796,6 +797,20 @@ dependencies = [
  "version_check 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "encoded-words"
+version = "0.1.0"
+source = "git+https://github.com/async-email/encoded-words#019e833f0c9ea7d4b0b693aab44e66d78d18f1d0"
+dependencies = [
+ "base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "charset 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "encoding_rs 0.8.20 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "thiserror 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "encoding"
 version = "0.2.33"
@@ -2760,6 +2775,24 @@ dependencies = [
  "wincolor 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "thiserror"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "thiserror-impl 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "thread-local-object"
 version = "0.1.0"
@@ -3428,6 +3461,7 @@ dependencies = [
 "checksum ed25519-dalek 1.0.0-pre.2 (registry+https://github.com/rust-lang/crates.io-index)" = "845aaacc16f01178f33349e7c992ecd0cee095aa5e577f0f4dee35971bd36455"
 "checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3"
 "checksum email 0.0.21 (git+https://github.com/deltachat/rust-email)" = "<none>"
+"checksum encoded-words 0.1.0 (git+https://github.com/async-email/encoded-words)" = "<none>"
 "checksum encoding 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "6b0d943856b990d12d3b55b359144ff341533e516d94098b1d3fc1ac666d36ec"
 "checksum encoding-index-japanese 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)" = "04e8b2ff42e9a05335dbf8b5c6f7567e5591d0d916ccef4e0b1710d32a0d0c91"
 "checksum encoding-index-korean 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)" = "4dc33fb8e6bcba213fe2f14275f0963fd16f0a02c878e3095ecfdf5bee529d81"
@@ -3644,6 +3678,8 @@ dependencies = [
 "checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9"
 "checksum termcolor 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "adc4587ead41bf016f11af03e55a624c06568b5a19db4e90fde573d805074f83"
 "checksum termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "96d6098003bde162e4277c70665bd87c326f5a0c3f3fbfb285787fa482d54e6e"
+"checksum thiserror 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)" = "6f357d1814b33bc2dc221243f8424104bfe72dbe911d5b71b3816a2dff1c977e"
+"checksum thiserror-impl 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)" = "eb2e25d25307eb8436894f727aba8f65d07adf02e5b35a13cebed48bd282bfef"
 "checksum thread-local-object 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7da3caa820d0308c84c8654f6cafd81cc3195d45433311cbe22fcf44fc8be071"
 "checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b"
 "checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f"

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.0.0-beta.9"
+version = "1.0.0-beta.10"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL"
@@ -54,6 +54,7 @@ rustls = "0.16.0"
 webpki-roots = "0.18.0"
 webpki = "0.21.0"
 mailparse = "0.10.1"
+encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
 
 [dev-dependencies]
 tempfile = "3.0"

README.md

@@ -87,6 +87,15 @@ $ cargo test --all
 $ cargo build -p deltachat_ffi --release
 ```
 
+## Debugging environment variables 
+
+- `DCC_IMAP_DEBUG`: if set IMAP protocol commands and responses will be
+  printed
+
+- `DCC_MIME_DEBUG`: if set outgoing and incoming message will be printed 
+
+
+
 ### Expensive tests
 
 Some tests are expensive and marked with `#[ignore]`, to run these

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.0.0-beta.9"
+version = "1.0.0-beta.10"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/deltachat.h

@@ -2602,6 +2602,25 @@ dc_lot_t*        dc_chatlist_get_summary     (const dc_chatlist_t* chatlist, siz
 dc_context_t*    dc_chatlist_get_context     (dc_chatlist_t* chatlist);
 
 
+/**
+ * Get info summary for a chat, in json format. 
+ *
+ * The returned json string has the following key/values: 
+ *
+ * id: chat id 
+ * name: chat/group name 
+ * color: color of this chat 
+ * last-message-from: who sent the last message 
+ * last-message-text: message (truncated) 
+ * last-message-state: DC_STATE* constant
+ * last-message-date: 
+ * avatar-path: path-to-blobfile 
+ * is_verified: yes/no
+
+ * @return a utf8-encoded json string containing all requested info. Must be freed using dc_str_unref().  NULL is never returned.
+ */
+char*            dc_chat_get_info_json       (dc_context_t* context, size_t chat_id);
+
 /**
  * @class dc_chat_t
  *

deltachat-ffi/src/lib.rs

@@ -2377,6 +2377,27 @@ pub unsafe extern "C" fn dc_chat_is_sending_locations(chat: *mut dc_chat_t) -> l
     ffi_chat.chat.is_sending_locations() as libc::c_int
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_chat_get_info_json(
+    context: *mut dc_context_t,
+    chat_id: u32,
+) -> *mut libc::c_char {
+    if context.is_null() {
+        eprintln!("ignoring careless call to dc_chat_get_info_json()");
+        return "".strdup();
+    }
+    let ffi_context = &*context;
+    ffi_context
+        .with_inner(|ctx| match chat::get_info_json(ctx, chat_id) {
+            Ok(s) => s.strdup(),
+            Err(err) => {
+                error!(ctx, "get_info_json({}) returned: {}", chat_id, err);
+                return "".strdup();
+            }
+        })
+        .unwrap_or_else(|_| "".strdup())
+}
+
 // dc_msg_t
 
 /// FFI struct for [dc_msg_t]

python/src/deltachat/chat.py

@@ -2,6 +2,7 @@
 
 import mimetypes
 import calendar
+import json
 from datetime import datetime
 import os
 from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array
@@ -242,6 +243,12 @@ class Chat(object):
         """
         return lib.dc_marknoticed_chat(self._dc_context, self.id)
 
+    def get_summary(self):
+        """ return dictionary with summary information. """
+        dc_res = lib.dc_chat_get_info_json(self._dc_context, self.id)
+        s = from_dc_charpointer(dc_res)
+        return json.loads(s)
+
     # ------  group management API ------------------------------
 
     def add_contact(self, contact):
@@ -324,6 +331,18 @@ class Chat(object):
             return None
         return from_dc_charpointer(dc_res)
 
+    def get_color(self):
+        """return the color of the chat.
+        :returns: color as 0x00rrggbb
+        """
+        return lib.dc_chat_get_color(self._dc_chat)
+
+    def get_subtitle(self):
+        """return the subtitle of the chat
+        :returns: the subtitle
+        """
+        return from_dc_charpointer(lib.dc_chat_get_subtitle(self._dc_chat))
+
     # ------  location streaming API ------------------------------
 
     def is_sending_locations(self):
@@ -332,6 +351,12 @@ class Chat(object):
         """
         return lib.dc_is_sending_locations_to_chat(self._dc_context, self.id)
 
+    def is_archived(self):
+        """return True if this chat is archived.
+        :returns: True if archived.
+        """
+        return lib.dc_chat_get_archived(self._dc_chat)
+
     def enable_sending_locations(self, seconds):
         """enable sending locations for this chat.
 

python/src/deltachat/message.py

@@ -58,8 +58,6 @@ class Message(object):
 
     def set_text(self, text):
         """set text of this message. """
-        assert self.id > 0, "message not prepared"
-        assert self.is_out_preparing()
         lib.dc_msg_set_text(self._dc_msg, as_dc_charpointer(text))
 
     @props.with_doc

python/tests/test_account.py

@@ -155,6 +155,18 @@ class TestOfflineChat:
         chat.set_name("title2")
         assert chat.get_name() == "title2"
 
+        d = chat.get_summary()
+        print(d)
+        assert d["id"] == chat.id
+        assert d["type"] == chat.get_type()
+        assert d["name"] == chat.get_name()
+        assert d["archived"] == chat.is_archived()
+        # assert d["param"] == chat.param
+        assert d["color"] == chat.get_color()
+        assert d["profile_image"] == "" if chat.get_profile_image() is None else chat.get_profile_image()
+        assert d["subtitle"] == chat.get_subtitle()
+        assert d["draft"] == "" if chat.get_draft() is None else chat.get_draft()
+
     def test_group_chat_creation_with_translation(self, ac1):
         ac1.set_stock_translation(const.DC_STR_NEWGROUPDRAFT, "xyz %1$s")
         ac1._evlogger.consume_events()
@@ -430,6 +442,32 @@ class TestOnlineAccount:
         assert self_addr not in ev[2]
         ev = ac1._evlogger.get_matching("DC_EVENT_DELETED_BLOB_FILE")
 
+    def test_prepare_file_with_unicode(self, acfactory, lp):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        chat = self.get_chat(ac1, ac2)
+
+        lp.sec("ac1: prepare and send attachment + text to ac2")
+        blobdir = ac1.get_blobdir()
+        basename = "somedäüta.txt"
+        p = os.path.join(blobdir, basename)
+        with open(p, "w") as f:
+            f.write("some data")
+        msg = Message.new_empty(ac1, "file")
+        msg.set_text("hello ä world")
+        msg.set_file(p)
+        message = chat.prepare_message(msg)
+        assert message.is_out_preparing()
+        assert message.text == "hello ä world"
+        chat.send_prepared(message)
+
+        lp.sec("ac2: receive message")
+        ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
+        assert ev[2] > const.DC_CHAT_ID_LAST_SPECIAL
+        msg = ac2.get_message_by_id(ev[1])
+        assert msg.text == "hello ä world"
+        assert open(msg.filename).read() == "some data"
+        assert msg.filename.endswith(basename)
+
     def test_mvbox_sentbox_threads(self, acfactory, lp):
         lp.sec("ac1: start with mvbox thread")
         ac1 = acfactory.get_online_configuring_account(mvbox=True, sentbox=True)
@@ -634,6 +672,32 @@ class TestOnlineAccount:
         ev = ac1._evlogger.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
         assert not msg.is_encrypted()
 
+    def test_send_first_message_as_long_unicode_with_cr(self, acfactory, lp):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        ac2.set_config("save_mime_headers", "1")
+
+        lp.sec("ac1: create chat with ac2")
+        chat = self.get_chat(ac1, ac2, both_created=True)
+
+        lp.sec("sending multi-line non-unicode message from ac1 to ac2")
+        text1 = "hello\nworld"
+        msg_out = chat.send_text(text1)
+        assert not msg_out.is_encrypted()
+
+        lp.sec("sending multi-line unicode text message from ac1 to ac2")
+        text2 = "äalis\nthis is ßßÄ"
+        msg_out = chat.send_text(text2)
+        assert not msg_out.is_encrypted()
+
+        lp.sec("wait for ac2 to receive multi-line non-unicode message")
+        msg_in = ac2.wait_next_incoming_message()
+        assert msg_in.text == text1
+
+        lp.sec("wait for ac2 to receive multi-line unicode message")
+        msg_in = ac2.wait_next_incoming_message()
+        assert msg_in.text == text2
+        assert ac1.get_config("addr") in msg_in.chat.get_name()
+
     def test_reply_encrypted(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 

src/chat.rs

@@ -4,6 +4,7 @@ use std::path::{Path, PathBuf};
 
 use itertools::Itertools;
 use num_traits::FromPrimitive;
+use serde_json::json;
 
 use crate::blob::{BlobError, BlobObject};
 use crate::chatlist::*;
@@ -1903,6 +1904,54 @@ pub fn forward_msgs(context: &Context, msg_ids: &[MsgId], chat_id: u32) -> Resul
     Ok(())
 }
 
+pub fn get_info_json(context: &Context, chat_id: u32) -> Result<String, Error> {
+    let chat = Chat::load_from_db(context, chat_id).unwrap();
+
+    // ToDo:
+    // - [x] id
+    // - [x] type
+    // - [x] name
+    // - [x] archived
+    // - [x] color
+    // - [x] profileImage
+    // - [x] subtitle
+    // - [x] draft,
+    // - [ ] deaddrop,
+    // - [ ] summary,
+    // - [ ] lastUpdated,
+    // - [ ] freshMessageCounter,
+    // - [ ] email
+
+    let profile_image = match chat.get_profile_image(context) {
+        Some(path) => path.into_os_string().into_string().unwrap(),
+        None => "".to_string(),
+    };
+
+    let draft = match get_draft(context, chat_id) {
+        Ok(message) => match message {
+            Some(m) => m.text.unwrap_or_else(|| "".to_string()),
+            None => "".to_string(),
+        },
+        Err(_) => "".to_string(),
+    };
+
+    let s = json!({
+         "id": chat.id,
+         "type": chat.typ as u32,
+         "name": chat.name,
+         "archived": chat.archived,
+         "param": chat.param.to_string(),
+         "gossiped_timestamp": chat.gossiped_timestamp,
+         "is_sending_locations": chat.is_sending_locations,
+         "color": chat.get_color(context),
+         "profile_image": profile_image,
+         "subtitle": chat.get_subtitle(context),
+         "draft": draft
+    });
+
+    Ok(s.to_string())
+}
+
 pub fn get_chat_contact_cnt(context: &Context, chat_id: u32) -> usize {
     context
         .sql

src/dc_receive_imf.rs

@@ -47,6 +47,11 @@ pub fn dc_receive_imf(
         server_uid,
     );
 
+    if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
+        info!(context, "dc_receive_imf: incoming message mime-body:");
+        println!("{}", String::from_utf8_lossy(imf_raw));
+    }
+
     let mime_parser = MimeParser::from_bytes(context, imf_raw);
     let mut mime_parser = if let Err(err) = mime_parser {
         warn!(context, "dc_receive_imf parse error: {}", err);

src/dc_strencode.rs

@@ -1,68 +0,0 @@
-use itertools::Itertools;
-
-/// Encode non-ascii-strings as `=?UTF-8?Q?Bj=c3=b6rn_Petersen?=`.
-/// Belongs to RFC 2047: https://tools.ietf.org/html/rfc2047
-///
-/// We do not fold at position 72; this would result in empty words as `=?utf-8?Q??=` which are correct,
-/// but cannot be displayed by some mail programs (eg. Android Stock Mail).
-/// however, this is not needed, as long as _one_ word is not longer than 72 characters.
-/// _if_ it is, the display may get weird.  This affects the subject only.
-/// the best solution wor all this would be if libetpan encodes the line as only libetpan knowns when a header line is full.
-///
-/// @param to_encode Null-terminated UTF-8-string to encode.
-/// @return Returns the encoded string which must be free()'d when no longed needed.
-///     On errors, NULL is returned.
-pub fn dc_encode_header_words(input: impl AsRef<str>) -> String {
-    let mut result = String::default();
-    for (_, group) in &input.as_ref().chars().group_by(|c| c.is_whitespace()) {
-        let word: String = group.collect();
-        result.push_str(&quote_word(&word.as_bytes()));
-    }
-
-    result
-}
-
-fn must_encode(byte: u8) -> bool {
-    static SPECIALS: &[u8] = b",:!\"#$@[\\]^`{|}~=?_";
-
-    SPECIALS.iter().any(|b| *b == byte)
-}
-
-fn quote_word(word: &[u8]) -> String {
-    let mut result = String::default();
-    let mut encoded = false;
-
-    for byte in word {
-        let byte = *byte;
-        if byte >= 128 || must_encode(byte) {
-            result.push_str(&format!("={:2X}", byte));
-            encoded = true;
-        } else if byte == b' ' {
-            result.push('_');
-            encoded = true;
-        } else {
-            result.push(byte as _);
-        }
-    }
-
-    if encoded {
-        result = format!("=?utf-8?Q?{}?=", &result);
-    }
-    result
-}
-
-/* ******************************************************************************
- * Encode/decode header words, RFC 2047
- ******************************************************************************/
-
-pub fn dc_needs_ext_header(to_check: impl AsRef<str>) -> bool {
-    let to_check = to_check.as_ref();
-
-    if to_check.is_empty() {
-        return false;
-    }
-
-    to_check.chars().any(|c| {
-        !c.is_ascii_alphanumeric() && c != '-' && c != '_' && c != '.' && c != '~' && c != '%'
-    })
-}

src/imap/mod.rs

@@ -201,112 +201,107 @@ impl Imap {
         self.should_reconnect.store(true, Ordering::Relaxed)
     }
 
-    fn setup_handle_if_needed(&self, context: &Context) -> Result<()> {
-        task::block_on(async move {
-            if self.config.read().await.imap_server.is_empty() {
-                return Err(Error::InTeardown);
-            }
+    async fn setup_handle_if_needed(&self, context: &Context) -> Result<()> {
+        if self.config.read().await.imap_server.is_empty() {
+            return Err(Error::InTeardown);
+        }
 
-            if self.should_reconnect() {
-                self.unsetup_handle(context).await;
-                self.should_reconnect.store(false, Ordering::Relaxed);
-            } else if self.is_connected().await {
-                return Ok(());
-            }
+        if self.should_reconnect() {
+            self.unsetup_handle(context).await;
+            self.should_reconnect.store(false, Ordering::Relaxed);
+        } else if self.is_connected().await {
+            return Ok(());
+        }
 
-            let server_flags = self.config.read().await.server_flags as i32;
+        let server_flags = self.config.read().await.server_flags as i32;
 
-            let connection_res: ImapResult<Client> =
-                if (server_flags & (DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_PLAIN)) != 0 {
-                    let config = self.config.read().await;
-                    let imap_server: &str = config.imap_server.as_ref();
-                    let imap_port = config.imap_port;
+        let connection_res: ImapResult<Client> =
+            if (server_flags & (DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_PLAIN)) != 0 {
+                let config = self.config.read().await;
+                let imap_server: &str = config.imap_server.as_ref();
+                let imap_port = config.imap_port;
 
-                    match Client::connect_insecure((imap_server, imap_port)).await {
-                        Ok(client) => {
-                            if (server_flags & DC_LP_IMAP_SOCKET_STARTTLS) != 0 {
-                                client.secure(imap_server, config.certificate_checks).await
-                            } else {
-                                Ok(client)
-                            }
-                        }
-                        Err(err) => Err(err),
-                    }
-                } else {
-                    let config = self.config.read().await;
-                    let imap_server: &str = config.imap_server.as_ref();
-                    let imap_port = config.imap_port;
-
-                    Client::connect_secure(
-                        (imap_server, imap_port),
-                        imap_server,
-                        config.certificate_checks,
-                    )
-                    .await
-                };
-
-            let login_res = match connection_res {
-                Ok(client) => {
-                    let config = self.config.read().await;
-                    let imap_user: &str = config.imap_user.as_ref();
-                    let imap_pw: &str = config.imap_pw.as_ref();
-
-                    if (server_flags & DC_LP_AUTH_OAUTH2) != 0 {
-                        let addr: &str = config.addr.as_ref();
-
-                        if let Some(token) =
-                            dc_get_oauth2_access_token(context, addr, imap_pw, true)
-                        {
-                            let auth = OAuth2 {
-                                user: imap_user.into(),
-                                access_token: token,
-                            };
-                            client.authenticate("XOAUTH2", &auth).await
+                match Client::connect_insecure((imap_server, imap_port)).await {
+                    Ok(client) => {
+                        if (server_flags & DC_LP_IMAP_SOCKET_STARTTLS) != 0 {
+                            client.secure(imap_server, config.certificate_checks).await
                         } else {
-                            return Err(Error::OauthError);
+                            Ok(client)
                         }
-                    } else {
-                        client.login(imap_user, imap_pw).await
                     }
+                    Err(err) => Err(err),
                 }
-                Err(err) => {
-                    let message = {
-                        let config = self.config.read().await;
-                        let imap_server: &str = config.imap_server.as_ref();
-                        let imap_port = config.imap_port;
-                        context.stock_string_repl_str2(
-                            StockMessage::ServerResponse,
-                            format!("{}:{}", imap_server, imap_port),
-                            err.to_string(),
-                        )
-                    };
-                    // IMAP connection failures are reported to users
-                    emit_event!(context, Event::ErrorNetwork(message));
-                    return Err(Error::ConnectionFailed(err.to_string()));
-                }
+            } else {
+                let config = self.config.read().await;
+                let imap_server: &str = config.imap_server.as_ref();
+                let imap_port = config.imap_port;
+
+                Client::connect_secure(
+                    (imap_server, imap_port),
+                    imap_server,
+                    config.certificate_checks,
+                )
+                .await
             };
 
-            self.should_reconnect.store(false, Ordering::Relaxed);
+        let login_res = match connection_res {
+            Ok(client) => {
+                let config = self.config.read().await;
+                let imap_user: &str = config.imap_user.as_ref();
+                let imap_pw: &str = config.imap_pw.as_ref();
 
-            match login_res {
-                Ok(session) => {
-                    *self.session.lock().await = Some(session);
-                    Ok(())
-                }
-                Err((err, _)) => {
-                    let imap_user = self.config.read().await.imap_user.to_owned();
-                    let message =
-                        context.stock_string_repl_str(StockMessage::CannotLogin, &imap_user);
+                if (server_flags & DC_LP_AUTH_OAUTH2) != 0 {
+                    let addr: &str = config.addr.as_ref();
 
-                    emit_event!(
-                        context,
-                        Event::ErrorNetwork(format!("{} ({})", message, err))
-                    );
-                    self.trigger_reconnect();
-                    Err(Error::LoginFailed(format!("cannot login as {}", imap_user)))
+                    if let Some(token) = dc_get_oauth2_access_token(context, addr, imap_pw, true) {
+                        let auth = OAuth2 {
+                            user: imap_user.into(),
+                            access_token: token,
+                        };
+                        client.authenticate("XOAUTH2", &auth).await
+                    } else {
+                        return Err(Error::OauthError);
+                    }
+                } else {
+                    client.login(imap_user, imap_pw).await
                 }
             }
-        })
+            Err(err) => {
+                let message = {
+                    let config = self.config.read().await;
+                    let imap_server: &str = config.imap_server.as_ref();
+                    let imap_port = config.imap_port;
+                    context.stock_string_repl_str2(
+                        StockMessage::ServerResponse,
+                        format!("{}:{}", imap_server, imap_port),
+                        err.to_string(),
+                    )
+                };
+                // IMAP connection failures are reported to users
+                emit_event!(context, Event::ErrorNetwork(message));
+                return Err(Error::ConnectionFailed(err.to_string()));
+            }
+        };
+
+        self.should_reconnect.store(false, Ordering::Relaxed);
+
+        match login_res {
+            Ok(session) => {
+                *self.session.lock().await = Some(session);
+                Ok(())
+            }
+            Err((err, _)) => {
+                let imap_user = self.config.read().await.imap_user.to_owned();
+                let message = context.stock_string_repl_str(StockMessage::CannotLogin, &imap_user);
+
+                emit_event!(
+                    context,
+                    Event::ErrorNetwork(format!("{} ({})", message, err))
+                );
+                self.trigger_reconnect();
+                Err(Error::LoginFailed(format!("cannot login as {}", imap_user)))
+            }
+        }
     }
 
     async fn unsetup_handle(&self, context: &Context) {
@@ -387,7 +382,7 @@ impl Imap {
                 config.server_flags = server_flags;
             }
 
-            if let Err(err) = self.setup_handle_if_needed(context) {
+            if let Err(err) = self.setup_handle_if_needed(context).await {
                 warn!(context, "failed to setup imap handle: {}", err);
                 self.free_connect_params().await;
                 return false;
@@ -449,10 +444,9 @@ impl Imap {
                 // probably shutdown
                 return Err(Error::InTeardown);
             }
-            while self
-                .fetch_from_single_folder(context, &watch_folder)
-                .await?
-            {
+            self.setup_handle_if_needed(context).await?;
+
+            while self.fetch_new_messages(context, &watch_folder).await? {
                 // We fetch until no more new messages are there.
             }
             Ok(())
@@ -560,7 +554,7 @@ impl Imap {
         })
     }
 
-    async fn fetch_from_single_folder<S: AsRef<str>>(
+    async fn fetch_new_messages<S: AsRef<str>>(
         &self,
         context: &Context,
         folder: S,
@@ -593,9 +587,11 @@ impl Imap {
         for msg in &list {
             let cur_uid = msg.uid.unwrap_or_default();
             if cur_uid <= last_seen_uid {
-                warn!(
+                // seems that at least dovecot sends the last available UID
+                // even if we asked for higher UID+N:*
+                info!(
                     context,
-                    "unexpected uid {}, last seen was {}", cur_uid, last_seen_uid
+                    "fetch_new_messages: ignoring uid {}, last seen was {}", cur_uid, last_seen_uid
                 );
                 continue;
             }
@@ -740,7 +736,7 @@ impl Imap {
                 return Err(Error::IdleAbilityMissing);
             }
 
-            self.setup_handle_if_needed(context)?;
+            self.setup_handle_if_needed(context).await?;
 
             self.select_folder(context, watch_folder.clone()).await?;
 
@@ -886,9 +882,9 @@ impl Imap {
                     // will not find any new.
 
                     if let Some(ref watch_folder) = watch_folder {
-                        match self.fetch_from_single_folder(context, watch_folder).await {
+                        match self.fetch_new_messages(context, watch_folder).await {
                             Ok(res) => {
-                                info!(context, "fetch_from_single_folder returned {:?}", res);
+                                info!(context, "fetch_new_messages returned {:?}", res);
                                 if res {
                                     break;
                                 }
@@ -1322,13 +1318,17 @@ impl Imap {
         task::block_on(async move {
             info!(context, "emptying folder {}", folder);
 
+            // we want to report all error to the user
+            // (no retry should be attempted)
             if folder.is_empty() {
                 error!(context, "cannot perform empty, folder not set");
                 return;
             }
+            if let Err(err) = self.setup_handle_if_needed(context).await {
+                error!(context, "could not setup imap connection: {:?}", err);
+                return;
+            }
             if let Err(err) = self.select_folder(context, Some(&folder)).await {
-                // we want to report all error to the user
-                // (no retry should be attempted)
                 error!(
                     context,
                     "Could not select {} for expunging: {:?}", folder, err

src/imap/select_folder.rs

@@ -34,6 +34,7 @@ impl Imap {
             let mut cfg = self.config.write().await;
             cfg.selected_folder = None;
             cfg.selected_folder_needs_expunge = false;
+            self.trigger_reconnect();
             return Err(Error::NoSession);
         }
 
@@ -61,6 +62,7 @@ impl Imap {
                             info!(context, "close/expunge succeeded");
                         }
                         Err(err) => {
+                            self.trigger_reconnect();
                             return Err(Error::CloseExpungeFailed(err));
                         }
                     }

src/imap_client.rs

@@ -11,8 +11,6 @@ use async_tls::client::TlsStream;
 
 use crate::login_param::{dc_build_tls_config, CertificateChecks};
 
-const DCC_IMAP_DEBUG: &str = "DCC_IMAP_DEBUG";
-
 #[derive(Debug)]
 pub(crate) enum Client {
     Secure(ImapClient<TlsStream<TcpStream>>),
@@ -42,7 +40,7 @@ impl Client {
         let tls_connector: async_tls::TlsConnector = Arc::new(tls_config).into();
         let tls_stream = tls_connector.connect(domain.as_ref(), stream)?.await?;
         let mut client = ImapClient::new(tls_stream);
-        if std::env::var(DCC_IMAP_DEBUG).is_ok() {
+        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
             client.debug = true;
         }
 
@@ -58,7 +56,7 @@ impl Client {
         let stream = TcpStream::connect(addr).await?;
 
         let mut client = ImapClient::new(stream);
-        if std::env::var(DCC_IMAP_DEBUG).is_ok() {
+        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
             client.debug = true;
         }
         let _greeting = client

src/job.rs

@@ -192,6 +192,10 @@ impl Job {
                     // was sent we need to mark it in the database ASAP as we
                     // otherwise might send it twice.
                     let mut smtp = context.smtp.lock().unwrap();
+                    if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
+                        info!(context, "smtp-sending out mime message:");
+                        println!("{}", String::from_utf8_lossy(&body));
+                    }
                     match smtp.send(context, recipients_list, body, self.job_id) {
                         Err(crate::smtp::send::Error::SendError(err)) => {
                             // Remote error, retry later.

src/lib.rs

@@ -1,5 +1,6 @@
 #![deny(clippy::correctness, missing_debug_implementations, clippy::all)]
-#![warn(
+// for now we hide warnings to not clutter/hide errors during "cargo clippy"
+#![allow(
     clippy::type_complexity,
     clippy::cognitive_complexity,
     clippy::too_many_arguments,
@@ -75,8 +76,13 @@ mod dehtml;
 pub mod dc_array;
 pub mod dc_receive_imf;
 mod dc_simplify;
-mod dc_strencode;
 pub mod dc_tools;
 
+/// if set imap/incoming and smtp/outgoing MIME messages will be printed
+pub const DCC_MIME_DEBUG: &str = "DCC_MIME_DEBUG";
+
+/// if set IMAP protocol commands and responses will be printed
+pub const DCC_IMAP_DEBUG: &str = "DCC_IMAP_DEBUG";
+
 #[cfg(test)]
 mod test_utils;

src/mimefactory.rs

@@ -6,7 +6,6 @@ use crate::config::Config;
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::{get_version_str, Context};
-use crate::dc_strencode::*;
 use crate::dc_tools::*;
 use crate::e2ee::*;
 use crate::error::Error;
@@ -333,13 +332,19 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             Loaded::Message => {
                 match self.chat {
                     Some(ref chat) => {
-                        let raw_subject = message::get_summarytext_by_raw(
+                        let raw = message::get_summarytext_by_raw(
                             self.msg.type_0,
                             self.msg.text.as_ref(),
                             &self.msg.param,
                             32,
                             self.context,
                         );
+                        let mut lines = raw.lines();
+                        let raw_subject = if let Some(line) = lines.next() {
+                            line
+                        } else {
+                            ""
+                        };
 
                         let afwd_email = self.msg.param.exists(Param::Forwarded);
                         let fwd = if afwd_email { "Fwd: " } else { "" };
@@ -377,7 +382,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let mut unprotected_headers: Vec<Header> = Vec::new();
 
         let from = Address::new_mailbox_with_name(
-            dc_encode_header_words(&self.from_displayname),
+            encode_words(&self.from_displayname),
             self.from_addr.clone(),
         );
 
@@ -389,7 +394,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                 to.push(Address::new_mailbox(addr.clone()));
             } else {
                 to.push(Address::new_mailbox_with_name(
-                    dc_encode_header_words(name),
+                    encode_words(name),
                     addr.clone(),
                 ));
             }
@@ -445,7 +450,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let e2ee_guranteed = self.is_e2ee_guranteed();
         let mut encrypt_helper = EncryptHelper::new(self.context)?;
 
-        let subject = dc_encode_header_words(subject_str);
+        let subject = encode_words(&subject_str);
 
         let mut message = match self.loaded {
             Loaded::Message => {
@@ -605,7 +610,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         if chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup {
             protected_headers.push(Header::new("Chat-Group-ID".into(), chat.grpid.clone()));
 
-            let encoded = dc_encode_header_words(&chat.name);
+            let encoded = encode_words(&chat.name);
             protected_headers.push(Header::new("Chat-Group-Name".into(), encoded));
 
             match command {
@@ -975,20 +980,18 @@ fn build_body_file(
         }
     };
 
-    let needs_ext = dc_needs_ext_header(&filename_to_send);
-
     // create mime part, for Content-Disposition, see RFC 2183.
     // `Content-Disposition: attachment` seems not to make a difference to `Content-Disposition: inline`
     // at least on tested Thunderbird and Gma'l in 2017.
     // But I've heard about problems with inline and outl'k, so we just use the attachment-type until we
     // run into other problems ...
-    let cd_value = if needs_ext {
-        format!("attachment; filename=\"{}\"", &filename_to_send)
-    } else {
+    let cd_value = if needs_encoding(&filename_to_send) {
         format!(
             "attachment; filename*=\"{}\"",
-            dc_encode_header_words(&filename_to_send)
+            encode_words(&filename_to_send)
         )
+    } else {
+        format!("attachment; filename=\"{}\"", &filename_to_send)
     };
 
     let body = std::fs::read(blob.to_abs_path())?;
@@ -1022,3 +1025,23 @@ fn is_file_size_okay(context: &Context, msg: &Message) -> bool {
         None => false,
     }
 }
+
+/* ******************************************************************************
+ * Encode/decode header words, RFC 2047
+ ******************************************************************************/
+
+fn encode_words(word: &str) -> String {
+    encoded_words::encode(word, None, encoded_words::EncodingFlag::Shortest, None)
+}
+
+pub fn needs_encoding(to_check: impl AsRef<str>) -> bool {
+    let to_check = to_check.as_ref();
+
+    if to_check.is_empty() {
+        return false;
+    }
+
+    to_check.chars().any(|c| {
+        !c.is_ascii_alphanumeric() && c != '-' && c != '_' && c != '.' && c != '~' && c != '%'
+    })
+}