use encoded-words crate, which friedel ported from python

CHANGELOG.md

@@ -1,14 +1,6 @@
 # Changelog 
 
-## 1.0.0-beta.11
-
-- trigger reconnect more often on imap error states.  Should fix an 
-  issue observed when trying to empty a folder.  @hpk42
-
-- un-split qr tests: we fixed qr-securejoin protocol flakyness 
-  last weeks. @hpk42
-
-## 1.0.0-beta.10
+## 1.0.0-beta.10 (pending)
 
 - fix grpid-determination from in-reply-to and references headers. @hpk42
 
@@ -18,11 +10,6 @@
 
 - remove last unsafe code from dc_receive_imf :) @hpk42
 
-- add experimental new dc_chat_get_info_json FFI/API so that desktop devs
-  can play with using it. @jikstra
-
-- fix encoding of subjects and attachment-filenames @hpk42
-  @dignifiedquire . 
 
 ## 1.0.0-beta.9
 

Cargo.lock

@@ -607,7 +607,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat"
-version = "1.0.0-beta.11"
+version = "1.0.0-beta.9"
 dependencies = [
  "async-imap 0.1.1 (git+https://github.com/async-email/async-imap)",
  "async-std 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -683,9 +683,9 @@ dependencies = [
 
 [[package]]
 name = "deltachat_ffi"
-version = "1.0.0-beta.11"
+version = "1.0.0-beta.9"
 dependencies = [
- "deltachat 1.0.0-beta.11",
+ "deltachat 1.0.0-beta.9",
  "deltachat-provider-database 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "human-panic 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -800,7 +800,7 @@ dependencies = [
 [[package]]
 name = "encoded-words"
 version = "0.1.0"
-source = "git+https://github.com/async-email/encoded-words#2631c258183620f6d976abffabbfc2dcc697d793"
+source = "git+https://github.com/async-email/encoded-words#019e833f0c9ea7d4b0b693aab44e66d78d18f1d0"
 dependencies = [
  "base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "charset 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.0.0-beta.11"
+version = "1.0.0-beta.9"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL"

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.0.0-beta.11"
+version = "1.0.0-beta.9"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/deltachat.h

@@ -2602,25 +2602,6 @@ dc_lot_t*        dc_chatlist_get_summary     (const dc_chatlist_t* chatlist, siz
 dc_context_t*    dc_chatlist_get_context     (dc_chatlist_t* chatlist);
 
 
-/**
- * Get info summary for a chat, in json format. 
- *
- * The returned json string has the following key/values: 
- *
- * id: chat id 
- * name: chat/group name 
- * color: color of this chat 
- * last-message-from: who sent the last message 
- * last-message-text: message (truncated) 
- * last-message-state: DC_STATE* constant
- * last-message-date: 
- * avatar-path: path-to-blobfile 
- * is_verified: yes/no
-
- * @return a utf8-encoded json string containing all requested info. Must be freed using dc_str_unref().  NULL is never returned.
- */
-char*            dc_chat_get_info_json       (dc_context_t* context, size_t chat_id);
-
 /**
  * @class dc_chat_t
  *

deltachat-ffi/src/lib.rs

@@ -2377,27 +2377,6 @@ pub unsafe extern "C" fn dc_chat_is_sending_locations(chat: *mut dc_chat_t) -> l
     ffi_chat.chat.is_sending_locations() as libc::c_int
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_chat_get_info_json(
-    context: *mut dc_context_t,
-    chat_id: u32,
-) -> *mut libc::c_char {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_chat_get_info_json()");
-        return "".strdup();
-    }
-    let ffi_context = &*context;
-    ffi_context
-        .with_inner(|ctx| match chat::get_info_json(ctx, chat_id) {
-            Ok(s) => s.strdup(),
-            Err(err) => {
-                error!(ctx, "get_info_json({}) returned: {}", chat_id, err);
-                return "".strdup();
-            }
-        })
-        .unwrap_or_else(|_| "".strdup())
-}
-
 // dc_msg_t
 
 /// FFI struct for [dc_msg_t]

python/src/deltachat/chat.py

@@ -2,7 +2,6 @@
 
 import mimetypes
 import calendar
-import json
 from datetime import datetime
 import os
 from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array
@@ -243,12 +242,6 @@ class Chat(object):
         """
         return lib.dc_marknoticed_chat(self._dc_context, self.id)
 
-    def get_summary(self):
-        """ return dictionary with summary information. """
-        dc_res = lib.dc_chat_get_info_json(self._dc_context, self.id)
-        s = from_dc_charpointer(dc_res)
-        return json.loads(s)
-
     # ------  group management API ------------------------------
 
     def add_contact(self, contact):
@@ -331,18 +324,6 @@ class Chat(object):
             return None
         return from_dc_charpointer(dc_res)
 
-    def get_color(self):
-        """return the color of the chat.
-        :returns: color as 0x00rrggbb
-        """
-        return lib.dc_chat_get_color(self._dc_chat)
-
-    def get_subtitle(self):
-        """return the subtitle of the chat
-        :returns: the subtitle
-        """
-        return from_dc_charpointer(lib.dc_chat_get_subtitle(self._dc_chat))
-
     # ------  location streaming API ------------------------------
 
     def is_sending_locations(self):
@@ -351,12 +332,6 @@ class Chat(object):
         """
         return lib.dc_is_sending_locations_to_chat(self._dc_context, self.id)
 
-    def is_archived(self):
-        """return True if this chat is archived.
-        :returns: True if archived.
-        """
-        return lib.dc_chat_get_archived(self._dc_chat)
-
     def enable_sending_locations(self, seconds):
         """enable sending locations for this chat.
 

python/tests/test_account.py

@@ -155,18 +155,6 @@ class TestOfflineChat:
         chat.set_name("title2")
         assert chat.get_name() == "title2"
 
-        d = chat.get_summary()
-        print(d)
-        assert d["id"] == chat.id
-        assert d["type"] == chat.get_type()
-        assert d["name"] == chat.get_name()
-        assert d["archived"] == chat.is_archived()
-        # assert d["param"] == chat.param
-        assert d["color"] == chat.get_color()
-        assert d["profile_image"] == "" if chat.get_profile_image() is None else chat.get_profile_image()
-        assert d["subtitle"] == chat.get_subtitle()
-        assert d["draft"] == "" if chat.get_draft() is None else chat.get_draft()
-
     def test_group_chat_creation_with_translation(self, ac1):
         ac1.set_stock_translation(const.DC_STR_NEWGROUPDRAFT, "xyz %1$s")
         ac1._evlogger.consume_events()

python/tox.ini

@@ -7,7 +7,11 @@ envlist =
 
 [testenv]
 commands = 
-    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx {posargs:tests} 
+    # (some qr tests are pretty heavy in terms of send/received
+    # messages and async-imap's likely has concurrency problems, 
+    # eg https://github.com/async-email/async-imap/issues/4 )
+    pytest -n6 --reruns 3 --reruns-delay 5 -v -rsXx -k "not qr" {posargs:tests} 
+    pytest -n6 --reruns 5 --reruns-delay 5 -v -rsXx -k "qr" {posargs:tests} 
     # python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 

src/chat.rs

@@ -4,7 +4,6 @@ use std::path::{Path, PathBuf};
 
 use itertools::Itertools;
 use num_traits::FromPrimitive;
-use serde_json::json;
 
 use crate::blob::{BlobError, BlobObject};
 use crate::chatlist::*;
@@ -1904,54 +1903,6 @@ pub fn forward_msgs(context: &Context, msg_ids: &[MsgId], chat_id: u32) -> Resul
     Ok(())
 }
 
-pub fn get_info_json(context: &Context, chat_id: u32) -> Result<String, Error> {
-    let chat = Chat::load_from_db(context, chat_id).unwrap();
-
-    // ToDo:
-    // - [x] id
-    // - [x] type
-    // - [x] name
-    // - [x] archived
-    // - [x] color
-    // - [x] profileImage
-    // - [x] subtitle
-    // - [x] draft,
-    // - [ ] deaddrop,
-    // - [ ] summary,
-    // - [ ] lastUpdated,
-    // - [ ] freshMessageCounter,
-    // - [ ] email
-
-    let profile_image = match chat.get_profile_image(context) {
-        Some(path) => path.into_os_string().into_string().unwrap(),
-        None => "".to_string(),
-    };
-
-    let draft = match get_draft(context, chat_id) {
-        Ok(message) => match message {
-            Some(m) => m.text.unwrap_or_else(|| "".to_string()),
-            None => "".to_string(),
-        },
-        Err(_) => "".to_string(),
-    };
-
-    let s = json!({
-         "id": chat.id,
-         "type": chat.typ as u32,
-         "name": chat.name,
-         "archived": chat.archived,
-         "param": chat.param.to_string(),
-         "gossiped_timestamp": chat.gossiped_timestamp,
-         "is_sending_locations": chat.is_sending_locations,
-         "color": chat.get_color(context),
-         "profile_image": profile_image,
-         "subtitle": chat.get_subtitle(context),
-         "draft": draft
-    });
-
-    Ok(s.to_string())
-}
-
 pub fn get_chat_contact_cnt(context: &Context, chat_id: u32) -> usize {
     context
         .sql

src/imap/mod.rs

@@ -201,107 +201,112 @@ impl Imap {
         self.should_reconnect.store(true, Ordering::Relaxed)
     }
 
-    async fn setup_handle_if_needed(&self, context: &Context) -> Result<()> {
-        if self.config.read().await.imap_server.is_empty() {
-            return Err(Error::InTeardown);
-        }
-
-        if self.should_reconnect() {
-            self.unsetup_handle(context).await;
-            self.should_reconnect.store(false, Ordering::Relaxed);
-        } else if self.is_connected().await {
-            return Ok(());
-        }
-
-        let server_flags = self.config.read().await.server_flags as i32;
-
-        let connection_res: ImapResult<Client> =
-            if (server_flags & (DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_PLAIN)) != 0 {
-                let config = self.config.read().await;
-                let imap_server: &str = config.imap_server.as_ref();
-                let imap_port = config.imap_port;
-
-                match Client::connect_insecure((imap_server, imap_port)).await {
-                    Ok(client) => {
-                        if (server_flags & DC_LP_IMAP_SOCKET_STARTTLS) != 0 {
-                            client.secure(imap_server, config.certificate_checks).await
-                        } else {
-                            Ok(client)
-                        }
-                    }
-                    Err(err) => Err(err),
-                }
-            } else {
-                let config = self.config.read().await;
-                let imap_server: &str = config.imap_server.as_ref();
-                let imap_port = config.imap_port;
-
-                Client::connect_secure(
-                    (imap_server, imap_port),
-                    imap_server,
-                    config.certificate_checks,
-                )
-                .await
-            };
-
-        let login_res = match connection_res {
-            Ok(client) => {
-                let config = self.config.read().await;
-                let imap_user: &str = config.imap_user.as_ref();
-                let imap_pw: &str = config.imap_pw.as_ref();
-
-                if (server_flags & DC_LP_AUTH_OAUTH2) != 0 {
-                    let addr: &str = config.addr.as_ref();
-
-                    if let Some(token) = dc_get_oauth2_access_token(context, addr, imap_pw, true) {
-                        let auth = OAuth2 {
-                            user: imap_user.into(),
-                            access_token: token,
-                        };
-                        client.authenticate("XOAUTH2", &auth).await
-                    } else {
-                        return Err(Error::OauthError);
-                    }
-                } else {
-                    client.login(imap_user, imap_pw).await
-                }
+    fn setup_handle_if_needed(&self, context: &Context) -> Result<()> {
+        task::block_on(async move {
+            if self.config.read().await.imap_server.is_empty() {
+                return Err(Error::InTeardown);
             }
-            Err(err) => {
-                let message = {
+
+            if self.should_reconnect() {
+                self.unsetup_handle(context).await;
+                self.should_reconnect.store(false, Ordering::Relaxed);
+            } else if self.is_connected().await {
+                return Ok(());
+            }
+
+            let server_flags = self.config.read().await.server_flags as i32;
+
+            let connection_res: ImapResult<Client> =
+                if (server_flags & (DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_PLAIN)) != 0 {
                     let config = self.config.read().await;
                     let imap_server: &str = config.imap_server.as_ref();
                     let imap_port = config.imap_port;
-                    context.stock_string_repl_str2(
-                        StockMessage::ServerResponse,
-                        format!("{}:{}", imap_server, imap_port),
-                        err.to_string(),
+
+                    match Client::connect_insecure((imap_server, imap_port)).await {
+                        Ok(client) => {
+                            if (server_flags & DC_LP_IMAP_SOCKET_STARTTLS) != 0 {
+                                client.secure(imap_server, config.certificate_checks).await
+                            } else {
+                                Ok(client)
+                            }
+                        }
+                        Err(err) => Err(err),
+                    }
+                } else {
+                    let config = self.config.read().await;
+                    let imap_server: &str = config.imap_server.as_ref();
+                    let imap_port = config.imap_port;
+
+                    Client::connect_secure(
+                        (imap_server, imap_port),
+                        imap_server,
+                        config.certificate_checks,
                     )
+                    .await
                 };
-                // IMAP connection failures are reported to users
-                emit_event!(context, Event::ErrorNetwork(message));
-                return Err(Error::ConnectionFailed(err.to_string()));
-            }
-        };
 
-        self.should_reconnect.store(false, Ordering::Relaxed);
+            let login_res = match connection_res {
+                Ok(client) => {
+                    let config = self.config.read().await;
+                    let imap_user: &str = config.imap_user.as_ref();
+                    let imap_pw: &str = config.imap_pw.as_ref();
 
-        match login_res {
-            Ok(session) => {
-                *self.session.lock().await = Some(session);
-                Ok(())
-            }
-            Err((err, _)) => {
-                let imap_user = self.config.read().await.imap_user.to_owned();
-                let message = context.stock_string_repl_str(StockMessage::CannotLogin, &imap_user);
+                    if (server_flags & DC_LP_AUTH_OAUTH2) != 0 {
+                        let addr: &str = config.addr.as_ref();
 
-                emit_event!(
-                    context,
-                    Event::ErrorNetwork(format!("{} ({})", message, err))
-                );
-                self.trigger_reconnect();
-                Err(Error::LoginFailed(format!("cannot login as {}", imap_user)))
+                        if let Some(token) =
+                            dc_get_oauth2_access_token(context, addr, imap_pw, true)
+                        {
+                            let auth = OAuth2 {
+                                user: imap_user.into(),
+                                access_token: token,
+                            };
+                            client.authenticate("XOAUTH2", &auth).await
+                        } else {
+                            return Err(Error::OauthError);
+                        }
+                    } else {
+                        client.login(imap_user, imap_pw).await
+                    }
+                }
+                Err(err) => {
+                    let message = {
+                        let config = self.config.read().await;
+                        let imap_server: &str = config.imap_server.as_ref();
+                        let imap_port = config.imap_port;
+                        context.stock_string_repl_str2(
+                            StockMessage::ServerResponse,
+                            format!("{}:{}", imap_server, imap_port),
+                            err.to_string(),
+                        )
+                    };
+                    // IMAP connection failures are reported to users
+                    emit_event!(context, Event::ErrorNetwork(message));
+                    return Err(Error::ConnectionFailed(err.to_string()));
+                }
+            };
+
+            self.should_reconnect.store(false, Ordering::Relaxed);
+
+            match login_res {
+                Ok(session) => {
+                    *self.session.lock().await = Some(session);
+                    Ok(())
+                }
+                Err((err, _)) => {
+                    let imap_user = self.config.read().await.imap_user.to_owned();
+                    let message =
+                        context.stock_string_repl_str(StockMessage::CannotLogin, &imap_user);
+
+                    emit_event!(
+                        context,
+                        Event::ErrorNetwork(format!("{} ({})", message, err))
+                    );
+                    self.trigger_reconnect();
+                    Err(Error::LoginFailed(format!("cannot login as {}", imap_user)))
+                }
             }
-        }
+        })
     }
 
     async fn unsetup_handle(&self, context: &Context) {
@@ -382,7 +387,7 @@ impl Imap {
                 config.server_flags = server_flags;
             }
 
-            if let Err(err) = self.setup_handle_if_needed(context).await {
+            if let Err(err) = self.setup_handle_if_needed(context) {
                 warn!(context, "failed to setup imap handle: {}", err);
                 self.free_connect_params().await;
                 return false;
@@ -444,9 +449,10 @@ impl Imap {
                 // probably shutdown
                 return Err(Error::InTeardown);
             }
-            self.setup_handle_if_needed(context).await?;
-
-            while self.fetch_new_messages(context, &watch_folder).await? {
+            while self
+                .fetch_from_single_folder(context, &watch_folder)
+                .await?
+            {
                 // We fetch until no more new messages are there.
             }
             Ok(())
@@ -554,7 +560,7 @@ impl Imap {
         })
     }
 
-    async fn fetch_new_messages<S: AsRef<str>>(
+    async fn fetch_from_single_folder<S: AsRef<str>>(
         &self,
         context: &Context,
         folder: S,
@@ -587,11 +593,9 @@ impl Imap {
         for msg in &list {
             let cur_uid = msg.uid.unwrap_or_default();
             if cur_uid <= last_seen_uid {
-                // seems that at least dovecot sends the last available UID
-                // even if we asked for higher UID+N:*
-                info!(
+                warn!(
                     context,
-                    "fetch_new_messages: ignoring uid {}, last seen was {}", cur_uid, last_seen_uid
+                    "unexpected uid {}, last seen was {}", cur_uid, last_seen_uid
                 );
                 continue;
             }
@@ -736,7 +740,7 @@ impl Imap {
                 return Err(Error::IdleAbilityMissing);
             }
 
-            self.setup_handle_if_needed(context).await?;
+            self.setup_handle_if_needed(context)?;
 
             self.select_folder(context, watch_folder.clone()).await?;
 
@@ -882,9 +886,9 @@ impl Imap {
                     // will not find any new.
 
                     if let Some(ref watch_folder) = watch_folder {
-                        match self.fetch_new_messages(context, watch_folder).await {
+                        match self.fetch_from_single_folder(context, watch_folder).await {
                             Ok(res) => {
-                                info!(context, "fetch_new_messages returned {:?}", res);
+                                info!(context, "fetch_from_single_folder returned {:?}", res);
                                 if res {
                                     break;
                                 }
@@ -1318,17 +1322,13 @@ impl Imap {
         task::block_on(async move {
             info!(context, "emptying folder {}", folder);
 
-            // we want to report all error to the user
-            // (no retry should be attempted)
             if folder.is_empty() {
                 error!(context, "cannot perform empty, folder not set");
                 return;
             }
-            if let Err(err) = self.setup_handle_if_needed(context).await {
-                error!(context, "could not setup imap connection: {:?}", err);
-                return;
-            }
             if let Err(err) = self.select_folder(context, Some(&folder)).await {
+                // we want to report all error to the user
+                // (no retry should be attempted)
                 error!(
                     context,
                     "Could not select {} for expunging: {:?}", folder, err

src/imap/select_folder.rs

@@ -34,7 +34,6 @@ impl Imap {
             let mut cfg = self.config.write().await;
             cfg.selected_folder = None;
             cfg.selected_folder_needs_expunge = false;
-            self.trigger_reconnect();
             return Err(Error::NoSession);
         }
 
@@ -62,7 +61,6 @@ impl Imap {
                             info!(context, "close/expunge succeeded");
                         }
                         Err(err) => {
-                            self.trigger_reconnect();
                             return Err(Error::CloseExpungeFailed(err));
                         }
                     }