test_encrypt_decrypt_fuzz_alice: load secret key directly from file

Simplified test using pregenerated key without signature.

.github/workflows/code-quality.yml

@@ -45,22 +45,4 @@ jobs:
       - uses: actions-rs/clippy-check@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          args: --all-features
-
-  doc:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: nightly-2019-11-06
-          override: true
-      - uses: actions-rs/cargo@v1
-        with:
-          command: doc
-          args: --no-deps -p deltachat
-      - uses: actions-rs/cargo@v1
-        with:
-          command: doc
-          args: --no-deps -p deltachat_ffi
+          args: --all-features

CHANGELOG.md

@@ -1,5 +1,52 @@
 # Changelog 
 
+## 1.25.0
+
+- save traffic by downloading only messages that are really displayed #1236
+
+- change generated key type to Ed25519, these keys are much shorter
+  than RSA keys, which results in saving traffic and speed improvements #1287
+
+- improve key handling #1237 #1240 #1242 #1247
+
+- mute handling, apis are dc_set_chat_mute_duration()
+  dc_chat_is_muted() and dc_chat_get_remaining_mute_duration() #1143
+
+- pinning chats, new apis are dc_set_chat_visibility() and
+  dc_chat_get_visibility() #1248
+
+- add dc_provider_new_from_email() api that queries the new, integrated
+  provider-database #1207
+
+- account creation by scanning a qr code
+  in the DCACCOUNT scheme (https://mailadm.readthedocs.io),
+  new api is dc_set_config_from_qr() #1249
+
+- if possible, dc_join_securejoin(), returns the new chat-id immediately
+  and does the handshake in background #1225
+
+- update imap and smtp dependencies #1115
+
+- check for MOVE capability before using MOVE command #1263
+
+- allow inline attachments from RFC 2183 #1280
+
+- fix updating names from incoming mails #1298
+
+- fix error messages shown on import #1234
+
+- directly attempt to re-connect if the smtp connection is maybe stale #1296
+
+- improve adding group members #1291
+
+- improve rust-api #1261
+
+- cleanup #1302 #1283 #1282 #1276 #1270-#1274 #1267 #1258-#1260
+  #1257 #1239 #1231 #1224
+
+- update spec #1286 #1291
+
+
 ## 1.0.0-beta.24
 
 - fix oauth2/gmail bug introduced in beta23 (not used in releases) #1219

Cargo.lock

@@ -627,7 +627,7 @@ dependencies = [
 
 [[package]]
 name = "deltachat"
-version = "1.0.0-beta.24"
+version = "1.25.0"
 dependencies = [
  "async-imap 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "async-native-tls 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -693,9 +693,9 @@ dependencies = [
 
 [[package]]
 name = "deltachat_ffi"
-version = "1.0.0-beta.24"
+version = "1.25.0"
 dependencies = [
- "deltachat 1.0.0-beta.24",
+ "deltachat 1.25.0",
  "failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)",
  "human-panic 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.0.0-beta.24"
+version = "1.25.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.0.0-beta.24"
+version = "1.25.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/deltachat.h

@@ -364,6 +364,12 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    also show all mails of confirmed contacts,
  *                    DC_SHOW_EMAILS_ALL (2)=
  *                    also show mails of unconfirmed contacts in the deaddrop.
+ * - `key_gen_type` = DC_KEY_GEN_DEFAULT (0)=
+ *                    generate recommended key type (default),
+ *                    DC_KEY_GEN_RSA2048 (1)=
+ *                    generate RSA 2048 keypair
+ *                    DC_KEY_GEN_ED25519 (2)=
+ *                    generate Ed25519 keypair
  * - `save_mime_headers` = 1=save mime headers
  *                    and make dc_get_mime_headers() work for subsequent calls,
  *                    0=do not save mime headers (default)
@@ -918,7 +924,7 @@ int             dc_preconfigure_keypair        (dc_context_t* context, const cha
  *   or "Not now".
  *   The UI can also offer a "Close" button that calls dc_marknoticed_contact() then.
  * - DC_CHAT_ID_ARCHIVED_LINK (6) - this special chat is present if the user has
- *   archived _any_ chat using dc_archive_chat(). The UI should show a link as
+ *   archived _any_ chat using dc_set_chat_visibility(). The UI should show a link as
  *   "Show archived chats", if the user clicks this item, the UI should show a
  *   list of all archived chats that can be created by this function hen using
  *   the DC_GCL_ARCHIVED_ONLY flag.
@@ -1372,25 +1378,18 @@ uint32_t        dc_get_next_media            (dc_context_t* context, uint32_t ms
 
 
 /**
- * Archive or unarchive a chat.
+ * Set chat visibility to pinned, archived or normal.
  *
- * Archived chats are not included in the default chatlist returned
- * by dc_get_chatlist().  Instead, if there are _any_ archived chats,
- * the pseudo-chat with the chat_id DC_CHAT_ID_ARCHIVED_LINK will be added the the
- * end of the chatlist.
- *
- * - To get a list of archived chats, use dc_get_chatlist() with the flag DC_GCL_ARCHIVED_ONLY.
- * - To find out the archived state of a given chat, use dc_chat_get_archived()
- * - Messages in archived chats are marked as being noticed, so they do not count as "fresh"
- * - Calling this function usually results in the event #DC_EVENT_MSGS_CHANGED
+ * Calling this function usually results in the event #DC_EVENT_MSGS_CHANGED
+ * See @ref DC_CHAT_VISIBILITY for detailed information about the visibilities.
  *
  * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
- * @param chat_id The ID of the chat to archive or unarchive.
- * @param archive 1=archive chat, 0=unarchive chat, all other values are reserved for future use
+ * @param chat_id The ID of the chat to change the visibility for.
+ * @param visibility one of @ref DC_CHAT_VISIBILITY
  * @return None.
  */
-void            dc_archive_chat              (dc_context_t* context, uint32_t chat_id, int archive);
+void            dc_set_chat_visibility       (dc_context_t* context, uint32_t chat_id, int visibility);
 
 
 /**
@@ -2846,21 +2845,14 @@ uint32_t        dc_chat_get_color            (const dc_chat_t* chat);
 
 
 /**
- * Get archived state.
- *
- * - 0 = normal chat, not archived, not sticky.
- * - 1 = chat archived
- * - 2 = chat sticky (reserved for future use, if you do not support this value, just treat the chat as a normal one)
- *
- * To archive or unarchive chats, use dc_archive_chat().
- * If chats are archived, this should be shown in the UI by a little icon or text,
- * eg. the search will also return archived chats.
+ * Get visibility of chat.
+ * See @ref DC_CHAT_VISIBILITY for detailed information about the visibilities.
  *
  * @memberof dc_chat_t
  * @param chat The chat object.
- * @return Archived state.
+ * @return One of @ref DC_CHAT_VISIBILITY
  */
-int             dc_chat_get_archived         (const dc_chat_t* chat);
+int             dc_chat_get_visibility       (const dc_chat_t* chat);
 
 
 /**
@@ -3773,7 +3765,7 @@ int             dc_contact_is_verified       (dc_contact_t* contact);
  *     accessor functions.  If no provider info is found, NULL will be
  *     returned.
  */
-dc_provider_t*  dc_provider_new_from_email            (const dc_context_t*, const char* email);
+dc_provider_t*  dc_provider_new_from_email            (const dc_context_t* context, const char* email);
 
 
 /**
@@ -4532,6 +4524,8 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_EVENT_RETURNS_STRING(e)   ((e)==DC_EVENT_GET_STRING) // not used anymore
 char*           dc_get_version_str           (void); // deprecated
 void            dc_array_add_id              (dc_array_t*, uint32_t); // deprecated
+#define dc_archive_chat(a,b,c)  dc_set_chat_visibility((a), (b), (c)? 1 : 0) // not used anymore
+#define dc_chat_get_archived(a) (dc_chat_get_visibility((a))==1? 1 : 0)      // not used anymore
 
 
 /*
@@ -4541,6 +4535,13 @@ void            dc_array_add_id              (dc_array_t*, uint32_t); // depreca
 #define DC_SHOW_EMAILS_ACCEPTED_CONTACTS 1
 #define DC_SHOW_EMAILS_ALL               2
 
+/*
+ * Values for dc_get|set_config("key_gen_type")
+ */
+#define DC_KEY_GEN_DEFAULT 0
+#define DC_KEY_GEN_RSA2048 1
+#define DC_KEY_GEN_ED25519 2
+
 
 /**
  * @defgroup DC_PROVIDER_STATUS DC_PROVIDER_STATUS
@@ -4597,6 +4598,48 @@ void            dc_array_add_id              (dc_array_t*, uint32_t); // depreca
  */
 
 
+/**
+ * @defgroup DC_CHAT_VISIBILITY DC_CHAT_VISIBILITY
+ *
+ * These constants describe the visibility of a chat.
+ * The chat visibiliry can be get using dc_chat_get_visibility()
+ * and set using dc_set_chat_visibility().
+ *
+ * @addtogroup DC_CHAT_VISIBILITY
+ * @{
+ */
+
+/**
+ * Chats with normal visibility are not archived and are shown below all pinned chats.
+ * Archived chats, that receive new messages automatically become normal chats.
+ */
+#define         DC_CHAT_VISIBILITY_NORMAL      0
+
+/**
+ * Archived chats are not included in the default chatlist returned by dc_get_chatlist().
+ * Instead, if there are _any_ archived chats, the pseudo-chat
+ * with the chat_id DC_CHAT_ID_ARCHIVED_LINK will be added the the end of the chatlist.
+ *
+ * The UI typically shows a little icon or chats beside archived chats in the chatlist,
+ * this is needed as eg. the search will also return archived chats.
+ *
+ * If archived chats receive new messages, they become normal chats again.
+ *
+ * To get a list of archived chats, use dc_get_chatlist() with the flag DC_GCL_ARCHIVED_ONLY.
+ */
+#define         DC_CHAT_VISIBILITY_ARCHIVED    1
+
+/**
+ * Pinned chats are included in the default chatlist. moreover,
+ * they are always the first items, whether they have fresh messages or not.
+ */
+#define         DC_CHAT_VISIBILITY_PINNED      2
+
+/**
+ * @}
+ */
+
+
 /*
  * TODO: Strings need some doumentation about used placeholders.
  *

deltachat-ffi/src/lib.rs

@@ -25,8 +25,7 @@ use std::time::{Duration, SystemTime};
 use libc::uintptr_t;
 use num_traits::{FromPrimitive, ToPrimitive};
 
-use deltachat::chat::ChatId;
-use deltachat::chat::MuteDuration;
+use deltachat::chat::{ChatId, ChatVisibility, MuteDuration};
 use deltachat::constants::DC_MSG_ID_LAST_SPECIAL;
 use deltachat::contact::Contact;
 use deltachat::context::Context;
@@ -489,9 +488,7 @@ pub unsafe extern "C" fn dc_configure(context: *mut dc_context_t) {
         return;
     }
     let ffi_context = &*context;
-    ffi_context
-        .with_inner(|ctx| configure::configure(ctx))
-        .unwrap_or(())
+    ffi_context.with_inner(|ctx| ctx.configure()).unwrap_or(())
 }
 
 #[no_mangle]
@@ -502,7 +499,7 @@ pub unsafe extern "C" fn dc_is_configured(context: *mut dc_context_t) -> libc::c
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| configure::dc_is_configured(ctx) as libc::c_int)
+        .with_inner(|ctx| ctx.is_configured() as libc::c_int)
         .unwrap_or(0)
 }
 
@@ -1189,28 +1186,32 @@ pub unsafe extern "C" fn dc_get_next_media(
 }
 
 #[no_mangle]
-pub unsafe extern "C" fn dc_archive_chat(
+pub unsafe extern "C" fn dc_set_chat_visibility(
     context: *mut dc_context_t,
     chat_id: u32,
     archive: libc::c_int,
 ) {
     if context.is_null() {
-        eprintln!("ignoring careless call to dc_archive_chat()");
+        eprintln!("ignoring careless call to dc_set_chat_visibility()");
         return;
     }
     let ffi_context = &*context;
-    let archive = if archive == 0 {
-        false
-    } else if archive == 1 {
-        true
-    } else {
-        return;
+    let visibility = match archive {
+        0 => ChatVisibility::Normal,
+        1 => ChatVisibility::Archived,
+        2 => ChatVisibility::Pinned,
+        _ => {
+            ffi_context.warning(
+                "ignoring careless call to dc_set_chat_visibility(): unknown archived state",
+            );
+            return;
+        }
     };
     ffi_context
         .with_inner(|ctx| {
             ChatId::new(chat_id)
-                .set_archived(ctx, archive)
-                .log_err(ffi_context, "Failed archive chat")
+                .set_visibility(ctx, visibility)
+                .log_err(ffi_context, "Failed setting chat visibility")
                 .unwrap_or(())
         })
         .unwrap_or(())
@@ -2454,13 +2455,17 @@ pub unsafe extern "C" fn dc_chat_get_color(chat: *mut dc_chat_t) -> u32 {
 }
 
 #[no_mangle]
-pub unsafe extern "C" fn dc_chat_get_archived(chat: *mut dc_chat_t) -> libc::c_int {
+pub unsafe extern "C" fn dc_chat_get_visibility(chat: *mut dc_chat_t) -> libc::c_int {
     if chat.is_null() {
-        eprintln!("ignoring careless call to dc_chat_get_archived()");
+        eprintln!("ignoring careless call to dc_chat_get_visibility()");
         return 0;
     }
     let ffi_chat = &*chat;
-    ffi_chat.chat.is_archived() as libc::c_int
+    match ffi_chat.chat.visibility {
+        ChatVisibility::Normal => 0,
+        ChatVisibility::Archived => 1,
+        ChatVisibility::Pinned => 2,
+    }
 }
 
 #[no_mangle]
@@ -3222,7 +3227,7 @@ pub unsafe extern "C" fn dc_lot_get_text1(lot: *mut dc_lot_t) -> *mut libc::c_ch
     }
 
     let lot = &*lot;
-    strdup_opt(lot.get_text1())
+    lot.get_text1().strdup()
 }
 
 #[no_mangle]
@@ -3233,7 +3238,7 @@ pub unsafe extern "C" fn dc_lot_get_text2(lot: *mut dc_lot_t) -> *mut libc::c_ch
     }
 
     let lot = &*lot;
-    strdup_opt(lot.get_text2())
+    lot.get_text2().strdup()
 }
 
 #[no_mangle]
@@ -3318,13 +3323,6 @@ impl<T: Default, E: std::fmt::Display> ResultExt<T, E> for Result<T, E> {
     }
 }
 
-unsafe fn strdup_opt(s: Option<impl AsRef<str>>) -> *mut libc::c_char {
-    match s {
-        Some(s) => s.as_ref().strdup(),
-        None => ptr::null_mut(),
-    }
-}
-
 trait ResultNullableExt<T> {
     fn into_raw(self) -> *mut T;
 }

deltachat-ffi/src/string.rs

@@ -1,5 +1,6 @@
 use failure::Fail;
 use std::ffi::{CStr, CString};
+use std::ptr;
 
 /// Duplicates a string
 ///
@@ -144,7 +145,7 @@ pub trait CStringExt {
 
 impl CStringExt for CString {}
 
-/// Convenience methods to make transitioning from raw C strings easier.
+/// Convenience methods to turn strings into C strings.
 ///
 /// To interact with (legacy) C APIs we often need to convert from
 /// Rust strings to raw C strings.  This can be clumsy to do correctly
@@ -174,6 +175,35 @@ impl<T: AsRef<str>> StrExt for T {
     }
 }
 
+/// Convenience methods to turn optional strings into C strings.
+///
+/// This is the same as the [StrExt] trait but a different trait name
+/// to work around the type system not allowing to implement [StrExt]
+/// for `Option<impl StrExt>` When we already have an [StrExt] impl
+/// for `AsRef<&str>`.
+///
+/// When the [Option] is [Option::Some] this behaves just like
+/// [StrExt::strdup], when it is [Option::None] a null pointer is
+/// returned.
+pub trait OptStrExt {
+    /// Allocate a new raw C `*char` version of this string, or NULL.
+    ///
+    /// See [StrExt::strdup] for details.
+    unsafe fn strdup(&self) -> *mut libc::c_char;
+}
+
+impl<T: AsRef<str>> OptStrExt for Option<T> {
+    unsafe fn strdup(&self) -> *mut libc::c_char {
+        match self {
+            Some(s) => {
+                let tmp = CString::yolo(s.as_ref());
+                dc_strdup(tmp.as_ptr())
+            }
+            None => ptr::null_mut(),
+        }
+    }
+}
+
 pub fn to_string_lossy(s: *const libc::c_char) -> String {
     if s.is_null() {
         return "".into();
@@ -347,4 +377,19 @@ mod tests {
             assert_eq!(cmp, 0);
         }
     }
+
+    #[test]
+    fn test_strdup_opt_string() {
+        unsafe {
+            let s = Some("hello");
+            let c = s.strdup();
+            let cmp = strcmp(c, b"hello\x00" as *const u8 as *const libc::c_char);
+            free(c as *mut libc::c_void);
+            assert_eq!(cmp, 0);
+
+            let s: Option<&str> = None;
+            let c = s.strdup();
+            assert_eq!(c, ptr::null_mut());
+        }
+    }
 }

examples/repl/cmdline.rs

@@ -1,7 +1,7 @@
 use std::path::Path;
 use std::str::FromStr;
 
-use deltachat::chat::{self, Chat, ChatId};
+use deltachat::chat::{self, Chat, ChatId, ChatVisibility};
 use deltachat::chatlist::*;
 use deltachat::constants::*;
 use deltachat::contact::*;
@@ -371,6 +371,8 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
                  listmedia\n\
                  archive <chat-id>\n\
                  unarchive <chat-id>\n\
+                 pin <chat-id>\n\
+                 unpin <chat-id>\n\
                  delchat <chat-id>\n\
                  ===========================Message commands==\n\
                  listmsgs <query>\n\
@@ -511,14 +513,19 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
                 for i in (0..cnt).rev() {
                     let chat = Chat::load_from_db(context, chatlist.get_chat_id(i))?;
                     println!(
-                        "{}#{}: {} [{} fresh]",
+                        "{}#{}: {} [{} fresh] {}",
                         chat_prefix(&chat),
                         chat.get_id(),
                         chat.get_name(),
                         chat.get_id().get_fresh_msg_cnt(context),
+                        match chat.visibility {
+                            ChatVisibility::Normal => "",
+                            ChatVisibility::Archived => "📦",
+                            ChatVisibility::Pinned => "📌",
+                        },
                     );
                     let lot = chatlist.get_summary(context, i, Some(&chat));
-                    let statestr = if chat.is_archived() {
+                    let statestr = if chat.visibility == ChatVisibility::Archived {
                         " [Archived]"
                     } else {
                         match lot.get_state() {
@@ -842,10 +849,18 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
             }
             print!("\n");
         }
-        "archive" | "unarchive" => {
+        "archive" | "unarchive" | "pin" | "unpin" => {
             ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
             let chat_id = ChatId::new(arg1.parse()?);
-            chat_id.set_archived(context, arg0 == "archive")?;
+            chat_id.set_visibility(
+                context,
+                match arg0 {
+                    "archive" => ChatVisibility::Archived,
+                    "unarchive" | "unpin" => ChatVisibility::Normal,
+                    "pin" => ChatVisibility::Pinned,
+                    _ => panic!("Unexpected command (This should never happen)"),
+                },
+            )?;
         }
         "delchat" => {
             ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
@@ -969,7 +984,10 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
         }
         "setqr" => {
             ensure!(!arg1.is_empty(), "Argument <qr-content> missing.");
-            set_config_from_qr(context, arg1);
+            match set_config_from_qr(context, arg1) {
+                Ok(()) => println!("Config set from QR code, you can now call 'configure'"),
+                Err(err) => println!("Cannot set config from QR code: {:?}", err),
+            }
         }
         "providerinfo" => {
             ensure!(!arg1.is_empty(), "Argument <addr> missing.");

examples/repl/main.rs

@@ -22,7 +22,6 @@ use std::sync::{Arc, Mutex, RwLock};
 
 use deltachat::chat::ChatId;
 use deltachat::config;
-use deltachat::configure::*;
 use deltachat::context::*;
 use deltachat::job::*;
 use deltachat::oauth2::*;
@@ -263,7 +262,7 @@ const DB_COMMANDS: [&str; 11] = [
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 24] = [
+const CHAT_COMMANDS: [&str; 26] = [
     "listchats",
     "listarchived",
     "chat",
@@ -287,6 +286,8 @@ const CHAT_COMMANDS: [&str; 24] = [
     "listmedia",
     "archive",
     "unarchive",
+    "pin",
+    "unpin",
     "delchat",
 ];
 const MESSAGE_COMMANDS: [&str; 8] = [
@@ -461,7 +462,7 @@ fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, failu
         }
         "configure" => {
             start_threads(ctx.clone());
-            configure(&ctx.read().unwrap());
+            ctx.read().unwrap().configure();
         }
         "oauth2" => {
             if let Some(addr) = ctx.read().unwrap().get_config(config::Config::Addr) {

examples/simple.rs

@@ -7,7 +7,6 @@ use tempfile::tempdir;
 use deltachat::chat;
 use deltachat::chatlist::*;
 use deltachat::config;
-use deltachat::configure::*;
 use deltachat::contact::*;
 use deltachat::context::*;
 use deltachat::job::{
@@ -77,7 +76,7 @@ fn main() {
     ctx.set_config(config::Config::Addr, Some("d@testrun.org"))
         .unwrap();
     ctx.set_config(config::Config::MailPw, Some(&pw)).unwrap();
-    configure(&ctx);
+    ctx.configure();
 
     thread::sleep(duration);
 

python/CHANGELOG

@@ -1,3 +1,10 @@
+0.800.0
+-------
+
+- use latest core 1.25.0 
+
+- refine tests and some internal changes to core bindings
+
 0.700.0
 ---------
 

python/doc/_static/custom.css

@@ -15,3 +15,7 @@ div.globaltoc {
 img.logo {
     height: 120px;
 }
+
+div.footer {
+    display: none;
+}

python/doc/conf.py

@@ -55,7 +55,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'deltachat'
-copyright = u'2018, holger krekel and contributors'
+copyright = u'2020, holger krekel and contributors'
 
 
 # The language for content autogenerated by Sphinx. Refer to documentation

python/doc/index.rst

@@ -1,15 +1,14 @@
 deltachat python bindings
 =========================
 
-The ``deltachat`` Python package provides two bindings for the core Rust-library
-of the https://delta.chat messaging ecosystem:
+The ``deltachat`` Python package provides two layers of bindings for the
+core Rust-library of the https://delta.chat messaging ecosystem:
 
 - :doc:`api` is a high level interface to deltachat-core which aims
   to be memory safe and thoroughly tested through continous tox/pytest runs.
 
-- :doc:`capi` is a lowlevel CFFI-binding to the previous
-  `deltachat-core C-API <https://c.delta.chat>`_ (so far the Rust library
-  replicates exactly the same C-level API).
+- :doc:`lapi` is a lowlevel CFFI-binding to the `Rust Core
+  <https://github.com/deltachat/deltachat-core-rust>`_.
 
 
 
@@ -28,7 +27,6 @@ getting started
    links
    changelog
    api
-   capi
    lapi
 
 ..

python/setup.py

@@ -18,7 +18,7 @@ def main():
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'six'],
+        install_requires=['cffi>=1.0.0', 'pluggy'],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],

python/src/deltachat/account.py

@@ -4,13 +4,9 @@ from __future__ import print_function
 import atexit
 import threading
 import os
-import re
 import time
 from array import array
-try:
-    from queue import Queue, Empty
-except ImportError:
-    from Queue import Queue, Empty
+from queue import Queue
 
 import deltachat
 from . import const
@@ -19,6 +15,8 @@ from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array, DCLot
 from .chat import Chat
 from .message import Message
 from .contact import Contact
+from .eventlogger import EventLogger
+from .hookspec import get_plugin_manager, hookimpl
 
 
 class Account(object):
@@ -26,14 +24,13 @@ class Account(object):
     by the underlying deltachat core library.  All public Account methods are
     meant to be memory-safe and return memory-safe objects.
     """
-    def __init__(self, db_path, logid=None, eventlogging=True, os_name=None, debug=True):
+    def __init__(self, db_path, logid=None, os_name=None, debug=True):
         """ initialize account object.
 
         :param db_path: a path to the account database. The database
                         will be created if it doesn't exist.
         :param logid: an optional logging prefix that should be used with
                       the default internal logging.
-        :param eventlogging: if False no eventlogging and no context callback will be configured
         :param os_name: this will be put to the X-Mailer header in outgoing messages
         :param debug: turn on debug logging for events.
         """
@@ -41,19 +38,26 @@ class Account(object):
             lib.dc_context_new(lib.py_dc_callback, ffi.NULL, as_dc_charpointer(os_name)),
             _destroy_dc_context,
         )
-        if eventlogging:
-            self._evlogger = EventLogger(self._dc_context, logid, debug)
-            deltachat.set_context_callback(self._dc_context, self._process_event)
-            self._threads = IOThreads(self._dc_context, self._evlogger._log_event)
-        else:
-            self._threads = IOThreads(self._dc_context)
+        self._evlogger = EventLogger(self, logid, debug)
+        self._threads = IOThreads(self._dc_context, self._evlogger._log_event)
 
+        # register event call back and initialize plugin system
+        def _ll_event(ctx, evt_name, data1, data2):
+            assert ctx == self._dc_context
+            self.pluggy.hook.process_low_level_event(
+                account=self, event_name=evt_name, data1=data1, data2=data2
+            )
+
+        self.pluggy = get_plugin_manager()
+        self.pluggy.register(self._evlogger)
+        deltachat.set_context_callback(self._dc_context, _ll_event)
+
+        # open database
         if hasattr(db_path, "encode"):
             db_path = db_path.encode("utf8")
         if not lib.dc_open(self._dc_context, db_path, ffi.NULL):
             raise ValueError("Could not dc_open: {}".format(db_path))
         self._configkeys = self.get_config("sys.config_keys").split()
-        self._imex_events = Queue()
         atexit.register(self.shutdown)
 
     # def __del__(self):
@@ -177,11 +181,6 @@ class Account(object):
             raise ValueError("no flags set")
         lib.dc_empty_server(self._dc_context, flags)
 
-    def get_infostring(self):
-        """ return info of the configured account. """
-        self.check_is_configured()
-        return from_dc_charpointer(lib.dc_get_info(self._dc_context))
-
     def get_latest_backupfile(self, backupdir):
         """ return the latest backup file in a given directory.
         """
@@ -382,27 +381,12 @@ class Account(object):
             raise RuntimeError("found more than one new file")
         return export_files[0]
 
-    def _imex_events_clear(self):
-        try:
-            while True:
-                self._imex_events.get_nowait()
-        except Empty:
-            pass
-
     def _export(self, path, imex_cmd):
-        self._imex_events_clear()
-        lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
-        if not self._threads.is_started():
-            lib.dc_perform_imap_jobs(self._dc_context)
-        files_written = []
-        while True:
-            ev = self._imex_events.get()
-            if isinstance(ev, str):
-                files_written.append(ev)
-            elif isinstance(ev, bool):
-                if not ev:
-                    raise ValueError("export failed, exp-files: {}".format(files_written))
-                return files_written
+        with ImexTracker(self) as imex_tracker:
+            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
+            if not self._threads.is_started():
+                lib.dc_perform_imap_jobs(self._dc_context)
+            return imex_tracker.wait_finish()
 
     def import_self_keys(self, path):
         """ Import private keys found in the `path` directory.
@@ -420,12 +404,11 @@ class Account(object):
         self._import(path, imex_cmd=12)
 
     def _import(self, path, imex_cmd):
-        self._imex_events_clear()
-        lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
-        if not self._threads.is_started():
-            lib.dc_perform_imap_jobs(self._dc_context)
-        if not self._imex_events.get():
-            raise ValueError("import from path '{}' failed".format(path))
+        with ImexTracker(self) as imex_tracker:
+            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
+            if not self._threads.is_started():
+                lib.dc_perform_imap_jobs(self._dc_context)
+            imex_tracker.wait_finish()
 
     def initiate_key_transfer(self):
         """return setup code after a Autocrypt setup message
@@ -528,24 +511,7 @@ class Account(object):
             deltachat.clear_context_callback(self._dc_context)
             del self._dc_context
             atexit.unregister(self.shutdown)
-
-    def _process_event(self, ctx, evt_name, data1, data2):
-        assert ctx == self._dc_context
-        if hasattr(self, "_evlogger"):
-            self._evlogger(evt_name, data1, data2)
-            method = getattr(self, "on_" + evt_name.lower(), None)
-            if method is not None:
-                method(data1, data2)
-        return 0
-
-    def on_dc_event_imex_progress(self, data1, data2):
-        if data1 == 1000:
-            self._imex_events.put(True)
-        elif data1 == 0:
-            self._imex_events.put(False)
-
-    def on_dc_event_imex_file_written(self, data1, data2):
-        self._imex_events.put(data1)
+        self.pluggy.unregister(self._evlogger)
 
     def set_location(self, latitude=0.0, longitude=0.0, accuracy=0.0):
         """set a new location. It effects all chats where we currently
@@ -562,6 +528,41 @@ class Account(object):
             raise ValueError("no chat is streaming locations")
 
 
+class ImexTracker:
+    def __init__(self, account):
+        self._imex_events = Queue()
+        self.account = account
+
+    def __enter__(self):
+        self.account.pluggy.register(self)
+        return self
+
+    def __exit__(self, *args):
+        self.account.pluggy.unregister(self)
+
+    @hookimpl
+    def process_low_level_event(self, account, event_name, data1, data2):
+        # there could be multiple accounts instantiated
+        if self.account is not account:
+            return
+        if event_name == "DC_EVENT_IMEX_PROGRESS":
+            self._imex_events.put(data1)
+        elif event_name == "DC_EVENT_IMEX_FILE_WRITTEN":
+            self._imex_events.put(data1)
+
+    def wait_finish(self, progress_timeout=60):
+        """ Return list of written files, raise ValueError if ExportFailed. """
+        files_written = []
+        while True:
+            ev = self._imex_events.get(timeout=progress_timeout)
+            if isinstance(ev, str):
+                files_written.append(ev)
+            elif ev == 0:
+                raise ValueError("export failed, exp-files: {}".format(files_written))
+            elif ev == 1000:
+                return files_written
+
+
 class IOThreads:
     def __init__(self, dc_context, log_event=lambda *args: None):
         self._dc_context = dc_context
@@ -642,80 +643,6 @@ class IOThreads:
         self._log_event("py-bindings-info", 0, "SMTP THREAD FINISHED")
 
 
-class EventLogger:
-    _loglock = threading.RLock()
-
-    def __init__(self, dc_context, logid=None, debug=True):
-        self._dc_context = dc_context
-        self._event_queue = Queue()
-        self._debug = debug
-        if logid is None:
-            logid = str(self._dc_context).strip(">").split()[-1]
-        self.logid = logid
-        self._timeout = None
-        self.init_time = time.time()
-
-    def __call__(self, evt_name, data1, data2):
-        self._log_event(evt_name, data1, data2)
-        self._event_queue.put((evt_name, data1, data2))
-
-    def set_timeout(self, timeout):
-        self._timeout = timeout
-
-    def consume_events(self, check_error=True):
-        while not self._event_queue.empty():
-            self.get()
-
-    def get(self, timeout=None, check_error=True):
-        timeout = timeout or self._timeout
-        ev = self._event_queue.get(timeout=timeout)
-        if check_error and ev[0] == "DC_EVENT_ERROR":
-            raise ValueError("{}({!r},{!r})".format(*ev))
-        return ev
-
-    def ensure_event_not_queued(self, event_name_regex):
-        __tracebackhide__ = True
-        rex = re.compile("(?:{}).*".format(event_name_regex))
-        while 1:
-            try:
-                ev = self._event_queue.get(False)
-            except Empty:
-                break
-            else:
-                assert not rex.match(ev[0]), "event found {}".format(ev)
-
-    def get_matching(self, event_name_regex, check_error=True, timeout=None):
-        self._log("-- waiting for event with regex: {} --".format(event_name_regex))
-        rex = re.compile("(?:{}).*".format(event_name_regex))
-        while 1:
-            ev = self.get(timeout=timeout, check_error=check_error)
-            if rex.match(ev[0]):
-                return ev
-
-    def get_info_matching(self, regex):
-        rex = re.compile("(?:{}).*".format(regex))
-        while 1:
-            ev = self.get_matching("DC_EVENT_INFO")
-            if rex.match(ev[2]):
-                return ev
-
-    def _log_event(self, evt_name, data1, data2):
-        # don't show events that are anyway empty impls now
-        if evt_name == "DC_EVENT_GET_STRING":
-            return
-        if self._debug:
-            evpart = "{}({!r},{!r})".format(evt_name, data1, data2)
-            self._log(evpart)
-
-    def _log(self, msg):
-        t = threading.currentThread()
-        tname = getattr(t, "name", t)
-        if tname == "MainThread":
-            tname = "MAIN"
-        with self._loglock:
-            print("{:2.2f} [{}-{}] {}".format(time.time() - self.init_time, tname, self.logid, msg))
-
-
 def _destroy_dc_context(dc_context, dc_context_unref=lib.dc_context_unref):
     # destructor for dc_context
     dc_context_unref(dc_context)

python/src/deltachat/chat.py

@@ -423,7 +423,7 @@ class Chat(object):
         """return True if this chat is archived.
         :returns: True if archived.
         """
-        return lib.dc_chat_get_archived(self._dc_chat)
+        return lib.dc_chat_get_visibility(self._dc_chat) == const.DC_CHAT_VISIBILITY_ARCHIVED
 
     def enable_sending_locations(self, seconds):
         """enable sending locations for this chat.

python/src/deltachat/const.py

@@ -18,6 +18,7 @@ DC_QR_ASK_VERIFYGROUP = 202
 DC_QR_FPR_OK = 210
 DC_QR_FPR_MISMATCH = 220
 DC_QR_FPR_WITHOUT_ADDR = 230
+DC_QR_ACCOUNT = 250
 DC_QR_ADDR = 320
 DC_QR_TEXT = 330
 DC_QR_URL = 332
@@ -102,9 +103,15 @@ DC_EVENT_FILE_COPIED = 2055
 DC_EVENT_IS_OFFLINE = 2081
 DC_EVENT_GET_STRING = 2091
 DC_STR_SELFNOTINGRP = 21
+DC_KEY_GEN_DEFAULT = 0
+DC_KEY_GEN_RSA2048 = 1
+DC_KEY_GEN_ED25519 = 2
 DC_PROVIDER_STATUS_OK = 1
 DC_PROVIDER_STATUS_PREPARATION = 2
 DC_PROVIDER_STATUS_BROKEN = 3
+DC_CHAT_VISIBILITY_NORMAL = 0
+DC_CHAT_VISIBILITY_ARCHIVED = 1
+DC_CHAT_VISIBILITY_PINNED = 2
 DC_STR_NOMESSAGES = 1
 DC_STR_SELF = 2
 DC_STR_DRAFT = 3
@@ -157,7 +164,7 @@ DC_STR_COUNT = 68
 
 def read_event_defines(f):
     rex = re.compile(r'#define\s+((?:DC_EVENT|DC_QR|DC_MSG|DC_LP|DC_EMPTY|DC_CERTCK|DC_STATE|DC_STR|'
-                     r'DC_CONTACT_ID|DC_GCL|DC_CHAT|DC_PROVIDER)_\S+)\s+([x\d]+).*')
+                     r'DC_CONTACT_ID|DC_GCL|DC_CHAT|DC_PROVIDER|DC_KEY_GEN)_\S+)\s+([x\d]+).*')
     for line in f:
         m = rex.match(line)
         if m:

python/src/deltachat/eventlogger.py

@@ -0,0 +1,81 @@
+import threading
+import re
+import time
+from queue import Queue, Empty
+from .hookspec import hookimpl
+
+
+class EventLogger:
+    _loglock = threading.RLock()
+
+    def __init__(self, account, logid=None, debug=True):
+        self.account = account
+        self._event_queue = Queue()
+        self._debug = debug
+        if logid is None:
+            logid = str(self.account._dc_context).strip(">").split()[-1]
+        self.logid = logid
+        self._timeout = None
+        self.init_time = time.time()
+
+    @hookimpl
+    def process_low_level_event(self, account, event_name, data1, data2):
+        if self.account == account:
+            self._log_event(event_name, data1, data2)
+            self._event_queue.put((event_name, data1, data2))
+
+    def set_timeout(self, timeout):
+        self._timeout = timeout
+
+    def consume_events(self, check_error=True):
+        while not self._event_queue.empty():
+            self.get(check_error=check_error)
+
+    def get(self, timeout=None, check_error=True):
+        timeout = timeout or self._timeout
+        ev = self._event_queue.get(timeout=timeout)
+        if check_error and ev[0] == "DC_EVENT_ERROR":
+            raise ValueError("{}({!r},{!r})".format(*ev))
+        return ev
+
+    def ensure_event_not_queued(self, event_name_regex):
+        __tracebackhide__ = True
+        rex = re.compile("(?:{}).*".format(event_name_regex))
+        while 1:
+            try:
+                ev = self._event_queue.get(False)
+            except Empty:
+                break
+            else:
+                assert not rex.match(ev[0]), "event found {}".format(ev)
+
+    def get_matching(self, event_name_regex, check_error=True, timeout=None):
+        self._log("-- waiting for event with regex: {} --".format(event_name_regex))
+        rex = re.compile("(?:{}).*".format(event_name_regex))
+        while 1:
+            ev = self.get(timeout=timeout, check_error=check_error)
+            if rex.match(ev[0]):
+                return ev
+
+    def get_info_matching(self, regex):
+        rex = re.compile("(?:{}).*".format(regex))
+        while 1:
+            ev = self.get_matching("DC_EVENT_INFO")
+            if rex.match(ev[2]):
+                return ev
+
+    def _log_event(self, evt_name, data1, data2):
+        # don't show events that are anyway empty impls now
+        if evt_name == "DC_EVENT_GET_STRING":
+            return
+        if self._debug:
+            evpart = "{}({!r},{!r})".format(evt_name, data1, data2)
+            self._log(evpart)
+
+    def _log(self, msg):
+        t = threading.currentThread()
+        tname = getattr(t, "name", t)
+        if tname == "MainThread":
+            tname = "MAIN"
+        with self._loglock:
+            print("{:2.2f} [{}-{}] {}".format(time.time() - self.init_time, tname, self.logid, msg))

python/src/deltachat/hookspec.py

@@ -0,0 +1,25 @@
+""" Hooks for python bindings """
+
+import pluggy
+
+name = "deltachat"
+
+hookspec = pluggy.HookspecMarker(name)
+hookimpl = pluggy.HookimplMarker(name)
+_plugin_manager = None
+
+
+def get_plugin_manager():
+    global _plugin_manager
+    if _plugin_manager is None:
+        _plugin_manager = pluggy.PluginManager(name)
+        _plugin_manager.add_hookspecs(DeltaChatHookSpecs)
+    return _plugin_manager
+
+
+class DeltaChatHookSpecs:
+    """ Plugin Hook specifications for Python bindings to Delta Chat CFFI. """
+
+    @hookspec
+    def process_low_level_event(self, account, event_name, data1, data2):
+        """ process a CFFI low level events for a given account. """

python/tests/conftest.py

@@ -1,5 +1,6 @@
 from __future__ import print_function
 import os
+import sys
 import py
 import pytest
 import requests
@@ -7,6 +8,7 @@ import time
 from deltachat import Account
 from deltachat import const
 from deltachat.capi import lib
+from _pytest.monkeypatch import MonkeyPatch
 import tempfile
 
 
@@ -43,11 +45,14 @@ def pytest_report_header(config, startdir):
     summary = []
 
     t = tempfile.mktemp()
+    m = MonkeyPatch()
     try:
-        ac = Account(t, eventlogging=False)
+        m.setattr(sys.stdout, "write", lambda x: len(x))
+        ac = Account(t)
         info = ac.get_info()
         ac.shutdown()
     finally:
+        m.undo()
         os.remove(t)
     summary.extend(['Deltachat core={} sqlite={}'.format(
          info['deltachat_core_version'],
@@ -213,9 +218,10 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir):
             return ac, dict(configdict)
 
         def get_online_configuring_account(self, mvbox=False, sentbox=False,
-                                           pre_generated_key=True):
+                                           pre_generated_key=True, config={}):
             ac, configdict = self.get_online_config(
                 pre_generated_key=pre_generated_key)
+            configdict.update(config)
             ac.configure(**configdict)
             ac.start_threads(mvbox=mvbox, sentbox=sentbox)
             return ac

python/tests/test_account.py

@@ -65,11 +65,6 @@ class TestOfflineAccountBasic:
         with pytest.raises(ValueError):
             ac1.get_self_contact()
 
-    def test_get_info(self, acfactory):
-        ac1 = acfactory.get_configured_offline_account()
-        out = ac1.get_infostring()
-        assert "number_of_chats=0" in out
-
     def test_selfcontact_configured(self, acfactory):
         ac1 = acfactory.get_configured_offline_account()
         me = ac1.get_self_contact()
@@ -428,12 +423,6 @@ class TestOfflineChat:
 
 
 class TestOnlineAccount:
-    @pytest.mark.ignored
-    def test_configure_generate_key(self, acfactory):
-        # A slow test which will generate a new key.
-        ac = acfactory.get_one_online_account(pre_generated_key=False)
-        ac.check_is_configured()
-
     def get_chat(self, ac1, ac2, both_created=False):
         c2 = ac1.create_contact(email=ac2.get_config("addr"))
         chat = ac1.create_chat_by_contact(c2)
@@ -442,6 +431,45 @@ class TestOnlineAccount:
             ac2.create_chat_by_contact(ac2.create_contact(email=ac1.get_config("addr")))
         return chat
 
+    @pytest.mark.ignored
+    def test_configure_generate_key(self, acfactory, lp):
+        # A slow test which will generate new keys.
+        ac1 = acfactory.get_online_configuring_account(
+            pre_generated_key=False,
+            config={"key_gen_type": str(const.DC_KEY_GEN_RSA2048)}
+        )
+        ac2 = acfactory.get_online_configuring_account(
+            pre_generated_key=False,
+            config={"key_gen_type": str(const.DC_KEY_GEN_ED25519)}
+        )
+        wait_configuration_progress(ac1, 1000)
+        wait_configuration_progress(ac2, 1000)
+        chat = self.get_chat(ac1, ac2, both_created=True)
+
+        lp.sec("ac1: send unencrypted message to ac2")
+        chat.send_text("message1")
+        lp.sec("ac2: waiting for message from ac1")
+        ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG")
+        msg_in = ac2.get_message_by_id(ev[2])
+        assert msg_in.text == "message1"
+        assert not msg_in.is_encrypted()
+
+        lp.sec("ac2: send encrypted message to ac1")
+        msg_in.chat.send_text("message2")
+        lp.sec("ac1: waiting for message from ac2")
+        ev = ac1._evlogger.get_matching("DC_EVENT_INCOMING_MSG")
+        msg2_in = ac1.get_message_by_id(ev[2])
+        assert msg2_in.text == "message2"
+        assert msg2_in.is_encrypted()
+
+        lp.sec("ac1: send encrypted message to ac2")
+        msg2_in.chat.send_text("message3")
+        lp.sec("ac2: waiting for message from ac1")
+        ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG")
+        msg3_in = ac1.get_message_by_id(ev[2])
+        assert msg3_in.text == "message3"
+        assert msg3_in.is_encrypted()
+
     def test_configure_canceled(self, acfactory):
         ac1 = acfactory.get_online_configuring_account()
         wait_configuration_progress(ac1, 200)

python/tests/test_lowlevel.py

@@ -2,7 +2,6 @@ from __future__ import print_function
 from deltachat import capi, cutil, const, set_context_callback, clear_context_callback
 from deltachat.capi import ffi
 from deltachat.capi import lib
-from deltachat.account import EventLogger
 
 
 def test_empty_context():
@@ -18,21 +17,13 @@ def test_callback_None2int():
 
 
 def test_dc_close_events(tmpdir):
-    ctx = ffi.gc(
-        capi.lib.dc_context_new(capi.lib.py_dc_callback, ffi.NULL, ffi.NULL),
-        lib.dc_context_unref,
-    )
-    evlog = EventLogger(ctx)
-    evlog.set_timeout(5)
-    set_context_callback(
-        ctx,
-        lambda ctx, evt_name, data1, data2: evlog(evt_name, data1, data2)
-    )
+    from deltachat.account import Account
     p = tmpdir.join("hello.db")
-    lib.dc_open(ctx, p.strpath.encode("ascii"), ffi.NULL)
-    capi.lib.dc_close(ctx)
+    ac1 = Account(p.strpath)
+    ac1.shutdown()
 
     def find(info_string):
+        evlog = ac1._evlogger
         while 1:
             ev = evlog.get_matching("DC_EVENT_INFO", check_error=False)
             data2 = ev[2]

scripts/proxy.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+# Examples:
+#
+# Original server that doesn't use SSL:
+# ./proxy.py 8080 imap.nauta.cu 143
+# ./proxy.py 8081 smtp.nauta.cu 25
+#
+# Original server that uses SSL:
+# ./proxy.py 8080 testrun.org 993 --ssl
+# ./proxy.py 8081 testrun.org 465 --ssl
+
+from datetime import datetime
+import argparse
+import selectors
+import ssl
+import socket
+import socketserver
+
+
+class Proxy(socketserver.ThreadingTCPServer):
+    allow_reuse_address = True
+
+    def __init__(self, proxy_host, proxy_port, real_host, real_port, use_ssl):
+        self.real_host = real_host
+        self.real_port = real_port
+        self.use_ssl = use_ssl
+        super().__init__((proxy_host, proxy_port), RequestHandler)
+
+
+class RequestHandler(socketserver.BaseRequestHandler):
+
+    def handle(self):
+        print('{} - {} CONNECTED.'.format(datetime.now(), self.client_address))
+
+        total = 0
+        real_server = (self.server.real_host, self.server.real_port)
+        with socket.create_connection(real_server) as sock:
+            if self.server.use_ssl:
+                context = ssl.create_default_context()
+                sock = context.wrap_socket(
+                    sock, server_hostname=real_server[0])
+
+            forward = {self.request: sock, sock: self.request}
+
+            sel = selectors.DefaultSelector()
+            sel.register(self.request, selectors.EVENT_READ,
+                         self.client_address)
+            sel.register(sock, selectors.EVENT_READ, real_server)
+
+            active = True
+            while active:
+                events = sel.select()
+                for key, mask in events:
+                    print('\n{} - {} wrote:'.format(datetime.now(), key.data))
+                    data = key.fileobj.recv(1024)
+                    received = len(data)
+                    total += received
+                    print(data)
+                    print('{} Bytes\nTotal: {} Bytes'.format(received, total))
+                    if data:
+                        forward[key.fileobj].sendall(data)
+                    else:
+                        print('\nCLOSING CONNECTION.\n\n')
+                        forward[key.fileobj].close()
+                        key.fileobj.close()
+                        active = False
+
+
+if __name__ == '__main__':
+    p = argparse.ArgumentParser(description='Simple Python Proxy')
+    p.add_argument(
+        "proxy_port", help="the port where the proxy will listen", type=int)
+    p.add_argument('host', help="the real host")
+    p.add_argument('port', help="the port of the real host", type=int)
+    p.add_argument("--ssl", help="use ssl to connect to the real host",
+                   action="store_true")
+    args = p.parse_args()
+
+    with Proxy('', args.proxy_port, args.host, args.port, args.ssl) as proxy:
+        proxy.serve_forever()

set_core_version.py

@@ -35,7 +35,7 @@ if __name__ == "__main__":
     if len(sys.argv) < 2:
         for x in ("Cargo.toml", "deltachat-ffi/Cargo.toml"):
             print("{}: {}".format(x, read_toml_version(x)))
-        raise SystemExit("need argument: new version, example 1.0.0-beta.27")
+        raise SystemExit("need argument: new version, example: 1.25.0")
     newversion = sys.argv[1]
     if newversion.count(".") < 2:
         raise SystemExit("need at least two dots in version")
@@ -45,7 +45,7 @@ if __name__ == "__main__":
     assert core_toml == ffi_toml, (core_toml, ffi_toml)
 
     for line in open("CHANGELOG.md"):
-        ## 1.0.0-beta5
+        ## 1.25.0
         if line.startswith("## "):
             if line[2:].strip().startswith(newversion):
                 break

spec.md

@@ -1,6 +1,6 @@
 # Chat-over-Email specification
 
-Version 0.20.0
+Version 0.30.0
 
 This document describes how emails can be used
 to implement typical messenger functions
@@ -17,6 +17,9 @@ while staying compatible to existing MUAs.
     - [Change group name](#change-group-name)
     - [Set group image](#set-group-image)
 - [Set profile image](#set-profile-image)
+- [Locations](#locations)
+    - [User locations](#user-locations)
+    - [Points of interest](#points-of-interest)
 - [Miscellaneous](#miscellaneous)
 
 
@@ -29,8 +32,7 @@ Messages SHOULD be encrypted by the
 Meta data (at least the subject and all chat-headers) SHOULD be encrypted
 by the [Memoryhole](https://github.com/autocrypt/memoryhole) standard.
 If Memoryhole is not used,
-the subject of encrypted messages SHOULD be replaced by the string
-`Chat: Encrypted message` where the part after the colon MAY be localized.
+the subject of encrypted messages SHOULD be replaced by the string `...`.
 
 
 # Outgoing messages
@@ -113,6 +115,7 @@ but MUAs typically expose the sender in the UI.
 Groups are chats with usually more than one recipient,
 each defined by an email-address.
 The sender plus the recipients are the group members.
+All group members form the member list.
 
 To allow different groups with the same members,
 groups are identified by a group-id.
@@ -135,8 +138,7 @@ The group-name MUST be written to `Chat-Group-Name` header
 to join a group chat on a second device any time).
 
 The `Subject` header of outgoing group messages
-SHOULD start with the characters `Chat:`
-followed by the group-name and a colon followed by an excerpt of the message.
+SHOULD be set to the group-name.
 
 To identify the group-id on replies from normal MUAs,
 the group-id MUST also be added to the message-id of outgoing messages.
@@ -177,12 +179,22 @@ to a normal single-user chat with the email-address given in `From`.
 
 ## Add and remove members
 
-Messenger clients MUST construct the member list
-from the `From`/`To` headers only on the first group message
-or if they see a `Chat-Group-Member-Added`
-or `Chat-Group-Member-Removed` action header.
-Both headers MUST have the email-address
-of the added or removed member as the value.
+Messenger clients MUST init the member list
+from the `From`/`To` headers on the first group message.
+
+When a member is added later,
+a `Chat-Group-Member-Added` action header must be sent
+with the value set to the email-address of the added member.
+When receiving a `Chat-Group-Member-Added` header, however,
+_all missing_ members  the `From`/`To` headers has to be added.
+This is to mitigate problems when receiving messages
+in different orders, esp. on creating new groups.
+
+To remove a member, a `Chat-Group-Member-Removed` header must be sent
+with the value set to the email-address of the member to remove.
+When receiving a `Chat-Group-Member-Removed` header,
+only exaxtly the given member has to be removed from the member list.
+
 Messenger clients MUST NOT construct the member list
 on other group messages
 (this is to avoid accidentally altered To-lists in normal MUAs;
@@ -332,6 +344,64 @@ To save data, it is RECOMMENDED to add a `Chat-User-Avatar` header
 only on image changes.
 
 
+# Locations
+
+Locations can be attachted to messages using
+[standard kml-files](https://www.opengeospatial.org/standards/kml/)
+with well-known names.
+
+
+## User locations
+
+To send the location of the sender,
+the app can attach a file with the name `location.kml`.
+The file can contain one or more locations.
+Apps that support location streaming will typically collect some location events
+and send them together in one file.
+As each location has an independent timestamp,
+the apps can show the location as a track.
+
+Note that the `addr` attribute inside the  `location.kml` file
+MUST match the users email-address.
+Otherwise, the file is discarded silently;
+this is to protect against getting wrong locations,
+eg. forwarded from a normal MUA.
+
+    <?xml version="1.0" encoding="UTF-8"?>
+    <kml xmlns="http://www.opengis.net/kml/2.2">
+      <Document addr="ndh@deltachat.de">
+        <Placemark>
+          <Timestamp><when>2020-01-11T20:40:19Z</when></Timestamp>
+          <Point><coordinates accuracy="1.2">1.234,5.678</coordinates></Point>
+        </Placemark>
+        <Placemark>
+          <Timestamp><when>2020-01-11T20:40:25Z</when></Timestamp>
+          <Point><coordinates accuracy="5.4">7.654,3.21</coordinates></Point>
+        </Placemark>
+      </Document>
+    </kml>
+
+
+## Points of interest
+
+To send an "Point of interest", a POI,
+use a normal message and attach a file with the name  `message.kml`.
+In contrast to user locations, this file should contain only one location
+and an address-attribute is not needed -
+as the location belongs to the message content,
+it is fine if the location is detected on forwarding etc.
+
+    <?xml version="1.0" encoding="UTF-8"?>
+    <kml xmlns="http://www.opengis.net/kml/2.2">
+      <Document>
+        <Placemark>
+          <Timestamp><when>2020-01-01T20:40:19Z</when></Timestamp>
+          <Point><coordinates accuracy="1.2">1.234,5.678</coordinates></Point>
+        </Placemark>
+      </Document>
+    </kml>
+
+
 # Miscellaneous
 
 Messengers SHOULD use the header `In-Reply-To` as usual.
@@ -368,4 +438,4 @@ as the sending time of the message as indicated by its Date header,
 or the time of first receipt if that date is in the future or unavailable.
 
 
-Copyright © 2017-2019 Delta Chat contributors.
+Copyright © 2017-2020 Delta Chat contributors.

src/chat.rs

@@ -137,14 +137,18 @@ impl ChatId {
     }
 
     /// Archives or unarchives a chat.
-    pub fn set_archived(self, context: &Context, new_archived: bool) -> Result<(), Error> {
+    pub fn set_visibility(
+        self,
+        context: &Context,
+        visibility: ChatVisibility,
+    ) -> Result<(), Error> {
         ensure!(
             !self.is_special(),
             "bad chat_id, can not be special chat: {}",
             self
         );
 
-        if new_archived {
+        if visibility == ChatVisibility::Archived {
             sql::execute(
                 context,
                 &context.sql,
@@ -157,7 +161,7 @@ impl ChatId {
             context,
             &context.sql,
             "UPDATE chats SET archived=? WHERE id=?;",
-            params![new_archived, self],
+            params![visibility, self],
         )?;
 
         context.call_cb(Event::MsgsChanged {
@@ -168,13 +172,13 @@ impl ChatId {
         Ok(())
     }
 
-    // note that unarchive() is not the same as set_archived(false) -
-    // eg. unarchive() does not send events as done for set_archived(false).
+    // note that unarchive() is not the same as set_visibility(Normal) -
+    // eg. unarchive() does not modify pinned chats and does not send events.
     pub fn unarchive(self, context: &Context) -> Result<(), Error> {
         sql::execute(
             context,
             &context.sql,
-            "UPDATE chats SET archived=0 WHERE id=?",
+            "UPDATE chats SET archived=0 WHERE id=? and archived=1",
             params![self],
         )?;
         Ok(())
@@ -421,7 +425,7 @@ pub struct Chat {
     pub id: ChatId,
     pub typ: Chattype,
     pub name: String,
-    archived: bool,
+    pub visibility: ChatVisibility,
     pub grpid: String,
     blocked: Blocked,
     pub param: Params,
@@ -445,7 +449,7 @@ impl Chat {
                     name: row.get::<_, String>(1)?,
                     grpid: row.get::<_, String>(2)?,
                     param: row.get::<_, String>(3)?.parse().unwrap_or_default(),
-                    archived: row.get(4)?,
+                    visibility: row.get(4)?,
                     blocked: row.get::<_, Option<_>>(5)?.unwrap_or_default(),
                     is_sending_locations: row.get(6)?,
                     mute_duration: row.get(7)?,
@@ -656,7 +660,7 @@ impl Chat {
             id: self.id,
             type_: self.typ as u32,
             name: self.name.clone(),
-            archived: self.archived,
+            archived: self.visibility == ChatVisibility::Archived,
             param: self.param.to_string(),
             gossiped_timestamp: self.get_gossiped_timestamp(context),
             is_sending_locations: self.is_sending_locations,
@@ -668,9 +672,8 @@ impl Chat {
         })
     }
 
-    /// Returns true if the chat is archived.
-    pub fn is_archived(&self) -> bool {
-        self.archived
+    pub fn get_visibility(&self) -> ChatVisibility {
+        self.visibility
     }
 
     pub fn is_unpromoted(&self) -> bool {
@@ -928,6 +931,40 @@ impl Chat {
     }
 }
 
+#[derive(Debug, Copy, Eq, PartialEq, Clone, Serialize, Deserialize)]
+pub enum ChatVisibility {
+    Normal,
+    Archived,
+    Pinned,
+}
+
+impl rusqlite::types::ToSql for ChatVisibility {
+    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
+        let visibility = match &self {
+            ChatVisibility::Normal => 0,
+            ChatVisibility::Archived => 1,
+            ChatVisibility::Pinned => 2,
+        };
+        let val = rusqlite::types::Value::Integer(visibility);
+        let out = rusqlite::types::ToSqlOutput::Owned(val);
+        Ok(out)
+    }
+}
+
+impl rusqlite::types::FromSql for ChatVisibility {
+    fn column_result(value: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
+        i64::column_result(value).and_then(|val| {
+            match val {
+                2 => Ok(ChatVisibility::Pinned),
+                1 => Ok(ChatVisibility::Archived),
+                0 => Ok(ChatVisibility::Normal),
+                // fallback to to Normal for unknown values, may happen eg. on imports created by a newer version.
+                _ => Ok(ChatVisibility::Normal),
+            }
+        })
+    }
+}
+
 /// The current state of a chat.
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 #[non_exhaustive]
@@ -1077,7 +1114,7 @@ pub fn create_by_contact_id(context: &Context, contact_id: u32) -> Result<ChatId
     Ok(chat_id)
 }
 
-pub fn update_saved_messages_icon(context: &Context) -> Result<(), Error> {
+pub(crate) fn update_saved_messages_icon(context: &Context) -> Result<(), Error> {
     // if there is no saved-messages chat, there is nothing to update. this is no error.
     if let Ok((chat_id, _)) = lookup_by_contact_id(context, DC_CONTACT_ID_SELF) {
         let icon = include_bytes!("../assets/icon-saved-messages.png");
@@ -1091,7 +1128,7 @@ pub fn update_saved_messages_icon(context: &Context) -> Result<(), Error> {
     Ok(())
 }
 
-pub fn update_device_icon(context: &Context) -> Result<(), Error> {
+pub(crate) fn update_device_icon(context: &Context) -> Result<(), Error> {
     // if there is no device-chat, there is nothing to update. this is no error.
     if let Ok((chat_id, _)) = lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE) {
         let icon = include_bytes!("../assets/icon-device.png");
@@ -1125,13 +1162,13 @@ fn update_special_chat_name(
     Ok(())
 }
 
-pub fn update_special_chat_names(context: &Context) -> Result<(), Error> {
+pub(crate) fn update_special_chat_names(context: &Context) -> Result<(), Error> {
     update_special_chat_name(context, DC_CONTACT_ID_DEVICE, StockMessage::DeviceMessages)?;
     update_special_chat_name(context, DC_CONTACT_ID_SELF, StockMessage::SavedMessages)?;
     Ok(())
 }
 
-pub fn create_or_lookup_by_contact_id(
+pub(crate) fn create_or_lookup_by_contact_id(
     context: &Context,
     contact_id: u32,
     create_blocked: Blocked,
@@ -1147,33 +1184,31 @@ pub fn create_or_lookup_by_contact_id(
     let contact = Contact::load_from_db(context, contact_id)?;
     let chat_name = contact.get_display_name();
 
-    sql::execute(
-        context,
-        &context.sql,
-        "INSERT INTO chats (type, name, param, blocked, grpid, created_timestamp) VALUES(?, ?, ?, ?, ?, ?)",
+    context
+        .sql
+        .start_stmt("create_or_lookup_by_contact_id transaction");
+    context.sql.with_conn(|conn| {
+        let tx = conn.transaction()?;
+        tx.execute(
+        "INSERT INTO chats (type, name, param, blocked, created_timestamp) VALUES(?, ?, ?, ?, ?)",
         params![
-            100,
+            Chattype::Single,
             chat_name,
             match contact_id {
                 DC_CONTACT_ID_SELF => "K=1".to_string(), // K = Param::Selftalk
                 DC_CONTACT_ID_DEVICE => "D=1".to_string(), // D = Param::Devicetalk
-                _ => "".to_string()
+                _ => "".to_string(),
             },
             create_blocked as u8,
-            contact.get_addr(),
             time(),
         ]
-    )?;
-
-    let row_id = sql::get_rowid(context, &context.sql, "chats", "grpid", contact.get_addr());
-    let chat_id = ChatId::new(row_id);
-
-    sql::execute(
-        context,
-        &context.sql,
-        "INSERT INTO chats_contacts (chat_id, contact_id) VALUES(?, ?)",
-        params![chat_id, contact_id],
-    )?;
+        )?;
+        tx.execute(
+            "INSERT INTO chats_contacts (chat_id, contact_id) VALUES((SELECT last_insert_rowid()), ?)",
+            params![contact_id])?;
+        tx.commit()?;
+        Ok(())
+    })?;
 
     if contact_id == DC_CONTACT_ID_SELF {
         update_saved_messages_icon(context)?;
@@ -1181,10 +1216,10 @@ pub fn create_or_lookup_by_contact_id(
         update_device_icon(context)?;
     }
 
-    Ok((chat_id, create_blocked))
+    lookup_by_contact_id(context, contact_id)
 }
 
-pub fn lookup_by_contact_id(
+pub(crate) fn lookup_by_contact_id(
     context: &Context,
     contact_id: u32,
 ) -> Result<(ChatId, Blocked), Error> {
@@ -1238,7 +1273,7 @@ pub fn prepare_msg<'a>(
     Ok(msg_id)
 }
 
-pub fn msgtype_has_file(msgtype: Viewtype) -> bool {
+pub(crate) fn msgtype_has_file(msgtype: Viewtype) -> bool {
     match msgtype {
         Viewtype::Unknown => false,
         Viewtype::Text => false,
@@ -1715,9 +1750,11 @@ pub fn create_group_chat(
     Ok(chat_id)
 }
 
-/* you MUST NOT modify this or the following strings */
-// Context functions to work with chats
-pub fn add_to_chat_contacts_table(context: &Context, chat_id: ChatId, contact_id: u32) -> bool {
+pub(crate) fn add_to_chat_contacts_table(
+    context: &Context,
+    chat_id: ChatId,
+    contact_id: u32,
+) -> bool {
     // add a contact to a chat; the function does not check the type or if any of the record exist or are already
     // added to the chat!
     sql::execute(
@@ -1729,6 +1766,22 @@ pub fn add_to_chat_contacts_table(context: &Context, chat_id: ChatId, contact_id
     .is_ok()
 }
 
+pub(crate) fn remove_from_chat_contacts_table(
+    context: &Context,
+    chat_id: ChatId,
+    contact_id: u32,
+) -> bool {
+    // remove a contact from the chats_contact table unconditionally
+    // the function does not check the type or if the record exist
+    sql::execute(
+        context,
+        &context.sql,
+        "DELETE FROM chats_contacts WHERE chat_id=? AND contact_id=?",
+        params![chat_id, contact_id as i32],
+    )
+    .is_ok()
+}
+
 /// Adds a contact to the chat.
 pub fn add_contact_to_chat(context: &Context, chat_id: ChatId, contact_id: u32) -> bool {
     match add_contact_to_chat_ex(context, chat_id, contact_id, false) {
@@ -1849,7 +1902,7 @@ fn real_group_exists(context: &Context, chat_id: ChatId) -> bool {
         .unwrap_or_default()
 }
 
-pub fn reset_gossiped_timestamp(context: &Context, chat_id: ChatId) -> Result<(), Error> {
+pub(crate) fn reset_gossiped_timestamp(context: &Context, chat_id: ChatId) -> Result<(), Error> {
     set_gossiped_timestamp(context, chat_id, 0)
 }
 
@@ -1866,7 +1919,7 @@ pub fn get_gossiped_timestamp(context: &Context, chat_id: ChatId) -> i64 {
         .unwrap_or_default()
 }
 
-pub fn set_gossiped_timestamp(
+pub(crate) fn set_gossiped_timestamp(
     context: &Context,
     chat_id: ChatId,
     timestamp: i64,
@@ -1886,7 +1939,7 @@ pub fn set_gossiped_timestamp(
     Ok(())
 }
 
-pub fn shall_attach_selfavatar(context: &Context, chat_id: ChatId) -> Result<bool, Error> {
+pub(crate) fn shall_attach_selfavatar(context: &Context, chat_id: ChatId) -> Result<bool, Error> {
     // versions before 12/2019 already allowed to set selfavatar, however, it was never sent to others.
     // to avoid sending out previously set selfavatars unexpectedly we added this additional check.
     // it can be removed after some time.
@@ -2040,14 +2093,7 @@ pub fn remove_contact_from_chat(
                         });
                     }
                 }
-                if sql::execute(
-                    context,
-                    &context.sql,
-                    "DELETE FROM chats_contacts WHERE chat_id=? AND contact_id=?;",
-                    params![chat_id, contact_id as i32],
-                )
-                .is_ok()
-                {
+                if remove_from_chat_contacts_table(context, chat_id, contact_id) {
                     context.call_cb(Event::ChatModified(chat_id));
                     success = true;
                 }
@@ -2075,7 +2121,10 @@ fn set_group_explicitly_left(context: &Context, grpid: impl AsRef<str>) -> Resul
     Ok(())
 }
 
-pub fn is_group_explicitly_left(context: &Context, grpid: impl AsRef<str>) -> Result<bool, Error> {
+pub(crate) fn is_group_explicitly_left(
+    context: &Context,
+    grpid: impl AsRef<str>,
+) -> Result<bool, Error> {
     context
         .sql
         .exists(
@@ -2297,7 +2346,7 @@ pub fn forward_msgs(context: &Context, msg_ids: &[MsgId], chat_id: ChatId) -> Re
     Ok(())
 }
 
-pub fn get_chat_contact_cnt(context: &Context, chat_id: ChatId) -> usize {
+pub(crate) fn get_chat_contact_cnt(context: &Context, chat_id: ChatId) -> usize {
     context
         .sql
         .query_get_value::<_, isize>(
@@ -2308,7 +2357,7 @@ pub fn get_chat_contact_cnt(context: &Context, chat_id: ChatId) -> usize {
         .unwrap_or_default() as usize
 }
 
-pub fn get_chat_cnt(context: &Context) -> usize {
+pub(crate) fn get_chat_cnt(context: &Context) -> usize {
     if context.sql.is_open() {
         /* no database, no chats - this is no error (needed eg. for information) */
         context
@@ -2423,7 +2472,7 @@ pub fn was_device_msg_ever_added(context: &Context, label: &str) -> Result<bool,
 //   no wrong information are shown in the device chat
 // - deletion in `devmsglabels` makes sure,
 //   deleted messages are resetted and useful messages can be added again
-pub fn delete_and_reset_all_device_msgs(context: &Context) -> Result<(), Error> {
+pub(crate) fn delete_and_reset_all_device_msgs(context: &Context) -> Result<(), Error> {
     context.sql.execute(
         "DELETE FROM msgs WHERE from_id=?;",
         params![DC_CONTACT_ID_DEVICE],
@@ -2437,7 +2486,7 @@ pub fn delete_and_reset_all_device_msgs(context: &Context) -> Result<(), Error>
 /// Adds an informational message to chat.
 ///
 /// For example, it can be a message showing that a member was added to a group.
-pub fn add_info_msg(context: &Context, chat_id: ChatId, text: impl AsRef<str>) {
+pub(crate) fn add_info_msg(context: &Context, chat_id: ChatId, text: impl AsRef<str>) {
     let rfc724_mid = dc_create_outgoing_rfc724_mid(None, "@device");
 
     if context.sql.execute(
@@ -2560,7 +2609,7 @@ mod tests {
         let chat = Chat::load_from_db(&t.ctx, chat_id).unwrap();
         assert_eq!(chat.id, chat_id);
         assert!(chat.is_self_talk());
-        assert!(!chat.archived);
+        assert!(chat.visibility == ChatVisibility::Normal);
         assert!(!chat.is_device_talk());
         assert!(chat.can_send());
         assert_eq!(chat.name, t.ctx.stock_str(StockMessage::SavedMessages));
@@ -2574,7 +2623,7 @@ mod tests {
         assert_eq!(DC_CHAT_ID_DEADDROP, 1);
         assert!(chat.id.is_deaddrop());
         assert!(!chat.is_self_talk());
-        assert!(!chat.archived);
+        assert!(chat.visibility == ChatVisibility::Normal);
         assert!(!chat.is_device_talk());
         assert!(!chat.can_send());
         assert_eq!(chat.name, t.ctx.stock_str(StockMessage::DeadDrop));
@@ -2780,35 +2829,134 @@ mod tests {
         assert_eq!(DC_GCL_NO_SPECIALS, 0x02);
 
         // archive first chat
-        assert!(chat_id1.set_archived(&t.ctx, true).is_ok());
-        assert!(Chat::load_from_db(&t.ctx, chat_id1).unwrap().is_archived());
-        assert!(!Chat::load_from_db(&t.ctx, chat_id2).unwrap().is_archived());
+        assert!(chat_id1
+            .set_visibility(&t.ctx, ChatVisibility::Archived)
+            .is_ok());
+        assert!(
+            Chat::load_from_db(&t.ctx, chat_id1)
+                .unwrap()
+                .get_visibility()
+                == ChatVisibility::Archived
+        );
+        assert!(
+            Chat::load_from_db(&t.ctx, chat_id2)
+                .unwrap()
+                .get_visibility()
+                == ChatVisibility::Normal
+        );
         assert_eq!(get_chat_cnt(&t.ctx), 2);
         assert_eq!(chatlist_len(&t.ctx, 0), 2); // including DC_CHAT_ID_ARCHIVED_LINK now
         assert_eq!(chatlist_len(&t.ctx, DC_GCL_NO_SPECIALS), 1);
         assert_eq!(chatlist_len(&t.ctx, DC_GCL_ARCHIVED_ONLY), 1);
 
         // archive second chat
-        assert!(chat_id2.set_archived(&t.ctx, true).is_ok());
-        assert!(Chat::load_from_db(&t.ctx, chat_id1).unwrap().is_archived());
-        assert!(Chat::load_from_db(&t.ctx, chat_id2).unwrap().is_archived());
+        assert!(chat_id2
+            .set_visibility(&t.ctx, ChatVisibility::Archived)
+            .is_ok());
+        assert!(
+            Chat::load_from_db(&t.ctx, chat_id1)
+                .unwrap()
+                .get_visibility()
+                == ChatVisibility::Archived
+        );
+        assert!(
+            Chat::load_from_db(&t.ctx, chat_id2)
+                .unwrap()
+                .get_visibility()
+                == ChatVisibility::Archived
+        );
         assert_eq!(get_chat_cnt(&t.ctx), 2);
         assert_eq!(chatlist_len(&t.ctx, 0), 1); // only DC_CHAT_ID_ARCHIVED_LINK now
         assert_eq!(chatlist_len(&t.ctx, DC_GCL_NO_SPECIALS), 0);
         assert_eq!(chatlist_len(&t.ctx, DC_GCL_ARCHIVED_ONLY), 2);
 
         // archive already archived first chat, unarchive second chat two times
-        assert!(chat_id1.set_archived(&t.ctx, true).is_ok());
-        assert!(chat_id2.set_archived(&t.ctx, false).is_ok());
-        assert!(chat_id2.set_archived(&t.ctx, false).is_ok());
-        assert!(Chat::load_from_db(&t.ctx, chat_id1).unwrap().is_archived());
-        assert!(!Chat::load_from_db(&t.ctx, chat_id2).unwrap().is_archived());
+        assert!(chat_id1
+            .set_visibility(&t.ctx, ChatVisibility::Archived)
+            .is_ok());
+        assert!(chat_id2
+            .set_visibility(&t.ctx, ChatVisibility::Normal)
+            .is_ok());
+        assert!(chat_id2
+            .set_visibility(&t.ctx, ChatVisibility::Normal)
+            .is_ok());
+        assert!(
+            Chat::load_from_db(&t.ctx, chat_id1)
+                .unwrap()
+                .get_visibility()
+                == ChatVisibility::Archived
+        );
+        assert!(
+            Chat::load_from_db(&t.ctx, chat_id2)
+                .unwrap()
+                .get_visibility()
+                == ChatVisibility::Normal
+        );
         assert_eq!(get_chat_cnt(&t.ctx), 2);
         assert_eq!(chatlist_len(&t.ctx, 0), 2);
         assert_eq!(chatlist_len(&t.ctx, DC_GCL_NO_SPECIALS), 1);
         assert_eq!(chatlist_len(&t.ctx, DC_GCL_ARCHIVED_ONLY), 1);
     }
 
+    fn get_chats_from_chat_list(ctx: &Context, listflags: usize) -> Vec<ChatId> {
+        let chatlist = Chatlist::try_load(ctx, listflags, None, None).unwrap();
+        let mut result = Vec::new();
+        for chatlist_index in 0..chatlist.len() {
+            result.push(chatlist.get_chat_id(chatlist_index))
+        }
+        result
+    }
+
+    #[test]
+    fn test_pinned() {
+        let t = dummy_context();
+
+        // create 3 chats, wait 1 second in between to get a reliable order (we order by time)
+        let mut msg = Message::new(Viewtype::Text);
+        msg.text = Some("foo".to_string());
+        let msg_id = add_device_msg(&t.ctx, None, Some(&mut msg)).unwrap();
+        let chat_id1 = message::Message::load_from_db(&t.ctx, msg_id)
+            .unwrap()
+            .chat_id;
+        std::thread::sleep(std::time::Duration::from_millis(1000));
+        let chat_id2 = create_by_contact_id(&t.ctx, DC_CONTACT_ID_SELF).unwrap();
+        std::thread::sleep(std::time::Duration::from_millis(1000));
+        let chat_id3 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo").unwrap();
+
+        let chatlist = get_chats_from_chat_list(&t.ctx, DC_GCL_NO_SPECIALS);
+        assert_eq!(chatlist, vec![chat_id3, chat_id2, chat_id1]);
+
+        // pin
+        assert!(chat_id1
+            .set_visibility(&t.ctx, ChatVisibility::Pinned)
+            .is_ok());
+        assert_eq!(
+            Chat::load_from_db(&t.ctx, chat_id1)
+                .unwrap()
+                .get_visibility(),
+            ChatVisibility::Pinned
+        );
+
+        // check if chat order changed
+        let chatlist = get_chats_from_chat_list(&t.ctx, DC_GCL_NO_SPECIALS);
+        assert_eq!(chatlist, vec![chat_id1, chat_id3, chat_id2]);
+
+        // unpin
+        assert!(chat_id1
+            .set_visibility(&t.ctx, ChatVisibility::Normal)
+            .is_ok());
+        assert_eq!(
+            Chat::load_from_db(&t.ctx, chat_id1)
+                .unwrap()
+                .get_visibility(),
+            ChatVisibility::Normal
+        );
+
+        // check if chat order changed back
+        let chatlist = get_chats_from_chat_list(&t.ctx, DC_GCL_NO_SPECIALS);
+        assert_eq!(chatlist, vec![chat_id3, chat_id2, chat_id1]);
+    }
+
     #[test]
     fn test_set_chat_name() {
         let t = dummy_context();

src/chatlist.rs

@@ -60,7 +60,7 @@ impl Chatlist {
     ///   or "Not now".
     ///   The UI can also offer a "Close" button that calls dc_marknoticed_contact() then.
     /// - DC_CHAT_ID_ARCHIVED_LINK (6) - this special chat is present if the user has
-    ///   archived *any* chat using dc_archive_chat(). The UI should show a link as
+    ///   archived *any* chat using dc_set_chat_visibility(). The UI should show a link as
     ///   "Show archived chats", if the user clicks this item, the UI should show a
     ///   list of all archived chats that can be created by this function hen using
     ///   the DC_GCL_ARCHIVED_ONLY flag.
@@ -127,13 +127,13 @@ impl Chatlist {
                                SELECT MAX(timestamp)
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?))
+                                  AND (hidden=0 OR state=?1))
                  WHERE c.id>9
                    AND c.blocked=0
-                   AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)
+                   AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?2)
                  GROUP BY c.id
-                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                params![MessageState::OutDraft, query_contact_id as i32],
+                 ORDER BY c.archived=?3 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                params![MessageState::OutDraft, query_contact_id as i32, ChatVisibility::Pinned],
                 process_row,
                 process_rows,
             )?
@@ -199,13 +199,13 @@ impl Chatlist {
                                SELECT MAX(timestamp)
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?))
+                                  AND (hidden=0 OR state=?1))
                  WHERE c.id>9
                    AND c.blocked=0
-                   AND c.archived=0
+                   AND NOT c.archived=?2
                  GROUP BY c.id
-                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                params![MessageState::OutDraft],
+                 ORDER BY c.archived=?3 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                params![MessageState::OutDraft, ChatVisibility::Archived, ChatVisibility::Pinned],
                 process_row,
                 process_rows,
             )?;
@@ -392,7 +392,9 @@ mod tests {
         let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None).unwrap();
         assert_eq!(chats.len(), 0);
 
-        chat_id1.set_archived(&t.ctx, true).ok();
+        chat_id1
+            .set_visibility(&t.ctx, ChatVisibility::Archived)
+            .ok();
         let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None).unwrap();
         assert_eq!(chats.len(), 1);
     }

src/config.rs

@@ -62,6 +62,9 @@ pub enum Config {
     #[strum(props(default = "0"))] // also change ShowEmails.default() on changes
     ShowEmails,
 
+    #[strum(props(default = "0"))]
+    KeyGenType,
+
     SaveMimeHeaders,
     ConfiguredAddr,
     ConfiguredMailServer,

src/configure/mod.rs

@@ -32,26 +32,28 @@ macro_rules! progress {
     };
 }
 
-// connect
-pub fn configure(context: &Context) {
-    if context.has_ongoing() {
-        warn!(context, "There is already another ongoing process running.",);
-        return;
+impl Context {
+    /// Starts a configuration job.
+    pub fn configure(&self) {
+        if self.has_ongoing() {
+            warn!(self, "There is already another ongoing process running.",);
+            return;
+        }
+        job_kill_action(self, job::Action::ConfigureImap);
+        job_add(self, job::Action::ConfigureImap, 0, Params::new(), 0);
     }
-    job_kill_action(context, job::Action::ConfigureImap);
-    job_add(context, job::Action::ConfigureImap, 0, Params::new(), 0);
-}
 
-/// Check if the context is already configured.
-pub fn dc_is_configured(context: &Context) -> bool {
-    context.sql.get_raw_config_bool(context, "configured")
+    /// Checks if the context is already configured.
+    pub fn is_configured(&self) -> bool {
+        self.sql.get_raw_config_bool(self, "configured")
+    }
 }
 
 /*******************************************************************************
  * Configure JOB
  ******************************************************************************/
 #[allow(non_snake_case, unused_must_use, clippy::cognitive_complexity)]
-pub fn JobConfigureImap(context: &Context) -> job::Status {
+pub(crate) fn JobConfigureImap(context: &Context) -> job::Status {
     if !context.sql.is_open() {
         error!(context, "Cannot configure, database not opened.",);
         progress!(context, 0);

src/constants.rs

@@ -57,6 +57,20 @@ impl Default for ShowEmails {
     }
 }
 
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[repr(u8)]
+pub enum KeyGenType {
+    Default = 0,
+    Rsa2048 = 1,
+    Ed25519 = 2,
+}
+
+impl Default for KeyGenType {
+    fn default() -> Self {
+        KeyGenType::Default
+    }
+}
+
 pub const DC_HANDSHAKE_CONTINUE_NORMAL_PROCESSING: i32 = 0x01;
 pub const DC_HANDSHAKE_STOP_NORMAL_PROCESSING: i32 = 0x02;
 pub const DC_HANDSHAKE_ADD_DELETE_JOB: i32 = 0x04;

src/contact.rs

@@ -60,7 +60,7 @@ pub struct Contact {
     /// to access this field.
     authname: String,
 
-    /// E-Mail-Address of the contact. It is recommended to use `Contact::get_addr`` to access this field.
+    /// E-Mail-Address of the contact. It is recommended to use `Contact::get_addr` to access this field.
     addr: String,
 
     /// Blocked state. Use dc_contact_is_blocked to access this field.
@@ -146,7 +146,7 @@ impl Origin {
 }
 
 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
-pub enum Modifier {
+pub(crate) enum Modifier {
     None,
     Modified,
     Created,
@@ -300,9 +300,31 @@ impl Contact {
     }
 
     /// Lookup a contact and create it if it does not exist yet.
+    /// The contact is identified by the email-address, a name and an "origin" can be given.
+    ///
+    /// The "origin" is where the address comes from -
+    /// from-header, cc-header, addressbook, qr, manual-edit etc.
+    /// In general, "better" origins overwrite the names of "worse" origins -
+    /// Eg. if we got a name in cc-header and later in from-header, the name will change -
+    /// this does not happen the other way round.
+    ///
+    /// The "best" origin are manually created contacts -
+    /// names given manually can only be overwritten by further manual edits
+    /// (until they are set empty again or reset to the name seen in the From-header).
+    ///
+    /// These manually edited names are _never_ used for sending on the wire -
+    /// this should avoid sending sth. as "Mama" or "Daddy" to some 3rd party.
+    /// Instead, for the wire, we use so called "authnames"
+    /// that can only be set and updated by a From-header.
+    ///
+    /// The different names used in the function are:
+    /// - "name": name passed as function argument, belonging to the given origin
+    /// - "row_name": current name used in the database, typically set to "name"
+    /// - "row_authname": name as authorized from a contact, set only through a From-header
+    /// Depending on the origin, both, "row_name" and "row_authname" are updated from "name".
     ///
     /// Returns the contact_id and a `Modifier` value indicating if a modification occured.
-    pub fn add_or_lookup(
+    pub(crate) fn add_or_lookup(
         context: &Context,
         name: impl AsRef<str>,
         addr: impl AsRef<str>,
@@ -356,7 +378,9 @@ impl Contact {
 
                 if !name.as_ref().is_empty() {
                     if !row_name.is_empty() {
-                        if origin >= row_origin && name.as_ref() != row_name {
+                        if (origin >= row_origin || row_name == row_authname)
+                            && name.as_ref() != row_name
+                        {
                             update_name = true;
                         }
                     } else {
@@ -365,6 +389,9 @@ impl Contact {
                     if origin == Origin::IncomingUnknownFrom && name.as_ref() != row_authname {
                         update_authname = true;
                     }
+                } else if origin == Origin::ManuallyCreated && !row_authname.is_empty() {
+                    // no name given on manual edit, this will update the name to the authname
+                    update_name = true;
                 }
 
                 Ok((row_id, row_name, row_addr, row_origin, row_authname))
@@ -375,16 +402,22 @@ impl Contact {
                 update_addr = true;
             }
             if update_name || update_authname || update_addr || origin > row_origin {
+                let new_name = if update_name {
+                    if !name.as_ref().is_empty() {
+                        name.as_ref()
+                    } else {
+                        &row_authname
+                    }
+                } else {
+                    &row_name
+                };
+
                 sql::execute(
                     context,
                     &context.sql,
                     "UPDATE contacts SET name=?, addr=?, origin=?, authname=? WHERE id=?;",
                     params![
-                        if update_name {
-                            name.as_ref()
-                        } else {
-                            &row_name
-                        },
+                        new_name,
                         if update_addr { addr } else { &row_addr },
                         if origin > row_origin {
                             origin
@@ -402,11 +435,13 @@ impl Contact {
                 .ok();
 
                 if update_name {
+                    // Update the contact name also if it is used as a group name.
+                    // This is one of the few duplicated data, however, getting the chat list is easier this way.
                     sql::execute(
                     context,
                     &context.sql,
                     "UPDATE chats SET name=? WHERE type=? AND id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?);",
-                    params![name.as_ref(), Chattype::Single, row_id]
+                    params![new_name, Chattype::Single, row_id]
                 ).ok();
                 }
                 sth_modified = Modifier::Modified;
@@ -984,7 +1019,7 @@ fn set_block_contact(context: &Context, contact_id: u32, new_blocking: bool) {
     }
 }
 
-pub fn set_profile_image(
+pub(crate) fn set_profile_image(
     context: &Context,
     contact_id: u32,
     profile_image: &AvatarAction,
@@ -1001,7 +1036,6 @@ pub fn set_profile_image(
             contact.param.remove(Param::ProfileImage);
             true
         }
-        AvatarAction::None => false,
     };
     if changed {
         contact.update_param(context)?;
@@ -1281,6 +1315,7 @@ mod tests {
     fn test_remote_authnames() {
         let t = dummy_context();
 
+        // incoming mail `From: bob1 <bob@example.org>` - this should init authname and name
         let (contact_id, sth_modified) = Contact::add_or_lookup(
             &t.ctx,
             "bob1",
@@ -1295,6 +1330,7 @@ mod tests {
         assert_eq!(contact.get_name(), "bob1");
         assert_eq!(contact.get_display_name(), "bob1");
 
+        // incoming mail `From: bob2 <bob@example.org>` - this should update authname and name
         let (contact_id, sth_modified) = Contact::add_or_lookup(
             &t.ctx,
             "bob2",
@@ -1309,16 +1345,15 @@ mod tests {
         assert_eq!(contact.get_name(), "bob2");
         assert_eq!(contact.get_display_name(), "bob2");
 
-        let (contact_id, sth_modified) =
-            Contact::add_or_lookup(&t.ctx, "bob3", "bob@example.org", Origin::ManuallyCreated)
-                .unwrap();
+        // manually edit name to "bob3" - authname should be still be "bob2" a given in `From:` above
+        let contact_id = Contact::create(&t.ctx, "bob3", "bob@example.org").unwrap();
         assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
-        assert_eq!(sth_modified, Modifier::Modified);
         let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
         assert_eq!(contact.get_authname(), "bob2");
         assert_eq!(contact.get_name(), "bob3");
         assert_eq!(contact.get_display_name(), "bob3");
 
+        // incoming mail `From: bob4 <bob@example.org>` - this should update authname, manually given name is still "bob3"
         let (contact_id, sth_modified) = Contact::add_or_lookup(
             &t.ctx,
             "bob4",
@@ -1334,6 +1369,81 @@ mod tests {
         assert_eq!(contact.get_display_name(), "bob3");
     }
 
+    #[test]
+    fn test_remote_authnames_create_empty() {
+        let t = dummy_context();
+
+        // manually create "claire@example.org" without a given name
+        let contact_id = Contact::create(&t.ctx, "", "claire@example.org").unwrap();
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_authname(), "");
+        assert_eq!(contact.get_name(), "");
+        assert_eq!(contact.get_display_name(), "claire@example.org");
+
+        // incoming mail `From: claire1 <claire@example.org>` - this should update authname and name
+        let (contact_id_same, sth_modified) = Contact::add_or_lookup(
+            &t.ctx,
+            "claire1",
+            "claire@example.org",
+            Origin::IncomingUnknownFrom,
+        )
+        .unwrap();
+        assert_eq!(contact_id, contact_id_same);
+        assert_eq!(sth_modified, Modifier::Modified);
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_authname(), "claire1");
+        assert_eq!(contact.get_name(), "claire1");
+        assert_eq!(contact.get_display_name(), "claire1");
+
+        // incoming mail `From: claire2 <claire@example.org>` - this should update authname and name
+        let (contact_id_same, sth_modified) = Contact::add_or_lookup(
+            &t.ctx,
+            "claire2",
+            "claire@example.org",
+            Origin::IncomingUnknownFrom,
+        )
+        .unwrap();
+        assert_eq!(contact_id, contact_id_same);
+        assert_eq!(sth_modified, Modifier::Modified);
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_authname(), "claire2");
+        assert_eq!(contact.get_name(), "claire2");
+        assert_eq!(contact.get_display_name(), "claire2");
+    }
+
+    #[test]
+    fn test_remote_authnames_edit_empty() {
+        let t = dummy_context();
+
+        // manually create "dave@example.org"
+        let contact_id = Contact::create(&t.ctx, "dave1", "dave@example.org").unwrap();
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_authname(), "");
+        assert_eq!(contact.get_name(), "dave1");
+        assert_eq!(contact.get_display_name(), "dave1");
+
+        // incoming mail `From: dave2 <dave@example.org>` - this should update authname
+        Contact::add_or_lookup(
+            &t.ctx,
+            "dave2",
+            "dave@example.org",
+            Origin::IncomingUnknownFrom,
+        )
+        .unwrap();
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_authname(), "dave2");
+        assert_eq!(contact.get_name(), "dave1");
+        assert_eq!(contact.get_display_name(), "dave1");
+
+        // manually clear the name
+        Contact::create(&t.ctx, "", "dave@example.org").unwrap();
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_authname(), "dave2");
+        assert_eq!(contact.get_name(), "dave2");
+        assert_eq!(contact.get_display_name(), "dave2");
+    }
+
     #[test]
     fn test_addr_cmp() {
         assert!(addr_cmp("AA@AA.ORG", "aa@aa.ORG"));

src/context.rs

@@ -51,7 +51,7 @@ pub struct Context {
     cb: Box<ContextCallback>,
     pub os_name: Option<String>,
     pub cmdline_sel_chat_id: Arc<RwLock<ChatId>>,
-    pub bob: Arc<RwLock<BobStatus>>,
+    pub(crate) bob: Arc<RwLock<BobStatus>>,
     pub last_smeared_timestamp: RwLock<i64>,
     pub running_state: Arc<RwLock<RunningState>>,
     /// Mutex to avoid generating the key for the user more than once.
@@ -478,14 +478,14 @@ impl Default for RunningState {
 }
 
 #[derive(Debug, Default)]
-pub struct BobStatus {
+pub(crate) struct BobStatus {
     pub expects: i32,
     pub status: i32,
     pub qr_scan: Option<Lot>,
 }
 
 #[derive(Debug, PartialEq)]
-pub enum PerformJobsNeeded {
+pub(crate) enum PerformJobsNeeded {
     Not,
     AtOnce,
     AvoidDos,
@@ -502,7 +502,7 @@ pub struct SmtpState {
     pub idle: bool,
     pub suspended: bool,
     pub doing_jobs: bool,
-    pub perform_jobs_needed: PerformJobsNeeded,
+    pub(crate) perform_jobs_needed: PerformJobsNeeded,
     pub probe_network: bool,
 }
 

src/dc_receive_imf.rs

@@ -181,8 +181,8 @@ pub fn dc_receive_imf(
         );
     }
 
-    if mime_parser.user_avatar != AvatarAction::None {
-        match contact::set_profile_image(&context, from_id, &mime_parser.user_avatar) {
+    if let Some(avatar_action) = &mime_parser.user_avatar {
+        match contact::set_profile_image(&context, from_id, avatar_action) {
             Ok(()) => {
                 context.call_cb(Event::ChatModified(chat_id));
             }
@@ -860,21 +860,21 @@ fn create_or_lookup_group(
 
                 mime_parser.is_system_message = SystemMessage::GroupNameChanged;
             } else if let Some(value) = mime_parser.get(HeaderDef::ChatContent) {
-                if value == "group-avatar-changed" && mime_parser.group_avatar != AvatarAction::None
-                {
-                    // this is just an explicit message containing the group-avatar,
-                    // apart from that, the group-avatar is send along with various other messages
-                    mime_parser.is_system_message = SystemMessage::GroupImageChanged;
-                    better_msg = context.stock_system_msg(
-                        if mime_parser.group_avatar == AvatarAction::Delete {
-                            StockMessage::MsgGrpImgDeleted
-                        } else {
-                            StockMessage::MsgGrpImgChanged
-                        },
-                        "",
-                        "",
-                        from_id as u32,
-                    )
+                if value == "group-avatar-changed" {
+                    if let Some(avatar_action) = &mime_parser.group_avatar {
+                        // this is just an explicit message containing the group-avatar,
+                        // apart from that, the group-avatar is send along with various other messages
+                        mime_parser.is_system_message = SystemMessage::GroupImageChanged;
+                        better_msg = context.stock_system_msg(
+                            match avatar_action {
+                                AvatarAction::Delete => StockMessage::MsgGrpImgDeleted,
+                                AvatarAction::Change(_) => StockMessage::MsgGrpImgChanged,
+                            },
+                            "",
+                            "",
+                            from_id as u32,
+                        )
+                    }
                 }
             }
         }
@@ -985,7 +985,7 @@ fn create_or_lookup_group(
     // XXX insert code in a different PR :)
 
     // execute group commands
-    if X_MrAddToGrp.is_some() || X_MrRemoveFromGrp.is_some() {
+    if X_MrAddToGrp.is_some() {
         recreate_member_list = true;
     } else if X_MrGrpNameChanged {
         if let Some(ref grpname) = grpname {
@@ -1004,17 +1004,16 @@ fn create_or_lookup_group(
             }
         }
     }
-    if mime_parser.group_avatar != AvatarAction::None {
+    if let Some(avatar_action) = &mime_parser.group_avatar {
         info!(context, "group-avatar change for {}", chat_id);
         if let Ok(mut chat) = Chat::load_from_db(context, chat_id) {
-            match &mime_parser.group_avatar {
+            match avatar_action {
                 AvatarAction::Change(profile_image) => {
                     chat.param.set(Param::ProfileImage, profile_image);
                 }
                 AvatarAction::Delete => {
                     chat.param.remove(Param::ProfileImage);
                 }
-                AvatarAction::None => {}
             };
             chat.update_param(context)?;
             send_EVENT_CHAT_MODIFIED = true;
@@ -1022,39 +1021,32 @@ fn create_or_lookup_group(
     }
 
     // add members to group/check members
-    // for recreation: we should add a timestamp
     if recreate_member_list {
-        // TODO: the member list should only be recreated if the corresponding message is newer
-        // than the one that is responsible for the current member list, see
-        // https://github.com/deltachat/deltachat-core/issues/127
-
-        let skip = X_MrRemoveFromGrp.as_ref();
-        sql::execute(
-            context,
-            &context.sql,
-            "DELETE FROM chats_contacts WHERE chat_id=?;",
-            params![chat_id],
-        )
-        .ok();
-        if skip.is_none() || !addr_cmp(&self_addr, skip.unwrap()) {
+        if !chat::is_contact_in_chat(context, chat_id, DC_CONTACT_ID_SELF) {
             chat::add_to_chat_contacts_table(context, chat_id, DC_CONTACT_ID_SELF);
         }
-        if from_id > DC_CHAT_ID_LAST_SPECIAL
+        if from_id > DC_CONTACT_ID_LAST_SPECIAL
             && !Contact::addr_equals_contact(context, &self_addr, from_id as u32)
-            && (skip.is_none()
-                || !Contact::addr_equals_contact(context, skip.unwrap(), from_id as u32))
+            && !chat::is_contact_in_chat(context, chat_id, from_id)
         {
             chat::add_to_chat_contacts_table(context, chat_id, from_id as u32);
         }
         for &to_id in to_ids.iter() {
             info!(context, "adding to={:?} to chat id={}", to_id, chat_id);
             if !Contact::addr_equals_contact(context, &self_addr, to_id)
-                && (skip.is_none() || !Contact::addr_equals_contact(context, skip.unwrap(), to_id))
+                && !chat::is_contact_in_chat(context, chat_id, to_id)
             {
                 chat::add_to_chat_contacts_table(context, chat_id, to_id);
             }
         }
         send_EVENT_CHAT_MODIFIED = true;
+    } else if let Some(removed_addr) = X_MrRemoveFromGrp {
+        let contact_id = Contact::lookup_id_by_addr(context, removed_addr);
+        if contact_id != 0 {
+            info!(context, "remove {:?} from chat id={}", contact_id, chat_id);
+            chat::remove_from_chat_contacts_table(context, chat_id, contact_id);
+        }
+        send_EVENT_CHAT_MODIFIED = true;
     }
 
     if send_EVENT_CHAT_MODIFIED {
@@ -1472,7 +1464,7 @@ fn is_known_rfc724_mid_in_list(context: &Context, mid_list: &str) -> bool {
         return false;
     }
 
-    if let Ok(ids) = mailparse::addrparse(mid_list) {
+    if let Ok(ids) = mailparse::msgidparse(mid_list) {
         for id in ids.iter() {
             if is_known_rfc724_mid(context, id) {
                 return true;
@@ -1484,8 +1476,7 @@ fn is_known_rfc724_mid_in_list(context: &Context, mid_list: &str) -> bool {
 }
 
 /// Check if a message is a reply to a known message (messenger or non-messenger).
-fn is_known_rfc724_mid(context: &Context, rfc724_mid: &mailparse::MailAddr) -> bool {
-    let addr = extract_single_from_addr(rfc724_mid);
+fn is_known_rfc724_mid(context: &Context, rfc724_mid: &str) -> bool {
     context
         .sql
         .exists(
@@ -1493,7 +1484,7 @@ fn is_known_rfc724_mid(context: &Context, rfc724_mid: &mailparse::MailAddr) -> b
              LEFT JOIN chats c ON m.chat_id=c.id  \
              WHERE m.rfc724_mid=?  \
              AND m.chat_id>9 AND c.blocked=0;",
-            params![addr],
+            params![rfc724_mid],
         )
         .unwrap_or_default()
 }
@@ -1520,7 +1511,7 @@ fn is_reply_to_messenger_message(context: &Context, mime_parser: &MimeMessage) -
 }
 
 pub(crate) fn is_msgrmsg_rfc724_mid_in_list(context: &Context, mid_list: &str) -> bool {
-    if let Ok(ids) = mailparse::addrparse(mid_list) {
+    if let Ok(ids) = mailparse::msgidparse(mid_list) {
         for id in ids.iter() {
             if is_msgrmsg_rfc724_mid(context, id) {
                 return true;
@@ -1530,21 +1521,13 @@ pub(crate) fn is_msgrmsg_rfc724_mid_in_list(context: &Context, mid_list: &str) -
     false
 }
 
-fn extract_single_from_addr(addr: &mailparse::MailAddr) -> &String {
-    match addr {
-        mailparse::MailAddr::Group(infos) => &infos.addrs[0].addr,
-        mailparse::MailAddr::Single(info) => &info.addr,
-    }
-}
-
 /// Check if a message is a reply to any messenger message.
-fn is_msgrmsg_rfc724_mid(context: &Context, rfc724_mid: &mailparse::MailAddr) -> bool {
-    let addr = extract_single_from_addr(rfc724_mid);
+fn is_msgrmsg_rfc724_mid(context: &Context, rfc724_mid: &str) -> bool {
     context
         .sql
         .exists(
             "SELECT id FROM msgs  WHERE rfc724_mid=?  AND msgrmsg!=0  AND chat_id>9;",
-            params![addr],
+            params![rfc724_mid],
         )
         .unwrap_or_default()
 }

src/e2ee.rs

@@ -8,6 +8,7 @@ use num_traits::FromPrimitive;
 
 use crate::aheader::*;
 use crate::config::Config;
+use crate::constants::KeyGenType;
 use crate::context::Context;
 use crate::dc_tools::EmailAddress;
 use crate::error::*;
@@ -211,11 +212,11 @@ fn load_or_generate_self_public_key(
     }
 
     let start = std::time::Instant::now();
-    info!(
-        context,
-        "Generating keypair with {} bits, e={} ...", 2048, 65537,
-    );
-    let keypair = pgp::create_keypair(EmailAddress::new(self_addr.as_ref())?)?;
+
+    let keygen_type =
+        KeyGenType::from_i32(context.get_config_int(Config::KeyGenType)).unwrap_or_default();
+    info!(context, "Generating keypair with type {}", keygen_type);
+    let keypair = pgp::create_keypair(EmailAddress::new(self_addr.as_ref())?, keygen_type)?;
     key::store_self_keypair(context, &keypair, KeyPairUse::Default)?;
     info!(
         context,

src/headerdef.rs

@@ -26,7 +26,6 @@ pub enum HeaderDef {
     ChatGroupName,
     ChatGroupNameChanged,
     ChatVerified,
-    ChatGroupImage, // deprecated
     ChatGroupAvatar,
     ChatUserAvatar,
     ChatVoiceMessage,

src/imap/client.rs

@@ -0,0 +1,104 @@
+use async_imap::{
+    error::{Error as ImapError, Result as ImapResult},
+    Client as ImapClient,
+};
+use async_native_tls::TlsStream;
+use async_std::net::{self, TcpStream};
+
+use super::session::Session;
+use crate::login_param::{dc_build_tls, CertificateChecks};
+
+#[derive(Debug)]
+pub(crate) enum Client {
+    Secure(ImapClient<TlsStream<TcpStream>>),
+    Insecure(ImapClient<TcpStream>),
+}
+
+impl Client {
+    pub async fn connect_secure<A: net::ToSocketAddrs, S: AsRef<str>>(
+        addr: A,
+        domain: S,
+        certificate_checks: CertificateChecks,
+    ) -> ImapResult<Self> {
+        let stream = TcpStream::connect(addr).await?;
+        let tls = dc_build_tls(certificate_checks);
+        let tls_stream = tls.connect(domain.as_ref(), stream).await?;
+        let mut client = ImapClient::new(tls_stream);
+        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
+            client.debug = true;
+        }
+
+        let _greeting = client
+            .read_response()
+            .await
+            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
+
+        Ok(Client::Secure(client))
+    }
+
+    pub async fn connect_insecure<A: net::ToSocketAddrs>(addr: A) -> ImapResult<Self> {
+        let stream = TcpStream::connect(addr).await?;
+
+        let mut client = ImapClient::new(stream);
+        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
+            client.debug = true;
+        }
+        let _greeting = client
+            .read_response()
+            .await
+            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
+
+        Ok(Client::Insecure(client))
+    }
+
+    pub async fn secure<S: AsRef<str>>(
+        self,
+        domain: S,
+        certificate_checks: CertificateChecks,
+    ) -> ImapResult<Client> {
+        match self {
+            Client::Insecure(client) => {
+                let tls = dc_build_tls(certificate_checks);
+                let client_sec = client.secure(domain, tls).await?;
+
+                Ok(Client::Secure(client_sec))
+            }
+            // Nothing to do
+            Client::Secure(_) => Ok(self),
+        }
+    }
+
+    pub async fn authenticate<A: async_imap::Authenticator, S: AsRef<str>>(
+        self,
+        auth_type: S,
+        authenticator: &A,
+    ) -> Result<Session, (ImapError, Client)> {
+        match self {
+            Client::Secure(i) => match i.authenticate(auth_type, authenticator).await {
+                Ok(session) => Ok(Session::Secure(session)),
+                Err((err, c)) => Err((err, Client::Secure(c))),
+            },
+            Client::Insecure(i) => match i.authenticate(auth_type, authenticator).await {
+                Ok(session) => Ok(Session::Insecure(session)),
+                Err((err, c)) => Err((err, Client::Insecure(c))),
+            },
+        }
+    }
+
+    pub async fn login<U: AsRef<str>, P: AsRef<str>>(
+        self,
+        username: U,
+        password: P,
+    ) -> Result<Session, (ImapError, Client)> {
+        match self {
+            Client::Secure(i) => match i.login(username, password).await {
+                Ok(session) => Ok(Session::Secure(session)),
+                Err((err, c)) => Err((err, Client::Secure(c))),
+            },
+            Client::Insecure(i) => match i.login(username, password).await {
+                Ok(session) => Ok(Session::Insecure(session)),
+                Err((err, c)) => Err((err, Client::Insecure(c))),
+            },
+        }
+    }
+}

src/imap/idle.rs

@@ -1,15 +1,17 @@
 use super::Imap;
 
-use async_imap::extensions::idle::IdleResponse;
+use async_imap::extensions::idle::{Handle as ImapIdleHandle, IdleResponse};
+use async_native_tls::TlsStream;
+use async_std::net::TcpStream;
 use async_std::prelude::*;
 use async_std::task;
 use std::sync::atomic::Ordering;
 use std::time::{Duration, SystemTime};
 
 use crate::context::Context;
-use crate::imap_client::*;
 
 use super::select_folder;
+use super::session::Session;
 
 type Result<T> = std::result::Result<T, Error>;
 
@@ -37,6 +39,27 @@ impl From<select_folder::Error> for Error {
     }
 }
 
+#[derive(Debug)]
+pub(crate) enum IdleHandle {
+    Secure(ImapIdleHandle<TlsStream<TcpStream>>),
+    Insecure(ImapIdleHandle<TcpStream>),
+}
+
+impl Session {
+    pub fn idle(self) -> IdleHandle {
+        match self {
+            Session::Secure(i) => {
+                let h = i.idle();
+                IdleHandle::Secure(h)
+            }
+            Session::Insecure(i) => {
+                let h = i.idle();
+                IdleHandle::Insecure(h)
+            }
+        }
+    }
+}
+
 impl Imap {
     pub fn can_idle(&self) -> bool {
         task::block_on(async move { self.config.read().await.can_idle })

src/imap/mod.rs

@@ -22,7 +22,6 @@ use crate::dc_receive_imf::{
 };
 use crate::events::Event;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
-use crate::imap_client::*;
 use crate::job::{job_add, Action};
 use crate::login_param::{CertificateChecks, LoginParam};
 use crate::message::{self, update_server_uid};
@@ -30,8 +29,13 @@ use crate::oauth2::dc_get_oauth2_access_token;
 use crate::param::Params;
 use crate::stock::StockMessage;
 
+mod client;
 mod idle;
 pub mod select_folder;
+mod session;
+
+use client::Client;
+use session::Session;
 
 type Result<T> = std::result::Result<T, Error>;
 
@@ -594,7 +598,7 @@ impl Imap {
 
         let mut list = if let Some(ref mut session) = &mut *self.session.lock().await {
             // fetch messages with larger UID than the last one seen
-            // (`UID FETCH lastseenuid+1:*)`, see RFC 4549
+            // `(UID FETCH lastseenuid+1:*)`, see RFC 4549
             let set = format!("{}:*", last_seen_uid + 1);
             match session.uid_fetch(set, PREFETCH_FLAGS).await {
                 Ok(list) => list,
@@ -751,25 +755,10 @@ impl Imap {
             return Err(Error::Other("Could not get IMAP session".to_string()));
         };
 
-        if msgs.is_empty() {
-            warn!(
-                context,
-                "Message #{} does not exist in folder \"{}\".",
-                server_uid,
-                folder.as_ref()
-            );
-        } else {
-            let msg = &msgs[0];
-
+        if let Some(msg) = msgs.first() {
             // XXX put flags into a set and pass them to dc_receive_imf
-            let is_deleted = msg.flags().any(|flag| match flag {
-                Flag::Deleted => true,
-                _ => false,
-            });
-            let is_seen = msg.flags().any(|flag| match flag {
-                Flag::Seen => true,
-                _ => false,
-            });
+            let is_deleted = msg.flags().any(|flag| flag == Flag::Deleted);
+            let is_seen = msg.flags().any(|flag| flag == Flag::Seen);
 
             if !is_deleted && msg.body().is_some() {
                 let body = msg.body().unwrap_or_default();
@@ -785,6 +774,13 @@ impl Imap {
                     );
                 }
             }
+        } else {
+            warn!(
+                context,
+                "Message #{} does not exist in folder \"{}\".",
+                server_uid,
+                folder.as_ref()
+            );
         }
 
         Ok(())
@@ -995,7 +991,6 @@ impl Imap {
         })
     }
 
-    // only returns 0 on connection problems; we should try later again in this case *
     pub fn delete_msg(
         &self,
         context: &Context,
@@ -1311,17 +1306,6 @@ fn precheck_imf(context: &Context, rfc724_mid: &str, server_folder: &str, server
     }
 }
 
-fn parse_message_id(value: &str) -> crate::error::Result<String> {
-    let addrs = mailparse::addrparse(value)
-        .map_err(|err| format_err!("failed to parse message id {:?}", err))?;
-
-    if let Some(info) = addrs.extract_single_info() {
-        return Ok(info.addr);
-    }
-
-    bail!("could not parse message_id: {}", value);
-}
-
 fn get_fetch_headers(prefetch_msg: &Fetch) -> Result<Vec<mailparse::MailHeader>> {
     let header_bytes = match prefetch_msg.header() {
         Some(header_bytes) => header_bytes,
@@ -1333,7 +1317,7 @@ fn get_fetch_headers(prefetch_msg: &Fetch) -> Result<Vec<mailparse::MailHeader>>
 
 fn prefetch_get_message_id(headers: &[mailparse::MailHeader]) -> Result<String> {
     if let Some(message_id) = headers.get_header_value(HeaderDef::MessageId)? {
-        Ok(parse_message_id(&message_id)?)
+        Ok(crate::mimeparser::parse_message_id(&message_id)?)
     } else {
         Err(Error::Other("prefetch: No message ID found".to_string()))
     }
@@ -1389,20 +1373,3 @@ fn prefetch_should_download(
     let show = show && !blocked_contact;
     Ok(show)
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_parse_message_id() {
-        assert_eq!(
-            parse_message_id("Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org").unwrap(),
-            "Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org"
-        );
-        assert_eq!(
-            parse_message_id("<Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org>").unwrap(),
-            "Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org"
-        );
-    }
-}

src/imap/session.rs

@@ -1,122 +1,18 @@
 use async_imap::{
-    error::{Error as ImapError, Result as ImapResult},
-    extensions::idle::Handle as ImapIdleHandle,
+    error::Result as ImapResult,
     types::{Capabilities, Fetch, Mailbox, Name},
-    Client as ImapClient, Session as ImapSession,
+    Session as ImapSession,
 };
 use async_native_tls::TlsStream;
-use async_std::net::{self, TcpStream};
+use async_std::net::TcpStream;
 use async_std::prelude::*;
 
-use crate::login_param::{dc_build_tls, CertificateChecks};
-
-#[derive(Debug)]
-pub(crate) enum Client {
-    Secure(ImapClient<TlsStream<TcpStream>>),
-    Insecure(ImapClient<TcpStream>),
-}
-
 #[derive(Debug)]
 pub(crate) enum Session {
     Secure(ImapSession<TlsStream<TcpStream>>),
     Insecure(ImapSession<TcpStream>),
 }
 
-#[derive(Debug)]
-pub(crate) enum IdleHandle {
-    Secure(ImapIdleHandle<TlsStream<TcpStream>>),
-    Insecure(ImapIdleHandle<TcpStream>),
-}
-
-impl Client {
-    pub async fn connect_secure<A: net::ToSocketAddrs, S: AsRef<str>>(
-        addr: A,
-        domain: S,
-        certificate_checks: CertificateChecks,
-    ) -> ImapResult<Self> {
-        let stream = TcpStream::connect(addr).await?;
-        let tls = dc_build_tls(certificate_checks);
-        let tls_stream = tls.connect(domain.as_ref(), stream).await?;
-        let mut client = ImapClient::new(tls_stream);
-        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
-            client.debug = true;
-        }
-
-        let _greeting = client
-            .read_response()
-            .await
-            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
-
-        Ok(Client::Secure(client))
-    }
-
-    pub async fn connect_insecure<A: net::ToSocketAddrs>(addr: A) -> ImapResult<Self> {
-        let stream = TcpStream::connect(addr).await?;
-
-        let mut client = ImapClient::new(stream);
-        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
-            client.debug = true;
-        }
-        let _greeting = client
-            .read_response()
-            .await
-            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
-
-        Ok(Client::Insecure(client))
-    }
-
-    pub async fn secure<S: AsRef<str>>(
-        self,
-        domain: S,
-        certificate_checks: CertificateChecks,
-    ) -> ImapResult<Client> {
-        match self {
-            Client::Insecure(client) => {
-                let tls = dc_build_tls(certificate_checks);
-                let client_sec = client.secure(domain, tls).await?;
-
-                Ok(Client::Secure(client_sec))
-            }
-            // Nothing to do
-            Client::Secure(_) => Ok(self),
-        }
-    }
-
-    pub async fn authenticate<A: async_imap::Authenticator, S: AsRef<str>>(
-        self,
-        auth_type: S,
-        authenticator: &A,
-    ) -> Result<Session, (ImapError, Client)> {
-        match self {
-            Client::Secure(i) => match i.authenticate(auth_type, authenticator).await {
-                Ok(session) => Ok(Session::Secure(session)),
-                Err((err, c)) => Err((err, Client::Secure(c))),
-            },
-            Client::Insecure(i) => match i.authenticate(auth_type, authenticator).await {
-                Ok(session) => Ok(Session::Insecure(session)),
-                Err((err, c)) => Err((err, Client::Insecure(c))),
-            },
-        }
-    }
-
-    pub async fn login<U: AsRef<str>, P: AsRef<str>>(
-        self,
-        username: U,
-        password: P,
-    ) -> Result<Session, (ImapError, Client)> {
-        match self {
-            Client::Secure(i) => match i.login(username, password).await {
-                Ok(session) => Ok(Session::Secure(session)),
-                Err((err, c)) => Err((err, Client::Secure(c))),
-            },
-            Client::Insecure(i) => match i.login(username, password).await {
-                Ok(session) => Ok(Session::Insecure(session)),
-                Err((err, c)) => Err((err, Client::Insecure(c))),
-            },
-        }
-    }
-}
-
 impl Session {
     pub async fn capabilities(&mut self) -> ImapResult<Capabilities> {
         let res = match self {
@@ -227,19 +123,6 @@ impl Session {
         Ok(res)
     }
 
-    pub fn idle(self) -> IdleHandle {
-        match self {
-            Session::Secure(i) => {
-                let h = i.idle();
-                IdleHandle::Secure(h)
-            }
-            Session::Insecure(i) => {
-                let h = i.idle();
-                IdleHandle::Insecure(h)
-            }
-        }
-    }
-
     pub async fn uid_store<S1, S2>(&mut self, uid_set: S1, query: S2) -> ImapResult<Vec<Fetch>>
     where
         S1: AsRef<str>,

src/imex.rs

@@ -10,7 +10,6 @@ use crate::blob::BlobObject;
 use crate::chat;
 use crate::chat::delete_and_reset_all_device_msgs;
 use crate::config::Config;
-use crate::configure::*;
 use crate::constants::*;
 use crate::context::Context;
 use crate::dc_tools::*;
@@ -414,7 +413,7 @@ fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Resul
     );
 
     ensure!(
-        !dc_is_configured(context),
+        !context.is_configured(),
         "Cannot import backups to accounts in use."
     );
     context.sql.close(&context);

src/job.rs

@@ -193,9 +193,31 @@ impl Job {
             Err(crate::smtp::send::Error::SendError(err)) => {
                 // Remote error, retry later.
                 warn!(context, "SMTP failed to send: {}", err);
-                smtp.disconnect();
                 self.pending_error = Some(err.to_string());
-                Status::RetryLater
+
+                let res = match err {
+                    async_smtp::smtp::error::Error::Permanent(_) => {
+                        Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
+                    }
+                    async_smtp::smtp::error::Error::Transient(_) => {
+                        // We got a transient 4xx response from SMTP server.
+                        // Give some time until the server-side error maybe goes away.
+                        Status::RetryLater
+                    }
+                    _ => {
+                        if smtp.has_maybe_stale_connection() {
+                            info!(context, "stale connection? immediately reconnecting");
+                            Status::RetryNow
+                        } else {
+                            Status::RetryLater
+                        }
+                    }
+                };
+
+                // this clears last_success info
+                smtp.disconnect();
+
+                res
             }
             Err(crate::smtp::send::Error::EnvelopeError(err)) => {
                 // Local error, job is invalid, do not retry.
@@ -919,52 +941,10 @@ fn job_perform(context: &Context, thread: Thread, probe_network: bool) {
             suspend_smtp_thread(context, true);
         }
 
-        let try_res = (0..2)
-            .map(|tries| {
-                info!(
-                    context,
-                    "{} performs immediate try {} of job {}", thread, tries, job
-                );
-
-                let try_res = match job.action {
-                    Action::Unknown => Status::Finished(Err(format_err!("Unknown job id found"))),
-                    Action::SendMsgToSmtp => job.SendMsgToSmtp(context),
-                    Action::EmptyServer => job.EmptyServer(context),
-                    Action::DeleteMsgOnImap => job.DeleteMsgOnImap(context),
-                    Action::MarkseenMsgOnImap => job.MarkseenMsgOnImap(context),
-                    Action::MarkseenMdnOnImap => job.MarkseenMdnOnImap(context),
-                    Action::MoveMsg => job.MoveMsg(context),
-                    Action::SendMdn => job.SendMdn(context),
-                    Action::ConfigureImap => JobConfigureImap(context),
-                    Action::ImexImap => match JobImexImap(context, &job) {
-                        Ok(()) => Status::Finished(Ok(())),
-                        Err(err) => {
-                            error!(context, "{}", err);
-                            Status::Finished(Err(err))
-                        }
-                    },
-                    Action::MaybeSendLocations => location::JobMaybeSendLocations(context, &job),
-                    Action::MaybeSendLocationsEnded => {
-                        location::JobMaybeSendLocationsEnded(context, &mut job)
-                    }
-                    Action::Housekeeping => {
-                        sql::housekeeping(context);
-                        Status::Finished(Ok(()))
-                    }
-                };
-
-                info!(
-                    context,
-                    "{} finished immediate try {} of job {}", thread, tries, job
-                );
-
-                try_res
-            })
-            .find(|try_res| match try_res {
-                Status::RetryNow => false,
-                _ => true,
-            })
-            .unwrap_or(Status::RetryNow);
+        let try_res = match perform_job_action(context, &mut job, thread, 0) {
+            Status::RetryNow => perform_job_action(context, &mut job, thread, 1),
+            x => x,
+        };
 
         if Action::ConfigureImap == job.action || Action::ImexImap == job.action {
             context
@@ -1055,6 +1035,45 @@ fn job_perform(context: &Context, thread: Thread, probe_network: bool) {
     }
 }
 
+fn perform_job_action(context: &Context, mut job: &mut Job, thread: Thread, tries: u32) -> Status {
+    info!(
+        context,
+        "{} begin immediate try {} of job {}", thread, tries, job
+    );
+
+    let try_res = match job.action {
+        Action::Unknown => Status::Finished(Err(format_err!("Unknown job id found"))),
+        Action::SendMsgToSmtp => job.SendMsgToSmtp(context),
+        Action::EmptyServer => job.EmptyServer(context),
+        Action::DeleteMsgOnImap => job.DeleteMsgOnImap(context),
+        Action::MarkseenMsgOnImap => job.MarkseenMsgOnImap(context),
+        Action::MarkseenMdnOnImap => job.MarkseenMdnOnImap(context),
+        Action::MoveMsg => job.MoveMsg(context),
+        Action::SendMdn => job.SendMdn(context),
+        Action::ConfigureImap => JobConfigureImap(context),
+        Action::ImexImap => match JobImexImap(context, &job) {
+            Ok(()) => Status::Finished(Ok(())),
+            Err(err) => {
+                error!(context, "{}", err);
+                Status::Finished(Err(err))
+            }
+        },
+        Action::MaybeSendLocations => location::JobMaybeSendLocations(context, &job),
+        Action::MaybeSendLocationsEnded => location::JobMaybeSendLocationsEnded(context, &mut job),
+        Action::Housekeeping => {
+            sql::housekeeping(context);
+            Status::Finished(Ok(()))
+        }
+    };
+
+    info!(
+        context,
+        "{} finished immediate try {} of job {}", thread, tries, job
+    );
+
+    try_res
+}
+
 fn get_backoff_time_offset(tries: u32) -> i64 {
     let n = 2_i32.pow(tries - 1) * 60;
     let mut rng = thread_rng();

src/lib.rs

@@ -27,23 +27,22 @@ pub(crate) mod events;
 pub use events::*;
 
 mod aheader;
-pub mod blob;
+mod blob;
 pub mod chat;
 pub mod chatlist;
 pub mod config;
-pub mod configure;
+mod configure;
 pub mod constants;
 pub mod contact;
 pub mod context;
 mod e2ee;
 mod imap;
-mod imap_client;
 pub mod imex;
 #[macro_use]
 pub mod job;
 mod job_thread;
 pub mod key;
-pub mod keyring;
+mod keyring;
 pub mod location;
 mod login_param;
 pub mod lot;

src/location.rs

@@ -364,6 +364,7 @@ fn is_marker(txt: &str) -> bool {
     txt.len() == 1 && !txt.starts_with(' ')
 }
 
+/// Deletes all locations from the database.
 pub fn delete_all(context: &Context) -> Result<(), Error> {
     sql::execute(context, &context.sql, "DELETE FROM locations;", params![])?;
     context.call_cb(Event::LocationChanged(None));
@@ -547,7 +548,7 @@ pub fn save(
 }
 
 #[allow(non_snake_case)]
-pub fn JobMaybeSendLocations(context: &Context, _job: &Job) -> job::Status {
+pub(crate) fn JobMaybeSendLocations(context: &Context, _job: &Job) -> job::Status {
     let now = time();
     let mut continue_streaming = false;
     info!(
@@ -638,7 +639,7 @@ pub fn JobMaybeSendLocations(context: &Context, _job: &Job) -> job::Status {
 }
 
 #[allow(non_snake_case)]
-pub fn JobMaybeSendLocationsEnded(context: &Context, job: &mut Job) -> job::Status {
+pub(crate) fn JobMaybeSendLocationsEnded(context: &Context, job: &mut Job) -> job::Status {
     // this function is called when location-streaming _might_ have ended for a chat.
     // the function checks, if location-streaming is really ended;
     // if so, a device-message is added if not yet done.

src/message.rs

@@ -161,7 +161,7 @@ pub struct InvalidMsgId;
     Deserialize,
 )]
 #[repr(u8)]
-pub enum MessengerMessage {
+pub(crate) enum MessengerMessage {
     No = 0,
     Yes = 1,
 

src/mimefactory.rs

@@ -68,7 +68,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
     pub fn from_msg(
         context: &'a Context,
         msg: &'b Message,
-        add_selfavatar: bool,
+        attach_selfavatar: bool,
     ) -> Result<MimeFactory<'a, 'b>, Error> {
         let chat = Chat::load_from_db(context, msg.chat_id)?;
 
@@ -156,7 +156,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             references,
             req_mdn,
             last_added_location_id: 0,
-            attach_selfavatar: add_selfavatar,
+            attach_selfavatar,
             context,
         };
         Ok(factory)
@@ -638,7 +638,6 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let command = self.msg.param.get_cmd();
         let mut placeholdertext = None;
         let mut meta_part = None;
-        let mut add_compatibility_header = false;
 
         if chat.typ == Chattype::VerifiedGroup {
             protected_headers.push(Header::new("Chat-Verified".to_string(), "1".to_string()));
@@ -681,7 +680,6 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                             "vg-member-added".to_string(),
                         ));
                     }
-                    add_compatibility_header = true;
                 }
                 SystemMessage::GroupNameChanged => {
                     let value_to_add = self.msg.param.get(Param::Arg).unwrap_or_default();
@@ -702,7 +700,6 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                             "0".to_string(),
                         ));
                     }
-                    add_compatibility_header = true;
                 }
                 _ => {}
             }
@@ -770,18 +767,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
             let (mail, filename_as_sent) = build_body_file(context, &meta, "group-image")?;
             meta_part = Some(mail);
-            protected_headers.push(Header::new(
-                "Chat-Group-Avatar".into(),
-                filename_as_sent.clone(),
-            ));
-
-            // add the old group-image headers for versions <=0.973 resp. <=beta.15 (december 2019)
-            // image deletion is not supported in the compatibility layer.
-            // this can be removed some time after releasing 1.0,
-            // grep for #DeprecatedAvatar to get the place where compatibility parsing takes place.
-            if add_compatibility_header {
-                protected_headers.push(Header::new("Chat-Group-Image".into(), filename_as_sent));
-            }
+            protected_headers.push(Header::new("Chat-Group-Avatar".into(), filename_as_sent));
         }
 
         if self.msg.viewtype == Viewtype::Sticker {
@@ -880,7 +866,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
         if self.attach_selfavatar {
             match context.get_config(Config::Selfavatar) {
-                Some(path) => match build_selfavatar_file(context, path) {
+                Some(path) => match build_selfavatar_file(context, &path) {
                     Ok((part, filename)) => {
                         parts.push(part);
                         protected_headers.push(Header::new("Chat-User-Avatar".into(), filename))
@@ -1075,7 +1061,7 @@ fn build_body_file(
     Ok((mail, filename_to_send))
 }
 
-fn build_selfavatar_file(context: &Context, path: String) -> Result<(PartBuilder, String), Error> {
+fn build_selfavatar_file(context: &Context, path: &str) -> Result<(PartBuilder, String), Error> {
     let blob = BlobObject::from_path(context, path)?;
     let filename_to_send = match blob.suffix() {
         Some(suffix) => format!("avatar.{}", suffix),

src/mimeparser.rs

@@ -5,6 +5,7 @@ use lettre_email::mime::{self, Mime};
 use mailparse::{DispositionType, MailAddr, MailHeaderMap};
 
 use crate::aheader::Aheader;
+use crate::bail;
 use crate::blob::BlobObject;
 use crate::config::Config;
 use crate::constants::Viewtype;
@@ -24,7 +25,6 @@ use crate::peerstate::Peerstate;
 use crate::securejoin::handle_degrade_event;
 use crate::simplify::*;
 use crate::stock::StockMessage;
-use crate::{bail, ensure};
 
 /// A parsed MIME message.
 ///
@@ -46,28 +46,17 @@ pub struct MimeMessage {
     pub is_system_message: SystemMessage,
     pub location_kml: Option<location::Kml>,
     pub message_kml: Option<location::Kml>,
-    pub user_avatar: AvatarAction,
-    pub group_avatar: AvatarAction,
+    pub(crate) user_avatar: Option<AvatarAction>,
+    pub(crate) group_avatar: Option<AvatarAction>,
     pub(crate) reports: Vec<Report>,
 }
 
 #[derive(Debug, PartialEq)]
-pub enum AvatarAction {
-    None,
+pub(crate) enum AvatarAction {
     Delete,
     Change(String),
 }
 
-impl AvatarAction {
-    pub fn is_change(&self) -> bool {
-        match self {
-            AvatarAction::None => false,
-            AvatarAction::Delete => false,
-            AvatarAction::Change(_) => true,
-        }
-    }
-}
-
 #[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql)]
 #[repr(i32)]
 pub enum SystemMessage {
@@ -163,8 +152,8 @@ impl MimeMessage {
             is_system_message: SystemMessage::Unknown,
             location_kml: None,
             message_kml: None,
-            user_avatar: AvatarAction::None,
-            group_avatar: AvatarAction::None,
+            user_avatar: None,
+            group_avatar: None,
         };
         parser.parse_mime_recursive(context, &mail)?;
         parser.parse_headers(context)?;
@@ -202,10 +191,6 @@ impl MimeMessage {
     fn parse_avatar_headers(&mut self) {
         if let Some(header_value) = self.get(HeaderDef::ChatGroupAvatar).cloned() {
             self.group_avatar = self.avatar_action_from_header(header_value);
-        } else if let Some(header_value) = self.get(HeaderDef::ChatGroupImage).cloned() {
-            // parse the old group-image headers for versions <=0.973 resp. <=beta.15 (december 2019)
-            // grep for #DeprecatedAvatar to get the place where a compatibility header is generated.
-            self.group_avatar = self.avatar_action_from_header(header_value);
         }
 
         if let Some(header_value) = self.get(HeaderDef::ChatUserAvatar).cloned() {
@@ -360,9 +345,9 @@ impl MimeMessage {
         Ok(())
     }
 
-    fn avatar_action_from_header(&mut self, header_value: String) -> AvatarAction {
+    fn avatar_action_from_header(&mut self, header_value: String) -> Option<AvatarAction> {
         if header_value == "0" {
-            return AvatarAction::Delete;
+            Some(AvatarAction::Delete)
         } else {
             let mut i = 0;
             while i != self.parts.len() {
@@ -370,7 +355,7 @@ impl MimeMessage {
                 if let Some(part_filename) = &part.org_filename {
                     if part_filename == &header_value {
                         if let Some(blob) = part.param.get(Param::File) {
-                            let res = AvatarAction::Change(blob.to_string());
+                            let res = Some(AvatarAction::Change(blob.to_string()));
                             self.parts.remove(i);
                             return res;
                         }
@@ -379,8 +364,8 @@ impl MimeMessage {
                 }
                 i += 1;
             }
+            None
         }
-        AvatarAction::None
     }
 
     pub fn was_encrypted(&self) -> bool {
@@ -593,60 +578,64 @@ impl MimeMessage {
         let (mime_type, msg_type) = get_mime_type(mail)?;
         let raw_mime = mail.ctype.mimetype.to_lowercase();
 
-        let filename = get_attachment_filename(mail);
+        let filename = get_attachment_filename(mail)?;
 
         let old_part_count = self.parts.len();
 
-        if let Ok(filename) = filename {
-            self.do_add_single_file_part(
-                context,
-                msg_type,
-                mime_type,
-                &raw_mime,
-                &mail.get_body_raw()?,
-                &filename,
-            );
-        } else {
-            match mime_type.type_() {
-                mime::IMAGE | mime::AUDIO | mime::VIDEO | mime::APPLICATION => {
-                    bail!("missing attachment");
-                }
-                mime::TEXT | mime::HTML => {
-                    let decoded_data = match mail.get_body() {
-                        Ok(decoded_data) => decoded_data,
-                        Err(err) => {
-                            warn!(context, "Invalid body parsed {:?}", err);
-                            // Note that it's not always an error - might be no data
-                            return Ok(false);
-                        }
-                    };
-
-                    let (simplified_txt, is_forwarded) = if decoded_data.is_empty() {
-                        ("".into(), false)
-                    } else {
-                        let is_html = mime_type == mime::TEXT_HTML;
-                        let out = if is_html {
-                            dehtml(&decoded_data)
-                        } else {
-                            decoded_data.clone()
+        match filename {
+            Some(filename) => {
+                self.do_add_single_file_part(
+                    context,
+                    msg_type,
+                    mime_type,
+                    &raw_mime,
+                    &mail.get_body_raw()?,
+                    &filename,
+                );
+            }
+            None => {
+                match mime_type.type_() {
+                    mime::IMAGE | mime::AUDIO | mime::VIDEO | mime::APPLICATION => {
+                        warn!(context, "Missing attachment");
+                        return Ok(false);
+                    }
+                    mime::TEXT | mime::HTML => {
+                        let decoded_data = match mail.get_body() {
+                            Ok(decoded_data) => decoded_data,
+                            Err(err) => {
+                                warn!(context, "Invalid body parsed {:?}", err);
+                                // Note that it's not always an error - might be no data
+                                return Ok(false);
+                            }
                         };
-                        simplify(out, self.has_chat_version())
-                    };
 
-                    if !simplified_txt.is_empty() {
-                        let mut part = Part::default();
-                        part.typ = Viewtype::Text;
-                        part.mimetype = Some(mime_type);
-                        part.msg = simplified_txt;
-                        part.msg_raw = Some(decoded_data);
-                        self.do_add_single_part(part);
-                    }
+                        let (simplified_txt, is_forwarded) = if decoded_data.is_empty() {
+                            ("".into(), false)
+                        } else {
+                            let is_html = mime_type == mime::TEXT_HTML;
+                            let out = if is_html {
+                                dehtml(&decoded_data)
+                            } else {
+                                decoded_data.clone()
+                            };
+                            simplify(out, self.has_chat_version())
+                        };
 
-                    if is_forwarded {
-                        self.is_forwarded = true;
+                        if !simplified_txt.is_empty() {
+                            let mut part = Part::default();
+                            part.typ = Viewtype::Text;
+                            part.mimetype = Some(mime_type);
+                            part.msg = simplified_txt;
+                            part.msg_raw = Some(decoded_data);
+                            self.do_add_single_part(part);
+                        }
+
+                        if is_forwarded {
+                            self.is_forwarded = true;
+                        }
                     }
+                    _ => {}
                 }
-                _ => {}
             }
         }
 
@@ -752,7 +741,7 @@ impl MimeMessage {
 
     pub fn get_rfc724_mid(&self) -> Option<String> {
         self.get(HeaderDef::MessageId)
-            .and_then(|msgid| parse_message_id(msgid))
+            .and_then(|msgid| parse_message_id(msgid).ok())
     }
 
     fn merge_headers(headers: &mut HashMap<String, String>, fields: &[mailparse::MailHeader<'_>]) {
@@ -790,14 +779,16 @@ impl MimeMessage {
                 .get_header_value(HeaderDef::OriginalMessageId)
                 .ok()
                 .flatten()
-                .and_then(|v| parse_message_id(&v))
+                .and_then(|v| parse_message_id(&v).ok())
             {
                 let additional_message_ids = report_fields
                     .get_header_value(HeaderDef::AdditionalMessageIds)
                     .ok()
                     .flatten()
                     .map_or_else(Vec::new, |v| {
-                        v.split(' ').filter_map(parse_message_id).collect()
+                        v.split(' ')
+                            .filter_map(|s| parse_message_id(s).ok())
+                            .collect()
                     });
 
                 return Ok(Some(Report {
@@ -920,14 +911,20 @@ pub(crate) struct Report {
     additional_message_ids: Vec<String>,
 }
 
-fn parse_message_id(field: &str) -> Option<String> {
-    if let Ok(addrs) = mailparse::addrparse(field) {
-        // Assume the message id is a single id in the form of <id>
-        if let mailparse::MailAddr::Single(mailparse::SingleInfo { ref addr, .. }) = addrs[0] {
-            return Some(addr.clone());
+pub(crate) fn parse_message_id(value: &str) -> crate::error::Result<String> {
+    let ids = mailparse::msgidparse(value)
+        .map_err(|err| format_err!("failed to parse message id {:?}", err))?;
+
+    if ids.len() == 1 {
+        let id = &ids[0];
+        if id.starts_with('<') && id.ends_with('>') {
+            Ok(id.chars().skip(1).take(id.len() - 2).collect())
+        } else {
+            bail!("message-ID {} is not enclosed in < and >", value);
         }
+    } else {
+        bail!("could not parse message_id: {}", value);
     }
-    None
 }
 
 fn is_known(key: &str) -> bool {
@@ -1001,45 +998,48 @@ fn is_attachment_disposition(mail: &mailparse::ParsedMail<'_>) -> bool {
     false
 }
 
-fn get_attachment_filename(mail: &mailparse::ParsedMail) -> Result<String> {
+/// Tries to get attachment filename.
+///
+/// If filename is explitictly specified in Content-Disposition, it is
+/// returned. If Content-Disposition is "attachment" but filename is
+/// not specified, filename is guessed. If Content-Disposition cannot
+/// be parsed, returns an error.
+fn get_attachment_filename(mail: &mailparse::ParsedMail) -> Result<Option<String>> {
     // try to get file name from
     //    `Content-Disposition: ... filename*=...`
     // or `Content-Disposition: ... filename*0*=... filename*1*=... filename*2*=...`
     // or `Content-Disposition: ... filename=...`
 
     let ct = mail.get_content_disposition()?;
-    ensure!(
-        ct.disposition == DispositionType::Attachment,
-        "disposition not an attachment: {:?}",
-        ct.disposition
-    );
 
-    let mut desired_filename = ct
+    let desired_filename: Option<String> = ct
         .params
         .iter()
         .filter(|(key, _value)| key.starts_with("filename"))
-        .fold(String::new(), |mut acc, (_key, value)| {
-            acc += value;
-            acc
+        .fold(None, |acc, (_key, value)| {
+            if let Some(acc) = acc {
+                Some(acc + value)
+            } else {
+                Some(value.to_string())
+            }
         });
 
-    if desired_filename.is_empty() {
-        if let Some(param) = ct.params.get("name") {
-            // might be a wrongly encoded filename
-            desired_filename = param.to_string();
-        }
-    }
+    let desired_filename =
+        desired_filename.or_else(|| ct.params.get("name").map(|s| s.to_string()));
 
-    // if there is still no filename, guess one
-    if desired_filename.is_empty() {
+    // If there is no filename, but part is an attachment, guess filename
+    if ct.disposition == DispositionType::Attachment && desired_filename.is_none() {
         if let Some(subtype) = mail.ctype.mimetype.split('/').nth(1) {
-            desired_filename = format!("file.{}", subtype,);
+            Ok(Some(format!("file.{}", subtype,)))
         } else {
-            bail!("could not determine filename: {:?}", ct.disposition);
+            bail!(
+                "could not determine attachment filename: {:?}",
+                ct.disposition
+            );
         }
+    } else {
+        Ok(desired_filename)
     }
-
-    Ok(desired_filename)
 }
 
 // returned addresses are normalized and lowercased.
@@ -1093,6 +1093,15 @@ mod tests {
     use super::*;
     use crate::test_utils::*;
 
+    impl AvatarAction {
+        pub fn is_change(&self) -> bool {
+            match self {
+                AvatarAction::Delete => false,
+                AvatarAction::Change(_) => true,
+            }
+        }
+    }
+
     #[test]
     fn test_dc_mimeparser_crash() {
         let context = dummy_context();
@@ -1151,10 +1160,12 @@ mod tests {
     fn test_get_attachment_filename() {
         let raw = include_bytes!("../test-data/message/html_attach.eml");
         let mail = mailparse::parse_mail(raw).unwrap();
-        assert!(get_attachment_filename(&mail).is_err());
-        assert!(get_attachment_filename(&mail.subparts[0]).is_err());
+        assert!(get_attachment_filename(&mail).unwrap().is_none());
+        assert!(get_attachment_filename(&mail.subparts[0])
+            .unwrap()
+            .is_none());
         let filename = get_attachment_filename(&mail.subparts[1]).unwrap();
-        assert_eq!(filename, "test.html")
+        assert_eq!(filename, Some("test.html".to_string()))
     }
 
     #[test]
@@ -1240,29 +1251,29 @@ mod tests {
 
         let raw = include_bytes!("../test-data/message/mail_attach_txt.eml");
         let mimeparser = MimeMessage::from_bytes(&t.ctx, &raw[..]).unwrap();
-        assert_eq!(mimeparser.user_avatar, AvatarAction::None);
-        assert_eq!(mimeparser.group_avatar, AvatarAction::None);
+        assert_eq!(mimeparser.user_avatar, None);
+        assert_eq!(mimeparser.group_avatar, None);
 
         let raw = include_bytes!("../test-data/message/mail_with_user_avatar.eml");
         let mimeparser = MimeMessage::from_bytes(&t.ctx, &raw[..]).unwrap();
         assert_eq!(mimeparser.parts.len(), 1);
         assert_eq!(mimeparser.parts[0].typ, Viewtype::Text);
-        assert!(mimeparser.user_avatar.is_change());
-        assert_eq!(mimeparser.group_avatar, AvatarAction::None);
+        assert!(mimeparser.user_avatar.unwrap().is_change());
+        assert_eq!(mimeparser.group_avatar, None);
 
         let raw = include_bytes!("../test-data/message/mail_with_user_avatar_deleted.eml");
         let mimeparser = MimeMessage::from_bytes(&t.ctx, &raw[..]).unwrap();
         assert_eq!(mimeparser.parts.len(), 1);
         assert_eq!(mimeparser.parts[0].typ, Viewtype::Text);
-        assert_eq!(mimeparser.user_avatar, AvatarAction::Delete);
-        assert_eq!(mimeparser.group_avatar, AvatarAction::None);
+        assert_eq!(mimeparser.user_avatar, Some(AvatarAction::Delete));
+        assert_eq!(mimeparser.group_avatar, None);
 
         let raw = include_bytes!("../test-data/message/mail_with_user_and_group_avatars.eml");
         let mimeparser = MimeMessage::from_bytes(&t.ctx, &raw[..]).unwrap();
         assert_eq!(mimeparser.parts.len(), 1);
         assert_eq!(mimeparser.parts[0].typ, Viewtype::Text);
-        assert!(mimeparser.user_avatar.is_change());
-        assert!(mimeparser.group_avatar.is_change());
+        assert!(mimeparser.user_avatar.unwrap().is_change());
+        assert!(mimeparser.group_avatar.unwrap().is_change());
 
         // if the Chat-User-Avatar header is missing, the avatar become a normal attachment
         let raw = include_bytes!("../test-data/message/mail_with_user_and_group_avatars.eml");
@@ -1271,8 +1282,8 @@ mod tests {
         let mimeparser = MimeMessage::from_bytes(&t.ctx, raw.as_bytes()).unwrap();
         assert_eq!(mimeparser.parts.len(), 1);
         assert_eq!(mimeparser.parts[0].typ, Viewtype::Image);
-        assert_eq!(mimeparser.user_avatar, AvatarAction::None);
-        assert!(mimeparser.group_avatar.is_change());
+        assert_eq!(mimeparser.user_avatar, None);
+        assert!(mimeparser.group_avatar.unwrap().is_change());
     }
 
     #[test]
@@ -1489,4 +1500,40 @@ Additional-Message-IDs: <foo@example.com> <foo@example.net>\n\
             &["foo@example.com", "foo@example.net"]
         );
     }
+
+    #[test]
+    fn test_parse_inline_attachment() {
+        let context = dummy_context();
+        let raw = br#"Date: Thu, 13 Feb 2020 22:41:20 +0000 (UTC)
+From: sender@example.com
+To: receiver@example.com
+Subject: Mail with inline attachment
+MIME-Version: 1.0
+Content-Type: multipart/mixed;
+	boundary="----=_Part_25_46172632.1581201680436"
+
+------=_Part_25_46172632.1581201680436
+Content-Type: text/plain; charset=utf-8
+
+Hello!
+
+------=_Part_25_46172632.1581201680436
+Content-Type: application/pdf; name="some_pdf.pdf"
+Content-Transfer-Encoding: base64
+Content-Disposition: inline; filename="some_pdf.pdf"
+
+JVBERi0xLjUKJcOkw7zDtsOfCjIgMCBvYmoKPDwvTGVuZ3RoIDMgMCBSL0ZpbHRlci9GbGF0ZURl
+Y29kZT4+CnN0cmVhbQp4nGVOuwoCMRDs8xVbC8aZvC4Hx4Hno7ATAhZi56MTtPH33YtXiLKQ3ZnM
+MDYyMDYxNTE1RTlDOEE4Cj4+CnN0YXJ0eHJlZgo4Mjc4CiUlRU9GCg==
+------=_Part_25_46172632.1581201680436--
+"#;
+
+        let message = MimeMessage::from_bytes(&context.ctx, &raw[..]).unwrap();
+        assert_eq!(
+            message.get_subject(),
+            Some("Mail with inline attachment".to_string())
+        );
+
+        assert_eq!(message.parts.len(), 2);
+    }
 }

src/peerstate.rs

@@ -316,6 +316,25 @@ impl<'a> Peerstate<'a> {
                 self.recalc_fingerprint();
                 self.to_save = Some(ToSave::All)
             }
+
+            // This is non-standard.
+            //
+            // According to Autocrypt 1.1.0 gossip headers SHOULD NOT
+            // contain encryption preference, but we include it into
+            // Autocrypt-Gossip and apply it one way (from
+            // "nopreference" to "mutual").
+            //
+            // This is compatible to standard clients, because they
+            // can't distinguish it from the case where we have
+            // contacted the client in the past and received this
+            // preference via Autocrypt header.
+            if self.last_seen_autocrypt == 0
+                && self.prefer_encrypt == EncryptPreference::NoPreference
+                && gossip_header.prefer_encrypt == EncryptPreference::Mutual
+            {
+                self.prefer_encrypt = EncryptPreference::Mutual;
+                self.to_save = Some(ToSave::All);
+            }
         };
     }
 
@@ -326,7 +345,15 @@ impl<'a> Peerstate<'a> {
             let header = Aheader::new(
                 self.addr.clone(),
                 public_key,
-                EncryptPreference::NoPreference,
+                // Autocrypt 1.1.0 specification says that
+                // `prefer-encrypt` attribute SHOULD NOT be included,
+                // but we include it anyway to propagate encryption
+                // preference to new members in group chats.
+                if self.last_seen_autocrypt > 0 {
+                    self.prefer_encrypt
+                } else {
+                    EncryptPreference::NoPreference
+                },
             );
             Some(header.to_string())
         } else {

src/pgp.rs

@@ -16,6 +16,7 @@ use pgp::types::{
 };
 use rand::{thread_rng, CryptoRng, Rng};
 
+use crate::constants::KeyGenType;
 use crate::dc_tools::EmailAddress;
 use crate::error::Result;
 use crate::key::*;
@@ -147,10 +148,18 @@ pub struct KeyPair {
 }
 
 /// Create a new key pair.
-pub(crate) fn create_keypair(addr: EmailAddress) -> std::result::Result<KeyPair, PgpKeygenError> {
+pub(crate) fn create_keypair(
+    addr: EmailAddress,
+    keygen_type: KeyGenType,
+) -> std::result::Result<KeyPair, PgpKeygenError> {
+    let (secret_key_type, public_key_type) = match keygen_type {
+        KeyGenType::Rsa2048 => (PgpKeyType::Rsa(2048), PgpKeyType::Rsa(2048)),
+        KeyGenType::Ed25519 | KeyGenType::Default => (PgpKeyType::EdDSA, PgpKeyType::ECDH),
+    };
+
     let user_id = format!("<{}>", addr);
     let key_params = SecretKeyParamsBuilder::default()
-        .key_type(PgpKeyType::Rsa(2048))
+        .key_type(secret_key_type)
         .can_create_certificates(true)
         .can_sign(true)
         .primary_user_id(user_id)
@@ -173,7 +182,7 @@ pub(crate) fn create_keypair(addr: EmailAddress) -> std::result::Result<KeyPair,
         ])
         .subkey(
             SubkeyParamsBuilder::default()
-                .key_type(PgpKeyType::Rsa(2048))
+                .key_type(public_key_type)
                 .can_encrypt(true)
                 .passphrase(None)
                 .build()
@@ -387,8 +396,16 @@ mod tests {
     #[test]
     #[ignore] // is too expensive
     fn test_create_keypair() {
-        let keypair0 = create_keypair(EmailAddress::new("foo@bar.de").unwrap()).unwrap();
-        let keypair1 = create_keypair(EmailAddress::new("two@zwo.de").unwrap()).unwrap();
+        let keypair0 = create_keypair(
+            EmailAddress::new("foo@bar.de").unwrap(),
+            KeyGenType::Default,
+        )
+        .unwrap();
+        let keypair1 = create_keypair(
+            EmailAddress::new("two@zwo.de").unwrap(),
+            KeyGenType::Default,
+        )
+        .unwrap();
         assert_ne!(keypair0.public, keypair1.public);
     }
 
@@ -556,4 +573,88 @@ mod tests {
         .unwrap();
         assert_eq!(plain, CLEARTEXT);
     }
+
+    fn test_encrypt_decrypt_fuzz(key_type: KeyGenType) {
+        // sending messages from Alice to Bob
+        let alice = create_keypair(EmailAddress::new("a@e.org").unwrap(), key_type).unwrap();
+        let bob = create_keypair(EmailAddress::new("b@e.org").unwrap(), key_type).unwrap();
+
+        let alice_secret = Key::from(alice.secret.clone());
+        let alice_public = Key::from(alice.public.clone());
+        let bob_secret = Key::from(bob.secret.clone());
+        let bob_public = Key::from(bob.public.clone());
+
+        let plain: &[u8] = b"just a test";
+
+        let mut encr_keyring = Keyring::default();
+        encr_keyring.add_ref(&bob_public);
+
+        let mut decr_keyring = Keyring::default();
+        decr_keyring.add_ref(&bob_secret);
+        let mut validate_keyring = Keyring::default();
+        validate_keyring.add_ref(&alice_public);
+
+        for _ in 0..1000 {
+            let ctext = pk_encrypt(plain.as_ref(), &encr_keyring, Some(&alice_secret)).unwrap();
+            let plain2 =
+                pk_decrypt(ctext.as_ref(), &decr_keyring, &validate_keyring, None).unwrap();
+            assert_eq!(plain2, plain);
+        }
+    }
+
+    #[test]
+    #[ignore]
+    fn test_encrypt_decrypt_fuzz_rsa() {
+        test_encrypt_decrypt_fuzz(KeyGenType::Rsa2048)
+    }
+
+    #[test]
+    fn test_encrypt_decrypt_fuzz_ecc() {
+        test_encrypt_decrypt_fuzz(KeyGenType::Ed25519)
+    }
+
+    #[test]
+    fn test_encrypt_decrypt_fuzz_alice() -> Result<()> {
+        let plain: &[u8] = b"just a test";
+        let lit_msg = Message::new_literal_bytes("", plain);
+
+        let mut rng = thread_rng();
+
+        let enc_key =
+            SignedPublicKey::from_base64(include_str!("../test-data/key/alice-public.asc"))
+                .unwrap();
+        let enc_subkey = &enc_key.public_subkeys[0];
+        assert!(enc_subkey.is_encryption_key());
+        let pkeys: Vec<&SignedPublicSubKey> = vec![&enc_subkey];
+
+        let skey = SignedSecretKey::from_base64(include_str!("../test-data/key/alice-secret.asc"))
+            .unwrap();
+        let skeys: Vec<&SignedSecretKey> = vec![&skey];
+
+        for _ in 0..1000 {
+            let msg = lit_msg.encrypt_to_keys(
+                &mut rng,
+                pgp::crypto::sym::SymmetricKeyAlgorithm::AES128,
+                &pkeys[..],
+            )?;
+            let ctext = msg.to_armored_string(None)?;
+
+            println!("{}", ctext);
+
+            let (decryptor, _) = msg.decrypt(|| "".into(), || "".into(), &skeys[..])?;
+            let msgs = decryptor.collect::<pgp::errors::Result<Vec<_>>>()?;
+            ensure!(!msgs.is_empty(), "No valid messages found");
+
+            let dec_msg = &msgs[0];
+
+            let plain2: Vec<u8> = match dec_msg.get_content()? {
+                Some(content) => content,
+                None => bail!("Decrypted message is empty"),
+            };
+
+            assert_eq!(plain2, plain);
+        }
+
+        Ok(())
+    }
 }

src/provider/data.rs

@@ -20,6 +20,16 @@ lazy_static::lazy_static! {
         ],
     };
 
+    // aol.md: aol.com
+    static ref P_AOL: Provider = Provider {
+        status: Status::PREPARATION,
+        before_login_hint: "To log in to AOL with Delta Chat, you need to set up an app password in the AOL web interface.",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/aol",
+        server: vec![
+        ],
+    };
+
     // autistici.org.md: autistici.org
     static ref P_AUTISTICI_ORG: Provider = Provider {
         status: Status::OK,
@@ -65,6 +75,16 @@ lazy_static::lazy_static! {
         ],
     };
 
+    // fastmail.md: fastmail.com
+    static ref P_FASTMAIL: Provider = Provider {
+        status: Status::PREPARATION,
+        before_login_hint: "You must create an app-specific password for Delta Chat before you can log in.",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/fastmail",
+        server: vec![
+        ],
+    };
+
     // freenet.de.md: freenet.de
     static ref P_FREENET_DE: Provider = Provider {
         status: Status::OK,
@@ -165,12 +185,45 @@ lazy_static::lazy_static! {
         ],
     };
 
+    // protonmail.md: protonmail.com, protonmail.ch
+    static ref P_PROTONMAIL: Provider = Provider {
+        status: Status::BROKEN,
+        before_login_hint: "Protonmail does not offer the standard IMAP e-mail protocol, so you cannot log in with Delta Chat to Protonmail.",
+        after_login_hint: "To use Delta Chat with Protonmail, the IMAP bridge must be running in the background. If you have connectivity issues, double check whether it works as expected.",
+        overview_page: "https://providers.delta.chat/protonmail",
+        server: vec![
+        ],
+    };
+
     // riseup.net.md: riseup.net
     // - skipping provider with status OK and no special things to do
 
     // rogers.com.md: rogers.com
     // - skipping provider with status OK and no special things to do
 
+    // t-online.md: t-online.de, magenta.de
+    static ref P_T_ONLINE: Provider = Provider {
+        status: Status::PREPARATION,
+        before_login_hint: "To use Delta Chat with a T-Online email address, you need to create an app password in the web interface.",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/t-online",
+        server: vec![
+        ],
+    };
+
+    // testrun.md: testrun.org
+    static ref P_TESTRUN: Provider = Provider {
+        status: Status::OK,
+        before_login_hint: "",
+        after_login_hint: "",
+        overview_page: "https://providers.delta.chat/testrun",
+        server: vec![
+            Server { protocol: IMAP, socket: SSL, hostname: "testrun.org", port: 993, username_pattern: EMAIL },
+            Server { protocol: IMAP, socket: STARTTLS, hostname: "testrun.org", port: 143, username_pattern: EMAIL },
+            Server { protocol: SMTP, socket: STARTTLS, hostname: "testrun.org", port: 587, username_pattern: EMAIL },
+        ],
+    };
+
     // tiscali.it.md: tiscali.it
     static ref P_TISCALI_IT: Provider = Provider {
         status: Status::OK,
@@ -229,15 +282,14 @@ lazy_static::lazy_static! {
         ],
     };
 
-    // zoho.com.md: zoho.com
-    // - skipping provider with status OK and no special things to do
-
     pub static ref PROVIDER_DATA: HashMap<&'static str, &'static Provider> = [
         ("aktivix.org", &*P_AKTIVIX_ORG),
+        ("aol.com", &*P_AOL),
         ("autistici.org", &*P_AUTISTICI_ORG),
         ("bluewin.ch", &*P_BLUEWIN_CH),
         ("example.com", &*P_EXAMPLE_COM),
         ("example.org", &*P_EXAMPLE_COM),
+        ("fastmail.com", &*P_FASTMAIL),
         ("freenet.de", &*P_FREENET_DE),
         ("gmail.com", &*P_GMAIL),
         ("googlemail.com", &*P_GMAIL),
@@ -260,6 +312,11 @@ lazy_static::lazy_static! {
         ("outlook.com.tr", &*P_OUTLOOK_COM),
         ("live.com", &*P_OUTLOOK_COM),
         ("posteo.de", &*P_POSTEO),
+        ("protonmail.com", &*P_PROTONMAIL),
+        ("protonmail.ch", &*P_PROTONMAIL),
+        ("t-online.de", &*P_T_ONLINE),
+        ("magenta.de", &*P_T_ONLINE),
+        ("testrun.org", &*P_TESTRUN),
         ("tiscali.it", &*P_TISCALI_IT),
         ("web.de", &*P_WEB_DE),
         ("email.de", &*P_WEB_DE),

src/provider/mod.rs

@@ -1,3 +1,5 @@
+//! [Provider database](https://providers.delta.chat/) module
+
 mod data;
 
 use crate::dc_tools::EmailAddress;

src/smtp/mod.rs

@@ -2,7 +2,7 @@
 
 pub mod send;
 
-use std::time::Duration;
+use std::time::{Duration, Instant};
 
 use async_smtp::smtp::client::net::*;
 use async_smtp::*;
@@ -53,8 +53,14 @@ pub type Result<T> = std::result::Result<T, Error>;
 pub struct Smtp {
     #[debug_stub(some = "SmtpTransport")]
     transport: Option<smtp::SmtpTransport>,
+
     /// Email address we are sending from.
     from: Option<EmailAddress>,
+
+    /// Timestamp of last successful send/receive network interaction
+    /// (eg connect or send succeeded). On initialization and disconnect
+    /// it is set to None.
+    last_success: Option<Instant>,
 }
 
 impl Smtp {
@@ -68,6 +74,17 @@ impl Smtp {
         if let Some(mut transport) = self.transport.take() {
             async_std::task::block_on(transport.close()).ok();
         }
+        self.last_success = None;
+    }
+
+    /// Return true if smtp was connected but is not known to
+    /// have been successfully used the last 60 seconds
+    pub fn has_maybe_stale_connection(&self) -> bool {
+        if let Some(last_success) = self.last_success {
+            Instant::now().duration_since(last_success).as_secs() > 60
+        } else {
+            false
+        }
     }
 
     /// Check whether we are connected.
@@ -161,6 +178,7 @@ impl Smtp {
         trans.connect().await.map_err(Error::ConnectionFailure)?;
 
         self.transport = Some(trans);
+        self.last_success = Some(Instant::now());
         context.call_cb(Event::SmtpConnected(format!(
             "SMTP-LOGIN as {} ok",
             lp.send_user,

src/smtp/send.rs

@@ -53,6 +53,8 @@ impl Smtp {
                 "Message len={} was smtp-sent to {}",
                 message_len, recipients_display
             )));
+            self.last_success = Some(std::time::Instant::now());
+
             Ok(())
         } else {
             warn!(

src/sql.rs

@@ -113,16 +113,16 @@ impl Sql {
         self.with_conn(|conn| conn.execute(sql, params).map_err(Into::into))
     }
 
-    fn with_conn<T, G>(&self, g: G) -> Result<T>
+    pub fn with_conn<T, G>(&self, g: G) -> Result<T>
     where
-        G: FnOnce(&Connection) -> Result<T>,
+        G: FnOnce(&mut Connection) -> Result<T>,
     {
         let res = match &*self.pool.read().unwrap() {
             Some(pool) => {
-                let conn = pool.get()?;
+                let mut conn = pool.get()?;
 
                 // Only one process can make changes to the database at one time.
-                // busy_timeout defines, that if a seconds process wants write access,
+                // busy_timeout defines, that if a second process wants write access,
                 // this second process will wait some milliseconds
                 // and try over until it gets write access or the given timeout is elapsed.
                 // If the second process does not get write access within the given timeout,
@@ -130,7 +130,7 @@ impl Sql {
                 // (without a busy_timeout, sqlite3_step() would return SQLITE_BUSY _at once_)
                 conn.busy_timeout(Duration::from_secs(10))?;
 
-                g(&conn)
+                g(&mut conn)
             }
             None => Err(Error::SqlNoConnection),
         };
@@ -360,7 +360,7 @@ impl Sql {
             .and_then(|r| r.parse().ok())
     }
 
-    fn start_stmt(&self, stmt: impl AsRef<str>) {
+    pub fn start_stmt(&self, stmt: impl AsRef<str>) {
         if let Some(query) = self.in_use.get_cloned() {
             let bt = backtrace::Backtrace::new();
             eprintln!("old query: {}", query);
@@ -893,6 +893,11 @@ fn open(
             )?;
             sql.set_raw_config_int(context, "dbversion", 62)?;
         }
+        if dbversion < 63 {
+            info!(context, "[migration] v63");
+            sql.execute("UPDATE chats SET grpid='' WHERE type=100", NO_PARAMS)?;
+            sql.set_raw_config_int(context, "dbversion", 63)?;
+        }
 
         // (2) updates that require high-level objects
         // (the structure is complete now and all objects are usable)

test-data/key/alice-public.asc

@@ -1 +1 @@
-xsBNBF425W8BCADLIbltPzG1vk/V2ov2+eBeJJJnRu1kJHdo6e3oNB+HTIxVde5+7Uq8tTEDZB1O7m9NBUFrXr7UYQsA/86G2jmsyWKTzIu1O/t5kdcNDqsNcTVZAhBu2ixYsYVc3ws6kJONjpXLtD2u3P7vEXU3INiOb2JrBQDT8/ubEm1xas/UirYnP5DMaH068IHRdVEYs9ULFaD5scw1m/94buXYZ1CRt/2hT8iRrtBi6ki8kArnhsZC2Xr0+jRQNMUnG5k7Bwi6saCqVmd7IlqSM6MbfYank30Gi/UyDmyIrOk7daTg6WIqgiVOTHav65EK/aUvvjlr+awM+C+u35rQytzyTitZABEBAAHNEzxhbGljZUBleGFtcGxlLmNvbT7CwIkEEAEIADMCGQEFAl425ZQCGwMECwkIBwYVCAkKCwIDFgIBFiEEsBJRVVptIGB7DRLzYuJiDHjRb8EACgkQYuJiDHjRb8HiZQf+PLDxzWchkHAdQFbxxtoXj66aiknofjlRWHDWvUG4nULZ15tjDjnv3z22Meldr8kSV4r1+ejhLFHou9gTzAYk7eAxiybDd8AJOdK+ZgK/Nn7xjdO+HTZLhNdi+R7EektDyf8WDNktEaS8pZc74VKu4984ESi4PoqVxqGHRiSisH4cw4b2pQYxp32BkIdil7sWnqRUEoCpMoKdw2h0N7/lm+rS7/JR9cdjXaVzy1dYTqAVsTL1FTGy4osOKGOyQbkP+Cm6uNq7kC/Bt+fefsb+c2JycmI1uwdvnG7PoFslKv3lRnfkNSmrcIYlJHUl5z0yAgliophr5fqMfzQpO4zMc87ATQReNuVvAQgAuNjE1i+g4v25UNDPIMgXODU4WztE30074gQs5sZa0DQnDUMsdWc2g1o060YZDojMYJQAtBjlW1Dz8FEE7WsLNohGtRyUWmIgNxE5CpodjpwIZ0MdO4Aji0YM+g+WsOSS8kiHMs+dMFfQJuNKjujGFaMIciSaMMrUmPtzkQ/o8NEJs2Aftw90fpVR+M7Mue3++rcEX09ntbgqkgm8SV6OIrOY2kfILudtybocgYkCTeNVqz5VFXuxrnT4ceyFQ64JkwsZxb+X/pCm4V5Q2TbKRwtdonU8HfAz0nAd5tsNeGmf/dPLOKBCxlNEme399YmzWrT+kJBp7CIH5jlWQKyuLwARAQABwsB2BBgBCAAgBQJeNuWUAhsMFiEEsBJRVVptIGB7DRLzYuJiDHjRb8EACgkQYuJiDHjRb8HrEgf/Xu8eRPPdskwtyd98y64teidBpkHuIjuZKJpNyy2HhdGXQwYbNIzwINg0EJ+u2nkreNF/h2Lu+/saqI8Dai02dpYXjvxJIlCgP2os7sNhVaZSaS4XmmJjkHCfZuIKblZypKDJVc5AceZxrtvUbgG+94+H3zeRWVAA30S5ep6YPvxigvhmQah/sdzY7708/jd9uXcCbkP47PBaXCpuPiYLb3t7z8mOteJb7LOZUmSI1efiLDLTGj7ofkdDfA7E6/nF/1+nq+UIDWqljwiUzeNIJsFlZRa/9/uDEjcQbaDe9/knBs7k9pEDZX5u8SSwSED75L+OvRpFWenp4SSKvd2BUw==
+mDMEXlh13RYJKwYBBAHaRw8BAQdAzfVIAleCXMJrq8VeLlEVof6ITCviMktKjmcBKAu4m5C0GUFsaWNlIDxhbGljZUBleGFtcGxlLm9yZz6IkAQTFggAOBYhBC5vossjtTLXKGNLWGSwj2Gp7ZRDBQJeWHXdAhsDBQsJCAcCBhUKCQgLAgQWAgMBAh4BAheAAAoJEGSwj2Gp7ZRDE3oA/i4MCyDMTsjWqDZoQwX/A/GoTO2/V0wKPhjJJy/8m2pMAPkBjOnGOtx2SZpQvJGTa9h804RY6iDrRuI8A/8tEEXAA7g4BF5Ydd0SCisGAQQBl1UBBQEBB0AG7cjWy2SFAU8KnltlubVW67rFiyfp01JrRe6Xqy22HQMBCAeIeAQYFggAIBYhBC5vossjtTLXKGNLWGSwj2Gp7ZRDBQJeWHXdAhsMAAoJEGSwj2Gp7ZRDLo8BAObE8GnsGVwKzNqCvHeWgJsqhjS3C6gvSlV3tEm9XmF6AQDXucIyVfoBwoyMh2h6cSn/ATn5QJb35pgo+ivp3jsMAg==

test-data/key/alice-secret.asc

@@ -1 +1 @@
-xcLYBF425W8BCADLIbltPzG1vk/V2ov2+eBeJJJnRu1kJHdo6e3oNB+HTIxVde5+7Uq8tTEDZB1O7m9NBUFrXr7UYQsA/86G2jmsyWKTzIu1O/t5kdcNDqsNcTVZAhBu2ixYsYVc3ws6kJONjpXLtD2u3P7vEXU3INiOb2JrBQDT8/ubEm1xas/UirYnP5DMaH068IHRdVEYs9ULFaD5scw1m/94buXYZ1CRt/2hT8iRrtBi6ki8kArnhsZC2Xr0+jRQNMUnG5k7Bwi6saCqVmd7IlqSM6MbfYank30Gi/UyDmyIrOk7daTg6WIqgiVOTHav65EK/aUvvjlr+awM+C+u35rQytzyTitZABEBAAEAB/oDQFnwdrd7+jza5nGhFWTS/PDe+FKqbK8AneXx9ouepcoFQCr+Gxw8IwZS0JJrhgOADxp59n1FdvwvGukaXXnY2yxZw0dlMj2XN49ipR51y58X+qF6tMFK9iR1VRif6lqCRIr/RLZMCzuFZhkjNcJhnUTNA7p8qgYX+FaKHzSOaVat/v0kIUHUcZDkREWPUESYDmc1Nv6FXhB0WBiTsBglF+fq5Rm7UWPSmA59Cr7BrW8DctbzTh0+6bkzum2xdOcZ59nuTZa+IKcReI1+kVne5JPNFNJ2tP2f9GSSlL7u+NBtx3zRxZgAotXcJK9cVNIWtegqf+2hoLvm7m2CkWKRBADglpC7TpjV+8wJH+KuyGQ7jepqzf5EHwMrK2i6lPnnmoi0nkKvkklvtdcC7FoFGtLCDJ7vwlUdeN+itDxPlP8bbbUabcy0lLuzyGOVt5NwYXgIuPicpdt2ZTJgvChd9oWi1DG8pVpm+EMJZPyYVEpvDGl6q95oktrytbqjASZbBQQA54roJnwBcptLMTrttDrglULX7ciSKY5HXN1c0rqZn1dTKB1nPYB26hNbu6lZ8ixSOyZm3KwpeDUNW7A3hyzXOfoGFPaddH6WMSFFsGGC/orRVxnuPZLr3UJ3uFX7J0JOav90n/6A4YmS7uImRAG4/vTrAbEfmlBl5msHVUaYh0UD/jSX22JLenO1o8pNU04JQl3lQ4mWY6MvgTyCvpchTzDDva+wdOBTUeVUmb/KqYkYBq98tXl1VnGnNpeEymUISSi60RjaXDhbg7a3ELV0yvvWcBN9zreyyINuCU5OmNefPRvPt4Co12KtIxPACByFNTevzPKbrXd1cyhHOxAuqfzLRbDNEzxhbGljZUBleGFtcGxlLmNvbT7CwIkEEAEIADMCGQEFAl425ZQCGwMECwkIBwYVCAkKCwIDFgIBFiEEsBJRVVptIGB7DRLzYuJiDHjRb8EACgkQYuJiDHjRb8HiZQf+PLDxzWchkHAdQFbxxtoXj66aiknofjlRWHDWvUG4nULZ15tjDjnv3z22Meldr8kSV4r1+ejhLFHou9gTzAYk7eAxiybDd8AJOdK+ZgK/Nn7xjdO+HTZLhNdi+R7EektDyf8WDNktEaS8pZc74VKu4984ESi4PoqVxqGHRiSisH4cw4b2pQYxp32BkIdil7sWnqRUEoCpMoKdw2h0N7/lm+rS7/JR9cdjXaVzy1dYTqAVsTL1FTGy4osOKGOyQbkP+Cm6uNq7kC/Bt+fefsb+c2JycmI1uwdvnG7PoFslKv3lRnfkNSmrcIYlJHUl5z0yAgliophr5fqMfzQpO4zMc8fC2AReNuVvAQgAuNjE1i+g4v25UNDPIMgXODU4WztE30074gQs5sZa0DQnDUMsdWc2g1o060YZDojMYJQAtBjlW1Dz8FEE7WsLNohGtRyUWmIgNxE5CpodjpwIZ0MdO4Aji0YM+g+WsOSS8kiHMs+dMFfQJuNKjujGFaMIciSaMMrUmPtzkQ/o8NEJs2Aftw90fpVR+M7Mue3++rcEX09ntbgqkgm8SV6OIrOY2kfILudtybocgYkCTeNVqz5VFXuxrnT4ceyFQ64JkwsZxb+X/pCm4V5Q2TbKRwtdonU8HfAz0nAd5tsNeGmf/dPLOKBCxlNEme399YmzWrT+kJBp7CIH5jlWQKyuLwARAQABAAf/YmpfWp5fLZvjJ8kVDqIZ4r5LNB+5Sp7nbC3G7lPblBDAXgpOyG9ckdDcbguTWa6yChWizkCXFOhkCKZKVlHw1Wb3JoSB5CFsf4U29pMZe41N2BTeoohV5Fg2nojgNWxtZHwDJ6VsTonidGH9l1sN5AU6gPNF+QZ07MKsRCbRYi0yMgX064gwZXRtkm8AECz8ay1wDzoBy14ALe9aDClafVwfxdYUcxDBqtvjLhGeTWX5lMMAQ1Ix8D0Gp4r0Zvtl+oxlTSZFAt9m6sbRBbJf4LJjRQh07aWF2gUOiyIyz7YymYdwsyFnCPn2Aj84uRdqYCekAUfzBeNTBukUQq1DYQQA3BeH38pnr34m0UyD/tCrTvrX60MOJVvFuaTQw+IgY4XmT9UiiiqYMaoLfzxeevdMCQ9EtMdXUTjI27/II3dR5Obg6J0QTybj78IKPbH8Vdlg0etllRjC3bV/M4a5UcXPKG6W5CvB0UJg7eqn/8wUqwiL9x+hZoLy+nU5rCAjzZEEANcBK8Vy9eBxkKmEfH/mChDSKE82ua0xdQuZiTvvGedUYG3ucH4rAlkZaZcZrtJTod1BKhAhDBrjxk/yLCjK3z5JDsacdDGGfaqga3zdPBJubWE7f4mg6uYVs04Uf90YVY7t0LEQAh7i9QYiIqUOJDy3L8y3+bNgNz2r1p8pFd+/A/0YoYE0YDgABbLKmBQFoWjF3Op7P+k6Z4ENK4Me1fkNSAU451QX7ZduI7i3pGTM06bXG2umhTI1lg48ZveMRk1vBezHU+ThnciEkuhYafnq7NRdkEtI20MyFmN7dZF8LQ/joYKsJbeSG5svj8f1ue2eHkiIIlTtDqVUTizDU3ddlzUZwsB2BBgBCAAgBQJeNuWUAhsMFiEEsBJRVVptIGB7DRLzYuJiDHjRb8EACgkQYuJiDHjRb8HrEgf/Xu8eRPPdskwtyd98y64teidBpkHuIjuZKJpNyy2HhdGXQwYbNIzwINg0EJ+u2nkreNF/h2Lu+/saqI8Dai02dpYXjvxJIlCgP2os7sNhVaZSaS4XmmJjkHCfZuIKblZypKDJVc5AceZxrtvUbgG+94+H3zeRWVAA30S5ep6YPvxigvhmQah/sdzY7708/jd9uXcCbkP47PBaXCpuPiYLb3t7z8mOteJb7LOZUmSI1efiLDLTGj7ofkdDfA7E6/nF/1+nq+UIDWqljwiUzeNIJsFlZRa/9/uDEjcQbaDe9/knBs7k9pEDZX5u8SSwSED75L+OvRpFWenp4SSKvd2BUw==
+lFgEXlh13RYJKwYBBAHaRw8BAQdAzfVIAleCXMJrq8VeLlEVof6ITCviMktKjmcBKAu4m5AAAQDMpCY4sD5/DUR0jRjGC5WstwShz1q+5Vofo5mY9+XRXRA3tBlBbGljZSA8YWxpY2VAZXhhbXBsZS5vcmc+iJAEExYIADgWIQQub6LLI7Uy1yhjS1hksI9hqe2UQwUCXlh13QIbAwULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRBksI9hqe2UQxN6AP4uDAsgzE7I1qg2aEMF/wPxqEztv1dMCj4YyScv/JtqTAD5AYzpxjrcdkmaULyRk2vYfNOEWOog60biPAP/LRBFwAOcXQReWHXdEgorBgEEAZdVAQUBAQdABu3I1stkhQFPCp5bZbm1Vuu6xYsn6dNSa0Xul6stth0DAQgHAAD/X9y9I/JFBeArkgR3U363cWXXxMCWftS+BDwM9zE4PrgQb4h4BBgWCAAgFiEELm+iyyO1MtcoY0tYZLCPYantlEMFAl5Ydd0CGwwACgkQZLCPYantlEMujwEA5sTwaewZXArM2oK8d5aAmyqGNLcLqC9KVXe0Sb1eYXoBANe5wjJV+gHCjIyHaHpxKf8BOflAlvfmmCj6K+neOwwC