fix #616 -- allow invalid certs for smtp and imap connections -- this is the behaviour of C-core.

it's eg used in server-filters for detecting DC messages

.gitignore

@@ -16,6 +16,7 @@ python/.tox
 *.egg-info
 __pycache__
 python/src/deltachat/capi*.so
+python/.venv/
 
 python/liveconfig*
 

deltachat-ffi/src/lib.rs

@@ -275,11 +275,11 @@ pub unsafe extern "C" fn dc_is_open(context: *mut dc_context_t) -> libc::c_int {
         eprintln!("ignoring careless call to dc_is_open()");
         return 0;
     }
-    let ffi_context = &mut *context;
+    let ffi_context = &*context;
     let inner_guard = ffi_context.inner.read().unwrap();
     match *inner_guard {
-        Some(_) => 0,
-        None => 1,
+        Some(_) => 1,
+        None => 0,
     }
 }
 
@@ -1563,7 +1563,13 @@ pub unsafe extern "C" fn dc_initiate_key_transfer(context: *mut dc_context_t) ->
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| dc_imex::dc_initiate_key_transfer(ctx))
+        .with_inner(|ctx| match dc_imex::dc_initiate_key_transfer(ctx) {
+            Ok(res) => res.strdup(),
+            Err(err) => {
+                error!(ctx, "dc_initiate_key_transfer(): {}", err);
+                ptr::null_mut()
+            }
+        })
         .unwrap_or_else(|_| ptr::null_mut())
 }
 

examples/repl/cmdline.rs

@@ -96,16 +96,7 @@ pub unsafe fn dc_reset_tables(context: &Context, bits: i32) -> i32 {
 fn dc_poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<(), Error> {
     let data = dc_read_file(context, filename)?;
 
-    unsafe {
-        dc_receive_imf(
-            context,
-            data.as_ptr() as *const _,
-            data.len(),
-            "import",
-            0,
-            0,
-        )
-    };
+    unsafe { dc_receive_imf(context, &data, "import", 0, 0) };
     Ok(())
 }
 
@@ -414,18 +405,13 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
                  ============================================="
             ),
         },
-        "initiate-key-transfer" => {
-            let setup_code = dc_initiate_key_transfer(context);
-            if !setup_code.is_null() {
-                println!(
-                    "Setup code for the transferred setup message: {}",
-                    as_str(setup_code),
-                );
-                free(setup_code as *mut libc::c_void);
-            } else {
-                bail!("Failed to generate setup code");
-            };
-        }
+        "initiate-key-transfer" => match dc_initiate_key_transfer(context) {
+            Ok(setup_code) => println!(
+                "Setup code for the transferred setup message: {}",
+                setup_code,
+            ),
+            Err(err) => bail!("Failed to generate setup code: {}", err),
+        },
         "get-setupcodebegin" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let msg_id: u32 = arg1.parse()?;

proptest-regressions/dc_strencode.txt

@@ -5,3 +5,4 @@
 # It is recommended to check this file in to source control so that
 # everyone who runs the test benefits from these saved cases.
 cc 679506fe9ac59df773f8cfa800fdab5f0a32fe49d2ab370394000a1aa5bc2a72 # shrinks to buf = "%0A"
+cc e34960438edb2426904b44fb4215154e7e2880f2fd1c3183b98bfcc76fec4882 # shrinks to input = " 0"

python/src/deltachat/account.py

@@ -22,7 +22,7 @@ class Account(object):
     by the underlying deltachat c-library.  All public Account methods are
     meant to be memory-safe and return memory-safe objects.
     """
-    def __init__(self, db_path, logid=None, eventlogging=True):
+    def __init__(self, db_path, logid=None, eventlogging=True, debug=True):
         """ initialize account object.
 
         :param db_path: a path to the account database. The database
@@ -30,13 +30,14 @@ class Account(object):
         :param logid: an optional logging prefix that should be used with
                       the default internal logging.
         :param eventlogging: if False no eventlogging and no context callback will be configured
+        :param debug: turn on debug logging for events.
         """
         self._dc_context = ffi.gc(
             lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
             _destroy_dc_context,
         )
         if eventlogging:
-            self._evlogger = EventLogger(self._dc_context, logid)
+            self._evlogger = EventLogger(self._dc_context, logid, debug)
             deltachat.set_context_callback(self._dc_context, self._process_event)
             self._threads = IOThreads(self._dc_context, self._evlogger._log_event)
         else:

python/tests/test_lowlevel.py

@@ -130,3 +130,21 @@ def test_get_info_open(tmpdir):
     info = cutil.from_dc_charpointer(lib.dc_get_info(ctx))
     assert 'deltachat_core_version' in info
     assert 'database_dir' in info
+
+
+def test_is_open_closed():
+    ctx = ffi.gc(
+        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_unref,
+    )
+    assert lib.dc_is_open(ctx) == 0
+
+
+def test_is_open_actually_open(tmpdir):
+    ctx = ffi.gc(
+        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_unref,
+    )
+    db_fname = tmpdir.join("test.db")
+    lib.dc_open(ctx, db_fname.strpath.encode("ascii"), ffi.NULL)
+    assert lib.dc_is_open(ctx) == 1

spec.md

@@ -117,7 +117,8 @@ The sender plus the recipients are the group members.
 To allow different groups with the same members,
 groups are identified by a group-id.
 The group-id MUST be created only from the characters
-`0`-`9`, `A`-`Z`, `a`-`z` `_` and `-`.
+`0`-`9`, `A`-`Z`, `a`-`z` `_` and `-`
+and MUST have a length of at least 11 characters.
 
 Groups MUST have a group-name.
 The group-name is any non-zero-length UTF-8 string.
@@ -144,9 +145,9 @@ The message-id MUST have the format `Gr.<group-id>.<unique data>`.
     From: member1@domain
     To: member2@domain, member3@domain
     Chat-Version: 1.0
-    Chat-Group-ID: 1234xyZ
+    Chat-Group-ID: 12345uvwxyZ
     Chat-Group-Name: My Group
-    Message-ID: Gr.1234xyZ.0001@domain
+    Message-ID: Gr.12345uvwxyZ.0001@domain
     Subject: Chat: My Group: Hello group ...
 
     Hello group - this group contains three members
@@ -196,10 +197,10 @@ and the message SHOULD appear as a message or action from the sender.
     From: member1@domain
     To: member2@domain, member3@domain, member4@domain
     Chat-Version: 1.0
-    Chat-Group-ID: 1234xyZ
+    Chat-Group-ID: 12345uvwxyZ
     Chat-Group-Name: My Group
     Chat-Group-Member-Added: member4@domain
-    Message-ID: Gr.1234xyZ.0002@domain
+    Message-ID: Gr.12345uvwxyZ.0002@domain
     Subject: Chat: My Group: Hello, ...
 
     Hello, I've added member4@domain to our group.  Now we have 4 members.
@@ -209,10 +210,10 @@ To remove a member:
     From: member1@domain
     To: member2@domain, member3@domain
     Chat-Version: 1.0
-    Chat-Group-ID: 1234xyZ
+    Chat-Group-ID: 12345uvwxyZ
     Chat-Group-Name: My Group
     Chat-Group-Member-Removed: member4@domain
-    Message-ID: Gr.1234xyZ.0003@domain
+    Message-ID: Gr.12345uvwxyZ.0003@domain
     Subject: Chat: My Group: Hello, ...
 
     Hello, I've removed member4@domain from our group.  Now we have 3 members.
@@ -233,10 +234,10 @@ and the message SHOULD appear as a message or action from the sender.
     From: member1@domain
     To: member2@domain, member3@domain
     Chat-Version: 1.0
-    Chat-Group-ID: 1234xyZ
+    Chat-Group-ID: 12345uvwxyZ
     Chat-Group-Name: Our Group
     Chat-Group-Name-Changed: My Group
-    Message-ID: Gr.1234xyZ.0004@domain
+    Message-ID: Gr.12345uvwxyZ.0004@domain
     Subject: Chat: Our Group: Hello, ...
 
     Hello, I've changed the group name from "My Group" to "Our Group".
@@ -262,10 +263,10 @@ and the message SHOULD appear as a message or action from the sender.
     From: member1@domain
     To: member2@domain, member3@domain
     Chat-Version: 1.0
-    Chat-Group-ID: 1234xyZ
+    Chat-Group-ID: 12345uvwxyZ
     Chat-Group-Name: Our Group
     Chat-Group-Image: image.jpg
-    Message-ID: Gr.1234xyZ.0005@domain
+    Message-ID: Gr.12345uvwxyZ.0005@domain
     Subject: Chat: Our Group: Hello, ...
     Content-Type: multipart/mixed; boundary="==break=="
 

src/dc_imex.rs

@@ -75,11 +75,9 @@ pub fn dc_imex_has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Resu
     }
 }
 
-pub unsafe fn dc_initiate_key_transfer(context: &Context) -> *mut libc::c_char {
+pub fn dc_initiate_key_transfer(context: &Context) -> Result<String> {
     let mut msg: Message;
-    if !dc_alloc_ongoing(context) {
-        return std::ptr::null_mut();
-    }
+    ensure!(dc_alloc_ongoing(context), "could not allocate ongoing");
     let setup_code = dc_create_setup_code(context);
     /* this may require a keypair to be created. this may take a second ... */
     if !context
@@ -149,7 +147,7 @@ pub unsafe fn dc_initiate_key_transfer(context: &Context) -> *mut libc::c_char {
     }
     dc_free_ongoing(context);
 
-    setup_code.strdup()
+    Ok(setup_code)
 }
 
 /// Renders HTML body of a setup file message.
@@ -244,14 +242,15 @@ pub fn dc_continue_key_transfer(context: &Context, msg_id: u32, setup_code: &str
     if let Some(filename) = msg.get_file(context) {
         if let Ok(buf) = dc_read_file(context, filename) {
             let norm_sc = CString::yolo(dc_normalize_setup_code(setup_code));
-            let armored_key: String;
             unsafe {
-                let sc = dc_decrypt_setup_file(context, norm_sc.as_ptr(), buf.as_ptr().cast());
-                ensure!(!sc.is_null(), "bad setup code");
-                armored_key = to_string(sc);
-                free(sc as *mut libc::c_void);
+                if let Ok(armored_key) =
+                    dc_decrypt_setup_file(context, norm_sc.as_ptr(), buf.as_ptr().cast())
+                {
+                    set_self_key(context, &armored_key, true, true)?;
+                } else {
+                    bail!("Bad setup code.")
+                }
             }
-            set_self_key(context, &armored_key, true, true)?;
             Ok(())
         } else {
             bail!("Cannot read Autocrypt Setup Message file.");
@@ -333,7 +332,7 @@ pub unsafe fn dc_decrypt_setup_file(
     context: &Context,
     passphrase: *const libc::c_char,
     filecontent: *const libc::c_char,
-) -> *mut libc::c_char {
+) -> Result<String> {
     let fc_buf: *mut libc::c_char;
     let mut fc_headerline = String::default();
     let mut fc_base64: *const libc::c_char = ptr::null();
@@ -341,7 +340,8 @@ pub unsafe fn dc_decrypt_setup_file(
     let mut binary_bytes: libc::size_t = 0;
     let mut indx: libc::size_t = 0;
 
-    let mut payload: *mut libc::c_char = ptr::null_mut();
+    let mut payload: Result<String> = Err(format_err!("Failed to decrypt"));
+
     fc_buf = dc_strdup(filecontent);
     if dc_split_armored_data(
         fc_buf,
@@ -369,12 +369,10 @@ pub unsafe fn dc_decrypt_setup_file(
                 as_str(passphrase),
                 std::slice::from_raw_parts(binary as *const u8, binary_bytes),
             ) {
-                Ok(plain) => {
-                    let payload_c = CString::new(plain).unwrap();
-                    payload = strdup(payload_c.as_ptr());
-                }
+                Ok(plain) => payload = Ok(String::from_utf8(plain).unwrap()),
                 Err(err) => {
                     error!(context, "Failed to decrypt message: {:?}", err);
+                    payload = Err(err);
                 }
             }
         }
@@ -554,7 +552,7 @@ fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Resul
 The macro avoids weird values of 0% or 100% while still working. */
 #[allow(non_snake_case)]
 unsafe fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
-    let mut ok_to_continue: bool;
+    let mut ok_to_continue = true;
     let mut success = false;
 
     let mut delete_dest_file: libc::c_int = 0;
@@ -571,54 +569,55 @@ unsafe fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()>
 
     sql::try_execute(context, &context.sql, "VACUUM;").ok();
     context.sql.close(context);
-    let mut closed = true;
     info!(
         context,
         "Backup \"{}\" to \"{}\".",
         context.get_dbfile().display(),
         dest_path_filename.display(),
     );
-    if dc_copy_file(context, context.get_dbfile(), &dest_path_filename) {
-        context.sql.open(&context, &context.get_dbfile(), 0);
-        closed = false;
-        /* add all files as blobs to the database copy (this does not require the source to be locked, neigher the destination as it is used only here) */
-        /*for logging only*/
-        let sql = Sql::new();
-        if sql.open(context, &dest_path_filename, 0) {
-            if !sql.table_exists("backup_blobs") {
-                if sql::execute(
-                    context,
-                    &sql,
-                    "CREATE TABLE backup_blobs (id INTEGER PRIMARY KEY, file_name, file_content);",
-                    params![],
-                )
-                .is_err()
-                {
-                    /* error already logged */
-                    ok_to_continue = false;
-                } else {
-                    ok_to_continue = true;
-                }
-            } else {
-                ok_to_continue = true;
+    let copied = dc_copy_file(context, context.get_dbfile(), &dest_path_filename);
+    context.sql.open(&context, &context.get_dbfile(), 0);
+    if !copied {
+        let s = dest_path_filename.to_string_lossy().to_string();
+        bail!(
+            "could not copy file from {:?} to {:?}",
+            context.get_dbfile(),
+            s
+        );
+    }
+    /* add all files as blobs to the database copy (this does not require the source to be locked, neigher the destination as it is used only here) */
+    /*for logging only*/
+    let sql = Sql::new();
+    if sql.open(context, &dest_path_filename, 0) {
+        if !sql.table_exists("backup_blobs") {
+            if sql::execute(
+                context,
+                &sql,
+                "CREATE TABLE backup_blobs (id INTEGER PRIMARY KEY, file_name, file_content);",
+                params![],
+            )
+            .is_err()
+            {
+                /* error already logged */
+                ok_to_continue = false;
             }
-            if ok_to_continue {
-                let mut total_files_cnt = 0;
-                let dir = context.get_blobdir();
-                if let Ok(dir_handle) = std::fs::read_dir(&dir) {
-                    total_files_cnt += dir_handle.filter(|r| r.is_ok()).count();
+        }
+        if ok_to_continue {
+            let mut total_files_cnt = 0;
+            let dir = context.get_blobdir();
+            if let Ok(dir_handle) = std::fs::read_dir(&dir) {
+                total_files_cnt += dir_handle.filter(|r| r.is_ok()).count();
 
-                    info!(context, "EXPORT: total_files_cnt={}", total_files_cnt);
-                    if total_files_cnt > 0 {
-                        // scan directory, pass 2: copy files
-                        if let Ok(dir_handle) = std::fs::read_dir(&dir) {
-                            sql.prepare(
+                info!(context, "EXPORT: total_files_cnt={}", total_files_cnt);
+                if total_files_cnt > 0 {
+                    // scan directory, pass 2: copy files
+                    if let Ok(dir_handle) = std::fs::read_dir(&dir) {
+                        sql.prepare(
                                     "INSERT INTO backup_blobs (file_name, file_content) VALUES (?, ?);",
                                     move |mut stmt, _| {
                                         let mut processed_files_cnt = 0;
                                         for entry in dir_handle {
                                             if entry.is_err() {
-                                                ok_to_continue = true;
                                                 break;
                                             }
                                             let entry = entry.unwrap();
@@ -677,39 +676,34 @@ unsafe fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()>
                                         Ok(())
                                     }
                                 ).unwrap();
-                        } else {
-                            error!(
-                                context,
-                                "Backup: Cannot copy from blob-directory \"{}\".",
-                                context.get_blobdir().display(),
-                            );
-                        }
                     } else {
-                        info!(context, "Backup: No files to copy.",);
-                        ok_to_continue = true;
-                    }
-                    if ok_to_continue {
-                        if sql
-                            .set_config_int(context, "backup_time", now as i32)
-                            .is_ok()
-                        {
-                            context.call_cb(Event::ImexFileWritten(dest_path_filename.clone()));
-                            success = true;
-                        }
+                        error!(
+                            context,
+                            "Backup: Cannot copy from blob-directory \"{}\".",
+                            context.get_blobdir().display(),
+                        );
                     }
                 } else {
-                    error!(
-                        context,
-                        "Backup: Cannot get info for blob-directory \"{}\".",
-                        context.get_blobdir().display(),
-                    );
-                };
-            }
+                    info!(context, "Backup: No files to copy.",);
+                }
+                if ok_to_continue {
+                    if sql
+                        .set_config_int(context, "backup_time", now as i32)
+                        .is_ok()
+                    {
+                        context.call_cb(Event::ImexFileWritten(dest_path_filename.clone()));
+                        success = true;
+                    }
+                }
+            } else {
+                error!(
+                    context,
+                    "Backup: Cannot get info for blob-directory \"{}\".",
+                    context.get_blobdir().display(),
+                );
+            };
         }
     }
-    if closed {
-        context.sql.open(&context, &context.get_dbfile(), 0);
-    }
     if 0 != delete_dest_file {
         dc_delete_file(context, &dest_path_filename);
     }

src/dc_mimeparser.rs

@@ -1422,7 +1422,6 @@ mod tests {
     }
 
     #[test]
-    #[ignore]
     fn test_dc_mimeparser_crash() {
         let context = dummy_context();
         let raw = include_bytes!("../test-data/message/issue_523.txt");

src/dc_receive_imf.rs

@@ -39,8 +39,7 @@ enum CreateEvent {
 /// Receive a message and add it to the database.
 pub unsafe fn dc_receive_imf(
     context: &Context,
-    imf_raw_not_terminated: *const libc::c_char,
-    imf_raw_bytes: libc::size_t,
+    imf_raw: &[u8],
     server_folder: impl AsRef<str>,
     server_uid: u32,
     flags: u32,
@@ -61,9 +60,8 @@ pub unsafe fn dc_receive_imf(
     // we use mailmime_parse() through dc_mimeparser (both call mailimf_struct_multiple_parse()
     // somewhen, I did not found out anything that speaks against this approach yet)
 
-    let body = std::slice::from_raw_parts(imf_raw_not_terminated as *const u8, imf_raw_bytes);
     let mut mime_parser = MimeParser::new(context);
-    mime_parser.parse(body);
+    mime_parser.parse(imf_raw);
 
     if mime_parser.header.is_empty() {
         // Error - even adding an empty record won't help as we do not know the message ID
@@ -204,8 +202,7 @@ pub unsafe fn dc_receive_imf(
         if let Err(err) = add_parts(
             context,
             &mut mime_parser,
-            imf_raw_not_terminated,
-            imf_raw_bytes,
+            imf_raw,
             incoming,
             &mut incoming_origin,
             server_folder.as_ref(),
@@ -292,8 +289,7 @@ pub unsafe fn dc_receive_imf(
 unsafe fn add_parts(
     context: &Context,
     mut mime_parser: &mut MimeParser,
-    imf_raw_not_terminated: *const libc::c_char,
-    imf_raw_bytes: libc::size_t,
+    imf_raw: &[u8],
     incoming: i32,
     incoming_origin: &mut Origin,
     server_folder: impl AsRef<str>,
@@ -683,10 +679,7 @@ unsafe fn add_parts(
                         part.bytes,
                         *hidden,
                         if save_mime_headers {
-                            Some(String::from_utf8_lossy(std::slice::from_raw_parts(
-                                imf_raw_not_terminated as *const u8,
-                                imf_raw_bytes,
-                            )))
+                            Some(String::from_utf8_lossy(imf_raw))
                         } else {
                             None
                         },

src/dc_strencode.rs

@@ -1,3 +1,4 @@
+use itertools::Itertools;
 use std::borrow::Cow;
 use std::ffi::CString;
 use std::ptr;
@@ -5,7 +6,6 @@ use std::ptr;
 use charset::Charset;
 use libc::{free, strlen};
 use mmime::mailmime::decode::mailmime_encoded_phrase_parse;
-use mmime::mmapstring::*;
 use mmime::other::*;
 use percent_encoding::{percent_decode, utf8_percent_encode, AsciiSet, CONTROLS};
 
@@ -25,195 +25,49 @@ use crate::dc_tools::*;
  * @return Returns the encoded string which must be free()'d when no longed needed.
  *     On errors, NULL is returned.
  */
-pub unsafe fn dc_encode_header_words(to_encode_r: impl AsRef<str>) -> String {
-    let to_encode =
-        CString::new(to_encode_r.as_ref().as_bytes()).expect("invalid cstring to_encode");
-
-    let mut ok_to_continue = true;
-    let mut ret_str: *mut libc::c_char = ptr::null_mut();
-    let mut cur: *const libc::c_char = to_encode.as_ptr();
-    let mmapstr: *mut MMAPString = mmap_string_new(b"\x00" as *const u8 as *const libc::c_char);
-    if mmapstr.is_null() {
-        ok_to_continue = false;
-    }
-    loop {
-        if !ok_to_continue {
-            if !mmapstr.is_null() {
-                mmap_string_free(mmapstr);
-            }
-            break;
-        } else {
-            if *cur as libc::c_int != '\u{0}' as i32 {
-                let begin: *const libc::c_char;
-                let mut end: *const libc::c_char;
-                let mut do_quote: bool;
-                let mut quote_words: libc::c_int;
-                begin = cur;
-                end = begin;
-                quote_words = 0i32;
-                do_quote = true;
-                while *cur as libc::c_int != '\u{0}' as i32 {
-                    get_word(cur, &mut cur, &mut do_quote);
-                    if !do_quote {
-                        break;
-                    }
-                    quote_words = 1i32;
-                    end = cur;
-                    if *cur as libc::c_int != '\u{0}' as i32 {
-                        cur = cur.offset(1isize)
-                    }
-                }
-                if 0 != quote_words {
-                    if !quote_word(
-                        mmapstr,
-                        begin,
-                        end.wrapping_offset_from(begin) as libc::size_t,
-                    ) {
-                        ok_to_continue = false;
-                        continue;
-                    }
-                    if *end as libc::c_int == ' ' as i32 || *end as libc::c_int == '\t' as i32 {
-                        if mmap_string_append_c(mmapstr, *end).is_null() {
-                            ok_to_continue = false;
-                            continue;
-                        }
-                        end = end.offset(1isize)
-                    }
-                    if *end as libc::c_int != '\u{0}' as i32 {
-                        if mmap_string_append_len(
-                            mmapstr,
-                            end,
-                            cur.wrapping_offset_from(end) as libc::size_t,
-                        )
-                        .is_null()
-                        {
-                            ok_to_continue = false;
-                            continue;
-                        }
-                    }
-                } else if mmap_string_append_len(
-                    mmapstr,
-                    begin,
-                    cur.wrapping_offset_from(begin) as libc::size_t,
-                )
-                .is_null()
-                {
-                    ok_to_continue = false;
-                    continue;
-                }
-                if !(*cur as libc::c_int == ' ' as i32 || *cur as libc::c_int == '\t' as i32) {
-                    continue;
-                }
-                if mmap_string_append_c(mmapstr, *cur).is_null() {
-                    ok_to_continue = false;
-                    continue;
-                }
-                cur = cur.offset(1isize);
-            } else {
-                ret_str = strdup((*mmapstr).str_0);
-                ok_to_continue = false;
-            }
-        }
+pub fn dc_encode_header_words(input: impl AsRef<str>) -> String {
+    let mut result = String::default();
+    for (_, group) in &input.as_ref().chars().group_by(|c| c.is_whitespace()) {
+        let word: String = group.collect();
+        result.push_str(&quote_word(&word.as_bytes()));
     }
 
-    let s = to_string(ret_str);
-    free(ret_str.cast());
-    s
+    result
 }
 
-unsafe fn quote_word(
-    mmapstr: *mut MMAPString,
-    word: *const libc::c_char,
-    size: libc::size_t,
-) -> bool {
-    let mut cur: *const libc::c_char;
-    let mut i = 0;
-    let mut hex: [libc::c_char; 4] = [0; 4];
-    // let mut col: libc::c_int = 0i32;
-    if mmap_string_append(mmapstr, b"=?utf-8?Q?\x00".as_ptr().cast()).is_null() {
-        return false;
-    }
+fn must_encode(byte: u8) -> bool {
+    static SPECIALS: &[u8] = b",:!\"#$@[\\]^`{|}~=?_";
 
-    // col = (*mmapstr).len as libc::c_int;
-    cur = word;
-    while i < size {
-        let mut do_quote_char = false;
-        match *cur as u8 as char {
-            ',' | ':' | '!' | '"' | '#' | '$' | '@' | '[' | '\\' | ']' | '^' | '`' | '{' | '|'
-            | '}' | '~' | '=' | '?' | '_' => do_quote_char = true,
-            _ => {
-                if *cur as u8 >= 128 {
-                    do_quote_char = true;
-                }
-            }
-        }
-        if do_quote_char {
-            print_hex(hex.as_mut_ptr(), cur);
-            if mmap_string_append(mmapstr, hex.as_mut_ptr()).is_null() {
-                return false;
-            }
-        // col += 3i32
-        } else {
-            if *cur as libc::c_int == ' ' as i32 {
-                if mmap_string_append_c(mmapstr, '_' as i32 as libc::c_char).is_null() {
-                    return false;
-                }
-            } else if mmap_string_append_c(mmapstr, *cur).is_null() {
-                return false;
-            }
-            // col += 3i32
-        }
-        cur = cur.offset(1isize);
-        i = i.wrapping_add(1)
-    }
-    if mmap_string_append(mmapstr, b"?=\x00" as *const u8 as *const libc::c_char).is_null() {
-        return false;
-    }
-
-    true
+    SPECIALS.into_iter().any(|b| *b == byte)
 }
 
-unsafe fn get_word(
-    begin: *const libc::c_char,
-    pend: *mut *const libc::c_char,
-    pto_be_quoted: *mut bool,
-) {
-    let mut cur: *const libc::c_char = begin;
-    while *cur as libc::c_int != ' ' as i32
-        && *cur as libc::c_int != '\t' as i32
-        && *cur as libc::c_int != '\u{0}' as i32
-    {
-        cur = cur.offset(1isize)
+fn quote_word(word: &[u8]) -> String {
+    let mut result = String::default();
+    let mut encoded = false;
+
+    for byte in word {
+        let byte = *byte;
+        if byte >= 128 || must_encode(byte) {
+            result.push_str(&format!("={:2X}", byte));
+            encoded = true;
+        } else if byte == b' ' {
+            result.push('_');
+            encoded = true;
+        } else {
+            result.push(byte as _);
+        }
     }
-    *pto_be_quoted = to_be_quoted(begin, cur.wrapping_offset_from(begin) as libc::size_t);
-    *pend = cur;
+
+    if encoded {
+        result = format!("=?utf-8?Q?{}?=", &result);
+    }
+    result
 }
 
 /* ******************************************************************************
  * Encode/decode header words, RFC 2047
  ******************************************************************************/
 
-/* see comment below */
-unsafe fn to_be_quoted(word: *const libc::c_char, size: libc::size_t) -> bool {
-    let mut cur: *const libc::c_char = word;
-    let mut i = 0;
-    while i < size {
-        match *cur as libc::c_int {
-            44 | 58 | 33 | 34 | 35 | 36 | 64 | 91 | 92 | 93 | 94 | 96 | 123 | 124 | 125 | 126
-            | 61 | 63 | 95 => return true,
-            _ => {
-                if *cur as libc::c_uchar as libc::c_int >= 128i32 {
-                    return true;
-                }
-            }
-        }
-        cur = cur.offset(1isize);
-        i = i.wrapping_add(1)
-    }
-
-    false
-}
-
 pub unsafe fn dc_decode_header_words(in_0: *const libc::c_char) -> *mut libc::c_char {
     if in_0.is_null() {
         return ptr::null_mut();
@@ -319,15 +173,6 @@ pub fn dc_decode_ext_header(to_decode: &[u8]) -> Cow<str> {
     String::from_utf8_lossy(to_decode)
 }
 
-unsafe fn print_hex(target: *mut libc::c_char, cur: *const libc::c_char) {
-    assert!(!target.is_null());
-    assert!(!cur.is_null());
-
-    let bytes = std::slice::from_raw_parts(cur as *const _, strlen(cur));
-    let raw = CString::yolo(format!("={}", &hex::encode_upper(bytes)[..2]));
-    libc::memcpy(target as *mut _, raw.as_ptr() as *const _, 4);
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -429,18 +274,6 @@ mod tests {
         assert_eq!(dc_needs_ext_header("a b"), true);
     }
 
-    #[test]
-    fn test_print_hex() {
-        let mut hex: [libc::c_char; 4] = [0; 4];
-        let cur = b"helloworld" as *const u8 as *const libc::c_char;
-        unsafe { print_hex(hex.as_mut_ptr(), cur) };
-        assert_eq!(to_string(hex.as_ptr() as *const _), "=68");
-
-        let cur = b":" as *const u8 as *const libc::c_char;
-        unsafe { print_hex(hex.as_mut_ptr(), cur) };
-        assert_eq!(to_string(hex.as_ptr() as *const _), "=3A");
-    }
-
     use proptest::prelude::*;
 
     proptest! {
@@ -456,5 +289,13 @@ mod tests {
             // make sure this never panics
             let _decoded = dc_decode_ext_header(&buf);
         }
+
+        #[test]
+        fn test_dc_header_roundtrip(input: String) {
+            let encoded = dc_encode_header_words(&input);
+            let decoded = dc_decode_header_words_safe(&encoded);
+
+            assert_eq!(input, decoded);
+        }
     }
 }

src/dc_tools.rs

@@ -9,7 +9,7 @@ use std::time::SystemTime;
 use std::{fmt, fs, ptr};
 
 use chrono::{Local, TimeZone};
-use libc::{memcpy, strcpy, strlen, uintptr_t};
+use libc::{memcpy, strlen};
 use mmime::clist::*;
 use mmime::mailimf::types::*;
 use rand::{thread_rng, Rng};
@@ -149,66 +149,6 @@ pub(crate) fn dc_truncate(buf: &str, approx_chars: usize, do_unwrap: bool) -> Co
     }
 }
 
-#[allow(non_snake_case)]
-pub(crate) unsafe fn dc_truncate_n_unwrap_str(
-    buf: *mut libc::c_char,
-    approx_characters: libc::c_int,
-    do_unwrap: libc::c_int,
-) {
-    /* Function unwraps the given string and removes unnecessary whitespace.
-    Function stops processing after approx_characters are processed.
-    (as we're using UTF-8, for simplicity, we cut the string only at whitespaces). */
-    /* a single line is truncated `...` instead of `[...]` (the former is typically also used by the UI to fit strings in a rectangle) */
-    let ellipse_utf8: *const libc::c_char = if 0 != do_unwrap {
-        b" ...\x00" as *const u8 as *const libc::c_char
-    } else {
-        b" [...]\x00" as *const u8 as *const libc::c_char
-    };
-    let mut lastIsCharacter: libc::c_int = 0;
-    /* force unsigned - otherwise the `> ' '` comparison will fail */
-    let mut p1: *mut libc::c_uchar = buf as *mut libc::c_uchar;
-    while 0 != *p1 {
-        if *p1 as libc::c_int > ' ' as i32 {
-            lastIsCharacter = 1
-        } else if 0 != lastIsCharacter {
-            let used_bytes = (p1 as uintptr_t).wrapping_sub(buf as uintptr_t) as libc::size_t;
-            if dc_utf8_strnlen(buf, used_bytes) >= approx_characters as usize {
-                let buf_bytes = strlen(buf);
-                if buf_bytes.wrapping_sub(used_bytes) >= strlen(ellipse_utf8) {
-                    strcpy(p1 as *mut libc::c_char, ellipse_utf8);
-                }
-                break;
-            } else {
-                lastIsCharacter = 0;
-                if 0 != do_unwrap {
-                    *p1 = ' ' as i32 as libc::c_uchar
-                }
-            }
-        } else if 0 != do_unwrap {
-            *p1 = '\r' as i32 as libc::c_uchar
-        }
-        p1 = p1.offset(1isize)
-    }
-    if 0 != do_unwrap {
-        dc_remove_cr_chars(buf);
-    };
-}
-
-unsafe fn dc_utf8_strnlen(s: *const libc::c_char, n: libc::size_t) -> libc::size_t {
-    if s.is_null() {
-        return 0;
-    }
-
-    let mut j: libc::size_t = 0;
-    for i in 0..n {
-        if *s.add(i) as libc::c_int & 0xc0 != 0x80 {
-            j = j.wrapping_add(1)
-        }
-    }
-
-    j
-}
-
 pub(crate) unsafe fn dc_str_from_clist(
     list: *const clist,
     delimiter: *const libc::c_char,

src/e2ee.rs

@@ -182,7 +182,8 @@ impl EncryptHelper {
                         let opt_field = (*field).fld_data.fld_optional_field;
                         if !opt_field.is_null() && !(*opt_field).fld_name.is_null() {
                             let fld_name = to_string_lossy((*opt_field).fld_name);
-                            if fld_name.starts_with("Secure-Join") || fld_name.starts_with("Chat-")
+                            if fld_name.starts_with("Secure-Join")
+                                || (fld_name.starts_with("Chat-") && fld_name != "Chat-Version")
                             {
                                 move_to_encrypted = true;
                             }
@@ -586,7 +587,7 @@ unsafe fn decrypt_recursive(
         {
             for cur_data in (*(*mime).mm_data.mm_multipart.mm_mp_list).into_iter() {
                 let mut decrypted_mime: *mut Mailmime = ptr::null_mut();
-                if decrypt_part(
+                let decrypted = match decrypt_part(
                     context,
                     cur_data as *mut Mailmime,
                     private_keyring,
@@ -594,6 +595,10 @@ unsafe fn decrypt_recursive(
                     ret_valid_signatures,
                     &mut decrypted_mime,
                 ) {
+                    Ok(res) => res,
+                    Err(err) => bail!("decrypt_part: {}", err),
+                };
+                if decrypted {
                     if (*ret_gossip_headers).is_null() && ret_valid_signatures.len() > 0 {
                         let mut dummy: libc::size_t = 0;
                         let mut test: *mut mailimf_fields = ptr::null_mut();
@@ -659,113 +664,120 @@ unsafe fn decrypt_part(
     public_keyring_for_validate: &Keyring,
     ret_valid_signatures: &mut HashSet<String>,
     ret_decrypted_mime: *mut *mut Mailmime,
-) -> bool {
-    let mut ok_to_continue = true;
+) -> Result<bool> {
     let mime_data: *mut mailmime_data;
     let mut mime_transfer_encoding: libc::c_int = MAILMIME_MECHANISM_BINARY as libc::c_int;
-    /* mmap_string_unref()'d if set */
-    let mut transfer_decoding_buffer: *mut libc::c_char = ptr::null_mut();
-    /* must not be free()'d */
-    let mut decoded_data: *const libc::c_char = ptr::null_mut();
-    let mut decoded_data_bytes: libc::size_t = 0;
     let mut sth_decrypted = false;
 
+    let cleanup = |transfer_decoding_buffer: *mut libc::c_char| {
+        if !transfer_decoding_buffer.is_null() {
+            mmap_string_unref(transfer_decoding_buffer);
+        }
+    };
+
     *ret_decrypted_mime = ptr::null_mut();
     mime_data = (*mime).mm_data.mm_single;
     /* MAILMIME_DATA_FILE indicates, the data is in a file; AFAIK this is not used on parsing */
-    if !((*mime_data).dt_type != MAILMIME_DATA_TEXT as libc::c_int
+    if (*mime_data).dt_type != MAILMIME_DATA_TEXT as libc::c_int
         || (*mime_data).dt_data.dt_text.dt_data.is_null()
-        || (*mime_data).dt_data.dt_text.dt_length <= 0)
+        || (*mime_data).dt_data.dt_text.dt_length <= 0
     {
-        if !(*mime).mm_mime_fields.is_null() {
-            for cur_data in (*(*(*mime).mm_mime_fields).fld_list).into_iter() {
-                let field: *mut mailmime_field = cur_data as *mut _;
-                if (*field).fld_type == MAILMIME_FIELD_TRANSFER_ENCODING as libc::c_int
-                    && !(*field).fld_data.fld_encoding.is_null()
-                {
-                    mime_transfer_encoding = (*(*field).fld_data.fld_encoding).enc_type
-                }
+        return Ok(false);
+    }
+    if !(*mime).mm_mime_fields.is_null() {
+        for cur_data in (*(*(*mime).mm_mime_fields).fld_list).into_iter() {
+            let field: *mut mailmime_field = cur_data as *mut _;
+            if (*field).fld_type == MAILMIME_FIELD_TRANSFER_ENCODING as libc::c_int
+                && !(*field).fld_data.fld_encoding.is_null()
+            {
+                mime_transfer_encoding = (*(*field).fld_data.fld_encoding).enc_type
             }
         }
-        /* regard `Content-Transfer-Encoding:` */
-        if mime_transfer_encoding == MAILMIME_MECHANISM_7BIT as libc::c_int
-            || mime_transfer_encoding == MAILMIME_MECHANISM_8BIT as libc::c_int
-            || mime_transfer_encoding == MAILMIME_MECHANISM_BINARY as libc::c_int
+    }
+    /* regarding `Content-Transfer-Encoding:` */
+    /* mmap_string_unref()'d if set */
+    let mut transfer_decoding_buffer: *mut libc::c_char = ptr::null_mut();
+    let decoded_data: *const libc::c_char;
+    let mut decoded_data_bytes: libc::size_t = 0;
+    if mime_transfer_encoding == MAILMIME_MECHANISM_7BIT as libc::c_int
+        || mime_transfer_encoding == MAILMIME_MECHANISM_8BIT as libc::c_int
+        || mime_transfer_encoding == MAILMIME_MECHANISM_BINARY as libc::c_int
+    {
+        decoded_data = (*mime_data).dt_data.dt_text.dt_data;
+        decoded_data_bytes = (*mime_data).dt_data.dt_text.dt_length;
+        if decoded_data.is_null() || decoded_data_bytes <= 0 {
+            /* no error - but no data */
+            return Ok(false);
+        }
+    } else {
+        let r: libc::c_int;
+        let mut current_index: libc::size_t = 0;
+        r = mailmime_part_parse(
+            (*mime_data).dt_data.dt_text.dt_data,
+            (*mime_data).dt_data.dt_text.dt_length,
+            &mut current_index,
+            mime_transfer_encoding,
+            &mut transfer_decoding_buffer,
+            &mut decoded_data_bytes,
+        );
+        if r != MAILIMF_NO_ERROR as libc::c_int
+            || transfer_decoding_buffer.is_null()
+            || decoded_data_bytes <= 0
         {
-            decoded_data = (*mime_data).dt_data.dt_text.dt_data;
-            decoded_data_bytes = (*mime_data).dt_data.dt_text.dt_length;
-            if decoded_data.is_null() || decoded_data_bytes <= 0 {
-                /* no error - but no data */
-                ok_to_continue = false;
+            cleanup(transfer_decoding_buffer);
+            bail!("mailmime_part_parse returned error or invalid data");
+        }
+        decoded_data = transfer_decoding_buffer;
+    }
+
+    /* encrypted, decoded data in decoded_data now ... */
+    if has_decrypted_pgp_armor(decoded_data, decoded_data_bytes as libc::c_int) {
+        let add_signatures = if ret_valid_signatures.is_empty() {
+            Some(ret_valid_signatures)
+        } else {
+            None
+        };
+
+        /*if we already have fingerprints, do not add more; this ensures, only the fingerprints from the outer-most part are collected */
+        let plain = match dc_pgp_pk_decrypt(
+            std::slice::from_raw_parts(decoded_data as *const u8, decoded_data_bytes),
+            &private_keyring,
+            &public_keyring_for_validate,
+            add_signatures,
+        ) {
+            Ok(plain) => plain,
+            Err(err) => {
+                cleanup(transfer_decoding_buffer);
+                bail!("could not decrypt: {}", err)
+            }
+        };
+        let plain_bytes = plain.len();
+        let plain_buf = plain.as_ptr() as *const libc::c_char;
+
+        let mut index: libc::size_t = 0;
+        let mut decrypted_mime: *mut Mailmime = ptr::null_mut();
+        if mailmime_parse(
+            plain_buf as *const _,
+            plain_bytes,
+            &mut index,
+            &mut decrypted_mime,
+        ) != MAIL_NO_ERROR as libc::c_int
+            || decrypted_mime.is_null()
+        {
+            if !decrypted_mime.is_null() {
+                mailmime_free(decrypted_mime);
             }
         } else {
-            let r: libc::c_int;
-            let mut current_index: libc::size_t = 0;
-            r = mailmime_part_parse(
-                (*mime_data).dt_data.dt_text.dt_data,
-                (*mime_data).dt_data.dt_text.dt_length,
-                &mut current_index,
-                mime_transfer_encoding,
-                &mut transfer_decoding_buffer,
-                &mut decoded_data_bytes,
-            );
-            if r != MAILIMF_NO_ERROR as libc::c_int
-                || transfer_decoding_buffer.is_null()
-                || decoded_data_bytes <= 0
-            {
-                ok_to_continue = false;
-            } else {
-                decoded_data = transfer_decoding_buffer;
-            }
-        }
-        if ok_to_continue {
-            /* encrypted, decoded data in decoded_data now ... */
-            if has_decrypted_pgp_armor(decoded_data, decoded_data_bytes as libc::c_int) {
-                let add_signatures = if ret_valid_signatures.is_empty() {
-                    Some(ret_valid_signatures)
-                } else {
-                    None
-                };
-
-                /*if we already have fingerprints, do not add more; this ensures, only the fingerprints from the outer-most part are collected */
-                if let Ok(plain) = dc_pgp_pk_decrypt(
-                    std::slice::from_raw_parts(decoded_data as *const u8, decoded_data_bytes),
-                    &private_keyring,
-                    &public_keyring_for_validate,
-                    add_signatures,
-                ) {
-                    let plain_bytes = plain.len();
-                    let plain_buf = plain.as_ptr() as *const libc::c_char;
-
-                    let mut index: libc::size_t = 0;
-                    let mut decrypted_mime: *mut Mailmime = ptr::null_mut();
-                    if mailmime_parse(
-                        plain_buf as *const _,
-                        plain_bytes,
-                        &mut index,
-                        &mut decrypted_mime,
-                    ) != MAIL_NO_ERROR as libc::c_int
-                        || decrypted_mime.is_null()
-                    {
-                        if !decrypted_mime.is_null() {
-                            mailmime_free(decrypted_mime);
-                        }
-                    } else {
-                        *ret_decrypted_mime = decrypted_mime;
-                        sth_decrypted = true;
-                    }
-                    std::mem::forget(plain);
-                }
-            }
+            *ret_decrypted_mime = decrypted_mime;
+            sth_decrypted = true;
         }
+        std::mem::forget(plain);
     }
     //mailmime_substitute(mime, new_mime);
     //s. mailprivacy_gnupg.c::pgp_decrypt()
-    if !transfer_decoding_buffer.is_null() {
-        mmap_string_unref(transfer_decoding_buffer);
-    }
+    cleanup(transfer_decoding_buffer);
 
-    sth_decrypted
+    Ok(sth_decrypted)
 }
 
 unsafe fn has_decrypted_pgp_armor(str__: *const libc::c_char, mut str_bytes: libc::c_int) -> bool {

src/imap.rs

@@ -110,6 +110,8 @@ impl Client {
     ) -> imap::error::Result<Self> {
         let stream = net::TcpStream::connect(addr)?;
         let tls = native_tls::TlsConnector::builder()
+            // see also: https://github.com/deltachat/deltachat-core-rust/issues/203
+            .danger_accept_invalid_certs(true)
             .danger_accept_invalid_hostnames(true)
             .build()
             .unwrap();
@@ -953,14 +955,7 @@ impl Imap {
             if !is_deleted && msg.body().is_some() {
                 let body = msg.body().unwrap();
                 unsafe {
-                    dc_receive_imf(
-                        context,
-                        body.as_ptr() as *const libc::c_char,
-                        body.len(),
-                        folder.as_ref(),
-                        server_uid,
-                        flags as u32,
-                    );
+                    dc_receive_imf(context, &body, folder.as_ref(), server_uid, flags as u32);
                 }
             }
         }

src/message.rs

@@ -2,7 +2,6 @@ use std::path::{Path, PathBuf};
 use std::ptr;
 
 use deltachat_derive::{FromSql, ToSql};
-use libc::free;
 
 use crate::chat::{self, Chat};
 use crate::constants::*;
@@ -108,18 +107,6 @@ impl Message {
             msg.hidden = row.get(18)?;
             msg.location_id = row.get(19)?;
             msg.chat_blocked = row.get::<_, Option<Blocked>>(20)?.unwrap_or_default();
-            if msg.chat_blocked == Blocked::Deaddrop {
-                if let Some(ref text) = msg.text {
-                    unsafe {
-                        let ptr = text.strdup();
-
-                        dc_truncate_n_unwrap_str(ptr, 256, 0);
-
-                        msg.text = Some(to_string(ptr));
-                        free(ptr.cast());
-                    }
-                }
-            };
             Ok(msg)
         })
     }

src/smtp.rs

@@ -70,8 +70,9 @@ impl Smtp {
         let port = lp.send_port as u16;
 
         let tls = native_tls::TlsConnector::builder()
-            // FIXME: unfortunately this is needed to make things work on macos + testrun.org
+            // see also: https://github.com/deltachat/deltachat-core-rust/issues/203
             .danger_accept_invalid_hostnames(true)
+            .danger_accept_invalid_certs(true)
             .min_protocol_version(Some(DEFAULT_TLS_PROTOCOLS[0]))
             .build()
             .unwrap();

tests/stress.rs

@@ -209,7 +209,8 @@ unsafe fn stress_functions(context: &Context) {
 
     assert!(preferencrypt_0.is_null());
     free(buf_1 as *mut libc::c_void);
-    buf_1 = dc_decrypt_setup_file(context, S_EM_SETUPCODE, S_EM_SETUPFILE);
+    let decrypted = dc_decrypt_setup_file(context, S_EM_SETUPCODE, S_EM_SETUPFILE).unwrap();
+    buf_1 = decrypted.strdup();
     assert!(!buf_1.is_null());
     assert!(dc_split_armored_data(
         buf_1,