Use the first subkey for encryption instead of the primary key

resorting the chatlist on changing drafts has some ux issues.
eg. when the chatlist is visible together with the input field,
if may come to flickering resorting during input
or to a resorting just when the user leave the chat
as this might trigger set_draft().

but also on mobiles, the resorting is visible and a bit unexpected.
also it is unclear what happens when a chat with a draft is entered
and left without modifications.

the solution proposed here is to ignore draft on sorting
while still showing them in the chatlist
if they're newer as the last message.

a possible disadvantage is
that the date for the chat with a draft does not follow the ordering
(the ordering is by the last message),
however, the date is not shown as a "primary sort" criterion or so,
so it might be that this is completely okay.
also, of course, it affects only draft :)

.circleci/config.yml

@@ -4,6 +4,9 @@ executors:
     docker:
       - image: filecoin/rust:latest
     working_directory: /mnt/crate
+  doxygen:
+    docker:
+      - image: hrektts/doxygen
 
 restore-workspace: &restore-workspace
   attach_workspace:
@@ -113,6 +116,18 @@ jobs:
           target: "aarch64-linux-android"
 
 
+  build_doxygen:
+    executor: doxygen
+    steps:
+      - checkout
+      - run: bash ci_scripts/run-doxygen.sh
+      - run: mkdir -p workspace/c-docs 
+      - run: cp -av deltachat-ffi/{html,xml} workspace/c-docs/
+      - persist_to_workspace:
+          root: workspace
+          paths:
+            - c-docs
+
   build_test_docs_wheel:
     docker:
       - image: deltachat/coredeps
@@ -148,7 +163,7 @@ jobs:
           at: workspace
       - run: pyenv global 3.5.2
       - run: ls -laR workspace
-      - run: ci_scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse
+      - run: ci_scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
 
   clippy:
     executor: default
@@ -166,18 +181,21 @@ workflows:
   test:
     jobs:
       - cargo_fetch
+      - build_doxygen
+
       - build_test_docs_wheel:
           requires:
             - cargo_fetch
       - upload_docs_wheels:
           requires:
             - build_test_docs_wheel
+            - build_doxygen 
       - rustfmt:
           requires:
             - cargo_fetch
       - clippy:
           requires:
-            - cargo_fetch
+           - cargo_fetch
 
       # Linux Desktop 64bit
       - test_x86_64-unknown-linux-gnu:

CHANGELOG.md

@@ -0,0 +1,88 @@
+# Changelog 
+
+## 1.0.0-beta.7
+
+- fix location-streaming #782
+
+- fix display of messages that could not be decrypted #785
+ 
+- fix smtp MAILER-DAEMON bug #786 
+
+- fix a logging of durations #783
+
+- add more error logging #779
+
+- do not panic on some bad utf-8 mime #776
+
+## 1.0.0-beta.6
+
+- fix chatlist.get_msg_id to return id, instead of wrongly erroring
+
+## 1.0.0-beta.5
+
+- fix dc_get_msg() to return empty messages when asked for special ones 
+
+## 1.0.0-beta.4
+
+- fix more than one sending of autocrypt setup message
+
+- fix recognition of mailto-address-qr-codes, add tests
+
+- tune down error to warning when adding self to chat
+
+## 1.0.0-beta.3
+
+- add back `dc_empty_server()` #682
+
+- if `show_emails` is set to `DC_SHOW_EMAILS_ALL`,
+  email-based contact requests are added to the chatlist directly
+
+- fix IMAP hangs #717 and cleanups
+
+- several rPGP fixes
+
+- code streamlining and rustifications
+
+
+## 1.0.0-beta.2
+
+- https://c.delta.chat docs are now regenerated again through our CI 
+
+- several rPGP cleanups, security fixes and better multi-platform support 
+
+- reconnect on io errors and broken pipes (imap)
+
+- probe SMTP with real connection not just setup
+
+- various imap/smtp related fixes
+
+- use to_string_lossy in most places instead of relying on valid utf-8
+  encodings
+ 
+- rework, rustify and test autoconfig-reading and parsing 
+
+- some rustifications/boolifications of c-ints 
+
+
+## 1.0.0-beta.1 
+
+- first beta of the Delta Chat Rust core library. many fixes of crashes
+  and other issues compared to 1.0.0-alpha.5.
+
+- Most code is now "rustified" and does not do manual memory allocation anymore. 
+
+- The `DC_EVENT_GET_STRING` event is not used anymore, removing the last
+  event where the core requested a return value from the event callback. 
+
+  Please now use `dc_set_stock_translation()` API for core messages
+  to be properly localized. 
+
+- Deltachat FFI docs are automatically generated and available here: 
+  https://c.delta.chat 
+
+- New events ImapMessageMoved and ImapMessageDeleted
+
+For a full list of changes, please see our closed Pull Requests: 
+
+https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
+

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "deltachat"
-version = "1.0.0-alpha.5"
-authors = ["dignifiedquire <dignifiedquire@gmail.com>"]
+version = "1.0.0-beta.7"
+authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL"
 
@@ -10,7 +10,7 @@ deltachat_derive = { path = "./deltachat_derive" }
 mmime = { version = "0.1.2", path = "./mmime" }
 
 libc = "0.2.51"
-pgp = { version = "0.2", default-features = false }
+pgp = { version = "0.2.3", default-features = false }
 hex = "0.3.2"
 sha2 = "0.8.0"
 rand = "0.6.5"
@@ -19,8 +19,8 @@ reqwest = "0.9.15"
 num-derive = "0.2.5"
 num-traits = "0.2.6"
 native-tls = "0.2.3"
-lettre = "0.9.0"
-imap = { git = "https://github.com/jonhoo/rust-imap", rev = "281d2eb8ab50dc656ceff2ae749ca5045f334e15" }
+lettre = { git = "https://github.com/deltachat/lettre", branch = "master" }
+imap = { git = "https://github.com/deltachat/rust-imap", branch = "master" }
 base64 = "0.10"
 charset = "0.1"
 percent-encoding = "2.0"
@@ -48,6 +48,7 @@ escaper = "0.1.0"
 bitflags = "1.1.0"
 jetscii = "0.4.4"
 debug_stub_derive = "0.3.0"
+sanitize-filename = "0.2.1"
 
 [dev-dependencies]
 tempfile = "3.0"

Xargo.toml

@@ -1,6 +0,0 @@
-[dependencies.std]
-features = ["panic-unwind"]
-
-# if using `cargo test`
-[dependencies.test]
-stage = 1

ci_scripts/ci_upload.sh

@@ -7,9 +7,9 @@ fi
 
 set -xe
 
-#DOXYDOCDIR=${1:?directory where doxygen docs to be found}
 PYDOCDIR=${1:?directory with python docs}
 WHEELHOUSEDIR=${2:?directory with pre-built wheels}
+DOXYDOCDIR=${3:?directory where doxygen docs to be found}
 
 export BRANCH=${CIRCLE_BRANCH:?specify branch for uploading purposes}
 
@@ -22,10 +22,11 @@ rsync -avz \
   delta@py.delta.chat:build/${BRANCH}
 
 # C docs to c.delta.chat
-#rsync -avz \
-#  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
-#  "$DOXYDOCDIR/html/" \
-#  delta@py.delta.chat:build-c/${BRANCH}
+ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@c.delta.chat mkdir -p build-c/${BRANCH}
+rsync -avz \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  "$DOXYDOCDIR/html/" \
+  delta@c.delta.chat:build-c/${BRANCH}
 
 echo -----------------------
 echo upload wheels 

ci_scripts/run-doxygen.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -ex
+
+cd deltachat-ffi 
+doxygen 
+

ci_scripts/run-python.sh

@@ -43,8 +43,8 @@ if [ -n "$TESTS" ]; then
     # we split out qr-tests run to minimize likelyness of flaky tests
     # (some qr tests are pretty heavy in terms of send/received
     # messages and rust's imap code likely has concurrency problems) 
-    tox --workdir "$TOXWORKDIR" -e py37 -- -k "not qr"
-    tox --workdir "$TOXWORKDIR" -e py37 -- -k "qr"
+    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "not qr"
+    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "qr"
     unset DCC_PY_LIVECONFIG
     tox --workdir "$TOXWORKDIR" -p4 -e lint,py35,py36,doc
     tox --workdir "$TOXWORKDIR" -e auditwheels

deltachat-ffi/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
 name = "deltachat_ffi"
-version = "1.0.0-alpha.5"
+version = "1.0.0-beta.7"
 description = "Deltachat FFI"
-authors = ["dignifiedquire <dignifiedquire@gmail.com>"]
+authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 readme = "README.md"
 license = "MIT OR Apache-2.0"

deltachat-ffi/deltachat.h

@@ -78,8 +78,7 @@ typedef struct _dc_provider dc_provider_t;
  *
  * The example above uses "pthreads",
  * however, you can also use anything else for thread handling.
- * NB: The deltachat-core library itself does not create any threads on its own,
- * however, functions, unless stated otherwise, are thread-safe.
+ * All deltachat-core-functions, unless stated otherwise, are thread-safe.
  *
  * After that you can  **define and open a database.**
  * The database is a normal sqlite-file and is created as needed:
@@ -135,7 +134,7 @@ typedef struct _dc_provider dc_provider_t;
  *
  *     printf("Message %i: %s\n", i+1, text);
  *
- *     free(text);
+ *     dc_str_unref(text);
  *     dc_msg_unref(msg);
  * }
  * dc_array_unref(msglist);
@@ -321,7 +320,7 @@ int             dc_is_open                   (const dc_context_t* context);
  * @memberof dc_context_t
  * @param context The context object as created by dc_context_new().
  * @return Blob directory associated with the context object, empty string if unset or on errors. NULL is never returned.
- *     The returned string must be free()'d.
+ *     The returned string must be released using dc_str_unref().
  */
 char*           dc_get_blobdir               (const dc_context_t* context);
 
@@ -339,6 +338,8 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `send_pw`      = SMTP-password, guessed if left out
  * - `send_port`    = SMTP-port, guessed if left out
  * - `server_flags` = IMAP-/SMTP-flags as a combination of @ref DC_LP flags, guessed if left out
+ * - `imap_certificate_checks` = how to check IMAP certificates, one of the @ref DC_CERTCK flags, defaults to #DC_CERTCK_AUTO (0)
+ * - `smtp_certificate_checks` = how to check SMTP certificates, one of the @ref DC_CERTCK flags, defaults to #DC_CERTCK_AUTO (0)
  * - `displayname`  = Own name to use when sending messages.  MUAs are allowed to spread this way eg. using CC, defaults to empty
  * - `selfstatus`   = Own status to display eg. in email footers, defaults to a standard text
  * - `selfavatar`   = File containing avatar. Will be copied to blob directory.
@@ -397,11 +398,24 @@ int             dc_set_config                (dc_context_t* context, const char*
  * @param context The context object as created by dc_context_new(). For querying system values, this can be NULL.
  * @param key The key to query.
  * @return Returns current value of "key", if "key" is unset, the default
- *     value is returned.  The returned value must be free()'d, NULL is never
+ *     value is returned.  The returned value must be released using dc_str_unref(), NULL is never
  *     returned.  If there is an error an empty string will be returned.
  */
 char*           dc_get_config                (dc_context_t* context, const char* key);
 
+/**
+ * Set stock string translation. 
+ *
+ * The function will emit warnings if it returns an error state. 
+ *
+ * @memberof dc_context_t
+ * @param context The context object
+ * @param stock_id   the integer id of the stock message (DC_STR_*)
+ * @param stock_msg  the message to be used 
+ * @return int (==0 on error, 1 on success)
+ */
+int             dc_set_stock_translation(dc_context_t* context, uint32_t stock_id, const char* stock_msg);
+
 
 /**
  * Get information about the context.
@@ -415,7 +429,7 @@ char*           dc_get_config                (dc_context_t* context, const char*
  *
  * @memberof dc_context_t
  * @param context The context as created by dc_context_new().
- * @return String which must be free()'d after usage.  Never returns NULL.
+ * @return String which must be released using dc_str_unref() after usage.  Never returns NULL.
  */
 char*           dc_get_info                  (dc_context_t* context);
 
@@ -446,6 +460,7 @@ char*           dc_get_info                  (dc_context_t* context);
  *     `https://localhost:PORT/PATH`, `urn:ietf:wg:oauth:2.0:oob`
  *     (the latter just displays the code the user can copy+paste then)
  * @return URL that can be opened in the browser to start OAuth2.
+ *     Returned strings must be released using dc_str_unref().
  *     If OAuth2 is not possible for the given e-mail-address, NULL is returned.
  */
 char*           dc_get_oauth2_url            (dc_context_t* context, const char* addr, const char* redirect_uri);
@@ -486,15 +501,9 @@ char*           dc_get_oauth2_url            (dc_context_t* context, const char*
  * To interrupt a configuration prematurely, use dc_stop_ongoing_process();
  * this is not needed if #DC_EVENT_CONFIGURE_PROGRESS reports success.
  *
- * On a successfull configuration,
- * the core makes a copy of the parameters mentioned above:
- * the original parameters as are never modified by the core.
- *
- * UI-implementors should keep this in mind -
- * eg. if the UI wants to prefill a configure-edit-dialog with these parameters,
- * the UI should reset them if the user cancels the dialog
- * after a configure-attempts has failed.
- * Otherwise the parameters may not reflect the current configuation.
+ * If #DC_EVENT_CONFIGURE_PROGRESS reports failure,
+ * the core continues to use the last working configuration
+ * and parameters as `addr`, `mail_pw` etc. are set to that.
  *
  * @memberof dc_context_t
  * @param context The context object as created by dc_context_new().
@@ -694,6 +703,7 @@ void            dc_perform_mvbox_idle        (dc_context_t* context);
  */
 void            dc_interrupt_mvbox_idle      (dc_context_t* context);
 
+
 /**
  * Execute pending sentbox-jobs.
  * This function and dc_perform_sentbox_fetch() and dc_perform_sentbox_idle()
@@ -1082,6 +1092,27 @@ uint32_t        dc_send_text_msg             (dc_context_t* context, uint32_t ch
 void            dc_set_draft                 (dc_context_t* context, uint32_t chat_id, dc_msg_t* msg);
 
 
+/**
+ * Add a message to the device-chat.
+ * Device-messages usually contain update information
+ * and some hints that are added during the program runs, multi-device etc.
+ *
+ * Device-messages may be added from the core,
+ * however, with this function, this can be done from the ui as well.
+ * If needed, the device-chat is created before.
+ *
+ * Sends the event #DC_EVENT_MSGS_CHANGED on success.
+ * To check, if a given chat is a device-chat, see dc_chat_is_device_talk()
+ *
+ * @memberof dc_context_t
+ * @param context The context as created by dc_context_new().
+ * @param msg Message to be added to the device-chat.
+ *     The message appears to the user as an incoming message.
+ * @return The ID of the added message.
+ */
+uint32_t        dc_add_device_msg            (dc_context_t* context, dc_msg_t* msg);
+
+
 /**
  * Get draft for a chat, if any.
  * See dc_set_draft() for more details about drafts.
@@ -1197,7 +1228,7 @@ void            dc_marknoticed_all_chats     (dc_context_t* context);
  * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
  * @param chat_id The chat ID to get all messages with media from.
- * @param msg_type Specify a message type to query here, one of the DC_MSG_* constats.
+ * @param msg_type Specify a message type to query here, one of the @ref DC_MSG constants.
  * @param msg_type2 Alternative message type to search for. 0 to skip.
  * @param msg_type3 Alternative message type to search for. 0 to skip.
  * @return An array with messages from the given chat ID that have the wanted message types.
@@ -1465,7 +1496,7 @@ int             dc_set_chat_profile_image    (dc_context_t* context, uint32_t ch
  * @memberof dc_context_t
  * @param context The context object as created by dc_context_new().
  * @param msg_id The message id for which information should be generated
- * @return Text string, must be free()'d after usage
+ * @return Text string, must be released using dc_str_unref() after usage
  */
 char*           dc_get_msg_info              (dc_context_t* context, uint32_t msg_id);
 
@@ -1479,7 +1510,7 @@ char*           dc_get_msg_info              (dc_context_t* context, uint32_t ms
  * @memberof dc_context_t
  * @param context The context object as created by dc_context_new().
  * @param msg_id The message id, must be the id of an incoming message.
- * @return Raw headers as a multi-line string, must be free()'d after usage.
+ * @return Raw headers as a multi-line string, must be released using dc_str_unref() after usage.
  *     Returns NULL if there are no headers saved for the given message,
  *     eg. because of save_mime_headers is not set
  *     or the message is not incoming.
@@ -1499,6 +1530,16 @@ char*           dc_get_mime_headers          (dc_context_t* context, uint32_t ms
  */
 void            dc_delete_msgs               (dc_context_t* context, const uint32_t* msg_ids, int msg_cnt);
 
+/**
+ * Empty IMAP server folder: delete all messages. 
+ *
+ * @memberof dc_context_t
+ * @param context The context object as created by dc_context_new()
+ * @param flags What to delete, a combination of the @ref DC_EMPTY flags
+ * @return None.
+ */
+void            dc_empty_server              (dc_context_t* context, uint32_t flags);
+
 
 /**
  * Forward messages to another chat.
@@ -1720,7 +1761,7 @@ void            dc_block_contact             (dc_context_t* context, uint32_t co
  * @memberof dc_context_t
  * @param context The context object as created by dc_context_new().
  * @param contact_id ID of the contact to get the encryption info for.
- * @return Multi-line text, must be free()'d after usage.
+ * @return Multi-line text, must be released using dc_str_unref() after usage.
  */
 char*           dc_get_contact_encrinfo      (dc_context_t* context, uint32_t contact_id);
 
@@ -1856,7 +1897,8 @@ void            dc_imex                      (dc_context_t* context, int what, c
  * @memberof dc_context_t
  * @param context The context as created by dc_context_new().
  * @param dir Directory to search backups in.
- * @return String with the backup file, typically given to dc_imex(), returned strings must be free()'d.
+ * @return String with the backup file, typically given to dc_imex(),
+ *     returned strings must be released using dc_str_unref().
  *     The function returns NULL if no backup was found.
  */
 char*           dc_imex_has_backup           (dc_context_t* context, const char* dir);
@@ -1904,7 +1946,7 @@ char*           dc_imex_has_backup           (dc_context_t* context, const char*
  *
  * @memberof dc_context_t
  * @param context The context object.
- * @return The setup code. Must be free()'d after usage.
+ * @return The setup code. Must be released using dc_str_unref() after usage.
  *     On errors, eg. if the message could not be sent, NULL is returned.
  */
 char*           dc_initiate_key_transfer     (dc_context_t* context);
@@ -2009,7 +2051,7 @@ dc_lot_t*       dc_check_qr                  (dc_context_t* context, const char*
  *     If set to 0, the setup-Verified-contact-protocol is offered in the QR code.
  * @return Text that should go to the QR code,
  *     On errors, an empty QR code is returned, NULL is never returned.
- *     The returned string must be free()'d after usage.
+ *     The returned string must be released using dc_str_unref() after usage.
  */
 char*           dc_get_securejoin_qr         (dc_context_t* context, uint32_t chat_id);
 
@@ -2179,6 +2221,21 @@ dc_array_t* dc_get_locations                (dc_context_t* context, uint32_t cha
 void        dc_delete_all_locations         (dc_context_t* context);
 
 
+/**
+ * Release a string returned by another deltachat-core function.
+ * - Strings returned by any deltachat-core-function
+ *   MUST NOT be released by the standard free() function;
+ *   always use dc_str_unref() for this purpose.
+ * - dc_str_unref() MUST NOT be called for strings not returned by deltachat-core.
+ * - dc_str_unref() MUST NOT be called for other objectes returned by deltachat-core.
+ *
+ * @memberof dc_context_t
+ * @param str The string to release.
+ * @return None.
+ */
+void dc_str_unref (char* str);
+
+
 /**
  * @class dc_array_t
  *
@@ -2319,7 +2376,7 @@ uint32_t         dc_array_get_msg_id         (const dc_array_t* array, size_t in
  * @param index Index of the item. Must be between 0 and dc_array_get_cnt()-1.
  * @return Marker-character of the item at the given index.
  *     NULL if there is no marker-character bound to the given item.
- *     The returned value must be free()'d after usage.
+ *     The returned value must be released using dc_str_unref() after usage.
  */
 char*            dc_array_get_marker         (const dc_array_t* array, size_t index);
 
@@ -2570,24 +2627,22 @@ int             dc_chat_get_type             (const dc_chat_t* chat);
  *
  * To change the name, use dc_set_chat_name()
  *
- * See also: dc_chat_get_subtitle()
- *
  * @memberof dc_chat_t
  * @param chat The chat object.
- * @return Chat name as a string. Must be free()'d after usage. Never NULL.
+ * @return Chat name as a string. Must be released using dc_str_unref() after usage. Never NULL.
  */
 char*           dc_chat_get_name             (const dc_chat_t* chat);
 
 
-/**
+/*
  * Get a subtitle for a chat.  The subtitle is eg. the email-address or the
  * number of group members.
  *
- * See also: dc_chat_get_name()
+ * Deprecated function. Subtitles should be created in the ui
+ * where plural forms and other specials can be handled more gracefully.
  *
- * @memberof dc_chat_t
  * @param chat The chat object to calulate the subtitle for.
- * @return Subtitle as a string. Must be free()'d after usage. Never NULL.
+ * @return Subtitle as a string. Must be released using dc_str_unref() after usage. Never NULL.
  */
 char*           dc_chat_get_subtitle         (const dc_chat_t* chat);
 
@@ -2603,7 +2658,7 @@ char*           dc_chat_get_subtitle         (const dc_chat_t* chat);
  * @param chat The chat object.
  * @return Path and file if the profile image, if any.
  *     NULL otherwise.
- *     Must be free()'d after usage.
+ *     Must be released using dc_str_unref() after usage.
  */
 char*           dc_chat_get_profile_image    (const dc_chat_t* chat);
 
@@ -2675,6 +2730,39 @@ int             dc_chat_is_unpromoted        (const dc_chat_t* chat);
 int             dc_chat_is_self_talk         (const dc_chat_t* chat);
 
 
+/**
+ * Check if a chat is a device-talk.
+ * Device-talks contain update information
+ * and some hints that are added during the program runs, multi-device etc.
+ *
+ * From the ui view, device-talks are not very special,
+ * the user can delete and forward messages, archive the chat, set notifications etc.
+ *
+ * Messages may be added from the core to the device chat,
+ * so the chat just pops up as usual.
+ * However, if needed the ui can also add messages using dc_add_device_msg()
+ *
+ * @memberof dc_chat_t
+ * @param chat The chat object.
+ * @return 1=chat is device-talk, 0=chat is no device-talk
+ */
+int             dc_chat_is_device_talk       (const dc_chat_t* chat);
+
+
+/**
+ * Check if messages can be sent to a give chat.
+ * This is not true eg. for the deaddrop or for the device-talk, cmp. dc_chat_is_device_talk().
+ *
+ * Calling dc_send_msg() for these chats will fail
+ * and the ui may decide to hide input controls therefore.
+ *
+ * @memberof dc_chat_t
+ * @param chat The chat object.
+ * @return 1=chat is writable, 0=chat is not writable
+ */
+int             dc_chat_can_send              (const dc_chat_t* chat);
+
+
 /**
  * Check if a chat is verified.  Verified chats contain only verified members
  * and encryption is alwasy enabled.  Verified chats are created using
@@ -2907,7 +2995,7 @@ int64_t          dc_msg_get_sort_timestamp     (const dc_msg_t* msg);
  *
  * @memberof dc_msg_t
  * @param msg The message object.
- * @return Message text. The result must be free()'d. Never returns NULL.
+ * @return Message text. The result must be released using dc_str_unref(). Never returns NULL.
  */
 char*           dc_msg_get_text               (const dc_msg_t* msg);
 
@@ -2921,9 +3009,9 @@ char*           dc_msg_get_text               (const dc_msg_t* msg);
  *
  * @memberof dc_msg_t
  * @param msg The message object.
- * @return Full path, file name and extension of the file associated with the
- *     message.  If there is no file associated with the message, an emtpy
- *     string is returned.  NULL is never returned and the returned value must be free()'d.
+ * @return Full path, file name and extension of the file associated with the message.
+ *     If there is no file associated with the message, an emtpy string is returned.
+ *     NULL is never returned and the returned value must be released using dc_str_unref().
  */
 char*           dc_msg_get_file               (const dc_msg_t* msg);
 
@@ -2936,7 +3024,7 @@ char*           dc_msg_get_file               (const dc_msg_t* msg);
  * @param msg The message object.
  * @return Base file name plus extension without part.  If there is no file
  *     associated with the message, an empty string is returned.  The returned
- *     value must be free()'d.
+ *     value must be released using dc_str_unref().
  */
 char*           dc_msg_get_filename           (const dc_msg_t* msg);
 
@@ -2948,7 +3036,8 @@ char*           dc_msg_get_filename           (const dc_msg_t* msg);
  *
  * @memberof dc_msg_t
  * @param msg The message object.
- * @return String containing the mime type. Must be free()'d after usage. NULL is never returned.
+ * @return String containing the mime type.
+ *     Must be released using dc_str_unref() after usage. NULL is never returned.
  */
 char*           dc_msg_get_filemime           (const dc_msg_t* msg);
 
@@ -3056,7 +3145,8 @@ dc_lot_t*       dc_msg_get_summary            (const dc_msg_t* msg, const dc_cha
  * @memberof dc_msg_t
  * @param msg The message object.
  * @param approx_characters Rough length of the expected string.
- * @return A summary for the given messages. The returned string must be free()'d.
+ * @return A summary for the given messages.
+ *     The returned string must be released using dc_str_unref().
  *     Returns an empty string on errors, never returns NULL.
  */
 char*           dc_msg_get_summarytext        (const dc_msg_t* msg, int approx_characters);
@@ -3201,7 +3291,7 @@ int             dc_msg_is_setupmessage        (const dc_msg_t* msg);
  * @memberof dc_msg_t
  * @param msg The message object.
  * @return Typically, the first two digits of the setup code or an empty string if unknown.
- *     NULL is never returned. Must be free()'d when done.
+ *     NULL is never returned. Must be released using dc_str_unref() when done.
  */
 char*           dc_msg_get_setupcodebegin     (const dc_msg_t* msg);
 
@@ -3328,7 +3418,8 @@ void            dc_msg_latefiling_mediasize   (dc_msg_t* msg, int width, int hei
 
 
 #define         DC_CONTACT_ID_SELF           1
-#define         DC_CONTACT_ID_DEVICE         2
+#define         DC_CONTACT_ID_INFO           2 // centered messages as "member added", used in all chats
+#define         DC_CONTACT_ID_DEVICE         5 // messages "update info" in the device-chat
 #define         DC_CONTACT_ID_LAST_SPECIAL   9
 
 
@@ -3358,7 +3449,8 @@ uint32_t        dc_contact_get_id            (const dc_contact_t* contact);
  *
  * @memberof dc_contact_t
  * @param contact The contact object.
- * @return String with the email address, must be free()'d. Never returns NULL.
+ * @return String with the email address,
+ *     must be released using dc_str_unref(). Never returns NULL.
  */
 char*           dc_contact_get_addr          (const dc_contact_t* contact);
 
@@ -3372,7 +3464,8 @@ char*           dc_contact_get_addr          (const dc_contact_t* contact);
  *
  * @memberof dc_contact_t
  * @param contact The contact object.
- * @return String with the name to display, must be free()'d. Empty string if unset, never returns NULL.
+ * @return String with the name to display, must be released using dc_str_unref().
+ *     Empty string if unset, never returns NULL.
  */
 char*           dc_contact_get_name          (const dc_contact_t* contact);
 
@@ -3386,7 +3479,8 @@ char*           dc_contact_get_name          (const dc_contact_t* contact);
  *
  * @memberof dc_contact_t
  * @param contact The contact object.
- * @return String with the name to display, must be free()'d. Never returns NULL.
+ * @return String with the name to display, must be released using dc_str_unref().
+ *     Never returns NULL.
  */
 char*           dc_contact_get_display_name  (const dc_contact_t* contact);
 
@@ -3402,7 +3496,8 @@ char*           dc_contact_get_display_name  (const dc_contact_t* contact);
  *
  * @memberof dc_contact_t
  * @param contact The contact object.
- * @return Summary string, must be free()'d. Never returns NULL.
+ * @return Summary string, must be released using dc_str_unref().
+ *     Never returns NULL.
  */
 char*           dc_contact_get_name_n_addr   (const dc_contact_t* contact);
 
@@ -3414,7 +3509,8 @@ char*           dc_contact_get_name_n_addr   (const dc_contact_t* contact);
  *
  * @memberof dc_contact_t
  * @param contact The contact object.
- * @return String with the name to display, must be free()'d. Never returns NULL.
+ * @return String with the name to display, must be released using dc_str_unref().
+ *     Never returns NULL.
  */
 char*           dc_contact_get_first_name    (const dc_contact_t* contact);
 
@@ -3428,7 +3524,7 @@ char*           dc_contact_get_first_name    (const dc_contact_t* contact);
  * @param contact The contact object.
  * @return Path and file if the profile image, if any.
  *     NULL otherwise.
- *     Must be free()'d after usage.
+ *     Must be released using dc_str_unref() after usage.
  */
 char*           dc_contact_get_profile_image (const dc_contact_t* contact);
 
@@ -3509,7 +3605,7 @@ dc_provider_t*  dc_provider_new_from_email            (const char* email);
  *
  * @memberof dc_provider_t
  * @param provider The dc_provider_t struct.
- * @return A string which must be free()d.
+ * @return A string which must be released using dc_str_unref().
  */
 char*           dc_provider_get_overview_page         (const dc_provider_t* provider);
 
@@ -3521,7 +3617,7 @@ char*           dc_provider_get_overview_page         (const dc_provider_t* prov
  *
  * @memberof dc_provider_t
  * @param provider The dc_provider_t struct.
- * @return A string which must be free()d.
+ * @return A string which must be released using dc_str_unref().
  */
 char*           dc_provider_get_name                  (const dc_provider_t* provider);
 
@@ -3534,7 +3630,7 @@ char*           dc_provider_get_name                  (const dc_provider_t* prov
  *
  * @memberof dc_provider_t
  * @param provider The dc_provider_t struct.
- * @return A string which must be free()d.
+ * @return A string which must be released using dc_str_unref().
  */
 char*           dc_provider_get_markdown              (const dc_provider_t* provider);
 
@@ -3546,7 +3642,7 @@ char*           dc_provider_get_markdown              (const dc_provider_t* prov
  *
  * @memberof dc_provider_t
  * @param provider The dc_provider_t struct.
- * @return A string which must be free()d.
+ * @return A string which must be released using dc_str_unref().
  */
 char*           dc_provider_get_status_date           (const dc_provider_t* provider);
 
@@ -3608,7 +3704,9 @@ void            dc_lot_unref             (dc_lot_t* lot);
  *
  * @memberof dc_lot_t
  * @param lot The lot object.
- * @return A string, the string may be empty and the returned value must be free()'d. NULL if there is no such string.
+ * @return A string, the string may be empty
+ *     and the returned value must be released using dc_str_unref().
+ *     NULL if there is no such string.
  */
 char*           dc_lot_get_text1         (const dc_lot_t* lot);
 
@@ -3617,10 +3715,10 @@ char*           dc_lot_get_text1         (const dc_lot_t* lot);
  * Get second string. The meaning of the string is defined by the creator of the object.
  *
  * @memberof dc_lot_t
- *
  * @param lot The lot object.
- *
- * @return A string, the string may be empty and the returned value must be free()'d	. NULL if there is no such string.
+ * @return A string, the string may be empty
+ *     and the returned value must be released using dc_str_unref().
+ *     NULL if there is no such string.
  */
 char*           dc_lot_get_text2         (const dc_lot_t* lot);
 
@@ -3641,9 +3739,7 @@ int             dc_lot_get_text1_meaning (const dc_lot_t* lot);
  * Get the associated state. The meaning of the state is defined by the creator of the object.
  *
  * @memberof dc_lot_t
- *
  * @param lot The lot object.
- *
  * @return The state as defined by the creator of the object. 0 if there is not state or on errors.
  */
 int             dc_lot_get_state         (const dc_lot_t* lot);
@@ -3665,9 +3761,7 @@ uint32_t        dc_lot_get_id            (const dc_lot_t* lot);
  * The meaning of the timestamp is defined by the creator of the object.
  *
  * @memberof dc_lot_t
- *
  * @param lot The lot object.
- *
  * @return The timestamp as defined by the creator of the object. 0 if there is not timestamp or on errors.
  */
 int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
@@ -3719,6 +3813,14 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_MSG_GIF       21
 
 
+/**
+ * Message containing a sticker, similar to image.
+ * If possible, the ui should display the image without borders in a transparent way.
+ * A click on a sticker will offer to install the sticker set in some future.
+ */
+#define DC_MSG_STICKER     23
+
+
 /**
  * Message containing an Audio file.
  * File and duration are set via dc_msg_set_file(), dc_msg_set_duration()
@@ -3836,6 +3938,67 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_LP_IMAP_SOCKET_FLAGS (DC_LP_IMAP_SOCKET_STARTTLS|DC_LP_IMAP_SOCKET_SSL|DC_LP_IMAP_SOCKET_PLAIN) // if none of these flags are set, the default is chosen
 #define DC_LP_SMTP_SOCKET_FLAGS (DC_LP_SMTP_SOCKET_STARTTLS|DC_LP_SMTP_SOCKET_SSL|DC_LP_SMTP_SOCKET_PLAIN) // if none of these flags are set, the default is chosen
 
+/**
+ * @defgroup DC_CERTCK DC_CERTCK
+ *
+ * These constants configure TLS certificate checks for IMAP and SMTP connections.
+ *
+ * These constants are set via dc_set_config()
+ * using keys "imap_certificate_checks" and "smtp_certificate_checks".
+ *
+ * @addtogroup DC_CERTCK
+ * @{
+ */
+
+/**
+ * Configure certificate checks automatically.
+ */
+#define DC_CERTCK_AUTO 0
+
+/**
+ * Strictly check TLS certificates;
+ * require that both the certificate and hostname are valid.
+ */
+#define DC_CERTCK_STRICT 1
+
+/**
+ * Accept invalid hostnames, but not invalid certificates.
+ */
+#define DC_CERTCK_ACCEPT_INVALID_HOSTNAMES 2
+
+/**
+ * Accept invalid certificates, including self-signed ones
+ * or having incorrect hostname.
+ */
+#define DC_CERTCK_ACCEPT_INVALID_CERTIFICATES 3
+
+/**
+ * @}
+ */
+
+
+/**
+ * @defgroup DC_EMPTY DC_EMPTY
+ *
+ * These constants configure emptying imap folders with dc_empty_server()
+ *
+ * @addtogroup DC_EMPTY
+ * @{
+ */
+
+/**
+ * Clear all mvbox messages.
+ */
+#define DC_EMPTY_MVBOX 0x01
+
+/**
+ * Clear all INBOX messages.
+ */
+#define DC_EMPTY_INBOX 0x02
+
+/**
+ * @}
+ */
 
 
 /**
@@ -3850,7 +4013,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @{
  */
 
-
 /**
  * The library-user may write an informational string to the log.
  * Passed to the callback given to dc_context_new().
@@ -3859,7 +4021,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
- *     Must not be free()'d or modified and is valid only until the callback returns.
+ *     Must not be unref'd or modified and is valid only until the callback returns.
  * @return 0
  */
 #define DC_EVENT_INFO                     100
@@ -3870,7 +4032,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
- *     Must not be free()'d or modified and is valid only until the callback returns.
+ *     Must not be unref'd or modified and is valid only until the callback returns.
  * @return 0
  */
 #define DC_EVENT_SMTP_CONNECTED           101
@@ -3881,7 +4043,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
- *     Must not be free()'d or modified and is valid only until the callback returns.
+ *     Must not be unref'd or modified and is valid only until the callback returns.
  * @return 0
  */
 #define DC_EVENT_IMAP_CONNECTED           102
@@ -3891,11 +4053,60 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
- *     Must not be free()'d or modified and is valid only until the callback returns.
+ *     Must not be unref'd or modified and is valid only until the callback returns.
  * @return 0
  */
 #define DC_EVENT_SMTP_MESSAGE_SENT        103
 
+/**
+ * Emitted when a message was successfully marked as deleted on the IMAP server.
+ *
+ * @param data1 0
+ * @param data2 (const char*) Info string in english language.
+ *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
+ */
+#define DC_EVENT_IMAP_MESSAGE_DELETED   104
+
+/**
+ * Emitted when a message was successfully moved on IMAP.
+ *
+ * @param data1 0
+ * @param data2 (const char*) Info string in english language.
+ *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
+ */
+#define DC_EVENT_IMAP_MESSAGE_MOVED   105
+
+/**
+ * Emitted when an IMAP folder was emptied. 
+ *
+ * @param data1 0
+ * @param data2 (const char*) folder name. 
+ *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
+ */
+#define DC_EVENT_IMAP_FOLDER_EMPTIED  106
+
+/**
+ * Emitted when a new blob file was successfully written 
+ *
+ * @param data1 0
+ * @param data2 (const char*) path name 
+ *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
+ */
+#define DC_EVENT_NEW_BLOB_FILE 150
+
+/**
+ * Emitted when a blob file was successfully deleted 
+ *
+ * @param data1 0
+ * @param data2 (const char*) path name 
+ *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
+ */
+#define DC_EVENT_DELETED_BLOB_FILE 151
 
 /**
  * The library-user should write a warning string to the log.
@@ -3905,7 +4116,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 0
  * @param data2 (const char*) Warning string in english language.
- *     Must not be free()'d or modified and is valid only until the callback returns.
+ *     Must not be unref'd or modified and is valid only until the callback returns.
  * @return 0
  */
 #define DC_EVENT_WARNING                  300
@@ -3925,9 +4136,10 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * in a messasge box then.
  *
  * @param data1 0
- * @param data2 (const char*) Error string, always set, never NULL. Frequent error strings are
- *     localized using #DC_EVENT_GET_STRING, however, most error strings will be in english language.
- *     Must not be free()'d or modified and is valid only until the callback returns.
+ * @param data2 (const char*) Error string, always set, never NULL.
+ *     Some error strings are taken from dc_set_stock_translation(),
+ *     however, most error strings will be in english language.
+ *     Must not be unref'd or modified and is valid only until the callback returns.
  * @return 0
  */
 #define DC_EVENT_ERROR                    400
@@ -3951,7 +4163,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 (int) 1=first/new network error, should be reported the user;
  *     0=subsequent network error, should be logged only
  * @param data2 (const char*) Error string, always set, never NULL.
- *     Must not be free()'d or modified and is valid only until the callback returns.
+ *     Must not be unref'd or modified and is valid only until the callback returns.
  * @return 0
  */
 #define DC_EVENT_ERROR_NETWORK            401
@@ -3966,7 +4178,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
- *     Must not be free()'d or modified
+ *     Must not be unref'd or modified
  *     and is valid only until the callback returns.
  * @return 0
  */
@@ -4097,7 +4309,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * services.
  *
  * @param data1 (const char*) Path and file name.
- *     Must not be free()'d or modified and is valid only until the callback returns.
+ *     Must not be unref'd or modified and is valid only until the callback returns.
  * @param data2 0
  * @return 0
  */
@@ -4138,36 +4350,21 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_EVENT_SECUREJOIN_JOINER_PROGRESS       2061
 
 
-// the following events are functions that should be provided by the frontends
-
-
-/**
- * Requeste a localized string from the frontend.
- *
- * @param data1 (int) ID of the string to request, one of the DC_STR_* constants.
- * @param data2 (int) The count. If the requested string contains a placeholder for a numeric value,
- *     the ui may use this value to return different strings on different plural forms.
- * @return (const char*) Null-terminated UTF-8 string.
- *     The string will be free()'d by the core,
- *     so it must be allocated using malloc() or a compatible function.
- *     Return 0 if the ui cannot provide the requested string
- *     the core will use a default string in english language then.
- */
-#define DC_EVENT_GET_STRING               2091
-
 /**
  * @}
  */
 
-#define DC_EVENT_FILE_COPIED         2055 // deprecated
-#define DC_EVENT_IS_OFFLINE          2081 // deprecated
-#define DC_ERROR_SEE_STRING          0    // deprecated
-#define DC_ERROR_SELF_NOT_IN_GROUP   1    // deprecated
-#define DC_STR_SELFNOTINGRP          21   // deprecated
+
+#define DC_EVENT_FILE_COPIED         2055 // not used anymore
+#define DC_EVENT_IS_OFFLINE          2081 // not used anymore
+#define DC_EVENT_GET_STRING          2091 // not used anymore, use dc_set_stock_translation()
+#define DC_ERROR_SEE_STRING          0    // not used anymore
+#define DC_ERROR_SELF_NOT_IN_GROUP   1    // not used anymore
+#define DC_STR_SELFNOTINGRP          21   // not used anymore
 #define DC_EVENT_DATA1_IS_STRING(e)  ((e)==DC_EVENT_IMEX_FILE_WRITTEN || (e)==DC_EVENT_FILE_COPIED)
 #define DC_EVENT_DATA2_IS_STRING(e)  ((e)>=100 && (e)<=499)
-#define DC_EVENT_RETURNS_INT(e)      ((e)==DC_EVENT_IS_OFFLINE)
-#define DC_EVENT_RETURNS_STRING(e)   ((e)==DC_EVENT_GET_STRING)
+#define DC_EVENT_RETURNS_INT(e)      ((e)==DC_EVENT_IS_OFFLINE) // not used anymore
+#define DC_EVENT_RETURNS_STRING(e)   ((e)==DC_EVENT_GET_STRING) // not used anymore
 char*           dc_get_version_str           (void); // deprecated
 void            dc_array_add_id              (dc_array_t*, uint32_t); // deprecated
 
@@ -4269,10 +4466,8 @@ void            dc_array_add_id              (dc_array_t*, uint32_t); // depreca
 #define DC_STR_MSGLOCATIONENABLED         64
 #define DC_STR_MSGLOCATIONDISABLED        65
 #define DC_STR_LOCATION                   66
-#define DC_STR_COUNT                      66
-
-void dc_str_unref (char*);
-
+#define DC_STR_STICKER                    67
+#define DC_STR_COUNT                      67
 
 /*
  * @}

deltachat-ffi/src/lib.rs

@@ -22,11 +22,14 @@ use std::sync::RwLock;
 use libc::uintptr_t;
 use num_traits::{FromPrimitive, ToPrimitive};
 
+use deltachat::constants::DC_MSG_ID_LAST_SPECIAL;
 use deltachat::contact::Contact;
 use deltachat::context::Context;
 use deltachat::dc_tools::{
-    as_path, as_str, dc_strdup, to_string, to_string_lossy, OsStrExt, StrExt,
+    as_path, dc_strdup, to_opt_string_lossy, to_string_lossy, OsStrExt, StrExt,
 };
+use deltachat::message::MsgId;
+use deltachat::stock::StockMessage;
 use deltachat::*;
 
 // as C lacks a good and portable error handling,
@@ -125,20 +128,28 @@ impl ContextWrapper {
                     | Event::SmtpConnected(msg)
                     | Event::ImapConnected(msg)
                     | Event::SmtpMessageSent(msg)
+                    | Event::ImapMessageDeleted(msg)
+                    | Event::ImapMessageMoved(msg)
+                    | Event::ImapFolderEmptied(msg)
+                    | Event::NewBlobFile(msg)
+                    | Event::DeletedBlobFile(msg)
                     | Event::Warning(msg)
                     | Event::Error(msg)
                     | Event::ErrorNetwork(msg)
                     | Event::ErrorSelfNotInGroup(msg) => {
-                        let data2 = CString::new(msg).unwrap();
+                        let data2 = CString::new(msg).unwrap_or_default();
                         ffi_cb(self, event_id, 0, data2.as_ptr() as uintptr_t)
                     }
                     Event::MsgsChanged { chat_id, msg_id }
                     | Event::IncomingMsg { chat_id, msg_id }
                     | Event::MsgDelivered { chat_id, msg_id }
                     | Event::MsgFailed { chat_id, msg_id }
-                    | Event::MsgRead { chat_id, msg_id } => {
-                        ffi_cb(self, event_id, chat_id as uintptr_t, msg_id as uintptr_t)
-                    }
+                    | Event::MsgRead { chat_id, msg_id } => ffi_cb(
+                        self,
+                        event_id,
+                        chat_id as uintptr_t,
+                        msg_id.to_u32() as uintptr_t,
+                    ),
                     Event::ChatModified(chat_id) => ffi_cb(self, event_id, chat_id as uintptr_t, 0),
                     Event::ContactsChanged(id) | Event::LocationChanged(id) => {
                         let id = id.unwrap_or_default();
@@ -148,7 +159,7 @@ impl ContextWrapper {
                         ffi_cb(self, event_id, progress as uintptr_t, 0)
                     }
                     Event::ImexFileWritten(file) => {
-                        let data1 = file.to_c_string().unwrap();
+                        let data1 = file.to_c_string().unwrap_or_default();
                         ffi_cb(self, event_id, data1.as_ptr() as uintptr_t, 0)
                     }
                     Event::SecurejoinInviterProgress {
@@ -164,12 +175,6 @@ impl ContextWrapper {
                         contact_id as uintptr_t,
                         progress as uintptr_t,
                     ),
-                    Event::GetString { id, count } => ffi_cb(
-                        self,
-                        event_id,
-                        id.to_u32().unwrap_or_default() as uintptr_t,
-                        count as uintptr_t,
-                    ),
                 }
             }
             None => 0,
@@ -306,11 +311,14 @@ pub unsafe extern "C" fn dc_set_config(
         return 0;
     }
     let ffi_context = &*context;
-    match config::Config::from_str(as_str(key)) {
+    match config::Config::from_str(&to_string_lossy(key)) {
         // When ctx.set_config() fails it already logged the error.
         // TODO: Context::set_config() should not log this
         Ok(key) => ffi_context
-            .with_inner(|ctx| ctx.set_config(key, as_opt_str(value)).is_ok() as libc::c_int)
+            .with_inner(|ctx| {
+                ctx.set_config(key, to_opt_string_lossy(value).as_ref().map(|x| x.as_str()))
+                    .is_ok() as libc::c_int
+            })
             .unwrap_or(0),
         Err(_) => {
             ffi_context.error("dc_set_config(): invalid key");
@@ -329,7 +337,7 @@ pub unsafe extern "C" fn dc_get_config(
         return "".strdup();
     }
     let ffi_context = &*context;
-    match config::Config::from_str(as_str(key)) {
+    match config::Config::from_str(&to_string_lossy(key)) {
         Ok(key) => ffi_context
             .with_inner(|ctx| ctx.get_config(key).unwrap_or_default().strdup())
             .unwrap_or_else(|_| "".strdup()),
@@ -340,6 +348,35 @@ pub unsafe extern "C" fn dc_get_config(
     }
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_set_stock_translation(
+    context: *mut dc_context_t,
+    stock_id: u32,
+    stock_msg: *mut libc::c_char,
+) -> libc::c_int {
+    if context.is_null() || stock_msg.is_null() {
+        eprintln!("ignoring careless call to dc_set_stock_string");
+        return 0;
+    }
+    let msg = to_string_lossy(stock_msg);
+    let ffi_context = &*context;
+    ffi_context
+        .with_inner(|ctx| match StockMessage::from_u32(stock_id) {
+            Some(id) => match ctx.set_stock_translation(id, msg) {
+                Ok(()) => 1,
+                Err(err) => {
+                    warn!(ctx, "set_stock_translation failed: {}", err);
+                    0
+                }
+            },
+            None => {
+                warn!(ctx, "invalid stock message id {}", stock_id);
+                0
+            }
+        })
+        .unwrap_or(0)
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_info(context: *mut dc_context_t) -> *mut libc::c_char {
     if context.is_null() {
@@ -377,8 +414,8 @@ pub unsafe extern "C" fn dc_get_oauth2_url(
         return ptr::null_mut(); // NULL explicitly defined as "unknown"
     }
     let ffi_context = &*context;
-    let addr = to_string(addr);
-    let redirect = to_string(redirect);
+    let addr = to_string_lossy(addr);
+    let redirect = to_string_lossy(redirect);
     ffi_context
         .with_inner(|ctx| match oauth2::dc_get_oauth2_url(ctx, addr, redirect) {
             Some(res) => res.strdup(),
@@ -620,22 +657,24 @@ pub unsafe extern "C" fn dc_get_chatlist(
         return ptr::null_mut();
     }
     let ffi_context = &*context;
-    let qs = if query_str.is_null() {
-        None
-    } else {
-        Some(as_str(query_str))
-    };
+    let qs = to_opt_string_lossy(query_str);
+
     let qi = if query_id == 0 { None } else { Some(query_id) };
     ffi_context
-        .with_inner(
-            |ctx| match chatlist::Chatlist::try_load(ctx, flags as usize, qs, qi) {
+        .with_inner(|ctx| {
+            match chatlist::Chatlist::try_load(
+                ctx,
+                flags as usize,
+                qs.as_ref().map(|x| x.as_str()),
+                qi,
+            ) {
                 Ok(list) => {
                     let ffi_list = ChatlistWrapper { context, list };
                     Box::into_raw(Box::new(ffi_list))
                 }
                 Err(_) => ptr::null_mut(),
-            },
-        )
+            }
+        })
         .unwrap_or_else(|_| ptr::null_mut())
 }
 
@@ -648,7 +687,8 @@ pub unsafe extern "C" fn dc_create_chat_by_msg_id(context: *mut dc_context_t, ms
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            chat::create_by_msg_id(ctx, msg_id).unwrap_or_log_default(ctx, "Failed to create chat")
+            chat::create_by_msg_id(ctx, MsgId::new(msg_id))
+                .unwrap_or_log_default(ctx, "Failed to create chat")
         })
         .unwrap_or(0)
 }
@@ -682,10 +722,7 @@ pub unsafe extern "C" fn dc_get_chat_id_by_contact_id(
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| {
-            chat::get_by_contact_id(ctx, contact_id)
-                .unwrap_or_log_default(ctx, "Failed to get chat")
-        })
+        .with_inner(|ctx| chat::get_by_contact_id(ctx, contact_id).unwrap_or(0))
         .unwrap_or(0)
 }
 
@@ -706,6 +743,7 @@ pub unsafe extern "C" fn dc_prepare_msg(
             chat::prepare_msg(ctx, chat_id, &mut ffi_msg.message)
                 .unwrap_or_log_default(ctx, "Failed to prepare message")
         })
+        .map(|msg_id| msg_id.to_u32())
         .unwrap_or(0)
 }
 
@@ -726,6 +764,7 @@ pub unsafe extern "C" fn dc_send_msg(
             chat::send_msg(ctx, chat_id, &mut ffi_msg.message)
                 .unwrap_or_log_default(ctx, "Failed to send message")
         })
+        .map(|msg_id| msg_id.to_u32())
         .unwrap_or(0)
 }
 
@@ -744,6 +783,7 @@ pub unsafe extern "C" fn dc_send_text_msg(
     ffi_context
         .with_inner(|ctx| {
             chat::send_text_msg(ctx, chat_id, text_to_send)
+                .map(|msg_id| msg_id.to_u32())
                 .unwrap_or_log_default(ctx, "Failed to send text message")
         })
         .unwrap_or(0)
@@ -771,6 +811,23 @@ pub unsafe extern "C" fn dc_set_draft(
         .unwrap_or(())
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_add_device_msg(context: *mut dc_context_t, msg: *mut dc_msg_t) -> u32 {
+    if context.is_null() || msg.is_null() {
+        eprintln!("ignoring careless call to dc_add_device_msg()");
+        return 0;
+    }
+    let ffi_context = &mut *context;
+    let ffi_msg = &mut *msg;
+    ffi_context
+        .with_inner(|ctx| {
+            chat::add_device_msg(ctx, &mut ffi_msg.message)
+                .unwrap_or_log_default(ctx, "Failed to add device message")
+        })
+        .map(|msg_id| msg_id.to_u32())
+        .unwrap_or(0)
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_draft(context: *mut dc_context_t, chat_id: u32) -> *mut dc_msg_t {
     if context.is_null() {
@@ -808,9 +865,19 @@ pub unsafe extern "C" fn dc_get_chat_msgs(
         return ptr::null_mut();
     }
     let ffi_context = &*context;
+    let marker_flag = if marker1before <= DC_MSG_ID_LAST_SPECIAL {
+        None
+    } else {
+        Some(MsgId::new(marker1before))
+    };
     ffi_context
         .with_inner(|ctx| {
-            let arr = dc_array_t::from(chat::get_chat_msgs(ctx, chat_id, flags, marker1before));
+            let arr = dc_array_t::from(
+                chat::get_chat_msgs(ctx, chat_id, flags, marker_flag)
+                    .iter()
+                    .map(|msg_id| msg_id.to_u32())
+                    .collect::<Vec<u32>>(),
+            );
             Box::into_raw(Box::new(arr))
         })
         .unwrap_or_else(|_| ptr::null_mut())
@@ -854,7 +921,12 @@ pub unsafe extern "C" fn dc_get_fresh_msgs(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            let arr = dc_array_t::from(ctx.get_fresh_msgs());
+            let arr = dc_array_t::from(
+                ctx.get_fresh_msgs()
+                    .iter()
+                    .map(|msg_id| msg_id.to_u32())
+                    .collect::<Vec<u32>>(),
+            );
             Box::into_raw(Box::new(arr))
         })
         .unwrap_or_else(|_| ptr::null_mut())
@@ -916,13 +988,12 @@ pub unsafe extern "C" fn dc_get_chat_media(
         from_prim(or_msg_type3).expect(&format!("incorrect or_msg_type3 = {}", or_msg_type3));
     ffi_context
         .with_inner(|ctx| {
-            let arr = dc_array_t::from(chat::get_chat_media(
-                ctx,
-                chat_id,
-                msg_type,
-                or_msg_type2,
-                or_msg_type3,
-            ));
+            let arr = dc_array_t::from(
+                chat::get_chat_media(ctx, chat_id, msg_type, or_msg_type2, or_msg_type3)
+                    .iter()
+                    .map(|msg_id| msg_id.to_u32())
+                    .collect::<Vec<u32>>(),
+            );
             Box::into_raw(Box::new(arr))
         })
         .unwrap_or_else(|_| ptr::null_mut())
@@ -955,7 +1026,16 @@ pub unsafe extern "C" fn dc_get_next_media(
         from_prim(or_msg_type3).expect(&format!("incorrect or_msg_type3 = {}", or_msg_type3));
     ffi_context
         .with_inner(|ctx| {
-            chat::get_next_media(ctx, msg_id, direction, msg_type, or_msg_type2, or_msg_type3)
+            chat::get_next_media(
+                ctx,
+                MsgId::new(msg_id),
+                direction,
+                msg_type,
+                or_msg_type2,
+                or_msg_type3,
+            )
+            .map(|msg_id| msg_id.to_u32())
+            .unwrap_or(0)
         })
         .unwrap_or(0)
 }
@@ -1026,7 +1106,12 @@ pub unsafe extern "C" fn dc_search_msgs(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            let arr = dc_array_t::from(ctx.search_msgs(chat_id, as_str(query)));
+            let arr = dc_array_t::from(
+                ctx.search_msgs(chat_id, to_string_lossy(query))
+                    .iter()
+                    .map(|msg_id| msg_id.to_u32())
+                    .collect::<Vec<u32>>(),
+            );
             Box::into_raw(Box::new(arr))
         })
         .unwrap_or_else(|_| ptr::null_mut())
@@ -1068,7 +1153,7 @@ pub unsafe extern "C" fn dc_create_group_chat(
     };
     ffi_context
         .with_inner(|ctx| {
-            chat::create_group_chat(ctx, verified, as_str(name))
+            chat::create_group_chat(ctx, verified, to_string_lossy(name))
                 .unwrap_or_log_default(ctx, "Failed to create group chat")
         })
         .unwrap_or(0)
@@ -1140,7 +1225,7 @@ pub unsafe extern "C" fn dc_set_chat_name(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            chat::set_chat_name(ctx, chat_id, as_str(name))
+            chat::set_chat_name(ctx, chat_id, to_string_lossy(name))
                 .map(|_| 1)
                 .unwrap_or_log_default(ctx, "Failed to set chat name")
         })
@@ -1160,15 +1245,9 @@ pub unsafe extern "C" fn dc_set_chat_profile_image(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            chat::set_chat_profile_image(ctx, chat_id, {
-                if image.is_null() {
-                    ""
-                } else {
-                    as_str(image)
-                }
-            })
-            .map(|_| 1)
-            .unwrap_or_log_default(ctx, "Failed to set profile image")
+            chat::set_chat_profile_image(ctx, chat_id, to_string_lossy(image))
+                .map(|_| 1)
+                .unwrap_or_log_default(ctx, "Failed to set profile image")
         })
         .unwrap_or(0)
 }
@@ -1184,7 +1263,7 @@ pub unsafe extern "C" fn dc_get_msg_info(
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| message::get_msg_info(ctx, msg_id).strdup())
+        .with_inner(|ctx| message::get_msg_info(ctx, MsgId::new(msg_id)).strdup())
         .unwrap_or_else(|_| ptr::null_mut())
 }
 
@@ -1200,7 +1279,7 @@ pub unsafe extern "C" fn dc_get_mime_headers(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            message::get_mime_headers(ctx, msg_id)
+            message::get_mime_headers(ctx, MsgId::new(msg_id))
                 .map(|s| s.strdup())
                 .unwrap_or_else(|| ptr::null_mut())
         })
@@ -1218,11 +1297,21 @@ pub unsafe extern "C" fn dc_delete_msgs(
         return;
     }
     let ffi_context = &*context;
-
-    let ids = std::slice::from_raw_parts(msg_ids, msg_cnt as usize);
-
+    let msg_ids = convert_and_prune_message_ids(msg_ids, msg_cnt);
     ffi_context
-        .with_inner(|ctx| message::delete_msgs(ctx, ids))
+        .with_inner(|ctx| message::delete_msgs(ctx, &msg_ids[..]))
+        .unwrap_or(())
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_empty_server(context: *mut dc_context_t, flags: u32) {
+    if context.is_null() || flags == 0 {
+        eprintln!("ignoring careless call to dc_empty_server()");
+        return;
+    }
+    let ffi_context = &*context;
+    ffi_context
+        .with_inner(|ctx| message::dc_empty_server(ctx, flags))
         .unwrap_or(())
 }
 
@@ -1241,12 +1330,14 @@ pub unsafe extern "C" fn dc_forward_msgs(
         eprintln!("ignoring careless call to dc_forward_msgs()");
         return;
     }
-    let ids = std::slice::from_raw_parts(msg_ids, msg_cnt as usize);
-
+    let msg_ids = convert_and_prune_message_ids(msg_ids, msg_cnt);
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| chat::forward_msgs(ctx, ids, chat_id))
-        .unwrap_or(())
+        .with_inner(|ctx| {
+            chat::forward_msgs(ctx, &msg_ids[..], chat_id)
+                .unwrap_or_log_default(ctx, "Failed to forward message")
+        })
+        .unwrap_or_default()
 }
 
 #[no_mangle]
@@ -1271,11 +1362,10 @@ pub unsafe extern "C" fn dc_markseen_msgs(
         eprintln!("ignoring careless call to dc_markseen_msgs()");
         return;
     }
-    let ids = std::slice::from_raw_parts(msg_ids, msg_cnt as usize);
-
+    let msg_ids = convert_and_prune_message_ids(msg_ids, msg_cnt);
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| message::markseen_msgs(ctx, ids))
+        .with_inner(|ctx| message::markseen_msgs(ctx, &msg_ids[..]))
         .ok();
 }
 
@@ -1290,12 +1380,10 @@ pub unsafe extern "C" fn dc_star_msgs(
         eprintln!("ignoring careless call to dc_star_msgs()");
         return;
     }
-
-    let ids = std::slice::from_raw_parts(msg_ids, msg_cnt as usize);
-
+    let msg_ids = convert_and_prune_message_ids(msg_ids, msg_cnt);
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| message::star_msgs(ctx, ids, star == 1))
+        .with_inner(|ctx| message::star_msgs(ctx, &msg_ids[..], star == 1))
         .ok();
 }
 
@@ -1308,11 +1396,23 @@ pub unsafe extern "C" fn dc_get_msg(context: *mut dc_context_t, msg_id: u32) ->
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            let message = match message::Message::load_from_db(ctx, msg_id) {
+            let message = match message::Message::load_from_db(ctx, MsgId::new(msg_id)) {
                 Ok(msg) => msg,
                 Err(e) => {
-                    error!(ctx, "Error getting msg #{}: {}", msg_id, e);
-                    return ptr::null_mut();
+                    if msg_id <= constants::DC_MSG_ID_LAST_SPECIAL {
+                        // C-core API returns empty messages, do the same
+                        warn!(
+                            ctx,
+                            "dc_get_msg called with special msg_id={}, returning empty msg", msg_id
+                        );
+                        message::Message::default()
+                    } else {
+                        error!(
+                            ctx,
+                            "dc_get_msg could not retrieve msg_id {}: {}", msg_id, e
+                        );
+                        return ptr::null_mut();
+                    }
                 }
             };
             let ffi_msg = MessageWrapper { context, message };
@@ -1328,7 +1428,7 @@ pub unsafe extern "C" fn dc_may_be_valid_addr(addr: *const libc::c_char) -> libc
         return 0;
     }
 
-    contact::may_be_valid_addr(as_str(addr)) as libc::c_int
+    contact::may_be_valid_addr(&to_string_lossy(addr)) as libc::c_int
 }
 
 #[no_mangle]
@@ -1342,7 +1442,7 @@ pub unsafe extern "C" fn dc_lookup_contact_id_by_addr(
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| Contact::lookup_id_by_addr(ctx, as_str(addr)))
+        .with_inner(|ctx| Contact::lookup_id_by_addr(ctx, to_string_lossy(addr)))
         .unwrap_or(0)
 }
 
@@ -1357,12 +1457,14 @@ pub unsafe extern "C" fn dc_create_contact(
         return 0;
     }
     let ffi_context = &*context;
-    let name = if name.is_null() { "" } else { as_str(name) };
+    let name = to_string_lossy(name);
     ffi_context
-        .with_inner(|ctx| match Contact::create(ctx, name, as_str(addr)) {
-            Ok(id) => id,
-            Err(_) => 0,
-        })
+        .with_inner(
+            |ctx| match Contact::create(ctx, name, to_string_lossy(addr)) {
+                Ok(id) => id,
+                Err(_) => 0,
+            },
+        )
         .unwrap_or(0)
 }
 
@@ -1378,7 +1480,7 @@ pub unsafe extern "C" fn dc_add_address_book(
     let ffi_context = &*context;
     ffi_context
         .with_inner(
-            |ctx| match Contact::add_address_book(ctx, as_str(addr_book)) {
+            |ctx| match Contact::add_address_book(ctx, to_string_lossy(addr_book)) {
                 Ok(cnt) => cnt as libc::c_int,
                 Err(_) => 0,
             },
@@ -1397,11 +1499,7 @@ pub unsafe extern "C" fn dc_get_contacts(
         return ptr::null_mut();
     }
     let ffi_context = &*context;
-    let query = if query.is_null() {
-        None
-    } else {
-        Some(as_str(query))
-    };
+    let query = to_opt_string_lossy(query);
     ffi_context
         .with_inner(|ctx| match Contact::get_all(ctx, flags, query) {
             Ok(contacts) => Box::into_raw(Box::new(dc_array_t::from(contacts))),
@@ -1538,7 +1636,7 @@ pub unsafe extern "C" fn dc_imex(
 
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| imex::imex(ctx, what, as_opt_str(param1)))
+        .with_inner(|ctx| imex::imex(ctx, what, to_opt_string_lossy(param1)))
         .ok();
 }
 
@@ -1553,7 +1651,7 @@ pub unsafe extern "C" fn dc_imex_has_backup(
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| match imex::has_backup(ctx, as_str(dir)) {
+        .with_inner(|ctx| match imex::has_backup(ctx, to_string_lossy(dir)) {
             Ok(res) => res.strdup(),
             Err(err) => {
                 error!(ctx, "dc_imex_has_backup: {}", err);
@@ -1596,15 +1694,16 @@ pub unsafe extern "C" fn dc_continue_key_transfer(
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(
-            |ctx| match imex::continue_key_transfer(ctx, msg_id, as_str(setup_code)) {
+        .with_inner(|ctx| {
+            match imex::continue_key_transfer(ctx, MsgId::new(msg_id), &to_string_lossy(setup_code))
+            {
                 Ok(()) => 1,
                 Err(err) => {
                     error!(ctx, "dc_continue_key_transfer: {}", err);
                     0
                 }
-            },
-        )
+            }
+        })
         .unwrap_or(0)
 }
 
@@ -1615,9 +1714,7 @@ pub unsafe extern "C" fn dc_stop_ongoing_process(context: *mut dc_context_t) {
         return;
     }
     let ffi_context = &*context;
-    ffi_context
-        .with_inner(|ctx| configure::dc_stop_ongoing_process(ctx))
-        .ok();
+    ffi_context.with_inner(|ctx| ctx.stop_ongoing()).ok();
 }
 
 #[no_mangle]
@@ -1632,7 +1729,7 @@ pub unsafe extern "C" fn dc_check_qr(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            let lot = qr::check_qr(ctx, as_str(qr));
+            let lot = qr::check_qr(ctx, to_string_lossy(qr));
             Box::into_raw(Box::new(lot))
         })
         .unwrap_or_else(|_| ptr::null_mut())
@@ -1668,7 +1765,7 @@ pub unsafe extern "C" fn dc_join_securejoin(
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| securejoin::dc_join_securejoin(ctx, as_str(qr)))
+        .with_inner(|ctx| securejoin::dc_join_securejoin(ctx, &to_string_lossy(qr)))
         .unwrap_or(0)
 }
 
@@ -1717,7 +1814,7 @@ pub unsafe extern "C" fn dc_set_location(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| location::set(ctx, latitude, longitude, accuracy))
-        .unwrap_or(0)
+        .unwrap_or(false) as _
 }
 
 #[no_mangle]
@@ -2010,7 +2107,11 @@ pub unsafe extern "C" fn dc_chatlist_get_msg_id(
         return 0;
     }
     let ffi_list = &*chatlist;
-    ffi_list.list.get_msg_id(index as usize)
+    ffi_list
+        .list
+        .get_msg_id(index as usize)
+        .map(|msg_id| msg_id.to_u32())
+        .unwrap_or(0)
 }
 
 #[no_mangle]
@@ -2131,7 +2232,7 @@ pub unsafe extern "C" fn dc_chat_get_profile_image(chat: *mut dc_chat_t) -> *mut
     let ffi_context = &*ffi_chat.context;
     ffi_context
         .with_inner(|ctx| match ffi_chat.chat.get_profile_image(ctx) {
-            Some(p) => p.to_str().unwrap().to_string().strdup(),
+            Some(p) => p.to_string_lossy().strdup(),
             None => ptr::null_mut(),
         })
         .unwrap_or_else(|_| ptr::null_mut())
@@ -2180,6 +2281,26 @@ pub unsafe extern "C" fn dc_chat_is_self_talk(chat: *mut dc_chat_t) -> libc::c_i
     ffi_chat.chat.is_self_talk() as libc::c_int
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_chat_is_device_talk(chat: *mut dc_chat_t) -> libc::c_int {
+    if chat.is_null() {
+        eprintln!("ignoring careless call to dc_chat_is_device_talk()");
+        return 0;
+    }
+    let ffi_chat = &*chat;
+    ffi_chat.chat.is_device_talk() as libc::c_int
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_chat_can_send(chat: *mut dc_chat_t) -> libc::c_int {
+    if chat.is_null() {
+        eprintln!("ignoring careless call to dc_chat_can_send()");
+        return 0;
+    }
+    let ffi_chat = &*chat;
+    ffi_chat.chat.can_send() as libc::c_int
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_chat_is_verified(chat: *mut dc_chat_t) -> libc::c_int {
     if chat.is_null() {
@@ -2252,7 +2373,7 @@ pub unsafe extern "C" fn dc_msg_get_id(msg: *mut dc_msg_t) -> u32 {
         return 0;
     }
     let ffi_msg = &*msg;
-    ffi_msg.message.get_id()
+    ffi_msg.message.get_id().to_u32()
 }
 
 #[no_mangle]
@@ -2476,7 +2597,7 @@ pub unsafe extern "C" fn dc_msg_get_summarytext(
         .with_inner(|ctx| {
             ffi_msg
                 .message
-                .get_summarytext(ctx, approx_characters.try_into().unwrap())
+                .get_summarytext(ctx, approx_characters.try_into().unwrap_or_default())
         })
         .unwrap_or_default()
         .strdup()
@@ -2583,8 +2704,7 @@ pub unsafe extern "C" fn dc_msg_set_text(msg: *mut dc_msg_t, text: *const libc::
         return;
     }
     let ffi_msg = &mut *msg;
-    // TODO: {text} equal to NULL is treated as "", which is strange. Does anyone rely on it?
-    ffi_msg.message.set_text(as_opt_str(text).map(Into::into))
+    ffi_msg.message.set_text(to_opt_string_lossy(text))
 }
 
 #[no_mangle]
@@ -2598,7 +2718,10 @@ pub unsafe extern "C" fn dc_msg_set_file(
         return;
     }
     let ffi_msg = &mut *msg;
-    ffi_msg.message.set_file(as_str(file), as_opt_str(filemime))
+    ffi_msg.message.set_file(
+        to_string_lossy(file),
+        to_opt_string_lossy(filemime).as_ref().map(|x| x.as_str()),
+    )
 }
 
 #[no_mangle]
@@ -2768,7 +2891,7 @@ pub unsafe extern "C" fn dc_contact_get_profile_image(
             ffi_contact
                 .contact
                 .get_profile_image(ctx)
-                .map(|p| p.to_str().unwrap().to_string().strdup())
+                .map(|p| p.to_string_lossy().strdup())
                 .unwrap_or_else(|| std::ptr::null_mut())
         })
         .unwrap_or_else(|_| ptr::null_mut())
@@ -2893,14 +3016,6 @@ pub unsafe extern "C" fn dc_str_unref(s: *mut libc::c_char) {
     libc::free(s as *mut _)
 }
 
-fn as_opt_str<'a>(s: *const libc::c_char) -> Option<&'a str> {
-    if s.is_null() {
-        return None;
-    }
-
-    Some(as_str(s))
-}
-
 pub mod providers;
 
 pub trait ResultExt<T> {
@@ -2945,3 +3060,14 @@ impl<T, E> ResultNullableExt<T> for Result<T, E> {
         }
     }
 }
+
+fn convert_and_prune_message_ids(msg_ids: *const u32, msg_cnt: libc::c_int) -> Vec<MsgId> {
+    let ids = unsafe { std::slice::from_raw_parts(msg_ids, msg_cnt as usize) };
+    let msg_ids: Vec<MsgId> = ids
+        .iter()
+        .filter(|id| **id > DC_MSG_ID_LAST_SPECIAL)
+        .map(|id| MsgId::new(*id))
+        .collect();
+
+    msg_ids
+}

deltachat-ffi/src/providers.rs

@@ -2,7 +2,7 @@ extern crate deltachat_provider_database;
 
 use std::ptr;
 
-use deltachat::dc_tools::{as_str, StrExt};
+use deltachat::dc_tools::{to_string_lossy, StrExt};
 use deltachat_provider_database::StatusState;
 
 #[no_mangle]
@@ -12,7 +12,7 @@ pub type dc_provider_t = deltachat_provider_database::Provider;
 pub unsafe extern "C" fn dc_provider_new_from_domain(
     domain: *const libc::c_char,
 ) -> *const dc_provider_t {
-    match deltachat_provider_database::get_provider_info(as_str(domain)) {
+    match deltachat_provider_database::get_provider_info(&to_string_lossy(domain)) {
         Some(provider) => provider,
         None => ptr::null(),
     }
@@ -22,7 +22,8 @@ pub unsafe extern "C" fn dc_provider_new_from_domain(
 pub unsafe extern "C" fn dc_provider_new_from_email(
     email: *const libc::c_char,
 ) -> *const dc_provider_t {
-    let domain = deltachat_provider_database::get_domain_from_email(as_str(email));
+    let email = to_string_lossy(email);
+    let domain = deltachat_provider_database::get_domain_from_email(&email);
     match deltachat_provider_database::get_provider_info(domain) {
         Some(provider) => provider,
         None => ptr::null(),

examples/repl/cmdline.rs

@@ -4,7 +4,6 @@ use std::str::FromStr;
 use deltachat::chat::{self, Chat};
 use deltachat::chatlist::*;
 use deltachat::config;
-use deltachat::configure::*;
 use deltachat::constants::*;
 use deltachat::contact::*;
 use deltachat::context::*;
@@ -15,7 +14,7 @@ use deltachat::imex::*;
 use deltachat::job::*;
 use deltachat::location;
 use deltachat::lot::LotState;
-use deltachat::message::{self, Message, MessageState};
+use deltachat::message::{self, Message, MessageState, MsgId};
 use deltachat::peerstate::*;
 use deltachat::qr::*;
 use deltachat::sql;
@@ -87,7 +86,7 @@ pub unsafe fn dc_reset_tables(context: &Context, bits: i32) -> i32 {
 
     context.call_cb(Event::MsgsChanged {
         chat_id: 0,
-        msg_id: 0,
+        msg_id: MsgId::new(0),
     });
 
     1
@@ -119,13 +118,13 @@ fn poke_spec(context: &Context, spec: *const libc::c_char) -> libc::c_int {
 
     /* if `spec` is given, remember it for later usage; if it is not given, try to use the last one */
     if !spec.is_null() {
-        real_spec = to_string(spec);
+        real_spec = to_string_lossy(spec);
         context
             .sql
-            .set_config(context, "import_spec", Some(&real_spec))
+            .set_raw_config(context, "import_spec", Some(&real_spec))
             .unwrap();
     } else {
-        let rs = context.sql.get_config(context, "import_spec");
+        let rs = context.sql.get_raw_config(context, "import_spec");
         if rs.is_none() {
             error!(context, "Import: No file or folder given.");
             return 0;
@@ -171,7 +170,7 @@ fn poke_spec(context: &Context, spec: *const libc::c_char) -> libc::c_int {
     if read_cnt > 0 {
         context.call_cb(Event::MsgsChanged {
             chat_id: 0,
-            msg_id: 0,
+            msg_id: MsgId::new(0),
         });
     }
     1
@@ -193,9 +192,9 @@ unsafe fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
     let msgtext = msg.get_text();
     info!(
         context,
-        "{}#{}{}{}: {} (Contact#{}): {} {}{}{}{} [{}]",
+        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{} [{}]",
         prefix.as_ref(),
-        msg.get_id() as libc::c_int,
+        msg.get_id(),
         if msg.get_showpadlock() { "🔒" } else { "" },
         if msg.has_location() { "📍" } else { "" },
         &contact_name,
@@ -212,22 +211,27 @@ unsafe fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
             "[FRESH]"
         },
         if msg.is_info() { "[INFO]" } else { "" },
+        if msg.is_forwarded() {
+            "[FORWARDED]"
+        } else {
+            ""
+        },
         statestr,
         &temp2,
     );
 }
 
-unsafe fn log_msglist(context: &Context, msglist: &Vec<u32>) -> Result<(), Error> {
+unsafe fn log_msglist(context: &Context, msglist: &Vec<MsgId>) -> Result<(), Error> {
     let mut lines_out = 0;
     for &msg_id in msglist {
-        if msg_id == 9 as libc::c_uint {
+        if msg_id.is_daymarker() {
             info!(
                 context,
                 "--------------------------------------------------------------------------------"
             );
 
             lines_out += 1
-        } else if msg_id > 0 {
+        } else if !msg_id.is_special() {
             if lines_out == 0 {
                 info!(
                     context,
@@ -236,7 +240,7 @@ unsafe fn log_msglist(context: &Context, msglist: &Vec<u32>) -> Result<(), Error
                 lines_out += 1
             }
             let msg = Message::load_from_db(context, msg_id)?;
-            log_msg(context, "Msg", &msg);
+            log_msg(context, "", &msg);
         }
     }
     if lines_out > 0 {
@@ -349,6 +353,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
                  configure\n\
                  connect\n\
                  disconnect\n\
+                 interrupt\n\
                  maybenetwork\n\
                  housekeeping\n\
                  help imex (Import/Export)\n\
@@ -374,6 +379,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
                  sendimage <file> [<text>]\n\
                  sendfile <file> [<text>]\n\
                  draft [<text>]\n\
+                 devicemsg <text>\n\
                  listmedia\n\
                  archive <chat-id>\n\
                  unarchive <chat-id>\n\
@@ -400,6 +406,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
                  checkqr <qr-content>\n\
                  event <event-id to test>\n\
                  fileinfo <file>\n\
+                 emptyserver <flags> (1=MVBOX 2=INBOX)\n\
                  clear -- clear screen\n\
                  exit or quit\n\
                  ============================================="
@@ -414,17 +421,17 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
         },
         "get-setupcodebegin" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let msg_id: u32 = arg1.parse()?;
+            let msg_id: MsgId = MsgId::new(arg1.parse()?);
             let msg = Message::load_from_db(context, msg_id)?;
             if msg.is_setupmessage() {
                 let setupcodebegin = msg.get_setupcodebegin(context);
                 println!(
-                    "The setup code for setup message Msg#{} starts with: {}",
+                    "The setup code for setup message {} starts with: {}",
                     msg_id,
                     setupcodebegin.unwrap_or_default(),
                 );
             } else {
-                bail!("Msg#{} is no setup message.", msg_id,);
+                bail!("{} is no setup message.", msg_id,);
             }
         }
         "continue-key-transfer" => {
@@ -432,7 +439,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
                 !arg1.is_empty() && !arg2.is_empty(),
                 "Arguments <msg-id> <setup-code> expected"
             );
-            continue_key_transfer(context, arg1.parse()?, &arg2)?;
+            continue_key_transfer(context, MsgId::new(arg1.parse()?), &arg2)?;
         }
         "has-backup" => {
             has_backup(context, blobdir)?;
@@ -471,7 +478,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
             ensure!(0 != dc_reset_tables(context, bits), "Reset failed");
         }
         "stop" => {
-            dc_stop_ongoing_process(context);
+            context.stop_ongoing();
         }
         "set" => {
             ensure!(!arg1.is_empty(), "Argument <key> missing.");
@@ -488,6 +495,9 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
         "info" => {
             println!("{:#?}", context.get_info());
         }
+        "interrupt" => {
+            interrupt_imap_idle(context);
+        }
         "maybenetwork" => {
             maybe_network(context);
         }
@@ -512,15 +522,12 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
 
                 for i in (0..cnt).rev() {
                     let chat = Chat::load_from_db(context, chatlist.get_chat_id(i))?;
-                    let temp_subtitle = chat.get_subtitle(context);
-                    let temp_name = chat.get_name();
                     info!(
                         context,
-                        "{}#{}: {} [{}] [{} fresh]",
+                        "{}#{}: {} [{} fresh]",
                         chat_prefix(&chat),
                         chat.get_id(),
-                        temp_name,
-                        temp_subtitle,
+                        chat.get_name(),
                         chat::get_fresh_msg_cnt(context, chat.get_id()),
                     );
                     let lot = chatlist.get_summary(context, i, Some(&chat));
@@ -577,21 +584,35 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
             ensure!(sel_chat.is_some(), "Failed to select chat");
             let sel_chat = sel_chat.as_ref().unwrap();
 
-            let msglist = chat::get_chat_msgs(context, sel_chat.get_id(), 0x1, 0);
-            let temp2 = sel_chat.get_subtitle(context);
-            let temp_name = sel_chat.get_name();
+            let msglist = chat::get_chat_msgs(context, sel_chat.get_id(), 0x1, None);
+            let members = chat::get_chat_contacts(context, sel_chat.id);
+            let subtitle = if sel_chat.is_device_talk() {
+                "device-talk".to_string()
+            } else if sel_chat.get_type() == Chattype::Single && members.len() >= 1 {
+                let contact = Contact::get_by_id(context, members[0])?;
+                contact.get_addr().to_string()
+            } else {
+                format!("{} member(s)", members.len())
+            };
             info!(
                 context,
-                "{}#{}: {} [{}]{}",
+                "{}#{}: {} [{}]{}{}",
                 chat_prefix(sel_chat),
                 sel_chat.get_id(),
-                temp_name,
-                temp2,
+                sel_chat.get_name(),
+                subtitle,
                 if sel_chat.is_sending_locations() {
                     "📍"
                 } else {
                     ""
                 },
+                match sel_chat.get_profile_image(context) {
+                    Some(icon) => match icon.to_str() {
+                        Some(icon) => format!(" Icon: {}", icon),
+                        _ => " Icon: Err".to_string(),
+                    },
+                    _ => "".to_string(),
+                },
             );
             log_msglist(context, &msglist)?;
             if let Some(draft) = chat::get_draft(context, sel_chat.get_id())? {
@@ -613,7 +634,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
         }
         "createchatbymsg" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing");
-            let msg_id: u32 = arg1.parse()?;
+            let msg_id = MsgId::new(arg1.parse()?);
             let chat_id = chat::create_by_msg_id(context, msg_id)?;
             let chat = Chat::load_from_db(context, chat_id)?;
 
@@ -705,7 +726,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
                 let marker = location.marker.as_ref().unwrap_or(&default_marker);
                 info!(
                     context,
-                    "Loc#{}: {}: lat={} lng={} acc={} Chat#{} Contact#{} Msg#{} {}",
+                    "Loc#{}: {}: lat={} lng={} acc={} Chat#{} Contact#{} {} {}",
                     location.location_id,
                     dc_timestamp_to_str(location.timestamp),
                     location.latitude,
@@ -742,7 +763,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
             let longitude = arg2.parse()?;
 
             let continue_streaming = location::set(context, latitude, longitude, 0.);
-            if 0 != continue_streaming {
+            if continue_streaming {
                 println!("Success, streaming should be continued.");
             } else {
                 println!("Success, streaming can be stoppped.");
@@ -809,6 +830,15 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
                 println!("Draft deleted.");
             }
         }
+        "devicemsg" => {
+            ensure!(
+                !arg1.is_empty(),
+                "Please specify text to add as device message."
+            );
+            let mut msg = Message::new(Viewtype::Text);
+            msg.set_text(Some(arg1.to_string()));
+            chat::add_device_msg(context, &mut msg)?;
+        }
         "listmedia" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
 
@@ -822,9 +852,9 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
             println!("{} images or videos: ", images.len());
             for (i, data) in images.iter().enumerate() {
                 if 0 == i {
-                    print!("Msg#{}", data);
+                    print!("{}", data);
                 } else {
-                    print!(", Msg#{}", data);
+                    print!(", {}", data);
                 }
             }
             print!("\n");
@@ -845,7 +875,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
         }
         "msginfo" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let id = arg1.parse()?;
+            let id = MsgId::new(arg1.parse()?);
             let res = message::get_msg_info(context, id);
             println!("{}", res);
         }
@@ -861,27 +891,27 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
                 "Arguments <msg-id> <chat-id> expected"
             );
 
-            let mut msg_ids = [0; 1];
+            let mut msg_ids = [MsgId::new(0); 1];
             let chat_id = arg2.parse()?;
-            msg_ids[0] = arg1.parse()?;
-            chat::forward_msgs(context, &msg_ids, chat_id);
+            msg_ids[0] = MsgId::new(arg1.parse()?);
+            chat::forward_msgs(context, &msg_ids, chat_id)?;
         }
         "markseen" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let mut msg_ids = [0; 1];
-            msg_ids[0] = arg1.parse()?;
+            let mut msg_ids = [MsgId::new(0); 1];
+            msg_ids[0] = MsgId::new(arg1.parse()?);
             message::markseen_msgs(context, &msg_ids);
         }
         "star" | "unstar" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let mut msg_ids = [0; 1];
-            msg_ids[0] = arg1.parse()?;
+            let mut msg_ids = [MsgId::new(0); 1];
+            msg_ids[0] = MsgId::new(arg1.parse()?);
             message::star_msgs(context, &msg_ids, arg0 == "star");
         }
         "delmsg" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let mut ids = [0; 1];
-            ids[0] = arg1.parse()?;
+            let mut ids = [MsgId::new(0); 1];
+            ids[0] = MsgId::new(arg1.parse()?);
             message::delete_msgs(context, &ids);
         }
         "listcontacts" | "contacts" | "listverified" => {
@@ -972,6 +1002,11 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
                 bail!("Command failed.");
             }
         }
+        "emptyserver" => {
+            ensure!(!arg1.is_empty(), "Argument <flags> missing");
+
+            message::dc_empty_server(context, arg1.parse()?);
+        }
         "" => (),
         _ => bail!("Unknown command: \"{}\" type ? for help.", arg0),
     }

examples/repl/main.rs

@@ -43,7 +43,6 @@ use self::cmdline::*;
 
 fn receive_event(_context: &Context, event: Event) -> libc::uintptr_t {
     match event {
-        Event::GetString { .. } => {}
         Event::Info(msg) => {
             /* do not show the event as this would fill the screen */
             println!("{}", msg);

examples/simple.rs

@@ -35,93 +35,90 @@ fn cb(_ctx: &Context, event: Event) -> usize {
 }
 
 fn main() {
-    unsafe {
-        let dir = tempdir().unwrap();
-        let dbfile = dir.path().join("db.sqlite");
-        println!("creating database {:?}", dbfile);
-        let ctx =
-            Context::new(Box::new(cb), "FakeOs".into(), dbfile).expect("Failed to create context");
-        let running = Arc::new(RwLock::new(true));
-        let info = ctx.get_info();
-        let duration = time::Duration::from_millis(4000);
-        println!("info: {:#?}", info);
+    let dir = tempdir().unwrap();
+    let dbfile = dir.path().join("db.sqlite");
+    println!("creating database {:?}", dbfile);
+    let ctx =
+        Context::new(Box::new(cb), "FakeOs".into(), dbfile).expect("Failed to create context");
+    let running = Arc::new(RwLock::new(true));
+    let info = ctx.get_info();
+    let duration = time::Duration::from_millis(4000);
+    println!("info: {:#?}", info);
+
+    let ctx = Arc::new(ctx);
+    let ctx1 = ctx.clone();
+    let r1 = running.clone();
+    let t1 = thread::spawn(move || {
+        while *r1.read().unwrap() {
+            perform_imap_jobs(&ctx1);
+            if *r1.read().unwrap() {
+                perform_imap_fetch(&ctx1);
 
-        let ctx = Arc::new(ctx);
-        let ctx1 = ctx.clone();
-        let r1 = running.clone();
-        let t1 = thread::spawn(move || {
-            while *r1.read().unwrap() {
-                perform_imap_jobs(&ctx1);
                 if *r1.read().unwrap() {
-                    perform_imap_fetch(&ctx1);
-
-                    if *r1.read().unwrap() {
-                        perform_imap_idle(&ctx1);
-                    }
+                    perform_imap_idle(&ctx1);
                 }
             }
-        });
-
-        let ctx1 = ctx.clone();
-        let r1 = running.clone();
-        let t2 = thread::spawn(move || {
-            while *r1.read().unwrap() {
-                perform_smtp_jobs(&ctx1);
-                if *r1.read().unwrap() {
-                    perform_smtp_idle(&ctx1);
-                }
-            }
-        });
-
-        println!("configuring");
-        let args = std::env::args().collect::<Vec<String>>();
-        assert_eq!(args.len(), 2, "missing password");
-        let pw = args[1].clone();
-        ctx.set_config(config::Config::Addr, Some("d@testrun.org"))
-            .unwrap();
-        ctx.set_config(config::Config::MailPw, Some(&pw)).unwrap();
-        configure(&ctx);
-
-        thread::sleep(duration);
-
-        println!("sending a message");
-        let contact_id =
-            Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com").unwrap();
-        let chat_id = chat::create_by_contact_id(&ctx, contact_id).unwrap();
-        chat::send_text_msg(&ctx, chat_id, "Hi, here is my first message!".into()).unwrap();
-
-        println!("fetching chats..");
-        let chats = Chatlist::try_load(&ctx, 0, None, None).unwrap();
-
-        for i in 0..chats.len() {
-            let summary = chats.get_summary(&ctx, 0, None);
-            let text1 = summary.get_text1();
-            let text2 = summary.get_text2();
-            println!("chat: {} - {:?} - {:?}", i, text1, text2,);
         }
+    });
 
-        thread::sleep(duration);
+    let ctx1 = ctx.clone();
+    let r1 = running.clone();
+    let t2 = thread::spawn(move || {
+        while *r1.read().unwrap() {
+            perform_smtp_jobs(&ctx1);
+            if *r1.read().unwrap() {
+                perform_smtp_idle(&ctx1);
+            }
+        }
+    });
 
-        // let msglist = dc_get_chat_msgs(&ctx, chat_id, 0, 0);
-        // for i in 0..dc_array_get_cnt(msglist) {
-        //     let msg_id = dc_array_get_id(msglist, i);
-        //     let msg = dc_get_msg(context, msg_id);
-        //     let text = CStr::from_ptr(dc_msg_get_text(msg)).unwrap();
-        //     println!("Message {}: {}\n", i + 1, text.to_str().unwrap());
-        //     dc_msg_unref(msg);
-        // }
-        // dc_array_unref(msglist);
+    println!("configuring");
+    let args = std::env::args().collect::<Vec<String>>();
+    assert_eq!(args.len(), 2, "missing password");
+    let pw = args[1].clone();
+    ctx.set_config(config::Config::Addr, Some("d@testrun.org"))
+        .unwrap();
+    ctx.set_config(config::Config::MailPw, Some(&pw)).unwrap();
+    configure(&ctx);
 
-        println!("stopping threads");
+    thread::sleep(duration);
 
-        *running.clone().write().unwrap() = false;
-        deltachat::job::interrupt_imap_idle(&ctx);
-        deltachat::job::interrupt_smtp_idle(&ctx);
+    println!("sending a message");
+    let contact_id = Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com").unwrap();
+    let chat_id = chat::create_by_contact_id(&ctx, contact_id).unwrap();
+    chat::send_text_msg(&ctx, chat_id, "Hi, here is my first message!".into()).unwrap();
 
-        println!("joining");
-        t1.join().unwrap();
-        t2.join().unwrap();
+    println!("fetching chats..");
+    let chats = Chatlist::try_load(&ctx, 0, None, None).unwrap();
 
-        println!("closing");
+    for i in 0..chats.len() {
+        let summary = chats.get_summary(&ctx, 0, None);
+        let text1 = summary.get_text1();
+        let text2 = summary.get_text2();
+        println!("chat: {} - {:?} - {:?}", i, text1, text2,);
     }
+
+    thread::sleep(duration);
+
+    // let msglist = dc_get_chat_msgs(&ctx, chat_id, 0, 0);
+    // for i in 0..dc_array_get_cnt(msglist) {
+    //     let msg_id = dc_array_get_id(msglist, i);
+    //     let msg = dc_get_msg(context, msg_id);
+    //     let text = CStr::from_ptr(dc_msg_get_text(msg)).unwrap();
+    //     println!("Message {}: {}\n", i + 1, text.to_str().unwrap());
+    //     dc_msg_unref(msg);
+    // }
+    // dc_array_unref(msglist);
+
+    println!("stopping threads");
+
+    *running.clone().write().unwrap() = false;
+    deltachat::job::interrupt_imap_idle(&ctx);
+    deltachat::job::interrupt_smtp_idle(&ctx);
+
+    println!("joining");
+    t1.join().unwrap();
+    t2.join().unwrap();
+
+    println!("closing");
 }

mmime/src/charconv.rs

@@ -17,12 +17,12 @@ pub unsafe fn charconv(
     assert!(!fromcode.is_null(), "invalid fromcode");
     assert!(!s.is_null(), "invalid input string");
     if let Some(encoding) =
-        charset::Charset::for_label(CStr::from_ptr(fromcode).to_str().unwrap().as_bytes())
+        charset::Charset::for_label(CStr::from_ptr(fromcode).to_string_lossy().as_bytes())
     {
         let data = std::slice::from_raw_parts(s as *const u8, strlen(s));
 
         let (res, _, _) = encoding.decode(data);
-        let res_c = CString::new(res.as_bytes()).unwrap();
+        let res_c = CString::new(res.as_bytes()).unwrap_or_default();
         *result = strdup(res_c.as_ptr()) as *mut _;
 
         MAIL_CHARCONV_NO_ERROR as libc::c_int

mmime/src/display.rs

@@ -123,7 +123,7 @@ unsafe fn display_mime_discrete_type(mut discrete_type: *mut mailmime_discrete_t
         _ => {}
     };
 }
-unsafe fn display_mime_data(mut data: *mut mailmime_data) {
+pub unsafe fn display_mime_data(mut data: *mut mailmime_data) {
     match (*data).dt_type {
         0 => {
             println!(

mmime/src/mailimf/write_generic.rs

@@ -1598,7 +1598,7 @@ unsafe fn mailimf_date_time_write_driver(
         (*date_time).dt_sec,
         (*date_time).dt_zone,
     );
-    let date_str_c = std::ffi::CString::new(date_str).unwrap();
+    let date_str_c = std::ffi::CString::new(date_str).unwrap_or_default();
     let r = mailimf_string_write_driver(
         do_write,
         data,

mmime/src/mailmime/types_helper.rs

@@ -848,7 +848,7 @@ pub unsafe fn mailmime_generate_boundary() -> *mut libc::c_char {
         hex::encode(&std::process::id().to_le_bytes()[..2])
     );
 
-    let c = std::ffi::CString::new(raw).unwrap();
+    let c = std::ffi::CString::new(raw).unwrap_or_default();
     strdup(c.as_ptr())
 }
 

mmime/src/mailmime/write_generic.rs

@@ -338,7 +338,7 @@ unsafe fn mailmime_disposition_param_write_driver(
         4 => {
             let value = (*param).pa_data.pa_size as u32;
             let raw = format!("{}", value);
-            let raw_c = std::ffi::CString::new(raw).unwrap();
+            let raw_c = std::ffi::CString::new(raw).unwrap_or_default();
             sizestr = strdup(raw_c.as_ptr());
             len = strlen(b"size=\x00" as *const u8 as *const libc::c_char)
                 .wrapping_add(strlen(sizestr))
@@ -542,7 +542,7 @@ unsafe fn mailmime_version_write_driver(
     }
 
     let raw = format!("{}.{}", (version >> 16) as i32, (version & 0xffff) as i32);
-    let raw_c = std::ffi::CString::new(raw).unwrap();
+    let raw_c = std::ffi::CString::new(raw).unwrap_or_default();
     let mut versionstr = strdup(raw_c.as_ptr());
     r = mailimf_string_write_driver(do_write, data, col, versionstr, strlen(versionstr));
     if r != MAILIMF_NO_ERROR as libc::c_int {
@@ -1515,8 +1515,8 @@ pub unsafe fn mailmime_data_write_driver(
         }
         1 => {
             let filename = CStr::from_ptr((*mime_data).dt_data.dt_filename)
-                .to_str()
-                .unwrap();
+                .to_string_lossy()
+                .into_owned();
             if let Ok(file) = std::fs::File::open(filename) {
                 if let Ok(mut text) = memmap::MmapOptions::new().map_copy(&file) {
                     if 0 != (*mime_data).dt_encoded {
@@ -1797,7 +1797,7 @@ pub unsafe fn mailmime_quoted_printable_write_driver(
                             start = text.offset(i as isize).offset(1isize);
 
                             let raw = format!("={:02X}", (ch as libc::c_int));
-                            let raw_c = std::ffi::CString::new(raw).unwrap();
+                            let raw_c = std::ffi::CString::new(raw).unwrap_or_default();
                             let mut hexstr = strdup(raw_c.as_ptr());
                             r = mailimf_string_write_driver(
                                 do_write,
@@ -1822,7 +1822,7 @@ pub unsafe fn mailmime_quoted_printable_write_driver(
                         }
                         start = text.offset(i as isize).offset(1isize);
                         let raw = format!("={:02X}", ch as libc::c_int);
-                        let raw_c = std::ffi::CString::new(raw).unwrap();
+                        let raw_c = std::ffi::CString::new(raw).unwrap_or_default();
                         let mut hexstr = strdup(raw_c.as_ptr());
                         r = mailimf_string_write_driver(
                             do_write,
@@ -1866,7 +1866,7 @@ pub unsafe fn mailmime_quoted_printable_write_driver(
                     }
                     start = text.offset(i as isize);
                     let raw = format!("={:02X}", b'\r' as i32);
-                    let raw_c = std::ffi::CString::new(raw).unwrap();
+                    let raw_c = std::ffi::CString::new(raw).unwrap_or_default();
                     let mut hexstr = strdup(raw_c.as_ptr());
                     r = mailimf_string_write_driver(do_write, data, col, hexstr, 3i32 as size_t);
                     if r != MAILIMF_NO_ERROR as libc::c_int {
@@ -1890,7 +1890,7 @@ pub unsafe fn mailmime_quoted_printable_write_driver(
                         "={:02X}\r\n",
                         *text.offset(i.wrapping_sub(1i32 as libc::size_t) as isize) as libc::c_int
                     );
-                    let raw_c = std::ffi::CString::new(raw).unwrap();
+                    let raw_c = std::ffi::CString::new(raw).unwrap_or_default();
                     let mut hexstr = strdup(raw_c.as_ptr());
 
                     r = mailimf_string_write_driver(do_write, data, col, hexstr, strlen(hexstr));
@@ -1917,7 +1917,7 @@ pub unsafe fn mailmime_quoted_printable_write_driver(
                         "={:02X}\r\n",
                         *text.offset(i.wrapping_sub(2i32 as libc::size_t) as isize) as libc::c_int
                     );
-                    let raw_c = std::ffi::CString::new(raw).unwrap();
+                    let raw_c = std::ffi::CString::new(raw).unwrap_or_default();
                     let mut hexstr = strdup(raw_c.as_ptr());
 
                     r = mailimf_string_write_driver(do_write, data, col, hexstr, strlen(hexstr));
@@ -1938,7 +1938,7 @@ pub unsafe fn mailmime_quoted_printable_write_driver(
                         (*text.offset(i.wrapping_sub(2i32 as libc::size_t) as isize) as u8 as char),
                         b'\r' as i32
                     );
-                    let raw_c = std::ffi::CString::new(raw).unwrap();
+                    let raw_c = std::ffi::CString::new(raw).unwrap_or_default();
                     let mut hexstr = strdup(raw_c.as_ptr());
 
                     r = mailimf_string_write_driver(do_write, data, col, hexstr, strlen(hexstr));

mmime/src/other.rs

@@ -8,10 +8,14 @@ use crate::mailmime::types_helper::*;
 
 pub(crate) use libc::{
     calloc, close, free, isalpha, isdigit, malloc, memcmp, memcpy, memmove, memset, realloc,
-    strcpy, strlen, strncmp, strncpy,
+    strcpy, strlen, strncmp, strncpy, strnlen,
 };
 
 pub(crate) unsafe fn strcasecmp(s1: *const libc::c_char, s2: *const libc::c_char) -> libc::c_int {
+    if s1.is_null() || s2.is_null() {
+        return 1;
+    }
+
     let s1 = std::ffi::CStr::from_ptr(s1)
         .to_string_lossy()
         .to_lowercase();
@@ -30,16 +34,25 @@ pub(crate) unsafe fn strncasecmp(
     s2: *const libc::c_char,
     n: libc::size_t,
 ) -> libc::c_int {
-    let s1 = std::ffi::CStr::from_ptr(s1)
-        .to_string_lossy()
-        .to_lowercase();
-    let s2 = std::ffi::CStr::from_ptr(s2)
-        .to_string_lossy()
-        .to_lowercase();
-    let m1 = std::cmp::min(n, s1.len());
-    let m2 = std::cmp::min(n, s2.len());
+    if s1.is_null() || s2.is_null() {
+        return 1;
+    }
 
-    if s1[..m1] == s2[..m2] {
+    // s1 and s2 might not be null terminated.
+
+    let s1_slice =
+        std::slice::from_raw_parts(s1 as *const u8, strnlen(s1 as *const libc::c_char, n));
+    let s2_slice =
+        std::slice::from_raw_parts(s2 as *const u8, strnlen(s2 as *const libc::c_char, n));
+
+    let s1 = std::ffi::CStr::from_bytes_with_nul_unchecked(s1_slice)
+        .to_string_lossy()
+        .to_lowercase();
+    let s2 = std::ffi::CStr::from_bytes_with_nul_unchecked(s2_slice)
+        .to_string_lossy()
+        .to_lowercase();
+
+    if s1 == s2 {
         0
     } else {
         1
@@ -1682,6 +1695,14 @@ mod tests {
                 4,
             )
         });
+
+        assert_eq!(0, unsafe {
+            strncasecmp(
+                CString::new("hell").unwrap().as_ptr(),
+                CString::new("Hell").unwrap().as_ptr(),
+                100_000_000,
+            )
+        });
     }
 
     #[test]

proptest-regressions/dc_mimeparser.txt

@@ -0,0 +1,7 @@
+# Seeds for failure cases proptest has generated in the past. It is
+# automatically read and these particular cases re-run before any
+# novel cases are generated.
+#
+# It is recommended to check this file in to source control so that
+# everyone who runs the test benefits from these saved cases.
+cc 03cab93c6d1f3a8245f63cf84dacb307944294fe6333c1e38f078a6600659c7a # shrinks to data = "a\t0aA\ta\t0 \ta\t0 \ta a\t\ta  A\tAA0a0a 0\t a\t aA   \t a\t  A0\t  AAa\taA0\taAAaA\t0\taa0a\ta Aa Aaaa A0A\t  a aA 0\t  A\t0\t0\t\t\t\t\t\tA \t\t a\tA Aa aAA0A0AA0aaA A\t\t  aa0\ta\t  \tAa\taA\t00  AA  A a\tA a aAAa \t 00\t0 \t\t  a A  0\t\t\t aAA Aa \taAAAA0a A0A\t\t1\\E$\t$R\tc\t^=\t\"\tQ<Uk\t\t>A\t\t&\t}v&\tM^\'|\tW5?dn\t\t+\t\tP\te`\t\t>:Brlq--?\t$#Q\tK=zJ\tb\"9*.\"\t`\tF&T*\tBs,\tg\'*\\\t:\t?l$\t\t|A\"HR:Hk\t\\KkV\t\t{=&!^e%:|_*0wV\t[$`\t:\t$%f\t\t[!\"Y. \tP\t\th\'?\'/?%:++NfQo#@\"+?\t(\\??{\t\'\'$Dzj\t0.?{s4?&?Y=/yj]Z=\t4n\t?Ja\"\t{I\t$\t;I}_8V\t&\t?N\'\tI2/\t9.\tFT%9%`\'\tz\to7Y\t|AXP&@G12g\t\'w\t\t%??\t\"h$?F\"\"6%q\\\\{\tT\t\"]$87$}h\'\t<\t$\tc%U:mT2:<v\t#Rl!;U\t\t\"^D\tRZ(BZ{n\t%[$fgH`\t{B}:*\t\t%%*F`%W\t//B}PQ\t\tsu2\tLz<1*!p-X\tnKv&&0\thm4<n\\.\\/.w\'\t<)E1g3#ood\t`?\t\\({N?,8_?h\ty\t0%\t*$A\t\t*w-ViQUj\tTiH\t%\t%&0p\'\'\tA%r**Fo\'Z\\\tNI*ely4=I?}$.o`\t$\ts\'&lt\t\",:~=Nv/0O%=+<LS\t%P\'\t$r%d.\t{G{/L:\t&I&8-`cy*\"{\t/%fP9.P<\t\t\'/`\t\t`\t\t`!t:\t::\t\tW\'X^\t@.uL&a\tN\t\t\t.\t?0*\tvUK>UJ\\\tQbj%w]P=Js\t\"R\t&s^?</=%\t\'VI:\" kT`{b*<\t\tF&?Y\t\t:/\t!C\'e0`\t\t\tx-\t*\\N\\wwu.I4\tb/y\t\"P:P%\"\t\tQ][\\\t^x\t\t):\t\t&s\t$1-\t\t\tXt`0\t;\t/UeCP*\"G\t\t\':\tk@91Hf&#\t(Uzz&`p2;\t{]\t\"I_%=\\%,XD\"\'06QfCd<;$:q^\t8:\"*\"H\t\to\t&xK/\t\ty0/<}<j<|*`~::b\t=S.=}=Ki\t<Y.\'{\tf\t{Ub*H%(_4*?*\tn2\t/\'\t\t\t/,4]\tt\t<y\t\t\tWi\t\tT&\"\t\t\t\t\t=/1Wu?\t\'A\"W-P\t$?e\\\t`\t6`vD\t8`\t\tccD J\tY&jI//?_\t\\j\t_\tsiq$\t?9\tQ\t.^%&..?%Jm??A%+<\tN&*\t.g\tS$W\"\"\tMpq\t\t:&\\\thTz&<iz%/%T&\'F\t\\]&\t\t}\t\t\tXr=dtp`:+<,\t%60Y*<(&K*kJ\todO\t=<V?&\tMU/\"\t= Y!)<\tV\t9\t)\t&v8q{\t\t&pT\t3\ttB,qcq\'i$;gv%j_%M_{[\"&;\t\t\t.B;6Y\\%\t\"\tY/a\t\\`:?\t<\t?]\taNwD;\\\t%l*74%5T?QS :~yR/E*R\t\t=u\t\\\t\t.Q<;\\\t_S/{&F$^\tw_>\'h=|%\t\t:W!\\<U2\'$\tb&`\t=|=L9\t\t\t\\WZ:! }{\t ;\t;\t\t 0.*\t.%\"\'r%{<Mh_t[[}\t-\tJo\"b/AC*-=`=T\tz$2\tC\t\t/k{47\"\t\t,q%\tZ\tT3\t\tf>%\t\'?%@^tx\t7\"1Bk{b{n\t\"Pj3\tHc\t\tt\tY<\t#?\tSh\\yk/N\\\t8 7=R4*9Cw&m\t\\-\'f\t|\'#t(Etu.Hdu(E\t%&v:\'aqW~A5\t\t w.s{J%lX<\"\t\'*%m<&:/B<&\':U}$&`.{)\t\t6S\t:/$*kQ-Z\t^\'t${/tH*\'v\t3\t=\t\tDyp:B\t`I_R&4SO\t\t&-j=*.\t87&\'e*( \t\t\t\'<$\\DJ<$p?{}\'&\tv\t\\Xk<Y:Y`!$K{\tF&\tzd\t\t*i$\tj\'\t<)R*\t%?\t!.\t=\"@#~:=*\t\tXO=_T,1\"\'.%%\"`{\\:\t\"\tfkeOb/\'$I~\ta\t|&\t[\\KK\"1&Z\t<k\t\t)%\'-~\"2n\tj\tW?*<@w{g%d\ta\\\'\'I\t;:ySR%ke:4\tc\t$=\t&9P]x4\tJ=\t6C6%a\t`0\tF\tm-\tTr\t}\t\tQum\t&@\typ|w2&\t\t3`i&t\t\tT5\"\t.&b&e*/==1.\'*\\[U*\tqPL%?$-0/}~|q`\t\t}\t$\tq==o+T$\'!H\t\ti&um\"?\"%%\t/\'p\tg>?{0{J{\t\t/\t\t{zKZ&>=\t[\"1h<H%z/8,/]s\tv{7\t\t:j*H,M//\t\t\td\'.)\t"

python/README.rst

@@ -97,7 +97,7 @@ If you want to run "liveconfig" functional tests you can set
   chat devs.
 
 - or the path of a file that contains two lines, each describing
-  via "addr=... mail_pwd=..." a test account login that will
+  via "addr=... mail_pw=..." a test account login that will
   be used for the live tests.
 
 With ``DCC_PY_LIVECONFIG`` set pytest invocations will use real

python/doc/_templates/globaltoc.html

@@ -6,7 +6,6 @@
   <li><a href="{{ pathto('install') }}">install</a></li>
   <li><a href="{{ pathto('api') }}">high level API</a></li>
   <li><a href="{{ pathto('lapi') }}">low level API</a></li>
-  <li><a href="{{ pathto('capi') }}">C deltachat.h</a></li>
 </ul>
 <b>external links:</b>
 <ul>

python/doc/api.rst

@@ -8,8 +8,8 @@ high level API reference
 
 - :class:`deltachat.account.Account` (your main entry point, creates the
   other classes)
-- :class:`deltachat.chatting.Contact`
-- :class:`deltachat.chatting.Chat`
+- :class:`deltachat.contact.Contact`
+- :class:`deltachat.chat.Chat`
 - :class:`deltachat.message.Message`
 
 Account
@@ -22,13 +22,13 @@ Account
 Contact
 -------
 
-.. autoclass:: deltachat.chatting.Contact
+.. autoclass:: deltachat.contact.Contact
     :members:
 
 Chat
 ----
 
-.. autoclass:: deltachat.chatting.Chat
+.. autoclass:: deltachat.chat.Chat
     :members:
 
 Message

python/doc/lapi.rst

@@ -2,7 +2,13 @@
 low level API reference
 ===================================
 
-for full C-docs, defines and function checkout :doc:`capi`
+for full doxygen-generated C-docs, defines and functions please checkout
+
+https://c.delta.chat
+
+
+Python low-level capi calls
+---------------------------
 
 
 .. automodule:: deltachat.capi.lib

python/src/deltachat/account.py

@@ -1,6 +1,7 @@
 """ Account class implementation. """
 
 from __future__ import print_function
+import atexit
 import threading
 import re
 import time
@@ -14,12 +15,14 @@ import deltachat
 from . import const
 from .capi import ffi, lib
 from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array, DCLot
-from .chatting import Contact, Chat, Message
+from .chat import Chat
+from .message import Message
+from .contact import Contact
 
 
 class Account(object):
     """ Each account is tied to a sqlite database file which is fully managed
-    by the underlying deltachat c-library.  All public Account methods are
+    by the underlying deltachat core library.  All public Account methods are
     meant to be memory-safe and return memory-safe objects.
     """
     def __init__(self, db_path, logid=None, eventlogging=True, debug=True):
@@ -49,9 +52,10 @@ class Account(object):
             raise ValueError("Could not dc_open: {}".format(db_path))
         self._configkeys = self.get_config("sys.config_keys").split()
         self._imex_events = Queue()
+        atexit.register(self.shutdown)
 
-    def __del__(self):
-        self.shutdown()
+    # def __del__(self):
+    #    self.shutdown()
 
     def _check_config_key(self, name):
         if name not in self._configkeys:
@@ -69,6 +73,18 @@ class Account(object):
             d[key.lower()] = value
         return d
 
+    def set_stock_translation(self, id, string):
+        """ set stock translation string.
+
+        :param id: id of stock string (const.DC_STR_*)
+        :param value: string to set as new transalation
+        :returns: None
+        """
+        string = string.encode("utf8")
+        res = lib.dc_set_stock_translation(self._dc_context, id, string)
+        if res == 0:
+            raise ValueError("could not set translation string")
+
     def set_config(self, name, value):
         """ set configuration values.
 
@@ -121,11 +137,30 @@ class Account(object):
         if not self.is_configured():
             raise ValueError("need to configure first")
 
+    def empty_server_folders(self, inbox=False, mvbox=False):
+        """ empty server folders. """
+        flags = 0
+        if inbox:
+            flags |= const.DC_EMPTY_INBOX
+        if mvbox:
+            flags |= const.DC_EMPTY_MVBOX
+        if not flags:
+            raise ValueError("no flags set")
+        lib.dc_empty_server(self._dc_context, flags)
+
     def get_infostring(self):
         """ return info of the configured account. """
         self.check_is_configured()
         return from_dc_charpointer(lib.dc_get_info(self._dc_context))
 
+    def get_latest_backupfile(self, backupdir):
+        """ return the latest backup file in a given directory.
+        """
+        res = lib.dc_imex_has_backup(self._dc_context, as_dc_charpointer(backupdir))
+        if res == ffi.NULL:
+            return None
+        return from_dc_charpointer(res)
+
     def get_blobdir(self):
         """ return the directory for files.
 
@@ -135,9 +170,9 @@ class Account(object):
         return from_dc_charpointer(lib.dc_get_blobdir(self._dc_context))
 
     def get_self_contact(self):
-        """ return this account's identity as a :class:`deltachat.chatting.Contact`.
+        """ return this account's identity as a :class:`deltachat.contact.Contact`.
 
-        :returns: :class:`deltachat.chatting.Contact`
+        :returns: :class:`deltachat.contact.Contact`
         """
         self.check_is_configured()
         return Contact(self._dc_context, const.DC_CONTACT_ID_SELF)
@@ -149,7 +184,7 @@ class Account(object):
 
         :param email: email-address (text type)
         :param name: display name for this contact (optional)
-        :returns: :class:`deltachat.chatting.Contact` instance.
+        :returns: :class:`deltachat.contact.Contact` instance.
         """
         name = as_dc_charpointer(name)
         email = as_dc_charpointer(email)
@@ -175,7 +210,7 @@ class Account(object):
                       whose name or e-mail matches query.
         :param only_verified: if true only return verified contacts.
         :param with_self: if true the self-contact is also returned.
-        :returns: list of :class:`deltachat.chatting.Contact` objects.
+        :returns: list of :class:`deltachat.contact.Contact` objects.
         """
         flags = 0
         query = as_dc_charpointer(query)
@@ -193,7 +228,7 @@ class Account(object):
         """ create or get an existing 1:1 chat object for the specified contact or contact id.
 
         :param contact: chat_id (int) or contact object.
-        :returns: a :class:`deltachat.chatting.Chat` object.
+        :returns: a :class:`deltachat.chat.Chat` object.
         """
         if hasattr(contact, "id"):
             if contact._dc_context != self._dc_context:
@@ -210,7 +245,7 @@ class Account(object):
         the specified message.
 
         :param message: messsage id or message instance.
-        :returns: a :class:`deltachat.chatting.Chat` object.
+        :returns: a :class:`deltachat.chat.Chat` object.
         """
         if hasattr(message, "id"):
             if self._dc_context != message._dc_context:
@@ -228,7 +263,7 @@ class Account(object):
         Chats are unpromoted until the first message is sent.
 
         :param verified: if true only verified contacts can be added.
-        :returns: a :class:`deltachat.chatting.Chat` object.
+        :returns: a :class:`deltachat.chat.Chat` object.
         """
         bytes_name = name.encode("utf8")
         chat_id = lib.dc_create_group_chat(self._dc_context, int(verified), bytes_name)
@@ -237,7 +272,7 @@ class Account(object):
     def get_chats(self):
         """ return list of chats.
 
-        :returns: a list of :class:`deltachat.chatting.Chat` objects.
+        :returns: a list of :class:`deltachat.chat.Chat` objects.
         """
         dc_chatlist = ffi.gc(
             lib.dc_get_chatlist(self._dc_context, 0, ffi.NULL, 0),
@@ -255,9 +290,24 @@ class Account(object):
         return Chat(self, const.DC_CHAT_ID_DEADDROP)
 
     def get_message_by_id(self, msg_id):
-        """ return Message instance. """
+        """ return Message instance.
+        :param msg_id: integer id of this message.
+        :returns: :class:`deltachat.message.Message` instance.
+        """
         return Message.from_db(self, msg_id)
 
+    def get_chat_by_id(self, chat_id):
+        """ return Chat instance.
+        :param chat_id: integer id of this chat.
+        :returns: :class:`deltachat.chat.Chat` instance.
+        :raises: ValueError if chat does not exist.
+        """
+        res = lib.dc_get_chat(self._dc_context, chat_id)
+        if res == ffi.NULL:
+            raise ValueError("cannot get chat with id={}".format(chat_id))
+        lib.dc_chat_unref(res)
+        return Chat(self, chat_id)
+
     def mark_seen_messages(self, messages):
         """ mark the given set of messages as seen.
 
@@ -274,7 +324,7 @@ class Account(object):
         """ Forward list of messages to a chat.
 
         :param messages: list of :class:`deltachat.message.Message` object.
-        :param chat: :class:`deltachat.chatting.Chat` object.
+        :param chat: :class:`deltachat.chat.Chat` object.
         :returns: None
         """
         msg_ids = [msg.id for msg in messages]
@@ -386,7 +436,7 @@ class Account(object):
         """ setup contact and return a Chat after contact is established.
 
         Note that this function may block for a long time as messages are exchanged
-        with the emitter of the QR code.  On success a :class:`deltachat.chatting.Chat` instance
+        with the emitter of the QR code.  On success a :class:`deltachat.chat.Chat` instance
         is returned.
         :param qr: valid "setup contact" QR code (all other QR codes will result in an exception)
         """
@@ -400,7 +450,7 @@ class Account(object):
         """ join a chat group through a QR code.
 
         Note that this function may block for a long time as messages are exchanged
-        with the emitter of the QR code.  On success a :class:`deltachat.chatting.Chat` instance
+        with the emitter of the QR code.  On success a :class:`deltachat.chat.Chat` instance
         is returned which is the chat that we just joined.
 
         :param qr: valid "join-group" QR code (all other QR codes will result in an exception)
@@ -411,6 +461,9 @@ class Account(object):
             raise ValueError("could not join group")
         return Chat(self, chat_id)
 
+    def stop_ongoing(self):
+        lib.dc_stop_ongoing_process(self._dc_context)
+
     #
     # meta API for start/stop and event based processing
     #
@@ -432,8 +485,9 @@ class Account(object):
 
     def stop_threads(self, wait=True):
         """ stop IMAP/SMTP threads. """
-        lib.dc_stop_ongoing_process(self._dc_context)
-        self._threads.stop(wait=wait)
+        if self._threads.is_started():
+            self.stop_ongoing()
+            self._threads.stop(wait=wait)
 
     def shutdown(self, wait=True):
         """ stop threads and close and remove underlying dc_context and callbacks. """
@@ -444,6 +498,7 @@ class Account(object):
             self.stop_threads(wait=wait)  # to wait for threads
             deltachat.clear_context_callback(self._dc_context)
             del self._dc_context
+            atexit.unregister(self.shutdown)
 
     def _process_event(self, ctx, evt_name, data1, data2):
         assert ctx == self._dc_context
@@ -463,6 +518,20 @@ class Account(object):
     def on_dc_event_imex_file_written(self, data1, data2):
         self._imex_events.put(data1)
 
+    def set_location(self, latitude=0.0, longitude=0.0, accuracy=0.0):
+        """set a new location. It effects all chats where we currently
+        have enabled location streaming.
+
+        :param latitude: float (use 0.0 if not known)
+        :param longitude: float (use 0.0 if not known)
+        :param accuracy: float (use 0.0 if not known)
+        :raises: ValueError if no chat is currently streaming locations
+        :returns: None
+        """
+        dc_res = lib.dc_set_location(self._dc_context, latitude, longitude, accuracy)
+        if dc_res == 0:
+            raise ValueError("no chat is streaming locations")
+
 
 class IOThreads:
     def __init__(self, dc_context, log_event=lambda *args: None):
@@ -574,11 +643,11 @@ class EventLogger:
             else:
                 assert not rex.match(ev[0]), "event found {}".format(ev)
 
-    def get_matching(self, event_name_regex, check_error=True):
+    def get_matching(self, event_name_regex, check_error=True, timeout=None):
         self._log("-- waiting for event with regex: {} --".format(event_name_regex))
         rex = re.compile("(?:{}).*".format(event_name_regex))
         while 1:
-            ev = self.get()
+            ev = self.get(timeout=timeout, check_error=check_error)
             if rex.match(ev[0]):
                 return ev
 

python/src/deltachat/chat.py

@@ -1,58 +1,15 @@
-""" chatting related objects: Contact, Chat, Message. """
+""" Chat and Location related API. """
 
 import mimetypes
+import calendar
+from datetime import datetime
 import os
-from . import props
 from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array
 from .capi import lib, ffi
 from . import const
 from .message import Message
 
 
-class Contact(object):
-    """ Delta-Chat Contact.
-
-    You obtain instances of it through :class:`deltachat.account.Account`.
-    """
-    def __init__(self, dc_context, id):
-        self._dc_context = dc_context
-        self.id = id
-
-    def __eq__(self, other):
-        return self._dc_context == other._dc_context and self.id == other.id
-
-    def __ne__(self, other):
-        return not (self == other)
-
-    def __repr__(self):
-        return "<Contact id={} addr={} dc_context={}>".format(self.id, self.addr, self._dc_context)
-
-    @property
-    def _dc_contact(self):
-        return ffi.gc(
-            lib.dc_get_contact(self._dc_context, self.id),
-            lib.dc_contact_unref
-        )
-
-    @props.with_doc
-    def addr(self):
-        """ normalized e-mail address for this account. """
-        return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
-
-    @props.with_doc
-    def display_name(self):
-        """ display name for this contact. """
-        return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
-
-    def is_blocked(self):
-        """ Return True if the contact is blocked. """
-        return lib.dc_contact_is_blocked(self._dc_contact)
-
-    def is_verified(self):
-        """ Return True if the contact is verified. """
-        return lib.dc_contact_is_verified(self._dc_contact)
-
-
 class Chat(object):
     """ Chat object which manages members and through which you can send and retrieve messages.
 
@@ -312,9 +269,10 @@ class Chat(object):
     def get_contacts(self):
         """ get all contacts for this chat.
         :params: contact object.
-        :returns: list of :class:`deltachat.chatting.Contact` objects for this chat
+        :returns: list of :class:`deltachat.contact.Contact` objects for this chat
 
         """
+        from .contact import Contact
         dc_array = ffi.gc(
             lib.dc_get_chat_contacts(self._dc_context, self.id),
             lib.dc_array_unref
@@ -365,3 +323,62 @@ class Chat(object):
         if dc_res == ffi.NULL:
             return None
         return from_dc_charpointer(dc_res)
+
+    # ------  location streaming API ------------------------------
+
+    def is_sending_locations(self):
+        """return True if this chat has location-sending enabled currently.
+        :returns: True if location sending is enabled.
+        """
+        return lib.dc_is_sending_locations_to_chat(self._dc_context, self.id)
+
+    def enable_sending_locations(self, seconds):
+        """enable sending locations for this chat.
+
+        all subsequent messages will carry a location with them.
+        """
+        lib.dc_send_locations_to_chat(self._dc_context, self.id, seconds)
+
+    def get_locations(self, contact=None, timestamp_from=None, timestamp_to=None):
+        """return list of locations for the given contact in the given timespan.
+
+        :param contact: the contact for which locations shall be returned.
+        :param timespan_from: a datetime object or None (indicating "since beginning")
+        :param timespan_to: a datetime object or None (indicating up till now)
+        :returns: list of :class:`deltachat.chat.Location` objects.
+        """
+        if timestamp_from is None:
+            time_from = 0
+        else:
+            time_from = calendar.timegm(timestamp_from.utctimetuple())
+        if timestamp_to is None:
+            time_to = 0
+        else:
+            time_to = calendar.timegm(timestamp_to.utctimetuple())
+
+        if contact is None:
+            contact_id = 0
+        else:
+            contact_id = contact.id
+
+        dc_array = lib.dc_get_locations(self._dc_context, self.id, contact_id, time_from, time_to)
+        return [
+            Location(
+                latitude=lib.dc_array_get_latitude(dc_array, i),
+                longitude=lib.dc_array_get_longitude(dc_array, i),
+                accuracy=lib.dc_array_get_accuracy(dc_array, i),
+                timestamp=datetime.utcfromtimestamp(lib.dc_array_get_timestamp(dc_array, i)))
+            for i in range(lib.dc_array_get_cnt(dc_array))
+        ]
+
+
+class Location:
+    def __init__(self, latitude, longitude, accuracy, timestamp):
+        assert isinstance(timestamp, datetime)
+        self.latitude = latitude
+        self.longitude = longitude
+        self.accuracy = accuracy
+        self.timestamp = timestamp
+
+    def __eq__(self, other):
+        return self.__dict__ == other.__dict__

python/src/deltachat/const.py

@@ -8,9 +8,6 @@ from os.path import join as joinpath
 # this works well when you in a git-checkout
 # run "python deltachat/const.py" to regenerate events
 # begin const generated
-DC_PROVIDER_STATUS_OK = 1
-DC_PROVIDER_STATUS_PREPARATION = 2
-DC_PROVIDER_STATUS_BROKEN = 3
 DC_GCL_ARCHIVED_ONLY = 0x01
 DC_GCL_NO_SPECIALS = 0x02
 DC_GCL_ADD_ALLDONE_HINT = 0x04
@@ -50,19 +47,40 @@ DC_STATE_OUT_FAILED = 24
 DC_STATE_OUT_DELIVERED = 26
 DC_STATE_OUT_MDN_RCVD = 28
 DC_CONTACT_ID_SELF = 1
-DC_CONTACT_ID_DEVICE = 2
+DC_CONTACT_ID_INFO = 2
+DC_CONTACT_ID_DEVICE = 5
 DC_CONTACT_ID_LAST_SPECIAL = 9
 DC_MSG_TEXT = 10
 DC_MSG_IMAGE = 20
 DC_MSG_GIF = 21
+DC_MSG_STICKER = 23
 DC_MSG_AUDIO = 40
 DC_MSG_VOICE = 41
 DC_MSG_VIDEO = 50
 DC_MSG_FILE = 60
+DC_LP_AUTH_OAUTH2 = 0x2
+DC_LP_AUTH_NORMAL = 0x4
+DC_LP_IMAP_SOCKET_STARTTLS = 0x100
+DC_LP_IMAP_SOCKET_SSL = 0x200
+DC_LP_IMAP_SOCKET_PLAIN = 0x400
+DC_LP_SMTP_SOCKET_STARTTLS = 0x10000
+DC_LP_SMTP_SOCKET_SSL = 0x20000
+DC_LP_SMTP_SOCKET_PLAIN = 0x40000
+DC_CERTCK_AUTO = 0
+DC_CERTCK_STRICT = 1
+DC_CERTCK_ACCEPT_INVALID_HOSTNAMES = 2
+DC_CERTCK_ACCEPT_INVALID_CERTIFICATES = 3
+DC_EMPTY_MVBOX = 0x01
+DC_EMPTY_INBOX = 0x02
 DC_EVENT_INFO = 100
 DC_EVENT_SMTP_CONNECTED = 101
 DC_EVENT_IMAP_CONNECTED = 102
 DC_EVENT_SMTP_MESSAGE_SENT = 103
+DC_EVENT_IMAP_MESSAGE_DELETED = 104
+DC_EVENT_IMAP_MESSAGE_MOVED = 105
+DC_EVENT_IMAP_FOLDER_EMPTIED = 106
+DC_EVENT_NEW_BLOB_FILE = 150
+DC_EVENT_DELETED_BLOB_FILE = 151
 DC_EVENT_WARNING = 300
 DC_EVENT_ERROR = 400
 DC_EVENT_ERROR_NETWORK = 401
@@ -80,15 +98,65 @@ DC_EVENT_IMEX_PROGRESS = 2051
 DC_EVENT_IMEX_FILE_WRITTEN = 2052
 DC_EVENT_SECUREJOIN_INVITER_PROGRESS = 2060
 DC_EVENT_SECUREJOIN_JOINER_PROGRESS = 2061
-DC_EVENT_GET_STRING = 2091
 DC_EVENT_FILE_COPIED = 2055
 DC_EVENT_IS_OFFLINE = 2081
+DC_EVENT_GET_STRING = 2091
+DC_STR_SELFNOTINGRP = 21
+DC_PROVIDER_STATUS_OK = 1
+DC_PROVIDER_STATUS_PREPARATION = 2
+DC_PROVIDER_STATUS_BROKEN = 3
+DC_STR_NOMESSAGES = 1
+DC_STR_SELF = 2
+DC_STR_DRAFT = 3
+DC_STR_MEMBER = 4
+DC_STR_CONTACT = 6
+DC_STR_VOICEMESSAGE = 7
+DC_STR_DEADDROP = 8
+DC_STR_IMAGE = 9
+DC_STR_VIDEO = 10
+DC_STR_AUDIO = 11
+DC_STR_FILE = 12
+DC_STR_STATUSLINE = 13
+DC_STR_NEWGROUPDRAFT = 14
+DC_STR_MSGGRPNAME = 15
+DC_STR_MSGGRPIMGCHANGED = 16
+DC_STR_MSGADDMEMBER = 17
+DC_STR_MSGDELMEMBER = 18
+DC_STR_MSGGROUPLEFT = 19
+DC_STR_GIF = 23
+DC_STR_ENCRYPTEDMSG = 24
+DC_STR_E2E_AVAILABLE = 25
+DC_STR_ENCR_TRANSP = 27
+DC_STR_ENCR_NONE = 28
+DC_STR_CANTDECRYPT_MSG_BODY = 29
+DC_STR_FINGERPRINTS = 30
+DC_STR_READRCPT = 31
+DC_STR_READRCPT_MAILBODY = 32
+DC_STR_MSGGRPIMGDELETED = 33
+DC_STR_E2E_PREFERRED = 34
+DC_STR_CONTACT_VERIFIED = 35
+DC_STR_CONTACT_NOT_VERIFIED = 36
+DC_STR_CONTACT_SETUP_CHANGED = 37
+DC_STR_ARCHIVEDCHATS = 40
+DC_STR_STARREDMSGS = 41
+DC_STR_AC_SETUP_MSG_SUBJECT = 42
+DC_STR_AC_SETUP_MSG_BODY = 43
+DC_STR_SELFTALK_SUBTITLE = 50
+DC_STR_CANNOT_LOGIN = 60
+DC_STR_SERVER_RESPONSE = 61
+DC_STR_MSGACTIONBYUSER = 62
+DC_STR_MSGACTIONBYME = 63
+DC_STR_MSGLOCATIONENABLED = 64
+DC_STR_MSGLOCATIONDISABLED = 65
+DC_STR_LOCATION = 66
+DC_STR_STICKER = 67
+DC_STR_COUNT = 67
 # end const generated
 
 
 def read_event_defines(f):
-    rex = re.compile(r'#define\s+((?:DC_EVENT_|DC_QR|DC_MSG|DC_STATE_|'
-                     r'DC_CONTACT_ID_|DC_GCL|DC_CHAT|DC_PROVIDER)\S+)\s+([x\d]+).*')
+    rex = re.compile(r'#define\s+((?:DC_EVENT|DC_QR|DC_MSG|DC_LP|DC_EMPTY|DC_CERTCK|DC_STATE|DC_STR|'
+                     r'DC_CONTACT_ID|DC_GCL|DC_CHAT|DC_PROVIDER)_\S+)\s+([x\d]+).*')
     for line in f:
         m = rex.match(line)
         if m:

python/src/deltachat/contact.py

@@ -0,0 +1,49 @@
+""" Contact object. """
+
+from . import props
+from .cutil import from_dc_charpointer
+from .capi import lib, ffi
+
+
+class Contact(object):
+    """ Delta-Chat Contact.
+
+    You obtain instances of it through :class:`deltachat.account.Account`.
+    """
+    def __init__(self, dc_context, id):
+        self._dc_context = dc_context
+        self.id = id
+
+    def __eq__(self, other):
+        return self._dc_context == other._dc_context and self.id == other.id
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __repr__(self):
+        return "<Contact id={} addr={} dc_context={}>".format(self.id, self.addr, self._dc_context)
+
+    @property
+    def _dc_contact(self):
+        return ffi.gc(
+            lib.dc_get_contact(self._dc_context, self.id),
+            lib.dc_contact_unref
+        )
+
+    @props.with_doc
+    def addr(self):
+        """ normalized e-mail address for this account. """
+        return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
+
+    @props.with_doc
+    def display_name(self):
+        """ display name for this contact. """
+        return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
+
+    def is_blocked(self):
+        """ Return True if the contact is blocked. """
+        return lib.dc_contact_is_blocked(self._dc_contact)
+
+    def is_verified(self):
+        """ Return True if the contact is verified. """
+        return lib.dc_contact_is_verified(self._dc_contact)

python/src/deltachat/message.py

@@ -1,4 +1,4 @@
-""" chatting related objects: Contact, Chat, Message. """
+""" The Message object. """
 
 import os
 import shutil
@@ -13,7 +13,7 @@ class Message(object):
     """ Message object.
 
     You obtain instances of it through :class:`deltachat.account.Account` or
-    :class:`deltachat.chatting.Chat`.
+    :class:`deltachat.chat.Chat`.
     """
     def __init__(self, account, dc_msg):
         self.account = account
@@ -109,6 +109,10 @@ class Message(object):
         """ return True if this message was encrypted. """
         return bool(lib.dc_msg_get_showpadlock(self._dc_msg))
 
+    def is_forwarded(self):
+        """ return True if this message was forwarded. """
+        return bool(lib.dc_msg_is_forwarded(self._dc_msg))
+
     def get_message_info(self):
         """ Return informational text for a single message.
 
@@ -165,18 +169,18 @@ class Message(object):
     def chat(self):
         """chat this message was posted in.
 
-        :returns: :class:`deltachat.chatting.Chat` object
+        :returns: :class:`deltachat.chat.Chat` object
         """
-        from .chatting import Chat
+        from .chat import Chat
         chat_id = lib.dc_msg_get_chat_id(self._dc_msg)
         return Chat(self.account, chat_id)
 
     def get_sender_contact(self):
         """return the contact of who wrote the message.
 
-        :returns: :class:`deltachat.chatting.Contact` instance
+        :returns: :class:`deltachat.chat.Contact` instance
         """
-        from .chatting import Contact
+        from .contact import Contact
         contact_id = lib.dc_msg_get_from_id(self._dc_msg)
         return Contact(self._dc_context, contact_id)
 

python/tests/conftest.py

@@ -4,6 +4,7 @@ import pytest
 import requests
 import time
 from deltachat import Account
+from deltachat import const
 from deltachat.capi import lib
 import tempfile
 
@@ -150,6 +151,11 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig):
             lib.dc_set_config(ac._dc_context, b"configured", b"1")
             return ac
 
+        def peek_online_config(self):
+            if not session_liveconfig:
+                pytest.skip("specify DCC_PY_LIVECONFIG or --liveconfig")
+            return session_liveconfig.get(self.live_count)
+
         def get_online_config(self):
             if not session_liveconfig:
                 pytest.skip("specify DCC_PY_LIVECONFIG or --liveconfig")
@@ -157,6 +163,11 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig):
             self.live_count += 1
             if "e2ee_enabled" not in configdict:
                 configdict["e2ee_enabled"] = "1"
+
+            # Enable strict certificate checks for online accounts
+            configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+            configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+
             tmpdb = tmpdir.join("livedb%d" % self.live_count)
             ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
             ac._evlogger.init_time = self.init_time
@@ -169,6 +180,12 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig):
             ac.start_threads(mvbox=mvbox, sentbox=sentbox)
             return ac
 
+        def get_one_online_account(self):
+            ac1 = self.get_online_configuring_account()
+            wait_successful_IMAP_SMTP_connection(ac1)
+            wait_configuration_progress(ac1, 1000)
+            return ac1
+
         def get_two_online_accounts(self):
             ac1 = self.get_online_configuring_account()
             ac2 = self.get_online_configuring_account()

python/tests/test_account.py

@@ -1,6 +1,8 @@
 from __future__ import print_function
 import pytest
 import os
+import queue
+import time
 from deltachat import const, Account
 from deltachat.message import Message
 from datetime import datetime, timedelta
@@ -19,6 +21,7 @@ class TestOfflineAccountBasic:
         d = ac1.get_info()
         assert d["arch"]
         assert d["number_of_chats"] == "0"
+        assert d["bcc_self"] == "1"
 
     def test_is_not_configured(self, acfactory):
         ac1 = acfactory.get_unconfigured_account()
@@ -37,6 +40,11 @@ class TestOfflineAccountBasic:
         ac1 = acfactory.get_unconfigured_account()
         assert "save_mime_headers" in ac1.get_config("sys.config_keys").split()
 
+    def test_has_bccself(self, acfactory):
+        ac1 = acfactory.get_unconfigured_account()
+        assert "bcc_self" in ac1.get_config("sys.config_keys").split()
+        assert ac1.get_config("bcc_self") == "1"
+
     def test_selfcontact_if_unconfigured(self, acfactory):
         ac1 = acfactory.get_unconfigured_account()
         with pytest.raises(ValueError):
@@ -93,7 +101,7 @@ class TestOfflineContact:
         ac1 = acfactory.get_configured_offline_account()
         contact1 = ac1.create_contact(email="some1@example.com", name="some1")
         chat = ac1.create_chat_by_contact(contact1)
-        msg = chat.send_text("one messae")
+        msg = chat.send_text("one message")
         assert not ac1.delete_contact(contact1)
         assert not msg.filemime
 
@@ -114,6 +122,12 @@ class TestOfflineChat:
         str(chat1)
         repr(chat1)
 
+    def test_chat_by_id(self, chat1):
+        chat2 = chat1.account.get_chat_by_id(chat1.id)
+        assert chat2 == chat1
+        with pytest.raises(ValueError):
+            chat1.account.get_chat_by_id(123123)
+
     def test_chat_idempotent(self, chat1, ac1):
         contact1 = chat1.get_contacts()[0]
         chat2 = ac1.create_chat_by_contact(contact1.id)
@@ -141,6 +155,27 @@ class TestOfflineChat:
         chat.set_name("title2")
         assert chat.get_name() == "title2"
 
+    def test_group_chat_creation_with_translation(self, ac1):
+        ac1.set_stock_translation(const.DC_STR_NEWGROUPDRAFT, "xyz %1$s")
+        ac1._evlogger.consume_events()
+        with pytest.raises(ValueError):
+            ac1.set_stock_translation(const.DC_STR_NEWGROUPDRAFT, "xyz %2$s")
+        ac1._evlogger.get_matching("DC_EVENT_WARNING")
+        with pytest.raises(ValueError):
+            ac1.set_stock_translation(500, "xyz %1$s")
+        ac1._evlogger.get_matching("DC_EVENT_WARNING")
+        contact1 = ac1.create_contact("some1@hello.com", name="some1")
+        contact2 = ac1.create_contact("some2@hello.com", name="some2")
+        chat = ac1.create_group_chat(name="title1")
+        chat.add_contact(contact1)
+        chat.add_contact(contact2)
+        assert chat.get_name() == "title1"
+        assert contact1 in chat.get_contacts()
+        assert contact2 in chat.get_contacts()
+        assert not chat.is_promoted()
+        msg = chat.get_draft()
+        assert msg.text == "xyz title1"
+
     @pytest.mark.parametrize("verified", [True, False])
     def test_group_chat_qr(self, acfactory, ac1, verified):
         ac2 = acfactory.get_configured_offline_account()
@@ -222,7 +257,9 @@ class TestOfflineChat:
             chat1.send_image(path="notexists")
         fn = data.get_path("d.png")
         lp.sec("sending image")
+        chat1.account._evlogger.consume_events()
         msg = chat1.send_image(fn)
+        chat1.account._evlogger.get_matching("DC_EVENT_NEW_BLOB_FILE")
         assert msg.is_image()
         assert msg
         assert msg.id > 0
@@ -335,12 +372,20 @@ class TestOfflineChat:
 
 
 class TestOnlineAccount:
-    def get_chat(self, ac1, ac2):
+    def get_chat(self, ac1, ac2, both_created=False):
         c2 = ac1.create_contact(email=ac2.get_config("addr"))
         chat = ac1.create_chat_by_contact(c2)
         assert chat.id > const.DC_CHAT_ID_LAST_SPECIAL
+        if both_created:
+            ac2.create_chat_by_contact(ac2.create_contact(email=ac1.get_config("addr")))
         return chat
 
+    def test_configure_canceled(self, acfactory):
+        ac1 = acfactory.get_online_configuring_account()
+        wait_configuration_progress(ac1, 200)
+        ac1.stop_ongoing()
+        wait_configuration_progress(ac1, 0, 0)
+
     def test_export_import_self_keys(self, acfactory, tmpdir):
         ac1, ac2 = acfactory.get_two_online_accounts()
         dir = tmpdir.mkdir("exportdir")
@@ -351,18 +396,36 @@ class TestOnlineAccount:
         ac1._evlogger.consume_events()
         ac1.import_self_keys(dir.strpath)
 
-    def test_one_account_send(self, acfactory):
+    def test_one_account_send_bcc_setting(self, acfactory, lp):
         ac1 = acfactory.get_online_configuring_account()
-        c2 = ac1.create_contact(email=ac1.get_config("addr"))
+        ac2_config = acfactory.peek_online_config()
+        c2 = ac1.create_contact(email=ac2_config["addr"])
         chat = ac1.create_chat_by_contact(c2)
         assert chat.id > const.DC_CHAT_ID_LAST_SPECIAL
         wait_successful_IMAP_SMTP_connection(ac1)
         wait_configuration_progress(ac1, 1000)
 
+        lp.sec("send out message with bcc to ourselves")
         msg_out = chat.send_text("message2")
-        # wait for own account to receive
-        ev = ac1._evlogger.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
-        assert ev[1] == msg_out.id
+        ev = ac1._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
+        assert ev[2] == msg_out.id
+        # wait for send out (BCC)
+        assert ac1.get_config("bcc_self") == "1"
+        self_addr = ac1.get_config("addr")
+        ev = ac1._evlogger.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
+        assert self_addr in ev[2]
+        ev = ac1._evlogger.get_matching("DC_EVENT_DELETED_BLOB_FILE")
+
+        ac1._evlogger.consume_events()
+        lp.sec("send out message without bcc")
+        ac1.set_config("bcc_self", "0")
+        msg_out = chat.send_text("message3")
+        assert not msg_out.is_forwarded()
+        ev = ac1._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
+        assert ev[2] == msg_out.id
+        ev = ac1._evlogger.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
+        assert self_addr not in ev[2]
+        ev = ac1._evlogger.get_matching("DC_EVENT_DELETED_BLOB_FILE")
 
     def test_mvbox_sentbox_threads(self, acfactory):
         ac1 = acfactory.get_online_configuring_account(mvbox=True, sentbox=True)
@@ -374,6 +437,17 @@ class TestOnlineAccount:
         ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
         assert ev[2] > const.DC_CHAT_ID_LAST_SPECIAL
 
+    def test_move_works(self, acfactory):
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.get_online_configuring_account(mvbox=True)
+        wait_configuration_progress(ac2, 1000)
+        wait_configuration_progress(ac1, 1000)
+        chat = self.get_chat(ac1, ac2)
+        chat.send_text("message1")
+        ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
+        assert ev[2] > const.DC_CHAT_ID_LAST_SPECIAL
+        ev = ac2._evlogger.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
+
     def test_forward_messages(self, acfactory):
         ac1, ac2 = acfactory.get_two_online_accounts()
         chat = self.get_chat(ac1, ac2)
@@ -389,6 +463,7 @@ class TestOnlineAccount:
         # check the message arrived in contact-requests/deaddrop
         chat2 = msg_in.chat
         assert msg_in in chat2.get_messages()
+        assert not msg_in.is_forwarded()
         assert chat2.is_deaddrop()
         assert chat2 == ac2.get_deaddrop_chat()
         chat3 = ac2.create_group_chat("newgroup")
@@ -396,9 +471,47 @@ class TestOnlineAccount:
         ac2.forward_messages([msg_in], chat3)
         assert chat3.is_promoted()
         messages = chat3.get_messages()
+        msg = messages[-1]
+        assert msg.is_forwarded()
         ac2.delete_messages(messages)
         assert not chat3.get_messages()
 
+    def test_forward_own_message(self, acfactory, lp):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        chat = self.get_chat(ac1, ac2, both_created=True)
+
+        lp.sec("sending message")
+        msg_out = chat.send_text("message2")
+
+        lp.sec("receiving message")
+        ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG")
+        msg_in = ac2.get_message_by_id(ev[2])
+        assert msg_in.text == "message2"
+        assert not msg_in.is_forwarded()
+
+        lp.sec("ac1: creating group chat, and forward own message")
+        group = ac1.create_group_chat("newgroup2")
+        group.add_contact(ac1.create_contact(ac2.get_config("addr")))
+        ac1.forward_messages([msg_out], group)
+
+        # wait for other account to receive
+        ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG")
+        msg_in = ac2.get_message_by_id(ev[2])
+        assert msg_in.text == "message2"
+        assert msg_in.is_forwarded()
+
+    def test_send_self_message_and_empty_folder(self, acfactory, lp):
+        ac1 = acfactory.get_one_online_account()
+        lp.sec("ac1: create self chat")
+        chat = ac1.create_chat_by_contact(ac1.get_self_contact())
+        chat.send_text("hello")
+        ac1._evlogger.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
+        ac1.empty_server_folders(inbox=True, mvbox=True)
+        ev = ac1._evlogger.get_matching("DC_EVENT_IMAP_FOLDER_EMPTIED")
+        assert ev[2] == "DeltaChat"
+        ev = ac1._evlogger.get_matching("DC_EVENT_IMAP_FOLDER_EMPTIED")
+        assert ev[2] == "INBOX"
+
     def test_send_and_receive_message_markseen(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
@@ -418,6 +531,7 @@ class TestOnlineAccount:
         assert ev[2] == msg_out.id
         msg_in = ac2.get_message_by_id(msg_out.id)
         assert msg_in.text == "message1"
+        assert not msg_in.is_forwarded()
 
         lp.sec("check the message arrived in contact-requets/deaddrop")
         chat2 = msg_in.chat
@@ -444,6 +558,14 @@ class TestOnlineAccount:
         lp.step("2")
         assert msg_out.is_out_mdn_received()
 
+        lp.sec("check that a second call to mark_seen does not create change or smtp job")
+        ac2._evlogger.consume_events()
+        ac2.mark_seen_messages([msg_in])
+        try:
+            ac2._evlogger.get_matching("DC_EVENT_MSG_READ", timeout=0.01)
+        except queue.Empty:
+            pass  # mark_seen_messages() has generated events before it returns
+
     def test_send_and_receive_will_encrypt_decrypt(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
@@ -472,6 +594,14 @@ class TestOnlineAccount:
         assert msg_back.text == "message-back"
         assert msg_back.is_encrypted()
 
+        lp.sec("create group chat with two members, one of which has no encrypt state")
+        chat = ac1.create_group_chat("encryption test")
+        chat.add_contact(ac1.create_contact(ac2.get_config("addr")))
+        chat.add_contact(ac1.create_contact("notexisting@testrun.org"))
+        msg = chat.send_text("test not encrypt")
+        ev = ac1._evlogger.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
+        assert not msg.is_encrypted()
+
     def test_saved_mime_on_received_message(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
@@ -512,18 +642,29 @@ class TestOnlineAccount:
         assert os.path.exists(msg_in.filename)
         assert os.stat(msg_in.filename).st_size == os.stat(path).st_size
 
-    def test_import_export_online_all(self, acfactory, tmpdir):
+    def test_import_export_online_all(self, acfactory, tmpdir, lp):
         ac1 = acfactory.get_online_configuring_account()
         wait_configuration_progress(ac1, 1000)
 
+        lp.sec("create some chat content")
         contact1 = ac1.create_contact("some1@hello.com", name="some1")
         chat = ac1.create_chat_by_contact(contact1)
         chat.send_text("msg1")
         backupdir = tmpdir.mkdir("backup")
+
+        lp.sec("export all to {}".format(backupdir))
         path = ac1.export_all(backupdir.strpath)
         assert os.path.exists(path)
+        t = time.time()
 
+        lp.sec("get fresh empty account")
         ac2 = acfactory.get_unconfigured_account()
+
+        lp.sec("get latest backup file")
+        path2 = ac2.get_latest_backupfile(backupdir.strpath)
+        assert path2 == path
+
+        lp.sec("import backup and check it's proper")
         ac2.import_all(path)
         contacts = ac2.get_contacts(query="some1")
         assert len(contacts) == 1
@@ -534,6 +675,18 @@ class TestOnlineAccount:
         assert len(messages) == 1
         assert messages[0].text == "msg1"
 
+        pytest.xfail("cannot export twice yet, probably due to interrupt_idle failing")
+        # wait until a second passed since last backup
+        # because get_latest_backupfile() shall return the latest backup
+        # from a UI it's unlikely anyone manages to export two
+        # backups in one second.
+        time.sleep(max(0, 1 - (time.time() - t)))
+        lp.sec("Second-time export all to {}".format(backupdir))
+        path2 = ac1.export_all(backupdir.strpath)
+        assert os.path.exists(path2)
+        assert path2 != path
+        assert ac2.get_latest_backupfile(backupdir.strpath) == path2
+
     def test_ac_setup_message(self, acfactory, lp):
         # note that the receiving account needs to be configured and running
         # before ther setup message is send. DC does not read old messages
@@ -559,6 +712,27 @@ class TestOnlineAccount:
         msg.continue_key_transfer(setup_code)
         assert ac1.get_info()["fingerprint"] == ac2.get_info()["fingerprint"]
 
+    def test_ac_setup_message_twice(self, acfactory, lp):
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.clone_online_account(ac1)
+        ac2._evlogger.set_timeout(30)
+        wait_configuration_progress(ac2, 1000)
+        wait_configuration_progress(ac1, 1000)
+        lp.sec("trigger ac setup message but ignore")
+        assert ac1.get_info()["fingerprint"] != ac2.get_info()["fingerprint"]
+        ac1.initiate_key_transfer()
+        ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
+
+        lp.sec("trigger second ac setup message, wait for receive ")
+        setup_code2 = ac1.initiate_key_transfer()
+        ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
+        msg = ac2.get_message_by_id(ev[2])
+        assert msg.is_setup_message()
+        assert msg.get_setupcodebegin() == setup_code2[:2]
+        lp.sec("process second setup message")
+        msg.continue_key_transfer(setup_code2)
+        assert ac1.get_info()["fingerprint"] == ac2.get_info()["fingerprint"]
+
     def test_qr_setup_contact(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
         lp.sec("ac1: create QR code and let ac2 scan it, starting the securejoin")
@@ -576,6 +750,9 @@ class TestOnlineAccount:
         lp.sec("ac2: start QR-code based join-group protocol")
         ch = ac2.qr_join_chat(qr)
         assert ch.id >= 10
+        # check that at least some of the handshake messages are deleted
+        ac1._evlogger.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
+        ac2._evlogger.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
         wait_securejoin_inviter_progress(ac1, 1000)
 
     def test_qr_verified_group_and_chatting(self, acfactory, lp):
@@ -663,6 +840,54 @@ class TestOnlineAccount:
         assert chat1b.get_profile_image() is None
         assert chat.get_profile_image() is None
 
+    def test_send_receive_locations(self, acfactory, lp):
+        now = datetime.utcnow()
+        ac1, ac2 = acfactory.get_two_online_accounts()
+
+        lp.sec("ac1: create chat with ac2")
+        chat1 = self.get_chat(ac1, ac2)
+        chat2 = self.get_chat(ac2, ac1)
+
+        assert not chat1.is_sending_locations()
+        with pytest.raises(ValueError):
+            ac1.set_location(latitude=0.0, longitude=10.0)
+
+        ac1._evlogger.consume_events()
+        ac2._evlogger.consume_events()
+
+        lp.sec("ac1: enable location sending in chat")
+        chat1.enable_sending_locations(seconds=100)
+        assert chat1.is_sending_locations()
+        ac1._evlogger.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
+
+        ac1.set_location(latitude=2.0, longitude=3.0, accuracy=0.5)
+        ac1._evlogger.get_matching("DC_EVENT_LOCATION_CHANGED")
+        chat1.send_text("hello")
+        ac1._evlogger.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
+
+        lp.sec("ac2: wait for incoming location message")
+        ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG")  # "enabled-location streaming"
+
+        # currently core emits location changed before event_incoming message
+        ac2._evlogger.get_matching("DC_EVENT_LOCATION_CHANGED")
+        ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG")  # text message with location
+
+        locations = chat2.get_locations()
+        assert len(locations) == 1
+        assert locations[0].latitude == 2.0
+        assert locations[0].longitude == 3.0
+        assert locations[0].accuracy == 0.5
+        assert locations[0].timestamp > now
+
+        contact = ac2.create_contact(ac1.get_config("addr"))
+        locations2 = chat2.get_locations(contact=contact)
+        assert len(locations2) == 1
+        assert locations2 == locations
+
+        contact = ac2.create_contact("nonexisting@example.org")
+        locations3 = chat2.get_locations(contact=contact)
+        assert not locations3
+
 
 class TestOnlineConfigureFails:
     def test_invalid_password(self, acfactory):

python/tests/test_lowlevel.py

@@ -95,6 +95,13 @@ def test_markseen_invalid_message_ids(acfactory):
     ac1._evlogger.ensure_event_not_queued("DC_EVENT_WARNING|DC_EVENT_ERROR")
 
 
+def test_get_special_message_id_returns_empty_message(acfactory):
+    ac1 = acfactory.get_configured_offline_account()
+    for i in range(1, 10):
+        msg = ac1.get_message_by_id(i)
+        assert msg.id == 0
+
+
 def test_provider_info():
     provider = lib.dc_provider_new_from_email(cutil.as_dc_charpointer("ex@example.com"))
     assert cutil.from_dc_charpointer(

run-integration-tests.sh

@@ -23,7 +23,7 @@ if [ $? != 0 ]; then
 fi
 
 pushd python
-if [ -e "./liveconfig" && -z "$DCC_PY_LIVECONFIG" ]; then
+if [ -e "./liveconfig" -a -z "$DCC_PY_LIVECONFIG" ]; then
     export DCC_PY_LIVECONFIG=liveconfig
 fi
 tox "$@"

set_core_version.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import re
+import pathlib
+import subprocess
+
+rex = re.compile(r'version = "(\S+)"')
+
+def read_toml_version(relpath):
+    p = pathlib.Path(relpath)
+    assert p.exists()
+    for line in open(str(p)):
+        m = rex.match(line)
+        if m is not None:
+            return m.group(1)
+    raise ValueError("no version found in {}".format(relpath))
+
+def replace_toml_version(relpath, newversion):
+    p = pathlib.Path(relpath)
+    assert p.exists()
+    tmp_path = str(p) + "_tmp"
+    with open(tmp_path, "w") as f:
+        for line in open(str(p)):
+            m = rex.match(line)
+            if m is not None:
+                f.write('version = "{}"\n'.format(newversion))
+            else:
+                f.write(line)
+    os.rename(tmp_path, str(p))
+
+if __name__ == "__main__":
+
+    if len(sys.argv) < 2:
+        raise SystemExit("need argument: new version, example 1.0.0-beta.27")
+    newversion = sys.argv[1]
+    if newversion.count(".") < 2:
+        raise SystemExit("need at least two dots in version")
+
+    core_toml = read_toml_version("Cargo.toml")
+    ffi_toml = read_toml_version("deltachat-ffi/Cargo.toml")
+    assert core_toml == ffi_toml, (core_toml, ffi_toml)
+
+    for line in open("CHANGELOG.md"):
+        ## 1.0.0-beta5
+        if line.startswith("## "):
+            if line[2:].strip().startswith(newversion):
+                break
+    else:
+        raise SystemExit("CHANGELOG.md contains no entry for version: {}".format(newversion))
+
+    replace_toml_version("Cargo.toml", newversion)
+    replace_toml_version("deltachat-ffi/Cargo.toml", newversion)
+
+    subprocess.call(["cargo", "update", "-p", "deltachat"])
+
+    print("after commit make sure to: ")
+    print("")
+    print("   git tag {}".format(newversion))
+    print("")

spec.md

@@ -334,9 +334,7 @@ only on image changes.
 
 # Miscellaneous
 
-Messengers SHOULD use the header `Chat-Predecessor`
-instead of `In-Reply-To` as the latter one results
-in infinite threads on typical MUAs.
+Messengers SHOULD use the header `In-Reply-To` as usual.
 
 Messengers SHOULD add a `Chat-Voice-message: 1` header
 if an attached audio file is a voice message.
@@ -346,7 +344,7 @@ to specify the duration of attached audio or video files.
 The value MUST be the duration in milliseconds.
 This allows the receiver to show the time without knowing the file format.
 
-    Chat-Predecessor: foo123@domain
+    In-Reply-To: Gr.12345uvwxyZ.0005@domain
     Chat-Voice-Message: 1
     Chat-Duration: 10000
 

src/aheader.rs

@@ -79,27 +79,21 @@ impl Aheader {
                 let optional_field = unsafe { (*field).fld_data.fld_optional_field };
                 if !optional_field.is_null()
                     && unsafe { !(*optional_field).fld_name.is_null() }
-                    && unsafe { CStr::from_ptr((*optional_field).fld_name).to_str().unwrap() }
+                    && unsafe { CStr::from_ptr((*optional_field).fld_name).to_string_lossy() }
                         == "Autocrypt"
                 {
-                    let value = unsafe {
-                        CStr::from_ptr((*optional_field).fld_value)
-                            .to_str()
-                            .unwrap()
-                    };
+                    let value =
+                        unsafe { CStr::from_ptr((*optional_field).fld_value).to_string_lossy() };
 
-                    match Self::from_str(value) {
-                        Ok(test) => {
-                            if addr_cmp(&test.addr, wanted_from) {
-                                if fine_header.is_none() {
-                                    fine_header = Some(test);
-                                } else {
-                                    // TODO: figure out what kind of error case this is
-                                    return None;
-                                }
+                    if let Ok(test) = Self::from_str(&value) {
+                        if addr_cmp(&test.addr, wanted_from) {
+                            if fine_header.is_none() {
+                                fine_header = Some(test);
+                            } else {
+                                // TODO: figure out what kind of error case this is
+                                return None;
                             }
                         }
-                        _ => {}
                     }
                 }
             }
@@ -131,9 +125,9 @@ impl str::FromStr for Aheader {
 
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         let mut attributes: BTreeMap<String, String> = s
-            .split(";")
+            .split(';')
             .filter_map(|a| {
-                let attribute: Vec<&str> = a.trim().splitn(2, "=").collect();
+                let attribute: Vec<&str> = a.trim().splitn(2, '=').collect();
                 if attribute.len() < 2 {
                     return None;
                 }
@@ -178,7 +172,7 @@ impl str::FromStr for Aheader {
 
         // Autocrypt-Level0: unknown attributes starting with an underscore can be safely ignored
         // Autocrypt-Level0: unknown attribute, treat the header as invalid
-        if attributes.keys().find(|k| !k.starts_with("_")).is_some() {
+        if attributes.keys().any(|k| !k.starts_with('_')) {
             return Err(());
         }
 

src/blob.rs

@@ -0,0 +1,674 @@
+use std::ffi::OsStr;
+use std::fmt;
+use std::fs;
+use std::io::Write;
+use std::path::{Path, PathBuf};
+
+use crate::context::Context;
+use crate::events::Event;
+
+/// Represents a file in the blob directory.
+///
+/// The object has a name, which will always be valid UTF-8.  Having a
+/// blob object does not imply the respective file exists, however
+/// when using one of the `create*()` methods a unique file is
+/// created.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct BlobObject<'a> {
+    blobdir: &'a Path,
+    name: String,
+}
+
+impl<'a> BlobObject<'a> {
+    /// Creates a new blob object with a unique name.
+    ///
+    /// Creates a new file in the blob directory.  The name will be
+    /// derived from the platform-agnostic basename of the suggested
+    /// name, followed by a random number and followed by a possible
+    /// extension.  The `data` will be written into the file without
+    /// race-conditions.
+    ///
+    /// # Errors
+    ///
+    /// [BlobErrorKind::CreateFailure] is used when the file could not
+    /// be created.  You can expect [BlobError.cause] to contain an
+    /// underlying error.
+    ///
+    /// [BlobErrorKind::WriteFailure] is used when the file could not
+    /// be written to.  You can expect [BlobError.cause] to contain an
+    /// underlying error.
+    pub fn create(
+        context: &'a Context,
+        suggested_name: impl AsRef<str>,
+        data: &[u8],
+    ) -> std::result::Result<BlobObject<'a>, BlobError> {
+        let blobdir = context.get_blobdir();
+        let (stem, ext) = BlobObject::sanitise_name(suggested_name.as_ref());
+        let (name, mut file) = BlobObject::create_new_file(&blobdir, &stem, &ext)?;
+        file.write_all(data)
+            .map_err(|err| BlobError::new_write_failure(blobdir, &name, err))?;
+        let blob = BlobObject {
+            blobdir,
+            name: format!("$BLOBDIR/{}", name),
+        };
+        context.call_cb(Event::NewBlobFile(blob.as_name().to_string()));
+        Ok(blob)
+    }
+
+    // Creates a new file, returning a tuple of the name and the handle.
+    fn create_new_file(dir: &Path, stem: &str, ext: &str) -> Result<(String, fs::File), BlobError> {
+        let max_attempt = 15;
+        let mut name = format!("{}{}", stem, ext);
+        for attempt in 0..max_attempt {
+            let path = dir.join(&name);
+            match fs::OpenOptions::new()
+                .create_new(true)
+                .write(true)
+                .open(&path)
+            {
+                Ok(file) => return Ok((name, file)),
+                Err(err) => {
+                    if attempt == max_attempt {
+                        return Err(BlobError::new_create_failure(dir, &name, err));
+                    } else {
+                        name = format!("{}-{}{}", stem, rand::random::<u32>(), ext);
+                    }
+                }
+            }
+        }
+        Err(BlobError::new_create_failure(
+            dir,
+            &name,
+            format_err!("Unreachable code - supposedly"),
+        ))
+    }
+
+    /// Creates a new blob object with unique name by copying an existing file.
+    ///
+    /// This creates a new blob as described in [BlobObject::create]
+    /// but also copies an existing file into it.  This is done in a
+    /// in way which avoids race-conditions when multiple files are
+    /// concurrently created.
+    ///
+    /// # Errors
+    ///
+    /// In addition to the errors in [BlobObject::create] the
+    /// [BlobErrorKind::CopyFailure] is used when the data can not be
+    /// copied.
+    pub fn create_and_copy(
+        context: &'a Context,
+        src: impl AsRef<Path>,
+    ) -> std::result::Result<BlobObject<'a>, BlobError> {
+        let mut src_file = fs::File::open(src.as_ref()).map_err(|err| {
+            BlobError::new_copy_failure(context.get_blobdir(), "", src.as_ref(), err)
+        })?;
+        let (stem, ext) = BlobObject::sanitise_name(&src.as_ref().to_string_lossy());
+        let (name, mut dst_file) = BlobObject::create_new_file(context.get_blobdir(), &stem, &ext)?;
+        std::io::copy(&mut src_file, &mut dst_file).map_err(|err| {
+            {
+                // Attempt to remove the failed file, swallow errors resulting from that.
+                let path = context.get_blobdir().join(&name);
+                fs::remove_file(path).ok();
+            }
+            BlobError::new_copy_failure(context.get_blobdir(), &name, src.as_ref(), err)
+        })?;
+        let blob = BlobObject {
+            blobdir: context.get_blobdir(),
+            name: format!("$BLOBDIR/{}", name),
+        };
+        context.call_cb(Event::NewBlobFile(blob.as_name().to_string()));
+        Ok(blob)
+    }
+
+    /// Creates a blob from a file, possibly copying it to the blobdir.
+    ///
+    /// If the source file is not a path to into the blob directory
+    /// the file will be copied into the blob directory first.  If the
+    /// source file is already in the blobdir it will not be copied
+    /// and only be created if it is a valid blobname, that is no
+    /// subdirectory is used and [BlobObject::sanitise_name] does not
+    /// modify the filename.
+    ///
+    /// # Errors
+    ///
+    /// This merely delegates to the [BlobObject::create_and_copy] and
+    /// the [BlobObject::from_path] methods.  See those for possible
+    /// errors.
+    pub fn create_from_path(
+        context: &Context,
+        src: impl AsRef<Path>,
+    ) -> std::result::Result<BlobObject, BlobError> {
+        match src.as_ref().starts_with(context.get_blobdir()) {
+            true => BlobObject::from_path(context, src),
+            false => BlobObject::create_and_copy(context, src),
+        }
+    }
+
+    /// Returns a [BlobObject] for an existing blob from a path.
+    ///
+    /// The path must designate a file directly in the blobdir and
+    /// must use a valid blob name.  That is after sanitisation the
+    /// name must still be the same, that means it must be valid UTF-8
+    /// and not have any special characters in it.
+    ///
+    /// # Errors
+    ///
+    /// [BlobErrorKind::WrongBlobdir] is used if the path is not in
+    /// the blob directory.
+    ///
+    /// [BlobErrorKind::WrongName] is used if the file name does not
+    /// remain identical after sanitisation.
+    pub fn from_path(
+        context: &Context,
+        path: impl AsRef<Path>,
+    ) -> std::result::Result<BlobObject, BlobError> {
+        let rel_path = path
+            .as_ref()
+            .strip_prefix(context.get_blobdir())
+            .map_err(|_| BlobError::new_wrong_blobdir(context.get_blobdir(), path.as_ref()))?;
+        if !BlobObject::is_acceptible_blob_name(&rel_path) {
+            return Err(BlobError::new_wrong_name(path.as_ref()));
+        }
+        let name = rel_path
+            .to_str()
+            .ok_or_else(|| BlobError::new_wrong_name(path.as_ref()))?;
+        BlobObject::from_name(context, name.to_string())
+    }
+
+    /// Returns a [BlobObject] for an existing blob.
+    ///
+    /// The `name` may optionally be prefixed with the `$BLOBDIR/`
+    /// prefixed, as returned by [BlobObject::as_name].  This is how
+    /// you want to create a [BlobObject] for a filename read from the
+    /// database.
+    ///
+    /// # Errors
+    ///
+    /// [BlobErrorKind::WrongName] is used if the name is not a valid
+    /// blobname, i.e. if [BlobObject::sanitise_name] does modify the
+    /// provided name.
+    pub fn from_name(
+        context: &'a Context,
+        name: String,
+    ) -> std::result::Result<BlobObject<'a>, BlobError> {
+        let name: String = match name.starts_with("$BLOBDIR/") {
+            true => name.splitn(2, '/').last().unwrap().to_string(),
+            false => name,
+        };
+        if !BlobObject::is_acceptible_blob_name(&name) {
+            return Err(BlobError::new_wrong_name(name));
+        }
+        Ok(BlobObject {
+            blobdir: context.get_blobdir(),
+            name: format!("$BLOBDIR/{}", name),
+        })
+    }
+
+    /// Returns the absolute path to the blob in the filesystem.
+    pub fn to_abs_path(&self) -> PathBuf {
+        let fname = Path::new(&self.name).strip_prefix("$BLOBDIR/").unwrap();
+        self.blobdir.join(fname)
+    }
+
+    /// Returns the blob name, as stored in the database.
+    ///
+    /// This returns the blob in the `$BLOBDIR/<name>` format used in
+    /// the database.  Do not use this unless you're about to store
+    /// this string in the database or [Params].  Eventually even
+    /// those conversions should be handled by the type system.
+    ///
+    /// [Params]: crate::param::Params
+    pub fn as_name(&self) -> &str {
+        &self.name
+    }
+
+    /// Returns the filename of the blob.
+    pub fn as_file_name(&self) -> &str {
+        self.name.rsplitn(2, '/').next().unwrap()
+    }
+
+    /// The path relative in the blob directory.
+    pub fn as_rel_path(&self) -> &Path {
+        Path::new(self.as_file_name())
+    }
+
+    /// Returns the extension of the blob.
+    ///
+    /// If a blob's filename has an extension, it is always guaranteed
+    /// to be lowercase.
+    pub fn suffix(&self) -> Option<&str> {
+        let ext = self.name.rsplitn(2, '.').next();
+        if ext == Some(&self.name) {
+            None
+        } else {
+            ext
+        }
+    }
+
+    /// Create a safe name based on a messy input string.
+    ///
+    /// The safe name will be a valid filename on Unix and Windows and
+    /// not contain any path separators.  The input can contain path
+    /// segments separated by either Unix or Windows path separators,
+    /// the rightmost non-empty segment will be used as name,
+    /// sanitised for special characters.
+    ///
+    /// The resulting name is returned as a tuple, the first part
+    /// being the stem or basename and the second being an extension,
+    /// including the dot.  E.g. "foo.txt" is returned as `("foo",
+    /// ".txt")` while "bar" is returned as `("bar", "")`.
+    ///
+    /// The extension part will always be lowercased.
+    fn sanitise_name(name: &str) -> (String, String) {
+        let mut name = name.to_string();
+        for part in name.rsplit('/') {
+            if part.len() > 0 {
+                name = part.to_string();
+                break;
+            }
+        }
+        for part in name.rsplit('\\') {
+            if part.len() > 0 {
+                name = part.to_string();
+                break;
+            }
+        }
+        let opts = sanitize_filename::Options {
+            truncate: true,
+            windows: true,
+            replacement: "",
+        };
+
+        let clean = sanitize_filename::sanitize_with_options(name, opts);
+        let mut iter = clean.rsplitn(2, '.');
+        let mut ext = iter.next().unwrap_or_default().to_string();
+        let mut stem = iter.next().unwrap_or_default().to_string();
+        ext.truncate(32);
+        stem.truncate(64);
+        match stem.len() {
+            0 => (ext, "".to_string()),
+            _ => (stem, format!(".{}", ext).to_lowercase()),
+        }
+    }
+
+    /// Checks whether a name is a valid blob name.
+    ///
+    /// This is slightly less strict than stanitise_name, presumably
+    /// someone already created a file with such a name so we just
+    /// ensure it's not actually a path in disguise is actually utf-8.
+    fn is_acceptible_blob_name(name: impl AsRef<OsStr>) -> bool {
+        let uname = match name.as_ref().to_str() {
+            Some(name) => name,
+            None => return false,
+        };
+        if uname.find('/').is_some() {
+            return false;
+        }
+        if uname.find('\\').is_some() {
+            return false;
+        }
+        if uname.find('\0').is_some() {
+            return false;
+        }
+        true
+    }
+}
+
+impl<'a> fmt::Display for BlobObject<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "$BLOBDIR/{}", self.name)
+    }
+}
+
+/// Errors for the [BlobObject].
+///
+/// To keep the return type small and thus the happy path fast this
+/// stores everything on the heap.
+#[derive(Debug)]
+pub struct BlobError {
+    inner: Box<BlobErrorInner>,
+}
+
+#[derive(Debug)]
+struct BlobErrorInner {
+    kind: BlobErrorKind,
+    data: BlobErrorData,
+    backtrace: failure::Backtrace,
+}
+
+/// Error kind for [BlobError].
+///
+/// Each error kind has associated data in the [BlobErrorData].
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum BlobErrorKind {
+    /// Failed to create the blob.
+    CreateFailure,
+    /// Failed to write data to blob.
+    WriteFailure,
+    /// Failed to copy data to blob.
+    CopyFailure,
+    /// Blob is not in the blobdir.
+    WrongBlobdir,
+    /// Blob has a bad name.
+    ///
+    /// E.g. the name is not sanitised correctly or contains a
+    /// sub-directory.
+    WrongName,
+}
+
+/// Associated data for each [BlobError] error kind.
+///
+/// This is not stored directly on the [BlobErrorKind] so that the
+/// kind can stay trivially Copy and Eq.  It is however possible to
+/// create a [BlobError] with mismatching [BlobErrorKind] and
+/// [BlobErrorData], don't do that.
+///
+/// Any blobname stored here is the bare name, without the `$BLOBDIR`
+/// prefix.  All data is owned so that errors do not need to be tied
+/// to any lifetimes.
+#[derive(Debug)]
+enum BlobErrorData {
+    CreateFailure {
+        blobdir: PathBuf,
+        blobname: String,
+        cause: failure::Error,
+    },
+    WriteFailure {
+        blobdir: PathBuf,
+        blobname: String,
+        cause: failure::Error,
+    },
+    CopyFailure {
+        blobdir: PathBuf,
+        blobname: String,
+        src: PathBuf,
+        cause: failure::Error,
+    },
+    WrongBlobdir {
+        blobdir: PathBuf,
+        src: PathBuf,
+    },
+    WrongName {
+        blobname: PathBuf,
+    },
+}
+
+impl BlobError {
+    pub fn kind(&self) -> BlobErrorKind {
+        self.inner.kind
+    }
+
+    fn new_create_failure(
+        blobdir: impl Into<PathBuf>,
+        blobname: impl Into<String>,
+        cause: impl Into<failure::Error>,
+    ) -> BlobError {
+        BlobError {
+            inner: Box::new(BlobErrorInner {
+                kind: BlobErrorKind::CreateFailure,
+                data: BlobErrorData::CreateFailure {
+                    blobdir: blobdir.into(),
+                    blobname: blobname.into(),
+                    cause: cause.into(),
+                },
+                backtrace: failure::Backtrace::new(),
+            }),
+        }
+    }
+
+    fn new_write_failure(
+        blobdir: impl Into<PathBuf>,
+        blobname: impl Into<String>,
+        cause: impl Into<failure::Error>,
+    ) -> BlobError {
+        BlobError {
+            inner: Box::new(BlobErrorInner {
+                kind: BlobErrorKind::WriteFailure,
+                data: BlobErrorData::WriteFailure {
+                    blobdir: blobdir.into(),
+                    blobname: blobname.into(),
+                    cause: cause.into(),
+                },
+                backtrace: failure::Backtrace::new(),
+            }),
+        }
+    }
+
+    fn new_copy_failure(
+        blobdir: impl Into<PathBuf>,
+        blobname: impl Into<String>,
+        src: impl Into<PathBuf>,
+        cause: impl Into<failure::Error>,
+    ) -> BlobError {
+        BlobError {
+            inner: Box::new(BlobErrorInner {
+                kind: BlobErrorKind::CopyFailure,
+                data: BlobErrorData::CopyFailure {
+                    blobdir: blobdir.into(),
+                    blobname: blobname.into(),
+                    src: src.into(),
+                    cause: cause.into(),
+                },
+                backtrace: failure::Backtrace::new(),
+            }),
+        }
+    }
+
+    fn new_wrong_blobdir(blobdir: impl Into<PathBuf>, src: impl Into<PathBuf>) -> BlobError {
+        BlobError {
+            inner: Box::new(BlobErrorInner {
+                kind: BlobErrorKind::WrongBlobdir,
+                data: BlobErrorData::WrongBlobdir {
+                    blobdir: blobdir.into(),
+                    src: src.into(),
+                },
+                backtrace: failure::Backtrace::new(),
+            }),
+        }
+    }
+
+    fn new_wrong_name(blobname: impl Into<PathBuf>) -> BlobError {
+        BlobError {
+            inner: Box::new(BlobErrorInner {
+                kind: BlobErrorKind::WrongName,
+                data: BlobErrorData::WrongName {
+                    blobname: blobname.into(),
+                },
+                backtrace: failure::Backtrace::new(),
+            }),
+        }
+    }
+}
+
+impl fmt::Display for BlobError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        // Match on the data rather than kind, they are equivalent for
+        // identifying purposes but contain the actual data we need.
+        match &self.inner.data {
+            BlobErrorData::CreateFailure {
+                blobdir, blobname, ..
+            } => write!(
+                f,
+                "Failed to create blob {} in {}",
+                blobname,
+                blobdir.display()
+            ),
+            BlobErrorData::WriteFailure {
+                blobdir, blobname, ..
+            } => write!(
+                f,
+                "Failed to write data to blob {} in {}",
+                blobname,
+                blobdir.display()
+            ),
+            BlobErrorData::CopyFailure {
+                blobdir,
+                blobname,
+                src,
+                ..
+            } => write!(
+                f,
+                "Failed to copy data from {} to blob {} in {}",
+                src.display(),
+                blobname,
+                blobdir.display(),
+            ),
+            BlobErrorData::WrongBlobdir { blobdir, src } => write!(
+                f,
+                "File path {} is not in blobdir {}",
+                src.display(),
+                blobdir.display(),
+            ),
+            BlobErrorData::WrongName { blobname } => {
+                write!(f, "Blob has a bad name: {}", blobname.display(),)
+            }
+        }
+    }
+}
+
+impl failure::Fail for BlobError {
+    fn cause(&self) -> Option<&dyn failure::Fail> {
+        match &self.inner.data {
+            BlobErrorData::CreateFailure { cause, .. }
+            | BlobErrorData::WriteFailure { cause, .. }
+            | BlobErrorData::CopyFailure { cause, .. } => Some(cause.as_fail()),
+            _ => None,
+        }
+    }
+
+    fn backtrace(&self) -> Option<&failure::Backtrace> {
+        Some(&self.inner.backtrace)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use crate::test_utils::*;
+
+    #[test]
+    fn test_create() {
+        let t = dummy_context();
+        let blob = BlobObject::create(&t.ctx, "foo", b"hello").unwrap();
+        let fname = t.ctx.get_blobdir().join("foo");
+        let data = fs::read(fname).unwrap();
+        assert_eq!(data, b"hello");
+        assert_eq!(blob.as_name(), "$BLOBDIR/foo");
+        assert_eq!(blob.to_abs_path(), t.ctx.get_blobdir().join("foo"));
+    }
+
+    #[test]
+    fn test_lowercase_ext() {
+        let t = dummy_context();
+        let blob = BlobObject::create(&t.ctx, "foo.TXT", b"hello").unwrap();
+        assert_eq!(blob.as_name(), "$BLOBDIR/foo.txt");
+    }
+
+    #[test]
+    fn test_as_file_name() {
+        let t = dummy_context();
+        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
+        assert_eq!(blob.as_file_name(), "foo.txt");
+    }
+
+    #[test]
+    fn test_as_rel_path() {
+        let t = dummy_context();
+        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
+        assert_eq!(blob.as_rel_path(), Path::new("foo.txt"));
+    }
+
+    #[test]
+    fn test_suffix() {
+        let t = dummy_context();
+        let foo = BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
+        assert_eq!(foo.suffix(), Some("txt"));
+        let bar = BlobObject::create(&t.ctx, "bar", b"world").unwrap();
+        assert_eq!(bar.suffix(), None);
+    }
+
+    #[test]
+    fn test_create_dup() {
+        let t = dummy_context();
+        BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
+        let foo = t.ctx.get_blobdir().join("foo.txt");
+        assert!(foo.exists());
+        BlobObject::create(&t.ctx, "foo.txt", b"world").unwrap();
+        for dirent in fs::read_dir(t.ctx.get_blobdir()).unwrap() {
+            let fname = dirent.unwrap().file_name();
+            if fname == foo.file_name().unwrap() {
+                assert_eq!(fs::read(&foo).unwrap(), b"hello");
+            } else {
+                let name = fname.to_str().unwrap();
+                assert!(name.starts_with("foo"));
+                assert!(name.ends_with(".txt"));
+            }
+        }
+    }
+
+    #[test]
+    fn test_create_long_names() {
+        let t = dummy_context();
+        let s = "1".repeat(150);
+        let blob = BlobObject::create(&t.ctx, &s, b"data").unwrap();
+        let blobname = blob.as_name().split('/').last().unwrap();
+        assert!(blobname.len() < 128);
+    }
+
+    #[test]
+    fn test_create_and_copy() {
+        let t = dummy_context();
+        let src = t.dir.path().join("src");
+        fs::write(&src, b"boo").unwrap();
+        let blob = BlobObject::create_and_copy(&t.ctx, &src).unwrap();
+        assert_eq!(blob.as_name(), "$BLOBDIR/src");
+        let data = fs::read(blob.to_abs_path()).unwrap();
+        assert_eq!(data, b"boo");
+
+        let whoops = t.dir.path().join("whoops");
+        assert!(BlobObject::create_and_copy(&t.ctx, &whoops).is_err());
+        let whoops = t.ctx.get_blobdir().join("whoops");
+        assert!(!whoops.exists());
+    }
+
+    #[test]
+    fn test_create_from_path() {
+        let t = dummy_context();
+
+        let src_ext = t.dir.path().join("external");
+        fs::write(&src_ext, b"boo").unwrap();
+        let blob = BlobObject::create_from_path(&t.ctx, &src_ext).unwrap();
+        assert_eq!(blob.as_name(), "$BLOBDIR/external");
+        let data = fs::read(blob.to_abs_path()).unwrap();
+        assert_eq!(data, b"boo");
+
+        let src_int = t.ctx.get_blobdir().join("internal");
+        fs::write(&src_int, b"boo").unwrap();
+        let blob = BlobObject::create_from_path(&t.ctx, &src_int).unwrap();
+        assert_eq!(blob.as_name(), "$BLOBDIR/internal");
+        let data = fs::read(blob.to_abs_path()).unwrap();
+        assert_eq!(data, b"boo");
+    }
+    #[test]
+    fn test_create_from_name_long() {
+        let t = dummy_context();
+        let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
+        fs::write(&src_ext, b"boo").unwrap();
+        let blob = BlobObject::create_from_path(&t.ctx, &src_ext).unwrap();
+        assert_eq!(
+            blob.as_name(),
+            "$BLOBDIR/autocrypt-setup-message-4137848473.html"
+        );
+    }
+
+    #[test]
+    fn test_is_blob_name() {
+        assert!(BlobObject::is_acceptible_blob_name("foo"));
+        assert!(BlobObject::is_acceptible_blob_name("foo.txt"));
+        assert!(BlobObject::is_acceptible_blob_name("f".repeat(128)));
+        assert!(!BlobObject::is_acceptible_blob_name("foo/bar"));
+        assert!(!BlobObject::is_acceptible_blob_name("foo\\bar"));
+        assert!(!BlobObject::is_acceptible_blob_name("foo\x00bar"));
+    }
+}

src/chatlist.rs

@@ -4,7 +4,7 @@ use crate::contact::*;
 use crate::context::*;
 use crate::error::Result;
 use crate::lot::Lot;
-use crate::message::Message;
+use crate::message::{Message, MsgId};
 use crate::stock::StockMessage;
 
 /// An object representing a single chatlist in memory.
@@ -34,7 +34,7 @@ use crate::stock::StockMessage;
 #[derive(Debug)]
 pub struct Chatlist {
     /// Stores pairs of `chat_id, message_id`
-    ids: Vec<(u32, u32)>,
+    ids: Vec<(u32, MsgId)>,
 }
 
 impl Chatlist {
@@ -86,25 +86,12 @@ impl Chatlist {
         query: Option<&str>,
         query_contact_id: Option<u32>,
     ) -> Result<Self> {
-        let mut add_archived_link_item = 0;
-
-        // select with left join and minimum:
-        // - the inner select must use `hidden` and _not_ `m.hidden`
-        //   which would refer the outer select and take a lot of time
-        // - `GROUP BY` is needed several messages may have the same timestamp
-        // - the list starts with the newest chats
-        // nb: the query currently shows messages from blocked contacts in groups.
-        // however, for normal-groups, this is okay as the message is also returned by dc_get_chat_msgs()
-        // (otherwise it would be hard to follow conversations, wa and tg do the same)
-        // for the deaddrop, however, they should really be hidden, however, _currently_ the deaddrop is not
-        // shown at all permanent in the chatlist.
+        let mut add_archived_link_item = false;
 
         let process_row = |row: &rusqlite::Row| {
-            let chat_id: i32 = row.get(0)?;
-            // TODO: verify that it is okay for this to be Null
-            let msg_id: i32 = row.get(1).unwrap_or_default();
-
-            Ok((chat_id as u32, msg_id as u32))
+            let chat_id: u32 = row.get(0)?;
+            let msg_id: MsgId = row.get(1).unwrap_or_default();
+            Ok((chat_id, msg_id))
         };
 
         let process_rows = |rows: rusqlite::MappedRows<_>| {
@@ -112,36 +99,63 @@ impl Chatlist {
                 .map_err(Into::into)
         };
 
-        // nb: the query currently shows messages from blocked contacts in groups.
-        // however, for normal-groups, this is okay as the message is also returned by dc_get_chat_msgs()
-        // (otherwise it would be hard to follow conversations, wa and tg do the same)
-        // for the deaddrop, however, they should really be hidden, however, _currently_ the deaddrop is not
+        // select with left join and minimum:
+        //
+        // - the inner select must use `hidden` and _not_ `m.hidden`
+        //   which would refer the outer select and take a lot of time
+        // - `GROUP BY` is needed several messages may have the same
+        //   timestamp
+        // - the list starts with the newest chats
+        //
+        // nb: the query currently shows messages from blocked
+        // contacts in groups.  however, for normal-groups, this is
+        // okay as the message is also returned by dc_get_chat_msgs()
+        // (otherwise it would be hard to follow conversations, wa and
+        // tg do the same) for the deaddrop, however, they should
+        // really be hidden, however, _currently_ the deaddrop is not
         // shown at all permanent in the chatlist.
-
         let mut ids = if let Some(query_contact_id) = query_contact_id {
             // show chats shared with a given contact
             context.sql.query_map(
-            "SELECT c.id, m.id FROM chats c  LEFT JOIN msgs m         \
-             ON c.id=m.chat_id        \
-             AND m.timestamp=( SELECT MAX(timestamp)   \
-             FROM msgs  WHERE chat_id=c.id    \
-             AND (hidden=0 OR (hidden=1 AND state=19))) WHERE c.id>9   \
-             AND c.blocked=0 AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)  \
-             GROUP BY c.id  ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;",
-            params![query_contact_id as i32],
-            process_row,
-            process_rows,
-        )?
+                concat!(
+                    "SELECT c.id, m.id",
+                    " FROM chats c",
+                    " LEFT JOIN msgs m",
+                    "        ON c.id=m.chat_id",
+                    "       AND m.timestamp=(",
+                    "               SELECT MAX(timestamp)",
+                    "                 FROM msgs",
+                    "                WHERE chat_id=c.id",
+                    "                  AND hidden=0)",
+                    " WHERE c.id>9",
+                    "   AND c.blocked=0",
+                    "   AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)",
+                    " GROUP BY c.id",
+                    " ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;"
+                ),
+                params![query_contact_id as i32],
+                process_row,
+                process_rows,
+            )?
         } else if 0 != listflags & DC_GCL_ARCHIVED_ONLY {
             // show archived chats
             context.sql.query_map(
-                "SELECT c.id, m.id FROM chats c  LEFT JOIN msgs m         \
-                 ON c.id=m.chat_id        \
-                 AND m.timestamp=( SELECT MAX(timestamp)   \
-                 FROM msgs  WHERE chat_id=c.id    \
-                 AND (hidden=0 OR (hidden=1 AND state=19))) WHERE c.id>9   \
-                 AND c.blocked=0 AND c.archived=1  GROUP BY c.id  \
-                 ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;",
+                concat!(
+                    "SELECT c.id, m.id",
+                    " FROM chats c",
+                    " LEFT JOIN msgs m",
+                    "        ON c.id=m.chat_id",
+                    "       AND m.timestamp=(",
+                    "               SELECT MAX(timestamp)",
+                    "                 FROM msgs",
+                    "                WHERE chat_id=c.id",
+                    "                  AND hidden=0)",
+                    " WHERE c.id>9",
+                    "   AND c.blocked=0",
+                    "   AND c.archived=1",
+                    " GROUP BY c.id",
+                    " ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;"
+                ),
                 params![],
                 process_row,
                 process_rows,
@@ -152,13 +166,22 @@ impl Chatlist {
 
             let str_like_cmd = format!("%{}%", query);
             context.sql.query_map(
-                "SELECT c.id, m.id FROM chats c  LEFT JOIN msgs m         \
-                 ON c.id=m.chat_id        \
-                 AND m.timestamp=( SELECT MAX(timestamp)   \
-                 FROM msgs  WHERE chat_id=c.id    \
-                 AND (hidden=0 OR (hidden=1 AND state=19))) WHERE c.id>9   \
-                 AND c.blocked=0 AND c.name LIKE ?  \
-                 GROUP BY c.id  ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;",
+                concat!(
+                    "SELECT c.id, m.id",
+                    " FROM chats c",
+                    " LEFT JOIN msgs m",
+                    "        ON c.id=m.chat_id",
+                    "       AND m.timestamp=(",
+                    "               SELECT MAX(timestamp)",
+                    "                 FROM msgs",
+                    "                WHERE chat_id=c.id",
+                    "                  AND hidden=0)",
+                    " WHERE c.id>9",
+                    "   AND c.blocked=0",
+                    "   AND c.name LIKE ?",
+                    " GROUP BY c.id",
+                    " ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;"
+                ),
                 params![str_like_cmd],
                 process_row,
                 process_rows,
@@ -166,34 +189,40 @@ impl Chatlist {
         } else {
             //  show normal chatlist
             let mut ids = context.sql.query_map(
-                "SELECT c.id, m.id FROM chats c  \
-                 LEFT JOIN msgs m         \
-                 ON c.id=m.chat_id        \
-                 AND m.timestamp=( SELECT MAX(timestamp)   \
-                 FROM msgs  WHERE chat_id=c.id    \
-                 AND (hidden=0 OR (hidden=1 AND state=19))) WHERE c.id>9   \
-                 AND c.blocked=0 AND c.archived=0  \
-                 GROUP BY c.id  \
-                 ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;",
+                concat!(
+                    "SELECT c.id, m.id",
+                    " FROM chats c",
+                    " LEFT JOIN msgs m",
+                    "        ON c.id=m.chat_id",
+                    "       AND m.timestamp=(",
+                    "               SELECT MAX(timestamp)",
+                    "                 FROM msgs",
+                    "                WHERE chat_id=c.id",
+                    "                  AND hidden=0)",
+                    " WHERE c.id>9",
+                    "   AND c.blocked=0",
+                    "   AND c.archived=0",
+                    " GROUP BY c.id",
+                    " ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;"
+                ),
                 params![],
                 process_row,
                 process_rows,
             )?;
             if 0 == listflags & DC_GCL_NO_SPECIALS {
-                let last_deaddrop_fresh_msg_id = get_last_deaddrop_fresh_msg(context);
-                if last_deaddrop_fresh_msg_id > 0 {
-                    ids.push((1, last_deaddrop_fresh_msg_id));
+                if let Some(last_deaddrop_fresh_msg_id) = get_last_deaddrop_fresh_msg(context) {
+                    ids.insert(0, (DC_CHAT_ID_DEADDROP, last_deaddrop_fresh_msg_id));
                 }
-                add_archived_link_item = 1;
+                add_archived_link_item = true;
             }
             ids
         };
 
-        if 0 != add_archived_link_item && dc_get_archived_cnt(context) > 0 {
+        if add_archived_link_item && dc_get_archived_cnt(context) > 0 {
             if ids.is_empty() && 0 != listflags & DC_GCL_ADD_ALLDONE_HINT {
-                ids.push((DC_CHAT_ID_ALLDONE_HINT, 0));
+                ids.push((DC_CHAT_ID_ALLDONE_HINT, MsgId::new(0)));
             }
-            ids.push((DC_CHAT_ID_ARCHIVED_LINK, 0));
+            ids.push((DC_CHAT_ID_ARCHIVED_LINK, MsgId::new(0)));
         }
 
         Ok(Chatlist { ids })
@@ -221,12 +250,9 @@ impl Chatlist {
     /// Get a single message ID of a chatlist.
     ///
     /// To get the message object from the message ID, use dc_get_msg().
-    pub fn get_msg_id(&self, index: usize) -> u32 {
-        if index >= self.ids.len() {
-            return 0;
-        }
-
-        self.ids[index].1
+    pub fn get_msg_id(&self, index: usize) -> Result<MsgId> {
+        ensure!(index < self.ids.len(), "Chatlist index out of range");
+        Ok(self.ids[index].1)
     }
 
     /// Get a summary for a chatlist index.
@@ -258,34 +284,38 @@ impl Chatlist {
         let chat_loaded: Chat;
         let chat = if let Some(chat) = chat {
             chat
+        } else if let Ok(chat) = Chat::load_from_db(context, self.ids[index].0) {
+            chat_loaded = chat;
+            &chat_loaded
         } else {
-            if let Ok(chat) = Chat::load_from_db(context, self.ids[index].0) {
-                chat_loaded = chat;
-                &chat_loaded
-            } else {
-                return ret;
-            }
+            return ret;
         };
 
         let lastmsg_id = self.ids[index].1;
         let mut lastcontact = None;
 
-        let lastmsg = if 0 != lastmsg_id {
-            if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id) {
-                if lastmsg.from_id != 1 as libc::c_uint
-                    && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup)
-                {
-                    lastcontact = Contact::load_from_db(context, lastmsg.from_id).ok();
-                }
-
-                Some(lastmsg)
-            } else {
-                None
+        let mut lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id) {
+            if lastmsg.from_id != DC_CONTACT_ID_SELF
+                && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup)
+            {
+                lastcontact = Contact::load_from_db(context, lastmsg.from_id).ok();
             }
+
+            Some(lastmsg)
         } else {
             None
         };
 
+        if let Ok(draft) = get_draft(context, chat.id) {
+            if draft.is_some()
+                && (lastmsg.is_none()
+                    || draft.as_ref().unwrap().timestamp_sort
+                        > lastmsg.as_ref().unwrap().timestamp_sort)
+            {
+                lastmsg = draft;
+            }
+        }
+
         if chat.id == DC_CHAT_ID_ARCHIVED_LINK {
             ret.text2 = None;
         } else if lastmsg.is_none() || lastmsg.as_ref().unwrap().from_id == DC_CONTACT_ID_UNDEFINED
@@ -310,19 +340,21 @@ pub fn dc_get_archived_cnt(context: &Context) -> u32 {
         .unwrap_or_default()
 }
 
-fn get_last_deaddrop_fresh_msg(context: &Context) -> u32 {
-    // We have an index over the state-column, this should be sufficient as there are typically
-    // only few fresh messages.
-    context
-        .sql
-        .query_get_value(
-            context,
-            "SELECT m.id  FROM msgs m  LEFT JOIN chats c ON c.id=m.chat_id  \
-             WHERE m.state=10   \
-             AND m.hidden=0    \
-             AND c.blocked=2 \
-             ORDER BY m.timestamp DESC, m.id DESC;",
-            params![],
-        )
-        .unwrap_or_default()
+fn get_last_deaddrop_fresh_msg(context: &Context) -> Option<MsgId> {
+    // We have an index over the state-column, this should be
+    // sufficient as there are typically only few fresh messages.
+    context.sql.query_get_value(
+        context,
+        concat!(
+            "SELECT m.id",
+            " FROM msgs m",
+            " LEFT JOIN chats c",
+            "        ON c.id=m.chat_id",
+            " WHERE m.state=10",
+            "   AND m.hidden=0",
+            "   AND c.blocked=2",
+            " ORDER BY m.timestamp DESC, m.id DESC;"
+        ),
+        params![],
+    )
 }

src/config.rs

@@ -19,10 +19,12 @@ pub enum Config {
     MailUser,
     MailPw,
     MailPort,
+    ImapCertificateChecks,
     SendServer,
     SendUser,
     SendPw,
     SendPort,
+    SmtpCertificateChecks,
     ServerFlags,
     #[strum(props(default = "INBOX"))]
     ImapFolder,
@@ -30,6 +32,8 @@ pub enum Config {
     Selfstatus,
     Selfavatar,
     #[strum(props(default = "1"))]
+    BccSelf,
+    #[strum(props(default = "1"))]
     E2eeEnabled,
     #[strum(props(default = "1"))]
     MdnsEnabled,
@@ -41,7 +45,7 @@ pub enum Config {
     MvboxWatch,
     #[strum(props(default = "1"))]
     MvboxMove,
-    #[strum(props(default = "0"))]
+    #[strum(props(default = "0"))] // also change ShowEmails.default() on changes
     ShowEmails,
     SaveMimeHeaders,
     ConfiguredAddr,
@@ -50,10 +54,12 @@ pub enum Config {
     ConfiguredMailPw,
     ConfiguredMailPort,
     ConfiguredMailSecurity,
+    ConfiguredImapCertificateChecks,
     ConfiguredSendServer,
     ConfiguredSendUser,
     ConfiguredSendPw,
     ConfiguredSendPort,
+    ConfiguredSmtpCertificateChecks,
     ConfiguredServerFlags,
     ConfiguredSendSecurity,
     ConfiguredE2EEEnabled,
@@ -72,13 +78,13 @@ impl Context {
     pub fn get_config(&self, key: Config) -> Option<String> {
         let value = match key {
             Config::Selfavatar => {
-                let rel_path = self.sql.get_config(self, key);
-                rel_path.map(|p| dc_get_abs_path(self, &p).to_str().unwrap().to_string())
+                let rel_path = self.sql.get_raw_config(self, key);
+                rel_path.map(|p| dc_get_abs_path(self, &p).to_string_lossy().into_owned())
             }
             Config::SysVersion => Some((&*DC_VERSION_STR).clone()),
             Config::SysMsgsizeMaxRecommended => Some(format!("{}", 24 * 1024 * 1024 / 4 * 3)),
             Config::SysConfigKeys => Some(get_config_keys_string()),
-            _ => self.sql.get_config(self, key),
+            _ => self.sql.get_raw_config(self, key),
         };
 
         if value.is_some() {
@@ -92,6 +98,16 @@ impl Context {
         }
     }
 
+    pub fn get_config_int(&self, key: Config) -> i32 {
+        self.get_config(key)
+            .and_then(|s| s.parse().ok())
+            .unwrap_or_default()
+    }
+
+    pub fn get_config_bool(&self, key: Config) -> bool {
+        self.get_config_int(key) != 0
+    }
+
     /// Set the given config key.
     /// If `None` is passed as a value the value is cleared and set to the default if there is one.
     pub fn set_config(&self, key: Config, value: Option<&str>) -> Result<(), Error> {
@@ -99,20 +115,20 @@ impl Context {
             Config::Selfavatar if value.is_some() => {
                 let rel_path = std::fs::canonicalize(value.unwrap())?;
                 self.sql
-                    .set_config(self, key, Some(&rel_path.to_string_lossy()))
+                    .set_raw_config(self, key, Some(&rel_path.to_string_lossy()))
             }
             Config::InboxWatch => {
-                let ret = self.sql.set_config(self, key, value);
+                let ret = self.sql.set_raw_config(self, key, value);
                 interrupt_imap_idle(self);
                 ret
             }
             Config::SentboxWatch => {
-                let ret = self.sql.set_config(self, key, value);
+                let ret = self.sql.set_raw_config(self, key, value);
                 interrupt_sentbox_idle(self);
                 ret
             }
             Config::MvboxWatch => {
-                let ret = self.sql.set_config(self, key, value);
+                let ret = self.sql.set_raw_config(self, key, value);
                 interrupt_mvbox_idle(self);
                 ret
             }
@@ -124,9 +140,9 @@ impl Context {
                     value
                 };
 
-                self.sql.set_config(self, key, val)
+                self.sql.set_raw_config(self, key, val)
             }
-            _ => self.sql.set_config(self, key, value),
+            _ => self.sql.set_raw_config(self, key, value),
         }
     }
 }

src/configure/auto_mozilla.rs

@@ -1,10 +1,9 @@
-use libc::free;
 use quick_xml;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
 use crate::constants::*;
 use crate::context::Context;
-use crate::dc_tools::*;
+use crate::error::Error;
 use crate::login_param::LoginParam;
 
 use super::read_autoconf_file;
@@ -12,52 +11,56 @@ use super::read_autoconf_file;
  * Thunderbird's Autoconfigure
  ******************************************************************************/
 /* documentation: https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration */
-#[repr(C)]
-struct moz_autoconfigure_t<'a> {
-    pub in_0: &'a LoginParam,
+struct MozAutoconfigure<'a> {
+    pub in_emailaddr: &'a str,
     pub in_emaildomain: &'a str,
     pub in_emaillocalpart: &'a str,
     pub out: LoginParam,
-    pub out_imap_set: libc::c_int,
-    pub out_smtp_set: libc::c_int,
-    pub tag_server: libc::c_int,
-    pub tag_config: libc::c_int,
+    pub out_imap_set: bool,
+    pub out_smtp_set: bool,
+    pub tag_server: MozServer,
+    pub tag_config: MozConfigTag,
 }
 
-pub unsafe fn moz_autoconfigure(
-    context: &Context,
-    url: &str,
-    param_in: &LoginParam,
-) -> Option<LoginParam> {
-    let xml_raw = read_autoconf_file(context, url);
-    if xml_raw.is_null() {
-        return None;
-    }
+#[derive(PartialEq)]
+enum MozServer {
+    Undefined,
+    Imap,
+    Smtp,
+}
 
-    // Split address into local part and domain part.
-    let p = param_in.addr.find("@");
-    if p.is_none() {
-        free(xml_raw as *mut libc::c_void);
-        return None;
-    }
-    let (in_emaillocalpart, in_emaildomain) = param_in.addr.split_at(p.unwrap());
-    let in_emaildomain = &in_emaildomain[1..];
+enum MozConfigTag {
+    Undefined,
+    Hostname,
+    Port,
+    Sockettype,
+    Username,
+}
 
-    let mut reader = quick_xml::Reader::from_str(as_str(xml_raw));
+pub fn moz_parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result<LoginParam, Error> {
+    let mut reader = quick_xml::Reader::from_str(xml_raw);
     reader.trim_text(true);
 
-    let mut buf = Vec::new();
+    // Split address into local part and domain part.
+    let p = match in_emailaddr.find('@') {
+        Some(i) => i,
+        None => bail!("Email address {} does not contain @", in_emailaddr),
+    };
+    let (in_emaillocalpart, in_emaildomain) = in_emailaddr.split_at(p);
+    let in_emaildomain = &in_emaildomain[1..];
 
-    let mut moz_ac = moz_autoconfigure_t {
-        in_0: param_in,
+    let mut moz_ac = MozAutoconfigure {
+        in_emailaddr,
         in_emaildomain,
         in_emaillocalpart,
         out: LoginParam::new(),
-        out_imap_set: 0,
-        out_smtp_set: 0,
-        tag_server: 0,
-        tag_config: 0,
+        out_imap_set: false,
+        out_smtp_set: false,
+        tag_server: MozServer::Undefined,
+        tag_config: MozConfigTag::Undefined,
     };
+
+    let mut buf = Vec::new();
     loop {
         match reader.read_event(&mut buf) {
             Ok(quick_xml::events::Event::Start(ref e)) => {
@@ -68,8 +71,7 @@ pub unsafe fn moz_autoconfigure(
                 moz_autoconfigure_text_cb(e, &mut moz_ac, &reader)
             }
             Err(e) => {
-                error!(
-                    context,
+                bail!(
                     "Configure xml: Error at position {}: {:?}",
                     reader.buffer_position(),
                     e
@@ -87,23 +89,36 @@ pub unsafe fn moz_autoconfigure(
         || moz_ac.out.send_port == 0
     {
         let r = moz_ac.out.to_string();
-        warn!(context, "Bad or incomplete autoconfig: {}", r,);
-        free(xml_raw as *mut libc::c_void);
-        return None;
+        bail!("Bad or incomplete autoconfig: {}", r,);
     }
 
-    free(xml_raw as *mut libc::c_void);
-    Some(moz_ac.out)
+    Ok(moz_ac.out)
+}
+
+pub fn moz_autoconfigure(
+    context: &Context,
+    url: &str,
+    param_in: &LoginParam,
+) -> Option<LoginParam> {
+    let xml_raw = read_autoconf_file(context, url)?;
+
+    match moz_parse_xml(&param_in.addr, &xml_raw) {
+        Err(err) => {
+            warn!(context, "{}", err);
+            None
+        }
+        Ok(lp) => Some(lp),
+    }
 }
 
 fn moz_autoconfigure_text_cb<B: std::io::BufRead>(
     event: &BytesText,
-    moz_ac: &mut moz_autoconfigure_t,
+    moz_ac: &mut MozAutoconfigure,
     reader: &quick_xml::Reader<B>,
 ) {
     let val = event.unescape_and_decode(reader).unwrap_or_default();
 
-    let addr = &moz_ac.in_0.addr;
+    let addr = moz_ac.in_emailaddr;
     let email_local = moz_ac.in_emaillocalpart;
     let email_domain = moz_ac.in_emaildomain;
 
@@ -113,12 +128,12 @@ fn moz_autoconfigure_text_cb<B: std::io::BufRead>(
         .replace("%EMAILLOCALPART%", email_local)
         .replace("%EMAILDOMAIN%", email_domain);
 
-    if moz_ac.tag_server == 1 {
-        match moz_ac.tag_config {
-            10 => moz_ac.out.mail_server = val,
-            11 => moz_ac.out.mail_port = val.parse().unwrap_or_default(),
-            12 => moz_ac.out.mail_user = val,
-            13 => {
+    match moz_ac.tag_server {
+        MozServer::Imap => match moz_ac.tag_config {
+            MozConfigTag::Hostname => moz_ac.out.mail_server = val,
+            MozConfigTag::Port => moz_ac.out.mail_port = val.parse().unwrap_or_default(),
+            MozConfigTag::Username => moz_ac.out.mail_user = val,
+            MozConfigTag::Sockettype => {
                 let val_lower = val.to_lowercase();
                 if val_lower == "ssl" {
                     moz_ac.out.server_flags |= DC_LP_IMAP_SOCKET_SSL as i32
@@ -131,13 +146,12 @@ fn moz_autoconfigure_text_cb<B: std::io::BufRead>(
                 }
             }
             _ => {}
-        }
-    } else if moz_ac.tag_server == 2 {
-        match moz_ac.tag_config {
-            10 => moz_ac.out.send_server = val,
-            11 => moz_ac.out.send_port = val.parse().unwrap_or_default(),
-            12 => moz_ac.out.send_user = val,
-            13 => {
+        },
+        MozServer::Smtp => match moz_ac.tag_config {
+            MozConfigTag::Hostname => moz_ac.out.send_server = val,
+            MozConfigTag::Port => moz_ac.out.send_port = val.parse().unwrap_or_default(),
+            MozConfigTag::Username => moz_ac.out.send_user = val,
+            MozConfigTag::Sockettype => {
                 let val_lower = val.to_lowercase();
                 if val_lower == "ssl" {
                     moz_ac.out.server_flags |= DC_LP_SMTP_SOCKET_SSL as i32
@@ -150,29 +164,34 @@ fn moz_autoconfigure_text_cb<B: std::io::BufRead>(
                 }
             }
             _ => {}
-        }
+        },
+        MozServer::Undefined => {}
     }
 }
 
-fn moz_autoconfigure_endtag_cb(event: &BytesEnd, moz_ac: &mut moz_autoconfigure_t) {
+fn moz_autoconfigure_endtag_cb(event: &BytesEnd, moz_ac: &mut MozAutoconfigure) {
     let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
 
     if tag == "incomingserver" {
-        moz_ac.tag_server = 0;
-        moz_ac.tag_config = 0;
-        moz_ac.out_imap_set = 1;
+        if moz_ac.tag_server == MozServer::Imap {
+            moz_ac.out_imap_set = true;
+        }
+        moz_ac.tag_server = MozServer::Undefined;
+        moz_ac.tag_config = MozConfigTag::Undefined;
     } else if tag == "outgoingserver" {
-        moz_ac.tag_server = 0;
-        moz_ac.tag_config = 0;
-        moz_ac.out_smtp_set = 1;
+        if moz_ac.tag_server == MozServer::Smtp {
+            moz_ac.out_smtp_set = true;
+        }
+        moz_ac.tag_server = MozServer::Undefined;
+        moz_ac.tag_config = MozConfigTag::Undefined;
     } else {
-        moz_ac.tag_config = 0;
+        moz_ac.tag_config = MozConfigTag::Undefined;
     }
 }
 
 fn moz_autoconfigure_starttag_cb<B: std::io::BufRead>(
     event: &BytesStart,
-    moz_ac: &mut moz_autoconfigure_t,
+    moz_ac: &mut MozAutoconfigure,
     reader: &quick_xml::Reader<B>,
 ) {
     let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
@@ -189,25 +208,115 @@ fn moz_autoconfigure_starttag_cb<B: std::io::BufRead>(
                 .unwrap_or_default()
                 .to_lowercase();
 
-            if typ == "imap" && moz_ac.out_imap_set == 0 {
-                1
+            if typ == "imap" && !moz_ac.out_imap_set {
+                MozServer::Imap
             } else {
-                0
+                MozServer::Undefined
             }
         } else {
-            0
+            MozServer::Undefined
         };
-        moz_ac.tag_config = 0;
+        moz_ac.tag_config = MozConfigTag::Undefined;
     } else if tag == "outgoingserver" {
-        moz_ac.tag_server = if moz_ac.out_smtp_set == 0 { 2 } else { 0 };
-        moz_ac.tag_config = 0;
+        moz_ac.tag_server = if !moz_ac.out_smtp_set {
+            MozServer::Smtp
+        } else {
+            MozServer::Undefined
+        };
+        moz_ac.tag_config = MozConfigTag::Undefined;
     } else if tag == "hostname" {
-        moz_ac.tag_config = 10;
+        moz_ac.tag_config = MozConfigTag::Hostname;
     } else if tag == "port" {
-        moz_ac.tag_config = 11;
+        moz_ac.tag_config = MozConfigTag::Port;
     } else if tag == "sockettype" {
-        moz_ac.tag_config = 13;
+        moz_ac.tag_config = MozConfigTag::Sockettype;
     } else if tag == "username" {
-        moz_ac.tag_config = 12;
+        moz_ac.tag_config = MozConfigTag::Username;
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_outlook_autoconfig() {
+        // Copied from https://autoconfig.thunderbird.net/v1.1/outlook.com on 2019-10-11
+        let xml_raw =
+"<clientConfig version=\"1.1\">
+  <emailProvider id=\"outlook.com\">
+    <domain>hotmail.com</domain>
+    <domain>hotmail.co.uk</domain>
+    <domain>hotmail.co.jp</domain>
+    <domain>hotmail.com.br</domain>
+    <domain>hotmail.de</domain>
+    <domain>hotmail.fr</domain>
+    <domain>hotmail.it</domain>
+    <domain>hotmail.es</domain>
+    <domain>live.com</domain>
+    <domain>live.co.uk</domain>
+    <domain>live.co.jp</domain>
+    <domain>live.de</domain>
+    <domain>live.fr</domain>
+    <domain>live.it</domain>
+    <domain>live.jp</domain>
+    <domain>msn.com</domain>
+    <domain>outlook.com</domain>
+    <displayName>Outlook.com (Microsoft)</displayName>
+    <displayShortName>Outlook</displayShortName>
+    <incomingServer type=\"exchange\">
+      <hostname>outlook.office365.com</hostname>
+      <port>443</port>
+      <username>%EMAILADDRESS%</username>
+      <socketType>SSL</socketType>
+      <authentication>OAuth2</authentication>
+      <owaURL>https://outlook.office365.com/owa/</owaURL>
+      <ewsURL>https://outlook.office365.com/ews/exchange.asmx</ewsURL>
+      <useGlobalPreferredServer>true</useGlobalPreferredServer>
+    </incomingServer>
+    <incomingServer type=\"imap\">
+      <hostname>outlook.office365.com</hostname>
+      <port>993</port>
+      <socketType>SSL</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </incomingServer>
+    <incomingServer type=\"pop3\">
+      <hostname>outlook.office365.com</hostname>
+      <port>995</port>
+      <socketType>SSL</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+      <pop3>
+        <leaveMessagesOnServer>true</leaveMessagesOnServer>
+        <!-- Outlook.com docs specifically mention that POP3 deletes have effect on the main inbox on webmail and IMAP -->
+      </pop3>
+    </incomingServer>
+    <outgoingServer type=\"smtp\">
+      <hostname>smtp.office365.com</hostname>
+      <port>587</port>
+      <socketType>STARTTLS</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </outgoingServer>
+    <documentation url=\"http://windows.microsoft.com/en-US/windows/outlook/send-receive-from-app\">
+      <descr lang=\"en\">Set up an email app with Outlook.com</descr>
+    </documentation>
+  </emailProvider>
+  <webMail>
+    <loginPage url=\"https://www.outlook.com/\"/>
+    <loginPageInfo url=\"https://www.outlook.com/\">
+      <username>%EMAILADDRESS%</username>
+      <usernameField id=\"i0116\" name=\"login\"/>
+      <passwordField id=\"i0118\" name=\"passwd\"/>
+      <loginButton id=\"idSIButton9\" name=\"SI\"/>
+    </loginPageInfo>
+  </webMail>
+</clientConfig>";
+        let res = moz_parse_xml("example@outlook.com", xml_raw).expect("XML parsing failed");
+        assert_eq!(res.mail_server, "outlook.office365.com");
+        assert_eq!(res.mail_port, 993);
+        assert_eq!(res.send_server, "smtp.office365.com");
+        assert_eq!(res.send_port, 587);
     }
 }

src/configure/auto_outlook.rs

@@ -1,214 +1,249 @@
-use std::ptr;
-
-use libc::free;
 use quick_xml;
-use quick_xml::events::{BytesEnd, BytesStart, BytesText};
+use quick_xml::events::BytesEnd;
 
 use crate::constants::*;
 use crate::context::Context;
-use crate::dc_tools::*;
+use crate::error::Error;
 use crate::login_param::LoginParam;
 
 use super::read_autoconf_file;
-/* ******************************************************************************
- * Outlook's Autodiscover
- ******************************************************************************/
-#[repr(C)]
-struct outlk_autodiscover_t<'a> {
-    pub in_0: &'a LoginParam,
+
+/// Outlook's Autodiscover
+struct OutlookAutodiscover {
     pub out: LoginParam,
-    pub out_imap_set: libc::c_int,
-    pub out_smtp_set: libc::c_int,
-    pub tag_config: libc::c_int,
-    pub config: [*mut libc::c_char; 6],
-    pub redirect: *mut libc::c_char,
+    pub out_imap_set: bool,
+    pub out_smtp_set: bool,
+    pub config_type: Option<String>,
+    pub config_server: String,
+    pub config_port: i32,
+    pub config_ssl: String,
+    pub config_redirecturl: Option<String>,
 }
 
-pub unsafe fn outlk_autodiscover(
-    context: &Context,
-    url__: &str,
-    param_in: &LoginParam,
-) -> Option<LoginParam> {
-    let mut xml_raw: *mut libc::c_char = ptr::null_mut();
-    let mut url = url__.strdup();
-    let mut outlk_ad = outlk_autodiscover_t {
-        in_0: param_in,
+enum ParsingResult {
+    LoginParam(LoginParam),
+    RedirectUrl(String),
+}
+
+fn outlk_parse_xml(xml_raw: &str) -> Result<ParsingResult, Error> {
+    let mut outlk_ad = OutlookAutodiscover {
         out: LoginParam::new(),
-        out_imap_set: 0,
-        out_smtp_set: 0,
-        tag_config: 0,
-        config: [ptr::null_mut(); 6],
-        redirect: ptr::null_mut(),
+        out_imap_set: false,
+        out_smtp_set: false,
+        config_type: None,
+        config_server: String::new(),
+        config_port: 0,
+        config_ssl: String::new(),
+        config_redirecturl: None,
     };
-    let ok_to_continue;
-    let mut i = 0;
+
+    let mut reader = quick_xml::Reader::from_str(&xml_raw);
+    reader.trim_text(true);
+
+    let mut buf = Vec::new();
+
+    let mut current_tag: Option<String> = None;
+
     loop {
-        if !(i < 10) {
-            ok_to_continue = true;
-            break;
-        }
-        libc::memset(
-            &mut outlk_ad as *mut outlk_autodiscover_t as *mut libc::c_void,
-            0,
-            ::std::mem::size_of::<outlk_autodiscover_t>(),
-        );
-        xml_raw = read_autoconf_file(context, as_str(url));
-        if xml_raw.is_null() {
-            ok_to_continue = false;
-            break;
-        }
+        match reader.read_event(&mut buf) {
+            Ok(quick_xml::events::Event::Start(ref e)) => {
+                let tag = String::from_utf8_lossy(e.name()).trim().to_lowercase();
 
-        let mut reader = quick_xml::Reader::from_str(as_str(xml_raw));
-        reader.trim_text(true);
+                if tag == "protocol" {
+                    outlk_ad.config_type = None;
+                    outlk_ad.config_server = String::new();
+                    outlk_ad.config_port = 0;
+                    outlk_ad.config_ssl = String::new();
+                    outlk_ad.config_redirecturl = None;
 
-        let mut buf = Vec::new();
-
-        loop {
-            match reader.read_event(&mut buf) {
-                Ok(quick_xml::events::Event::Start(ref e)) => {
-                    outlk_autodiscover_starttag_cb(e, &mut outlk_ad)
+                    current_tag = None;
+                } else {
+                    current_tag = Some(tag);
                 }
-                Ok(quick_xml::events::Event::End(ref e)) => {
-                    outlk_autodiscover_endtag_cb(e, &mut outlk_ad)
-                }
-                Ok(quick_xml::events::Event::Text(ref e)) => {
-                    outlk_autodiscover_text_cb(e, &mut outlk_ad, &reader)
-                }
-                Err(e) => {
-                    error!(
-                        context,
-                        "Configure xml: Error at position {}: {:?}",
-                        reader.buffer_position(),
-                        e
-                    );
-                }
-                Ok(quick_xml::events::Event::Eof) => break,
-                _ => (),
             }
-            buf.clear();
-        }
+            Ok(quick_xml::events::Event::End(ref e)) => {
+                outlk_autodiscover_endtag_cb(e, &mut outlk_ad);
+                current_tag = None;
+            }
+            Ok(quick_xml::events::Event::Text(ref e)) => {
+                let val = e.unescape_and_decode(&reader).unwrap_or_default();
 
-        if !(!outlk_ad.config[5].is_null()
-            && 0 != *outlk_ad.config[5usize].offset(0isize) as libc::c_int)
-        {
-            ok_to_continue = true;
-            break;
+                if let Some(ref tag) = current_tag {
+                    match tag.as_str() {
+                        "type" => {
+                            outlk_ad.config_type = Some(val.trim().to_lowercase().to_string())
+                        }
+                        "server" => outlk_ad.config_server = val.trim().to_string(),
+                        "port" => outlk_ad.config_port = val.trim().parse().unwrap_or_default(),
+                        "ssl" => outlk_ad.config_ssl = val.trim().to_string(),
+                        "redirecturl" => outlk_ad.config_redirecturl = Some(val.trim().to_string()),
+                        _ => {}
+                    };
+                }
+            }
+            Err(e) => {
+                bail!(
+                    "Configure xml: Error at position {}: {:?}",
+                    reader.buffer_position(),
+                    e
+                );
+            }
+            Ok(quick_xml::events::Event::Eof) => break,
+            _ => (),
         }
-        free(url as *mut libc::c_void);
-        url = dc_strdup(outlk_ad.config[5usize]);
-
-        outlk_clean_config(&mut outlk_ad);
-        free(xml_raw as *mut libc::c_void);
-        xml_raw = ptr::null_mut();
-        i += 1;
+        buf.clear();
     }
 
-    if ok_to_continue {
+    // XML redirect via redirecturl
+    if outlk_ad.config_redirecturl.is_none()
+        || outlk_ad.config_redirecturl.as_ref().unwrap().is_empty()
+    {
         if outlk_ad.out.mail_server.is_empty()
             || outlk_ad.out.mail_port == 0
             || outlk_ad.out.send_server.is_empty()
             || outlk_ad.out.send_port == 0
         {
             let r = outlk_ad.out.to_string();
-            warn!(context, "Bad or incomplete autoconfig: {}", r,);
-            free(url as *mut libc::c_void);
-            free(xml_raw as *mut libc::c_void);
-            outlk_clean_config(&mut outlk_ad);
+            bail!("Bad or incomplete autoconfig: {}", r,);
+        }
+        Ok(ParsingResult::LoginParam(outlk_ad.out))
+    } else {
+        Ok(ParsingResult::RedirectUrl(
+            outlk_ad.config_redirecturl.unwrap(),
+        ))
+    }
+}
 
+pub fn outlk_autodiscover(
+    context: &Context,
+    url: &str,
+    _param_in: &LoginParam,
+) -> Option<LoginParam> {
+    let mut url = url.to_string();
+    /* Follow up to 10 xml-redirects (http-redirects are followed in read_autoconf_file() */
+    for _i in 0..10 {
+        if let Some(xml_raw) = read_autoconf_file(context, &url) {
+            match outlk_parse_xml(&xml_raw) {
+                Err(err) => {
+                    warn!(context, "{}", err);
+                    return None;
+                }
+                Ok(res) => match res {
+                    ParsingResult::RedirectUrl(redirect_url) => url = redirect_url,
+                    ParsingResult::LoginParam(login_param) => return Some(login_param),
+                },
+            }
+        } else {
             return None;
         }
     }
-    free(url as *mut libc::c_void);
-    free(xml_raw as *mut libc::c_void);
-    outlk_clean_config(&mut outlk_ad);
-    Some(outlk_ad.out)
+    None
 }
 
-unsafe fn outlk_clean_config(mut outlk_ad: *mut outlk_autodiscover_t) {
-    for i in 0..6 {
-        free((*outlk_ad).config[i] as *mut libc::c_void);
-        (*outlk_ad).config[i] = ptr::null_mut();
-    }
-}
-
-fn outlk_autodiscover_text_cb<B: std::io::BufRead>(
-    event: &BytesText,
-    outlk_ad: &mut outlk_autodiscover_t,
-    reader: &quick_xml::Reader<B>,
-) {
-    let val = event.unescape_and_decode(reader).unwrap_or_default();
-
-    unsafe {
-        free(outlk_ad.config[outlk_ad.tag_config as usize].cast());
-        outlk_ad.config[outlk_ad.tag_config as usize] = val.trim().strdup();
-    }
-}
-
-unsafe fn outlk_autodiscover_endtag_cb(event: &BytesEnd, outlk_ad: &mut outlk_autodiscover_t) {
+fn outlk_autodiscover_endtag_cb(event: &BytesEnd, outlk_ad: &mut OutlookAutodiscover) {
     let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
 
     if tag == "protocol" {
-        if !outlk_ad.config[1].is_null() {
-            let port = dc_atoi_null_is_0(outlk_ad.config[3]);
-            let ssl_on = (!outlk_ad.config[4].is_null()
-                && strcasecmp(
-                    outlk_ad.config[4],
-                    b"on\x00" as *const u8 as *const libc::c_char,
-                ) == 0) as libc::c_int;
-            let ssl_off = (!outlk_ad.config[4].is_null()
-                && strcasecmp(
-                    outlk_ad.config[4],
-                    b"off\x00" as *const u8 as *const libc::c_char,
-                ) == 0) as libc::c_int;
-            if strcasecmp(
-                outlk_ad.config[1],
-                b"imap\x00" as *const u8 as *const libc::c_char,
-            ) == 0
-                && outlk_ad.out_imap_set == 0
-            {
-                outlk_ad.out.mail_server = to_string(outlk_ad.config[2]);
+        if let Some(type_) = &outlk_ad.config_type {
+            let port = outlk_ad.config_port;
+            let ssl_on = outlk_ad.config_ssl == "on";
+            let ssl_off = outlk_ad.config_ssl == "off";
+            if type_ == "imap" && !outlk_ad.out_imap_set {
+                outlk_ad.out.mail_server =
+                    std::mem::replace(&mut outlk_ad.config_server, String::new());
                 outlk_ad.out.mail_port = port;
-                if 0 != ssl_on {
+                if ssl_on {
                     outlk_ad.out.server_flags |= DC_LP_IMAP_SOCKET_SSL as i32
-                } else if 0 != ssl_off {
+                } else if ssl_off {
                     outlk_ad.out.server_flags |= DC_LP_IMAP_SOCKET_PLAIN as i32
                 }
-                outlk_ad.out_imap_set = 1
-            } else if strcasecmp(
-                outlk_ad.config[1usize],
-                b"smtp\x00" as *const u8 as *const libc::c_char,
-            ) == 0
-                && outlk_ad.out_smtp_set == 0
-            {
-                outlk_ad.out.send_server = to_string(outlk_ad.config[2]);
-                outlk_ad.out.send_port = port;
-                if 0 != ssl_on {
+                outlk_ad.out_imap_set = true
+            } else if type_ == "smtp" && !outlk_ad.out_smtp_set {
+                outlk_ad.out.send_server =
+                    std::mem::replace(&mut outlk_ad.config_server, String::new());
+                outlk_ad.out.send_port = outlk_ad.config_port;
+                if ssl_on {
                     outlk_ad.out.server_flags |= DC_LP_SMTP_SOCKET_SSL as i32
-                } else if 0 != ssl_off {
+                } else if ssl_off {
                     outlk_ad.out.server_flags |= DC_LP_SMTP_SOCKET_PLAIN as i32
                 }
-                outlk_ad.out_smtp_set = 1
+                outlk_ad.out_smtp_set = true
             }
         }
-        outlk_clean_config(outlk_ad);
     }
-    outlk_ad.tag_config = 0;
 }
 
-fn outlk_autodiscover_starttag_cb(event: &BytesStart, outlk_ad: &mut outlk_autodiscover_t) {
-    let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
+#[cfg(test)]
+mod tests {
+    use super::*;
 
-    if tag == "protocol" {
-        unsafe { outlk_clean_config(outlk_ad) };
-    } else if tag == "type" {
-        outlk_ad.tag_config = 1
-    } else if tag == "server" {
-        outlk_ad.tag_config = 2
-    } else if tag == "port" {
-        outlk_ad.tag_config = 3
-    } else if tag == "ssl" {
-        outlk_ad.tag_config = 4
-    } else if tag == "redirecturl" {
-        outlk_ad.tag_config = 5
-    };
+    #[test]
+    fn test_parse_redirect() {
+        let res = outlk_parse_xml("
+<?xml version=\"1.0\" encoding=\"utf-8\"?>
+  <Autodiscover xmlns=\"http://schemas.microsoft.com/exchange/autodiscover/responseschema/2006\">
+    <Response xmlns=\"http://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a\">
+      <Account>
+        <AccountType>email</AccountType>
+        <Action>redirectUrl</Action>
+        <RedirectUrl>https://mail.example.com/autodiscover/autodiscover.xml</RedirectUrl>
+      </Account>
+    </Response>
+  </Autodiscover>
+ ").expect("XML is not parsed successfully");
+        match res {
+            ParsingResult::LoginParam(_lp) => {
+                panic!("redirecturl is not found");
+            }
+            ParsingResult::RedirectUrl(url) => {
+                assert_eq!(
+                    url,
+                    "https://mail.example.com/autodiscover/autodiscover.xml"
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn test_parse_loginparam() {
+        let res = outlk_parse_xml(
+            "\
+<?xml version=\"1.0\" encoding=\"utf-8\"?>
+<Autodiscover xmlns=\"http://schemas.microsoft.com/exchange/autodiscover/responseschema/2006\">
+  <Response xmlns=\"http://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a\">
+    <Account>
+      <AccountType>email</AccountType>
+      <Action>settings</Action>
+      <Protocol>
+        <Type>IMAP</Type>
+        <Server>example.com</Server>
+        <Port>993</Port>
+        <SSL>on</SSL>
+        <AuthRequired>on</AuthRequired>
+      </Protocol>
+      <Protocol>
+        <Type>SMTP</Type>
+        <Server>smtp.example.com</Server>
+        <Port>25</Port>
+        <SSL>off</SSL>
+        <AuthRequired>on</AuthRequired>
+      </Protocol>
+    </Account>
+  </Response>
+</Autodiscover>",
+        )
+        .expect("XML is not parsed successfully");
+
+        match res {
+            ParsingResult::LoginParam(lp) => {
+                assert_eq!(lp.mail_server, "example.com");
+                assert_eq!(lp.mail_port, 993);
+                assert_eq!(lp.send_server, "smtp.example.com");
+                assert_eq!(lp.send_port, 25);
+            }
+            ParsingResult::RedirectUrl(_) => {
+                panic!("RedirectUrl is not expected");
+            }
+        }
+    }
 }

src/constants.rs

@@ -25,7 +25,6 @@ impl Default for MoveState {
 
 // some defaults
 const DC_E2EE_DEFAULT_ENABLED: i32 = 1;
-pub const DC_MDNS_DEFAULT_ENABLED: i32 = 1;
 const DC_INBOX_WATCH_DEFAULT: i32 = 1;
 const DC_SENTBOX_WATCH_DEFAULT: i32 = 1;
 const DC_MVBOX_WATCH_DEFAULT: i32 = 1;
@@ -45,6 +44,20 @@ impl Default for Blocked {
     }
 }
 
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[repr(u8)]
+pub enum ShowEmails {
+    Off = 0,
+    AcceptedContacts = 1,
+    All = 2,
+}
+
+impl Default for ShowEmails {
+    fn default() -> Self {
+        ShowEmails::Off // also change Config.ShowEmails props(default) on changes
+    }
+}
+
 pub const DC_IMAP_SEEN: u32 = 0x1;
 
 pub const DC_HANDSHAKE_CONTINUE_NORMAL_PROCESSING: i32 = 0x01;
@@ -55,7 +68,7 @@ pub const DC_GCL_ARCHIVED_ONLY: usize = 0x01;
 pub const DC_GCL_NO_SPECIALS: usize = 0x02;
 pub const DC_GCL_ADD_ALLDONE_HINT: usize = 0x04;
 
-const DC_GCM_ADDDAYMARKER: usize = 0x01;
+pub const DC_GCM_ADDDAYMARKER: u32 = 0x01;
 
 pub const DC_GCL_VERIFIED_ONLY: usize = 0x01;
 pub const DC_GCL_ADD_SELF: usize = 0x02;
@@ -107,7 +120,7 @@ impl Default for Chattype {
 }
 
 pub const DC_MSG_ID_MARKER1: u32 = 1;
-const DC_MSG_ID_DAYMARKER: u32 = 9;
+pub const DC_MSG_ID_DAYMARKER: u32 = 9;
 pub const DC_MSG_ID_LAST_SPECIAL: u32 = 9;
 
 /// approx. max. length returned by dc_msg_get_text()
@@ -117,11 +130,17 @@ const DC_MAX_GET_INFO_LEN: usize = 100000;
 
 pub const DC_CONTACT_ID_UNDEFINED: u32 = 0;
 pub const DC_CONTACT_ID_SELF: u32 = 1;
-pub const DC_CONTACT_ID_DEVICE: u32 = 2;
+pub const DC_CONTACT_ID_INFO: u32 = 2;
+pub const DC_CONTACT_ID_DEVICE: u32 = 5;
 pub const DC_CONTACT_ID_LAST_SPECIAL: u32 = 9;
 
 pub const DC_CREATE_MVBOX: usize = 1;
 
+// Flags for empty server job
+
+pub const DC_EMPTY_MVBOX: u32 = 0x01;
+pub const DC_EMPTY_INBOX: u32 = 0x02;
+
 // Flags for configuring IMAP and SMTP servers.
 // These flags are optional
 // and may be set together with the username, password etc.
@@ -130,23 +149,23 @@ pub const DC_CREATE_MVBOX: usize = 1;
 /// Force OAuth2 authorization. This flag does not skip automatic configuration.
 /// Before calling configure() with DC_LP_AUTH_OAUTH2 set,
 /// the user has to confirm access at the URL returned by dc_get_oauth2_url().
-pub const DC_LP_AUTH_OAUTH2: usize = 0x2;
+pub const DC_LP_AUTH_OAUTH2: i32 = 0x2;
 
 /// Force NORMAL authorization, this is the default.
 /// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_AUTH_NORMAL: usize = 0x4;
+pub const DC_LP_AUTH_NORMAL: i32 = 0x4;
 
 /// Connect to IMAP via STARTTLS.
 /// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_IMAP_SOCKET_STARTTLS: usize = 0x100;
+pub const DC_LP_IMAP_SOCKET_STARTTLS: i32 = 0x100;
 
 /// Connect to IMAP via SSL.
 /// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_IMAP_SOCKET_SSL: usize = 0x200;
+pub const DC_LP_IMAP_SOCKET_SSL: i32 = 0x200;
 
 /// Connect to IMAP unencrypted, this should not be used.
 /// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_IMAP_SOCKET_PLAIN: usize = 0x400;
+pub const DC_LP_IMAP_SOCKET_PLAIN: i32 = 0x400;
 
 /// Connect to SMTP via STARTTLS.
 /// If this flag is set, automatic configuration is skipped.
@@ -161,9 +180,9 @@ pub const DC_LP_SMTP_SOCKET_SSL: usize = 0x20000;
 pub const DC_LP_SMTP_SOCKET_PLAIN: usize = 0x40000;
 
 /// if none of these flags are set, the default is chosen
-pub const DC_LP_AUTH_FLAGS: usize = (DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL);
+pub const DC_LP_AUTH_FLAGS: i32 = (DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL);
 /// if none of these flags are set, the default is chosen
-pub const DC_LP_IMAP_SOCKET_FLAGS: usize =
+pub const DC_LP_IMAP_SOCKET_FLAGS: i32 =
     (DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_SSL | DC_LP_IMAP_SOCKET_PLAIN);
 /// if none of these flags are set, the default is chosen
 pub const DC_LP_SMTP_SOCKET_FLAGS: usize =
@@ -195,6 +214,11 @@ pub enum Viewtype {
     /// and retrieved via dc_msg_get_file(), dc_msg_get_width(), dc_msg_get_height().
     Gif = 21,
 
+    /// Message containing a sticker, similar to image.
+    /// If possible, the ui should display the image without borders in a transparent way.
+    /// A click on a sticker will offer to install the sticker set in some future.
+    Sticker = 23,
+
     /// Message containing an Audio file.
     /// File and duration are set via dc_msg_set_file(), dc_msg_set_duration()
     /// and retrieved via dc_msg_get_file(), dc_msg_get_duration().
@@ -247,13 +271,8 @@ const DC_ERROR_SEE_STRING: usize = 0; // deprecated;
 const DC_ERROR_SELF_NOT_IN_GROUP: usize = 1; // deprecated;
 const DC_STR_SELFNOTINGRP: usize = 21; // deprecated;
 
-/// Values for dc_get|set_config("show_emails")
-const DC_SHOW_EMAILS_OFF: usize = 0;
-const DC_SHOW_EMAILS_ACCEPTED_CONTACTS: usize = 1;
-const DC_SHOW_EMAILS_ALL: usize = 2;
-
 // TODO: Strings need some doumentation about used placeholders.
-// These constants are used to request strings using #DC_EVENT_GET_STRING.
+// These constants are used to set stock translation strings
 
 const DC_STR_NOMESSAGES: usize = 1;
 const DC_STR_SELF: usize = 2;
@@ -299,7 +318,8 @@ const DC_STR_MSGACTIONBYME: usize = 63;
 const DC_STR_MSGLOCATIONENABLED: usize = 64;
 const DC_STR_MSGLOCATIONDISABLED: usize = 65;
 const DC_STR_LOCATION: usize = 66;
-const DC_STR_COUNT: usize = 66;
+const DC_STR_STICKER: usize = 67;
+const DC_STR_COUNT: usize = 67;
 
 pub const DC_JOB_DELETE_MSG_ON_IMAP: i32 = 110;
 

src/contact.rs

@@ -14,7 +14,7 @@ use crate::error::Result;
 use crate::events::Event;
 use crate::key::*;
 use crate::login_param::LoginParam;
-use crate::message::MessageState;
+use crate::message::{MessageState, MsgId};
 use crate::peerstate::*;
 use crate::sql;
 use crate::stock::StockMessage;
@@ -153,7 +153,16 @@ impl Contact {
                 blocked: false,
                 origin: Origin::Unknown,
             };
-
+            return Ok(contact);
+        } else if contact_id == DC_CONTACT_ID_DEVICE {
+            let contact = Contact {
+                id: contact_id,
+                name: context.stock_str(StockMessage::DeviceMessages).into(),
+                authname: "".into(),
+                addr: "device@localhost".into(),
+                blocked: false,
+                origin: Origin::Unknown,
+            };
             return Ok(contact);
         }
 
@@ -243,7 +252,7 @@ impl Contact {
         {
             context.call_cb(Event::MsgsChanged {
                 chat_id: 0,
-                msg_id: 0,
+                msg_id: MsgId::new(0),
             });
         }
     }
@@ -264,7 +273,7 @@ impl Contact {
             .unwrap_or_default();
 
         if addr_normalized == addr_self {
-            return 1;
+            return DC_CONTACT_ID_SELF;
         }
 
         context.sql.query_get_value(
@@ -301,7 +310,7 @@ impl Contact {
             .unwrap_or_default();
 
         if addr == addr_self {
-            return Ok((1, sth_modified));
+            return Ok((DC_CONTACT_ID_SELF, sth_modified));
         }
 
         if !may_be_valid_addr(&addr) {
@@ -390,20 +399,18 @@ impl Contact {
                 }
                 sth_modified = Modifier::Modified;
             }
+        } else if sql::execute(
+            context,
+            &context.sql,
+            "INSERT INTO contacts (name, addr, origin) VALUES(?, ?, ?);",
+            params![name.as_ref(), addr, origin,],
+        )
+        .is_ok()
+        {
+            row_id = sql::get_rowid(context, &context.sql, "contacts", "addr", addr);
+            sth_modified = Modifier::Created;
         } else {
-            if sql::execute(
-                context,
-                &context.sql,
-                "INSERT INTO contacts (name, addr, origin) VALUES(?, ?, ?);",
-                params![name.as_ref(), addr, origin,],
-            )
-            .is_ok()
-            {
-                row_id = sql::get_rowid(context, &context.sql, "contacts", "addr", addr);
-                sth_modified = Modifier::Created;
-            } else {
-                error!(context, "Cannot add contact.");
-            }
+            error!(context, "Cannot add contact.");
         }
 
         Ok((row_id, sth_modified))
@@ -827,7 +834,7 @@ impl Contact {
         if let Ok(contact) = Contact::load_from_db(context, contact_id) {
             if !contact.addr.is_empty() {
                 let normalized_addr = addr_normalize(addr.as_ref());
-                if &contact.addr == &normalized_addr {
+                if contact.addr == normalized_addr {
                     return true;
                 }
             }
@@ -963,9 +970,9 @@ pub fn normalize_name(full_name: impl AsRef<str>) -> String {
     if len > 0 {
         let firstchar = full_name.as_bytes()[0];
         let lastchar = full_name.as_bytes()[len - 1];
-        if firstchar == '\'' as u8 && lastchar == '\'' as u8
-            || firstchar == '\"' as u8 && lastchar == '\"' as u8
-            || firstchar == '<' as u8 && lastchar == '>' as u8
+        if firstchar == b'\'' && lastchar == b'\''
+            || firstchar == b'\"' && lastchar == b'\"'
+            || firstchar == b'<' && lastchar == b'>'
         {
             full_name = &full_name[1..len - 1];
         }

src/context.rs

@@ -6,6 +6,7 @@ use std::sync::{Arc, Condvar, Mutex, RwLock};
 use libc::uintptr_t;
 
 use crate::chat::*;
+use crate::config::Config;
 use crate::constants::*;
 use crate::contact::*;
 use crate::error::*;
@@ -16,7 +17,7 @@ use crate::job_thread::JobThread;
 use crate::key::*;
 use crate::login_param::LoginParam;
 use crate::lot::Lot;
-use crate::message::{self, Message};
+use crate::message::{self, Message, MsgId};
 use crate::param::Params;
 use crate::smtp::*;
 use crate::sql::Sql;
@@ -58,12 +59,13 @@ pub struct Context {
     pub running_state: Arc<RwLock<RunningState>>,
     /// Mutex to avoid generating the key for the user more than once.
     pub generating_key_mutex: Mutex<()>,
+    pub translated_stockstrings: RwLock<HashMap<usize, String>>,
 }
 
 #[derive(Debug, PartialEq, Eq)]
 pub struct RunningState {
     pub ongoing_running: bool,
-    pub shall_stop_ongoing: bool,
+    shall_stop_ongoing: bool,
 }
 
 /// Return some info about deltachat-core
@@ -140,10 +142,11 @@ impl Context {
             probe_imap_network: Arc::new(RwLock::new(false)),
             perform_inbox_jobs_needed: Arc::new(RwLock::new(false)),
             generating_key_mutex: Mutex::new(()),
+            translated_stockstrings: RwLock::new(HashMap::new()),
         };
 
         ensure!(
-            ctx.sql.open(&ctx, &ctx.dbfile, 0),
+            ctx.sql.open(&ctx, &ctx.dbfile, false),
             "Failed opening sqlite database"
         );
 
@@ -162,31 +165,84 @@ impl Context {
         (*self.cb)(self, event)
     }
 
+    /*******************************************************************************
+     * Ongoing process allocation/free/check
+     ******************************************************************************/
+
+    pub fn alloc_ongoing(&self) -> bool {
+        if self.has_ongoing() {
+            warn!(self, "There is already another ongoing process running.",);
+
+            false
+        } else {
+            let s_a = self.running_state.clone();
+            let mut s = s_a.write().unwrap();
+
+            s.ongoing_running = true;
+            s.shall_stop_ongoing = false;
+
+            true
+        }
+    }
+
+    pub fn free_ongoing(&self) {
+        let s_a = self.running_state.clone();
+        let mut s = s_a.write().unwrap();
+
+        s.ongoing_running = false;
+        s.shall_stop_ongoing = true;
+    }
+
+    pub fn has_ongoing(&self) -> bool {
+        let s_a = self.running_state.clone();
+        let s = s_a.read().unwrap();
+
+        s.ongoing_running || !s.shall_stop_ongoing
+    }
+
+    /// Signal an ongoing process to stop.
+    pub fn stop_ongoing(&self) {
+        let s_a = self.running_state.clone();
+        let mut s = s_a.write().unwrap();
+
+        if s.ongoing_running && !s.shall_stop_ongoing {
+            info!(self, "Signaling the ongoing process to stop ASAP.",);
+            s.shall_stop_ongoing = true;
+        } else {
+            info!(self, "No ongoing process to stop.",);
+        };
+    }
+
+    pub fn shall_stop_ongoing(&self) -> bool {
+        self.running_state
+            .clone()
+            .read()
+            .unwrap()
+            .shall_stop_ongoing
+    }
+
+    /*******************************************************************************
+     * UI chat/message related API
+     ******************************************************************************/
+
     pub fn get_info(&self) -> HashMap<&'static str, String> {
         let unset = "0";
         let l = LoginParam::from_database(self, "");
         let l2 = LoginParam::from_database(self, "configured_");
-        let displayname = self.sql.get_config(self, "displayname");
+        let displayname = self.get_config(Config::Displayname);
         let chats = get_chat_cnt(self) as usize;
         let real_msgs = message::get_real_msg_cnt(self) as usize;
         let deaddrop_msgs = message::get_deaddrop_msg_cnt(self) as usize;
         let contacts = Contact::get_real_cnt(self) as usize;
-        let is_configured = self
-            .sql
-            .get_config_int(self, "configured")
-            .unwrap_or_default();
+        let is_configured = self.get_config_int(Config::Configured);
         let dbversion = self
             .sql
-            .get_config_int(self, "dbversion")
+            .get_raw_config_int(self, "dbversion")
             .unwrap_or_default();
-        let e2ee_enabled = self
-            .sql
-            .get_config_int(self, "e2ee_enabled")
-            .unwrap_or_else(|| 1);
-        let mdns_enabled = self
-            .sql
-            .get_config_int(self, "mdns_enabled")
-            .unwrap_or_else(|| 1);
+
+        let e2ee_enabled = self.get_config_int(Config::E2eeEnabled);
+        let mdns_enabled = self.get_config_int(Config::MdnsEnabled);
+        let bcc_self = self.get_config_int(Config::BccSelf);
 
         let prv_key_cnt: Option<isize> =
             self.sql
@@ -204,33 +260,22 @@ impl Context {
             "<Not yet calculated>".into()
         };
 
-        let inbox_watch = self
-            .sql
-            .get_config_int(self, "inbox_watch")
-            .unwrap_or_else(|| 1);
-        let sentbox_watch = self
-            .sql
-            .get_config_int(self, "sentbox_watch")
-            .unwrap_or_else(|| 1);
-        let mvbox_watch = self
-            .sql
-            .get_config_int(self, "mvbox_watch")
-            .unwrap_or_else(|| 1);
-        let mvbox_move = self
-            .sql
-            .get_config_int(self, "mvbox_move")
-            .unwrap_or_else(|| 1);
+        let inbox_watch = self.get_config_int(Config::InboxWatch);
+        let sentbox_watch = self.get_config_int(Config::SentboxWatch);
+        let mvbox_watch = self.get_config_int(Config::MvboxWatch);
+        let mvbox_move = self.get_config_int(Config::MvboxMove);
         let folders_configured = self
             .sql
-            .get_config_int(self, "folders_configured")
+            .get_raw_config_int(self, "folders_configured")
             .unwrap_or_default();
+
         let configured_sentbox_folder = self
             .sql
-            .get_config(self, "configured_sentbox_folder")
+            .get_raw_config(self, "configured_sentbox_folder")
             .unwrap_or_else(|| "<unset>".to_string());
         let configured_mvbox_folder = self
             .sql
-            .get_config(self, "configured_mvbox_folder")
+            .get_raw_config(self, "configured_mvbox_folder")
             .unwrap_or_else(|| "<unset>".to_string());
 
         let mut res = get_info();
@@ -254,6 +299,7 @@ impl Context {
         res.insert("configured_mvbox_folder", configured_mvbox_folder);
         res.insert("mdns_enabled", mdns_enabled.to_string());
         res.insert("e2ee_enabled", e2ee_enabled.to_string());
+        res.insert("bcc_self", bcc_self.to_string());
         res.insert(
             "private_key_count",
             prv_key_cnt.unwrap_or_default().to_string(),
@@ -267,33 +313,39 @@ impl Context {
         res
     }
 
-    pub fn get_fresh_msgs(&self) -> Vec<u32> {
+    pub fn get_fresh_msgs(&self) -> Vec<MsgId> {
         let show_deaddrop = 0;
-
         self.sql
             .query_map(
-                "SELECT m.id FROM msgs m LEFT JOIN contacts ct \
-                 ON m.from_id=ct.id LEFT JOIN chats c ON m.chat_id=c.id WHERE m.state=?   \
-                 AND m.hidden=0   \
-                 AND m.chat_id>?   \
-                 AND ct.blocked=0   \
-                 AND (c.blocked=0 OR c.blocked=?) ORDER BY m.timestamp DESC,m.id DESC;",
+                concat!(
+                    "SELECT m.id",
+                    " FROM msgs m",
+                    " LEFT JOIN contacts ct",
+                    "        ON m.from_id=ct.id",
+                    " LEFT JOIN chats c",
+                    "        ON m.chat_id=c.id",
+                    " WHERE m.state=?",
+                    "   AND m.hidden=0",
+                    "   AND m.chat_id>?",
+                    "   AND ct.blocked=0",
+                    "   AND (c.blocked=0 OR c.blocked=?)",
+                    " ORDER BY m.timestamp DESC,m.id DESC;"
+                ),
                 &[10, 9, if 0 != show_deaddrop { 2 } else { 0 }],
-                |row| row.get(0),
+                |row| row.get::<_, MsgId>(0),
                 |rows| {
                     let mut ret = Vec::new();
                     for row in rows {
-                        let id: u32 = row?;
-                        ret.push(id);
+                        ret.push(row?);
                     }
                     Ok(ret)
                 },
             )
-            .unwrap()
+            .unwrap_or_default()
     }
 
     #[allow(non_snake_case)]
-    pub fn search_msgs(&self, chat_id: u32, query: impl AsRef<str>) -> Vec<u32> {
+    pub fn search_msgs(&self, chat_id: u32, query: impl AsRef<str>) -> Vec<MsgId> {
         let real_query = query.as_ref().trim();
         if real_query.is_empty() {
             return Vec::new();
@@ -302,25 +354,43 @@ impl Context {
         let strLikeBeg = format!("{}%", real_query);
 
         let query = if 0 != chat_id {
-            "SELECT m.id, m.timestamp FROM msgs m LEFT JOIN contacts ct ON m.from_id=ct.id WHERE m.chat_id=?  \
-         AND m.hidden=0  \
-         AND ct.blocked=0 AND (txt LIKE ? OR ct.name LIKE ?) ORDER BY m.timestamp,m.id;"
+            concat!(
+                "SELECT m.id AS id, m.timestamp AS timestamp",
+                " FROM msgs m",
+                " LEFT JOIN contacts ct",
+                "        ON m.from_id=ct.id",
+                " WHERE m.chat_id=?",
+                "   AND m.hidden=0",
+                "   AND ct.blocked=0",
+                "   AND (txt LIKE ? OR ct.name LIKE ?)",
+                " ORDER BY m.timestamp,m.id;"
+            )
         } else {
-            "SELECT m.id, m.timestamp FROM msgs m LEFT JOIN contacts ct ON m.from_id=ct.id \
-         LEFT JOIN chats c ON m.chat_id=c.id WHERE m.chat_id>9 AND m.hidden=0  \
-         AND (c.blocked=0 OR c.blocked=?) \
-         AND ct.blocked=0 AND (m.txt LIKE ? OR ct.name LIKE ?) ORDER BY m.timestamp DESC,m.id DESC;"
+            concat!(
+                "SELECT m.id AS id, m.timestamp AS timestamp",
+                " FROM msgs m",
+                " LEFT JOIN contacts ct",
+                "        ON m.from_id=ct.id",
+                " LEFT JOIN chats c",
+                "        ON m.chat_id=c.id",
+                " WHERE m.chat_id>9",
+                "   AND m.hidden=0",
+                "   AND (c.blocked=0 OR c.blocked=?)",
+                "   AND ct.blocked=0",
+                "   AND (m.txt LIKE ? OR ct.name LIKE ?)",
+                " ORDER BY m.timestamp DESC,m.id DESC;"
+            )
         };
 
         self.sql
             .query_map(
                 query,
                 params![chat_id as i32, &strLikeInText, &strLikeBeg],
-                |row| row.get::<_, i32>(0),
+                |row| row.get::<_, MsgId>("id"),
                 |rows| {
                     let mut ret = Vec::new();
                     for id in rows {
-                        ret.push(id? as u32);
+                        ret.push(id?);
                     }
                     Ok(ret)
                 },
@@ -333,7 +403,7 @@ impl Context {
     }
 
     pub fn is_sentbox(&self, folder_name: impl AsRef<str>) -> bool {
-        let sentbox_name = self.sql.get_config(self, "configured_sentbox_folder");
+        let sentbox_name = self.sql.get_raw_config(self, "configured_sentbox_folder");
         if let Some(name) = sentbox_name {
             name == folder_name.as_ref()
         } else {
@@ -342,7 +412,7 @@ impl Context {
     }
 
     pub fn is_mvbox(&self, folder_name: impl AsRef<str>) -> bool {
-        let mvbox_name = self.sql.get_config(self, "configured_mvbox_folder");
+        let mvbox_name = self.sql.get_raw_config(self, "configured_mvbox_folder");
 
         if let Some(name) = mvbox_name {
             name == folder_name.as_ref()
@@ -351,13 +421,8 @@ impl Context {
         }
     }
 
-    pub fn do_heuristics_moves(&self, folder: &str, msg_id: u32) {
-        if self
-            .sql
-            .get_config_int(self, "mvbox_move")
-            .unwrap_or_else(|| 1)
-            == 0
-        {
+    pub fn do_heuristics_moves(&self, folder: &str, msg_id: MsgId) {
+        if !self.get_config_bool(Config::MvboxMove) {
             return;
         }
 
@@ -381,7 +446,7 @@ impl Context {
                 job_add(
                     self,
                     Action::MoveMsg,
-                    msg.id as libc::c_int,
+                    msg.id.to_u32() as i32,
                     Params::new(),
                     0,
                 );

src/dc_mimeparser.rs

@@ -11,9 +11,9 @@ use mmime::mailmime::content::*;
 use mmime::mailmime::disposition::*;
 use mmime::mailmime::types::*;
 use mmime::mailmime::*;
-use mmime::mmapstring::*;
 use mmime::other::*;
 
+use crate::blob::BlobObject;
 use crate::constants::Viewtype;
 use crate::contact::*;
 use crate::context::Context;
@@ -25,6 +25,7 @@ use crate::error::Error;
 use crate::location;
 use crate::param::*;
 use crate::stock::StockMessage;
+use crate::wrapmime;
 
 #[derive(Debug)]
 pub struct MimeParser<'a> {
@@ -116,18 +117,30 @@ impl<'a> MimeParser<'a> {
         );
 
         if r == MAILIMF_NO_ERROR as libc::c_int && !self.mimeroot.is_null() {
-            let (encrypted, signatures, gossipped_addr) =
-                e2ee::try_decrypt(self.context, self.mimeroot)?;
-            self.encrypted = encrypted;
-            self.signatures = signatures;
-            self.gossipped_addr = gossipped_addr;
+            match e2ee::try_decrypt(self.context, self.mimeroot) {
+                Ok((encrypted, signatures, gossipped_addr)) => {
+                    self.encrypted = encrypted;
+                    self.signatures = signatures;
+                    self.gossipped_addr = gossipped_addr;
+                }
+                Err(err) => {
+                    // continue with the current, still encrypted, mime tree.
+                    // unencrypted parts will be replaced by an error message
+                    // that is added as "the message" to the chat then.
+                    //
+                    // if we just return here, the header is missing
+                    // and the caller cannot display the message
+                    // and try to assign the message to a chat
+                    warn!(self.context, "decryption failed: {}", err);
+                }
+            }
+
             self.parse_mime_recursive(self.mimeroot);
 
             if let Some(field) = self.lookup_field("Subject") {
                 if (*field).fld_type == MAILIMF_FIELD_SUBJECT as libc::c_int {
                     let subj = (*(*field).fld_data.fld_subject).sbj_value;
-
-                    self.subject = as_opt_str(subj).map(dc_decode_header_words_safe);
+                    self.subject = to_opt_string_lossy(subj).map(|x| dc_decode_header_words(&x));
                 }
             }
 
@@ -164,21 +177,17 @@ impl<'a> MimeParser<'a> {
                         }
                     }
                 }
-            } else {
-                if let Some(optional_field) = self.lookup_optional_field("Chat-Content") {
-                    if optional_field == "location-streaming-enabled" {
-                        self.is_system_message = SystemMessage::LocationStreamingEnabled;
-                    }
+            } else if let Some(optional_field) = self.lookup_optional_field("Chat-Content") {
+                if optional_field == "location-streaming-enabled" {
+                    self.is_system_message = SystemMessage::LocationStreamingEnabled;
                 }
             }
             if self.lookup_field("Chat-Group-Image").is_some() && !self.parts.is_empty() {
                 let textpart = &self.parts[0];
-                if textpart.typ == Viewtype::Text {
-                    if self.parts.len() >= 2 {
-                        let imgpart = &mut self.parts[1];
-                        if imgpart.typ == Viewtype::Image {
-                            imgpart.is_meta = true;
-                        }
+                if textpart.typ == Viewtype::Text && self.parts.len() >= 2 {
+                    let imgpart = &mut self.parts[1];
+                    if imgpart.typ == Viewtype::Image {
+                        imgpart.is_meta = true;
                     }
                 }
             }
@@ -189,6 +198,7 @@ impl<'a> MimeParser<'a> {
                     textpart.typ == Viewtype::Text
                         && (filepart.typ == Viewtype::Image
                             || filepart.typ == Viewtype::Gif
+                            || filepart.typ == Viewtype::Sticker
                             || filepart.typ == Viewtype::Audio
                             || filepart.typ == Viewtype::Voice
                             || filepart.typ == Viewtype::Video
@@ -256,6 +266,14 @@ impl<'a> MimeParser<'a> {
                         part_mut.typ = Viewtype::Voice;
                     }
                 }
+                if self.parts[0].typ == Viewtype::Image {
+                    if let Some(content_type) = self.lookup_optional_field("Chat-Content") {
+                        if content_type == "sticker" {
+                            let part_mut = &mut self.parts[0];
+                            part_mut.typ = Viewtype::Sticker;
+                        }
+                    }
+                }
                 let part = &self.parts[0];
                 if part.typ == Viewtype::Audio
                     || part.typ == Viewtype::Voice
@@ -277,7 +295,7 @@ impl<'a> MimeParser<'a> {
                     if self.get_last_nonmeta().is_some() {
                         let mut mb_list: *mut mailimf_mailbox_list = ptr::null_mut();
                         let mut index_0 = 0;
-                        let dn_field_c = CString::new(dn_field).unwrap();
+                        let dn_field_c = CString::new(dn_field).unwrap_or_default();
 
                         if mailimf_mailbox_list_parse(
                             dn_field_c.as_ptr(),
@@ -287,12 +305,12 @@ impl<'a> MimeParser<'a> {
                         ) == MAILIMF_NO_ERROR as libc::c_int
                             && !mb_list.is_null()
                         {
-                            if let Some(dn_to_addr) = mailimf_find_first_addr(mb_list) {
+                            if let Some(dn_to_addr) = wrapmime::mailimf_find_first_addr(mb_list) {
                                 if let Some(from_field) = self.lookup_field("From") {
                                     if (*from_field).fld_type == MAILIMF_FIELD_FROM as libc::c_int
                                         && !(*from_field).fld_data.fld_from.is_null()
                                     {
-                                        let from_addr = mailimf_find_first_addr(
+                                        let from_addr = wrapmime::mailimf_find_first_addr(
                                             (*(*from_field).fld_data.fld_from).frm_mb_list,
                                         );
                                         if let Some(from_addr) = from_addr {
@@ -590,10 +608,10 @@ impl<'a> MimeParser<'a> {
             return false;
         }
 
-        let mut decoded_data = match mailmime_transfer_decode(mime) {
+        let mut decoded_data = match wrapmime::mailmime_transfer_decode(mime) {
             Ok(decoded_data) => decoded_data,
             Err(_) => {
-                // Note that it's now always an error - might be no data
+                // Note that it's not always an error - might be no data
                 return false;
             }
         };
@@ -601,14 +619,9 @@ impl<'a> MimeParser<'a> {
         let old_part_count = self.parts.len();
 
         /* regard `Content-Transfer-Encoding:` */
-        let mut ok_to_continue = true;
         let mut desired_filename = String::default();
-        let mut simplifier: Option<Simplify> = None;
         match mime_type {
             DC_MIMETYPE_TEXT_PLAIN | DC_MIMETYPE_TEXT_HTML => {
-                if simplifier.is_none() {
-                    simplifier = Some(Simplify::new());
-                }
                 /* get from `Content-Type: text/...; charset=utf-8`; must not be free()'d */
                 let charset = mailmime_content_charset_get((*mime).mm_content_type);
                 if !charset.is_null()
@@ -616,49 +629,47 @@ impl<'a> MimeParser<'a> {
                     && strcmp(charset, b"UTF-8\x00" as *const u8 as *const libc::c_char) != 0i32
                 {
                     if let Some(encoding) =
-                        Charset::for_label(CStr::from_ptr(charset).to_str().unwrap().as_bytes())
+                        Charset::for_label(CStr::from_ptr(charset).to_string_lossy().as_bytes())
                     {
                         let (res, _, _) = encoding.decode(&decoded_data);
                         if res.is_empty() {
                             /* no error - but nothing to add */
-                            ok_to_continue = false;
-                        } else {
-                            decoded_data = res.as_bytes().to_vec()
+                            return false;
                         }
+                        decoded_data = res.as_bytes().to_vec()
                     } else {
                         warn!(
                             self.context,
                             "Cannot convert {} bytes from \"{}\" to \"utf-8\".",
                             decoded_data.len(),
-                            as_str(charset),
+                            to_string_lossy(charset),
                         );
                     }
                 }
-                if ok_to_continue {
-                    /* check header directly as is_send_by_messenger is not yet set up */
-                    let is_msgrmsg = self.lookup_optional_field("Chat-Version").is_some();
+                /* check header directly as is_send_by_messenger is not yet set up */
+                let is_msgrmsg = self.lookup_optional_field("Chat-Version").is_some();
 
-                    let simplified_txt = if decoded_data.is_empty() {
-                        "".into()
-                    } else {
-                        let input = std::string::String::from_utf8_lossy(&decoded_data);
-                        let is_html = mime_type == 70;
+                let mut simplifier = Simplify::new();
+                let simplified_txt = if decoded_data.is_empty() {
+                    "".into()
+                } else {
+                    let input = std::string::String::from_utf8_lossy(&decoded_data);
+                    let is_html = mime_type == 70;
 
-                        simplifier.unwrap().simplify(&input, is_html, is_msgrmsg)
-                    };
-                    if !simplified_txt.is_empty() {
-                        let mut part = Part::default();
-                        part.typ = Viewtype::Text;
-                        part.mimetype = mime_type;
-                        part.msg = Some(simplified_txt);
-                        part.msg_raw =
-                            Some(std::string::String::from_utf8_lossy(&decoded_data).to_string());
-                        self.do_add_single_part(part);
-                    }
+                    simplifier.simplify(&input, is_html, is_msgrmsg)
+                };
+                if !simplified_txt.is_empty() {
+                    let mut part = Part::default();
+                    part.typ = Viewtype::Text;
+                    part.mimetype = mime_type;
+                    part.msg = Some(simplified_txt);
+                    part.msg_raw =
+                        Some(std::string::String::from_utf8_lossy(&decoded_data).to_string());
+                    self.do_add_single_part(part);
+                }
 
-                    if simplifier.unwrap().is_forwarded {
-                        self.is_forwarded = true;
-                    }
+                if simplifier.is_forwarded {
+                    self.is_forwarded = true;
                 }
             }
             DC_MIMETYPE_IMAGE
@@ -704,7 +715,7 @@ impl<'a> MimeParser<'a> {
                                         // might be a wrongly encoded filename
                                         let s = to_string_lossy((*dsp_param).pa_data.pa_filename);
                                         // this is used only if the parts buffer stays empty
-                                        desired_filename = dc_decode_header_words_safe(&s)
+                                        desired_filename = dc_decode_header_words(&s)
                                     }
                                 }
                             }
@@ -730,37 +741,21 @@ impl<'a> MimeParser<'a> {
                     if !(*mime).mm_content_type.is_null()
                         && !(*(*mime).mm_content_type).ct_subtype.is_null()
                     {
-                        desired_filename =
-                            format!("file.{}", as_str((*(*mime).mm_content_type).ct_subtype));
-                    } else {
-                        ok_to_continue = false;
-                    }
-                }
-                if ok_to_continue {
-                    if desired_filename.starts_with("location")
-                        && desired_filename.ends_with(".kml")
-                    {
-                        if !decoded_data.is_empty() {
-                            let d = std::string::String::from_utf8_lossy(&decoded_data);
-                            self.location_kml = location::Kml::parse(self.context, &d).ok();
-                        }
-                    } else if desired_filename.starts_with("message")
-                        && desired_filename.ends_with(".kml")
-                    {
-                        if !decoded_data.is_empty() {
-                            let d = std::string::String::from_utf8_lossy(&decoded_data);
-                            self.message_kml = location::Kml::parse(self.context, &d).ok();
-                        }
-                    } else if !decoded_data.is_empty() {
-                        self.do_add_single_file_part(
-                            msg_type,
-                            mime_type,
-                            raw_mime.as_ref(),
-                            &decoded_data,
-                            &desired_filename,
+                        desired_filename = format!(
+                            "file.{}",
+                            to_string_lossy((*(*mime).mm_content_type).ct_subtype)
                         );
+                    } else {
+                        return false;
                     }
                 }
+                self.do_add_single_file_part(
+                    msg_type,
+                    mime_type,
+                    raw_mime.as_ref(),
+                    &decoded_data,
+                    &desired_filename,
+                );
             }
             _ => {}
         }
@@ -774,37 +769,72 @@ impl<'a> MimeParser<'a> {
         mime_type: libc::c_int,
         raw_mime: Option<&String>,
         decoded_data: &[u8],
-        desired_filename: &str,
+        filename: &str,
     ) {
-        /* create a free file name to use */
-        let path_filename = dc_get_fine_path_filename(self.context, "$BLOBDIR", desired_filename);
-
-        /* copy data to file */
-        if dc_write_file(self.context, &path_filename, decoded_data) {
-            let mut part = Part::default();
-            part.typ = msg_type;
-            part.mimetype = mime_type;
-            part.bytes = decoded_data.len() as libc::c_int;
-            part.param.set(Param::File, path_filename.to_string_lossy());
-            if let Some(raw_mime) = raw_mime {
-                part.param.set(Param::MimeType, raw_mime);
-            }
-
-            if mime_type == DC_MIMETYPE_IMAGE {
-                if let Ok((width, height)) = dc_get_filemeta(decoded_data) {
-                    part.param.set_int(Param::Width, width as i32);
-                    part.param.set_int(Param::Height, height as i32);
-                }
-            }
-            self.do_add_single_part(part);
+        if decoded_data.is_empty() {
+            return;
         }
+        // treat location/message kml file attachments specially
+        if filename.ends_with(".kml") {
+            // XXX what if somebody sends eg an "location-highlights.kml"
+            // attachment unrelated to location streaming?
+            if filename.starts_with("location") || filename.starts_with("message") {
+                let parsed = location::Kml::parse(self.context, decoded_data)
+                    .map_err(|err| {
+                        warn!(self.context, "failed to parse kml part: {}", err);
+                    })
+                    .ok();
+                if filename.starts_with("location") {
+                    self.location_kml = parsed;
+                } else {
+                    self.message_kml = parsed;
+                }
+                return;
+            }
+        }
+        /* we have a regular file attachment,
+        write decoded data to new blob object */
+
+        let blob = match BlobObject::create(self.context, filename, decoded_data) {
+            Ok(blob) => blob,
+            Err(err) => {
+                error!(
+                    self.context,
+                    "Could not add blob for mime part {}, error {}", filename, err
+                );
+                return;
+            }
+        };
+
+        /* create and register Mime part referencing the new Blob object */
+        let mut part = Part::default();
+        part.typ = msg_type;
+        part.mimetype = mime_type;
+        part.bytes = decoded_data.len() as libc::c_int;
+        part.param.set(Param::File, blob.as_name());
+        if let Some(raw_mime) = raw_mime {
+            part.param.set(Param::MimeType, raw_mime);
+        }
+
+        if mime_type == DC_MIMETYPE_IMAGE {
+            if let Ok((width, height)) = dc_get_filemeta(decoded_data) {
+                part.param.set_int(Param::Width, width as i32);
+                part.param.set_int(Param::Height, height as i32);
+            }
+        }
+        self.do_add_single_part(part);
     }
 
     fn do_add_single_part(&mut self, mut part: Part) {
-        if self.encrypted && self.signatures.len() > 0 {
-            part.param.set_int(Param::GuranteeE2ee, 1);
-        } else if self.encrypted {
-            part.param.set_int(Param::ErroneousE2ee, 0x2);
+        if self.encrypted {
+            if self.signatures.len() > 0 {
+                part.param.set_int(Param::GuaranteeE2ee, 1);
+            } else {
+                // XXX if the message was encrypted but not signed
+                // it's not neccessarily an error we need to signal.
+                // we could just treat it as if it was not encrypted.
+                part.param.set_int(Param::ErroneousE2ee, 0x2);
+            }
         }
         self.parts.push(part);
     }
@@ -832,7 +862,7 @@ impl<'a> MimeParser<'a> {
         let mut fld_from: *const mailimf_from = ptr::null();
 
         /* get From: and check there is exactly one sender */
-        let fld = mailimf_find_field(self.header_root, MAILIMF_FIELD_FROM as libc::c_int);
+        let fld = wrapmime::mailimf_find_field(self.header_root, MAILIMF_FIELD_FROM as libc::c_int);
         if !(fld.is_null()
             || {
                 fld_from = (*fld).fld_data.fld_from;
@@ -849,12 +879,11 @@ impl<'a> MimeParser<'a> {
             }) as *mut mailimf_mailbox;
 
             if !mb.is_null() {
-                let from_addr_norm = addr_normalize(as_str((*mb).mb_addr_spec));
-                let recipients = mailimf_get_recipients(self.header_root);
-                if recipients.len() == 1 {
-                    if recipients.contains(from_addr_norm) {
-                        sender_equals_recipient = true;
-                    }
+                let from_addr = to_string_lossy((*mb).mb_addr_spec);
+                let from_addr_norm = addr_normalize(&from_addr);
+                let recipients = wrapmime::mailimf_get_recipients(self.header_root);
+                if recipients.len() == 1 && recipients.contains(from_addr_norm) {
+                    sender_equals_recipient = true;
                 }
             }
         }
@@ -881,7 +910,7 @@ impl<'a> MimeParser<'a> {
             unsafe {
                 let fld_message_id = (*field).fld_data.fld_message_id;
                 if !fld_message_id.is_null() {
-                    return Some(to_string((*fld_message_id).mid_value));
+                    return Some(to_string_lossy((*fld_message_id).mid_value));
                 }
             }
         }
@@ -911,22 +940,6 @@ pub struct Part {
     pub param: Params,
 }
 
-pub fn mailimf_find_first_addr(mb_list: *const mailimf_mailbox_list) -> Option<String> {
-    if mb_list.is_null() {
-        return None;
-    }
-
-    for cur in unsafe { (*(*mb_list).mb_list).into_iter() } {
-        let mb = cur as *mut mailimf_mailbox;
-        if !mb.is_null() && !unsafe { (*mb).mb_addr_spec.is_null() } {
-            let addr = unsafe { as_str((*mb).mb_addr_spec) };
-            return Some(addr_normalize(addr).to_string());
-        }
-    }
-
-    None
-}
-
 unsafe fn hash_header(out: &mut HashMap<String, *mut mailimf_field>, in_0: *const mailimf_fields) {
     if in_0.is_null() {
         return;
@@ -997,26 +1010,25 @@ unsafe fn mailmime_get_mime_type(mime: *mut Mailmime) -> (libc::c_int, Viewtype,
                     ) == 0i32
                     {
                         return (DC_MIMETYPE_TEXT_PLAIN, Viewtype::Text, None);
-                    } else {
-                        if strcmp(
-                            (*c).ct_subtype,
-                            b"html\x00" as *const u8 as *const libc::c_char,
-                        ) == 0i32
-                        {
-                            return (DC_MIMETYPE_TEXT_HTML, Viewtype::Text, None);
-                        }
+                    } else if strcmp(
+                        (*c).ct_subtype,
+                        b"html\x00" as *const u8 as *const libc::c_char,
+                    ) == 0i32
+                    {
+                        return (DC_MIMETYPE_TEXT_HTML, Viewtype::Text, None);
                     }
                 }
 
-                let raw_mime = reconcat_mime(Some("text"), as_opt_str((*c).ct_subtype));
+                let raw_mime = reconcat_mime(Some("text"), to_opt_string_lossy((*c).ct_subtype));
                 (DC_MIMETYPE_FILE, Viewtype::File, Some(raw_mime))
             }
             MAILMIME_DISCRETE_TYPE_IMAGE => {
-                let subtype = as_opt_str((*c).ct_subtype);
-                let msg_type = match subtype {
+                let subtype = to_opt_string_lossy((*c).ct_subtype);
+                let msg_type = match subtype.as_ref().map(|x| x.as_str()) {
                     Some("gif") => Viewtype::Gif,
                     Some("svg+xml") => {
-                        let raw_mime = reconcat_mime(Some("image"), as_opt_str((*c).ct_subtype));
+                        let raw_mime =
+                            reconcat_mime(Some("image"), to_opt_string_lossy((*c).ct_subtype));
                         return (DC_MIMETYPE_FILE, Viewtype::File, Some(raw_mime));
                     }
                     _ => Viewtype::Image,
@@ -1026,11 +1038,11 @@ unsafe fn mailmime_get_mime_type(mime: *mut Mailmime) -> (libc::c_int, Viewtype,
                 (DC_MIMETYPE_IMAGE, msg_type, Some(raw_mime))
             }
             MAILMIME_DISCRETE_TYPE_AUDIO => {
-                let raw_mime = reconcat_mime(Some("audio"), as_opt_str((*c).ct_subtype));
+                let raw_mime = reconcat_mime(Some("audio"), to_opt_string_lossy((*c).ct_subtype));
                 (DC_MIMETYPE_AUDIO, Viewtype::Audio, Some(raw_mime))
             }
             MAILMIME_DISCRETE_TYPE_VIDEO => {
-                let raw_mime = reconcat_mime(Some("video"), as_opt_str((*c).ct_subtype));
+                let raw_mime = reconcat_mime(Some("video"), to_opt_string_lossy((*c).ct_subtype));
                 (DC_MIMETYPE_VIDEO, Viewtype::Video, Some(raw_mime))
             }
             _ => {
@@ -1041,13 +1053,15 @@ unsafe fn mailmime_get_mime_type(mime: *mut Mailmime) -> (libc::c_int, Viewtype,
                         b"autocrypt-setup\x00" as *const u8 as *const libc::c_char,
                     ) == 0i32
                 {
-                    let raw_mime = reconcat_mime(None, as_opt_str((*c).ct_subtype));
+                    let raw_mime = reconcat_mime(None, to_opt_string_lossy((*c).ct_subtype));
                     return (DC_MIMETYPE_AC_SETUP_FILE, Viewtype::File, Some(raw_mime));
                 }
 
                 let raw_mime = reconcat_mime(
-                    as_opt_str((*(*(*c).ct_type).tp_data.tp_discrete_type).dt_extension),
-                    as_opt_str((*c).ct_subtype),
+                    to_opt_string_lossy((*(*(*c).ct_type).tp_data.tp_discrete_type).dt_extension)
+                        .as_ref()
+                        .map(|x| x.as_str()),
+                    to_opt_string_lossy((*c).ct_subtype),
                 );
 
                 (DC_MIMETYPE_FILE, Viewtype::File, Some(raw_mime))
@@ -1057,9 +1071,9 @@ unsafe fn mailmime_get_mime_type(mime: *mut Mailmime) -> (libc::c_int, Viewtype,
             if (*(*(*c).ct_type).tp_data.tp_composite_type).ct_type
                 == MAILMIME_COMPOSITE_TYPE_MULTIPART as libc::c_int
             {
-                let subtype = as_opt_str((*c).ct_subtype);
+                let subtype = to_opt_string_lossy((*c).ct_subtype);
 
-                let mime_type = match subtype {
+                let mime_type = match subtype.as_ref().map(|x| x.as_str()) {
                     Some("alternative") => DC_MIMETYPE_MP_ALTERNATIVE,
                     Some("related") => DC_MIMETYPE_MP_RELATED,
                     Some("encrypted") => {
@@ -1094,9 +1108,9 @@ unsafe fn mailmime_get_mime_type(mime: *mut Mailmime) -> (libc::c_int, Viewtype,
     }
 }
 
-fn reconcat_mime(typ: Option<&str>, subtype: Option<&str>) -> String {
+fn reconcat_mime(typ: Option<&str>, subtype: Option<String>) -> String {
     let typ = typ.unwrap_or("application");
-    let subtype = subtype.unwrap_or("octet-stream");
+    let subtype = subtype.unwrap_or("octet-stream".to_string());
 
     format!("{}/{}", typ, subtype)
 }
@@ -1148,214 +1162,6 @@ pub unsafe fn mailmime_find_ct_parameter(
     ptr::null_mut()
 }
 
-pub unsafe fn mailmime_transfer_decode(mime: *mut Mailmime) -> Result<Vec<u8>, Error> {
-    ensure!(!mime.is_null(), "invalid inputs");
-
-    let mut mime_transfer_encoding = MAILMIME_MECHANISM_BINARY as libc::c_int;
-
-    let mime_data = (*mime).mm_data.mm_single;
-    if !(*mime).mm_mime_fields.is_null() {
-        for cur in (*(*(*mime).mm_mime_fields).fld_list).into_iter() {
-            let field = cur as *mut mailmime_field;
-
-            if !field.is_null()
-                && (*field).fld_type == MAILMIME_FIELD_TRANSFER_ENCODING as libc::c_int
-                && !(*field).fld_data.fld_encoding.is_null()
-            {
-                mime_transfer_encoding = (*(*field).fld_data.fld_encoding).enc_type;
-                break;
-            }
-        }
-    }
-
-    if mime_transfer_encoding == MAILMIME_MECHANISM_7BIT as libc::c_int
-        || mime_transfer_encoding == MAILMIME_MECHANISM_8BIT as libc::c_int
-        || mime_transfer_encoding == MAILMIME_MECHANISM_BINARY as libc::c_int
-    {
-        let decoded_data = (*mime_data).dt_data.dt_text.dt_data;
-        let decoded_data_bytes = (*mime_data).dt_data.dt_text.dt_length;
-
-        if decoded_data.is_null() || decoded_data_bytes <= 0 {
-            bail!("No data to decode found");
-        } else {
-            let result = std::slice::from_raw_parts(decoded_data as *const u8, decoded_data_bytes);
-            return Ok(result.to_vec());
-        }
-    }
-
-    let mut current_index = 0;
-    let mut transfer_decoding_buffer = ptr::null_mut();
-    let mut decoded_data_bytes = 0;
-
-    let r = mailmime_part_parse(
-        (*mime_data).dt_data.dt_text.dt_data,
-        (*mime_data).dt_data.dt_text.dt_length,
-        &mut current_index,
-        mime_transfer_encoding,
-        &mut transfer_decoding_buffer,
-        &mut decoded_data_bytes,
-    );
-
-    if r == MAILIMF_NO_ERROR as libc::c_int
-        && !transfer_decoding_buffer.is_null()
-        && decoded_data_bytes > 0
-    {
-        let result =
-            std::slice::from_raw_parts(transfer_decoding_buffer as *const u8, decoded_data_bytes)
-                .to_vec();
-        mmap_string_unref(transfer_decoding_buffer);
-
-        return Ok(result);
-    }
-
-    Err(format_err!("Failed to to decode"))
-}
-
-pub fn mailimf_get_recipients(imffields: *mut mailimf_fields) -> HashSet<String> {
-    /* returned addresses are normalized. */
-    let mut recipients: HashSet<String> = Default::default();
-
-    for cur in unsafe { (*(*imffields).fld_list).into_iter() } {
-        let fld = cur as *mut mailimf_field;
-
-        let fld_to: *mut mailimf_to;
-        let fld_cc: *mut mailimf_cc;
-
-        let mut addr_list: *mut mailimf_address_list = ptr::null_mut();
-        if fld.is_null() {
-            continue;
-        }
-
-        let fld = unsafe { *fld };
-
-        // TODO match on enums /rtn
-        match fld.fld_type {
-            13 => {
-                fld_to = unsafe { fld.fld_data.fld_to };
-                if !fld_to.is_null() {
-                    addr_list = unsafe { (*fld_to).to_addr_list };
-                }
-            }
-            14 => {
-                fld_cc = unsafe { fld.fld_data.fld_cc };
-                if !fld_cc.is_null() {
-                    addr_list = unsafe { (*fld_cc).cc_addr_list };
-                }
-            }
-            _ => {}
-        }
-
-        if !addr_list.is_null() {
-            for cur2 in unsafe { &(*(*addr_list).ad_list) } {
-                let adr = cur2 as *mut mailimf_address;
-
-                if adr.is_null() {
-                    continue;
-                }
-                let adr = unsafe { *adr };
-
-                if adr.ad_type == MAILIMF_ADDRESS_MAILBOX as libc::c_int {
-                    mailimf_get_recipients_add_addr(&mut recipients, unsafe {
-                        adr.ad_data.ad_mailbox
-                    });
-                } else if adr.ad_type == MAILIMF_ADDRESS_GROUP as libc::c_int {
-                    let group = unsafe { adr.ad_data.ad_group };
-                    if !group.is_null() && unsafe { !(*group).grp_mb_list.is_null() } {
-                        for cur3 in unsafe { &(*(*(*group).grp_mb_list).mb_list) } {
-                            mailimf_get_recipients_add_addr(
-                                &mut recipients,
-                                cur3 as *mut mailimf_mailbox,
-                            );
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-    recipients
-}
-
-fn mailimf_get_recipients_add_addr(recipients: &mut HashSet<String>, mb: *mut mailimf_mailbox) {
-    if !mb.is_null() {
-        let addr_norm = addr_normalize(as_str(unsafe { (*mb).mb_addr_spec }));
-        recipients.insert(addr_norm.into());
-    }
-}
-
-/*the result is a pointer to mime, must not be freed*/
-pub fn mailimf_find_field(
-    header: *mut mailimf_fields,
-    wanted_fld_type: libc::c_int,
-) -> *mut mailimf_field {
-    if header.is_null() {
-        return ptr::null_mut();
-    }
-
-    let header = unsafe { (*header) };
-    if header.fld_list.is_null() {
-        return ptr::null_mut();
-    }
-
-    for cur in unsafe { &(*header.fld_list) } {
-        let field = cur as *mut mailimf_field;
-        if !field.is_null() {
-            if unsafe { (*field).fld_type } == wanted_fld_type {
-                return field;
-            }
-        }
-    }
-
-    ptr::null_mut()
-}
-
-/*the result is a pointer to mime, must not be freed*/
-pub unsafe fn mailmime_find_mailimf_fields(mime: *mut Mailmime) -> *mut mailimf_fields {
-    if mime.is_null() {
-        return ptr::null_mut();
-    }
-
-    match (*mime).mm_type as _ {
-        MAILMIME_MULTIPLE => {
-            for cur_data in (*(*mime).mm_data.mm_multipart.mm_mp_list).into_iter() {
-                let header = mailmime_find_mailimf_fields(cur_data as *mut _);
-                if !header.is_null() {
-                    return header;
-                }
-            }
-        }
-        MAILMIME_MESSAGE => return (*mime).mm_data.mm_message.mm_fields,
-        _ => {}
-    }
-
-    ptr::null_mut()
-}
-
-pub unsafe fn mailimf_find_optional_field(
-    header: *mut mailimf_fields,
-    wanted_fld_name: *const libc::c_char,
-) -> *mut mailimf_optional_field {
-    if header.is_null() || (*header).fld_list.is_null() {
-        return ptr::null_mut();
-    }
-    for cur_data in (*(*header).fld_list).into_iter() {
-        let field: *mut mailimf_field = cur_data as *mut _;
-
-        if (*field).fld_type == MAILIMF_FIELD_OPTIONAL_FIELD as libc::c_int {
-            let optional_field: *mut mailimf_optional_field = (*field).fld_data.fld_optional_field;
-            if !optional_field.is_null()
-                && !(*optional_field).fld_name.is_null()
-                && !(*optional_field).fld_value.is_null()
-                && strcasecmp((*optional_field).fld_name, wanted_fld_name) == 0i32
-            {
-                return optional_field;
-            }
-        }
-    }
-
-    ptr::null_mut()
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -1371,14 +1177,13 @@ mod tests {
             let mut mime: *mut Mailmime = ptr::null_mut();
             let mut dummy = 0;
             let res = mailmime_parse(txt, strlen(txt), &mut dummy, &mut mime);
-
             assert_eq!(res, MAIL_NO_ERROR as libc::c_int);
             assert!(!mime.is_null());
 
-            let fields: *mut mailimf_fields = mailmime_find_mailimf_fields(mime);
+            let fields: *mut mailimf_fields = wrapmime::mailmime_find_mailimf_fields(mime);
             assert!(!fields.is_null());
 
-            let mut of_a: *mut mailimf_optional_field = mailimf_find_optional_field(
+            let mut of_a: *mut mailimf_optional_field = wrapmime::mailimf_find_optional_field(
                 fields,
                 b"fielda\x00" as *const u8 as *const libc::c_char,
             );
@@ -1398,7 +1203,7 @@ mod tests {
                 "ValueA",
             );
 
-            of_a = mailimf_find_optional_field(
+            of_a = wrapmime::mailimf_find_optional_field(
                 fields,
                 b"FIELDA\x00" as *const u8 as *const libc::c_char,
             );
@@ -1418,7 +1223,7 @@ mod tests {
                 "ValueA",
             );
 
-            let of_b: *mut mailimf_optional_field = mailimf_find_optional_field(
+            let of_b: *mut mailimf_optional_field = wrapmime::mailimf_find_optional_field(
                 fields,
                 b"FieldB\x00" as *const u8 as *const libc::c_char,
             );
@@ -1441,17 +1246,24 @@ mod tests {
         let context = dummy_context();
         let raw = include_bytes!("../test-data/message/issue_523.txt");
         let mut mimeparser = MimeParser::new(&context.ctx);
-        unsafe { mimeparser.parse(&raw[..]).unwrap() };
+        unsafe {
+            mimeparser.parse(&raw[..]).unwrap();
+        };
         assert_eq!(mimeparser.subject, None);
         assert_eq!(mimeparser.parts.len(), 1);
     }
 
     proptest! {
+        #[ignore]
         #[test]
         fn test_dc_mailmime_parse_crash_fuzzy(data in "[!-~\t ]{2000,}") {
             let context = dummy_context();
             let mut mimeparser = MimeParser::new(&context.ctx);
-            unsafe { mimeparser.parse(data.as_bytes()).unwrap() };
+
+            // parsing should always succeed
+            // but the returned header will normally be empty on random data
+            assert!(unsafe {mimeparser.parse(data.as_bytes()).is_ok()});
+            assert!(mimeparser.header.is_empty());
         }
     }
 
@@ -1480,7 +1292,7 @@ mod tests {
     fn test_mimeparser_with_context() {
         unsafe {
             let context = dummy_context();
-            let raw = b"Content-Type: multipart/mixed; boundary=\"==break==\";\nSubject: outer-subject\nX-Special-A: special-a\nFoo: Bar\nChat-Version: 0.0\n\n--==break==\nContent-Type: text/plain; protected-headers=\"v1\";\nSubject: inner-subject\nX-Special-B: special-b\nFoo: Xy\nChat-Version: 1.0\n\ntest1\n\n--==break==--\n\n\x00";
+            let raw = b"From: hello\nContent-Type: multipart/mixed; boundary=\"==break==\";\nSubject: outer-subject\nX-Special-A: special-a\nFoo: Bar\nChat-Version: 0.0\n\n--==break==\nContent-Type: text/plain; protected-headers=\"v1\";\nSubject: inner-subject\nX-Special-B: special-b\nFoo: Xy\nChat-Version: 1.0\n\ntest1\n\n--==break==--\n\n\x00";
             let mut mimeparser = MimeParser::new(&context.ctx);
             mimeparser.parse(&raw[..]).unwrap();
 

src/dc_simplify.rs

@@ -10,18 +10,16 @@ pub struct Simplify {
 ///
 /// Also return whether not-standard (rfc3676, §4.3) footer is found.
 fn find_message_footer(lines: &[&str]) -> (usize, bool) {
-    for ix in 0..lines.len() {
-        let line = lines[ix];
-
+    for (ix, &line) in lines.iter().enumerate() {
         // quoted-printable may encode `-- ` to `-- =20` which is converted
         // back to `--  `
-        match line.as_ref() {
+        match line {
             "-- " | "--  " => return (ix, false),
             "--" | "---" | "----" => return (ix, true),
             _ => (),
         }
     }
-    return (lines.len(), false);
+    (lines.len(), false)
 }
 
 impl Simplify {
@@ -103,10 +101,8 @@ impl Simplify {
             if let Some(last_quoted_line) = l_lastQuotedLine {
                 l_last = last_quoted_line;
                 is_cut_at_end = true;
-                if l_last > 1 {
-                    if is_empty_line(lines[l_last - 1]) {
-                        l_last -= 1
-                    }
+                if l_last > 1 && is_empty_line(lines[l_last - 1]) {
+                    l_last -= 1
                 }
                 if l_last > 1 {
                     let line = lines[l_last - 1];
@@ -146,8 +142,8 @@ impl Simplify {
             ret += "[...]";
         }
         /* we write empty lines only in case and non-empty line follows */
-        let mut pending_linebreaks: libc::c_int = 0i32;
-        let mut content_lines_added: libc::c_int = 0i32;
+        let mut pending_linebreaks = 0;
+        let mut content_lines_added = 0;
         for l in l_first..l_last {
             let line = lines[l];
             if is_empty_line(line) {
@@ -205,7 +201,7 @@ fn is_quoted_headline(buf: &str) -> bool {
 }
 
 fn is_plain_quote(buf: &str) -> bool {
-    buf.starts_with(">")
+    buf.starts_with('>')
 }
 
 #[cfg(test)]

src/dc_strencode.rs

@@ -4,7 +4,7 @@ use std::ffi::CString;
 use std::ptr;
 
 use charset::Charset;
-use libc::{free, strlen};
+use libc::free;
 use mmime::mailmime::decode::mailmime_encoded_phrase_parse;
 use mmime::other::*;
 use percent_encoding::{percent_decode, utf8_percent_encode, AsciiSet, CONTROLS};
@@ -68,28 +68,7 @@ fn quote_word(word: &[u8]) -> String {
  * Encode/decode header words, RFC 2047
  ******************************************************************************/
 
-pub unsafe fn dc_decode_header_words(in_0: *const libc::c_char) -> *mut libc::c_char {
-    if in_0.is_null() {
-        return ptr::null_mut();
-    }
-    let mut out: *mut libc::c_char = ptr::null_mut();
-    let mut cur_token = 0;
-    let r: libc::c_int = mailmime_encoded_phrase_parse(
-        b"iso-8859-1\x00" as *const u8 as *const libc::c_char,
-        in_0,
-        strlen(in_0),
-        &mut cur_token,
-        b"utf-8\x00" as *const u8 as *const libc::c_char,
-        &mut out,
-    );
-    if r != MAILIMF_NO_ERROR as libc::c_int || out.is_null() {
-        out = dc_strdup(in_0)
-    }
-
-    out
-}
-
-pub fn dc_decode_header_words_safe(input: &str) -> String {
+pub(crate) fn dc_decode_header_words(input: &str) -> String {
     static FROM_ENCODING: &[u8] = b"iso-8859-1\x00";
     static TO_ENCODING: &[u8] = b"utf-8\x00";
     let mut out = ptr::null_mut();
@@ -107,7 +86,7 @@ pub fn dc_decode_header_words_safe(input: &str) -> String {
         if r as u32 != MAILIMF_NO_ERROR || out.is_null() {
             input.to_string()
         } else {
-            let res = to_string(out);
+            let res = to_string_lossy(out);
             free(out.cast());
             res
         }
@@ -177,60 +156,31 @@ pub fn dc_decode_ext_header(to_decode: &[u8]) -> Cow<str> {
 mod tests {
     use super::*;
 
-    use libc::strcmp;
-    use std::ffi::CStr;
-
     #[test]
     fn test_dc_decode_header_words() {
-        unsafe {
-            let mut buf1: *mut libc::c_char = dc_decode_header_words(
-                b"=?utf-8?B?dGVzdMOkw7bDvC50eHQ=?=\x00" as *const u8 as *const libc::c_char,
-            );
-            assert_eq!(
-                strcmp(
-                    buf1,
-                    b"test\xc3\xa4\xc3\xb6\xc3\xbc.txt\x00" as *const u8 as *const libc::c_char
-                ),
-                0
-            );
-            free(buf1 as *mut libc::c_void);
+        assert_eq!(
+            dc_decode_header_words("=?utf-8?B?dGVzdMOkw7bDvC50eHQ=?="),
+            std::string::String::from_utf8(b"test\xc3\xa4\xc3\xb6\xc3\xbc.txt".to_vec()).unwrap(),
+        );
 
-            buf1 =
-                dc_decode_header_words(b"just ascii test\x00" as *const u8 as *const libc::c_char);
-            assert_eq!(CStr::from_ptr(buf1).to_str().unwrap(), "just ascii test");
-            free(buf1 as *mut libc::c_void);
+        assert_eq!(dc_decode_header_words("just ascii test"), "just ascii test");
 
-            assert_eq!(dc_encode_header_words("abcdef"), "abcdef");
+        assert_eq!(dc_encode_header_words("abcdef"), "abcdef");
 
-            let r = dc_encode_header_words(
-                std::string::String::from_utf8(b"test\xc3\xa4\xc3\xb6\xc3\xbc.txt".to_vec())
-                    .unwrap(),
+        let r = dc_encode_header_words(
+            std::string::String::from_utf8(b"test\xc3\xa4\xc3\xb6\xc3\xbc.txt".to_vec()).unwrap(),
+        );
+        assert!(r.starts_with("=?utf-8"));
+
+        assert_eq!(
+            dc_decode_header_words(&r),
+            std::string::String::from_utf8(b"test\xc3\xa4\xc3\xb6\xc3\xbc.txt".to_vec()).unwrap(),
+        );
+
+        assert_eq!(
+                dc_decode_header_words("=?ISO-8859-1?Q?attachment=3B=0D=0A_filename=3D?= =?ISO-8859-1?Q?=22test=E4=F6=FC=2Etxt=22=3B=0D=0A_size=3D39?="),
+                std::string::String::from_utf8(b"attachment;\r\n filename=\"test\xc3\xa4\xc3\xb6\xc3\xbc.txt\";\r\n size=39".to_vec()).unwrap(),
             );
-            assert!(r.starts_with("=?utf-8"));
-
-            buf1 = r.strdup();
-            let buf2: *mut libc::c_char = dc_decode_header_words(buf1);
-            assert_eq!(
-                strcmp(
-                    buf2,
-                    b"test\xc3\xa4\xc3\xb6\xc3\xbc.txt\x00" as *const u8 as *const libc::c_char
-                ),
-                0
-            );
-            free(buf2 as *mut libc::c_void);
-
-            buf1 = dc_decode_header_words(
-                b"=?ISO-8859-1?Q?attachment=3B=0D=0A_filename=3D?= =?ISO-8859-1?Q?=22test=E4=F6=FC=2Etxt=22=3B=0D=0A_size=3D39?=\x00" as *const u8 as *const libc::c_char
-            );
-            assert_eq!(
-                strcmp(
-                    buf1,
-                    b"attachment;\r\n filename=\"test\xc3\xa4\xc3\xb6\xc3\xbc.txt\";\r\n size=39\x00" as *const u8 as *const libc::c_char,
-
-                ),
-                0
-            );
-        }
     }
 
     #[test]
@@ -293,7 +243,7 @@ mod tests {
         #[test]
         fn test_dc_header_roundtrip(input: String) {
             let encoded = dc_encode_header_words(&input);
-            let decoded = dc_decode_header_words_safe(&encoded);
+            let decoded = dc_decode_header_words(&encoded);
 
             assert_eq!(input, decoded);
         }

src/dc_tools.rs

@@ -1,8 +1,9 @@
 //! Some tools and enhancements to the used libraries, there should be
 //! no references to Context and other "larger" entities here.
 
+use core::cmp::max;
 use std::borrow::Cow;
-use std::ffi::{CStr, CString, OsString};
+use std::ffi::{CStr, CString};
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 use std::time::SystemTime;
@@ -16,6 +17,7 @@ use rand::{thread_rng, Rng};
 
 use crate::context::Context;
 use crate::error::Error;
+use crate::events::Event;
 
 pub(crate) fn dc_exactly_one_bit_set(v: libc::c_int) -> bool {
     0 != v && 0 == v & (v - 1)
@@ -28,11 +30,11 @@ pub(crate) fn dc_exactly_one_bit_set(v: libc::c_int) -> bool {
 /// # Examples
 ///
 /// ```
-/// use deltachat::dc_tools::{dc_strdup, to_string};
+/// use deltachat::dc_tools::{dc_strdup, to_string_lossy};
 /// unsafe {
 ///     let str_a = b"foobar\x00" as *const u8 as *const libc::c_char;
 ///     let str_a_copy = dc_strdup(str_a);
-///     assert_eq!(to_string(str_a_copy), "foobar");
+///     assert_eq!(to_string_lossy(str_a_copy), "foobar");
 ///     assert_ne!(str_a, str_a_copy);
 /// }
 /// ```
@@ -49,83 +51,6 @@ pub unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char {
     ret
 }
 
-pub(crate) fn dc_atoi_null_is_0(s: *const libc::c_char) -> libc::c_int {
-    if !s.is_null() {
-        as_str(s).parse().unwrap_or_default()
-    } else {
-        0
-    }
-}
-
-unsafe fn dc_ltrim(buf: *mut libc::c_char) {
-    let mut len: libc::size_t;
-    let mut cur: *const libc::c_uchar;
-    if !buf.is_null() && 0 != *buf as libc::c_int {
-        len = strlen(buf);
-        cur = buf as *const libc::c_uchar;
-        while 0 != *cur as libc::c_int && 0 != libc::isspace(*cur as libc::c_int) {
-            cur = cur.offset(1isize);
-            len = len.wrapping_sub(1)
-        }
-        if buf as *const libc::c_uchar != cur {
-            libc::memmove(
-                buf as *mut libc::c_void,
-                cur as *const libc::c_void,
-                len.wrapping_add(1),
-            );
-        }
-    };
-}
-
-unsafe fn dc_rtrim(buf: *mut libc::c_char) {
-    let mut len: libc::size_t;
-    let mut cur: *mut libc::c_uchar;
-    if !buf.is_null() && 0 != *buf as libc::c_int {
-        len = strlen(buf);
-        cur = (buf as *mut libc::c_uchar)
-            .offset(len as isize)
-            .offset(-1isize);
-        while cur != buf as *mut libc::c_uchar && 0 != libc::isspace(*cur as libc::c_int) {
-            cur = cur.offset(-1isize);
-            len = len.wrapping_sub(1)
-        }
-        *cur.offset(
-            (if 0 != libc::isspace(*cur as libc::c_int) {
-                0
-            } else {
-                1
-            }) as isize,
-        ) = '\u{0}' as i32 as libc::c_uchar
-    };
-}
-
-pub(crate) unsafe fn dc_trim(buf: *mut libc::c_char) {
-    dc_ltrim(buf);
-    dc_rtrim(buf);
-}
-
-/* remove all \r characters from string */
-pub(crate) unsafe fn dc_remove_cr_chars(buf: *mut libc::c_char) {
-    /* search for first `\r` */
-    let mut p1: *const libc::c_char = buf;
-    while 0 != *p1 {
-        if *p1 as libc::c_int == '\r' as i32 {
-            break;
-        }
-        p1 = p1.offset(1isize)
-    }
-    /* p1 is `\r` or null-byte; start removing `\r` */
-    let mut p2: *mut libc::c_char = p1 as *mut libc::c_char;
-    while 0 != *p1 {
-        if *p1 as libc::c_int != '\r' as i32 {
-            *p2 = *p1;
-            p2 = p2.offset(1isize)
-        }
-        p1 = p1.offset(1isize)
-    }
-    *p2 = 0 as libc::c_char;
-}
-
 /// Shortens a string to a specified length and adds "..." or "[...]" to the end of
 /// the shortened string.
 pub(crate) fn dc_truncate(buf: &str, approx_chars: usize, do_unwrap: bool) -> Cow<str> {
@@ -140,7 +65,7 @@ pub(crate) fn dc_truncate(buf: &str, approx_chars: usize, do_unwrap: bool) -> Co
             .unwrap_or_default();
 
         if let Some(index) = buf[..end_pos].rfind(|c| c == ' ' || c == '\n') {
-            Cow::Owned(format!("{}{}", &buf[..index + 1], ellipse))
+            Cow::Owned(format!("{}{}", &buf[..=index], ellipse))
         } else {
             Cow::Owned(format!("{}{}", &buf[..end_pos], ellipse))
         }
@@ -149,36 +74,18 @@ pub(crate) fn dc_truncate(buf: &str, approx_chars: usize, do_unwrap: bool) -> Co
     }
 }
 
-pub(crate) unsafe fn dc_str_from_clist(
-    list: *const clist,
-    delimiter: *const libc::c_char,
-) -> *mut libc::c_char {
+pub(crate) fn dc_str_from_clist(list: *const clist, delimiter: &str) -> String {
     let mut res = String::new();
 
     if !list.is_null() {
-        let mut cur: *mut clistiter = (*list).first;
-        while !cur.is_null() {
-            let rfc724_mid = (if !cur.is_null() {
-                (*cur).data
-            } else {
-                ptr::null_mut()
-            }) as *const libc::c_char;
-
-            if !rfc724_mid.is_null() {
-                if !res.is_empty() && !delimiter.is_null() {
-                    res += as_str(delimiter);
-                }
-                res += as_str(rfc724_mid);
-            }
-            cur = if !cur.is_null() {
-                (*cur).next
-            } else {
-                ptr::null_mut()
+        for rfc724_mid in unsafe { (*list).into_iter() } {
+            if !res.is_empty() {
+                res += delimiter;
             }
+            res += &to_string_lossy(rfc724_mid as *const libc::c_char);
         }
     }
-
-    res.strdup()
+    res
 }
 
 pub(crate) fn dc_str_to_clist(str: &str, delimiter: &str) -> *mut clist {
@@ -247,13 +154,15 @@ pub(crate) fn dc_timestamp_from_date(date_time: *mut mailimf_date_time) -> i64 {
  ******************************************************************************/
 
 pub fn dc_timestamp_to_str(wanted: i64) -> String {
-    let ts = chrono::Utc.timestamp(wanted, 0);
+    let ts = Local.timestamp(wanted, 0);
     ts.format("%Y.%m.%d %H:%M:%S").to_string()
 }
 
 pub(crate) fn dc_gm2local_offset() -> i64 {
+    /* returns the offset that must be _added_ to an UTC/GMT-time to create the localtime.
+    the function may return negative values. */
     let lt = Local::now();
-    ((lt.offset().local_minus_utc() / (60 * 60)) * 100) as i64
+    lt.offset().local_minus_utc() as i64
 }
 
 /* timesmearing */
@@ -333,21 +242,19 @@ fn encode_66bits_as_base64(v1: u32, v2: u32, fill: u32) -> String {
         enc.write_u8(((fill & 0x3) as u8) << 6).unwrap();
         enc.finish().unwrap();
     }
-    assert_eq!(wrapped_writer.pop(), Some('A' as u8)); // Remove last "A"
+    assert_eq!(wrapped_writer.pop(), Some(b'A')); // Remove last "A"
     String::from_utf8(wrapped_writer).unwrap()
 }
 
 pub(crate) fn dc_create_incoming_rfc724_mid(
     message_timestamp: i64,
     contact_id_from: u32,
-    contact_ids_to: &Vec<u32>,
+    contact_ids_to: &[u32],
 ) -> Option<String> {
-    if contact_ids_to.is_empty() {
-        return None;
-    }
-    /* find out the largest receiver ID (we could also take the smallest, but it should be unique) */
-    let largest_id_to = contact_ids_to.iter().max().copied().unwrap_or_default();
+    /* create a deterministic rfc724_mid from input such that
+    repeatedly calling it with the same input results in the same Message-id */
 
+    let largest_id_to = contact_ids_to.iter().max().copied().unwrap_or_default();
     let result = format!(
         "{}-{}-{}@stub",
         message_timestamp, contact_id_from, largest_id_to
@@ -396,9 +303,9 @@ pub(crate) fn dc_extract_grpid_from_rfc724_mid_list(list: *const clist) -> *mut
     if !list.is_null() {
         unsafe {
             for cur in (*list).into_iter() {
-                let mid = as_str(cur as *const libc::c_char);
+                let mid = to_string_lossy(cur as *const libc::c_char);
 
-                if let Some(grpid) = dc_extract_grpid_from_rfc724_mid(mid) {
+                if let Some(grpid) = dc_extract_grpid_from_rfc724_mid(&mid) {
                     return grpid.strdup();
                 }
             }
@@ -415,12 +322,41 @@ pub(crate) fn dc_ensure_no_slash_safe(path: &str) -> &str {
     path
 }
 
-/// Function modifies the given buffer and replaces all characters not valid in filenames by a "-".
-fn validate_filename(filename: &str) -> String {
-    filename
-        .replace('/', "-")
-        .replace('\\', "-")
-        .replace(':', "-")
+// Function returns a sanitized basename that does not contain
+// win/linux path separators and also not any non-ascii chars
+fn get_safe_basename(filename: &str) -> String {
+    // return the (potentially mangled) basename of the input filename
+    // this might be a path that comes in from another operating system
+    let mut index: usize = 0;
+
+    if let Some(unix_index) = filename.rfind('/') {
+        index = unix_index + 1;
+    }
+    if let Some(win_index) = filename.rfind('\\') {
+        index = max(index, win_index + 1);
+    }
+    if index >= filename.len() {
+        "nobasename".to_string()
+    } else {
+        // we don't allow any non-ascii to be super-safe
+        filename[index..].replace(|c: char| !c.is_ascii() || c == ':', "-")
+    }
+}
+
+pub fn dc_derive_safe_stem_ext(filename: &str) -> (String, String) {
+    let basename = get_safe_basename(&filename);
+    let (mut stem, mut ext) = if let Some(index) = basename.rfind('.') {
+        (
+            basename[0..index].to_string(),
+            basename[index..].to_string(),
+        )
+    } else {
+        (basename, "".to_string())
+    };
+    // limit length of stem and ext
+    stem.truncate(32);
+    ext.truncate(32);
+    (stem, ext)
 }
 
 // the returned suffix is lower-case
@@ -456,10 +392,6 @@ pub(crate) fn dc_get_abs_path<P: AsRef<std::path::Path>>(
     }
 }
 
-pub(crate) fn dc_file_exist(context: &Context, path: impl AsRef<std::path::Path>) -> bool {
-    dc_get_abs_path(context, &path).exists()
-}
-
 pub(crate) fn dc_get_filebytes(context: &Context, path: impl AsRef<std::path::Path>) -> u64 {
     let path_abs = dc_get_abs_path(context, &path);
     match fs::metadata(&path_abs) {
@@ -482,10 +414,14 @@ pub(crate) fn dc_delete_file(context: &Context, path: impl AsRef<std::path::Path
         return false;
     }
 
+    let dpath = format!("{}", path.as_ref().to_string_lossy());
     match fs::remove_file(path_abs) {
-        Ok(_) => true,
-        Err(_err) => {
-            warn!(context, "Cannot delete \"{}\".", path.as_ref().display());
+        Ok(_) => {
+            context.call_cb(Event::DeletedBlobFile(dpath));
+            true
+        }
+        Err(err) => {
+            warn!(context, "Cannot delete \"{}\": {}", dpath, err);
             false
         }
     }
@@ -493,20 +429,55 @@ pub(crate) fn dc_delete_file(context: &Context, path: impl AsRef<std::path::Path
 
 pub(crate) fn dc_copy_file(
     context: &Context,
-    src: impl AsRef<std::path::Path>,
-    dest: impl AsRef<std::path::Path>,
+    src_path: impl AsRef<std::path::Path>,
+    dest_path: impl AsRef<std::path::Path>,
 ) -> bool {
-    let src_abs = dc_get_abs_path(context, &src);
-    let dest_abs = dc_get_abs_path(context, &dest);
-    match fs::copy(&src_abs, &dest_abs) {
+    let src_abs = dc_get_abs_path(context, &src_path);
+    let mut src_file = match fs::File::open(&src_abs) {
+        Ok(file) => file,
+        Err(err) => {
+            warn!(
+                context,
+                "failed to open for read '{}': {}",
+                src_abs.display(),
+                err
+            );
+            return false;
+        }
+    };
+
+    let dest_abs = dc_get_abs_path(context, &dest_path);
+    let mut dest_file = match fs::OpenOptions::new()
+        .create_new(true)
+        .write(true)
+        .open(&dest_abs)
+    {
+        Ok(file) => file,
+        Err(err) => {
+            warn!(
+                context,
+                "failed to open for write '{}': {}",
+                dest_abs.display(),
+                err
+            );
+            return false;
+        }
+    };
+
+    match std::io::copy(&mut src_file, &mut dest_file) {
         Ok(_) => true,
-        Err(_) => {
+        Err(err) => {
             error!(
                 context,
-                "Cannot copy \"{}\" to \"{}\".",
-                src.as_ref().display(),
-                dest.as_ref().display(),
+                "Cannot copy \"{}\" to \"{}\": {}",
+                src_abs.display(),
+                dest_abs.display(),
+                err
             );
+            {
+                // Attempt to remove the failed file, swallow errors resulting from that.
+                fs::remove_file(dest_abs).ok();
+            }
             false
         }
     }
@@ -517,11 +488,12 @@ pub(crate) fn dc_create_folder(context: &Context, path: impl AsRef<std::path::Pa
     if !path_abs.exists() {
         match fs::create_dir_all(path_abs) {
             Ok(_) => true,
-            Err(_err) => {
+            Err(err) => {
                 warn!(
                     context,
-                    "Cannot create directory \"{}\".",
+                    "Cannot create directory \"{}\": {}",
                     path.as_ref().display(),
+                    err
                 );
                 false
             }
@@ -534,12 +506,13 @@ pub(crate) fn dc_create_folder(context: &Context, path: impl AsRef<std::path::Pa
 /// Write a the given content to provied file path.
 pub(crate) fn dc_write_file(context: &Context, path: impl AsRef<Path>, buf: &[u8]) -> bool {
     let path_abs = dc_get_abs_path(context, &path);
-    if let Err(_err) = fs::write(&path_abs, buf) {
+    if let Err(err) = fs::write(&path_abs, buf) {
         warn!(
             context,
-            "Cannot write {} bytes to \"{}\".",
+            "Cannot write {} bytes to \"{}\": {}",
             buf.len(),
             path.as_ref().display(),
+            err
         );
         false
     } else {
@@ -558,92 +531,53 @@ pub fn dc_read_file<P: AsRef<std::path::Path>>(
         Err(err) => {
             warn!(
                 context,
-                "Cannot read \"{}\" or file is empty.",
-                path.as_ref().display()
+                "Cannot read \"{}\" or file is empty: {}",
+                path.as_ref().display(),
+                err
             );
             Err(err.into())
         }
     }
 }
 
-pub(crate) fn dc_get_fine_path_filename(
+pub fn dc_open_file<P: AsRef<std::path::Path>>(
     context: &Context,
-    folder: impl AsRef<Path>,
-    desired_filename_suffix: impl AsRef<str>,
-) -> PathBuf {
-    let now = time();
+    path: P,
+) -> Result<std::fs::File, Error> {
+    let path_abs = dc_get_abs_path(context, &path);
 
-    let folder = PathBuf::from(folder.as_ref());
-    // XXX sanitize desired_filename eg using
-    // https://github.com/kardeiz/sanitize-filename/blob/master/src/lib.rs
-    let suffix = validate_filename(desired_filename_suffix.as_ref());
-    let file_name = PathBuf::from(suffix);
-    let extension = file_name.extension().map(|c| c.clone());
-
-    for i in 0..100_000 {
-        let ret = if i == 0 {
-            let mut folder = folder.clone();
-            folder.push(&file_name);
-            folder
-        } else {
-            let idx = if i < 100 { i } else { now + i };
-            let file_name = if let Some(stem) = file_name.file_stem() {
-                let mut stem = stem.to_os_string();
-                stem.push(format!("-{}", idx));
-                stem
-            } else {
-                OsString::from(idx.to_string())
-            };
-            let mut folder = folder.clone();
-            folder.push(file_name);
-            if let Some(ext) = extension {
-                folder.set_extension(&ext);
-            }
-            folder
-        };
-
-        if !dc_file_exist(context, &ret) {
-            // fine filename found
-            return ret;
+    match fs::File::open(&path_abs) {
+        Ok(bytes) => Ok(bytes),
+        Err(err) => {
+            warn!(
+                context,
+                "Cannot read \"{}\" or file is empty: {}",
+                path.as_ref().display(),
+                err
+            );
+            Err(err.into())
         }
     }
-
-    panic!("Something is really wrong, you need to clean up your disk");
 }
 
-pub(crate) fn dc_is_blobdir_path(context: &Context, path: impl AsRef<str>) -> bool {
-    context
-        .get_blobdir()
-        .to_str()
-        .map(|s| path.as_ref().starts_with(s))
-        .unwrap_or_default()
-        || path.as_ref().starts_with("$BLOBDIR")
-}
+pub(crate) fn dc_get_next_backup_path(
+    folder: impl AsRef<Path>,
+    backup_time: i64,
+) -> Result<PathBuf, Error> {
+    let folder = PathBuf::from(folder.as_ref());
+    let stem = chrono::NaiveDateTime::from_timestamp(backup_time, 0)
+        .format("delta-chat-%Y-%m-%d")
+        .to_string();
 
-fn dc_make_rel_path(context: &Context, path: &mut String) {
-    if context
-        .get_blobdir()
-        .to_str()
-        .map(|s| path.starts_with(s))
-        .unwrap_or_default()
-    {
-        *path = path.replace(context.get_blobdir().to_str().unwrap(), "$BLOBDIR");
+    // 64 backup files per day should be enough for everyone
+    for i in 0..64 {
+        let mut path = folder.clone();
+        path.push(format!("{}-{}.bak", stem, i));
+        if !path.exists() {
+            return Ok(path);
+        }
     }
-}
-
-pub(crate) fn dc_make_rel_and_copy(context: &Context, path: &mut String) -> bool {
-    if dc_is_blobdir_path(context, &path) {
-        dc_make_rel_path(context, path);
-        return true;
-    }
-    let blobdir_path = dc_get_fine_path_filename(context, "$BLOBDIR", &path);
-    if dc_copy_file(context, &path, &blobdir_path) {
-        *path = blobdir_path.to_string_lossy().to_string();
-        dc_make_rel_path(context, path);
-        return true;
-    }
-
-    false
+    bail!("could not create backup file, disk full?");
 }
 
 /// Error type for the [OsStrExt] trait
@@ -791,22 +725,6 @@ impl<T: AsRef<str>> StrExt for T {
     }
 }
 
-pub fn to_string(s: *const libc::c_char) -> String {
-    if s.is_null() {
-        return "".into();
-    }
-
-    let cstr = unsafe { CStr::from_ptr(s) };
-
-    cstr.to_str().map(|s| s.to_string()).unwrap_or_else(|err| {
-        panic!(
-            "Non utf8 string: '{:?}' ({:?})",
-            cstr.to_string_lossy(),
-            err
-        );
-    })
-}
-
 pub fn to_string_lossy(s: *const libc::c_char) -> String {
     if s.is_null() {
         return "".into();
@@ -814,30 +732,15 @@ pub fn to_string_lossy(s: *const libc::c_char) -> String {
 
     let cstr = unsafe { CStr::from_ptr(s) };
 
-    cstr.to_str()
-        .map(|s| s.to_string())
-        .unwrap_or_else(|_| cstr.to_string_lossy().to_string())
+    cstr.to_string_lossy().to_string()
 }
 
-pub fn as_str<'a>(s: *const libc::c_char) -> &'a str {
-    as_str_safe(s).unwrap_or_else(|err| panic!("{}", err))
-}
-
-/// Converts a C string to either a Rust `&str` or `None` if  it is a null pointer.
-pub fn as_opt_str<'a>(s: *const libc::c_char) -> Option<&'a str> {
+pub fn to_opt_string_lossy(s: *const libc::c_char) -> Option<String> {
     if s.is_null() {
         return None;
     }
-    Some(as_str(s))
-}
 
-fn as_str_safe<'a>(s: *const libc::c_char) -> Result<&'a str, Error> {
-    assert!(!s.is_null(), "cannot be used on null pointers");
-
-    let cstr = unsafe { CStr::from_ptr(s) };
-
-    cstr.to_str()
-        .map_err(|err| format_err!("Non utf8 string: '{:?}' ({:?})", cstr.to_bytes(), err))
+    Some(to_string_lossy(s))
 }
 
 /// Convert a C `*char` pointer to a [std::path::Path] slice.
@@ -876,7 +779,11 @@ pub fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path {
 #[allow(dead_code)]
 fn as_path_unicode<'a>(s: *const libc::c_char) -> &'a std::path::Path {
     assert!(!s.is_null(), "cannot be used on null pointers");
-    std::path::Path::new(as_str(s))
+
+    let cstr = unsafe { CStr::from_ptr(s) };
+    let str = cstr.to_str().unwrap_or_else(|err| panic!("{}", err));
+
+    std::path::Path::new(str)
 }
 
 pub(crate) fn time() -> i64 {
@@ -1021,54 +928,6 @@ mod tests {
         }
     }
 
-    #[test]
-    fn test_dc_ltrim() {
-        unsafe {
-            let html: *const libc::c_char =
-                b"\r\r\nline1<br>\r\n\r\n\r\rline2\n\r\x00" as *const u8 as *const libc::c_char;
-            let out: *mut libc::c_char = strndup(html, strlen(html) as libc::c_ulong);
-
-            dc_ltrim(out);
-
-            assert_eq!(
-                CStr::from_ptr(out as *const libc::c_char).to_str().unwrap(),
-                "line1<br>\r\n\r\n\r\rline2\n\r"
-            );
-        }
-    }
-
-    #[test]
-    fn test_dc_rtrim() {
-        unsafe {
-            let html: *const libc::c_char =
-                b"\r\r\nline1<br>\r\n\r\n\r\rline2\n\r\x00" as *const u8 as *const libc::c_char;
-            let out: *mut libc::c_char = strndup(html, strlen(html) as libc::c_ulong);
-
-            dc_rtrim(out);
-
-            assert_eq!(
-                CStr::from_ptr(out as *const libc::c_char).to_str().unwrap(),
-                "\r\r\nline1<br>\r\n\r\n\r\rline2"
-            );
-        }
-    }
-
-    #[test]
-    fn test_dc_trim() {
-        unsafe {
-            let html: *const libc::c_char =
-                b"\r\r\nline1<br>\r\n\r\n\r\rline2\n\r\x00" as *const u8 as *const libc::c_char;
-            let out: *mut libc::c_char = strndup(html, strlen(html) as libc::c_ulong);
-
-            dc_trim(out);
-
-            assert_eq!(
-                CStr::from_ptr(out as *const libc::c_char).to_str().unwrap(),
-                "line1<br>\r\n\r\n\r\rline2"
-            );
-        }
-    }
-
     #[test]
     fn test_rust_ftoa() {
         assert_eq!("1.22", format!("{}", 1.22));
@@ -1144,21 +1003,6 @@ mod tests {
         }
     }
 
-    fn strndup(s: *const libc::c_char, n: libc::c_ulong) -> *mut libc::c_char {
-        if s.is_null() {
-            return std::ptr::null_mut();
-        }
-
-        let end = std::cmp::min(n as usize, unsafe { strlen(s) });
-        unsafe {
-            let result = libc::malloc(end + 1);
-            memcpy(result, s as *const _, end);
-            std::ptr::write_bytes(result.offset(end as isize), b'\x00', 1);
-
-            result as *mut _
-        }
-    }
-
     #[test]
     fn test_dc_str_to_clist_1() {
         unsafe {
@@ -1174,17 +1018,11 @@ mod tests {
         unsafe {
             let list: *mut clist = dc_str_to_clist("foo bar test", " ");
             assert_eq!((*list).count, 3);
-            let str: *mut libc::c_char =
-                dc_str_from_clist(list, b" \x00" as *const u8 as *const libc::c_char);
-
-            assert_eq!(
-                CStr::from_ptr(str as *const libc::c_char).to_str().unwrap(),
-                "foo bar test"
-            );
+            let str = dc_str_from_clist(list, " ");
+            assert_eq!(str, "foo bar test");
 
             clist_free_content(list);
             clist_free(list);
-            free(str as *mut libc::c_void);
         }
     }
 
@@ -1345,6 +1183,25 @@ mod tests {
         assert_eq!(grpid, Some("1234567890123456"));
     }
 
+    #[test]
+    fn test_dc_create_outgoing_rfc724_mid() {
+        // create a normal message-id
+        let mid = dc_create_outgoing_rfc724_mid(None, "foo@bar.de");
+        assert!(mid.starts_with("Mr."));
+        assert!(mid.ends_with("bar.de"));
+        assert!(dc_extract_grpid_from_rfc724_mid(mid.as_str()).is_none());
+
+        // create a message-id containing a group-id
+        let grpid = dc_create_id();
+        let mid = dc_create_outgoing_rfc724_mid(Some(&grpid), "foo@bar.de");
+        assert!(mid.starts_with("Gr."));
+        assert!(mid.ends_with("bar.de"));
+        assert_eq!(
+            dc_extract_grpid_from_rfc724_mid(mid.as_str()),
+            Some(grpid.as_str())
+        );
+    }
+
     #[test]
     fn test_emailaddress_parse() {
         assert_eq!(EmailAddress::new("").is_ok(), false);
@@ -1409,38 +1266,32 @@ mod tests {
     fn test_dc_create_incoming_rfc724_mid() {
         let res = dc_create_incoming_rfc724_mid(123, 45, &vec![6, 7]);
         assert_eq!(res, Some("123-45-7@stub".into()));
+        let res = dc_create_incoming_rfc724_mid(123, 45, &vec![]);
+        assert_eq!(res, Some("123-45-0@stub".into()));
     }
 
     #[test]
-    fn test_dc_make_rel_path() {
-        let t = dummy_context();
-        let mut foo: String = t
-            .ctx
-            .get_blobdir()
-            .join("foo")
-            .to_string_lossy()
-            .into_owned();
-        dc_make_rel_path(&t.ctx, &mut foo);
-        assert_eq!(foo, format!("$BLOBDIR{}foo", std::path::MAIN_SEPARATOR));
-    }
-
-    #[test]
-    fn test_strndup() {
-        unsafe {
-            let res = strndup(b"helloworld\x00" as *const u8 as *const libc::c_char, 4);
-            assert_eq!(
-                to_string(res),
-                to_string(b"hell\x00" as *const u8 as *const libc::c_char)
-            );
-            assert_eq!(strlen(res), 4);
-            free(res as *mut _);
-        }
+    fn test_file_get_safe_basename() {
+        assert_eq!(get_safe_basename("12312/hello"), "hello");
+        assert_eq!(get_safe_basename("12312\\hello"), "hello");
+        assert_eq!(get_safe_basename("//12312\\hello"), "hello");
+        assert_eq!(get_safe_basename("//123:12\\hello"), "hello");
+        assert_eq!(get_safe_basename("//123:12/\\\\hello"), "hello");
+        assert_eq!(get_safe_basename("//123:12//hello"), "hello");
+        assert_eq!(get_safe_basename("//123:12//"), "nobasename");
+        assert_eq!(get_safe_basename("//123:12/"), "nobasename");
+        assert!(get_safe_basename("123\x012.hello").ends_with(".hello"));
     }
 
     #[test]
     fn test_file_handling() {
         let t = dummy_context();
         let context = &t.ctx;
+        let dc_file_exist = |ctx: &Context, fname: &str| {
+            ctx.get_blobdir()
+                .join(Path::new(fname).file_name().unwrap())
+                .exists()
+        };
 
         assert!(!dc_delete_file(context, "$BLOBDIR/lkqwjelqkwlje"));
         if dc_file_exist(context, "$BLOBDIR/foobar")
@@ -1464,13 +1315,13 @@ mod tests {
             .to_string_lossy()
             .to_string();
 
-        assert!(dc_is_blobdir_path(context, &abs_path));
-
-        assert!(dc_is_blobdir_path(context, "$BLOBDIR/fofo",));
-        assert!(!dc_is_blobdir_path(context, "/BLOBDIR/fofo",));
         assert!(dc_file_exist(context, &abs_path));
 
         assert!(dc_copy_file(context, "$BLOBDIR/foobar", "$BLOBDIR/dada",));
+
+        // attempting to copy a second time should fail
+        assert!(!dc_copy_file(context, "$BLOBDIR/foobar", "$BLOBDIR/dada",));
+
         assert_eq!(dc_get_filebytes(context, "$BLOBDIR/dada",), 7);
 
         let buf = dc_read_file(context, "$BLOBDIR/dada").unwrap();
@@ -1483,14 +1334,12 @@ mod tests {
         assert!(dc_create_folder(context, "$BLOBDIR/foobar-folder"));
         assert!(dc_file_exist(context, "$BLOBDIR/foobar-folder",));
         assert!(!dc_delete_file(context, "$BLOBDIR/foobar-folder"));
-        let fn0 = dc_get_fine_path_filename(context, "$BLOBDIR", "foobar.dadada");
-        assert_eq!(fn0, PathBuf::from("$BLOBDIR/foobar.dadada"));
 
+        let fn0 = "$BLOBDIR/data.data";
         assert!(dc_write_file(context, &fn0, b"content"));
-        let fn1 = dc_get_fine_path_filename(context, "$BLOBDIR", "foobar.dadada");
-        assert_eq!(fn1, PathBuf::from("$BLOBDIR/foobar-1.dadada"));
 
         assert!(dc_delete_file(context, &fn0));
+        assert!(!dc_file_exist(context, &fn0));
     }
 
     #[test]
@@ -1506,19 +1355,4 @@ mod tests {
         let listflags: u32 = DC_GCL_VERIFIED_ONLY.try_into().unwrap();
         assert!(listflags_has(listflags, DC_GCL_ADD_SELF) == false);
     }
-
-    #[test]
-    fn test_dc_remove_cr_chars() {
-        unsafe {
-            let input = "foo\r\nbar".strdup();
-            dc_remove_cr_chars(input);
-            assert_eq!("foo\nbar", to_string(input));
-            free(input.cast());
-
-            let input = "\rfoo\r\rbar\r".strdup();
-            dc_remove_cr_chars(input);
-            assert_eq!("foobar", to_string(input));
-            free(input.cast());
-        }
-    }
 }

src/e2ee.rs

@@ -22,14 +22,13 @@ use num_traits::FromPrimitive;
 use crate::aheader::*;
 use crate::config::Config;
 use crate::context::Context;
-use crate::dc_mimeparser::*;
 use crate::dc_tools::*;
 use crate::error::*;
 use crate::key::*;
 use crate::keyring::*;
 use crate::mimefactory::MimeFactory;
 use crate::peerstate::*;
-use crate::pgp::*;
+use crate::pgp;
 use crate::securejoin::handle_degrade_event;
 use crate::wrapmime;
 use crate::wrapmime::*;
@@ -47,12 +46,9 @@ pub struct EncryptHelper {
 
 impl EncryptHelper {
     pub fn new(context: &Context) -> Result<EncryptHelper> {
-        let prefer_encrypt = context
-            .sql
-            .get_config_int(&context, "e2ee_enabled")
-            .and_then(EncryptPreference::from_i32)
-            .unwrap_or_default();
-
+        let prefer_encrypt =
+            EncryptPreference::from_i32(context.get_config_int(Config::E2eeEnabled))
+                .unwrap_or_default();
         let addr = match context.get_config(Config::ConfiguredAddr) {
             None => {
                 bail!("addr not configured!");
@@ -215,7 +211,7 @@ impl EncryptHelper {
                 "could not write/allocate"
             );
 
-            let ctext = dc_pgp_pk_encrypt(
+            let ctext = pgp::pk_encrypt(
                 std::slice::from_raw_parts((*plain).str_0 as *const u8, (*plain).len),
                 &keyring,
                 sign_key.as_ref(),
@@ -266,103 +262,80 @@ pub fn try_decrypt(
     context: &Context,
     in_out_message: *mut Mailmime,
 ) -> Result<(bool, HashSet<String>, HashSet<String>)> {
+    // just a pointer into mailmime structure, must not be freed
+    let imffields = mailmime_find_mailimf_fields(in_out_message);
+    ensure!(
+        !in_out_message.is_null() && !imffields.is_null(),
+        "corrupt invalid mime inputs"
+    );
+
+    let from = wrapmime::get_field_from(imffields)?;
+    let message_time = wrapmime::get_field_date(imffields)?;
+
+    let mut peerstate = None;
+    let autocryptheader = Aheader::from_imffields(&from, imffields);
+
+    if message_time > 0 {
+        peerstate = Peerstate::from_addr(context, &context.sql, &from);
+
+        if let Some(ref mut peerstate) = peerstate {
+            if let Some(ref header) = autocryptheader {
+                peerstate.apply_header(&header, message_time);
+                peerstate.save_to_db(&context.sql, false)?;
+            } else if message_time > peerstate.last_seen_autocrypt
+                && !contains_report(in_out_message)
+            {
+                peerstate.degrade_encryption(message_time);
+                peerstate.save_to_db(&context.sql, false)?;
+            }
+        } else if let Some(ref header) = autocryptheader {
+            let p = Peerstate::from_header(context, header, message_time);
+            p.save_to_db(&context.sql, true)?;
+            peerstate = Some(p);
+        }
+    }
+    /* possibly perform decryption */
+    let mut private_keyring = Keyring::default();
+    let mut public_keyring_for_validate = Keyring::default();
     let mut encrypted = false;
     let mut signatures = HashSet::default();
     let mut gossipped_addr = HashSet::default();
 
-    // just a pointer into mailmime structure, must not be freed
-    let imffields = unsafe { mailmime_find_mailimf_fields(in_out_message) };
-    let mut message_time = 0;
-    let mut from = None;
-    let mut private_keyring = Keyring::default();
-    let mut public_keyring_for_validate = Keyring::default();
-    let mut gossip_headers = ptr::null_mut();
+    let self_addr = context.get_config(Config::ConfiguredAddr);
 
-    // XXX do wrapmime:: helper for the next block
-    if !(in_out_message.is_null() || imffields.is_null()) {
-        let mut field = mailimf_find_field(imffields, MAILIMF_FIELD_FROM as libc::c_int);
-
-        if !field.is_null() && unsafe { !(*field).fld_data.fld_from.is_null() } {
-            let mb_list = unsafe { (*(*field).fld_data.fld_from).frm_mb_list };
-            from = mailimf_find_first_addr(mb_list);
-        }
-
-        field = mailimf_find_field(imffields, MAILIMF_FIELD_ORIG_DATE as libc::c_int);
-        if !field.is_null() && unsafe { !(*field).fld_data.fld_orig_date.is_null() } {
-            let orig_date = unsafe { (*field).fld_data.fld_orig_date };
-
-            if !orig_date.is_null() {
-                let dt = unsafe { (*orig_date).dt_date_time };
-                message_time = dc_timestamp_from_date(dt);
-                if message_time != 0 && message_time > time() {
-                    message_time = time()
+    if let Some(self_addr) = self_addr {
+        if private_keyring.load_self_private_for_decrypting(context, self_addr, &context.sql) {
+            if peerstate.as_ref().map(|p| p.last_seen).unwrap_or_else(|| 0) == 0 {
+                peerstate = Peerstate::from_addr(&context, &context.sql, &from);
+            }
+            if let Some(ref peerstate) = peerstate {
+                if peerstate.degrade_event.is_some() {
+                    handle_degrade_event(context, &peerstate)?;
+                }
+                if let Some(ref key) = peerstate.gossip_key {
+                    public_keyring_for_validate.add_ref(key);
+                }
+                if let Some(ref key) = peerstate.public_key {
+                    public_keyring_for_validate.add_ref(key);
                 }
             }
-        }
-        let mut peerstate = None;
-        let autocryptheader = from
-            .as_ref()
-            .and_then(|from| Aheader::from_imffields(from, imffields));
-        if message_time > 0 {
-            if let Some(ref from) = from {
-                peerstate = Peerstate::from_addr(context, &context.sql, from);
 
-                if let Some(ref mut peerstate) = peerstate {
-                    if let Some(ref header) = autocryptheader {
-                        peerstate.apply_header(&header, message_time);
-                        peerstate.save_to_db(&context.sql, false).unwrap();
-                    } else if message_time > peerstate.last_seen_autocrypt
-                        && !contains_report(in_out_message)
-                    {
-                        peerstate.degrade_encryption(message_time);
-                        peerstate.save_to_db(&context.sql, false).unwrap();
-                    }
-                } else if let Some(ref header) = autocryptheader {
-                    let p = Peerstate::from_header(context, header, message_time);
-                    p.save_to_db(&context.sql, true).unwrap();
-                    peerstate = Some(p);
-                }
-            }
-        }
-        /* load private key for decryption */
-        let self_addr = context.get_config(Config::ConfiguredAddr);
-        if let Some(self_addr) = self_addr {
-            if private_keyring.load_self_private_for_decrypting(context, self_addr, &context.sql) {
-                if peerstate.as_ref().map(|p| p.last_seen).unwrap_or_else(|| 0) == 0 {
-                    peerstate =
-                        Peerstate::from_addr(&context, &context.sql, &from.unwrap_or_default());
-                }
-                if let Some(ref peerstate) = peerstate {
-                    if peerstate.degrade_event.is_some() {
-                        handle_degrade_event(context, &peerstate)?;
-                    }
-                    if let Some(ref key) = peerstate.gossip_key {
-                        public_keyring_for_validate.add_ref(key);
-                    }
-                    if let Some(ref key) = peerstate.public_key {
-                        public_keyring_for_validate.add_ref(key);
-                    }
-                }
-
-                encrypted = decrypt_if_autocrypt_message(
-                    context,
-                    in_out_message,
-                    &private_keyring,
-                    &public_keyring_for_validate,
-                    &mut signatures,
-                    &mut gossip_headers,
-                )?;
-                if !gossip_headers.is_null() {
-                    gossipped_addr =
-                        update_gossip_peerstates(context, message_time, imffields, gossip_headers)?;
-                }
+            let mut gossip_headers = ptr::null_mut();
+            encrypted = decrypt_if_autocrypt_message(
+                context,
+                in_out_message,
+                &private_keyring,
+                &public_keyring_for_validate,
+                &mut signatures,
+                &mut gossip_headers,
+            )?;
+            if !gossip_headers.is_null() {
+                gossipped_addr =
+                    update_gossip_peerstates(context, message_time, imffields, gossip_headers)?;
+                unsafe { mailimf_fields_free(gossip_headers) };
             }
         }
     }
-    if !gossip_headers.is_null() {
-        unsafe { mailimf_fields_free(gossip_headers) };
-    }
-
     Ok((encrypted, signatures, gossipped_addr))
 }
 
@@ -425,7 +398,7 @@ fn load_or_generate_self_public_key(context: &Context, self_addr: impl AsRef<str
         context,
         "Generating keypair with {} bits, e={} ...", 2048, 65537,
     );
-    match dc_pgp_create_keypair(&self_addr) {
+    match pgp::create_keypair(&self_addr) {
         Some((public_key, private_key)) => {
             match dc_key_save_self_keypair(
                 context,
@@ -476,7 +449,7 @@ fn update_gossip_peerstates(
 
             let optional_field = unsafe { *optional_field };
             if !optional_field.fld_name.is_null()
-                && as_str(optional_field.fld_name) == "Autocrypt-Gossip"
+                && to_string_lossy(optional_field.fld_name) == "Autocrypt-Gossip"
             {
                 let value = to_string_lossy(optional_field.fld_value);
                 let gossip_header = Aheader::from_str(&value);
@@ -560,7 +533,7 @@ fn decrypt_if_autocrypt_message(
     // XXX better return parsed headers so that upstream
     // does not need to dive into mmime-stuff again.
     unsafe {
-        if (*ret_gossip_headers).is_null() && ret_valid_signatures.len() > 0 {
+        if (*ret_gossip_headers).is_null() && !ret_valid_signatures.is_empty() {
             let mut dummy: libc::size_t = 0;
             let mut test: *mut mailimf_fields = ptr::null_mut();
             if mailimf_envelope_and_optional_fields_parse(
@@ -600,22 +573,16 @@ fn decrypt_part(
         mime_transfer_encoding = enc;
     }
 
-    let (decoded_data, decoded_data_bytes) =
-        wrapmime::decode_dt_data(mime_data, mime_transfer_encoding)?;
+    let data: Vec<u8> = wrapmime::decode_dt_data(mime_data, mime_transfer_encoding)?;
 
-    // encrypted, non-NULL decoded data in decoded_data now ...
-    // Note that we need to take care of freeing decoded_data ourself,
-    // after encryption has been attempted.
     let mut ret_decrypted_mime = ptr::null_mut();
 
-    ensure!(!decoded_data.is_null(), "Missing data");
-    let data = unsafe { std::slice::from_raw_parts(decoded_data as *const u8, decoded_data_bytes) };
-    if has_decrypted_pgp_armor(data) {
+    if has_decrypted_pgp_armor(&data) {
         // we should only have one decryption happening
         ensure!(ret_valid_signatures.is_empty(), "corrupt signatures");
 
-        let plain = match dc_pgp_pk_decrypt(
-            data,
+        let plain = match pgp::pk_decrypt(
+            &data,
             &private_keyring,
             &public_keyring_for_validate,
             Some(ret_valid_signatures),
@@ -624,10 +591,7 @@ fn decrypt_part(
                 ensure!(!ret_valid_signatures.is_empty(), "no valid signatures");
                 plain
             }
-            Err(err) => {
-                unsafe { mmap_string_unref(decoded_data) };
-                bail!("could not decrypt: {}", err)
-            }
+            Err(err) => bail!("could not decrypt: {}", err),
         };
         let plain_bytes = plain.len();
         let plain_buf = plain.as_ptr() as *const libc::c_char;
@@ -655,7 +619,6 @@ fn decrypt_part(
             ret_decrypted_mime = decrypted_mime;
         }
     }
-    unsafe { mmap_string_unref(decoded_data) };
 
     Ok(ret_decrypted_mime)
 }
@@ -692,7 +655,7 @@ fn contains_report(mime: *mut Mailmime) -> bool {
 
         if tp_type == MAILMIME_TYPE_COMPOSITE_TYPE as libc::c_int
             && ct_type == MAILMIME_COMPOSITE_TYPE_MULTIPART as libc::c_int
-            && as_str(unsafe { (*mime.mm_content_type).ct_subtype }) == "report"
+            && to_string_lossy(unsafe { (*mime.mm_content_type).ct_subtype }) == "report"
         {
             return true;
         }
@@ -723,12 +686,12 @@ fn contains_report(mime: *mut Mailmime) -> bool {
 /// If this succeeds you are also guaranteed that the
 /// [Config::ConfiguredAddr] is configured, this address is returned.
 pub fn ensure_secret_key_exists(context: &Context) -> Result<String> {
-    let self_addr = context
-        .get_config(Config::ConfiguredAddr)
-        .ok_or(format_err!(concat!(
+    let self_addr = context.get_config(Config::ConfiguredAddr).ok_or_else(|| {
+        format_err!(concat!(
             "Failed to get self address, ",
             "cannot ensure secret key if not configured."
-        )))?;
+        ))
+    })?;
     load_or_generate_self_public_key(context, &self_addr)?;
     Ok(self_addr)
 }

src/error.rs

@@ -30,6 +30,10 @@ pub enum Error {
     Base64Decode(base64::DecodeError),
     #[fail(display = "{:?}", _0)]
     FromUtf8(std::string::FromUtf8Error),
+    #[fail(display = "{}", _0)]
+    BlobError(#[cause] crate::blob::BlobError),
+    #[fail(display = "Invalid Message ID.")]
+    InvalidMsgId,
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -70,12 +74,6 @@ impl From<std::str::Utf8Error> for Error {
     }
 }
 
-impl From<std::string::FromUtf8Error> for Error {
-    fn from(err: std::string::FromUtf8Error) -> Error {
-        Error::FromUtf8(err)
-    }
-}
-
 impl From<image_meta::ImageError> for Error {
     fn from(err: image_meta::ImageError) -> Error {
         Error::Image(err)
@@ -94,6 +92,24 @@ impl From<pgp::errors::Error> for Error {
     }
 }
 
+impl From<std::string::FromUtf8Error> for Error {
+    fn from(err: std::string::FromUtf8Error) -> Error {
+        Error::FromUtf8(err)
+    }
+}
+
+impl From<crate::blob::BlobError> for Error {
+    fn from(err: crate::blob::BlobError) -> Error {
+        Error::BlobError(err)
+    }
+}
+
+impl From<crate::message::InvalidMsgId> for Error {
+    fn from(_err: crate::message::InvalidMsgId) -> Error {
+        Error::InvalidMsgId
+    }
+}
+
 #[macro_export]
 macro_rules! bail {
     ($e:expr) => {

src/events.rs

@@ -2,7 +2,7 @@ use std::path::PathBuf;
 
 use strum::EnumProperty;
 
-use crate::stock::StockMessage;
+use crate::message::MsgId;
 
 impl Event {
     /// Returns the corresponding Event id.
@@ -42,6 +42,36 @@ pub enum Event {
     #[strum(props(id = "103"))]
     SmtpMessageSent(String),
 
+    /// Emitted when an IMAP message has been marked as deleted
+    ///
+    /// @return 0
+    #[strum(props(id = "104"))]
+    ImapMessageDeleted(String),
+
+    /// Emitted when an IMAP message has been moved
+    ///
+    /// @return 0
+    #[strum(props(id = "105"))]
+    ImapMessageMoved(String),
+
+    /// Emitted when an IMAP folder was emptied
+    ///
+    /// @return 0
+    #[strum(props(id = "106"))]
+    ImapFolderEmptied(String),
+
+    /// Emitted when an new file in the $BLOBDIR was created
+    ///
+    /// @return 0
+    #[strum(props(id = "150"))]
+    NewBlobFile(String),
+
+    /// Emitted when an new file in the $BLOBDIR was created
+    ///
+    /// @return 0
+    #[strum(props(id = "151"))]
+    DeletedBlobFile(String),
+
     /// The library-user should write a warning string to the log.
     /// Passed to the callback given to dc_context_new().
     ///
@@ -103,7 +133,7 @@ pub enum Event {
     ///
     /// @return 0
     #[strum(props(id = "2000"))]
-    MsgsChanged { chat_id: u32, msg_id: u32 },
+    MsgsChanged { chat_id: u32, msg_id: MsgId },
 
     /// There is a fresh message. Typically, the user will show an notification
     /// when receiving this message.
@@ -112,28 +142,28 @@ pub enum Event {
     ///
     /// @return 0
     #[strum(props(id = "2005"))]
-    IncomingMsg { chat_id: u32, msg_id: u32 },
+    IncomingMsg { chat_id: u32, msg_id: MsgId },
 
     /// A single message is sent successfully. State changed from  DC_STATE_OUT_PENDING to
     /// DC_STATE_OUT_DELIVERED, see dc_msg_get_state().
     ///
     /// @return 0
     #[strum(props(id = "2010"))]
-    MsgDelivered { chat_id: u32, msg_id: u32 },
+    MsgDelivered { chat_id: u32, msg_id: MsgId },
 
     /// A single message could not be sent. State changed from DC_STATE_OUT_PENDING or DC_STATE_OUT_DELIVERED to
     /// DC_STATE_OUT_FAILED, see dc_msg_get_state().
     ///
     /// @return 0
     #[strum(props(id = "2012"))]
-    MsgFailed { chat_id: u32, msg_id: u32 },
+    MsgFailed { chat_id: u32, msg_id: MsgId },
 
     /// A single message is read by the receiver. State changed from DC_STATE_OUT_DELIVERED to
     /// DC_STATE_OUT_MDN_RCVD, see dc_msg_get_state().
     ///
     /// @return 0
     #[strum(props(id = "2015"))]
-    MsgRead { chat_id: u32, msg_id: u32 },
+    MsgRead { chat_id: u32, msg_id: MsgId },
 
     /// Chat changed.  The name or the image of a chat group was changed or members were added or removed.
     /// Or the verify state of a chat has changed.
@@ -213,17 +243,4 @@ pub enum Event {
     /// @return 0
     #[strum(props(id = "2061"))]
     SecurejoinJoinerProgress { contact_id: u32, progress: usize },
-
-    // the following events are functions that should be provided by the frontends
-    /// Requeste a localized string from the frontend.
-    /// @param data1 (int) ID of the string to request, one of the DC_STR_/// constants.
-    /// @param data2 (int) The count. If the requested string contains a placeholder for a numeric value,
-    ///     the ui may use this value to return different strings on different plural forms.
-    /// @return (const char*) Null-terminated UTF-8 string.
-    ///     The string will be free()'d by the core,
-    ///     so it must be allocated using malloc() or a compatible function.
-    ///     Return 0 if the ui cannot provide the requested string
-    ///     the core will use a default string in english language then.
-    #[strum(props(id = "2091"))]
-    GetString { id: StockMessage, count: usize },
 }

src/imex.rs

@@ -1,24 +1,25 @@
-use std::ffi::CString;
+use core::cmp::{max, min};
 use std::path::Path;
-use std::ptr;
 
 use num_traits::FromPrimitive;
 use rand::{thread_rng, Rng};
 
+use crate::blob::BlobObject;
 use crate::chat;
 use crate::config::Config;
 use crate::configure::*;
 use crate::constants::*;
 use crate::context::Context;
+use crate::dc_mimeparser::SystemMessage;
 use crate::dc_tools::*;
 use crate::e2ee;
 use crate::error::*;
 use crate::events::Event;
 use crate::job::*;
 use crate::key::*;
-use crate::message::Message;
+use crate::message::{Message, MsgId};
 use crate::param::*;
-use crate::pgp::*;
+use crate::pgp;
 use crate::sql::{self, Sql};
 use crate::stock::StockMessage;
 
@@ -74,7 +75,7 @@ pub fn imex(context: &Context, what: ImexMode, param1: Option<impl AsRef<Path>>)
     job_add(context, Action::ImexImap, 0, param, 0);
 }
 
-/// Returns the filename of the backup if found, nullptr otherwise.
+/// Returns the filename of the backup found (otherwise an error)
 pub fn has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Result<String> {
     let dir_name = dir_name.as_ref();
     let dir_iter = std::fs::read_dir(dir_name)?;
@@ -88,14 +89,16 @@ pub fn has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Result<Strin
                 let name = name.to_string_lossy();
                 if name.starts_with("delta-chat") && name.ends_with(".bak") {
                     let sql = Sql::new();
-                    if sql.open(context, &path, 0x1) {
-                        let curr_backup_time =
-                            sql.get_config_int(context, "backup_time")
-                                .unwrap_or_default() as u64;
+                    if sql.open(context, &path, true) {
+                        let curr_backup_time = sql
+                            .get_raw_config_int(context, "backup_time")
+                            .unwrap_or_default();
                         if curr_backup_time > newest_backup_time {
                             newest_backup_path = Some(path);
                             newest_backup_time = curr_backup_time;
                         }
+                        info!(context, "backup_time of {} is {}", name, curr_backup_time);
+                        sql.close(&context);
                     }
                 }
             }
@@ -104,82 +107,54 @@ pub fn has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Result<Strin
     }
     match newest_backup_path {
         Some(path) => Ok(path.to_string_lossy().into_owned()),
-        None => bail!("no backup found"),
+        None => bail!("no backup found in {}", dir_name.display()),
     }
 }
 
 pub fn initiate_key_transfer(context: &Context) -> Result<String> {
+    ensure!(context.alloc_ongoing(), "could not allocate ongoing");
+    let res = do_initiate_key_transfer(context);
+    context.free_ongoing();
+    res
+}
+
+fn do_initiate_key_transfer(context: &Context) -> Result<String> {
     let mut msg: Message;
-    ensure!(dc_alloc_ongoing(context), "could not allocate ongoing");
     let setup_code = create_setup_code(context);
     /* this may require a keypair to be created. this may take a second ... */
-    if !context
-        .running_state
-        .clone()
-        .read()
-        .unwrap()
-        .shall_stop_ongoing
-    {
-        if let Ok(ref setup_file_content) = render_setup_file(context, &setup_code) {
-            /* encrypting may also take a while ... */
-            if !context
-                .running_state
-                .clone()
-                .read()
-                .unwrap()
-                .shall_stop_ongoing
-            {
-                let setup_file_name =
-                    dc_get_fine_path_filename(context, "$BLOBDIR", "autocrypt-setup-message.html");
-                if dc_write_file(context, &setup_file_name, setup_file_content.as_bytes()) {
-                    if let Ok(chat_id) = chat::create_by_contact_id(context, 1) {
-                        msg = Message::default();
-                        msg.type_0 = Viewtype::File;
-                        msg.param
-                            .set(Param::File, setup_file_name.to_string_lossy());
+    ensure!(!context.shall_stop_ongoing(), "canceled");
+    let setup_file_content = render_setup_file(context, &setup_code)?;
+    /* encrypting may also take a while ... */
+    ensure!(!context.shall_stop_ongoing(), "canceled");
+    let setup_file_blob = BlobObject::create(
+        context,
+        "autocrypt-setup-message.html",
+        setup_file_content.as_bytes(),
+    )?;
 
-                        msg.param
-                            .set(Param::MimeType, "application/autocrypt-setup");
-                        msg.param.set_int(Param::Cmd, 6);
-                        msg.param
-                            .set_int(Param::ForcePlaintext, DC_FP_NO_AUTOCRYPT_HEADER);
+    let chat_id = chat::create_by_contact_id(context, DC_CONTACT_ID_SELF)?;
+    msg = Message::default();
+    msg.type_0 = Viewtype::File;
+    msg.param.set(Param::File, setup_file_blob.as_name());
 
-                        if !context
-                            .running_state
-                            .clone()
-                            .read()
-                            .unwrap()
-                            .shall_stop_ongoing
-                        {
-                            if let Ok(msg_id) = chat::send_msg(context, chat_id, &mut msg) {
-                                info!(context, "Wait for setup message being sent ...",);
-                                loop {
-                                    if context
-                                        .running_state
-                                        .clone()
-                                        .read()
-                                        .unwrap()
-                                        .shall_stop_ongoing
-                                    {
-                                        break;
-                                    }
-                                    std::thread::sleep(std::time::Duration::from_secs(1));
-                                    if let Ok(msg) = Message::load_from_db(context, msg_id) {
-                                        if msg.is_sent() {
-                                            info!(context, "... setup message sent.",);
-                                            break;
-                                        }
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
+    msg.param
+        .set(Param::MimeType, "application/autocrypt-setup");
+    msg.param.set_cmd(SystemMessage::AutocryptSetupMessage);
+    msg.param
+        .set_int(Param::ForcePlaintext, DC_FP_NO_AUTOCRYPT_HEADER);
+
+    ensure!(!context.shall_stop_ongoing(), "canceled");
+    let msg_id = chat::send_msg(context, chat_id, &mut msg)?;
+    info!(context, "Wait for setup message being sent ...",);
+    while !context.shall_stop_ongoing() {
+        std::thread::sleep(std::time::Duration::from_secs(1));
+        if let Ok(msg) = Message::load_from_db(context, msg_id) {
+            if msg.is_sent() {
+                info!(context, "... setup message sent.",);
+                break;
             }
         }
     }
-    dc_free_ongoing(context);
-
     Ok(setup_code)
 }
 
@@ -194,16 +169,12 @@ pub fn render_setup_file(context: &Context, passphrase: &str) -> Result<String>
     let self_addr = e2ee::ensure_secret_key_exists(context)?;
     let private_key = Key::from_self_private(context, self_addr, &context.sql)
         .ok_or(format_err!("Failed to get private key."))?;
-    let ac_headers = match context
-        .sql
-        .get_config_int(context, Config::E2eeEnabled)
-        .unwrap_or(1)
-    {
-        0 => None,
-        _ => Some(("Autocrypt-Prefer-Encrypt", "mutual")),
+    let ac_headers = match context.get_config_bool(Config::E2eeEnabled) {
+        false => None,
+        true => Some(("Autocrypt-Prefer-Encrypt", "mutual")),
     };
     let private_key_asc = private_key.to_asc(ac_headers);
-    let encr = dc_pgp_symm_encrypt(&passphrase, private_key_asc.as_bytes())?;
+    let encr = pgp::symm_encrypt(&passphrase, private_key_asc.as_bytes())?;
 
     let replacement = format!(
         concat!(
@@ -259,32 +230,22 @@ pub fn create_setup_code(_context: &Context) -> String {
     ret
 }
 
-pub fn continue_key_transfer(context: &Context, msg_id: u32, setup_code: &str) -> Result<()> {
-    ensure!(msg_id > DC_MSG_ID_LAST_SPECIAL, "wrong id");
+pub fn continue_key_transfer(context: &Context, msg_id: MsgId, setup_code: &str) -> Result<()> {
+    ensure!(!msg_id.is_special(), "wrong id");
 
-    let msg = Message::load_from_db(context, msg_id);
-    if msg.is_err() {
-        bail!("Message is no Autocrypt Setup Message.");
-    }
-    let msg = msg.unwrap();
+    let msg = Message::load_from_db(context, msg_id)?;
     ensure!(
         msg.is_setupmessage(),
         "Message is no Autocrypt Setup Message."
     );
 
     if let Some(filename) = msg.get_file(context) {
-        if let Ok(ref mut buf) = dc_read_file(context, filename) {
-            let sc = normalize_setup_code(setup_code);
-            if let Ok(armored_key) = decrypt_setup_file(context, sc, buf) {
-                set_self_key(context, &armored_key, true, true)?;
-            } else {
-                bail!("Bad setup code.")
-            }
+        let file = dc_open_file(context, filename)?;
+        let sc = normalize_setup_code(setup_code);
+        let armored_key = decrypt_setup_file(context, &sc, file)?;
+        set_self_key(context, &armored_key, true, true)?;
 
-            Ok(())
-        } else {
-            bail!("Cannot read Autocrypt Setup Message file.");
-        }
+        Ok(())
     } else {
         bail!("Message is no Autocrypt Setup Message.");
     }
@@ -316,7 +277,7 @@ fn set_self_key(
             };
             context
                 .sql
-                .set_config_int(context, "e2ee_enabled", e2ee_enabled)?;
+                .set_raw_config_int(context, "e2ee_enabled", e2ee_enabled)?;
         }
         None => {
             if prefer_encrypt_required {
@@ -349,7 +310,7 @@ fn set_self_key(
         context,
         &public_key,
         &private_key,
-        self_addr.unwrap(),
+        self_addr.unwrap_or_default(),
         set_default,
         &context.sql,
     ) {
@@ -358,53 +319,15 @@ fn set_self_key(
     Ok(())
 }
 
-fn decrypt_setup_file(
+fn decrypt_setup_file<T: std::io::Read + std::io::Seek>(
     _context: &Context,
-    passphrase: impl AsRef<str>,
-    filecontent: &mut [u8],
+    passphrase: &str,
+    file: T,
 ) -> Result<String> {
-    let mut fc_headerline = String::default();
-    let mut fc_base64: *const libc::c_char = ptr::null();
+    let plain_bytes = pgp::symm_decrypt(passphrase, file)?;
+    let plain_text = std::string::String::from_utf8(plain_bytes)?;
 
-    let split_result = unsafe {
-        dc_split_armored_data(
-            filecontent.as_mut_ptr().cast(),
-            &mut fc_headerline,
-            ptr::null_mut(),
-            ptr::null_mut(),
-            &mut fc_base64,
-        )
-    };
-
-    if !split_result || fc_headerline != "-----BEGIN PGP MESSAGE-----" || fc_base64.is_null() {
-        bail!("Invalid armored data");
-    }
-
-    // convert base64 to binary
-    let base64_encoded =
-        unsafe { std::slice::from_raw_parts(fc_base64 as *const u8, libc::strlen(fc_base64)) };
-
-    let data = base64_decode(&base64_encoded)?;
-
-    // decrypt symmetrically
-    let payload = dc_pgp_symm_decrypt(passphrase.as_ref(), &data)?;
-    let payload_str = String::from_utf8(payload)?;
-
-    Ok(payload_str)
-}
-
-/// Decode the base64 encoded slice. Handles line breaks.
-fn base64_decode(input: &[u8]) -> Result<Vec<u8>> {
-    use std::io::Read;
-    let c = std::io::Cursor::new(input);
-    let lr = pgp::line_reader::LineReader::new(c);
-    let br = pgp::base64_reader::Base64Reader::new(lr);
-    let mut reader = pgp::base64_decoder::Base64Decoder::new(br);
-
-    let mut data = Vec::new();
-    reader.read_to_end(&mut data)?;
-
-    Ok(data)
+    Ok(plain_text)
 }
 
 pub fn normalize_setup_code(s: &str) -> String {
@@ -422,7 +345,7 @@ pub fn normalize_setup_code(s: &str) -> String {
 
 #[allow(non_snake_case)]
 pub fn job_do_DC_JOB_IMEX_IMAP(context: &Context, job: &Job) -> Result<()> {
-    ensure!(dc_alloc_ongoing(context), "could not allocate ongoing");
+    ensure!(context.alloc_ongoing(), "could not allocate ongoing");
     let what: Option<ImexMode> = job.param.get_int(Param::Cmd).and_then(ImexMode::from_i32);
     let param = job.param.get(Param::Arg).unwrap_or_default();
 
@@ -434,7 +357,7 @@ pub fn job_do_DC_JOB_IMEX_IMAP(context: &Context, job: &Job) -> Result<()> {
     if what == Some(ImexMode::ExportBackup) || what == Some(ImexMode::ExportSelfKeys) {
         // before we export anything, make sure the private key exists
         if e2ee::ensure_secret_key_exists(context).is_err() {
-            dc_free_ongoing(context);
+            context.free_ongoing();
             bail!("Cannot create private key or private key not available.");
         } else {
             dc_create_folder(context, &param);
@@ -450,7 +373,7 @@ pub fn job_do_DC_JOB_IMEX_IMAP(context: &Context, job: &Job) -> Result<()> {
             bail!("unknown IMEX type");
         }
     };
-    dc_free_ongoing(context);
+    context.free_ongoing();
     match success {
         Ok(()) => {
             info!(context, "IMEX successfully completed");
@@ -477,10 +400,10 @@ fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Resul
         !dc_is_configured(context),
         "Cannot import backups to accounts in use."
     );
-    &context.sql.close(&context);
+    context.sql.close(&context);
     dc_delete_file(context, context.get_dbfile());
     ensure!(
-        !dc_file_exist(context, context.get_dbfile()),
+        !context.get_dbfile().exists(),
         "Cannot delete old database."
     );
 
@@ -491,7 +414,7 @@ fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Resul
     /* error already logged */
     /* re-open copied database file */
     ensure!(
-        context.sql.open(&context, &context.get_dbfile(), 0),
+        context.sql.open(&context, &context.get_dbfile(), false),
         "could not re-open db"
     );
 
@@ -514,20 +437,9 @@ fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Resul
             Ok((name, blob))
         },
         |files| {
-            let mut processed_files_cnt = 0;
-
-            for file in files {
+            for (processed_files_cnt, file) in files.enumerate() {
                 let (file_name, file_blob) = file?;
-                if context
-                    .running_state
-                    .clone()
-                    .read()
-                    .unwrap()
-                    .shall_stop_ongoing
-                {
-                    bail!("received stop signal");
-                }
-                processed_files_cnt += 1;
+                ensure!(!context.shall_stop_ongoing(), "received stop signal");
                 let mut permille = processed_files_cnt * 1000 / total_files_cnt;
                 if permille < 10 {
                     permille = 10
@@ -568,163 +480,110 @@ fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Resul
 /* the FILE_PROGRESS macro calls the callback with the permille of files processed.
 The macro avoids weird values of 0% or 100% while still working. */
 fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
-    let mut ok_to_continue = true;
-    let mut success = false;
-
-    let mut delete_dest_file: libc::c_int = 0;
     // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
-    // FIXME: we should write to a temporary file first and rename it on success. this would guarantee the backup is complete. however, currently it is not clear it the import exists in the long run (may be replaced by a restore-from-imap)
+    // FIXME: we should write to a temporary file first and rename it on success. this would guarantee the backup is complete.
+    // let dest_path_filename = dc_get_next_backup_file(context, dir, res);
     let now = time();
-    let res = chrono::NaiveDateTime::from_timestamp(now as i64, 0)
-        .format("delta-chat-%Y-%m-%d.bak")
-        .to_string();
-
-    let dest_path_filename = dc_get_fine_path_filename(context, dir, res);
+    let dest_path_filename = dc_get_next_backup_path(dir, now)?;
+    let dest_path_string = dest_path_filename.to_string_lossy().to_string();
 
     sql::housekeeping(context);
 
     sql::try_execute(context, &context.sql, "VACUUM;").ok();
+
+    // we close the database during the copy of the dbfile
     context.sql.close(context);
     info!(
         context,
-        "Backup \"{}\" to \"{}\".",
+        "Backup '{}' to '{}'.",
         context.get_dbfile().display(),
         dest_path_filename.display(),
     );
     let copied = dc_copy_file(context, context.get_dbfile(), &dest_path_filename);
-    context.sql.open(&context, &context.get_dbfile(), 0);
+    context.sql.open(&context, &context.get_dbfile(), false);
+
     if !copied {
-        let s = dest_path_filename.to_string_lossy().to_string();
         bail!(
-            "could not copy file from {:?} to {:?}",
-            context.get_dbfile(),
-            s
+            "could not copy file from '{}' to '{}'",
+            context.get_dbfile().display(),
+            dest_path_string
         );
     }
-    /* add all files as blobs to the database copy (this does not require the source to be locked, neigher the destination as it is used only here) */
-    /*for logging only*/
-    let sql = Sql::new();
-    if sql.open(context, &dest_path_filename, 0) {
-        if !sql.table_exists("backup_blobs") {
-            if sql::execute(
-                context,
-                &sql,
-                "CREATE TABLE backup_blobs (id INTEGER PRIMARY KEY, file_name, file_content);",
-                params![],
-            )
-            .is_err()
-            {
-                /* error already logged */
-                ok_to_continue = false;
-            }
+    let dest_sql = Sql::new();
+    ensure!(
+        dest_sql.open(context, &dest_path_filename, false),
+        "could not open exported database {}",
+        dest_path_string
+    );
+    let res = match add_files_to_export(context, &dest_sql) {
+        Err(err) => {
+            dc_delete_file(context, &dest_path_filename);
+            error!(context, "backup failed: {}", err);
+            Err(err)
         }
-        if ok_to_continue {
-            let mut total_files_cnt = 0;
-            let dir = context.get_blobdir();
-            if let Ok(dir_handle) = std::fs::read_dir(&dir) {
-                total_files_cnt += dir_handle.filter(|r| r.is_ok()).count();
+        Ok(()) => {
+            dest_sql.set_raw_config_int(context, "backup_time", now as i32)?;
+            context.call_cb(Event::ImexFileWritten(dest_path_filename.clone()));
+            Ok(())
+        }
+    };
+    dest_sql.close(context);
 
-                info!(context, "EXPORT: total_files_cnt={}", total_files_cnt);
-                if total_files_cnt > 0 {
-                    // scan directory, pass 2: copy files
-                    if let Ok(dir_handle) = std::fs::read_dir(&dir) {
-                        sql.prepare(
-                                    "INSERT INTO backup_blobs (file_name, file_content) VALUES (?, ?);",
-                                    move |mut stmt, _| {
-                                        let mut processed_files_cnt = 0;
-                                        for entry in dir_handle {
-                                            if entry.is_err() {
-                                                break;
-                                            }
-                                            let entry = entry.unwrap();
-                                            if context
-                                                .running_state
-                                                .clone()
-                                                .read()
-                                                .unwrap()
-                                                .shall_stop_ongoing
-                                            {
-                                                delete_dest_file = 1;
-                                                ok_to_continue = false;
-                                                break;
-                                            } else {
-                                                processed_files_cnt += 1;
-                                                let mut permille =
-                                                    processed_files_cnt * 1000 / total_files_cnt;
-                                                if permille < 10 {
-                                                    permille = 10;
-                                                }
-                                                if permille > 990 {
-                                                    permille = 990;
-                                                }
-                                                context.call_cb(Event::ImexProgress(permille));
+    res
+}
 
-                                                let name_f = entry.file_name();
-                                                let name = name_f.to_string_lossy();
-                                                if name.starts_with("delta-chat") && name.ends_with(".bak")
-                                                {
-                                                    continue;
-                                                } else {
-                                                    info!(context, "EXPORTing filename={}", name);
-                                                    let curr_path_filename = context.get_blobdir().join(entry.file_name());
-                                                    if let Ok(buf) =
-                                                        dc_read_file(context, &curr_path_filename)
-                                                    {
-                                                        if buf.is_empty() {
-                                                            continue;
-                                                        }
-                                                        if stmt.execute(params![name, buf]).is_err() {
-                                                            error!(
-                                                                context,
-                                                                "Disk full? Cannot add file \"{}\" to backup.",
-                                                                curr_path_filename.display(),
-                                                            );
-                                                            /* this is not recoverable! writing to the sqlite database should work! */
-                                                            ok_to_continue = false;
-                                                            break;
-                                                        }
-                                                    } else {
-                                                        continue;
-                                                    }
-                                                }
-                                            }
-                                        }
-                                        Ok(())
-                                    }
-                                ).unwrap();
-                    } else {
-                        error!(
-                            context,
-                            "Backup: Cannot copy from blob-directory \"{}\".",
-                            context.get_blobdir().display(),
-                        );
-                    }
-                } else {
-                    info!(context, "Backup: No files to copy.",);
-                }
-                if ok_to_continue {
-                    if sql
-                        .set_config_int(context, "backup_time", now as i32)
-                        .is_ok()
-                    {
-                        context.call_cb(Event::ImexFileWritten(dest_path_filename.clone()));
-                        success = true;
-                    }
-                }
-            } else {
-                error!(
-                    context,
-                    "Backup: Cannot get info for blob-directory \"{}\".",
-                    context.get_blobdir().display(),
+fn add_files_to_export(context: &Context, sql: &Sql) -> Result<()> {
+    // add all files as blobs to the database copy (this does not require
+    // the source to be locked, neigher the destination as it is used only here)
+    if !sql.table_exists("backup_blobs") {
+        sql::execute(
+            context,
+            &sql,
+            "CREATE TABLE backup_blobs (id INTEGER PRIMARY KEY, file_name, file_content);",
+            params![],
+        )?
+    }
+    // copy all files from BLOBDIR into backup-db
+    let mut total_files_cnt = 0;
+    let dir = context.get_blobdir();
+    let dir_handle = std::fs::read_dir(&dir)?;
+    total_files_cnt += dir_handle.filter(|r| r.is_ok()).count();
+
+    info!(context, "EXPORT: total_files_cnt={}", total_files_cnt);
+    // scan directory, pass 2: copy files
+    let dir_handle = std::fs::read_dir(&dir)?;
+    sql.prepare(
+        "INSERT INTO backup_blobs (file_name, file_content) VALUES (?, ?);",
+        |mut stmt, _| {
+            let mut processed_files_cnt = 0;
+            for entry in dir_handle {
+                let entry = entry?;
+                ensure!(
+                    !context.shall_stop_ongoing(),
+                    "canceled during export-files"
                 );
-            };
-        }
-    }
-    if 0 != delete_dest_file {
-        dc_delete_file(context, &dest_path_filename);
-    }
+                processed_files_cnt += 1;
+                let permille = max(min(processed_files_cnt * 1000 / total_files_cnt, 990), 10);
+                context.call_cb(Event::ImexProgress(permille));
 
-    ensure!(success, "failed");
+                let name_f = entry.file_name();
+                let name = name_f.to_string_lossy();
+                if name.starts_with("delta-chat") && name.ends_with(".bak") {
+                    continue;
+                }
+                info!(context, "EXPORT: copying filename={}", name);
+                let curr_path_filename = context.get_blobdir().join(entry.file_name());
+                if let Ok(buf) = dc_read_file(context, &curr_path_filename) {
+                    if buf.is_empty() {
+                        continue;
+                    }
+                    // bail out if we can't insert
+                    stmt.execute(params![name, buf])?;
+                }
+            }
+            Ok(())
+        },
+    )?;
     Ok(())
 }
 
@@ -739,81 +598,48 @@ fn import_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     Maybe we should make the "default" key handlong also a little bit smarter
     (currently, the last imported key is the standard key unless it contains the string "legacy" in its name) */
     let mut set_default: bool;
-    let mut key: String;
     let mut imported_cnt = 0;
 
     let dir_name = dir.as_ref().to_string_lossy();
-    if let Ok(dir_handle) = std::fs::read_dir(&dir) {
-        for entry in dir_handle {
-            let entry_fn = match entry {
-                Err(err) => {
-                    info!(context, "file-dir error: {}", err);
-                    break;
+    let dir_handle = std::fs::read_dir(&dir)?;
+    for entry in dir_handle {
+        let entry_fn = entry?.file_name();
+        let name_f = entry_fn.to_string_lossy();
+        let path_plus_name = dir.as_ref().join(&entry_fn);
+        match dc_get_filesuffix_lc(&name_f) {
+            Some(suffix) => {
+                if suffix != "asc" {
+                    continue;
                 }
-                Ok(res) => res.file_name(),
-            };
-            let name_f = entry_fn.to_string_lossy();
-            let path_plus_name = dir.as_ref().join(&entry_fn);
-            match dc_get_filesuffix_lc(&name_f) {
-                Some(suffix) => {
-                    if suffix != "asc" {
-                        continue;
-                    }
-                    set_default = if name_f.contains("legacy") {
-                        info!(context, "found legacy key '{}'", path_plus_name.display());
-                        false
-                    } else {
-                        true
-                    }
+                set_default = if name_f.contains("legacy") {
+                    info!(context, "found legacy key '{}'", path_plus_name.display());
+                    false
+                } else {
+                    true
                 }
-                None => {
+            }
+            None => {
+                continue;
+            }
+        }
+        match dc_read_file(context, &path_plus_name) {
+            Ok(buf) => {
+                let armored = std::string::String::from_utf8_lossy(&buf);
+                if let Err(err) = set_self_key(context, &armored, set_default, false) {
+                    error!(context, "set_self_key: {}", err);
                     continue;
                 }
             }
-            let ccontent = if let Ok(content) = dc_read_file(context, &path_plus_name) {
-                key = String::from_utf8_lossy(&content).to_string();
-                CString::new(content).unwrap()
-            } else {
-                continue;
-            };
-
-            /* only import if we have a private key */
-            let mut buf2_headerline = String::default();
-            let split_res: bool;
-            unsafe {
-                let buf2 = dc_strdup(ccontent.as_ptr());
-                split_res = dc_split_armored_data(
-                    buf2,
-                    &mut buf2_headerline,
-                    ptr::null_mut(),
-                    ptr::null_mut(),
-                    ptr::null_mut(),
-                );
-                libc::free(buf2 as *mut libc::c_void);
-            }
-            if split_res
-                && buf2_headerline.contains("-----BEGIN PGP PUBLIC KEY BLOCK-----")
-                && !key.contains("-----BEGIN PGP PRIVATE KEY BLOCK")
-            {
-                info!(context, "ignoring public key file '{}", name_f);
-                // it's fine: DC exports public with private
-                continue;
-            }
-            if let Err(err) = set_self_key(context, &key, set_default, false) {
-                error!(context, "set_self_key: {}", err);
-                continue;
-            }
-            imported_cnt += 1
+            Err(_) => continue,
         }
-        ensure!(
-            imported_cnt > 0,
-            "No private keys found in \"{}\".",
-            dir_name
-        );
-        return Ok(());
-    } else {
-        bail!("Import: Cannot open directory \"{}\".", dir_name);
+        imported_cnt += 1;
     }
+    ensure!(
+        imported_cnt > 0,
+        "No private keys found in \"{}\".",
+        dir_name
+    );
+    Ok(())
 }
 
 fn export_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
@@ -856,7 +682,7 @@ fn export_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
         },
     )?;
 
-    ensure!(export_errors == 0, "errors while importing");
+    ensure!(export_errors == 0, "errors while exporting keys");
     Ok(())
 }
 
@@ -893,7 +719,9 @@ fn export_key_to_asc_file(
 mod tests {
     use super::*;
 
+    use crate::pgp::{split_armored_data, HEADER_AUTOCRYPT, HEADER_SETUPCODE};
     use crate::test_utils::*;
+    use ::pgp::armor::BlockType;
 
     #[test]
     fn test_render_setup_file() {
@@ -915,19 +743,12 @@ mod tests {
         assert!(msg.contains("-----END PGP MESSAGE-----\n"));
     }
 
-    fn ac_setup_msg_cb(ctx: &Context, evt: Event) -> libc::uintptr_t {
-        match evt {
-            Event::GetString {
-                id: StockMessage::AcSetupMsgBody,
-                ..
-            } => unsafe { "hello\r\nthere".strdup() as usize },
-            _ => logging_cb(ctx, evt),
-        }
-    }
-
     #[test]
-    fn otest_render_setup_file_newline_replace() {
-        let t = test_context(Some(Box::new(ac_setup_msg_cb)));
+    fn test_render_setup_file_newline_replace() {
+        let t = dummy_context();
+        t.ctx
+            .set_stock_translation(StockMessage::AcSetupMsgBody, "hello\r\nthere".to_string())
+            .unwrap();
         configure_alice_keypair(&t.ctx);
         let msg = render_setup_file(&t.ctx, "pw").unwrap();
         println!("{}", &msg);
@@ -983,47 +804,26 @@ mod tests {
         let ctx = dummy_context();
         let context = &ctx.ctx;
 
-        let mut headerline = String::default();
-        let mut setupcodebegin = ptr::null();
-        let mut preferencrypt = ptr::null();
+        let buf_1 = S_EM_SETUPFILE.as_bytes().to_vec();
+        let (typ, headers, base64) = split_armored_data(&buf_1).unwrap();
+        assert_eq!(typ, BlockType::Message);
+        assert!(S_EM_SETUPCODE.starts_with(headers.get(HEADER_SETUPCODE).unwrap()));
+        assert!(headers.get(HEADER_AUTOCRYPT).is_none());
 
-        let mut buf_1 = S_EM_SETUPFILE.to_string();
+        assert!(!base64.is_empty());
 
-        unsafe {
-            assert!(dc_split_armored_data(
-                buf_1.as_mut_ptr().cast(),
-                &mut headerline,
-                &mut setupcodebegin,
-                &mut preferencrypt,
-                ptr::null_mut(),
-            ));
-        }
-        assert_eq!(headerline, "-----BEGIN PGP MESSAGE-----");
-        assert!(!setupcodebegin.is_null());
+        let setup_file = S_EM_SETUPFILE.to_string();
+        let decrypted = decrypt_setup_file(
+            context,
+            S_EM_SETUPCODE,
+            std::io::Cursor::new(setup_file.as_bytes()),
+        )
+        .unwrap();
 
-        // TODO: verify that this is the right check
-        assert!(S_EM_SETUPCODE.starts_with(as_str(setupcodebegin)));
+        let (typ, headers, _base64) = split_armored_data(decrypted.as_bytes()).unwrap();
 
-        assert!(preferencrypt.is_null());
-
-        let mut setup_file = S_EM_SETUPFILE.to_string();
-        let mut decrypted = unsafe {
-            decrypt_setup_file(context, S_EM_SETUPCODE, setup_file.as_bytes_mut()).unwrap()
-        };
-
-        unsafe {
-            assert!(dc_split_armored_data(
-                decrypted.as_mut_ptr().cast(),
-                &mut headerline,
-                &mut setupcodebegin,
-                &mut preferencrypt,
-                ptr::null_mut(),
-            ));
-        }
-
-        assert_eq!(headerline, "-----BEGIN PGP PRIVATE KEY BLOCK-----");
-        assert!(setupcodebegin.is_null());
-        assert!(!preferencrypt.is_null());
-        assert_eq!(as_str(preferencrypt), "mutual",);
+        assert_eq!(typ, BlockType::PrivateKey);
+        assert_eq!(headers.get(HEADER_AUTOCRYPT), Some(&"mutual".to_string()));
+        assert!(headers.get(HEADER_SETUPCODE).is_none());
     }
 }

src/job.rs

@@ -3,16 +3,20 @@ use std::time::Duration;
 use deltachat_derive::{FromSql, ToSql};
 use rand::{thread_rng, Rng};
 
+use crate::blob::BlobObject;
 use crate::chat;
+use crate::config::Config;
 use crate::configure::*;
 use crate::constants::*;
 use crate::context::Context;
 use crate::dc_tools::*;
+use crate::error::Error;
 use crate::events::Event;
 use crate::imap::*;
 use crate::imex::*;
 use crate::location;
 use crate::login_param::LoginParam;
+use crate::message::MsgId;
 use crate::message::{self, Message, MessageState};
 use crate::mimefactory::{vec_contains_lowercase, Loaded, MimeFactory};
 use crate::param::*;
@@ -40,6 +44,7 @@ pub enum Action {
 
     // Jobs in the INBOX-thread, range from DC_IMAP_THREAD..DC_IMAP_THREAD+999
     Housekeeping = 105, // low priority ...
+    EmptyServer = 107,
     DeleteMsgOnImap = 110,
     MarkseenMdnOnImap = 120,
     MarkseenMsgOnImap = 130,
@@ -71,6 +76,7 @@ impl From<Action> for Thread {
 
             Housekeeping => Thread::Imap,
             DeleteMsgOnImap => Thread::Imap,
+            EmptyServer => Thread::Imap,
             MarkseenMdnOnImap => Thread::Imap,
             MarkseenMsgOnImap => Thread::Imap,
             MoveMsg => Thread::Imap,
@@ -129,22 +135,21 @@ impl Job {
         if !context.smtp.lock().unwrap().is_connected() {
             let loginparam = LoginParam::from_database(context, "configured_");
             let connected = context.smtp.lock().unwrap().connect(context, &loginparam);
-
-            if !connected {
-                self.try_again_later(3i32, None);
+            if connected.is_err() {
+                self.try_again_later(3, None);
                 return;
             }
         }
 
-        if let Some(filename) = self.param.get(Param::File) {
-            if let Ok(body) = dc_read_file(context, filename) {
+        if let Some(filename) = self.param.get_path(Param::File, context).unwrap_or(None) {
+            if let Ok(body) = dc_read_file(context, &filename) {
                 if let Some(recipients) = self.param.get(Param::Recipients) {
                     let recipients_list = recipients
-                        .split("\x1e")
+                        .split('\x1e')
                         .filter_map(|addr| match lettre::EmailAddress::new(addr.to_string()) {
                             Ok(addr) => Some(addr),
                             Err(err) => {
-                                eprintln!("WARNING: invalid recipient: {} {:?}", addr, err);
+                                warn!(context, "invalid recipient: {} {:?}", addr, err);
                                 None
                             }
                         })
@@ -153,42 +158,50 @@ impl Job {
                     /* if there is a msg-id and it does not exist in the db, cancel sending.
                     this happends if dc_delete_msgs() was called
                     before the generated mime was sent out */
-                    if 0 != self.foreign_id && !message::exists(context, self.foreign_id) {
+                    if 0 != self.foreign_id
+                        && !message::exists(context, MsgId::new(self.foreign_id))
+                    {
                         warn!(
                             context,
-                            "Message {} for job {} does not exist", self.foreign_id, self.job_id,
+                            "Not sending Message {} as it was deleted", self.foreign_id
                         );
                         return;
                     };
 
                     // hold the smtp lock during sending of a job and
                     // its ok/error response processing. Note that if a message
-                    // was sent we need to mark it in the database as we
+                    // was sent we need to mark it in the database ASAP as we
                     // otherwise might send it twice.
-                    let mut sock = context.smtp.lock().unwrap();
-                    if 0 == sock.send(context, recipients_list, body) {
-                        sock.disconnect();
-                        self.try_again_later(-1i32, sock.error.clone());
-                    } else {
-                        dc_delete_file(context, filename);
-                        if 0 != self.foreign_id {
-                            message::update_msg_state(
-                                context,
-                                self.foreign_id,
-                                MessageState::OutDelivered,
-                            );
-                            let chat_id: i32 = context
-                                .sql
-                                .query_get_value(
+                    let mut smtp = context.smtp.lock().unwrap();
+                    match smtp.send(context, recipients_list, body, self.job_id) {
+                        Err(err) => {
+                            smtp.disconnect();
+                            warn!(context, "smtp failed: {}", err);
+                            self.try_again_later(-1, Some(err.to_string()));
+                        }
+                        Ok(()) => {
+                            // smtp success, update db ASAP, then delete smtp file
+                            if 0 != self.foreign_id {
+                                message::update_msg_state(
                                     context,
-                                    "SELECT chat_id FROM msgs WHERE id=?",
-                                    params![self.foreign_id as i32],
-                                )
-                                .unwrap_or_default();
-                            context.call_cb(Event::MsgDelivered {
-                                chat_id: chat_id as u32,
-                                msg_id: self.foreign_id,
-                            });
+                                    MsgId::new(self.foreign_id),
+                                    MessageState::OutDelivered,
+                                );
+                                let chat_id: i32 = context
+                                    .sql
+                                    .query_get_value(
+                                        context,
+                                        "SELECT chat_id FROM msgs WHERE id=?",
+                                        params![self.foreign_id as i32],
+                                    )
+                                    .unwrap_or_default();
+                                context.call_cb(Event::MsgDelivered {
+                                    chat_id: chat_id as u32,
+                                    msg_id: MsgId::new(self.foreign_id),
+                                });
+                            }
+                            // now also delete the generated file
+                            dc_delete_file(context, filename);
                         }
                     }
                 } else {
@@ -199,7 +212,7 @@ impl Job {
     }
 
     // this value does not increase the number of tries
-    fn try_again_later(&mut self, try_again: libc::c_int, pending_error: Option<String>) {
+    fn try_again_later(&mut self, try_again: i32, pending_error: Option<String>) {
         self.try_again = try_again;
         self.pending_error = pending_error;
     }
@@ -208,23 +221,18 @@ impl Job {
     fn do_DC_JOB_MOVE_MSG(&mut self, context: &Context) {
         let inbox = context.inbox.read().unwrap();
 
-        if !inbox.is_connected() {
-            connect_to_inbox(context, &inbox);
-            if !inbox.is_connected() {
-                self.try_again_later(3, None);
-                return;
-            }
-        }
-        if let Ok(msg) = Message::load_from_db(context, self.foreign_id) {
+        if let Ok(msg) = Message::load_from_db(context, MsgId::new(self.foreign_id)) {
             if context
                 .sql
-                .get_config_int(context, "folders_configured")
+                .get_raw_config_int(context, "folders_configured")
                 .unwrap_or_default()
                 < 3
             {
                 inbox.configure_folders(context, 0x1i32);
             }
-            let dest_folder = context.sql.get_config(context, "configured_mvbox_folder");
+            let dest_folder = context
+                .sql
+                .get_raw_config(context, "configured_mvbox_folder");
 
             if let Some(dest_folder) = dest_folder {
                 let server_folder = msg.server_folder.as_ref().unwrap();
@@ -258,29 +266,21 @@ impl Job {
     fn do_DC_JOB_DELETE_MSG_ON_IMAP(&mut self, context: &Context) {
         let inbox = context.inbox.read().unwrap();
 
-        if let Ok(mut msg) = Message::load_from_db(context, self.foreign_id) {
+        if let Ok(mut msg) = Message::load_from_db(context, MsgId::new(self.foreign_id)) {
             if !msg.rfc724_mid.is_empty() {
                 /* eg. device messages have no Message-ID */
-                let mut delete_from_server = true;
-                if message::rfc724_mid_cnt(context, &msg.rfc724_mid) != 1 {
+                if message::rfc724_mid_cnt(context, &msg.rfc724_mid) > 1 {
                     info!(
                         context,
                         "The message is deleted from the server when all parts are deleted.",
                     );
-                    delete_from_server = false;
-                }
-                /* if this is the last existing part of the message, we delete the message from the server */
-                if delete_from_server {
-                    if !inbox.is_connected() {
-                        connect_to_inbox(context, &inbox);
-                        if !inbox.is_connected() {
-                            self.try_again_later(3i32, None);
-                            return;
-                        }
-                    }
+                } else {
+                    /* if this is the last existing part of the message,
+                    we delete the message from the server */
                     let mid = msg.rfc724_mid;
                     let server_folder = msg.server_folder.as_ref().unwrap();
-                    if 0 == inbox.delete_msg(context, &mid, server_folder, &mut msg.server_uid) {
+                    let res = inbox.delete_msg(context, &mid, server_folder, &mut msg.server_uid);
+                    if res == ImapResult::RetryLater {
                         self.try_again_later(-1i32, None);
                         return;
                     }
@@ -290,41 +290,43 @@ impl Job {
         }
     }
 
+    #[allow(non_snake_case)]
+    fn do_DC_JOB_EMPTY_SERVER(&mut self, context: &Context) {
+        let inbox = context.inbox.read().unwrap();
+        if self.foreign_id & DC_EMPTY_MVBOX > 0 {
+            if let Some(mvbox_folder) = context
+                .sql
+                .get_raw_config(context, "configured_mvbox_folder")
+            {
+                inbox.empty_folder(context, &mvbox_folder);
+            }
+        }
+        if self.foreign_id & DC_EMPTY_INBOX > 0 {
+            inbox.empty_folder(context, "INBOX");
+        }
+    }
+
     #[allow(non_snake_case)]
     fn do_DC_JOB_MARKSEEN_MSG_ON_IMAP(&mut self, context: &Context) {
         let inbox = context.inbox.read().unwrap();
 
-        if !inbox.is_connected() {
-            connect_to_inbox(context, &inbox);
-            if !inbox.is_connected() {
-                self.try_again_later(3i32, None);
-                return;
-            }
-        }
-        if let Ok(msg) = Message::load_from_db(context, self.foreign_id) {
-            let server_folder = msg.server_folder.as_ref().unwrap();
-            match inbox.set_seen(context, server_folder, msg.server_uid) {
-                ImapResult::Failed => {}
+        if let Ok(msg) = Message::load_from_db(context, MsgId::new(self.foreign_id)) {
+            let folder = msg.server_folder.as_ref().unwrap();
+            match inbox.set_seen(context, folder, msg.server_uid) {
                 ImapResult::RetryLater => {
                     self.try_again_later(3i32, None);
                 }
-                _ => {
+                ImapResult::AlreadyDone => {}
+                ImapResult::Success | ImapResult::Failed => {
+                    // XXX the message might just have been moved
+                    // we want to send out an MDN anyway
+                    // The job will not be retried so locally
+                    // there is no risk of double-sending MDNs.
                     if 0 != msg.param.get_int(Param::WantsMdn).unwrap_or_default()
-                        && 0 != context
-                            .sql
-                            .get_config_int(context, "mdns_enabled")
-                            .unwrap_or_else(|| 1)
+                        && context.get_config_bool(Config::MdnsEnabled)
                     {
-                        let folder = msg.server_folder.as_ref().unwrap();
-
-                        match inbox.set_mdnsent(context, folder, msg.server_uid) {
-                            ImapResult::RetryLater => {
-                                self.try_again_later(3i32, None);
-                            }
-                            ImapResult::Success => {
-                                send_mdn(context, msg.id);
-                            }
-                            ImapResult::Failed | ImapResult::AlreadyDone => {}
+                        if let Err(err) = send_mdn(context, msg.id) {
+                            warn!(context, "could not send out mdn for {}: {}", msg.id, err);
                         }
                     }
                 }
@@ -341,31 +343,26 @@ impl Job {
             .to_string();
         let uid = self.param.get_int(Param::ServerUid).unwrap_or_default() as u32;
         let inbox = context.inbox.read().unwrap();
-
-        if !inbox.is_connected() {
-            connect_to_inbox(context, &inbox);
-            if !inbox.is_connected() {
-                self.try_again_later(3, None);
-                return;
-            }
-        }
-        if inbox.set_seen(context, &folder, uid) == ImapResult::Failed {
+        if inbox.set_seen(context, &folder, uid) == ImapResult::RetryLater {
             self.try_again_later(3i32, None);
+            return;
         }
         if 0 != self.param.get_int(Param::AlsoMove).unwrap_or_default() {
             if context
                 .sql
-                .get_config_int(context, "folders_configured")
+                .get_raw_config_int(context, "folders_configured")
                 .unwrap_or_default()
                 < 3
             {
                 inbox.configure_folders(context, 0x1i32);
             }
-            let dest_folder = context.sql.get_config(context, "configured_mvbox_folder");
+            let dest_folder = context
+                .sql
+                .get_raw_config(context, "configured_mvbox_folder");
             if let Some(dest_folder) = dest_folder {
                 let mut dest_uid = 0;
                 if ImapResult::RetryLater
-                    == inbox.mv(context, folder, uid, dest_folder, &mut dest_uid)
+                    == inbox.mv(context, &folder, uid, &dest_folder, &mut dest_uid)
                 {
                     self.try_again_later(3, None);
                 }
@@ -392,12 +389,7 @@ pub fn perform_imap_fetch(context: &Context) {
     if 0 == connect_to_inbox(context, &inbox) {
         return;
     }
-    if context
-        .sql
-        .get_config_int(context, "inbox_watch")
-        .unwrap_or_else(|| 1)
-        == 0
-    {
+    if !context.get_config_bool(Config::InboxWatch) {
         info!(context, "INBOX-watch disabled.",);
         return;
     }
@@ -410,7 +402,7 @@ pub fn perform_imap_fetch(context: &Context) {
     info!(
         context,
         "INBOX-fetch done in {:.4} ms.",
-        start.elapsed().as_nanos() as f64 / 1000.0,
+        start.elapsed().as_nanos() as f64 / 1_000_000.0,
     );
 }
 
@@ -432,29 +424,23 @@ pub fn perform_imap_idle(context: &Context) {
 }
 
 pub fn perform_mvbox_fetch(context: &Context) {
-    let use_network = context
-        .sql
-        .get_config_int(context, "mvbox_watch")
-        .unwrap_or_else(|| 1);
+    let use_network = context.get_config_bool(Config::MvboxWatch);
 
     context
         .mvbox_thread
         .write()
         .unwrap()
-        .fetch(context, use_network == 1);
+        .fetch(context, use_network);
 }
 
 pub fn perform_mvbox_idle(context: &Context) {
-    let use_network = context
-        .sql
-        .get_config_int(context, "mvbox_watch")
-        .unwrap_or_else(|| 1);
+    let use_network = context.get_config_bool(Config::MvboxWatch);
 
     context
         .mvbox_thread
         .read()
         .unwrap()
-        .idle(context, use_network == 1);
+        .idle(context, use_network);
 }
 
 pub fn interrupt_mvbox_idle(context: &Context) {
@@ -462,29 +448,23 @@ pub fn interrupt_mvbox_idle(context: &Context) {
 }
 
 pub fn perform_sentbox_fetch(context: &Context) {
-    let use_network = context
-        .sql
-        .get_config_int(context, "sentbox_watch")
-        .unwrap_or_else(|| 1);
+    let use_network = context.get_config_bool(Config::SentboxWatch);
 
     context
         .sentbox_thread
         .write()
         .unwrap()
-        .fetch(context, use_network == 1);
+        .fetch(context, use_network);
 }
 
 pub fn perform_sentbox_idle(context: &Context) {
-    let use_network = context
-        .sql
-        .get_config_int(context, "sentbox_watch")
-        .unwrap_or_else(|| 1);
+    let use_network = context.get_config_bool(Config::SentboxWatch);
 
     context
         .sentbox_thread
         .read()
         .unwrap()
-        .idle(context, use_network == 1);
+        .idle(context, use_network);
 }
 
 pub fn interrupt_sentbox_idle(context: &Context) {
@@ -601,109 +581,104 @@ pub fn job_action_exists(context: &Context, action: Action) -> bool {
 
 /* special case for DC_JOB_SEND_MSG_TO_SMTP */
 #[allow(non_snake_case)]
-pub fn job_send_msg(context: &Context, msg_id: u32) -> libc::c_int {
-    let mut success = 0;
+pub fn job_send_msg(context: &Context, msg_id: MsgId) -> Result<(), Error> {
+    let mut mimefactory = MimeFactory::load_msg(context, msg_id)?;
 
-    /* load message data */
-    let mimefactory = MimeFactory::load_msg(context, msg_id);
-    if mimefactory.is_err() {
-        warn!(
-            context,
-            "Cannot load data to send, maybe the message is deleted in between.",
-        );
-    } else {
-        let mut mimefactory = mimefactory.unwrap();
-        // no redo, no IMAP. moreover, as the data does not exist, there is no need in calling dc_set_msg_failed()
-        if chat::msgtype_has_file(mimefactory.msg.type_0) {
-            let file_param = mimefactory
-                .msg
-                .param
-                .get(Param::File)
-                .map(|s| s.to_string());
-            if let Some(pathNfilename) = file_param {
-                if (mimefactory.msg.type_0 == Viewtype::Image
-                    || mimefactory.msg.type_0 == Viewtype::Gif)
-                    && !mimefactory.msg.param.exists(Param::Width)
-                {
-                    mimefactory.msg.param.set_int(Param::Width, 0);
-                    mimefactory.msg.param.set_int(Param::Height, 0);
-
-                    if let Ok(buf) = dc_read_file(context, pathNfilename) {
-                        if let Ok((width, height)) = dc_get_filemeta(&buf) {
-                            mimefactory.msg.param.set_int(Param::Width, width as i32);
-                            mimefactory.msg.param.set_int(Param::Height, height as i32);
-                        }
-                    }
-                    mimefactory.msg.save_param_to_disk(context);
-                }
-            }
-        }
-        /* create message */
-        if let Err(msg) = unsafe { mimefactory.render() } {
-            let e = msg.to_string();
-            message::set_msg_failed(context, msg_id, Some(e));
-        } else if 0
-            != mimefactory
-                .msg
-                .param
-                .get_int(Param::GuranteeE2ee)
-                .unwrap_or_default()
-            && !mimefactory.out_encrypted
-        {
-            /* unrecoverable */
-            warn!(
-                context,
-                "e2e encryption unavailable {} - {:?}",
-                msg_id,
-                mimefactory.msg.param.get_int(Param::GuranteeE2ee),
-            );
-            message::set_msg_failed(
-                context,
-                msg_id,
-                Some("End-to-end-encryption unavailable unexpectedly."),
-            );
-        } else {
-            if !vec_contains_lowercase(&mimefactory.recipients_addr, &mimefactory.from_addr) {
-                mimefactory.recipients_names.push("".to_string());
-                mimefactory
-                    .recipients_addr
-                    .push(mimefactory.from_addr.to_string());
-            }
-            if mimefactory.out_gossiped {
-                chat::set_gossiped_timestamp(context, mimefactory.msg.chat_id, time());
-            }
-            if 0 != mimefactory.out_last_added_location_id {
-                if let Err(err) =
-                    location::set_kml_sent_timestamp(context, mimefactory.msg.chat_id, time())
-                {
-                    error!(context, "Failed to set kml sent_timestamp: {:?}", err);
-                }
-                if !mimefactory.msg.hidden {
-                    if let Err(err) = location::set_msg_location_id(
-                        context,
-                        mimefactory.msg.id,
-                        mimefactory.out_last_added_location_id,
-                    ) {
-                        error!(context, "Failed to set msg_location_id: {:?}", err);
-                    }
-                }
-            }
-            if mimefactory.out_encrypted
-                && mimefactory
-                    .msg
-                    .param
-                    .get_int(Param::GuranteeE2ee)
-                    .unwrap_or_default()
-                    == 0
+    if chat::msgtype_has_file(mimefactory.msg.type_0) {
+        let file_param = mimefactory.msg.param.get_path(Param::File, context)?;
+        if let Some(pathNfilename) = file_param {
+            if (mimefactory.msg.type_0 == Viewtype::Image
+                || mimefactory.msg.type_0 == Viewtype::Gif)
+                && !mimefactory.msg.param.exists(Param::Width)
             {
-                mimefactory.msg.param.set_int(Param::GuranteeE2ee, 1);
+                mimefactory.msg.param.set_int(Param::Width, 0);
+                mimefactory.msg.param.set_int(Param::Height, 0);
+
+                if let Ok(buf) = dc_read_file(context, pathNfilename) {
+                    if let Ok((width, height)) = dc_get_filemeta(&buf) {
+                        mimefactory.msg.param.set_int(Param::Width, width as i32);
+                        mimefactory.msg.param.set_int(Param::Height, height as i32);
+                    }
+                }
                 mimefactory.msg.save_param_to_disk(context);
             }
-            success = add_smtp_job(context, Action::SendMsgToSmtp, &mut mimefactory);
         }
     }
 
-    success
+    /* create message */
+    if let Err(msg) = unsafe { mimefactory.render() } {
+        let e = msg.to_string();
+        message::set_msg_failed(context, msg_id, Some(e));
+        return Err(msg);
+    }
+    if 0 != mimefactory
+        .msg
+        .param
+        .get_int(Param::GuaranteeE2ee)
+        .unwrap_or_default()
+        && !mimefactory.out_encrypted
+    {
+        /* unrecoverable */
+        message::set_msg_failed(
+            context,
+            msg_id,
+            Some("End-to-end-encryption unavailable unexpectedly."),
+        );
+        bail!(
+            "e2e encryption unavailable {} - {:?}",
+            msg_id,
+            mimefactory.msg.param.get_int(Param::GuaranteeE2ee),
+        );
+    }
+    if context.get_config_bool(Config::BccSelf)
+        && !vec_contains_lowercase(&mimefactory.recipients_addr, &mimefactory.from_addr)
+    {
+        mimefactory.recipients_names.push("".to_string());
+        mimefactory
+            .recipients_addr
+            .push(mimefactory.from_addr.to_string());
+    }
+
+    if mimefactory.recipients_addr.is_empty() {
+        warn!(
+            context,
+            "message {} has no recipient, skipping smtp-send", msg_id
+        );
+        return Ok(());
+    }
+
+    if mimefactory.out_gossiped {
+        chat::set_gossiped_timestamp(context, mimefactory.msg.chat_id, time());
+    }
+    if 0 != mimefactory.out_last_added_location_id {
+        if let Err(err) = location::set_kml_sent_timestamp(context, mimefactory.msg.chat_id, time())
+        {
+            error!(context, "Failed to set kml sent_timestamp: {:?}", err);
+        }
+        if !mimefactory.msg.hidden {
+            if let Err(err) = location::set_msg_location_id(
+                context,
+                mimefactory.msg.id,
+                mimefactory.out_last_added_location_id,
+            ) {
+                error!(context, "Failed to set msg_location_id: {:?}", err);
+            }
+        }
+    }
+    if mimefactory.out_encrypted
+        && mimefactory
+            .msg
+            .param
+            .get_int(Param::GuaranteeE2ee)
+            .unwrap_or_default()
+            == 0
+    {
+        mimefactory.msg.param.set_int(Param::GuaranteeE2ee, 1);
+        mimefactory.msg.save_param_to_disk(context);
+    }
+    add_smtp_job(context, Action::SendMsgToSmtp, &mut mimefactory)?;
+
+    Ok(())
 }
 
 pub fn perform_imap_jobs(context: &Context) {
@@ -793,13 +768,13 @@ fn job_perform(context: &Context, thread: Thread, probe_network: bool) {
         // - they can be re-executed one time AT_ONCE, but they are not save in the database for later execution
         if Action::ConfigureImap == job.action || Action::ImexImap == job.action {
             job_kill_action(context, job.action);
-            &context
+            context
                 .sentbox_thread
                 .clone()
                 .read()
                 .unwrap()
                 .suspend(context);
-            &context
+            context
                 .mvbox_thread
                 .clone()
                 .read()
@@ -818,12 +793,13 @@ fn job_perform(context: &Context, thread: Thread, probe_network: bool) {
                     warn!(context, "Unknown job id found");
                 }
                 Action::SendMsgToSmtp => job.do_DC_JOB_SEND(context),
+                Action::EmptyServer => job.do_DC_JOB_EMPTY_SERVER(context),
                 Action::DeleteMsgOnImap => job.do_DC_JOB_DELETE_MSG_ON_IMAP(context),
                 Action::MarkseenMsgOnImap => job.do_DC_JOB_MARKSEEN_MSG_ON_IMAP(context),
                 Action::MarkseenMdnOnImap => job.do_DC_JOB_MARKSEEN_MDN_ON_IMAP(context),
                 Action::MoveMsg => job.do_DC_JOB_MOVE_MSG(context),
                 Action::SendMdn => job.do_DC_JOB_SEND(context),
-                Action::ConfigureImap => unsafe { dc_job_do_DC_JOB_CONFIGURE_IMAP(context) },
+                Action::ConfigureImap => dc_job_do_DC_JOB_CONFIGURE_IMAP(context),
                 Action::ImexImap => match job_do_DC_JOB_IMEX_IMAP(context, &job) {
                     Ok(()) => {}
                     Err(err) => {
@@ -903,7 +879,11 @@ fn job_perform(context: &Context, thread: Thread, probe_network: bool) {
                 }
             } else {
                 if job.action == Action::SendMsgToSmtp {
-                    message::set_msg_failed(context, job.foreign_id, job.pending_error.as_ref());
+                    message::set_msg_failed(
+                        context,
+                        MsgId::new(job.foreign_id),
+                        job.pending_error.as_ref(),
+                    );
                 }
                 job.delete(context);
             }
@@ -924,8 +904,7 @@ fn job_perform(context: &Context, thread: Thread, probe_network: bool) {
 #[allow(non_snake_case)]
 fn get_backoff_time_offset(c_tries: libc::c_int) -> i64 {
     // results in ~3 weeks for the last backoff timespan
-    let mut N = 2_i32.pow((c_tries - 1) as u32);
-    N = N * 60;
+    let N = 2_i32.pow((c_tries - 1) as u32) * 60;
     let mut rng = thread_rng();
     let n: i32 = rng.gen();
     let mut seconds = n % (N + 1);
@@ -947,7 +926,7 @@ fn suspend_smtp_thread(context: &Context, suspend: bool) {
     }
 }
 
-fn connect_to_inbox(context: &Context, inbox: &Imap) -> libc::c_int {
+pub fn connect_to_inbox(context: &Context, inbox: &Imap) -> libc::c_int {
     let ret_connected = dc_connect_to_configured_imap(context, inbox);
     if 0 != ret_connected {
         inbox.set_watch_folder("INBOX".into());
@@ -955,51 +934,44 @@ fn connect_to_inbox(context: &Context, inbox: &Imap) -> libc::c_int {
     ret_connected
 }
 
-fn send_mdn(context: &Context, msg_id: u32) {
-    if let Ok(mut mimefactory) = MimeFactory::load_mdn(context, msg_id) {
-        if unsafe { mimefactory.render() }.is_ok() {
-            add_smtp_job(context, Action::SendMdn, &mut mimefactory);
-        }
-    }
+fn send_mdn(context: &Context, msg_id: MsgId) -> Result<(), Error> {
+    let mut mimefactory = MimeFactory::load_mdn(context, msg_id)?;
+    unsafe { mimefactory.render()? };
+    add_smtp_job(context, Action::SendMdn, &mut mimefactory)?;
+
+    Ok(())
 }
 
 #[allow(non_snake_case)]
-fn add_smtp_job(context: &Context, action: Action, mimefactory: &MimeFactory) -> libc::c_int {
-    let mut success: libc::c_int = 0i32;
+fn add_smtp_job(context: &Context, action: Action, mimefactory: &MimeFactory) -> Result<(), Error> {
+    ensure!(
+        !mimefactory.recipients_addr.is_empty(),
+        "no recipients for smtp job set"
+    );
     let mut param = Params::new();
-    let path_filename = dc_get_fine_path_filename(context, "$BLOBDIR", &mimefactory.rfc724_mid);
     let bytes = unsafe {
         std::slice::from_raw_parts(
             (*mimefactory.out).str_0 as *const u8,
             (*mimefactory.out).len,
         )
     };
-    if !dc_write_file(context, &path_filename, bytes) {
-        error!(
-            context,
-            "Could not write message <{}> to \"{}\".",
-            mimefactory.rfc724_mid,
-            path_filename.display(),
-        );
-    } else {
-        info!(context, "add_smtp_job file written: {:?}", path_filename);
-        let recipients = mimefactory.recipients_addr.join("\x1e");
-        param.set(Param::File, path_filename.to_string_lossy());
-        param.set(Param::Recipients, &recipients);
-        job_add(
-            context,
-            action,
-            (if mimefactory.loaded == Loaded::Message {
-                mimefactory.msg.id
-            } else {
-                0
-            }) as libc::c_int,
-            param,
-            0,
-        );
-        success = 1;
-    }
-    success
+    let blob = BlobObject::create(context, &mimefactory.rfc724_mid, bytes)?;
+    let recipients = mimefactory.recipients_addr.join("\x1e");
+    param.set(Param::File, blob.as_name());
+    param.set(Param::Recipients, &recipients);
+    job_add(
+        context,
+        action,
+        (if mimefactory.loaded == Loaded::Message {
+            mimefactory.msg.id.to_u32() as i32
+        } else {
+            0
+        }) as libc::c_int,
+        param,
+        0,
+    );
+
+    Ok(())
 }
 
 pub fn job_add(
@@ -1050,8 +1022,9 @@ pub fn interrupt_smtp_idle(context: &Context) {
 }
 
 pub fn interrupt_imap_idle(context: &Context) {
-    info!(context, "Interrupting IMAP-IDLE...",);
+    info!(context, "Interrupting INBOX-IDLE...",);
 
     *context.perform_inbox_jobs_needed.write().unwrap() = true;
+
     context.inbox.read().unwrap().interrupt_idle();
 }

src/job_thread.rs

@@ -15,7 +15,7 @@ pub struct JobThread {
 #[derive(Clone, Debug, Default)]
 pub struct JobState {
     idle: bool,
-    jobs_needed: i32,
+    jobs_needed: bool,
     suspended: bool,
     using_handle: bool,
 }
@@ -58,7 +58,7 @@ impl JobThread {
 
     pub fn interrupt_idle(&self, context: &Context) {
         {
-            self.state.0.lock().unwrap().jobs_needed = 1;
+            self.state.0.lock().unwrap().jobs_needed = true;
         }
 
         info!(context, "Interrupting {}-IDLE...", self.name);
@@ -116,14 +116,14 @@ impl JobThread {
         if ret_connected {
             if context
                 .sql
-                .get_config_int(context, "folders_configured")
+                .get_raw_config_int(context, "folders_configured")
                 .unwrap_or_default()
                 < 3
             {
                 self.imap.configure_folders(context, 0x1);
             }
 
-            if let Some(mvbox_name) = context.sql.get_config(context, self.folder_config_name) {
+            if let Some(mvbox_name) = context.sql.get_raw_config(context, self.folder_config_name) {
                 self.imap.set_watch_folder(mvbox_name);
             } else {
                 self.imap.disconnect(context);
@@ -139,13 +139,13 @@ impl JobThread {
             let &(ref lock, ref cvar) = &*self.state.clone();
             let mut state = lock.lock().unwrap();
 
-            if 0 != state.jobs_needed {
+            if state.jobs_needed {
                 info!(
                     context,
                     "{}-IDLE will not be started as it was interrupted while not ideling.",
                     self.name,
                 );
-                state.jobs_needed = 0;
+                state.jobs_needed = false;
                 return;
             }
 

src/key.rs

@@ -163,8 +163,8 @@ impl Key {
 
     pub fn to_bytes(&self) -> Vec<u8> {
         match self {
-            Key::Public(k) => k.to_bytes().unwrap(),
-            Key::Secret(k) => k.to_bytes().unwrap(),
+            Key::Public(k) => k.to_bytes().unwrap_or_default(),
+            Key::Secret(k) => k.to_bytes().unwrap_or_default(),
         }
     }
 
@@ -180,15 +180,15 @@ impl Key {
 
         let encoded = base64::encode(&buf);
         encoded
-            .as_bytes()
-            .chunks(break_every)
-            .fold(String::new(), |mut res, buf| {
-                // safe because we are using a base64 encoded string
-                res += unsafe { std::str::from_utf8_unchecked(buf) };
-                res += " ";
+            .chars()
+            .enumerate()
+            .fold(String::new(), |mut res, (i, c)| {
+                if i > 0 && i % break_every == 0 {
+                    res.push(' ')
+                }
+                res.push(c);
                 res
             })
-            .trim()
             .to_string()
     }
 
@@ -218,10 +218,10 @@ impl Key {
         let file_content = self.to_asc(None).into_bytes();
 
         if dc_write_file(context, &file, &file_content) {
-            return true;
+            true
         } else {
             error!(context, "Cannot write key to {}", file.as_ref().display());
-            return false;
+            false
         }
     }
 
@@ -381,7 +381,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     #[test]
     #[ignore] // is too expensive
     fn test_from_slice_roundtrip() {
-        let (public_key, private_key) = crate::pgp::dc_pgp_create_keypair("hello").unwrap();
+        let (public_key, private_key) = crate::pgp::create_keypair("hello").unwrap();
 
         let binary = public_key.to_bytes();
         let public_key2 = Key::from_slice(&binary, KeyType::Public).expect("invalid public key");
@@ -416,7 +416,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     #[test]
     #[ignore] // is too expensive
     fn test_ascii_roundtrip() {
-        let (public_key, private_key) = crate::pgp::dc_pgp_create_keypair("hello").unwrap();
+        let (public_key, private_key) = crate::pgp::create_keypair("hello").unwrap();
 
         let s = public_key.to_armored_string(None).unwrap();
         let (public_key2, _) =

src/lib.rs

@@ -30,6 +30,7 @@ pub(crate) mod events;
 pub use events::*;
 
 mod aheader;
+pub mod blob;
 pub mod chat;
 pub mod chatlist;
 pub mod config;
@@ -57,7 +58,7 @@ pub mod qr;
 pub mod securejoin;
 mod smtp;
 pub mod sql;
-mod stock;
+pub mod stock;
 mod token;
 #[macro_use]
 mod wrapmime;

src/location.rs

@@ -3,13 +3,15 @@ use quick_xml;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
 use crate::chat;
+use crate::config::Config;
 use crate::constants::*;
 use crate::context::*;
+use crate::dc_mimeparser::SystemMessage;
 use crate::dc_tools::*;
 use crate::error::Error;
 use crate::events::Event;
 use crate::job::*;
-use crate::message::Message;
+use crate::message::{Message, MsgId};
 use crate::param::*;
 use crate::sql;
 use crate::stock::StockMessage;
@@ -60,14 +62,11 @@ impl Kml {
         Default::default()
     }
 
-    pub fn parse(context: &Context, content: impl AsRef<str>) -> Result<Self, Error> {
-        ensure!(
-            content.as_ref().len() <= (1 * 1024 * 1024),
-            "A kml-files with {} bytes is larger than reasonably expected.",
-            content.as_ref().len()
-        );
+    pub fn parse(context: &Context, content: &[u8]) -> Result<Self, Error> {
+        ensure!(content.len() <= 1024 * 1024, "kml-file is too large");
 
-        let mut reader = quick_xml::Reader::from_str(content.as_ref());
+        let to_parse = String::from_utf8_lossy(content);
+        let mut reader = quick_xml::Reader::from_str(&to_parse);
         reader.trim_text(true);
 
         let mut kml = Kml::new();
@@ -194,10 +193,8 @@ impl Kml {
 // location streaming
 pub fn send_locations_to_chat(context: &Context, chat_id: u32, seconds: i64) {
     let now = time();
-    let mut msg: Message;
-    let is_sending_locations_before: bool;
-    if !(seconds < 0 || chat_id <= 9i32 as libc::c_uint) {
-        is_sending_locations_before = is_sending_locations_to_chat(context, chat_id);
+    if !(seconds < 0 || chat_id <= DC_CHAT_ID_LAST_SPECIAL) {
+        let is_sending_locations_before = is_sending_locations_to_chat(context, chat_id);
         if sql::execute(
             context,
             &context.sql,
@@ -214,19 +211,19 @@ pub fn send_locations_to_chat(context: &Context, chat_id: u32, seconds: i64) {
         .is_ok()
         {
             if 0 != seconds && !is_sending_locations_before {
-                msg = Message::new(Viewtype::Text);
+                let mut msg = Message::new(Viewtype::Text);
                 msg.text =
                     Some(context.stock_system_msg(StockMessage::MsgLocationEnabled, "", "", 0));
-                msg.param.set_int(Param::Cmd, 8);
-                chat::send_msg(context, chat_id, &mut msg).unwrap();
+                msg.param.set_cmd(SystemMessage::LocationStreamingEnabled);
+                chat::send_msg(context, chat_id, &mut msg).unwrap_or_default();
             } else if 0 == seconds && is_sending_locations_before {
                 let stock_str =
                     context.stock_system_msg(StockMessage::MsgLocationDisabled, "", "", 0);
-                chat::add_device_msg(context, chat_id, stock_str);
+                chat::add_info_msg(context, chat_id, stock_str);
             }
             context.call_cb(Event::ChatModified(chat_id));
             if 0 != seconds {
-                schedule_MAYBE_SEND_LOCATIONS(context, 0i32);
+                schedule_MAYBE_SEND_LOCATIONS(context, false);
                 job_add(
                     context,
                     Action::MaybeSendLocationsEnded,
@@ -240,8 +237,8 @@ pub fn send_locations_to_chat(context: &Context, chat_id: u32, seconds: i64) {
 }
 
 #[allow(non_snake_case)]
-fn schedule_MAYBE_SEND_LOCATIONS(context: &Context, flags: i32) {
-    if 0 != flags & 0x1 || !job_action_exists(context, Action::MaybeSendLocations) {
+fn schedule_MAYBE_SEND_LOCATIONS(context: &Context, force_schedule: bool) {
+    if force_schedule || !job_action_exists(context, Action::MaybeSendLocations) {
         job_add(context, Action::MaybeSendLocations, 0, Params::new(), 60);
     };
 }
@@ -256,9 +253,9 @@ pub fn is_sending_locations_to_chat(context: &Context, chat_id: u32) -> bool {
         .unwrap_or_default()
 }
 
-pub fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64) -> libc::c_int {
+pub fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64) -> bool {
     if latitude == 0.0 && longitude == 0.0 {
-        return 1;
+        return true;
     }
     let mut continue_streaming = false;
 
@@ -278,7 +275,7 @@ pub fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64) -> l
                         accuracy,
                         time(),
                         chat_id,
-                        1,
+                        DC_CONTACT_ID_SELF,
                     ]
             ) {
                 warn!(context, "failed to store location {:?}", err);
@@ -287,12 +284,12 @@ pub fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64) -> l
             }
         }
         if continue_streaming {
-            context.call_cb(Event::LocationChanged(Some(1)));
+            context.call_cb(Event::LocationChanged(Some(DC_CONTACT_ID_SELF)));
         };
-        schedule_MAYBE_SEND_LOCATIONS(context, 0);
+        schedule_MAYBE_SEND_LOCATIONS(context, false);
     }
 
-    continue_streaming as libc::c_int
+    continue_streaming
 }
 
 pub fn get_range(
@@ -359,7 +356,7 @@ pub fn get_range(
 }
 
 fn is_marker(txt: &str) -> bool {
-    txt.len() == 1 && txt.chars().next().unwrap() != ' '
+    txt.len() == 1 && !txt.starts_with(' ')
 }
 
 pub fn delete_all(context: &Context) -> Result<(), Error> {
@@ -369,14 +366,10 @@ pub fn delete_all(context: &Context) -> Result<(), Error> {
 }
 
 pub fn get_kml(context: &Context, chat_id: u32) -> Result<(String, u32), Error> {
-    let now = time();
-    let mut location_count = 0;
-    let mut ret = String::new();
     let mut last_added_location_id = 0;
 
     let self_addr = context
-        .sql
-        .get_config(context, "configured_addr")
+        .get_config(Config::ConfiguredAddr)
         .unwrap_or_default();
 
     let (locations_send_begin, locations_send_until, locations_last_sent) = context.sql.query_row(
@@ -389,21 +382,24 @@ pub fn get_kml(context: &Context, chat_id: u32) -> Result<(String, u32), Error>
             Ok((send_begin, send_until, last_sent))
         })?;
 
-    if !(locations_send_begin == 0 || now > locations_send_until) {
+    let now = time();
+    let mut location_count = 0;
+    let mut ret = String::new();
+    if locations_send_begin != 0 && now <= locations_send_until {
         ret += &format!(
             "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<kml xmlns=\"http://www.opengis.net/kml/2.2\">\n<Document addr=\"{}\">\n",
             self_addr,
         );
 
         context.sql.query_map(
-            "SELECT id, latitude, longitude, accuracy, timestamp\
+            "SELECT id, latitude, longitude, accuracy, timestamp \
              FROM locations  WHERE from_id=? \
              AND timestamp>=? \
              AND (timestamp>=? OR timestamp=(SELECT MAX(timestamp) FROM locations WHERE from_id=?)) \
              AND independent=0 \
              GROUP BY timestamp \
              ORDER BY timestamp;",
-            params![1, locations_send_begin, locations_last_sent, 1],
+            params![DC_CONTACT_ID_SELF, locations_send_begin, locations_last_sent, DC_CONTACT_ID_SELF],
             |row| {
                 let location_id: i32 = row.get(0)?;
                 let latitude: f64 = row.get(1)?;
@@ -417,7 +413,7 @@ pub fn get_kml(context: &Context, chat_id: u32) -> Result<(String, u32), Error>
                 for row in rows {
                     let (location_id, latitude, longitude, accuracy, timestamp) = row?;
                     ret += &format!(
-                        "<Placemark><Timestamp><when>{}</when></Timestamp><Point><coordinates accuracy=\"{}\">{},{}</coordinates></Point></Placemark>\n\x00",
+                        "<Placemark><Timestamp><when>{}</when></Timestamp><Point><coordinates accuracy=\"{}\">{},{}</coordinates></Point></Placemark>\n",
                         timestamp,
                         accuracy,
                         longitude,
@@ -429,10 +425,10 @@ pub fn get_kml(context: &Context, chat_id: u32) -> Result<(String, u32), Error>
                 Ok(())
             }
         )?;
+        ret += "</Document>\n</kml>";
     }
 
     ensure!(location_count > 0, "No locations processed");
-    ret += "</Document>\n</kml>";
 
     Ok((ret, last_added_location_id))
 }
@@ -476,12 +472,16 @@ pub fn set_kml_sent_timestamp(
     Ok(())
 }
 
-pub fn set_msg_location_id(context: &Context, msg_id: u32, location_id: u32) -> Result<(), Error> {
+pub fn set_msg_location_id(
+    context: &Context,
+    msg_id: MsgId,
+    location_id: u32,
+) -> Result<(), Error> {
     sql::execute(
         context,
         &context.sql,
         "UPDATE msgs SET location_id=? WHERE id=?;",
-        params![location_id, msg_id as i32],
+        params![location_id, msg_id],
     )?;
 
     Ok(())
@@ -492,9 +492,9 @@ pub fn save(
     chat_id: u32,
     contact_id: u32,
     locations: &[Location],
-    independent: i32,
+    independent: bool,
 ) -> Result<u32, Error> {
-    ensure!(chat_id > 9, "Invalid chat id");
+    ensure!(chat_id > DC_CHAT_ID_LAST_SPECIAL, "Invalid chat id");
     context.sql.prepare2(
         "SELECT id FROM locations WHERE timestamp=? AND from_id=?",
         "INSERT INTO locations\
@@ -507,7 +507,7 @@ pub fn save(
             for location in locations {
                 let exists = stmt_test.exists(params![location.timestamp, contact_id as i32])?;
 
-                if 0 != independent || !exists {
+                if independent || !exists {
                     stmt_insert.execute(params![
                         location.timestamp,
                         contact_id as i32,
@@ -540,7 +540,7 @@ pub fn save(
 #[allow(non_snake_case)]
 pub fn job_do_DC_JOB_MAYBE_SEND_LOCATIONS(context: &Context, _job: &Job) {
     let now = time();
-    let mut continue_streaming: libc::c_int = 1;
+    let mut continue_streaming = false;
     info!(
         context,
         " ----------------- MAYBE_SEND_LOCATIONS -------------- ",
@@ -555,7 +555,7 @@ pub fn job_do_DC_JOB_MAYBE_SEND_LOCATIONS(context: &Context, _job: &Job) {
             let chat_id: i32 = row.get(0)?;
             let locations_send_begin: i64 = row.get(1)?;
             let locations_last_sent: i64 = row.get(2)?;
-            continue_streaming = 1;
+            continue_streaming = true;
 
             // be a bit tolerant as the timer may not align exactly with time(NULL)
             if now - locations_last_sent < (60 - 3) {
@@ -585,7 +585,11 @@ pub fn job_do_DC_JOB_MAYBE_SEND_LOCATIONS(context: &Context, _job: &Job) {
                         .into_iter()
                         .filter_map(|(chat_id, locations_send_begin, locations_last_sent)| {
                             if !stmt_locations
-                                .exists(params![1, locations_send_begin, locations_last_sent,])
+                                .exists(params![
+                                    DC_CONTACT_ID_SELF,
+                                    locations_send_begin,
+                                    locations_last_sent,
+                                ])
                                 .unwrap_or_default()
                             {
                                 // if there is no new location, there's nothing to send.
@@ -603,7 +607,7 @@ pub fn job_do_DC_JOB_MAYBE_SEND_LOCATIONS(context: &Context, _job: &Job) {
                                 // and dc_set_location() is typically called periodically, this is ok)
                                 let mut msg = Message::new(Viewtype::Text);
                                 msg.hidden = true;
-                                msg.param.set_int(Param::Cmd, 9);
+                                msg.param.set_cmd(SystemMessage::LocationOnly);
                                 Some((chat_id, msg))
                             }
                         })
@@ -615,11 +619,11 @@ pub fn job_do_DC_JOB_MAYBE_SEND_LOCATIONS(context: &Context, _job: &Job) {
 
         for (chat_id, mut msg) in msgs.into_iter() {
             // TODO: better error handling
-            chat::send_msg(context, chat_id as u32, &mut msg).unwrap();
+            chat::send_msg(context, chat_id as u32, &mut msg).unwrap_or_default();
         }
     }
-    if 0 != continue_streaming {
-        schedule_MAYBE_SEND_LOCATIONS(context, 0x1);
+    if continue_streaming {
+        schedule_MAYBE_SEND_LOCATIONS(context, true);
     }
 }
 
@@ -647,7 +651,7 @@ pub fn job_do_DC_JOB_MAYBE_SEND_LOC_ENDED(context: &Context, job: &mut Job) {
                     params![chat_id as i32],
                 ).is_ok() {
                     let stock_str = context.stock_system_msg(StockMessage::MsgLocationDisabled, "", "", 0);
-                    chat::add_device_msg(context, chat_id, stock_str);
+                    chat::add_info_msg(context, chat_id, stock_str);
                     context.call_cb(Event::ChatModified(chat_id));
                 }
             }
@@ -665,9 +669,9 @@ mod tests {
         let context = dummy_context();
 
         let xml =
-            "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<kml xmlns=\"http://www.opengis.net/kml/2.2\">\n<Document addr=\"user@example.org\">\n<Placemark><Timestamp><when>2019-03-06T21:09:57Z</when></Timestamp><Point><coordinates accuracy=\"32.000000\">9.423110,53.790302</coordinates></Point></Placemark>\n<PlaceMARK>\n<Timestamp><WHEN > \n\t2018-12-13T22:11:12Z\t</WHEN></Timestamp><Point><coordinates aCCuracy=\"2.500000\"> 19.423110 \t , \n 63.790302\n </coordinates></Point></PlaceMARK>\n</Document>\n</kml>";
+            b"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<kml xmlns=\"http://www.opengis.net/kml/2.2\">\n<Document addr=\"user@example.org\">\n<Placemark><Timestamp><when>2019-03-06T21:09:57Z</when></Timestamp><Point><coordinates accuracy=\"32.000000\">9.423110,53.790302</coordinates></Point></Placemark>\n<PlaceMARK>\n<Timestamp><WHEN > \n\t2018-12-13T22:11:12Z\t</WHEN></Timestamp><Point><coordinates aCCuracy=\"2.500000\"> 19.423110 \t , \n 63.790302\n </coordinates></Point></PlaceMARK>\n</Document>\n</kml>";
 
-        let kml = Kml::parse(&context.ctx, &xml).expect("parsing failed");
+        let kml = Kml::parse(&context.ctx, xml).expect("parsing failed");
 
         assert!(kml.addr.is_some());
         assert_eq!(kml.addr.as_ref().unwrap(), "user@example.org",);

src/login_param.rs

@@ -4,6 +4,22 @@ use std::fmt;
 use crate::context::Context;
 use crate::error::Error;
 
+#[derive(Copy, Clone, Debug, Display, FromPrimitive)]
+#[repr(i32)]
+#[strum(serialize_all = "snake_case")]
+pub enum CertificateChecks {
+    Automatic = 0,
+    Strict = 1,
+    AcceptInvalidHostnames = 2,
+    AcceptInvalidCertificates = 3,
+}
+
+impl Default for CertificateChecks {
+    fn default() -> Self {
+        Self::Automatic
+    }
+}
+
 #[derive(Default, Debug)]
 pub struct LoginParam {
     pub addr: String,
@@ -11,10 +27,14 @@ pub struct LoginParam {
     pub mail_user: String,
     pub mail_pw: String,
     pub mail_port: i32,
+    /// IMAP TLS options: whether to allow invalid certificates and/or invalid hostnames
+    pub imap_certificate_checks: CertificateChecks,
     pub send_server: String,
     pub send_user: String,
     pub send_pw: String,
     pub send_port: i32,
+    /// SMTP TLS options: whether to allow invalid certificates and/or invalid hostnames
+    pub smtp_certificate_checks: CertificateChecks,
     pub server_flags: i32,
 }
 
@@ -31,37 +51,53 @@ impl LoginParam {
 
         let key = format!("{}addr", prefix);
         let addr = sql
-            .get_config(context, key)
+            .get_raw_config(context, key)
             .unwrap_or_default()
             .trim()
             .to_string();
 
         let key = format!("{}mail_server", prefix);
-        let mail_server = sql.get_config(context, key).unwrap_or_default();
+        let mail_server = sql.get_raw_config(context, key).unwrap_or_default();
 
         let key = format!("{}mail_port", prefix);
-        let mail_port = sql.get_config_int(context, key).unwrap_or_default();
+        let mail_port = sql.get_raw_config_int(context, key).unwrap_or_default();
 
         let key = format!("{}mail_user", prefix);
-        let mail_user = sql.get_config(context, key).unwrap_or_default();
+        let mail_user = sql.get_raw_config(context, key).unwrap_or_default();
 
         let key = format!("{}mail_pw", prefix);
-        let mail_pw = sql.get_config(context, key).unwrap_or_default();
+        let mail_pw = sql.get_raw_config(context, key).unwrap_or_default();
+
+        let key = format!("{}imap_certificate_checks", prefix);
+        let imap_certificate_checks =
+            if let Some(certificate_checks) = sql.get_raw_config_int(context, key) {
+                num_traits::FromPrimitive::from_i32(certificate_checks).unwrap()
+            } else {
+                Default::default()
+            };
 
         let key = format!("{}send_server", prefix);
-        let send_server = sql.get_config(context, key).unwrap_or_default();
+        let send_server = sql.get_raw_config(context, key).unwrap_or_default();
 
         let key = format!("{}send_port", prefix);
-        let send_port = sql.get_config_int(context, key).unwrap_or_default();
+        let send_port = sql.get_raw_config_int(context, key).unwrap_or_default();
 
         let key = format!("{}send_user", prefix);
-        let send_user = sql.get_config(context, key).unwrap_or_default();
+        let send_user = sql.get_raw_config(context, key).unwrap_or_default();
 
         let key = format!("{}send_pw", prefix);
-        let send_pw = sql.get_config(context, key).unwrap_or_default();
+        let send_pw = sql.get_raw_config(context, key).unwrap_or_default();
+
+        let key = format!("{}smtp_certificate_checks", prefix);
+        let smtp_certificate_checks =
+            if let Some(certificate_checks) = sql.get_raw_config_int(context, key) {
+                num_traits::FromPrimitive::from_i32(certificate_checks).unwrap()
+            } else {
+                Default::default()
+            };
 
         let key = format!("{}server_flags", prefix);
-        let server_flags = sql.get_config_int(context, key).unwrap_or_default();
+        let server_flags = sql.get_raw_config_int(context, key).unwrap_or_default();
 
         LoginParam {
             addr: addr.to_string(),
@@ -69,10 +105,12 @@ impl LoginParam {
             mail_user,
             mail_pw,
             mail_port,
+            imap_certificate_checks,
             send_server,
             send_user,
             send_pw,
             send_port,
+            smtp_certificate_checks,
             server_flags,
         }
     }
@@ -91,34 +129,40 @@ impl LoginParam {
         let sql = &context.sql;
 
         let key = format!("{}addr", prefix);
-        sql.set_config(context, key, Some(&self.addr))?;
+        sql.set_raw_config(context, key, Some(&self.addr))?;
 
         let key = format!("{}mail_server", prefix);
-        sql.set_config(context, key, Some(&self.mail_server))?;
+        sql.set_raw_config(context, key, Some(&self.mail_server))?;
 
         let key = format!("{}mail_port", prefix);
-        sql.set_config_int(context, key, self.mail_port)?;
+        sql.set_raw_config_int(context, key, self.mail_port)?;
 
         let key = format!("{}mail_user", prefix);
-        sql.set_config(context, key, Some(&self.mail_user))?;
+        sql.set_raw_config(context, key, Some(&self.mail_user))?;
 
         let key = format!("{}mail_pw", prefix);
-        sql.set_config(context, key, Some(&self.mail_pw))?;
+        sql.set_raw_config(context, key, Some(&self.mail_pw))?;
+
+        let key = format!("{}imap_certificate_checks", prefix);
+        sql.set_raw_config_int(context, key, self.imap_certificate_checks as i32)?;
 
         let key = format!("{}send_server", prefix);
-        sql.set_config(context, key, Some(&self.send_server))?;
+        sql.set_raw_config(context, key, Some(&self.send_server))?;
 
         let key = format!("{}send_port", prefix);
-        sql.set_config_int(context, key, self.send_port)?;
+        sql.set_raw_config_int(context, key, self.send_port)?;
 
         let key = format!("{}send_user", prefix);
-        sql.set_config(context, key, Some(&self.send_user))?;
+        sql.set_raw_config(context, key, Some(&self.send_user))?;
 
         let key = format!("{}send_pw", prefix);
-        sql.set_config(context, key, Some(&self.send_pw))?;
+        sql.set_raw_config(context, key, Some(&self.send_pw))?;
+
+        let key = format!("{}smtp_certificate_checks", prefix);
+        sql.set_raw_config_int(context, key, self.smtp_certificate_checks as i32)?;
 
         let key = format!("{}server_flags", prefix);
-        sql.set_config_int(context, key, self.server_flags)?;
+        sql.set_raw_config_int(context, key, self.server_flags)?;
 
         Ok(())
     }
@@ -133,16 +177,18 @@ impl fmt::Display for LoginParam {
 
         write!(
             f,
-            "{} {}:{}:{}:{} {}:{}:{}:{} {}",
+            "{} imap:{}:{}:{}:{}:cert_{} smtp:{}:{}:{}:{}:cert_{} {}",
             unset_empty(&self.addr),
             unset_empty(&self.mail_user),
             if !self.mail_pw.is_empty() { pw } else { unset },
             unset_empty(&self.mail_server),
             self.mail_port,
+            self.imap_certificate_checks,
             unset_empty(&self.send_user),
             if !self.send_pw.is_empty() { pw } else { unset },
             unset_empty(&self.send_server),
             self.send_port,
+            self.smtp_certificate_checks,
             flags_readable,
         )
     }
@@ -204,3 +250,41 @@ fn get_readable_flags(flags: i32) -> String {
 
     res
 }
+
+pub fn dc_build_tls(
+    certificate_checks: CertificateChecks,
+) -> Result<native_tls::TlsConnector, native_tls::Error> {
+    let mut tls_builder = native_tls::TlsConnector::builder();
+    match certificate_checks {
+        CertificateChecks::Automatic => {
+            // Same as AcceptInvalidCertificates for now.
+            // TODO: use provider database when it becomes available
+            tls_builder
+                .danger_accept_invalid_hostnames(true)
+                .danger_accept_invalid_certs(true)
+        }
+        CertificateChecks::Strict => &mut tls_builder,
+        CertificateChecks::AcceptInvalidHostnames => {
+            tls_builder.danger_accept_invalid_hostnames(true)
+        }
+        CertificateChecks::AcceptInvalidCertificates => tls_builder
+            .danger_accept_invalid_hostnames(true)
+            .danger_accept_invalid_certs(true),
+    }
+    .build()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_certificate_checks_display() {
+        use std::string::ToString;
+
+        assert_eq!(
+            "accept_invalid_hostnames".to_string(),
+            CertificateChecks::AcceptInvalidHostnames.to_string()
+        );
+    }
+}

src/message.rs

@@ -1,7 +1,7 @@
 use std::path::{Path, PathBuf};
-use std::ptr;
 
 use deltachat_derive::{FromSql, ToSql};
+use failure::Fail;
 
 use crate::chat::{self, Chat};
 use crate::constants::*;
@@ -18,9 +18,134 @@ use crate::pgp::*;
 use crate::sql;
 use crate::stock::StockMessage;
 
-/// In practice, the user additionally cuts the string himself pixel-accurate.
+// In practice, the user additionally cuts the string themselves
+// pixel-accurate.
 const SUMMARY_CHARACTERS: usize = 160;
 
+/// Message ID, including reserved IDs.
+///
+/// Some message IDs are reserved to identify special message types.
+/// This type can represent both the special as well as normal
+/// messages.
+#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)]
+pub struct MsgId(u32);
+
+impl MsgId {
+    /// Create a new [MsgId].
+    pub fn new(id: u32) -> MsgId {
+        MsgId(id)
+    }
+
+    /// Create a new unset [MsgId].
+    pub fn new_unset() -> MsgId {
+        MsgId(0)
+    }
+
+    /// Whether the message ID signifies a special message.
+    ///
+    /// This kind of message ID can not be used for real messages.
+    pub fn is_special(&self) -> bool {
+        match self.0 {
+            0..=DC_MSG_ID_LAST_SPECIAL => true,
+            _ => false,
+        }
+    }
+
+    /// Whether the message ID is unset.
+    ///
+    /// When a message is created it initially has a ID of `0`, which
+    /// is filled in by a real message ID once the message is saved in
+    /// the database.  This returns true while the message has not
+    /// been saved and thus not yet been given an actual message ID.
+    ///
+    /// When this is `true`, [MsgId::is_special] will also always be
+    /// `true`.
+    pub fn is_unset(&self) -> bool {
+        self.0 == 0
+    }
+
+    /// Whether the message ID is the special marker1 marker.
+    ///
+    /// See the docs of the `dc_get_chat_msgs` C API for details.
+    pub fn is_marker1(&self) -> bool {
+        self.0 == DC_MSG_ID_MARKER1
+    }
+
+    /// Whether the message ID is the special day marker.
+    ///
+    /// See the docs of the `dc_get_chat_msgs` C API for details.
+    pub fn is_daymarker(&self) -> bool {
+        self.0 == DC_MSG_ID_DAYMARKER
+    }
+
+    /// Bad evil escape hatch.
+    ///
+    /// Avoid using this, eventually types should be cleaned up enough
+    /// that it is no longer necessary.
+    pub fn to_u32(&self) -> u32 {
+        self.0
+    }
+}
+
+impl std::fmt::Display for MsgId {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        // Would be nice if we could use match here, but no computed values in ranges.
+        if self.0 == DC_MSG_ID_MARKER1 {
+            write!(f, "Msg#Marker1")
+        } else if self.0 == DC_MSG_ID_DAYMARKER {
+            write!(f, "Msg#DayMarker")
+        } else if self.0 <= DC_MSG_ID_LAST_SPECIAL {
+            write!(f, "Msg#UnknownSpecial")
+        } else {
+            write!(f, "Msg#{}", self.0)
+        }
+    }
+}
+
+/// Allow converting [MsgId] to an SQLite type.
+///
+/// This allows you to directly store [MsgId] into the database.
+///
+/// # Errors
+///
+/// This **does** ensure that no special message IDs are written into
+/// the database and the conversion will fail if this is not the case.
+impl rusqlite::types::ToSql for MsgId {
+    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
+        if self.0 <= DC_MSG_ID_LAST_SPECIAL {
+            return Err(rusqlite::Error::ToSqlConversionFailure(Box::new(
+                InvalidMsgId.compat(),
+            )));
+        }
+        let val = rusqlite::types::Value::Integer(self.0 as i64);
+        let out = rusqlite::types::ToSqlOutput::Owned(val);
+        Ok(out)
+    }
+}
+
+/// Allow converting an SQLite integer directly into [MsgId].
+impl rusqlite::types::FromSql for MsgId {
+    fn column_result(value: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
+        // Would be nice if we could use match here, but alas.
+        i64::column_result(value).and_then(|val| {
+            if 0 <= val && val <= std::u32::MAX as i64 {
+                Ok(MsgId::new(val as u32))
+            } else {
+                Err(rusqlite::types::FromSqlError::OutOfRange(val))
+            }
+        })
+    }
+}
+
+/// Message ID was invalid.
+///
+/// This usually occurs when trying to use a message ID of
+/// [DC_MSG_ID_LAST_SPECIAL] or below in a situation where this is not
+/// possible.
+#[derive(Debug, Fail)]
+#[fail(display = "Invalid Message ID.")]
+pub struct InvalidMsgId;
+
 /// An object representing a single message in memory.
 /// The message object is not updated.
 /// If you want an update, you have to recreate the object.
@@ -30,7 +155,7 @@ const SUMMARY_CHARACTERS: usize = 160;
 /// approx. max. length returned by dc_get_msg_info()
 #[derive(Debug, Clone, Default)]
 pub struct Message {
-    pub(crate) id: u32,
+    pub(crate) id: MsgId,
     pub(crate) from_id: u32,
     pub(crate) to_id: u32,
     pub(crate) chat_id: u32,
@@ -62,69 +187,105 @@ impl Message {
         msg
     }
 
-    pub fn load_from_db(context: &Context, id: u32) -> Result<Message, Error> {
+    pub fn load_from_db(context: &Context, id: MsgId) -> Result<Message, Error> {
+        ensure!(
+            !id.is_special(),
+            "Can not load special message IDs from DB."
+        );
         context.sql.query_row(
-        "SELECT  \
-         m.id,rfc724_mid,m.mime_in_reply_to,m.server_folder,m.server_uid,m.move_state,m.chat_id,  \
-         m.from_id,m.to_id,m.timestamp,m.timestamp_sent,m.timestamp_rcvd, m.type,m.state,m.msgrmsg,m.txt,  \
-         m.param,m.starred,m.hidden,m.location_id, c.blocked  \
-         FROM msgs m \
-         LEFT JOIN chats c ON c.id=m.chat_id WHERE m.id=?;",
-        params![id as i32],
-        |row| {
-            let mut msg = Message::default();
-            msg.id = row.get::<_, i32>(0)? as u32;
-            msg.rfc724_mid = row.get::<_, String>(1)?;
-            msg.in_reply_to = row.get::<_, Option<String>>(2)?;
-            msg.server_folder = row.get::<_, Option<String>>(3)?;
-            msg.server_uid = row.get(4)?;
-            msg.move_state = row.get(5)?;
-            msg.chat_id = row.get(6)?;
-            msg.from_id = row.get(7)?;
-            msg.to_id = row.get(8)?;
-            msg.timestamp_sort = row.get(9)?;
-            msg.timestamp_sent = row.get(10)?;
-            msg.timestamp_rcvd = row.get(11)?;
-            msg.type_0 = row.get(12)?;
-            msg.state = row.get(13)?;
-            msg.is_dc_message = row.get(14)?;
+            concat!(
+                "SELECT",
+                "    m.id AS id,",
+                "    rfc724_mid AS rfc724mid,",
+                "    m.mime_in_reply_to AS mime_in_reply_to,",
+                "    m.server_folder AS server_folder,",
+                "    m.server_uid AS server_uid,",
+                "    m.move_state as move_state,",
+                "    m.chat_id AS chat_id,",
+                "    m.from_id AS from_id,",
+                "    m.to_id AS to_id,",
+                "    m.timestamp AS timestamp,",
+                "    m.timestamp_sent AS timestamp_sent,",
+                "    m.timestamp_rcvd AS timestamp_rcvd,",
+                "    m.type AS type,",
+                "    m.state AS state,",
+                "    m.msgrmsg AS msgrmsg,",
+                "    m.txt AS txt,",
+                "    m.param AS param,",
+                "    m.starred AS starred,",
+                "    m.hidden AS hidden,",
+                "    m.location_id AS location,",
+                "    c.blocked AS blocked",
+                " FROM msgs m LEFT JOIN chats c ON c.id=m.chat_id",
+                " WHERE m.id=?;"
+            ),
+            params![id],
+            |row| {
+                let mut msg = Message::default();
+                // msg.id = row.get::<_, AnyMsgId>("id")?;
+                msg.id = row.get("id")?;
+                msg.rfc724_mid = row.get::<_, String>("rfc724mid")?;
+                msg.in_reply_to = row.get::<_, Option<String>>("mime_in_reply_to")?;
+                msg.server_folder = row.get::<_, Option<String>>("server_folder")?;
+                msg.server_uid = row.get("server_uid")?;
+                msg.move_state = row.get("move_state")?;
+                msg.chat_id = row.get("chat_id")?;
+                msg.from_id = row.get("from_id")?;
+                msg.to_id = row.get("to_id")?;
+                msg.timestamp_sort = row.get("timestamp")?;
+                msg.timestamp_sent = row.get("timestamp_sent")?;
+                msg.timestamp_rcvd = row.get("timestamp_rcvd")?;
+                msg.type_0 = row.get("type")?;
+                msg.state = row.get("state")?;
+                msg.is_dc_message = row.get("msgrmsg")?;
 
-            let text;
-            if let rusqlite::types::ValueRef::Text(buf) = row.get_raw(15) {
-                if let Ok(t) = String::from_utf8(buf.to_vec()) {
-                    text = t;
+                let text;
+                if let rusqlite::types::ValueRef::Text(buf) = row.get_raw("txt") {
+                    if let Ok(t) = String::from_utf8(buf.to_vec()) {
+                        text = t;
+                    } else {
+                        warn!(
+                            context,
+                            concat!(
+                                "dc_msg_load_from_db: could not get ",
+                                "text column as non-lossy utf8 id {}"
+                            ),
+                            id
+                        );
+                        text = String::from_utf8_lossy(buf).into_owned();
+                    }
                 } else {
-                    warn!(context, "dc_msg_load_from_db: could not get text column as non-lossy utf8 id {}", id);
-                    text = String::from_utf8_lossy(buf).into_owned();
+                    text = "".to_string();
                 }
-            } else {
-                text = "".to_string();
-            }
-            msg.text = Some(text);
+                msg.text = Some(text);
 
-            msg.param = row.get::<_, String>(16)?.parse().unwrap_or_default();
-            msg.starred = row.get(17)?;
-            msg.hidden = row.get(18)?;
-            msg.location_id = row.get(19)?;
-            msg.chat_blocked = row.get::<_, Option<Blocked>>(20)?.unwrap_or_default();
-            Ok(msg)
-        })
+                msg.param = row.get::<_, String>("param")?.parse().unwrap_or_default();
+                msg.starred = row.get("starred")?;
+                msg.hidden = row.get("hidden")?;
+                msg.location_id = row.get("location")?;
+                msg.chat_blocked = row
+                    .get::<_, Option<Blocked>>("blocked")?
+                    .unwrap_or_default();
+
+                Ok(msg)
+            },
+        )
     }
 
-    pub fn delete_from_db(context: &Context, msg_id: u32) {
+    pub fn delete_from_db(context: &Context, msg_id: MsgId) {
         if let Ok(msg) = Message::load_from_db(context, msg_id) {
             sql::execute(
                 context,
                 &context.sql,
                 "DELETE FROM msgs WHERE id=?;",
-                params![msg.id as i32],
+                params![msg.id],
             )
             .ok();
             sql::execute(
                 context,
                 &context.sql,
                 "DELETE FROM msgs_mdns WHERE msg_id=?;",
-                params![msg.id as i32],
+                params![msg.id],
             )
             .ok();
         }
@@ -145,9 +306,7 @@ impl Message {
     }
 
     pub fn get_file(&self, context: &Context) -> Option<PathBuf> {
-        self.param
-            .get(Param::File)
-            .map(|f| dc_get_abs_path(context, f))
+        self.param.get_path(Param::File, context).unwrap_or(None)
     }
 
     /// Check if a message has a location bound to it.
@@ -190,7 +349,7 @@ impl Message {
         }
     }
 
-    pub fn get_id(&self) -> u32 {
+    pub fn get_id(&self) -> MsgId {
         self.id
     }
 
@@ -237,8 +396,9 @@ impl Message {
 
     pub fn get_filebytes(&self, context: &Context) -> u64 {
         self.param
-            .get(Param::File)
-            .map(|file| dc_get_filebytes(context, &file))
+            .get_path(Param::File, context)
+            .unwrap_or(None)
+            .map(|path| dc_get_filebytes(context, &path))
             .unwrap_or_default()
     }
 
@@ -255,7 +415,7 @@ impl Message {
     }
 
     pub fn get_showpadlock(&self) -> bool {
-        self.param.get_int(Param::GuranteeE2ee).unwrap_or_default() != 0
+        self.param.get_int(Param::GuaranteeE2ee).unwrap_or_default() != 0
     }
 
     pub fn get_summary(&mut self, context: &Context, chat: Option<&Chat>) -> Lot {
@@ -316,11 +476,19 @@ impl Message {
 
     pub fn is_info(&self) -> bool {
         let cmd = self.param.get_cmd();
-        self.from_id == DC_CONTACT_ID_DEVICE as libc::c_uint
-            || self.to_id == DC_CONTACT_ID_DEVICE as libc::c_uint
+        self.from_id == DC_CONTACT_ID_INFO as libc::c_uint
+            || self.to_id == DC_CONTACT_ID_INFO as libc::c_uint
             || cmd != SystemMessage::Unknown && cmd != SystemMessage::AutocryptSetupMessage
     }
 
+    /// Whether the message is still being created.
+    ///
+    /// Messages with attachments might be created before the
+    /// attachment is ready.  In this case some more restrictions on
+    /// the attachment apply, e.g. if the file to be attached is still
+    /// being written to or otherwise will still change it can not be
+    /// copied to the blobdir.  Thus those attachments need to be
+    /// created immediately in the blobdir with a valid filename.
     pub fn is_increation(&self) -> bool {
         chat::msgtype_has_file(self.type_0) && self.state == MessageState::OutPreparing
     }
@@ -339,23 +507,10 @@ impl Message {
         }
 
         if let Some(filename) = self.get_file(context) {
-            if let Ok(mut buf) = dc_read_file(context, filename) {
-                unsafe {
-                    // just a pointer inside buf, MUST NOT be free()'d
-                    let mut buf_headerline = String::default();
-                    // just a pointer inside buf, MUST NOT be free()'d
-                    let mut buf_setupcodebegin = ptr::null();
-
-                    if dc_split_armored_data(
-                        buf.as_mut_ptr().cast(),
-                        &mut buf_headerline,
-                        &mut buf_setupcodebegin,
-                        ptr::null_mut(),
-                        ptr::null_mut(),
-                    ) && buf_headerline == "-----BEGIN PGP MESSAGE-----"
-                        && !buf_setupcodebegin.is_null()
-                    {
-                        return Some(to_string(buf_setupcodebegin));
+            if let Ok(ref buf) = dc_read_file(context, filename) {
+                if let Ok((typ, headers, _)) = split_armored_data(buf) {
+                    if typ == pgp::armor::BlockType::Message {
+                        return headers.get(crate::pgp::HEADER_SETUPCODE).cloned();
                     }
                 }
             }
@@ -406,7 +561,7 @@ impl Message {
             context,
             &context.sql,
             "UPDATE msgs SET param=? WHERE id=?;",
-            params![self.param.to_string(), self.id as i32],
+            params![self.param.to_string(), self.id],
         )
         .is_ok()
     }
@@ -494,12 +649,10 @@ impl Lot {
                     } else {
                         self.text1 = None;
                     }
+                } else if let Some(contact) = contact {
+                    self.text1 = Some(contact.get_first_name().into());
                 } else {
-                    if let Some(contact) = contact {
-                        self.text1 = Some(contact.get_first_name().into());
-                    } else {
-                        self.text1 = None;
-                    }
+                    self.text1 = None;
                 }
                 self.text1_meaning = Meaning::Text1Username;
             }
@@ -518,7 +671,7 @@ impl Lot {
     }
 }
 
-pub fn get_msg_info(context: &Context, msg_id: u32) -> String {
+pub fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
     let mut ret = String::new();
 
     let msg = Message::load_from_db(context, msg_id);
@@ -526,19 +679,19 @@ pub fn get_msg_info(context: &Context, msg_id: u32) -> String {
         return ret;
     }
 
-    let msg = msg.unwrap();
+    let msg = msg.unwrap_or_default();
 
     let rawtxt: Option<String> = context.sql.query_get_value(
         context,
         "SELECT txt_raw FROM msgs WHERE id=?;",
-        params![msg_id as i32],
+        params![msg_id],
     );
 
     if rawtxt.is_none() {
-        ret += &format!("Cannot load message #{}.", msg_id as usize);
+        ret += &format!("Cannot load message {}.", msg_id);
         return ret;
     }
-    let rawtxt = rawtxt.unwrap();
+    let rawtxt = rawtxt.unwrap_or_default();
     let rawtxt = dc_truncate(rawtxt.trim(), 100000, false);
 
     let fts = dc_timestamp_to_str(msg.get_timestamp());
@@ -561,14 +714,14 @@ pub fn get_msg_info(context: &Context, msg_id: u32) -> String {
         ret += "\n";
     }
 
-    if msg.from_id == 2 || msg.to_id == 2 {
+    if msg.from_id == DC_CONTACT_ID_INFO || msg.to_id == DC_CONTACT_ID_INFO {
         // device-internal message, no further details needed
         return ret;
     }
 
     if let Ok(rows) = context.sql.query_map(
         "SELECT contact_id, timestamp_sent FROM msgs_mdns WHERE msg_id=?;",
-        params![msg_id as i32],
+        params![msg_id],
         |row| {
             let contact_id: i32 = row.get(0)?;
             let ts: i64 = row.get(1)?;
@@ -613,14 +766,13 @@ pub fn get_msg_info(context: &Context, msg_id: u32) -> String {
         if 0 != e2ee_errors & 0x2 {
             ret += ", Encrypted, no valid signature";
         }
-    } else if 0 != msg.param.get_int(Param::GuranteeE2ee).unwrap_or_default() {
+    } else if 0 != msg.param.get_int(Param::GuaranteeE2ee).unwrap_or_default() {
         ret += ", Encrypted";
     }
 
     ret += "\n";
-    match msg.param.get(Param::Error) {
-        Some(err) => ret += &format!("Error: {}", err),
-        _ => {}
+    if let Some(err) = msg.param.get(Param::Error) {
+        ret += &format!("Error: {}", err)
     }
 
     if let Some(path) = msg.get_file(context) {
@@ -679,21 +831,21 @@ pub fn guess_msgtype_from_suffix(path: &Path) -> Option<(Viewtype, &str)> {
     Some(info)
 }
 
-pub fn get_mime_headers(context: &Context, msg_id: u32) -> Option<String> {
+pub fn get_mime_headers(context: &Context, msg_id: MsgId) -> Option<String> {
     context.sql.query_get_value(
         context,
         "SELECT mime_headers FROM msgs WHERE id=?;",
-        params![msg_id as i32],
+        params![msg_id],
     )
 }
 
-pub fn delete_msgs(context: &Context, msg_ids: &[u32]) {
-    for msg_id in msg_ids.into_iter() {
+pub fn delete_msgs(context: &Context, msg_ids: &[MsgId]) {
+    for msg_id in msg_ids.iter() {
         update_msg_chat_id(context, *msg_id, DC_CHAT_ID_TRASH);
         job_add(
             context,
             Action::DeleteMsgOnImap,
-            *msg_id as libc::c_int,
+            msg_id.to_u32() as i32,
             Params::new(),
             0,
         );
@@ -702,35 +854,45 @@ pub fn delete_msgs(context: &Context, msg_ids: &[u32]) {
     if !msg_ids.is_empty() {
         context.call_cb(Event::MsgsChanged {
             chat_id: 0,
-            msg_id: 0,
+            msg_id: MsgId::new(0),
         });
         job_kill_action(context, Action::Housekeeping);
         job_add(context, Action::Housekeeping, 0, Params::new(), 10);
     };
 }
 
-fn update_msg_chat_id(context: &Context, msg_id: u32, chat_id: u32) -> bool {
+fn update_msg_chat_id(context: &Context, msg_id: MsgId, chat_id: u32) -> bool {
     sql::execute(
         context,
         &context.sql,
         "UPDATE msgs SET chat_id=? WHERE id=?;",
-        params![chat_id as i32, msg_id as i32],
+        params![chat_id as i32, msg_id],
     )
     .is_ok()
 }
 
-pub fn markseen_msgs(context: &Context, msg_ids: &[u32]) -> bool {
+pub fn markseen_msgs(context: &Context, msg_ids: &[MsgId]) -> bool {
     if msg_ids.is_empty() {
         return false;
     }
 
     let msgs = context.sql.prepare(
-        "SELECT m.state, c.blocked  FROM msgs m  LEFT JOIN chats c ON c.id=m.chat_id  WHERE m.id=? AND m.chat_id>9",
+        concat!(
+            "SELECT",
+            "    m.state AS state,",
+            "    c.blocked AS blocked",
+            " FROM msgs m LEFT JOIN chats c ON c.id=m.chat_id",
+            " WHERE m.id=? AND m.chat_id>9"
+        ),
         |mut stmt, _| {
             let mut res = Vec::with_capacity(msg_ids.len());
-            for id in msg_ids.into_iter() {
-                let query_res = stmt.query_row(params![*id as i32], |row| {
-                    Ok((row.get::<_, MessageState>(0)?, row.get::<_, Option<Blocked>>(1)?.unwrap_or_default()))
+            for id in msg_ids.iter() {
+                let query_res = stmt.query_row(params![*id], |row| {
+                    Ok((
+                        row.get::<_, MessageState>("state")?,
+                        row.get::<_, Option<Blocked>>("blocked")?
+                            .unwrap_or_default(),
+                    ))
                 });
                 if let Err(rusqlite::Error::QueryReturnedNoRows) = query_res {
                     continue;
@@ -740,7 +902,7 @@ pub fn markseen_msgs(context: &Context, msg_ids: &[u32]) -> bool {
             }
 
             Ok(res)
-        }
+        },
     );
 
     if msgs.is_err() {
@@ -748,18 +910,18 @@ pub fn markseen_msgs(context: &Context, msg_ids: &[u32]) -> bool {
         return false;
     }
     let mut send_event = false;
-    let msgs = msgs.unwrap();
+    let msgs = msgs.unwrap_or_default();
 
     for (id, curr_state, curr_blocked) in msgs.into_iter() {
         if curr_blocked == Blocked::Not {
             if curr_state == MessageState::InFresh || curr_state == MessageState::InNoticed {
                 update_msg_state(context, *id, MessageState::InSeen);
-                info!(context, "Seen message #{}.", id);
+                info!(context, "Seen message {}.", id);
 
                 job_add(
                     context,
                     Action::MarkseenMsgOnImap,
-                    *id as i32,
+                    id.to_u32() as i32,
                     Params::new(),
                     0,
                 );
@@ -774,32 +936,32 @@ pub fn markseen_msgs(context: &Context, msg_ids: &[u32]) -> bool {
     if send_event {
         context.call_cb(Event::MsgsChanged {
             chat_id: 0,
-            msg_id: 0,
+            msg_id: MsgId::new(0),
         });
     }
 
     true
 }
 
-pub fn update_msg_state(context: &Context, msg_id: u32, state: MessageState) -> bool {
+pub fn update_msg_state(context: &Context, msg_id: MsgId, state: MessageState) -> bool {
     sql::execute(
         context,
         &context.sql,
         "UPDATE msgs SET state=? WHERE id=?;",
-        params![state, msg_id as i32],
+        params![state, msg_id],
     )
     .is_ok()
 }
 
-pub fn star_msgs(context: &Context, msg_ids: &[u32], star: bool) -> bool {
+pub fn star_msgs(context: &Context, msg_ids: &[MsgId], star: bool) -> bool {
     if msg_ids.is_empty() {
         return false;
     }
     context
         .sql
         .prepare("UPDATE msgs SET starred=? WHERE id=?;", |mut stmt, _| {
-            for msg_id in msg_ids.into_iter() {
-                stmt.execute(params![star as i32, *msg_id as i32])?;
+            for msg_id in msg_ids.iter() {
+                stmt.execute(params![star as i32, *msg_id])?;
             }
             Ok(())
         })
@@ -818,6 +980,7 @@ pub fn get_summarytext_by_raw(
     let prefix = match viewtype {
         Viewtype::Image => context.stock_str(StockMessage::Image).into_owned(),
         Viewtype::Gif => context.stock_str(StockMessage::Gif).into_owned(),
+        Viewtype::Sticker => context.stock_str(StockMessage::Sticker).into_owned(),
         Viewtype::Video => context.stock_str(StockMessage::Video).into_owned(),
         Viewtype::Voice => context.stock_str(StockMessage::VoiceMessage).into_owned(),
         Viewtype::Audio | Viewtype::File => {
@@ -827,17 +990,14 @@ pub fn get_summarytext_by_raw(
                     .stock_str(StockMessage::AcSetupMsgSubject)
                     .to_string()
             } else {
-                let file_name: String = if let Some(file_path) = param.get(Param::File) {
-                    if let Some(file_name) = Path::new(file_path).file_name() {
-                        Some(file_name.to_string_lossy().into_owned())
-                    } else {
-                        None
-                    }
-                } else {
-                    None
-                }
-                .unwrap_or("ErrFileName".to_string());
-
+                let file_name: String = param
+                    .get_path(Param::File, context)
+                    .unwrap_or(None)
+                    .and_then(|path| {
+                        path.file_name()
+                            .map(|fname| fname.to_string_lossy().into_owned())
+                    })
+                    .unwrap_or_else(|| String::from("ErrFileName"));
                 let label = context.stock_str(if viewtype == Viewtype::Audio {
                     StockMessage::Audio
                 } else {
@@ -861,7 +1021,9 @@ pub fn get_summarytext_by_raw(
     }
 
     if let Some(text) = text {
-        if prefix.is_empty() {
+        if text.as_ref().is_empty() {
+            prefix
+        } else if prefix.is_empty() {
             dc_truncate(text.as_ref(), approx_characters, true).to_string()
         } else {
             let tmp = format!("{} – {}", prefix, text.as_ref());
@@ -879,8 +1041,8 @@ pub fn get_summarytext_by_raw(
 
 // Context functions to work with messages
 
-pub fn exists(context: &Context, msg_id: u32) -> bool {
-    if msg_id <= DC_CHAT_ID_LAST_SPECIAL {
+pub fn exists(context: &Context, msg_id: MsgId) -> bool {
+    if msg_id.is_special() {
         return false;
     }
 
@@ -909,7 +1071,7 @@ pub fn update_msg_move_state(context: &Context, rfc724_mid: &str, state: MoveSta
     .is_ok()
 }
 
-pub fn set_msg_failed(context: &Context, msg_id: u32, error: Option<impl AsRef<str>>) {
+pub fn set_msg_failed(context: &Context, msg_id: MsgId, error: Option<impl AsRef<str>>) {
     if let Ok(mut msg) = Message::load_from_db(context, msg_id) {
         if msg.state.can_fail() {
             msg.state = MessageState::OutFailed;
@@ -923,7 +1085,7 @@ pub fn set_msg_failed(context: &Context, msg_id: u32, error: Option<impl AsRef<s
             context,
             &context.sql,
             "UPDATE msgs SET state=?, param=? WHERE id=?;",
-            params![msg.state, msg.param.to_string(), msg_id as i32],
+            params![msg.state, msg.param.to_string(), msg_id],
         )
         .is_ok()
         {
@@ -935,38 +1097,39 @@ pub fn set_msg_failed(context: &Context, msg_id: u32, error: Option<impl AsRef<s
     }
 }
 
-/// returns true if an event should be send
+/// returns Some if an event should be send
 pub fn mdn_from_ext(
     context: &Context,
     from_id: u32,
     rfc724_mid: &str,
     timestamp_sent: i64,
-    ret_chat_id: &mut u32,
-    ret_msg_id: &mut u32,
-) -> bool {
-    if from_id <= 9 || rfc724_mid.is_empty() || *ret_chat_id != 0 || *ret_msg_id != 0 {
-        return false;
+) -> Option<(u32, MsgId)> {
+    if from_id <= 9 || rfc724_mid.is_empty() {
+        return None;
     }
 
-    let mut read_by_all = false;
-
     if let Ok((msg_id, chat_id, chat_type, msg_state)) = context.sql.query_row(
-        "SELECT m.id, c.id, c.type, m.state FROM msgs m  \
-         LEFT JOIN chats c ON m.chat_id=c.id  \
-         WHERE rfc724_mid=? AND from_id=1  \
-         ORDER BY m.id;",
+        concat!(
+            "SELECT",
+            "    m.id AS msg_id,",
+            "    c.id AS chat_id,",
+            "    c.type AS type,",
+            "    m.state AS state",
+            " FROM msgs m LEFT JOIN chats c ON m.chat_id=c.id",
+            " WHERE rfc724_mid=? AND from_id=1",
+            " ORDER BY m.id;"
+        ),
         params![rfc724_mid],
         |row| {
             Ok((
-                row.get::<_, i32>(0)?,
-                row.get::<_, i32>(1)?,
-                row.get::<_, Chattype>(2)?,
-                row.get::<_, MessageState>(3)?,
+                row.get::<_, MsgId>("msg_id")?,
+                row.get::<_, u32>("chat_id")?,
+                row.get::<_, Chattype>("type")?,
+                row.get::<_, MessageState>("state")?,
             ))
         },
     ) {
-        *ret_msg_id = msg_id as u32;
-        *ret_chat_id = chat_id as u32;
+        let mut read_by_all = false;
 
         // if already marked as MDNS_RCVD msgstate_can_fail() returns false.
         // however, it is important, that ret_msg_id is set above as this
@@ -976,20 +1139,20 @@ pub fn mdn_from_ext(
                 .sql
                 .exists(
                     "SELECT contact_id FROM msgs_mdns WHERE msg_id=? AND contact_id=?;",
-                    params![*ret_msg_id as i32, from_id as i32,],
+                    params![msg_id, from_id as i32,],
                 )
                 .unwrap_or_default();
 
             if !mdn_already_in_table {
                 context.sql.execute(
                     "INSERT INTO msgs_mdns (msg_id, contact_id, timestamp_sent) VALUES (?, ?, ?);",
-                    params![*ret_msg_id as i32, from_id as i32, timestamp_sent],
-                ).unwrap(); // TODO: better error handling
+                    params![msg_id, from_id as i32, timestamp_sent],
+                ).unwrap_or_default(); // TODO: better error handling
             }
 
             // Normal chat? that's quite easy.
             if chat_type == Chattype::Single {
-                update_msg_state(context, *ret_msg_id, MessageState::OutMdnRcvd);
+                update_msg_state(context, msg_id, MessageState::OutMdnRcvd);
                 read_by_all = true;
             } else {
                 // send event about new state
@@ -998,7 +1161,7 @@ pub fn mdn_from_ext(
                     .query_get_value::<_, isize>(
                         context,
                         "SELECT COUNT(*) FROM msgs_mdns WHERE msg_id=?;",
-                        params![*ret_msg_id as i32],
+                        params![msg_id],
                     )
                     .unwrap_or_default() as usize;
                 /*
@@ -1014,16 +1177,19 @@ pub fn mdn_from_ext(
                 (S=Sender, R=Recipient)
                  */
                 // for rounding, SELF is already included!
-                let soll_cnt = (chat::get_chat_contact_cnt(context, *ret_chat_id) + 1) / 2;
+                let soll_cnt = (chat::get_chat_contact_cnt(context, chat_id) + 1) / 2;
                 if ist_cnt >= soll_cnt {
-                    update_msg_state(context, *ret_msg_id, MessageState::OutMdnRcvd);
+                    update_msg_state(context, msg_id, MessageState::OutMdnRcvd);
                     read_by_all = true;
                 } // else wait for more receipts
             }
         }
+        return match read_by_all {
+            true => Some((chat_id, msg_id)),
+            false => None,
+        };
     }
-
-    read_by_all
+    None
 }
 
 /// The number of messages assigned to real chat (!=deaddrop, !=trash)
@@ -1077,7 +1243,7 @@ pub fn rfc724_mid_cnt(context: &Context, rfc724_mid: &str) -> libc::c_int {
 pub(crate) fn rfc724_mid_exists(
     context: &Context,
     rfc724_mid: &str,
-) -> Result<(String, u32, u32), Error> {
+) -> Result<(String, u32, MsgId), Error> {
     ensure!(!rfc724_mid.is_empty(), "empty rfc724_mid");
 
     context.sql.query_row(
@@ -1086,7 +1252,7 @@ pub(crate) fn rfc724_mid_exists(
         |row| {
             let server_folder = row.get::<_, Option<String>>(0)?.unwrap_or_default();
             let server_uid = row.get(1)?;
-            let msg_id = row.get(2)?;
+            let msg_id: MsgId = row.get(2)?;
 
             Ok((server_folder, server_uid, msg_id))
         },
@@ -1110,6 +1276,12 @@ pub fn update_server_uid(
     }
 }
 
+#[allow(dead_code)]
+pub fn dc_empty_server(context: &Context, flags: u32) {
+    job_kill_action(context, Action::EmptyServer);
+    job_add(context, Action::EmptyServer, flags as i32, Params::new(), 0);
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -1145,4 +1317,121 @@ mod tests {
         let _msg2 = Message::load_from_db(ctx, msg_id).unwrap();
         assert_eq!(_msg2.get_filemime(), None);
     }
+
+    #[test]
+    pub fn test_get_summarytext_by_raw() {
+        let d = test::dummy_context();
+        let ctx = &d.ctx;
+
+        let some_text = Some("bla bla".to_string());
+        let empty_text = Some("".to_string());
+        let no_text: Option<String> = None;
+
+        let mut some_file = Params::new();
+        some_file.set(Param::File, "foo.bar");
+
+        assert_eq!(
+            get_summarytext_by_raw(
+                Viewtype::Text,
+                some_text.as_ref(),
+                &mut Params::new(),
+                50,
+                &ctx
+            ),
+            "bla bla" // for simple text, the type is not added to the summary
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(Viewtype::Image, no_text.as_ref(), &mut some_file, 50, &ctx,),
+            "Image" // file names are not added for images
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(Viewtype::Video, no_text.as_ref(), &mut some_file, 50, &ctx,),
+            "Video" // file names are not added for videos
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(Viewtype::Gif, no_text.as_ref(), &mut some_file, 50, &ctx,),
+            "GIF" // file names are not added for GIFs
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(
+                Viewtype::Sticker,
+                no_text.as_ref(),
+                &mut some_file,
+                50,
+                &ctx,
+            ),
+            "Sticker" // file names are not added for stickers
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(
+                Viewtype::Voice,
+                empty_text.as_ref(),
+                &mut some_file,
+                50,
+                &ctx,
+            ),
+            "Voice message" // file names are not added for voice messages, empty text is skipped
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(Viewtype::Voice, no_text.as_ref(), &mut some_file, 50, &ctx),
+            "Voice message" // file names are not added for voice messages
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(
+                Viewtype::Voice,
+                some_text.as_ref(),
+                &mut some_file,
+                50,
+                &ctx
+            ),
+            "Voice message \u{2013} bla bla" // `\u{2013}` explicitly checks for "EN DASH"
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(Viewtype::Audio, no_text.as_ref(), &mut some_file, 50, &ctx),
+            "Audio \u{2013} foo.bar" // file name is added for audio
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(
+                Viewtype::Audio,
+                empty_text.as_ref(),
+                &mut some_file,
+                50,
+                &ctx,
+            ),
+            "Audio \u{2013} foo.bar" // file name is added for audio, empty text is not added
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(
+                Viewtype::Audio,
+                some_text.as_ref(),
+                &mut some_file,
+                50,
+                &ctx
+            ),
+            "Audio \u{2013} foo.bar \u{2013} bla bla" // file name and text added for audio
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(Viewtype::File, some_text.as_ref(), &mut some_file, 50, &ctx),
+            "File \u{2013} foo.bar \u{2013} bla bla" // file name is added for files
+        );
+
+        let mut asm_file = Params::new();
+        asm_file.set(Param::File, "foo.bar");
+        asm_file.set_cmd(SystemMessage::AutocryptSetupMessage);
+        assert_eq!(
+            get_summarytext_by_raw(Viewtype::File, no_text.as_ref(), &mut asm_file, 50, &ctx),
+            "Autocrypt Setup Message" // file name is not added for autocrypt setup messages
+        );
+    }
 }

src/mimefactory.rs

@@ -1,4 +1,3 @@
-use std::path::Path;
 use std::ptr;
 
 use chrono::TimeZone;
@@ -13,15 +12,17 @@ use mmime::mmapstring::*;
 use mmime::other::*;
 
 use crate::chat::{self, Chat};
+use crate::config::Config;
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::{get_version_str, Context};
-use crate::dc_mimeparser::{mailmime_find_mailimf_fields, SystemMessage};
+use crate::dc_mimeparser::SystemMessage;
 use crate::dc_strencode::*;
 use crate::dc_tools::*;
 use crate::e2ee::*;
 use crate::error::Error;
 use crate::location;
+use crate::message::MsgId;
 use crate::message::{self, Message};
 use crate::param::*;
 use crate::stock::StockMessage;
@@ -59,11 +60,13 @@ pub struct MimeFactory<'a> {
 
 impl<'a> MimeFactory<'a> {
     fn new(context: &'a Context, msg: Message) -> Self {
-        let cget = |context: &Context, name: &str| context.sql.get_config(context, name);
         MimeFactory {
-            from_addr: cget(&context, "configured_addr").unwrap_or_default(),
-            from_displayname: cget(&context, "displayname").unwrap_or_default(),
-            selfstatus: cget(&context, "selfstatus")
+            from_addr: context
+                .get_config(Config::ConfiguredAddr)
+                .unwrap_or_default(),
+            from_displayname: context.get_config(Config::Displayname).unwrap_or_default(),
+            selfstatus: context
+                .get_config(Config::Selfstatus)
                 .unwrap_or_else(|| context.stock_str(StockMessage::StatusLine).to_string()),
             recipients_names: Vec::with_capacity(5),
             recipients_addr: Vec::with_capacity(5),
@@ -105,16 +108,11 @@ impl<'a> MimeFactory<'a> {
         Ok(())
     }
 
-    pub fn load_mdn(context: &'a Context, msg_id: u32) -> Result<MimeFactory, Error> {
-        if 0 == context
-            .sql
-            .get_config_int(context, "mdns_enabled")
-            .unwrap_or_else(|| 1)
-        {
-            // MDNs not enabled - check this is late, in the job. the use may have changed its
-            // choice while offline ...
-
-            bail!("MDNs disabled ")
+    pub fn load_mdn(context: &'a Context, msg_id: MsgId) -> Result<MimeFactory, Error> {
+        if !context.get_config_bool(Config::MdnsEnabled) {
+            // MDNs not enabled - check this is late, in the job. the
+            // user may have changed its choice while offline ...
+            bail!("MDNs meanwhile disabled")
         }
 
         let msg = Message::load_from_db(context, msg_id)?;
@@ -141,52 +139,35 @@ impl<'a> MimeFactory<'a> {
     }
 
     /*******************************************************************************
-     * Render
+     * Render a basic email
      ******************************************************************************/
-    // restrict unsafe to parts, introduce wrapmime helpers where appropriate
+    // XXX restrict unsafe to parts, introduce wrapmime helpers where appropriate
     pub unsafe fn render(&mut self) -> Result<(), Error> {
         if self.loaded == Loaded::Nothing || !self.out.is_null() {
             bail!("Invalid use of mimefactory-object.");
         }
         let context = &self.context;
+        let from = wrapmime::new_mailbox_list(&self.from_displayname, &self.from_addr);
 
-        /* create basic mail
-         *************************************************************************/
-
-        let from: *mut mailimf_mailbox_list = mailimf_mailbox_list_new_empty();
-        mailimf_mailbox_list_add(
-            from,
-            mailimf_mailbox_new(
-                if !self.from_displayname.is_empty() {
-                    dc_encode_header_words(&self.from_displayname).strdup()
-                } else {
-                    ptr::null_mut()
-                },
-                self.from_addr.strdup(),
-            ),
-        );
-        let mut to: *mut mailimf_address_list = ptr::null_mut();
-        if !self.recipients_names.is_empty() && !self.recipients_addr.is_empty() {
-            to = mailimf_address_list_new_empty();
-            let name_iter = self.recipients_names.iter();
-            let addr_iter = self.recipients_addr.iter();
-            for (name, addr) in name_iter.zip(addr_iter) {
-                mailimf_address_list_add(
-                    to,
-                    mailimf_address_new(
-                        MAILIMF_ADDRESS_MAILBOX as libc::c_int,
-                        mailimf_mailbox_new(
-                            if !name.is_empty() {
-                                dc_encode_header_words(&name).strdup()
-                            } else {
-                                ptr::null_mut()
-                            },
-                            addr.strdup(),
-                        ),
-                        ptr::null_mut(),
+        let to = mailimf_address_list_new_empty();
+        let name_iter = self.recipients_names.iter();
+        let addr_iter = self.recipients_addr.iter();
+        for (name, addr) in name_iter.zip(addr_iter) {
+            mailimf_address_list_add(
+                to,
+                mailimf_address_new(
+                    MAILIMF_ADDRESS_MAILBOX as libc::c_int,
+                    mailimf_mailbox_new(
+                        if !name.is_empty() {
+                            dc_encode_header_words(&name).strdup()
+                        } else {
+                            ptr::null_mut()
+                        },
+                        addr.strdup(),
                     ),
-                );
-            }
+                    ptr::null_mut(),
+                ),
+            );
         }
         let references_list = if !self.references.is_empty() {
             dc_str_to_clist(&self.references, " ")
@@ -198,6 +179,7 @@ impl<'a> MimeFactory<'a> {
         } else {
             ptr::null_mut()
         };
+
         let imf_fields = mailimf_fields_new_with_data_all(
             mailimf_get_date(self.timestamp as i64),
             from,
@@ -275,7 +257,7 @@ impl<'a> MimeFactory<'a> {
                         e2ee_guaranteed = self
                             .msg
                             .param
-                            .get_int(Param::GuranteeE2ee)
+                            .get_int(Param::GuaranteeE2ee)
                             .unwrap_or_default()
                             != 0;
                     }
@@ -402,15 +384,8 @@ impl<'a> MimeFactory<'a> {
                                     &fingerprint,
                                 );
                             }
-                            match msg.param.get(Param::Arg4) {
-                                Some(id) => {
-                                    wrapmime::new_custom_field(
-                                        imf_fields,
-                                        "Secure-Join-Group",
-                                        &id,
-                                    );
-                                }
-                                None => {}
+                            if let Some(id) = msg.param.get(Param::Arg4) {
+                                wrapmime::new_custom_field(imf_fields, "Secure-Join-Group", &id);
                             };
                         }
                     }
@@ -435,6 +410,10 @@ impl<'a> MimeFactory<'a> {
                     }
                 }
 
+                if self.msg.type_0 == Viewtype::Sticker {
+                    wrapmime::new_custom_field(imf_fields, "Chat-Content", "sticker");
+                }
+
                 if self.msg.type_0 == Viewtype::Voice
                     || self.msg.type_0 == Viewtype::Audio
                     || self.msg.type_0 == Viewtype::Video
@@ -505,6 +484,7 @@ impl<'a> MimeFactory<'a> {
                 if !meta_part.is_null() {
                     mailmime_smart_add_part(message, meta_part);
                 }
+
                 if self.msg.param.exists(Param::SetLatitude) {
                     let param = &self.msg.param;
                     let kml_file = location::get_message_kml(
@@ -521,18 +501,21 @@ impl<'a> MimeFactory<'a> {
                 }
 
                 if location::is_sending_locations_to_chat(context, self.msg.chat_id) {
-                    if let Ok((kml_file, last_added_location_id)) =
-                        location::get_kml(context, self.msg.chat_id)
-                    {
-                        wrapmime::add_filename_part(
-                            message,
-                            "location.kml",
-                            "application/vnd.google-earth.kml+xml",
-                            &kml_file,
-                        )?;
-                        if !self.msg.param.exists(Param::SetLatitude) {
-                            // otherwise, the independent location is already filed
-                            self.out_last_added_location_id = last_added_location_id;
+                    match location::get_kml(context, self.msg.chat_id) {
+                        Ok((kml_content, last_added_location_id)) => {
+                            wrapmime::add_filename_part(
+                                message,
+                                "location.kml",
+                                "application/vnd.google-earth.kml+xml",
+                                &kml_content,
+                            )?;
+                            if !self.msg.param.exists(Param::SetLatitude) {
+                                // otherwise, the independent location is already filed
+                                self.out_last_added_location_id = last_added_location_id;
+                            }
+                        }
+                        Err(err) => {
+                            warn!(context, "mimefactory: could not get location: {}", err);
                         }
                     }
                 }
@@ -559,7 +542,7 @@ impl<'a> MimeFactory<'a> {
                     != self
                         .msg
                         .param
-                        .get_int(Param::GuranteeE2ee)
+                        .get_int(Param::GuaranteeE2ee)
                         .unwrap_or_default()
                 {
                     self.context
@@ -594,7 +577,6 @@ impl<'a> MimeFactory<'a> {
                 wrapmime::set_body_text(mach_mime_part, &message_text2)?;
                 mailmime_add_part(multipart, mach_mime_part);
                 force_plaintext = DC_FP_NO_AUTOCRYPT_HEADER;
-                info!(context, "sending MDM {:?}", message_text2);
                 /* currently, we do not send MDNs encrypted:
                 - in a multi-device-setup that is not set up properly, MDNs would disturb the communication as they
                   are send automatically which may lead to spreading outdated Autocrypt headers.
@@ -645,7 +627,7 @@ impl<'a> MimeFactory<'a> {
         );
 
         /*just a pointer into mailmime structure, must not be freed*/
-        let imffields_unprotected = mailmime_find_mailimf_fields(message);
+        let imffields_unprotected = wrapmime::mailmime_find_mailimf_fields(message);
         ensure!(
             !imffields_unprotected.is_null(),
             "could not find mime fields"
@@ -657,17 +639,18 @@ impl<'a> MimeFactory<'a> {
             let aheader = encrypt_helper.get_aheader().to_string();
             wrapmime::new_custom_field(imffields_unprotected, "Autocrypt", &aheader);
         }
-        let mut finalized = false;
-        if force_plaintext == 0 {
-            finalized = encrypt_helper.try_encrypt(
+        let finalized = if force_plaintext == 0 {
+            encrypt_helper.try_encrypt(
                 self,
                 e2ee_guaranteed,
                 min_verified,
                 do_gossip,
                 message,
                 imffields_unprotected,
-            )?;
-        }
+            )?
+        } else {
+            false
+        };
         if !finalized {
             self.finalize_mime_message(message, false, false)?;
         }
@@ -675,9 +658,7 @@ impl<'a> MimeFactory<'a> {
         Ok(())
     }
 
-    pub fn load_msg(context: &Context, msg_id: u32) -> Result<MimeFactory, Error> {
-        ensure!(msg_id > DC_CHAT_ID_LAST_SPECIAL, "Invalid chat id");
-
+    pub fn load_msg(context: &Context, msg_id: MsgId) -> Result<MimeFactory, Error> {
         let msg = Message::load_from_db(context, msg_id)?;
         let chat = Chat::load_from_db(context, msg.chat_id)?;
         let mut factory = MimeFactory::new(context, msg);
@@ -692,31 +673,28 @@ impl<'a> MimeFactory<'a> {
                 .push(factory.from_displayname.to_string());
             factory.recipients_addr.push(factory.from_addr.to_string());
         } else {
-            context
-                .sql
-                .query_map(
-                    "SELECT c.authname, c.addr  \
-                     FROM chats_contacts cc  \
-                     LEFT JOIN contacts c ON cc.contact_id=c.id  \
-                     WHERE cc.chat_id=? AND cc.contact_id>9;",
-                    params![factory.msg.chat_id as i32],
-                    |row| {
-                        let authname: String = row.get(0)?;
-                        let addr: String = row.get(1)?;
-                        Ok((authname, addr))
-                    },
-                    |rows| {
-                        for row in rows {
-                            let (authname, addr) = row?;
-                            if !vec_contains_lowercase(&factory.recipients_addr, &addr) {
-                                factory.recipients_addr.push(addr);
-                                factory.recipients_names.push(authname);
-                            }
+            context.sql.query_map(
+                "SELECT c.authname, c.addr  \
+                 FROM chats_contacts cc  \
+                 LEFT JOIN contacts c ON cc.contact_id=c.id  \
+                 WHERE cc.chat_id=? AND cc.contact_id>9;",
+                params![factory.msg.chat_id as i32],
+                |row| {
+                    let authname: String = row.get(0)?;
+                    let addr: String = row.get(1)?;
+                    Ok((authname, addr))
+                },
+                |rows| {
+                    for row in rows {
+                        let (authname, addr) = row?;
+                        if !vec_contains_lowercase(&factory.recipients_addr, &addr) {
+                            factory.recipients_addr.push(addr);
+                            factory.recipients_names.push(authname);
                         }
-                        Ok(())
-                    },
-                )
-                .unwrap();
+                    }
+                    Ok(())
+                },
+            )?;
 
             let command = factory.msg.param.get_cmd();
             let msg = &factory.msg;
@@ -726,8 +704,7 @@ impl<'a> MimeFactory<'a> {
                 let email_to_remove = msg.param.get(Param::Arg).unwrap_or_default();
 
                 let self_addr = context
-                    .sql
-                    .get_config(context, "configured_addr")
+                    .get_config(Config::ConfiguredAddr)
                     .unwrap_or_default();
 
                 if !email_to_remove.is_empty() && email_to_remove != self_addr {
@@ -739,17 +716,14 @@ impl<'a> MimeFactory<'a> {
             }
             if command != SystemMessage::AutocryptSetupMessage
                 && command != SystemMessage::SecurejoinMessage
-                && 0 != context
-                    .sql
-                    .get_config_int(context, "mdns_enabled")
-                    .unwrap_or_else(|| 1)
+                && context.get_config_bool(Config::MdnsEnabled)
             {
                 factory.req_mdn = true;
             }
         }
         let row = context.sql.query_row(
             "SELECT mime_in_reply_to, mime_references FROM msgs WHERE id=?",
-            params![factory.msg.id as i32],
+            params![factory.msg.id],
             |row| {
                 let in_reply_to: String = row.get(0)?;
                 let references: String = row.get(1)?;
@@ -822,26 +796,21 @@ fn build_body_file(
     msg: &Message,
     base_name: &str,
 ) -> Result<(*mut Mailmime, String), Error> {
-    let path_filename = match msg.param.get(Param::File) {
-        None => {
-            bail!("msg has no filename");
-        }
-        Some(path) => path,
-    };
-    let suffix = dc_get_filesuffix_lc(path_filename).unwrap_or_else(|| "dat".into());
+    let blob = msg
+        .param
+        .get_blob(Param::File, context, true)?
+        .ok_or_else(|| format_err!("msg has no filename"))?;
+    let suffix = blob.suffix().unwrap_or("dat");
 
-    /* get file name to use for sending
-    (for privacy purposes, we do not transfer the original filenames eg. for images;
-    these names are normally not needed and contain timestamps, running numbers etc.) */
-    let filename_to_send = match msg.type_0 {
+    // Get file name to use for sending.  For privacy purposes, we do
+    // not transfer the original filenames eg. for images; these names
+    // are normally not needed and contain timestamps, running numbers
+    // etc.
+    let filename_to_send: String = match msg.type_0 {
         Viewtype::Voice => chrono::Utc
             .timestamp(msg.timestamp_sort as i64, 0)
-            .format(&format!("voice-message_%Y-%m-%d_%H-%M-%S.{}", suffix))
+            .format(&format!("voice-message_%Y-%m-%d_%H-%M-%S.{}", &suffix))
             .to_string(),
-        Viewtype::Audio => Path::new(path_filename)
-            .file_name()
-            .map(|c| c.to_string_lossy().to_string())
-            .unwrap_or_default(),
         Viewtype::Image | Viewtype::Gif => format!(
             "{}.{}",
             if base_name.is_empty() {
@@ -852,18 +821,14 @@ fn build_body_file(
             &suffix,
         ),
         Viewtype::Video => format!("video.{}", &suffix),
-        _ => Path::new(path_filename)
-            .file_name()
-            .map(|c| c.to_string_lossy().to_string())
-            .unwrap_or_default(),
+        _ => blob.as_file_name().to_string(),
     };
 
     /* check mimetype */
     let mimetype = match msg.param.get(Param::MimeType) {
         Some(mtype) => mtype,
         None => {
-            let path = Path::new(path_filename);
-            if let Some(res) = message::guess_msgtype_from_suffix(&path) {
+            if let Some(res) = message::guess_msgtype_from_suffix(blob.as_rel_path()) {
                 res.1
             } else {
                 "application/octet-stream"
@@ -923,15 +888,13 @@ fn build_body_file(
         wrapmime::append_ct_param(content, "name", &filename_encoded)?;
 
         let mime_sub = mailmime_new_empty(content, mime_fields);
-        let abs_path = dc_get_abs_path(context, path_filename)
-            .to_c_string()
-            .unwrap();
+        let abs_path = blob.to_abs_path().to_c_string()?;
         mailmime_set_body_file(mime_sub, dc_strdup(abs_path.as_ptr()));
         Ok((mime_sub, filename_to_send))
     }
 }
 
-pub(crate) fn vec_contains_lowercase(vec: &Vec<String>, part: &str) -> bool {
+pub(crate) fn vec_contains_lowercase(vec: &[String], part: &str) -> bool {
     let partlc = part.to_lowercase();
     for cur in vec.iter() {
         if cur.to_lowercase() == partlc {
@@ -942,13 +905,11 @@ pub(crate) fn vec_contains_lowercase(vec: &Vec<String>, part: &str) -> bool {
 }
 
 fn is_file_size_okay(context: &Context, msg: &Message) -> bool {
-    let mut file_size_okay = true;
-    let path = msg.param.get(Param::File).unwrap_or_default();
-    let bytes = dc_get_filebytes(context, &path);
-
-    if bytes > (49 * 1024 * 1024 / 4 * 3) {
-        file_size_okay = false;
+    match msg.param.get_path(Param::File, context).unwrap_or(None) {
+        Some(path) => {
+            let bytes = dc_get_filebytes(context, &path);
+            bytes <= (49 * 1024 * 1024 / 4 * 3)
+        }
+        None => false,
     }
-
-    file_size_okay
 }

src/oauth2.rs

@@ -7,6 +7,7 @@ use crate::context::Context;
 use crate::dc_tools::*;
 
 const OAUTH2_GMAIL: Oauth2 = Oauth2 {
+    // see https://developers.google.com/identity/protocols/OAuth2InstalledApp
     client_id: "959970109878-4mvtgf6feshskf7695nfln6002mom908.apps.googleusercontent.com",
     get_code: "https://accounts.google.com/o/oauth2/auth?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&response_type=code&scope=https%3A%2F%2Fmail.google.com%2F%20email&access_type=offline",
     init_token: "https://accounts.google.com/o/oauth2/token?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&code=$CODE&grant_type=authorization_code",
@@ -15,6 +16,7 @@ const OAUTH2_GMAIL: Oauth2 = Oauth2 {
 };
 
 const OAUTH2_YANDEX: Oauth2 = Oauth2 {
+    // see https://tech.yandex.com/oauth/doc/dg/reference/auto-code-client-docpage/
     client_id: "c4d0b6735fc8420a816d7e1303469341",
     get_code: "https://oauth.yandex.com/authorize?client_id=$CLIENT_ID&response_type=code&scope=mail%3Aimap_full%20mail%3Asmtp&force_confirm=true",
     init_token: "https://oauth.yandex.com/token?grant_type=authorization_code&code=$CODE&client_id=$CLIENT_ID&client_secret=58b8c6e94cf44fbe952da8511955dacf",
@@ -50,7 +52,7 @@ pub fn dc_get_oauth2_url(
     if let Some(oauth2) = Oauth2::from_address(addr) {
         if context
             .sql
-            .set_config(
+            .set_raw_config(
                 context,
                 "oauth2_pending_redirect_uri",
                 Some(redirect_uri.as_ref()),
@@ -82,17 +84,18 @@ pub fn dc_get_oauth2_access_token(
 
         // read generated token
         if !regenerate && !is_expired(context) {
-            let access_token = context.sql.get_config(context, "oauth2_access_token");
+            let access_token = context.sql.get_raw_config(context, "oauth2_access_token");
             if access_token.is_some() {
                 // success
                 return access_token;
             }
         }
 
-        let refresh_token = context.sql.get_config(context, "oauth2_refresh_token");
+        // generate new token: build & call auth url
+        let refresh_token = context.sql.get_raw_config(context, "oauth2_refresh_token");
         let refresh_token_for = context
             .sql
-            .get_config(context, "oauth2_refresh_token_for")
+            .get_raw_config(context, "oauth2_refresh_token_for")
             .unwrap_or_else(|| "unset".into());
 
         let (redirect_uri, token_url, update_redirect_uri_on_success) =
@@ -101,7 +104,7 @@ pub fn dc_get_oauth2_access_token(
                 (
                     context
                         .sql
-                        .get_config(context, "oauth2_pending_redirect_uri")
+                        .get_raw_config(context, "oauth2_pending_redirect_uri")
                         .unwrap_or_else(|| "unset".into()),
                     oauth2.init_token,
                     true,
@@ -114,20 +117,43 @@ pub fn dc_get_oauth2_access_token(
                 (
                     context
                         .sql
-                        .get_config(context, "oauth2_redirect_uri")
+                        .get_raw_config(context, "oauth2_redirect_uri")
                         .unwrap_or_else(|| "unset".into()),
                     oauth2.refresh_token,
                     false,
                 )
             };
-        let mut token_url = replace_in_uri(&token_url, "$CLIENT_ID", oauth2.client_id);
-        token_url = replace_in_uri(&token_url, "$REDIRECT_URI", &redirect_uri);
-        token_url = replace_in_uri(&token_url, "$CODE", code.as_ref());
-        if let Some(ref token) = refresh_token {
-            token_url = replace_in_uri(&token_url, "$REFRESH_TOKEN", token);
+
+        // to allow easier specification of different configurations,
+        // token_url is in GET-method-format, sth. as https://domain?param1=val1&param2=val2 -
+        // convert this to POST-format ...
+        let mut parts = token_url.splitn(2, '?');
+        let post_url = parts.next().unwrap_or_default();
+        let post_args = parts.next().unwrap_or_default();
+        let mut post_param = HashMap::new();
+        for key_value_pair in post_args.split('&') {
+            let mut parts = key_value_pair.splitn(2, '=');
+            let key = parts.next().unwrap_or_default();
+            let mut value = parts.next().unwrap_or_default();
+
+            if value == "$CLIENT_ID" {
+                value = oauth2.client_id;
+            } else if value == "$REDIRECT_URI" {
+                value = &redirect_uri;
+            } else if value == "$CODE" {
+                value = code.as_ref();
+            } else if value == "$REFRESH_TOKEN" && refresh_token.is_some() {
+                value = refresh_token.as_ref().unwrap();
+            }
+
+            post_param.insert(key, value);
         }
 
-        let response = reqwest::Client::new().post(&token_url).send();
+        // ... and POST
+        let response = reqwest::Client::new()
+            .post(post_url)
+            .form(&post_param)
+            .send();
         if response.is_err() {
             warn!(
                 context,
@@ -139,13 +165,14 @@ pub fn dc_get_oauth2_access_token(
         if !response.status().is_success() {
             warn!(
                 context,
-                "Error calling OAuth2 at {}: {:?}",
+                "Unsuccessful response when calling OAuth2 at {}: {:?}",
                 token_url,
                 response.status()
             );
             return None;
         }
 
+        // generate new token: parse returned json
         let parsed: reqwest::Result<Response> = response.json();
         if parsed.is_err() {
             warn!(
@@ -155,15 +182,17 @@ pub fn dc_get_oauth2_access_token(
             return None;
         }
         println!("response: {:?}", &parsed);
+
+        // update refresh_token if given, typically on the first round, but we update it later as well.
         let response = parsed.unwrap();
         if let Some(ref token) = response.refresh_token {
             context
                 .sql
-                .set_config(context, "oauth2_refresh_token", Some(token))
+                .set_raw_config(context, "oauth2_refresh_token", Some(token))
                 .ok();
             context
                 .sql
-                .set_config(context, "oauth2_refresh_token_for", Some(code.as_ref()))
+                .set_raw_config(context, "oauth2_refresh_token_for", Some(code.as_ref()))
                 .ok();
         }
 
@@ -172,7 +201,7 @@ pub fn dc_get_oauth2_access_token(
         if let Some(ref token) = response.access_token {
             context
                 .sql
-                .set_config(context, "oauth2_access_token", Some(token))
+                .set_raw_config(context, "oauth2_access_token", Some(token))
                 .ok();
             let expires_in = response
                 .expires_in
@@ -181,13 +210,13 @@ pub fn dc_get_oauth2_access_token(
                 .unwrap_or_else(|| 0);
             context
                 .sql
-                .set_config_int64(context, "oauth2_timestamp_expires", expires_in)
+                .set_raw_config_int64(context, "oauth2_timestamp_expires", expires_in)
                 .ok();
 
             if update_redirect_uri_on_success {
                 context
                     .sql
-                    .set_config(context, "oauth2_redirect_uri", Some(redirect_uri.as_ref()))
+                    .set_raw_config(context, "oauth2_redirect_uri", Some(redirect_uri.as_ref()))
                     .ok();
             }
         } else {
@@ -207,14 +236,8 @@ pub fn dc_get_oauth2_addr(
     addr: impl AsRef<str>,
     code: impl AsRef<str>,
 ) -> Option<String> {
-    let oauth2 = Oauth2::from_address(addr.as_ref());
-    if oauth2.is_none() {
-        return None;
-    }
-    let oauth2 = oauth2.unwrap();
-    if oauth2.get_userinfo.is_none() {
-        return None;
-    }
+    let oauth2 = Oauth2::from_address(addr.as_ref())?;
+    oauth2.get_userinfo?;
 
     if let Some(access_token) =
         dc_get_oauth2_access_token(context, addr.as_ref(), code.as_ref(), false)
@@ -274,7 +297,7 @@ impl Oauth2 {
             return None;
         }
 
-        let parsed: reqwest::Result<HashMap<String, String>> = response.json();
+        let parsed: reqwest::Result<HashMap<String, serde_json::Value>> = response.json();
         if parsed.is_err() {
             warn!(
                 context,
@@ -283,11 +306,13 @@ impl Oauth2 {
             return None;
         }
         if let Ok(response) = parsed {
+            // serde_json::Value.as_str() removes the quotes of json-strings
             let addr = response.get("email");
             if addr.is_none() {
                 warn!(context, "E-mail missing in userinfo.");
+                return None;
             }
-
+            let addr = addr.unwrap().as_str();
             addr.map(|addr| addr.to_string())
         } else {
             warn!(context, "Failed to parse userinfo.");
@@ -299,7 +324,7 @@ impl Oauth2 {
 fn is_expired(context: &Context) -> bool {
     let expire_timestamp = context
         .sql
-        .get_config_int64(context, "oauth2_timestamp_expires")
+        .get_raw_config_int64(context, "oauth2_timestamp_expires")
         .unwrap_or_default();
 
     if expire_timestamp <= 0 {

src/param.rs

@@ -1,9 +1,12 @@
 use std::collections::BTreeMap;
 use std::fmt;
+use std::path::PathBuf;
 use std::str;
 
 use num_traits::FromPrimitive;
 
+use crate::blob::{BlobError, BlobObject};
+use crate::context::Context;
 use crate::dc_mimeparser::SystemMessage;
 use crate::error;
 
@@ -12,65 +15,75 @@ use crate::error;
 #[repr(u8)]
 pub enum Param {
     /// For messages and jobs
-    File = 'f' as u8,
+    File = b'f',
     /// For Messages
-    Width = 'w' as u8,
+    Width = b'w',
     /// For Messages
-    Height = 'h' as u8,
+    Height = b'h',
     /// For Messages
-    Duration = 'd' as u8,
+    Duration = b'd',
     /// For Messages
-    MimeType = 'm' as u8,
+    MimeType = b'm',
     /// For Messages: message is encryoted, outgoing: guarantee E2EE or the message is not send
-    GuranteeE2ee = 'c' as u8,
+    GuaranteeE2ee = b'c',
     /// For Messages: decrypted with validation errors or without mutual set, if neither
     /// 'c' nor 'e' are preset, the messages is only transport encrypted.
-    ErroneousE2ee = 'e' as u8,
+    ErroneousE2ee = b'e',
     /// For Messages: force unencrypted message, either `ForcePlaintext::AddAutocryptHeader` (1),
     /// `ForcePlaintext::NoAutocryptHeader` (2) or 0.
-    ForcePlaintext = 'u' as u8,
+    ForcePlaintext = b'u',
     /// For Messages
-    WantsMdn = 'r' as u8,
+    WantsMdn = b'r',
     /// For Messages
-    Forwarded = 'a' as u8,
+    Forwarded = b'a',
     /// For Messages
-    Cmd = 'S' as u8,
+    Cmd = b'S',
     /// For Messages
-    Arg = 'E' as u8,
+    Arg = b'E',
     /// For Messages
-    Arg2 = 'F' as u8,
+    Arg2 = b'F',
     /// For Messages
-    Arg3 = 'G' as u8,
+    Arg3 = b'G',
     /// For Messages
-    Arg4 = 'H' as u8,
+    Arg4 = b'H',
     /// For Messages
-    Error = 'L' as u8,
+    Error = b'L',
     /// For Messages: space-separated list of messaged IDs of forwarded copies.
-    PrepForwards = 'P' as u8,
+    ///
+    /// This is used when a [Message] is in the
+    /// [MessageState::OutPending] state but is already forwarded.
+    /// In this case the forwarded messages are written to the
+    /// database and their message IDs are added to this parameter of
+    /// the original message, which is also saved in the database.
+    /// When the original message is then finally sent this parameter
+    /// is used to also send all the forwarded messages.
+    PrepForwards = b'P',
     /// For Jobs
-    SetLatitude = 'l' as u8,
+    SetLatitude = b'l',
     /// For Jobs
-    SetLongitude = 'n' as u8,
+    SetLongitude = b'n',
     /// For Jobs
-    ServerFolder = 'Z' as u8,
+    ServerFolder = b'Z',
     /// For Jobs
-    ServerUid = 'z' as u8,
+    ServerUid = b'z',
     /// For Jobs
-    AlsoMove = 'M' as u8,
+    AlsoMove = b'M',
     /// For Jobs: space-separated list of message recipients
-    Recipients = 'R' as u8,
+    Recipients = b'R',
     // For Groups
-    Unpromoted = 'U' as u8,
+    Unpromoted = b'U',
     // For Groups and Contacts
-    ProfileImage = 'i' as u8,
+    ProfileImage = b'i',
     // For Chats
-    Selftalk = 'K' as u8,
+    Selftalk = b'K',
+    // For Chats
+    Devicetalk = b'D',
     // For QR
-    Auth = 's' as u8,
+    Auth = b's',
     // For QR
-    GroupId = 'x' as u8,
+    GroupId = b'x',
     // For QR
-    GroupName = 'g' as u8,
+    GroupName = b'g',
 }
 
 /// Possible values for `Param::ForcePlaintext`.
@@ -122,8 +135,8 @@ impl str::FromStr for Params {
             ensure!(key.is_some(), "Missing key");
             ensure!(value.is_some(), "Missing value");
 
-            let key = key.unwrap().trim();
-            let value = value.unwrap().trim();
+            let key = key.unwrap_or_default().trim();
+            let value = value.unwrap_or_default().trim();
 
             if let Some(key) = Param::from_u8(key.as_bytes()[0]) {
                 inner.insert(key, value.to_string());
@@ -186,11 +199,82 @@ impl Params {
             .unwrap_or_default()
     }
 
+    /// Set the parameter behind `Param::Cmd`.
+    pub fn set_cmd(&mut self, value: SystemMessage) {
+        self.set_int(Param::Cmd, value as i32);
+    }
+
     /// Get the given parameter and parse as `f64`.
     pub fn get_float(&self, key: Param) -> Option<f64> {
         self.get(key).and_then(|s| s.parse().ok())
     }
 
+    /// Gets the given parameter and parse as [ParamsFile].
+    ///
+    /// See also [Params::get_blob] and [Params::get_path] which may
+    /// be more convenient.
+    pub fn get_file<'a>(
+        &self,
+        key: Param,
+        context: &'a Context,
+    ) -> Result<Option<ParamsFile<'a>>, BlobError> {
+        let val = match self.get(key) {
+            Some(val) => val,
+            None => return Ok(None),
+        };
+        ParamsFile::from_param(context, val).map(|file| Some(file))
+    }
+
+    /// Gets the parameter and returns a [BlobObject] for it.
+    ///
+    /// This parses the parameter value as a [ParamsFile] and than
+    /// tries to return a [BlobObject] for that file.  If the file is
+    /// not yet a valid blob, one will be created by copying the file
+    /// only if `create` is set to `true`, otherwise the a [BlobError]
+    /// will result.
+    ///
+    /// Note that in the [ParamsFile::FsPath] case the blob can be
+    /// created without copying if the path already referes to a valid
+    /// blob.  If so a [BlobObject] will be returned regardless of the
+    /// `create` argument.
+    pub fn get_blob<'a>(
+        &self,
+        key: Param,
+        context: &'a Context,
+        create: bool,
+    ) -> Result<Option<BlobObject<'a>>, BlobError> {
+        let val = match self.get(key) {
+            Some(val) => val,
+            None => return Ok(None),
+        };
+        let file = ParamsFile::from_param(context, val)?;
+        let blob = match file {
+            ParamsFile::FsPath(path) => match create {
+                true => BlobObject::create_from_path(context, path)?,
+                false => BlobObject::from_path(context, path)?,
+            },
+            ParamsFile::Blob(blob) => blob,
+        };
+        Ok(Some(blob))
+    }
+
+    /// Gets the parameter and returns a [PathBuf] for it.
+    ///
+    /// This parses the parameter value as a [ParamsFile] and returns
+    /// a [PathBuf] to the file.
+    pub fn get_path(&self, key: Param, context: &Context) -> Result<Option<PathBuf>, BlobError> {
+        let val = match self.get(key) {
+            Some(val) => val,
+            None => return Ok(None),
+        };
+        let file = ParamsFile::from_param(context, val)?;
+        let path = match file {
+            ParamsFile::FsPath(path) => path,
+            ParamsFile::Blob(blob) => blob.to_abs_path(),
+        };
+        Ok(Some(path))
+    }
+
     /// Set the given paramter to the passed in `i32`.
     pub fn set_int(&mut self, key: Param, value: i32) -> &mut Self {
         self.set(key, format!("{}", value));
@@ -204,10 +288,42 @@ impl Params {
     }
 }
 
+/// The value contained in [Param::File].
+///
+/// Because the only way to construct this object is from a valid
+/// UTF-8 string it is always safe to convert the value contained
+/// within the [ParamsFile::FsPath] back to a [String] or [&str].
+/// Despite the type itself does not guarantee this.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum ParamsFile<'a> {
+    FsPath(PathBuf),
+    Blob(BlobObject<'a>),
+}
+
+impl<'a> ParamsFile<'a> {
+    /// Parse the [Param::File] value into an object.
+    ///
+    /// If the value was stored into the [Params] correctly this
+    /// should not fail.
+    pub fn from_param(context: &'a Context, src: &str) -> Result<ParamsFile<'a>, BlobError> {
+        let param = match src.starts_with("$BLOBDIR/") {
+            true => ParamsFile::Blob(BlobObject::from_name(context, src.to_string())?),
+            false => ParamsFile::FsPath(PathBuf::from(src)),
+        };
+        Ok(param)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
 
+    use std::fs;
+    use std::path::Path;
+
+    use crate::blob::BlobErrorKind;
+    use crate::test_utils::*;
+
     #[test]
     fn test_dc_param() {
         let mut p1: Params = "\r\n\r\na=1\nf=2\n\nc = 3 ".parse().unwrap();
@@ -225,7 +341,7 @@ mod tests {
 
         p1.set(Param::Forwarded, "foo")
             .set_int(Param::File, 2)
-            .remove(Param::GuranteeE2ee)
+            .remove(Param::GuaranteeE2ee)
             .set_int(Param::Duration, 4);
 
         assert_eq!(p1.to_string(), "a=foo\nd=4\nf=2");
@@ -251,4 +367,64 @@ mod tests {
             .unwrap();
         assert_eq!(p1.get(Param::Forwarded).unwrap(), "cli%40deltachat.de");
     }
+
+    #[test]
+    fn test_params_file_fs_path() {
+        let t = dummy_context();
+        if let ParamsFile::FsPath(p) = ParamsFile::from_param(&t.ctx, "/foo/bar/baz").unwrap() {
+            assert_eq!(p, Path::new("/foo/bar/baz"));
+        } else {
+            assert!(false, "Wrong enum variant");
+        }
+    }
+
+    #[test]
+    fn test_params_file_blob() {
+        let t = dummy_context();
+        if let ParamsFile::Blob(b) = ParamsFile::from_param(&t.ctx, "$BLOBDIR/foo").unwrap() {
+            assert_eq!(b.as_name(), "$BLOBDIR/foo");
+        } else {
+            assert!(false, "Wrong enum variant");
+        }
+    }
+
+    // Tests for Params::get_file(), Params::get_path() and Params::get_blob().
+    #[test]
+    fn test_params_get_fileparam() {
+        let t = dummy_context();
+        let fname = t.dir.path().join("foo");
+        let mut p = Params::new();
+        p.set(Param::File, fname.to_str().unwrap());
+
+        let file = p.get_file(Param::File, &t.ctx).unwrap().unwrap();
+        assert_eq!(file, ParamsFile::FsPath(fname.clone()));
+
+        let path = p.get_path(Param::File, &t.ctx).unwrap().unwrap();
+        assert_eq!(path, fname);
+
+        // Blob does not exist yet, expect BlobError.
+        let err = p.get_blob(Param::File, &t.ctx, false).unwrap_err();
+        assert_eq!(err.kind(), BlobErrorKind::WrongBlobdir);
+
+        fs::write(fname, b"boo").unwrap();
+        let blob = p.get_blob(Param::File, &t.ctx, true).unwrap().unwrap();
+        assert_eq!(
+            blob,
+            BlobObject::from_name(&t.ctx, "foo".to_string()).unwrap()
+        );
+
+        // Blob in blobdir, expect blob.
+        let bar = t.ctx.get_blobdir().join("bar");
+        p.set(Param::File, bar.to_str().unwrap());
+        let blob = p.get_blob(Param::File, &t.ctx, false).unwrap().unwrap();
+        assert_eq!(
+            blob,
+            BlobObject::from_name(&t.ctx, "bar".to_string()).unwrap()
+        );
+
+        p.remove(Param::File);
+        assert!(p.get_file(Param::File, &t.ctx).unwrap().is_none());
+        assert!(p.get_path(Param::File, &t.ctx).unwrap().is_none());
+        assert!(p.get_blob(Param::File, &t.ctx, false).unwrap().is_none());
+    }
 }

src/peerstate.rs

@@ -400,7 +400,8 @@ impl<'a> Peerstate<'a> {
                     &self.verified_key_fingerprint,
                     &self.addr,
                 ],
-            )?
+            )?;
+            reset_gossiped_timestamp(self.context, 0);
         } else if self.to_save == Some(ToSave::Timestamps) {
             sql::execute(
                 self.context,
@@ -416,10 +417,6 @@ impl<'a> Peerstate<'a> {
             )?;
         }
 
-        if self.to_save == Some(ToSave::All) || create {
-            reset_gossiped_timestamp(self.context, 0);
-        }
-
         Ok(())
     }
 
@@ -475,7 +472,7 @@ mod tests {
             "failed to save to db"
         );
 
-        let peerstate_new = Peerstate::from_addr(&ctx.ctx, &ctx.ctx.sql, addr.into())
+        let peerstate_new = Peerstate::from_addr(&ctx.ctx, &ctx.ctx.sql, addr)
             .expect("failed to load peerstate from db");
 
         // clear to_save, as that is not persissted
@@ -487,6 +484,44 @@ mod tests {
         assert_eq!(peerstate, peerstate_new2);
     }
 
+    #[test]
+    fn test_peerstate_double_create() {
+        let ctx = crate::test_utils::dummy_context();
+        let addr = "hello@mail.com";
+
+        let pub_key = crate::key::Key::from_base64(
+            include_str!("../test-data/key/public.asc"),
+            KeyType::Public,
+        )
+        .unwrap();
+
+        let peerstate = Peerstate {
+            context: &ctx.ctx,
+            addr: Some(addr.into()),
+            last_seen: 10,
+            last_seen_autocrypt: 11,
+            prefer_encrypt: EncryptPreference::Mutual,
+            public_key: Some(pub_key.clone()),
+            public_key_fingerprint: Some(pub_key.fingerprint()),
+            gossip_key: None,
+            gossip_timestamp: 12,
+            gossip_key_fingerprint: None,
+            verified_key: None,
+            verified_key_fingerprint: None,
+            to_save: Some(ToSave::All),
+            degrade_event: None,
+        };
+
+        assert!(
+            peerstate.save_to_db(&ctx.ctx.sql, true).is_ok(),
+            "failed to save"
+        );
+        assert!(
+            peerstate.save_to_db(&ctx.ctx.sql, true).is_ok(),
+            "double-call with create failed"
+        );
+    }
+
     #[test]
     fn test_peerstate_with_empty_gossip_key_save_to_db() {
         let ctx = crate::test_utils::dummy_context();
@@ -520,7 +555,7 @@ mod tests {
             "failed to save"
         );
 
-        let peerstate_new = Peerstate::from_addr(&ctx.ctx, &ctx.ctx.sql, addr.into())
+        let peerstate_new = Peerstate::from_addr(&ctx.ctx, &ctx.ctx.sql, addr)
             .expect("failed to load peerstate from db");
 
         // clear to_save, as that is not persissted

src/pgp.rs

@@ -1,137 +1,53 @@
-use std::collections::HashSet;
+use std::collections::{BTreeMap, HashSet};
 use std::convert::TryInto;
 use std::io::Cursor;
-use std::ptr;
 
-use libc::{strchr, strlen, strncmp, strspn, strstr};
+use pgp::armor::BlockType;
 use pgp::composed::{
     Deserializable, KeyType as PgpKeyType, Message, SecretKeyParamsBuilder, SignedPublicKey,
-    SignedSecretKey, SubkeyParamsBuilder,
+    SignedPublicSubKey, SignedSecretKey, SubkeyParamsBuilder,
 };
 use pgp::crypto::{HashAlgorithm, SymmetricKeyAlgorithm};
 use pgp::types::{CompressionAlgorithm, KeyTrait, SecretKeyTrait, StringToKey};
 use rand::thread_rng;
 
-use crate::dc_tools::*;
 use crate::error::Error;
 use crate::key::*;
 use crate::keyring::*;
 
-pub unsafe fn dc_split_armored_data(
-    buf: *mut libc::c_char,
-    ret_headerline: *mut String,
-    ret_setupcodebegin: *mut *const libc::c_char,
-    ret_preferencrypt: *mut *const libc::c_char,
-    ret_base64: *mut *const libc::c_char,
-) -> bool {
-    let mut success = false;
-    let mut line_chars: libc::size_t = 0;
-    let mut line: *mut libc::c_char = buf;
-    let mut p1: *mut libc::c_char = buf;
-    let mut p2: *mut libc::c_char;
-    let mut headerline: *mut libc::c_char = ptr::null_mut();
-    let mut base64: *mut libc::c_char = ptr::null_mut();
-    if !ret_setupcodebegin.is_null() {
-        *ret_setupcodebegin = ptr::null_mut();
-    }
-    if !ret_preferencrypt.is_null() {
-        *ret_preferencrypt = ptr::null();
-    }
-    if !ret_base64.is_null() {
-        *ret_base64 = ptr::null();
-    }
-    if !buf.is_null() {
-        dc_remove_cr_chars(buf);
-        while 0 != *p1 {
-            if *p1 as libc::c_int == '\n' as i32 {
-                *line.offset(line_chars as isize) = 0i32 as libc::c_char;
-                if headerline.is_null() {
-                    dc_trim(line);
-                    if strncmp(
-                        line,
-                        b"-----BEGIN \x00" as *const u8 as *const libc::c_char,
-                        1,
-                    ) == 0i32
-                        && strncmp(
-                            &mut *line.offset(strlen(line).wrapping_sub(5) as isize),
-                            b"-----\x00" as *const u8 as *const libc::c_char,
-                            5,
-                        ) == 0i32
-                    {
-                        headerline = line;
-                        *ret_headerline = as_str(headerline).to_string();
-                    }
-                } else if strspn(line, b"\t\r\n \x00" as *const u8 as *const libc::c_char)
-                    == strlen(line)
-                {
-                    base64 = p1.offset(1isize);
-                    break;
-                } else {
-                    p2 = strchr(line, ':' as i32);
-                    if p2.is_null() {
-                        *line.offset(line_chars as isize) = '\n' as i32 as libc::c_char;
-                        base64 = line;
-                        break;
-                    } else {
-                        *p2 = 0i32 as libc::c_char;
-                        dc_trim(line);
-                        if strcasecmp(
-                            line,
-                            b"Passphrase-Begin\x00" as *const u8 as *const libc::c_char,
-                        ) == 0i32
-                        {
-                            p2 = p2.offset(1isize);
-                            dc_trim(p2);
-                            if !ret_setupcodebegin.is_null() {
-                                *ret_setupcodebegin = p2
-                            }
-                        } else if strcasecmp(
-                            line,
-                            b"Autocrypt-Prefer-Encrypt\x00" as *const u8 as *const libc::c_char,
-                        ) == 0i32
-                        {
-                            p2 = p2.offset(1isize);
-                            dc_trim(p2);
-                            if !ret_preferencrypt.is_null() {
-                                *ret_preferencrypt = p2
-                            }
-                        }
-                    }
-                }
-                p1 = p1.offset(1isize);
-                line = p1;
-                line_chars = 0;
-            } else {
-                p1 = p1.offset(1isize);
-                line_chars = line_chars.wrapping_add(1)
-            }
-        }
-        if !(headerline.is_null() || base64.is_null()) {
-            /* now, line points to beginning of base64 data, search end */
-            /*the trailing space makes sure, this is not a normal base64 sequence*/
-            p1 = strstr(base64, b"-----END \x00" as *const u8 as *const libc::c_char);
-            if !(p1.is_null()
-                || strncmp(
-                    p1.offset(9isize),
-                    headerline.offset(11isize),
-                    strlen(headerline.offset(11isize)),
-                ) != 0i32)
-            {
-                *p1 = 0i32 as libc::c_char;
-                dc_trim(base64);
-                if !ret_base64.is_null() {
-                    *ret_base64 = base64
-                }
-                success = true;
-            }
-        }
-    }
+pub const HEADER_AUTOCRYPT: &str = "autocrypt-prefer-encrypt";
+pub const HEADER_SETUPCODE: &str = "passphrase-begin";
 
-    success
+/// Split data from PGP Armored Data as defined in https://tools.ietf.org/html/rfc4880#section-6.2.
+///
+/// Returns (type, headers, base64 encoded body).
+pub fn split_armored_data(
+    buf: &[u8],
+) -> Result<(BlockType, BTreeMap<String, String>, Vec<u8>), Error> {
+    use std::io::Read;
+
+    let cursor = Cursor::new(buf);
+    let mut dearmor = pgp::armor::Dearmor::new(cursor);
+
+    let mut bytes = Vec::with_capacity(buf.len());
+
+    dearmor.read_to_end(&mut bytes)?;
+    ensure!(dearmor.typ.is_some(), "Failed to parse type");
+
+    let typ = dearmor.typ.unwrap();
+
+    // normalize headers
+    let headers = dearmor
+        .headers
+        .into_iter()
+        .map(|(key, value)| (key.trim().to_lowercase(), value.trim().to_string()))
+        .collect();
+
+    Ok((typ, headers, bytes))
 }
 
 /// Create a new key pair.
-pub fn dc_pgp_create_keypair(addr: impl AsRef<str>) -> Option<(Key, Key)> {
+pub fn create_keypair(addr: impl AsRef<str>) -> Option<(Key, Key)> {
     let user_id = format!("<{}>", addr.as_ref());
 
     let key_params = SecretKeyParamsBuilder::default()
@@ -181,18 +97,29 @@ pub fn dc_pgp_create_keypair(addr: impl AsRef<str>) -> Option<(Key, Key)> {
     Some((Key::Public(public_key), Key::Secret(private_key)))
 }
 
-pub fn dc_pgp_pk_encrypt(
+/// Select subkey of the public key to use for encryption.
+///
+/// Currently the first subkey is selected.
+fn select_pk_for_encryption(key: &SignedPublicKey) -> Option<&SignedPublicSubKey> {
+    key.public_subkeys.iter().find(|_k|
+        // TODO: check if it is an encryption subkey
+        true)
+}
+
+pub fn pk_encrypt(
     plain: &[u8],
     public_keys_for_encryption: &Keyring,
     private_key_for_signing: Option<&Key>,
 ) -> Result<String, Error> {
     let lit_msg = Message::new_literal_bytes("", plain);
-    let pkeys: Vec<&SignedPublicKey> = public_keys_for_encryption
+    let pkeys: Vec<&SignedPublicSubKey> = public_keys_for_encryption
         .keys()
-        .into_iter()
+        .iter()
         .filter_map(|key| {
-            let k: &Key = &key;
-            k.try_into().ok()
+            key.as_ref()
+                .try_into()
+                .ok()
+                .and_then(select_pk_for_encryption)
         })
         .collect();
 
@@ -218,7 +145,7 @@ pub fn dc_pgp_pk_encrypt(
     Ok(encoded_msg)
 }
 
-pub fn dc_pgp_pk_decrypt(
+pub fn pk_decrypt(
     ctext: &[u8],
     private_keys_for_decryption: &Keyring,
     public_keys_for_validation: &Keyring,
@@ -267,7 +194,7 @@ pub fn dc_pgp_pk_decrypt(
 }
 
 /// Symmetric encryption.
-pub fn dc_pgp_symm_encrypt(passphrase: &str, plain: &[u8]) -> Result<String, Error> {
+pub fn symm_encrypt(passphrase: &str, plain: &[u8]) -> Result<String, Error> {
     let mut rng = thread_rng();
     let lit_msg = Message::new_literal_bytes("", plain);
 
@@ -281,8 +208,11 @@ pub fn dc_pgp_symm_encrypt(passphrase: &str, plain: &[u8]) -> Result<String, Err
 }
 
 /// Symmetric decryption.
-pub fn dc_pgp_symm_decrypt(passphrase: &str, ctext: &[u8]) -> Result<Vec<u8>, Error> {
-    let enc_msg = Message::from_bytes(Cursor::new(ctext))?;
+pub fn symm_decrypt<T: std::io::Read + std::io::Seek>(
+    passphrase: &str,
+    ctext: T,
+) -> Result<Vec<u8>, Error> {
+    let (enc_msg, _) = Message::from_armor_single(ctext)?;
     let decryptor = enc_msg.decrypt_with_password(|| passphrase.into())?;
 
     let msgs = decryptor.collect::<Result<Vec<_>, _>>()?;
@@ -293,3 +223,35 @@ pub fn dc_pgp_symm_decrypt(passphrase: &str, ctext: &[u8]) -> Result<Vec<u8>, Er
         None => bail!("Decrypted message is empty"),
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_split_armored_data_1() {
+        let (typ, _headers, base64) = split_armored_data(
+            b"-----BEGIN PGP MESSAGE-----\nNoVal:\n\naGVsbG8gd29ybGQ=\n-----END PGP MESSAGE----",
+        )
+        .unwrap();
+
+        assert_eq!(typ, BlockType::Message);
+        assert!(!base64.is_empty());
+        assert_eq!(
+            std::string::String::from_utf8(base64).unwrap(),
+            "hello world"
+        );
+    }
+
+    #[test]
+    fn test_split_armored_data_2() {
+        let (typ, headers, base64) = split_armored_data(
+            b"-----BEGIN PGP PRIVATE KEY BLOCK-----\nAutocrypt-Prefer-Encrypt: mutual \n\naGVsbG8gd29ybGQ=\n-----END PGP PRIVATE KEY BLOCK-----"
+        )
+            .unwrap();
+
+        assert_eq!(typ, BlockType::PrivateKey);
+        assert!(!base64.is_empty());
+        assert_eq!(headers.get(HEADER_AUTOCRYPT), Some(&"mutual".to_string()));
+    }
+}

src/qr.rs

@@ -90,7 +90,8 @@ fn decode_openpgp(context: &Context, qr: &str) -> Lot {
 
     // what is up with that param name?
     let name = if let Some(encoded_name) = param.get(Param::SetLongitude) {
-        match percent_decode_str(encoded_name).decode_utf8() {
+        let encoded_name = encoded_name.replace("+", "%20"); // sometimes spaces are encoded as `+`
+        match percent_decode_str(&encoded_name).decode_utf8() {
             Ok(name) => name.to_string(),
             Err(err) => return format_err!("Invalid name: {}", err).into(),
         }
@@ -104,7 +105,8 @@ fn decode_openpgp(context: &Context, qr: &str) -> Lot {
 
     let grpname = if grpid.is_some() {
         if let Some(encoded_name) = param.get(Param::GroupName) {
-            match percent_decode_str(encoded_name).decode_utf8() {
+            let encoded_name = encoded_name.replace("+", "%20"); // sometimes spaces are encoded as `+`
+            match percent_decode_str(&encoded_name).decode_utf8() {
                 Ok(name) => Some(name.to_string()),
                 Err(err) => return format_err!("Invalid group name: {}", err).into(),
             }
@@ -148,7 +150,7 @@ fn decode_openpgp(context: &Context, qr: &str) -> Lot {
             let (id, _) = chat::create_or_lookup_by_contact_id(context, lot.id, Blocked::Deaddrop)
                 .unwrap_or_default();
 
-            chat::add_device_msg(
+            chat::add_info_msg(
                 context,
                 id,
                 format!("{} verified.", peerstate.addr.unwrap_or_default()),
@@ -186,7 +188,7 @@ fn decode_mailto(context: &Context, qr: &str) -> Lot {
     let addr = if let Some(query_index) = payload.find('?') {
         &payload[..query_index]
     } else {
-        return format_err!("Invalid mailto found").into();
+        payload
     };
 
     let addr = match normalize_address(addr) {
@@ -405,13 +407,21 @@ mod tests {
             &ctx.ctx,
             "mailto:stress@test.local?subject=hello&body=world",
         );
-
         println!("{:?}", res);
         assert_eq!(res.get_state(), LotState::QrAddr);
         assert_ne!(res.get_id(), 0);
-
         let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
         assert_eq!(contact.get_addr(), "stress@test.local");
+
+        let res = check_qr(&ctx.ctx, "mailto:no-questionmark@example.org");
+        assert_eq!(res.get_state(), LotState::QrAddr);
+        assert_ne!(res.get_id(), 0);
+        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
+        assert_eq!(contact.get_addr(), "no-questionmark@example.org");
+
+        let res = check_qr(&ctx.ctx, "mailto:no-addr");
+        assert_eq!(res.get_state(), LotState::QrError);
+        assert!(res.get_text1().is_some());
     }
 
     #[test]
@@ -434,12 +444,13 @@ mod tests {
 
         let res = check_qr(
             &ctx.ctx,
-            "OPENPGP4FPR:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&g=testtesttest&x=h-0oKQf2CDK&i=9JEXlxAqGM0&s=0V7LzL9cxRL"
+            "OPENPGP4FPR:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&g=test%20%3F+test%20%21&x=h-0oKQf2CDK&i=9JEXlxAqGM0&s=0V7LzL9cxRL"
         );
 
         println!("{:?}", res);
         assert_eq!(res.get_state(), LotState::QrAskVerifyGroup);
         assert_ne!(res.get_id(), 0);
+        assert_eq!(res.get_text1().unwrap(), "test ? test !");
 
         let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
         assert_eq!(contact.get_addr(), "cli@deltachat.de");
@@ -451,7 +462,7 @@ mod tests {
 
         let res = check_qr(
             &ctx.ctx,
-            "OPENPGP4FPR:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&n=&i=TbnwJ6lSvD5&s=0ejvbdFSQxB"
+            "OPENPGP4FPR:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&n=J%C3%B6rn%20P.+P.&i=TbnwJ6lSvD5&s=0ejvbdFSQxB"
         );
 
         println!("{:?}", res);
@@ -460,5 +471,6 @@ mod tests {
 
         let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
         assert_eq!(contact.get_addr(), "cli@deltachat.de");
+        assert_eq!(contact.get_name(), "Jörn P. P.");
     }
 }

src/securejoin.rs

@@ -3,7 +3,6 @@ use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC};
 use crate::aheader::EncryptPreference;
 use crate::chat::{self, Chat};
 use crate::config::*;
-use crate::configure::*;
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::Context;
@@ -82,10 +81,7 @@ pub fn dc_get_securejoin_qr(context: &Context, group_chat_id: u32) -> Option<Str
         }
     };
 
-    let self_name = context
-        .sql
-        .get_config(context, "displayname")
-        .unwrap_or_default();
+    let self_name = context.get_config(Config::Displayname).unwrap_or_default();
 
     fingerprint = match get_self_fingerprint(context) {
         Some(fp) => fp,
@@ -160,7 +156,7 @@ pub fn dc_join_securejoin(context: &Context, qr: &str) -> u32 {
             bob.qr_scan = None;
 
             if ongoing_allocated {
-                dc_free_ongoing(context);
+                context.free_ongoing();
             }
             ret_chat_id as u32
         };
@@ -173,7 +169,7 @@ pub fn dc_join_securejoin(context: &Context, qr: &str) -> u32 {
 
     info!(context, "Requesting secure-join ...",);
     ensure_secret_key_exists(context).ok();
-    if !dc_alloc_ongoing(context) {
+    if !context.alloc_ongoing() {
         return cleanup(&context, contact_chat_id, false, join_vg);
     }
     let qr_scan = check_qr(context, &qr);
@@ -187,7 +183,7 @@ pub fn dc_join_securejoin(context: &Context, qr: &str) -> u32 {
         error!(context, "Unknown contact.",);
         return cleanup(&context, contact_chat_id, true, join_vg);
     }
-    if check_exit(context) {
+    if context.shall_stop_ongoing() {
         return cleanup(&context, contact_chat_id, true, join_vg);
     }
     join_vg = qr_scan.get_state() == LotState::QrAskVerifyGroup;
@@ -213,7 +209,7 @@ pub fn dc_join_securejoin(context: &Context, qr: &str) -> u32 {
         info!(context, "Taking protocol shortcut.");
         context.bob.write().unwrap().expects = DC_VC_CONTACT_CONFIRM;
         joiner_progress!(context, chat_id_2_contact_id(context, contact_chat_id), 400);
-        let own_fingerprint = get_self_fingerprint(context).unwrap();
+        let own_fingerprint = get_self_fingerprint(context).unwrap_or_default();
         send_handshake_msg(
             context,
             contact_chat_id,
@@ -243,21 +239,12 @@ pub fn dc_join_securejoin(context: &Context, qr: &str) -> u32 {
     }
 
     // Bob -> Alice
-    while !check_exit(&context) {
+    while !context.shall_stop_ongoing() {
         std::thread::sleep(std::time::Duration::new(0, 3_000_000));
     }
     cleanup(&context, contact_chat_id, true, join_vg)
 }
 
-fn check_exit(context: &Context) -> bool {
-    context
-        .running_state
-        .clone()
-        .read()
-        .unwrap()
-        .shall_stop_ongoing
-}
-
 fn send_handshake_msg(
     context: &Context,
     contact_chat_id: u32,
@@ -270,7 +257,7 @@ fn send_handshake_msg(
     msg.type_0 = Viewtype::Text;
     msg.text = Some(format!("Secure-Join: {}", step));
     msg.hidden = true;
-    msg.param.set_int(Param::Cmd, 7);
+    msg.param.set_cmd(SystemMessage::SecurejoinMessage);
     if step.is_empty() {
         msg.param.remove(Param::Arg);
     } else {
@@ -291,10 +278,10 @@ fn send_handshake_msg(
             ForcePlaintext::AddAutocryptHeader as i32,
         );
     } else {
-        msg.param.set_int(Param::GuranteeE2ee, 1);
+        msg.param.set_int(Param::GuaranteeE2ee, 1);
     }
     // TODO. handle cleanup on error
-    chat::send_msg(context, contact_chat_id, &mut msg).unwrap();
+    chat::send_msg(context, contact_chat_id, &mut msg).unwrap_or_default();
 }
 
 fn chat_id_2_contact_id(context: &Context, contact_chat_id: u32) -> u32 {
@@ -639,7 +626,7 @@ pub fn handle_securejoin_handshake(
 
 fn end_bobs_joining(context: &Context, status: libc::c_int) {
     context.bob.write().unwrap().status = status;
-    dc_stop_ongoing_process(context);
+    context.stop_ongoing();
 }
 
 fn secure_connection_established(context: &Context, contact_chat_id: u32) {
@@ -651,7 +638,7 @@ fn secure_connection_established(context: &Context, contact_chat_id: u32) {
         "?"
     };
     let msg = context.stock_string_repl_str(StockMessage::ContactVerified, addr);
-    chat::add_device_msg(context, contact_chat_id, msg);
+    chat::add_info_msg(context, contact_chat_id, msg);
     emit_event!(context, Event::ChatModified(contact_chat_id));
 }
 
@@ -667,7 +654,7 @@ fn could_not_establish_secure_connection(context: &Context, contact_chat_id: u32
         },
     );
 
-    chat::add_device_msg(context, contact_chat_id, &msg);
+    chat::add_info_msg(context, contact_chat_id, &msg);
     error!(context, "{} ({})", &msg, details);
 }
 
@@ -678,7 +665,9 @@ fn mark_peer_as_verified(context: &Context, fingerprint: impl AsRef<str>) -> Res
         if peerstate.set_verified(1, fingerprint.as_ref(), 2) {
             peerstate.prefer_encrypt = EncryptPreference::Mutual;
             peerstate.to_save = Some(ToSave::All);
-            peerstate.save_to_db(&context.sql, false).unwrap();
+            peerstate
+                .save_to_db(&context.sql, false)
+                .unwrap_or_default();
             return Ok(());
         }
     }
@@ -696,7 +685,7 @@ fn encrypted_and_signed(mimeparser: &MimeParser, expected_fingerprint: impl AsRe
     if !mimeparser.encrypted {
         warn!(mimeparser.context, "Message not encrypted.",);
         false
-    } else if mimeparser.signatures.len() <= 0 {
+    } else if mimeparser.signatures.is_empty() {
         warn!(mimeparser.context, "Message not signed.",);
         false
     } else if expected_fingerprint.as_ref().is_empty() {
@@ -746,7 +735,7 @@ pub fn handle_degrade_event(context: &Context, peerstate: &Peerstate) -> Result<
             };
             let msg = context.stock_string_repl_str(StockMessage::ContactSetupChanged, peeraddr);
 
-            chat::add_device_msg(context, contact_chat_id, msg);
+            chat::add_info_msg(context, contact_chat_id, msg);
             emit_event!(context, Event::ChatModified(contact_chat_id));
         }
     }

src/smtp.rs

@@ -3,8 +3,9 @@ use lettre::*;
 
 use crate::constants::*;
 use crate::context::Context;
+use crate::error::Error;
 use crate::events::Event;
-use crate::login_param::LoginParam;
+use crate::login_param::{dc_build_tls, LoginParam};
 use crate::oauth2::*;
 
 #[derive(DebugStub)]
@@ -14,7 +15,6 @@ pub struct Smtp {
     transport_connected: bool,
     /// Email address we are sending from.
     from: Option<EmailAddress>,
-    pub error: Option<String>,
 }
 
 impl Smtp {
@@ -24,7 +24,6 @@ impl Smtp {
             transport: None,
             transport_connected: false,
             from: None,
-            error: None,
         }
     }
 
@@ -45,56 +44,59 @@ impl Smtp {
     }
 
     /// Connect using the provided login params
-    pub fn connect(&mut self, context: &Context, lp: &LoginParam) -> bool {
+    pub fn connect(&mut self, context: &Context, lp: &LoginParam) -> Result<(), Error> {
         if self.is_connected() {
             warn!(context, "SMTP already connected.");
-            return true;
+            return Ok(());
         }
 
         if lp.send_server.is_empty() || lp.send_port == 0 {
             context.call_cb(Event::ErrorNetwork("SMTP bad parameters.".into()));
+            bail!("SMTP Bad parameters");
         }
 
-        self.from = if let Ok(addr) = EmailAddress::new(lp.addr.clone()) {
-            Some(addr)
-        } else {
-            None
+        self.from = match EmailAddress::new(lp.addr.clone()) {
+            Ok(addr) => Some(addr),
+            Err(err) => {
+                bail!("invalid login address {}: {}", lp.addr, err);
+            }
         };
 
-        if self.from.is_none() {
-            // TODO: print error
-            return false;
-        }
-
         let domain = &lp.send_server;
         let port = lp.send_port as u16;
 
-        let tls = native_tls::TlsConnector::builder()
-            // see also: https://github.com/deltachat/deltachat-core-rust/issues/203
-            .danger_accept_invalid_hostnames(true)
-            .danger_accept_invalid_certs(true)
-            .min_protocol_version(Some(DEFAULT_TLS_PROTOCOLS[0]))
-            .build()
-            .unwrap();
-
+        let tls = dc_build_tls(lp.smtp_certificate_checks).unwrap();
         let tls_parameters = ClientTlsParameters::new(domain.to_string(), tls);
 
-        let creds = if 0 != lp.server_flags & (DC_LP_AUTH_OAUTH2 as i32) {
+        let (creds, mechanism) = if 0 != lp.server_flags & (DC_LP_AUTH_OAUTH2 as i32) {
             // oauth2
             let addr = &lp.addr;
             let send_pw = &lp.send_pw;
             let access_token = dc_get_oauth2_access_token(context, addr, send_pw, false);
-            if access_token.is_none() {
-                return false;
-            }
+            ensure!(
+                access_token.is_some(),
+                "could not get oaut2_access token addr={}",
+                addr
+            );
             let user = &lp.send_user;
-
-            lettre::smtp::authentication::Credentials::new(user.to_string(), access_token.unwrap())
+            (
+                lettre::smtp::authentication::Credentials::new(
+                    user.to_string(),
+                    access_token.unwrap_or_default(),
+                ),
+                vec![lettre::smtp::authentication::Mechanism::Xoauth2],
+            )
         } else {
             // plain
             let user = lp.send_user.clone();
             let pw = lp.send_pw.clone();
-            lettre::smtp::authentication::Credentials::new(user, pw)
+            (
+                lettre::smtp::authentication::Credentials::new(user, pw),
+                vec![
+                    lettre::smtp::authentication::Mechanism::Plain,
+                    lettre::smtp::authentication::Mechanism::Login,
+                ],
+            )
         };
 
         let security = if 0
@@ -110,52 +112,78 @@ impl Smtp {
                 let client = client
                     .smtp_utf8(true)
                     .credentials(creds)
+                    .authentication_mechanism(mechanism)
                     .connection_reuse(lettre::smtp::ConnectionReuseParameters::ReuseUnlimited);
-                self.transport = Some(client.transport());
-                context.call_cb(Event::SmtpConnected(format!(
-                    "SMTP-LOGIN as {} ok",
-                    lp.send_user,
-                )));
-                true
+                let mut trans = client.transport();
+                match trans.connect() {
+                    Ok(()) => {
+                        self.transport = Some(trans);
+                        self.transport_connected = true;
+                        context.call_cb(Event::SmtpConnected(format!(
+                            "SMTP-LOGIN as {} ok",
+                            lp.send_user,
+                        )));
+                        return Ok(());
+                    }
+                    Err(err) => {
+                        bail!("SMTP: failed to connect {:?}", err);
+                    }
+                }
             }
             Err(err) => {
-                warn!(context, "SMTP: failed to establish connection {:?}", err);
-                false
+                bail!("SMTP: failed to setup connection {:?}", err);
             }
         }
     }
 
+    /// SMTP-Send a prepared mail to recipients.
+    /// on successful send out Ok() is returned.
     pub fn send<'a>(
         &mut self,
         context: &Context,
         recipients: Vec<EmailAddress>,
-        body: Vec<u8>,
-    ) -> usize {
+        message: Vec<u8>,
+        job_id: u32,
+    ) -> Result<(), Error> {
+        let message_len = message.len();
+
+        let recipients_display = recipients
+            .iter()
+            .map(|x| format!("{}", x))
+            .collect::<Vec<String>>()
+            .join(",");
+
         if let Some(ref mut transport) = self.transport {
-            let envelope = Envelope::new(self.from.clone(), recipients).expect("invalid envelope");
+            let envelope = match Envelope::new(self.from.clone(), recipients) {
+                Ok(env) => env,
+                Err(err) => {
+                    bail!("{}", err);
+                }
+            };
             let mail = SendableEmail::new(
                 envelope,
-                "mail-id".into(), // TODO: random id
-                body,
+                format!("{}", job_id), // only used for internal logging
+                message,
             );
 
             match transport.send(mail) {
                 Ok(_) => {
-                    context.call_cb(Event::SmtpMessageSent(
-                        "Message was sent to SMTP server".into(),
-                    ));
+                    context.call_cb(Event::SmtpMessageSent(format!(
+                        "Message len={} was smtp-sent to {}",
+                        message_len, recipients_display
+                    )));
                     self.transport_connected = true;
-                    1
+                    Ok(())
                 }
                 Err(err) => {
-                    warn!(context, "SMTP failed to send message: {}", err);
-                    self.error = Some(format!("{}", err));
-                    0
+                    bail!("SMTP failed len={}: error: {}", message_len, err);
                 }
             }
         } else {
-            // TODO: log error
-            0
+            bail!(
+                "uh? SMTP has no transport,  failed to send to {:?}",
+                recipients_display
+            );
         }
     }
 }

src/sql.rs

@@ -1,5 +1,6 @@
 use std::collections::HashSet;
 use std::sync::{Arc, RwLock};
+use std::time::Duration;
 
 use rusqlite::{Connection, OpenFlags, Statement, NO_PARAMS};
 use thread_local_object::ThreadLocal;
@@ -10,8 +11,6 @@ use crate::error::{Error, Result};
 use crate::param::*;
 use crate::peerstate::*;
 
-const DC_OPEN_READONLY: usize = 0x01;
-
 /// A wrapper around the underlying Sqlite3 object.
 #[derive(DebugStub)]
 pub struct Sql {
@@ -41,8 +40,8 @@ impl Sql {
     }
 
     // return true on success, false on failure
-    pub fn open(&self, context: &Context, dbfile: &std::path::Path, flags: libc::c_int) -> bool {
-        match open(context, self, dbfile, flags) {
+    pub fn open(&self, context: &Context, dbfile: &std::path::Path, readonly: bool) -> bool {
+        match open(context, self, dbfile, readonly) {
             Ok(_) => true,
             Err(Error::SqlAlreadyOpen) => false,
             Err(_) => {
@@ -68,6 +67,16 @@ impl Sql {
         let res = match &*self.pool.read().unwrap() {
             Some(pool) => {
                 let conn = pool.get()?;
+
+                // Only one process can make changes to the database at one time.
+                // busy_timeout defines, that if a seconds process wants write access,
+                // this second process will wait some milliseconds
+                // and try over until it gets write access or the given timeout is elapsed.
+                // If the second process does not get write access within the given timeout,
+                // sqlite3_step() will return the error SQLITE_BUSY.
+                // (without a busy_timeout, sqlite3_step() would return SQLITE_BUSY _at once_)
+                conn.busy_timeout(Duration::from_secs(10))?;
+
                 g(&conn)
             }
             None => Err(Error::SqlNoConnection),
@@ -83,8 +92,7 @@ impl Sql {
         self.start_stmt(sql.to_string());
         self.with_conn(|conn| {
             let stmt = conn.prepare(sql)?;
-            let res = g(stmt, conn)?;
-            Ok(res)
+            g(stmt, conn)
         })
     }
 
@@ -97,8 +105,7 @@ impl Sql {
             let stmt1 = conn.prepare(sql1)?;
             let stmt2 = conn.prepare(sql2)?;
 
-            let res = g(stmt1, stmt2, conn)?;
-            Ok(res)
+            g(stmt1, stmt2, conn)
         })
     }
 
@@ -182,14 +189,14 @@ impl Sql {
     ///
     /// Setting `None` deletes the value.  On failure an error message
     /// will already have been logged.
-    pub fn set_config(
+    pub fn set_raw_config(
         &self,
         context: &Context,
         key: impl AsRef<str>,
         value: Option<&str>,
     ) -> Result<()> {
         if !self.is_open() {
-            error!(context, "set_config(): Database not ready.");
+            error!(context, "set_raw_config(): Database not ready.");
             return Err(Error::SqlNoConnection);
         }
 
@@ -223,14 +230,14 @@ impl Sql {
         match res {
             Ok(_) => Ok(()),
             Err(err) => {
-                error!(context, "set_config(): Cannot change value. {:?}", &err);
-                Err(err.into())
+                error!(context, "set_raw_config(): Cannot change value. {:?}", &err);
+                Err(err)
             }
         }
     }
 
     /// Get configuration options from the database.
-    pub fn get_config(&self, context: &Context, key: impl AsRef<str>) -> Option<String> {
+    pub fn get_raw_config(&self, context: &Context, key: impl AsRef<str>) -> Option<String> {
         if !self.is_open() || key.as_ref().is_empty() {
             return None;
         }
@@ -241,44 +248,46 @@ impl Sql {
         )
     }
 
-    pub fn set_config_int(
+    pub fn set_raw_config_int(
         &self,
         context: &Context,
         key: impl AsRef<str>,
         value: i32,
     ) -> Result<()> {
-        self.set_config(context, key, Some(&format!("{}", value)))
+        self.set_raw_config(context, key, Some(&format!("{}", value)))
     }
 
-    pub fn get_config_int(&self, context: &Context, key: impl AsRef<str>) -> Option<i32> {
-        self.get_config(context, key).and_then(|s| s.parse().ok())
+    pub fn get_raw_config_int(&self, context: &Context, key: impl AsRef<str>) -> Option<i32> {
+        self.get_raw_config(context, key)
+            .and_then(|s| s.parse().ok())
     }
 
-    pub fn get_config_bool(&self, context: &Context, key: impl AsRef<str>) -> bool {
+    pub fn get_raw_config_bool(&self, context: &Context, key: impl AsRef<str>) -> bool {
         // Not the most obvious way to encode bool as string, but it is matter
         // of backward compatibility.
-        self.get_config_int(context, key).unwrap_or_default() > 0
+        self.get_raw_config_int(context, key).unwrap_or_default() > 0
     }
 
-    pub fn set_config_bool<T>(&self, context: &Context, key: T, value: bool) -> Result<()>
+    pub fn set_raw_config_bool<T>(&self, context: &Context, key: T, value: bool) -> Result<()>
     where
         T: AsRef<str>,
     {
         let value = if value { Some("1") } else { None };
-        self.set_config(context, key, value)
+        self.set_raw_config(context, key, value)
     }
 
-    pub fn set_config_int64(
+    pub fn set_raw_config_int64(
         &self,
         context: &Context,
         key: impl AsRef<str>,
         value: i64,
     ) -> Result<()> {
-        self.set_config(context, key, Some(&format!("{}", value)))
+        self.set_raw_config(context, key, Some(&format!("{}", value)))
     }
 
-    pub fn get_config_int64(&self, context: &Context, key: impl AsRef<str>) -> Option<i64> {
-        self.get_config(context, key).and_then(|r| r.parse().ok())
+    pub fn get_raw_config_int64(&self, context: &Context, key: impl AsRef<str>) -> Option<i64> {
+        self.get_raw_config(context, key)
+            .and_then(|r| r.parse().ok())
     }
 
     fn start_stmt(&self, stmt: impl AsRef<str>) {
@@ -307,7 +316,7 @@ fn open(
     context: &Context,
     sql: &Sql,
     dbfile: impl AsRef<std::path::Path>,
-    flags: libc::c_int,
+    readonly: bool,
 ) -> Result<()> {
     if sql.is_open() {
         error!(
@@ -319,7 +328,7 @@ fn open(
     }
 
     let mut open_flags = OpenFlags::SQLITE_OPEN_NO_MUTEX;
-    if 0 != (flags & DC_OPEN_READONLY as i32) {
+    if readonly {
         open_flags.insert(OpenFlags::SQLITE_OPEN_READ_ONLY);
     } else {
         open_flags.insert(OpenFlags::SQLITE_OPEN_READ_WRITE);
@@ -330,7 +339,7 @@ fn open(
         .with_init(|c| c.execute_batch("PRAGMA secure_delete=on;"));
     let pool = r2d2::Pool::builder()
         .min_idle(Some(2))
-        .max_size(4)
+        .max_size(10)
         .connection_timeout(std::time::Duration::new(60, 0))
         .build(mgr)?;
 
@@ -338,7 +347,7 @@ fn open(
         *sql.pool.write().unwrap() = Some(pool);
     }
 
-    if 0 == flags & DC_OPEN_READONLY as i32 {
+    if !readonly {
         let mut exists_before_update = 0;
         let mut dbversion_before_update = 0;
         /* Init tables to dbversion=0 */
@@ -374,8 +383,8 @@ fn open(
             )?;
             sql.execute(
                 "INSERT INTO contacts (id,name,origin) VALUES \
-                 (1,'self',262144), (2,'device',262144), (3,'rsvd',262144), \
-                 (4,'rsvd',262144), (5,'rsvd',262144), (6,'rsvd',262144), \
+                 (1,'self',262144), (2,'info',262144), (3,'rsvd',262144), \
+                 (4,'rsvd',262144), (5,'device',262144), (6,'rsvd',262144), \
                  (7,'rsvd',262144), (8,'rsvd',262144), (9,'rsvd',262144);",
                 params![],
             )?;
@@ -466,11 +475,13 @@ fn open(
                 // cannot create the tables - maybe we cannot write?
                 return Err(Error::SqlFailedToOpen);
             } else {
-                sql.set_config_int(context, "dbversion", 0)?;
+                sql.set_raw_config_int(context, "dbversion", 0)?;
             }
         } else {
             exists_before_update = 1;
-            dbversion_before_update = sql.get_config_int(context, "dbversion").unwrap_or_default();
+            dbversion_before_update = sql
+                .get_raw_config_int(context, "dbversion")
+                .unwrap_or_default();
         }
 
         // (1) update low-level database structure.
@@ -482,6 +493,7 @@ fn open(
         let mut update_file_paths = 0;
 
         if dbversion < 1 {
+            info!(context, "[migration] v1");
             sql.execute(
                 "CREATE TABLE leftgrps ( id INTEGER PRIMARY KEY, grpid TEXT DEFAULT '');",
                 params![],
@@ -491,17 +503,19 @@ fn open(
                 params![],
             )?;
             dbversion = 1;
-            sql.set_config_int(context, "dbversion", 1)?;
+            sql.set_raw_config_int(context, "dbversion", 1)?;
         }
         if dbversion < 2 {
+            info!(context, "[migration] v2");
             sql.execute(
                 "ALTER TABLE contacts ADD COLUMN authname TEXT DEFAULT '';",
                 params![],
             )?;
             dbversion = 2;
-            sql.set_config_int(context, "dbversion", 2)?;
+            sql.set_raw_config_int(context, "dbversion", 2)?;
         }
         if dbversion < 7 {
+            info!(context, "[migration] v7");
             sql.execute(
                 "CREATE TABLE keypairs (\
                  id INTEGER PRIMARY KEY, \
@@ -513,9 +527,10 @@ fn open(
                 params![],
             )?;
             dbversion = 7;
-            sql.set_config_int(context, "dbversion", 7)?;
+            sql.set_raw_config_int(context, "dbversion", 7)?;
         }
         if dbversion < 10 {
+            info!(context, "[migration] v10");
             sql.execute(
                 "CREATE TABLE acpeerstates (\
                  id INTEGER PRIMARY KEY, \
@@ -531,9 +546,10 @@ fn open(
                 params![],
             )?;
             dbversion = 10;
-            sql.set_config_int(context, "dbversion", 10)?;
+            sql.set_raw_config_int(context, "dbversion", 10)?;
         }
         if dbversion < 12 {
+            info!(context, "[migration] v12");
             sql.execute(
                 "CREATE TABLE msgs_mdns ( msg_id INTEGER,  contact_id INTEGER);",
                 params![],
@@ -543,9 +559,10 @@ fn open(
                 params![],
             )?;
             dbversion = 12;
-            sql.set_config_int(context, "dbversion", 12)?;
+            sql.set_raw_config_int(context, "dbversion", 12)?;
         }
         if dbversion < 17 {
+            info!(context, "[migration] v17");
             sql.execute(
                 "ALTER TABLE chats ADD COLUMN archived INTEGER DEFAULT 0;",
                 params![],
@@ -557,18 +574,20 @@ fn open(
             )?;
             sql.execute("CREATE INDEX msgs_index5 ON msgs (starred);", params![])?;
             dbversion = 17;
-            sql.set_config_int(context, "dbversion", 17)?;
+            sql.set_raw_config_int(context, "dbversion", 17)?;
         }
         if dbversion < 18 {
+            info!(context, "[migration] v18");
             sql.execute(
                 "ALTER TABLE acpeerstates ADD COLUMN gossip_timestamp INTEGER DEFAULT 0;",
                 params![],
             )?;
             sql.execute("ALTER TABLE acpeerstates ADD COLUMN gossip_key;", params![])?;
             dbversion = 18;
-            sql.set_config_int(context, "dbversion", 18)?;
+            sql.set_raw_config_int(context, "dbversion", 18)?;
         }
         if dbversion < 27 {
+            info!(context, "[migration] v27");
             sql.execute("DELETE FROM msgs WHERE chat_id=1 OR chat_id=2;", params![])?;
             sql.execute(
                 "CREATE INDEX chats_contacts_index2 ON chats_contacts (contact_id);",
@@ -583,9 +602,10 @@ fn open(
                 params![],
             )?;
             dbversion = 27;
-            sql.set_config_int(context, "dbversion", 27)?;
+            sql.set_raw_config_int(context, "dbversion", 27)?;
         }
         if dbversion < 34 {
+            info!(context, "[migration] v34");
             sql.execute(
                 "ALTER TABLE msgs ADD COLUMN hidden INTEGER DEFAULT 0;",
                 params![],
@@ -612,9 +632,10 @@ fn open(
             )?;
             recalc_fingerprints = 1;
             dbversion = 34;
-            sql.set_config_int(context, "dbversion", 34)?;
+            sql.set_raw_config_int(context, "dbversion", 34)?;
         }
         if dbversion < 39 {
+            info!(context, "[migration] v39");
             sql.execute(
                 "CREATE TABLE tokens ( id INTEGER PRIMARY KEY, namespc INTEGER DEFAULT 0, foreign_id INTEGER DEFAULT 0, token TEXT DEFAULT '', timestamp INTEGER DEFAULT 0);",
                 params![]
@@ -642,32 +663,37 @@ fn open(
                 )?;
             }
             dbversion = 39;
-            sql.set_config_int(context, "dbversion", 39)?;
+            sql.set_raw_config_int(context, "dbversion", 39)?;
         }
         if dbversion < 40 {
+            info!(context, "[migration] v40");
             sql.execute(
                 "ALTER TABLE jobs ADD COLUMN thread INTEGER DEFAULT 0;",
                 params![],
             )?;
             dbversion = 40;
-            sql.set_config_int(context, "dbversion", 40)?;
+            sql.set_raw_config_int(context, "dbversion", 40)?;
         }
         if dbversion < 41 {
+            info!(context, "[migration] v41");
             update_file_paths = 1;
             dbversion = 41;
-            sql.set_config_int(context, "dbversion", 41)?;
+            sql.set_raw_config_int(context, "dbversion", 41)?;
         }
         if dbversion < 42 {
+            info!(context, "[migration] v42");
             sql.execute("UPDATE msgs SET txt='' WHERE type!=10", params![])?;
             dbversion = 42;
-            sql.set_config_int(context, "dbversion", 42)?;
+            sql.set_raw_config_int(context, "dbversion", 42)?;
         }
         if dbversion < 44 {
+            info!(context, "[migration] v44");
             sql.execute("ALTER TABLE msgs ADD COLUMN mime_headers TEXT;", params![])?;
             dbversion = 44;
-            sql.set_config_int(context, "dbversion", 44)?;
+            sql.set_raw_config_int(context, "dbversion", 44)?;
         }
         if dbversion < 46 {
+            info!(context, "[migration] v46");
             sql.execute(
                 "ALTER TABLE msgs ADD COLUMN mime_in_reply_to TEXT;",
                 params![],
@@ -677,7 +703,7 @@ fn open(
                 params![],
             )?;
             dbversion = 46;
-            sql.set_config_int(context, "dbversion", 46)?;
+            sql.set_raw_config_int(context, "dbversion", 46)?;
         }
         if dbversion < 47 {
             info!(context, "[migration] v47");
@@ -686,7 +712,7 @@ fn open(
                 params![],
             )?;
             dbversion = 47;
-            sql.set_config_int(context, "dbversion", 47)?;
+            sql.set_raw_config_int(context, "dbversion", 47)?;
         }
         if dbversion < 48 {
             info!(context, "[migration] v48");
@@ -696,7 +722,7 @@ fn open(
             )?;
 
             dbversion = 48;
-            sql.set_config_int(context, "dbversion", 48)?;
+            sql.set_raw_config_int(context, "dbversion", 48)?;
         }
         if dbversion < 49 {
             info!(context, "[migration] v49");
@@ -705,15 +731,15 @@ fn open(
                 params![],
             )?;
             dbversion = 49;
-            sql.set_config_int(context, "dbversion", 49)?;
+            sql.set_raw_config_int(context, "dbversion", 49)?;
         }
         if dbversion < 50 {
             info!(context, "[migration] v50");
             if 0 != exists_before_update {
-                sql.set_config_int(context, "show_emails", 2)?;
+                sql.set_raw_config_int(context, "show_emails", 2)?;
             }
             dbversion = 50;
-            sql.set_config_int(context, "dbversion", 50)?;
+            sql.set_raw_config_int(context, "dbversion", 50)?;
         }
         if dbversion < 53 {
             info!(context, "[migration] v53");
@@ -746,7 +772,7 @@ fn open(
                 params![],
             )?;
             dbversion = 53;
-            sql.set_config_int(context, "dbversion", 53)?;
+            sql.set_raw_config_int(context, "dbversion", 53)?;
         }
         if dbversion < 54 {
             info!(context, "[migration] v54");
@@ -756,18 +782,20 @@ fn open(
             )?;
             sql.execute("CREATE INDEX msgs_index6 ON msgs (location_id);", params![])?;
             dbversion = 54;
-            sql.set_config_int(context, "dbversion", 54)?;
+            sql.set_raw_config_int(context, "dbversion", 54)?;
         }
         if dbversion < 55 {
+            info!(context, "[migration] v55");
             sql.execute(
                 "ALTER TABLE locations ADD COLUMN independent INTEGER DEFAULT 0;",
                 params![],
             )?;
 
-            sql.set_config_int(context, "dbversion", 55)?;
+            sql.set_raw_config_int(context, "dbversion", 55)?;
         }
 
         if 0 != recalc_fingerprints {
+            info!(context, "[migration] recalc fingerprints");
             sql.query_map(
                 "SELECT addr FROM acpeerstates;",
                 params![],
@@ -777,7 +805,7 @@ fn open(
                         if let Some(ref mut peerstate) = Peerstate::from_addr(context, sql, &addr?)
                         {
                             peerstate.recalc_fingerprint();
-                            peerstate.save_to_db(sql, false).unwrap();
+                            peerstate.save_to_db(sql, false)?;
                         }
                     }
                     Ok(())
@@ -788,11 +816,9 @@ fn open(
             // versions before 2018-08 save the absolute paths in the database files at "param.f=";
             // for newer versions, we copy files always to the blob directory and store relative paths.
             // this snippet converts older databases and can be removed after some time.
-
-            info!(context, "[open] update file paths");
-
+            info!(context, "[migration] update file paths");
             let repl_from = sql
-                .get_config(context, "backup_for")
+                .get_raw_config(context, "backup_for")
                 .unwrap_or_else(|| context.get_blobdir().to_string_lossy().into());
 
             let repl_from = dc_ensure_no_slash_safe(&repl_from);
@@ -812,7 +838,7 @@ fn open(
                 NO_PARAMS,
             )?;
 
-            sql.set_config(context, "backup_for", None)?;
+            sql.set_raw_config(context, "backup_for", None)?;
         }
     }
 
@@ -1133,12 +1159,8 @@ mod test {
         maybe_add_file(&mut files, "$BLOBDIR/world.txt");
         maybe_add_file(&mut files, "world2.txt");
 
-        assert!(is_file_in_use(&mut files, None, "hello"));
-        assert!(!is_file_in_use(&mut files, Some(".txt"), "hello"));
-        assert!(is_file_in_use(
-            &mut files,
-            Some("-suffix"),
-            "world.txt-suffix"
-        ));
+        assert!(is_file_in_use(&files, None, "hello"));
+        assert!(!is_file_in_use(&files, Some(".txt"), "hello"));
+        assert!(is_file_in_use(&files, Some("-suffix"), "world.txt-suffix"));
     }
 }

src/stock.rs

@@ -5,8 +5,7 @@ use strum_macros::EnumProperty;
 
 use crate::contact::*;
 use crate::context::Context;
-use crate::dc_tools::*;
-use crate::events::Event;
+use crate::error::Error;
 
 /// Stock strings
 ///
@@ -97,7 +96,7 @@ pub enum StockMessage {
     SelfTalkSubTitle = 50,
     #[strum(props(fallback = "Cannot login as %1$s."))]
     CannotLogin = 60,
-    #[strum(props(fallback = "Response from %1$s: %2$s"))]
+    #[strum(props(fallback = "Could not connect to %1$s: %2$s"))]
     ServerResponse = 61,
     #[strum(props(fallback = "%1$s by %2$s."))]
     MsgActionByUser = 62,
@@ -109,6 +108,10 @@ pub enum StockMessage {
     MsgLocationDisabled = 65,
     #[strum(props(fallback = "Location"))]
     Location = 66,
+    #[strum(props(fallback = "Sticker"))]
+    Sticker = 67,
+    #[strum(props(fallback = "Device Messages"))]
+    DeviceMessages = 68,
 }
 
 impl StockMessage {
@@ -116,24 +119,52 @@ impl StockMessage {
     ///
     /// These could be used in logging calls, so no logging here.
     fn fallback(&self) -> &'static str {
-        self.get_str("fallback").unwrap()
+        self.get_str("fallback").unwrap_or_default()
     }
 }
 
 impl Context {
+    /// Set the stock string for the [StockMessage].
+    ///
+    pub fn set_stock_translation(
+        &self,
+        id: StockMessage,
+        stockstring: String,
+    ) -> Result<(), Error> {
+        if stockstring.contains("%1") && !id.fallback().contains("%1") {
+            bail!(
+                "translation {} contains invalid %1 placeholder, default is {}",
+                stockstring,
+                id.fallback()
+            );
+        }
+        if stockstring.contains("%2") && !id.fallback().contains("%2") {
+            bail!(
+                "translation {} contains invalid %2 placeholder, default is {}",
+                stockstring,
+                id.fallback()
+            );
+        }
+        self.translated_stockstrings
+            .write()
+            .unwrap()
+            .insert(id as usize, stockstring);
+        Ok(())
+    }
+
     /// Return the stock string for the [StockMessage].
     ///
-    /// If the context callback responds with a string to use, e.g. a
-    /// translation, then this string will be returned.  Otherwise a
-    /// default (English) string is returned.
+    /// Return a translation (if it was set with set_stock_translation before)
+    /// or a default (English) string.
     pub fn stock_str(&self, id: StockMessage) -> Cow<str> {
-        let ptr = self.call_cb(Event::GetString { id, count: 0 }) as *mut libc::c_char;
-        if ptr.is_null() {
-            Cow::Borrowed(id.fallback())
-        } else {
-            let ret = to_string(ptr);
-            unsafe { libc::free(ptr as *mut libc::c_void) };
-            Cow::Owned(ret)
+        match self
+            .translated_stockstrings
+            .read()
+            .unwrap()
+            .get(&(id as usize))
+        {
+            Some(ref x) => Cow::Owned(x.to_string()),
+            None => Cow::Borrowed(id.fallback()),
         }
     }
 
@@ -163,7 +194,7 @@ impl Context {
     /// placeholders with the string in `insert` and does the same for
     /// `%2$s`, `%2$d` and `%2$@` for `insert2`.
     /// (the `%1$@` variant is used on iOS, the other are used on Android and Desktop)
-    fn stock_string_repl_str2(
+    pub fn stock_string_repl_str2(
         &self,
         id: StockMessage,
         insert: impl AsRef<str>,
@@ -237,7 +268,6 @@ mod tests {
     use crate::test_utils::*;
 
     use crate::constants::DC_CONTACT_ID_SELF;
-    use libc::uintptr_t;
 
     use num_traits::ToPrimitive;
 
@@ -253,25 +283,31 @@ mod tests {
     }
 
     #[test]
-    fn test_stock_str() {
+    fn test_set_stock_translation() {
         let t = dummy_context();
-        assert_eq!(t.ctx.stock_str(StockMessage::NoMessages), "No messages.");
-    }
-
-    fn test_stock_str_no_fallback_cb(_ctx: &Context, evt: Event) -> uintptr_t {
-        match evt {
-            Event::GetString {
-                id: StockMessage::NoMessages,
-                ..
-            } => unsafe { "Hello there".strdup() as usize },
-            _ => 0,
-        }
+        t.ctx
+            .set_stock_translation(StockMessage::NoMessages, "xyz".to_string())
+            .unwrap();
+        assert_eq!(t.ctx.stock_str(StockMessage::NoMessages), "xyz")
     }
 
     #[test]
-    fn test_stock_str_no_fallback() {
-        let t = test_context(Some(Box::new(test_stock_str_no_fallback_cb)));
-        assert_eq!(t.ctx.stock_str(StockMessage::NoMessages), "Hello there");
+    fn test_set_stock_translation_wrong_replacements() {
+        let t = dummy_context();
+        assert!(t
+            .ctx
+            .set_stock_translation(StockMessage::NoMessages, "xyz %1$s ".to_string())
+            .is_err());
+        assert!(t
+            .ctx
+            .set_stock_translation(StockMessage::NoMessages, "xyz %2$s ".to_string())
+            .is_err());
+    }
+
+    #[test]
+    fn test_stock_str() {
+        let t = dummy_context();
+        assert_eq!(t.ctx.stock_str(StockMessage::NoMessages), "No messages.");
     }
 
     #[test]
@@ -300,7 +336,7 @@ mod tests {
         assert_eq!(
             t.ctx
                 .stock_string_repl_str2(StockMessage::ServerResponse, "foo", "bar"),
-            "Response from foo: bar"
+            "Could not connect to foo: bar"
         );
     }
 
@@ -349,9 +385,7 @@ mod tests {
         let contact_id = {
             Contact::create(&t.ctx, "Alice", "alice@example.com")
                 .expect("Failed to create contact Alice");
-            let id =
-                Contact::create(&t.ctx, "Bob", "bob@example.com").expect("failed to create bob");
-            id
+            Contact::create(&t.ctx, "Bob", "bob@example.com").expect("failed to create bob")
         };
         assert_eq!(
             t.ctx.stock_system_msg(

src/top_evil_rs.py

@@ -14,22 +14,27 @@ if __name__ == "__main__":
         s = re.sub(r"(?m)///.*$", "", s)  # remove comments
         unsafe = s.count("unsafe")
         free = s.count("free(")
-        gotoblocks = s.count("ok_to_continue") + s.count('OK_TO_CONTINUE')
+        unsafe_fn = s.count("unsafe fn")
         chars = s.count("c_char") + s.count("CStr")
-        filestats.append((fn, unsafe, free, gotoblocks, chars))
+        libc = s.count("libc")
+        filestats.append((fn, unsafe, free, unsafe_fn, chars, libc))
 
-    sum_unsafe, sum_free, sum_gotoblocks, sum_chars = 0, 0, 0, 0
+    sum_unsafe, sum_free, sum_unsafe_fn, sum_chars, sum_libc = 0, 0, 0, 0, 0
 
-    for fn, unsafe, free, gotoblocks, chars in reversed(sorted(filestats, key=lambda x: sum(x[1:]))):
-        print("{0: <25} unsafe: {1: >3} free: {2: >3} ok_to_cont: {3: >3} chars: {4: >3}".format(str(fn), unsafe, free, gotoblocks, chars))
+    for fn, unsafe, free, unsafe_fn, chars, libc in reversed(sorted(filestats, key=lambda x: sum(x[1:]))):
+        if unsafe + free + unsafe_fn + chars + libc == 0:
+            continue
+        print("{0: <25} unsafe: {1: >3} free: {2: >3} unsafe-fn: {3: >3} chars: {4: >3} libc: {5: >3}".format(str(fn), unsafe, free, unsafe_fn, chars, libc))
         sum_unsafe += unsafe
         sum_free += free
-        sum_gotoblocks += gotoblocks
+        sum_unsafe_fn += unsafe_fn
         sum_chars += chars
+        sum_libc += libc
 
 
     print()
     print("total unsafe:", sum_unsafe)
     print("total free:", sum_free)
-    print("total ok_to_continue:", sum_gotoblocks)
+    print("total unsafe-fn:", sum_unsafe_fn)
     print("total c_chars:", sum_chars)
+    print("total libc:", sum_libc)

src/wrapmime.rs

@@ -1,9 +1,14 @@
+use std::collections::HashSet;
 use std::ffi::CString;
 use std::ptr;
 
+use crate::contact::addr_normalize;
+use crate::dc_strencode::*;
 use crate::dc_tools::*;
 use crate::error::Error;
 use mmime::clist::*;
+// use mmime::display::*;
+use mmime::mailimf::mailimf_msg_id_parse;
 use mmime::mailimf::types::*;
 use mmime::mailimf::types_helper::*;
 use mmime::mailmime::content::*;
@@ -11,6 +16,7 @@ use mmime::mailmime::disposition::*;
 use mmime::mailmime::types::*;
 use mmime::mailmime::types_helper::*;
 use mmime::mailmime::*;
+use mmime::mmapstring::*;
 use mmime::other::*;
 
 #[macro_export]
@@ -36,13 +42,30 @@ pub fn get_ct_subtype(mime: *mut Mailmime) -> Option<String> {
         let ct: *mut mailmime_content = (*mime).mm_content_type;
 
         if !ct.is_null() && !(*ct).ct_subtype.is_null() {
-            Some(to_string((*ct).ct_subtype))
+            Some(to_string_lossy((*ct).ct_subtype))
         } else {
             None
         }
     }
 }
 
+pub fn parse_message_id(message_id: &str) -> Result<String, Error> {
+    let mut dummy = 0;
+    let c_message_id = CString::new(message_id).unwrap_or_default();
+    let c_ptr = c_message_id.as_ptr();
+    let mut rfc724_mid_c = std::ptr::null_mut();
+    if unsafe { mailimf_msg_id_parse(c_ptr, libc::strlen(c_ptr), &mut dummy, &mut rfc724_mid_c) }
+        == MAIL_NO_ERROR as libc::c_int
+        && !rfc724_mid_c.is_null()
+    {
+        let res = to_string_lossy(rfc724_mid_c);
+        unsafe { libc::free(rfc724_mid_c.cast()) };
+        Ok(res)
+    } else {
+        bail!("could not parse message_id: {}", message_id);
+    }
+}
+
 pub fn get_autocrypt_mime(
     mime_undetermined: *mut Mailmime,
 ) -> Result<(*mut Mailmime, *mut Mailmime), Error> {
@@ -88,6 +111,193 @@ pub fn has_decryptable_data(mime_data: *mut mailmime_data) -> bool {
     }
 }
 
+pub fn get_field_from(imffields: *mut mailimf_fields) -> Result<String, Error> {
+    let field = mailimf_find_field(imffields, MAILIMF_FIELD_FROM as libc::c_int);
+    if !field.is_null() && unsafe { !(*field).fld_data.fld_from.is_null() } {
+        let mb_list = unsafe { (*(*field).fld_data.fld_from).frm_mb_list };
+        if let Some(addr) = mailimf_find_first_addr(mb_list) {
+            return Ok(addr);
+        }
+    }
+    bail!("not From field found");
+}
+
+pub fn get_field_date(imffields: *mut mailimf_fields) -> Result<i64, Error> {
+    let field = mailimf_find_field(imffields, MAILIMF_FIELD_ORIG_DATE as libc::c_int);
+    let mut message_time = 0;
+
+    if !field.is_null() && unsafe { !(*field).fld_data.fld_orig_date.is_null() } {
+        let orig_date = unsafe { (*field).fld_data.fld_orig_date };
+
+        if !orig_date.is_null() {
+            let dt = unsafe { (*orig_date).dt_date_time };
+            message_time = dc_timestamp_from_date(dt);
+            if message_time != 0 && message_time > time() {
+                message_time = time()
+            }
+        }
+    }
+
+    Ok(message_time)
+}
+
+fn mailimf_get_recipients_add_addr(recipients: &mut HashSet<String>, mb: *mut mailimf_mailbox) {
+    if !mb.is_null() {
+        let addr = to_string_lossy(unsafe { (*mb).mb_addr_spec });
+        let addr_norm = addr_normalize(&addr);
+        recipients.insert(addr_norm.into());
+    }
+}
+
+/*the result is a pointer to mime, must not be freed*/
+pub fn mailimf_find_field(
+    header: *mut mailimf_fields,
+    wanted_fld_type: libc::c_int,
+) -> *mut mailimf_field {
+    if header.is_null() {
+        return ptr::null_mut();
+    }
+
+    let header = unsafe { (*header) };
+    if header.fld_list.is_null() {
+        return ptr::null_mut();
+    }
+
+    for cur in unsafe { &(*header.fld_list) } {
+        let field = cur as *mut mailimf_field;
+        if !field.is_null() {
+            if unsafe { (*field).fld_type } == wanted_fld_type {
+                return field;
+            }
+        }
+    }
+
+    ptr::null_mut()
+}
+
+/*the result is a pointer to mime, must not be freed*/
+pub fn mailmime_find_mailimf_fields(mime: *mut Mailmime) -> *mut mailimf_fields {
+    if mime.is_null() {
+        return ptr::null_mut();
+    }
+
+    match unsafe { (*mime).mm_type as _ } {
+        MAILMIME_MULTIPLE => {
+            for cur_data in unsafe { (*(*mime).mm_data.mm_multipart.mm_mp_list).into_iter() } {
+                let header = mailmime_find_mailimf_fields(cur_data as *mut _);
+                if !header.is_null() {
+                    return header;
+                }
+            }
+        }
+        MAILMIME_MESSAGE => return unsafe { (*mime).mm_data.mm_message.mm_fields },
+        _ => {}
+    }
+
+    ptr::null_mut()
+}
+
+pub unsafe fn mailimf_find_optional_field(
+    header: *mut mailimf_fields,
+    wanted_fld_name: *const libc::c_char,
+) -> *mut mailimf_optional_field {
+    if header.is_null() || (*header).fld_list.is_null() {
+        return ptr::null_mut();
+    }
+    for cur_data in (*(*header).fld_list).into_iter() {
+        let field: *mut mailimf_field = cur_data as *mut _;
+
+        if (*field).fld_type == MAILIMF_FIELD_OPTIONAL_FIELD as libc::c_int {
+            let optional_field: *mut mailimf_optional_field = (*field).fld_data.fld_optional_field;
+            if !optional_field.is_null()
+                && !(*optional_field).fld_name.is_null()
+                && !(*optional_field).fld_value.is_null()
+                && strcasecmp((*optional_field).fld_name, wanted_fld_name) == 0i32
+            {
+                return optional_field;
+            }
+        }
+    }
+
+    ptr::null_mut()
+}
+
+pub fn mailimf_get_recipients(imffields: *mut mailimf_fields) -> HashSet<String> {
+    /* returned addresses are normalized. */
+    let mut recipients: HashSet<String> = Default::default();
+
+    for cur in unsafe { (*(*imffields).fld_list).into_iter() } {
+        let fld = cur as *mut mailimf_field;
+
+        let fld_to: *mut mailimf_to;
+        let fld_cc: *mut mailimf_cc;
+
+        let mut addr_list: *mut mailimf_address_list = ptr::null_mut();
+        if fld.is_null() {
+            continue;
+        }
+
+        let fld = unsafe { *fld };
+
+        // TODO match on enums /rtn
+        match fld.fld_type {
+            13 => {
+                fld_to = unsafe { fld.fld_data.fld_to };
+                if !fld_to.is_null() {
+                    addr_list = unsafe { (*fld_to).to_addr_list };
+                }
+            }
+            14 => {
+                fld_cc = unsafe { fld.fld_data.fld_cc };
+                if !fld_cc.is_null() {
+                    addr_list = unsafe { (*fld_cc).cc_addr_list };
+                }
+            }
+            _ => {}
+        }
+
+        if !addr_list.is_null() {
+            for cur2 in unsafe { &(*(*addr_list).ad_list) } {
+                let adr = cur2 as *mut mailimf_address;
+
+                if adr.is_null() {
+                    continue;
+                }
+                let adr = unsafe { *adr };
+
+                if adr.ad_type == MAILIMF_ADDRESS_MAILBOX as libc::c_int {
+                    mailimf_get_recipients_add_addr(&mut recipients, unsafe {
+                        adr.ad_data.ad_mailbox
+                    });
+                } else if adr.ad_type == MAILIMF_ADDRESS_GROUP as libc::c_int {
+                    let group = unsafe { adr.ad_data.ad_group };
+                    if !group.is_null() && unsafe { !(*group).grp_mb_list.is_null() } {
+                        for cur3 in unsafe { &(*(*(*group).grp_mb_list).mb_list) } {
+                            mailimf_get_recipients_add_addr(
+                                &mut recipients,
+                                cur3 as *mut mailimf_mailbox,
+                            );
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    recipients
+}
+
+pub fn mailmime_transfer_decode(mime: *mut Mailmime) -> Result<Vec<u8>, Error> {
+    ensure!(!mime.is_null(), "invalid inputs");
+
+    let mime_transfer_encoding =
+        get_mime_transfer_encoding(mime).unwrap_or(MAILMIME_MECHANISM_BINARY as i32);
+
+    let mime_data = unsafe { (*mime).mm_data.mm_single };
+
+    decode_dt_data(mime_data, mime_transfer_encoding)
+}
+
 pub fn get_mime_transfer_encoding(mime: *mut Mailmime) -> Option<libc::c_int> {
     unsafe {
         let mm_mime_fields = (*mime).mm_mime_fields;
@@ -108,48 +318,77 @@ pub fn get_mime_transfer_encoding(mime: *mut Mailmime) -> Option<libc::c_int> {
 pub fn decode_dt_data(
     mime_data: *mut mailmime_data,
     mime_transfer_encoding: libc::c_int,
-) -> Result<(*mut libc::c_char, libc::size_t), Error> {
+) -> Result<Vec<u8>, Error> {
     // Decode data according to mime_transfer_encoding
     // returns Ok with a (decoded_data,decoded_data_bytes) pointer
     // where the caller must make sure to free it.
     // It may return Ok(ptr::null_mut(), 0)
-
-    let mut transfer_decoding_buffer: *mut libc::c_char = ptr::null_mut();
-    let decoded_data: *mut libc::c_char;
-    let mut decoded_data_bytes: libc::size_t = 0;
     if mime_transfer_encoding == MAILMIME_MECHANISM_7BIT as libc::c_int
         || mime_transfer_encoding == MAILMIME_MECHANISM_8BIT as libc::c_int
         || mime_transfer_encoding == MAILMIME_MECHANISM_BINARY as libc::c_int
     {
-        unsafe {
-            decoded_data = (*mime_data).dt_data.dt_text.dt_data as *mut _;
-            decoded_data_bytes = (*mime_data).dt_data.dt_text.dt_length;
-        }
-        ensure!(
-            !decoded_data.is_null() && decoded_data_bytes > 0,
-            "could not decode mime message"
-        );
-    } else {
-        let mut current_index: libc::size_t = 0;
-        unsafe {
-            let r = mailmime_part_parse(
-                (*mime_data).dt_data.dt_text.dt_data,
-                (*mime_data).dt_data.dt_text.dt_length,
-                &mut current_index,
-                mime_transfer_encoding,
-                &mut transfer_decoding_buffer,
-                &mut decoded_data_bytes,
-            );
-            if r != MAILIMF_NO_ERROR as libc::c_int
-                || transfer_decoding_buffer.is_null()
-                || decoded_data_bytes <= 0
-            {
-                bail!("mailmime_part_parse returned error or invalid data");
-            }
-            decoded_data = transfer_decoding_buffer;
+        let decoded_data = unsafe { (*mime_data).dt_data.dt_text.dt_data };
+        let decoded_data_bytes = unsafe { (*mime_data).dt_data.dt_text.dt_length };
+
+        if decoded_data.is_null() || decoded_data_bytes == 0 {
+            bail!("No data to decode found");
+        } else {
+            let result = unsafe {
+                std::slice::from_raw_parts(decoded_data as *const u8, decoded_data_bytes)
+            };
+            return Ok(result.to_vec());
         }
     }
-    Ok((decoded_data, decoded_data_bytes))
+    // unsafe { display_mime_data(mime_data) };
+
+    let mut current_index = 0;
+    let mut transfer_decoding_buffer = ptr::null_mut();
+    let mut decoded_data_bytes = 0;
+
+    let r = unsafe {
+        mailmime_part_parse(
+            (*mime_data).dt_data.dt_text.dt_data,
+            (*mime_data).dt_data.dt_text.dt_length,
+            &mut current_index,
+            mime_transfer_encoding,
+            &mut transfer_decoding_buffer,
+            &mut decoded_data_bytes,
+        )
+    };
+
+    if r == MAILIMF_NO_ERROR as libc::c_int
+        && !transfer_decoding_buffer.is_null()
+        && decoded_data_bytes > 0
+    {
+        let result = unsafe {
+            std::slice::from_raw_parts(transfer_decoding_buffer as *const u8, decoded_data_bytes)
+        }
+        .to_vec();
+        // we return a fresh vec and transfer_decoding_buffer is not used or passed anywhere
+        // so it's safe to free it right away, as mailman_part_parse has
+        // allocated it fresh.
+        unsafe { mmap_string_unref(transfer_decoding_buffer) };
+
+        return Ok(result);
+    }
+
+    Err(format_err!("Failed to to decode"))
+}
+
+pub fn mailimf_find_first_addr(mb_list: *const mailimf_mailbox_list) -> Option<String> {
+    if mb_list.is_null() {
+        return None;
+    }
+
+    for cur in unsafe { (*(*mb_list).mb_list).into_iter() } {
+        let mb = cur as *mut mailimf_mailbox;
+        if !mb.is_null() && !unsafe { (*mb).mb_addr_spec.is_null() } {
+            let addr = unsafe { to_string_lossy((*mb).mb_addr_spec) };
+            return Some(addr_normalize(&addr).to_string());
+        }
+    }
+
+    None
 }
 
 /**************************************
@@ -210,8 +449,8 @@ pub fn append_ct_param(
     value: &str,
 ) -> Result<(), Error> {
     unsafe {
-        let name_c = CString::new(name).unwrap();
-        let value_c = CString::new(value).unwrap();
+        let name_c = CString::new(name).unwrap_or_default();
+        let value_c = CString::new(value).unwrap_or_default();
 
         clist_append!(
             (*content).ct_parameters,
@@ -225,7 +464,7 @@ pub fn append_ct_param(
 }
 
 pub fn new_content_type(content_type: &str) -> Result<*mut mailmime_content, Error> {
-    let ct = CString::new(content_type).unwrap();
+    let ct = CString::new(content_type).unwrap_or_default();
     let content: *mut mailmime_content;
     // mailmime_content_new_with_str only parses but does not retain/own ct
     unsafe {
@@ -251,7 +490,9 @@ pub fn content_type_needs_encoding(content: *const mailmime_content) -> bool {
         if (*(*content).ct_type).tp_type == MAILMIME_TYPE_COMPOSITE_TYPE as libc::c_int {
             let composite = (*(*content).ct_type).tp_data.tp_composite_type;
             match (*composite).ct_type as u32 {
-                MAILMIME_COMPOSITE_TYPE_MESSAGE => as_str((*content).ct_subtype) != "rfc822",
+                MAILMIME_COMPOSITE_TYPE_MESSAGE => {
+                    to_string_lossy((*content).ct_subtype) != "rfc822"
+                }
                 MAILMIME_COMPOSITE_TYPE_MULTIPART => false,
                 _ => false,
             }
@@ -261,6 +502,24 @@ pub fn content_type_needs_encoding(content: *const mailmime_content) -> bool {
     }
 }
 
+pub fn new_mailbox_list(displayname: &str, addr: &str) -> *mut mailimf_mailbox_list {
+    let mbox: *mut mailimf_mailbox_list = unsafe { mailimf_mailbox_list_new_empty() };
+    unsafe {
+        mailimf_mailbox_list_add(
+            mbox,
+            mailimf_mailbox_new(
+                if !displayname.is_empty() {
+                    dc_encode_header_words(&displayname).strdup()
+                } else {
+                    ptr::null_mut()
+                },
+                addr.strdup(),
+            ),
+        );
+    }
+    mbox
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -280,4 +539,16 @@ mod tests {
             new_content_type("application/pgp-encrypted").unwrap()
         ));
     }
+
+    #[test]
+    fn test_parse_message_id() {
+        assert_eq!(
+            parse_message_id("Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org").unwrap(),
+            "Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org"
+        );
+        assert_eq!(
+            parse_message_id("<Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org>").unwrap(),
+            "Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org"
+        );
+    }
 }

tests/stress.rs

@@ -1,18 +1,15 @@
 //! Stress some functions for testing; if used as a lib, this file is obsolete.
 
 use std::collections::HashSet;
-use std::ptr;
 
 use deltachat::chat::{self, Chat};
 use deltachat::config;
 use deltachat::contact::*;
 use deltachat::context::*;
-use deltachat::dc_tools::*;
 use deltachat::keyring::*;
 use deltachat::oauth2::*;
-use deltachat::pgp::*;
+use deltachat::pgp;
 use deltachat::Event;
-use libc::{free, strcmp, strdup};
 use tempfile::{tempdir, TempDir};
 
 /* some data used for testing
@@ -50,135 +47,6 @@ unsafe fn stress_functions(context: &Context) {
     assert!(res.contains(" configured_send_port "));
     assert!(res.contains(" configured_server_flags "));
 
-    let mut buf_0: *mut libc::c_char;
-    let mut headerline = String::default();
-    let mut setupcodebegin: *const libc::c_char = ptr::null();
-    let mut preferencrypt: *const libc::c_char = ptr::null();
-    let mut base64: *const libc::c_char = ptr::null();
-    buf_0 = strdup(
-        b"-----BEGIN PGP MESSAGE-----\nNoVal:\n\ndata\n-----END PGP MESSAGE-----\x00" as *const u8
-            as *const libc::c_char,
-    );
-    let ok = dc_split_armored_data(
-        buf_0,
-        &mut headerline,
-        &mut setupcodebegin,
-        ptr::null_mut(),
-        &mut base64,
-    );
-    assert!(ok);
-    assert!(!headerline.is_empty());
-    assert_eq!(headerline, "-----BEGIN PGP MESSAGE-----");
-
-    assert!(!base64.is_null());
-    assert_eq!(as_str(base64 as *const libc::c_char), "data",);
-
-    free(buf_0 as *mut libc::c_void);
-
-    buf_0 =
-        strdup(b"-----BEGIN PGP MESSAGE-----\n\ndat1\n-----END PGP MESSAGE-----\n-----BEGIN PGP MESSAGE-----\n\ndat2\n-----END PGP MESSAGE-----\x00"
-                   as *const u8 as *const libc::c_char);
-    let ok = dc_split_armored_data(
-        buf_0,
-        &mut headerline,
-        &mut setupcodebegin,
-        ptr::null_mut(),
-        &mut base64,
-    );
-
-    assert!(ok);
-    assert_eq!(headerline, "-----BEGIN PGP MESSAGE-----");
-
-    assert!(!base64.is_null());
-    assert_eq!(as_str(base64 as *const libc::c_char), "dat1",);
-
-    free(buf_0 as *mut libc::c_void);
-
-    buf_0 = strdup(
-        b"foo \n -----BEGIN PGP MESSAGE----- \n base64-123 \n  -----END PGP MESSAGE-----\x00"
-            as *const u8 as *const libc::c_char,
-    );
-    let ok = dc_split_armored_data(
-        buf_0,
-        &mut headerline,
-        &mut setupcodebegin,
-        ptr::null_mut(),
-        &mut base64,
-    );
-
-    assert!(ok);
-    assert_eq!(headerline, "-----BEGIN PGP MESSAGE-----");
-    assert!(setupcodebegin.is_null());
-
-    assert!(!base64.is_null());
-    assert_eq!(as_str(base64 as *const libc::c_char), "base64-123",);
-
-    free(buf_0 as *mut libc::c_void);
-
-    buf_0 = strdup(b"foo-----BEGIN PGP MESSAGE-----\x00" as *const u8 as *const libc::c_char);
-    let ok = dc_split_armored_data(
-        buf_0,
-        &mut headerline,
-        &mut setupcodebegin,
-        ptr::null_mut(),
-        &mut base64,
-    );
-
-    assert!(!ok);
-    free(buf_0 as *mut libc::c_void);
-    buf_0 =
-        strdup(b"foo \n -----BEGIN PGP MESSAGE-----\n  Passphrase-BeGIN  :  23 \n  \n base64-567 \r\n abc \n  -----END PGP MESSAGE-----\n\n\n\x00"
-                   as *const u8 as *const libc::c_char);
-    let ok = dc_split_armored_data(
-        buf_0,
-        &mut headerline,
-        &mut setupcodebegin,
-        ptr::null_mut(),
-        &mut base64,
-    );
-    assert!(ok);
-    assert_eq!(headerline, "-----BEGIN PGP MESSAGE-----");
-
-    assert!(!setupcodebegin.is_null());
-    assert_eq!(
-        strcmp(
-            setupcodebegin,
-            b"23\x00" as *const u8 as *const libc::c_char,
-        ),
-        0
-    );
-
-    assert!(!base64.is_null());
-    assert_eq!(as_str(base64 as *const libc::c_char), "base64-567 \n abc",);
-
-    free(buf_0 as *mut libc::c_void);
-
-    buf_0 =
-        strdup(b"-----BEGIN PGP PRIVATE KEY BLOCK-----\n Autocrypt-Prefer-Encrypt :  mutual \n\nbase64\n-----END PGP PRIVATE KEY BLOCK-----\x00"
-                   as *const u8 as *const libc::c_char);
-    let ok = dc_split_armored_data(
-        buf_0,
-        &mut headerline,
-        ptr::null_mut(),
-        &mut preferencrypt,
-        &mut base64,
-    );
-    assert!(ok);
-    assert_eq!(headerline, "-----BEGIN PGP PRIVATE KEY BLOCK-----");
-    assert!(!preferencrypt.is_null());
-    assert_eq!(
-        strcmp(
-            preferencrypt,
-            b"mutual\x00" as *const u8 as *const libc::c_char,
-        ),
-        0
-    );
-
-    assert!(!base64.is_null());
-    assert_eq!(as_str(base64 as *const libc::c_char), "base64",);
-
-    free(buf_0 as *mut libc::c_void);
-
     // Cant check, no configured context
     // assert!(dc_is_configured(context) != 0, "Missing configured context");
 
@@ -232,11 +100,11 @@ unsafe fn stress_functions(context: &Context) {
 #[test]
 #[ignore] // is too expensive
 fn test_encryption_decryption() {
-    let (public_key, private_key) = dc_pgp_create_keypair("foo@bar.de").unwrap();
+    let (public_key, private_key) = pgp::create_keypair("foo@bar.de").unwrap();
 
     private_key.split_key().unwrap();
 
-    let (public_key2, private_key2) = dc_pgp_create_keypair("two@zwo.de").unwrap();
+    let (public_key2, private_key2) = pgp::create_keypair("two@zwo.de").unwrap();
 
     assert_ne!(public_key, public_key2);
 
@@ -245,11 +113,11 @@ fn test_encryption_decryption() {
     keyring.add_owned(public_key.clone());
     keyring.add_ref(&public_key2);
 
-    let ctext_signed = dc_pgp_pk_encrypt(original_text, &keyring, Some(&private_key)).unwrap();
+    let ctext_signed = pgp::pk_encrypt(original_text, &keyring, Some(&private_key)).unwrap();
     assert!(!ctext_signed.is_empty());
     assert!(ctext_signed.starts_with("-----BEGIN PGP MESSAGE-----"));
 
-    let ctext_unsigned = dc_pgp_pk_encrypt(original_text, &keyring, None).unwrap();
+    let ctext_unsigned = pgp::pk_encrypt(original_text, &keyring, None).unwrap();
     assert!(!ctext_unsigned.is_empty());
     assert!(ctext_unsigned.starts_with("-----BEGIN PGP MESSAGE-----"));
 
@@ -264,7 +132,7 @@ fn test_encryption_decryption() {
 
     let mut valid_signatures: HashSet<String> = Default::default();
 
-    let plain = dc_pgp_pk_decrypt(
+    let plain = pgp::pk_decrypt(
         ctext_signed.as_bytes(),
         &keyring,
         &public_keyring,
@@ -278,7 +146,7 @@ fn test_encryption_decryption() {
     valid_signatures.clear();
 
     let empty_keyring = Keyring::default();
-    let plain = dc_pgp_pk_decrypt(
+    let plain = pgp::pk_decrypt(
         ctext_signed.as_bytes(),
         &keyring,
         &empty_keyring,
@@ -290,7 +158,7 @@ fn test_encryption_decryption() {
 
     valid_signatures.clear();
 
-    let plain = dc_pgp_pk_decrypt(
+    let plain = pgp::pk_decrypt(
         ctext_signed.as_bytes(),
         &keyring,
         &public_keyring2,
@@ -304,7 +172,7 @@ fn test_encryption_decryption() {
 
     public_keyring2.add_ref(&public_key);
 
-    let plain = dc_pgp_pk_decrypt(
+    let plain = pgp::pk_decrypt(
         ctext_signed.as_bytes(),
         &keyring,
         &public_keyring2,
@@ -316,7 +184,7 @@ fn test_encryption_decryption() {
 
     valid_signatures.clear();
 
-    let plain = dc_pgp_pk_decrypt(
+    let plain = pgp::pk_decrypt(
         ctext_unsigned.as_bytes(),
         &keyring,
         &public_keyring,
@@ -333,8 +201,7 @@ fn test_encryption_decryption() {
     let mut public_keyring = Keyring::default();
     public_keyring.add_ref(&public_key);
 
-    let plain =
-        dc_pgp_pk_decrypt(ctext_signed.as_bytes(), &keyring, &public_keyring, None).unwrap();
+    let plain = pgp::pk_decrypt(ctext_signed.as_bytes(), &keyring, &public_keyring, None).unwrap();
 
     assert_eq!(plain, original_text);
 }