use automatic serialization, thanks @link2xt for the tip

This effectively reverts
https://github.com/deltachat/deltachat-core-rust/pull/964 for chat.rs,
which in that PR was thought to fix something.  So maybe something is
still broken?  But after improving tests the previous code seems to be
correct.

- Update Python bindings to not always use dc_prepare_msg path when
  sending messages with attachements.  When using dc_prepare_msg the
  blobs need to be created in the blobdir since they will not get
  copied and many tests where not doing this.

- Add a test that ensures that calling dc_prepare_msg with a
  file **not** in the blobdir fails.

- Add a test that ensures that calling dc_send_msg directly with a
  file **not** in the blobdir copies the file to the blobdir.  This
  test cheats a little by knowing what the filename in the blobdir
  will be which is implementation-dependent and thus a bit brittle.
  But for now it proves correct behaviour so let's go with this.

- Improve the test_forward_increation test to ensure that the
  in-creation file only has it's final state before calling
  dc_send_msg.  This checks the correct file data is sent out and not
  the preparing data, this fails with the chat.rs changes in
  #964 (reverted here to make this work again).  Also fix the test to
  actually create the in-creation file in the blobdir.

- Fix test_send_file_twice_unicode_filename_mangling to not use
  in-creation.  It was not creating it's files in the blobdir and that
  is an error when using in-creation and it didn't seem it was trying
  to test something about the in-creation logic (which is tested in
  test_increation.py already).

- Fix Message._msgtate code which presumably was not used before?

- Rename `BlobObject::create_from_path` to
  `BlobObject::new_from_path`.  All the `BlobObject::create*` calls
  now always create new files which is much more consistent.  APIs
  should do what is obious.

Cargo.lock

@@ -116,6 +116,31 @@ dependencies = [
  "pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "async-smtp"
+version = "0.1.0"
+source = "git+https://github.com/async-email/async-smtp#f6bf23d29fa0a09db2bce557a0e2c13c3d851f2f"
+dependencies = [
+ "async-std 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "async-tls 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "async-trait 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
+ "base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "bufstream 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "fast_chemail 0.9.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "hostname 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
+ "nom 5.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "pin-project 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rustls 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rustls-connector 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_derive 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
+ "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)",
+ "snafu 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "webpki-roots 0.18.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "async-std"
 version = "1.2.0"
@@ -161,6 +186,16 @@ dependencies = [
  "webpki-roots 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "async-trait"
+version = "0.1.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "atty"
 version = "0.2.13"
@@ -610,6 +645,7 @@ name = "deltachat"
 version = "1.0.0-beta.12"
 dependencies = [
  "async-imap 0.1.1 (git+https://github.com/async-email/async-imap)",
+ "async-smtp 0.1.0 (git+https://github.com/async-email/async-smtp)",
  "async-std 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "async-tls 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "backtrace 0.3.40 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -628,7 +664,6 @@ dependencies = [
  "image-meta 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "itertools 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "lettre 0.9.2 (git+https://github.com/deltachat/lettre?branch=feat/mail)",
  "lettre_email 0.9.2 (git+https://github.com/deltachat/lettre?branch=feat/mail)",
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
  "mailparse 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -760,6 +795,11 @@ dependencies = [
  "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "doc-comment"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "dtoa"
 version = "0.4.4"
@@ -1198,11 +1238,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 
 [[package]]
 name = "hostname"
-version = "0.1.5"
+version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
- "winutil 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "match_cfg 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -1415,17 +1456,9 @@ name = "lettre"
 version = "0.9.2"
 source = "git+https://github.com/deltachat/lettre?branch=feat/mail#00ba9db544059ddd60048f0b85d5052e4bf605da"
 dependencies = [
- "base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "bufstream 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "fast_chemail 0.9.6 (registry+https://github.com/rust-lang/crates.io-index)",
- "hostname 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
- "nom 5.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "rustls 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "rustls-connector 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde_derive 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
- "serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)",
  "webpki-roots 0.18.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
@@ -1521,6 +1554,11 @@ dependencies = [
  "quoted_printable 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "match_cfg"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
 [[package]]
 name = "matches"
 version = "0.1.8"
@@ -2607,6 +2645,25 @@ name = "smallvec"
 version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
+[[package]]
+name = "snafu"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "doc-comment 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "snafu-derive 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "snafu-derive"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "proc-macro2 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
+ "quote 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "syn 1.0.11 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "sourcefile"
 version = "0.1.4"
@@ -3324,14 +3381,6 @@ dependencies = [
  "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
-[[package]]
-name = "winutil"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-dependencies = [
- "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
-]
-
 [[package]]
 name = "ws2_32-sys"
 version = "0.2.1"
@@ -3393,9 +3442,11 @@ dependencies = [
 "checksum async-attributes 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "efd3d156917d94862e779f356c5acae312b08fd3121e792c857d7928c8088423"
 "checksum async-imap 0.1.1 (git+https://github.com/async-email/async-imap)" = "<none>"
 "checksum async-macros 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "644a5a8de80f2085a1e7e57cd1544a2a7438f6e003c0790999bd43b92a77cdb2"
+"checksum async-smtp 0.1.0 (git+https://github.com/async-email/async-smtp)" = "<none>"
 "checksum async-std 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "513ee3c49800679a319912340f5601afda9e72848d7dea3a48bab489e8c1a46f"
 "checksum async-task 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "de6bd58f7b9cc49032559422595c81cbfcf04db2f2133592f70af19e258a1ced"
 "checksum async-tls 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6ce6977f57fa68da77ffe5542950d47e9c23d65f5bc7cb0a9f8700996913eec7"
+"checksum async-trait 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)" = "8b6dd385bb33043b833ba049048d57bdbb4d654a121ed68c71871ca51ff67070"
 "checksum atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "1803c647a3ec87095e7ae7acfca019e98de5ec9a7d01343f611cf3152ed71a90"
 "checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
 "checksum backtrace 0.3.40 (registry+https://github.com/rust-lang/crates.io-index)" = "924c76597f0d9ca25d762c25a4d369d51267536465dc5064bdf0eb073ed477ea"
@@ -3457,6 +3508,7 @@ dependencies = [
 "checksum difference 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198"
 "checksum digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5"
 "checksum dirs 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901"
+"checksum doc-comment 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "923dea538cea0aa3025e8685b20d6ee21ef99c4f77e954a30febbaac5ec73a97"
 "checksum dtoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "ea57b42383d091c85abcc2706240b94ab2a8fa1fc81c10ff23c4de06e2a90b5e"
 "checksum ed25519-dalek 1.0.0-pre.2 (registry+https://github.com/rust-lang/crates.io-index)" = "845aaacc16f01178f33349e7c992ecd0cee095aa5e577f0f4dee35971bd36455"
 "checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3"
@@ -3510,7 +3562,7 @@ dependencies = [
 "checksum heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205"
 "checksum hermit-abi 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "307c3c9f937f38e3534b1d6447ecf090cafcc9744e4a6360e8b037b2cf5af120"
 "checksum hex 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "023b39be39e3a2da62a94feb433e91e8bcd37676fbc8bea371daf52b7a769a3e"
-"checksum hostname 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "21ceb46a83a85e824ef93669c8b390009623863b5c195d1ba747292c0c72f94e"
+"checksum hostname 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "dc5260e6c63877196b6fca5a7fb4eaff751134045ad3415716192baa36f5b9a0"
 "checksum http 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)" = "d6ccf5ede3a895d8856620237b2f02972c1bbc78d2965ad7fe8838d4a0ed41f0"
 "checksum http-body 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6741c859c1b2463a423a1dbce98d418e6c3c3fc720fb0d45528657320920292d"
 "checksum httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9"
@@ -3544,6 +3596,7 @@ dependencies = [
 "checksum lru-cache 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c"
 "checksum mach 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "86dd2487cdfea56def77b88438a2c915fb45113c5319bfe7e14306ca4cd0b0e1"
 "checksum mailparse 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c00eb97cc18f08aaadd02808328dcc9be94948d8e5b54adbfd3414d2f87f7bf1"
+"checksum match_cfg 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4"
 "checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
 "checksum maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00"
 "checksum md-5 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a18af3dcaf2b0219366cdb4e2af65a6101457b415c3d1a5c71dd9c2b7c77b9c8"
@@ -3657,6 +3710,8 @@ dependencies = [
 "checksum slice-deque 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "ffddf594f5f597f63533d897427a570dbaa9feabaaa06595b74b71b7014507d7"
 "checksum smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6"
 "checksum smallvec 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4ecf3b85f68e8abaa7555aa5abdb1153079387e60b718283d732f03897fcfc86"
+"checksum snafu 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "41207ca11f96a62cd34e6b7fdf73d322b25ae3848eb9d38302169724bb32cf27"
+"checksum snafu-derive 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4c5e338c8b0577457c9dda8e794b6ad7231c96e25b1b0dd5842d52249020c1c0"
 "checksum sourcefile 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4bf77cb82ba8453b42b6ae1d692e4cdc92f9a47beaf89a847c8be83f4e328ad3"
 "checksum spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
 "checksum stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dba1a27d3efae4351c8051072d619e3ade2820635c3958d826bfea39d59b54c8"
@@ -3743,7 +3798,6 @@ dependencies = [
 "checksum wincolor 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "eeb06499a3a4d44302791052df005d5232b927ed1a9658146d842165c4de7767"
 "checksum wincolor 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "96f5016b18804d24db43cebf3c77269e7569b8954a8464501c216cc5e070eaa9"
 "checksum winreg 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9"
-"checksum winutil 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7daf138b6b14196e3830a588acf1e86966c694d3e8fb026fb105b8b5dca07e6e"
 "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e"
 "checksum x25519-dalek 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7ee1585dc1484373cbc1cee7aafda26634665cf449436fd6e24bfd1fad230538"
 "checksum yaml-rust 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "65923dd1784f44da1d2c3dbbc5e822045628c590ba72123e1c73d3c230c4434d"

Cargo.toml

@@ -17,7 +17,7 @@ smallvec = "0.6.9"
 reqwest = { version = "0.9.15", default-features = false, features = ["rustls-tls"] }
 num-derive = "0.2.5"
 num-traits = "0.2.6"
-lettre = { git = "https://github.com/deltachat/lettre", branch = "feat/mail" }
+async-smtp = { git = "https://github.com/async-email/async-smtp" }
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "feat/mail" }
 async-imap = { git = "https://github.com/async-email/async-imap", branch="master" }
 async-tls = "0.6"

deltachat-ffi/deltachat.h

@@ -4050,11 +4050,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  */
 #define DC_CERTCK_STRICT 1
 
-/**
- * Accept invalid hostnames, but not invalid certificates.
- */
-#define DC_CERTCK_ACCEPT_INVALID_HOSTNAMES 2
-
 /**
  * Accept invalid certificates, including self-signed ones
  * or having incorrect hostname.

deltachat-ffi/src/lib.rs

@@ -29,6 +29,8 @@ use deltachat::message::MsgId;
 use deltachat::stock::StockMessage;
 use deltachat::*;
 
+mod dc_array;
+
 mod string;
 use self::string::*;
 

python/src/deltachat/chat.py

@@ -109,6 +109,30 @@ class Chat(object):
 
     # ------  chat messaging API ------------------------------
 
+    def send_msg(self, msg):
+        """send a message by using a ready Message object.
+
+        :param msg: a :class:`deltachat.message.Message` instance
+           previously returned by
+           e.g. :meth:`deltachat.message.Message.new_empty` or
+           :meth:`prepare_file`.
+        :raises ValueError: if message can not be sent.
+
+        :returns: a :class:`deltachat.message.Message` instance as
+           sent out.  This is the same object as was passed in, which
+           has been modified with the new state of the core.
+        """
+        if msg.is_out_preparing():
+            assert msg.id != 0
+            # get a fresh copy of dc_msg, the core needs it
+            msg = Message.from_db(self.account, msg.id)
+        sent_id = lib.dc_send_msg(self._dc_context, self.id, msg._dc_msg)
+        if sent_id == 0:
+            raise ValueError("message could not be sent")
+        # modify message in place to avoid bad state for the caller
+        msg._dc_msg = Message.from_db(self.account, sent_id)._dc_msg
+        return msg
+
     def send_text(self, text):
         """ send a text message and return the resulting Message instance.
 
@@ -130,9 +154,12 @@ class Chat(object):
         :raises ValueError: if message can not be send/chat does not exist.
         :returns: the resulting :class:`deltachat.message.Message` instance
         """
-        msg = self.prepare_message_file(path=path, mime_type=mime_type)
-        self.send_prepared(msg)
-        return msg
+        msg = Message.new_empty(self.account, view_type="file")
+        msg.set_file(path, mime_type)
+        sent_id = lib.dc_send_msg(self._dc_context, self.id, msg._dc_msg)
+        if sent_id == 0:
+            raise ValueError("message could not be sent")
+        return Message.from_db(self.account, sent_id)
 
     def send_image(self, path):
         """ send an image message and return the resulting Message instance.
@@ -142,9 +169,12 @@ class Chat(object):
         :returns: the resulting :class:`deltachat.message.Message` instance
         """
         mime_type = mimetypes.guess_type(path)[0]
-        msg = self.prepare_message_file(path=path, mime_type=mime_type, view_type="image")
-        self.send_prepared(msg)
-        return msg
+        msg = Message.new_empty(self.account, view_type="image")
+        msg.set_file(path, mime_type)
+        sent_id = lib.dc_send_msg(self._dc_context, self.id, msg._dc_msg)
+        if sent_id == 0:
+            raise ValueError("message could not be sent")
+        return Message.from_db(self.account, sent_id)
 
     def prepare_message(self, msg):
         """ create a new prepared message.

python/src/deltachat/const.py

@@ -68,7 +68,6 @@ DC_LP_SMTP_SOCKET_SSL = 0x20000
 DC_LP_SMTP_SOCKET_PLAIN = 0x40000
 DC_CERTCK_AUTO = 0
 DC_CERTCK_STRICT = 1
-DC_CERTCK_ACCEPT_INVALID_HOSTNAMES = 2
 DC_CERTCK_ACCEPT_INVALID_CERTIFICATES = 3
 DC_EMPTY_MVBOX = 0x01
 DC_EMPTY_INBOX = 0x02

python/src/deltachat/message.py

@@ -174,7 +174,7 @@ class Message(object):
     @property
     def _msgstate(self):
         if self.id == 0:
-            dc_msg = self.message._dc_msg
+            dc_msg = self._dc_msg
         else:
             # load message from db to get a fresh/current state
             dc_msg = ffi.gc(

python/tests/test_account.py

@@ -456,10 +456,7 @@ class TestOnlineAccount:
             msg1 = Message.new_empty(ac1, "file")
             msg1.set_text("withfile")
             msg1.set_file(p)
-            message = chat.prepare_message(msg1)
-            assert message.is_out_preparing()
-            assert message.text == "withfile"
-            chat.send_prepared(message)
+            chat.send_msg(msg1)
 
             lp.sec("ac2: receive message")
             ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")

python/tests/test_increation.py

@@ -1,10 +1,49 @@
 from __future__ import print_function
+
+import os.path
+import shutil
+
+import pytest
 from filecmp import cmp
-from deltachat import const
+
 from conftest import wait_configuration_progress, wait_msgs_changed
+from deltachat import const
 
 
 class TestOnlineInCreation:
+    def test_increation_not_blobdir(self, tmpdir, acfactory, lp):
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.get_online_configuring_account()
+        wait_configuration_progress(ac1, 1000)
+        wait_configuration_progress(ac2, 1000)
+
+        c2 = ac1.create_contact(email=ac2.get_config("addr"))
+        chat = ac1.create_chat_by_contact(c2)
+
+        lp.sec("Creating in-creation file outside of blobdir")
+        assert tmpdir.strpath != ac1.get_blobdir()
+        src = tmpdir.join('file.txt').ensure(file=1)
+        with pytest.raises(Exception):
+            chat.prepare_message_file(src.strpath)
+
+    def test_no_increation_copies_to_blobdir(self, tmpdir, acfactory, lp):
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.get_online_configuring_account()
+        wait_configuration_progress(ac1, 1000)
+        wait_configuration_progress(ac2, 1000)
+
+        c2 = ac1.create_contact(email=ac2.get_config("addr"))
+        chat = ac1.create_chat_by_contact(c2)
+
+        lp.sec("Creating file outside of blobdir")
+        assert tmpdir.strpath != ac1.get_blobdir()
+        src = tmpdir.join('file.txt')
+        src.write("hello there\n")
+        chat.send_file(src.strpath)
+
+        blob_src = os.path.join(ac1.get_blobdir(), 'file.txt')
+        assert os.path.exists(blob_src), "file.txt not copied to blobdir"
+
     def test_forward_increation(self, acfactory, data, lp):
         ac1 = acfactory.get_online_configuring_account()
         ac2 = acfactory.get_online_configuring_account()
@@ -17,7 +56,10 @@ class TestOnlineInCreation:
         wait_msgs_changed(ac1, 0, 0)  # why no chat id?
 
         lp.sec("create a message with a file in creation")
-        path = data.get_path("d.png")
+        orig = data.get_path("d.png")
+        path = os.path.join(ac1.get_blobdir(), 'd.png')
+        with open(path, "x") as fp:
+            fp.write("preparing")
         prepared_original = chat.prepare_message_file(path)
         assert prepared_original.is_out_preparing()
         wait_msgs_changed(ac1, chat.id, prepared_original.id)
@@ -38,6 +80,7 @@ class TestOnlineInCreation:
 
         lp.sec("finish creating the file and send it")
         assert prepared_original.is_out_preparing()
+        shutil.copyfile(orig, path)
         chat.send_prepared(prepared_original)
         assert prepared_original.is_out_pending() or prepared_original.is_out_delivered()
         wait_msgs_changed(ac1, chat.id, prepared_original.id)
@@ -59,11 +102,11 @@ class TestOnlineInCreation:
         ev1 = ac2._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
         assert ev1[1] > const.DC_CHAT_ID_LAST_SPECIAL
         received_original = ac2.get_message_by_id(ev1[2])
-        assert cmp(received_original.filename, path, False)
+        assert cmp(received_original.filename, orig, shallow=False)
 
         lp.sec("wait2 for original or forwarded messages to arrive")
         ev2 = ac2._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
         assert ev2[1] > const.DC_CHAT_ID_LAST_SPECIAL
         assert ev2[1] != ev1[1]
         received_copy = ac2.get_message_by_id(ev2[2])
-        assert cmp(received_copy.filename, path, False)
+        assert cmp(received_copy.filename, orig, shallow=False)

python/tox.ini

@@ -65,8 +65,8 @@ commands =
 
 
 [pytest]
-addopts = -v -rs 
-python_files = tests/test_*.py 
+addopts = -v -ra
+python_files = tests/test_*.py
 norecursedirs = .tox 
 xfail_strict=true
 timeout = 60 

src/blob.rs

@@ -159,7 +159,7 @@ impl<'a> BlobObject<'a> {
     /// This merely delegates to the [BlobObject::create_and_copy] and
     /// the [BlobObject::from_path] methods.  See those for possible
     /// errors.
-    pub fn create_from_path(
+    pub fn new_from_path(
         context: &Context,
         src: impl AsRef<Path>,
     ) -> std::result::Result<BlobObject, BlobError> {
@@ -559,14 +559,14 @@ mod tests {
 
         let src_ext = t.dir.path().join("external");
         fs::write(&src_ext, b"boo").unwrap();
-        let blob = BlobObject::create_from_path(&t.ctx, &src_ext).unwrap();
+        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/external");
         let data = fs::read(blob.to_abs_path()).unwrap();
         assert_eq!(data, b"boo");
 
         let src_int = t.ctx.get_blobdir().join("internal");
         fs::write(&src_int, b"boo").unwrap();
-        let blob = BlobObject::create_from_path(&t.ctx, &src_int).unwrap();
+        let blob = BlobObject::new_from_path(&t.ctx, &src_int).unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/internal");
         let data = fs::read(blob.to_abs_path()).unwrap();
         assert_eq!(data, b"boo");
@@ -576,7 +576,7 @@ mod tests {
         let t = dummy_context();
         let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
         fs::write(&src_ext, b"boo").unwrap();
-        let blob = BlobObject::create_from_path(&t.ctx, &src_ext).unwrap();
+        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).unwrap();
         assert_eq!(
             blob.as_name(),
             "$BLOBDIR/autocrypt-setup-message-4137848473.html"

src/chat.rs

@@ -725,20 +725,11 @@ fn prepare_msg_blob(context: &Context, msg: &mut Message) -> Result<(), Error> {
     if msg.type_0 == Viewtype::Text {
         // the caller should check if the message text is empty
     } else if msgtype_has_file(msg.type_0) {
-        let blob = if let Some(f) = msg.param.get_file(Param::File, context)? {
-            match f {
-                ParamsFile::Blob(blob) => blob,
-                ParamsFile::FsPath(path) => {
-                    // path is outside the blobdir, let's copy
-                    let blob = BlobObject::create_and_copy(context, path)?;
-                    msg.param.set(Param::File, blob.as_name());
-
-                    blob
-                }
-            }
-        } else {
-            bail!("Attachment missing for message of type #{}", msg.type_0);
-        };
+        let blob = msg
+            .param
+            .get_blob(Param::File, context, !msg.is_increation())?
+            .ok_or_else(|| format_err!("Attachment missing for message of type #{}", msg.type_0))?;
+        msg.param.set(Param::File, blob.as_name());
 
         if msg.type_0 == Viewtype::File || msg.type_0 == Viewtype::Image {
             // Correct the type, take care not to correct already very special

src/config.rs

@@ -128,7 +128,7 @@ impl Context {
     pub fn set_config(&self, key: Config, value: Option<&str>) -> crate::sql::Result<()> {
         match key {
             Config::Selfavatar if value.is_some() => {
-                let blob = BlobObject::create_from_path(&self, value.unwrap())?;
+                let blob = BlobObject::new_from_path(&self, value.unwrap())?;
                 self.sql.set_raw_config(self, key, Some(blob.as_name()))
             }
             Config::InboxWatch => {

src/configure/mod.rs

@@ -485,8 +485,12 @@ fn try_imap_connection(
 
 fn try_imap_one_param(context: &Context, param: &LoginParam) -> Option<bool> {
     let inf = format!(
-        "imap: {}@{}:{} flags=0x{:x}",
-        param.mail_user, param.mail_server, param.mail_port, param.server_flags
+        "imap: {}@{}:{} flags=0x{:x} certificate_checks={}",
+        param.mail_user,
+        param.mail_server,
+        param.mail_port,
+        param.server_flags,
+        param.imap_certificate_checks
     );
     info!(context, "Trying: {}", inf);
     if context

src/contact.rs

@@ -861,14 +861,14 @@ impl Contact {
             .unwrap_or_default() as usize
     }
 
-    pub fn get_origin_by_id(context: &Context, contact_id: u32, ret_blocked: &mut i32) -> Origin {
+    pub fn get_origin_by_id(context: &Context, contact_id: u32, ret_blocked: &mut bool) -> Origin {
         let mut ret = Origin::Unknown;
-        *ret_blocked = 0;
+        *ret_blocked = false;
 
         if let Ok(contact) = Contact::load_from_db(context, contact_id) {
             /* we could optimize this by loading only the needed fields */
             if contact.blocked {
-                *ret_blocked = 1;
+                *ret_blocked = true;
             } else {
                 ret = contact.origin;
             }

src/dc_receive_imf.rs

@@ -12,6 +12,7 @@ use crate::context::Context;
 use crate::dc_tools::*;
 use crate::error::Result;
 use crate::events::Event;
+use crate::headerdef::HeaderDef;
 use crate::job::*;
 use crate::location;
 use crate::message::{self, MessageState, MsgId};
@@ -60,21 +61,21 @@ pub fn dc_receive_imf(
         mime_parser.unwrap()
     };
 
-    if mime_parser.header.is_empty() {
-        // Error - even adding an empty record won't help as we do not know the message ID
-        warn!(context, "No header.");
+    if mime_parser.get(HeaderDef::From_).is_none() {
+        // Error - even adding an empty record won't help as we do not know the sender
+        warn!(context, "No From header.");
         return;
     }
 
     // the function returns the number of created messages in the database
-    let mut incoming = 1;
+    let mut incoming = true;
     let mut incoming_origin = Origin::Unknown;
     let mut to_self = false;
     let mut from_id = 0u32;
-    let mut from_id_blocked = 0;
+    let mut from_id_blocked = false;
     let mut to_id = 0u32;
     let mut chat_id = 0;
-    let mut hidden = 0;
+    let mut hidden = false;
 
     let mut needs_delete_job = false;
     let mut insert_msg_id = MsgId::new_unset();
@@ -114,7 +115,7 @@ pub fn dc_receive_imf(
         }
     };
 
-    if let Some(value) = mime_parser.lookup_field("Date") {
+    if let Some(value) = mime_parser.get(HeaderDef::Date) {
         // is not yet checked against bad times! we do this later if we have the database information.
         sent_timestamp = mailparse::dateparse(value).unwrap_or_default();
     }
@@ -122,7 +123,7 @@ pub fn dc_receive_imf(
     // get From: and check if it is known (for known From:'s we add the other To:/Cc: in the 3rd pass)
     // or if From: is equal to SELF (in this case, it is any outgoing messages,
     // we do not check Return-Path any more as this is unreliable, see issue #150
-    if let Some(field_from) = mime_parser.lookup_field("From") {
+    if let Some(field_from) = mime_parser.get(HeaderDef::From_) {
         let mut check_self = false;
         let mut from_list = Vec::with_capacity(16);
         dc_add_or_lookup_contacts_by_address_list(
@@ -133,7 +134,7 @@ pub fn dc_receive_imf(
             &mut check_self,
         );
         if check_self {
-            incoming = 0;
+            incoming = false;
             if mime_parser.sender_equals_recipient() {
                 from_id = DC_CONTACT_ID_SELF;
             }
@@ -147,11 +148,11 @@ pub fn dc_receive_imf(
     }
 
     // Make sure, to_ids starts with the first To:-address (Cc: is added in the loop below pass)
-    if let Some(field) = mime_parser.lookup_field("To") {
+    if let Some(field) = mime_parser.get(HeaderDef::To) {
         dc_add_or_lookup_contacts_by_address_list(
             context,
             &field,
-            if 0 == incoming {
+            if !incoming {
                 Origin::OutgoingTo
             } else if incoming_origin.is_verified() {
                 Origin::IncomingTo
@@ -277,7 +278,7 @@ fn add_parts(
     context: &Context,
     mut mime_parser: &mut MimeParser,
     imf_raw: &[u8],
-    incoming: i32,
+    incoming: bool,
     incoming_origin: &mut Origin,
     server_folder: impl AsRef<str>,
     server_uid: u32,
@@ -285,8 +286,8 @@ fn add_parts(
     rfc724_mid: &str,
     sent_timestamp: &mut i64,
     from_id: &mut u32,
-    from_id_blocked: i32,
-    hidden: &mut i32,
+    from_id_blocked: bool,
+    hidden: &mut bool,
     chat_id: &mut u32,
     to_id: &mut u32,
     flags: u32,
@@ -307,11 +308,11 @@ fn add_parts(
     // collect the rest information, CC: is added to the to-list, BCC: is ignored
     // (we should not add BCC to groups as this would split groups. We could add them as "known contacts",
     // however, the benefit is very small and this may leak data that is expected to be hidden)
-    if let Some(fld_cc) = mime_parser.lookup_field("Cc") {
+    if let Some(fld_cc) = mime_parser.get(HeaderDef::Cc) {
         dc_add_or_lookup_contacts_by_address_list(
             context,
             fld_cc,
-            if 0 == incoming {
+            if !incoming {
                 Origin::OutgoingCc
             } else if incoming_origin.is_verified() {
                 Origin::IncomingCc
@@ -338,22 +339,22 @@ fn add_parts(
 
     // 1 or 0 for yes/no
     msgrmsg = mime_parser.has_chat_version() as _;
-    if msgrmsg == 0 && 0 != dc_is_reply_to_messenger_message(context, mime_parser) {
+    if msgrmsg == 0 && is_reply_to_messenger_message(context, mime_parser) {
         // 2=no, but is reply to messenger message
         msgrmsg = 2;
     }
     // incoming non-chat messages may be discarded;
     // maybe this can be optimized later, by checking the state before the message body is downloaded
-    let mut allow_creation = 1;
+    let mut allow_creation = true;
     let show_emails =
         ShowEmails::from_i32(context.get_config_int(Config::ShowEmails)).unwrap_or_default();
     if mime_parser.is_system_message != SystemMessage::AutocryptSetupMessage && msgrmsg == 0 {
         // this message is a classic email not a chat-message nor a reply to one
         if show_emails == ShowEmails::Off {
             *chat_id = DC_CHAT_ID_TRASH;
-            allow_creation = 0
+            allow_creation = false
         } else if show_emails == ShowEmails::AcceptedContacts {
-            allow_creation = 0
+            allow_creation = false
         }
     }
 
@@ -361,7 +362,7 @@ fn add_parts(
     // - outgoing messages introduce a chat with the first to: address if they are sent by a messenger
     // - incoming messages introduce a chat only for known contacts if they are sent by a messenger
     // (of course, the user can add other chats manually later)
-    if 0 != incoming {
+    if incoming {
         state = if 0 != flags & DC_IMAP_SEEN {
             MessageState::InSeen
         } else {
@@ -372,15 +373,15 @@ fn add_parts(
 
         // handshake messages must be processed _before_ chats are created
         // (eg. contacs may be marked as verified)
-        if mime_parser.lookup_field("Secure-Join").is_some() {
+        if mime_parser.get(HeaderDef::SecureJoin).is_some() {
             // avoid discarding by show_emails setting
             msgrmsg = 1;
             *chat_id = 0;
-            allow_creation = 1;
+            allow_creation = true;
             match handle_securejoin_handshake(context, mime_parser, *from_id) {
                 Ok(ret) => {
                     if ret.hide_this_msg {
-                        *hidden = 1;
+                        *hidden = true;
                         *needs_delete_job = ret.delete_this_msg;
                         state = MessageState::InSeen;
                     }
@@ -449,7 +450,7 @@ fn add_parts(
             if 0 != test_normal_chat_id {
                 *chat_id = test_normal_chat_id;
                 chat_id_blocked = test_normal_chat_id_blocked;
-            } else if 0 != allow_creation {
+            } else if allow_creation {
                 let (id, bl) =
                     chat::create_or_lookup_by_contact_id(context, *from_id, create_blocked)
                         .unwrap_or_default();
@@ -460,7 +461,7 @@ fn add_parts(
                 if Blocked::Not == create_blocked {
                     chat::unblock(context, *chat_id);
                     chat_id_blocked = Blocked::Not;
-                } else if 0 != dc_is_reply_to_known_message(context, mime_parser) {
+                } else if is_reply_to_known_message(context, mime_parser) {
                     //  we do not want any chat to be created implicitly.  Because of the origin-scale-up,
                     // the contact requests will pop up and this should be just fine.
                     Contact::scaleup_origin_by_id(context, *from_id, Origin::IncomingReplyTo);
@@ -523,7 +524,7 @@ fn add_parts(
                     chat_id_blocked = Blocked::Not;
                 }
             }
-            if *chat_id == 0 && 0 != allow_creation {
+            if *chat_id == 0 && allow_creation {
                 let create_blocked = if 0 != msgrmsg && !Contact::is_blocked_load(context, *to_id) {
                     Blocked::Not
                 } else {
@@ -569,7 +570,7 @@ fn add_parts(
         *chat_id,
         *from_id,
         *sent_timestamp,
-        if 0 != flags & DC_IMAP_SEEN { 0 } else { 1 },
+        0 == flags & DC_IMAP_SEEN,
         &mut sort_timestamp,
         sent_timestamp,
         &mut rcvd_timestamp,
@@ -581,11 +582,11 @@ fn add_parts(
     // if the mime-headers should be saved, find out its size
     // (the mime-header ends with an empty line)
     let save_mime_headers = context.get_config_bool(Config::SaveMimeHeaders);
-    if let Some(raw) = mime_parser.lookup_field("In-Reply-To") {
+    if let Some(raw) = mime_parser.get(HeaderDef::InReplyTo) {
         mime_in_reply_to = raw.clone();
     }
 
-    if let Some(raw) = mime_parser.lookup_field("References") {
+    if let Some(raw) = mime_parser.get(HeaderDef::References) {
         mime_references = raw.clone();
     }
 
@@ -604,8 +605,9 @@ fn add_parts(
          bytes, hidden, mime_headers,  mime_in_reply_to, mime_references) \
          VALUES (?,?,?,?,?,?, ?,?,?,?,?,?, ?,?,?,?,?,?, ?,?);",
         |mut stmt, conn| {
-            for i in 0..icnt {
-                let part = &mut mime_parser.parts[i];
+            let subject = mime_parser.get_subject().unwrap_or_default();
+
+            for part in mime_parser.parts.iter_mut() {
                 if part.is_meta {
                     continue;
                 }
@@ -614,7 +616,7 @@ fn add_parts(
                     && icnt == 1
                     && (part.msg == "-location-" || part.msg.is_empty())
                 {
-                    *hidden = 1;
+                    *hidden = true;
                     if state == MessageState::InFresh {
                         state = MessageState::InNoticed;
                     }
@@ -622,11 +624,6 @@ fn add_parts(
 
                 if part.typ == Viewtype::Text {
                     let msg_raw = part.msg_raw.as_ref().cloned().unwrap_or_default();
-                    let subject = mime_parser
-                        .subject
-                        .as_ref()
-                        .map(|s| s.to_string())
-                        .unwrap_or_else(|| "".into());
                     txt_raw = Some(format!("{}\n\n{}", subject, msg_raw));
                 }
                 if mime_parser.is_system_message != SystemMessage::Unknown {
@@ -680,8 +677,8 @@ fn add_parts(
     // check event to send
     if *chat_id == DC_CHAT_ID_TRASH {
         *create_event_to_send = None;
-    } else if 0 != incoming && state == MessageState::InFresh {
-        if 0 != from_id_blocked {
+    } else if incoming && state == MessageState::InFresh {
+        if from_id_blocked {
             *create_event_to_send = None;
         } else if Blocked::Not != chat_id_blocked {
             *create_event_to_send = Some(CreateEvent::MsgsChanged);
@@ -699,7 +696,7 @@ fn save_locations(
     chat_id: u32,
     from_id: u32,
     insert_msg_id: MsgId,
-    hidden: i32,
+    hidden: bool,
 ) {
     if chat_id <= DC_CHAT_ID_LAST_SPECIAL {
         return;
@@ -712,7 +709,7 @@ fn save_locations(
         let newest_location_id =
             location::save(context, chat_id, from_id, locations, true).unwrap_or_default();
         if 0 != newest_location_id
-            && 0 == hidden
+            && !hidden
             && location::set_msg_location_id(context, insert_msg_id, newest_location_id).is_ok()
         {
             location_id_written = true;
@@ -728,7 +725,7 @@ fn save_locations(
                     let newest_location_id =
                         location::save(context, chat_id, from_id, locations, false)
                             .unwrap_or_default();
-                    if newest_location_id != 0 && hidden == 0 && !location_id_written {
+                    if newest_location_id != 0 && !hidden && !location_id_written {
                         if let Err(err) = location::set_msg_location_id(
                             context,
                             insert_msg_id,
@@ -752,7 +749,7 @@ fn calc_timestamps(
     chat_id: u32,
     from_id: u32,
     message_timestamp: i64,
-    is_fresh_msg: i32,
+    is_fresh_msg: bool,
     sort_timestamp: &mut i64,
     sent_timestamp: &mut i64,
     rcvd_timestamp: &mut i64,
@@ -763,7 +760,7 @@ fn calc_timestamps(
         *sent_timestamp = *rcvd_timestamp
     }
     *sort_timestamp = message_timestamp;
-    if 0 != is_fresh_msg {
+    if is_fresh_msg {
         let last_msg_time: Option<i64> = context.sql.query_get_value(
             context,
             "SELECT MAX(timestamp) FROM msgs WHERE chat_id=? and from_id!=? AND timestamp>=?",
@@ -795,19 +792,18 @@ fn calc_timestamps(
 fn create_or_lookup_group(
     context: &Context,
     mime_parser: &mut MimeParser,
-    allow_creation: i32,
+    allow_creation: bool,
     create_blocked: Blocked,
     from_id: u32,
     to_ids: &[u32],
 ) -> Result<(u32, Blocked)> {
     let mut chat_id_blocked = Blocked::Not;
-    let mut grpname = None;
     let to_ids_cnt = to_ids.len();
-    let mut recreate_member_list = 0;
-    let mut send_EVENT_CHAT_MODIFIED = 0;
+    let mut recreate_member_list = false;
+    let mut send_EVENT_CHAT_MODIFIED = false;
     let mut X_MrRemoveFromGrp = None;
     let mut X_MrAddToGrp = None;
-    let mut X_MrGrpNameChanged = 0;
+    let mut X_MrGrpNameChanged = false;
     let mut X_MrGrpImageChanged = "".to_string();
     let mut better_msg: String = From::from("");
 
@@ -818,20 +814,21 @@ fn create_or_lookup_group(
     set_better_msg(mime_parser, &better_msg);
 
     let mut grpid = "".to_string();
-    if let Some(optional_field) = mime_parser.lookup_field("Chat-Group-ID") {
+    if let Some(optional_field) = mime_parser.get(HeaderDef::ChatGroupId) {
         grpid = optional_field.clone();
     }
 
     if grpid.is_empty() {
-        if let Some(value) = mime_parser.lookup_field("Message-ID") {
+        if let Some(value) = mime_parser.get(HeaderDef::MessageId) {
             if let Some(extracted_grpid) = dc_extract_grpid_from_rfc724_mid(&value) {
                 grpid = extracted_grpid.to_string();
             }
         }
         if grpid.is_empty() {
-            if let Some(extracted_grpid) = get_grpid_from_list(mime_parser, "In-Reply-To") {
+            if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::InReplyTo) {
                 grpid = extracted_grpid;
-            } else if let Some(extracted_grpid) = get_grpid_from_list(mime_parser, "References") {
+            } else if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::References)
+            {
                 grpid = extracted_grpid;
             } else {
                 return create_or_lookup_adhoc_group(
@@ -850,19 +847,15 @@ fn create_or_lookup_group(
         }
     }
 
-    if let Some(optional_field) = mime_parser.lookup_field("Chat-Group-Name").cloned() {
-        grpname = Some(optional_field);
-    }
-    let field = mime_parser
-        .lookup_field("Chat-Group-Member-Removed")
-        .cloned();
-    if let Some(optional_field) = field {
+    let grpname = mime_parser.get(HeaderDef::ChatGroupName).cloned();
+
+    if let Some(optional_field) = mime_parser.get(HeaderDef::ChatGroupMemberRemoved).cloned() {
         X_MrRemoveFromGrp = Some(optional_field);
         mime_parser.is_system_message = SystemMessage::MemberRemovedFromGroup;
-        let left_group = (Contact::lookup_id_by_addr(context, X_MrRemoveFromGrp.as_ref().unwrap())
-            == from_id as u32) as i32;
+        let left_group = Contact::lookup_id_by_addr(context, X_MrRemoveFromGrp.as_ref().unwrap())
+            == from_id as u32;
         better_msg = context.stock_system_msg(
-            if 0 != left_group {
+            if left_group {
                 StockMessage::MsgGroupLeft
             } else {
                 StockMessage::MsgDelMember
@@ -872,11 +865,11 @@ fn create_or_lookup_group(
             from_id as u32,
         )
     } else {
-        let field = mime_parser.lookup_field("Chat-Group-Member-Added").cloned();
+        let field = mime_parser.get(HeaderDef::ChatGroupMemberAdded).cloned();
         if let Some(optional_field) = field {
             X_MrAddToGrp = Some(optional_field);
             mime_parser.is_system_message = SystemMessage::MemberAddedToGroup;
-            if let Some(optional_field) = mime_parser.lookup_field("Chat-Group-Image").cloned() {
+            if let Some(optional_field) = mime_parser.get(HeaderDef::ChatGroupImage).cloned() {
                 X_MrGrpImageChanged = optional_field;
             }
             better_msg = context.stock_system_msg(
@@ -886,9 +879,9 @@ fn create_or_lookup_group(
                 from_id as u32,
             )
         } else {
-            let field = mime_parser.lookup_field("Chat-Group-Name-Changed");
+            let field = mime_parser.get(HeaderDef::ChatGroupNameChanged);
             if let Some(field) = field {
-                X_MrGrpNameChanged = 1;
+                X_MrGrpNameChanged = true;
                 better_msg = context.stock_system_msg(
                     StockMessage::MsgGrpName,
                     field,
@@ -901,8 +894,7 @@ fn create_or_lookup_group(
                 );
 
                 mime_parser.is_system_message = SystemMessage::GroupNameChanged;
-            } else if let Some(optional_field) =
-                mime_parser.lookup_field("Chat-Group-Image").cloned()
+            } else if let Some(optional_field) = mime_parser.get(HeaderDef::ChatGroupImage).cloned()
             {
                 // fld_value is a pointer somewhere into mime_parser, must not be freed
                 X_MrGrpImageChanged = optional_field;
@@ -937,7 +929,7 @@ fn create_or_lookup_group(
         // check if the sender is a member of the existing group -
         // if not, we'll recreate the group list
         if !chat::is_contact_in_chat(context, chat_id, from_id as u32) {
-            recreate_member_list = 1;
+            recreate_member_list = true;
         }
     }
 
@@ -958,7 +950,7 @@ fn create_or_lookup_group(
                 || X_MrAddToGrp.is_some() && addr_cmp(&self_addr, X_MrAddToGrp.as_ref().unwrap()))
     {
         // group does not exist but should be created
-        let create_verified = if mime_parser.lookup_field("Chat-Verified").is_some() {
+        let create_verified = if mime_parser.get(HeaderDef::ChatVerified).is_some() {
             if let Err(err) =
                 check_verified_properties(context, mime_parser, from_id as u32, to_ids)
             {
@@ -971,7 +963,7 @@ fn create_or_lookup_group(
             VerifiedStatus::Unverified
         };
 
-        if allow_creation == 0 {
+        if !allow_creation {
             info!(context, "creating group forbidden by caller");
             return Ok((0, Blocked::Not));
         }
@@ -984,7 +976,7 @@ fn create_or_lookup_group(
             create_verified,
         );
         chat_id_blocked = create_blocked;
-        recreate_member_list = 1;
+        recreate_member_list = true;
     }
 
     // again, check chat_id
@@ -1009,8 +1001,8 @@ fn create_or_lookup_group(
 
     // execute group commands
     if X_MrAddToGrp.is_some() || X_MrRemoveFromGrp.is_some() {
-        recreate_member_list = 1;
-    } else if 0 != X_MrGrpNameChanged {
+        recreate_member_list = true;
+    } else if X_MrGrpNameChanged {
         if let Some(ref grpname) = grpname {
             if grpname.len() < 200 {
                 info!(context, "updating grpname for chat {}", chat_id);
@@ -1063,14 +1055,14 @@ fn create_or_lookup_group(
                     None => chat.param.remove(Param::ProfileImage),
                 };
                 chat.update_param(context)?;
-                send_EVENT_CHAT_MODIFIED = 1;
+                send_EVENT_CHAT_MODIFIED = true;
             }
         }
     }
 
     // add members to group/check members
     // for recreation: we should add a timestamp
-    if 0 != recreate_member_list {
+    if recreate_member_list {
         // TODO: the member list should only be recreated if the corresponding message is newer
         // than the one that is responsible for the current member list, see
         // https://github.com/deltachat/deltachat-core/issues/127
@@ -1100,11 +1092,11 @@ fn create_or_lookup_group(
                 chat::add_to_chat_contacts_table(context, chat_id, to_id);
             }
         }
-        send_EVENT_CHAT_MODIFIED = 1;
+        send_EVENT_CHAT_MODIFIED = true;
         chat::reset_gossiped_timestamp(context, chat_id);
     }
 
-    if 0 != send_EVENT_CHAT_MODIFIED {
+    if send_EVENT_CHAT_MODIFIED {
         context.call_cb(Event::ChatModified(chat_id));
     }
 
@@ -1134,8 +1126,8 @@ fn create_or_lookup_group(
 }
 
 /// try extract a grpid from a message-id list header value
-fn get_grpid_from_list(mime_parser: &MimeParser, header_key: &str) -> Option<String> {
-    if let Some(value) = mime_parser.lookup_field(header_key) {
+fn extract_grpid(mime_parser: &MimeParser, headerdef: HeaderDef) -> Option<String> {
+    if let Some(value) = mime_parser.get(headerdef) {
         for part in value.split(',').map(str::trim) {
             if !part.is_empty() {
                 if let Some(extracted_grpid) = dc_extract_grpid_from_rfc724_mid(part) {
@@ -1151,7 +1143,7 @@ fn get_grpid_from_list(mime_parser: &MimeParser, header_key: &str) -> Option<Str
 fn create_or_lookup_adhoc_group(
     context: &Context,
     mime_parser: &MimeParser,
-    allow_creation: i32,
+    allow_creation: bool,
     create_blocked: Blocked,
     from_id: u32,
     to_ids: &[u32],
@@ -1204,7 +1196,7 @@ fn create_or_lookup_adhoc_group(
         }
     }
 
-    if allow_creation == 0 {
+    if !allow_creation {
         info!(context, "creating ad-hoc group prevented from caller");
         return Ok((0, Blocked::Not));
     }
@@ -1223,11 +1215,9 @@ fn create_or_lookup_adhoc_group(
         return Ok((0, Blocked::Not));
     }
     // use subject as initial chat name
-    let grpname = if let Some(subject) = mime_parser.subject.as_ref().filter(|s| !s.is_empty()) {
-        subject.to_string()
-    } else {
+    let grpname = mime_parser.get_subject().unwrap_or_else(|| {
         context.stock_string_repl_int(StockMessage::Member, member_ids.len() as i32)
-    };
+    });
 
     // create group record
     let new_chat_id = create_group_record(
@@ -1395,7 +1385,7 @@ fn check_verified_properties(
 ) -> Result<()> {
     let contact = Contact::load_from_db(context, from_id)?;
 
-    ensure!(mimeparser.encrypted, "This message is not encrypted.");
+    ensure!(mimeparser.was_encrypted(), "This message is not encrypted.");
 
     // ensure, the contact is verified
     // and the message is signed with a verified key of the sender.
@@ -1488,23 +1478,23 @@ fn set_better_msg(mime_parser: &mut MimeParser, better_msg: impl AsRef<str>) {
     }
 }
 
-fn dc_is_reply_to_known_message(context: &Context, mime_parser: &MimeParser) -> i32 {
+fn is_reply_to_known_message(context: &Context, mime_parser: &MimeParser) -> bool {
     /* check if the message is a reply to a known message; the replies are identified by the Message-ID from
     `In-Reply-To`/`References:` (to support non-Delta-Clients) */
 
-    if let Some(field) = mime_parser.lookup_field("In-Reply-To") {
+    if let Some(field) = mime_parser.get(HeaderDef::InReplyTo) {
         if is_known_rfc724_mid_in_list(context, &field) {
-            return 1;
+            return true;
         }
     }
 
-    if let Some(field) = mime_parser.lookup_field("References") {
+    if let Some(field) = mime_parser.get(HeaderDef::References) {
         if is_known_rfc724_mid_in_list(context, &field) {
-            return 1;
+            return true;
         }
     }
 
-    0
+    false
 }
 
 fn is_known_rfc724_mid_in_list(context: &Context, mid_list: &str) -> bool {
@@ -1538,26 +1528,25 @@ fn is_known_rfc724_mid(context: &Context, rfc724_mid: &mailparse::MailAddr) -> b
         .unwrap_or_default()
 }
 
-fn dc_is_reply_to_messenger_message(context: &Context, mime_parser: &MimeParser) -> i32 {
-    /* function checks, if the message defined by mime_parser references a message send by us from Delta Chat.
-    This is similar to is_reply_to_known_message() but
-    - checks also if any of the referenced IDs are send by a messenger
-    - it is okay, if the referenced messages are moved to trash here
-    - no check for the Chat-* headers (function is only called if it is no messenger message itself) */
-
-    if let Some(value) = mime_parser.lookup_field("In-Reply-To") {
+/// Checks if the message defined by mime_parser references a message send by us from Delta Chat.
+/// This is similar to is_reply_to_known_message() but
+/// - checks also if any of the referenced IDs are send by a messenger
+/// - it is okay, if the referenced messages are moved to trash here
+/// - no check for the Chat-* headers (function is only called if it is no messenger message itself)
+fn is_reply_to_messenger_message(context: &Context, mime_parser: &MimeParser) -> bool {
+    if let Some(value) = mime_parser.get(HeaderDef::InReplyTo) {
         if is_msgrmsg_rfc724_mid_in_list(context, &value) {
-            return 1;
+            return true;
         }
     }
 
-    if let Some(value) = mime_parser.lookup_field("References") {
+    if let Some(value) = mime_parser.get(HeaderDef::References) {
         if is_msgrmsg_rfc724_mid_in_list(context, &value) {
-            return 1;
+            return true;
         }
     }
 
-    0
+    false
 }
 
 fn is_msgrmsg_rfc724_mid_in_list(context: &Context, mid_list: &str) -> bool {
@@ -1690,9 +1679,9 @@ mod tests {
                     \n\
                     hello\x00";
         let mimeparser = MimeParser::from_bytes(&context.ctx, &raw[..]).unwrap();
-        assert_eq!(get_grpid_from_list(&mimeparser, "In-Reply-To"), None);
+        assert_eq!(extract_grpid(&mimeparser, HeaderDef::InReplyTo), None);
         let grpid = Some("HcxyMARjyJy".to_string());
-        assert_eq!(get_grpid_from_list(&mimeparser, "References"), grpid);
+        assert_eq!(extract_grpid(&mimeparser, HeaderDef::References), grpid);
     }
 
     #[test]
@@ -1706,7 +1695,7 @@ mod tests {
                     hello\x00";
         let mimeparser = MimeParser::from_bytes(&context.ctx, &raw[..]).unwrap();
         let grpid = Some("HcxyMARjyJy".to_string());
-        assert_eq!(get_grpid_from_list(&mimeparser, "In-Reply-To"), grpid);
-        assert_eq!(get_grpid_from_list(&mimeparser, "References"), grpid);
+        assert_eq!(extract_grpid(&mimeparser, HeaderDef::InReplyTo), grpid);
+        assert_eq!(extract_grpid(&mimeparser, HeaderDef::References), grpid);
     }
 }

src/headerdef.rs

@@ -0,0 +1,43 @@
+#[derive(Debug, Display, Clone, PartialEq, Eq, EnumVariantNames)]
+#[strum(serialize_all = "kebab_case")]
+#[allow(dead_code)]
+pub enum HeaderDef {
+    MessageId,
+    Subject,
+    Date,
+    From_,
+    To,
+    Cc,
+    Disposition,
+    OriginalMessageId,
+    ListId,
+    References,
+    InReplyTo,
+    Precedence,
+    ChatVersion,
+    ChatGroupId,
+    ChatGroupName,
+    ChatGroupNameChanged,
+    ChatVerified,
+    ChatGroupImage,
+    ChatVoiceMessage,
+    ChatGroupMemberRemoved,
+    ChatGroupMemberAdded,
+    ChatContent,
+    ChatDuration,
+    ChatDispositionNotificationTo,
+    AutocryptSetupMessage,
+    SecureJoin,
+    SecureJoinGroup,
+    SecureJoinFingerprint,
+    SecureJoinInvitenumber,
+    SecureJoinAuth,
+    _TestHeader,
+}
+
+impl HeaderDef {
+    /// Returns the corresponding Event id.
+    pub fn get_headername(&self) -> String {
+        self.to_string()
+    }
+}

src/imap/idle.rs

@@ -18,6 +18,9 @@ pub enum Error {
     #[fail(display = "IMAP IDLE protocol failed to init/complete")]
     IdleProtocolFailed(#[cause] async_imap::error::Error),
 
+    #[fail(display = "IMAP IDLE protocol timed out")]
+    IdleTimeout(#[cause] async_std::future::TimeoutError),
+
     #[fail(display = "IMAP server does not have IDLE capability")]
     IdleAbilityMissing,
 
@@ -91,7 +94,17 @@ impl Imap {
                                 }
                             }
                         }
-                        match handle.done().await {
+                        // if we can't properly terminate the idle
+                        // protocol let's break the connection.
+                        let res =
+                            async_std::future::timeout(Duration::from_secs(15), handle.done())
+                                .await
+                                .map_err(|err| {
+                                    self.trigger_reconnect();
+                                    Error::IdleTimeout(err)
+                                })?;
+
+                        match res {
                             Ok(session) => {
                                 *self.session.lock().await = Some(Session::Secure(session));
                             }
@@ -135,7 +148,17 @@ impl Imap {
                                 }
                             }
                         }
-                        match handle.done().await {
+                        // if we can't properly terminate the idle
+                        // protocol let's break the connection.
+                        let res =
+                            async_std::future::timeout(Duration::from_secs(15), handle.done())
+                                .await
+                                .map_err(|err| {
+                                    self.trigger_reconnect();
+                                    Error::IdleTimeout(err)
+                                })?;
+
+                        match res {
                             Ok(session) => {
                                 *self.session.lock().await = Some(Session::Insecure(session));
                             }

src/job.rs

@@ -167,13 +167,15 @@ impl Job {
                 if let Some(recipients) = self.param.get(Param::Recipients) {
                     let recipients_list = recipients
                         .split('\x1e')
-                        .filter_map(|addr| match lettre::EmailAddress::new(addr.to_string()) {
-                            Ok(addr) => Some(addr),
-                            Err(err) => {
-                                warn!(context, "invalid recipient: {} {:?}", addr, err);
-                                None
-                            }
-                        })
+                        .filter_map(
+                            |addr| match async_smtp::EmailAddress::new(addr.to_string()) {
+                                Ok(addr) => Some(addr),
+                                Err(err) => {
+                                    warn!(context, "invalid recipient: {} {:?}", addr, err);
+                                    None
+                                }
+                            },
+                        )
                         .collect::<Vec<_>>();
 
                     /* if there is a msg-id and it does not exist in the db, cancel sending.
@@ -198,11 +200,11 @@ impl Job {
                         info!(context, "smtp-sending out mime message:");
                         println!("{}", String::from_utf8_lossy(&body));
                     }
-                    match smtp.send(context, recipients_list, body, self.job_id) {
+                    match task::block_on(smtp.send(context, recipients_list, body, self.job_id)) {
                         Err(crate::smtp::send::Error::SendError(err)) => {
                             // Remote error, retry later.
-                            smtp.disconnect();
                             info!(context, "SMTP failed to send: {}", err);
+                            smtp.disconnect();
                             self.try_again_later(TryAgain::AtOnce, Some(err.to_string()));
                         }
                         Err(crate::smtp::send::Error::EnvelopeError(err)) => {

src/lib.rs

@@ -33,6 +33,8 @@ mod log;
 #[macro_use]
 pub mod error;
 
+pub mod headerdef;
+
 pub(crate) mod events;
 pub use events::*;
 
@@ -73,7 +75,6 @@ mod token;
 mod wrapmime;
 mod dehtml;
 
-pub mod dc_array;
 pub mod dc_receive_imf;
 mod dc_simplify;
 pub mod dc_tools;

src/login_param.rs

@@ -16,7 +16,11 @@ use webpki_roots;
 pub enum CertificateChecks {
     Automatic = 0,
     Strict = 1,
-    AcceptInvalidHostnames = 2,
+
+    /// Same as AcceptInvalidCertificates
+    /// Previously known as AcceptInvalidHostnames, now deprecated.
+    AcceptInvalidCertificates2 = 2,
+
     AcceptInvalidCertificates = 3,
 }
 
@@ -288,14 +292,8 @@ pub fn dc_build_tls_config(certificate_checks: CertificateChecks) -> rustls::Cli
                 .dangerous()
                 .set_certificate_verifier(Arc::new(NoCertificateVerification {}));
         }
-        CertificateChecks::AcceptInvalidCertificates => {
-            // TODO: only accept invalid certs
-            config
-                .dangerous()
-                .set_certificate_verifier(Arc::new(NoCertificateVerification {}));
-        }
-        CertificateChecks::AcceptInvalidHostnames => {
-            // TODO: only accept invalid hostnames
+        CertificateChecks::AcceptInvalidCertificates
+        | CertificateChecks::AcceptInvalidCertificates2 => {
             config
                 .dangerous()
                 .set_certificate_verifier(Arc::new(NoCertificateVerification {}));
@@ -313,8 +311,8 @@ mod tests {
         use std::string::ToString;
 
         assert_eq!(
-            "accept_invalid_hostnames".to_string(),
-            CertificateChecks::AcceptInvalidHostnames.to_string()
+            "accept_invalid_certificates".to_string(),
+            CertificateChecks::AcceptInvalidCertificates.to_string()
         );
     }
 }

src/mimeparser.rs

@@ -14,6 +14,7 @@ use crate::dc_simplify::*;
 use crate::dc_tools::*;
 use crate::e2ee;
 use crate::error::Result;
+use crate::headerdef::HeaderDef;
 use crate::job::{job_add, Action};
 use crate::location;
 use crate::message;
@@ -28,10 +29,8 @@ use crate::{bail, ensure};
 pub struct MimeParser<'a> {
     pub context: &'a Context,
     pub parts: Vec<Part>,
-    pub header: HashMap<String, String>,
-    pub subject: Option<String>,
+    header: HashMap<String, String>,
     pub decrypting_failed: bool,
-    pub encrypted: bool,
     pub signatures: HashSet<String>,
     pub gossipped_addr: HashSet<String>,
     pub is_forwarded: bool,
@@ -73,9 +72,9 @@ impl<'a> MimeParser<'a> {
         let mut parser = MimeParser {
             parts: Vec::new(),
             header: Default::default(),
-            subject: None,
             decrypting_failed: false,
-            encrypted: false,
+
+            // only non-empty if it was a valid autocrypt message
             signatures: Default::default(),
             gossipped_addr: Default::default(),
             is_forwarded: false,
@@ -102,7 +101,6 @@ impl<'a> MimeParser<'a> {
         let mail = match e2ee::try_decrypt(parser.context, &mail, message_time) {
             Ok((raw, signatures)) => {
                 // Valid autocrypt message, encrypted
-                parser.encrypted = raw.is_some();
                 parser.signatures = signatures;
 
                 if let Some(raw) = raw {
@@ -147,11 +145,7 @@ impl<'a> MimeParser<'a> {
     }
 
     fn parse_headers(&mut self) -> Result<()> {
-        if let Some(field) = self.lookup_field("Subject") {
-            self.subject = Some(field.clone());
-        }
-
-        if self.lookup_field("Autocrypt-Setup-Message").is_some() {
+        if self.get(HeaderDef::AutocryptSetupMessage).is_some() {
             let has_setup_file = self.parts.iter().any(|p| {
                 p.mimetype.is_some() && p.mimetype.as_ref().unwrap().as_ref() == MIME_AC_SETUP_FILE
             });
@@ -182,12 +176,12 @@ impl<'a> MimeParser<'a> {
                     }
                 }
             }
-        } else if let Some(value) = self.lookup_field("Chat-Content") {
+        } else if let Some(value) = self.get(HeaderDef::ChatContent) {
             if value == "location-streaming-enabled" {
                 self.is_system_message = SystemMessage::LocationStreamingEnabled;
             }
         }
-        if self.lookup_field("Chat-Group-Image").is_some() && !self.parts.is_empty() {
+        if self.get(HeaderDef::ChatGroupImage).is_some() && !self.parts.is_empty() {
             let textpart = &self.parts[0];
             if textpart.typ == Viewtype::Text && self.parts.len() >= 2 {
                 let imgpart = &mut self.parts[1];
@@ -225,7 +219,7 @@ impl<'a> MimeParser<'a> {
                 std::mem::replace(&mut self.parts[0], filepart);
             }
         }
-        if let Some(ref subject) = self.subject {
+        if let Some(ref subject) = self.get_subject() {
             let mut prepend_subject = 1i32;
             if !self.decrypting_failed {
                 let colon = subject.find(':');
@@ -262,13 +256,13 @@ impl<'a> MimeParser<'a> {
         }
         if self.parts.len() == 1 {
             if self.parts[0].typ == Viewtype::Audio
-                && self.lookup_field("Chat-Voice-Message").is_some()
+                && self.get(HeaderDef::ChatVoiceMessage).is_some()
             {
                 let part_mut = &mut self.parts[0];
                 part_mut.typ = Viewtype::Voice;
             }
             if self.parts[0].typ == Viewtype::Image {
-                if let Some(value) = self.lookup_field("Chat-Content") {
+                if let Some(value) = self.get(HeaderDef::ChatContent) {
                     if value == "sticker" {
                         let part_mut = &mut self.parts[0];
                         part_mut.typ = Viewtype::Sticker;
@@ -280,7 +274,7 @@ impl<'a> MimeParser<'a> {
                 || part.typ == Viewtype::Voice
                 || part.typ == Viewtype::Video
             {
-                if let Some(field_0) = self.lookup_field("Chat-Duration") {
+                if let Some(field_0) = self.get(HeaderDef::ChatDuration) {
                     let duration_ms = field_0.parse().unwrap_or_default();
                     if duration_ms > 0 && duration_ms < 24 * 60 * 60 * 1000 {
                         let part_mut = &mut self.parts[0];
@@ -290,12 +284,12 @@ impl<'a> MimeParser<'a> {
             }
         }
         if !self.decrypting_failed {
-            if let Some(dn_field) = self.lookup_field("Chat-Disposition-Notification-To") {
+            if let Some(dn_field) = self.get(HeaderDef::ChatDispositionNotificationTo) {
                 if self.get_last_nonmeta().is_some() {
                     let addrs = mailparse::addrparse(&dn_field).unwrap();
 
                     if let Some(dn_to_addr) = addrs.first() {
-                        if let Some(from_field) = self.lookup_field("From") {
+                        if let Some(from_field) = self.get(HeaderDef::From_) {
                             let from_addrs = mailparse::addrparse(&from_field).unwrap();
 
                             if let Some(from_addr) = from_addrs.first() {
@@ -316,7 +310,7 @@ impl<'a> MimeParser<'a> {
             let mut part = Part::default();
             part.typ = Viewtype::Text;
 
-            if let Some(ref subject) = self.subject {
+            if let Some(ref subject) = self.get_subject() {
                 if !self.has_chat_version() {
                     part.msg = subject.to_string();
                 }
@@ -336,12 +330,27 @@ impl<'a> MimeParser<'a> {
         self.parts.iter_mut().rev().find(|part| !part.is_meta)
     }
 
+    pub fn was_encrypted(&self) -> bool {
+        !self.signatures.is_empty()
+    }
+
     pub(crate) fn has_chat_version(&self) -> bool {
         self.header.contains_key("chat-version")
     }
 
-    pub fn lookup_field(&self, field_name: &str) -> Option<&String> {
-        self.header.get(&field_name.to_lowercase())
+    pub(crate) fn get_subject(&self) -> Option<String> {
+        if let Some(s) = self.get(HeaderDef::Subject) {
+            if s.is_empty() {
+                return None;
+            }
+            Some(s.to_string())
+        } else {
+            None
+        }
+    }
+
+    pub fn get(&self, headerdef: HeaderDef) -> Option<&String> {
+        self.header.get(&headerdef.get_headername())
     }
 
     fn parse_mime_recursive(&mut self, mail: &mailparse::ParsedMail<'_>) -> Result<bool> {
@@ -445,6 +454,9 @@ impl<'a> MimeParser<'a> {
                 }
             }
             (mime::MULTIPART, "encrypted") => {
+                // we currently do not try to decrypt non-autocrypt messages
+                // at all. If we see an encrypted part, we set
+                // decrypting_failed.
                 let msg_body = self.context.stock_str(StockMessage::CantDecryptMsgBody);
                 let txt = format!("[{}]", msg_body);
 
@@ -631,25 +643,18 @@ impl<'a> MimeParser<'a> {
     }
 
     fn do_add_single_part(&mut self, mut part: Part) {
-        if self.encrypted {
-            if !self.signatures.is_empty() {
-                part.param.set_int(Param::GuaranteeE2ee, 1);
-            } else {
-                // XXX if the message was encrypted but not signed
-                // it's not neccessarily an error we need to signal.
-                // we could just treat it as if it was not encrypted.
-                part.param.set_int(Param::ErroneousE2ee, 0x2);
-            }
+        if self.was_encrypted() {
+            part.param.set_int(Param::GuaranteeE2ee, 1);
         }
         self.parts.push(part);
     }
 
     pub fn is_mailinglist_message(&self) -> bool {
-        if self.lookup_field("List-Id").is_some() {
+        if self.get(HeaderDef::ListId).is_some() {
             return true;
         }
 
-        if let Some(precedence) = self.lookup_field("Precedence") {
+        if let Some(precedence) = self.get(HeaderDef::Precedence) {
             precedence == "list" || precedence == "bulk"
         } else {
             false
@@ -658,7 +663,7 @@ impl<'a> MimeParser<'a> {
 
     pub fn sender_equals_recipient(&self) -> bool {
         /* get From: and check there is exactly one sender */
-        if let Some(field) = self.lookup_field("From") {
+        if let Some(field) = self.get(HeaderDef::From_) {
             if let Ok(addrs) = mailparse::addrparse(field) {
                 if addrs.len() != 1 {
                     return false;
@@ -689,11 +694,11 @@ impl<'a> MimeParser<'a> {
     }
 
     pub fn get_rfc724_mid(&self) -> Option<String> {
-        // get Message-ID from header
-        if let Some(field) = self.lookup_field("Message-ID") {
-            return parse_message_id(field);
+        if let Some(msgid) = self.get(HeaderDef::MessageId) {
+            parse_message_id(msgid)
+        } else {
+            None
         }
-        None
     }
 
     fn hash_header(&mut self, fields: &[mailparse::MailHeader<'_>]) {
@@ -723,9 +728,10 @@ impl<'a> MimeParser<'a> {
         let (report_fields, _) = mailparse::parse_headers(&report_body)?;
 
         // must be present
-        if let Some(_disposition) = report_fields.get_first_value("Disposition").ok().flatten() {
+        let disp = HeaderDef::Disposition.get_headername();
+        if let Some(_disposition) = report_fields.get_first_value(&disp).ok().flatten() {
             if let Some(original_message_id) = report_fields
-                .get_first_value("Original-Message-ID")
+                .get_first_value(&HeaderDef::OriginalMessageId.get_headername())
                 .ok()
                 .flatten()
                 .and_then(|v| parse_message_id(&v))
@@ -1016,7 +1022,7 @@ mod tests {
         let raw = include_bytes!("../test-data/message/issue_523.txt");
         let mimeparser = MimeParser::from_bytes(&context.ctx, &raw[..]).unwrap();
 
-        assert_eq!(mimeparser.subject, None);
+        assert_eq!(mimeparser.get_subject(), None);
         assert_eq!(mimeparser.parts.len(), 1);
     }
 
@@ -1104,14 +1110,14 @@ mod tests {
         let raw = b"From: hello\n\
                     Content-Type: multipart/mixed; boundary=\"==break==\";\n\
                     Subject: outer-subject\n\
-                    X-Special-A: special-a\n\
-                    Foo: Bar\nChat-Version: 0.0\n\
+                    Secure-Join-Group: no\n\
+                    Test-Header: Bar\nChat-Version: 0.0\n\
                     \n\
                     --==break==\n\
                     Content-Type: text/plain; protected-headers=\"v1\";\n\
                     Subject: inner-subject\n\
-                    X-Special-B: special-b\n\
-                    Foo: Xy\n\
+                    SecureBar-Join-Group: yes\n\
+                    Test-Header: Xy\n\
                     Chat-Version: 1.0\n\
                     \n\
                     test1\n\
@@ -1121,15 +1127,18 @@ mod tests {
                     \x00";
         let mimeparser = MimeParser::from_bytes(&context.ctx, &raw[..]).unwrap();
 
-        assert_eq!(mimeparser.subject, Some("inner-subject".into()));
+        // test that we treat Subject as a protected header that can
+        // bubble upwards
+        assert_eq!(mimeparser.get_subject(), Some("inner-subject".into()));
 
-        let of = mimeparser.lookup_field("X-Special-A").unwrap();
-        assert_eq!(of, "special-a");
+        let of = mimeparser.get(HeaderDef::SecureJoinGroup).unwrap();
+        assert_eq!(of, "no");
 
-        let of = mimeparser.lookup_field("Foo").unwrap();
+        // unprotected headers do not bubble upwards
+        let of = mimeparser.get(HeaderDef::_TestHeader).unwrap();
         assert_eq!(of, "Bar");
 
-        let of = mimeparser.lookup_field("Chat-Version").unwrap();
+        let of = mimeparser.get(HeaderDef::ChatVersion).unwrap();
         assert_eq!(of, "1.0");
         assert_eq!(mimeparser.parts.len(), 1);
     }

src/param.rs

@@ -250,7 +250,7 @@ impl Params {
         let file = ParamsFile::from_param(context, val)?;
         let blob = match file {
             ParamsFile::FsPath(path) => match create {
-                true => BlobObject::create_from_path(context, path)?,
+                true => BlobObject::new_from_path(context, path)?,
                 false => BlobObject::from_path(context, path)?,
             },
             ParamsFile::Blob(blob) => blob,

src/securejoin.rs

@@ -11,6 +11,7 @@ use crate::context::Context;
 use crate::e2ee::*;
 use crate::error::Error;
 use crate::events::Event;
+use crate::headerdef::HeaderDef;
 use crate::key::*;
 use crate::lot::LotState;
 use crate::message::Message;
@@ -350,7 +351,7 @@ pub(crate) fn handle_securejoin_handshake(
         "handle_securejoin_handshake(): called with special contact id"
     );
     let step = mimeparser
-        .lookup_field("Secure-Join")
+        .get(HeaderDef::SecureJoin)
         .ok_or_else(|| format_err!("This message is not a Secure-Join message"))?;
 
     info!(
@@ -378,7 +379,7 @@ pub(crate) fn handle_securejoin_handshake(
             // it just ensures, we have Bobs key now. If we do _not_ have the key because eg. MitM has removed it,
             // send_message() will fail with the error "End-to-end-encryption unavailable unexpectedly.", so, there is no additional check needed here.
             // verify that the `Secure-Join-Invitenumber:`-header matches invitenumber written to the QR code
-            let invitenumber = match mimeparser.lookup_field("Secure-Join-Invitenumber") {
+            let invitenumber = match mimeparser.get(HeaderDef::SecureJoinInvitenumber) {
                 Some(n) => n,
                 None => {
                     warn!(context, "Secure-join denied (invitenumber missing).",);
@@ -422,7 +423,7 @@ pub(crate) fn handle_securejoin_handshake(
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
-                    if mimeparser.encrypted {
+                    if mimeparser.was_encrypted() {
                         "No valid signature."
                     } else {
                         "Not encrypted."
@@ -467,7 +468,7 @@ pub(crate) fn handle_securejoin_handshake(
             ====  Step 6 in "Out-of-band verified groups" protocol  ====
             ============================================================ */
             // verify that Secure-Join-Fingerprint:-header matches the fingerprint of Bob
-            let fingerprint = match mimeparser.lookup_field("Secure-Join-Fingerprint") {
+            let fingerprint = match mimeparser.get(HeaderDef::SecureJoinFingerprint) {
                 Some(fp) => fp,
                 None => {
                     could_not_establish_secure_connection(
@@ -496,7 +497,7 @@ pub(crate) fn handle_securejoin_handshake(
             }
             info!(context, "Fingerprint verified.",);
             // verify that the `Secure-Join-Auth:`-header matches the secret written to the QR code
-            let auth_0 = match mimeparser.lookup_field("Secure-Join-Auth") {
+            let auth_0 = match mimeparser.get(HeaderDef::SecureJoinAuth) {
                 Some(auth) => auth,
                 None => {
                     could_not_establish_secure_connection(
@@ -526,7 +527,7 @@ pub(crate) fn handle_securejoin_handshake(
             inviter_progress!(context, contact_id, 600);
             if join_vg {
                 let field_grpid = mimeparser
-                    .lookup_field("Secure-Join-Group")
+                    .get(HeaderDef::SecureJoinGroup)
                     .map(|s| s.as_str())
                     .unwrap_or_else(|| "");
                 let (group_chat_id, _, _) = chat::get_chat_id_by_grpid(context, field_grpid);
@@ -600,7 +601,7 @@ pub(crate) fn handle_securejoin_handshake(
             Contact::scaleup_origin_by_id(context, contact_id, Origin::SecurejoinJoined);
             emit_event!(context, Event::ContactsChanged(None));
             let cg_member_added = mimeparser
-                .lookup_field("Chat-Group-Member-Added")
+                .get(HeaderDef::ChatGroupMemberAdded)
                 .map(|s| s.as_str())
                 .unwrap_or_else(|| "");
             if join_vg && !addr_equals_self(context, cg_member_added) {
@@ -635,7 +636,7 @@ pub(crate) fn handle_securejoin_handshake(
                 inviter_progress!(context, contact_id, 800);
                 inviter_progress!(context, contact_id, 1000);
                 let field_grpid = mimeparser
-                    .lookup_field("Secure-Join-Group")
+                    .get(HeaderDef::SecureJoinGroup)
                     .map(|s| s.as_str())
                     .unwrap_or_else(|| "");
                 let (group_chat_id, _, _) = chat::get_chat_id_by_grpid(context, &field_grpid);
@@ -717,7 +718,7 @@ fn mark_peer_as_verified(context: &Context, fingerprint: impl AsRef<str>) -> Res
  ******************************************************************************/
 
 fn encrypted_and_signed(mimeparser: &MimeParser, expected_fingerprint: impl AsRef<str>) -> bool {
-    if !mimeparser.encrypted {
+    if !mimeparser.was_encrypted() {
         warn!(mimeparser.context, "Message not encrypted.",);
         false
     } else if mimeparser.signatures.is_empty() {

src/smtp/mod.rs

@@ -2,8 +2,10 @@
 
 pub mod send;
 
-use lettre::smtp::client::net::*;
-use lettre::*;
+use async_smtp::smtp::client::net::*;
+use async_smtp::*;
+
+use async_std::task;
 
 use crate::constants::*;
 use crate::context::Context;
@@ -19,12 +21,12 @@ pub enum Error {
     InvalidLoginAddress {
         address: String,
         #[cause]
-        error: lettre::error::Error,
+        error: async_smtp::error::Error,
     },
     #[fail(display = "SMTP failed to connect: {:?}", _0)]
-    ConnectionFailure(#[cause] lettre::smtp::error::Error),
+    ConnectionFailure(#[cause] async_smtp::smtp::error::Error),
     #[fail(display = "SMTP: failed to setup connection {:?}", _0)]
-    ConnectionSetupFailure(#[cause] lettre::smtp::error::Error),
+    ConnectionSetupFailure(#[cause] async_smtp::smtp::error::Error),
     #[fail(display = "SMTP: oauth2 error {:?}", _0)]
     Oauth2Error { address: String },
 }
@@ -34,8 +36,7 @@ pub type Result<T> = std::result::Result<T, Error>;
 #[derive(Default, DebugStub)]
 pub struct Smtp {
     #[debug_stub(some = "SmtpTransport")]
-    transport: Option<lettre::smtp::SmtpTransport>,
-    transport_connected: bool,
+    transport: Option<async_smtp::smtp::SmtpTransport>,
     /// Email address we are sending from.
     from: Option<EmailAddress>,
 }
@@ -48,16 +49,12 @@ impl Smtp {
 
     /// Disconnect the SMTP transport and drop it entirely.
     pub fn disconnect(&mut self) {
-        if self.transport.is_none() || !self.transport_connected {
-            return;
+        if let Some(ref mut transport) = self.transport.take() {
+            transport.close();
         }
-
-        let mut transport = self.transport.take().unwrap();
-        transport.close();
-        self.transport_connected = false;
     }
 
-    /// Check if a connection already exists.
+    /// check whether we are connected
     pub fn is_connected(&self) -> bool {
         self.transport.is_some()
     }
@@ -99,21 +96,21 @@ impl Smtp {
             }
             let user = &lp.send_user;
             (
-                lettre::smtp::authentication::Credentials::new(
+                async_smtp::smtp::authentication::Credentials::new(
                     user.to_string(),
                     access_token.unwrap_or_default(),
                 ),
-                vec![lettre::smtp::authentication::Mechanism::Xoauth2],
+                vec![async_smtp::smtp::authentication::Mechanism::Xoauth2],
             )
         } else {
             // plain
             let user = lp.send_user.clone();
             let pw = lp.send_pw.clone();
             (
-                lettre::smtp::authentication::Credentials::new(user, pw),
+                async_smtp::smtp::authentication::Credentials::new(user, pw),
                 vec![
-                    lettre::smtp::authentication::Mechanism::Plain,
-                    lettre::smtp::authentication::Mechanism::Login,
+                    async_smtp::smtp::authentication::Mechanism::Plain,
+                    async_smtp::smtp::authentication::Mechanism::Login,
                 ],
             )
         };
@@ -121,24 +118,26 @@ impl Smtp {
         let security = if 0
             != lp.server_flags & (DC_LP_SMTP_SOCKET_STARTTLS | DC_LP_SMTP_SOCKET_PLAIN) as i32
         {
-            lettre::smtp::ClientSecurity::Opportunistic(tls_parameters)
+            async_smtp::smtp::ClientSecurity::Opportunistic(tls_parameters)
         } else {
-            lettre::smtp::ClientSecurity::Wrapper(tls_parameters)
+            async_smtp::smtp::ClientSecurity::Wrapper(tls_parameters)
         };
 
-        let client = lettre::smtp::SmtpClient::new((domain.as_str(), port), security)
-            .map_err(Error::ConnectionSetupFailure)?;
+        let client = task::block_on(async_smtp::smtp::SmtpClient::with_security(
+            (domain.as_str(), port),
+            security,
+        ))
+        .map_err(Error::ConnectionSetupFailure)?;
 
         let client = client
             .smtp_utf8(true)
             .credentials(creds)
             .authentication_mechanism(mechanism)
-            .connection_reuse(lettre::smtp::ConnectionReuseParameters::ReuseUnlimited);
-        let mut trans = client.transport();
-        trans.connect().map_err(Error::ConnectionFailure)?;
+            .connection_reuse(async_smtp::smtp::ConnectionReuseParameters::ReuseUnlimited);
+        let mut trans = client.into_transport();
+        task::block_on(trans.connect()).map_err(Error::ConnectionFailure)?;
 
         self.transport = Some(trans);
-        self.transport_connected = true;
         context.call_cb(Event::SmtpConnected(format!(
             "SMTP-LOGIN as {} ok",
             lp.send_user,

src/smtp/send.rs

@@ -1,7 +1,7 @@
 //! # SMTP message sending
 
 use super::Smtp;
-use lettre::*;
+use async_smtp::*;
 
 use crate::context::Context;
 use crate::events::Event;
@@ -11,9 +11,9 @@ pub type Result<T> = std::result::Result<T, Error>;
 #[derive(Debug, Fail)]
 pub enum Error {
     #[fail(display = "Envelope error: {}", _0)]
-    EnvelopeError(#[cause] lettre::error::Error),
+    EnvelopeError(#[cause] async_smtp::error::Error),
     #[fail(display = "Send error: {}", _0)]
-    SendError(#[cause] lettre::smtp::error::Error),
+    SendError(#[cause] async_smtp::smtp::error::Error),
     #[fail(display = "SMTP has no transport")]
     NoTransport,
 }
@@ -21,7 +21,7 @@ pub enum Error {
 impl Smtp {
     /// Send a prepared mail to recipients.
     /// On successful send out Ok() is returned.
-    pub fn send(
+    pub async fn send(
         &mut self,
         context: &Context,
         recipients: Vec<EmailAddress>,
@@ -36,22 +36,21 @@ impl Smtp {
             .collect::<Vec<String>>()
             .join(",");
 
+        let envelope =
+            Envelope::new(self.from.clone(), recipients).map_err(Error::EnvelopeError)?;
+        let mail = SendableEmail::new(
+            envelope,
+            format!("{}", job_id), // only used for internal logging
+            message,
+        );
         if let Some(ref mut transport) = self.transport {
-            let envelope =
-                Envelope::new(self.from.clone(), recipients).map_err(Error::EnvelopeError)?;
-            let mail = SendableEmail::new(
-                envelope,
-                format!("{}", job_id), // only used for internal logging
-                message,
-            );
-
-            transport.send(mail).map_err(Error::SendError)?;
+            transport.send(mail).await.map_err(Error::SendError)?;
 
             context.call_cb(Event::SmtpMessageSent(format!(
                 "Message len={} was smtp-sent to {}",
                 message_len, recipients_display
             )));
-            self.transport_connected = true;
+
             Ok(())
         } else {
             warn!(