Download & decrypt uploaded attachements.

- Parse the "Chat-Upload-Url" header
- Add message::schedule_download function to schedule download job
- In the job download, decrypt, save

CHANGELOG.md

@@ -1,5 +1,28 @@
 # Changelog 
 
+## 1.35.0
+
+- enable strict-tls from a new provider-db setting #1587
+
+- new subject 'Message from USER' for one-to-one chats #1395
+
+- recode images #1563
+
+- improve reconnect handling #1549 #1580
+
+- improve importing addresses #1544
+
+- improve configure and folder detection #1539 #1548
+
+- improve test suite #1559 #1564 #1580 #1581 #1582 #1584 #1588:
+
+- fix ad-hoc groups #1566
+
+- preventions against being marked as spam #1575
+
+- refactorings #1542 #1569
+
+
 ## 1.34.0
 
 - new api for io, thread and event handling #1356,

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.34.0"
+version = "1.35.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
@@ -12,12 +12,12 @@ license = "MPL-2.0"
 deltachat_derive = { path = "./deltachat_derive" }
 
 libc = "0.2.51"
-pgp = { version = "0.5.1", default-features = false } 
+pgp = { version = "0.6.0", default-features = false } 
 hex = "0.4.0"
-sha2 = "0.8.0"
+sha2 = "0.9.0"
 rand = "0.7.0"
 smallvec = "1.0.0"
-surf = { version = "2.0.0-alpha.2", default-features = false, features = ["h1-client"] }
+surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
 num-derive = "0.3.0"
 num-traits = "0.2.6"
 async-smtp = { version = "0.3" }
@@ -25,8 +25,8 @@ email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 async-imap = "0.3.1"
 async-native-tls = { version = "0.3.3" }
-async-std = { version = "1.6.0", features = ["unstable"] }
-base64 = "0.11"
+async-std = { version = "1.6.1", features = ["unstable"] }
+base64 = "0.12"
 charset = "0.1"
 percent-encoding = "2.0"
 serde = { version = "1.0", features = ["derive"] }
@@ -35,42 +35,42 @@ chrono = "0.4.6"
 indexmap = "1.3.0"
 lazy_static = "1.4.0"
 regex = "1.1.6"
-rusqlite = { version = "0.22", features = ["bundled"] }
-r2d2_sqlite = "0.15.0"
+rusqlite = { version = "0.23", features = ["bundled"] }
+r2d2_sqlite = "0.16.0"
 r2d2 = "0.8.5"
-strum = "0.16.0"
-strum_macros = "0.16.0"
+strum = "0.18.0"
+strum_macros = "0.18.0"
 backtrace = "0.3.33"
 byteorder = "1.3.1"
 itertools = "0.8.0"
 image-meta = "0.1.0"
-quick-xml = "0.17.1"
+quick-xml = "0.18.1"
 escaper = "0.1.0"
 bitflags = "1.1.0"
-debug_stub_derive = "0.3.0"
 sanitize-filename = "0.2.1"
 stop-token = { version = "0.1.1", features = ["unstable"] }
 mailparse = "0.12.1"
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
 native-tls = "0.2.3"
-image = { version = "0.22.4", default-features=false, features = ["gif_codec", "jpeg", "ico", "png_codec", "pnm", "webp", "bmp"] }
+image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
 futures = "0.3.4"
 thiserror = "1.0.14"
 anyhow = "1.0.28"
 async-trait = "0.1.31"
 url = "2.1.1"
 
-pretty_env_logger = { version = "0.3.1", optional = true }
+pretty_env_logger = { version = "0.4.0", optional = true }
 log = {version = "0.4.8", optional = true }
 rustyline = { version = "4.1.0", optional = true }
 ansi_term = { version = "0.12.1", optional = true }
+open = { version = "1.4.0", optional = true }
 
 
 [dev-dependencies]
 tempfile = "3.0"
 pretty_assertions = "0.6.1"
-pretty_env_logger = "0.3.0"
-proptest = "0.9.4"
+pretty_env_logger = "0.4.0"
+proptest = "0.10"
 async-std = { version = "1.6.0", features = ["unstable", "attributes"] }
 smol = "0.1.10"
 
@@ -94,7 +94,7 @@ required-features = ["repl"]
 [features]
 default = []
 internals = []
-repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term"]
+repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "open"]
 vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored"]
 nightly = ["pgp/nightly"]
 

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.34.0"
+version = "1.35.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/deltachat.h

@@ -3163,6 +3163,54 @@ int             dc_msg_is_setupmessage        (const dc_msg_t* msg);
 char*           dc_msg_get_setupcodebegin     (const dc_msg_t* msg);
 
 
+/**
+ * Check if the message is completely downloaded
+ * or if some further action is needed.
+ *
+ * The function returns one of:
+ * - @ref DC_DOWNLOAD_NO_URL      - The message does not need any further download action
+ *                                  and should be rendered as usual.
+ * - @ref DC_DOWNLOAD_AVAILABLE   - There is additional content to download.
+ *                                  Tn addition to the usual message rendering,
+ *                                  the UI shall show a download button that starts dc_schedule_download()
+ * - @ref DC_DOWNLOAD_IN_PROGRESS - Download was started with dc_schedule_download() and is still in progress.
+ *                                  On progress changes and if the download fails or succeeds,
+ *                                  the event @ref DC_EVENT_DOWNLOAD_PROGRESS will be emitted.
+ * - @ref DC_DOWNLOAD_DONE        - Download finished successfully
+ * - @ref DC_DOWNLOAD_FAILURE     - Download error, the user may start over calling dc_schedule_download() again.
+ *
+ * @memberof dc_msg_t
+ * @param msg The message object.
+ * @return One of the @ref DC_DOWNLOAD values
+ */
+int dc_msg_download_status(const dc_msg_t* msg);
+
+
+/**
+ * Advices the core to start downloading a message.
+ * This function is typically called when the user hits the "Download" button
+ * that is shown by the UI in case dc_msg_download_status()
+ * returns @ref DC_DOWNLOAD_AVAILABLE or @ref DC_DOWNLOAD_FAILURE.
+ *
+ * The UI may want to show a file selector and let the user chose a download location.
+ * The file name in the file selector may be prefilled using dc_msg_get_filename().
+ *
+ * During the download, the progress, errors and success
+ * are reported using @ref DC_EVENT_DOWNLOAD_PROGRESS.
+ *
+ * Once the @ref DC_EVENT_DOWNLOAD_PROGRESS reports success,
+ * The file can be accessed as usual using dc_msg_get_file().
+ *
+ * @memberof dc_context_t
+ * @param context The context object.
+ * @param path Path to the destination file.
+ *     You can specify NULL here to download
+ *     to a reasonable file name in the internal blob-directory.
+ * @param msg_id Message-ID to download the content for.
+ */
+void dc_schedule_download(dc_context_t* context, int msg_id, const char* path);
+
+
 /**
  * Set the text of a message object.
  * This does not alter any information in the database; this may be done by dc_send_msg() later.
@@ -4231,6 +4279,16 @@ void dc_event_unref(dc_event_t* event);
  */
 #define DC_EVENT_SECUREJOIN_JOINER_PROGRESS       2061
 
+
+/**
+ * Inform about the progress of a download started by dc_schedule_download().
+ *
+ * @param data1 (int) Message-ID the progress is reported for.
+ * @param data2 (int) 0=error, 1-999=progress in permille, 1000=success and done
+ */
+#define DC_EVENT_DOWNLOAD_PROGRESS       2070
+
+
 /**
  * @}
  */
@@ -4370,6 +4428,29 @@ void dc_event_unref(dc_event_t* event);
  */
 
 
+/**
+ * @defgroup DC_DOWNLOAD DC_DOWNLOAD
+ *
+ * These constants describe the download state of a message.
+ * The download state can be retrieved using dc_msg_download_status()
+ * and usually changes after calling dc_schedule_download().
+ *
+ * @addtogroup DC_DOWNLOAD
+ * @{
+ */
+
+#define DC_DOWNLOAD_NO_URL      10 ///< Download not needed, see dc_msg_download_status() for details.
+#define DC_DOWNLOAD_AVAILABLE   20 ///< Download available, see dc_msg_download_status() for details.
+#define DC_DOWNLOAD_IN_PROGRESS 30 ///< Download in progress, see dc_msg_download_status() for details.
+#define DC_DOWNLOAD_DONE        40 ///< Download done, see dc_msg_download_status() for details.
+#define DC_DOWNLOAD_FAILURE     50 ///< Download failed, see dc_msg_download_status() for details.
+
+/**
+ * @}
+ */
+
+
+
 /*
  * TODO: Strings need some doumentation about used placeholders.
  *
@@ -4434,6 +4515,13 @@ void dc_event_unref(dc_event_t* event);
  * @}
  */
 
+#ifdef PY_CFFI_INC
+/* Helper utility to locate the header file when building python bindings. */
+char* _dc_header_file_location(void) {
+    return __FILE__;
+}
+#endif
+
 
 #ifdef __cplusplus
 }

examples/repl/cmdline.rs

@@ -370,6 +370,8 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  ===========================Message commands==\n\
                  listmsgs <query>\n\
                  msginfo <msg-id>\n\
+                 openfile <msg-id>\n\
+                 download <msg-id>\n\
                  listfresh\n\
                  forward <msg-id> <chat-id>\n\
                  markseen <msg-id>\n\
@@ -890,6 +892,25 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let res = message::get_msg_info(&context, id).await;
             println!("{}", res);
         }
+        "openfile" => {
+            ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
+            let id = MsgId::new(arg1.parse()?);
+            let msg = Message::load_from_db(&context, id).await?;
+            let filepath = msg.get_file(&context);
+            ensure!(filepath.is_some(), "Message has no file.");
+            let filepath = filepath.unwrap();
+            open::that(filepath)?;
+        }
+        "download" => {
+            ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
+            let id = MsgId::new(arg1.parse()?);
+            let path = if !arg2.is_empty() {
+                Some(arg2.into())
+            } else {
+                None
+            };
+            message::schedule_download(&context, id, path).await?;
+        }
         "listfresh" => {
             let msglist = context.get_fresh_msgs().await;
 

examples/repl/main.rs

@@ -186,9 +186,11 @@ const CHAT_COMMANDS: [&str; 26] = [
     "unpin",
     "delchat",
 ];
-const MESSAGE_COMMANDS: [&str; 8] = [
+const MESSAGE_COMMANDS: [&str; 10] = [
     "listmsgs",
     "msginfo",
+    "openfile",
+    "download",
     "listfresh",
     "forward",
     "markseen",

python/src/deltachat/_build.py

@@ -1,65 +1,64 @@
 import distutils.ccompiler
 import distutils.log
 import distutils.sysconfig
-import tempfile
-import platform
 import os
-import cffi
+import platform
+import re
 import shutil
-from os.path import dirname as dn
+import subprocess
+import tempfile
+import textwrap
+import types
 from os.path import abspath
+from os.path import dirname as dn
+
+import cffi
 
 
-def ffibuilder():
-    projdir = os.environ.get('DCC_RS_DEV')
-    if not projdir:
-        p = dn(dn(dn(dn(abspath(__file__)))))
-        projdir = os.environ["DCC_RS_DEV"] = p
-    target = os.environ.get('DCC_RS_TARGET', 'release')
-    if projdir:
-        if platform.system() == 'Darwin':
-            libs = ['resolv', 'dl']
-            extra_link_args = [
-                    '-framework', 'CoreFoundation',
-                    '-framework', 'CoreServices',
-                    '-framework', 'Security',
-            ]
-        elif platform.system() == 'Linux':
-            libs = ['rt', 'dl', 'm']
-            extra_link_args = []
-        else:
-            raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
-        target_dir = os.environ.get("CARGO_TARGET_DIR")
-        if target_dir is None:
-            target_dir = os.path.join(projdir, 'target')
-        objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
-        assert os.path.exists(objs[0]), objs
-        incs = [os.path.join(projdir, 'deltachat-ffi')]
+def local_build_flags(projdir, target):
+    """Construct build flags for building against a checkout.
+
+    :param projdir: The root directory of the deltachat-core-rust project.
+    :param target: The rust build target, `debug` or `release`.
+    """
+    flags = types.SimpleNamespace()
+    if platform.system() == 'Darwin':
+        flags.libs = ['resolv', 'dl']
+        flags.extra_link_args = [
+                '-framework', 'CoreFoundation',
+                '-framework', 'CoreServices',
+                '-framework', 'Security',
+        ]
+    elif platform.system() == 'Linux':
+        flags.libs = ['rt', 'dl', 'm']
+        flags.extra_link_args = []
     else:
-        libs = ['deltachat']
-        objs = []
-        incs = []
-        extra_link_args = []
-    builder = cffi.FFI()
-    builder.set_source(
-        'deltachat.capi',
-        """
-            #include <deltachat.h>
-            int dc_event_has_string_data(int e)
-            {
-                return DC_EVENT_DATA2_IS_STRING(e);
-            }
-        """,
-        include_dirs=incs,
-        libraries=libs,
-        extra_objects=objs,
-        extra_link_args=extra_link_args,
-    )
-    builder.cdef("""
-        typedef int... time_t;
-        void free(void *ptr);
-        extern int dc_event_has_string_data(int);
-    """)
+        raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
+    target_dir = os.environ.get("CARGO_TARGET_DIR")
+    if target_dir is None:
+        target_dir = os.path.join(projdir, 'target')
+    flags.objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
+    assert os.path.exists(flags.objs[0]), flags.objs
+    flags.incs = [os.path.join(projdir, 'deltachat-ffi')]
+    return flags
+
+
+def system_build_flags():
+    """Construct build flags for building against an installed libdeltachat."""
+    flags = types.SimpleNamespace()
+    flags.libs = ['deltachat']
+    flags.objs = []
+    flags.incs = []
+    flags.extra_link_args = []
+
+
+def extract_functions(flags):
+    """Extract the function definitions from deltachat.h.
+
+    This creates a .h file with a single `#include <deltachat.h>` line
+    in it.  It then runs the C preprocessor to create an output file
+    which contains all function definitions found in `deltachat.h`.
+    """
     distutils.log.set_verbosity(distutils.log.INFO)
     cc = distutils.ccompiler.new_compiler(force=True)
     distutils.sysconfig.customize_compiler(cc)
@@ -71,13 +70,133 @@ def ffibuilder():
             src_fp.write('#include <deltachat.h>')
         cc.preprocess(source=src_name,
                       output_file=dst_name,
-                      include_dirs=incs,
+                      include_dirs=flags.incs,
                       macros=[('PY_CFFI', '1')])
         with open(dst_name, "r") as dst_fp:
-            builder.cdef(dst_fp.read())
+            return dst_fp.read()
     finally:
         shutil.rmtree(tmpdir)
 
+
+def find_header(flags):
+    """Use the compiler to find the deltachat.h header location.
+
+    This uses a small utility in deltachat.h to find the location of
+    the header file location.
+    """
+    distutils.log.set_verbosity(distutils.log.INFO)
+    cc = distutils.ccompiler.new_compiler(force=True)
+    distutils.sysconfig.customize_compiler(cc)
+    tmpdir = tempfile.mkdtemp()
+    try:
+        src_name = os.path.join(tmpdir, "where.c")
+        obj_name = os.path.join(tmpdir, "where.o")
+        dst_name = os.path.join(tmpdir, "where")
+        with open(src_name, "w") as src_fp:
+            src_fp.write(textwrap.dedent("""
+                #include <stdio.h>
+                #include <deltachat.h>
+
+                int main(void) {
+                    printf("%s", _dc_header_file_location());
+                    return 0;
+                }
+            """))
+        cwd = os.getcwd()
+        try:
+            os.chdir(tmpdir)
+            cc.compile(sources=["where.c"],
+                       include_dirs=flags.incs,
+                       macros=[("PY_CFFI_INC", "1")])
+        finally:
+            os.chdir(cwd)
+        cc.link_executable(objects=[obj_name],
+                           output_progname="where",
+                           output_dir=tmpdir)
+        return subprocess.check_output(dst_name)
+    finally:
+        shutil.rmtree(tmpdir)
+
+
+def extract_defines(flags):
+    """Extract the required #DEFINEs from deltachat.h.
+
+    Since #DEFINEs are interpreted by the C preprocessor we can not
+    use the compiler to extract these and need to parse the header
+    file ourselves.
+
+    The defines are returned in a string that can be passed to CFFIs
+    cdef() method.
+    """
+    header = find_header(flags)
+    defines_re = re.compile(r"""
+        \#define\s+   # The start of a define.
+        (             # Begin capturing group which captures the define name.
+            (?:       # A nested group which is not captured, this allows us
+                      #    to build the list of prefixes to extract without
+                      #    creation another capture group.
+                DC_EVENT
+                | DC_QR
+                | DC_MSG
+                | DC_LP
+                | DC_EMPTY
+                | DC_CERTCK
+                | DC_STATE
+                | DC_STR
+                | DC_CONTACT_ID
+                | DC_GCL
+                | DC_CHAT
+                | DC_PROVIDER
+                | DC_KEY_GEN
+            )         # End of prefix matching
+            _[\w_]+   # Match the suffix, e.g. _RSA2048 in DC_KEY_GEN_RSA2048
+        )             # Close the capturing group, this contains
+                      # the entire name e.g. DC_MSG_TEXT.
+        \s+\S+        # Ensure there is whitespace followed by a value.
+    """, re.VERBOSE)
+    defines = []
+    with open(header) as fp:
+        for line in fp:
+            match = defines_re.match(line)
+            if match:
+                defines.append(match.group(1))
+    return '\n'.join('#define {} ...'.format(d) for d in defines)
+
+
+def ffibuilder():
+    projdir = os.environ.get('DCC_RS_DEV')
+    if not projdir:
+        p = dn(dn(dn(dn(abspath(__file__)))))
+        projdir = os.environ["DCC_RS_DEV"] = p
+    target = os.environ.get('DCC_RS_TARGET', 'release')
+    if projdir:
+        flags = local_build_flags(projdir, target)
+    else:
+        flags = system_build_flags()
+    builder = cffi.FFI()
+    builder.set_source(
+        'deltachat.capi',
+        """
+            #include <deltachat.h>
+            int dc_event_has_string_data(int e)
+            {
+                return DC_EVENT_DATA2_IS_STRING(e);
+            }
+        """,
+        include_dirs=flags.incs,
+        libraries=flags.libs,
+        extra_objects=flags.objs,
+        extra_link_args=flags.extra_link_args,
+    )
+    builder.cdef("""
+        typedef int... time_t;
+        void free(void *ptr);
+        extern int dc_event_has_string_data(int);
+    """)
+    function_defs = extract_functions(flags)
+    defines = extract_defines(flags)
+    builder.cdef(function_defs)
+    builder.cdef(defines)
     return builder
 
 

python/src/deltachat/const.py

@@ -1,197 +1,7 @@
-import sys
-import re
-import os
-from os.path import dirname, abspath
-from os.path import join as joinpath
-
-# the following const are generated from deltachat.h
-# this works well when you in a git-checkout
-# run "python deltachat/const.py" to regenerate events
-# begin const generated
-DC_GCL_ARCHIVED_ONLY = 0x01
-DC_GCL_NO_SPECIALS = 0x02
-DC_GCL_ADD_ALLDONE_HINT = 0x04
-DC_GCL_FOR_FORWARDING = 0x08
-DC_GCL_VERIFIED_ONLY = 0x01
-DC_GCL_ADD_SELF = 0x02
-DC_QR_ASK_VERIFYCONTACT = 200
-DC_QR_ASK_VERIFYGROUP = 202
-DC_QR_FPR_OK = 210
-DC_QR_FPR_MISMATCH = 220
-DC_QR_FPR_WITHOUT_ADDR = 230
-DC_QR_ACCOUNT = 250
-DC_QR_ADDR = 320
-DC_QR_TEXT = 330
-DC_QR_URL = 332
-DC_QR_ERROR = 400
-DC_CHAT_ID_DEADDROP = 1
-DC_CHAT_ID_TRASH = 3
-DC_CHAT_ID_MSGS_IN_CREATION = 4
-DC_CHAT_ID_STARRED = 5
-DC_CHAT_ID_ARCHIVED_LINK = 6
-DC_CHAT_ID_ALLDONE_HINT = 7
-DC_CHAT_ID_LAST_SPECIAL = 9
-DC_CHAT_TYPE_UNDEFINED = 0
-DC_CHAT_TYPE_SINGLE = 100
-DC_CHAT_TYPE_GROUP = 120
-DC_CHAT_TYPE_VERIFIED_GROUP = 130
-DC_MSG_ID_MARKER1 = 1
-DC_MSG_ID_DAYMARKER = 9
-DC_MSG_ID_LAST_SPECIAL = 9
-DC_STATE_UNDEFINED = 0
-DC_STATE_IN_FRESH = 10
-DC_STATE_IN_NOTICED = 13
-DC_STATE_IN_SEEN = 16
-DC_STATE_OUT_PREPARING = 18
-DC_STATE_OUT_DRAFT = 19
-DC_STATE_OUT_PENDING = 20
-DC_STATE_OUT_FAILED = 24
-DC_STATE_OUT_DELIVERED = 26
-DC_STATE_OUT_MDN_RCVD = 28
-DC_CONTACT_ID_SELF = 1
-DC_CONTACT_ID_INFO = 2
-DC_CONTACT_ID_DEVICE = 5
-DC_CONTACT_ID_LAST_SPECIAL = 9
-DC_MSG_TEXT = 10
-DC_MSG_IMAGE = 20
-DC_MSG_GIF = 21
-DC_MSG_STICKER = 23
-DC_MSG_AUDIO = 40
-DC_MSG_VOICE = 41
-DC_MSG_VIDEO = 50
-DC_MSG_FILE = 60
-DC_LP_AUTH_OAUTH2 = 0x2
-DC_LP_AUTH_NORMAL = 0x4
-DC_LP_IMAP_SOCKET_STARTTLS = 0x100
-DC_LP_IMAP_SOCKET_SSL = 0x200
-DC_LP_IMAP_SOCKET_PLAIN = 0x400
-DC_LP_SMTP_SOCKET_STARTTLS = 0x10000
-DC_LP_SMTP_SOCKET_SSL = 0x20000
-DC_LP_SMTP_SOCKET_PLAIN = 0x40000
-DC_CERTCK_AUTO = 0
-DC_CERTCK_STRICT = 1
-DC_CERTCK_ACCEPT_INVALID_CERTIFICATES = 3
-DC_EMPTY_MVBOX = 0x01
-DC_EMPTY_INBOX = 0x02
-DC_EVENT_INFO = 100
-DC_EVENT_SMTP_CONNECTED = 101
-DC_EVENT_IMAP_CONNECTED = 102
-DC_EVENT_SMTP_MESSAGE_SENT = 103
-DC_EVENT_IMAP_MESSAGE_DELETED = 104
-DC_EVENT_IMAP_MESSAGE_MOVED = 105
-DC_EVENT_IMAP_FOLDER_EMPTIED = 106
-DC_EVENT_NEW_BLOB_FILE = 150
-DC_EVENT_DELETED_BLOB_FILE = 151
-DC_EVENT_WARNING = 300
-DC_EVENT_ERROR = 400
-DC_EVENT_ERROR_NETWORK = 401
-DC_EVENT_ERROR_SELF_NOT_IN_GROUP = 410
-DC_EVENT_MSGS_CHANGED = 2000
-DC_EVENT_INCOMING_MSG = 2005
-DC_EVENT_MSG_DELIVERED = 2010
-DC_EVENT_MSG_FAILED = 2012
-DC_EVENT_MSG_READ = 2015
-DC_EVENT_CHAT_MODIFIED = 2020
-DC_EVENT_CONTACTS_CHANGED = 2030
-DC_EVENT_LOCATION_CHANGED = 2035
-DC_EVENT_CONFIGURE_PROGRESS = 2041
-DC_EVENT_IMEX_PROGRESS = 2051
-DC_EVENT_IMEX_FILE_WRITTEN = 2052
-DC_EVENT_SECUREJOIN_INVITER_PROGRESS = 2060
-DC_EVENT_SECUREJOIN_JOINER_PROGRESS = 2061
-DC_EVENT_FILE_COPIED = 2055
-DC_EVENT_IS_OFFLINE = 2081
-DC_EVENT_GET_STRING = 2091
-DC_STR_SELFNOTINGRP = 21
-DC_KEY_GEN_DEFAULT = 0
-DC_KEY_GEN_RSA2048 = 1
-DC_KEY_GEN_ED25519 = 2
-DC_PROVIDER_STATUS_OK = 1
-DC_PROVIDER_STATUS_PREPARATION = 2
-DC_PROVIDER_STATUS_BROKEN = 3
-DC_CHAT_VISIBILITY_NORMAL = 0
-DC_CHAT_VISIBILITY_ARCHIVED = 1
-DC_CHAT_VISIBILITY_PINNED = 2
-DC_STR_NOMESSAGES = 1
-DC_STR_SELF = 2
-DC_STR_DRAFT = 3
-DC_STR_VOICEMESSAGE = 7
-DC_STR_DEADDROP = 8
-DC_STR_IMAGE = 9
-DC_STR_VIDEO = 10
-DC_STR_AUDIO = 11
-DC_STR_FILE = 12
-DC_STR_STATUSLINE = 13
-DC_STR_NEWGROUPDRAFT = 14
-DC_STR_MSGGRPNAME = 15
-DC_STR_MSGGRPIMGCHANGED = 16
-DC_STR_MSGADDMEMBER = 17
-DC_STR_MSGDELMEMBER = 18
-DC_STR_MSGGROUPLEFT = 19
-DC_STR_GIF = 23
-DC_STR_ENCRYPTEDMSG = 24
-DC_STR_E2E_AVAILABLE = 25
-DC_STR_ENCR_TRANSP = 27
-DC_STR_ENCR_NONE = 28
-DC_STR_CANTDECRYPT_MSG_BODY = 29
-DC_STR_FINGERPRINTS = 30
-DC_STR_READRCPT = 31
-DC_STR_READRCPT_MAILBODY = 32
-DC_STR_MSGGRPIMGDELETED = 33
-DC_STR_E2E_PREFERRED = 34
-DC_STR_CONTACT_VERIFIED = 35
-DC_STR_CONTACT_NOT_VERIFIED = 36
-DC_STR_CONTACT_SETUP_CHANGED = 37
-DC_STR_ARCHIVEDCHATS = 40
-DC_STR_STARREDMSGS = 41
-DC_STR_AC_SETUP_MSG_SUBJECT = 42
-DC_STR_AC_SETUP_MSG_BODY = 43
-DC_STR_CANNOT_LOGIN = 60
-DC_STR_SERVER_RESPONSE = 61
-DC_STR_MSGACTIONBYUSER = 62
-DC_STR_MSGACTIONBYME = 63
-DC_STR_MSGLOCATIONENABLED = 64
-DC_STR_MSGLOCATIONDISABLED = 65
-DC_STR_LOCATION = 66
-DC_STR_STICKER = 67
-DC_STR_DEVICE_MESSAGES = 68
-DC_STR_COUNT = 68
-# end const generated
+from .capi import lib
 
 
-def read_event_defines(f):
-    rex = re.compile(r'#define\s+((?:DC_EVENT|DC_QR|DC_MSG|DC_LP|DC_EMPTY|DC_CERTCK|DC_STATE|DC_STR|'
-                     r'DC_CONTACT_ID|DC_GCL|DC_CHAT|DC_PROVIDER|DC_KEY_GEN)_\S+)\s+([x\d]+).*')
-    for line in f:
-        m = rex.match(line)
-        if m:
-            yield m.groups()
-
-
-if __name__ == "__main__":
-    here = abspath(__file__).rstrip("oc")
-    here_dir = dirname(here)
-    if len(sys.argv) >= 2:
-        deltah = sys.argv[1]
-    else:
-        deltah = joinpath(dirname(dirname(dirname(here_dir))), "deltachat-ffi", "deltachat.h")
-    assert os.path.exists(deltah)
-
-    lines = []
-    skip_to_end = False
-    for orig_line in open(here):
-        if skip_to_end:
-            if not orig_line.startswith("# end const"):
-                continue
-            skip_to_end = False
-        lines.append(orig_line)
-        if orig_line.startswith("# begin const"):
-            with open(deltah) as f:
-                for name, item in read_event_defines(f):
-                    lines.append("{} = {}\n".format(name, item))
-            skip_to_end = True
-
-    tmpname = here + ".tmp"
-    with open(tmpname, "w") as f:
-        f.write("".join(lines))
-    os.rename(tmpname, here)
+for name in dir(lib):
+    if name.startswith("DC_"):
+        globals()[name] = getattr(lib, name)
+del name

python/src/deltachat/testplugin.py

@@ -359,6 +359,10 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
 
             bot_ac, bot_cfg = self.get_online_config()
 
+            # Avoid starting ac so we don't interfere with the bot operating on
+            # the same database.
+            self._accounts.remove(bot_ac)
+
             args = [
                 sys.executable,
                 "-u",

src/blob.rs

@@ -60,7 +60,7 @@ impl<'a> BlobObject<'a> {
             .map_err(|err| BlobError::WriteFailure {
                 blobdir: blobdir.to_path_buf(),
                 blobname: name.clone(),
-                cause: err,
+                cause: err.into(),
             })?;
         let blob = BlobObject {
             blobdir,
@@ -375,7 +375,7 @@ impl<'a> BlobObject<'a> {
         img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
             blobdir: context.get_blobdir().to_path_buf(),
             blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err,
+            cause: err.into(),
         })?;
 
         Ok(())
@@ -413,7 +413,7 @@ impl<'a> BlobObject<'a> {
         img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
             blobdir: context.get_blobdir().to_path_buf(),
             blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err,
+            cause: err.into(),
         })?;
 
         Ok(())
@@ -441,7 +441,7 @@ pub enum BlobError {
         blobdir: PathBuf,
         blobname: String,
         #[source]
-        cause: std::io::Error,
+        cause: anyhow::Error,
     },
     #[error("Failed to copy data from {} to blob {blobname} in {}", .src.display(), .blobdir.display())]
     CopyFailure {

src/dc_receive_imf.rs

@@ -313,8 +313,6 @@ async fn add_parts(
 ) -> Result<()> {
     let mut state: MessageState;
     let mut chat_id_blocked = Blocked::Not;
-    let mut sort_timestamp = 0;
-    let mut rcvd_timestamp = 0;
     let mut mime_in_reply_to = String::new();
     let mut mime_references = String::new();
     let mut incoming_origin = incoming_origin;
@@ -601,17 +599,9 @@ async fn add_parts(
     }
     // correct message_timestamp, it should not be used before,
     // however, we cannot do this earlier as we need from_id to be set
-    calc_timestamps(
-        context,
-        *chat_id,
-        from_id,
-        *sent_timestamp,
-        !seen,
-        &mut sort_timestamp,
-        sent_timestamp,
-        &mut rcvd_timestamp,
-    )
-    .await;
+    let rcvd_timestamp = time();
+    let sort_timestamp = calc_sort_timestamp(context, *sent_timestamp, *chat_id, !seen).await;
+    *sent_timestamp = std::cmp::min(*sent_timestamp, rcvd_timestamp);
 
     // unarchive chat
     chat_id.unarchive(context).await?;
@@ -837,41 +827,38 @@ async fn save_locations(
     }
 }
 
-#[allow(clippy::too_many_arguments)]
-async fn calc_timestamps(
+async fn calc_sort_timestamp(
     context: &Context,
-    chat_id: ChatId,
-    from_id: u32,
     message_timestamp: i64,
+    chat_id: ChatId,
     is_fresh_msg: bool,
-    sort_timestamp: &mut i64,
-    sent_timestamp: &mut i64,
-    rcvd_timestamp: &mut i64,
-) {
-    *rcvd_timestamp = time();
-    *sent_timestamp = message_timestamp;
-    if *sent_timestamp > *rcvd_timestamp {
-        *sent_timestamp = *rcvd_timestamp
-    }
-    *sort_timestamp = message_timestamp;
+) -> i64 {
+    let mut sort_timestamp = message_timestamp;
+
+    // get newest non fresh message for this chat
+    // update sort_timestamp if less than that
     if is_fresh_msg {
         let last_msg_time: Option<i64> = context
             .sql
             .query_get_value(
                 context,
-                "SELECT MAX(timestamp) FROM msgs WHERE chat_id=? and from_id!=? AND timestamp>=?",
-                paramsv![chat_id, from_id as i32, *sort_timestamp],
+                "SELECT MAX(timestamp) FROM msgs WHERE chat_id=? AND state>?",
+                paramsv![chat_id, MessageState::InFresh],
             )
             .await;
+
         if let Some(last_msg_time) = last_msg_time {
-            if last_msg_time > 0 && *sort_timestamp <= last_msg_time {
-                *sort_timestamp = last_msg_time + 1;
+            if last_msg_time > sort_timestamp {
+                sort_timestamp = last_msg_time;
             }
         }
     }
-    if *sort_timestamp >= dc_smeared_time(context).await {
-        *sort_timestamp = dc_create_smeared_timestamp(context).await;
+
+    if sort_timestamp >= dc_smeared_time(context).await {
+        sort_timestamp = dc_create_smeared_timestamp(context).await;
     }
+
+    sort_timestamp
 }
 
 /// This function tries extracts the group-id from the message and returns the

src/headerdef.rs

@@ -34,6 +34,7 @@ pub enum HeaderDef {
     ChatContent,
     ChatDuration,
     ChatDispositionNotificationTo,
+    ChatUploadUrl,
     Autocrypt,
     AutocryptSetupMessage,
     SecureJoin,

src/imap/mod.rs

@@ -314,9 +314,15 @@ impl Imap {
     }
 
     async fn unsetup_handle(&mut self, context: &Context) {
+        // Close folder if messages should be expunged
+        if let Err(err) = self.close_folder(context).await {
+            warn!(context, "failed to close folder: {:?}", err);
+        }
+
+        // Logout from the server
         if let Some(mut session) = self.session.take() {
-            if let Err(err) = session.close().await {
-                warn!(context, "failed to close connection: {:?}", err);
+            if let Err(err) = session.logout().await {
+                warn!(context, "failed to logout: {:?}", err);
             }
         }
         self.connected = false;

src/imap/select_folder.rs

@@ -27,7 +27,7 @@ impl Imap {
     ///
     /// CLOSE is considerably faster than an EXPUNGE, see
     /// https://tools.ietf.org/html/rfc3501#section-6.4.2
-    async fn close_folder(&mut self, context: &Context) -> Result<()> {
+    pub(super) async fn close_folder(&mut self, context: &Context) -> Result<()> {
         if let Some(ref folder) = self.config.selected_folder {
             info!(context, "Expunge messages in \"{}\".", folder);
 

src/job.rs

@@ -3,6 +3,7 @@
 //! This module implements a job queue maintained in the SQLite database
 //! and job types.
 
+use std::env;
 use std::fmt;
 use std::future::Future;
 
@@ -31,6 +32,7 @@ use crate::message::{self, Message, MessageState};
 use crate::mimefactory::MimeFactory;
 use crate::param::*;
 use crate::smtp::Smtp;
+use crate::upload::{download_message_file, generate_upload_url, upload_file};
 use crate::{scheduler::InterruptInfo, sql};
 
 // results in ~3 weeks for the last backoff timespan
@@ -107,6 +109,8 @@ pub enum Action {
     MaybeSendLocationsEnded = 5007,
     SendMdn = 5010,
     SendMsgToSmtp = 5901, // ... high priority
+
+    DownloadMessageFile = 7000,
 }
 
 impl Default for Action {
@@ -133,6 +137,9 @@ impl From<Action> for Thread {
             MaybeSendLocationsEnded => Thread::Smtp,
             SendMdn => Thread::Smtp,
             SendMsgToSmtp => Thread::Smtp,
+
+            // TODO: Where does downloading fit in the thread architecture?
+            DownloadMessageFile => Thread::Imap,
         }
     }
 }
@@ -190,7 +197,7 @@ impl Job {
     /// Saves the job to the database, creating a new entry if necessary.
     ///
     /// The Job is consumed by this method.
-    pub async fn save(self, context: &Context) -> Result<()> {
+    pub(crate) async fn save(self, context: &Context) -> Result<()> {
         let thread: Thread = self.action.into();
 
         info!(context, "saving job for {}-thread: {:?}", thread, self);
@@ -329,7 +336,15 @@ impl Job {
         }
     }
 
-    pub async fn send_msg_to_smtp(&mut self, context: &Context, smtp: &mut Smtp) -> Status {
+    pub(crate) async fn send_msg_to_smtp(&mut self, context: &Context, smtp: &mut Smtp) -> Status {
+        // Upload file to HTTP if set in params.
+        if let (Some(upload_url), Ok(Some(upload_path))) = (
+            self.param.get_upload_url(),
+            self.param.get_upload_path(context),
+        ) {
+            job_try!(upload_file(context, upload_url.to_string(), upload_path).await);
+        }
+
         //  SMTP server, if not yet done
         if !smtp.is_connected().await {
             let loginparam = LoginParam::from_database(context, "configured_").await;
@@ -658,6 +673,13 @@ impl Job {
             }
         }
     }
+
+    pub(crate) async fn download_message_file(&mut self, context: &Context) -> Status {
+        let msg_id = MsgId::new(self.foreign_id);
+        let download_path = job_try!(self.param.get_upload_path(context));
+        job_try!(download_message_file(context, msg_id, download_path).await);
+        Status::Finished(Ok(()))
+    }
 }
 
 /// Delete all pending jobs with the given action.
@@ -726,7 +748,23 @@ pub async fn send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<Job
         }
     };
 
-    let mimefactory = MimeFactory::from_msg(context, &msg, attach_selfavatar).await?;
+    let mut mimefactory = MimeFactory::from_msg(context, &msg, attach_selfavatar).await?;
+
+    // Prepare file upload if DCC_UPLOAD_URL env variable is set.
+    // See upload-server folder for an example server impl.
+    // Here a new URL is generated, which the mimefactory includes in the message instead of the
+    // actual attachement. The upload then happens in the smtp send job.
+    let upload = if let Some(file) = msg.get_file(context) {
+        if let Ok(endpoint) = env::var("DCC_UPLOAD_URL") {
+            let upload_url = generate_upload_url(context, endpoint);
+            mimefactory.set_upload_url(upload_url.clone());
+            Some((upload_url, file))
+        } else {
+            None
+        }
+    } else {
+        None
+    };
 
     let mut recipients = mimefactory.recipients();
 
@@ -818,13 +856,17 @@ pub async fn send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<Job
     param.set(Param::File, blob.as_name());
     param.set(Param::Recipients, &recipients);
 
+    if let Some((upload_url, upload_path)) = upload {
+        param.set_upload_url(upload_url);
+        param.set_upload_path(upload_path);
+    }
+
     let job = create(Action::SendMsgToSmtp, msg_id.to_u32() as i32, param, 0)?;
 
     Ok(Some(job))
 }
 
-#[derive(Debug)]
-pub enum Connection<'a> {
+pub(crate) enum Connection<'a> {
     Inbox(&'a mut Imap),
     Smtp(&'a mut Smtp),
 }
@@ -979,6 +1021,7 @@ async fn perform_job_action(
             sql::housekeeping(context).await;
             Status::Finished(Ok(()))
         }
+        Action::DownloadMessageFile => job.download_message_file(context).await,
     };
 
     info!(
@@ -1035,7 +1078,8 @@ pub async fn add(context: &Context, job: Job) {
             | Action::OldDeleteMsgOnImap
             | Action::DeleteMsgOnImap
             | Action::MarkseenMsgOnImap
-            | Action::MoveMsg => {
+            | Action::MoveMsg
+            | Action::DownloadMessageFile => {
                 info!(context, "interrupt: imap");
                 context
                     .interrupt_inbox(InterruptInfo::new(false, None))

src/lib.rs

@@ -11,8 +11,6 @@ extern crate rusqlite;
 extern crate strum;
 #[macro_use]
 extern crate strum_macros;
-#[macro_use]
-extern crate debug_stub_derive;
 
 pub trait ToSql: rusqlite::ToSql + Send + Sync {}
 
@@ -69,6 +67,7 @@ mod simplify;
 mod smtp;
 pub mod stock;
 mod token;
+pub(crate) mod upload;
 #[macro_use]
 mod dehtml;
 

src/login_param.rs

@@ -130,10 +130,6 @@ impl LoginParam {
         }
     }
 
-    pub fn addr_str(&self) -> &str {
-        self.addr.as_str()
-    }
-
     /// Save this loginparam to the database.
     pub async fn save_to_database(
         &self,

src/message.rs

@@ -1545,6 +1545,33 @@ pub async fn update_server_uid(
     }
 }
 
+/// Schedule attachement download for a message.
+pub async fn schedule_download(
+    context: &Context,
+    msg_id: MsgId,
+    path: Option<PathBuf>,
+) -> Result<(), Error> {
+    let msg = Message::load_from_db(context, msg_id).await?;
+    if let Some(_upload_url) = msg.param.get_upload_url() {
+        // TODO: Check if message was already downloaded.
+        let mut params = Params::new();
+        if let Some(path) = path {
+            params.set_upload_path(path);
+        }
+        job::add(
+            context,
+            job::Job::new(Action::DownloadMessageFile, msg_id.to_u32(), params, 0),
+        )
+        .await;
+    } else {
+        warn!(
+            context,
+            "Tried to schedule download for message {} which has no uploads", msg_id
+        );
+    }
+    Ok(())
+}
+
 #[allow(dead_code)]
 pub async fn dc_empty_server(context: &Context, flags: u32) {
     job::kill_action(context, Action::EmptyServer).await;

src/mimefactory.rs

@@ -50,6 +50,7 @@ pub struct MimeFactory<'a, 'b> {
     context: &'a Context,
     last_added_location_id: u32,
     attach_selfavatar: bool,
+    upload_url: Option<String>,
 }
 
 /// Result of rendering a message, ready to be submitted to a send job.
@@ -159,6 +160,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             last_added_location_id: 0,
             attach_selfavatar,
             context,
+            upload_url: None,
         };
         Ok(factory)
     }
@@ -206,6 +208,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             req_mdn: false,
             last_added_location_id: 0,
             attach_selfavatar: false,
+            upload_url: None,
         };
 
         Ok(res)
@@ -409,6 +412,10 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             .collect()
     }
 
+    pub fn set_upload_url(&mut self, upload_url: String) {
+        self.upload_url = Some(upload_url)
+    }
+
     pub async fn render(mut self) -> Result<RenderedEmail, Error> {
         // Headers that are encrypted
         // - Chat-*, except Chat-Version
@@ -879,11 +886,21 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             }
         };
 
+        // if upload url is present: add as header and to message text
+        // TODO: make text part translatable (or remove)
+        let upload_url_text = if let Some(ref upload_url) = self.upload_url {
+            protected_headers.push(Header::new("Chat-Upload-Url".into(), upload_url.clone()));
+            Some(format!("\n\nFile attachement: {}", upload_url.clone()))
+        } else {
+            None
+        };
+
         let footer = &self.selfstatus;
         let message_text = format!(
-            "{}{}{}{}{}",
+            "{}{}{}{}{}{}",
             fwdhint.unwrap_or_default(),
             escape_message_footer_marks(final_text),
+            upload_url_text.unwrap_or_default(),
             if !final_text.is_empty() && !footer.is_empty() {
                 "\r\n\r\n"
             } else {
@@ -899,8 +916,8 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             .body(message_text);
         let mut parts = Vec::new();
 
-        // add attachment part
-        if chat::msgtype_has_file(self.msg.viewtype) {
+        // add attachment part, skip if upload url was provided
+        if chat::msgtype_has_file(self.msg.viewtype) && self.upload_url.is_none() {
             if !is_file_size_okay(context, &self.msg).await {
                 bail!(
                     "Message exceeds the recommended {} MB.",

src/mimeparser.rs

@@ -334,6 +334,13 @@ impl MimeMessage {
             }
         }
 
+        let upload_url = self.get(HeaderDef::ChatUploadUrl).map(|v| v.to_string());
+        if let Some(upload_url) = upload_url {
+            for part in self.parts.iter_mut() {
+                part.param.set_upload_url(upload_url.clone());
+            }
+        }
+
         self.parse_attachments();
 
         // See if an MDN is requested from the other side

src/param.rs

@@ -120,6 +120,12 @@ pub enum Param {
 
     /// For MDN-sending job
     MsgId = b'I',
+
+    /// For messages that have a HTTP file upload instead of attachement
+    UploadUrl = b'y',
+
+    /// For messages that have a HTTP file upload instead of attachement: Path to local file
+    UploadPath = b'Y',
 }
 
 /// Possible values for `Param::ForcePlaintext`.
@@ -317,6 +323,23 @@ impl Params {
         Ok(Some(path))
     }
 
+    pub fn get_upload_url(&self) -> Option<&str> {
+        self.get(Param::UploadUrl)
+    }
+
+    pub fn get_upload_path(&self, context: &Context) -> Result<Option<PathBuf>, BlobError> {
+        self.get_path(Param::UploadPath, context)
+    }
+
+    pub fn set_upload_path(&mut self, path: PathBuf) {
+        // TODO: Remove unwrap? May panic for invalid UTF8 in path.
+        self.set(Param::UploadPath, path.to_str().unwrap());
+    }
+
+    pub fn set_upload_url(&mut self, url: impl AsRef<str>) {
+        self.set(Param::UploadUrl, url);
+    }
+
     pub fn get_msg_id(&self) -> Option<MsgId> {
         self.get(Param::MsgId)
             .and_then(|x| x.parse::<u32>().ok())

src/pgp.rs

@@ -10,6 +10,7 @@ use pgp::composed::{
     SignedPublicSubKey, SignedSecretKey, SubkeyParamsBuilder,
 };
 use pgp::crypto::{HashAlgorithm, SymmetricKeyAlgorithm};
+use pgp::ser::Serialize;
 use pgp::types::{
     CompressionAlgorithm, KeyTrait, Mpi, PublicKeyTrait, SecretKeyTrait, StringToKey,
 };
@@ -322,8 +323,22 @@ pub async fn pk_decrypt(
     Ok(content)
 }
 
-/// Symmetric encryption.
+/// Symmetric encryption with armored base64 text output.
 pub async fn symm_encrypt(passphrase: &str, plain: &[u8]) -> Result<String> {
+    let message = symm_encrypt_to_message(passphrase, plain).await?;
+    let encoded_msg = message.to_armored_string(None)?;
+    Ok(encoded_msg)
+}
+
+/// Symmetric encryption with binary output.
+pub async fn symm_encrypt_bytes(passphrase: &str, plain: &[u8]) -> Result<Vec<u8>> {
+    let message = symm_encrypt_to_message(passphrase, plain).await?;
+    let mut buf = Vec::new();
+    message.to_writer(&mut buf)?;
+    Ok(buf)
+}
+
+async fn symm_encrypt_to_message(passphrase: &str, plain: &[u8]) -> Result<Message> {
     let lit_msg = Message::new_literal_bytes("", plain);
     let passphrase = passphrase.to_string();
 
@@ -332,24 +347,33 @@ pub async fn symm_encrypt(passphrase: &str, plain: &[u8]) -> Result<String> {
         let s2k = StringToKey::new_default(&mut rng);
         let msg =
             lit_msg.encrypt_with_password(&mut rng, s2k, Default::default(), || passphrase)?;
-
-        let encoded_msg = msg.to_armored_string(None)?;
-
-        Ok(encoded_msg)
+        Ok(msg)
     })
     .await
 }
 
-/// Symmetric decryption.
+/// Symmetric decryption from armored text.
 pub async fn symm_decrypt<T: std::io::Read + std::io::Seek>(
     passphrase: &str,
     ctext: T,
 ) -> Result<Vec<u8>> {
     let (enc_msg, _) = Message::from_armor_single(ctext)?;
+    symm_decrypt_from_message(enc_msg, passphrase).await
+}
 
+/// Symmetric decryption from bytes.
+pub async fn symm_decrypt_bytes<T: std::io::Read + std::io::Seek>(
+    passphrase: &str,
+    cbytes: T,
+) -> Result<Vec<u8>> {
+    let enc_msg = Message::from_bytes(cbytes)?;
+    symm_decrypt_from_message(enc_msg, passphrase).await
+}
+
+async fn symm_decrypt_from_message(message: Message, passphrase: &str) -> Result<Vec<u8>> {
     let passphrase = passphrase.to_string();
     async_std::task::spawn_blocking(move || {
-        let decryptor = enc_msg.decrypt_with_password(|| passphrase)?;
+        let decryptor = message.decrypt_with_password(|| passphrase)?;
 
         let msgs = decryptor.collect::<pgp::errors::Result<Vec<_>>>()?;
         ensure!(!msgs.is_empty(), "No valid messages found");

src/scheduler.rs

@@ -109,7 +109,7 @@ async fn fetch(ctx: &Context, connection: &mut Imap) {
             // fetch
             if let Err(err) = connection.fetch(&ctx, &watch_folder).await {
                 connection.trigger_reconnect();
-                error!(ctx, "{}", err);
+                warn!(ctx, "{}", err);
             }
         }
         None => {
@@ -131,7 +131,7 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
             // fetch
             if let Err(err) = connection.fetch(&ctx, &watch_folder).await {
                 connection.trigger_reconnect();
-                error!(ctx, "{}", err);
+                warn!(ctx, "{}", err);
             }
 
             // idle
@@ -141,7 +141,7 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
                     .await
                     .unwrap_or_else(|err| {
                         connection.trigger_reconnect();
-                        error!(ctx, "{}", err);
+                        warn!(ctx, "{}", err);
                         InterruptInfo::new(false, None)
                     })
             } else {
@@ -149,7 +149,7 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
             }
         }
         None => {
-            warn!(ctx, "Can not watch inbox folder, not set");
+            warn!(ctx, "Can not watch {} folder, not set", folder);
             connection.fake_idle(&ctx, None).await
         }
     }
@@ -471,7 +471,6 @@ impl SmtpConnectionState {
     }
 }
 
-#[derive(Debug)]
 struct SmtpConnectionHandlers {
     connection: Smtp,
     stop_receiver: Receiver<()>,

src/smtp/mod.rs

@@ -45,9 +45,8 @@ pub enum Error {
 
 pub type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Default, DebugStub)]
-pub struct Smtp {
-    #[debug_stub(some = "SmtpTransport")]
+#[derive(Default)]
+pub(crate) struct Smtp {
     transport: Option<smtp::SmtpTransport>,
 
     /// Email address we are sending from.
@@ -180,7 +179,7 @@ impl Smtp {
                 .stock_string_repl_str2(
                     StockMessage::ServerResponse,
                     format!("SMTP {}:{}", domain, port),
-                    err.to_string(),
+                    format!("{}, ({:?})", err.to_string(), err),
                 )
                 .await;
 

src/sql.rs

@@ -49,7 +49,7 @@ pub enum Error {
 pub type Result<T> = std::result::Result<T, Error>;
 
 /// A wrapper around the underlying Sqlite3 object.
-#[derive(DebugStub)]
+#[derive(Debug)]
 pub struct Sql {
     pool: RwLock<Option<r2d2::Pool<r2d2_sqlite::SqliteConnectionManager>>>,
 }

src/upload.rs

@@ -0,0 +1,142 @@
+use crate::blob::BlobObject;
+// use crate::constants::Viewtype;
+use crate::context::Context;
+use crate::error::{bail, format_err, Result};
+use crate::message::{Message, MsgId};
+use crate::pgp::{symm_decrypt_bytes, symm_encrypt_bytes};
+use async_std::fs;
+use async_std::path::PathBuf;
+use rand::Rng;
+use std::io::Cursor;
+use url::Url;
+
+/// Upload file to a HTTP upload endpoint.
+pub async fn upload_file(
+    context: &Context,
+    url: impl AsRef<str>,
+    filepath: PathBuf,
+) -> Result<String> {
+    let (passphrase, url) = parse_upload_url(url)?;
+
+    let content = fs::read(filepath).await?;
+    let encrypted = symm_encrypt_bytes(&passphrase, &content).await?;
+
+    // TODO: Use tokens for upload.
+    info!(context, "uploading encrypted file to {}", &url);
+    let response = surf::put(url).body_bytes(encrypted).await;
+    if let Err(err) = response {
+        bail!("Upload failed: {}", err);
+    }
+    let mut response = response.unwrap();
+    match response.body_string().await {
+        Ok(string) => Ok(string),
+        Err(err) => bail!("Invalid response from upload: {}", err),
+    }
+}
+
+pub async fn download_message_file(
+    context: &Context,
+    msg_id: MsgId,
+    download_path: Option<PathBuf>,
+) -> Result<()> {
+    let mut message = Message::load_from_db(context, msg_id).await?;
+    let upload_url = message
+        .param
+        .get_upload_url()
+        .ok_or_else(|| format_err!("Message has no upload URL"))?;
+
+    let (passphrase, url) = parse_upload_url(upload_url)?;
+
+    let filename: String = url
+        .path_segments()
+        .ok_or_else(|| format_err!("Invalid upload URL"))?
+        .last()
+        .ok_or_else(|| format_err!("Invalid upload URL"))?
+        .to_string();
+
+    let data = download_file(context, url, passphrase).await?;
+    let saved_path = if let Some(download_path) = download_path {
+        fs::write(&download_path, data).await?;
+        download_path.to_string_lossy().to_string()
+    } else {
+        let blob = BlobObject::create(context, filename.clone(), &data)
+            .await
+            .map_err(|err| {
+                format_err!(
+                    "Could not add blob for file download {}, error {}",
+                    filename,
+                    err
+                )
+            })?;
+        blob.as_name().to_string()
+    };
+    info!(context, "saved download to: {:?}", saved_path);
+
+    // TODO: Support getting the mime type.
+    let filemime = None;
+
+    message.set_file(saved_path, filemime);
+    message.save_param_to_disk(context).await;
+
+    Ok(())
+}
+
+/// Download and decrypt a file from a HTTP endpoint.
+pub async fn download_file(
+    context: &Context,
+    url: impl AsRef<str>,
+    passphrase: String,
+) -> Result<Vec<u8>> {
+    info!(context, "downloading file from {}", &url.as_ref());
+    let response = surf::get(url).recv_bytes().await;
+    if let Err(err) = response {
+        bail!("Download failed: {}", err);
+    }
+    let bytes = response.unwrap();
+    info!(context, "download complete, len: {}", bytes.len());
+    let reader = Cursor::new(bytes);
+    let decrypted = symm_decrypt_bytes(&passphrase, reader).await?;
+    Ok(decrypted)
+}
+
+/// Parse a URL from a string and take out the hash fragment.
+fn parse_upload_url(url: impl AsRef<str>) -> Result<(String, Url)> {
+    let mut url = url::Url::parse(url.as_ref())?;
+    let passphrase = url.fragment();
+    if passphrase.is_none() {
+        bail!("Missing passphrase for upload URL");
+    }
+    let passphrase = passphrase.unwrap().to_string();
+    url.set_fragment(None);
+    Ok((passphrase, url))
+}
+
+/// Generate a random URL based on the provided endpoint.
+pub fn generate_upload_url(_context: &Context, mut endpoint: String) -> String {
+    // equals at least 16 random bytes (base32 takes 160% of binary size).
+    const FILENAME_LEN: usize = 26;
+    // equals at least 32 random bytes.
+    const PASSPHRASE_LEN: usize = 52;
+
+    if endpoint.ends_with('/') {
+        endpoint.pop();
+    }
+    let passphrase = generate_token_string(PASSPHRASE_LEN);
+    let filename = generate_token_string(FILENAME_LEN);
+    format!("{}/{}#{}", endpoint, filename, passphrase)
+}
+
+/// Generate a random string encoded in base32.
+/// Len is the desired string length of the result.
+/// TODO: There's likely better methods to create random tokens.
+pub fn generate_token_string(len: usize) -> String {
+    const CROCKFORD_ALPHABET: &[u8] = b"0123456789abcdefghjkmnpqrstvwxyz";
+    let mut rng = rand::thread_rng();
+    let token: String = (0..len)
+        .map(|_| {
+            let idx = rng.gen_range(0, CROCKFORD_ALPHABET.len());
+            CROCKFORD_ALPHABET[idx] as char
+        })
+        .collect();
+    token
+}

upload-server/.gitignore

@@ -0,0 +1,4 @@
+uploads
+node_modules
+yarn*
+.gitfoo

upload-server/README.md

@@ -0,0 +1,17 @@
+# deltachat-upload-server
+
+Demo server for the HTTP file upload feature.
+
+### Usage
+
+```
+npm install
+node server.js
+```
+
+Configure with environment variables:
+* `UPLOAD_PATH`: Path to upload files to (default: `./uploads`)
+* `PORT`: Port to listen on (default: `8080`)
+* `HOSTNAME`: Hostname to listen on (default: `0.0.0.0`)
+* `BASEURL`: Base URL for generated links (default: `http://[hostname]:[port]/`)
+

upload-server/package.json

@@ -0,0 +1,13 @@
+{
+  "name": "deltachat-upload-server",
+  "version": "1.0.0",
+  "main": "server.js",
+  "license": "MIT",
+  "scripts": {
+    "start": "node server.js"
+  },
+  "dependencies": {
+    "base32": "^0.0.6",
+    "express": "^4.17.1"
+  }
+}

upload-server/server.js

@@ -0,0 +1,73 @@
+const p = require('path')
+const express = require('express')
+const fs = require('fs')
+const { pipeline } = require('stream')
+
+const app = express()
+
+const config = {
+  path: process.env.UPLOAD_PATH || p.resolve('./uploads'),
+  port: process.env.PORT || 8080,
+  hostname: process.env.HOSTNAME || '0.0.0.0',
+  baseurl: process.env.BASE_URL
+}
+
+if (!config.baseurl) config.baseurl = `http://${config.hostname}:${config.port}/`
+if (!config.baseurl.endsWith('/')) config.baseurl = config.baseurl + '/'
+
+if (!fs.existsSync(config.path)) {
+  fs.mkdirSync(config.path, { recursive: true })
+}
+
+app.use('/:filename', checkFilenameMiddleware)
+app.put('/:filename', (req, res) => {
+  const uploadpath = req.uploadpath
+  const filename = req.params.filename
+  fs.stat(uploadpath, (err, stat) => {
+    if (err && err.code !== 'ENOENT') {
+      console.error('error', err.message)
+      return res.code(500).send('internal server error')
+    }
+    if (stat) return res.status(500).send('filename in use')
+
+    const ws = fs.createWriteStream(uploadpath)
+    pipeline(req, ws, err => {
+      if (err) {
+        console.error('error', err.message)
+        return res.status(500).send('internal server error')
+      }
+      console.log('file uploaded: ' + uploadpath)
+      const url = config.baseurl + filename
+      res.end(url)
+    })
+  })
+})
+
+app.get('/:filename', (req, res) => {
+  const uploadpath = req.uploadpath
+  const rs = fs.createReadStream(uploadpath)
+  res.setHeader('content-type', 'application/octet-stream')
+  pipeline(rs, res, err => {
+    if (err) console.error('error', err.message)
+    if (err) return res.status(500).send(err.message)
+  })
+})
+
+function checkFilenameMiddleware (req, res, next) {
+  const filename = req.params.filename
+  if (!filename) return res.status(500).send('missing filename')
+  if (!filename.match(/^[a-zA-Z0-9]{26,32}$/)) {
+    return res.status(500).send('illegal filename')
+  }
+  const uploadpath = p.normalize(p.join(config.path, req.params.filename))
+  if (!uploadpath.startsWith(config.path)) {
+    return res.code(500).send('bad request')
+  }
+  req.uploadpath = uploadpath
+  next()
+}
+
+app.listen(config.port, err => {
+  if (err) console.error(err)
+  else console.log(`Listening on ${config.baseurl}`)
+})