mark 30-times runs tests as ignored by default

.circleci/config.yml

@@ -138,6 +138,12 @@ jobs:
             - py-docs
             - wheelhouse
 
+  remote_tests_rust:
+    machine: true
+    steps:
+      - checkout
+      - run: ci_scripts/remote_tests_rust.sh
+    
   remote_tests_python:
     machine: true
     steps:
@@ -172,6 +178,11 @@ workflows:
     jobs:
       # - cargo_fetch
 
+      - remote_tests_rust:
+          filters:
+            tags:
+              only: /.*/
+
       - remote_tests_python:
           filters:
             tags:
@@ -180,9 +191,8 @@ workflows:
       - remote_python_packaging:
           requires: 
              - remote_tests_python
+             - remote_tests_rust 
           filters:
-            branches:
-              only: master
             tags:
               only: /.*/
 
@@ -191,8 +201,6 @@ workflows:
              - remote_python_packaging
              - build_doxygen
           filters:
-            branches:
-              only: master
             tags:
               only: /.*/
       # - rustfmt:
@@ -204,8 +212,6 @@ workflows:
 
       - build_doxygen:
           filters:
-            branches:
-              only: master
             tags:
               only: /.*/
 

.github/workflows/ci.yml

@@ -1,89 +0,0 @@
-name: Rust CI
-
-on:
-  pull_request:
-  push:
-    branches:
-      - master
-      - staging
-      - trying
-
-jobs:
-
-  fmt:
-    name: Rustfmt
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: 1.43.1
-          override: true
-      - run: rustup component add rustfmt
-      - uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
-
-  run_clippy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-            toolchain: 1.43.1
-            components: clippy
-            override: true
-      - uses: actions-rs/clippy-check@v1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-
-
-  build_and_test:
-    name: Build and test
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, windows-latest, macOS-latest]
-        rust: [nightly, 1.43.1]
-
-    steps:
-    - uses: actions/checkout@master
-
-    - name: Install ${{ matrix.rust }}
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: ${{ matrix.rust }}
-        override: true
-
-    - name: Cache cargo registry
-      uses: actions/cache@v2
-      with:
-        path: ~/.cargo/registry
-        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}
-
-    - name: Cache cargo index
-      uses: actions/cache@v2
-      with:
-        path: ~/.cargo/git
-        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}
-
-    - name: Cache cargo build
-      uses: actions/cache@v2
-      with:
-        path: target
-        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-build-target-${{ hashFiles('**/Cargo.toml') }}
-
-    - name: check
-      uses: actions-rs/cargo@v1
-      with:
-        command:  check
-        args: --workspace --all --bins --examples --tests
-
-    - name: tests
-      uses: actions-rs/cargo@v1
-      with:
-        command: test
-        args: --workspace
-          

.github/workflows/code-quality.yml

@@ -0,0 +1,48 @@
+on: push
+name: Code Quality
+jobs:
+
+  check:
+    name: Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly-2020-03-19
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          command: check
+          args: --workspace --examples --tests --all-features
+
+  fmt:
+    name: Rustfmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly-2020-03-19
+          override: true
+      - run: rustup component add rustfmt
+      - uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
+  run_clippy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+            toolchain: nightly-2020-03-19
+            components: clippy
+            override: true
+      - uses: actions-rs/clippy-check@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          args: --all-features

CHANGELOG.md

@@ -1,28 +1,5 @@
 # Changelog 
 
-## 1.35.0
-
-- enable strict-tls from a new provider-db setting #1587
-
-- new subject 'Message from USER' for one-to-one chats #1395
-
-- recode images #1563
-
-- improve reconnect handling #1549 #1580
-
-- improve importing addresses #1544
-
-- improve configure and folder detection #1539 #1548
-
-- improve test suite #1559 #1564 #1580 #1581 #1582 #1584 #1588:
-
-- fix ad-hoc groups #1566
-
-- preventions against being marked as spam #1575
-
-- refactorings #1542 #1569
-
-
 ## 1.34.0
 
 - new api for io, thread and event handling #1356,

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.35.0"
+version = "1.34.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
@@ -12,12 +12,12 @@ license = "MPL-2.0"
 deltachat_derive = { path = "./deltachat_derive" }
 
 libc = "0.2.51"
-pgp = { version = "0.6.0", default-features = false } 
+pgp = { version = "0.5.1", default-features = false } 
 hex = "0.4.0"
-sha2 = "0.9.0"
+sha2 = "0.8.0"
 rand = "0.7.0"
 smallvec = "1.0.0"
-surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
+surf = { version = "2.0.0-alpha.2", default-features = false, features = ["h1-client"] }
 num-derive = "0.3.0"
 num-traits = "0.2.6"
 async-smtp = { version = "0.3" }
@@ -25,8 +25,8 @@ email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 async-imap = "0.3.1"
 async-native-tls = { version = "0.3.3" }
-async-std = { version = "1.6.1", features = ["unstable"] }
-base64 = "0.12"
+async-std = { version = "1.6.0", features = ["unstable"] }
+base64 = "0.11"
 charset = "0.1"
 percent-encoding = "2.0"
 serde = { version = "1.0", features = ["derive"] }
@@ -35,42 +35,42 @@ chrono = "0.4.6"
 indexmap = "1.3.0"
 lazy_static = "1.4.0"
 regex = "1.1.6"
-rusqlite = { version = "0.23", features = ["bundled"] }
-r2d2_sqlite = "0.16.0"
+rusqlite = { version = "0.22", features = ["bundled"] }
+r2d2_sqlite = "0.15.0"
 r2d2 = "0.8.5"
-strum = "0.18.0"
-strum_macros = "0.18.0"
+strum = "0.16.0"
+strum_macros = "0.16.0"
 backtrace = "0.3.33"
 byteorder = "1.3.1"
 itertools = "0.8.0"
 image-meta = "0.1.0"
-quick-xml = "0.18.1"
+quick-xml = "0.17.1"
 escaper = "0.1.0"
 bitflags = "1.1.0"
+debug_stub_derive = "0.3.0"
 sanitize-filename = "0.2.1"
 stop-token = { version = "0.1.1", features = ["unstable"] }
 mailparse = "0.12.1"
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
 native-tls = "0.2.3"
-image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
+image = { version = "0.22.4", default-features=false, features = ["gif_codec", "jpeg", "ico", "png_codec", "pnm", "webp", "bmp"] }
 futures = "0.3.4"
 thiserror = "1.0.14"
 anyhow = "1.0.28"
 async-trait = "0.1.31"
 url = "2.1.1"
 
-pretty_env_logger = { version = "0.4.0", optional = true }
+pretty_env_logger = { version = "0.3.1", optional = true }
 log = {version = "0.4.8", optional = true }
 rustyline = { version = "4.1.0", optional = true }
 ansi_term = { version = "0.12.1", optional = true }
-open = { version = "1.4.0", optional = true }
 
 
 [dev-dependencies]
 tempfile = "3.0"
 pretty_assertions = "0.6.1"
-pretty_env_logger = "0.4.0"
-proptest = "0.10"
+pretty_env_logger = "0.3.0"
+proptest = "0.9.4"
 async-std = { version = "1.6.0", features = ["unstable", "attributes"] }
 smol = "0.1.10"
 
@@ -94,7 +94,9 @@ required-features = ["repl"]
 [features]
 default = []
 internals = []
-repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "open"]
+repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term"]
 vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored"]
 nightly = ["pgp/nightly"]
 
+[patch.crates-io]
+smol = { git = "https://github.com/dignifiedquire/smol-1", branch = "isolate-nix" }

appveyor.yml

@@ -0,0 +1,19 @@
+environment:
+  matrix:
+    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
+
+install:
+  - appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
+  - rustup-init -yv --default-toolchain nightly-2020-03-19
+  - set PATH=%PATH%;%USERPROFILE%\.cargo\bin
+  - rustc -vV
+  - cargo -vV
+
+build: false
+
+test_script:
+  - cargo test --release --all
+
+cache:
+  - target
+  - C:\Users\appveyor\.cargo\registry

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.35.0"
+version = "1.34.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/deltachat.h

@@ -3163,54 +3163,6 @@ int             dc_msg_is_setupmessage        (const dc_msg_t* msg);
 char*           dc_msg_get_setupcodebegin     (const dc_msg_t* msg);
 
 
-/**
- * Check if the message is completely downloaded
- * or if some further action is needed.
- *
- * The function returns one of:
- * - @ref DC_DOWNLOAD_NO_URL      - The message does not need any further download action
- *                                  and should be rendered as usual.
- * - @ref DC_DOWNLOAD_AVAILABLE   - There is additional content to download.
- *                                  Tn addition to the usual message rendering,
- *                                  the UI shall show a download button that starts dc_schedule_download()
- * - @ref DC_DOWNLOAD_IN_PROGRESS - Download was started with dc_schedule_download() and is still in progress.
- *                                  On progress changes and if the download fails or succeeds,
- *                                  the event @ref DC_EVENT_DOWNLOAD_PROGRESS will be emitted.
- * - @ref DC_DOWNLOAD_DONE        - Download finished successfully
- * - @ref DC_DOWNLOAD_FAILURE     - Download error, the user may start over calling dc_schedule_download() again.
- *
- * @memberof dc_msg_t
- * @param msg The message object.
- * @return One of the @ref DC_DOWNLOAD values
- */
-int dc_msg_download_status(const dc_msg_t* msg);
-
-
-/**
- * Advices the core to start downloading a message.
- * This function is typically called when the user hits the "Download" button
- * that is shown by the UI in case dc_msg_download_status()
- * returns @ref DC_DOWNLOAD_AVAILABLE or @ref DC_DOWNLOAD_FAILURE.
- *
- * The UI may want to show a file selector and let the user chose a download location.
- * The file name in the file selector may be prefilled using dc_msg_get_filename().
- *
- * During the download, the progress, errors and success
- * are reported using @ref DC_EVENT_DOWNLOAD_PROGRESS.
- *
- * Once the @ref DC_EVENT_DOWNLOAD_PROGRESS reports success,
- * The file can be accessed as usual using dc_msg_get_file().
- *
- * @memberof dc_context_t
- * @param context The context object.
- * @param path Path to the destination file.
- *     You can specify NULL here to download
- *     to a reasonable file name in the internal blob-directory.
- * @param msg_id Message-ID to download the content for.
- */
-void dc_schedule_download(dc_context_t* context, int msg_id, const char* path);
-
-
 /**
  * Set the text of a message object.
  * This does not alter any information in the database; this may be done by dc_send_msg() later.
@@ -4279,16 +4231,6 @@ void dc_event_unref(dc_event_t* event);
  */
 #define DC_EVENT_SECUREJOIN_JOINER_PROGRESS       2061
 
-
-/**
- * Inform about the progress of a download started by dc_schedule_download().
- *
- * @param data1 (int) Message-ID the progress is reported for.
- * @param data2 (int) 0=error, 1-999=progress in permille, 1000=success and done
- */
-#define DC_EVENT_DOWNLOAD_PROGRESS       2070
-
-
 /**
  * @}
  */
@@ -4428,29 +4370,6 @@ void dc_event_unref(dc_event_t* event);
  */
 
 
-/**
- * @defgroup DC_DOWNLOAD DC_DOWNLOAD
- *
- * These constants describe the download state of a message.
- * The download state can be retrieved using dc_msg_download_status()
- * and usually changes after calling dc_schedule_download().
- *
- * @addtogroup DC_DOWNLOAD
- * @{
- */
-
-#define DC_DOWNLOAD_NO_URL      10 ///< Download not needed, see dc_msg_download_status() for details.
-#define DC_DOWNLOAD_AVAILABLE   20 ///< Download available, see dc_msg_download_status() for details.
-#define DC_DOWNLOAD_IN_PROGRESS 30 ///< Download in progress, see dc_msg_download_status() for details.
-#define DC_DOWNLOAD_DONE        40 ///< Download done, see dc_msg_download_status() for details.
-#define DC_DOWNLOAD_FAILURE     50 ///< Download failed, see dc_msg_download_status() for details.
-
-/**
- * @}
- */
-
-
-
 /*
  * TODO: Strings need some doumentation about used placeholders.
  *
@@ -4508,20 +4427,12 @@ void dc_event_unref(dc_event_t* event);
 #define DC_STR_DEVICE_MESSAGES_HINT       70
 #define DC_STR_WELCOME_MESSAGE            71
 #define DC_STR_UNKNOWN_SENDER_FOR_CHAT    72
-#define DC_STR_SUBJECT_FOR_NEW_CONTACT    73
-#define DC_STR_COUNT                      73
+#define DC_STR_COUNT                      72
 
 /*
  * @}
  */
 
-#ifdef PY_CFFI_INC
-/* Helper utility to locate the header file when building python bindings. */
-char* _dc_header_file_location(void) {
-    return __FILE__;
-}
-#endif
-
 
 #ifdef __cplusplus
 }

examples/repl/cmdline.rs

@@ -370,8 +370,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  ===========================Message commands==\n\
                  listmsgs <query>\n\
                  msginfo <msg-id>\n\
-                 openfile <msg-id>\n\
-                 download <msg-id>\n\
                  listfresh\n\
                  forward <msg-id> <chat-id>\n\
                  markseen <msg-id>\n\
@@ -892,25 +890,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let res = message::get_msg_info(&context, id).await;
             println!("{}", res);
         }
-        "openfile" => {
-            ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let id = MsgId::new(arg1.parse()?);
-            let msg = Message::load_from_db(&context, id).await?;
-            let filepath = msg.get_file(&context);
-            ensure!(filepath.is_some(), "Message has no file.");
-            let filepath = filepath.unwrap();
-            open::that(filepath)?;
-        }
-        "download" => {
-            ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let id = MsgId::new(arg1.parse()?);
-            let path = if !arg2.is_empty() {
-                Some(arg2.into())
-            } else {
-                None
-            };
-            message::schedule_download(&context, id, path).await?;
-        }
         "listfresh" => {
             let msglist = context.get_fresh_msgs().await;
 

examples/repl/main.rs

@@ -186,11 +186,9 @@ const CHAT_COMMANDS: [&str; 26] = [
     "unpin",
     "delchat",
 ];
-const MESSAGE_COMMANDS: [&str; 10] = [
+const MESSAGE_COMMANDS: [&str; 8] = [
     "listmsgs",
     "msginfo",
-    "openfile",
-    "download",
     "listfresh",
     "forward",
     "markseen",

python/examples/echo_and_quit.py

@@ -12,7 +12,7 @@ class EchoPlugin:
             message.account.shutdown()
         else:
             # unconditionally accept the chat
-            message.create_chat()
+            message.accept_sender_contact()
             addr = message.get_sender_contact().addr
             if message.is_system_message():
                 message.chat.send_text("echoing system message from {}:\n{}".format(addr, message))

python/examples/group_tracking.py

@@ -12,7 +12,7 @@ class GroupTrackingPlugin:
             message.account.shutdown()
         else:
             # unconditionally accept the chat
-            message.create_chat()
+            message.accept_sender_contact()
             addr = message.get_sender_contact().addr
             text = message.text
             message.chat.send_text("echoing from {}:\n{}".format(addr, text))

python/examples/test_examples.py

@@ -26,15 +26,15 @@ def test_echo_quit_plugin(acfactory, lp):
 
     lp.sec("sending a message to the bot")
     bot_contact = ac1.create_contact(botproc.addr)
-    bot_chat = bot_contact.create_chat()
-    bot_chat.send_text("hello")
+    ch1 = ac1.create_chat_by_contact(bot_contact)
+    ch1.send_text("hello")
 
-    lp.sec("waiting for the reply message from the bot to arrive")
+    lp.sec("waiting for the bot-reply to arrive")
     reply = ac1._evtracker.wait_next_incoming_message()
-    assert reply.chat == bot_chat
     assert "hello" in reply.text
+    assert reply.chat == ch1
     lp.sec("send quit sequence")
-    bot_chat.send_text("/quit")
+    ch1.send_text("/quit")
     botproc.wait()
 
 
@@ -47,8 +47,8 @@ def test_group_tracking_plugin(acfactory, lp):
     botproc.fnmatch_lines("""
         *ac_configure_completed*
     """)
-    ac1.add_account_plugin(FFIEventLogger(ac1))
-    ac2.add_account_plugin(FFIEventLogger(ac2))
+    ac1.add_account_plugin(FFIEventLogger(ac1, "ac1"))
+    ac2.add_account_plugin(FFIEventLogger(ac2, "ac2"))
 
     lp.sec("creating bot test group with bot")
     bot_contact = ac1.create_contact(botproc.addr)

python/setup.py

@@ -18,7 +18,7 @@ def main():
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient'],
+        install_requires=['cffi>=1.0.0', 'pluggy'],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],

python/src/deltachat/__init__.py

@@ -60,8 +60,7 @@ def run_cmdline(argv=None, account_plugins=None):
     ac = Account(args.db)
 
     if args.show_ffi:
-        ac.set_config("displayname", "bot")
-        log = events.FFIEventLogger(ac)
+        log = events.FFIEventLogger(ac, "bot")
         ac.add_account_plugin(log)
 
     for plugin in account_plugins or []:
@@ -77,8 +76,8 @@ def run_cmdline(argv=None, account_plugins=None):
         ac.set_config("mvbox_move", "0")
         ac.set_config("mvbox_watch", "0")
         ac.set_config("sentbox_watch", "0")
-        configtracker = ac.configure()
-        configtracker.wait_finish()
+        ac.configure()
+        ac.wait_configure_finish()
 
     # start IO threads and configure if neccessary
     ac.start_io()

python/src/deltachat/_build.py

@@ -1,64 +1,65 @@
 import distutils.ccompiler
 import distutils.log
 import distutils.sysconfig
-import os
-import platform
-import re
-import shutil
-import subprocess
 import tempfile
-import textwrap
-import types
-from os.path import abspath
-from os.path import dirname as dn
-
+import platform
+import os
 import cffi
+import shutil
+from os.path import dirname as dn
+from os.path import abspath
 
 
-def local_build_flags(projdir, target):
-    """Construct build flags for building against a checkout.
-
-    :param projdir: The root directory of the deltachat-core-rust project.
-    :param target: The rust build target, `debug` or `release`.
-    """
-    flags = types.SimpleNamespace()
-    if platform.system() == 'Darwin':
-        flags.libs = ['resolv', 'dl']
-        flags.extra_link_args = [
-                '-framework', 'CoreFoundation',
-                '-framework', 'CoreServices',
-                '-framework', 'Security',
-        ]
-    elif platform.system() == 'Linux':
-        flags.libs = ['rt', 'dl', 'm']
-        flags.extra_link_args = []
+def ffibuilder():
+    projdir = os.environ.get('DCC_RS_DEV')
+    if not projdir:
+        p = dn(dn(dn(dn(abspath(__file__)))))
+        projdir = os.environ["DCC_RS_DEV"] = p
+    target = os.environ.get('DCC_RS_TARGET', 'release')
+    if projdir:
+        if platform.system() == 'Darwin':
+            libs = ['resolv', 'dl']
+            extra_link_args = [
+                    '-framework', 'CoreFoundation',
+                    '-framework', 'CoreServices',
+                    '-framework', 'Security',
+            ]
+        elif platform.system() == 'Linux':
+            libs = ['rt', 'dl', 'm']
+            extra_link_args = []
+        else:
+            raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
+        target_dir = os.environ.get("CARGO_TARGET_DIR")
+        if target_dir is None:
+            target_dir = os.path.join(projdir, 'target')
+        objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
+        assert os.path.exists(objs[0]), objs
+        incs = [os.path.join(projdir, 'deltachat-ffi')]
     else:
-        raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
-    target_dir = os.environ.get("CARGO_TARGET_DIR")
-    if target_dir is None:
-        target_dir = os.path.join(projdir, 'target')
-    flags.objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
-    assert os.path.exists(flags.objs[0]), flags.objs
-    flags.incs = [os.path.join(projdir, 'deltachat-ffi')]
-    return flags
-
-
-def system_build_flags():
-    """Construct build flags for building against an installed libdeltachat."""
-    flags = types.SimpleNamespace()
-    flags.libs = ['deltachat']
-    flags.objs = []
-    flags.incs = []
-    flags.extra_link_args = []
-
-
-def extract_functions(flags):
-    """Extract the function definitions from deltachat.h.
-
-    This creates a .h file with a single `#include <deltachat.h>` line
-    in it.  It then runs the C preprocessor to create an output file
-    which contains all function definitions found in `deltachat.h`.
-    """
+        libs = ['deltachat']
+        objs = []
+        incs = []
+        extra_link_args = []
+    builder = cffi.FFI()
+    builder.set_source(
+        'deltachat.capi',
+        """
+            #include <deltachat.h>
+            int dc_event_has_string_data(int e)
+            {
+                return DC_EVENT_DATA2_IS_STRING(e);
+            }
+        """,
+        include_dirs=incs,
+        libraries=libs,
+        extra_objects=objs,
+        extra_link_args=extra_link_args,
+    )
+    builder.cdef("""
+        typedef int... time_t;
+        void free(void *ptr);
+        extern int dc_event_has_string_data(int);
+    """)
     distutils.log.set_verbosity(distutils.log.INFO)
     cc = distutils.ccompiler.new_compiler(force=True)
     distutils.sysconfig.customize_compiler(cc)
@@ -70,133 +71,13 @@ def extract_functions(flags):
             src_fp.write('#include <deltachat.h>')
         cc.preprocess(source=src_name,
                       output_file=dst_name,
-                      include_dirs=flags.incs,
+                      include_dirs=incs,
                       macros=[('PY_CFFI', '1')])
         with open(dst_name, "r") as dst_fp:
-            return dst_fp.read()
+            builder.cdef(dst_fp.read())
     finally:
         shutil.rmtree(tmpdir)
 
-
-def find_header(flags):
-    """Use the compiler to find the deltachat.h header location.
-
-    This uses a small utility in deltachat.h to find the location of
-    the header file location.
-    """
-    distutils.log.set_verbosity(distutils.log.INFO)
-    cc = distutils.ccompiler.new_compiler(force=True)
-    distutils.sysconfig.customize_compiler(cc)
-    tmpdir = tempfile.mkdtemp()
-    try:
-        src_name = os.path.join(tmpdir, "where.c")
-        obj_name = os.path.join(tmpdir, "where.o")
-        dst_name = os.path.join(tmpdir, "where")
-        with open(src_name, "w") as src_fp:
-            src_fp.write(textwrap.dedent("""
-                #include <stdio.h>
-                #include <deltachat.h>
-
-                int main(void) {
-                    printf("%s", _dc_header_file_location());
-                    return 0;
-                }
-            """))
-        cwd = os.getcwd()
-        try:
-            os.chdir(tmpdir)
-            cc.compile(sources=["where.c"],
-                       include_dirs=flags.incs,
-                       macros=[("PY_CFFI_INC", "1")])
-        finally:
-            os.chdir(cwd)
-        cc.link_executable(objects=[obj_name],
-                           output_progname="where",
-                           output_dir=tmpdir)
-        return subprocess.check_output(dst_name)
-    finally:
-        shutil.rmtree(tmpdir)
-
-
-def extract_defines(flags):
-    """Extract the required #DEFINEs from deltachat.h.
-
-    Since #DEFINEs are interpreted by the C preprocessor we can not
-    use the compiler to extract these and need to parse the header
-    file ourselves.
-
-    The defines are returned in a string that can be passed to CFFIs
-    cdef() method.
-    """
-    header = find_header(flags)
-    defines_re = re.compile(r"""
-        \#define\s+   # The start of a define.
-        (             # Begin capturing group which captures the define name.
-            (?:       # A nested group which is not captured, this allows us
-                      #    to build the list of prefixes to extract without
-                      #    creation another capture group.
-                DC_EVENT
-                | DC_QR
-                | DC_MSG
-                | DC_LP
-                | DC_EMPTY
-                | DC_CERTCK
-                | DC_STATE
-                | DC_STR
-                | DC_CONTACT_ID
-                | DC_GCL
-                | DC_CHAT
-                | DC_PROVIDER
-                | DC_KEY_GEN
-            )         # End of prefix matching
-            _[\w_]+   # Match the suffix, e.g. _RSA2048 in DC_KEY_GEN_RSA2048
-        )             # Close the capturing group, this contains
-                      # the entire name e.g. DC_MSG_TEXT.
-        \s+\S+        # Ensure there is whitespace followed by a value.
-    """, re.VERBOSE)
-    defines = []
-    with open(header) as fp:
-        for line in fp:
-            match = defines_re.match(line)
-            if match:
-                defines.append(match.group(1))
-    return '\n'.join('#define {} ...'.format(d) for d in defines)
-
-
-def ffibuilder():
-    projdir = os.environ.get('DCC_RS_DEV')
-    if not projdir:
-        p = dn(dn(dn(dn(abspath(__file__)))))
-        projdir = os.environ["DCC_RS_DEV"] = p
-    target = os.environ.get('DCC_RS_TARGET', 'release')
-    if projdir:
-        flags = local_build_flags(projdir, target)
-    else:
-        flags = system_build_flags()
-    builder = cffi.FFI()
-    builder.set_source(
-        'deltachat.capi',
-        """
-            #include <deltachat.h>
-            int dc_event_has_string_data(int e)
-            {
-                return DC_EVENT_DATA2_IS_STRING(e);
-            }
-        """,
-        include_dirs=flags.incs,
-        libraries=flags.libs,
-        extra_objects=flags.objs,
-        extra_link_args=flags.extra_link_args,
-    )
-    builder.cdef("""
-        typedef int... time_t;
-        void free(void *ptr);
-        extern int dc_event_has_string_data(int);
-    """)
-    function_defs = extract_functions(flags)
-    defines = extract_defines(flags)
-    builder.cdef(function_defs)
-    builder.cdef(defines)
     return builder
 
 

python/src/deltachat/account.py

@@ -28,7 +28,7 @@ class Account(object):
     """
     MissingCredentials = MissingCredentials
 
-    def __init__(self, db_path, os_name=None, logging=True):
+    def __init__(self, db_path, os_name=None, logging=True, logid=None):
         """ initialize account object.
 
         :param db_path: a path to the account database. The database
@@ -38,6 +38,7 @@ class Account(object):
         # initialize per-account plugin system
         self._pm = hookspec.PerAccount._make_plugin_manager()
         self._logging = logging
+        self.logid = logid
 
         self.add_account_plugin(self)
 
@@ -213,40 +214,22 @@ class Account(object):
         """
         return Contact(self, const.DC_CONTACT_ID_SELF)
 
-    def create_contact(self, obj, name=None):
-        """ create a (new) Contact or return an existing one.
+    def create_contact(self, email, name=None):
+        """ create a (new) Contact. If there already is a Contact
+        with that e-mail address, it is unblocked and its name is
+        updated.
 
-        Calling this method will always resulut in the same
-        underlying contact id.  If there already is a Contact
-        with that e-mail address, it is unblocked and its display
-        `name` is updated if specified.
-
-        :param obj: email-address, Account or Contact instance.
-        :param name: (optional) display name for this contact
+        :param email: email-address (text type)
+        :param name: display name for this contact (optional)
         :returns: :class:`deltachat.contact.Contact` instance.
         """
-        if isinstance(obj, Account):
-            if not obj.is_configured():
-                raise ValueError("can only add addresses from configured accounts")
-            addr, displayname = obj.get_config("addr"), obj.get_config("displayname")
-        elif isinstance(obj, Contact):
-            if obj.account != self:
-                raise ValueError("account mismatch {}".format(obj))
-            addr, displayname = obj.addr, obj.name
-        elif isinstance(obj, str):
-            displayname, addr = parseaddr(obj)
-        else:
-            raise TypeError("don't know how to create chat for %r" % (obj, ))
-
-        if name is None and displayname:
-            name = displayname
-        return self._create_contact(addr, name)
-
-    def _create_contact(self, addr, name):
+        realname, addr = parseaddr(email)
+        if name:
+            realname = name
+        realname = as_dc_charpointer(realname)
         addr = as_dc_charpointer(addr)
-        name = as_dc_charpointer(name)
-        contact_id = lib.dc_create_contact(self._dc_context, name, addr)
-        assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL, contact_id
+        contact_id = lib.dc_create_contact(self._dc_context, realname, addr)
+        assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
         return Contact(self, contact_id)
 
     def delete_contact(self, contact):
@@ -268,13 +251,6 @@ class Account(object):
         if contact_id:
             return self.get_contact_by_id(contact_id)
 
-    def get_contact_by_id(self, contact_id):
-        """ return Contact instance or None.
-        :param contact_id: integer id of this contact.
-        :returns: None or :class:`deltachat.contact.Contact` instance.
-        """
-        return Contact(self, contact_id)
-
     def get_contacts(self, query=None, with_self=False, only_verified=False):
         """ get a (filtered) list of contacts.
 
@@ -304,29 +280,53 @@ class Account(object):
         )
         yield from iter_array(dc_array, lambda x: Message.from_db(self, x))
 
-    def create_chat(self, obj):
-        """ Create a 1:1 chat with Account, Contact or e-mail address. """
-        return self.create_contact(obj).create_chat()
+    def create_chat_by_contact(self, contact):
+        """ create or get an existing 1:1 chat object for the specified contact or contact id.
 
-    def _create_chat_by_message_id(self, msg_id):
-        return Chat(self, lib.dc_create_chat_by_msg_id(self._dc_context, msg_id))
+        :param contact: chat_id (int) or contact object.
+        :returns: a :class:`deltachat.chat.Chat` object.
+        """
+        if hasattr(contact, "id"):
+            if contact.account != self:
+                raise ValueError("Contact belongs to a different Account")
+            contact_id = contact.id
+        else:
+            assert isinstance(contact, int)
+            contact_id = contact
+        chat_id = lib.dc_create_chat_by_contact_id(self._dc_context, contact_id)
+        return Chat(self, chat_id)
 
-    def create_group_chat(self, name, contacts=None, verified=False):
+    def create_chat_by_message(self, message):
+        """ create or get an existing chat object for the
+        the specified message.
+
+        If this message is in the deaddrop chat then
+        the sender will become an accepted contact.
+
+        :param message: messsage id or message instance.
+        :returns: a :class:`deltachat.chat.Chat` object.
+        """
+        if hasattr(message, "id"):
+            if message.account != self:
+                raise ValueError("Message belongs to a different Account")
+            msg_id = message.id
+        else:
+            assert isinstance(message, int)
+            msg_id = message
+        chat_id = lib.dc_create_chat_by_msg_id(self._dc_context, msg_id)
+        return Chat(self, chat_id)
+
+    def create_group_chat(self, name, verified=False):
         """ create a new group chat object.
 
         Chats are unpromoted until the first message is sent.
 
-        :param contacts: list of contacts to add
         :param verified: if true only verified contacts can be added.
         :returns: a :class:`deltachat.chat.Chat` object.
         """
         bytes_name = name.encode("utf8")
         chat_id = lib.dc_create_group_chat(self._dc_context, int(verified), bytes_name)
-        chat = Chat(self, chat_id)
-        if contacts is not None:
-            for contact in contacts:
-                chat.add_contact(contact)
-        return chat
+        return Chat(self, chat_id)
 
     def get_chats(self):
         """ return list of chats.
@@ -355,6 +355,13 @@ class Account(object):
         """
         return Message.from_db(self, msg_id)
 
+    def get_contact_by_id(self, contact_id):
+        """ return Contact instance or None.
+        :param contact_id: integer id of this contact.
+        :returns: None or :class:`deltachat.contact.Contact` instance.
+        """
+        return Contact(self, contact_id)
+
     def get_chat_by_id(self, chat_id):
         """ return Chat instance.
         :param chat_id: integer id of this chat.
@@ -561,24 +568,29 @@ class Account(object):
         :raises MissingCredentials: if `addr` and `mail_pw` values are not set.
         :raises ConfigureFailed: if the account could not be configured.
 
-        :returns: None
+        :returns: None (account is configured and with io-scheduling running)
         """
         if not self.is_configured():
             raise ValueError("account not configured, cannot start io")
         lib.dc_start_io(self._dc_context)
 
     def configure(self):
-        """ Start configuration process and return a Configtracker instance
-        on which you can block with wait_finish() to get a True/False success
-        value for the configuration process.
-        """
         assert not self.is_configured()
+        assert not hasattr(self, "_configtracker")
         if not self.get_config("addr") or not self.get_config("mail_pw"):
             raise MissingCredentials("addr or mail_pwd not set in config")
-        configtracker = ConfigureTracker(self)
-        self.add_account_plugin(configtracker)
+        if hasattr(self, "_configtracker"):
+            self.remove_account_plugin(self._configtracker)
+        self._configtracker = ConfigureTracker()
+        self.add_account_plugin(self._configtracker)
         lib.dc_configure(self._dc_context)
-        return configtracker
+
+    def wait_configure_finish(self):
+        try:
+            self._configtracker.wait_finish()
+        finally:
+            self.remove_account_plugin(self._configtracker)
+            del self._configtracker
 
     def is_started(self):
         return self._event_thread.is_alive() and bool(lib.dc_is_io_running(self._dc_context))

python/src/deltachat/chat.py

@@ -18,8 +18,6 @@ class Chat(object):
     """
 
     def __init__(self, account, id):
-        from .account import Account
-        assert isinstance(account, Account), repr(account)
         self.account = account
         self.id = id
 
@@ -330,34 +328,33 @@ class Chat(object):
 
     # ------  group management API ------------------------------
 
-    def add_contact(self, obj):
+    def add_contact(self, contact):
         """ add a contact to this chat.
 
-        :params obj: Contact, Account or e-mail address.
+        :params: contact object.
         :raises ValueError: if contact could not be added
         :returns: None
         """
-        contact = self.account.create_contact(obj)
         ret = lib.dc_add_contact_to_chat(self.account._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not add contact {!r} to chat".format(contact))
-        return contact
 
-    def remove_contact(self, obj):
+    def remove_contact(self, contact):
         """ remove a contact from this chat.
 
-        :params obj: Contact, Account or e-mail address.
+        :params: contact object.
         :raises ValueError: if contact could not be removed
         :returns: None
         """
-        contact = self.account.create_contact(obj)
         ret = lib.dc_remove_contact_from_chat(self.account._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not remove contact {!r} from chat".format(contact))
 
     def get_contacts(self):
         """ get all contacts for this chat.
+        :params: contact object.
         :returns: list of :class:`deltachat.contact.Contact` objects for this chat
+
         """
         from .contact import Contact
         dc_array = ffi.gc(
@@ -368,14 +365,6 @@ class Chat(object):
             dc_array, lambda id: Contact(self.account, id))
         )
 
-    def num_contacts(self):
-        """ return number of contacts in this chat. """
-        dc_array = ffi.gc(
-            lib.dc_get_chat_contacts(self.account._dc_context, self.id),
-            lib.dc_array_unref
-        )
-        return lib.dc_array_get_cnt(dc_array)
-
     def set_profile_image(self, img_path):
         """Set group profile image.
 

python/src/deltachat/const.py

@@ -1,7 +1,197 @@
-from .capi import lib
+import sys
+import re
+import os
+from os.path import dirname, abspath
+from os.path import join as joinpath
+
+# the following const are generated from deltachat.h
+# this works well when you in a git-checkout
+# run "python deltachat/const.py" to regenerate events
+# begin const generated
+DC_GCL_ARCHIVED_ONLY = 0x01
+DC_GCL_NO_SPECIALS = 0x02
+DC_GCL_ADD_ALLDONE_HINT = 0x04
+DC_GCL_FOR_FORWARDING = 0x08
+DC_GCL_VERIFIED_ONLY = 0x01
+DC_GCL_ADD_SELF = 0x02
+DC_QR_ASK_VERIFYCONTACT = 200
+DC_QR_ASK_VERIFYGROUP = 202
+DC_QR_FPR_OK = 210
+DC_QR_FPR_MISMATCH = 220
+DC_QR_FPR_WITHOUT_ADDR = 230
+DC_QR_ACCOUNT = 250
+DC_QR_ADDR = 320
+DC_QR_TEXT = 330
+DC_QR_URL = 332
+DC_QR_ERROR = 400
+DC_CHAT_ID_DEADDROP = 1
+DC_CHAT_ID_TRASH = 3
+DC_CHAT_ID_MSGS_IN_CREATION = 4
+DC_CHAT_ID_STARRED = 5
+DC_CHAT_ID_ARCHIVED_LINK = 6
+DC_CHAT_ID_ALLDONE_HINT = 7
+DC_CHAT_ID_LAST_SPECIAL = 9
+DC_CHAT_TYPE_UNDEFINED = 0
+DC_CHAT_TYPE_SINGLE = 100
+DC_CHAT_TYPE_GROUP = 120
+DC_CHAT_TYPE_VERIFIED_GROUP = 130
+DC_MSG_ID_MARKER1 = 1
+DC_MSG_ID_DAYMARKER = 9
+DC_MSG_ID_LAST_SPECIAL = 9
+DC_STATE_UNDEFINED = 0
+DC_STATE_IN_FRESH = 10
+DC_STATE_IN_NOTICED = 13
+DC_STATE_IN_SEEN = 16
+DC_STATE_OUT_PREPARING = 18
+DC_STATE_OUT_DRAFT = 19
+DC_STATE_OUT_PENDING = 20
+DC_STATE_OUT_FAILED = 24
+DC_STATE_OUT_DELIVERED = 26
+DC_STATE_OUT_MDN_RCVD = 28
+DC_CONTACT_ID_SELF = 1
+DC_CONTACT_ID_INFO = 2
+DC_CONTACT_ID_DEVICE = 5
+DC_CONTACT_ID_LAST_SPECIAL = 9
+DC_MSG_TEXT = 10
+DC_MSG_IMAGE = 20
+DC_MSG_GIF = 21
+DC_MSG_STICKER = 23
+DC_MSG_AUDIO = 40
+DC_MSG_VOICE = 41
+DC_MSG_VIDEO = 50
+DC_MSG_FILE = 60
+DC_LP_AUTH_OAUTH2 = 0x2
+DC_LP_AUTH_NORMAL = 0x4
+DC_LP_IMAP_SOCKET_STARTTLS = 0x100
+DC_LP_IMAP_SOCKET_SSL = 0x200
+DC_LP_IMAP_SOCKET_PLAIN = 0x400
+DC_LP_SMTP_SOCKET_STARTTLS = 0x10000
+DC_LP_SMTP_SOCKET_SSL = 0x20000
+DC_LP_SMTP_SOCKET_PLAIN = 0x40000
+DC_CERTCK_AUTO = 0
+DC_CERTCK_STRICT = 1
+DC_CERTCK_ACCEPT_INVALID_CERTIFICATES = 3
+DC_EMPTY_MVBOX = 0x01
+DC_EMPTY_INBOX = 0x02
+DC_EVENT_INFO = 100
+DC_EVENT_SMTP_CONNECTED = 101
+DC_EVENT_IMAP_CONNECTED = 102
+DC_EVENT_SMTP_MESSAGE_SENT = 103
+DC_EVENT_IMAP_MESSAGE_DELETED = 104
+DC_EVENT_IMAP_MESSAGE_MOVED = 105
+DC_EVENT_IMAP_FOLDER_EMPTIED = 106
+DC_EVENT_NEW_BLOB_FILE = 150
+DC_EVENT_DELETED_BLOB_FILE = 151
+DC_EVENT_WARNING = 300
+DC_EVENT_ERROR = 400
+DC_EVENT_ERROR_NETWORK = 401
+DC_EVENT_ERROR_SELF_NOT_IN_GROUP = 410
+DC_EVENT_MSGS_CHANGED = 2000
+DC_EVENT_INCOMING_MSG = 2005
+DC_EVENT_MSG_DELIVERED = 2010
+DC_EVENT_MSG_FAILED = 2012
+DC_EVENT_MSG_READ = 2015
+DC_EVENT_CHAT_MODIFIED = 2020
+DC_EVENT_CONTACTS_CHANGED = 2030
+DC_EVENT_LOCATION_CHANGED = 2035
+DC_EVENT_CONFIGURE_PROGRESS = 2041
+DC_EVENT_IMEX_PROGRESS = 2051
+DC_EVENT_IMEX_FILE_WRITTEN = 2052
+DC_EVENT_SECUREJOIN_INVITER_PROGRESS = 2060
+DC_EVENT_SECUREJOIN_JOINER_PROGRESS = 2061
+DC_EVENT_FILE_COPIED = 2055
+DC_EVENT_IS_OFFLINE = 2081
+DC_EVENT_GET_STRING = 2091
+DC_STR_SELFNOTINGRP = 21
+DC_KEY_GEN_DEFAULT = 0
+DC_KEY_GEN_RSA2048 = 1
+DC_KEY_GEN_ED25519 = 2
+DC_PROVIDER_STATUS_OK = 1
+DC_PROVIDER_STATUS_PREPARATION = 2
+DC_PROVIDER_STATUS_BROKEN = 3
+DC_CHAT_VISIBILITY_NORMAL = 0
+DC_CHAT_VISIBILITY_ARCHIVED = 1
+DC_CHAT_VISIBILITY_PINNED = 2
+DC_STR_NOMESSAGES = 1
+DC_STR_SELF = 2
+DC_STR_DRAFT = 3
+DC_STR_VOICEMESSAGE = 7
+DC_STR_DEADDROP = 8
+DC_STR_IMAGE = 9
+DC_STR_VIDEO = 10
+DC_STR_AUDIO = 11
+DC_STR_FILE = 12
+DC_STR_STATUSLINE = 13
+DC_STR_NEWGROUPDRAFT = 14
+DC_STR_MSGGRPNAME = 15
+DC_STR_MSGGRPIMGCHANGED = 16
+DC_STR_MSGADDMEMBER = 17
+DC_STR_MSGDELMEMBER = 18
+DC_STR_MSGGROUPLEFT = 19
+DC_STR_GIF = 23
+DC_STR_ENCRYPTEDMSG = 24
+DC_STR_E2E_AVAILABLE = 25
+DC_STR_ENCR_TRANSP = 27
+DC_STR_ENCR_NONE = 28
+DC_STR_CANTDECRYPT_MSG_BODY = 29
+DC_STR_FINGERPRINTS = 30
+DC_STR_READRCPT = 31
+DC_STR_READRCPT_MAILBODY = 32
+DC_STR_MSGGRPIMGDELETED = 33
+DC_STR_E2E_PREFERRED = 34
+DC_STR_CONTACT_VERIFIED = 35
+DC_STR_CONTACT_NOT_VERIFIED = 36
+DC_STR_CONTACT_SETUP_CHANGED = 37
+DC_STR_ARCHIVEDCHATS = 40
+DC_STR_STARREDMSGS = 41
+DC_STR_AC_SETUP_MSG_SUBJECT = 42
+DC_STR_AC_SETUP_MSG_BODY = 43
+DC_STR_CANNOT_LOGIN = 60
+DC_STR_SERVER_RESPONSE = 61
+DC_STR_MSGACTIONBYUSER = 62
+DC_STR_MSGACTIONBYME = 63
+DC_STR_MSGLOCATIONENABLED = 64
+DC_STR_MSGLOCATIONDISABLED = 65
+DC_STR_LOCATION = 66
+DC_STR_STICKER = 67
+DC_STR_DEVICE_MESSAGES = 68
+DC_STR_COUNT = 68
+# end const generated
 
 
-for name in dir(lib):
-    if name.startswith("DC_"):
-        globals()[name] = getattr(lib, name)
-del name
+def read_event_defines(f):
+    rex = re.compile(r'#define\s+((?:DC_EVENT|DC_QR|DC_MSG|DC_LP|DC_EMPTY|DC_CERTCK|DC_STATE|DC_STR|'
+                     r'DC_CONTACT_ID|DC_GCL|DC_CHAT|DC_PROVIDER|DC_KEY_GEN)_\S+)\s+([x\d]+).*')
+    for line in f:
+        m = rex.match(line)
+        if m:
+            yield m.groups()
+
+
+if __name__ == "__main__":
+    here = abspath(__file__).rstrip("oc")
+    here_dir = dirname(here)
+    if len(sys.argv) >= 2:
+        deltah = sys.argv[1]
+    else:
+        deltah = joinpath(dirname(dirname(dirname(here_dir))), "deltachat-ffi", "deltachat.h")
+    assert os.path.exists(deltah)
+
+    lines = []
+    skip_to_end = False
+    for orig_line in open(here):
+        if skip_to_end:
+            if not orig_line.startswith("# end const"):
+                continue
+            skip_to_end = False
+        lines.append(orig_line)
+        if orig_line.startswith("# begin const"):
+            with open(deltah) as f:
+                for name, item in read_event_defines(f):
+                    lines.append("{} = {}\n".format(name, item))
+            skip_to_end = True
+
+    tmpname = here + ".tmp"
+    with open(tmpname, "w") as f:
+        f.write("".join(lines))
+    os.rename(tmpname, here)

python/src/deltachat/contact.py

@@ -3,8 +3,6 @@
 from . import props
 from .cutil import from_dc_charpointer
 from .capi import lib, ffi
-from .chat import Chat
-from . import const
 
 
 class Contact(object):
@@ -13,8 +11,6 @@ class Contact(object):
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
     def __init__(self, account, id):
-        from .account import Account
-        assert isinstance(account, Account), repr(account)
         self.account = account
         self.id = id
 
@@ -40,13 +36,10 @@ class Contact(object):
         return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
 
     @props.with_doc
-    def name(self):
+    def display_name(self):
         """ display name for this contact. """
         return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
 
-    # deprecated alias
-    display_name = name
-
     def is_blocked(self):
         """ Return True if the contact is blocked. """
         return lib.dc_contact_is_blocked(self._dc_contact)
@@ -65,16 +58,6 @@ class Contact(object):
             return None
         return from_dc_charpointer(dc_res)
 
-    def create_chat(self):
-        """ create or get an existing 1:1 chat object for the specified contact or contact id.
-
-        :param contact: chat_id (int) or contact object.
-        :returns: a :class:`deltachat.chat.Chat` object.
-        """
-        dc_context = self.account._dc_context
-        chat_id = lib.dc_create_chat_by_contact_id(dc_context, self.id)
-        assert chat_id > const.DC_CHAT_ID_LAST_SPECIAL, chat_id
-        return Chat(self.account, chat_id)
-
-    # deprecated name
-    get_chat = create_chat
+    def get_chat(self):
+        """return 1:1 chat for this contact. """
+        return self.account.create_chat_by_contact(self)

python/src/deltachat/direct_imap.py

@@ -1,211 +1,141 @@
-"""
-Internal Python-level IMAP handling used by the testplugin
-and for cleaning up inbox/mvbox for each test function run.
-"""
-
-import io
-import email
-import ssl
+import imaplib
 import pathlib
-from imapclient import IMAPClient
-from imapclient.exceptions import IMAPClientError
-import deltachat
+from . import Account
+
+INBOX = "Inbox"
+SENT = "Sent"
+MVBOX = "DeltaChat"
+MVBOX_FALLBBACK = "INBOX/DeltaChat"
+DC_CONSTANT_MSG_MOVESTATE_PENDING = 1
+DC_CONSTANT_MSG_MOVESTATE_STAY = 2
+DC_CONSTANT_MSG_MOVESTATE_MOVING = 3
 
 
-SEEN = b'\\Seen'
-DELETED = b'\\Deleted'
-FLAGS = b'FLAGS'
-FETCH = b'FETCH'
-ALL = "1:*"
+def db_folder_attr(name):
+    def fget(s):
+        return s.db_folder.get(name, 1)
+
+    def fset(s, val):
+        s.db_folder[name] = val
+    return property(fget, fset, None, None)
 
 
-@deltachat.global_hookimpl
-def dc_account_extra_configure(account):
-    """ Reset the account (we reuse accounts across tests)
-    and make 'account.direct_imap' available for direct IMAP ops.
-    """
-    imap = DirectImap(account)
-    if imap.select_config_folder("mvbox"):
-        imap.delete(ALL, expunge=True)
-    assert imap.select_config_folder("inbox")
-    imap.delete(ALL, expunge=True)
-    setattr(account, "direct_imap", imap)
+class ImapConn():
+    def __init__(self, foldername, conn_info):
+        self.foldername = foldername
+        host, user, pw = conn_info
 
-
-@deltachat.global_hookimpl
-def dc_account_after_shutdown(account):
-    """ shutdown the imap connection if there is one. """
-    imap = getattr(account, "direct_imap", None)
-    if imap is not None:
-        imap.shutdown()
-        del account.direct_imap
-
-
-class DirectImap:
-    def __init__(self, account):
-        self.account = account
-        self.logid = account.get_config("displayname") or id(account)
-        self._idling = False
-        self.connect()
-
-    def connect(self):
-        ssl_context = ssl.create_default_context()
-
-        # don't check if certificate hostname doesn't match target hostname
-        ssl_context.check_hostname = False
-
-        # don't check if the certificate is trusted by a certificate authority
-        ssl_context.verify_mode = ssl.CERT_NONE
-
-        host = self.account.get_config("configured_mail_server")
-        user = self.account.get_config("addr")
-        pw = self.account.get_config("mail_pw")
-        self.conn = IMAPClient(host, ssl_context=ssl_context)
-        self.conn.login(user, pw)
-
-        self.select_folder("INBOX")
-
-    def shutdown(self):
+        self.connection = imaplib.IMAP4_SSL(host)
+        self.connection.login(user, pw)
+        messages = self.reselect_folder()
         try:
-            self.conn.idle_done()
-        except (OSError, IMAPClientError):
-            pass
-        try:
-            self.conn.logout()
-        except (OSError, IMAPClientError):
-            print("Could not logout direct_imap conn")
-
-    def select_folder(self, foldername):
-        assert not self._idling
-        return self.conn.select_folder(foldername)
-
-    def select_config_folder(self, config_name):
-        """ Return info about selected folder if it is
-        configured, otherwise None. """
-        if "_" not in config_name:
-            config_name = "configured_{}_folder".format(config_name)
-        foldername = self.account.get_config(config_name)
-        if foldername:
-            return self.select_folder(foldername)
-
-    def list_folders(self):
-        """ return list of all existing folder names"""
-        assert not self._idling
-        folders = []
-        for meta, sep, foldername in self.conn.list_folders():
-            folders.append(foldername)
-        return folders
-
-    def delete(self, range, expunge=True):
-        """ delete a range of messages (imap-syntax).
-        If expunge is true, perform the expunge-operation
-        to make sure the messages are really gone and not
-        just flagged as deleted.
-        """
-        self.conn.set_flags(range, [DELETED])
-        if expunge:
-            self.conn.expunge()
-
-    def get_all_messages(self):
-        assert not self._idling
-        return self.conn.fetch(ALL, [FLAGS])
-
-    def get_unread_messages(self):
-        assert not self._idling
-        res = self.conn.fetch(ALL, [FLAGS])
-        return [uid for uid in res
-                if SEEN not in res[uid][FLAGS]]
+            self.original_msg_count = int(messages[0])
+        except IndexError:
+            self.original_msg_count = 0
 
     def mark_all_read(self):
-        messages = self.get_unread_messages()
-        if messages:
-            res = self.conn.set_flags(messages, [SEEN])
-            print("marked seen:", messages, res)
+        self.reselect_folder()
+#        result, data = self.connection.uid('search', None, "(UNSEEN)")
+        result, data = self.connection.search(None, 'UnSeen')
+        try:
+            mails_uid = data[0].split()
+            print("New mails")
+
+#            self.connection.store(data[0].replace(' ',','),'+FLAGS','\Seen')
+            for e_id in mails_uid:
+                self.connection.store(e_id, '+FLAGS', '\\Seen')
+                print("marked:", e_id)
+
+            return True
+        except IndexError:
+            print("No unread")
+            return False
 
     def get_unread_cnt(self):
-        return len(self.get_unread_messages())
+        self.reselect_folder()
+#        result, data = self.connection.uid('search', None, "(UNSEEN)")
+        result, data = self.connection.search(None, 'UnSeen')
+        try:
+            mails_uid = data[0].split()
 
-    def dump_account_info(self, logfile):
-        def log(*args, **kwargs):
-            kwargs["file"] = logfile
-            print(*args, **kwargs)
+            return len(mails_uid)
+        except IndexError:
+            return 0
 
-        cursor = 0
-        for name, val in self.account.get_info().items():
-            entry = "{}={}".format(name.upper(), val)
-            if cursor + len(entry) > 80:
-                log("")
-                cursor = 0
-            log(entry, end=" ")
-            cursor += len(entry) + 1
-        log("")
+    def get_new_email_cnt(self):
+        messages = self.reselect_folder()
+        try:
+            return int(messages[0]) - self.original_msg_count
+        except IndexError:
+            return 0
 
-    def dump_imap_structures(self, dir, logfile):
-        assert not self._idling
-        stream = io.StringIO()
+    def reselect_folder(self):
+        status, messages = self.connection.select(self.foldername)
+        if status != "OK":
+            print("Incorrect mail box " + status + str(messages))
+            raise ConnectionError
+#        print("(Re-)Selected mailbox: " + status + " " + str(messages))
+        return messages
 
-        def log(*args, **kwargs):
-            kwargs["file"] = stream
-            print(*args, **kwargs)
+    def __del__(self):
+        try:
+            self.connection.close()
+        except Exception:
+            pass
+        try:
+            self.connection.logout()
+        except Exception:
+            print("Could not logout direct_imap conn")
 
-        empty_folders = []
-        for imapfolder in self.list_folders():
-            self.select_folder(imapfolder)
-            messages = list(self.get_all_messages())
-            if not messages:
-                empty_folders.append(imapfolder)
-                continue
 
-            log("---------", imapfolder, len(messages), "messages ---------")
-            # get message content without auto-marking it as seen
-            # fetching 'RFC822' would mark it as seen.
-            requested = [b'BODY.PEEK[HEADER]', FLAGS]
-            for uid, data in self.conn.fetch(messages, requested).items():
-                body_bytes = data[b'BODY[HEADER]']
-                flags = data[FLAGS]
-                path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
+def make_direct_imap(account, folder):
+    conn_info = (account.get_config("configured_mail_server"),
+                 account.get_config("addr"), account.get_config("mail_pw"))
+    # try:
+    #     return ImapConn(folder, conn_info=conn_info)
+    # except ConnectionError as e:
+    #     if folder == MVBOX:
+    #         account.log("Selecting " + MVBOX_FALLBBACK + " not " + MVBOX + " because connecting to the latter failed")
+    #         return ImapConn(MVBOX_FALLBBACK, conn_info=conn_info)
+    #     else:
+    #         raise e
+    if folder == MVBOX:
+        new_folder = account.get_config("configured_mvbox_folder")
+    else:
+        new_folder = folder
+    if new_folder != folder:
+        account.log("Making connection with " + new_folder + " not " + folder)
+    return ImapConn(new_folder, conn_info=conn_info)
+
+
+def print_imap_structure(database, dir="."):
+    print_imap_structure_ac(Account(database), dir)
+
+
+def print_imap_structure_ac(ac, dir="."):
+    acinfo = ac.logid + "-" + ac.get_config("addr")
+    print("================= ACCOUNT", acinfo, "=================")
+    print("----------------- CONFIG: -----------------")
+    print(ac.get_info())
+
+    for imapfolder in [INBOX, MVBOX, SENT, MVBOX_FALLBBACK]:
+        try:
+            imap = make_direct_imap(ac, imapfolder)
+            c = imap.connection
+            typ, data = c.search(None, 'ALL')
+            c._get_tagged_response
+            print("-----------------", imapfolder, "-----------------")
+            for num in data[0].split():
+                typ, data = c.fetch(num, '(RFC822)')
+                body = data[0][1]
+
+                typ, data = c.fetch(num, '(UID FLAGS)')
+                info = data[0]
+
+                path = pathlib.Path(dir).joinpath("IMAP-MESSAGES", acinfo, imapfolder)
                 path.mkdir(parents=True, exist_ok=True)
-                fn = path.joinpath(str(uid))
-                fn.write_bytes(body_bytes)
-                log("Message", uid, fn)
-                email_message = email.message_from_bytes(body_bytes)
-                log("Message", uid, flags, "Message-Id:", email_message.get("Message-Id"))
-
-        if empty_folders:
-            log("--------- EMPTY FOLDERS:", empty_folders)
-
-        print(stream.getvalue(), file=logfile)
-
-    def idle_start(self):
-        """ switch this connection to idle mode. non-blocking. """
-        assert not self._idling
-        res = self.conn.idle()
-        self._idling = True
-        return res
-
-    def idle_check(self, terminate=False):
-        """ (blocking) wait for next idle message from server. """
-        assert self._idling
-        self.account.log("imap-direct: calling idle_check")
-        res = self.conn.idle_check()
-        if terminate:
-            self.idle_done()
-        return res
-
-    def idle_wait_for_seen(self):
-        """ Return first message with SEEN flag
-        from a running idle-stream REtiurn.
-        """
-        while 1:
-            for item in self.idle_check():
-                if item[1] == FETCH:
-                    if item[2][0] == FLAGS:
-                        if SEEN in item[2][1]:
-                            return item[0]
-
-    def idle_done(self):
-        """ send idle-done to server if we are currently in idle mode. """
-        if self._idling:
-            res = self.conn.idle_done()
-            self._idling = False
-            return res
+                file = path.joinpath(str(info).replace("b'", "").replace("'", "").replace("\\", ""))
+                file.write_bytes(body)
+                print("Message", info, "saved as", file)
+        except Exception:
+            pass

python/src/deltachat/events.py

@@ -28,9 +28,13 @@ class FFIEventLogger:
     # to prevent garbled logging
     _loglock = threading.RLock()
 
-    def __init__(self, account):
+    def __init__(self, account, logid):
+        """
+        :param logid: an optional logging prefix that should be used with
+                      the default internal logging.
+        """
         self.account = account
-        self.logid = self.account.get_config("displayname")
+        self.logid = logid
         self.init_time = time.time()
 
     @account_hookimpl
@@ -123,12 +127,6 @@ class FFIEventTracker:
         if ev.data2 > 0:
             return self.account.get_message_by_id(ev.data2)
 
-    def wait_msg_delivered(self, msg):
-        ev = self.get_matching("DC_EVENT_MSG_DELIVERED")
-        assert ev.data1 == msg.chat.id
-        assert ev.data2 == msg.id
-        assert msg.is_out_delivered()
-
 
 class EventThread(threading.Thread):
     """ Event Thread for an account.

python/src/deltachat/hookspec.py

@@ -43,7 +43,7 @@ class PerAccount:
 
     @account_hookspec
     def ac_configure_completed(self, success):
-        """ Called after a configure process completed. """
+        """ Called when a configure process completed. """
 
     @account_hookspec
     def ac_incoming_message(self, message):
@@ -88,14 +88,6 @@ class Global:
     def dc_account_init(self, account):
         """ called when `Account::__init__()` function starts executing. """
 
-    @global_hookspec
-    def dc_account_extra_configure(self, account):
-        """ Called when account configuration successfully finished.
-
-        This hook can be used to perform extra work before
-        ac_configure_completed is called.
-        """
-
     @global_hookspec
     def dc_account_after_shutdown(self, account):
         """ Called after the account has been shutdown. """

python/src/deltachat/message.py

@@ -53,19 +53,15 @@ class Message(object):
             lib.dc_msg_unref
         ))
 
-    def create_chat(self):
-        """ create or get an existing chat (group) object for this message.
-
-        If the message is a deaddrop contact request
-        the sender will become an accepted contact.
-
-        :returns: a :class:`deltachat.chat.Chat` object.
+    def accept_sender_contact(self):
+        """ ensure that the sender is an accepted contact
+        and that the message has a non-deaddrop chat object.
         """
-        from .chat import Chat
-        chat_id = lib.dc_create_chat_by_msg_id(self.account._dc_context, self.id)
-        ctx = self.account._dc_context
-        self._dc_msg = ffi.gc(lib.dc_get_msg(ctx, self.id), lib.dc_msg_unref)
-        return Chat(self.account, chat_id)
+        self.account.create_chat_by_message(self)
+        self._dc_msg = ffi.gc(
+                lib.dc_get_msg(self.account._dc_context, self.id),
+                lib.dc_msg_unref
+        )
 
     @props.with_doc
     def text(self):

python/src/deltachat/testplugin.py

@@ -1,7 +1,6 @@
 from __future__ import print_function
 import os
 import sys
-import io
 import subprocess
 import queue
 import threading
@@ -13,11 +12,10 @@ import tempfile
 import pytest
 import requests
 
-from . import Account, const
+from . import Account, const, direct_imap
 from .capi import lib
 from .events import FFIEventLogger, FFIEventTracker
 from _pytest._code import Source
-from deltachat import direct_imap
 
 import deltachat
 
@@ -215,7 +213,6 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             self._generated_keys = ["alice", "bob", "charlie",
                                     "dom", "elena", "fiona"]
             self.set_logging_default(False)
-            deltachat.register_global_plugin(direct_imap)
 
         def finalize(self):
             while self._finalizers:
@@ -226,15 +223,13 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 acc = self._accounts.pop()
                 acc.shutdown()
                 acc.disable_logging()
-            deltachat.unregister_global_plugin(direct_imap)
 
         def make_account(self, path, logid, quiet=False):
-            ac = Account(path, logging=self._logging)
+            ac = Account(path, logging=self._logging, logid=logid)
             ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
             ac.addr = ac.get_self_contact().addr
-            ac.set_config("displayname", logid)
             if not quiet:
-                ac.add_account_plugin(FFIEventLogger(ac))
+                ac.add_account_plugin(FFIEventLogger(ac, logid=logid))
             self._accounts.append(ac)
             return ac
 
@@ -303,27 +298,31 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             configdict["mvbox_move"] = str(int(move))
             configdict["sentbox_watch"] = str(int(sentbox))
             ac.update_config(configdict)
-            ac._configtracker = ac.configure()
+            ac.configure()
             return ac
 
         def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
             ac1 = self.get_online_configuring_account(
                 pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
-            self.wait_configure_and_start_io()
+            ac1.wait_configure_finish()
+            ac1.start_io()
             return ac1
 
         def get_two_online_accounts(self, move=False, quiet=False):
-            ac1 = self.get_online_configuring_account(move=move, quiet=quiet)
+            ac1 = self.get_online_configuring_account(move=True, quiet=quiet)
             ac2 = self.get_online_configuring_account(quiet=quiet)
-            self.wait_configure_and_start_io()
+            ac1.wait_configure_finish()
+            ac1.start_io()
+            ac2.wait_configure_finish()
+            ac2.start_io()
             return ac1, ac2
 
-        def get_many_online_accounts(self, num, move=True):
-            accounts = [self.get_online_configuring_account(move=move, quiet=True)
+        def get_many_online_accounts(self, num, move=True, quiet=True):
+            accounts = [self.get_online_configuring_account(move=move, quiet=quiet)
                         for i in range(num)]
-            self.wait_configure_and_start_io()
             for acc in accounts:
-                acc.add_account_plugin(FFIEventLogger(acc))
+                acc._configtracker.wait_finish()
+                acc.start_io()
             return accounts
 
         def clone_online_account(self, account, pre_generated_key=True):
@@ -341,28 +340,14 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 mvbox_move=account.get_config("mvbox_move"),
                 sentbox_watch=account.get_config("sentbox_watch"),
             ))
-            ac._configtracker = ac.configure()
+            ac.configure()
             return ac
 
-        def wait_configure_and_start_io(self):
-            for acc in self._accounts:
-                if hasattr(acc, "_configtracker"):
-                    acc._configtracker.wait_finish()
-                    del acc._configtracker
-                if acc.is_configured() and not acc.is_started():
-                    acc.start_io()
-                print("{}: {} account was successfully setup".format(
-                    acc.get_config("displayname"), acc.get_config("addr")))
-
         def run_bot_process(self, module, ffi=True):
             fn = module.__file__
 
             bot_ac, bot_cfg = self.get_online_config()
 
-            # Avoid starting ac so we don't interfere with the bot operating on
-            # the same database.
-            self._accounts.remove(bot_ac)
-
             args = [
                 sys.executable,
                 "-u",
@@ -387,42 +372,12 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             self._finalizers.append(bot.kill)
             return bot
 
-        def dump_imap_summary(self, logfile):
-            for ac in self._accounts:
-                imap = getattr(ac, "direct_imap", None)
-                if imap is not None:
-                    try:
-                        imap.idle_done()
-                    except Exception:
-                        pass
-                    imap.dump_account_info(logfile=logfile)
-                    imap.dump_imap_structures(tmpdir, logfile=logfile)
-
-        def get_accepted_chat(self, ac1, ac2):
-            ac2.create_chat(ac1)
-            return ac1.create_chat(ac2)
-
-        def introduce_each_other(self, accounts, sending=True):
-            to_wait = []
-            for i, acc in enumerate(accounts):
-                for acc2 in accounts[i + 1:]:
-                    chat = self.get_accepted_chat(acc, acc2)
-                    if sending:
-                        chat.send_text("hi")
-                        to_wait.append(acc2)
-                        acc2.create_chat(acc).send_text("hi back")
-                        to_wait.append(acc)
-            for acc in to_wait:
-                acc._evtracker.wait_next_incoming_message()
-
     am = AccountMaker()
     request.addfinalizer(am.finalize)
     yield am
-    if hasattr(request.node, "rep_call") and request.node.rep_call.failed:
-        logfile = io.StringIO()
-        am.dump_imap_summary(logfile=logfile)
-        print(logfile.getvalue())
-        # request.node.add_report_section("call", "imap-server-state", s)
+    if request.node.rep_call.failed:
+        for ac in am._accounts:
+            direct_imap.print_imap_structure_ac(ac, tmpdir)
 
 
 class BotProcess:
@@ -490,7 +445,6 @@ def lp():
 
         def step(self, msg):
             print("-" * 5, "step " + msg, "-" * 5)
-
     return Printer()
 
 

python/src/deltachat/tracker.py

@@ -2,7 +2,7 @@
 from queue import Queue
 from threading import Event
 
-from .hookspec import account_hookimpl, Global
+from .hookspec import account_hookimpl
 
 
 class ImexFailed(RuntimeError):
@@ -40,14 +40,12 @@ class ConfigureFailed(RuntimeError):
 class ConfigureTracker:
     ConfigureFailed = ConfigureFailed
 
-    def __init__(self, account):
-        self.account = account
+    def __init__(self):
         self._configure_events = Queue()
         self._smtp_finished = Event()
         self._imap_finished = Event()
         self._ffi_events = []
         self._progress = Queue()
-        self._gm = Global._get_plugin_manager()
 
     @account_hookimpl
     def ac_process_ffi_event(self, ffi_event):
@@ -61,10 +59,7 @@ class ConfigureTracker:
 
     @account_hookimpl
     def ac_configure_completed(self, success):
-        if success:
-            self._gm.hook.dc_account_extra_configure(account=self.account)
         self._configure_events.put(success)
-        self.account.remove_account_plugin(self)
 
     def wait_smtp_connected(self):
         """ wait until smtp is configured. """

python/tests/stress_test_db.py

@@ -110,7 +110,7 @@ class AutoReplier:
         if self.current_sent >= self.num_send:
             self.report_func(self, ReportType.exit)
             return
-        message.create_chat()
+        message.accept_sender_contact()
         message.mark_seen()
         self.log("incoming message: {}".format(message))
 

python/tests/test_increation.py

@@ -36,7 +36,9 @@ def wait_msgs_changed(account, msgs_list):
 class TestOnlineInCreation:
     def test_increation_not_blobdir(self, tmpdir, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
-        chat = ac1.create_chat(ac2)
+
+        c2 = ac1.create_contact(email=ac2.get_config("addr"))
+        chat = ac1.create_chat_by_contact(c2)
 
         lp.sec("Creating in-creation file outside of blobdir")
         assert tmpdir.strpath != ac1.get_blobdir()
@@ -46,7 +48,9 @@ class TestOnlineInCreation:
 
     def test_no_increation_copies_to_blobdir(self, tmpdir, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
-        chat = ac1.create_chat(ac2)
+
+        c2 = ac1.create_contact(email=ac2.get_config("addr"))
+        chat = ac1.create_chat_by_contact(c2)
 
         lp.sec("Creating file outside of blobdir")
         assert tmpdir.strpath != ac1.get_blobdir()
@@ -60,7 +64,9 @@ class TestOnlineInCreation:
     def test_forward_increation(self, acfactory, data, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
-        chat = ac1.create_chat(ac2)
+        c2 = ac1.create_contact(email=ac2.get_config("addr"))
+        chat = ac1.create_chat_by_contact(c2)
+        assert chat.id >= const.DC_CHAT_ID_LAST_SPECIAL
         wait_msgs_changed(ac1, [(0, 0)])  # why no chat id?
 
         lp.sec("create a message with a file in creation")
@@ -74,7 +80,7 @@ class TestOnlineInCreation:
 
         lp.sec("forward the message while still in creation")
         chat2 = ac1.create_group_chat("newgroup")
-        chat2.add_contact(ac2)
+        chat2.add_contact(c2)
         wait_msgs_changed(ac1, [(0, 0)])  # why not chat id?
         ac1.forward_messages([prepared_original], chat2)
         # XXX there might be two EVENT_MSGS_CHANGED and only one of them

python/tests/test_lowlevel.py

@@ -69,8 +69,8 @@ def test_sig():
 def test_markseen_invalid_message_ids(acfactory):
     ac1 = acfactory.get_configured_offline_account()
 
-    contact1 = ac1.create_contact("some1@example.com", name="some1")
-    chat = contact1.create_chat()
+    contact1 = ac1.create_contact(email="some1@example.com", name="some1")
+    chat = ac1.create_chat_by_contact(contact1)
     chat.send_text("one messae")
     ac1._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
     msg_ids = [9]

spec.md

@@ -1,12 +1,10 @@
-# chat-mail specification
+# Chat-over-Email specification
 
-Version: 0.32.0
-Status:  In-progress 
-Format:  [Semantic Line Breaks](https://sembr.org/)
+Version 0.30.0
 
-This document roughly describes how chat-mail 
-apps use the standard e-mail system 
-to implement typical messenger functions.
+This document describes how emails can be used
+to implement typical messenger functions
+while staying compatible to existing MUAs.
 
 - [Encryption](#encryption)
 - [Outgoing messages](#outgoing-messages)
@@ -32,14 +30,17 @@ Messages SHOULD be encrypted by the
 `prefer-encrypt=mutual` MAY be set by default.
 
 Meta data (at least the subject and all chat-headers) SHOULD be encrypted
-by the [Protected Headers](https://www.ietf.org/id/draft-autocrypt-lamps-protected-headers-02.html) standard.
+by the [Memoryhole](https://github.com/autocrypt/memoryhole) standard.
+If Memoryhole is not used,
+the subject of encrypted messages SHOULD be replaced by the string `...`.
 
 
 # Outgoing messages
 
 Messengers MUST add a `Chat-Version: 1.0` header to outgoing messages.
 For filtering and smart appearance of the messages in normal MUAs,
-the `Subject` header SHOULD be `Message from <sender name>`.
+the `Subject` header SHOULD start with the characters `Chat:`
+and SHOULD be an excerpt of the message.
 Replies to messages MAY follow the typical `Re:`-format.
 
 The body MAY contain text which MUST have the content type `text/plain`
@@ -57,7 +58,7 @@ Full quotes, footers or sth. like that MUST NOT go to the user-text-part.
     To: rcpt@domain
     Chat-Version: 1.0
     Content-Type: text/plain
-    Subject: Message from sender@domain
+    Subject: Chat: Hello ...
 
     Hello world!
 

src/blob.rs

@@ -8,14 +8,11 @@ use async_std::prelude::*;
 use async_std::{fs, io};
 
 use image::GenericImageView;
-use num_traits::FromPrimitive;
 use thiserror::Error;
 
-use crate::config::Config;
-use crate::constants::*;
+use crate::constants::AVATAR_SIZE;
 use crate::context::Context;
 use crate::events::Event;
-use crate::message;
 
 /// Represents a file in the blob directory.
 ///
@@ -60,7 +57,7 @@ impl<'a> BlobObject<'a> {
             .map_err(|err| BlobError::WriteFailure {
                 blobdir: blobdir.to_path_buf(),
                 blobname: name.clone(),
-                cause: err.into(),
+                cause: err,
             })?;
         let blob = BlobObject {
             blobdir,
@@ -373,49 +370,11 @@ impl<'a> BlobObject<'a> {
         let img = img.thumbnail(AVATAR_SIZE, AVATAR_SIZE);
 
         img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
-            blobdir: context.get_blobdir().to_path_buf(),
-            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err.into(),
-        })?;
-
-        Ok(())
-    }
-
-    pub async fn recode_to_image_size(&self, context: &Context) -> Result<(), BlobError> {
-        let blob_abs = self.to_abs_path();
-        if message::guess_msgtype_from_suffix(Path::new(&blob_abs))
-            != Some((Viewtype::Image, "image/jpeg"))
-        {
-            return Ok(());
-        }
-
-        let img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
             blobdir: context.get_blobdir().to_path_buf(),
             blobname: blob_abs.to_str().unwrap_or_default().to_string(),
             cause: err,
         })?;
 
-        let img_wh = if MediaQuality::from_i32(context.get_config_int(Config::MediaQuality).await)
-            .unwrap_or_default()
-            == MediaQuality::Balanced
-        {
-            BALANCED_IMAGE_SIZE
-        } else {
-            WORSE_IMAGE_SIZE
-        };
-
-        if img.width() <= img_wh && img.height() <= img_wh {
-            return Ok(());
-        }
-
-        let img = img.thumbnail(img_wh, img_wh);
-
-        img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
-            blobdir: context.get_blobdir().to_path_buf(),
-            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err.into(),
-        })?;
-
         Ok(())
     }
 }
@@ -441,7 +400,7 @@ pub enum BlobError {
         blobdir: PathBuf,
         blobname: String,
         #[source]
-        cause: anyhow::Error,
+        cause: std::io::Error,
     },
     #[error("Failed to copy data from {} to blob {blobname} in {}", .src.display(), .blobdir.display())]
     CopyFailure {

src/chat.rs

@@ -39,9 +39,18 @@ impl ChatId {
         ChatId(id)
     }
 
+    /// A ChatID which indicates an error.
+    ///
+    /// This is transitional and should not be used in new code.  Do
+    /// not represent errors in a ChatId.
+    pub fn is_error(self) -> bool {
+        self.0 == 0
+    }
+
     /// An unset ChatId
     ///
-    /// This is transitional and should not be used in new code.
+    /// Like [ChatId::is_error], from which it is indistinguishable, this is
+    /// transitional and should not be used in new code.
     pub fn is_unset(self) -> bool {
         self.0 == 0
     }
@@ -422,35 +431,15 @@ impl ChatId {
     }
 
     async fn get_parent_mime_headers(self, context: &Context) -> Option<(String, String, String)> {
-        let collect =
-            |row: &rusqlite::Row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?));
-        let (packed, rfc724_mid, mime_in_reply_to, mime_references): (
-            String,
-            String,
-            String,
-            String,
-        ) = self
-            .parent_query(
-                context,
-                "param, rfc724_mid, mime_in_reply_to, mime_references",
-                collect,
-            )
-            .await
-            .ok()
-            .flatten()?;
-
-        let param = packed.parse::<Params>().ok()?;
-        if param.exists(Param::Error) {
-            // Do not reply to error messages.
-            //
-            // An error message could be a group chat message that we failed to decrypt and
-            // assigned to 1:1 chat. A reply to it will show up as a reply to group message
-            // on the other side. To avoid such situations, it is better not to reply to
-            // error messages at all.
-            None
-        } else {
-            Some((rfc724_mid, mime_in_reply_to, mime_references))
-        }
+        let collect = |row: &rusqlite::Row| Ok((row.get(0)?, row.get(1)?, row.get(2)?));
+        self.parent_query(
+            context,
+            "rfc724_mid, mime_in_reply_to, mime_references",
+            collect,
+        )
+        .await
+        .ok()
+        .flatten()
     }
 
     async fn parent_is_encrypted(self, context: &Context) -> Result<bool, Error> {
@@ -459,7 +448,7 @@ impl ChatId {
 
         if let Some(ref packed) = packed {
             let param = packed.parse::<Params>()?;
-            Ok(!param.exists(Param::Error) && param.exists(Param::GuaranteeE2ee))
+            Ok(param.exists(Param::GuaranteeE2ee))
         } else {
             // No messages
             Ok(false)
@@ -1352,12 +1341,6 @@ async fn prepare_msg_blob(context: &Context, msg: &mut Message) -> Result<(), Er
             .ok_or_else(|| {
                 format_err!("Attachment missing for message of type #{}", msg.viewtype)
             })?;
-
-        if msg.viewtype == Viewtype::Image {
-            if let Err(e) = blob.recode_to_image_size(context).await {
-                warn!(context, "Cannot recode image, using original data: {:?}", e);
-            }
-        }
         msg.param.set(Param::File, blob.as_name());
 
         if msg.viewtype == Viewtype::File || msg.viewtype == Viewtype::Image {
@@ -1947,18 +1930,19 @@ pub async fn create_group_chat(
         .sql
         .get_rowid(context, "chats", "grpid", grpid)
         .await?;
-
     let chat_id = ChatId::new(row_id);
-    if add_to_chat_contacts_table(context, chat_id, DC_CONTACT_ID_SELF).await {
-        let mut draft_msg = Message::new(Viewtype::Text);
-        draft_msg.set_text(Some(draft_txt));
-        chat_id.set_draft_raw(context, &mut draft_msg).await;
-    }
+    if !chat_id.is_error() {
+        if add_to_chat_contacts_table(context, chat_id, DC_CONTACT_ID_SELF).await {
+            let mut draft_msg = Message::new(Viewtype::Text);
+            draft_msg.set_text(Some(draft_txt));
+            chat_id.set_draft_raw(context, &mut draft_msg).await;
+        }
 
-    context.emit_event(Event::MsgsChanged {
-        msg_id: MsgId::new(0),
-        chat_id: ChatId::new(0),
-    });
+        context.emit_event(Event::MsgsChanged {
+            msg_id: MsgId::new(0),
+            chat_id: ChatId::new(0),
+        });
+    }
 
     Ok(chat_id)
 }

src/constants.rs

@@ -227,10 +227,6 @@ pub const DC_BOB_SUCCESS: i32 = 1;
 // max. width/height of an avatar
 pub const AVATAR_SIZE: u32 = 192;
 
-// max. width/height of images
-pub const BALANCED_IMAGE_SIZE: u32 = 1280;
-pub const WORSE_IMAGE_SIZE: u32 = 640;
-
 // this value can be increased if the folder configuration is changed and must be redone on next program start
 pub const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3;
 

src/dc_receive_imf.rs

@@ -10,7 +10,7 @@ use crate::constants::*;
 use crate::contact::*;
 use crate::context::Context;
 use crate::dc_tools::*;
-use crate::error::{bail, ensure, format_err, Result};
+use crate::error::{bail, ensure, Result};
 use crate::events::Event;
 use crate::headerdef::HeaderDef;
 use crate::job::{self, Action};
@@ -313,6 +313,8 @@ async fn add_parts(
 ) -> Result<()> {
     let mut state: MessageState;
     let mut chat_id_blocked = Blocked::Not;
+    let mut sort_timestamp = 0;
+    let mut rcvd_timestamp = 0;
     let mut mime_in_reply_to = String::new();
     let mut mime_references = String::new();
     let mut incoming_origin = incoming_origin;
@@ -599,9 +601,17 @@ async fn add_parts(
     }
     // correct message_timestamp, it should not be used before,
     // however, we cannot do this earlier as we need from_id to be set
-    let rcvd_timestamp = time();
-    let sort_timestamp = calc_sort_timestamp(context, *sent_timestamp, *chat_id, !seen).await;
-    *sent_timestamp = std::cmp::min(*sent_timestamp, rcvd_timestamp);
+    calc_timestamps(
+        context,
+        *chat_id,
+        from_id,
+        *sent_timestamp,
+        !seen,
+        &mut sort_timestamp,
+        sent_timestamp,
+        &mut rcvd_timestamp,
+    )
+    .await;
 
     // unarchive chat
     chat_id.unarchive(context).await?;
@@ -739,31 +749,6 @@ async fn add_parts(
         }
     }
 
-    async fn update_last_subject(
-        context: &Context,
-        chat_id: ChatId,
-        mime_parser: &MimeMessage,
-    ) -> Result<()> {
-        let mut chat = Chat::load_from_db(context, chat_id).await?;
-        chat.param.set(
-            Param::LastSubject,
-            mime_parser
-                .get_subject()
-                .ok_or_else(|| format_err!("No subject in email"))?,
-        );
-        chat.update_param(context).await?;
-        Ok(())
-    }
-    update_last_subject(context, chat_id, mime_parser)
-        .await
-        .unwrap_or_else(|e| {
-            warn!(
-                context,
-                "Could not update LastSubject of chat: {}",
-                e.to_string()
-            )
-        });
-
     Ok(())
 }
 
@@ -827,38 +812,41 @@ async fn save_locations(
     }
 }
 
-async fn calc_sort_timestamp(
+#[allow(clippy::too_many_arguments)]
+async fn calc_timestamps(
     context: &Context,
-    message_timestamp: i64,
     chat_id: ChatId,
+    from_id: u32,
+    message_timestamp: i64,
     is_fresh_msg: bool,
-) -> i64 {
-    let mut sort_timestamp = message_timestamp;
-
-    // get newest non fresh message for this chat
-    // update sort_timestamp if less than that
+    sort_timestamp: &mut i64,
+    sent_timestamp: &mut i64,
+    rcvd_timestamp: &mut i64,
+) {
+    *rcvd_timestamp = time();
+    *sent_timestamp = message_timestamp;
+    if *sent_timestamp > *rcvd_timestamp {
+        *sent_timestamp = *rcvd_timestamp
+    }
+    *sort_timestamp = message_timestamp;
     if is_fresh_msg {
         let last_msg_time: Option<i64> = context
             .sql
             .query_get_value(
                 context,
-                "SELECT MAX(timestamp) FROM msgs WHERE chat_id=? AND state>?",
-                paramsv![chat_id, MessageState::InFresh],
+                "SELECT MAX(timestamp) FROM msgs WHERE chat_id=? and from_id!=? AND timestamp>=?",
+                paramsv![chat_id, from_id as i32, *sort_timestamp],
             )
             .await;
-
         if let Some(last_msg_time) = last_msg_time {
-            if last_msg_time > sort_timestamp {
-                sort_timestamp = last_msg_time;
+            if last_msg_time > 0 && *sort_timestamp <= last_msg_time {
+                *sort_timestamp = last_msg_time + 1;
             }
         }
     }
-
-    if sort_timestamp >= dc_smeared_time(context).await {
-        sort_timestamp = dc_create_smeared_timestamp(context).await;
+    if *sort_timestamp >= dc_smeared_time(context).await {
+        *sort_timestamp = dc_create_smeared_timestamp(context).await;
     }
-
-    sort_timestamp
 }
 
 /// This function tries extracts the group-id from the message and returns the
@@ -901,29 +889,32 @@ async fn create_or_lookup_group(
     }
 
     if grpid.is_empty() {
-        if let Some(extracted_grpid) = mime_parser
-            .get(HeaderDef::MessageId)
-            .and_then(|value| dc_extract_grpid_from_rfc724_mid(&value))
-        {
-            grpid = extracted_grpid.to_string();
-        } else if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::InReplyTo) {
-            grpid = extracted_grpid.to_string();
-        } else if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::References) {
-            grpid = extracted_grpid.to_string();
-        } else {
-            return create_or_lookup_adhoc_group(
-                context,
-                mime_parser,
-                allow_creation,
-                create_blocked,
-                from_id,
-                to_ids,
-            )
-            .await
-            .map_err(|err| {
-                info!(context, "could not create adhoc-group: {:?}", err);
-                err
-            });
+        if let Some(value) = mime_parser.get(HeaderDef::MessageId) {
+            if let Some(extracted_grpid) = dc_extract_grpid_from_rfc724_mid(&value) {
+                grpid = extracted_grpid.to_string();
+            }
+        }
+        if grpid.is_empty() {
+            if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::InReplyTo) {
+                grpid = extracted_grpid.to_string();
+            } else if let Some(extracted_grpid) = extract_grpid(mime_parser, HeaderDef::References)
+            {
+                grpid = extracted_grpid.to_string();
+            } else {
+                return create_or_lookup_adhoc_group(
+                    context,
+                    mime_parser,
+                    allow_creation,
+                    create_blocked,
+                    from_id,
+                    to_ids,
+                )
+                .await
+                .map_err(|err| {
+                    info!(context, "could not create adhoc-group: {:?}", err);
+                    err
+                });
+            }
         }
     }
     // now we have a grpid that is non-empty
@@ -1010,7 +1001,7 @@ async fn create_or_lookup_group(
     let (mut chat_id, chat_id_verified, _blocked) = chat::get_chat_id_by_grpid(context, &grpid)
         .await
         .unwrap_or((ChatId::new(0), false, Blocked::Not));
-    if !chat_id.is_unset() {
+    if !chat_id.is_error() {
         if chat_id_verified {
             if let Err(err) =
                 check_verified_properties(context, mime_parser, from_id as u32, to_ids).await
@@ -1041,7 +1032,7 @@ async fn create_or_lookup_group(
         .await
         .unwrap_or_default();
 
-    if chat_id.is_unset()
+    if chat_id.is_error()
             && !mime_parser.is_mailinglist_message()
             && !grpid.is_empty()
             && grpname.is_some()
@@ -1214,6 +1205,10 @@ async fn create_or_lookup_adhoc_group(
     from_id: u32,
     to_ids: &ContactIds,
 ) -> Result<(ChatId, Blocked)> {
+    // if we're here, no grpid was found, check if there is an existing
+    // ad-hoc group matching the to-list or if we should and can create one
+    // (we do not want to heuristically look at the likely mangled Subject)
+
     if mime_parser.is_mailinglist_message() {
         // XXX we could parse List-* headers and actually create and
         // manage a mailing list group, eventually
@@ -1224,10 +1219,6 @@ async fn create_or_lookup_adhoc_group(
         return Ok((ChatId::new(0), Blocked::Not));
     }
 
-    // if we're here, no grpid was found, check if there is an existing
-    // ad-hoc group matching the to-list or if we should and can create one
-    // (we do not want to heuristically look at the likely mangled Subject)
-
     let mut member_ids: Vec<u32> = to_ids.iter().copied().collect();
     if !member_ids.contains(&from_id) {
         member_ids.push(from_id);
@@ -1280,24 +1271,6 @@ async fn create_or_lookup_adhoc_group(
         return Ok((ChatId::new(0), Blocked::Not));
     }
 
-    if mime_parser.decrypting_failed {
-        // Do not create a new ad-hoc group if the message cannot be
-        // decrypted.
-        //
-        // The subject may be encrypted and contain a placeholder such
-        // as "...". Besides that, it is possible that the message was
-        // sent to a valid, yet unknown group, which was rejected
-        // because Chat-Group-Name, which is in the encrypted part,
-        // was not found. Generating a new ID in this case would
-        // result in creation of a twin group with a different group
-        // ID.
-        warn!(
-            context,
-            "not creating ad-hoc group for message that cannot be decrypted"
-        );
-        return Ok((ChatId::new(0), Blocked::Not));
-    }
-
     // we do not check if the message is a reply to another group, this may result in
     // chats with unclear member list. instead we create a new group in the following lines ...
 
@@ -1767,7 +1740,7 @@ mod tests {
     use crate::chat::ChatVisibility;
     use crate::chatlist::Chatlist;
     use crate::message::Message;
-    use crate::test_utils::{configured_offline_context, dummy_context};
+    use crate::test_utils::{dummy_context, TestContext};
 
     #[test]
     fn test_hex_hash() {
@@ -1863,6 +1836,23 @@ mod tests {
         assert!(!is_msgrmsg_rfc724_mid(&t.ctx, "nonexistant@message.id").await);
     }
 
+    async fn configured_offline_context() -> TestContext {
+        let t = dummy_context().await;
+        t.ctx
+            .set_config(Config::Addr, Some("alice@example.org"))
+            .await
+            .unwrap();
+        t.ctx
+            .set_config(Config::ConfiguredAddr, Some("alice@example.org"))
+            .await
+            .unwrap();
+        t.ctx
+            .set_config(Config::Configured, Some("1"))
+            .await
+            .unwrap();
+        t
+    }
+
     static MSGRMSG: &[u8] = b"From: Bob <bob@example.org>\n\
                     To: alice@example.org\n\
                     Chat-Version: 1.0\n\

src/headerdef.rs

@@ -34,7 +34,6 @@ pub enum HeaderDef {
     ChatContent,
     ChatDuration,
     ChatDispositionNotificationTo,
-    ChatUploadUrl,
     Autocrypt,
     AutocryptSetupMessage,
     SecureJoin,

src/imap/client.rs

@@ -7,7 +7,7 @@ use async_imap::{
 use async_std::net::{self, TcpStream};
 
 use super::session::Session;
-use crate::login_param::dc_build_tls;
+use crate::login_param::{dc_build_tls, CertificateChecks};
 
 use super::session::SessionStream;
 
@@ -78,10 +78,10 @@ impl Client {
     pub async fn connect_secure<A: net::ToSocketAddrs, S: AsRef<str>>(
         addr: A,
         domain: S,
-        strict_tls: bool,
+        certificate_checks: CertificateChecks,
     ) -> ImapResult<Self> {
         let stream = TcpStream::connect(addr).await?;
-        let tls = dc_build_tls(strict_tls);
+        let tls = dc_build_tls(certificate_checks);
         let tls_stream: Box<dyn SessionStream> =
             Box::new(tls.connect(domain.as_ref(), stream).await?);
         let mut client = ImapClient::new(tls_stream);
@@ -118,12 +118,16 @@ impl Client {
         })
     }
 
-    pub async fn secure<S: AsRef<str>>(self, domain: S, strict_tls: bool) -> ImapResult<Client> {
+    pub async fn secure<S: AsRef<str>>(
+        self,
+        domain: S,
+        certificate_checks: CertificateChecks,
+    ) -> ImapResult<Client> {
         if self.is_secure {
             Ok(self)
         } else {
             let Client { mut inner, .. } = self;
-            let tls = dc_build_tls(strict_tls);
+            let tls = dc_build_tls(certificate_checks);
             inner.run_command_and_check_ok("STARTTLS", None).await?;
 
             let stream = inner.into_inner();

src/imap/mod.rs

@@ -27,7 +27,6 @@ use crate::message::{self, update_server_uid};
 use crate::mimeparser;
 use crate::oauth2::dc_get_oauth2_access_token;
 use crate::param::Params;
-use crate::provider::get_provider_info;
 use crate::{scheduler::InterruptInfo, stock::StockMessage};
 
 mod client;
@@ -150,7 +149,7 @@ struct ImapConfig {
     pub imap_port: u16,
     pub imap_user: String,
     pub imap_pw: String,
-    pub strict_tls: bool,
+    pub certificate_checks: CertificateChecks,
     pub server_flags: usize,
     pub selected_folder: Option<String>,
     pub selected_mailbox: Option<Mailbox>,
@@ -170,7 +169,7 @@ impl Default for ImapConfig {
             imap_port: 0,
             imap_user: "".into(),
             imap_pw: "".into(),
-            strict_tls: false,
+            certificate_checks: Default::default(),
             server_flags: 0,
             selected_folder: None,
             selected_mailbox: None,
@@ -229,7 +228,7 @@ impl Imap {
                 match Client::connect_insecure((imap_server, imap_port)).await {
                     Ok(client) => {
                         if (server_flags & DC_LP_IMAP_SOCKET_STARTTLS) != 0 {
-                            client.secure(imap_server, config.strict_tls).await
+                            client.secure(imap_server, config.certificate_checks).await
                         } else {
                             Ok(client)
                         }
@@ -241,8 +240,12 @@ impl Imap {
                 let imap_server: &str = config.imap_server.as_ref();
                 let imap_port = config.imap_port;
 
-                Client::connect_secure((imap_server, imap_port), imap_server, config.strict_tls)
-                    .await
+                Client::connect_secure(
+                    (imap_server, imap_port),
+                    imap_server,
+                    config.certificate_checks,
+                )
+                .await
             };
 
         let login_res = match connection_res {
@@ -314,15 +317,9 @@ impl Imap {
     }
 
     async fn unsetup_handle(&mut self, context: &Context) {
-        // Close folder if messages should be expunged
-        if let Err(err) = self.close_folder(context).await {
-            warn!(context, "failed to close folder: {:?}", err);
-        }
-
-        // Logout from the server
         if let Some(mut session) = self.session.take() {
-            if let Err(err) = session.logout().await {
-                warn!(context, "failed to logout: {:?}", err);
+            if let Err(err) = session.close().await {
+                warn!(context, "failed to close connection: {:?}", err);
             }
         }
         self.connected = false;
@@ -382,15 +379,7 @@ impl Imap {
             config.imap_port = imap_port;
             config.imap_user = imap_user.to_string();
             config.imap_pw = imap_pw.to_string();
-            let provider = get_provider_info(&lp.addr);
-            config.strict_tls = match lp.imap_certificate_checks {
-                CertificateChecks::Automatic => {
-                    provider.map_or(false, |provider| provider.strict_tls)
-                }
-                CertificateChecks::Strict => true,
-                CertificateChecks::AcceptInvalidCertificates
-                | CertificateChecks::AcceptInvalidCertificates2 => false,
-            };
+            config.certificate_checks = lp.imap_certificate_checks;
             config.server_flags = server_flags;
         }
 

src/imap/select_folder.rs

@@ -27,7 +27,7 @@ impl Imap {
     ///
     /// CLOSE is considerably faster than an EXPUNGE, see
     /// https://tools.ietf.org/html/rfc3501#section-6.4.2
-    pub(super) async fn close_folder(&mut self, context: &Context) -> Result<()> {
+    async fn close_folder(&mut self, context: &Context) -> Result<()> {
         if let Some(ref folder) = self.config.selected_folder {
             info!(context, "Expunge messages in \"{}\".", folder);
 

src/job.rs

@@ -3,7 +3,6 @@
 //! This module implements a job queue maintained in the SQLite database
 //! and job types.
 
-use std::env;
 use std::fmt;
 use std::future::Future;
 
@@ -32,7 +31,6 @@ use crate::message::{self, Message, MessageState};
 use crate::mimefactory::MimeFactory;
 use crate::param::*;
 use crate::smtp::Smtp;
-use crate::upload::{download_message_file, generate_upload_url, upload_file};
 use crate::{scheduler::InterruptInfo, sql};
 
 // results in ~3 weeks for the last backoff timespan
@@ -109,8 +107,6 @@ pub enum Action {
     MaybeSendLocationsEnded = 5007,
     SendMdn = 5010,
     SendMsgToSmtp = 5901, // ... high priority
-
-    DownloadMessageFile = 7000,
 }
 
 impl Default for Action {
@@ -137,9 +133,6 @@ impl From<Action> for Thread {
             MaybeSendLocationsEnded => Thread::Smtp,
             SendMdn => Thread::Smtp,
             SendMsgToSmtp => Thread::Smtp,
-
-            // TODO: Where does downloading fit in the thread architecture?
-            DownloadMessageFile => Thread::Imap,
         }
     }
 }
@@ -197,7 +190,7 @@ impl Job {
     /// Saves the job to the database, creating a new entry if necessary.
     ///
     /// The Job is consumed by this method.
-    pub(crate) async fn save(self, context: &Context) -> Result<()> {
+    pub async fn save(self, context: &Context) -> Result<()> {
         let thread: Thread = self.action.into();
 
         info!(context, "saving job for {}-thread: {:?}", thread, self);
@@ -336,15 +329,7 @@ impl Job {
         }
     }
 
-    pub(crate) async fn send_msg_to_smtp(&mut self, context: &Context, smtp: &mut Smtp) -> Status {
-        // Upload file to HTTP if set in params.
-        if let (Some(upload_url), Ok(Some(upload_path))) = (
-            self.param.get_upload_url(),
-            self.param.get_upload_path(context),
-        ) {
-            job_try!(upload_file(context, upload_url.to_string(), upload_path).await);
-        }
-
+    pub async fn send_msg_to_smtp(&mut self, context: &Context, smtp: &mut Smtp) -> Status {
         //  SMTP server, if not yet done
         if !smtp.is_connected().await {
             let loginparam = LoginParam::from_database(context, "configured_").await;
@@ -513,12 +498,12 @@ impl Job {
     }
 
     async fn move_msg(&mut self, context: &Context, imap: &mut Imap) -> Status {
-        if let Err(err) = imap.connect_configured(context).await {
-            warn!(context, "could not connect: {:?}", err);
+        let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
+
+        if let Err(err) = imap.ensure_configured_folders(context, true).await {
+            warn!(context, "could not configure folders: {:?}", err);
             return Status::RetryLater;
         }
-
-        let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
         let dest_folder = context.get_config(Config::ConfiguredMvboxFolder).await;
 
         if let Some(dest_folder) = dest_folder {
@@ -553,11 +538,6 @@ impl Job {
     /// records pointing to the same message on the server, the job
     /// also removes the message on the server.
     async fn delete_msg_on_imap(&mut self, context: &Context, imap: &mut Imap) -> Status {
-        if let Err(err) = imap.connect_configured(context).await {
-            warn!(context, "could not connect: {:?}", err);
-            return Status::RetryLater;
-        }
-
         let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
 
         if !msg.rfc724_mid.is_empty() {
@@ -628,11 +608,6 @@ impl Job {
     }
 
     async fn empty_server(&mut self, context: &Context, imap: &mut Imap) -> Status {
-        if let Err(err) = imap.connect_configured(context).await {
-            warn!(context, "could not connect: {:?}", err);
-            return Status::RetryLater;
-        }
-
         if self.foreign_id & DC_EMPTY_MVBOX > 0 {
             if let Some(mvbox_folder) = &context.get_config(Config::ConfiguredMvboxFolder).await {
                 imap.empty_folder(context, &mvbox_folder).await;
@@ -645,11 +620,6 @@ impl Job {
     }
 
     async fn markseen_msg_on_imap(&mut self, context: &Context, imap: &mut Imap) -> Status {
-        if let Err(err) = imap.connect_configured(context).await {
-            warn!(context, "could not connect: {:?}", err);
-            return Status::RetryLater;
-        }
-
         let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
 
         let folder = msg.server_folder.as_ref().unwrap();
@@ -673,13 +643,6 @@ impl Job {
             }
         }
     }
-
-    pub(crate) async fn download_message_file(&mut self, context: &Context) -> Status {
-        let msg_id = MsgId::new(self.foreign_id);
-        let download_path = job_try!(self.param.get_upload_path(context));
-        job_try!(download_message_file(context, msg_id, download_path).await);
-        Status::Finished(Ok(()))
-    }
 }
 
 /// Delete all pending jobs with the given action.
@@ -748,23 +711,7 @@ pub async fn send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<Job
         }
     };
 
-    let mut mimefactory = MimeFactory::from_msg(context, &msg, attach_selfavatar).await?;
-
-    // Prepare file upload if DCC_UPLOAD_URL env variable is set.
-    // See upload-server folder for an example server impl.
-    // Here a new URL is generated, which the mimefactory includes in the message instead of the
-    // actual attachement. The upload then happens in the smtp send job.
-    let upload = if let Some(file) = msg.get_file(context) {
-        if let Ok(endpoint) = env::var("DCC_UPLOAD_URL") {
-            let upload_url = generate_upload_url(context, endpoint);
-            mimefactory.set_upload_url(upload_url.clone());
-            Some((upload_url, file))
-        } else {
-            None
-        }
-    } else {
-        None
-    };
+    let mimefactory = MimeFactory::from_msg(context, &msg, attach_selfavatar).await?;
 
     let mut recipients = mimefactory.recipients();
 
@@ -856,17 +803,13 @@ pub async fn send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<Job
     param.set(Param::File, blob.as_name());
     param.set(Param::Recipients, &recipients);
 
-    if let Some((upload_url, upload_path)) = upload {
-        param.set_upload_url(upload_url);
-        param.set_upload_path(upload_path);
-    }
-
     let job = create(Action::SendMsgToSmtp, msg_id.to_u32() as i32, param, 0)?;
 
     Ok(Some(job))
 }
 
-pub(crate) enum Connection<'a> {
+#[derive(Debug)]
+pub enum Connection<'a> {
     Inbox(&'a mut Imap),
     Smtp(&'a mut Smtp),
 }
@@ -1021,7 +964,6 @@ async fn perform_job_action(
             sql::housekeeping(context).await;
             Status::Finished(Ok(()))
         }
-        Action::DownloadMessageFile => job.download_message_file(context).await,
     };
 
     info!(
@@ -1078,8 +1020,7 @@ pub async fn add(context: &Context, job: Job) {
             | Action::OldDeleteMsgOnImap
             | Action::DeleteMsgOnImap
             | Action::MarkseenMsgOnImap
-            | Action::MoveMsg
-            | Action::DownloadMessageFile => {
+            | Action::MoveMsg => {
                 info!(context, "interrupt: imap");
                 context
                     .interrupt_inbox(InterruptInfo::new(false, None))

src/lib.rs

@@ -11,6 +11,8 @@ extern crate rusqlite;
 extern crate strum;
 #[macro_use]
 extern crate strum_macros;
+#[macro_use]
+extern crate debug_stub_derive;
 
 pub trait ToSql: rusqlite::ToSql + Send + Sync {}
 
@@ -67,7 +69,6 @@ mod simplify;
 mod smtp;
 pub mod stock;
 mod token;
-pub(crate) mod upload;
 #[macro_use]
 mod dehtml;
 

src/login_param.rs

@@ -130,6 +130,10 @@ impl LoginParam {
         }
     }
 
+    pub fn addr_str(&self) -> &str {
+        self.addr.as_str()
+    }
+
     /// Save this loginparam to the database.
     pub async fn save_to_database(
         &self,
@@ -273,15 +277,21 @@ fn get_readable_flags(flags: i32) -> String {
     res
 }
 
-pub fn dc_build_tls(strict_tls: bool) -> async_native_tls::TlsConnector {
+pub fn dc_build_tls(certificate_checks: CertificateChecks) -> async_native_tls::TlsConnector {
     let tls_builder = async_native_tls::TlsConnector::new();
-
-    if strict_tls {
-        tls_builder
-    } else {
-        tls_builder
+    match certificate_checks {
+        CertificateChecks::Automatic => {
+            // Same as AcceptInvalidCertificates for now.
+            // TODO: use provider database when it becomes available
+            tls_builder
+                .danger_accept_invalid_hostnames(true)
+                .danger_accept_invalid_certs(true)
+        }
+        CertificateChecks::Strict => tls_builder,
+        CertificateChecks::AcceptInvalidCertificates
+        | CertificateChecks::AcceptInvalidCertificates2 => tls_builder
             .danger_accept_invalid_hostnames(true)
-            .danger_accept_invalid_certs(true)
+            .danger_accept_invalid_certs(true),
     }
 }
 

src/message.rs

@@ -1545,33 +1545,6 @@ pub async fn update_server_uid(
     }
 }
 
-/// Schedule attachement download for a message.
-pub async fn schedule_download(
-    context: &Context,
-    msg_id: MsgId,
-    path: Option<PathBuf>,
-) -> Result<(), Error> {
-    let msg = Message::load_from_db(context, msg_id).await?;
-    if let Some(_upload_url) = msg.param.get_upload_url() {
-        // TODO: Check if message was already downloaded.
-        let mut params = Params::new();
-        if let Some(path) = path {
-            params.set_upload_path(path);
-        }
-        job::add(
-            context,
-            job::Job::new(Action::DownloadMessageFile, msg_id.to_u32(), params, 0),
-        )
-        .await;
-    } else {
-        warn!(
-            context,
-            "Tried to schedule download for message {} which has no uploads", msg_id
-        );
-    }
-    Ok(())
-}
-
 #[allow(dead_code)]
 pub async fn dc_empty_server(context: &Context, flags: u32) {
     job::kill_action(context, Action::EmptyServer).await;

src/mimefactory.rs

@@ -50,7 +50,6 @@ pub struct MimeFactory<'a, 'b> {
     context: &'a Context,
     last_added_location_id: u32,
     attach_selfavatar: bool,
-    upload_url: Option<String>,
 }
 
 /// Result of rendering a message, ready to be submitted to a send job.
@@ -160,7 +159,6 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             last_added_location_id: 0,
             attach_selfavatar,
             context,
-            upload_url: None,
         };
         Ok(factory)
     }
@@ -208,7 +206,6 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             req_mdn: false,
             last_added_location_id: 0,
             attach_selfavatar: false,
-            upload_url: None,
         };
 
         Ok(res)
@@ -354,47 +351,16 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                     };
                     format!("{}{}", re, chat.name)
                 } else {
-                    match chat.param.get(Param::LastSubject) {
-                        Some(last_subject) => {
-                            let subject_start = if last_subject.starts_with("Chat:") {
-                                0
-                            } else {
-                                // "Antw:" is the longest abbreviation in
-                                // https://en.wikipedia.org/wiki/List_of_email_subject_abbreviations#Abbreviations_in_other_languages,
-                                // so look at the first _5_ characters:
-                                match last_subject.chars().take(5).position(|c| c == ':') {
-                                    Some(prefix_end) => prefix_end + 1,
-                                    None => 0,
-                                }
-                            };
-                            format!(
-                                "Re: {}",
-                                last_subject
-                                    .chars()
-                                    .skip(subject_start)
-                                    .collect::<String>()
-                                    .trim()
-                            )
-                        }
-                        None => {
-                            let self_name = match self.context.get_config(Config::Displayname).await
-                            {
-                                Some(name) => name,
-                                None => self
-                                    .context
-                                    .get_config(Config::Addr)
-                                    .await
-                                    .unwrap_or_default(),
-                            };
-
-                            self.context
-                                .stock_string_repl_str(
-                                    StockMessage::SubjectForNewContact,
-                                    self_name,
-                                )
-                                .await
-                        }
-                    }
+                    let raw = message::get_summarytext_by_raw(
+                        self.msg.viewtype,
+                        self.msg.text.as_ref(),
+                        &self.msg.param,
+                        32,
+                        self.context,
+                    )
+                    .await;
+                    let raw_subject = raw.lines().next().unwrap_or_default();
+                    format!("Chat: {}", raw_subject)
                 }
             }
             Loaded::MDN { .. } => self
@@ -412,10 +378,6 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             .collect()
     }
 
-    pub fn set_upload_url(&mut self, upload_url: String) {
-        self.upload_url = Some(upload_url)
-    }
-
     pub async fn render(mut self) -> Result<RenderedEmail, Error> {
         // Headers that are encrypted
         // - Chat-*, except Chat-Version
@@ -447,8 +409,6 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             to.push(from.clone());
         }
 
-        unprotected_headers.push(Header::new("MIME-Version".into(), "1.0".into()));
-
         if !self.references.is_empty() {
             unprotected_headers.push(Header::new("References".into(), self.references.clone()));
         }
@@ -886,21 +846,11 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             }
         };
 
-        // if upload url is present: add as header and to message text
-        // TODO: make text part translatable (or remove)
-        let upload_url_text = if let Some(ref upload_url) = self.upload_url {
-            protected_headers.push(Header::new("Chat-Upload-Url".into(), upload_url.clone()));
-            Some(format!("\n\nFile attachement: {}", upload_url.clone()))
-        } else {
-            None
-        };
-
         let footer = &self.selfstatus;
         let message_text = format!(
-            "{}{}{}{}{}{}",
+            "{}{}{}{}{}",
             fwdhint.unwrap_or_default(),
             escape_message_footer_marks(final_text),
-            upload_url_text.unwrap_or_default(),
             if !final_text.is_empty() && !footer.is_empty() {
                 "\r\n\r\n"
             } else {
@@ -916,8 +866,8 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             .body(message_text);
         let mut parts = Vec::new();
 
-        // add attachment part, skip if upload url was provided
-        if chat::msgtype_has_file(self.msg.viewtype) && self.upload_url.is_none() {
+        // add attachment part
+        if chat::msgtype_has_file(self.msg.viewtype) {
             if !is_file_size_okay(context, &self.msg).await {
                 bail!(
                     "Message exceeds the recommended {} MB.",
@@ -1224,11 +1174,6 @@ pub fn needs_encoding(to_check: impl AsRef<str>) -> bool {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::chatlist::Chatlist;
-    use crate::dc_receive_imf::dc_receive_imf;
-    use crate::mimeparser::*;
-    use crate::test_utils::configured_offline_context;
-    use crate::test_utils::TestContext;
 
     #[test]
     fn test_render_email_address() {
@@ -1236,9 +1181,6 @@ mod tests {
         let addr = "x@y.org";
 
         assert!(!display_name.is_ascii());
-        assert!(!display_name
-            .chars()
-            .all(|c| c.is_ascii_alphanumeric() || c == ' '));
 
         let s = format!(
             "{}",
@@ -1250,25 +1192,6 @@ mod tests {
         assert_eq!(s, "=?utf-8?q?=C3=A4_space?= <x@y.org>");
     }
 
-    #[test]
-    fn test_render_email_address_noescape() {
-        let display_name = "a space";
-        let addr = "x@y.org";
-
-        assert!(display_name.is_ascii());
-        assert!(display_name
-            .chars()
-            .all(|c| c.is_ascii_alphanumeric() || c == ' '));
-
-        let s = format!(
-            "{}",
-            Address::new_mailbox_with_name(display_name.to_string(), addr.to_string())
-        );
-
-        // Addresses should not be unnecessarily be encoded, see https://github.com/deltachat/deltachat-core-rust/issues/1575:
-        assert_eq!(s, "a space <x@y.org>");
-    }
-
     #[test]
     fn test_render_rfc724_mid() {
         assert_eq!(
@@ -1311,192 +1234,4 @@ mod tests {
         assert!(needs_encoding(" "));
         assert!(needs_encoding("foo bar"));
     }
-
-    #[async_std::test]
-    async fn test_subject() {
-        // 1.: Receive a mail from an MUA or Delta Chat
-        assert_eq!(
-            msg_to_subject_str(
-                b"From: Bob <bob@example.org>\n\
-                To: alice@example.org\n\
-                Subject: Antw: Chat: hello\n\
-                Message-ID: <2222@example.org>\n\
-                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
-                \n\
-                hello\n"
-            )
-            .await,
-            "Re: Chat: hello"
-        );
-
-        assert_eq!(
-            msg_to_subject_str(
-                b"From: Bob <bob@example.org>\n\
-                To: alice@example.org\n\
-                Subject: Infos: 42\n\
-                Message-ID: <2222@example.org>\n\
-                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
-                \n\
-                hello\n"
-            )
-            .await,
-            "Re: Infos: 42"
-        );
-
-        // 2. Receive a message from Delta Chat when we did not send any messages before
-        assert_eq!(
-            msg_to_subject_str(
-                b"From: Charlie <charlie@example.org>\n\
-                To: alice@example.org\n\
-                Subject: Chat: hello\n\
-                Chat-Version: 1.0\n\
-                Message-ID: <2223@example.org>\n\
-                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
-                \n\
-                hello\n"
-            )
-            .await,
-            "Re: Chat: hello"
-        );
-
-        // 3. Send the first message to a new contact
-        let t = configured_offline_context().await;
-        assert_eq!(first_subject_str(t).await, "Message from alice@example.org");
-
-        let t = configured_offline_context().await;
-        t.ctx
-            .set_config(Config::Displayname, Some("Alice"))
-            .await
-            .unwrap();
-        assert_eq!(first_subject_str(t).await, "Message from Alice");
-
-        // 4. Receive messages with unicode characters and make sure that we do not panic (we do not care about the result)
-        msg_to_subject_str(
-            "From: Charlie <charlie@example.org>\n\
-            To: alice@example.org\n\
-            Subject: äääää\n\
-            Chat-Version: 1.0\n\
-            Message-ID: <2893@example.org>\n\
-            Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
-            \n\
-            hello\n"
-                .as_bytes(),
-        )
-        .await;
-
-        msg_to_subject_str(
-            "From: Charlie <charlie@example.org>\n\
-            To: alice@example.org\n\
-            Subject: aäääää\n\
-            Chat-Version: 1.0\n\
-            Message-ID: <2893@example.org>\n\
-            Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
-            \n\
-            hello\n"
-                .as_bytes(),
-        )
-        .await;
-    }
-
-    async fn first_subject_str(t: TestContext) -> String {
-        let contact_id =
-            Contact::add_or_lookup(&t.ctx, "Dave", "dave@example.org", Origin::ManuallyCreated)
-                .await
-                .unwrap()
-                .0;
-
-        let chat_id = chat::create_by_contact_id(&t.ctx, contact_id)
-            .await
-            .unwrap();
-
-        let mut new_msg = Message::new(Viewtype::Text);
-        new_msg.set_text(Some("Hi".to_string()));
-        new_msg.chat_id = chat_id;
-        chat::prepare_msg(&t.ctx, chat_id, &mut new_msg)
-            .await
-            .unwrap();
-
-        let mf = MimeFactory::from_msg(&t.ctx, &new_msg, false)
-            .await
-            .unwrap();
-
-        mf.subject_str().await
-    }
-
-    async fn msg_to_subject_str(imf_raw: &[u8]) -> String {
-        let t = configured_offline_context().await;
-        let new_msg = incoming_msg_to_reply_msg(imf_raw, &t.ctx).await;
-        let mf = MimeFactory::from_msg(&t.ctx, &new_msg, false)
-            .await
-            .unwrap();
-        mf.subject_str().await
-    }
-
-    // Creates a mimefactory for a message that replies "Hi" to the incoming message in `imf_raw`.
-    async fn incoming_msg_to_reply_msg(imf_raw: &[u8], context: &Context) -> Message {
-        context
-            .set_config(Config::ShowEmails, Some("2"))
-            .await
-            .unwrap();
-
-        dc_receive_imf(context, imf_raw, "INBOX", 1, false)
-            .await
-            .unwrap();
-
-        let chats = Chatlist::try_load(context, 0, None, None).await.unwrap();
-
-        let chat_id = chat::create_by_msg_id(context, chats.get_msg_id(0).unwrap())
-            .await
-            .unwrap();
-
-        let mut new_msg = Message::new(Viewtype::Text);
-        new_msg.set_text(Some("Hi".to_string()));
-        new_msg.chat_id = chat_id;
-        chat::prepare_msg(context, chat_id, &mut new_msg)
-            .await
-            .unwrap();
-
-        new_msg
-    }
-
-    #[async_std::test]
-    // This test could still be extended
-    async fn test_render_reply() {
-        let t = configured_offline_context().await;
-        let context = &t.ctx;
-
-        let msg = incoming_msg_to_reply_msg(
-            b"From: Charlie <charlie@example.org>\n\
-                To: alice@example.org\n\
-                Subject: Chat: hello\n\
-                Chat-Version: 1.0\n\
-                Message-ID: <2223@example.org>\n\
-                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
-                \n\
-                hello\n",
-            context,
-        )
-        .await;
-
-        let mimefactory = MimeFactory::from_msg(&t.ctx, &msg, false).await.unwrap();
-
-        let recipients = mimefactory.recipients();
-        assert_eq!(recipients, vec!["charlie@example.org"]);
-
-        let rendered_msg = mimefactory.render().await.unwrap();
-
-        let mail = mailparse::parse_mail(&rendered_msg.message).unwrap();
-        assert_eq!(
-            mail.headers
-                .iter()
-                .find(|h| h.get_key() == "MIME-Version")
-                .unwrap()
-                .get_value(),
-            "1.0"
-        );
-
-        let _mime_msg = MimeMessage::from_bytes(context, &rendered_msg.message)
-            .await
-            .unwrap();
-    }
 }

src/mimeparser.rs

@@ -334,13 +334,6 @@ impl MimeMessage {
             }
         }
 
-        let upload_url = self.get(HeaderDef::ChatUploadUrl).map(|v| v.to_string());
-        if let Some(upload_url) = upload_url {
-            for part in self.parts.iter_mut() {
-                part.param.set_upload_url(upload_url.clone());
-            }
-        }
-
         self.parse_attachments();
 
         // See if an MDN is requested from the other side
@@ -534,7 +527,6 @@ impl MimeMessage {
                 part.typ = Viewtype::Text;
                 part.msg_raw = Some(txt.clone());
                 part.msg = txt;
-                part.param.set(Param::Error, "Decryption failed");
 
                 self.parts.push(part);
 

src/param.rs

@@ -103,9 +103,6 @@ pub enum Param {
     /// For Chats
     Selftalk = b'K',
 
-    /// For Chats: So that on sending a new message we can sent the subject to "Re: <last subject>"
-    LastSubject = b't',
-
     /// For Chats
     Devicetalk = b'D',
 
@@ -120,12 +117,6 @@ pub enum Param {
 
     /// For MDN-sending job
     MsgId = b'I',
-
-    /// For messages that have a HTTP file upload instead of attachement
-    UploadUrl = b'y',
-
-    /// For messages that have a HTTP file upload instead of attachement: Path to local file
-    UploadPath = b'Y',
 }
 
 /// Possible values for `Param::ForcePlaintext`.
@@ -323,23 +314,6 @@ impl Params {
         Ok(Some(path))
     }
 
-    pub fn get_upload_url(&self) -> Option<&str> {
-        self.get(Param::UploadUrl)
-    }
-
-    pub fn get_upload_path(&self, context: &Context) -> Result<Option<PathBuf>, BlobError> {
-        self.get_path(Param::UploadPath, context)
-    }
-
-    pub fn set_upload_path(&mut self, path: PathBuf) {
-        // TODO: Remove unwrap? May panic for invalid UTF8 in path.
-        self.set(Param::UploadPath, path.to_str().unwrap());
-    }
-
-    pub fn set_upload_url(&mut self, url: impl AsRef<str>) {
-        self.set(Param::UploadUrl, url);
-    }
-
     pub fn get_msg_id(&self) -> Option<MsgId> {
         self.get(Param::MsgId)
             .and_then(|x| x.parse::<u32>().ok())

src/pgp.rs

@@ -10,7 +10,6 @@ use pgp::composed::{
     SignedPublicSubKey, SignedSecretKey, SubkeyParamsBuilder,
 };
 use pgp::crypto::{HashAlgorithm, SymmetricKeyAlgorithm};
-use pgp::ser::Serialize;
 use pgp::types::{
     CompressionAlgorithm, KeyTrait, Mpi, PublicKeyTrait, SecretKeyTrait, StringToKey,
 };
@@ -323,22 +322,8 @@ pub async fn pk_decrypt(
     Ok(content)
 }
 
-/// Symmetric encryption with armored base64 text output.
+/// Symmetric encryption.
 pub async fn symm_encrypt(passphrase: &str, plain: &[u8]) -> Result<String> {
-    let message = symm_encrypt_to_message(passphrase, plain).await?;
-    let encoded_msg = message.to_armored_string(None)?;
-    Ok(encoded_msg)
-}
-
-/// Symmetric encryption with binary output.
-pub async fn symm_encrypt_bytes(passphrase: &str, plain: &[u8]) -> Result<Vec<u8>> {
-    let message = symm_encrypt_to_message(passphrase, plain).await?;
-    let mut buf = Vec::new();
-    message.to_writer(&mut buf)?;
-    Ok(buf)
-}
-
-async fn symm_encrypt_to_message(passphrase: &str, plain: &[u8]) -> Result<Message> {
     let lit_msg = Message::new_literal_bytes("", plain);
     let passphrase = passphrase.to_string();
 
@@ -347,33 +332,24 @@ async fn symm_encrypt_to_message(passphrase: &str, plain: &[u8]) -> Result<Messa
         let s2k = StringToKey::new_default(&mut rng);
         let msg =
             lit_msg.encrypt_with_password(&mut rng, s2k, Default::default(), || passphrase)?;
-        Ok(msg)
+
+        let encoded_msg = msg.to_armored_string(None)?;
+
+        Ok(encoded_msg)
     })
     .await
 }
 
-/// Symmetric decryption from armored text.
+/// Symmetric decryption.
 pub async fn symm_decrypt<T: std::io::Read + std::io::Seek>(
     passphrase: &str,
     ctext: T,
 ) -> Result<Vec<u8>> {
     let (enc_msg, _) = Message::from_armor_single(ctext)?;
-    symm_decrypt_from_message(enc_msg, passphrase).await
-}
 
-/// Symmetric decryption from bytes.
-pub async fn symm_decrypt_bytes<T: std::io::Read + std::io::Seek>(
-    passphrase: &str,
-    cbytes: T,
-) -> Result<Vec<u8>> {
-    let enc_msg = Message::from_bytes(cbytes)?;
-    symm_decrypt_from_message(enc_msg, passphrase).await
-}
-
-async fn symm_decrypt_from_message(message: Message, passphrase: &str) -> Result<Vec<u8>> {
     let passphrase = passphrase.to_string();
     async_std::task::spawn_blocking(move || {
-        let decryptor = message.decrypt_with_password(|| passphrase)?;
+        let decryptor = enc_msg.decrypt_with_password(|| passphrase)?;
 
         let msgs = decryptor.collect::<pgp::errors::Result<Vec<_>>>()?;
         ensure!(!msgs.is_empty(), "No valid messages found");

src/provider/data.rs

@@ -19,7 +19,6 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "newyear.aktivix.org", port: 25, username_pattern: EMAIL },
         ],
         config_defaults: None,
-        strict_tls: false,
     };
 
     // aol.md: aol.com
@@ -31,7 +30,6 @@ lazy_static::lazy_static! {
         server: vec![
         ],
         config_defaults: None,
-        strict_tls: false,
     };
 
     // autistici.org.md: autistici.org
@@ -45,7 +43,6 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.autistici.org", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: None,
-        strict_tls: false,
     };
 
     // bluewin.ch.md: bluewin.ch
@@ -59,21 +56,6 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "smtpauths.bluewin.ch", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: None,
-        strict_tls: false,
-    };
-
-    // chello.at.md: chello.at
-    static ref P_CHELLO_AT: Provider = Provider {
-        status: Status::OK,
-        before_login_hint: "",
-        after_login_hint: "",
-        overview_page: "https://providers.delta.chat/chello-at",
-        server: vec![
-            Server { protocol: IMAP, socket: SSL, hostname: "mail.mymagenta.at", port: 993, username_pattern: EMAIL },
-            Server { protocol: SMTP, socket: SSL, hostname: "mail.mymagenta.at", port: 465, username_pattern: EMAIL },
-        ],
-        config_defaults: None,
-        strict_tls: false,
     };
 
     // comcast.md: xfinity.com, comcast.net
@@ -83,16 +65,7 @@ lazy_static::lazy_static! {
     // - skipping provider with status OK and no special things to do
 
     // disroot.md: disroot.org
-    static ref P_DISROOT: Provider = Provider {
-        status: Status::OK,
-        before_login_hint: "",
-        after_login_hint: "",
-        overview_page: "https://providers.delta.chat/disroot",
-        server: vec![
-        ],
-        config_defaults: None,
-        strict_tls: true,
-    };
+    // - skipping provider with status OK and no special things to do
 
     // example.com.md: example.com, example.org
     static ref P_EXAMPLE_COM: Provider = Provider {
@@ -105,7 +78,6 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.example.com", port: 1337, username_pattern: EMAIL },
         ],
         config_defaults: None,
-        strict_tls: false,
     };
 
     // fastmail.md: fastmail.com
@@ -117,19 +89,6 @@ lazy_static::lazy_static! {
         server: vec![
         ],
         config_defaults: None,
-        strict_tls: false,
-    };
-
-    // five.chat.md: five.chat
-    static ref P_FIVE_CHAT: Provider = Provider {
-        status: Status::OK,
-        before_login_hint: "",
-        after_login_hint: "",
-        overview_page: "https://providers.delta.chat/five-chat",
-        server: vec![
-        ],
-        config_defaults: None,
-        strict_tls: true,
     };
 
     // freenet.de.md: freenet.de
@@ -143,7 +102,6 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "mx.freenet.de", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
-        strict_tls: false,
     };
 
     // gmail.md: gmail.com, googlemail.com
@@ -157,7 +115,6 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.gmail.com", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: None,
-        strict_tls: true,
     };
 
     // gmx.net.md: gmx.net, gmx.de, gmx.at, gmx.ch, gmx.org, gmx.eu, gmx.info, gmx.biz, gmx.com
@@ -172,7 +129,6 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "mail.gmx.net", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
-        strict_tls: false,
     };
 
     // i.ua.md: i.ua
@@ -189,7 +145,6 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.mail.me.com", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
-        strict_tls: false,
     };
 
     // kolst.com.md: kolst.com
@@ -202,16 +157,7 @@ lazy_static::lazy_static! {
     // - skipping provider with status OK and no special things to do
 
     // mailbox.org.md: mailbox.org, secure.mailbox.org
-    static ref P_MAILBOX_ORG: Provider = Provider {
-        status: Status::OK,
-        before_login_hint: "",
-        after_login_hint: "",
-        overview_page: "https://providers.delta.chat/mailbox-org",
-        server: vec![
-        ],
-        config_defaults: None,
-        strict_tls: true,
-    };
+    // - skipping provider with status OK and no special things to do
 
     // nauta.cu.md: nauta.cu
     static ref P_NAUTA_CU: Provider = Provider {
@@ -232,7 +178,6 @@ lazy_static::lazy_static! {
             ConfigDefault { key: Config::E2eeEnabled, value: "0" },
             ConfigDefault { key: Config::MediaQuality, value: "1" },
         ]),
-        strict_tls: false,
     };
 
     // outlook.com.md: hotmail.com, outlook.com, office365.com, outlook.com.tr, live.com
@@ -246,10 +191,9 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp-mail.outlook.com", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
-        strict_tls: false,
     };
 
-    // posteo.md: posteo.de, posteo.af, posteo.at, posteo.be, posteo.ch, posteo.cl, posteo.co, posteo.co.uk, posteo.com.br, posteo.cr, posteo.cz, posteo.dk, posteo.ee, posteo.es, posteo.eu, posteo.fi, posteo.gl, posteo.gr, posteo.hn, posteo.hr, posteo.hu, posteo.ie, posteo.in, posteo.is, posteo.jp, posteo.la, posteo.li, posteo.lt, posteo.lu, posteo.me, posteo.mx, posteo.my, posteo.net, posteo.nl, posteo.no, posteo.nz, posteo.org, posteo.pe, posteo.pl, posteo.pm, posteo.pt, posteo.ro, posteo.ru, posteo.se, posteo.sg, posteo.si, posteo.tn, posteo.uk, posteo.us
+    // posteo.md: posteo.de
     static ref P_POSTEO: Provider = Provider {
         status: Status::OK,
         before_login_hint: "",
@@ -260,7 +204,6 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "posteo.de", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
-        strict_tls: true,
     };
 
     // protonmail.md: protonmail.com, protonmail.ch
@@ -272,35 +215,16 @@ lazy_static::lazy_static! {
         server: vec![
         ],
         config_defaults: None,
-        strict_tls: false,
     };
 
     // riseup.net.md: riseup.net
-    static ref P_RISEUP_NET: Provider = Provider {
-        status: Status::OK,
-        before_login_hint: "",
-        after_login_hint: "",
-        overview_page: "https://providers.delta.chat/riseup-net",
-        server: vec![
-        ],
-        config_defaults: None,
-        strict_tls: true,
-    };
+    // - skipping provider with status OK and no special things to do
 
     // rogers.com.md: rogers.com
     // - skipping provider with status OK and no special things to do
 
     // systemli.org.md: systemli.org
-    static ref P_SYSTEMLI_ORG: Provider = Provider {
-        status: Status::OK,
-        before_login_hint: "",
-        after_login_hint: "",
-        overview_page: "https://providers.delta.chat/systemli-org",
-        server: vec![
-        ],
-        config_defaults: None,
-        strict_tls: true,
-    };
+    // - skipping provider with status OK and no special things to do
 
     // t-online.md: t-online.de, magenta.de
     static ref P_T_ONLINE: Provider = Provider {
@@ -311,7 +235,6 @@ lazy_static::lazy_static! {
         server: vec![
         ],
         config_defaults: None,
-        strict_tls: false,
     };
 
     // testrun.md: testrun.org
@@ -326,7 +249,6 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "testrun.org", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
-        strict_tls: true,
     };
 
     // tiscali.it.md: tiscali.it
@@ -340,7 +262,6 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.tiscali.it", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: None,
-        strict_tls: false,
     };
 
     // ukr.net.md: ukr.net
@@ -361,13 +282,12 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.web.de", port: 587, username_pattern: EMAILLOCALPART },
         ],
         config_defaults: None,
-        strict_tls: false,
     };
 
     // yahoo.md: yahoo.com, yahoo.de, yahoo.it, yahoo.fr, yahoo.es, yahoo.se, yahoo.co.uk, yahoo.co.nz, yahoo.com.au, yahoo.com.ar, yahoo.com.br, yahoo.com.mx, ymail.com, rocketmail.com, yahoodns.net
     static ref P_YAHOO: Provider = Provider {
         status: Status::PREPARATION,
-        before_login_hint: "To use Delta Chat with your Yahoo email address you have to create an \"App-Password\" in the account security screen.",
+        before_login_hint: "To use Delta Chat with your Yahoo email address you have to allow \"less secure apps\" in the Yahoo webinterface.",
         after_login_hint: "",
         overview_page: "https://providers.delta.chat/yahoo",
         server: vec![
@@ -375,7 +295,6 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.mail.yahoo.com", port: 465, username_pattern: EMAIL },
         ],
         config_defaults: None,
-        strict_tls: false,
     };
 
     // yandex.ru.md: yandex.ru, yandex.com
@@ -387,7 +306,6 @@ lazy_static::lazy_static! {
         server: vec![
         ],
         config_defaults: None,
-        strict_tls: true,
     };
 
     // ziggo.nl.md: ziggo.nl
@@ -401,7 +319,6 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.ziggo.nl", port: 587, username_pattern: EMAIL },
         ],
         config_defaults: None,
-        strict_tls: false,
     };
 
     pub static ref PROVIDER_DATA: HashMap<&'static str, &'static Provider> = [
@@ -409,12 +326,9 @@ lazy_static::lazy_static! {
         ("aol.com", &*P_AOL),
         ("autistici.org", &*P_AUTISTICI_ORG),
         ("bluewin.ch", &*P_BLUEWIN_CH),
-        ("chello.at", &*P_CHELLO_AT),
-        ("disroot.org", &*P_DISROOT),
         ("example.com", &*P_EXAMPLE_COM),
         ("example.org", &*P_EXAMPLE_COM),
         ("fastmail.com", &*P_FASTMAIL),
-        ("five.chat", &*P_FIVE_CHAT),
         ("freenet.de", &*P_FREENET_DE),
         ("gmail.com", &*P_GMAIL),
         ("googlemail.com", &*P_GMAIL),
@@ -430,8 +344,6 @@ lazy_static::lazy_static! {
         ("icloud.com", &*P_ICLOUD),
         ("me.com", &*P_ICLOUD),
         ("mac.com", &*P_ICLOUD),
-        ("mailbox.org", &*P_MAILBOX_ORG),
-        ("secure.mailbox.org", &*P_MAILBOX_ORG),
         ("nauta.cu", &*P_NAUTA_CU),
         ("hotmail.com", &*P_OUTLOOK_COM),
         ("outlook.com", &*P_OUTLOOK_COM),
@@ -439,58 +351,8 @@ lazy_static::lazy_static! {
         ("outlook.com.tr", &*P_OUTLOOK_COM),
         ("live.com", &*P_OUTLOOK_COM),
         ("posteo.de", &*P_POSTEO),
-        ("posteo.af", &*P_POSTEO),
-        ("posteo.at", &*P_POSTEO),
-        ("posteo.be", &*P_POSTEO),
-        ("posteo.ch", &*P_POSTEO),
-        ("posteo.cl", &*P_POSTEO),
-        ("posteo.co", &*P_POSTEO),
-        ("posteo.co.uk", &*P_POSTEO),
-        ("posteo.com.br", &*P_POSTEO),
-        ("posteo.cr", &*P_POSTEO),
-        ("posteo.cz", &*P_POSTEO),
-        ("posteo.dk", &*P_POSTEO),
-        ("posteo.ee", &*P_POSTEO),
-        ("posteo.es", &*P_POSTEO),
-        ("posteo.eu", &*P_POSTEO),
-        ("posteo.fi", &*P_POSTEO),
-        ("posteo.gl", &*P_POSTEO),
-        ("posteo.gr", &*P_POSTEO),
-        ("posteo.hn", &*P_POSTEO),
-        ("posteo.hr", &*P_POSTEO),
-        ("posteo.hu", &*P_POSTEO),
-        ("posteo.ie", &*P_POSTEO),
-        ("posteo.in", &*P_POSTEO),
-        ("posteo.is", &*P_POSTEO),
-        ("posteo.jp", &*P_POSTEO),
-        ("posteo.la", &*P_POSTEO),
-        ("posteo.li", &*P_POSTEO),
-        ("posteo.lt", &*P_POSTEO),
-        ("posteo.lu", &*P_POSTEO),
-        ("posteo.me", &*P_POSTEO),
-        ("posteo.mx", &*P_POSTEO),
-        ("posteo.my", &*P_POSTEO),
-        ("posteo.net", &*P_POSTEO),
-        ("posteo.nl", &*P_POSTEO),
-        ("posteo.no", &*P_POSTEO),
-        ("posteo.nz", &*P_POSTEO),
-        ("posteo.org", &*P_POSTEO),
-        ("posteo.pe", &*P_POSTEO),
-        ("posteo.pl", &*P_POSTEO),
-        ("posteo.pm", &*P_POSTEO),
-        ("posteo.pt", &*P_POSTEO),
-        ("posteo.ro", &*P_POSTEO),
-        ("posteo.ru", &*P_POSTEO),
-        ("posteo.se", &*P_POSTEO),
-        ("posteo.sg", &*P_POSTEO),
-        ("posteo.si", &*P_POSTEO),
-        ("posteo.tn", &*P_POSTEO),
-        ("posteo.uk", &*P_POSTEO),
-        ("posteo.us", &*P_POSTEO),
         ("protonmail.com", &*P_PROTONMAIL),
         ("protonmail.ch", &*P_PROTONMAIL),
-        ("riseup.net", &*P_RISEUP_NET),
-        ("systemli.org", &*P_SYSTEMLI_ORG),
         ("t-online.de", &*P_T_ONLINE),
         ("magenta.de", &*P_T_ONLINE),
         ("testrun.org", &*P_TESTRUN),

src/provider/mod.rs

@@ -72,7 +72,6 @@ pub struct Provider {
     pub overview_page: &'static str,
     pub server: Vec<Server>,
     pub config_defaults: Option<Vec<ConfigDefault>>,
-    pub strict_tls: bool,
 }
 
 impl Provider {

src/provider/update.py

@@ -100,9 +100,6 @@ def process_data(data, file):
 
     config_defaults = process_config_defaults(data)
 
-    strict_tls = data.get("strict_tls", False)
-    strict_tls = "true" if strict_tls else "false"
-
     provider = ""
     before_login_hint = cleanstr(data.get("before_login_hint", ""))
     after_login_hint = cleanstr(data.get("after_login_hint", ""))
@@ -114,7 +111,6 @@ def process_data(data, file):
         provider += "        overview_page: \"" + file2url(file) + "\",\n"
         provider += "        server: vec![\n" + server + "        ],\n"
         provider += "        config_defaults: " + config_defaults + ",\n"
-        provider += "        strict_tls: " + strict_tls + ",\n"
         provider += "    };\n\n"
     else:
         raise TypeError("SMTP and IMAP must be specified together or left out both")
@@ -125,7 +121,7 @@ def process_data(data, file):
     # finally, add the provider
     global out_all, out_domains
     out_all += "    // " + file[file.rindex("/")+1:] + ": " + comment.strip(", ") + "\n"
-    if status == "OK" and before_login_hint == "" and after_login_hint == "" and server == "" and config_defaults == "None" and strict_tls == "false":
+    if status == "OK" and before_login_hint == "" and after_login_hint == "" and server == "" and config_defaults == "None":
         out_all += "    // - skipping provider with status OK and no special things to do\n\n"
     else:
         out_all += provider

src/scheduler.rs

@@ -61,12 +61,17 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
 
     let ctx1 = ctx.clone();
     let fut = async move {
-        started.send(()).await;
         let ctx = ctx1;
+        if let Err(err) = connection.connect_configured(&ctx).await {
+            error!(ctx, "{}", err);
+            return;
+        }
+
+        started.send(()).await;
 
         // track number of continously executed jobs
-        let mut jobs_loaded = 0;
-        let mut info = InterruptInfo::default();
+        let mut jobs_loaded: i32 = 0;
+        let mut info: InterruptInfo = Default::default();
         loop {
             match job::load_next(&ctx, Thread::Imap, &info).await {
                 Some(job) if jobs_loaded <= 20 => {
@@ -101,15 +106,10 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
 async fn fetch(ctx: &Context, connection: &mut Imap) {
     match ctx.get_config(Config::ConfiguredInboxFolder).await {
         Some(watch_folder) => {
-            if let Err(err) = connection.connect_configured(&ctx).await {
-                error!(ctx, "{}", err);
-                return;
-            }
-
             // fetch
             if let Err(err) = connection.fetch(&ctx, &watch_folder).await {
                 connection.trigger_reconnect();
-                warn!(ctx, "{}", err);
+                error!(ctx, "{}", err);
             }
         }
         None => {
@@ -122,16 +122,10 @@ async fn fetch(ctx: &Context, connection: &mut Imap) {
 async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> InterruptInfo {
     match ctx.get_config(folder).await {
         Some(watch_folder) => {
-            // connect and fake idle if unable to connect
-            if let Err(err) = connection.connect_configured(&ctx).await {
-                error!(ctx, "imap connection failed: {}", err);
-                return connection.fake_idle(&ctx, None).await;
-            }
-
             // fetch
             if let Err(err) = connection.fetch(&ctx, &watch_folder).await {
                 connection.trigger_reconnect();
-                warn!(ctx, "{}", err);
+                error!(ctx, "{}", err);
             }
 
             // idle
@@ -141,7 +135,7 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
                     .await
                     .unwrap_or_else(|err| {
                         connection.trigger_reconnect();
-                        warn!(ctx, "{}", err);
+                        error!(ctx, "{}", err);
                         InterruptInfo::new(false, None)
                     })
             } else {
@@ -149,7 +143,7 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
             }
         }
         None => {
-            warn!(ctx, "Can not watch {} folder, not set", folder);
+            warn!(ctx, "Can not watch inbox folder, not set");
             connection.fake_idle(&ctx, None).await
         }
     }
@@ -173,8 +167,13 @@ async fn simple_imap_loop(
     let ctx1 = ctx.clone();
 
     let fut = async move {
-        started.send(()).await;
         let ctx = ctx1;
+        if let Err(err) = connection.connect_configured(&ctx).await {
+            error!(ctx, "{}", err);
+            return;
+        }
+
+        started.send(()).await;
 
         loop {
             fetch_idle(&ctx, &mut connection, folder).await;
@@ -471,6 +470,7 @@ impl SmtpConnectionState {
     }
 }
 
+#[derive(Debug)]
 struct SmtpConnectionHandlers {
     connection: Smtp,
     stop_receiver: Receiver<()>,

src/smtp/mod.rs

@@ -10,9 +10,8 @@ use async_smtp::*;
 use crate::constants::*;
 use crate::context::Context;
 use crate::events::Event;
-use crate::login_param::{dc_build_tls, CertificateChecks, LoginParam};
+use crate::login_param::{dc_build_tls, LoginParam};
 use crate::oauth2::*;
-use crate::provider::get_provider_info;
 use crate::stock::StockMessage;
 
 /// SMTP write and read timeout in seconds.
@@ -45,8 +44,9 @@ pub enum Error {
 
 pub type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Default)]
-pub(crate) struct Smtp {
+#[derive(Default, DebugStub)]
+pub struct Smtp {
+    #[debug_stub(some = "SmtpTransport")]
     transport: Option<smtp::SmtpTransport>,
 
     /// Email address we are sending from.
@@ -113,14 +113,7 @@ impl Smtp {
         let domain = &lp.send_server;
         let port = lp.send_port as u16;
 
-        let provider = get_provider_info(&lp.addr);
-        let strict_tls = match lp.smtp_certificate_checks {
-            CertificateChecks::Automatic => provider.map_or(false, |provider| provider.strict_tls),
-            CertificateChecks::Strict => true,
-            CertificateChecks::AcceptInvalidCertificates
-            | CertificateChecks::AcceptInvalidCertificates2 => false,
-        };
-        let tls_config = dc_build_tls(strict_tls);
+        let tls_config = dc_build_tls(lp.smtp_certificate_checks);
         let tls_parameters = ClientTlsParameters::new(domain.to_string(), tls_config);
 
         let (creds, mechanism) = if 0 != lp.server_flags & (DC_LP_AUTH_OAUTH2 as i32) {
@@ -179,7 +172,7 @@ impl Smtp {
                 .stock_string_repl_str2(
                     StockMessage::ServerResponse,
                     format!("SMTP {}:{}", domain, port),
-                    format!("{}, ({:?})", err.to_string(), err),
+                    err.to_string(),
                 )
                 .await;
 

src/sql.rs

@@ -49,7 +49,7 @@ pub enum Error {
 pub type Result<T> = std::result::Result<T, Error>;
 
 /// A wrapper around the underlying Sqlite3 object.
-#[derive(Debug)]
+#[derive(DebugStub)]
 pub struct Sql {
     pool: RwLock<Option<r2d2::Pool<r2d2_sqlite::SqliteConnectionManager>>>,
 }

src/stock.rs

@@ -179,9 +179,6 @@ pub enum StockMessage {
 
     #[strum(props(fallback = "Unknown Sender for this chat. See 'info' for more details."))]
     UnknownSenderForChat = 72,
-
-    #[strum(props(fallback = "Message from %1$s"))]
-    SubjectForNewContact = 73,
 }
 
 /*

src/test_utils.rs

@@ -40,23 +40,6 @@ pub(crate) async fn dummy_context() -> TestContext {
     test_context().await
 }
 
-pub(crate) async fn configured_offline_context() -> TestContext {
-    let t = dummy_context().await;
-    t.ctx
-        .set_config(Config::Addr, Some("alice@example.org"))
-        .await
-        .unwrap();
-    t.ctx
-        .set_config(Config::ConfiguredAddr, Some("alice@example.org"))
-        .await
-        .unwrap();
-    t.ctx
-        .set_config(Config::Configured, Some("1"))
-        .await
-        .unwrap();
-    t
-}
-
 /// Load a pre-generated keypair for alice@example.com from disk.
 ///
 /// This saves CPU cycles by avoiding having to generate a key.

src/upload.rs

@@ -1,142 +0,0 @@
-use crate::blob::BlobObject;
-// use crate::constants::Viewtype;
-use crate::context::Context;
-use crate::error::{bail, format_err, Result};
-use crate::message::{Message, MsgId};
-use crate::pgp::{symm_decrypt_bytes, symm_encrypt_bytes};
-use async_std::fs;
-use async_std::path::PathBuf;
-use rand::Rng;
-use std::io::Cursor;
-use url::Url;
-
-/// Upload file to a HTTP upload endpoint.
-pub async fn upload_file(
-    context: &Context,
-    url: impl AsRef<str>,
-    filepath: PathBuf,
-) -> Result<String> {
-    let (passphrase, url) = parse_upload_url(url)?;
-
-    let content = fs::read(filepath).await?;
-    let encrypted = symm_encrypt_bytes(&passphrase, &content).await?;
-
-    // TODO: Use tokens for upload.
-    info!(context, "uploading encrypted file to {}", &url);
-    let response = surf::put(url).body_bytes(encrypted).await;
-    if let Err(err) = response {
-        bail!("Upload failed: {}", err);
-    }
-    let mut response = response.unwrap();
-    match response.body_string().await {
-        Ok(string) => Ok(string),
-        Err(err) => bail!("Invalid response from upload: {}", err),
-    }
-}
-
-pub async fn download_message_file(
-    context: &Context,
-    msg_id: MsgId,
-    download_path: Option<PathBuf>,
-) -> Result<()> {
-    let mut message = Message::load_from_db(context, msg_id).await?;
-    let upload_url = message
-        .param
-        .get_upload_url()
-        .ok_or_else(|| format_err!("Message has no upload URL"))?;
-
-    let (passphrase, url) = parse_upload_url(upload_url)?;
-
-    let filename: String = url
-        .path_segments()
-        .ok_or_else(|| format_err!("Invalid upload URL"))?
-        .last()
-        .ok_or_else(|| format_err!("Invalid upload URL"))?
-        .to_string();
-
-    let data = download_file(context, url, passphrase).await?;
-    let saved_path = if let Some(download_path) = download_path {
-        fs::write(&download_path, data).await?;
-        download_path.to_string_lossy().to_string()
-    } else {
-        let blob = BlobObject::create(context, filename.clone(), &data)
-            .await
-            .map_err(|err| {
-                format_err!(
-                    "Could not add blob for file download {}, error {}",
-                    filename,
-                    err
-                )
-            })?;
-        blob.as_name().to_string()
-    };
-    info!(context, "saved download to: {:?}", saved_path);
-
-    // TODO: Support getting the mime type.
-    let filemime = None;
-
-    message.set_file(saved_path, filemime);
-    message.save_param_to_disk(context).await;
-
-    Ok(())
-}
-
-/// Download and decrypt a file from a HTTP endpoint.
-pub async fn download_file(
-    context: &Context,
-    url: impl AsRef<str>,
-    passphrase: String,
-) -> Result<Vec<u8>> {
-    info!(context, "downloading file from {}", &url.as_ref());
-    let response = surf::get(url).recv_bytes().await;
-    if let Err(err) = response {
-        bail!("Download failed: {}", err);
-    }
-    let bytes = response.unwrap();
-    info!(context, "download complete, len: {}", bytes.len());
-    let reader = Cursor::new(bytes);
-    let decrypted = symm_decrypt_bytes(&passphrase, reader).await?;
-    Ok(decrypted)
-}
-
-/// Parse a URL from a string and take out the hash fragment.
-fn parse_upload_url(url: impl AsRef<str>) -> Result<(String, Url)> {
-    let mut url = url::Url::parse(url.as_ref())?;
-    let passphrase = url.fragment();
-    if passphrase.is_none() {
-        bail!("Missing passphrase for upload URL");
-    }
-    let passphrase = passphrase.unwrap().to_string();
-    url.set_fragment(None);
-    Ok((passphrase, url))
-}
-
-/// Generate a random URL based on the provided endpoint.
-pub fn generate_upload_url(_context: &Context, mut endpoint: String) -> String {
-    // equals at least 16 random bytes (base32 takes 160% of binary size).
-    const FILENAME_LEN: usize = 26;
-    // equals at least 32 random bytes.
-    const PASSPHRASE_LEN: usize = 52;
-
-    if endpoint.ends_with('/') {
-        endpoint.pop();
-    }
-    let passphrase = generate_token_string(PASSPHRASE_LEN);
-    let filename = generate_token_string(FILENAME_LEN);
-    format!("{}/{}#{}", endpoint, filename, passphrase)
-}
-
-/// Generate a random string encoded in base32.
-/// Len is the desired string length of the result.
-/// TODO: There's likely better methods to create random tokens.
-pub fn generate_token_string(len: usize) -> String {
-    const CROCKFORD_ALPHABET: &[u8] = b"0123456789abcdefghjkmnpqrstvwxyz";
-    let mut rng = rand::thread_rng();
-    let token: String = (0..len)
-        .map(|_| {
-            let idx = rng.gen_range(0, CROCKFORD_ALPHABET.len());
-            CROCKFORD_ALPHABET[idx] as char
-        })
-        .collect();
-    token
-}

upload-server/.gitignore

@@ -1,4 +0,0 @@
-uploads
-node_modules
-yarn*
-.gitfoo

upload-server/README.md

@@ -1,17 +0,0 @@
-# deltachat-upload-server
-
-Demo server for the HTTP file upload feature.
-
-### Usage
-
-```
-npm install
-node server.js
-```
-
-Configure with environment variables:
-* `UPLOAD_PATH`: Path to upload files to (default: `./uploads`)
-* `PORT`: Port to listen on (default: `8080`)
-* `HOSTNAME`: Hostname to listen on (default: `0.0.0.0`)
-* `BASEURL`: Base URL for generated links (default: `http://[hostname]:[port]/`)
-

upload-server/package.json

@@ -1,13 +0,0 @@
-{
-  "name": "deltachat-upload-server",
-  "version": "1.0.0",
-  "main": "server.js",
-  "license": "MIT",
-  "scripts": {
-    "start": "node server.js"
-  },
-  "dependencies": {
-    "base32": "^0.0.6",
-    "express": "^4.17.1"
-  }
-}

upload-server/server.js

@@ -1,73 +0,0 @@
-const p = require('path')
-const express = require('express')
-const fs = require('fs')
-const { pipeline } = require('stream')
-
-const app = express()
-
-const config = {
-  path: process.env.UPLOAD_PATH || p.resolve('./uploads'),
-  port: process.env.PORT || 8080,
-  hostname: process.env.HOSTNAME || '0.0.0.0',
-  baseurl: process.env.BASE_URL
-}
-
-if (!config.baseurl) config.baseurl = `http://${config.hostname}:${config.port}/`
-if (!config.baseurl.endsWith('/')) config.baseurl = config.baseurl + '/'
-
-if (!fs.existsSync(config.path)) {
-  fs.mkdirSync(config.path, { recursive: true })
-}
-
-app.use('/:filename', checkFilenameMiddleware)
-app.put('/:filename', (req, res) => {
-  const uploadpath = req.uploadpath
-  const filename = req.params.filename
-  fs.stat(uploadpath, (err, stat) => {
-    if (err && err.code !== 'ENOENT') {
-      console.error('error', err.message)
-      return res.code(500).send('internal server error')
-    }
-    if (stat) return res.status(500).send('filename in use')
-
-    const ws = fs.createWriteStream(uploadpath)
-    pipeline(req, ws, err => {
-      if (err) {
-        console.error('error', err.message)
-        return res.status(500).send('internal server error')
-      }
-      console.log('file uploaded: ' + uploadpath)
-      const url = config.baseurl + filename
-      res.end(url)
-    })
-  })
-})
-
-app.get('/:filename', (req, res) => {
-  const uploadpath = req.uploadpath
-  const rs = fs.createReadStream(uploadpath)
-  res.setHeader('content-type', 'application/octet-stream')
-  pipeline(rs, res, err => {
-    if (err) console.error('error', err.message)
-    if (err) return res.status(500).send(err.message)
-  })
-})
-
-function checkFilenameMiddleware (req, res, next) {
-  const filename = req.params.filename
-  if (!filename) return res.status(500).send('missing filename')
-  if (!filename.match(/^[a-zA-Z0-9]{26,32}$/)) {
-    return res.status(500).send('illegal filename')
-  }
-  const uploadpath = p.normalize(p.join(config.path, req.params.filename))
-  if (!uploadpath.startsWith(config.path)) {
-    return res.code(500).send('bad request')
-  }
-  req.uploadpath = uploadpath
-  next()
-}
-
-app.listen(config.port, err => {
-  if (err) console.error(err)
-  else console.log(`Listening on ${config.baseurl}`)
-})