fix: dont use self.error when it is not available

Corresponding C code:
d31c82478c/src/dc_context.h (L165)

.circleci/config.yml

@@ -120,32 +120,19 @@ jobs:
           paths:
             - c-docs
 
-  # build_test_docs_wheel:
-  #   docker:
-  #     - image: deltachat/coredeps
-  #       environment:
-  #           TESTS: 1
-  #           DOCS: 1
-  #   working_directory: /mnt/crate
-  #   steps:
-  #     - *restore-workspace
-  #     - *restore-cache
-  #     - run:
-  #         name: build docs, run tests and build wheels
-  #         command: ci_scripts/run-python.sh
-  #     - run:
-  #          name: copying docs and wheels to workspace
-  #          command: |
-  #            mkdir -p workspace/python
-  #            # cp -av docs workspace/c-docs
-  #            cp -av python/.docker-tox/wheelhouse workspace/
-  #            cp -av python/doc/_build/ workspace/py-docs
-  #     - persist_to_workspace:
-  #         root: workspace
-  #         paths:
-  #           # - c-docs
-  #           - py-docs
-  #           - wheelhouse
+  remote_python_packaging:
+    machine: true
+    steps:
+      - checkout
+      # the following commands on success produces
+      # workspace/{wheelhouse,py-docs} as artefact directories
+      - run: bash ci_scripts/remote_python_packaging.sh 
+      - persist_to_workspace:
+          root: workspace
+          paths:
+            # - c-docs
+            - py-docs
+            - wheelhouse
 
   remote_tests_rust:
     machine: true
@@ -157,10 +144,7 @@ jobs:
     machine: true
     steps:
       - checkout
-      #- attach_workspace:
-      #    at: workspace
       - run: ci_scripts/remote_tests_python.sh
-    # workspace/py-docs workspace/wheelhouse workspace/c-docs
 
   upload_docs_wheels:
     machine: true
@@ -168,6 +152,7 @@ jobs:
       - checkout
       - attach_workspace:
           at: workspace
+      - run: pyenv versions
       - run: pyenv global 3.5.2
       - run: ls -laR workspace
       - run: ci_scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
@@ -189,14 +174,31 @@ workflows:
     jobs:
       # - cargo_fetch
 
-      - remote_tests_rust
+      - remote_tests_rust:
+          filters:
+            tags:
+              only: /.*/
 
-      - remote_tests_python
+      - remote_tests_python:
+          filters:
+            tags:
+              only: /.*/
 
-      # - upload_docs_wheels:
-      #     requires:
-      #       - build_test_docs_wheel
-      #       - build_doxygen
+      - remote_python_packaging:
+          requires: 
+             - remote_tests_python
+             - remote_tests_rust 
+          filters:
+            tags:
+              only: /.*/
+
+      - upload_docs_wheels:
+          requires:
+             - remote_python_packaging
+             - build_doxygen
+          filters:
+            tags:
+              only: /.*/
       # - rustfmt:
       #     requires:
       #       - cargo_fetch
@@ -204,7 +206,10 @@ workflows:
       #     requires:
       #      - cargo_fetch
 
-      - build_doxygen
+      - build_doxygen:
+          filters:
+            tags:
+              only: /.*/
 
       # Linux Desktop 64bit
       # - test_x86_64-unknown-linux-gnu:

CHANGELOG.md

@@ -1,5 +1,109 @@
 # Changelog 
 
+## 1.0.0-beta.24
+
+- fix oauth2/gmail bug introduced in beta23 (not used in releases) #1219
+
+- fix panic when receiving eg. cyrillic filenames #1216
+
+- delete all consumed secure-join handshake messagess #1209 #1212
+
+- rust-level cleanups #1218 #1217 #1210 #1205
+
+- python-level cleanups #1204 #1202 #1201
+
+
+## 1.0.0-beta.23
+
+- #1197 fix imap-deletion of messages 
+
+- #1171 Combine multiple MDNs into a single mail, reducing traffic 
+
+- #1155 fix to not send out gossip always, reducing traffic
+
+- #1160 fix reply-to-encrypted determination 
+
+- #1182 Add "Auto-Submitted: auto-replied" header to MDNs
+
+- #1194 produce python wheels again, fix c/py.delta.chat
+  master-deployment 
+
+- rust-level housekeeping and improvements #1161 #1186 #1185 #1190 #1194 #1199 #1191 #1190 #1184 and more
+
+- #1063 clarify licensing 
+
+- #1147 use mailparse 0.10.2 
+
+
+## 1.0.0-beta.22
+
+- #1095 normalize email lineends to CRLF
+
+- #1095 enable link-time-optimization, saves eg. on android 11 mb
+
+- #1099 fix import regarding devicechats
+
+- #1092 improve logging
+
+- #1096 #1097 #1094 #1090 #1091 internal cleanups
+
+## 1.0.0-beta.21
+
+- #1078 #1082 ensure RFC compliance by producing 78 column lines for
+  encoded attachments. 
+
+- #1080 don't recreate and thus break group membership if an unknown 
+  sender (or mailer-daemon) sends a message referencing the group chat 
+
+- #1081 #1079 some internal cleanups 
+
+- update imap-proto dependency, to fix yandex/oauth 
+
+## 1.0.0-beta.20
+
+- #1074 fix OAUTH2/gmail
+- #1072 fix group members not appearing in contact list
+- #1071 never block interrupt_idle (thus hopefully also not on maybe_network())
+- #1069 reduce smtp-timeout to 30 seconds
+- #1066 #1065 avoid unwrap in dehtml, make literals more readable
+
+## 1.0.0-beta.19
+
+- #1058 timeout smtp-send if it doesn't complete in 15 minutes 
+
+- #1059 trim down logging
+
+## 1.0.0-beta.18
+
+- #1056 avoid panicking when we couldn't read imap-server's greeting
+  message 
+
+- #1055 avoid panicking when we don't have a selected folder
+
+- #1052 #1049 #1051 improve logging to add thread-id/name and
+  file/lineno to each info/warn message.
+
+- #1050 allow python bindings to initialize Account with "os_name".
+
+
+## 1.0.0-beta.17
+
+- #1044 implement avatar recoding to 192x192 in core to keep file sizes small. 
+
+- #1024 fix #1021 SQL/injection malformed Chat-Group-Name breakage
+
+- #1036 fix smtp crash by pulling in a fixed async-smtp 
+
+- #1039 fix read-receipts appearing as normal messages when you change
+  MDN settings 
+
+- #1040 do not panic on SystemTimeDifference
+
+- #1043 avoid potential crashes in malformed From/Chat-Disposition... headers  
+
+- #1045 #1041 #1038 #1035 #1034 #1029 #1025 various cleanups and doc
+  improvments 
+
 ## 1.0.0-beta.16
 
 - alleviate login problems with providers which only

Cargo.toml

@@ -1,9 +1,12 @@
 [package]
 name = "deltachat"
-version = "1.0.0-beta.16"
+version = "1.0.0-beta.24"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
-license = "MPL"
+license = "MPL-2.0"
+
+[profile.release]
+# lto = true
 
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
@@ -14,15 +17,15 @@ hex = "0.4.0"
 sha2 = "0.8.0"
 rand = "0.7.0"
 smallvec = "1.0.0"
-reqwest = { version = "0.9.15" }
+reqwest = { version = "0.10.0", features = ["blocking", "json"] }
 num-derive = "0.3.0"
 num-traits = "0.2.6"
-async-smtp = { git = "https://github.com/async-email/async-smtp", branch = "master" }
+async-smtp = "0.2"
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
-lettre_email = { git = "https://github.com/deltachat/lettre", branch = "native_tls" }
-async-imap = { git = "https://github.com/async-email/async-imap", branch="native_tls", default-features = false, features = ["tls_native"] }
-async-native-tls = "0.1.1"
-async-std = { version = "1.0", features = ["unstable"] }
+lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
+async-imap = "0.2"
+async-native-tls = "0.3.1"
+async-std = { version = "1.4", features = ["unstable"] }
 base64 = "0.11"
 charset = "0.1"
 percent-encoding = "2.0"
@@ -32,12 +35,10 @@ chrono = "0.4.6"
 failure = "0.1.5"
 failure_derive = "0.1.5"
 indexmap = "1.3.0"
-# TODO: make optional
-rustyline = "4.1.0"
 lazy_static = "1.4.0"
 regex = "1.1.6"
-rusqlite = { version = "0.20", features = ["bundled"] }
-r2d2_sqlite = "0.12.0"
+rusqlite = { version = "0.21", features = ["bundled"] }
+r2d2_sqlite = "0.13.0"
 r2d2 = "0.8.5"
 strum = "0.16.0"
 strum_macros = "0.16.0"
@@ -52,9 +53,13 @@ bitflags = "1.1.0"
 debug_stub_derive = "0.3.0"
 sanitize-filename = "0.2.1"
 stop-token = { version = "0.1.1", features = ["unstable"] }
-mailparse = "0.10.1"
+mailparse = "0.10.2"
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
 native-tls = "0.2.3"
+image = { version = "0.22.4", default-features=false, features = ["gif_codec", "jpeg", "ico", "png_codec", "pnm", "webp", "bmp"] }
+pretty_env_logger = "0.3.1"
+rustyline = { version = "4.1.0", optional = true }
+crossbeam = "0.7.3"
 
 [dev-dependencies]
 tempfile = "3.0"
@@ -75,10 +80,11 @@ path = "examples/simple.rs"
 [[example]]
 name = "repl"
 path = "examples/repl/main.rs"
+required-features = ["rustyline"]
 
 
 [features]
 default = ["nightly", "ringbuf"]
-vendored = ["native-tls/vendored", "reqwest/default-tls-vendored"]
+vendored = ["async-native-tls/vendored", "reqwest/native-tls-vendored", "async-smtp/native-tls-vendored"]
 nightly = ["pgp/nightly"]
 ringbuf = ["pgp/ringbuf"]

LICENSE

@@ -2,9 +2,6 @@ The files in this directory and under its subdirectories
 are (c) 2019 by Bjoern Petersen and contributors and released under the
 Mozilla Public License Version 2.0, see below for a copy.
 
-NOTE that the files in the "libs" directory are copyrighted by third parties
-and come with their own respective licenses.
-
 Mozilla Public License Version 2.0
 ==================================
 

README.md

@@ -114,3 +114,22 @@ $ cargo test -- --ignored
 [circle]: https://circleci.com/gh/deltachat/deltachat-core-rust/
 [appveyor-shield]: https://ci.appveyor.com/api/projects/status/lqpegel3ld4ipxj8/branch/master?style=flat-square
 [appveyor]: https://ci.appveyor.com/project/dignifiedquire/deltachat-core-rust/branch/master
+
+## Language bindings and frontend projects
+
+Language bindings are available for:
+
+- [C](https://c.delta.chat)
+- [Node.js](https://www.npmjs.com/package/deltachat-node)
+- [Python](https://py.delta.chat)
+- [Go](https://github.com/hugot/go-deltachat/)
+- **Java** and **Swift** (contained in the Android/iOS repos)
+
+The following "frontend" projects make use of the Rust-library
+or its language bindings:
+
+- [Android](https://github.com/deltachat/deltachat-android)
+- [iOS](https://github.com/deltachat/deltachat-ios)
+- [Desktop](https://github.com/deltachat/deltachat-desktop)
+- [Pidgin](https://code.ur.gs/lupine/purple-plugin-delta/)
+- several **Bots**

ci_scripts/ci_upload.sh

@@ -14,24 +14,22 @@ DOXYDOCDIR=${3:?directory where doxygen docs to be found}
 export BRANCH=${CIRCLE_BRANCH:?specify branch for uploading purposes}
 
 
-# DISABLED: python docs to py.delta.chat
-#ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
-#rsync -avz \
-#  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
-#  "$PYDOCDIR/html/" \
-#  delta@py.delta.chat:build/${BRANCH}
+# python docs to py.delta.chat
+ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
+rsync -avz \
+  --delete \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  "$PYDOCDIR/html/" \
+  delta@py.delta.chat:build/${BRANCH}
 
 # C docs to c.delta.chat
 ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@c.delta.chat mkdir -p build-c/${BRANCH}
 rsync -avz \
+  --delete \
   -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
   "$DOXYDOCDIR/html/" \
   delta@c.delta.chat:build-c/${BRANCH}
 
-exit 0
-
-# OUTDATED -- for re-use from python release-scripts 
-
 echo -----------------------
 echo upload wheels 
 echo -----------------------
@@ -39,6 +37,7 @@ echo -----------------------
 # Bundle external shared libraries into the wheels
 pushd $WHEELHOUSEDIR
 
+pip3 install -U pip
 pip3 install devpi-client
 devpi use https://m.devpi.net
 devpi login dc --password $DEVPI_LOGIN
@@ -50,6 +49,9 @@ devpi use dc/$N_BRANCH || {
     devpi use dc/$N_BRANCH
 }
 devpi index $N_BRANCH bases=/root/pypi
-devpi upload deltachat*.whl
+devpi upload deltachat*
 
 popd
+
+# remove devpi non-master dc indices if thy are too old
+python ci_scripts/cleanup_devpi_indices.py

ci_scripts/cleanup_devpi_indices.py

@@ -0,0 +1,69 @@
+"""
+Remove old "dc" indices except for master which always stays.
+
+"""
+from requests import Session
+import datetime
+import sys
+import subprocess
+
+MAXDAYS=7
+
+session = Session()
+session.headers["Accept"] = "application/json"
+
+
+def get_indexes(baseurl, username):
+    response = session.get(baseurl + username)
+    assert response.status_code == 200
+    result = response.json()["result"]
+    return result["indexes"]
+
+
+def get_projectnames(baseurl, username, indexname):
+    response = session.get(baseurl + username + "/" + indexname)
+    assert response.status_code == 200
+    result = response.json()["result"]
+    return result["projects"]
+
+
+def get_release_dates(baseurl, username, indexname, projectname):
+    response = session.get(baseurl + username + "/" + indexname + "/" + projectname)
+    assert response.status_code == 200
+    result = response.json()["result"]
+    dates = set()
+    for value in result.values():
+        if "+links" not in value:
+            continue
+        for link in value["+links"]:
+            for log in link["log"]:
+                dates.add(tuple(log["when"]))
+    return dates
+
+
+def run():
+    baseurl = "https://m.devpi.net/"
+    username = "dc"
+    for indexname in get_indexes(baseurl, username):
+        projectnames = get_projectnames(baseurl, username, indexname)
+        if indexname == "master" or not indexname:
+            continue
+        assert projectnames == ["deltachat"]
+        for projectname in projectnames:
+            dates = get_release_dates(baseurl, username, indexname, projectname)
+            if not dates:
+                print(
+                    "%s has no releases" % (baseurl + username + "/" + indexname),
+                    file=sys.stderr)
+                date = datetime.datetime.now()
+            else:
+                date = datetime.datetime(*max(dates))
+            if (datetime.datetime.now() - date) > datetime.timedelta(days=MAXDAYS):
+                assert username and indexname
+                url = baseurl + username + "/" + indexname
+                subprocess.check_call(["devpi", "index", "-y", "--delete", url])
+
+
+
+if __name__ == '__main__':
+    run()

ci_scripts/docker-coredeps/Dockerfile

@@ -5,14 +5,11 @@ RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
     echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
 ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
 
-# Install a recent Perl, needed to install OpenSSL
+# Install a recent Perl, needed to install the openssl crate 
 ADD deps/build_perl.sh /builder/build_perl.sh
+RUN rm /usr/bin/perl
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
 
-# Install OpenSSL
-ADD deps/build_openssl.sh /builder/build_openssl.sh
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_openssl.sh && cd .. && rm -r tmp1
-
 ENV PIP_DISABLE_PIP_VERSION_CHECK 1
 
 # Install python tools (auditwheels,tox, ...)

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -3,6 +3,9 @@
 set -e -x
 
 # Install Rust
-curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-09-12 -y
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-11-06 -y
 export PATH=/root/.cargo/bin:$PATH
 rustc --version
+
+# remove some 300-400 MB that we don't need for automated builds
+rm -rf /root/.rustup/toolchains/nightly-2019-11-06-x86_64-unknown-linux-gnu/share/

ci_scripts/docker-coredeps/deps/run_all.sh

@@ -1,49 +0,0 @@
-#!/bin/bash
-#
-# Build the Delta Chat C/Rust library
-#
-set -e -x
-
-# perform clean build of core and install 
-export TOXWORKDIR=.docker-tox
-
-# build core library
-
-cargo build --release -p deltachat_ffi
-
-# configure access to a base python and 
-# to several python interpreters needed by tox below
-export PATH=$PATH:/opt/python/cp35-cp35m/bin
-export PYTHONDONTWRITEBYTECODE=1
-pushd /bin
-ln -s /opt/python/cp27-cp27m/bin/python2.7
-ln -s /opt/python/cp36-cp36m/bin/python3.6
-ln -s /opt/python/cp37-cp37m/bin/python3.7
-popd
-
-#
-# run python tests
-#
-
-if [ -n "$TESTS" ]; then 
-
-    echo ----------------
-    echo run python tests
-    echo ----------------
-
-    pushd python 
-    # first run all tests ...
-    rm -rf tests/__pycache__
-    rm -rf src/deltachat/__pycache__
-    export PYTHONDONTWRITEBYTECODE=1
-    tox --workdir "$TOXWORKDIR" -e py27,py35,py36,py37
-    popd
-fi
-
-
-if [ -n "$DOCS" ]; then 
-    echo -----------------------
-    echo generating python docs
-    echo -----------------------
-    (cd python && tox --workdir "$TOXWORKDIR" -e doc) 
-fi

ci_scripts/remote_python_packaging.sh

@@ -0,0 +1,51 @@
+#!/bin/bash 
+
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+
+# we construct the BUILDDIR such that we can easily share the
+# CARGO_TARGET_DIR between runs ("..")
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+set -xe
+
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist 
+# we seem to need .git for setuptools_scm versioning 
+find .git >>.rsynclist
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+
+set +x
+
+# we have to create a remote file for the remote-docker run 
+# so we can do a simple ssh command with a TTY
+# so that when our job dies, all container-runs are aborted.
+# sidenote: the circle-ci machinery will kill ongoing jobs
+# if there are new commits and we want to ensure that 
+# everything is terminated/cleaned up and we have no orphaned
+# useless still-running docker-containers consuming resources. 
+
+ssh $SSHTARGET bash -c "cat >$BUILDDIR/exec_docker_run" <<_HERE
+    set +x -e
+    cd $BUILDDIR
+    export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
+
+    set -x
+
+    # run everything else inside docker 
+    docker run -e DCC_PY_LIVECONFIG \
+       --rm -it -v \$(pwd):/mnt -w /mnt \
+       deltachat/coredeps ci_scripts/run_all.sh
+
+_HERE
+
+echo "--- Running $CIRCLE_JOB remotely"
+
+ssh -t $SSHTARGET bash "$BUILDDIR/exec_docker_run"
+mkdir -p workspace 
+rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/wheelhouse/*manylinux1*" workspace/wheelhouse/
+rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/dist/*" workspace/wheelhouse/
+rsync -avz "$SSHTARGET:$BUILDDIR/python/doc/_build/" workspace/py-docs

ci_scripts/run-doxygen.sh

@@ -3,5 +3,4 @@
 set -ex
 
 cd deltachat-ffi 
-doxygen 
-
+PROJECT_NUMBER=$(git log -1 --format "%h (%cd)") doxygen 

ci_scripts/run_all.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+#
+# Build the Delta Chat Core Rust library, Python wheels and docs 
+
+set -e -x
+
+# Perform clean build of core and install.
+export TOXWORKDIR=.docker-tox
+
+# compile core lib
+
+export PATH=/root/.cargo/bin:$PATH
+cargo build --release -p deltachat_ffi
+# cargo test --all --all-features
+
+# Statically link against libdeltachat.a.
+export DCC_RS_DEV=$(pwd)
+
+# Configure access to a base python and to several python interpreters
+# needed by tox below.
+export PATH=$PATH:/opt/python/cp35-cp35m/bin
+export PYTHONDONTWRITEBYTECODE=1
+pushd /bin
+ln -s /opt/python/cp27-cp27m/bin/python2.7
+ln -s /opt/python/cp36-cp36m/bin/python3.6
+ln -s /opt/python/cp37-cp37m/bin/python3.7
+ln -s /opt/python/cp38-cp38/bin/python3.8
+popd
+
+pushd python
+# prepare a clean tox run
+rm -rf tests/__pycache__
+rm -rf src/deltachat/__pycache__
+mkdir -p $TOXWORKDIR
+
+# disable live-account testing to speed up test runs and wheel building
+# XXX we may switch on some live-tests on for better ensurances 
+# Note that the independent remote_tests_python step does all kinds of
+# live-testing already. 
+unset DCC_PY_LIVECONFIG 
+tox --workdir "$TOXWORKDIR" -e py35,py36,py37,py38,auditwheels
+popd
+
+
+echo -----------------------
+echo generating python docs
+echo -----------------------
+(cd python && tox --workdir "$TOXWORKDIR" -e doc)

deltachat-ffi/Cargo.toml

@@ -1,11 +1,11 @@
 [package]
 name = "deltachat_ffi"
-version = "1.0.0-beta.16"
+version = "1.0.0-beta.24"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 readme = "README.md"
-license = "MIT OR Apache-2.0"
+license = "MPL-2.0"
 
 keywords = ["deltachat", "chat", "openpgp", "email", "encryption"]
 categories = ["cryptography", "std", "email"]
@@ -21,6 +21,7 @@ libc = "0.2"
 human-panic = "1.0.1"
 num-traits = "0.2.6"
 failure = "0.1.6"
+serde_json = "1.0"
 
 [features]
 default = ["vendored", "nightly", "ringbuf"]

deltachat-ffi/deltachat.h

@@ -243,7 +243,7 @@ typedef uintptr_t (*dc_callback_t) (dc_context_t* context, int event, uintptr_t
  *     The object must be passed to the other context functions
  *     and must be freed using dc_context_unref() after usage.
  */
-dc_context_t*   dc_context_new               (dc_callback_t cb, void* userdata, const char* os_name);
+dc_context_t*   dc_context_new               (void* userdata, const char* os_name);
 
 
 /**
@@ -404,14 +404,14 @@ int             dc_set_config                (dc_context_t* context, const char*
 char*           dc_get_config                (dc_context_t* context, const char* key);
 
 /**
- * Set stock string translation. 
+ * Set stock string translation.
  *
- * The function will emit warnings if it returns an error state. 
+ * The function will emit warnings if it returns an error state.
  *
  * @memberof dc_context_t
  * @param context The context object
  * @param stock_id   the integer id of the stock message (DC_STR_*)
- * @param stock_msg  the message to be used 
+ * @param stock_msg  the message to be used
  * @return int (==0 on error, 1 on success)
  */
 int             dc_set_stock_translation(dc_context_t* context, uint32_t stock_id, const char* stock_msg);
@@ -538,307 +538,14 @@ int             dc_is_configured             (const dc_context_t* context);
 
 
 /**
- * Execute pending imap-jobs.
- * This function and dc_perform_imap_fetch() and dc_perform_imap_idle()
- * must be called from the same thread, typically in a loop.
- *
- * Example:
- *
- *     void* imap_thread_func(void* context)
- *     {
- *         while (true) {
- *             dc_perform_imap_jobs(context);
- *             dc_perform_imap_fetch(context);
- *             dc_perform_imap_idle(context);
- *         }
- *     }
- *
- *     // start imap-thread that runs forever
- *     pthread_t imap_thread;
- *     pthread_create(&imap_thread, NULL, imap_thread_func, context);
- *
- *     ... program runs ...
- *
- *     // network becomes available again -
- *     // the interrupt causes dc_perform_imap_idle() in the thread above
- *     // to return so that jobs are executed and messages are fetched.
- *     dc_maybe_network(context);
- *
- * @memberof dc_context_t
- * @param context The context as created by dc_context_new().
- * @return None.
+ * TODO: Document
  */
-void            dc_perform_imap_jobs         (dc_context_t* context);
-
+void            dc_context_run         (dc_context_t* context, dc_callback_t cb);
 
 /**
- * Fetch new messages, if any.
- * This function and dc_perform_imap_jobs() and dc_perform_imap_idle() must be called from the same thread,
- * typically in a loop.
- *
- * See dc_perform_imap_jobs() for an example.
- *
- * @memberof dc_context_t
- * @param context The context as created by dc_context_new().
- * @return None.
+ * TODO: Document
  */
-void            dc_perform_imap_fetch        (dc_context_t* context);
-
-
-/**
- * Wait for messages or jobs.
- * This function and dc_perform_imap_jobs() and dc_perform_imap_fetch() must be called from the same thread,
- * typically in a loop.
- *
- * You should call this function directly after calling dc_perform_imap_fetch().
- *
- * See dc_perform_imap_jobs() for an example.
- *
- * @memberof dc_context_t
- * @param context The context as created by dc_context_new().
- * @return None.
- */
-void            dc_perform_imap_idle         (dc_context_t* context);
-
-
-/**
- * Interrupt waiting for imap-jobs.
- * If dc_perform_imap_jobs(), dc_perform_imap_fetch() and dc_perform_imap_idle() are called in a loop,
- * calling this function causes imap-jobs to be executed and messages to be fetched.
- *
- * dc_interrupt_imap_idle() does _not_ interrupt dc_perform_imap_jobs() or dc_perform_imap_fetch().
- * If the imap-thread is inside one of these functions when dc_interrupt_imap_idle() is called, however,
- * the next call of the imap-thread to dc_perform_imap_idle() is interrupted immediately.
- *
- * Internally, this function is called whenever a imap-jobs should be processed
- * (delete message, markseen etc.).
- *
- * When you need to call this function just because to get jobs done after network changes,
- * use dc_maybe_network() instead.
- *
- * @memberof dc_context_t
- * @param context The context as created by dc_context_new().
- * @return None.
- */
-void            dc_interrupt_imap_idle       (dc_context_t* context);
-
-
-/**
- * Execute pending mvbox-jobs.
- * This function and dc_perform_mvbox_fetch() and dc_perform_mvbox_idle()
- * must be called from the same thread, typically in a loop.
- *
- * Example:
- *
- *     void* mvbox_thread_func(void* context)
- *     {
- *         while (true) {
- *             dc_perform_mvbox_jobs(context);
- *             dc_perform_mvbox_fetch(context);
- *             dc_perform_mvbox_idle(context);
- *         }
- *     }
- *
- *     // start mvbox-thread that runs forever
- *     pthread_t mvbox_thread;
- *     pthread_create(&mvbox_thread, NULL, mvbox_thread_func, context);
- *
- *     ... program runs ...
- *
- *     // network becomes available again -
- *     // the interrupt causes dc_perform_mvbox_idle() in the thread above
- *     // to return so that jobs are executed and messages are fetched.
- *     dc_maybe_network(context);
- *
- * @memberof dc_context_t
- * @param context The context as created by dc_context_new().
- * @return None.
- */
-void            dc_perform_mvbox_jobs         (dc_context_t* context);
-
-
-/**
- * Fetch new messages from the MVBOX, if any.
- * The MVBOX is a folder on the account where chat messages are moved to.
- * The moving is done to not disturb shared accounts that are used by both,
- * Delta Chat and a classical MUA.
- *
- * @memberof dc_context_t
- * @param context The context as created by dc_context_new().
- * @return None.
- */
-void            dc_perform_mvbox_fetch       (dc_context_t* context);
-
-
-/**
- * Wait for messages or jobs in the MVBOX-thread.
- * This function and dc_perform_mvbox_fetch().
- * must be called from the same thread, typically in a loop.
- *
- * You should call this function directly after calling dc_perform_mvbox_fetch().
- *
- * See dc_perform_mvbox_fetch() for an example.
- *
- * @memberof dc_context_t
- * @param context The context as created by dc_context_new().
- * @return None.
- */
-void            dc_perform_mvbox_idle        (dc_context_t* context);
-
-
-/**
- * Interrupt waiting for MVBOX-fetch.
- * dc_interrupt_mvbox_idle() does _not_ interrupt dc_perform_mvbox_fetch().
- * If the MVBOX-thread is inside this function when dc_interrupt_mvbox_idle() is called, however,
- * the next call of the MVBOX-thread to dc_perform_mvbox_idle() is interrupted immediately.
- *
- * Internally, this function is called whenever a imap-jobs should be processed.
- *
- * When you need to call this function just because to get jobs done after network changes,
- * use dc_maybe_network() instead.
- *
- * @memberof dc_context_t
- * @param context The context as created by dc_context_new().
- * @return None.
- */
-void            dc_interrupt_mvbox_idle      (dc_context_t* context);
-
-
-/**
- * Execute pending sentbox-jobs.
- * This function and dc_perform_sentbox_fetch() and dc_perform_sentbox_idle()
- * must be called from the same thread, typically in a loop.
- *
- * Example:
- *
- *     void* sentbox_thread_func(void* context)
- *     {
- *         while (true) {
- *             dc_perform_sentbox_jobs(context);
- *             dc_perform_sentbox_fetch(context);
- *             dc_perform_sentbox_idle(context);
- *         }
- *     }
- *
- *     // start sentbox-thread that runs forever
- *     pthread_t sentbox_thread;
- *     pthread_create(&sentbox_thread, NULL, sentbox_thread_func, context);
- *
- *     ... program runs ...
- *
- *     // network becomes available again -
- *     // the interrupt causes dc_perform_sentbox_idle() in the thread above
- *     // to return so that jobs are executed and messages are fetched.
- *     dc_maybe_network(context);
- *
- * @memberof dc_context_t
- * @param context The context as created by dc_context_new().
- * @return None.
- */
-void            dc_perform_sentbox_jobs         (dc_context_t* context);
-
-
-/**
- * Fetch new messages from the Sent folder, if any.
- * This function and dc_perform_sentbox_idle()
- * must be called from the same thread, typically in a loop.
- *
- * @memberof dc_context_t
- * @param context The context as created by dc_context_new().
- * @return None.
- */
-void            dc_perform_sentbox_fetch     (dc_context_t* context);
-
-
-/**
- * Wait for messages or jobs in the SENTBOX-thread.
- * This function and dc_perform_sentbox_fetch()
- * must be called from the same thread, typically in a loop.
- *
- * @memberof dc_context_t
- * @param context The context as created by dc_context_new().
- * @return None.
- */
-void            dc_perform_sentbox_idle      (dc_context_t* context);
-
-
-/**
- * Interrupt waiting for messages or jobs in the SENTBOX-thread.
- *
- * @memberof dc_context_t
- * @param context The context as created by dc_context_new().
- * @return None.
- */
-void            dc_interrupt_sentbox_idle    (dc_context_t* context);
-
-
-/**
- * Execute pending smtp-jobs.
- * This function and dc_perform_smtp_idle() must be called from the same thread,
- * typically in a loop.
- *
- * Example:
- *
- *     void* smtp_thread_func(void* context)
- *     {
- *         while (true) {
- *             dc_perform_smtp_jobs(context);
- *             dc_perform_smtp_idle(context);
- *         }
- *     }
- *
- *     // start smtp-thread that runs forever
- *     pthread_t smtp_thread;
- *     pthread_create(&smtp_thread, NULL, smtp_thread_func, context);
- *
- *     ... program runs ...
- *
- *     // network becomes available again -
- *     // the interrupt causes dc_perform_smtp_idle() in the thread above
- *     // to return so that jobs are executed
- *     dc_maybe_network(context);
- *
- * @memberof dc_context_t
- * @param context The context as created by dc_context_new().
- * @return None.
- */
-void            dc_perform_smtp_jobs         (dc_context_t* context);
-
-
-/**
- * Wait for smtp-jobs.
- * This function and dc_perform_smtp_jobs() must be called from the same thread,
- * typically in a loop.
- *
- * See dc_interrupt_smtp_idle() for an example.
- *
- * @memberof dc_context_t
- * @param context The context as created by dc_context_new().
- * @return None.
- */
-void            dc_perform_smtp_idle         (dc_context_t* context);
-
-
-/**
- * Interrupt waiting for smtp-jobs.
- * If dc_perform_smtp_jobs() and dc_perform_smtp_idle() are called in a loop,
- * calling this function causes jobs to be executed.
- *
- * dc_interrupt_smtp_idle() does _not_ interrupt dc_perform_smtp_jobs().
- * If the smtp-thread is inside this function when dc_interrupt_smtp_idle() is called, however,
- * the next call of the smtp-thread to dc_perform_smtp_idle() is interrupted immediately.
- *
- * Internally, this function is called whenever a message is to be sent.
- *
- * When you need to call this function just because to get jobs done after network changes,
- * use dc_maybe_network() instead.
- *
- * @memberof dc_context_t
- * @param context The context as created by dc_context_new().
- * @return None.
- */
-void            dc_interrupt_smtp_idle       (dc_context_t* context);
-
+void            dc_context_shutdown         (dc_context_t* context);
 
 /**
  * This function can be called whenever there is a hint
@@ -1603,7 +1310,7 @@ char*           dc_get_mime_headers          (dc_context_t* context, uint32_t ms
 void            dc_delete_msgs               (dc_context_t* context, const uint32_t* msg_ids, int msg_cnt);
 
 /**
- * Empty IMAP server folder: delete all messages. 
+ * Empty IMAP server folder: delete all messages.
  *
  * @memberof dc_context_t
  * @param context The context object as created by dc_context_new()
@@ -1756,7 +1463,7 @@ uint32_t        dc_create_contact            (dc_context_t* context, const char*
  * Trying to add email-addresses that are already in the contact list,
  * results in updating the name unless the name was changed manually by the user.
  * If any email-address or any name is really updated,
- * the event DC_EVENT_CONTACTS_CHANGED is sent.
+ * the event #DC_EVENT_CONTACTS_CHANGED is sent.
  *
  * To add a single contact entered by the user, you should prefer dc_create_contact(),
  * however, for adding a bunch of addresses, this function is _much_ faster.
@@ -2107,20 +1814,22 @@ dc_lot_t*       dc_check_qr                  (dc_context_t* context, const char*
 
 
 /**
- * Get QR code text that will offer an secure-join verification.
+ * Get QR code text that will offer an Setup-Contact or Verified-Group invitation.
  * The QR code is compatible to the OPENPGP4FPR format
  * so that a basic fingerprint comparison also works eg. with OpenKeychain.
  *
  * The scanning device will pass the scanned content to dc_check_qr() then;
- * if this function returns DC_QR_ASK_VERIFYCONTACT or DC_QR_ASK_VERIFYGROUP
+ * if dc_check_qr() returns DC_QR_ASK_VERIFYCONTACT or DC_QR_ASK_VERIFYGROUP
  * an out-of-band-verification can be joined using dc_join_securejoin()
  *
  * @memberof dc_context_t
  * @param context The context object.
  * @param chat_id If set to a group-chat-id,
- *     the group-join-protocol is offered in the QR code;
+ *     the Verified-Group-Invite protocol is offered in the QR code;
  *     works for verified groups as well as for normal groups.
- *     If set to 0, the setup-Verified-contact-protocol is offered in the QR code.
+ *     If set to 0, the Setup-Contact protocol is offered in the QR code.
+ *     See https://countermitm.readthedocs.io/en/latest/new.html
+ *     for details about both protocols.
  * @return Text that should go to the QR code,
  *     On errors, an empty QR code is returned, NULL is never returned.
  *     The returned string must be released using dc_str_unref() after usage.
@@ -2129,13 +1838,29 @@ char*           dc_get_securejoin_qr         (dc_context_t* context, uint32_t ch
 
 
 /**
- * Join an out-of-band-verification initiated on another device with dc_get_securejoin_qr().
+ * Continue a Setup-Contact or Verified-Group-Invite protocol
+ * started on another device with dc_get_securejoin_qr().
  * This function is typically called when dc_check_qr() returns
  * lot.state=DC_QR_ASK_VERIFYCONTACT or lot.state=DC_QR_ASK_VERIFYGROUP.
  *
- * This function takes some time and sends and receives several messages.
- * You should call it in a separate thread; if you want to abort it, you should
- * call dc_stop_ongoing_process().
+ * Depending on the given QR code,
+ * this function may takes some time and sends and receives several messages.
+ * Therefore, you should call it always in a separate thread;
+ * if you want to abort it, you should call dc_stop_ongoing_process().
+ *
+ * - If the given QR code starts the Setup-Contact protocol,
+ *   the function typically returns immediately
+ *   and the handshake runs in background.
+ *   Subsequent calls of dc_join_securejoin() will abort unfinished tasks.
+ *   The returned chat is the one-to-one opportunistic chat.
+ *   When the protocol has finished, an info-message is added to that chat.
+ * - If the given QR code starts the Verified-Group-Invite protocol,
+ *   the function waits until the protocol has finished.
+ *   This is because the verified group is not opportunistic
+ *   and can be created only when the contacts have verified each other.
+ *
+ * See https://countermitm.readthedocs.io/en/latest/new.html
+ * for details about both protocols.
  *
  * @memberof dc_context_t
  * @param context The context object
@@ -2143,6 +1868,9 @@ char*           dc_get_securejoin_qr         (dc_context_t* context, uint32_t ch
  *     to dc_check_qr().
  * @return Chat-id of the joined chat, the UI may redirect to the this chat.
  *     If the out-of-band verification failed or was aborted, 0 is returned.
+ *     A returned chat-id does not guarantee that the chat or the belonging contact is verified.
+ *     If needed, this be checked with dc_chat_is_verified() and dc_contact_is_verified(),
+ *     however, in practise, the UI will just listen to #DC_EVENT_CONTACTS_CHANGED unconditionally.
  */
 uint32_t        dc_join_securejoin           (dc_context_t* context, const char* qr);
 
@@ -2618,18 +2346,18 @@ dc_context_t*    dc_chatlist_get_context     (dc_chatlist_t* chatlist);
 
 
 /**
- * Get info summary for a chat, in json format. 
+ * Get info summary for a chat, in json format.
  *
- * The returned json string has the following key/values: 
+ * The returned json string has the following key/values:
  *
- * id: chat id 
- * name: chat/group name 
- * color: color of this chat 
- * last-message-from: who sent the last message 
- * last-message-text: message (truncated) 
+ * id: chat id
+ * name: chat/group name
+ * color: color of this chat
+ * last-message-from: who sent the last message
+ * last-message-text: message (truncated)
  * last-message-state: DC_STATE* constant
- * last-message-date: 
- * avatar-path: path-to-blobfile 
+ * last-message-date:
+ * avatar-path: path-to-blobfile
  * is_verified: yes/no
 
  * @return a utf8-encoded json string containing all requested info. Must be freed using dc_str_unref().  NULL is never returned.
@@ -2989,15 +2717,15 @@ int             dc_msg_get_viewtype           (const dc_msg_t* msg);
  * Get the state of a message.
  *
  * Incoming message states:
- * - DC_STATE_IN_FRESH (10) - Incoming _fresh_ message. Fresh messages are not noticed nor seen and are typically shown in notifications. Use dc_get_fresh_msgs() to get all fresh messages.
- * - DC_STATE_IN_NOTICED (13) - Incoming _noticed_ message. Eg. chat opened but message not yet read - noticed messages are not counted as unread but did not marked as read nor resulted in MDNs. Use dc_marknoticed_chat() or dc_marknoticed_contact() to mark messages as being noticed.
- * - DC_STATE_IN_SEEN (16) - Incoming message, really _seen_ by the user. Marked as read on IMAP and MDN may be send. Use dc_markseen_msgs() to mark messages as being seen.
+ * - DC_STATE_IN_FRESH (10) - Incoming _fresh_ message. Fresh messages are neither noticed nor seen and are typically shown in notifications. Use dc_get_fresh_msgs() to get all fresh messages.
+ * - DC_STATE_IN_NOTICED (13) - Incoming _noticed_ message. E.g. chat opened but message not yet read - noticed messages are not counted as unread but did not marked as read nor resulted in MDNs. Use dc_marknoticed_chat() or dc_marknoticed_contact() to mark messages as being noticed.
+ * - DC_STATE_IN_SEEN (16) - Incoming message, really _seen_ by the user. Marked as read on IMAP and MDN may be sent. Use dc_markseen_msgs() to mark messages as being seen.
  *
  * Outgoing message states:
  * - DC_STATE_OUT_PREPARING (18) - For files which need time to be prepared before they can be sent,
  *   the message enters this state before DC_STATE_OUT_PENDING.
  * - DC_STATE_OUT_DRAFT (19) - Message saved as draft using dc_set_draft()
- * - DC_STATE_OUT_PENDING (20) - The user has send the "send" button but the
+ * - DC_STATE_OUT_PENDING (20) - The user has pressed the "send" button but the
  *   message is not yet sent and is pending in some way. Maybe we're offline (no checkmark).
  * - DC_STATE_OUT_FAILED (24) - _Unrecoverable_ error (_recoverable_ errors result in pending messages), you'll receive the event #DC_EVENT_MSG_FAILED.
  * - DC_STATE_OUT_DELIVERED (26) - Outgoing message successfully delivered to server (one checkmark). Note, that already delivered messages may get into the state DC_STATE_OUT_FAILED if we get such a hint from the server.
@@ -4167,30 +3895,30 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_EVENT_IMAP_MESSAGE_MOVED   105
 
 /**
- * Emitted when an IMAP folder was emptied. 
+ * Emitted when an IMAP folder was emptied.
  *
  * @param data1 0
- * @param data2 (const char*) folder name. 
+ * @param data2 (const char*) folder name.
  *     Must not be unref'd or modified and is valid only until the callback returns.
  * @return 0
  */
 #define DC_EVENT_IMAP_FOLDER_EMPTIED  106
 
 /**
- * Emitted when a new blob file was successfully written 
+ * Emitted when a new blob file was successfully written
  *
  * @param data1 0
- * @param data2 (const char*) path name 
+ * @param data2 (const char*) path name
  *     Must not be unref'd or modified and is valid only until the callback returns.
  * @return 0
  */
 #define DC_EVENT_NEW_BLOB_FILE 150
 
 /**
- * Emitted when a blob file was successfully deleted 
+ * Emitted when a blob file was successfully deleted
  *
  * @param data1 0
- * @param data2 (const char*) path name 
+ * @param data2 (const char*) path name
  *     Must not be unref'd or modified and is valid only until the callback returns.
  * @return 0
  */
@@ -4347,7 +4075,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 
 
 /**
- * Contact(s) created, renamed, blocked or deleted.
+ * Contact(s) created, renamed, verified, blocked or deleted.
  *
  * @param data1 (int) If not 0, this is the contact_id of an added contact that should be selected.
  * @param data2 0

deltachat-ffi/src/dc_array.rs

@@ -2,7 +2,6 @@ use crate::location::Location;
 
 /* * the structure behind dc_array_t */
 #[derive(Debug, Clone)]
-#[allow(non_camel_case_types)]
 pub enum dc_array_t {
     Locations(Vec<Location>),
     Uint(Vec<u32>),

deltachat-ffi/src/lib.rs

@@ -4,12 +4,14 @@
     non_upper_case_globals,
     non_upper_case_globals,
     non_camel_case_types,
-    non_snake_case
+    clippy::missing_safety_doc,
+    clippy::expect_fun_call
 )]
 
 #[macro_use]
 extern crate human_panic;
 extern crate num_traits;
+extern crate serde_json;
 
 use std::collections::HashMap;
 use std::convert::TryInto;
@@ -22,6 +24,7 @@ use std::sync::RwLock;
 use libc::uintptr_t;
 use num_traits::{FromPrimitive, ToPrimitive};
 
+use deltachat::chat::ChatId;
 use deltachat::constants::DC_MSG_ID_LAST_SPECIAL;
 use deltachat::contact::Contact;
 use deltachat::context::Context;
@@ -54,7 +57,6 @@ use self::string::*;
 /// and protected by an [RwLock].  Other than that it needs to store
 /// the data which is passed into [dc_context_new].
 pub struct ContextWrapper {
-    cb: Option<dc_callback_t>,
     userdata: *mut libc::c_void,
     os_name: String,
     inner: RwLock<Option<context::Context>>,
@@ -88,7 +90,10 @@ impl ContextWrapper {
     ///
     /// This function makes it easy to log an error.
     unsafe fn error(&self, msg: &str) {
-        self.translate_cb(Event::Error(msg.to_string()));
+        self.with_inner(|ctx| {
+            ctx.call_cb(Event::Error(msg.to_string()));
+        })
+        .unwrap();
     }
 
     /// Unlock the context and execute a closure with it.
@@ -114,88 +119,19 @@ impl ContextWrapper {
         match guard.as_ref() {
             Some(ref ctx) => Ok(ctxfn(ctx)),
             None => {
-                unsafe { self.error("context not open") };
+                eprintln!("context not open");
                 Err(())
             }
         }
     }
 
-    /// Translates the callback from the rust style to the C-style version.
-    unsafe fn translate_cb(&self, event: Event) -> uintptr_t {
-        match self.cb {
-            Some(ffi_cb) => {
-                let event_id = event.as_id();
-                match event {
-                    Event::Info(msg)
-                    | Event::SmtpConnected(msg)
-                    | Event::ImapConnected(msg)
-                    | Event::SmtpMessageSent(msg)
-                    | Event::ImapMessageDeleted(msg)
-                    | Event::ImapMessageMoved(msg)
-                    | Event::ImapFolderEmptied(msg)
-                    | Event::NewBlobFile(msg)
-                    | Event::DeletedBlobFile(msg)
-                    | Event::Warning(msg)
-                    | Event::Error(msg)
-                    | Event::ErrorNetwork(msg)
-                    | Event::ErrorSelfNotInGroup(msg) => {
-                        let data2 = CString::new(msg).unwrap_or_default();
-                        ffi_cb(self, event_id, 0, data2.as_ptr() as uintptr_t)
-                    }
-                    Event::MsgsChanged { chat_id, msg_id }
-                    | Event::IncomingMsg { chat_id, msg_id }
-                    | Event::MsgDelivered { chat_id, msg_id }
-                    | Event::MsgFailed { chat_id, msg_id }
-                    | Event::MsgRead { chat_id, msg_id } => ffi_cb(
-                        self,
-                        event_id,
-                        chat_id as uintptr_t,
-                        msg_id.to_u32() as uintptr_t,
-                    ),
-                    Event::ChatModified(chat_id) => ffi_cb(self, event_id, chat_id as uintptr_t, 0),
-                    Event::ContactsChanged(id) | Event::LocationChanged(id) => {
-                        let id = id.unwrap_or_default();
-                        ffi_cb(self, event_id, id as uintptr_t, 0)
-                    }
-                    Event::ConfigureProgress(progress) | Event::ImexProgress(progress) => {
-                        ffi_cb(self, event_id, progress as uintptr_t, 0)
-                    }
-                    Event::ImexFileWritten(file) => {
-                        let data1 = file.to_c_string().unwrap_or_default();
-                        ffi_cb(self, event_id, data1.as_ptr() as uintptr_t, 0)
-                    }
-                    Event::SecurejoinInviterProgress {
-                        contact_id,
-                        progress,
-                    }
-                    | Event::SecurejoinJoinerProgress {
-                        contact_id,
-                        progress,
-                    } => ffi_cb(
-                        self,
-                        event_id,
-                        contact_id as uintptr_t,
-                        progress as uintptr_t,
-                    ),
-                    Event::SecurejoinMemberAdded {
-                        chat_id,
-                        contact_id,
-                    } => ffi_cb(
-                        self,
-                        event_id,
-                        chat_id as uintptr_t,
-                        contact_id as uintptr_t,
-                    ),
-                }
-            }
-            None => 0,
-        }
+    fn is_open(&self) -> bool {
+        self.inner.read().unwrap().is_some()
     }
 }
 
 #[no_mangle]
 pub unsafe extern "C" fn dc_context_new(
-    cb: Option<dc_callback_t>,
     userdata: *mut libc::c_void,
     os_name: *const libc::c_char,
 ) -> *mut dc_context_t {
@@ -207,11 +143,11 @@ pub unsafe extern "C" fn dc_context_new(
         to_string_lossy(os_name)
     };
     let ffi_ctx = ContextWrapper {
-        cb,
         userdata,
         os_name,
         inner: RwLock::new(None),
     };
+
     Box::into_raw(Box::new(ffi_ctx))
 }
 
@@ -249,17 +185,11 @@ pub unsafe extern "C" fn dc_open(
         return 0;
     }
     let ffi_context = &*context;
-    let rust_cb = move |_ctx: &Context, evt: Event| ffi_context.translate_cb(evt);
 
     let ctx = if blobdir.is_null() || *blobdir == 0 {
-        Context::new(
-            Box::new(rust_cb),
-            ffi_context.os_name.clone(),
-            as_path(dbfile).to_path_buf(),
-        )
+        Context::new(ffi_context.os_name.clone(), as_path(dbfile).to_path_buf())
     } else {
         Context::with_blobdir(
-            Box::new(rust_cb),
             ffi_context.os_name.clone(),
             as_path(dbfile).to_path_buf(),
             as_path(blobdir).to_path_buf(),
@@ -292,11 +222,7 @@ pub unsafe extern "C" fn dc_is_open(context: *mut dc_context_t) -> libc::c_int {
         return 0;
     }
     let ffi_context = &*context;
-    let inner_guard = ffi_context.inner.read().unwrap();
-    match *inner_guard {
-        Some(_) => 1,
-        None => 0,
-    }
+    ffi_context.is_open() as libc::c_int
 }
 
 #[no_mangle]
@@ -408,7 +334,7 @@ fn render_info(
 ) -> std::result::Result<String, std::fmt::Error> {
     let mut res = String::new();
     for (key, value) in &info {
-        write!(&mut res, "{}={}\n", key, value)?;
+        writeln!(&mut res, "{}={}", key, value)?;
     }
 
     Ok(res)
@@ -447,9 +373,7 @@ pub unsafe extern "C" fn dc_configure(context: *mut dc_context_t) {
         return;
     }
     let ffi_context = &*context;
-    ffi_context
-        .with_inner(|ctx| configure::configure(ctx))
-        .unwrap_or(())
+    ffi_context.with_inner(|ctx| ctx.configure()).unwrap_or(())
 }
 
 #[no_mangle]
@@ -460,188 +384,120 @@ pub unsafe extern "C" fn dc_is_configured(context: *mut dc_context_t) -> libc::c
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| configure::dc_is_configured(ctx) as libc::c_int)
+        .with_inner(|ctx| ctx.is_configured() as libc::c_int)
         .unwrap_or(0)
 }
 
 #[no_mangle]
-pub unsafe extern "C" fn dc_perform_imap_jobs(context: *mut dc_context_t) {
+pub unsafe extern "C" fn dc_context_run(context: *mut dc_context_t, cb: Option<dc_callback_t>) {
+    eprintln!("dc_context_run");
+
     if context.is_null() {
-        eprintln!("ignoring careless call to dc_perform_imap_jobs()");
+        eprintln!("ignoring careless call to dc_run()");
         return;
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| job::perform_inbox_jobs(ctx))
+        .with_inner(|ctx| {
+            eprintln!("RUN");
+            ctx.run(|_ctx, event| {
+                translate_cb(ffi_context, cb, event);
+            });
+            eprintln!("RUN DONE");
+        })
         .unwrap_or(())
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_perform_imap_fetch(context: *mut dc_context_t) {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_perform_imap_fetch()");
-        return;
+/// Translates the callback from the rust style to the C-style version.
+unsafe fn translate_cb(ctx: &ContextWrapper, ffi_cb: Option<dc_callback_t>, event: Event) {
+    if let Some(ffi_cb) = ffi_cb {
+        let event_id = event.as_id();
+        match event {
+            Event::Info(msg)
+            | Event::SmtpConnected(msg)
+            | Event::ImapConnected(msg)
+            | Event::SmtpMessageSent(msg)
+            | Event::ImapMessageDeleted(msg)
+            | Event::ImapMessageMoved(msg)
+            | Event::ImapFolderEmptied(msg)
+            | Event::NewBlobFile(msg)
+            | Event::DeletedBlobFile(msg)
+            | Event::Warning(msg)
+            | Event::Error(msg)
+            | Event::ErrorNetwork(msg)
+            | Event::ErrorSelfNotInGroup(msg) => {
+                let data2 = CString::new(msg).unwrap_or_default();
+                ffi_cb(ctx, event_id, 0, data2.as_ptr() as uintptr_t);
+            }
+            Event::MsgsChanged { chat_id, msg_id }
+            | Event::IncomingMsg { chat_id, msg_id }
+            | Event::MsgDelivered { chat_id, msg_id }
+            | Event::MsgFailed { chat_id, msg_id }
+            | Event::MsgRead { chat_id, msg_id } => {
+                ffi_cb(
+                    ctx,
+                    event_id,
+                    chat_id.to_u32() as uintptr_t,
+                    msg_id.to_u32() as uintptr_t,
+                );
+            }
+            Event::ChatModified(chat_id) => {
+                ffi_cb(ctx, event_id, chat_id.to_u32() as uintptr_t, 0);
+            }
+            Event::ContactsChanged(id) | Event::LocationChanged(id) => {
+                let id = id.unwrap_or_default();
+                ffi_cb(ctx, event_id, id as uintptr_t, 0);
+            }
+            Event::ConfigureProgress(progress) | Event::ImexProgress(progress) => {
+                ffi_cb(ctx, event_id, progress as uintptr_t, 0);
+            }
+            Event::ImexFileWritten(file) => {
+                let data1 = file.to_c_string().unwrap_or_default();
+                ffi_cb(ctx, event_id, data1.as_ptr() as uintptr_t, 0);
+            }
+            Event::SecurejoinInviterProgress {
+                contact_id,
+                progress,
+            }
+            | Event::SecurejoinJoinerProgress {
+                contact_id,
+                progress,
+            } => {
+                ffi_cb(
+                    ctx,
+                    event_id,
+                    contact_id as uintptr_t,
+                    progress as uintptr_t,
+                );
+            }
+            Event::SecurejoinMemberAdded {
+                chat_id,
+                contact_id,
+            } => {
+                ffi_cb(
+                    ctx,
+                    event_id,
+                    chat_id.to_u32() as uintptr_t,
+                    contact_id as uintptr_t,
+                );
+            }
+        }
     }
-    let ffi_context = &*context;
-    ffi_context
-        .with_inner(|ctx| job::perform_inbox_fetch(ctx))
-        .unwrap_or(())
 }
 
 #[no_mangle]
-pub unsafe extern "C" fn dc_perform_imap_idle(context: *mut dc_context_t) {
-    // TODO rename function in co-ordination with UIs
+pub unsafe extern "C" fn dc_context_shutdown(context: *mut dc_context_t) {
     if context.is_null() {
-        eprintln!("ignoring careless call to dc_perform_imap_idle()");
+        eprintln!("ignoring careless call to dc_shutdown()");
         return;
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| job::perform_inbox_idle(ctx))
-        .unwrap_or(())
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_interrupt_imap_idle(context: *mut dc_context_t) {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_interrupt_imap_idle()");
-        return;
-    }
-    let ffi_context = &*context;
-    ffi_context
-        .with_inner(|ctx| job::interrupt_inbox_idle(ctx, true))
-        .unwrap_or(())
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_perform_mvbox_fetch(context: *mut dc_context_t) {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_perform_mvbox_fetch()");
-        return;
-    }
-    let ffi_context = &*context;
-    ffi_context
-        .with_inner(|ctx| job::perform_mvbox_fetch(ctx))
-        .unwrap_or(())
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_perform_mvbox_jobs(context: *mut dc_context_t) {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_perform_mvbox_jobs()");
-        return;
-    }
-    let ffi_context = &*context;
-    ffi_context
-        .with_inner(|ctx| job::perform_mvbox_jobs(ctx))
-        .unwrap_or(())
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_perform_mvbox_idle(context: *mut dc_context_t) {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_perform_mvbox_idle()");
-        return;
-    }
-    let ffi_context = &*context;
-    ffi_context
-        .with_inner(|ctx| job::perform_mvbox_idle(ctx))
-        .unwrap_or(())
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_interrupt_mvbox_idle(context: *mut dc_context_t) {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_interrupt_mvbox_idle()");
-        return;
-    }
-    let ffi_context = &*context;
-    ffi_context
-        .with_inner(|ctx| job::interrupt_mvbox_idle(ctx))
-        .unwrap_or(())
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_perform_sentbox_fetch(context: *mut dc_context_t) {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_perform_sentbox_fetch()");
-        return;
-    }
-    let ffi_context = &*context;
-    ffi_context
-        .with_inner(|ctx| job::perform_sentbox_fetch(ctx))
-        .unwrap_or(())
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_perform_sentbox_jobs(context: *mut dc_context_t) {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_perform_sentbox_jobs()");
-        return;
-    }
-    let ffi_context = &*context;
-    ffi_context
-        .with_inner(|ctx| job::perform_sentbox_jobs(ctx))
-        .unwrap_or(())
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_perform_sentbox_idle(context: *mut dc_context_t) {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_perform_sentbox_idle()");
-        return;
-    }
-    let ffi_context = &*context;
-    ffi_context
-        .with_inner(|ctx| job::perform_sentbox_idle(ctx))
-        .unwrap_or(())
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_interrupt_sentbox_idle(context: *mut dc_context_t) {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_interrupt_sentbox_idle()");
-        return;
-    }
-    let ffi_context = &*context;
-    ffi_context
-        .with_inner(|ctx| job::interrupt_sentbox_idle(ctx))
-        .unwrap_or(())
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_perform_smtp_jobs(context: *mut dc_context_t) {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_perform_smtp_jobs()");
-        return;
-    }
-    let ffi_context = &*context;
-    ffi_context
-        .with_inner(|ctx| job::perform_smtp_jobs(ctx))
-        .unwrap_or(())
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_perform_smtp_idle(context: *mut dc_context_t) {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_perform_smtp_idle()");
-        return;
-    }
-    let ffi_context = &*context;
-    ffi_context
-        .with_inner(|ctx| job::perform_smtp_idle(ctx))
-        .unwrap_or(())
-}
-
-#[no_mangle]
-pub unsafe extern "C" fn dc_interrupt_smtp_idle(context: *mut dc_context_t) {
-    if context.is_null() {
-        eprintln!("ignoring careless call to dc_interrupt_smtp_idle()");
-        return;
-    }
-    let ffi_context = &*context;
-    ffi_context
-        .with_inner(|ctx| job::interrupt_smtp_idle(ctx))
+        .with_inner(|ctx| {
+            eprintln!("SHUTDOWN");
+            ctx.shutdown();
+            eprintln!("SHUTDOWN:DONE")
+        })
         .unwrap_or(())
 }
 
@@ -700,7 +556,9 @@ pub unsafe extern "C" fn dc_create_chat_by_msg_id(context: *mut dc_context_t, ms
     ffi_context
         .with_inner(|ctx| {
             chat::create_by_msg_id(ctx, MsgId::new(msg_id))
-                .unwrap_or_log_default(ctx, "Failed to create chat")
+                .log_err(ctx, "Failed to create chat from msg_id")
+                .map(|id| id.to_u32())
+                .unwrap_or(0)
         })
         .unwrap_or(0)
 }
@@ -718,7 +576,9 @@ pub unsafe extern "C" fn dc_create_chat_by_contact_id(
     ffi_context
         .with_inner(|ctx| {
             chat::create_by_contact_id(ctx, contact_id)
-                .unwrap_or_log_default(ctx, "Failed to create chat")
+                .log_err(ctx, "Failed to create chat from contact_id")
+                .map(|id| id.to_u32())
+                .unwrap_or(0)
         })
         .unwrap_or(0)
 }
@@ -734,7 +594,12 @@ pub unsafe extern "C" fn dc_get_chat_id_by_contact_id(
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| chat::get_by_contact_id(ctx, contact_id).unwrap_or(0))
+        .with_inner(|ctx| {
+            chat::get_by_contact_id(ctx, contact_id)
+                .log_err(ctx, "Failed to get chat for contact_id")
+                .map(|id| id.to_u32())
+                .unwrap_or(0)
+        })
         .unwrap_or(0)
 }
 
@@ -752,7 +617,7 @@ pub unsafe extern "C" fn dc_prepare_msg(
     let ffi_msg: &mut MessageWrapper = &mut *msg;
     ffi_context
         .with_inner(|ctx| {
-            chat::prepare_msg(ctx, chat_id, &mut ffi_msg.message)
+            chat::prepare_msg(ctx, ChatId::new(chat_id), &mut ffi_msg.message)
                 .unwrap_or_log_default(ctx, "Failed to prepare message")
         })
         .map(|msg_id| msg_id.to_u32())
@@ -773,7 +638,7 @@ pub unsafe extern "C" fn dc_send_msg(
     let ffi_msg = &mut *msg;
     ffi_context
         .with_inner(|ctx| {
-            chat::send_msg(ctx, chat_id, &mut ffi_msg.message)
+            chat::send_msg(ctx, ChatId::new(chat_id), &mut ffi_msg.message)
                 .unwrap_or_log_default(ctx, "Failed to send message")
         })
         .map(|msg_id| msg_id.to_u32())
@@ -794,7 +659,7 @@ pub unsafe extern "C" fn dc_send_text_msg(
     let text_to_send = to_string_lossy(text_to_send);
     ffi_context
         .with_inner(|ctx| {
-            chat::send_text_msg(ctx, chat_id, text_to_send)
+            chat::send_text_msg(ctx, ChatId::new(chat_id), text_to_send)
                 .map(|msg_id| msg_id.to_u32())
                 .unwrap_or_log_default(ctx, "Failed to send text message")
         })
@@ -819,7 +684,7 @@ pub unsafe extern "C" fn dc_set_draft(
         Some(&mut ffi_msg.message)
     };
     ffi_context
-        .with_inner(|ctx| chat::set_draft(ctx, chat_id, msg))
+        .with_inner(|ctx| ChatId::new(chat_id).set_draft(ctx, msg))
         .unwrap_or(())
 }
 
@@ -894,7 +759,7 @@ pub unsafe extern "C" fn dc_get_draft(context: *mut dc_context_t, chat_id: u32)
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| match chat::get_draft(ctx, chat_id) {
+        .with_inner(|ctx| match ChatId::new(chat_id).get_draft(ctx) {
             Ok(Some(draft)) => {
                 let ffi_msg = MessageWrapper {
                     context,
@@ -931,7 +796,7 @@ pub unsafe extern "C" fn dc_get_chat_msgs(
     ffi_context
         .with_inner(|ctx| {
             let arr = dc_array_t::from(
-                chat::get_chat_msgs(ctx, chat_id, flags, marker_flag)
+                chat::get_chat_msgs(ctx, ChatId::new(chat_id), flags, marker_flag)
                     .iter()
                     .map(|msg_id| msg_id.to_u32())
                     .collect::<Vec<u32>>(),
@@ -949,7 +814,7 @@ pub unsafe extern "C" fn dc_get_msg_cnt(context: *mut dc_context_t, chat_id: u32
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| chat::get_msg_cnt(ctx, chat_id) as libc::c_int)
+        .with_inner(|ctx| ChatId::new(chat_id).get_msg_cnt(ctx) as libc::c_int)
         .unwrap_or(0)
 }
 
@@ -964,7 +829,7 @@ pub unsafe extern "C" fn dc_get_fresh_msg_cnt(
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| chat::get_fresh_msg_cnt(ctx, chat_id) as libc::c_int)
+        .with_inner(|ctx| ChatId::new(chat_id).get_fresh_msg_cnt(ctx) as libc::c_int)
         .unwrap_or(0)
 }
 
@@ -999,7 +864,9 @@ pub unsafe extern "C" fn dc_marknoticed_chat(context: *mut dc_context_t, chat_id
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            chat::marknoticed_chat(ctx, chat_id).log_err(ctx, "Failed marknoticed chat")
+            chat::marknoticed_chat(ctx, ChatId::new(chat_id))
+                .log_err(ctx, "Failed marknoticed chat")
+                .unwrap_or(())
         })
         .unwrap_or(())
 }
@@ -1013,7 +880,9 @@ pub unsafe extern "C" fn dc_marknoticed_all_chats(context: *mut dc_context_t) {
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            chat::marknoticed_all_chats(ctx).log_err(ctx, "Failed marknoticed all chats")
+            chat::marknoticed_all_chats(ctx)
+                .log_err(ctx, "Failed marknoticed all chats")
+                .unwrap_or(())
         })
         .unwrap_or(())
 }
@@ -1047,10 +916,16 @@ pub unsafe extern "C" fn dc_get_chat_media(
     ffi_context
         .with_inner(|ctx| {
             let arr = dc_array_t::from(
-                chat::get_chat_media(ctx, chat_id, msg_type, or_msg_type2, or_msg_type3)
-                    .iter()
-                    .map(|msg_id| msg_id.to_u32())
-                    .collect::<Vec<u32>>(),
+                chat::get_chat_media(
+                    ctx,
+                    ChatId::new(chat_id),
+                    msg_type,
+                    or_msg_type2,
+                    or_msg_type3,
+                )
+                .iter()
+                .map(|msg_id| msg_id.to_u32())
+                .collect::<Vec<u32>>(),
             );
             Box::into_raw(Box::new(arr))
         })
@@ -1117,7 +992,12 @@ pub unsafe extern "C" fn dc_archive_chat(
         return;
     };
     ffi_context
-        .with_inner(|ctx| chat::archive(ctx, chat_id, archive).log_err(ctx, "Failed archive chat"))
+        .with_inner(|ctx| {
+            ChatId::new(chat_id)
+                .set_archived(ctx, archive)
+                .log_err(ctx, "Failed archive chat")
+                .unwrap_or(())
+        })
         .unwrap_or(())
 }
 
@@ -1129,7 +1009,12 @@ pub unsafe extern "C" fn dc_delete_chat(context: *mut dc_context_t, chat_id: u32
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| chat::delete(ctx, chat_id).log_err(ctx, "Failed chat delete"))
+        .with_inner(|ctx| {
+            ChatId::new(chat_id)
+                .delete(ctx)
+                .log_err(ctx, "Failed chat delete")
+                .unwrap_or(())
+        })
         .unwrap_or(())
 }
 
@@ -1145,7 +1030,7 @@ pub unsafe extern "C" fn dc_get_chat_contacts(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            let arr = dc_array_t::from(chat::get_chat_contacts(ctx, chat_id));
+            let arr = dc_array_t::from(chat::get_chat_contacts(ctx, ChatId::new(chat_id)));
             Box::into_raw(Box::new(arr))
         })
         .unwrap_or_else(|_| ptr::null_mut())
@@ -1165,7 +1050,7 @@ pub unsafe extern "C" fn dc_search_msgs(
     ffi_context
         .with_inner(|ctx| {
             let arr = dc_array_t::from(
-                ctx.search_msgs(chat_id, to_string_lossy(query))
+                ctx.search_msgs(ChatId::new(chat_id), to_string_lossy(query))
                     .iter()
                     .map(|msg_id| msg_id.to_u32())
                     .collect::<Vec<u32>>(),
@@ -1183,13 +1068,15 @@ pub unsafe extern "C" fn dc_get_chat(context: *mut dc_context_t, chat_id: u32) -
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| match chat::Chat::load_from_db(ctx, chat_id) {
-            Ok(chat) => {
-                let ffi_chat = ChatWrapper { context, chat };
-                Box::into_raw(Box::new(ffi_chat))
-            }
-            Err(_) => ptr::null_mut(),
-        })
+        .with_inner(
+            |ctx| match chat::Chat::load_from_db(ctx, ChatId::new(chat_id)) {
+                Ok(chat) => {
+                    let ffi_chat = ChatWrapper { context, chat };
+                    Box::into_raw(Box::new(ffi_chat))
+                }
+                Err(_) => ptr::null_mut(),
+            },
+        )
         .unwrap_or_else(|_| ptr::null_mut())
 }
 
@@ -1212,7 +1099,9 @@ pub unsafe extern "C" fn dc_create_group_chat(
     ffi_context
         .with_inner(|ctx| {
             chat::create_group_chat(ctx, verified, to_string_lossy(name))
-                .unwrap_or_log_default(ctx, "Failed to create group chat")
+                .log_err(ctx, "Failed to create group chat")
+                .map(|id| id.to_u32())
+                .unwrap_or(0)
         })
         .unwrap_or(0)
 }
@@ -1229,7 +1118,7 @@ pub unsafe extern "C" fn dc_is_contact_in_chat(
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| chat::is_contact_in_chat(ctx, chat_id, contact_id))
+        .with_inner(|ctx| chat::is_contact_in_chat(ctx, ChatId::new(chat_id), contact_id))
         .unwrap_or_default()
         .into()
 }
@@ -1246,7 +1135,9 @@ pub unsafe extern "C" fn dc_add_contact_to_chat(
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| chat::add_contact_to_chat(ctx, chat_id, contact_id) as libc::c_int)
+        .with_inner(|ctx| {
+            chat::add_contact_to_chat(ctx, ChatId::new(chat_id), contact_id) as libc::c_int
+        })
         .unwrap_or(0)
 }
 
@@ -1263,7 +1154,7 @@ pub unsafe extern "C" fn dc_remove_contact_from_chat(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            chat::remove_contact_from_chat(ctx, chat_id, contact_id)
+            chat::remove_contact_from_chat(ctx, ChatId::new(chat_id), contact_id)
                 .map(|_| 1)
                 .unwrap_or_log_default(ctx, "Failed to remove contact")
         })
@@ -1283,7 +1174,7 @@ pub unsafe extern "C" fn dc_set_chat_name(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            chat::set_chat_name(ctx, chat_id, to_string_lossy(name))
+            chat::set_chat_name(ctx, ChatId::new(chat_id), to_string_lossy(name))
                 .map(|_| 1)
                 .unwrap_or_log_default(ctx, "Failed to set chat name")
         })
@@ -1303,7 +1194,7 @@ pub unsafe extern "C" fn dc_set_chat_profile_image(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            chat::set_chat_profile_image(ctx, chat_id, to_string_lossy(image))
+            chat::set_chat_profile_image(ctx, ChatId::new(chat_id), to_string_lossy(image))
                 .map(|_| 1)
                 .unwrap_or_log_default(ctx, "Failed to set profile image")
         })
@@ -1339,7 +1230,7 @@ pub unsafe extern "C" fn dc_get_mime_headers(
         .with_inner(|ctx| {
             message::get_mime_headers(ctx, MsgId::new(msg_id))
                 .map(|s| s.strdup())
-                .unwrap_or_else(|| ptr::null_mut())
+                .unwrap_or_else(ptr::null_mut)
         })
         .unwrap_or_else(|_| ptr::null_mut())
 }
@@ -1392,7 +1283,7 @@ pub unsafe extern "C" fn dc_forward_msgs(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            chat::forward_msgs(ctx, &msg_ids[..], chat_id)
+            chat::forward_msgs(ctx, &msg_ids[..], ChatId::new(chat_id))
                 .unwrap_or_log_default(ctx, "Failed to forward message")
         })
         .unwrap_or_default()
@@ -1805,8 +1696,8 @@ pub unsafe extern "C" fn dc_get_securejoin_qr(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            securejoin::dc_get_securejoin_qr(ctx, chat_id)
-                .unwrap_or("".to_string())
+            securejoin::dc_get_securejoin_qr(ctx, ChatId::new(chat_id))
+                .unwrap_or_else(|| "".to_string())
                 .strdup()
         })
         .unwrap_or_else(|_| "".strdup())
@@ -1823,7 +1714,7 @@ pub unsafe extern "C" fn dc_join_securejoin(
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| securejoin::dc_join_securejoin(ctx, &to_string_lossy(qr)))
+        .with_inner(|ctx| securejoin::dc_join_securejoin(ctx, &to_string_lossy(qr)).to_u32())
         .unwrap_or(0)
 }
 
@@ -1839,7 +1730,9 @@ pub unsafe extern "C" fn dc_send_locations_to_chat(
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| location::send_locations_to_chat(ctx, chat_id, seconds as i64))
+        .with_inner(|ctx| {
+            location::send_locations_to_chat(ctx, ChatId::new(chat_id), seconds as i64)
+        })
         .ok();
 }
 
@@ -1854,7 +1747,9 @@ pub unsafe extern "C" fn dc_is_sending_locations_to_chat(
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| location::is_sending_locations_to_chat(ctx, chat_id) as libc::c_int)
+        .with_inner(|ctx| {
+            location::is_sending_locations_to_chat(ctx, ChatId::new(chat_id)) as libc::c_int
+        })
         .unwrap_or(0)
 }
 
@@ -1892,7 +1787,7 @@ pub unsafe extern "C" fn dc_get_locations(
         .with_inner(|ctx| {
             let res = location::get_range(
                 ctx,
-                chat_id,
+                ChatId::new(chat_id),
                 contact_id,
                 timestamp_begin as i64,
                 timestamp_end as i64,
@@ -2014,8 +1909,7 @@ pub unsafe extern "C" fn dc_array_get_chat_id(
         eprintln!("ignoring careless call to dc_array_get_chat_id()");
         return 0;
     }
-
-    (*array).get_location(index).chat_id
+    (*array).get_location(index).chat_id.to_u32()
 }
 #[no_mangle]
 pub unsafe extern "C" fn dc_array_get_contact_id(
@@ -2152,7 +2046,7 @@ pub unsafe extern "C" fn dc_chatlist_get_chat_id(
         return 0;
     }
     let ffi_list = &*chatlist;
-    ffi_list.list.get_chat_id(index as usize)
+    ffi_list.list.get_chat_id(index as usize).to_u32()
 }
 
 #[no_mangle]
@@ -2244,7 +2138,7 @@ pub unsafe extern "C" fn dc_chat_get_id(chat: *mut dc_chat_t) -> u32 {
         return 0;
     }
     let ffi_chat = &*chat;
-    ffi_chat.chat.get_id()
+    ffi_chat.chat.get_id().to_u32()
 }
 
 #[no_mangle]
@@ -2390,12 +2284,27 @@ pub unsafe extern "C" fn dc_chat_get_info_json(
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| match chat::get_info_json(ctx, chat_id) {
-            Ok(s) => s.strdup(),
-            Err(err) => {
-                error!(ctx, "get_info_json({}) returned: {}", chat_id, err);
-                return "".strdup();
-            }
+        .with_inner(|ctx| {
+            let chat = match chat::Chat::load_from_db(ctx, ChatId::new(chat_id)) {
+                Ok(chat) => chat,
+                Err(err) => {
+                    error!(ctx, "dc_get_chat_info_json() failed to load chat: {}", err);
+                    return "".strdup();
+                }
+            };
+            let info = match chat.get_info(ctx) {
+                Ok(info) => info,
+                Err(err) => {
+                    error!(
+                        ctx,
+                        "dc_get_chat_info_json() failed to get chat info: {}", err
+                    );
+                    return "".strdup();
+                }
+            };
+            serde_json::to_string(&info)
+                .unwrap_or_log_default(ctx, "dc_get_chat_info_json() failed to serialise to json")
+                .strdup()
         })
         .unwrap_or_else(|_| "".strdup())
 }
@@ -2472,7 +2381,7 @@ pub unsafe extern "C" fn dc_msg_get_chat_id(msg: *mut dc_msg_t) -> u32 {
         return 0;
     }
     let ffi_msg = &*msg;
-    ffi_msg.message.get_chat_id()
+    ffi_msg.message.get_chat_id().to_u32()
 }
 
 #[no_mangle]
@@ -2579,7 +2488,7 @@ pub unsafe extern "C" fn dc_msg_get_filemime(msg: *mut dc_msg_t) -> *mut libc::c
     if let Some(x) = ffi_msg.message.get_filemime() {
         x.strdup()
     } else {
-        return dc_strdup(ptr::null());
+        dc_strdup(ptr::null())
     }
 }
 
@@ -2971,7 +2880,7 @@ pub unsafe extern "C" fn dc_contact_get_profile_image(
                 .contact
                 .get_profile_image(ctx)
                 .map(|p| p.to_string_lossy().strdup())
-                .unwrap_or_else(|| std::ptr::null_mut())
+                .unwrap_or_else(std::ptr::null_mut)
         })
         .unwrap_or_else(|_| ptr::null_mut())
 }
@@ -3097,12 +3006,12 @@ pub unsafe extern "C" fn dc_str_unref(s: *mut libc::c_char) {
 
 pub mod providers;
 
-pub trait ResultExt<T> {
+pub trait ResultExt<T, E> {
     fn unwrap_or_log_default(self, context: &context::Context, message: &str) -> T;
-    fn log_err(&self, context: &context::Context, message: &str);
+    fn log_err(self, context: &context::Context, message: &str) -> Result<T, E>;
 }
 
-impl<T: Default, E: std::fmt::Display> ResultExt<T> for Result<T, E> {
+impl<T: Default, E: std::fmt::Display> ResultExt<T, E> for Result<T, E> {
     fn unwrap_or_log_default(self, context: &context::Context, message: &str) -> T {
         match self {
             Ok(t) => t,
@@ -3113,10 +3022,11 @@ impl<T: Default, E: std::fmt::Display> ResultExt<T> for Result<T, E> {
         }
     }
 
-    fn log_err(&self, context: &context::Context, message: &str) {
-        if let Err(err) = self {
-            error!(context, "{}: {}", message, err);
-        }
+    fn log_err(self, context: &context::Context, message: &str) -> Result<T, E> {
+        self.map_err(|err| {
+            warn!(context, "{}: {}", message, err);
+            err
+        })
     }
 }
 

deltachat-ffi/src/providers.rs

@@ -86,8 +86,6 @@ pub unsafe extern "C" fn dc_provider_get_status(provider: *const dc_provider_t)
 }
 
 #[no_mangle]
-pub unsafe extern "C" fn dc_provider_unref(_provider: *const dc_provider_t) {
-    ()
-}
+pub unsafe extern "C" fn dc_provider_unref(_provider: *const dc_provider_t) {}
 
 // TODO expose general provider overview url?

deltachat_derive/Cargo.toml

@@ -1,12 +1,13 @@
 [package]
 name = "deltachat_derive"
-version = "0.1.0"
-authors = ["Dmitry Bogatov <KAction@debian.org>"]
+version = "2.0.0"
+authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
+license = "MPL-2.0"
 
 [lib]
 proc-macro = true
 
 [dependencies]
-syn = "0.14.4"
-quote = "0.6.3"
+syn = "1.0.13"
+quote = "1.0.2"

examples/repl/cmdline.rs

@@ -1,7 +1,7 @@
 use std::path::Path;
 use std::str::FromStr;
 
-use deltachat::chat::{self, Chat};
+use deltachat::chat::{self, Chat, ChatId};
 use deltachat::chatlist::*;
 use deltachat::config;
 use deltachat::constants::*;
@@ -20,14 +20,14 @@ use deltachat::qr::*;
 use deltachat::sql;
 use deltachat::Event;
 
-/// Reset database tables. This function is called from Core cmdline.
+/// Reset database tables.
 /// Argument is a bitmask, executing single or multiple actions in one call.
 /// e.g. bitmask 7 triggers actions definded with bits 1, 2 and 4.
-pub fn dc_reset_tables(context: &Context, bits: i32) -> i32 {
-    info!(context, "Resetting tables ({})...", bits);
+fn dc_reset_tables(context: &Context, bits: i32) -> i32 {
+    println!("Resetting tables ({})...", bits);
     if 0 != bits & 1 {
         sql::execute(context, &context.sql, "DELETE FROM jobs;", params![]).unwrap();
-        info!(context, "(1) Jobs reset.");
+        println!("(1) Jobs reset.");
     }
     if 0 != bits & 2 {
         sql::execute(
@@ -37,11 +37,11 @@ pub fn dc_reset_tables(context: &Context, bits: i32) -> i32 {
             params![],
         )
         .unwrap();
-        info!(context, "(2) Peerstates reset.");
+        println!("(2) Peerstates reset.");
     }
     if 0 != bits & 4 {
         sql::execute(context, &context.sql, "DELETE FROM keypairs;", params![]).unwrap();
-        info!(context, "(4) Private keypairs reset.");
+        println!("(4) Private keypairs reset.");
     }
     if 0 != bits & 8 {
         sql::execute(
@@ -80,11 +80,11 @@ pub fn dc_reset_tables(context: &Context, bits: i32) -> i32 {
         )
         .unwrap();
         sql::execute(context, &context.sql, "DELETE FROM leftgrps;", params![]).unwrap();
-        info!(context, "(8) Rest but server config reset.");
+        println!("(8) Rest but server config reset.");
     }
 
     context.call_cb(Event::MsgsChanged {
-        chat_id: 0,
+        chat_id: ChatId::new(0),
         msg_id: MsgId::new(0),
     });
 
@@ -155,7 +155,7 @@ fn poke_spec(context: &Context, spec: Option<&str>) -> libc::c_int {
                 let name = name_f.to_string_lossy();
                 if name.ends_with(".eml") {
                     let path_plus_name = format!("{}/{}", &real_spec, name);
-                    info!(context, "Import: {}", path_plus_name);
+                    println!("Import: {}", path_plus_name);
                     if dc_poke_eml_file(context, path_plus_name).is_ok() {
                         read_cnt += 1
                     }
@@ -163,13 +163,10 @@ fn poke_spec(context: &Context, spec: Option<&str>) -> libc::c_int {
             }
         }
     }
-    info!(
-        context,
-        "Import: {} items read from \"{}\".", read_cnt, &real_spec
-    );
+    println!("Import: {} items read from \"{}\".", read_cnt, &real_spec);
     if read_cnt > 0 {
         context.call_cb(Event::MsgsChanged {
-            chat_id: 0,
+            chat_id: ChatId::new(0),
             msg_id: MsgId::new(0),
         });
     }
@@ -190,8 +187,7 @@ fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
     };
     let temp2 = dc_timestamp_to_str(msg.get_timestamp());
     let msgtext = msg.get_text();
-    info!(
-        context,
+    println!(
         "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{} [{}]",
         prefix.as_ref(),
         msg.get_id(),
@@ -225,16 +221,14 @@ fn log_msglist(context: &Context, msglist: &Vec<MsgId>) -> Result<(), Error> {
     let mut lines_out = 0;
     for &msg_id in msglist {
         if msg_id.is_daymarker() {
-            info!(
-                context,
+            println!(
                 "--------------------------------------------------------------------------------"
             );
 
             lines_out += 1
         } else if !msg_id.is_special() {
             if lines_out == 0 {
-                info!(
-                    context,
+                println!(
                     "--------------------------------------------------------------------------------",
                 );
                 lines_out += 1
@@ -244,8 +238,7 @@ fn log_msglist(context: &Context, msglist: &Vec<MsgId>) -> Result<(), Error> {
         }
     }
     if lines_out > 0 {
-        info!(
-            context,
+        println!(
             "--------------------------------------------------------------------------------"
         );
     }
@@ -295,7 +288,7 @@ fn log_contactlist(context: &Context, contacts: &Vec<u32>) {
                 );
             }
 
-            info!(context, "Contact#{}: {}{}", contact_id, line, line2);
+            println!("Contact#{}: {}{}", contact_id, line, line2);
         }
     }
 }
@@ -306,7 +299,7 @@ fn chat_prefix(chat: &Chat) -> &'static str {
 
 pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
     let chat_id = *context.cmdline_sel_chat_id.read().unwrap();
-    let mut sel_chat = if chat_id > 0 {
+    let mut sel_chat = if !chat_id.is_unset() {
         Chat::load_from_db(context, chat_id).ok()
     } else {
         None
@@ -491,7 +484,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
             println!("{:#?}", context.get_info());
         }
         "interrupt" => {
-            interrupt_inbox_idle(context, true);
+            interrupt_inbox_idle(context);
         }
         "maybenetwork" => {
             maybe_network(context);
@@ -510,20 +503,18 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
 
             let cnt = chatlist.len();
             if cnt > 0 {
-                info!(
-                    context,
+                println!(
                     "================================================================================"
                 );
 
                 for i in (0..cnt).rev() {
                     let chat = Chat::load_from_db(context, chatlist.get_chat_id(i))?;
-                    info!(
-                        context,
+                    println!(
                         "{}#{}: {} [{} fresh]",
                         chat_prefix(&chat),
                         chat.get_id(),
                         chat.get_name(),
-                        chat::get_fresh_msg_cnt(context, chat.get_id()),
+                        chat.get_id().get_fresh_msg_cnt(context),
                     );
                     let lot = chatlist.get_summary(context, i, Some(&chat));
                     let statestr = if chat.is_archived() {
@@ -540,8 +531,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
                     let timestr = dc_timestamp_to_str(lot.get_timestamp());
                     let text1 = lot.get_text1();
                     let text2 = lot.get_text2();
-                    info!(
-                        context,
+                    println!(
                         "{}{}{}{} [{}]{}",
                         text1.unwrap_or(""),
                         if text1.is_some() { ": " } else { "" },
@@ -554,14 +544,13 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
                             ""
                         },
                     );
-                    info!(
-                        context,
+                    println!(
                         "================================================================================"
                     );
                 }
             }
-            if location::is_sending_locations_to_chat(context, 0) {
-                info!(context, "Location streaming enabled.");
+            if location::is_sending_locations_to_chat(context, ChatId::new(0)) {
+                println!("Location streaming enabled.");
             }
             println!("{} chats", cnt);
         }
@@ -570,8 +559,8 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
                 bail!("Argument [chat-id] is missing.");
             }
             if !arg1.is_empty() {
-                let chat_id = arg1.parse()?;
-                println!("Selecting chat #{}", chat_id);
+                let chat_id = ChatId::new(arg1.parse()?);
+                println!("Selecting chat {}", chat_id);
                 sel_chat = Some(Chat::load_from_db(context, chat_id)?);
                 *context.cmdline_sel_chat_id.write().unwrap() = chat_id;
             }
@@ -589,8 +578,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
             } else {
                 format!("{} member(s)", members.len())
             };
-            info!(
-                context,
+            println!(
                 "{}#{}: {} [{}]{}{}",
                 chat_prefix(sel_chat),
                 sel_chat.get_id(),
@@ -610,14 +598,11 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
                 },
             );
             log_msglist(context, &msglist)?;
-            if let Some(draft) = chat::get_draft(context, sel_chat.get_id())? {
+            if let Some(draft) = sel_chat.get_id().get_draft(context)? {
                 log_msg(context, "Draft", &draft);
             }
 
-            println!(
-                "{} messages.",
-                chat::get_msg_cnt(context, sel_chat.get_id())
-            );
+            println!("{} messages.", sel_chat.get_id().get_msg_cnt(context));
             chat::marknoticed_chat(context, sel_chat.get_id())?;
         }
         "createchat" => {
@@ -693,7 +678,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
             ensure!(sel_chat.is_some(), "No chat selected.");
 
             let contacts = chat::get_chat_contacts(context, sel_chat.as_ref().unwrap().get_id());
-            info!(context, "Memberlist:");
+            println!("Memberlist:");
 
             log_contactlist(context, &contacts);
             println!(
@@ -719,8 +704,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
             let default_marker = "-".to_string();
             for location in &locations {
                 let marker = location.marker.as_ref().unwrap_or(&default_marker);
-                info!(
-                    context,
+                println!(
                     "Loc#{}: {}: lat={} lng={} acc={} Chat#{} Contact#{} {} {}",
                     location.location_id,
                     dc_timestamp_to_str(location.timestamp),
@@ -734,7 +718,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
                 );
             }
             if locations.is_empty() {
-                info!(context, "No locations.");
+                println!("No locations.");
             }
         }
         "sendlocations" => {
@@ -800,7 +784,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
             let chat = if let Some(ref sel_chat) = sel_chat {
                 sel_chat.get_id()
             } else {
-                0 as libc::c_uint
+                ChatId::new(0)
             };
 
             let msglist = context.search_msgs(chat, arg1);
@@ -814,14 +798,14 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
             if !arg1.is_empty() {
                 let mut draft = Message::new(Viewtype::Text);
                 draft.set_text(Some(arg1.to_string()));
-                chat::set_draft(
-                    context,
-                    sel_chat.as_ref().unwrap().get_id(),
-                    Some(&mut draft),
-                );
+                sel_chat
+                    .as_ref()
+                    .unwrap()
+                    .get_id()
+                    .set_draft(context, Some(&mut draft));
                 println!("Draft saved.");
             } else {
-                chat::set_draft(context, sel_chat.as_ref().unwrap().get_id(), None);
+                sel_chat.as_ref().unwrap().get_id().set_draft(context, None);
                 println!("Draft deleted.");
             }
         }
@@ -859,13 +843,13 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
         }
         "archive" | "unarchive" => {
             ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
-            let chat_id = arg1.parse()?;
-            chat::archive(context, chat_id, arg0 == "archive")?;
+            let chat_id = ChatId::new(arg1.parse()?);
+            chat_id.set_archived(context, arg0 == "archive")?;
         }
         "delchat" => {
             ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
-            let chat_id = arg1.parse()?;
-            chat::delete(context, chat_id)?;
+            let chat_id = ChatId::new(arg1.parse()?);
+            chat_id.delete(context)?;
         }
         "msginfo" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
@@ -886,7 +870,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
             );
 
             let mut msg_ids = [MsgId::new(0); 1];
-            let chat_id = arg2.parse()?;
+            let chat_id = ChatId::new(arg2.parse()?);
             msg_ids[0] = MsgId::new(arg1.parse()?);
             chat::forward_msgs(context, &msg_ids, chat_id)?;
         }

examples/repl/main.rs

@@ -9,21 +9,17 @@ extern crate deltachat;
 #[macro_use]
 extern crate failure;
 #[macro_use]
-extern crate lazy_static;
-#[macro_use]
 extern crate rusqlite;
 
 use std::borrow::Cow::{self, Borrowed, Owned};
 use std::io::{self, Write};
 use std::path::Path;
 use std::process::Command;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::{Arc, Mutex, RwLock};
+use std::sync::{Arc, RwLock};
 
+use deltachat::chat::ChatId;
 use deltachat::config;
-use deltachat::configure::*;
 use deltachat::context::*;
-use deltachat::job::*;
 use deltachat::oauth2::*;
 use deltachat::securejoin::*;
 use deltachat::Event;
@@ -41,7 +37,7 @@ use self::cmdline::*;
 
 // Event Handler
 
-fn receive_event(_context: &Context, event: Event) -> libc::uintptr_t {
+fn receive_event(_context: &Context, event: Event) {
     match event {
         Event::Info(msg) => {
             /* do not show the event as this would fill the screen */
@@ -111,111 +107,8 @@ fn receive_event(_context: &Context, event: Event) -> libc::uintptr_t {
             print!("\x1b[33m{{Received {:?}}}\n\x1b[0m", event);
         }
     }
-
-    0
 }
 
-// Threads for waiting for messages and for jobs
-
-lazy_static! {
-    static ref HANDLE: Arc<Mutex<Option<Handle>>> = Arc::new(Mutex::new(None));
-    static ref IS_RUNNING: AtomicBool = AtomicBool::new(true);
-}
-
-struct Handle {
-    handle_imap: Option<std::thread::JoinHandle<()>>,
-    handle_mvbox: Option<std::thread::JoinHandle<()>>,
-    handle_sentbox: Option<std::thread::JoinHandle<()>>,
-    handle_smtp: Option<std::thread::JoinHandle<()>>,
-}
-
-macro_rules! while_running {
-    ($code:block) => {
-        if IS_RUNNING.load(Ordering::Relaxed) {
-            $code
-        } else {
-            break;
-        }
-    };
-}
-
-fn start_threads(c: Arc<RwLock<Context>>) {
-    if HANDLE.clone().lock().unwrap().is_some() {
-        return;
-    }
-
-    println!("Starting threads");
-    IS_RUNNING.store(true, Ordering::Relaxed);
-
-    let ctx = c.clone();
-    let handle_imap = std::thread::spawn(move || loop {
-        while_running!({
-            perform_inbox_jobs(&ctx.read().unwrap());
-            perform_inbox_fetch(&ctx.read().unwrap());
-            while_running!({
-                let context = ctx.read().unwrap();
-                perform_inbox_idle(&context);
-            });
-        });
-    });
-
-    let ctx = c.clone();
-    let handle_mvbox = std::thread::spawn(move || loop {
-        while_running!({
-            perform_mvbox_fetch(&ctx.read().unwrap());
-            while_running!({
-                perform_mvbox_idle(&ctx.read().unwrap());
-            });
-        });
-    });
-
-    let ctx = c.clone();
-    let handle_sentbox = std::thread::spawn(move || loop {
-        while_running!({
-            perform_sentbox_fetch(&ctx.read().unwrap());
-            while_running!({
-                perform_sentbox_idle(&ctx.read().unwrap());
-            });
-        });
-    });
-
-    let ctx = c;
-    let handle_smtp = std::thread::spawn(move || loop {
-        while_running!({
-            perform_smtp_jobs(&ctx.read().unwrap());
-            while_running!({
-                perform_smtp_idle(&ctx.read().unwrap());
-            });
-        });
-    });
-
-    *HANDLE.clone().lock().unwrap() = Some(Handle {
-        handle_imap: Some(handle_imap),
-        handle_mvbox: Some(handle_mvbox),
-        handle_sentbox: Some(handle_sentbox),
-        handle_smtp: Some(handle_smtp),
-    });
-}
-
-fn stop_threads(context: &Context) {
-    if let Some(ref mut handle) = *HANDLE.clone().lock().unwrap() {
-        println!("Stopping threads");
-        IS_RUNNING.store(false, Ordering::Relaxed);
-
-        interrupt_inbox_idle(context, true);
-        interrupt_mvbox_idle(context);
-        interrupt_sentbox_idle(context);
-        interrupt_smtp_idle(context);
-
-        handle.handle_imap.take().unwrap().join().unwrap();
-        handle.handle_mvbox.take().unwrap().join().unwrap();
-        handle.handle_sentbox.take().unwrap().join().unwrap();
-        handle.handle_smtp.take().unwrap().join().unwrap();
-    }
-}
-
-// === The main loop
-
 struct DcHelper {
     completer: FilenameCompleter,
     highlighter: MatchingBracketHighlighter,
@@ -421,9 +314,7 @@ fn main_0(args: Vec<String>) -> Result<(), failure::Error> {
     }
     rl.save_history(".dc-history.txt")?;
     println!("history saved");
-    {
-        stop_threads(&ctx.read().unwrap());
-    }
+    ctx.read().unwrap().shutdown();
 
     Ok(())
 }
@@ -441,28 +332,20 @@ fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, failu
 
     match arg0 {
         "connect" => {
-            start_threads(ctx);
+            crossbeam::scope(|s| {
+                s.spawn(|_| ctx.read().unwrap().run());
+            })
+            .unwrap();
         }
         "disconnect" => {
-            stop_threads(&ctx.read().unwrap());
-        }
-        "smtp-jobs" => {
-            if HANDLE.clone().lock().unwrap().is_some() {
-                println!("smtp-jobs are already running in a thread.",);
-            } else {
-                perform_smtp_jobs(&ctx.read().unwrap());
-            }
-        }
-        "imap-jobs" => {
-            if HANDLE.clone().lock().unwrap().is_some() {
-                println!("inbox-jobs are already running in a thread.");
-            } else {
-                perform_inbox_jobs(&ctx.read().unwrap());
-            }
+            ctx.read().unwrap().shutdown();
         }
         "configure" => {
-            start_threads(ctx.clone());
-            configure(&ctx.read().unwrap());
+            crossbeam::scope(|s| {
+                s.spawn(|_| ctx.read().unwrap().run());
+            })
+            .unwrap();
+            ctx.read().unwrap().configure();
         }
         "oauth2" => {
             if let Some(addr) = ctx.read().unwrap().get_config(config::Config::Addr) {
@@ -485,10 +368,15 @@ fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, failu
             print!("\x1b[1;1H\x1b[2J");
         }
         "getqr" | "getbadqr" => {
-            start_threads(ctx.clone());
-            if let Some(mut qr) =
-                dc_get_securejoin_qr(&ctx.read().unwrap(), arg1.parse().unwrap_or_default())
-            {
+            crossbeam::scope(|s| {
+                s.spawn(|_| ctx.read().unwrap().run());
+            })
+            .unwrap();
+
+            if let Some(mut qr) = dc_get_securejoin_qr(
+                &ctx.read().unwrap(),
+                ChatId::new(arg1.parse().unwrap_or_default()),
+            ) {
                 if !qr.is_empty() {
                     if arg0 == "getbadqr" && qr.len() > 40 {
                         qr.replace_range(12..22, "0000000000")
@@ -504,8 +392,12 @@ fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, failu
             }
         }
         "joinqr" => {
-            start_threads(ctx.clone());
             if !arg0.is_empty() {
+                crossbeam::scope(|s| {
+                    s.spawn(|_| ctx.read().unwrap().run());
+                })
+                .unwrap();
+
                 dc_join_securejoin(&ctx.read().unwrap(), arg1);
             }
         }

examples/simple.rs

@@ -1,36 +1,27 @@
 extern crate deltachat;
 
-use std::sync::{Arc, RwLock};
 use std::{thread, time};
 use tempfile::tempdir;
 
 use deltachat::chat;
 use deltachat::chatlist::*;
 use deltachat::config;
-use deltachat::configure::*;
 use deltachat::contact::*;
 use deltachat::context::*;
-use deltachat::job::{
-    perform_inbox_fetch, perform_inbox_idle, perform_inbox_jobs, perform_smtp_idle,
-    perform_smtp_jobs,
-};
 use deltachat::Event;
 
-fn cb(_ctx: &Context, event: Event) -> usize {
+fn cb(_ctx: &Context, event: Event) {
     print!("[{:?}]", event);
 
     match event {
         Event::ConfigureProgress(progress) => {
             print!("  progress: {}\n", progress);
-            0
         }
         Event::Info(msg) | Event::Warning(msg) | Event::Error(msg) | Event::ErrorNetwork(msg) => {
             print!("  {}\n", msg);
-            0
         }
         _ => {
             print!("\n");
-            0
         }
     }
 }
@@ -39,77 +30,52 @@ fn main() {
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     println!("creating database {:?}", dbfile);
-    let ctx =
-        Context::new(Box::new(cb), "FakeOs".into(), dbfile).expect("Failed to create context");
-    let running = Arc::new(RwLock::new(true));
+    let ctx = Context::new("FakeOs".into(), dbfile).expect("Failed to create context");
     let info = ctx.get_info();
-    let duration = time::Duration::from_millis(4000);
+    let duration = time::Duration::from_millis(8000);
     println!("info: {:#?}", info);
 
-    let ctx = Arc::new(ctx);
-    let ctx1 = ctx.clone();
-    let r1 = running.clone();
-    let t1 = thread::spawn(move || {
-        while *r1.read().unwrap() {
-            perform_inbox_jobs(&ctx1);
-            if *r1.read().unwrap() {
-                perform_inbox_fetch(&ctx1);
+    crossbeam::scope(|s| {
+        let t1 = s.spawn(|_| {
+            ctx.run(cb);
+        });
 
-                if *r1.read().unwrap() {
-                    perform_inbox_idle(&ctx1);
-                }
-            }
+        println!("configuring");
+        let args = std::env::args().collect::<Vec<String>>();
+        assert_eq!(args.len(), 2, "missing password");
+        let pw = args[1].clone();
+        ctx.set_config(config::Config::Addr, Some("d@testrun.org"))
+            .unwrap();
+        ctx.set_config(config::Config::MailPw, Some(&pw)).unwrap();
+        ctx.configure();
+
+        thread::sleep(duration);
+
+        println!("sending a message");
+        let contact_id =
+            Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com").unwrap();
+        let chat_id = chat::create_by_contact_id(&ctx, contact_id).unwrap();
+        chat::send_text_msg(&ctx, chat_id, "Hi, here is my first message!".into()).unwrap();
+
+        println!("fetching chats..");
+        let chats = Chatlist::try_load(&ctx, 0, None, None).unwrap();
+
+        for i in 0..chats.len() {
+            let summary = chats.get_summary(&ctx, 0, None);
+            let text1 = summary.get_text1();
+            let text2 = summary.get_text2();
+            println!("chat: {} - {:?} - {:?}", i, text1, text2,);
         }
-    });
 
-    let ctx1 = ctx.clone();
-    let r1 = running.clone();
-    let t2 = thread::spawn(move || {
-        while *r1.read().unwrap() {
-            perform_smtp_jobs(&ctx1);
-            if *r1.read().unwrap() {
-                perform_smtp_idle(&ctx1);
-            }
-        }
-    });
+        thread::sleep(duration);
 
-    println!("configuring");
-    let args = std::env::args().collect::<Vec<String>>();
-    assert_eq!(args.len(), 2, "missing password");
-    let pw = args[1].clone();
-    ctx.set_config(config::Config::Addr, Some("d@testrun.org"))
-        .unwrap();
-    ctx.set_config(config::Config::MailPw, Some(&pw)).unwrap();
-    configure(&ctx);
+        println!("stopping threads");
+        ctx.shutdown();
 
-    thread::sleep(duration);
+        println!("joining");
+        t1.join().unwrap();
 
-    println!("sending a message");
-    let contact_id = Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com").unwrap();
-    let chat_id = chat::create_by_contact_id(&ctx, contact_id).unwrap();
-    chat::send_text_msg(&ctx, chat_id, "Hi, here is my first message!".into()).unwrap();
-
-    println!("fetching chats..");
-    let chats = Chatlist::try_load(&ctx, 0, None, None).unwrap();
-
-    for i in 0..chats.len() {
-        let summary = chats.get_summary(&ctx, 0, None);
-        let text1 = summary.get_text1();
-        let text2 = summary.get_text2();
-        println!("chat: {} - {:?} - {:?}", i, text1, text2,);
-    }
-
-    thread::sleep(duration);
-
-    println!("stopping threads");
-
-    *running.write().unwrap() = false;
-    deltachat::job::interrupt_inbox_idle(&ctx, true);
-    deltachat::job::interrupt_smtp_idle(&ctx);
-
-    println!("joining");
-    t1.join().unwrap();
-    t2.join().unwrap();
-
-    println!("closing");
+        println!("closing");
+    })
+    .unwrap();
 }

python/CHANGELOG

@@ -1,6 +1,10 @@
-0.600.1
+0.700.0
 ---------
 
+- lots of new Python APIs
+
+- use rust core-beta23 
+
 - introduce automatic versioning via setuptools_scm, 
   based on py-X.Y.Z tags 
 

python/README.rst

@@ -3,89 +3,54 @@ deltachat python bindings
 =========================
 
 This package provides bindings to the deltachat-core_ Rust -library
-which provides imap/smtp/crypto handling as well as chat/group/messages
-handling to Android, Desktop and IO user interfaces.
-
-Installing pre-built packages (linux-only)
-==========================================
-
-If you have a linux system you may install the ``deltachat`` binary "wheel" package
-without any "build-from-source" steps.
-
-1. `Install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
-   then create a fresh python environment and activate it in your shell::
-
-        virtualenv venv  # or: python -m venv
-        source venv/bin/activate
-
-   Afterwards, invoking ``python`` or ``pip install`` will only
-   modify files in your ``venv`` directory and leave your system installation
-   alone.
-
-2. Install the wheel for linux::
-
-        pip install deltachat
-
-    Verify it worked by typing::
-
-        python -c "import deltachat"
+which implements IMAP/SMTP/MIME/PGP e-mail standards and offers
+a low-level Chat/Contact/Message API to user interfaces and bots.
 
 
-Installing a wheel from a PR/branch
----------------------------------------
+Installing bindings from source (Updated: 20-Jan-2020)
+=========================================================
 
-For Linux, we automatically build wheels for all github PR branches
-and push them to a python package index. To install the latest github ``master`` branch::
+Install Rust and Cargo first.  Deltachat needs a specific nightly
+version, the easiest is probably to first install Rust stable from
+rustup and then use this to install the correct nightly version.
 
-    pip install -i https://m.devpi.net/dc/master deltachat
+Bootstrap Rust and Cargo by using rustup::
 
-.. note::
+   curl https://sh.rustup.rs -sSf | sh
 
-    If you can help to automate the building of wheels for Mac or Windows,
-    that'd be much appreciated! please then get
-    `in contact with us <https://delta.chat/en/contribute>`_.
+Then GIT clone the deltachat-core-rust repo and get the actual
+rust- and cargo-toolchain needed by deltachat::
 
+   git clone https://github.com/deltachat/deltachat-core-rust
+   cd deltachat-core-rust
+   rustup show
 
-Installing bindings from source
-===============================
+To install the Delta Chat Python bindings make sure you have Python3 installed.
+E.g. on Debian-based systems `apt install python3 python3-pip
+python3-venv` should give you a usable python installation.
 
-If you can't use "binary" method above then you need to compile
-to core deltachat library::
+Ensure you are in the deltachat-core-rust/python directory, create the
+virtual environment and activate it in your shell::
 
-    git clone https://github.com/deltachat/deltachat-core-rust
-    cd deltachat-core-rust
-    cd python
+   cd python
+   python3 -m venv venv  # or: virtualenv venv
+   source venv/bin/activate
 
-If you don't have one active, create and activate a python "virtualenv":
+You should now be able to build the python bindings using the supplied script::
 
-    python virtualenv venv  # or python -m venv
-    source venv/bin/activate
-
-Afterwards ``which python`` tells you that it comes out of the "venv"
-directory that contains all python install artifacts. Let's first
-install test tools::
-
-    pip install pytest pytest-timeout pytest-rerunfailures requests
-
-then cargo-build and install the deltachat bindings::
-
-    python install_python_bindings.py
+   ./install_python_bindings.py
 
+The installation might take a while, depending on your machine.
 The bindings will be installed in release mode but with debug symbols.
-The release mode is necessary because some tests generate RSA keys
-which is prohibitively slow in debug mode.
+The release mode is currently necessary because some tests generate RSA keys
+which is prohibitively slow in non-release mode.
 
-After successful binding installation you can finally run the tests::
+After successful binding installation you can install a few more
+Python packages before running the tests::
 
+    python -m pip install pytest pytest-timeout pytest-rerunfailures requests
     pytest -v tests
 
-.. note::
-
-    Some tests are sometimes failing/hanging because of
-    https://github.com/deltachat/deltachat-core-rust/issues/331
-    and
-    https://github.com/deltachat/deltachat-core-rust/issues/326
-
 
 running "live" tests (experimental)
 -----------------------------------
@@ -104,6 +69,38 @@ With ``DCC_PY_LIVECONFIG`` set pytest invocations will use real
 e-mail accounts and run through all functional "liveconfig" tests.
 
 
+Installing pre-built packages (Linux-only)
+========================================================
+
+If you have a Linux system you may try to install the ``deltachat`` binary "wheel" packages
+without any "build-from-source" steps.
+
+We suggest to `Install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
+then create a fresh Python virtual environment and activate it in your shell::
+
+        virtualenv venv  # or: python -m venv
+        source venv/bin/activate
+
+Afterwards, invoking ``python`` or ``pip install`` only
+modifies files in your ``venv`` directory and leaves
+your system installation alone.
+
+For Linux, we automatically build wheels for all github PR branches
+and push them to a python package index. To install the latest
+github ``master`` branch::
+
+    pip install --pre -i https://m.devpi.net/dc/master deltachat
+
+To verify it worked::
+
+    python -c "import deltachat"
+
+.. note::
+
+    If you can help to automate the building of wheels for Mac or Windows,
+    that'd be much appreciated! please then get
+    `in contact with us <https://delta.chat/en/contribute>`_.
+
 
 Code examples
 =============
@@ -115,15 +112,11 @@ You may look at `examples <https://py.delta.chat/examples.html>`_.
 .. _`deltachat-core`: https://github.com/deltachat/deltachat-core-rust
 
 
-Building manylinux1 wheels
-==========================
-
-.. note::
-
-   This section may not fully work.
+Building manylinux1 based wheels
+================================
 
 Building portable manylinux1 wheels which come with libdeltachat.so
-and all it's dependencies is easy using the provided docker tooling.
+can be done with docker-tooling.
 
 using docker pull / premade images
 ------------------------------------
@@ -136,9 +129,9 @@ organization::
 
 This docker image can be used to run tests and build Python wheels for all interpreters::
 
-    $ bash ci_scripts/ci_run.sh
-
-This command runs tests and build-wheel scripts in a docker container.
+    $ docker run -e DCC_PY_LIVECONFIG \
+       --rm -it -v \$(pwd):/mnt -w /mnt \
+       deltachat/coredeps ci_scripts/run_all.sh
 
 
 Optionally build your own docker image

python/fail_test.py

@@ -0,0 +1,7 @@
+from __future__ import print_function
+from deltachat import capi
+from deltachat.capi import ffi, lib
+
+if __name__ == "__main__":
+    ctx = capi.lib.dc_context_new(ffi.NULL, ffi.NULL)
+    lib.dc_context_shutdown(ctx)

python/setup.py

@@ -13,6 +13,7 @@ def main():
             "root": "..",
             "relative_to": __file__,
             'tag_regex': r'^(?P<prefix>py-)?(?P<version>[^\+]+)(?P<suffix>.*)?$',
+            'git_describe_command': "git describe --dirty --tags --long --match py-*.*",
         },
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,

python/src/deltachat/account.py

@@ -26,7 +26,7 @@ class Account(object):
     by the underlying deltachat core library.  All public Account methods are
     meant to be memory-safe and return memory-safe objects.
     """
-    def __init__(self, db_path, logid=None, eventlogging=True, debug=True):
+    def __init__(self, db_path, logid=None, eventlogging=True, os_name=None, debug=True):
         """ initialize account object.
 
         :param db_path: a path to the account database. The database
@@ -34,10 +34,11 @@ class Account(object):
         :param logid: an optional logging prefix that should be used with
                       the default internal logging.
         :param eventlogging: if False no eventlogging and no context callback will be configured
+        :param os_name: this will be put to the X-Mailer header in outgoing messages
         :param debug: turn on debug logging for events.
         """
         self._dc_context = ffi.gc(
-            lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+            lib.dc_context_new(ffi.NULL, as_dc_charpointer(os_name)),
             _destroy_dc_context,
         )
         if eventlogging:
@@ -379,8 +380,6 @@ class Account(object):
     def _export(self, path, imex_cmd):
         self._imex_events_clear()
         lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
-        if not self._threads.is_started():
-            lib.dc_perform_imap_jobs(self._dc_context)
         files_written = []
         while True:
             ev = self._imex_events.get()
@@ -409,8 +408,6 @@ class Account(object):
     def _import(self, path, imex_cmd):
         self._imex_events_clear()
         lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
-        if not self._threads.is_started():
-            lib.dc_perform_imap_jobs(self._dc_context)
         if not self._imex_events.get():
             raise ValueError("import from path '{}' failed".format(path))
 
@@ -489,7 +486,7 @@ class Account(object):
         ev = self._evlogger.get_matching("DC_EVENT_INCOMING_MSG")
         return self.get_message_by_id(ev[2])
 
-    def start_threads(self, mvbox=False, sentbox=False):
+    def start_threads(self):
         """ start IMAP/SMTP threads (and configure account if it hasn't happened).
 
         :raises: ValueError if 'addr' or 'mail_pw' are not configured.
@@ -497,7 +494,7 @@ class Account(object):
         """
         if not self.is_configured():
             self.configure()
-        self._threads.start(mvbox=mvbox, sentbox=sentbox)
+        self._threads.start()
 
     def stop_threads(self, wait=True):
         """ stop IMAP/SMTP threads. """
@@ -559,16 +556,9 @@ class IOThreads:
     def is_started(self):
         return len(self._name2thread) > 0
 
-    def start(self, imap=True, smtp=True, mvbox=False, sentbox=False):
+    def start(self):
         assert not self.is_started()
-        if imap:
-            self._start_one_thread("inbox", self.imap_thread_run)
-        if mvbox:
-            self._start_one_thread("mvbox", self.mvbox_thread_run)
-        if sentbox:
-            self._start_one_thread("sentbox", self.sentbox_thread_run)
-        if smtp:
-            self._start_one_thread("smtp", self.smtp_thread_run)
+        self._start_one_thread("deltachat", self.dc_thread_run)
 
     def _start_one_thread(self, name, func):
         self._name2thread[name] = t = threading.Thread(target=func, name=name)
@@ -576,48 +566,18 @@ class IOThreads:
         t.start()
 
     def stop(self, wait=False):
-        self._thread_quitflag = True
-        lib.dc_interrupt_imap_idle(self._dc_context)
-        lib.dc_interrupt_smtp_idle(self._dc_context)
-        lib.dc_interrupt_mvbox_idle(self._dc_context)
-        lib.dc_interrupt_sentbox_idle(self._dc_context)
+        lib.dc_context_shutdown(self._dc_context)
         if wait:
             for name, thread in self._name2thread.items():
                 thread.join()
 
-    def imap_thread_run(self):
-        self._log_event("py-bindings-info", 0, "INBOX THREAD START")
-        while not self._thread_quitflag:
-            lib.dc_perform_imap_jobs(self._dc_context)
-            if not self._thread_quitflag:
-                lib.dc_perform_imap_fetch(self._dc_context)
-            if not self._thread_quitflag:
-                lib.dc_perform_imap_idle(self._dc_context)
+    def dc_thread_run(self):
+        self._log_event("py-bindings-info", 0, "DC THREAD START")
+
+        lib.dc_context_run(self._dc_context, lib.py_dc_callback)
+
         self._log_event("py-bindings-info", 0, "INBOX THREAD FINISHED")
 
-    def mvbox_thread_run(self):
-        self._log_event("py-bindings-info", 0, "MVBOX THREAD START")
-        while not self._thread_quitflag:
-            lib.dc_perform_mvbox_jobs(self._dc_context)
-            lib.dc_perform_mvbox_fetch(self._dc_context)
-            lib.dc_perform_mvbox_idle(self._dc_context)
-        self._log_event("py-bindings-info", 0, "MVBOX THREAD FINISHED")
-
-    def sentbox_thread_run(self):
-        self._log_event("py-bindings-info", 0, "SENTBOX THREAD START")
-        while not self._thread_quitflag:
-            lib.dc_perform_sentbox_jobs(self._dc_context)
-            lib.dc_perform_sentbox_fetch(self._dc_context)
-            lib.dc_perform_sentbox_idle(self._dc_context)
-        self._log_event("py-bindings-info", 0, "SENTBOX THREAD FINISHED")
-
-    def smtp_thread_run(self):
-        self._log_event("py-bindings-info", 0, "SMTP THREAD START")
-        while not self._thread_quitflag:
-            lib.dc_perform_smtp_jobs(self._dc_context)
-            lib.dc_perform_smtp_idle(self._dc_context)
-        self._log_event("py-bindings-info", 0, "SMTP THREAD FINISHED")
-
 
 class EventLogger:
     _loglock = threading.RLock()

python/tests/conftest.py

@@ -182,7 +182,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig):
         def get_online_configuring_account(self, mvbox=False, sentbox=False):
             ac, configdict = self.get_online_config()
             ac.configure(**configdict)
-            ac.start_threads(mvbox=mvbox, sentbox=sentbox)
+            ac.start_threads()
             return ac
 
         def get_one_online_account(self):

python/tests/test_account.py

@@ -16,6 +16,14 @@ class TestOfflineAccountBasic:
         with pytest.raises(ValueError):
             Account(p.strpath)
 
+    def test_os_name(self, tmpdir):
+        p = tmpdir.join("hello.db")
+        # we can't easily test if os_name is used in X-Mailer
+        # outgoing messages without a full Online test
+        # but we at least check Account accepts the arg
+        ac1 = Account(p.strpath, os_name="solarpunk")
+        ac1.get_info()
+
     def test_getinfo(self, acfactory):
         ac1 = acfactory.get_unconfigured_account()
         d = ac1.get_info()
@@ -168,6 +176,7 @@ class TestOfflineChat:
         assert d["draft"] == "" if chat.get_draft() is None else chat.get_draft()
 
     def test_group_chat_creation_with_translation(self, ac1):
+        ac1.start_threads()
         ac1.set_stock_translation(const.DC_STR_NEWGROUPDRAFT, "xyz %1$s")
         ac1._evlogger.consume_events()
         with pytest.raises(ValueError):
@@ -187,6 +196,7 @@ class TestOfflineChat:
         assert not chat.is_promoted()
         msg = chat.get_draft()
         assert msg.text == "xyz title1"
+        ac1.stop_threads()
 
     @pytest.mark.parametrize("verified", [True, False])
     def test_group_chat_qr(self, acfactory, ac1, verified):
@@ -670,14 +680,26 @@ class TestOnlineAccount:
         chat2b.mark_noticed()
         assert chat2b.count_fresh_messages() == 0
 
-        lp.sec("mark message as seen on ac2, wait for changes on ac1")
-        ac2.mark_seen_messages([msg_in])
+        ac2._evlogger.consume_events()
+
+        lp.sec("sending a second message from ac1 to ac2")
+        msg_out2 = chat.send_text("message2")
+
+        lp.sec("wait for ac2 to receive second message")
+        ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG")
+        assert ev[2] == msg_out2.id
+        msg_in2 = ac2.get_message_by_id(msg_out2.id)
+
+        lp.sec("mark messages as seen on ac2, wait for changes on ac1")
+        ac2.mark_seen_messages([msg_in, msg_in2])
         lp.step("1")
-        ev = ac1._evlogger.get_matching("DC_EVENT_MSG_READ")
-        assert ev[1] > const.DC_CHAT_ID_LAST_SPECIAL
-        assert ev[2] > const.DC_MSG_ID_LAST_SPECIAL
+        for i in range(2):
+            ev = ac1._evlogger.get_matching("DC_EVENT_MSG_READ")
+            assert ev[1] > const.DC_CHAT_ID_LAST_SPECIAL
+            assert ev[2] > const.DC_MSG_ID_LAST_SPECIAL
         lp.step("2")
         assert msg_out.is_out_mdn_received()
+        assert msg_out2.is_out_mdn_received()
 
         lp.sec("check that a second call to mark_seen does not create change or smtp job")
         ac2._evlogger.consume_events()
@@ -687,6 +709,41 @@ class TestOnlineAccount:
         except queue.Empty:
             pass  # mark_seen_messages() has generated events before it returns
 
+    def test_mdn_asymetric(self, acfactory, lp):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+
+        lp.sec("ac1: create chat with ac2")
+        chat = self.get_chat(ac1, ac2, both_created=True)
+
+        # make sure mdns are enabled (usually enabled by default already)
+        ac1.set_config("mdns_enabled", "1")
+        ac2.set_config("mdns_enabled", "1")
+
+        lp.sec("sending text message from ac1 to ac2")
+        msg_out = chat.send_text("message1")
+
+        assert len(chat.get_messages()) == 1
+
+        lp.sec("disable ac1 MDNs")
+        ac1.set_config("mdns_enabled", "0")
+
+        lp.sec("wait for ac2 to receive message")
+        msg = ac2.wait_next_incoming_message()
+
+        assert len(msg.chat.get_messages()) == 1
+
+        lp.sec("ac2: mark incoming message as seen")
+        ac2.mark_seen_messages([msg])
+
+        lp.sec("ac1: waiting for incoming activity")
+        # MDN should be moved even though MDNs are already disabled
+        ac1._evlogger.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
+
+        assert len(chat.get_messages()) == 1
+
+        # MDN is received even though MDNs are already disabled
+        assert msg_out.is_out_mdn_received()
+
     def test_send_and_receive_will_encrypt_decrypt(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
@@ -786,7 +843,21 @@ class TestOnlineAccount:
         print("ac1: e2ee_enabled={}".format(ac1.get_config("e2ee_enabled")))
         print("ac2: e2ee_enabled={}".format(ac2.get_config("e2ee_enabled")))
         ac1.set_config("e2ee_enabled", "0")
-        chat.send_text("message2 -- should be encrypted")
+
+        # Set unprepared and unencrypted draft to test that it is not
+        # taken into account when determining whether last message is
+        # encrypted.
+        msg_draft = Message.new_empty(ac1, "text")
+        msg_draft.set_text("message2 -- should be encrypted")
+        chat.set_draft(msg_draft)
+
+        # Get the draft, prepare and send it.
+        msg_draft = chat.get_draft()
+        msg_out = chat.prepare_message(msg_draft)
+        chat.send_prepared(msg_out)
+
+        chat.set_draft(None)
+        assert chat.get_draft() is None
 
         lp.sec("wait for ac2 to receive message")
         ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG")
@@ -1134,29 +1205,46 @@ class TestOnlineAccount:
 class TestGroupStressTests:
     def test_group_many_members_add_leave_remove(self, acfactory, lp):
         lp.sec("creating and configuring five accounts")
-        ac1 = acfactory.get_online_configuring_account()
-        accounts = [acfactory.get_online_configuring_account() for i in range(3)]
-        wait_configuration_progress(ac1, 1000)
+        accounts = [acfactory.get_online_configuring_account() for i in range(5)]
         for acc in accounts:
             wait_configuration_progress(acc, 1000)
+        ac1 = accounts.pop()
+
+        lp.sec("ac1: setting up contacts with 4 other members")
+        contacts = []
+        for acc, name in zip(accounts, list("äöüsr")):
+            contact = ac1.create_contact(acc.get_config("addr"), name=name)
+            contacts.append(contact)
+
+            # make sure we accept the "hi" message
+            ac1.create_chat_by_contact(contact)
+
+            # make sure the other side accepts our messages
+            c1 = acc.create_contact(ac1.get_config("addr"), "ä member")
+            chat1 = acc.create_chat_by_contact(c1)
+
+            # send a message to get the contact key via autocrypt header
+            chat1.send_text("hi")
+            msg = ac1.wait_next_incoming_message()
+            assert msg.text == "hi"
+
+        # Save fifth account for later
+        ac5 = accounts.pop()
+        contact5 = contacts.pop()
 
         lp.sec("ac1: creating group chat with 3 other members")
         chat = ac1.create_group_chat("title1")
-        contacts = []
-        chars = list("äöüsr")
-        for acc in accounts:
-            contact = ac1.create_contact(acc.get_config("addr"), name=chars.pop())
-            contacts.append(contact)
+        for contact in contacts:
             chat.add_contact(contact)
-            # make sure the other side accepts our messages
-            c1 = acc.create_contact(ac1.get_config("addr"), "ä member")
-            acc.create_chat_by_contact(c1)
-
         assert not chat.is_promoted()
 
         lp.sec("ac1: send mesage to new group chat")
-        chat.send_text("hello")
+        msg = chat.send_text("hello")
         assert chat.is_promoted()
+        assert msg.is_encrypted()
+
+        gossiped_timestamp = chat.get_summary()["gossiped_timestamp"]
+        assert gossiped_timestamp > 0
 
         num_contacts = len(chat.get_contacts())
         assert num_contacts == 3 + 1
@@ -1168,14 +1256,49 @@ class TestGroupStressTests:
         print("chat is", msg.chat)
         assert len(msg.chat.get_contacts()) == 4
 
-        lp.sec("ac1: removing one contacts and checking things are right")
-        to_remove = msg.chat.get_contacts()[-1]
+        lp.sec("ac3: checking that 'ac4' is a known contact")
+        ac3 = accounts[1]
+        msg3 = ac3.wait_next_incoming_message()
+        assert msg3.text == "hello"
+        ac3_contacts = ac3.get_contacts()
+        assert len(ac3_contacts) == 3
+        ac4_contacts = ac3.get_contacts(query=accounts[2].get_config("addr"))
+        assert len(ac4_contacts) == 1
+
+        lp.sec("ac2: removing one contact")
+        to_remove = contacts[-1]
         msg.chat.remove_contact(to_remove)
 
+        lp.sec("ac1: receiving system message about contact removal")
         sysmsg = ac1.wait_next_incoming_message()
         assert to_remove.addr in sysmsg.text
         assert len(sysmsg.chat.get_contacts()) == 3
 
+        # Receiving message about removed contact does not reset gossip
+        assert chat.get_summary()["gossiped_timestamp"] == gossiped_timestamp
+
+        lp.sec("ac1: sending another message to the chat")
+        chat.send_text("hello2")
+        msg = ac2.wait_next_incoming_message()
+        assert msg.text == "hello2"
+        assert chat.get_summary()["gossiped_timestamp"] == gossiped_timestamp
+
+        lp.sec("ac1: adding fifth member to the chat")
+        chat.add_contact(contact5)
+        # Additng contact to chat resets gossiped_timestamp
+        assert chat.get_summary()["gossiped_timestamp"] >= gossiped_timestamp
+
+        lp.sec("ac2: receiving system message about contact addition")
+        sysmsg = ac2.wait_next_incoming_message()
+        assert contact5.addr in sysmsg.text
+        assert len(sysmsg.chat.get_contacts()) == 4
+
+        lp.sec("ac5: waiting for message about addition to the chat")
+        sysmsg = ac5.wait_next_incoming_message()
+        msg = sysmsg.chat.send_text("hello!")
+        # Message should be encrypted because keys of other members are gossiped
+        assert msg.is_encrypted()
+
 
 class TestOnlineConfigureFails:
     def test_invalid_password(self, acfactory):

python/tests/test_lowlevel.py

@@ -1,25 +1,48 @@
 from __future__ import print_function
+import threading
 from deltachat import capi, cutil, const, set_context_callback, clear_context_callback
 from deltachat.capi import ffi
 from deltachat.capi import lib
 from deltachat.account import EventLogger
 
 
+class EventThread(threading.Thread):
+    def __init__(self, dc_context):
+        self.dc_context = dc_context
+        super(EventThread, self).__init__()
+        self.setDaemon(1)
+
+    def run(self):
+        lib.dc_context_run(self.dc_context, lib.py_dc_callback)
+
+    def stop(self):
+        lib.dc_context_shutdown(self.dc_context)
+
+
 def test_empty_context():
-    ctx = capi.lib.dc_context_new(capi.ffi.NULL, capi.ffi.NULL, capi.ffi.NULL)
+    ctx = capi.lib.dc_context_new(capi.ffi.NULL, capi.ffi.NULL)
     capi.lib.dc_close(ctx)
 
 
 def test_callback_None2int():
-    ctx = capi.lib.dc_context_new(capi.lib.py_dc_callback, ffi.NULL, ffi.NULL)
+    ctx = capi.lib.dc_context_new(ffi.NULL, ffi.NULL)
     set_context_callback(ctx, lambda *args: None)
     capi.lib.dc_close(ctx)
     clear_context_callback(ctx)
 
+def test_start_stop_event_thread_basic():
+    print("1")
+    ctx = capi.lib.dc_context_new(ffi.NULL, ffi.NULL)
+    print("2")
+    ev_thread = EventThread(ctx)
+    print("3 -- starting event thread")
+    ev_thread.start()
+    print("4 -- stopping event thread")
+    ev_thread.stop()
 
 def test_dc_close_events(tmpdir):
     ctx = ffi.gc(
-        capi.lib.dc_context_new(capi.lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        capi.lib.dc_context_new(ffi.NULL, ffi.NULL),
         lib.dc_context_unref,
     )
     evlog = EventLogger(ctx)
@@ -28,6 +51,9 @@ def test_dc_close_events(tmpdir):
         ctx,
         lambda ctx, evt_name, data1, data2: evlog(evt_name, data1, data2)
     )
+    ev_thread = EventThread(ctx)
+    ev_thread.start()
+
     p = tmpdir.join("hello.db")
     lib.dc_open(ctx, p.strpath.encode("ascii"), ffi.NULL)
     capi.lib.dc_close(ctx)
@@ -46,11 +72,12 @@ def test_dc_close_events(tmpdir):
     find("disconnecting mvbox-thread")
     find("disconnecting SMTP")
     find("Database closed")
+    ev_thread.stop()
 
 
 def test_wrong_db(tmpdir):
     dc_context = ffi.gc(
-        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_new(ffi.NULL, ffi.NULL),
         lib.dc_context_unref,
     )
     p = tmpdir.join("hello.db")
@@ -62,7 +89,7 @@ def test_wrong_db(tmpdir):
 def test_empty_blobdir(tmpdir):
     # Apparently some client code expects this to be the same as passing NULL.
     ctx = ffi.gc(
-        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_new(ffi.NULL, ffi.NULL),
         lib.dc_context_unref,
     )
     db_fname = tmpdir.join("hello.db")
@@ -119,7 +146,7 @@ def test_provider_info_none():
 
 def test_get_info_closed():
     ctx = ffi.gc(
-        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_new(ffi.NULL, ffi.NULL),
         lib.dc_context_unref,
     )
     info = cutil.from_dc_charpointer(lib.dc_get_info(ctx))
@@ -129,7 +156,7 @@ def test_get_info_closed():
 
 def test_get_info_open(tmpdir):
     ctx = ffi.gc(
-        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_new(ffi.NULL, ffi.NULL),
         lib.dc_context_unref,
     )
     db_fname = tmpdir.join("test.db")
@@ -141,7 +168,7 @@ def test_get_info_open(tmpdir):
 
 def test_is_open_closed():
     ctx = ffi.gc(
-        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_new(ffi.NULL, ffi.NULL),
         lib.dc_context_unref,
     )
     assert lib.dc_is_open(ctx) == 0
@@ -149,7 +176,7 @@ def test_is_open_closed():
 
 def test_is_open_actually_open(tmpdir):
     ctx = ffi.gc(
-        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_new(ffi.NULL, ffi.NULL),
         lib.dc_context_unref,
     )
     db_fname = tmpdir.join("test.db")

python/tox.ini

@@ -8,7 +8,7 @@ envlist =
 [testenv]
 commands = 
     pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx {posargs:tests} 
-    # python tests/package_wheels.py {toxworkdir}/wheelhouse
+    python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 
     DCC_RS_DEV
@@ -69,7 +69,7 @@ addopts = -v -ra
 python_files = tests/test_*.py
 norecursedirs = .tox 
 xfail_strict=true
-timeout = 60 
+timeout = 90
 timeout_method = thread
 
 [flake8]

rust-toolchain

@@ -1 +1 @@
-nightly-2019-11-06
+nightly-2020-01-26

set_core_version.py

@@ -55,6 +55,7 @@ if __name__ == "__main__":
     replace_toml_version("Cargo.toml", newversion)
     replace_toml_version("deltachat-ffi/Cargo.toml", newversion)
 
+    subprocess.call(["cargo", "check"])
     subprocess.call(["git", "add", "-u"])
     # subprocess.call(["cargo", "update", "-p", "deltachat"])
 

spec.md

@@ -153,8 +153,8 @@ The message-id MUST have the format `Gr.<group-id>.<unique data>`.
     Hello group - this group contains three members
 
 Messengers adding the member list in the form `Name <email-address>`
-MUST take care only to spread the names authorized by the contacts themselves.
-Otherwise, names as _Daddy_ or _Honey_ may be spread
+MUST take care only to distribute the names authorized by the contacts themselves.
+Otherwise, names as _Daddy_ or _Honey_ may be distributed
 (this issue is also true for normal MUAs, however,
 for more contact- and chat-centralized apps
 such situations happen more frequently).
@@ -288,7 +288,7 @@ to add a `Chat-Group-Avatar` only on image changes.
 
 # Set profile image
 
-A user MAY have a profile-image that MAY be spread to their contacts.
+A user MAY have a profile-image that MAY be distributed to their contacts.
 To change or set the profile-image,
 the messenger MUST attach an image file to a message
 and MUST add the header `Chat-User-Avatar`
@@ -297,7 +297,7 @@ with the value set to the image name.
 To remove the profile-image,
 the messenger MUST add the header `Chat-User-Avatar: 0`.
 
-To spread the image,
+To distribute the image,
 the messenger MAY send the profile image
 together with the next mail to a given contact
 (to do this only once,

src/blob.rs

@@ -6,9 +6,13 @@ use std::fs;
 use std::io::Write;
 use std::path::{Path, PathBuf};
 
+use self::image::GenericImageView;
+use crate::constants::AVATAR_SIZE;
 use crate::context::Context;
 use crate::events::Event;
 
+extern crate image;
+
 /// Represents a file in the blob directory.
 ///
 /// The object has a name, which will always be valid UTF-8.  Having a
@@ -318,13 +322,12 @@ impl<'a> BlobObject<'a> {
 
         let clean = sanitize_filename::sanitize_with_options(name, opts);
         let mut iter = clean.splitn(2, '.');
-        let mut stem = iter.next().unwrap_or_default().to_string();
-        let mut ext = iter.next().unwrap_or_default().to_string();
-        stem.truncate(64);
-        ext.truncate(32);
-        match ext.len() {
-            0 => (stem, "".to_string()),
-            _ => (stem, format!(".{}", ext).to_lowercase()),
+        let stem: String = iter.next().unwrap_or_default().chars().take(64).collect();
+        let ext: String = iter.next().unwrap_or_default().chars().take(32).collect();
+        if ext.is_empty() {
+            (stem, "".to_string())
+        } else {
+            (stem, format!(".{}", ext).to_lowercase())
         }
     }
 
@@ -349,6 +352,31 @@ impl<'a> BlobObject<'a> {
         }
         true
     }
+
+    pub fn recode_to_avatar_size(&self, context: &Context) -> Result<(), BlobError> {
+        let blob_abs = self.to_abs_path();
+        let img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
+            blobdir: context.get_blobdir().to_path_buf(),
+            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+            cause: err,
+            backtrace: failure::Backtrace::new(),
+        })?;
+
+        if img.width() <= AVATAR_SIZE && img.height() <= AVATAR_SIZE {
+            return Ok(());
+        }
+
+        let img = img.thumbnail(AVATAR_SIZE, AVATAR_SIZE);
+
+        img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
+            blobdir: context.get_blobdir().to_path_buf(),
+            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+            cause: err,
+            backtrace: failure::Backtrace::new(),
+        })?;
+
+        Ok(())
+    }
 }
 
 impl<'a> fmt::Display for BlobObject<'a> {
@@ -382,6 +410,13 @@ pub enum BlobError {
         cause: std::io::Error,
         backtrace: failure::Backtrace,
     },
+    RecodeFailure {
+        blobdir: PathBuf,
+        blobname: String,
+        #[cause]
+        cause: image::ImageError,
+        backtrace: failure::Backtrace,
+    },
     WrongBlobdir {
         blobdir: PathBuf,
         src: PathBuf,
@@ -429,6 +464,9 @@ impl fmt::Display for BlobError {
                 blobname,
                 blobdir.display(),
             ),
+            BlobError::RecodeFailure {
+                blobdir, blobname, ..
+            } => write!(f, "Failed to recode {} in {}", blobname, blobdir.display(),),
             BlobError::WrongBlobdir { blobdir, src, .. } => write!(
                 f,
                 "File path {} is not in blobdir {}",
@@ -592,4 +630,11 @@ mod tests {
         assert!(!BlobObject::is_acceptible_blob_name("foo\\bar"));
         assert!(!BlobObject::is_acceptible_blob_name("foo\x00bar"));
     }
+
+    #[test]
+    fn test_sanitise_name() {
+        let (_, ext) =
+            BlobObject::sanitise_name("Я ЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯ.txt");
+        assert_eq!(ext, ".txt");
+    }
 }

src/chatlist.rs

@@ -36,7 +36,7 @@ use crate::stock::StockMessage;
 #[derive(Debug)]
 pub struct Chatlist {
     /// Stores pairs of `chat_id, message_id`
-    ids: Vec<(u32, MsgId)>,
+    ids: Vec<(ChatId, MsgId)>,
 }
 
 impl Chatlist {
@@ -91,7 +91,7 @@ impl Chatlist {
         let mut add_archived_link_item = false;
 
         let process_row = |row: &rusqlite::Row| {
-            let chat_id: u32 = row.get(0)?;
+            let chat_id: ChatId = row.get(0)?;
             let msg_id: MsgId = row.get(1).unwrap_or_default();
             Ok((chat_id, msg_id))
         };
@@ -162,6 +162,12 @@ impl Chatlist {
             let query = query.trim().to_string();
             ensure!(!query.is_empty(), "missing query");
 
+            // allow searching over special names that may change at any time
+            // when the ui calls set_stock_translation()
+            if let Err(err) = update_special_chat_names(context) {
+                warn!(context, "cannot update special chat names: {:?}", err)
+            }
+
             let str_like_cmd = format!("%{}%", query);
             context.sql.query_map(
                 "SELECT c.id, m.id
@@ -205,7 +211,10 @@ impl Chatlist {
             )?;
             if 0 == listflags & DC_GCL_NO_SPECIALS {
                 if let Some(last_deaddrop_fresh_msg_id) = get_last_deaddrop_fresh_msg(context) {
-                    ids.insert(0, (DC_CHAT_ID_DEADDROP, last_deaddrop_fresh_msg_id));
+                    ids.insert(
+                        0,
+                        (ChatId::new(DC_CHAT_ID_DEADDROP), last_deaddrop_fresh_msg_id),
+                    );
                 }
                 add_archived_link_item = true;
             }
@@ -214,9 +223,9 @@ impl Chatlist {
 
         if add_archived_link_item && dc_get_archived_cnt(context) > 0 {
             if ids.is_empty() && 0 != listflags & DC_GCL_ADD_ALLDONE_HINT {
-                ids.push((DC_CHAT_ID_ALLDONE_HINT, MsgId::new(0)));
+                ids.push((ChatId::new(DC_CHAT_ID_ALLDONE_HINT), MsgId::new(0)));
             }
-            ids.push((DC_CHAT_ID_ARCHIVED_LINK, MsgId::new(0)));
+            ids.push((ChatId::new(DC_CHAT_ID_ARCHIVED_LINK), MsgId::new(0)));
         }
 
         Ok(Chatlist { ids })
@@ -235,9 +244,9 @@ impl Chatlist {
     /// Get a single chat ID of a chatlist.
     ///
     /// To get the message object from the message ID, use dc_get_chat().
-    pub fn get_chat_id(&self, index: usize) -> u32 {
+    pub fn get_chat_id(&self, index: usize) -> ChatId {
         if index >= self.ids.len() {
-            return 0;
+            return ChatId::new(0);
         }
         self.ids[index].0
     }
@@ -301,7 +310,7 @@ impl Chatlist {
             None
         };
 
-        if chat.id == DC_CHAT_ID_ARCHIVED_LINK {
+        if chat.id.is_archived_link() {
             ret.text2 = None;
         } else if lastmsg.is_none() || lastmsg.as_ref().unwrap().from_id == DC_CONTACT_ID_UNDEFINED
         {
@@ -314,7 +323,7 @@ impl Chatlist {
     }
 }
 
-/// Get the number of archived chats
+/// Returns the number of archived chats
 pub fn dc_get_archived_cnt(context: &Context) -> u32 {
     context
         .sql
@@ -349,7 +358,6 @@ fn get_last_deaddrop_fresh_msg(context: &Context) -> Option<MsgId> {
 mod tests {
     use super::*;
 
-    use crate::chat;
     use crate::test_utils::*;
 
     #[test]
@@ -369,7 +377,7 @@ mod tests {
         // drafts are sorted to the top
         let mut msg = Message::new(Viewtype::Text);
         msg.set_text(Some("hello".to_string()));
-        set_draft(&t.ctx, chat_id2, Some(&mut msg));
+        chat_id2.set_draft(&t.ctx, Some(&mut msg));
         let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
         assert_eq!(chats.get_chat_id(0), chat_id2);
 
@@ -380,8 +388,31 @@ mod tests {
         let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None).unwrap();
         assert_eq!(chats.len(), 0);
 
-        chat::archive(&t.ctx, chat_id1, true).ok();
+        chat_id1.set_archived(&t.ctx, true).ok();
         let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None).unwrap();
         assert_eq!(chats.len(), 1);
     }
+
+    #[test]
+    fn test_search_special_chat_names() {
+        let t = dummy_context();
+        t.ctx.update_device_chats().unwrap();
+
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None).unwrap();
+        assert_eq!(chats.len(), 0);
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None).unwrap();
+        assert_eq!(chats.len(), 0);
+
+        t.ctx
+            .set_stock_translation(StockMessage::SavedMessages, "test-1234-save".to_string())
+            .unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None).unwrap();
+        assert_eq!(chats.len(), 1);
+
+        t.ctx
+            .set_stock_translation(StockMessage::DeviceMessages, "test-5678-babbel".to_string())
+            .unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None).unwrap();
+        assert_eq!(chats.len(), 1);
+    }
 }

src/config.rs

@@ -8,6 +8,7 @@ use crate::constants::DC_VERSION_STR;
 use crate::context::Context;
 use crate::dc_tools::*;
 use crate::job::*;
+use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::stock::StockMessage;
 use rusqlite::NO_PARAMS;
 
@@ -98,7 +99,7 @@ impl Context {
                 rel_path.map(|p| dc_get_abs_path(self, &p).to_string_lossy().into_owned())
             }
             Config::SysVersion => Some((&*DC_VERSION_STR).clone()),
-            Config::SysMsgsizeMaxRecommended => Some(format!("{}", 24 * 1024 * 1024 / 4 * 3)),
+            Config::SysMsgsizeMaxRecommended => Some(format!("{}", RECOMMENDED_FILE_SIZE)),
             Config::SysConfigKeys => Some(get_config_keys_string()),
             _ => self.sql.get_raw_config(self, key),
         };
@@ -136,6 +137,7 @@ impl Context {
                 match value {
                     Some(value) => {
                         let blob = BlobObject::new_from_path(&self, value)?;
+                        blob.recode_to_avatar_size(self)?;
                         self.sql.set_raw_config(self, key, Some(blob.as_name()))
                     }
                     None => self.sql.set_raw_config(self, key, None),
@@ -143,7 +145,7 @@ impl Context {
             }
             Config::InboxWatch => {
                 let ret = self.sql.set_raw_config(self, key, value);
-                interrupt_inbox_idle(self, true);
+                interrupt_inbox_idle(self);
                 ret
             }
             Config::SentboxWatch => {
@@ -189,7 +191,11 @@ mod tests {
     use std::str::FromStr;
     use std::string::ToString;
 
+    use crate::constants::AVATAR_SIZE;
     use crate::test_utils::*;
+    use image::GenericImageView;
+    use std::fs::File;
+    use std::io::Write;
 
     #[test]
     fn test_to_string() {
@@ -209,30 +215,55 @@ mod tests {
     }
 
     #[test]
-    fn test_selfavatar() -> failure::Fallible<()> {
+    fn test_selfavatar_outside_blobdir() {
         let t = dummy_context();
         let avatar_src = t.dir.path().join("avatar.jpg");
-        std::fs::write(&avatar_src, b"avatar")?;
+        let avatar_bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
+        File::create(&avatar_src)
+            .unwrap()
+            .write_all(avatar_bytes)
+            .unwrap();
         let avatar_blob = t.ctx.get_blobdir().join("avatar.jpg");
         assert!(!avatar_blob.exists());
         t.ctx
-            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))?;
+            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+            .unwrap();
         assert!(avatar_blob.exists());
-        assert_eq!(std::fs::read(&avatar_blob)?, b"avatar");
+        assert!(std::fs::metadata(&avatar_blob).unwrap().len() < avatar_bytes.len() as u64);
         let avatar_cfg = t.ctx.get_config(Config::Selfavatar);
         assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
-        Ok(())
+
+        let img = image::open(avatar_src).unwrap();
+        assert_eq!(img.width(), 1000);
+        assert_eq!(img.height(), 1000);
+
+        let img = image::open(avatar_blob).unwrap();
+        assert_eq!(img.width(), AVATAR_SIZE);
+        assert_eq!(img.height(), AVATAR_SIZE);
     }
 
     #[test]
-    fn test_selfavatar_in_blobdir() -> failure::Fallible<()> {
+    fn test_selfavatar_in_blobdir() {
         let t = dummy_context();
-        let avatar_src = t.ctx.get_blobdir().join("avatar.jpg");
-        std::fs::write(&avatar_src, b"avatar")?;
+        let avatar_src = t.ctx.get_blobdir().join("avatar.png");
+        let avatar_bytes = include_bytes!("../test-data/image/avatar900x900.png");
+        File::create(&avatar_src)
+            .unwrap()
+            .write_all(avatar_bytes)
+            .unwrap();
+
+        let img = image::open(&avatar_src).unwrap();
+        assert_eq!(img.width(), 900);
+        assert_eq!(img.height(), 900);
+
         t.ctx
-            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))?;
+            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+            .unwrap();
         let avatar_cfg = t.ctx.get_config(Config::Selfavatar);
         assert_eq!(avatar_cfg, avatar_src.to_str().map(|s| s.to_string()));
-        Ok(())
+
+        let img = image::open(avatar_src).unwrap();
+        assert_eq!(img.width(), AVATAR_SIZE);
+        assert_eq!(img.height(), AVATAR_SIZE);
     }
 }

src/configure/mod.rs

@@ -6,15 +6,16 @@ mod read_url;
 
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 
+use async_std::task;
+
 use crate::config::Config;
 use crate::constants::*;
 use crate::context::Context;
 use crate::dc_tools::*;
 use crate::e2ee;
-use crate::job::*;
+use crate::job;
 use crate::login_param::{CertificateChecks, LoginParam};
 use crate::oauth2::*;
-use crate::param::Params;
 
 use auto_mozilla::moz_autoconfigure;
 use auto_outlook::outlk_autodiscover;
@@ -29,34 +30,19 @@ macro_rules! progress {
     };
 }
 
-// connect
-pub fn configure(context: &Context) {
-    if context.has_ongoing() {
-        warn!(context, "There is already another ongoing process running.",);
-        return;
-    }
-    job_kill_action(context, Action::ConfigureImap);
-    job_add(context, Action::ConfigureImap, 0, Params::new(), 0);
-}
-
-/// Check if the context is already configured.
-pub fn dc_is_configured(context: &Context) -> bool {
-    context.sql.get_raw_config_bool(context, "configured")
-}
-
 /*******************************************************************************
  * Configure JOB
  ******************************************************************************/
-#[allow(non_snake_case, unused_must_use)]
-pub fn JobConfigureImap(context: &Context) {
+#[allow(non_snake_case, unused_must_use, clippy::cognitive_complexity)]
+pub fn JobConfigureImap(context: &Context) -> job::Status {
     if !context.sql.is_open() {
         error!(context, "Cannot configure, database not opened.",);
         progress!(context, 0);
-        return;
+        return job::Status::Finished(Err(format_err!("Database not opened")));
     }
     if !context.alloc_ongoing() {
         progress!(context, 0);
-        return;
+        return job::Status::Finished(Err(format_err!("Cannot allocated ongoing process")));
     }
     let mut success = false;
     let mut imap_connected_here = false;
@@ -198,10 +184,7 @@ pub fn JobConfigureImap(context: &Context) {
             7 => {
                 progress!(context, 310);
                 if param_autoconfig.is_none() {
-                    let url = format!(
-                        "https://{}{}/autodiscover/autodiscover.xml",
-                        "", param_domain
-                    );
+                    let url = format!("https://{}/autodiscover/autodiscover.xml", param_domain);
                     param_autoconfig = outlk_autodiscover(context, &url, &param).ok();
                 }
                 true
@@ -439,6 +422,7 @@ pub fn JobConfigureImap(context: &Context) {
 
     context.free_ongoing();
     progress!(context, if success { 1000 } else { 0 });
+    job::Status::Finished(Ok(()))
 }
 
 fn get_offline_autoconfig(context: &Context, param: &LoginParam) -> Option<LoginParam> {
@@ -540,13 +524,14 @@ fn try_imap_one_param(context: &Context, param: &LoginParam) -> Option<bool> {
         param.imap_certificate_checks
     );
     info!(context, "Trying: {}", inf);
-    if context
-        .inbox_thread
-        .read()
-        .unwrap()
-        .imap
-        .connect(context, &param)
-    {
+    if task::block_on(
+        context
+            .inbox_thread
+            .read()
+            .unwrap()
+            .imap
+            .connect(context, &param),
+    ) {
         info!(context, "success: {}", inf);
         return Some(true);
     }

src/configure/read_url.rs

@@ -11,10 +11,10 @@ pub type Result<T> = std::result::Result<T, Error>;
 pub fn read_url(context: &Context, url: &str) -> Result<String> {
     info!(context, "Requesting URL {}", url);
 
-    match reqwest::Client::new()
+    match reqwest::blocking::Client::new()
         .get(url)
         .send()
-        .and_then(|mut res| res.text())
+        .and_then(|res| res.text())
     {
         Ok(res) => Ok(res),
         Err(err) => {

src/constants.rs

@@ -1,5 +1,5 @@
 //! # Constants
-#![allow(non_camel_case_types, dead_code)]
+#![allow(dead_code)]
 
 use deltachat_derive::*;
 use lazy_static::lazy_static;
@@ -49,6 +49,8 @@ pub const DC_HANDSHAKE_CONTINUE_NORMAL_PROCESSING: i32 = 0x01;
 pub const DC_HANDSHAKE_STOP_NORMAL_PROCESSING: i32 = 0x02;
 pub const DC_HANDSHAKE_ADD_DELETE_JOB: i32 = 0x04;
 
+pub(crate) const DC_FROM_HANDSHAKE: i32 = 0x01;
+
 pub const DC_GCL_ARCHIVED_ONLY: usize = 0x01;
 pub const DC_GCL_NO_SPECIALS: usize = 0x02;
 pub const DC_GCL_ADD_ALLDONE_HINT: usize = 0x04;
@@ -61,10 +63,6 @@ pub const DC_GCL_ADD_SELF: usize = 0x02;
 // unchanged user avatars are resent to the recipients every some days
 pub const DC_RESEND_USER_AVATAR_DAYS: i64 = 14;
 
-// values for DC_PARAM_FORCE_PLAINTEXT
-pub(crate) const DC_FP_NO_AUTOCRYPT_HEADER: i32 = 2;
-pub(crate) const DC_FP_ADD_AUTOCRYPT_HEADER: i32 = 1;
-
 /// virtual chat showing all messages belonging to chats flagged with chats.blocked=2
 pub(crate) const DC_CHAT_ID_DEADDROP: u32 = 1;
 /// messages that should be deleted get this chat_id; the messages are deleted from the working thread later then. This is also needed as rfc724_mid should be preset as long as the message is not deleted on the server (otherwise it is downloaded again)
@@ -114,7 +112,7 @@ pub const DC_MSG_ID_LAST_SPECIAL: u32 = 9;
 /// approx. max. length returned by dc_msg_get_text()
 const DC_MAX_GET_TEXT_LEN: usize = 30000;
 /// approx. max. length returned by dc_get_msg_info()
-const DC_MAX_GET_INFO_LEN: usize = 100000;
+const DC_MAX_GET_INFO_LEN: usize = 100_000;
 
 pub const DC_CONTACT_ID_UNDEFINED: u32 = 0;
 pub const DC_CONTACT_ID_SELF: u32 = 1;
@@ -184,10 +182,14 @@ pub const DC_VC_CONTACT_CONFIRM: i32 = 6;
 pub const DC_BOB_ERROR: i32 = 0;
 pub const DC_BOB_SUCCESS: i32 = 1;
 
+// max. width/height of an avatar
+pub const AVATAR_SIZE: u32 = 192;
+
 #[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
 #[repr(i32)]
 pub enum Viewtype {
     Unknown = 0,
+
     /// Text message.
     /// The text of the message is set using dc_msg_set_text()
     /// and retrieved with dc_msg_get_text().

src/contact.rs

@@ -1,3 +1,5 @@
+//! Contacts module
+
 use std::path::PathBuf;
 
 use deltachat_derive::*;
@@ -5,6 +7,7 @@ use itertools::Itertools;
 use rusqlite;
 
 use crate::aheader::EncryptPreference;
+use crate::chat::ChatId;
 use crate::config::Config;
 use crate::constants::*;
 use crate::context::Context;
@@ -46,20 +49,26 @@ pub struct Contact {
     ///
     /// Normal contact IDs are larger than these special ones (larger than DC_CONTACT_ID_LAST_SPECIAL).
     pub id: u32,
+
     /// Contact name. It is recommended to use `Contact::get_name`,
     /// `Contact::get_display_name` or `Contact::get_name_n_addr` to access this field.
     /// May be empty, initially set to `authname`.
     name: String,
+
     /// Name authorized by the contact himself. Only this name may be spread to others,
     /// e.g. in To:-lists. May be empty. It is recommended to use `Contact::get_authname`,
     /// to access this field.
     authname: String,
+
     /// E-Mail-Address of the contact. It is recommended to use `Contact::get_addr`` to access this field.
     addr: String,
+
     /// Blocked state. Use dc_contact_is_blocked to access this field.
-    blocked: bool,
+    pub blocked: bool,
+
     /// The origin/source of the contact.
     pub origin: Origin,
+
     /// Parameters as Param::ProfileImage
     pub param: Params,
 }
@@ -71,38 +80,54 @@ pub struct Contact {
 #[repr(i32)]
 pub enum Origin {
     Unknown = 0,
+
     /// From: of incoming messages of unknown sender
     IncomingUnknownFrom = 0x10,
+
     /// Cc: of incoming messages of unknown sender
     IncomingUnknownCc = 0x20,
+
     /// To: of incoming messages of unknown sender
     IncomingUnknownTo = 0x40,
+
     /// address scanned but not verified
     UnhandledQrScan = 0x80,
+
     /// Reply-To: of incoming message of known sender
     IncomingReplyTo = 0x100,
+
     /// Cc: of incoming message of known sender
     IncomingCc = 0x200,
+
     /// additional To:'s of incoming message of known sender
     IncomingTo = 0x400,
+
     /// a chat was manually created for this user, but no message yet sent
     CreateChat = 0x800,
+
     /// message sent by us
     OutgoingBcc = 0x1000,
+
     /// message sent by us
     OutgoingCc = 0x2000,
+
     /// message sent by us
     OutgoingTo = 0x4000,
+
     /// internal use
     Internal = 0x40000,
+
     /// address is in our address book
     AdressBook = 0x80000,
+
     /// set on Alice's side for contacts like Bob that have scanned the QR code offered by her. Only means the contact has once been established using the "securejoin" procedure in the past, getting the current key verification status requires calling dc_contact_is_verified() !
-    SecurejoinInvited = 0x1000000,
+    SecurejoinInvited = 0x0100_0000,
+
     /// set on Bob's side for contacts scanned and verified from a QR code. Only means the contact has once been established using the "securejoin" procedure in the past, getting the current key verification status requires calling dc_contact_is_verified() !
-    SecurejoinJoined = 0x2000000,
-    /// contact added mannually by dc_create_contact(), this should be the largets origin as otherwise the user cannot modify the names
-    ManuallyCreated = 0x4000000,
+    SecurejoinJoined = 0x0200_0000,
+
+    /// contact added mannually by dc_create_contact(), this should be the largest origin as otherwise the user cannot modify the names
+    ManuallyCreated = 0x0400_0000,
 }
 
 impl Default for Origin {
@@ -112,14 +137,11 @@ impl Default for Origin {
 }
 
 impl Origin {
-    /// Contacts that are verified and known not to be spam.
-    pub fn is_verified(self) -> bool {
-        self as i32 >= 0x100
-    }
-
-    /// Contacts that are shown in the contact list.
-    pub fn include_in_contactlist(self) -> bool {
-        self as i32 >= DC_ORIGIN_MIN_CONTACT_LIST
+    /// Contacts that are known, i. e. they came in via accepted contacts or
+    /// themselves an accepted contact. Known contacts are shown in the
+    /// contact list when one creates a chat and wants to add members etc.
+    pub fn is_known(self) -> bool {
+        self >= Origin::IncomingReplyTo
     }
 }
 
@@ -241,7 +263,7 @@ impl Contact {
         .is_ok()
         {
             context.call_cb(Event::MsgsChanged {
-                chat_id: 0,
+                chat_id: ChatId::new(0),
                 msg_id: MsgId::new(0),
             });
         }
@@ -389,18 +411,30 @@ impl Contact {
                 }
                 sth_modified = Modifier::Modified;
             }
-        } else if sql::execute(
-            context,
-            &context.sql,
-            "INSERT INTO contacts (name, addr, origin) VALUES(?, ?, ?);",
-            params![name.as_ref(), addr, origin,],
-        )
-        .is_ok()
-        {
-            row_id = sql::get_rowid(context, &context.sql, "contacts", "addr", addr);
-            sth_modified = Modifier::Created;
         } else {
-            error!(context, "Cannot add contact.");
+            if origin == Origin::IncomingUnknownFrom {
+                update_authname = true;
+            }
+
+            if sql::execute(
+                context,
+                &context.sql,
+                "INSERT INTO contacts (name, addr, origin, authname) VALUES(?, ?, ?, ?);",
+                params![
+                    name.as_ref(),
+                    addr,
+                    origin,
+                    if update_authname { name.as_ref() } else { "" }
+                ],
+            )
+            .is_ok()
+            {
+                row_id = sql::get_rowid(context, &context.sql, "contacts", "addr", addr);
+                sth_modified = Modifier::Created;
+                info!(context, "added contact id={} addr={}", row_id, addr);
+            } else {
+                error!(context, "Cannot add contact.");
+            }
         }
 
         Ok((row_id, sth_modified))
@@ -484,7 +518,7 @@ impl Contact {
                 params![
                     self_addr,
                     DC_CONTACT_ID_LAST_SPECIAL as i32,
-                    0x100,
+                    Origin::IncomingReplyTo,
                     &s3str_like_cmd,
                     &s3str_like_cmd,
                     if flag_verified_only { 0 } else { 1 },
@@ -870,22 +904,6 @@ impl Contact {
             .unwrap_or_default() as usize
     }
 
-    pub fn get_origin_by_id(context: &Context, contact_id: u32, ret_blocked: &mut bool) -> Origin {
-        let mut ret = Origin::Unknown;
-        *ret_blocked = false;
-
-        if let Ok(contact) = Contact::load_from_db(context, contact_id) {
-            /* we could optimize this by loading only the needed fields */
-            if contact.blocked {
-                *ret_blocked = true;
-            } else {
-                ret = contact.origin;
-            }
-        }
-
-        ret
-    }
-
     pub fn real_exists_by_id(context: &Context, contact_id: u32) -> bool {
         if !context.sql.is_open() || contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
             return false;
@@ -969,7 +987,7 @@ fn set_block_contact(context: &Context, contact_id: u32, new_blocking: bool) {
 pub fn set_profile_image(
     context: &Context,
     contact_id: u32,
-    profile_image: AvatarAction,
+    profile_image: &AvatarAction,
 ) -> Result<()> {
     // the given profile image is expected to be already in the blob directory
     // as profile images can be set only by receiving messages, this should be always the case, however.
@@ -1259,6 +1277,63 @@ mod tests {
         assert!(!contact.is_blocked());
     }
 
+    #[test]
+    fn test_remote_authnames() {
+        let t = dummy_context();
+
+        let (contact_id, sth_modified) = Contact::add_or_lookup(
+            &t.ctx,
+            "bob1",
+            "bob@example.org",
+            Origin::IncomingUnknownFrom,
+        )
+        .unwrap();
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert_eq!(sth_modified, Modifier::Created);
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_authname(), "bob1");
+        assert_eq!(contact.get_name(), "bob1");
+        assert_eq!(contact.get_display_name(), "bob1");
+
+        let (contact_id, sth_modified) = Contact::add_or_lookup(
+            &t.ctx,
+            "bob2",
+            "bob@example.org",
+            Origin::IncomingUnknownFrom,
+        )
+        .unwrap();
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert_eq!(sth_modified, Modifier::Modified);
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_authname(), "bob2");
+        assert_eq!(contact.get_name(), "bob2");
+        assert_eq!(contact.get_display_name(), "bob2");
+
+        let (contact_id, sth_modified) =
+            Contact::add_or_lookup(&t.ctx, "bob3", "bob@example.org", Origin::ManuallyCreated)
+                .unwrap();
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert_eq!(sth_modified, Modifier::Modified);
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_authname(), "bob2");
+        assert_eq!(contact.get_name(), "bob3");
+        assert_eq!(contact.get_display_name(), "bob3");
+
+        let (contact_id, sth_modified) = Contact::add_or_lookup(
+            &t.ctx,
+            "bob4",
+            "bob@example.org",
+            Origin::IncomingUnknownFrom,
+        )
+        .unwrap();
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert_eq!(sth_modified, Modifier::Modified);
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_authname(), "bob4");
+        assert_eq!(contact.get_name(), "bob3");
+        assert_eq!(contact.get_display_name(), "bob3");
+    }
+
     #[test]
     fn test_addr_cmp() {
         assert!(addr_cmp("AA@AA.ORG", "aa@aa.ORG"));

src/context.rs

@@ -1,11 +1,15 @@
-//! Contacts module
+//! Context module
 
 use std::collections::HashMap;
 use std::ffi::OsString;
 use std::path::{Path, PathBuf};
-use std::sync::{Arc, Condvar, Mutex, RwLock};
+use std::sync::{
+    atomic::{AtomicBool, Ordering},
+    Arc, Condvar, Mutex, RwLock,
+};
 
-use libc::uintptr_t;
+use crossbeam::channel::select;
+use crossbeam::channel::{bounded, unbounded, Receiver, Sender};
 
 use crate::chat::*;
 use crate::config::Config;
@@ -14,12 +18,12 @@ use crate::contact::*;
 use crate::error::*;
 use crate::events::Event;
 use crate::imap::*;
-use crate::job::*;
+use crate::job;
 use crate::job_thread::JobThread;
 use crate::key::*;
 use crate::login_param::LoginParam;
 use crate::lot::Lot;
-use crate::message::{self, Message, MsgId};
+use crate::message::{self, Message, MessengerMessage, MsgId};
 use crate::param::Params;
 use crate::smtp::Smtp;
 use crate::sql::Sql;
@@ -32,14 +36,9 @@ use crate::sql::Sql;
 /// * `event` - One of the [Event] items.
 /// * `data1` - Depends on the event parameter, see [Event].
 /// * `data2` - Depends on the event parameter, see [Event].
-///
-/// # Returns
-///
-/// This callback must return 0 unless stated otherwise in the event
-/// description at [Event].
-pub type ContextCallback = dyn Fn(&Context, Event) -> uintptr_t + Send + Sync;
+pub type ContextCallback = dyn Fn(&Context, Event) -> () + Send + Sync;
 
-#[derive(DebugStub)]
+#[derive(Debug)]
 pub struct Context {
     /// Database file path
     dbfile: PathBuf,
@@ -54,16 +53,21 @@ pub struct Context {
     pub smtp: Arc<Mutex<Smtp>>,
     pub smtp_state: Arc<(Mutex<SmtpState>, Condvar)>,
     pub oauth2_critical: Arc<Mutex<()>>,
-    #[debug_stub = "Callback"]
-    cb: Box<ContextCallback>,
     pub os_name: Option<String>,
-    pub cmdline_sel_chat_id: Arc<RwLock<u32>>,
+    pub cmdline_sel_chat_id: Arc<RwLock<ChatId>>,
     pub bob: Arc<RwLock<BobStatus>>,
     pub last_smeared_timestamp: RwLock<i64>,
     pub running_state: Arc<RwLock<RunningState>>,
     /// Mutex to avoid generating the key for the user more than once.
     pub generating_key_mutex: Mutex<()>,
     pub translated_stockstrings: RwLock<HashMap<usize, String>>,
+
+    event_sender: Sender<Event>,
+    event_receiver: Receiver<Event>,
+    shutdown_sender: Sender<()>,
+    shutdown_receiver: Receiver<()>,
+
+    is_running: AtomicBool,
 }
 
 #[derive(Debug, PartialEq, Eq)]
@@ -82,23 +86,24 @@ pub fn get_info() -> HashMap<&'static str, String> {
     let mut res = HashMap::new();
     res.insert("deltachat_core_version", format!("v{}", &*DC_VERSION_STR));
     res.insert("sqlite_version", rusqlite::version().to_string());
-    res.insert(
-        "sqlite_thread_safe",
-        unsafe { rusqlite::ffi::sqlite3_threadsafe() }.to_string(),
-    );
-    res.insert(
-        "arch",
-        (std::mem::size_of::<*mut libc::c_void>())
-            .wrapping_mul(8)
-            .to_string(),
-    );
+    res.insert("arch", (std::mem::size_of::<usize>() * 8).to_string());
     res.insert("level", "awesome".into());
     res
 }
 
+macro_rules! while_running {
+    ($self:expr, $code:block) => {
+        if $self.is_running.load(Ordering::Relaxed) {
+            $code
+        } else {
+            break;
+        }
+    };
+}
+
 impl Context {
     /// Creates new context.
-    pub fn new(cb: Box<ContextCallback>, os_name: String, dbfile: PathBuf) -> Result<Context> {
+    pub fn new(os_name: String, dbfile: PathBuf) -> Result<Context> {
         let mut blob_fname = OsString::new();
         blob_fname.push(dbfile.file_name().unwrap_or_default());
         blob_fname.push("-blobs");
@@ -106,24 +111,22 @@ impl Context {
         if !blobdir.exists() {
             std::fs::create_dir_all(&blobdir)?;
         }
-        Context::with_blobdir(cb, os_name, dbfile, blobdir)
+        Context::with_blobdir(os_name, dbfile, blobdir)
     }
 
-    pub fn with_blobdir(
-        cb: Box<ContextCallback>,
-        os_name: String,
-        dbfile: PathBuf,
-        blobdir: PathBuf,
-    ) -> Result<Context> {
+    pub fn with_blobdir(os_name: String, dbfile: PathBuf, blobdir: PathBuf) -> Result<Context> {
         ensure!(
             blobdir.is_dir(),
             "Blobdir does not exist: {}",
             blobdir.display()
         );
+
+        let (event_sender, event_receiver) = unbounded();
+        let (shutdown_sender, shutdown_receiver) = bounded(0);
+
         let ctx = Context {
             blobdir,
             dbfile,
-            cb,
             os_name: Some(os_name),
             running_state: Arc::new(RwLock::new(Default::default())),
             sql: Sql::new(),
@@ -132,7 +135,7 @@ impl Context {
             oauth2_critical: Arc::new(Mutex::new(())),
             bob: Arc::new(RwLock::new(Default::default())),
             last_smeared_timestamp: RwLock::new(0),
-            cmdline_sel_chat_id: Arc::new(RwLock::new(0)),
+            cmdline_sel_chat_id: Arc::new(RwLock::new(ChatId::new(0))),
             inbox_thread: Arc::new(RwLock::new(JobThread::new(
                 "INBOX",
                 "configured_inbox_folder",
@@ -152,6 +155,11 @@ impl Context {
             perform_inbox_jobs_needed: Arc::new(RwLock::new(false)),
             generating_key_mutex: Mutex::new(()),
             translated_stockstrings: RwLock::new(HashMap::new()),
+            event_sender,
+            event_receiver,
+            shutdown_sender,
+            shutdown_receiver,
+            is_running: Default::default(),
         };
 
         ensure!(
@@ -172,8 +180,99 @@ impl Context {
         self.blobdir.as_path()
     }
 
-    pub fn call_cb(&self, event: Event) -> uintptr_t {
-        (*self.cb)(self, event)
+    pub fn call_cb(&self, event: Event) {
+        self.event_sender.send(event).unwrap();
+    }
+
+    /// Start the run loop.
+    pub fn run<F>(&self, cb: F)
+    where
+        F: Fn(&Context, Event) -> () + Send + Sync,
+    {
+        if self.is_running.swap(true, Ordering::Relaxed) {
+            panic!("Run can only be called once");
+        }
+
+        crossbeam::scope(|s| {
+            let imap_handle = s.spawn(|_| loop {
+                while_running!(self, {
+                    job::perform_inbox_jobs(self);
+                    while_running!(self, {
+                        job::perform_inbox_fetch(self);
+                        while_running!(self, { job::perform_inbox_idle(self) });
+                    });
+                });
+            });
+            let mvbox_handle = s.spawn(|_| loop {
+                while_running!(self, {
+                    job::perform_mvbox_fetch(self);
+                    while_running!(self, {
+                        job::perform_mvbox_idle(self);
+                    });
+                });
+            });
+            let sentbox_handle = s.spawn(|_| loop {
+                while_running!(self, {
+                    job::perform_sentbox_fetch(self);
+                    while_running!(self, {
+                        job::perform_sentbox_idle(self);
+                    });
+                });
+            });
+            let smtp_handle = s.spawn(|_| loop {
+                while_running!(self, {
+                    job::perform_smtp_jobs(self);
+                    while_running!(self, {
+                        job::perform_smtp_idle(self);
+                    });
+                });
+            });
+
+            loop {
+                select! {
+                    recv(self.event_receiver) -> event => {
+                        // This gurantees that the callback is always called from the thread
+                        // that called `run`.
+                        cb(self, event.unwrap())
+                    },
+                    recv(self.shutdown_receiver) -> _ => break,
+                }
+            }
+
+            imap_handle.join().unwrap();
+            mvbox_handle.join().unwrap();
+            sentbox_handle.join().unwrap();
+            smtp_handle.join().unwrap();
+        })
+        .unwrap()
+    }
+
+    /// Stop the run loop. Blocks until all threads have shutdown.
+    pub fn shutdown(&self) {
+        if !self.is_running.swap(false, Ordering::Relaxed) {
+            // already shutdown
+            return;
+        }
+        self.shutdown_sender.send(()).unwrap();
+
+        job::interrupt_inbox_idle(self);
+        job::interrupt_mvbox_idle(self);
+        job::interrupt_sentbox_idle(self);
+        job::interrupt_smtp_idle(self);
+    }
+
+    pub fn configure(&self) {
+        if self.has_ongoing() {
+            warn!(self, "There is already another ongoing process running.");
+            return;
+        }
+        job::job_kill_action(self, job::Action::ConfigureImap);
+        job::job_add(self, job::Action::ConfigureImap, 0, Params::new(), 0);
+    }
+
+    /// Check if the context is already configured.
+    pub fn is_configured(&self) -> bool {
+        self.sql.get_raw_config_bool(self, "configured")
     }
 
     /*******************************************************************************
@@ -361,7 +460,7 @@ impl Context {
     }
 
     #[allow(non_snake_case)]
-    pub fn search_msgs(&self, chat_id: u32, query: impl AsRef<str>) -> Vec<MsgId> {
+    pub fn search_msgs(&self, chat_id: ChatId, query: impl AsRef<str>) -> Vec<MsgId> {
         let real_query = query.as_ref().trim();
         if real_query.is_empty() {
             return Vec::new();
@@ -369,7 +468,7 @@ impl Context {
         let strLikeInText = format!("%{}%", real_query);
         let strLikeBeg = format!("{}%", real_query);
 
-        let query = if 0 != chat_id {
+        let query = if !chat_id.is_unset() {
             concat!(
                 "SELECT m.id AS id, m.timestamp AS timestamp",
                 " FROM msgs m",
@@ -401,7 +500,7 @@ impl Context {
         self.sql
             .query_map(
                 query,
-                params![chat_id as i32, &strLikeInText, &strLikeBeg],
+                params![chat_id, &strLikeInText, &strLikeBeg],
                 |row| row.get::<_, MsgId>("id"),
                 |rows| {
                     let mut ret = Vec::new();
@@ -452,15 +551,17 @@ impl Context {
                 return;
             }
 
-            // 1 = dc message, 2 = reply to dc message
-            if 0 != msg.is_dc_message {
-                job_add(
-                    self,
-                    Action::MoveMsg,
-                    msg.id.to_u32() as i32,
-                    Params::new(),
-                    0,
-                );
+            match msg.is_dc_message {
+                MessengerMessage::No => {}
+                MessengerMessage::Yes | MessengerMessage::Reply => {
+                    job::job_add(
+                        self,
+                        job::Action::MoveMsg,
+                        msg.id.to_u32() as i32,
+                        Params::new(),
+                        0,
+                    );
+                }
             }
         }
     }
@@ -533,7 +634,7 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         std::fs::write(&dbfile, b"123").unwrap();
-        let res = Context::new(Box::new(|_, _| 0), "FakeOs".into(), dbfile);
+        let res = Context::new("FakeOs".into(), dbfile);
         assert!(res.is_err());
     }
 
@@ -548,7 +649,7 @@ mod tests {
     fn test_blobdir_exists() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
-        Context::new(Box::new(|_, _| 0), "FakeOS".into(), dbfile).unwrap();
+        Context::new("FakeOS".into(), dbfile).unwrap();
         let blobdir = tmp.path().join("db.sqlite-blobs");
         assert!(blobdir.is_dir());
     }
@@ -559,7 +660,7 @@ mod tests {
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("db.sqlite-blobs");
         std::fs::write(&blobdir, b"123").unwrap();
-        let res = Context::new(Box::new(|_, _| 0), "FakeOS".into(), dbfile);
+        let res = Context::new("FakeOS".into(), dbfile);
         assert!(res.is_err());
     }
 
@@ -569,7 +670,7 @@ mod tests {
         let subdir = tmp.path().join("subdir");
         let dbfile = subdir.join("db.sqlite");
         let dbfile2 = dbfile.clone();
-        Context::new(Box::new(|_, _| 0), "FakeOS".into(), dbfile).unwrap();
+        Context::new("FakeOS".into(), dbfile).unwrap();
         assert!(subdir.is_dir());
         assert!(dbfile2.is_file());
     }
@@ -579,7 +680,7 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = PathBuf::new();
-        let res = Context::with_blobdir(Box::new(|_, _| 0), "FakeOS".into(), dbfile, blobdir);
+        let res = Context::with_blobdir("FakeOS".into(), dbfile, blobdir);
         assert!(res.is_err());
     }
 
@@ -588,7 +689,7 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("blobs");
-        let res = Context::with_blobdir(Box::new(|_, _| 0), "FakeOS".into(), dbfile, blobdir);
+        let res = Context::with_blobdir("FakeOS".into(), dbfile, blobdir);
         assert!(res.is_err());
     }
 

src/dc_simplify.rs

@@ -1,273 +0,0 @@
-use crate::dehtml::*;
-
-#[derive(Copy, Clone)]
-pub struct Simplify {
-    pub is_forwarded: bool,
-}
-
-/// Return index of footer line in vector of message lines, or vector length if
-/// no footer is found.
-///
-/// Also return whether not-standard (rfc3676, §4.3) footer is found.
-fn find_message_footer(lines: &[&str]) -> (usize, bool) {
-    for (ix, &line) in lines.iter().enumerate() {
-        // quoted-printable may encode `-- ` to `-- =20` which is converted
-        // back to `--  `
-        match line {
-            "-- " | "--  " => return (ix, false),
-            "--" | "---" | "----" => return (ix, true),
-            _ => (),
-        }
-    }
-    (lines.len(), false)
-}
-
-impl Simplify {
-    pub fn new() -> Self {
-        Simplify {
-            is_forwarded: false,
-        }
-    }
-
-    /// Simplify and normalise text: Remove quotes, signatures, unnecessary
-    /// lineends etc.
-    /// The data returned from simplify() must be free()'d when no longer used.
-    pub fn simplify(&mut self, input: &str, is_html: bool, is_msgrmsg: bool) -> String {
-        let mut out = if is_html {
-            dehtml(input)
-        } else {
-            input.to_string()
-        };
-
-        out.retain(|c| c != '\r');
-        out = self.simplify_plain_text(&out, is_msgrmsg);
-        out.retain(|c| c != '\r');
-
-        out
-    }
-
-    /**
-     * Simplify Plain Text
-     */
-    #[allow(non_snake_case, clippy::mut_range_bound, clippy::needless_range_loop)]
-    fn simplify_plain_text(&mut self, buf_terminated: &str, is_msgrmsg: bool) -> String {
-        /* This function ...
-        ... removes all text after the line `-- ` (footer mark)
-        ... removes full quotes at the beginning and at the end of the text -
-            these are all lines starting with the character `>`
-        ... remove a non-empty line before the removed quote (contains sth. like "On 2.9.2016, Bjoern wrote:" in different formats and lanugages) */
-        /* split the given buffer into lines */
-        let lines: Vec<_> = buf_terminated.split('\n').collect();
-        let mut l_first: usize = 0;
-        let mut is_cut_at_begin = false;
-        let (mut l_last, mut is_cut_at_end) = find_message_footer(&lines);
-
-        if l_last > l_first + 2 {
-            let line0 = lines[l_first];
-            let line1 = lines[l_first + 1];
-            let line2 = lines[l_first + 2];
-            if line0 == "---------- Forwarded message ----------"
-                && line1.starts_with("From: ")
-                && line2.is_empty()
-            {
-                self.is_forwarded = true;
-                l_first += 3
-            }
-        }
-        for l in l_first..l_last {
-            let line = lines[l];
-            if line == "-----"
-                || line == "_____"
-                || line == "====="
-                || line == "*****"
-                || line == "~~~~~"
-            {
-                l_last = l;
-                is_cut_at_end = true;
-                /* done */
-                break;
-            }
-        }
-        if !is_msgrmsg {
-            let mut l_lastQuotedLine = None;
-            for l in (l_first..l_last).rev() {
-                let line = lines[l];
-                if is_plain_quote(line) {
-                    l_lastQuotedLine = Some(l)
-                } else if !is_empty_line(line) {
-                    break;
-                }
-            }
-            if let Some(last_quoted_line) = l_lastQuotedLine {
-                l_last = last_quoted_line;
-                is_cut_at_end = true;
-                if l_last > 1 && is_empty_line(lines[l_last - 1]) {
-                    l_last -= 1
-                }
-                if l_last > 1 {
-                    let line = lines[l_last - 1];
-                    if is_quoted_headline(line) {
-                        l_last -= 1
-                    }
-                }
-            }
-        }
-        if !is_msgrmsg {
-            let mut l_lastQuotedLine_0 = None;
-            let mut hasQuotedHeadline = 0;
-            for l in l_first..l_last {
-                let line = lines[l];
-                if is_plain_quote(line) {
-                    l_lastQuotedLine_0 = Some(l)
-                } else if !is_empty_line(line) {
-                    if is_quoted_headline(line)
-                        && 0 == hasQuotedHeadline
-                        && l_lastQuotedLine_0.is_none()
-                    {
-                        hasQuotedHeadline = 1i32
-                    } else {
-                        /* non-quoting line found */
-                        break;
-                    }
-                }
-            }
-            if let Some(last_quoted_line) = l_lastQuotedLine_0 {
-                l_first = last_quoted_line + 1;
-                is_cut_at_begin = true
-            }
-        }
-        /* re-create buffer from the remaining lines */
-        let mut ret = String::new();
-        if is_cut_at_begin {
-            ret += "[...]";
-        }
-        /* we write empty lines only in case and non-empty line follows */
-        let mut pending_linebreaks = 0;
-        let mut content_lines_added = 0;
-        for l in l_first..l_last {
-            let line = lines[l];
-            if is_empty_line(line) {
-                pending_linebreaks += 1
-            } else {
-                if 0 != content_lines_added {
-                    if pending_linebreaks > 2i32 {
-                        pending_linebreaks = 2i32
-                    }
-                    while 0 != pending_linebreaks {
-                        ret += "\n";
-                        pending_linebreaks -= 1
-                    }
-                }
-                // the incoming message might contain invalid UTF8
-                ret += line;
-                content_lines_added += 1;
-                pending_linebreaks = 1i32
-            }
-        }
-        if is_cut_at_end && (!is_cut_at_begin || 0 != content_lines_added) {
-            ret += " [...]";
-        }
-
-        ret
-    }
-}
-
-/**
- * Tools
- */
-fn is_empty_line(buf: &str) -> bool {
-    // XXX: can it be simplified to buf.chars().all(|c| c.is_whitespace())?
-    //
-    // Strictly speaking, it is not equivalent (^A is not whitespace, but less than ' '),
-    // but having control sequences in email body?!
-    //
-    // See discussion at: https://github.com/deltachat/deltachat-core-rust/pull/402#discussion_r317062392
-    for c in buf.chars() {
-        if c > ' ' {
-            return false;
-        }
-    }
-
-    true
-}
-
-fn is_quoted_headline(buf: &str) -> bool {
-    /* This function may be called for the line _directly_ before a quote.
-    The function checks if the line contains sth. like "On 01.02.2016, xy@z wrote:" in various languages.
-    - Currently, we simply check if the last character is a ':'.
-    - Checking for the existence of an email address may fail (headlines may show the user's name instead of the address) */
-
-    buf.len() <= 80 && buf.ends_with(':')
-}
-
-fn is_plain_quote(buf: &str) -> bool {
-    buf.starts_with('>')
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use proptest::prelude::*;
-
-    proptest! {
-        #[test]
-        // proptest does not support [[:graphical:][:space:]] regex.
-        fn test_simplify_plain_text_fuzzy(input in "[!-~\t \n]+") {
-            let output = Simplify::new().simplify_plain_text(&input, true);
-            assert!(output.split('\n').all(|s| s != "-- "));
-        }
-    }
-
-    #[test]
-    fn test_simplify_trim() {
-        let mut simplify = Simplify::new();
-        let html = "\r\r\nline1<br>\r\n\r\n\r\rline2\n\r";
-        let plain = simplify.simplify(html, true, false);
-
-        assert_eq!(plain, "line1\nline2");
-    }
-
-    #[test]
-    fn test_simplify_parse_href() {
-        let mut simplify = Simplify::new();
-        let html = "<a href=url>text</a";
-        let plain = simplify.simplify(html, true, false);
-
-        assert_eq!(plain, "[text](url)");
-    }
-
-    #[test]
-    fn test_simplify_bold_text() {
-        let mut simplify = Simplify::new();
-        let html = "<!DOCTYPE name [<!DOCTYPE ...>]><!-- comment -->text <b><?php echo ... ?>bold</b><![CDATA[<>]]>";
-        let plain = simplify.simplify(html, true, false);
-
-        assert_eq!(plain, "text *bold*<>");
-    }
-
-    #[test]
-    fn test_simplify_html_encoded() {
-        let mut simplify = Simplify::new();
-        let html =
-                "&lt;&gt;&quot;&apos;&amp; &auml;&Auml;&ouml;&Ouml;&uuml;&Uuml;&szlig; foo&AElig;&ccedil;&Ccedil; &diams;&lrm;&rlm;&zwnj;&noent;&zwj;";
-
-        let plain = simplify.simplify(html, true, false);
-
-        assert_eq!(
-            plain,
-            "<>\"\'& äÄöÖüÜß fooÆçÇ \u{2666}\u{200e}\u{200f}\u{200c}&noent;\u{200d}"
-        );
-    }
-
-    #[test]
-    fn test_simplify_utilities() {
-        assert!(is_empty_line(" \t"));
-        assert!(is_empty_line(""));
-        assert!(is_empty_line(" \r"));
-        assert!(!is_empty_line(" x"));
-        assert!(is_plain_quote("> hello world"));
-        assert!(is_plain_quote(">>"));
-        assert!(!is_plain_quote("Life is pain"));
-        assert!(!is_plain_quote(""));
-    }
-}

src/dc_tools.rs

@@ -49,8 +49,8 @@ pub(crate) fn dc_truncate(buf: &str, approx_chars: usize, do_unwrap: bool) -> Co
 /// - harmonize together while being different enough
 /// (therefore, we cannot just use random rgb colors :)
 const COLORS: [u32; 16] = [
-    0xe56555, 0xf28c48, 0x8e85ee, 0x76c84d, 0x5bb6cc, 0x549cdd, 0xd25c99, 0xb37800, 0xf23030,
-    0x39b249, 0xbb243b, 0x964078, 0x66874f, 0x308ab9, 0x127ed0, 0xbe450c,
+    0xe5_65_55, 0xf2_8c_48, 0x8e_85_ee, 0x76_c8_4d, 0x5b_b6_cc, 0x54_9c_dd, 0xd2_5c_99, 0xb3_78_00,
+    0xf2_30_30, 0x39_b2_49, 0xbb_24_3b, 0x96_40_78, 0x66_87_4f, 0x30_8a_b9, 0x12_7e_d0, 0xbe_45_0c,
 ];
 
 pub(crate) fn dc_str_to_color(s: impl AsRef<str>) -> u32 {
@@ -59,7 +59,7 @@ pub(crate) fn dc_str_to_color(s: impl AsRef<str>) -> u32 {
     let bytes = str_lower.as_bytes();
     for (i, byte) in bytes.iter().enumerate() {
         checksum += (i + 1) * *byte as usize;
-        checksum %= 0xffffff;
+        checksum %= 0x00ff_ffff;
     }
     let color_index = checksum % COLORS.len();
 
@@ -222,45 +222,7 @@ pub(crate) fn dc_extract_grpid_from_rfc724_mid(mid: &str) -> Option<&str> {
     None
 }
 
-// Function returns a sanitized basename that does not contain
-// win/linux path separators and also not any non-ascii chars
-fn get_safe_basename(filename: &str) -> String {
-    // return the (potentially mangled) basename of the input filename
-    // this might be a path that comes in from another operating system
-    let mut index: usize = 0;
-
-    if let Some(unix_index) = filename.rfind('/') {
-        index = unix_index + 1;
-    }
-    if let Some(win_index) = filename.rfind('\\') {
-        index = max(index, win_index + 1);
-    }
-    if index >= filename.len() {
-        "nobasename".to_string()
-    } else {
-        // we don't allow any non-ascii to be super-safe
-        filename[index..].replace(|c: char| !c.is_ascii() || c == ':', "-")
-    }
-}
-
-pub fn dc_derive_safe_stem_ext(filename: &str) -> (String, String) {
-    let basename = get_safe_basename(&filename);
-    let (mut stem, mut ext) = if let Some(index) = basename.rfind('.') {
-        (
-            basename[0..index].to_string(),
-            basename[index..].to_string(),
-        )
-    } else {
-        (basename, "".to_string())
-    };
-    // limit length of stem and ext
-    stem.truncate(32);
-    ext.truncate(32);
-    (stem, ext)
-}
-
 // the returned suffix is lower-case
-#[allow(non_snake_case)]
 pub fn dc_get_filesuffix_lc(path_filename: impl AsRef<str>) -> Option<String> {
     Path::new(path_filename.as_ref())
         .extension()
@@ -381,11 +343,14 @@ pub(crate) fn dc_copy_file(
     }
 }
 
-pub(crate) fn dc_create_folder(context: &Context, path: impl AsRef<std::path::Path>) -> bool {
+pub(crate) fn dc_create_folder(
+    context: &Context,
+    path: impl AsRef<std::path::Path>,
+) -> Result<(), std::io::Error> {
     let path_abs = dc_get_abs_path(context, &path);
     if !path_abs.exists() {
         match fs::create_dir_all(path_abs) {
-            Ok(_) => true,
+            Ok(_) => Ok(()),
             Err(err) => {
                 warn!(
                     context,
@@ -393,11 +358,11 @@ pub(crate) fn dc_create_folder(context: &Context, path: impl AsRef<std::path::Pa
                     path.as_ref().display(),
                     err
                 );
-                false
+                Err(err)
             }
         }
     } else {
-        true
+        Ok(())
     }
 }
 
@@ -483,7 +448,7 @@ pub(crate) fn dc_get_next_backup_path(
 pub(crate) fn time() -> i64 {
     SystemTime::now()
         .duration_since(SystemTime::UNIX_EPOCH)
-        .unwrap()
+        .unwrap_or_default()
         .as_secs() as i64
 }
 
@@ -495,7 +460,7 @@ pub(crate) fn time() -> i64 {
 ///
 /// ```
 /// use deltachat::dc_tools::EmailAddress;
-/// let email = match EmailAddress::new("someone@example.com") {
+/// let email = match "someone@example.com".parse::<EmailAddress>() {
 ///     Ok(addr) => addr,
 ///     Err(e) => panic!("Error parsing address, error was {}", e),
 /// };
@@ -509,12 +474,6 @@ pub struct EmailAddress {
     pub domain: String,
 }
 
-impl EmailAddress {
-    pub fn new(input: &str) -> Result<Self, Error> {
-        input.parse::<EmailAddress>()
-    }
-}
-
 impl fmt::Display for EmailAddress {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(f, "{}@{}", self.local, self.domain)
@@ -651,7 +610,6 @@ mod tests {
         );
     }
 
-    #[test]
     #[test]
     fn test_dc_extract_grpid_from_rfc724_mid() {
         // Should return None if we pass invalid mid
@@ -706,29 +664,29 @@ mod tests {
 
     #[test]
     fn test_emailaddress_parse() {
-        assert_eq!(EmailAddress::new("").is_ok(), false);
+        assert_eq!("".parse::<EmailAddress>().is_ok(), false);
         assert_eq!(
-            EmailAddress::new("user@domain.tld").unwrap(),
+            "user@domain.tld".parse::<EmailAddress>().unwrap(),
             EmailAddress {
                 local: "user".into(),
                 domain: "domain.tld".into(),
             }
         );
-        assert_eq!(EmailAddress::new("uuu").is_ok(), false);
-        assert_eq!(EmailAddress::new("dd.tt").is_ok(), false);
-        assert_eq!(EmailAddress::new("tt.dd@uu").is_ok(), false);
-        assert_eq!(EmailAddress::new("u@d").is_ok(), false);
-        assert_eq!(EmailAddress::new("u@d.").is_ok(), false);
-        assert_eq!(EmailAddress::new("u@d.t").is_ok(), false);
+        assert_eq!("uuu".parse::<EmailAddress>().is_ok(), false);
+        assert_eq!("dd.tt".parse::<EmailAddress>().is_ok(), false);
+        assert_eq!("tt.dd@uu".parse::<EmailAddress>().is_ok(), false);
+        assert_eq!("u@d".parse::<EmailAddress>().is_ok(), false);
+        assert_eq!("u@d.".parse::<EmailAddress>().is_ok(), false);
+        assert_eq!("u@d.t".parse::<EmailAddress>().is_ok(), false);
         assert_eq!(
-            EmailAddress::new("u@d.tt").unwrap(),
+            "u@d.tt".parse::<EmailAddress>().unwrap(),
             EmailAddress {
                 local: "u".into(),
                 domain: "d.tt".into(),
             }
         );
-        assert_eq!(EmailAddress::new("u@.tt").is_ok(), false);
-        assert_eq!(EmailAddress::new("@d.tt").is_ok(), false);
+        assert_eq!("u@tt".parse::<EmailAddress>().is_ok(), false);
+        assert_eq!("@d.tt".parse::<EmailAddress>().is_ok(), false);
     }
 
     use proptest::prelude::*;
@@ -764,19 +722,6 @@ mod tests {
         }
     }
 
-    #[test]
-    fn test_file_get_safe_basename() {
-        assert_eq!(get_safe_basename("12312/hello"), "hello");
-        assert_eq!(get_safe_basename("12312\\hello"), "hello");
-        assert_eq!(get_safe_basename("//12312\\hello"), "hello");
-        assert_eq!(get_safe_basename("//123:12\\hello"), "hello");
-        assert_eq!(get_safe_basename("//123:12/\\\\hello"), "hello");
-        assert_eq!(get_safe_basename("//123:12//hello"), "hello");
-        assert_eq!(get_safe_basename("//123:12//"), "nobasename");
-        assert_eq!(get_safe_basename("//123:12/"), "nobasename");
-        assert!(get_safe_basename("123\x012.hello").ends_with(".hello"));
-    }
-
     #[test]
     fn test_file_handling() {
         let t = dummy_context();
@@ -825,7 +770,7 @@ mod tests {
 
         assert!(dc_delete_file(context, "$BLOBDIR/foobar"));
         assert!(dc_delete_file(context, "$BLOBDIR/dada"));
-        assert!(dc_create_folder(context, "$BLOBDIR/foobar-folder"));
+        assert!(dc_create_folder(context, "$BLOBDIR/foobar-folder").is_ok());
         assert!(dc_file_exist(context, "$BLOBDIR/foobar-folder",));
         assert!(!dc_delete_file(context, "$BLOBDIR/foobar-folder"));
 

src/dehtml.rs

@@ -139,13 +139,12 @@ fn dehtml_starttag_cb<B: std::io::BufRead>(
             dehtml.add_text = AddText::YesPreserveLineEnds;
         }
         "a" => {
-            if let Some(href) = event.html_attributes().find(|attr| {
-                attr.as_ref()
-                    .map(|a| String::from_utf8_lossy(a.key).trim().to_lowercase() == "href")
-                    .unwrap_or_default()
-            }) {
+            if let Some(href) = event
+                .html_attributes()
+                .filter_map(|attr| attr.ok())
+                .find(|attr| String::from_utf8_lossy(attr.key).trim().to_lowercase() == "href")
+            {
                 let href = href
-                    .unwrap()
                     .unescape_and_decode_value(reader)
                     .unwrap_or_default()
                     .to_lowercase();
@@ -189,4 +188,41 @@ mod tests {
             assert_eq!(dehtml(input), output);
         }
     }
+
+    #[test]
+    fn test_dehtml_parse_br() {
+        let html = "\r\r\nline1<br>\r\n\r\n\r\rline2\n\r";
+        let plain = dehtml(html);
+
+        assert_eq!(plain, "line1\n\r\r\rline2");
+    }
+
+    #[test]
+    fn test_dehtml_parse_href() {
+        let html = "<a href=url>text</a";
+        let plain = dehtml(html);
+
+        assert_eq!(plain, "[text](url)");
+    }
+
+    #[test]
+    fn test_dehtml_bold_text() {
+        let html = "<!DOCTYPE name [<!DOCTYPE ...>]><!-- comment -->text <b><?php echo ... ?>bold</b><![CDATA[<>]]>";
+        let plain = dehtml(html);
+
+        assert_eq!(plain, "text *bold*<>");
+    }
+
+    #[test]
+    fn test_dehtml_html_encoded() {
+        let html =
+                "&lt;&gt;&quot;&apos;&amp; &auml;&Auml;&ouml;&Ouml;&uuml;&Uuml;&szlig; foo&AElig;&ccedil;&Ccedil; &diams;&lrm;&rlm;&zwnj;&noent;&zwj;";
+
+        let plain = dehtml(html);
+
+        assert_eq!(
+            plain,
+            "<>\"\'& äÄöÖüÜß fooÆçÇ \u{2666}\u{200e}\u{200f}\u{200c}&noent;\u{200d}"
+        );
+    }
 }

src/e2ee.rs

@@ -2,7 +2,7 @@
 
 use std::collections::HashSet;
 
-use mailparse::MailHeaderMap;
+use mailparse::{MailHeaderMap, ParsedMail};
 use num_traits::FromPrimitive;
 
 use crate::aheader::*;
@@ -14,7 +14,6 @@ use crate::keyring::*;
 use crate::peerstate::*;
 use crate::pgp;
 use crate::securejoin::handle_degrade_event;
-use crate::wrapmime;
 
 #[derive(Debug)]
 pub struct EncryptHelper {
@@ -118,7 +117,7 @@ impl EncryptHelper {
 
 pub fn try_decrypt(
     context: &Context,
-    mail: &mailparse::ParsedMail<'_>,
+    mail: &ParsedMail<'_>,
     message_time: i64,
 ) -> Result<(Option<Vec<u8>>, HashSet<String>)> {
     let from = mail
@@ -232,9 +231,36 @@ fn load_or_generate_self_public_key(context: &Context, self_addr: impl AsRef<str
     }
 }
 
+/// Returns a reference to the encrypted payload and validates the autocrypt structure.
+fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Result<&'a ParsedMail<'b>> {
+    ensure!(
+        mail.ctype.mimetype == "multipart/encrypted",
+        "Not a multipart/encrypted message: {}",
+        mail.ctype.mimetype
+    );
+    ensure!(
+        mail.subparts.len() == 2,
+        "Invalid Autocrypt Level 1 Mime Parts"
+    );
+
+    ensure!(
+        mail.subparts[0].ctype.mimetype == "application/pgp-encrypted",
+        "Invalid Autocrypt Level 1 version part: {:?}",
+        mail.subparts[0].ctype,
+    );
+
+    ensure!(
+        mail.subparts[1].ctype.mimetype == "application/octet-stream",
+        "Invalid Autocrypt Level 1 encrypted part: {:?}",
+        mail.subparts[1].ctype
+    );
+
+    Ok(&mail.subparts[1])
+}
+
 fn decrypt_if_autocrypt_message<'a>(
     context: &Context,
-    mail: &mailparse::ParsedMail<'a>,
+    mail: &ParsedMail<'a>,
     private_keyring: &Keyring,
     public_keyring_for_validate: &Keyring,
     ret_valid_signatures: &mut HashSet<String>,
@@ -246,7 +272,7 @@ fn decrypt_if_autocrypt_message<'a>(
     //
     // Errors are returned for failures related to decryption of AC-messages.
 
-    let encrypted_data_part = match wrapmime::get_autocrypt_mime(mail) {
+    let encrypted_data_part = match get_autocrypt_mime(mail) {
         Err(_) => {
             // not an autocrypt mime message, abort and ignore
             return Ok(None);
@@ -267,7 +293,7 @@ fn decrypt_if_autocrypt_message<'a>(
 /// Returns Ok(None) if nothing encrypted was found.
 fn decrypt_part(
     _context: &Context,
-    mail: &mailparse::ParsedMail<'_>,
+    mail: &ParsedMail<'_>,
     private_keyring: &Keyring,
     public_keyring_for_validate: &Keyring,
     ret_valid_signatures: &mut HashSet<String>,
@@ -313,7 +339,7 @@ fn has_decrypted_pgp_armor(input: &[u8]) -> bool {
 /// However, Delta Chat itself has no problem with encrypted multipart/report
 /// parts and MUAs should be encouraged to encrpyt multipart/reports as well so
 /// that we could use the normal Autocrypt processing.
-fn contains_report(mail: &mailparse::ParsedMail<'_>) -> bool {
+fn contains_report(mail: &ParsedMail<'_>) -> bool {
     mail.ctype.mimetype == "multipart/report"
 }
 

src/error.rs

@@ -6,34 +6,52 @@ use lettre_email::mime;
 pub enum Error {
     #[fail(display = "{:?}", _0)]
     Failure(failure::Error),
+
     #[fail(display = "SQL error: {:?}", _0)]
     SqlError(#[cause] crate::sql::Error),
+
     #[fail(display = "{:?}", _0)]
     Io(std::io::Error),
+
     #[fail(display = "{:?}", _0)]
     Message(String),
+
     #[fail(display = "{:?}", _0)]
     Image(image_meta::ImageError),
+
     #[fail(display = "{:?}", _0)]
     Utf8(std::str::Utf8Error),
+
     #[fail(display = "PGP: {:?}", _0)]
     Pgp(pgp::errors::Error),
+
     #[fail(display = "Base64Decode: {:?}", _0)]
     Base64Decode(base64::DecodeError),
+
     #[fail(display = "{:?}", _0)]
     FromUtf8(std::string::FromUtf8Error),
+
     #[fail(display = "{}", _0)]
     BlobError(#[cause] crate::blob::BlobError),
+
     #[fail(display = "Invalid Message ID.")]
     InvalidMsgId,
+
     #[fail(display = "Watch folder not found {:?}", _0)]
     WatchFolderNotFound(String),
+
     #[fail(display = "Invalid Email: {:?}", _0)]
     MailParseError(#[cause] mailparse::MailParseError),
+
     #[fail(display = "Building invalid Email: {:?}", _0)]
     LettreError(#[cause] lettre_email::error::Error),
+
+    #[fail(display = "SMTP error: {:?}", _0)]
+    SmtpError(#[cause] async_smtp::error::Error),
+
     #[fail(display = "FromStr error: {:?}", _0)]
     FromStr(#[cause] mime::FromStrError),
+
     #[fail(display = "Not Configured")]
     NotConfigured,
 }

src/events.rs

@@ -4,6 +4,7 @@ use std::path::PathBuf;
 
 use strum::EnumProperty;
 
+use crate::chat::ChatId;
 use crate::message::MsgId;
 
 impl Event {
@@ -21,56 +22,38 @@ pub enum Event {
     /// The library-user may write an informational string to the log.
     /// Passed to the callback given to dc_context_new().
     /// This event should not be reported to the end-user using a popup or something like that.
-    ///
-    /// @return 0
     #[strum(props(id = "100"))]
     Info(String),
 
     /// Emitted when SMTP connection is established and login was successful.
-    ///
-    /// @return 0
     #[strum(props(id = "101"))]
     SmtpConnected(String),
 
     /// Emitted when IMAP connection is established and login was successful.
-    ///
-    /// @return 0
     #[strum(props(id = "102"))]
     ImapConnected(String),
 
     /// Emitted when a message was successfully sent to the SMTP server.
-    ///
-    /// @return 0
     #[strum(props(id = "103"))]
     SmtpMessageSent(String),
 
     /// Emitted when an IMAP message has been marked as deleted
-    ///
-    /// @return 0
     #[strum(props(id = "104"))]
     ImapMessageDeleted(String),
 
     /// Emitted when an IMAP message has been moved
-    ///
-    /// @return 0
     #[strum(props(id = "105"))]
     ImapMessageMoved(String),
 
     /// Emitted when an IMAP folder was emptied
-    ///
-    /// @return 0
     #[strum(props(id = "106"))]
     ImapFolderEmptied(String),
 
     /// Emitted when an new file in the $BLOBDIR was created
-    ///
-    /// @return 0
     #[strum(props(id = "150"))]
     NewBlobFile(String),
 
     /// Emitted when an new file in the $BLOBDIR was created
-    ///
-    /// @return 0
     #[strum(props(id = "151"))]
     DeletedBlobFile(String),
 
@@ -78,8 +61,6 @@ pub enum Event {
     /// Passed to the callback given to dc_context_new().
     ///
     /// This event should not be reported to the end-user using a popup or something like that.
-    ///
-    /// @return 0
     #[strum(props(id = "300"))]
     Warning(String),
 
@@ -94,8 +75,6 @@ pub enum Event {
     /// it might be better to delay showing these events until the function has really
     /// failed (returned false). It should be sufficient to report only the *last* error
     /// in a messasge box then.
-    ///
-    /// @return
     #[strum(props(id = "400"))]
     Error(String),
 
@@ -112,8 +91,6 @@ pub enum Event {
     /// Moreover, if the UI detects that the device is offline,
     /// it is probably more useful to report this to the user
     /// instead of the string from data2.
-    ///
-    /// @return 0
     #[strum(props(id = "401"))]
     ErrorNetwork(String),
 
@@ -122,8 +99,6 @@ pub enum Event {
     /// dc_set_chat_name(), dc_set_chat_profile_image(),
     /// dc_add_contact_to_chat(), dc_remove_contact_from_chat(),
     /// dc_send_text_msg() or another sending function.
-    ///
-    /// @return 0
     #[strum(props(id = "410"))]
     ErrorSelfNotInGroup(String),
 
@@ -132,54 +107,41 @@ pub enum Event {
     /// - Messages sent, received or removed
     /// - Chats created, deleted or archived
     /// - A draft has been set
-    ///
-    /// @return 0
     #[strum(props(id = "2000"))]
-    MsgsChanged { chat_id: u32, msg_id: MsgId },
+    MsgsChanged { chat_id: ChatId, msg_id: MsgId },
 
     /// There is a fresh message. Typically, the user will show an notification
     /// when receiving this message.
     ///
     /// There is no extra #DC_EVENT_MSGS_CHANGED event send together with this event.
-    ///
-    /// @return 0
     #[strum(props(id = "2005"))]
-    IncomingMsg { chat_id: u32, msg_id: MsgId },
+    IncomingMsg { chat_id: ChatId, msg_id: MsgId },
 
     /// A single message is sent successfully. State changed from  DC_STATE_OUT_PENDING to
     /// DC_STATE_OUT_DELIVERED, see dc_msg_get_state().
-    ///
-    /// @return 0
     #[strum(props(id = "2010"))]
-    MsgDelivered { chat_id: u32, msg_id: MsgId },
+    MsgDelivered { chat_id: ChatId, msg_id: MsgId },
 
     /// A single message could not be sent. State changed from DC_STATE_OUT_PENDING or DC_STATE_OUT_DELIVERED to
     /// DC_STATE_OUT_FAILED, see dc_msg_get_state().
-    ///
-    /// @return 0
     #[strum(props(id = "2012"))]
-    MsgFailed { chat_id: u32, msg_id: MsgId },
+    MsgFailed { chat_id: ChatId, msg_id: MsgId },
 
     /// A single message is read by the receiver. State changed from DC_STATE_OUT_DELIVERED to
     /// DC_STATE_OUT_MDN_RCVD, see dc_msg_get_state().
-    ///
-    /// @return 0
     #[strum(props(id = "2015"))]
-    MsgRead { chat_id: u32, msg_id: MsgId },
+    MsgRead { chat_id: ChatId, msg_id: MsgId },
 
     /// Chat changed.  The name or the image of a chat group was changed or members were added or removed.
     /// Or the verify state of a chat has changed.
     /// See dc_set_chat_name(), dc_set_chat_profile_image(), dc_add_contact_to_chat()
     /// and dc_remove_contact_from_chat().
-    ///
-    /// @return 0
     #[strum(props(id = "2020"))]
-    ChatModified(u32),
+    ChatModified(ChatId),
 
     /// Contact(s) created, renamed, blocked or deleted.
     ///
     /// @param data1 (int) If set, this is the contact_id of an added contact that should be selected.
-    /// @return 0
     #[strum(props(id = "2030"))]
     ContactsChanged(Option<u32>),
 
@@ -188,14 +150,12 @@ pub enum Event {
     /// @param data1 (u32) contact_id of the contact for which the location has changed.
     ///     If the locations of several contacts have been changed,
     ///     eg. after calling dc_delete_all_locations(), this parameter is set to `None`.
-    /// @return 0
     #[strum(props(id = "2035"))]
     LocationChanged(Option<u32>),
 
     /// Inform about the configuration progress started by configure().
     ///
     /// @param data1 (usize) 0=error, 1-999=progress in permille, 1000=success and done
-    /// @return 0
     #[strum(props(id = "2041"))]
     ConfigureProgress(usize),
 
@@ -203,7 +163,6 @@ pub enum Event {
     ///
     /// @param data1 (usize) 0=error, 1-999=progress in permille, 1000=success and done
     /// @param data2 0
-    /// @return 0
     #[strum(props(id = "2051"))]
     ImexProgress(usize),
 
@@ -214,7 +173,6 @@ pub enum Event {
     /// services.
     ///
     /// @param data2 0
-    /// @return 0
     #[strum(props(id = "2052"))]
     ImexFileWritten(PathBuf),
 
@@ -230,7 +188,6 @@ pub enum Event {
     ///     600=vg-/vc-request-with-auth received, vg-member-added/vc-contact-confirm sent, typically shown as "bob@addr verified".
     ///     800=vg-member-added-received received, shown as "bob@addr securely joined GROUP", only sent for the verified-group-protocol.
     ///     1000=Protocol finished for this contact.
-    /// @return 0
     #[strum(props(id = "2060"))]
     SecurejoinInviterProgress { contact_id: u32, progress: usize },
 
@@ -242,14 +199,12 @@ pub enum Event {
     /// @param data2 (int) Progress as:
     ///     400=vg-/vc-request-with-auth sent, typically shown as "alice@addr verified, introducing myself."
     ///     (Bob has verified alice and waits until Alice does the same for him)
-    /// @return 0
     #[strum(props(id = "2061"))]
     SecurejoinJoinerProgress { contact_id: u32, progress: usize },
 
     /// This event is sent out to the inviter when a joiner successfully joined a group.
     /// @param data1 (int) chat_id
     /// @param data2 (int) contact_id
-    /// @return 0
     #[strum(props(id = "2062"))]
-    SecurejoinMemberAdded { chat_id: u32, contact_id: u32 },
+    SecurejoinMemberAdded { chat_id: ChatId, contact_id: u32 },
 }

src/headerdef.rs

@@ -10,6 +10,10 @@ pub enum HeaderDef {
     Cc,
     Disposition,
     OriginalMessageId,
+
+    /// Delta Chat extension for message IDs in combined MDNs
+    AdditionalMessageIds,
+
     ListId,
     References,
     InReplyTo,

src/imap/idle.rs

@@ -80,18 +80,21 @@ impl Imap {
                         } else {
                             info!(context, "Idle entering wait-on-remote state");
                             match idle_wait.await {
-                                IdleResponse::NewData(_) => {
+                                Ok(IdleResponse::NewData(_)) => {
                                     info!(context, "Idle has NewData");
                                 }
                                 // TODO: idle_wait does not distinguish manual interrupts
                                 // from Timeouts if we would know it's a Timeout we could bail
                                 // directly and reconnect .
-                                IdleResponse::Timeout => {
+                                Ok(IdleResponse::Timeout) => {
                                     info!(context, "Idle-wait timeout or interruption");
                                 }
-                                IdleResponse::ManualInterrupt => {
+                                Ok(IdleResponse::ManualInterrupt) => {
                                     info!(context, "Idle wait was interrupted");
                                 }
+                                Err(err) => {
+                                    warn!(context, "Idle wait errored: {:?}", err);
+                                }
                             }
                         }
                         // if we can't properly terminate the idle
@@ -134,18 +137,21 @@ impl Imap {
                         } else {
                             info!(context, "Idle entering wait-on-remote state");
                             match idle_wait.await {
-                                IdleResponse::NewData(_) => {
+                                Ok(IdleResponse::NewData(_)) => {
                                     info!(context, "Idle has NewData");
                                 }
                                 // TODO: idle_wait does not distinguish manual interrupts
                                 // from Timeouts if we would know it's a Timeout we could bail
                                 // directly and reconnect .
-                                IdleResponse::Timeout => {
+                                Ok(IdleResponse::Timeout) => {
                                     info!(context, "Idle-wait timeout or interruption");
                                 }
-                                IdleResponse::ManualInterrupt => {
+                                Ok(IdleResponse::ManualInterrupt) => {
                                     info!(context, "Idle wait was interrupted");
                                 }
+                                Err(err) => {
+                                    warn!(context, "Idle wait errored: {:?}", err);
+                                }
                             }
                         }
                         // if we can't properly terminate the idle
@@ -241,7 +247,7 @@ impl Imap {
                 "IMAP-fake-IDLE done after {:.4}s",
                 SystemTime::now()
                     .duration_since(fake_idle_start_time)
-                    .unwrap()
+                    .unwrap_or_default()
                     .as_millis() as f64
                     / 1000.,
             );

src/imap/mod.rs

@@ -23,7 +23,6 @@ use crate::message::{self, update_server_uid};
 use crate::oauth2::dc_get_oauth2_access_token;
 use crate::param::Params;
 use crate::stock::StockMessage;
-use crate::wrapmime;
 
 mod idle;
 pub mod select_folder;
@@ -64,6 +63,9 @@ pub enum Error {
     #[fail(display = "IMAP select folder error")]
     SelectFolderError(#[cause] select_folder::Error),
 
+    #[fail(display = "No mailbox selected, folder: {:?}", _0)]
+    NoMailbox(String),
+
     #[fail(display = "IMAP other error: {:?}", _0)]
     Other(String),
 }
@@ -339,7 +341,7 @@ impl Imap {
         let param = LoginParam::from_database(context, "configured_");
         // the trailing underscore is correct
 
-        if self.connect(context, &param) {
+        if task::block_on(self.connect(context, &param)) {
             self.ensure_configured_folders(context, true)
         } else {
             Err(Error::ConnectionFailed(format!("{}", param)))
@@ -348,82 +350,80 @@ impl Imap {
 
     /// tries connecting to imap account using the specific login
     /// parameters
-    pub fn connect(&self, context: &Context, lp: &LoginParam) -> bool {
-        task::block_on(async move {
-            if lp.mail_server.is_empty() || lp.mail_user.is_empty() || lp.mail_pw.is_empty() {
-                return false;
-            }
+    pub async fn connect(&self, context: &Context, lp: &LoginParam) -> bool {
+        if lp.mail_server.is_empty() || lp.mail_user.is_empty() || lp.mail_pw.is_empty() {
+            return false;
+        }
 
-            {
-                let addr = &lp.addr;
-                let imap_server = &lp.mail_server;
-                let imap_port = lp.mail_port as u16;
-                let imap_user = &lp.mail_user;
-                let imap_pw = &lp.mail_pw;
-                let server_flags = lp.server_flags as usize;
+        {
+            let addr = &lp.addr;
+            let imap_server = &lp.mail_server;
+            let imap_port = lp.mail_port as u16;
+            let imap_user = &lp.mail_user;
+            let imap_pw = &lp.mail_pw;
+            let server_flags = lp.server_flags as usize;
 
-                let mut config = self.config.write().await;
-                config.addr = addr.to_string();
-                config.imap_server = imap_server.to_string();
-                config.imap_port = imap_port;
-                config.imap_user = imap_user.to_string();
-                config.imap_pw = imap_pw.to_string();
-                config.certificate_checks = lp.imap_certificate_checks;
-                config.server_flags = server_flags;
-            }
+            let mut config = self.config.write().await;
+            config.addr = addr.to_string();
+            config.imap_server = imap_server.to_string();
+            config.imap_port = imap_port;
+            config.imap_user = imap_user.to_string();
+            config.imap_pw = imap_pw.to_string();
+            config.certificate_checks = lp.imap_certificate_checks;
+            config.server_flags = server_flags;
+        }
 
-            if let Err(err) = self.setup_handle_if_needed(context).await {
-                warn!(context, "failed to setup imap handle: {}", err);
-                self.free_connect_params().await;
-                return false;
-            }
+        if let Err(err) = self.setup_handle_if_needed(context).await {
+            warn!(context, "failed to setup imap handle: {}", err);
+            self.free_connect_params().await;
+            return false;
+        }
 
-            let teardown = match &mut *self.session.lock().await {
-                Some(ref mut session) => match session.capabilities().await {
-                    Ok(caps) => {
-                        if !context.sql.is_open() {
-                            warn!(context, "IMAP-LOGIN as {} ok but ABORTING", lp.mail_user,);
-                            true
-                        } else {
-                            let can_idle = caps.has_str("IDLE");
-                            let has_xlist = caps.has_str("XLIST");
-                            let caps_list = caps.iter().fold(String::new(), |s, c| {
-                                if let Capability::Atom(x) = c {
-                                    s + &format!(" {}", x)
-                                } else {
-                                    s + &format!(" {:?}", c)
-                                }
-                            });
-
-                            self.config.write().await.can_idle = can_idle;
-                            self.config.write().await.has_xlist = has_xlist;
-                            *self.connected.lock().await = true;
-                            emit_event!(
-                                context,
-                                Event::ImapConnected(format!(
-                                    "IMAP-LOGIN as {}, capabilities: {}",
-                                    lp.mail_user, caps_list,
-                                ))
-                            );
-                            false
-                        }
-                    }
-                    Err(err) => {
-                        info!(context, "CAPABILITY command error: {}", err);
+        let teardown = match &mut *self.session.lock().await {
+            Some(ref mut session) => match session.capabilities().await {
+                Ok(caps) => {
+                    if !context.sql.is_open() {
+                        warn!(context, "IMAP-LOGIN as {} ok but ABORTING", lp.mail_user,);
                         true
+                    } else {
+                        let can_idle = caps.has_str("IDLE");
+                        let has_xlist = caps.has_str("XLIST");
+                        let caps_list = caps.iter().fold(String::new(), |s, c| {
+                            if let Capability::Atom(x) = c {
+                                s + &format!(" {}", x)
+                            } else {
+                                s + &format!(" {:?}", c)
+                            }
+                        });
+
+                        self.config.write().await.can_idle = can_idle;
+                        self.config.write().await.has_xlist = has_xlist;
+                        *self.connected.lock().await = true;
+                        emit_event!(
+                            context,
+                            Event::ImapConnected(format!(
+                                "IMAP-LOGIN as {}, capabilities: {}",
+                                lp.mail_user, caps_list,
+                            ))
+                        );
+                        false
                     }
-                },
-                None => true,
-            };
+                }
+                Err(err) => {
+                    info!(context, "CAPABILITY command error: {}", err);
+                    true
+                }
+            },
+            None => true,
+        };
 
-            if teardown {
-                self.disconnect(context);
+        if teardown {
+            self.disconnect(context);
 
-                false
-            } else {
-                true
-            }
-        })
+            false
+        } else {
+            true
+        }
     }
 
     pub fn disconnect(&self, context: &Context) {
@@ -481,7 +481,10 @@ impl Imap {
             let (uid_validity, last_seen_uid) = self.get_config_last_seen_uid(context, &folder);
 
             let config = self.config.read().await;
-            let mailbox = config.selected_mailbox.as_ref().expect("just selected");
+            let mailbox = config
+                .selected_mailbox
+                .as_ref()
+                .ok_or_else(|| Error::NoMailbox(folder.to_string()))?;
 
             let new_uid_validity = match mailbox.uid_validity {
                 Some(v) => v,
@@ -954,8 +957,8 @@ impl Imap {
                                 remote_message_id,
                                 message_id,
                             );
+                            *uid = 0;
                         }
-                        *uid = 0;
                     }
                     Err(err) => {
                         warn!(
@@ -1225,6 +1228,18 @@ fn precheck_imf(context: &Context, rfc724_mid: &str, server_folder: &str, server
     }
 }
 
+fn parse_message_id(message_id: &[u8]) -> crate::error::Result<String> {
+    let value = std::str::from_utf8(message_id)?;
+    let addrs = mailparse::addrparse(value)
+        .map_err(|err| format_err!("failed to parse message id {:?}", err))?;
+
+    if let Some(info) = addrs.extract_single_info() {
+        return Ok(info.addr);
+    }
+
+    bail!("could not parse message_id: {}", value);
+}
+
 fn prefetch_get_message_id(prefetch_msg: &Fetch) -> Result<String> {
     if prefetch_msg.envelope().is_none() {
         return Err(Error::Other(
@@ -1237,5 +1252,22 @@ fn prefetch_get_message_id(prefetch_msg: &Fetch) -> Result<String> {
         return Err(Error::Other("prefetch: No message ID found".to_string()));
     }
 
-    wrapmime::parse_message_id(&message_id.unwrap()).map_err(Into::into)
+    parse_message_id(&message_id.unwrap()).map_err(Into::into)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_message_id() {
+        assert_eq!(
+            parse_message_id(b"Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org").unwrap(),
+            "Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org"
+        );
+        assert_eq!(
+            parse_message_id(b"<Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org>").unwrap(),
+            "Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org"
+        );
+    }
 }

src/imap_client.rs

@@ -35,9 +35,8 @@ impl Client {
         certificate_checks: CertificateChecks,
     ) -> ImapResult<Self> {
         let stream = TcpStream::connect(addr).await?;
-        let tls = dc_build_tls(certificate_checks)?;
-        let tls_connector: async_native_tls::TlsConnector = tls.into();
-        let tls_stream = tls_connector.connect(domain.as_ref(), stream).await?;
+        let tls = dc_build_tls(certificate_checks);
+        let tls_stream = tls.connect(domain.as_ref(), stream).await?;
         let mut client = ImapClient::new(tls_stream);
         if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
             client.debug = true;
@@ -46,7 +45,7 @@ impl Client {
         let _greeting = client
             .read_response()
             .await
-            .expect("failed to read greeting");
+            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
 
         Ok(Client::Secure(client))
     }
@@ -61,7 +60,7 @@ impl Client {
         let _greeting = client
             .read_response()
             .await
-            .expect("failed to read greeting");
+            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
 
         Ok(Client::Insecure(client))
     }
@@ -73,10 +72,8 @@ impl Client {
     ) -> ImapResult<Client> {
         match self {
             Client::Insecure(client) => {
-                let tls = dc_build_tls(certificate_checks)?;
-                let tls_stream = tls.into();
-
-                let client_sec = client.secure(domain, &tls_stream).await?;
+                let tls = dc_build_tls(certificate_checks);
+                let client_sec = client.secure(domain, tls).await?;
 
                 Ok(Client::Secure(client_sec))
             }

src/imex.rs

@@ -8,8 +8,8 @@ use rand::{thread_rng, Rng};
 
 use crate::blob::BlobObject;
 use crate::chat;
+use crate::chat::delete_and_reset_all_device_msgs;
 use crate::config::Config;
-use crate::configure::*;
 use crate::constants::*;
 use crate::context::Context;
 use crate::dc_tools::*;
@@ -33,16 +33,19 @@ pub enum ImexMode {
     /// and `private-key-default.asc`, if there are more keys, they are written to files as
     /// `public-key-<id>.asc` and `private-key-<id>.asc`
     ExportSelfKeys = 1,
+
     /// Import private keys found in the directory given as `param1`.
     /// The last imported key is made the default keys unless its name contains the string `legacy`.
     /// Public keys are not imported.
     ImportSelfKeys = 2,
+
     /// Export a backup to the directory given as `param1`.
     /// The backup contains all contacts, chats, images and other data and device independent settings.
     /// The backup does not contain device dependent settings as ringtones or LED notification settings.
     /// The name of the backup is typically `delta-chat.<day>.bak`, if more than one backup is create on a day,
     /// the format is `delta-chat.<day>-<number>.bak`
     ExportBackup = 11,
+
     /// `param1` is the file (not: directory) to import. The file is normally
     /// created by DC_IMEX_EXPORT_BACKUP and detected by dc_imex_has_backup(). Importing a backup
     /// is only possible as long as the context is not configured or used in another way.
@@ -133,14 +136,16 @@ fn do_initiate_key_transfer(context: &Context) -> Result<String> {
 
     let chat_id = chat::create_by_contact_id(context, DC_CONTACT_ID_SELF)?;
     msg = Message::default();
-    msg.type_0 = Viewtype::File;
+    msg.viewtype = Viewtype::File;
     msg.param.set(Param::File, setup_file_blob.as_name());
 
     msg.param
         .set(Param::MimeType, "application/autocrypt-setup");
     msg.param.set_cmd(SystemMessage::AutocryptSetupMessage);
-    msg.param
-        .set_int(Param::ForcePlaintext, DC_FP_NO_AUTOCRYPT_HEADER);
+    msg.param.set_int(
+        Param::ForcePlaintext,
+        ForcePlaintext::NoAutocryptHeader as i32,
+    );
 
     ensure!(!context.shall_stop_ongoing(), "canceled");
     let msg_id = chat::send_msg(context, chat_id, &mut msg)?;
@@ -380,7 +385,7 @@ pub fn JobImexImap(context: &Context, job: &Job) -> Result<()> {
             context.free_ongoing();
             bail!("Cannot create private key or private key not available.");
         } else {
-            dc_create_folder(context, &param);
+            dc_create_folder(context, &param)?;
         }
     }
     let path = Path::new(param);
@@ -417,7 +422,7 @@ fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Resul
     );
 
     ensure!(
-        !dc_is_configured(context),
+        !context.is_configured(),
         "Cannot import backups to accounts in use."
     );
     context.sql.close(&context);
@@ -438,6 +443,8 @@ fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Resul
         "could not re-open db"
     );
 
+    delete_and_reset_all_device_msgs(&context)?;
+
     let total_files_cnt = context
         .sql
         .query_get_value::<_, isize>(context, "SELECT COUNT(*) FROM backup_blobs;", params![])

src/lib.rs

@@ -1,13 +1,5 @@
 #![deny(clippy::correctness, missing_debug_implementations, clippy::all)]
-// for now we hide warnings to not clutter/hide errors during "cargo clippy"
-#![allow(
-    clippy::type_complexity,
-    clippy::cognitive_complexity,
-    clippy::too_many_arguments
-)]
-#![allow(clippy::unreadable_literal, clippy::match_bool)]
-#![feature(ptr_wrapping_offset_from)]
-#![feature(drain_filter)]
+#![allow(clippy::match_bool)]
 
 #[macro_use]
 extern crate failure_derive;
@@ -24,7 +16,7 @@ extern crate strum_macros;
 extern crate debug_stub_derive;
 
 #[macro_use]
-mod log;
+pub mod log;
 #[macro_use]
 pub mod error;
 
@@ -46,6 +38,7 @@ mod e2ee;
 mod imap;
 mod imap_client;
 pub mod imex;
+#[macro_use]
 pub mod job;
 mod job_thread;
 pub mod key;
@@ -62,16 +55,15 @@ pub mod peerstate;
 pub mod pgp;
 pub mod qr;
 pub mod securejoin;
+mod simplify;
 mod smtp;
 pub mod sql;
 pub mod stock;
 mod token;
 #[macro_use]
-mod wrapmime;
 mod dehtml;
 
 pub mod dc_receive_imf;
-mod dc_simplify;
 pub mod dc_tools;
 
 /// if set imap/incoming and smtp/outgoing MIME messages will be printed

src/location.rs

@@ -4,14 +4,14 @@ use bitflags::bitflags;
 use quick_xml;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
-use crate::chat;
+use crate::chat::{self, ChatId};
 use crate::config::Config;
 use crate::constants::*;
 use crate::context::*;
 use crate::dc_tools::*;
 use crate::error::Error;
 use crate::events::Event;
-use crate::job::*;
+use crate::job::{self, job_action_exists, job_add, Job};
 use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::param::*;
@@ -28,7 +28,7 @@ pub struct Location {
     pub timestamp: i64,
     pub contact_id: u32,
     pub msg_id: u32,
-    pub chat_id: u32,
+    pub chat_id: ChatId,
     pub marker: Option<String>,
     pub independent: u32,
 }
@@ -193,9 +193,9 @@ impl Kml {
 }
 
 // location streaming
-pub fn send_locations_to_chat(context: &Context, chat_id: u32, seconds: i64) {
+pub fn send_locations_to_chat(context: &Context, chat_id: ChatId, seconds: i64) {
     let now = time();
-    if !(seconds < 0 || chat_id <= DC_CHAT_ID_LAST_SPECIAL) {
+    if !(seconds < 0 || chat_id.is_special()) {
         let is_sending_locations_before = is_sending_locations_to_chat(context, chat_id);
         if sql::execute(
             context,
@@ -207,7 +207,7 @@ pub fn send_locations_to_chat(context: &Context, chat_id: u32, seconds: i64) {
             params![
                 if 0 != seconds { now } else { 0 },
                 if 0 != seconds { now + seconds } else { 0 },
-                chat_id as i32,
+                chat_id,
             ],
         )
         .is_ok()
@@ -228,8 +228,8 @@ pub fn send_locations_to_chat(context: &Context, chat_id: u32, seconds: i64) {
                 schedule_MAYBE_SEND_LOCATIONS(context, false);
                 job_add(
                     context,
-                    Action::MaybeSendLocationsEnded,
-                    chat_id as i32,
+                    job::Action::MaybeSendLocationsEnded,
+                    chat_id.to_u32() as i32,
                     Params::new(),
                     seconds + 1,
                 );
@@ -240,17 +240,23 @@ pub fn send_locations_to_chat(context: &Context, chat_id: u32, seconds: i64) {
 
 #[allow(non_snake_case)]
 fn schedule_MAYBE_SEND_LOCATIONS(context: &Context, force_schedule: bool) {
-    if force_schedule || !job_action_exists(context, Action::MaybeSendLocations) {
-        job_add(context, Action::MaybeSendLocations, 0, Params::new(), 60);
+    if force_schedule || !job_action_exists(context, job::Action::MaybeSendLocations) {
+        job_add(
+            context,
+            job::Action::MaybeSendLocations,
+            0,
+            Params::new(),
+            60,
+        );
     };
 }
 
-pub fn is_sending_locations_to_chat(context: &Context, chat_id: u32) -> bool {
+pub fn is_sending_locations_to_chat(context: &Context, chat_id: ChatId) -> bool {
     context
         .sql
         .exists(
             "SELECT id  FROM chats  WHERE (? OR id=?)   AND locations_send_until>?;",
-            params![if chat_id == 0 { 1 } else { 0 }, chat_id as i32, time()],
+            params![if chat_id.is_unset() { 1 } else { 0 }, chat_id, time()],
         )
         .unwrap_or_default()
 }
@@ -296,7 +302,7 @@ pub fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64) -> b
 
 pub fn get_range(
     context: &Context,
-    chat_id: u32,
+    chat_id: ChatId,
     contact_id: u32,
     timestamp_from: i64,
     mut timestamp_to: i64,
@@ -314,8 +320,8 @@ pub fn get_range(
              AND (l.independent=1 OR (l.timestamp>=? AND l.timestamp<=?)) \
              ORDER BY l.timestamp DESC, l.id DESC, msg_id DESC;",
             params![
-                if chat_id == 0 { 1 } else { 0 },
-                chat_id as i32,
+                if chat_id.is_unset() { 1 } else { 0 },
+                chat_id,
                 if contact_id == 0 { 1 } else { 0 },
                 contact_id as i32,
                 timestamp_from,
@@ -365,7 +371,7 @@ pub fn delete_all(context: &Context) -> Result<(), Error> {
     Ok(())
 }
 
-pub fn get_kml(context: &Context, chat_id: u32) -> Result<(String, u32), Error> {
+pub fn get_kml(context: &Context, chat_id: ChatId) -> Result<(String, u32), Error> {
     let mut last_added_location_id = 0;
 
     let self_addr = context
@@ -374,7 +380,7 @@ pub fn get_kml(context: &Context, chat_id: u32) -> Result<(String, u32), Error>
 
     let (locations_send_begin, locations_send_until, locations_last_sent) = context.sql.query_row(
         "SELECT locations_send_begin, locations_send_until, locations_last_sent  FROM chats  WHERE id=?;",
-        params![chat_id as i32], |row| {
+        params![chat_id], |row| {
             let send_begin: i64 = row.get(0)?;
             let send_until: i64 = row.get(1)?;
             let last_sent: i64 = row.get(2)?;
@@ -459,14 +465,14 @@ pub fn get_message_kml(timestamp: i64, latitude: f64, longitude: f64) -> String
 
 pub fn set_kml_sent_timestamp(
     context: &Context,
-    chat_id: u32,
+    chat_id: ChatId,
     timestamp: i64,
 ) -> Result<(), Error> {
     sql::execute(
         context,
         &context.sql,
         "UPDATE chats SET locations_last_sent=? WHERE id=?;",
-        params![timestamp, chat_id as i32],
+        params![timestamp, chat_id],
     )?;
 
     Ok(())
@@ -489,12 +495,12 @@ pub fn set_msg_location_id(
 
 pub fn save(
     context: &Context,
-    chat_id: u32,
+    chat_id: ChatId,
     contact_id: u32,
     locations: &[Location],
     independent: bool,
 ) -> Result<u32, Error> {
-    ensure!(chat_id > DC_CHAT_ID_LAST_SPECIAL, "Invalid chat id");
+    ensure!(!chat_id.is_special(), "Invalid chat id");
     context
         .sql
         .prepare2(
@@ -514,7 +520,7 @@ pub fn save(
                         stmt_insert.execute(params![
                             location.timestamp,
                             contact_id as i32,
-                            chat_id as i32,
+                            chat_id,
                             location.latitude,
                             location.longitude,
                             location.accuracy,
@@ -542,7 +548,7 @@ pub fn save(
 }
 
 #[allow(non_snake_case)]
-pub fn JobMaybeSendLocations(context: &Context, _job: &Job) {
+pub fn JobMaybeSendLocations(context: &Context, _job: &Job) -> job::Status {
     let now = time();
     let mut continue_streaming = false;
     info!(
@@ -556,7 +562,7 @@ pub fn JobMaybeSendLocations(context: &Context, _job: &Job) {
          WHERE locations_send_until>?;",
         params![now],
         |row| {
-            let chat_id: i32 = row.get(0)?;
+            let chat_id: ChatId = row.get(0)?;
             let locations_send_begin: i64 = row.get(1)?;
             let locations_last_sent: i64 = row.get(2)?;
             continue_streaming = true;
@@ -623,44 +629,46 @@ pub fn JobMaybeSendLocations(context: &Context, _job: &Job) {
 
         for (chat_id, mut msg) in msgs.into_iter() {
             // TODO: better error handling
-            chat::send_msg(context, chat_id as u32, &mut msg).unwrap_or_default();
+            chat::send_msg(context, chat_id, &mut msg).unwrap_or_default();
         }
     }
     if continue_streaming {
         schedule_MAYBE_SEND_LOCATIONS(context, true);
     }
+    job::Status::Finished(Ok(()))
 }
 
 #[allow(non_snake_case)]
-pub fn JobMaybeSendLocationsEnded(context: &Context, job: &mut Job) {
+pub fn JobMaybeSendLocationsEnded(context: &Context, job: &mut Job) -> job::Status {
     // this function is called when location-streaming _might_ have ended for a chat.
     // the function checks, if location-streaming is really ended;
     // if so, a device-message is added if not yet done.
 
-    let chat_id = job.foreign_id;
+    let chat_id = ChatId::new(job.foreign_id);
 
-    if let Ok((send_begin, send_until)) = context.sql.query_row(
+    let (send_begin, send_until) = job_try!(context.sql.query_row(
         "SELECT locations_send_begin, locations_send_until  FROM chats  WHERE id=?",
-        params![chat_id as i32],
+        params![chat_id],
         |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?)),
-    ) {
-        if !(send_begin != 0 && time() <= send_until) {
-            // still streaming -
-            // may happen as several calls to dc_send_locations_to_chat()
-            // do not un-schedule pending DC_MAYBE_SEND_LOC_ENDED jobs
-            if !(send_begin == 0 && send_until == 0) {
-                // not streaming, device-message already sent
-                if context.sql.execute(
-                    "UPDATE chats    SET locations_send_begin=0, locations_send_until=0  WHERE id=?",
-                    params![chat_id as i32],
-                ).is_ok() {
-                    let stock_str = context.stock_system_msg(StockMessage::MsgLocationDisabled, "", "", 0);
-                    chat::add_info_msg(context, chat_id, stock_str);
-                    context.call_cb(Event::ChatModified(chat_id));
-                }
-            }
+    ));
+
+    if !(send_begin != 0 && time() <= send_until) {
+        // still streaming -
+        // may happen as several calls to dc_send_locations_to_chat()
+        // do not un-schedule pending DC_MAYBE_SEND_LOC_ENDED jobs
+        if !(send_begin == 0 && send_until == 0) {
+            // not streaming, device-message already sent
+            job_try!(context.sql.execute(
+                "UPDATE chats    SET locations_send_begin=0, locations_send_until=0  WHERE id=?",
+                params![chat_id],
+            ));
+
+            let stock_str = context.stock_system_msg(StockMessage::MsgLocationDisabled, "", "", 0);
+            chat::add_info_msg(context, chat_id, stock_str);
+            context.call_cb(Event::ChatModified(chat_id));
         }
     }
+    job::Status::Finished(Ok(()))
 }
 
 #[cfg(test)]

src/log.rs

@@ -7,7 +7,13 @@ macro_rules! info {
     };
     ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
         let formatted = format!($msg, $($args),*);
-        emit_event!($ctx, $crate::Event::Info(formatted));
+        let thread = ::std::thread::current();
+        let full = format!("{thid:?} {file}:{line}: {msg}",
+                           thid = thread.id(),
+                           file = file!(),
+                           line = line!(),
+                           msg = &formatted);
+        emit_event!($ctx, $crate::Event::Info(full));
     }};
 }
 
@@ -18,7 +24,13 @@ macro_rules! warn {
     };
     ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
         let formatted = format!($msg, $($args),*);
-        emit_event!($ctx, $crate::Event::Warning(formatted));
+        let thread = ::std::thread::current();
+        let full = format!("{thid:?} {file}:{line}: {msg}",
+                           thid = thread.id(),
+                           file = file!(),
+                           line = line!(),
+                           msg = &formatted);
+        emit_event!($ctx, $crate::Event::Warning(full));
     }};
 }
 

src/login_param.rs

@@ -258,10 +258,8 @@ fn get_readable_flags(flags: i32) -> String {
     res
 }
 
-pub fn dc_build_tls(
-    certificate_checks: CertificateChecks,
-) -> Result<native_tls::TlsConnector, native_tls::Error> {
-    let mut tls_builder = native_tls::TlsConnector::builder();
+pub fn dc_build_tls(certificate_checks: CertificateChecks) -> async_native_tls::TlsConnector {
+    let tls_builder = async_native_tls::TlsConnector::new();
     match certificate_checks {
         CertificateChecks::Automatic => {
             // Same as AcceptInvalidCertificates for now.
@@ -270,13 +268,12 @@ pub fn dc_build_tls(
                 .danger_accept_invalid_hostnames(true)
                 .danger_accept_invalid_certs(true)
         }
-        CertificateChecks::Strict => &mut tls_builder,
+        CertificateChecks::Strict => tls_builder,
         CertificateChecks::AcceptInvalidCertificates
         | CertificateChecks::AcceptInvalidCertificates2 => tls_builder
             .danger_accept_invalid_hostnames(true)
             .danger_accept_invalid_certs(true),
     }
-    .build()
 }
 
 #[cfg(test)]

src/lot.rs

@@ -73,20 +73,28 @@ pub enum LotState {
     // Qr States
     /// id=contact
     QrAskVerifyContact = 200,
+
     /// text1=groupname
     QrAskVerifyGroup = 202,
+
     /// id=contact
     QrFprOk = 210,
+
     /// id=contact
     QrFprMissmatch = 220,
+
     /// test1=formatted fingerprint
     QrFprWithoutAddr = 230,
+
     /// id=contact
     QrAddr = 320,
+
     /// text1=text
     QrText = 330,
+
     /// text1=URL
     QrUrl = 332,
+
     /// text1=error string
     QrError = 400,
 

src/message.rs

@@ -5,7 +5,7 @@ use std::path::{Path, PathBuf};
 use deltachat_derive::{FromSql, ToSql};
 use failure::Fail;
 
-use crate::chat::{self, Chat};
+use crate::chat::{self, Chat, ChatId};
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::*;
@@ -47,10 +47,7 @@ impl MsgId {
     ///
     /// This kind of message ID can not be used for real messages.
     pub fn is_special(self) -> bool {
-        match self.0 {
-            0..=DC_MSG_ID_LAST_SPECIAL => true,
-            _ => false,
-        }
+        self.0 <= DC_MSG_ID_LAST_SPECIAL
     }
 
     /// Whether the message ID is unset.
@@ -148,6 +145,22 @@ impl rusqlite::types::FromSql for MsgId {
 #[fail(display = "Invalid Message ID.")]
 pub struct InvalidMsgId;
 
+#[derive(Debug, Copy, Clone, PartialEq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[repr(u8)]
+pub enum MessengerMessage {
+    No = 0,
+    Yes = 1,
+
+    /// No, but reply to messenger message.
+    Reply = 2,
+}
+
+impl Default for MessengerMessage {
+    fn default() -> Self {
+        Self::No
+    }
+}
+
 /// An object representing a single message in memory.
 /// The message object is not updated.
 /// If you want an update, you have to recreate the object.
@@ -160,8 +173,8 @@ pub struct Message {
     pub(crate) id: MsgId,
     pub(crate) from_id: u32,
     pub(crate) to_id: u32,
-    pub(crate) chat_id: u32,
-    pub(crate) type_0: Viewtype,
+    pub(crate) chat_id: ChatId,
+    pub(crate) viewtype: Viewtype,
     pub(crate) state: MessageState,
     pub(crate) hidden: bool,
     pub(crate) timestamp_sort: i64,
@@ -172,8 +185,7 @@ pub struct Message {
     pub(crate) in_reply_to: Option<String>,
     pub(crate) server_folder: Option<String>,
     pub(crate) server_uid: u32,
-    // TODO: enum
-    pub(crate) is_dc_message: u32,
+    pub(crate) is_dc_message: MessengerMessage,
     pub(crate) starred: bool,
     pub(crate) chat_blocked: Blocked,
     pub(crate) location_id: u32,
@@ -183,7 +195,7 @@ pub struct Message {
 impl Message {
     pub fn new(viewtype: Viewtype) -> Self {
         let mut msg = Message::default();
-        msg.type_0 = viewtype;
+        msg.viewtype = viewtype;
 
         msg
     }
@@ -191,7 +203,8 @@ impl Message {
     pub fn load_from_db(context: &Context, id: MsgId) -> Result<Message, Error> {
         ensure!(
             !id.is_special(),
-            "Can not load special message IDs from DB."
+            "Can not load special message IDs from DB. ({:?})",
+            id
         );
         context
             .sql
@@ -236,7 +249,7 @@ impl Message {
                     msg.timestamp_sort = row.get("timestamp")?;
                     msg.timestamp_sent = row.get("timestamp_sent")?;
                     msg.timestamp_rcvd = row.get("timestamp_rcvd")?;
-                    msg.type_0 = row.get("type")?;
+                    msg.viewtype = row.get("type")?;
                     msg.state = row.get("state")?;
                     msg.is_dc_message = row.get("msgrmsg")?;
 
@@ -312,10 +325,10 @@ impl Message {
     }
 
     pub fn try_calc_and_set_dimensions(&mut self, context: &Context) -> Result<(), Error> {
-        if chat::msgtype_has_file(self.type_0) {
+        if chat::msgtype_has_file(self.viewtype) {
             let file_param = self.param.get_path(Param::File, context)?;
             if let Some(path_and_filename) = file_param {
-                if (self.type_0 == Viewtype::Image || self.type_0 == Viewtype::Gif)
+                if (self.viewtype == Viewtype::Image || self.viewtype == Viewtype::Gif)
                     && !self.param.exists(Param::Width)
                 {
                     self.param.set_int(Param::Width, 0);
@@ -385,16 +398,16 @@ impl Message {
         self.from_id
     }
 
-    pub fn get_chat_id(&self) -> u32 {
+    pub fn get_chat_id(&self) -> ChatId {
         if self.chat_blocked != Blocked::Not {
-            1
+            ChatId::new(DC_CHAT_ID_DEADDROP)
         } else {
             self.chat_id
         }
     }
 
     pub fn get_viewtype(&self) -> Viewtype {
-        self.type_0
+        self.viewtype
     }
 
     pub fn get_state(&self) -> MessageState {
@@ -474,7 +487,7 @@ impl Message {
 
     pub fn get_summarytext(&self, context: &Context, approx_characters: usize) -> String {
         get_summarytext_by_raw(
-            self.type_0,
+            self.viewtype,
             self.text.as_ref(),
             &self.param,
             approx_characters,
@@ -518,11 +531,11 @@ impl Message {
     /// copied to the blobdir.  Thus those attachments need to be
     /// created immediately in the blobdir with a valid filename.
     pub fn is_increation(&self) -> bool {
-        chat::msgtype_has_file(self.type_0) && self.state == MessageState::OutPreparing
+        chat::msgtype_has_file(self.viewtype) && self.state == MessageState::OutPreparing
     }
 
     pub fn is_setupmessage(&self) -> bool {
-        if self.type_0 != Viewtype::File {
+        if self.viewtype != Viewtype::File {
             return false;
         }
 
@@ -595,18 +608,48 @@ impl Message {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql)]
 #[repr(i32)]
 pub enum MessageState {
     Undefined = 0,
+
+    /// Incoming *fresh* message. Fresh messages are neither noticed
+    /// nor seen and are typically shown in notifications.
     InFresh = 10,
+
+    /// Incoming *noticed* message. E.g. chat opened but message not
+    /// yet read - noticed messages are not counted as unread but did
+    /// not marked as read nor resulted in MDNs.
     InNoticed = 13,
+
+    /// Incoming message, really *seen* by the user. Marked as read on
+    /// IMAP and MDN may be sent.
     InSeen = 16,
+
+    /// For files which need time to be prepared before they can be
+    /// sent, the message enters this state before
+    /// OutPending.
     OutPreparing = 18,
+
+    /// Message saved as draft.
     OutDraft = 19,
+
+    /// The user has pressed the "send" button but the message is not
+    /// yet sent and is pending in some way. Maybe we're offline (no
+    /// checkmark).
     OutPending = 20,
+
+    /// *Unrecoverable* error (*recoverable* errors result in pending
+    /// messages).
     OutFailed = 24,
+
+    /// Outgoing message successfully delivered to server (one
+    /// checkmark). Note, that already delivered messages may get into
+    /// the OutFailed state if we get such a hint from the server.
     OutDelivered = 26,
+
+    /// Outgoing message read by the recipient (two checkmarks; this
+    /// requires goodwill on the receiver's side)
     OutMdnRcvd = 28,
 }
 
@@ -616,6 +659,27 @@ impl Default for MessageState {
     }
 }
 
+impl std::fmt::Display for MessageState {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(
+            f,
+            "{}",
+            match self {
+                Self::Undefined => "Undefined",
+                Self::InFresh => "Fresh",
+                Self::InNoticed => "Noticed",
+                Self::InSeen => "Seen",
+                Self::OutPreparing => "Preparing",
+                Self::OutDraft => "Draft",
+                Self::OutPending => "Pending",
+                Self::OutFailed => "Failed",
+                Self::OutDelivered => "Delivered",
+                Self::OutMdnRcvd => "Read",
+            }
+        )
+    }
+}
+
 impl From<MessageState> for LotState {
     fn from(s: MessageState) -> Self {
         use MessageState::*;
@@ -671,7 +735,7 @@ impl Lot {
                 self.text1 = None;
                 self.text1_meaning = Meaning::None;
             } else {
-                if chat.id == DC_CHAT_ID_DEADDROP {
+                if chat.id.is_deaddrop() {
                     if let Some(contact) = contact {
                         self.text1 = Some(contact.get_display_name().into());
                     } else {
@@ -687,7 +751,7 @@ impl Lot {
         }
 
         self.text2 = Some(get_summarytext_by_raw(
-            msg.type_0,
+            msg.viewtype,
             msg.text.as_ref(),
             &msg.param,
             SUMMARY_CHARACTERS,
@@ -720,7 +784,7 @@ pub fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
         return ret;
     }
     let rawtxt = rawtxt.unwrap_or_default();
-    let rawtxt = dc_truncate(rawtxt.trim(), 100000, false);
+    let rawtxt = dc_truncate(rawtxt.trim(), 100_000, false);
 
     let fts = dc_timestamp_to_str(msg.get_timestamp());
     ret += &format!("Sent: {}", fts);
@@ -770,19 +834,7 @@ pub fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
         }
     }
 
-    ret += "State: ";
-    use MessageState::*;
-    match msg.state {
-        InFresh => ret += "Fresh",
-        InNoticed => ret += "Noticed",
-        InSeen => ret += "Seen",
-        OutDelivered => ret += "Delivered",
-        OutFailed => ret += "Failed",
-        OutMdnRcvd => ret += "Read",
-        OutPending => ret += "Pending",
-        OutPreparing => ret += "Preparing",
-        _ => ret += &format!("{}", msg.state),
-    }
+    ret += &format!("State: {}", msg.state);
 
     if msg.has_location() {
         ret += ", Location sent";
@@ -808,9 +860,9 @@ pub fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
         ret += &format!("\nFile: {}, {}, bytes\n", path.display(), bytes);
     }
 
-    if msg.type_0 != Viewtype::Text {
+    if msg.viewtype != Viewtype::Text {
         ret += "Type: ";
-        ret += &format!("{}", msg.type_0);
+        ret += &format!("{}", msg.viewtype);
         ret += "\n";
         ret += &format!("Mimetype: {}\n", &msg.get_filemime().unwrap_or_default());
     }
@@ -844,6 +896,7 @@ pub fn guess_msgtype_from_suffix(path: &Path) -> Option<(Viewtype, &str)> {
         "mp3" => (Viewtype::Audio, "audio/mpeg"),
         "aac" => (Viewtype::Audio, "audio/aac"),
         "mp4" => (Viewtype::Video, "video/mp4"),
+        "webm" => (Viewtype::Video, "video/webm"),
         "jpg" => (Viewtype::Image, "image/jpeg"),
         "jpeg" => (Viewtype::Image, "image/jpeg"),
         "jpe" => (Viewtype::Image, "image/jpeg"),
@@ -874,7 +927,7 @@ pub fn delete_msgs(context: &Context, msg_ids: &[MsgId]) {
                 delete_poi_location(context, msg.location_id);
             }
         }
-        update_msg_chat_id(context, *msg_id, DC_CHAT_ID_TRASH);
+        update_msg_chat_id(context, *msg_id, ChatId::new(DC_CHAT_ID_TRASH));
         job_add(
             context,
             Action::DeleteMsgOnImap,
@@ -886,7 +939,7 @@ pub fn delete_msgs(context: &Context, msg_ids: &[MsgId]) {
 
     if !msg_ids.is_empty() {
         context.call_cb(Event::MsgsChanged {
-            chat_id: 0,
+            chat_id: ChatId::new(0),
             msg_id: MsgId::new(0),
         });
         job_kill_action(context, Action::Housekeeping);
@@ -894,12 +947,12 @@ pub fn delete_msgs(context: &Context, msg_ids: &[MsgId]) {
     };
 }
 
-fn update_msg_chat_id(context: &Context, msg_id: MsgId, chat_id: u32) -> bool {
+fn update_msg_chat_id(context: &Context, msg_id: MsgId, chat_id: ChatId) -> bool {
     sql::execute(
         context,
         &context.sql,
         "UPDATE msgs SET chat_id=? WHERE id=?;",
-        params![chat_id as i32, msg_id],
+        params![chat_id, msg_id],
     )
     .is_ok()
 }
@@ -978,7 +1031,7 @@ pub fn markseen_msgs(context: &Context, msg_ids: &[MsgId]) -> bool {
 
     if send_event {
         context.call_cb(Event::MsgsChanged {
-            chat_id: 0,
+            chat_id: ChatId::new(0),
             msg_id: MsgId::new(0),
         });
     }
@@ -1089,14 +1142,14 @@ pub fn exists(context: &Context, msg_id: MsgId) -> bool {
         return false;
     }
 
-    let chat_id: Option<u32> = context.sql.query_get_value(
+    let chat_id: Option<ChatId> = context.sql.query_get_value(
         context,
         "SELECT chat_id FROM msgs WHERE id=?;",
         params![msg_id],
     );
 
     if let Some(chat_id) = chat_id {
-        chat_id != DC_CHAT_ID_TRASH
+        !chat_id.is_trash()
     } else {
         false
     }
@@ -1134,7 +1187,7 @@ pub fn mdn_from_ext(
     from_id: u32,
     rfc724_mid: &str,
     timestamp_sent: i64,
-) -> Option<(u32, MsgId)> {
+) -> Option<(ChatId, MsgId)> {
     if from_id <= DC_MSG_ID_LAST_SPECIAL || rfc724_mid.is_empty() {
         return None;
     }
@@ -1154,7 +1207,7 @@ pub fn mdn_from_ext(
         |row| {
             Ok((
                 row.get::<_, MsgId>("msg_id")?,
-                row.get::<_, u32>("chat_id")?,
+                row.get::<_, ChatId>("chat_id")?,
                 row.get::<_, Chattype>("type")?,
                 row.get::<_, MessageState>("state")?,
             ))

src/mimefactory.rs

@@ -11,36 +11,42 @@ use crate::dc_tools::*;
 use crate::e2ee::*;
 use crate::error::Error;
 use crate::location;
-use crate::message::MsgId;
 use crate::message::{self, Message};
 use crate::mimeparser::SystemMessage;
 use crate::param::*;
 use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
 use crate::stock::StockMessage;
 
-#[derive(Clone, Copy, Eq, PartialEq)]
+// attachments of 25 mb brutto should work on the majority of providers
+// (brutto examples: web.de=50, 1&1=40, t-online.de=32, gmail=25, posteo=50, yahoo=25, all-inkl=100).
+// as an upper limit, we double the size; the core won't send messages larger than this
+// to get the netto sizes, we subtract 1 mb header-overhead and the base64-overhead.
+pub const RECOMMENDED_FILE_SIZE: u64 = 24 * 1024 * 1024 / 4 * 3;
+const UPPER_LIMIT_FILE_SIZE: u64 = 49 * 1024 * 1024 / 4 * 3;
+
+#[derive(Debug, Clone)]
 pub enum Loaded {
-    Message,
-    MDN,
+    Message { chat: Chat },
+    MDN { additional_msg_ids: Vec<String> },
 }
 
 /// Helper to construct mime messages.
-#[derive(Clone)]
+#[derive(Debug, Clone)]
 pub struct MimeFactory<'a, 'b> {
-    pub from_addr: String,
-    pub from_displayname: String,
-    pub selfstatus: String,
-    pub recipients_names: Vec<String>,
-    pub recipients_addr: Vec<String>,
-    pub timestamp: i64,
-    pub loaded: Loaded,
-    pub msg: &'b Message,
-    pub chat: Option<Chat>,
-    pub increation: bool,
-    pub in_reply_to: String,
-    pub references: String,
-    pub req_mdn: bool,
-    pub context: &'a Context,
+    from_addr: String,
+    from_displayname: String,
+    selfstatus: String,
+
+    /// Vector of pairs of recipient name and address
+    recipients: Vec<(String, String)>,
+
+    timestamp: i64,
+    loaded: Loaded,
+    msg: &'b Message,
+    in_reply_to: String,
+    references: String,
+    req_mdn: bool,
+    context: &'a Context,
     last_added_location_id: u32,
     attach_selfavatar: bool,
 }
@@ -53,8 +59,6 @@ pub struct RenderedEmail {
     pub is_encrypted: bool,
     pub is_gossiped: bool,
     pub last_added_location_id: u32,
-    /// None for MDN, the message id otherwise
-    pub foreign_id: Option<MsgId>,
 
     pub from: String,
     pub recipients: Vec<String>,
@@ -71,44 +75,22 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
     ) -> Result<MimeFactory<'a, 'b>, Error> {
         let chat = Chat::load_from_db(context, msg.chat_id)?;
 
-        let mut factory = MimeFactory {
-            from_addr: context
-                .get_config(Config::ConfiguredAddr)
-                .unwrap_or_default(),
-            from_displayname: context.get_config(Config::Displayname).unwrap_or_default(),
-            selfstatus: context
-                .get_config(Config::Selfstatus)
-                .unwrap_or_else(|| context.stock_str(StockMessage::StatusLine).to_string()),
-            recipients_names: Vec::with_capacity(5),
-            recipients_addr: Vec::with_capacity(5),
-            timestamp: msg.timestamp_sort,
-            loaded: Loaded::Message,
-            msg,
-            chat: Some(chat),
-            increation: msg.is_increation(),
-            in_reply_to: String::default(),
-            references: String::default(),
-            req_mdn: false,
-            last_added_location_id: 0,
-            attach_selfavatar: add_selfavatar,
-            context,
-        };
-
-        // just set the chat above
-        let chat = factory.chat.as_ref().unwrap();
+        let from_addr = context
+            .get_config(Config::ConfiguredAddr)
+            .unwrap_or_default();
+        let from_displayname = context.get_config(Config::Displayname).unwrap_or_default();
+        let mut recipients = Vec::with_capacity(5);
+        let mut req_mdn = false;
 
         if chat.is_self_talk() {
-            factory
-                .recipients_names
-                .push(factory.from_displayname.to_string());
-            factory.recipients_addr.push(factory.from_addr.to_string());
+            recipients.push((from_displayname.to_string(), from_addr.to_string()));
         } else {
             context.sql.query_map(
                 "SELECT c.authname, c.addr  \
                  FROM chats_contacts cc  \
                  LEFT JOIN contacts c ON cc.contact_id=c.id  \
                  WHERE cc.chat_id=? AND cc.contact_id>9;",
-                params![factory.msg.chat_id as i32],
+                params![msg.chat_id],
                 |row| {
                     let authname: String = row.get(0)?;
                     let addr: String = row.get(1)?;
@@ -117,17 +99,15 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                 |rows| {
                     for row in rows {
                         let (authname, addr) = row?;
-                        if !vec_contains_lowercase(&factory.recipients_addr, &addr) {
-                            factory.recipients_addr.push(addr);
-                            factory.recipients_names.push(authname);
+                        if !recipients_contain_addr(&recipients, &addr) {
+                            recipients.push((authname, addr));
                         }
                     }
                     Ok(())
                 },
             )?;
 
-            let command = factory.msg.param.get_cmd();
-            let msg = &factory.msg;
+            let command = msg.param.get_cmd();
 
             /* for added members, the list is just fine */
             if command == SystemMessage::MemberRemovedFromGroup {
@@ -139,20 +119,19 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
                 if !email_to_remove.is_empty()
                     && !addr_cmp(email_to_remove, self_addr)
-                    && !vec_contains_lowercase(&factory.recipients_addr, &email_to_remove)
+                    && !recipients_contain_addr(&recipients, &email_to_remove)
                 {
-                    factory.recipients_names.push("".to_string());
-                    factory.recipients_addr.push(email_to_remove.to_string());
+                    recipients.push(("".to_string(), email_to_remove.to_string()));
                 }
             }
             if command != SystemMessage::AutocryptSetupMessage
                 && command != SystemMessage::SecurejoinMessage
                 && context.get_config_bool(Config::MdnsEnabled)
             {
-                factory.req_mdn = true;
+                req_mdn = true;
             }
         }
-        let row = context.sql.query_row(
+        let (in_reply_to, references) = context.sql.query_row(
             "SELECT mime_in_reply_to, mime_references FROM msgs WHERE id=?",
             params![msg.id],
             |row| {
@@ -164,39 +143,37 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                     render_rfc724_mid_list(&references),
                 ))
             },
-        );
-
-        match row {
-            Ok((in_reply_to, references)) => {
-                factory.in_reply_to = in_reply_to;
-                factory.references = references;
-            }
-            Err(err) => {
-                error!(
-                    context,
-                    "mimefactory: failed to load mime_in_reply_to: {:?}", err
-                );
-            }
-        }
+        )?;
 
+        let factory = MimeFactory {
+            from_addr,
+            from_displayname,
+            selfstatus: context
+                .get_config(Config::Selfstatus)
+                .unwrap_or_else(|| context.stock_str(StockMessage::StatusLine).to_string()),
+            recipients,
+            timestamp: msg.timestamp_sort,
+            loaded: Loaded::Message { chat },
+            msg,
+            in_reply_to,
+            references,
+            req_mdn,
+            last_added_location_id: 0,
+            attach_selfavatar: add_selfavatar,
+            context,
+        };
         Ok(factory)
     }
 
-    pub fn from_mdn(context: &'a Context, msg: &'b Message) -> Result<Self, Error> {
-        // MDNs not enabled - check this is late, in the job. the
-        // user may have changed its choice while offline ...
-        ensure!(
-            context.get_config_bool(Config::MdnsEnabled),
-            "MDNs meanwhile disabled"
-        );
+    pub fn from_mdn(
+        context: &'a Context,
+        msg: &'b Message,
+        additional_msg_ids: Vec<String>,
+    ) -> Result<Self, Error> {
+        ensure!(!msg.chat_id.is_special(), "Invalid chat id");
 
         let contact = Contact::load_from_db(context, msg.from_id)?;
 
-        // Do not send MDNs trash etc.; chats.blocked is already checked by the caller
-        // in dc_markseen_msgs()
-        ensure!(!contact.is_blocked(), "Contact blocked");
-        ensure!(msg.chat_id > DC_CHAT_ID_LAST_SPECIAL, "Invalid chat id");
-
         Ok(MimeFactory {
             context,
             from_addr: context
@@ -206,13 +183,13 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             selfstatus: context
                 .get_config(Config::Selfstatus)
                 .unwrap_or_else(|| context.stock_str(StockMessage::StatusLine).to_string()),
-            recipients_names: vec![contact.get_authname().to_string()],
-            recipients_addr: vec![contact.get_addr().to_string()],
+            recipients: vec![(
+                contact.get_authname().to_string(),
+                contact.get_addr().to_string(),
+            )],
             timestamp: dc_create_smeared_timestamp(context),
-            loaded: Loaded::MDN,
+            loaded: Loaded::MDN { additional_msg_ids },
             msg,
-            chat: None,
-            increation: false,
             in_reply_to: String::default(),
             references: String::default(),
             req_mdn: false,
@@ -228,10 +205,10 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             .ok_or_else(|| format_err!("Not configured"))?;
 
         Ok(self
-            .recipients_addr
+            .recipients
             .iter()
-            .filter(|addr| *addr != &self_addr)
-            .map(|addr| {
+            .filter(|(_, addr)| addr != &self_addr)
+            .map(|(_, addr)| {
                 (
                     Peerstate::from_addr(self.context, &self.context.sql, addr),
                     addr.as_str(),
@@ -240,10 +217,10 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             .collect())
     }
 
-    fn is_e2ee_guranteed(&self) -> bool {
-        match self.loaded {
-            Loaded::Message => {
-                if self.chat.as_ref().unwrap().typ == Chattype::VerifiedGroup {
+    fn is_e2ee_guaranteed(&self) -> bool {
+        match &self.loaded {
+            Loaded::Message { chat } => {
+                if chat.typ == Chattype::VerifiedGroup {
                     return true;
                 }
 
@@ -264,28 +241,26 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
                 false
             }
-            Loaded::MDN => false,
+            Loaded::MDN { .. } => false,
         }
     }
 
     fn min_verified(&self) -> PeerstateVerifiedStatus {
-        match self.loaded {
-            Loaded::Message => {
-                let chat = self.chat.as_ref().unwrap();
+        match &self.loaded {
+            Loaded::Message { chat } => {
                 if chat.typ == Chattype::VerifiedGroup {
                     PeerstateVerifiedStatus::BidirectVerified
                 } else {
                     PeerstateVerifiedStatus::Unverified
                 }
             }
-            Loaded::MDN => PeerstateVerifiedStatus::Unverified,
+            Loaded::MDN { .. } => PeerstateVerifiedStatus::Unverified,
         }
     }
 
     fn should_force_plaintext(&self) -> i32 {
-        match self.loaded {
-            Loaded::Message => {
-                let chat = self.chat.as_ref().unwrap();
+        match &self.loaded {
+            Loaded::Message { chat } => {
                 if chat.typ == Chattype::VerifiedGroup {
                     0
                 } else {
@@ -295,30 +270,28 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                         .unwrap_or_default()
                 }
             }
-            Loaded::MDN => DC_FP_NO_AUTOCRYPT_HEADER,
+            Loaded::MDN { .. } => ForcePlaintext::NoAutocryptHeader as i32,
         }
     }
 
     fn should_do_gossip(&self) -> bool {
-        match self.loaded {
-            Loaded::Message => {
-                let chat = self.chat.as_ref().unwrap();
+        match &self.loaded {
+            Loaded::Message { chat } => {
                 // beside key- and member-changes, force re-gossip every 48 hours
                 let gossiped_timestamp = chat.get_gossiped_timestamp(self.context);
-                if gossiped_timestamp == 0 || (gossiped_timestamp + (2 * 24 * 60 * 60)) > time() {
+                if time() > gossiped_timestamp + (2 * 24 * 60 * 60) {
                     return true;
                 }
 
                 self.msg.param.get_cmd() == SystemMessage::MemberAddedToGroup
             }
-            Loaded::MDN => false,
+            Loaded::MDN { .. } => false,
         }
     }
 
     fn grpimage(&self) -> Option<String> {
-        match self.loaded {
-            Loaded::Message => {
-                let chat = self.chat.as_ref().unwrap();
+        match &self.loaded {
+            Loaded::Message { chat } => {
                 let cmd = self.msg.param.get_cmd();
 
                 match cmd {
@@ -342,51 +315,37 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
                 None
             }
-            Loaded::MDN => None,
+            Loaded::MDN { .. } => None,
         }
     }
 
     fn subject_str(&self) -> String {
         match self.loaded {
-            Loaded::Message => {
-                match self.chat {
-                    Some(ref chat) => {
-                        let raw = message::get_summarytext_by_raw(
-                            self.msg.type_0,
-                            self.msg.text.as_ref(),
-                            &self.msg.param,
-                            32,
-                            self.context,
-                        );
-                        let mut lines = raw.lines();
-                        let raw_subject = if let Some(line) = lines.next() {
-                            line
-                        } else {
-                            ""
-                        };
-
-                        let afwd_email = self.msg.param.exists(Param::Forwarded);
-                        let fwd = if afwd_email { "Fwd: " } else { "" };
-
-                        if self.msg.param.get_cmd() == SystemMessage::AutocryptSetupMessage {
-                            // do not add the "Chat:" prefix for setup messages
-                            self.context
-                                .stock_str(StockMessage::AcSetupMsgSubject)
-                                .into_owned()
-                        } else if chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup
-                        {
-                            format!("Chat: {}: {}{}", chat.name, fwd, raw_subject)
-                        } else {
-                            format!("Chat: {}{}", fwd, raw_subject)
-                        }
-                    }
-                    None => String::new(),
+            Loaded::Message { ref chat } => {
+                if self.msg.param.get_cmd() == SystemMessage::AutocryptSetupMessage {
+                    self.context
+                        .stock_str(StockMessage::AcSetupMsgSubject)
+                        .into_owned()
+                } else if chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup {
+                    let re = if self.in_reply_to.is_empty() {
+                        ""
+                    } else {
+                        "Re: "
+                    };
+                    format!("{}{}", re, chat.name)
+                } else {
+                    let raw = message::get_summarytext_by_raw(
+                        self.msg.viewtype,
+                        self.msg.text.as_ref(),
+                        &self.msg.param,
+                        32,
+                        self.context,
+                    );
+                    let raw_subject = raw.lines().next().unwrap_or_default();
+                    format!("Chat: {}", raw_subject)
                 }
             }
-            Loaded::MDN => {
-                let e = self.context.stock_str(StockMessage::ReadRcpt);
-                format!("Chat: {}", e)
-            }
+            Loaded::MDN { .. } => self.context.stock_str(StockMessage::ReadRcpt).into_owned(),
         }
     }
 
@@ -405,10 +364,8 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             self.from_addr.clone(),
         );
 
-        let mut to = Vec::with_capacity(self.recipients_names.len());
-        let name_iter = self.recipients_names.iter();
-        let addr_iter = self.recipients_addr.iter();
-        for (name, addr) in name_iter.zip(addr_iter) {
+        let mut to = Vec::with_capacity(self.recipients.len());
+        for (name, addr) in self.recipients.iter() {
             if name.is_empty() {
                 to.push(Address::new_mailbox(addr.clone()));
             } else {
@@ -451,6 +408,13 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         ));
         unprotected_headers.push(Header::new("Chat-Version".to_string(), "1.0".to_string()));
 
+        if let Loaded::MDN { .. } = self.loaded {
+            unprotected_headers.push(Header::new(
+                "Auto-Submitted".to_string(),
+                "auto-replied".to_string(),
+            ));
+        }
+
         if self.req_mdn {
             // we use "Chat-Disposition-Notification-To"
             // because replies to "Disposition-Notification-To" are weird in many cases
@@ -465,19 +429,19 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let grpimage = self.grpimage();
         let force_plaintext = self.should_force_plaintext();
         let subject_str = self.subject_str();
-        let e2ee_guranteed = self.is_e2ee_guranteed();
+        let e2ee_guaranteed = self.is_e2ee_guaranteed();
         let mut encrypt_helper = EncryptHelper::new(self.context)?;
 
         let subject = encode_words(&subject_str);
 
         let mut message = match self.loaded {
-            Loaded::Message => {
+            Loaded::Message { .. } => {
                 self.render_message(&mut protected_headers, &mut unprotected_headers, &grpimage)?
             }
-            Loaded::MDN => self.render_mdn()?,
+            Loaded::MDN { .. } => self.render_mdn()?,
         };
 
-        if force_plaintext != DC_FP_NO_AUTOCRYPT_HEADER {
+        if force_plaintext != ForcePlaintext::NoAutocryptHeader as i32 {
             // unless determined otherwise we add the Autocrypt header
             let aheader = encrypt_helper.get_aheader().to_string();
             unprotected_headers.push(Header::new("Autocrypt".into(), aheader));
@@ -487,12 +451,12 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
         let peerstates = self.peerstates_for_recipients()?;
         let should_encrypt =
-            encrypt_helper.should_encrypt(self.context, e2ee_guranteed, &peerstates)?;
+            encrypt_helper.should_encrypt(self.context, e2ee_guaranteed, &peerstates)?;
         let is_encrypted = should_encrypt && force_plaintext == 0;
 
         let rfc724_mid = match self.loaded {
-            Loaded::Message => self.msg.rfc724_mid.clone(),
-            Loaded::MDN => dc_create_outgoing_rfc724_mid(None, &self.from_addr),
+            Loaded::Message { .. } => self.msg.rfc724_mid.clone(),
+            Loaded::MDN { .. } => dc_create_outgoing_rfc724_mid(None, &self.from_addr),
         };
 
         // we could also store the message-id in the protected headers
@@ -601,11 +565,9 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         };
 
         let MimeFactory {
-            recipients_addr,
+            recipients,
             from_addr,
             last_added_location_id,
-            msg,
-            loaded,
             ..
         } = self;
 
@@ -615,16 +577,53 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             is_encrypted,
             is_gossiped,
             last_added_location_id,
-            foreign_id: match loaded {
-                Loaded::Message => Some(msg.id),
-                Loaded::MDN => None,
-            },
-            recipients: recipients_addr,
+            recipients: recipients.into_iter().map(|(_, addr)| addr).collect(),
             from: from_addr,
             rfc724_mid,
         })
     }
 
+    fn get_message_kml_part(&self) -> Option<PartBuilder> {
+        let latitude = self.msg.param.get_float(Param::SetLatitude)?;
+        let longitude = self.msg.param.get_float(Param::SetLongitude)?;
+
+        let kml_file = location::get_message_kml(self.msg.timestamp_sort, latitude, longitude);
+        let part = PartBuilder::new()
+            .content_type(
+                &"application/vnd.google-earth.kml+xml"
+                    .parse::<mime::Mime>()
+                    .unwrap(),
+            )
+            .header((
+                "Content-Disposition",
+                "attachment; filename=\"message.kml\"",
+            ))
+            .body(kml_file);
+        Some(part)
+    }
+
+    fn get_location_kml_part(&mut self) -> Result<PartBuilder, Error> {
+        let (kml_content, last_added_location_id) =
+            location::get_kml(self.context, self.msg.chat_id)?;
+        let part = PartBuilder::new()
+            .content_type(
+                &"application/vnd.google-earth.kml+xml"
+                    .parse::<mime::Mime>()
+                    .unwrap(),
+            )
+            .header((
+                "Content-Disposition",
+                "attachment; filename=\"location.kml\"",
+            ))
+            .body(kml_content);
+        if !self.msg.param.exists(Param::SetLatitude) {
+            // otherwise, the independent location is already filed
+            self.last_added_location_id = last_added_location_id;
+        }
+        Ok(part)
+    }
+
+    #[allow(clippy::cognitive_complexity)]
     fn render_message(
         &mut self,
         protected_headers: &mut Vec<Header>,
@@ -632,7 +631,10 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         grpimage: &Option<String>,
     ) -> Result<PartBuilder, Error> {
         let context = self.context;
-        let chat = self.chat.as_ref().unwrap();
+        let chat = match &self.loaded {
+            Loaded::Message { chat } => chat,
+            Loaded::MDN { .. } => bail!("Attempt to render MDN as a message"),
+        };
         let command = self.msg.param.get_cmd();
         let mut placeholdertext = None;
         let mut meta_part = None;
@@ -666,7 +668,9 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                             email_to_add.into(),
                         ));
                     }
-                    if 0 != self.msg.param.get_int(Param::Arg2).unwrap_or_default() & 0x1 {
+                    if 0 != self.msg.param.get_int(Param::Arg2).unwrap_or_default()
+                        & DC_FROM_HANDSHAKE
+                    {
                         info!(
                             context,
                             "sending secure-join message \'{}\' >>>>>>>>>>>>>>>>>>>>>>>>>",
@@ -761,7 +765,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         if let Some(grpimage) = grpimage {
             info!(self.context, "setting group image '{}'", grpimage);
             let mut meta = Message::default();
-            meta.type_0 = Viewtype::Image;
+            meta.viewtype = Viewtype::Image;
             meta.param.set(Param::File, grpimage);
 
             let (mail, filename_as_sent) = build_body_file(context, &meta, "group-image")?;
@@ -780,15 +784,15 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             }
         }
 
-        if self.msg.type_0 == Viewtype::Sticker {
+        if self.msg.viewtype == Viewtype::Sticker {
             protected_headers.push(Header::new("Chat-Content".into(), "sticker".into()));
         }
 
-        if self.msg.type_0 == Viewtype::Voice
-            || self.msg.type_0 == Viewtype::Audio
-            || self.msg.type_0 == Viewtype::Video
+        if self.msg.viewtype == Viewtype::Voice
+            || self.msg.viewtype == Viewtype::Audio
+            || self.msg.viewtype == Viewtype::Video
         {
-            if self.msg.type_0 == Viewtype::Voice {
+            if self.msg.viewtype == Viewtype::Voice {
                 protected_headers.push(Header::new("Chat-Voice-Message".into(), "1".into()));
             }
             let duration_ms = self.msg.param.get_int(Param::Duration).unwrap_or_default();
@@ -839,16 +843,17 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         );
 
         // Message is sent as text/plain, with charset = utf-8
-        let mut parts = vec![PartBuilder::new()
+        let main_part = PartBuilder::new()
             .content_type(&mime::TEXT_PLAIN_UTF_8)
-            .body(message_text)];
+            .body(message_text);
+        let mut parts = Vec::new();
 
         // add attachment part
-        if chat::msgtype_has_file(self.msg.type_0) {
+        if chat::msgtype_has_file(self.msg.viewtype) {
             if !is_file_size_okay(context, &self.msg) {
                 bail!(
                     "Message exceeds the recommended {} MB.",
-                    24 * 1024 * 1024 / 4 * 3 / 1000 / 1000,
+                    RECOMMENDED_FILE_SIZE / 1_000_000,
                 );
             } else {
                 let (file_part, _) = build_body_file(context, &self.msg, "")?;
@@ -860,51 +865,15 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             parts.push(meta_part);
         }
 
-        if self.msg.param.exists(Param::SetLatitude) {
-            let param = &self.msg.param;
-            let kml_file = location::get_message_kml(
-                self.msg.timestamp_sort,
-                param.get_float(Param::SetLatitude).unwrap_or_default(),
-                param.get_float(Param::SetLongitude).unwrap_or_default(),
-            );
-            parts.push(
-                PartBuilder::new()
-                    .content_type(
-                        &"application/vnd.google-earth.kml+xml"
-                            .parse::<mime::Mime>()
-                            .unwrap(),
-                    )
-                    .header((
-                        "Content-Disposition",
-                        "attachment; filename=\"message.kml\"",
-                    ))
-                    .body(kml_file),
-            );
+        if let Some(msg_kml_part) = self.get_message_kml_part() {
+            parts.push(msg_kml_part);
         }
 
         if location::is_sending_locations_to_chat(context, self.msg.chat_id) {
-            match location::get_kml(context, self.msg.chat_id) {
-                Ok((kml_content, last_added_location_id)) => {
-                    parts.push(
-                        PartBuilder::new()
-                            .content_type(
-                                &"application/vnd.google-earth.kml+xml"
-                                    .parse::<mime::Mime>()
-                                    .unwrap(),
-                            )
-                            .header((
-                                "Content-Disposition",
-                                "attachment; filename=\"message.kml\"",
-                            ))
-                            .body(kml_content),
-                    );
-                    if !self.msg.param.exists(Param::SetLatitude) {
-                        // otherwise, the independent location is already filed
-                        self.last_added_location_id = last_added_location_id;
-                    }
-                }
+            match self.get_location_kml_part() {
+                Ok(part) => parts.push(part),
                 Err(err) => {
-                    warn!(context, "mimefactory: could not get location: {}", err);
+                    warn!(context, "mimefactory: could not send location: {}", err);
                 }
             }
         }
@@ -922,24 +891,24 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             }
         }
 
-        // Single part, render as regular message.
-        if parts.len() == 1 {
-            return Ok(parts.pop().unwrap());
+        if parts.is_empty() {
+            // Single part, render as regular message.
+            Ok(main_part)
+        } else {
+            // Multiple parts, render as multipart.
+            let mut message = PartBuilder::new().message_type(MimeMultipartType::Mixed);
+            message = message.child(main_part.build());
+            for part in parts.into_iter() {
+                message = message.child(part.build());
+            }
+            Ok(message)
         }
-
-        // Multiple parts, render as multipart.
-        let mut message = PartBuilder::new().message_type(MimeMultipartType::Mixed);
-        for part in parts.into_iter() {
-            message = message.child(part.build());
-        }
-
-        Ok(message)
     }
 
     /// Render an MDN
     fn render_mdn(&mut self) -> Result<PartBuilder, Error> {
         // RFC 6522, this also requires the `report-type` parameter which is equal
-        // to the MIME subtype of the second body part of the multipart/report */
+        // to the MIME subtype of the second body part of the multipart/report
         //
         // currently, we do not send MDNs encrypted:
         // - in a multi-device-setup that is not set up properly, MDNs would disturb the communication as they
@@ -950,6 +919,13 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         //   are forwarded for any reasons (eg. gmail always forwards to IMAP), we have no chance to decrypt them;
         //   this issue is fixed with 0.9.4
 
+        let additional_msg_ids = match &self.loaded {
+            Loaded::Message { .. } => bail!("Attempt to render a message as MDN"),
+            Loaded::MDN {
+                additional_msg_ids, ..
+            } => additional_msg_ids,
+        };
+
         let mut message = PartBuilder::new().header((
             "Content-Type".to_string(),
             "multipart/report; report-type=disposition-notification".to_string(),
@@ -991,10 +967,22 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             version, self.from_addr, self.from_addr, self.msg.rfc724_mid
         );
 
+        let extension_fields = if additional_msg_ids.is_empty() {
+            "".to_string()
+        } else {
+            "Additional-Message-IDs: ".to_string()
+                + &additional_msg_ids
+                    .iter()
+                    .map(|mid| render_rfc724_mid(&mid))
+                    .collect::<Vec<String>>()
+                    .join(" ")
+                + "\r\n"
+        };
+
         message = message.child(
             PartBuilder::new()
                 .content_type(&"message/disposition-notification".parse().unwrap())
-                .body(message_text2)
+                .body(message_text2 + &extension_fields)
                 .build(),
         );
 
@@ -1002,6 +990,20 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
     }
 }
 
+/// Returns base64-encoded buffer `buf` split into 78-bytes long
+/// chunks separated by CRLF.
+///
+/// This line length limit is an
+/// [RFC5322 requirement](https://tools.ietf.org/html/rfc5322#section-2.1.1).
+fn wrapped_base64_encode(buf: &[u8]) -> String {
+    let base64 = base64::encode(&buf);
+    let mut chars = base64.chars();
+    std::iter::repeat_with(|| chars.by_ref().take(78).collect::<String>())
+        .take_while(|s| !s.is_empty())
+        .collect::<Vec<_>>()
+        .join("\r\n")
+}
+
 fn build_body_file(
     context: &Context,
     msg: &Message,
@@ -1017,7 +1019,7 @@ fn build_body_file(
     // not transfer the original filenames eg. for images; these names
     // are normally not needed and contain timestamps, running numbers
     // etc.
-    let filename_to_send: String = match msg.type_0 {
+    let filename_to_send: String = match msg.viewtype {
         Viewtype::Voice => chrono::Utc
             .timestamp(msg.timestamp_sort as i64, 0)
             .format(&format!("voice-message_%Y-%m-%d_%H-%M-%S.{}", &suffix))
@@ -1062,7 +1064,7 @@ fn build_body_file(
     };
 
     let body = std::fs::read(blob.to_abs_path())?;
-    let encoded_body = base64::encode(&body);
+    let encoded_body = wrapped_base64_encode(&body);
 
     let mail = PartBuilder::new()
         .content_type(&mimetype)
@@ -1084,7 +1086,7 @@ fn build_selfavatar_file(context: &Context, path: String) -> Result<(PartBuilder
         None => mime::APPLICATION_OCTET_STREAM,
     };
     let body = std::fs::read(blob.to_abs_path())?;
-    let encoded_body = base64::encode(&body);
+    let encoded_body = wrapped_base64_encode(&body);
 
     let part = PartBuilder::new()
         .content_type(&mimetype)
@@ -1098,21 +1100,18 @@ fn build_selfavatar_file(context: &Context, path: String) -> Result<(PartBuilder
     Ok((part, filename_to_send))
 }
 
-pub(crate) fn vec_contains_lowercase(vec: &[String], part: &str) -> bool {
-    let partlc = part.to_lowercase();
-    for cur in vec.iter() {
-        if cur.to_lowercase() == partlc {
-            return true;
-        }
-    }
-    false
+fn recipients_contain_addr(recipients: &[(String, String)], addr: &str) -> bool {
+    let addr_lc = addr.to_lowercase();
+    recipients
+        .iter()
+        .any(|(_, cur)| cur.to_lowercase() == addr_lc)
 }
 
 fn is_file_size_okay(context: &Context, msg: &Message) -> bool {
     match msg.param.get_path(Param::File, context).unwrap_or(None) {
         Some(path) => {
             let bytes = dc_get_filebytes(context, &path);
-            bytes <= (49 * 1024 * 1024 / 4 * 3)
+            bytes <= UPPER_LIMIT_FILE_SIZE
         }
         None => false,
     }
@@ -1121,7 +1120,7 @@ fn is_file_size_okay(context: &Context, msg: &Message) -> bool {
 fn render_rfc724_mid(rfc724_mid: &str) -> String {
     let rfc724_mid = rfc724_mid.trim().to_string();
 
-    if rfc724_mid.chars().nth(0).unwrap_or_default() == '<' {
+    if rfc724_mid.chars().next().unwrap_or_default() == '<' {
         rfc724_mid
     } else {
         format!("<{}>", rfc724_mid)
@@ -1146,14 +1145,8 @@ fn encode_words(word: &str) -> String {
 }
 
 pub fn needs_encoding(to_check: impl AsRef<str>) -> bool {
-    let to_check = to_check.as_ref();
-
-    if to_check.is_empty() {
-        return false;
-    }
-
-    to_check.chars().any(|c| {
-        !c.is_ascii_alphanumeric() && c != '-' && c != '_' && c != '.' && c != '~' && c != '%'
+    !to_check.as_ref().chars().all(|c| {
+        c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.' || c == '~' || c == '%'
     })
 }
 
@@ -1203,4 +1196,21 @@ mod tests {
             "<123@q> <456@d>".to_string()
         );
     }
+
+    #[test]
+    fn test_wrapped_base64_encode() {
+        let input = b"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA";
+        let output =
+            "QUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQU\r\n\
+             FBQUFBQUFBQQ==";
+        assert_eq!(wrapped_base64_encode(input), output);
+    }
+
+    #[test]
+    fn test_needs_encoding() {
+        assert!(!needs_encoding(""));
+        assert!(!needs_encoding("foobar"));
+        assert!(needs_encoding(" "));
+        assert!(needs_encoding("foo bar"));
+    }
 }

src/mimeparser.rs

@@ -2,7 +2,7 @@ use std::collections::{HashMap, HashSet};
 
 use deltachat_derive::{FromSql, ToSql};
 use lettre_email::mime::{self, Mime};
-use mailparse::{DispositionType, MailHeaderMap};
+use mailparse::{DispositionType, MailAddr, MailHeaderMap};
 
 use crate::aheader::Aheader;
 use crate::blob::BlobObject;
@@ -10,23 +10,33 @@ use crate::config::Config;
 use crate::constants::Viewtype;
 use crate::contact::*;
 use crate::context::Context;
-use crate::dc_simplify::*;
 use crate::dc_tools::*;
+use crate::dehtml::dehtml;
 use crate::e2ee;
 use crate::error::Result;
+use crate::events::Event;
 use crate::headerdef::HeaderDef;
 use crate::job::{job_add, Action};
 use crate::location;
 use crate::message;
-use crate::message::MsgId;
 use crate::param::*;
 use crate::peerstate::Peerstate;
 use crate::securejoin::handle_degrade_event;
+use crate::simplify::*;
 use crate::stock::StockMessage;
 use crate::{bail, ensure};
 
+/// A parsed MIME message.
+///
+/// This represents the relevant information of a parsed MIME message
+/// for deltachat.  The original MIME message might have had more
+/// information but this representation should contain everything
+/// needed for deltachat's purposes.
+///
+/// It is created by parsing the raw data of an actual MIME message
+/// using the [MimeMessage::from_bytes] constructor.
 #[derive(Debug)]
-pub struct MimeParser<'a> {
+pub struct MimeMessage<'a> {
     pub context: &'a Context,
     pub parts: Vec<Part>,
     header: HashMap<String, String>,
@@ -39,9 +49,7 @@ pub struct MimeParser<'a> {
     pub message_kml: Option<location::Kml>,
     pub user_avatar: AvatarAction,
     pub group_avatar: AvatarAction,
-    reports: Vec<Report>,
-    mdns_enabled: bool,
-    parsed_protected_headers: bool,
+    pub(crate) reports: Vec<Report>,
 }
 
 #[derive(Debug, PartialEq)]
@@ -83,30 +91,9 @@ impl Default for SystemMessage {
 
 const MIME_AC_SETUP_FILE: &str = "application/autocrypt-setup";
 
-impl<'a> MimeParser<'a> {
+impl<'a> MimeMessage<'a> {
     pub fn from_bytes(context: &'a Context, body: &[u8]) -> Result<Self> {
         let mail = mailparse::parse_mail(body)?;
-        let mdns_enabled = context.get_config_bool(Config::MdnsEnabled);
-
-        let mut parser = MimeParser {
-            parts: Vec::new(),
-            header: Default::default(),
-            decrypting_failed: false,
-
-            // only non-empty if it was a valid autocrypt message
-            signatures: Default::default(),
-            gossipped_addr: Default::default(),
-            is_forwarded: false,
-            context,
-            reports: Vec::new(),
-            is_system_message: SystemMessage::Unknown,
-            location_kml: None,
-            message_kml: None,
-            user_avatar: AvatarAction::None,
-            group_avatar: AvatarAction::None,
-            mdns_enabled,
-            parsed_protected_headers: false,
-        };
 
         let message_time = mail
             .headers
@@ -114,18 +101,19 @@ impl<'a> MimeParser<'a> {
             .and_then(|v| mailparse::dateparse(&v).ok())
             .unwrap_or_default();
 
+        let mut headers = Default::default();
+
         // init known headers with what mailparse provided us
-        parser.merge_headers(&mail.headers);
+        MimeMessage::merge_headers(&mut headers, &mail.headers);
 
         // Memory location for a possible decrypted message.
         let mail_raw;
+        let mut gossipped_addr = Default::default();
 
-        let mail = match e2ee::try_decrypt(parser.context, &mail, message_time) {
+        let (mail, signatures) = match e2ee::try_decrypt(context, &mail, message_time) {
             Ok((raw, signatures)) => {
-                // Valid autocrypt message, encrypted
-                parser.signatures = signatures;
-
                 if let Some(raw) = raw {
+                    // Valid autocrypt message, encrypted
                     mail_raw = raw;
                     let decrypted_mail = mailparse::parse_mail(&mail_raw)?;
                     if std::env::var(crate::DCC_MIME_DEBUG).is_ok() {
@@ -137,17 +125,17 @@ impl<'a> MimeParser<'a> {
                     // "3.6 Key Gossip" of https://autocrypt.org/autocrypt-spec-1.1.0.pdf
                     let gossip_headers =
                         decrypted_mail.headers.get_all_values("Autocrypt-Gossip")?;
-                    parser.gossipped_addr =
+                    gossipped_addr =
                         update_gossip_peerstates(context, message_time, &mail, gossip_headers)?;
 
                     // let known protected headers from the decrypted
                     // part override the unencrypted top-level
-                    parser.merge_headers(&decrypted_mail.headers);
+                    MimeMessage::merge_headers(&mut headers, &decrypted_mail.headers);
 
-                    decrypted_mail
+                    (decrypted_mail, signatures)
                 } else {
                     // Message was not encrypted
-                    mail
+                    (mail, signatures)
                 }
             }
             Err(err) => {
@@ -158,24 +146,47 @@ impl<'a> MimeParser<'a> {
                 // if we just return here, the header is missing
                 // and the caller cannot display the message
                 // and try to assign the message to a chat
-                warn!(parser.context, "decryption failed: {}", err);
-                mail
+                warn!(context, "decryption failed: {}", err);
+                (mail, Default::default())
             }
         };
 
-        parser.parse_mime_recursive(&mail)?;
+        let mut parser = MimeMessage {
+            parts: Vec::new(),
+            header: headers,
+            decrypting_failed: false,
 
+            // only non-empty if it was a valid autocrypt message
+            signatures,
+            gossipped_addr,
+            is_forwarded: false,
+            context,
+            reports: Vec::new(),
+            is_system_message: SystemMessage::Unknown,
+            location_kml: None,
+            message_kml: None,
+            user_avatar: AvatarAction::None,
+            group_avatar: AvatarAction::None,
+        };
+        parser.parse_mime_recursive(&mail)?;
         parser.parse_headers()?;
 
         Ok(parser)
     }
 
-    fn parse_headers(&mut self) -> Result<()> {
+    /// Parses system messages.
+    fn parse_system_message_headers(&mut self) -> Result<()> {
         if self.get(HeaderDef::AutocryptSetupMessage).is_some() {
-            self.parts.drain_filter(|part| {
-                part.mimetype.is_some()
-                    && part.mimetype.as_ref().unwrap().as_ref() != MIME_AC_SETUP_FILE
-            });
+            self.parts = self
+                .parts
+                .iter()
+                .filter(|part| {
+                    part.mimetype.is_none()
+                        || part.mimetype.as_ref().unwrap().as_ref() == MIME_AC_SETUP_FILE
+                })
+                .cloned()
+                .collect();
+
             if self.parts.len() == 1 {
                 self.is_system_message = SystemMessage::AutocryptSetupMessage;
             } else {
@@ -186,7 +197,11 @@ impl<'a> MimeParser<'a> {
                 self.is_system_message = SystemMessage::LocationStreamingEnabled;
             }
         }
+        Ok(())
+    }
 
+    /// Parses avatar action headers.
+    fn parse_avatar_headers(&mut self) {
         if let Some(header_value) = self.get(HeaderDef::ChatGroupAvatar).cloned() {
             self.group_avatar = self.avatar_action_from_header(header_value);
         } else if let Some(header_value) = self.get(HeaderDef::ChatGroupImage).cloned() {
@@ -198,7 +213,13 @@ impl<'a> MimeParser<'a> {
         if let Some(header_value) = self.get(HeaderDef::ChatUserAvatar).cloned() {
             self.user_avatar = self.avatar_action_from_header(header_value);
         }
+    }
 
+    /// Squashes mutlipart chat messages with attachment into single-part messages.
+    ///
+    /// Delta Chat sends attachments, such as images, in two-part messages, with the first message
+    /// containing an explanation. If such a message is detected, first part can be safely dropped.
+    fn squash_attachment_parts(&mut self) {
         if self.has_chat_version() && self.parts.len() == 2 {
             let need_drop = {
                 let textpart = &self.parts[0];
@@ -211,7 +232,6 @@ impl<'a> MimeParser<'a> {
                         || filepart.typ == Viewtype::Voice
                         || filepart.typ == Viewtype::Video
                         || filepart.typ == Viewtype::File)
-                    && !filepart.is_meta
             };
 
             if need_drop {
@@ -227,41 +247,12 @@ impl<'a> MimeParser<'a> {
                 std::mem::replace(&mut self.parts[0], filepart);
             }
         }
-        if let Some(ref subject) = self.get_subject() {
-            let mut prepend_subject = 1i32;
-            if !self.decrypting_failed {
-                let colon = subject.find(':');
-                if colon == Some(2)
-                    || colon == Some(3)
-                    || self.has_chat_version()
-                    || subject.contains("Chat:")
-                {
-                    prepend_subject = 0i32
-                }
-            }
-            if 0 != prepend_subject {
-                let subj = if let Some(n) = subject.find('[') {
-                    &subject[0..n]
-                } else {
-                    subject
-                }
-                .trim();
+    }
 
-                if !subj.is_empty() {
-                    for part in self.parts.iter_mut() {
-                        if part.typ == Viewtype::Text {
-                            part.msg = format!("{} – {}", subj, part.msg);
-                            break;
-                        }
-                    }
-                }
-            }
-        }
-        if self.is_forwarded {
-            for part in self.parts.iter_mut() {
-                part.param.set_int(Param::Forwarded, 1);
-            }
-        }
+    /// Processes chat messages with attachments.
+    fn parse_attachments(&mut self) {
+        // Attachment messages should be squashed into a single part
+        // before calling this function.
         if self.parts.len() == 1 {
             if self.parts[0].typ == Viewtype::Audio
                 && self.get(HeaderDef::ChatVoiceMessage).is_some()
@@ -291,30 +282,71 @@ impl<'a> MimeParser<'a> {
                 }
             }
         }
-        if !self.decrypting_failed {
-            if let Some(dn_field) = self.get(HeaderDef::ChatDispositionNotificationTo) {
-                if self.get_last_nonmeta().is_some() {
-                    let addrs = mailparse::addrparse(&dn_field).unwrap();
+    }
 
-                    if let Some(dn_to_addr) = addrs.first() {
-                        if let Some(from_field) = self.get(HeaderDef::From_) {
-                            let from_addrs = mailparse::addrparse(&from_field).unwrap();
+    fn parse_headers(&mut self) -> Result<()> {
+        self.parse_system_message_headers()?;
+        self.parse_avatar_headers();
+        self.squash_attachment_parts();
 
-                            if let Some(from_addr) = from_addrs.first() {
-                                if compare_addrs(from_addr, dn_to_addr) {
-                                    if let Some(part_4) = self.get_last_nonmeta_mut() {
-                                        part_4.param.set_int(Param::WantsMdn, 1);
-                                    }
-                                }
-                            }
+        if let Some(ref subject) = self.get_subject() {
+            let mut prepend_subject = true;
+            if !self.decrypting_failed {
+                let colon = subject.find(':');
+                if colon == Some(2)
+                    || colon == Some(3)
+                    || self.has_chat_version()
+                    || subject.contains("Chat:")
+                {
+                    prepend_subject = false
+                }
+            }
+            if prepend_subject {
+                let subj = if let Some(n) = subject.find('[') {
+                    &subject[0..n]
+                } else {
+                    subject
+                }
+                .trim();
+
+                if !subj.is_empty() {
+                    for part in self.parts.iter_mut() {
+                        if part.typ == Viewtype::Text {
+                            part.msg = format!("{} – {}", subj, part.msg);
+                            break;
+                        }
+                    }
+                }
+            }
+        }
+        if self.is_forwarded {
+            for part in self.parts.iter_mut() {
+                part.param.set_int(Param::Forwarded, 1);
+            }
+        }
+
+        self.parse_attachments();
+
+        // See if an MDN is requested from the other side
+        if !self.decrypting_failed && !self.parts.is_empty() {
+            if let Some(ref dn_to_addr) =
+                self.parse_first_addr(HeaderDef::ChatDispositionNotificationTo)
+            {
+                if let Some(ref from_addr) = self.parse_first_addr(HeaderDef::From_) {
+                    if compare_addrs(from_addr, dn_to_addr) {
+                        if let Some(part) = self.parts.last_mut() {
+                            part.param.set_int(Param::WantsMdn, 1);
                         }
                     }
                 }
             }
         }
 
-        // Cleanup - and try to create at least an empty part if there are no parts yet
-        if self.get_last_nonmeta().is_none() && self.reports.is_empty() {
+        // If there were no parts, especially a non-DC mail user may
+        // just have send a message in the subject with an empty body.
+        // Besides, we want to show something in case our incoming-processing
+        // failed to properly handle an incoming message.
+        if self.parts.is_empty() && self.reports.is_empty() {
             let mut part = Part::default();
             part.typ = Viewtype::Text;
 
@@ -353,14 +385,6 @@ impl<'a> MimeParser<'a> {
         AvatarAction::None
     }
 
-    pub fn get_last_nonmeta(&self) -> Option<&Part> {
-        self.parts.iter().rev().find(|part| !part.is_meta)
-    }
-
-    pub fn get_last_nonmeta_mut(&mut self) -> Option<&mut Part> {
-        self.parts.iter_mut().rev().find(|part| !part.is_meta)
-    }
-
     pub fn was_encrypted(&self) -> bool {
         !self.signatures.is_empty()
     }
@@ -374,20 +398,29 @@ impl<'a> MimeParser<'a> {
     }
 
     pub(crate) fn get_subject(&self) -> Option<String> {
-        if let Some(s) = self.get(HeaderDef::Subject) {
-            if s.is_empty() {
-                return None;
-            }
-            Some(s.to_string())
-        } else {
-            None
-        }
+        self.get(HeaderDef::Subject)
+            .filter(|s| !s.is_empty())
+            .map(|s| s.to_string())
     }
 
     pub fn get(&self, headerdef: HeaderDef) -> Option<&String> {
         self.header.get(&headerdef.get_headername())
     }
 
+    fn parse_first_addr(&self, headerdef: HeaderDef) -> Option<MailAddr> {
+        if let Some(value) = self.get(headerdef.clone()) {
+            match mailparse::addrparse(&value) {
+                Ok(ref addrs) => {
+                    return addrs.first().cloned();
+                }
+                Err(err) => {
+                    warn!(self.context, "header {} parse error: {:?}", headerdef, err);
+                }
+            }
+        }
+        None
+    }
+
     fn parse_mime_recursive(&mut self, mail: &mailparse::ParsedMail<'_>) -> Result<bool> {
         if mail.ctype.params.get("protected-headers").is_some() {
             if mail.ctype.mimetype == "text/rfc822-headers" {
@@ -577,12 +610,16 @@ impl<'a> MimeParser<'a> {
                         }
                     };
 
-                    let mut simplifier = Simplify::new();
-                    let simplified_txt = if decoded_data.is_empty() {
-                        "".into()
+                    let (simplified_txt, is_forwarded) = if decoded_data.is_empty() {
+                        ("".into(), false)
                     } else {
                         let is_html = mime_type == mime::TEXT_HTML;
-                        simplifier.simplify(&decoded_data, is_html, self.has_chat_version())
+                        let out = if is_html {
+                            dehtml(&decoded_data)
+                        } else {
+                            decoded_data.clone()
+                        };
+                        simplify(out, self.has_chat_version())
                     };
 
                     if !simplified_txt.is_empty() {
@@ -594,7 +631,7 @@ impl<'a> MimeParser<'a> {
                         self.do_add_single_part(part);
                     }
 
-                    if simplifier.is_forwarded {
+                    if is_forwarded {
                         self.is_forwarded = true;
                     }
                 }
@@ -702,23 +739,20 @@ impl<'a> MimeParser<'a> {
     }
 
     pub fn get_rfc724_mid(&self) -> Option<String> {
-        if let Some(msgid) = self.get(HeaderDef::MessageId) {
-            parse_message_id(msgid)
-        } else {
-            None
-        }
+        self.get(HeaderDef::MessageId)
+            .and_then(|msgid| parse_message_id(msgid))
     }
 
-    fn merge_headers(&mut self, fields: &[mailparse::MailHeader<'_>]) {
+    fn merge_headers(headers: &mut HashMap<String, String>, fields: &[mailparse::MailHeader<'_>]) {
         for field in fields {
             if let Ok(key) = field.get_key() {
                 // lowercasing all headers is technically not correct, but makes things work better
                 let key = key.to_lowercase();
-                if !self.header.contains_key(&key) || // key already exists, only overwrite known types (protected headers)
+                if !headers.contains_key(&key) || // key already exists, only overwrite known types (protected headers)
                     is_known(&key) || key.starts_with("chat-")
                 {
                     if let Ok(value) = field.get_value() {
-                        self.header.insert(key, value);
+                        headers.insert(key, value);
                     }
                 }
             }
@@ -726,11 +760,6 @@ impl<'a> MimeParser<'a> {
     }
 
     fn process_report(&self, report: &mailparse::ParsedMail<'_>) -> Result<Option<Report>> {
-        // to get a clear functionality, do not show incoming MDNs if the options is disabled
-        if !self.mdns_enabled {
-            return Ok(None);
-        }
-
         // parse as mailheaders
         let report_body = report.subparts[1].get_body_raw()?;
         let (report_fields, _) = mailparse::parse_headers(&report_body)?;
@@ -744,47 +773,67 @@ impl<'a> MimeParser<'a> {
                 .flatten()
                 .and_then(|v| parse_message_id(&v))
             {
+                let additional_message_ids = report_fields
+                    .get_first_value(&HeaderDef::AdditionalMessageIds.get_headername())
+                    .ok()
+                    .flatten()
+                    .map_or_else(Vec::new, |v| {
+                        v.split(' ').filter_map(parse_message_id).collect()
+                    });
+
                 return Ok(Some(Report {
                     original_message_id,
+                    additional_message_ids,
                 }));
             }
         }
+        warn!(
+            self.context,
+            "ignoring unknown disposition-notification, Message-Id: {:?}",
+            report_fields.get_first_value("Message-ID").ok()
+        );
 
         Ok(None)
     }
 
-    // Handle reports (mainly MDNs)
+    /// Handle reports (only MDNs for now)
     pub fn handle_reports(
         &self,
         from_id: u32,
         sent_timestamp: i64,
-        rr_event_to_send: &mut Vec<(u32, MsgId)>,
         server_folder: impl AsRef<str>,
         server_uid: u32,
     ) {
+        if self.reports.is_empty() {
+            return;
+        }
+
+        let mut mdn_recognized = false;
         for report in &self.reports {
-            let mut mdn_consumed = false;
-
-            if let Some((chat_id, msg_id)) = message::mdn_from_ext(
-                self.context,
-                from_id,
-                &report.original_message_id,
-                sent_timestamp,
-            ) {
-                rr_event_to_send.push((chat_id, msg_id));
-                mdn_consumed = true;
-            }
-
-            if self.has_chat_version() || mdn_consumed {
-                let mut param = Params::new();
-                param.set(Param::ServerFolder, server_folder.as_ref());
-                param.set_int(Param::ServerUid, server_uid as i32);
-                if self.has_chat_version() && self.context.get_config_bool(Config::MvboxMove) {
-                    param.set_int(Param::AlsoMove, 1);
+            for original_message_id in
+                std::iter::once(&report.original_message_id).chain(&report.additional_message_ids)
+            {
+                if let Some((chat_id, msg_id)) = message::mdn_from_ext(
+                    self.context,
+                    from_id,
+                    original_message_id,
+                    sent_timestamp,
+                ) {
+                    self.context.call_cb(Event::MsgRead { chat_id, msg_id });
+                    mdn_recognized = true;
                 }
-                job_add(self.context, Action::MarkseenMdnOnImap, 0, param, 0);
             }
         }
+
+        if self.has_chat_version() || mdn_recognized {
+            let mut param = Params::new();
+            param.set(Param::ServerFolder, server_folder.as_ref());
+            param.set_int(Param::ServerUid, server_uid as i32);
+            if self.has_chat_version() && self.context.get_config_bool(Config::MvboxMove) {
+                param.set_int(Param::AlsoMove, 1);
+            }
+            job_add(self.context, Action::MarkseenMdnOnImap, 0, param, 0);
+        }
     }
 }
 
@@ -813,7 +862,11 @@ fn update_gossip_peerstates(
                 })));
             }
 
-            if recipients.as_ref().unwrap().contains(&header.addr) {
+            if recipients
+                .as_ref()
+                .unwrap()
+                .contains(&header.addr.to_lowercase())
+            {
                 let mut peerstate = Peerstate::from_addr(context, &context.sql, &header.addr);
                 if let Some(ref mut peerstate) = peerstate {
                     peerstate.apply_gossip(header, message_time);
@@ -843,8 +896,11 @@ fn update_gossip_peerstates(
 }
 
 #[derive(Debug)]
-struct Report {
+pub(crate) struct Report {
+    /// Original-Message-ID header
     original_message_id: String,
+    /// Additional-Message-IDs
+    additional_message_ids: Vec<String>,
 }
 
 fn parse_message_id(field: &str) -> Option<String> {
@@ -868,7 +924,6 @@ fn is_known(key: &str) -> bool {
 #[derive(Debug, Default, Clone)]
 pub struct Part {
     pub typ: Viewtype,
-    pub is_meta: bool,
     pub mimetype: Option<Mime>,
     pub msg: String,
     pub msg_raw: Option<String>,
@@ -970,7 +1025,7 @@ fn get_attachment_filename(mail: &mailparse::ParsedMail) -> Result<String> {
     Ok(desired_filename)
 }
 
-// returned addresses are normalized.
+// returned addresses are normalized and lowercased.
 fn get_recipients<S: AsRef<str>, T: Iterator<Item = (S, S)>>(headers: T) -> HashSet<String> {
     let mut recipients: HashSet<String> = Default::default();
 
@@ -982,11 +1037,11 @@ fn get_recipients<S: AsRef<str>, T: Iterator<Item = (S, S)>>(headers: T) -> Hash
                 for addr in addrs.iter() {
                     match addr {
                         mailparse::MailAddr::Single(ref info) => {
-                            recipients.insert(addr_normalize(&info.addr).into());
+                            recipients.insert(addr_normalize(&info.addr).to_lowercase());
                         }
                         mailparse::MailAddr::Group(ref infos) => {
                             for info in &infos.addrs {
-                                recipients.insert(addr_normalize(&info.addr).into());
+                                recipients.insert(addr_normalize(&info.addr).to_lowercase());
                             }
                         }
                     }
@@ -1020,34 +1075,22 @@ fn compare_addrs(a: &mailparse::MailAddr, b: &mailparse::MailAddr) -> bool {
 mod tests {
     use super::*;
     use crate::test_utils::*;
-    use proptest::prelude::*;
 
     #[test]
     fn test_dc_mimeparser_crash() {
         let context = dummy_context();
         let raw = include_bytes!("../test-data/message/issue_523.txt");
-        let mimeparser = MimeParser::from_bytes(&context.ctx, &raw[..]).unwrap();
+        let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]).unwrap();
 
         assert_eq!(mimeparser.get_subject(), None);
         assert_eq!(mimeparser.parts.len(), 1);
     }
 
-    proptest! {
-        #[ignore]
-        #[test]
-        fn test_dc_mailmime_parse_crash_fuzzy(data in "[!-~\t ]{2000,}") {
-            let context = dummy_context();
-
-            // just don't crash
-            let _ = MimeParser::from_bytes(&context.ctx, data.as_bytes());
-        }
-    }
-
     #[test]
     fn test_get_rfc724_mid_exists() {
         let context = dummy_context();
         let raw = include_bytes!("../test-data/message/mail_with_message_id.txt");
-        let mimeparser = MimeParser::from_bytes(&context.ctx, &raw[..]).unwrap();
+        let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]).unwrap();
 
         assert_eq!(
             mimeparser.get_rfc724_mid(),
@@ -1059,7 +1102,7 @@ mod tests {
     fn test_get_rfc724_mid_not_exists() {
         let context = dummy_context();
         let raw = include_bytes!("../test-data/message/issue_523.txt");
-        let mimeparser = MimeParser::from_bytes(&context.ctx, &raw[..]).unwrap();
+        let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]).unwrap();
         assert_eq!(mimeparser.get_rfc724_mid(), None);
     }
 
@@ -1067,7 +1110,7 @@ mod tests {
     fn test_get_recipients() {
         let context = dummy_context();
         let raw = include_bytes!("../test-data/message/mail_with_cc.txt");
-        let mimeparser = MimeParser::from_bytes(&context.ctx, &raw[..]).unwrap();
+        let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]).unwrap();
         let recipients = get_recipients(mimeparser.header.iter());
         assert!(recipients.contains("abc@bcd.com"));
         assert!(recipients.contains("def@def.de"));
@@ -1110,6 +1153,26 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_parse_first_addr() {
+        let context = dummy_context();
+        let raw = b"From: hello@one.org, world@two.org\n\
+                    Chat-Disposition-Notification-To: wrong\n\
+                    Content-Type: text/plain\n\
+                    Chat-Version: 1.0\n\
+                    \n\
+                    test1\n\
+                    ";
+
+        let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]).unwrap();
+
+        let of = mimeparser.parse_first_addr(HeaderDef::From_).unwrap();
+        assert_eq!(of, mailparse::addrparse("hello@one.org").unwrap()[0]);
+
+        let of = mimeparser.parse_first_addr(HeaderDef::ChatDispositionNotificationTo);
+        assert!(of.is_none());
+    }
+
     #[test]
     fn test_mimeparser_with_context() {
         let context = dummy_context();
@@ -1129,10 +1192,9 @@ mod tests {
                     test1\n\
                     \n\
                     --==break==--\n\
-                    \n\
-                    \x00";
+                    \n";
 
-        let mimeparser = MimeParser::from_bytes(&context.ctx, &raw[..]).unwrap();
+        let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]).unwrap();
 
         // non-overwritten headers do not bubble up
         let of = mimeparser.get(HeaderDef::SecureJoinGroup).unwrap();
@@ -1157,26 +1219,26 @@ mod tests {
         let t = dummy_context();
 
         let raw = include_bytes!("../test-data/message/mail_attach_txt.eml");
-        let mimeparser = MimeParser::from_bytes(&t.ctx, &raw[..]).unwrap();
+        let mimeparser = MimeMessage::from_bytes(&t.ctx, &raw[..]).unwrap();
         assert_eq!(mimeparser.user_avatar, AvatarAction::None);
         assert_eq!(mimeparser.group_avatar, AvatarAction::None);
 
         let raw = include_bytes!("../test-data/message/mail_with_user_avatar.eml");
-        let mimeparser = MimeParser::from_bytes(&t.ctx, &raw[..]).unwrap();
+        let mimeparser = MimeMessage::from_bytes(&t.ctx, &raw[..]).unwrap();
         assert_eq!(mimeparser.parts.len(), 1);
         assert_eq!(mimeparser.parts[0].typ, Viewtype::Text);
         assert!(mimeparser.user_avatar.is_change());
         assert_eq!(mimeparser.group_avatar, AvatarAction::None);
 
         let raw = include_bytes!("../test-data/message/mail_with_user_avatar_deleted.eml");
-        let mimeparser = MimeParser::from_bytes(&t.ctx, &raw[..]).unwrap();
+        let mimeparser = MimeMessage::from_bytes(&t.ctx, &raw[..]).unwrap();
         assert_eq!(mimeparser.parts.len(), 1);
         assert_eq!(mimeparser.parts[0].typ, Viewtype::Text);
         assert_eq!(mimeparser.user_avatar, AvatarAction::Delete);
         assert_eq!(mimeparser.group_avatar, AvatarAction::None);
 
         let raw = include_bytes!("../test-data/message/mail_with_user_and_group_avatars.eml");
-        let mimeparser = MimeParser::from_bytes(&t.ctx, &raw[..]).unwrap();
+        let mimeparser = MimeMessage::from_bytes(&t.ctx, &raw[..]).unwrap();
         assert_eq!(mimeparser.parts.len(), 1);
         assert_eq!(mimeparser.parts[0].typ, Viewtype::Text);
         assert!(mimeparser.user_avatar.is_change());
@@ -1186,10 +1248,225 @@ mod tests {
         let raw = include_bytes!("../test-data/message/mail_with_user_and_group_avatars.eml");
         let raw = String::from_utf8_lossy(raw).to_string();
         let raw = raw.replace("Chat-User-Avatar:", "Xhat-Xser-Xvatar:");
-        let mimeparser = MimeParser::from_bytes(&t.ctx, raw.as_bytes()).unwrap();
+        let mimeparser = MimeMessage::from_bytes(&t.ctx, raw.as_bytes()).unwrap();
         assert_eq!(mimeparser.parts.len(), 1);
         assert_eq!(mimeparser.parts[0].typ, Viewtype::Image);
         assert_eq!(mimeparser.user_avatar, AvatarAction::None);
         assert!(mimeparser.group_avatar.is_change());
     }
+
+    #[test]
+    fn test_mimeparser_message_kml() {
+        let context = dummy_context();
+        let raw = b"Chat-Version: 1.0\n\
+From: foo <foo@example.org>\n\
+To: bar <bar@example.org>\n\
+Subject: Location streaming\n\
+Content-Type: multipart/mixed; boundary=\"==break==\"\n\
+\n\
+\n\
+--==break==\n\
+Content-Type: text/plain; charset=utf-8\n\
+\n\
+--\n\
+Sent with my Delta Chat Messenger: https://delta.chat\n\
+\n\
+--==break==\n\
+Content-Type: application/vnd.google-earth.kml+xml\n\
+Content-Disposition: attachment; filename=\"message.kml\"\n\
+\n\
+<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n\
+<kml xmlns=\"http://www.opengis.net/kml/2.2\">\n\
+<Document addr=\"foo@example.org\">\n\
+<Placemark><Timestamp><when>XXX</when></Timestamp><Point><coordinates accuracy=\"48\">0.0,0.0</coordinates></Point></Placemark>\n\
+</Document>\n\
+</kml>\n\
+\n\
+--==break==--\n\
+;";
+
+        let mimeparser = MimeMessage::from_bytes(&context.ctx, &raw[..]).unwrap();
+        assert_eq!(
+            mimeparser.get_subject(),
+            Some("Location streaming".to_string())
+        );
+        assert!(mimeparser.location_kml.is_none());
+        assert!(mimeparser.message_kml.is_some());
+
+        // There is only one part because message.kml attachment is special
+        // and only goes into message_kml.
+        assert_eq!(mimeparser.parts.len(), 1);
+    }
+
+    #[test]
+    fn test_parse_mdn() {
+        let context = dummy_context();
+        let raw = b"Subject: =?utf-8?q?Chat=3A_Message_opened?=\n\
+Date: Mon, 10 Jan 2020 00:00:00 +0000\n\
+Chat-Version: 1.0\n\
+Message-ID: <bar@example.org>\n\
+To: Alice <alice@example.org>\n\
+From: Bob <bob@example.org>\n\
+Content-Type: multipart/report; report-type=disposition-notification;\n\t\
+boundary=\"kJBbU58X1xeWNHgBtTbMk80M5qnV4N\"\n\
+\n\
+\n\
+--kJBbU58X1xeWNHgBtTbMk80M5qnV4N\n\
+Content-Type: text/plain; charset=utf-8\n\
+\n\
+The \"Encrypted message\" message you sent was displayed on the screen of the recipient.\n\
+\n\
+This is no guarantee the content was read.\n\
+\n\
+\n\
+--kJBbU58X1xeWNHgBtTbMk80M5qnV4N\n\
+Content-Type: message/disposition-notification\n\
+\n\
+Reporting-UA: Delta Chat 1.0.0-beta.22\n\
+Original-Recipient: rfc822;bob@example.org\n\
+Final-Recipient: rfc822;bob@example.org\n\
+Original-Message-ID: <foo@example.org>\n\
+Disposition: manual-action/MDN-sent-automatically; displayed\n\
+\n\
+\n\
+--kJBbU58X1xeWNHgBtTbMk80M5qnV4N--\n\
+";
+
+        let message = MimeMessage::from_bytes(&context.ctx, &raw[..]).unwrap();
+        assert_eq!(
+            message.get_subject(),
+            Some("Chat: Message opened".to_string())
+        );
+
+        assert_eq!(message.parts.len(), 0);
+        assert_eq!(message.reports.len(), 1);
+    }
+
+    /// Test parsing multiple MDNs combined in a single message.
+    ///
+    /// RFC 6522 specifically allows MDNs to be nested inside
+    /// multipart MIME messages.
+    #[test]
+    fn test_parse_multiple_mdns() {
+        let context = dummy_context();
+        let raw = b"Subject: =?utf-8?q?Chat=3A_Message_opened?=\n\
+Date: Mon, 10 Jan 2020 00:00:00 +0000\n\
+Chat-Version: 1.0\n\
+Message-ID: <foo@example.org>\n\
+To: Alice <alice@example.org>\n\
+From: Bob <bob@example.org>\n\
+Content-Type: multipart/parallel; boundary=outer\n\
+\n\
+This is a multipart MDN.\n\
+\n\
+--outer\n\
+Content-Type: multipart/report; report-type=disposition-notification;\n\t\
+boundary=kJBbU58X1xeWNHgBtTbMk80M5qnV4N\n\
+\n\
+\n\
+--kJBbU58X1xeWNHgBtTbMk80M5qnV4N\n\
+Content-Type: text/plain; charset=utf-8\n\
+\n\
+The \"Encrypted message\" message you sent was displayed on the screen of the recipient.\n\
+\n\
+This is no guarantee the content was read.\n\
+\n\
+\n\
+--kJBbU58X1xeWNHgBtTbMk80M5qnV4N\n\
+Content-Type: message/disposition-notification\n\
+\n\
+Reporting-UA: Delta Chat 1.0.0-beta.22\n\
+Original-Recipient: rfc822;bob@example.org\n\
+Final-Recipient: rfc822;bob@example.org\n\
+Original-Message-ID: <bar@example.org>\n\
+Disposition: manual-action/MDN-sent-automatically; displayed\n\
+\n\
+\n\
+--kJBbU58X1xeWNHgBtTbMk80M5qnV4N--\n\
+--outer\n\
+Content-Type: multipart/report; report-type=disposition-notification;\n\t\
+boundary=zuOJlsTfZAukyawEPVdIgqWjaM9w2W\n\
+\n\
+\n\
+--zuOJlsTfZAukyawEPVdIgqWjaM9w2W\n\
+Content-Type: text/plain; charset=utf-8\n\
+\n\
+The \"Encrypted message\" message you sent was displayed on the screen of the recipient.\n\
+\n\
+This is no guarantee the content was read.\n\
+\n\
+\n\
+--zuOJlsTfZAukyawEPVdIgqWjaM9w2W\n\
+Content-Type: message/disposition-notification\n\
+\n\
+Reporting-UA: Delta Chat 1.0.0-beta.22\n\
+Original-Recipient: rfc822;bob@example.org\n\
+Final-Recipient: rfc822;bob@example.org\n\
+Original-Message-ID: <baz@example.org>\n\
+Disposition: manual-action/MDN-sent-automatically; displayed\n\
+\n\
+\n\
+--zuOJlsTfZAukyawEPVdIgqWjaM9w2W--\n\
+--outer--\n\
+";
+
+        let message = MimeMessage::from_bytes(&context.ctx, &raw[..]).unwrap();
+        assert_eq!(
+            message.get_subject(),
+            Some("Chat: Message opened".to_string())
+        );
+
+        assert_eq!(message.parts.len(), 0);
+        assert_eq!(message.reports.len(), 2);
+    }
+
+    #[test]
+    fn test_parse_mdn_with_additional_message_ids() {
+        let context = dummy_context();
+        let raw = b"Subject: =?utf-8?q?Chat=3A_Message_opened?=\n\
+Date: Mon, 10 Jan 2020 00:00:00 +0000\n\
+Chat-Version: 1.0\n\
+Message-ID: <bar@example.org>\n\
+To: Alice <alice@example.org>\n\
+From: Bob <bob@example.org>\n\
+Content-Type: multipart/report; report-type=disposition-notification;\n\t\
+boundary=\"kJBbU58X1xeWNHgBtTbMk80M5qnV4N\"\n\
+\n\
+\n\
+--kJBbU58X1xeWNHgBtTbMk80M5qnV4N\n\
+Content-Type: text/plain; charset=utf-8\n\
+\n\
+The \"Encrypted message\" message you sent was displayed on the screen of the recipient.\n\
+\n\
+This is no guarantee the content was read.\n\
+\n\
+\n\
+--kJBbU58X1xeWNHgBtTbMk80M5qnV4N\n\
+Content-Type: message/disposition-notification\n\
+\n\
+Reporting-UA: Delta Chat 1.0.0-beta.22\n\
+Original-Recipient: rfc822;bob@example.org\n\
+Final-Recipient: rfc822;bob@example.org\n\
+Original-Message-ID: <foo@example.org>\n\
+Disposition: manual-action/MDN-sent-automatically; displayed\n\
+Additional-Message-IDs: <foo@example.com> <foo@example.net>\n\
+\n\
+\n\
+--kJBbU58X1xeWNHgBtTbMk80M5qnV4N--\n\
+";
+
+        let message = MimeMessage::from_bytes(&context.ctx, &raw[..]).unwrap();
+        assert_eq!(
+            message.get_subject(),
+            Some("Chat: Message opened".to_string())
+        );
+
+        assert_eq!(message.parts.len(), 0);
+        assert_eq!(message.reports.len(), 1);
+        assert_eq!(message.reports[0].original_message_id, "foo@example.org");
+        assert_eq!(
+            &message.reports[0].additional_message_ids,
+            &["foo@example.com", "foo@example.net"]
+        );
+    }
 }

src/oauth2.rs

@@ -154,7 +154,7 @@ pub fn dc_get_oauth2_access_token(
         }
 
         // ... and POST
-        let response = reqwest::Client::new()
+        let response = reqwest::blocking::Client::new()
             .post(post_url)
             .form(&post_param)
             .send();
@@ -165,7 +165,7 @@ pub fn dc_get_oauth2_access_token(
             );
             return None;
         }
-        let mut response = response.unwrap();
+        let response = response.unwrap();
         if !response.status().is_success() {
             warn!(
                 context,
@@ -271,7 +271,8 @@ impl Oauth2 {
         {
             match domain {
                 "gmail.com" | "googlemail.com" => Some(OAUTH2_GMAIL),
-                "yandex.com" | "yandex.ru" | "yandex.ua" => Some(OAUTH2_YANDEX),
+                "yandex.com" | "yandex.by" | "yandex.kz" | "yandex.ru" | "yandex.ua" | "ya.ru"
+                | "narod.ru" => Some(OAUTH2_YANDEX),
                 _ => None,
             }
         } else {
@@ -290,12 +291,12 @@ impl Oauth2 {
         //   "verified_email": true,
         //   "picture": "https://lh4.googleusercontent.com/-Gj5jh_9R0BY/AAAAAAAAAAI/AAAAAAAAAAA/IAjtjfjtjNA/photo.jpg"
         // }
-        let response = reqwest::Client::new().get(&userinfo_url).send();
+        let response = reqwest::blocking::Client::new().get(&userinfo_url).send();
         if response.is_err() {
             warn!(context, "Error getting userinfo: {:?}", response);
             return None;
         }
-        let mut response = response.unwrap();
+        let response = response.unwrap();
         if !response.status().is_success() {
             warn!(context, "Error getting userinfo: {:?}", response.status());
             return None;
@@ -310,14 +311,19 @@ impl Oauth2 {
             return None;
         }
         if let Ok(response) = parsed {
-            // serde_json::Value.as_str() removes the quotes of json-strings
-            let addr = response.get("email");
-            if addr.is_none() {
+            // CAVE: serde_json::Value.as_str() removes the quotes of json-strings
+            // but serde_json::Value.to_string() does not!
+            if let Some(addr) = response.get("email") {
+                if let Some(s) = addr.as_str() {
+                    Some(s.to_string())
+                } else {
+                    warn!(context, "E-mail in userinfo is not a string: {}", addr);
+                    None
+                }
+            } else {
                 warn!(context, "E-mail missing in userinfo.");
-                return None;
+                None
             }
-            let addr = addr.unwrap().as_str();
-            addr.map(|addr| addr.to_string())
         } else {
             warn!(context, "Failed to parse userinfo.");
             None

src/param.rs

@@ -8,6 +8,7 @@ use num_traits::FromPrimitive;
 use crate::blob::{BlobError, BlobObject};
 use crate::context::Context;
 use crate::error;
+use crate::message::MsgId;
 use crate::mimeparser::SystemMessage;
 
 /// Available param keys.
@@ -16,36 +17,51 @@ use crate::mimeparser::SystemMessage;
 pub enum Param {
     /// For messages and jobs
     File = b'f',
+
     /// For Messages
     Width = b'w',
+
     /// For Messages
     Height = b'h',
+
     /// For Messages
     Duration = b'd',
+
     /// For Messages
     MimeType = b'm',
-    /// For Messages: message is encryoted, outgoing: guarantee E2EE or the message is not send
+
+    /// For Messages: message is encrypted, outgoing: guarantee E2EE or the message is not send
     GuaranteeE2ee = b'c',
+
     /// For Messages: decrypted with validation errors or without mutual set, if neither
     /// 'c' nor 'e' are preset, the messages is only transport encrypted.
     ErroneousE2ee = b'e',
+
     /// For Messages: force unencrypted message, either `ForcePlaintext::AddAutocryptHeader` (1),
     /// `ForcePlaintext::NoAutocryptHeader` (2) or 0.
     ForcePlaintext = b'u',
+
     /// For Messages
     WantsMdn = b'r',
+
     /// For Messages
     Forwarded = b'a',
+
     /// For Messages
     Cmd = b'S',
+
     /// For Messages
     Arg = b'E',
+
     /// For Messages
     Arg2 = b'F',
+
     /// For Messages
     Arg3 = b'G',
+
     /// For Messages
     Arg4 = b'H',
+
     /// For Messages
     Error = b'L',
 
@@ -62,32 +78,48 @@ pub enum Param {
     /// When the original message is then finally sent this parameter
     /// is used to also send all the forwarded messages.
     PrepForwards = b'P',
+
     /// For Jobs
     SetLatitude = b'l',
+
     /// For Jobs
     SetLongitude = b'n',
+
     /// For Jobs
     ServerFolder = b'Z',
+
     /// For Jobs
     ServerUid = b'z',
+
     /// For Jobs
     AlsoMove = b'M',
+
     /// For Jobs: space-separated list of message recipients
     Recipients = b'R',
-    // For Groups
+
+    /// For Groups
     Unpromoted = b'U',
-    // For Groups and Contacts
+
+    /// For Groups and Contacts
     ProfileImage = b'i',
-    // For Chats
+
+    /// For Chats
     Selftalk = b'K',
-    // For Chats
+
+    /// For Chats
     Devicetalk = b'D',
-    // For QR
+
+    /// For QR
     Auth = b's',
-    // For QR
+
+    /// For QR
     GroupId = b'x',
-    // For QR
+
+    /// For QR
     GroupName = b'g',
+
+    /// For MDN-sending job
+    MsgId = b'I',
 }
 
 /// Possible values for `Param::ForcePlaintext`.
@@ -284,6 +316,12 @@ impl Params {
         Ok(Some(path))
     }
 
+    pub fn get_msg_id(&self) -> Option<MsgId> {
+        self.get(Param::MsgId)
+            .and_then(|x| x.parse::<u32>().ok())
+            .map(MsgId::new)
+    }
+
     /// Set the given paramter to the passed in `i32`.
     pub fn set_int(&mut self, key: Param, value: i32) -> &mut Self {
         self.set(key, format!("{}", value));

src/peerstate.rs

@@ -5,7 +5,6 @@ use std::fmt;
 use num_traits::FromPrimitive;
 
 use crate::aheader::*;
-use crate::chat::*;
 use crate::constants::*;
 use crate::context::Context;
 use crate::key::*;
@@ -95,6 +94,7 @@ pub enum ToSave {
 pub enum DegradeEvent {
     /// Recoverable by an incoming encrypted mail.
     EncryptionPaused = 0x01,
+
     /// Recoverable by a new verify.
     FingerprintChanged = 0x02,
 }
@@ -417,7 +417,6 @@ impl<'a> Peerstate<'a> {
                     &self.addr,
                 ],
             )?;
-            reset_gossiped_timestamp(self.context, 0)?;
         } else if self.to_save == Some(ToSave::Timestamps) {
             sql::execute(
                 self.context,

src/pgp.rs

@@ -16,7 +16,7 @@ use pgp::types::{
 };
 use rand::{thread_rng, CryptoRng, Rng};
 
-use crate::error::Error;
+use crate::error::Result;
 use crate::key::*;
 use crate::keyring::*;
 
@@ -88,9 +88,7 @@ impl<'a> PublicKeyTrait for SignedPublicKeyOrSubkey<'a> {
 /// Split data from PGP Armored Data as defined in https://tools.ietf.org/html/rfc4880#section-6.2.
 ///
 /// Returns (type, headers, base64 encoded body).
-pub fn split_armored_data(
-    buf: &[u8],
-) -> Result<(BlockType, BTreeMap<String, String>, Vec<u8>), Error> {
+pub fn split_armored_data(buf: &[u8]) -> Result<(BlockType, BTreeMap<String, String>, Vec<u8>)> {
     use std::io::Read;
 
     let cursor = Cursor::new(buf);
@@ -194,7 +192,7 @@ pub fn pk_encrypt(
     plain: &[u8],
     public_keys_for_encryption: &Keyring,
     private_key_for_signing: Option<&Key>,
-) -> Result<String, Error> {
+) -> Result<String> {
     let lit_msg = Message::new_literal_bytes("", plain);
     let pkeys: Vec<SignedPublicKeyOrSubkey> = public_keys_for_encryption
         .keys()
@@ -236,7 +234,7 @@ pub fn pk_decrypt(
     private_keys_for_decryption: &Keyring,
     public_keys_for_validation: &Keyring,
     ret_signature_fingerprints: Option<&mut HashSet<String>>,
-) -> Result<Vec<u8>, Error> {
+) -> Result<Vec<u8>> {
     let (msg, _) = Message::from_armor_single(Cursor::new(ctext))?;
     let skeys: Vec<&SignedSecretKey> = private_keys_for_decryption
         .keys()
@@ -248,7 +246,7 @@ pub fn pk_decrypt(
         .collect();
 
     let (decryptor, _) = msg.decrypt(|| "".into(), || "".into(), &skeys[..])?;
-    let msgs = decryptor.collect::<Result<Vec<_>, _>>()?;
+    let msgs = decryptor.collect::<pgp::errors::Result<Vec<_>>>()?;
     ensure!(!msgs.is_empty(), "No valid messages found");
 
     let dec_msg = &msgs[0];
@@ -280,7 +278,7 @@ pub fn pk_decrypt(
 }
 
 /// Symmetric encryption.
-pub fn symm_encrypt(passphrase: &str, plain: &[u8]) -> Result<String, Error> {
+pub fn symm_encrypt(passphrase: &str, plain: &[u8]) -> Result<String> {
     let mut rng = thread_rng();
     let lit_msg = Message::new_literal_bytes("", plain);
 
@@ -297,11 +295,11 @@ pub fn symm_encrypt(passphrase: &str, plain: &[u8]) -> Result<String, Error> {
 pub fn symm_decrypt<T: std::io::Read + std::io::Seek>(
     passphrase: &str,
     ctext: T,
-) -> Result<Vec<u8>, Error> {
+) -> Result<Vec<u8>> {
     let (enc_msg, _) = Message::from_armor_single(ctext)?;
     let decryptor = enc_msg.decrypt_with_password(|| passphrase.into())?;
 
-    let msgs = decryptor.collect::<Result<Vec<_>, _>>()?;
+    let msgs = decryptor.collect::<pgp::errors::Result<Vec<_>>>()?;
     ensure!(!msgs.is_empty(), "No valid messages found");
 
     match msgs[0].get_content()? {

src/qr.rs

@@ -69,7 +69,7 @@ fn decode_openpgp(context: &Context, qr: &str) -> Lot {
         (fp, &rest[1..])
     }) {
         Some(pair) => pair,
-        None => return format_err!("Invalid OPENPGP4FPR found").into(),
+        None => (payload, ""),
     };
 
     // replace & with \n to match expected param format
@@ -83,11 +83,11 @@ fn decode_openpgp(context: &Context, qr: &str) -> Lot {
 
     let addr = if let Some(addr) = param.get(Param::Forwarded) {
         match normalize_address(addr) {
-            Ok(addr) => addr,
+            Ok(addr) => Some(addr),
             Err(err) => return err.into(),
         }
     } else {
-        return format_err!("Missing address").into();
+        None
     };
 
     // what is up with that param name?
@@ -157,7 +157,7 @@ fn decode_openpgp(context: &Context, qr: &str) -> Lot {
             lot.state = LotState::QrFprWithoutAddr;
             lot.text1 = Some(dc_format_fingerprint(&fingerprint));
         }
-    } else {
+    } else if let Some(addr) = addr {
         if grpid.is_some() && grpname.is_some() {
             lot.state = LotState::QrAskVerifyGroup;
             lot.text1 = grpname;
@@ -172,6 +172,8 @@ fn decode_openpgp(context: &Context, qr: &str) -> Lot {
         lot.fingerprint = Some(fingerprint);
         lot.invitenumber = invitenumber;
         lot.auth = auth;
+    } else {
+        return format_err!("Missing address").into();
     }
 
     lot
@@ -471,4 +473,24 @@ mod tests {
         assert_eq!(contact.get_addr(), "cli@deltachat.de");
         assert_eq!(contact.get_name(), "Jörn P. P.");
     }
+
+    #[test]
+    fn test_decode_openpgp_without_addr() {
+        let ctx = dummy_context();
+
+        let res = check_qr(
+            &ctx.ctx,
+            "OPENPGP4FPR:1234567890123456789012345678901234567890",
+        );
+        assert_eq!(res.get_state(), LotState::QrFprWithoutAddr);
+        assert_eq!(
+            res.get_text1().unwrap(),
+            "1234 5678 9012 3456 7890\n1234 5678 9012 3456 7890"
+        );
+        assert_eq!(res.get_id(), 0);
+
+        let res = check_qr(&ctx.ctx, "OPENPGP4FPR:12345678901234567890");
+        assert_eq!(res.get_state(), LotState::QrError);
+        assert_eq!(res.get_id(), 0);
+    }
 }

src/securejoin.rs

@@ -3,7 +3,7 @@
 use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC};
 
 use crate::aheader::EncryptPreference;
-use crate::chat::{self, Chat};
+use crate::chat::{self, Chat, ChatId};
 use crate::config::*;
 use crate::constants::*;
 use crate::contact::*;
@@ -65,15 +65,18 @@ macro_rules! get_qr_attr {
     };
 }
 
-pub fn dc_get_securejoin_qr(context: &Context, group_chat_id: u32) -> Option<String> {
-    /* =========================================================
+pub fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> Option<String> {
+    /*=======================================================
     ====             Alice - the inviter side            ====
     ====   Step 1 in "Setup verified contact" protocol   ====
-    ========================================================= */
+    =======================================================*/
 
     let fingerprint: String;
 
     ensure_secret_key_exists(context).ok();
+
+    // invitenumber will be used to allow starting the handshake,
+    // auth will be used to verify the fingerprint
     let invitenumber = token::lookup_or_new(context, token::Namespace::InviteNumber, group_chat_id);
     let auth = token::lookup_or_new(context, token::Namespace::Auth, group_chat_id);
     let self_addr = match context.get_config(Config::ConfiguredAddr) {
@@ -98,7 +101,8 @@ pub fn dc_get_securejoin_qr(context: &Context, group_chat_id: u32) -> Option<Str
     let self_name_urlencoded =
         utf8_percent_encode(&self_name, NON_ALPHANUMERIC_WITHOUT_DOT).to_string();
 
-    let qr = if 0 != group_chat_id {
+    let qr = if !group_chat_id.is_unset() {
+        // parameters used: a=g=x=i=s=
         if let Ok(chat) = Chat::load_from_db(context, group_chat_id) {
             let group_name = chat.get_name();
             let group_name_urlencoded =
@@ -118,6 +122,7 @@ pub fn dc_get_securejoin_qr(context: &Context, group_chat_id: u32) -> Option<Str
             return None;
         }
     } else {
+        // parameters used: a=n=i=s=
         Some(format!(
             "OPENPGP4FPR:{}#a={}&n={}&i={}&s={}",
             fingerprint, self_addr_urlencoded, self_name_urlencoded, &invitenumber, &auth,
@@ -138,36 +143,41 @@ fn get_self_fingerprint(context: &Context) -> Option<String> {
     None
 }
 
-pub fn dc_join_securejoin(context: &Context, qr: &str) -> u32 {
+/// Take a scanned QR-code and do the setup-contact/join-group handshake.
+/// See the ffi-documentation for more details.
+pub fn dc_join_securejoin(context: &Context, qr: &str) -> ChatId {
     let cleanup =
-        |context: &Context, contact_chat_id: u32, ongoing_allocated: bool, join_vg: bool| {
+        |context: &Context, contact_chat_id: ChatId, ongoing_allocated: bool, join_vg: bool| {
             let mut bob = context.bob.write().unwrap();
             bob.expects = 0;
-            let ret_chat_id = if bob.status == DC_BOB_SUCCESS {
+            let ret_chat_id: ChatId = if bob.status == DC_BOB_SUCCESS {
                 if join_vg {
                     chat::get_chat_id_by_grpid(
                         context,
                         bob.qr_scan.as_ref().unwrap().text2.as_ref().unwrap(),
                     )
+                    .unwrap_or((ChatId::new(0), false, Blocked::Not))
                     .0
                 } else {
                     contact_chat_id
                 }
             } else {
-                0
+                ChatId::new(0)
             };
             bob.qr_scan = None;
 
             if ongoing_allocated {
                 context.free_ongoing();
             }
-            ret_chat_id as u32
+            ret_chat_id
         };
-    /* ==========================================================
+
+    /*========================================================
     ====             Bob - the joiner's side             =====
     ====   Step 2 in "Setup verified contact" protocol   =====
-    ========================================================== */
-    let mut contact_chat_id: u32 = 0;
+    ========================================================*/
+
+    let mut contact_chat_id = ChatId::new(0);
     let mut join_vg: bool = false;
 
     info!(context, "Requesting secure-join ...",);
@@ -181,11 +191,13 @@ pub fn dc_join_securejoin(context: &Context, qr: &str) -> u32 {
         error!(context, "Unknown QR code.",);
         return cleanup(&context, contact_chat_id, true, join_vg);
     }
-    contact_chat_id = chat::create_by_contact_id(context, qr_scan.id).unwrap_or_default();
-    if contact_chat_id == 0 {
-        error!(context, "Unknown contact.",);
-        return cleanup(&context, contact_chat_id, true, join_vg);
-    }
+    contact_chat_id = match chat::create_by_contact_id(context, qr_scan.id) {
+        Ok(chat_id) => chat_id,
+        Err(_) => {
+            error!(context, "Unknown contact.");
+            return cleanup(&context, contact_chat_id, true, join_vg);
+        }
+    };
     if context.shall_stop_ongoing() {
         return cleanup(&context, contact_chat_id, true, join_vg);
     }
@@ -209,10 +221,14 @@ pub fn dc_join_securejoin(context: &Context, qr: &str) -> u32 {
             .unwrap(),
         contact_chat_id,
     ) {
+        // the scanned fingerprint matches Alice's key,
+        // we can proceed to step 4b) directly and save two mails
         info!(context, "Taking protocol shortcut.");
         context.bob.write().unwrap().expects = DC_VC_CONTACT_CONFIRM;
         joiner_progress!(context, chat_id_2_contact_id(context, contact_chat_id), 400);
         let own_fingerprint = get_self_fingerprint(context).unwrap_or_default();
+
+        // Bob -> Alice
         send_handshake_msg(
             context,
             contact_chat_id,
@@ -231,6 +247,8 @@ pub fn dc_join_securejoin(context: &Context, qr: &str) -> u32 {
         );
     } else {
         context.bob.write().unwrap().expects = DC_VC_AUTH_REQUIRED;
+
+        // Bob -> Alice
         send_handshake_msg(
             context,
             contact_chat_id,
@@ -241,24 +259,30 @@ pub fn dc_join_securejoin(context: &Context, qr: &str) -> u32 {
         );
     }
 
-    // Bob -> Alice
-    while !context.shall_stop_ongoing() {
-        // Don't sleep too long, the user is waiting.
-        std::thread::sleep(std::time::Duration::from_millis(200));
+    if join_vg {
+        // for a group-join, wait until the secure-join is done and the group is created
+        while !context.shall_stop_ongoing() {
+            std::thread::sleep(std::time::Duration::from_millis(200));
+        }
+        cleanup(&context, contact_chat_id, true, join_vg)
+    } else {
+        // for a one-to-one-chat, the chat is already known, return the chat-id,
+        // the verification runs in background
+        context.free_ongoing();
+        contact_chat_id
     }
-    cleanup(&context, contact_chat_id, true, join_vg)
 }
 
 fn send_handshake_msg(
     context: &Context,
-    contact_chat_id: u32,
+    contact_chat_id: ChatId,
     step: &str,
     param2: impl AsRef<str>,
     fingerprint: Option<String>,
     grpid: impl AsRef<str>,
 ) {
     let mut msg = Message::default();
-    msg.type_0 = Viewtype::Text;
+    msg.viewtype = Viewtype::Text;
     msg.text = Some(format!("Secure-Join: {}", step));
     msg.hidden = true;
     msg.param.set_cmd(SystemMessage::SecurejoinMessage);
@@ -288,7 +312,7 @@ fn send_handshake_msg(
     chat::send_msg(context, contact_chat_id, &mut msg).unwrap_or_default();
 }
 
-fn chat_id_2_contact_id(context: &Context, contact_chat_id: u32) -> u32 {
+fn chat_id_2_contact_id(context: &Context, contact_chat_id: ChatId) -> u32 {
     let contacts = chat::get_chat_contacts(context, contact_chat_id);
     if contacts.len() == 1 {
         contacts[0]
@@ -300,7 +324,7 @@ fn chat_id_2_contact_id(context: &Context, contact_chat_id: u32) -> u32 {
 fn fingerprint_equals_sender(
     context: &Context,
     fingerprint: impl AsRef<str>,
-    contact_chat_id: u32,
+    contact_chat_id: ChatId,
 ) -> bool {
     let contacts = chat::get_chat_contacts(context, contact_chat_id);
 
@@ -319,80 +343,115 @@ fn fingerprint_equals_sender(
     }
     false
 }
-
-pub(crate) struct HandshakeMessageStatus {
-    pub(crate) hide_this_msg: bool,
-    pub(crate) delete_this_msg: bool,
-    pub(crate) stop_ongoing_process: bool,
-    pub(crate) bob_securejoin_success: Option<bool>,
+#[derive(Fail, Debug)]
+pub(crate) enum HandshakeError {
+    #[fail(display = "Can not be called with special contact ID")]
+    SpecialContactId,
+    #[fail(display = "Not a Secure-Join message")]
+    NotSecureJoinMsg,
+    #[fail(
+        display = "Failed to look up or create chat for contact #{}",
+        contact_id
+    )]
+    NoChat {
+        contact_id: u32,
+        #[cause]
+        cause: Error,
+    },
+    #[fail(display = "Chat for group {} not found", group)]
+    ChatNotFound { group: String },
+    #[fail(display = "No configured self address found")]
+    NoSelfAddr,
 }
 
-impl Default for HandshakeMessageStatus {
-    fn default() -> Self {
-        Self {
-            hide_this_msg: true,
-            delete_this_msg: false,
-            stop_ongoing_process: false,
-            bob_securejoin_success: None,
-        }
-    }
+/// What to do with a Secure-Join handshake message after it was handled.
+pub(crate) enum HandshakeMessage {
+    /// The message has been fully handled and should be removed/delete.
+    Done,
+    /// The message should be ignored/hidden, but not removed/deleted.
+    Ignore,
+    /// The message should be further processed by incoming message handling.
+    Propagate,
 }
 
 /// Handle incoming secure-join handshake.
+///
+/// This function will update the securejoin state in [Context::bob]
+/// and also terminate the ongoing process using
+/// [Context::stop_ongoing] as required by the protocol.
+///
+/// A message which results in [Err] will be hidden from the user but
+/// not deleted, it may be a valid message for something else we are
+/// not aware off.  E.g. it could be part of a handshake performed by
+/// another DC app on the same account.
+///
+/// When handle_securejoin_handshake() is called,
+/// the message is not yet filed in the database;
+/// this is done by receive_imf() later on as needed.
 pub(crate) fn handle_securejoin_handshake(
     context: &Context,
-    mimeparser: &MimeParser,
+    mime_message: &MimeMessage,
     contact_id: u32,
-) -> Result<HandshakeMessageStatus, Error> {
+) -> Result<HandshakeMessage, HandshakeError> {
     let own_fingerprint: String;
 
-    ensure!(
-        contact_id > DC_CONTACT_ID_LAST_SPECIAL,
-        "handle_securejoin_handshake(): called with special contact id"
-    );
-    let step = mimeparser
+    if contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
+        return Err(HandshakeError::SpecialContactId);
+    }
+    let step = mime_message
         .get(HeaderDef::SecureJoin)
-        .ok_or_else(|| format_err!("This message is not a Secure-Join message"))?;
+        .ok_or(HandshakeError::NotSecureJoinMsg)?;
 
     info!(
         context,
         ">>>>>>>>>>>>>>>>>>>>>>>>> secure-join message \'{}\' received", step,
     );
 
-    let (contact_chat_id, contact_chat_id_blocked) =
-        chat::create_or_lookup_by_contact_id(context, contact_id, Blocked::Not).unwrap_or_default();
-
-    if contact_chat_id_blocked != Blocked::Not {
-        chat::unblock(context, contact_chat_id);
-    }
+    let contact_chat_id =
+        match chat::create_or_lookup_by_contact_id(context, contact_id, Blocked::Not) {
+            Ok((chat_id, blocked)) => {
+                if blocked != Blocked::Not {
+                    chat_id.unblock(context);
+                }
+                chat_id
+            }
+            Err(err) => {
+                return Err(HandshakeError::NoChat {
+                    contact_id,
+                    cause: err,
+                });
+            }
+        };
 
     let join_vg = step.starts_with("vg-");
-    let mut ret = HandshakeMessageStatus::default();
 
     match step.as_str() {
         "vg-request" | "vc-request" => {
-            /* =========================================================
+            /*=======================================================
             ====             Alice - the inviter side            ====
             ====   Step 3 in "Setup verified contact" protocol   ====
-            ========================================================= */
-            // this message may be unencrypted (Bob, the joinder and the sender, might not have Alice's key yet)
+            =======================================================*/
+
+            // this message may be unencrypted (Bob, the joiner and the sender, might not have Alice's key yet)
             // it just ensures, we have Bobs key now. If we do _not_ have the key because eg. MitM has removed it,
             // send_message() will fail with the error "End-to-end-encryption unavailable unexpectedly.", so, there is no additional check needed here.
             // verify that the `Secure-Join-Invitenumber:`-header matches invitenumber written to the QR code
-            let invitenumber = match mimeparser.get(HeaderDef::SecureJoinInvitenumber) {
+            let invitenumber = match mime_message.get(HeaderDef::SecureJoinInvitenumber) {
                 Some(n) => n,
                 None => {
-                    warn!(context, "Secure-join denied (invitenumber missing).",);
-                    return Ok(ret);
+                    warn!(context, "Secure-join denied (invitenumber missing)");
+                    return Ok(HandshakeMessage::Ignore);
                 }
             };
             if !token::exists(context, token::Namespace::InviteNumber, &invitenumber) {
-                warn!(context, "Secure-join denied (bad invitenumber).",);
-                return Ok(ret);
+                warn!(context, "Secure-join denied (bad invitenumber).");
+                return Ok(HandshakeMessage::Ignore);
             }
             info!(context, "Secure-join requested.",);
 
             inviter_progress!(context, contact_id, 300);
+
+            // Alice -> Bob
             send_handshake_msg(
                 context,
                 contact_chat_id,
@@ -401,8 +460,15 @@ pub(crate) fn handle_securejoin_handshake(
                 None,
                 "",
             );
+            Ok(HandshakeMessage::Done)
         }
         "vg-auth-required" | "vc-auth-required" => {
+            /*========================================================
+            ====             Bob - the joiner's side             =====
+            ====   Step 4 in "Setup verified contact" protocol   =====
+            ========================================================*/
+
+            // verify that Alice's Autocrypt key and fingerprint matches the QR-code
             let cond = {
                 let bob = context.bob.read().unwrap();
                 let scan = bob.qr_scan.as_ref();
@@ -412,26 +478,26 @@ pub(crate) fn handle_securejoin_handshake(
             };
 
             if cond {
-                warn!(context, "auth-required message out of sync.",);
+                warn!(context, "auth-required message out of sync.");
                 // no error, just aborted somehow or a mail from another handshake
-                return Ok(ret);
+                return Ok(HandshakeMessage::Ignore);
             }
             let scanned_fingerprint_of_alice = get_qr_attr!(context, fingerprint).to_string();
             let auth = get_qr_attr!(context, auth).to_string();
 
-            if !encrypted_and_signed(mimeparser, &scanned_fingerprint_of_alice) {
+            if !encrypted_and_signed(mime_message, &scanned_fingerprint_of_alice) {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
-                    if mimeparser.was_encrypted() {
+                    if mime_message.was_encrypted() {
                         "No valid signature."
                     } else {
                         "Not encrypted."
                     },
                 );
-                ret.stop_ongoing_process = true;
-                ret.bob_securejoin_success = Some(false);
-                return Ok(ret);
+                context.bob.write().unwrap().status = 0; // secure-join failed
+                context.stop_ongoing();
+                return Ok(HandshakeMessage::Ignore);
             }
             if !fingerprint_equals_sender(context, &scanned_fingerprint_of_alice, contact_chat_id) {
                 could_not_establish_secure_connection(
@@ -439,15 +505,16 @@ pub(crate) fn handle_securejoin_handshake(
                     contact_chat_id,
                     "Fingerprint mismatch on joiner-side.",
                 );
-                ret.stop_ongoing_process = true;
-                ret.bob_securejoin_success = Some(false);
-                return Ok(ret);
+                context.bob.write().unwrap().status = 0; // secure-join failed
+                context.stop_ongoing();
+                return Ok(HandshakeMessage::Ignore);
             }
             info!(context, "Fingerprint verified.",);
             own_fingerprint = get_self_fingerprint(context).unwrap();
             joiner_progress!(context, contact_id, 400);
             context.bob.write().unwrap().expects = DC_VC_CONTACT_CONFIRM;
 
+            // Bob -> Alice
             send_handshake_msg(
                 context,
                 contact_chat_id,
@@ -460,15 +527,17 @@ pub(crate) fn handle_securejoin_handshake(
                     "".to_string()
                 },
             );
+            Ok(HandshakeMessage::Done)
         }
         "vg-request-with-auth" | "vc-request-with-auth" => {
-            /* ============================================================
+            /*==========================================================
             ====              Alice - the inviter side              ====
             ====   Steps 5+6 in "Setup verified contact" protocol   ====
             ====  Step 6 in "Out-of-band verified groups" protocol  ====
-            ============================================================ */
+            ==========================================================*/
+
             // verify that Secure-Join-Fingerprint:-header matches the fingerprint of Bob
-            let fingerprint = match mimeparser.get(HeaderDef::SecureJoinFingerprint) {
+            let fingerprint = match mime_message.get(HeaderDef::SecureJoinFingerprint) {
                 Some(fp) => fp,
                 None => {
                     could_not_establish_secure_connection(
@@ -476,16 +545,16 @@ pub(crate) fn handle_securejoin_handshake(
                         contact_chat_id,
                         "Fingerprint not provided.",
                     );
-                    return Ok(ret);
+                    return Ok(HandshakeMessage::Ignore);
                 }
             };
-            if !encrypted_and_signed(mimeparser, &fingerprint) {
+            if !encrypted_and_signed(mime_message, &fingerprint) {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
                     "Auth not encrypted.",
                 );
-                return Ok(ret);
+                return Ok(HandshakeMessage::Ignore);
             }
             if !fingerprint_equals_sender(context, &fingerprint, contact_chat_id) {
                 could_not_establish_secure_connection(
@@ -493,11 +562,11 @@ pub(crate) fn handle_securejoin_handshake(
                     contact_chat_id,
                     "Fingerprint mismatch on inviter-side.",
                 );
-                return Ok(ret);
+                return Ok(HandshakeMessage::Ignore);
             }
             info!(context, "Fingerprint verified.",);
             // verify that the `Secure-Join-Auth:`-header matches the secret written to the QR code
-            let auth_0 = match mimeparser.get(HeaderDef::SecureJoinAuth) {
+            let auth_0 = match mime_message.get(HeaderDef::SecureJoinAuth) {
                 Some(auth) => auth,
                 None => {
                     could_not_establish_secure_connection(
@@ -505,12 +574,12 @@ pub(crate) fn handle_securejoin_handshake(
                         contact_chat_id,
                         "Auth not provided.",
                     );
-                    return Ok(ret);
+                    return Ok(HandshakeMessage::Ignore);
                 }
             };
             if !token::exists(context, token::Namespace::Auth, &auth_0) {
                 could_not_establish_secure_connection(context, contact_chat_id, "Auth invalid.");
-                return Ok(ret);
+                return Ok(HandshakeMessage::Ignore);
             }
             if mark_peer_as_verified(context, fingerprint).is_err() {
                 could_not_establish_secure_connection(
@@ -518,7 +587,7 @@ pub(crate) fn handle_securejoin_handshake(
                     contact_chat_id,
                     "Fingerprint mismatch on inviter-side.",
                 );
-                return Ok(ret);
+                return Ok(HandshakeMessage::Ignore);
             }
             Contact::scaleup_origin_by_id(context, contact_id, Origin::SecurejoinInvited);
             info!(context, "Auth verified.",);
@@ -526,49 +595,76 @@ pub(crate) fn handle_securejoin_handshake(
             emit_event!(context, Event::ContactsChanged(Some(contact_id)));
             inviter_progress!(context, contact_id, 600);
             if join_vg {
-                let field_grpid = mimeparser
-                    .get(HeaderDef::SecureJoinGroup)
-                    .map(|s| s.as_str())
-                    .unwrap_or_else(|| "");
-                let (group_chat_id, _, _) = chat::get_chat_id_by_grpid(context, field_grpid);
-                if group_chat_id == 0 {
-                    error!(context, "Chat {} not found.", &field_grpid);
-                    return Ok(ret);
-                } else if let Err(err) =
-                    chat::add_contact_to_chat_ex(context, group_chat_id, contact_id, true)
-                {
-                    error!(context, "failed to add contact: {}", err);
+                // the vg-member-added message is special:
+                // this is a normal Chat-Group-Member-Added message
+                // with an additional Secure-Join header
+                let field_grpid = match mime_message.get(HeaderDef::SecureJoinGroup) {
+                    Some(s) => s.as_str(),
+                    None => {
+                        warn!(context, "Missing Secure-Join-Group header");
+                        return Ok(HandshakeMessage::Ignore);
+                    }
+                };
+                match chat::get_chat_id_by_grpid(context, field_grpid) {
+                    Ok((group_chat_id, _, _)) => {
+                        if let Err(err) =
+                            chat::add_contact_to_chat_ex(context, group_chat_id, contact_id, true)
+                        {
+                            error!(context, "failed to add contact: {}", err);
+                        }
+                    }
+                    Err(err) => {
+                        error!(context, "Chat {} not found: {}", &field_grpid, err);
+                        return Err(HandshakeError::ChatNotFound {
+                            group: field_grpid.to_string(),
+                        });
+                    }
                 }
             } else {
+                // Alice -> Bob
                 send_handshake_msg(context, contact_chat_id, "vc-contact-confirm", "", None, "");
                 inviter_progress!(context, contact_id, 1000);
             }
+            Ok(HandshakeMessage::Done)
         }
         "vg-member-added" | "vc-contact-confirm" => {
-            if join_vg {
-                ret.hide_this_msg = false;
-            }
+            /*=======================================================
+            ====             Bob - the joiner's side             ====
+            ====   Step 7 in "Setup verified contact" protocol   ====
+            =======================================================*/
+            let abort_retval = if join_vg {
+                HandshakeMessage::Propagate
+            } else {
+                HandshakeMessage::Ignore
+            };
+
             if context.bob.read().unwrap().expects != DC_VC_CONTACT_CONFIRM {
                 info!(context, "Message belongs to a different handshake.",);
-                return Ok(ret);
+                return Ok(abort_retval);
             }
             let cond = {
                 let bob = context.bob.read().unwrap();
                 let scan = bob.qr_scan.as_ref();
-                scan.is_none() || join_vg && scan.unwrap().state != LotState::QrAskVerifyGroup
+                scan.is_none() || (join_vg && scan.unwrap().state != LotState::QrAskVerifyGroup)
             };
             if cond {
                 warn!(
                     context,
                     "Message out of sync or belongs to a different handshake.",
                 );
-                return Ok(ret);
+                return Ok(abort_retval);
             }
             let scanned_fingerprint_of_alice = get_qr_attr!(context, fingerprint).to_string();
 
             let vg_expect_encrypted = if join_vg {
                 let group_id = get_qr_attr!(context, text2).to_string();
-                let (_, is_verified_group, _) = chat::get_chat_id_by_grpid(context, group_id);
+                // This is buggy, is_verified_group will always be
+                // false since the group is created by receive_imf by
+                // the very handshake message we're handling now.  But
+                // only after we have returned.  It does not impact
+                // the security invariants of secure-join however.
+                let (_, is_verified_group, _) = chat::get_chat_id_by_grpid(context, &group_id)
+                    .unwrap_or((ChatId::new(0), false, Blocked::Not));
                 // when joining a non-verified group
                 // the vg-member-added message may be unencrypted
                 // when not all group members have keys or prefer encryption.
@@ -579,15 +675,15 @@ pub(crate) fn handle_securejoin_handshake(
                 true
             };
             if vg_expect_encrypted
-                && !encrypted_and_signed(mimeparser, &scanned_fingerprint_of_alice)
+                && !encrypted_and_signed(mime_message, &scanned_fingerprint_of_alice)
             {
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
                     "Contact confirm message not encrypted.",
                 );
-                ret.bob_securejoin_success = Some(false);
-                return Ok(ret);
+                context.bob.write().unwrap().status = 0;
+                return Ok(abort_retval);
             }
 
             if mark_peer_as_verified(context, &scanned_fingerprint_of_alice).is_err() {
@@ -596,21 +692,26 @@ pub(crate) fn handle_securejoin_handshake(
                     contact_chat_id,
                     "Fingerprint mismatch on joiner-side.",
                 );
-                return Ok(ret);
+                return Ok(abort_retval);
             }
             Contact::scaleup_origin_by_id(context, contact_id, Origin::SecurejoinJoined);
             emit_event!(context, Event::ContactsChanged(None));
-            let cg_member_added = mimeparser
+            let cg_member_added = mime_message
                 .get(HeaderDef::ChatGroupMemberAdded)
                 .map(|s| s.as_str())
                 .unwrap_or_else(|| "");
-            if join_vg && !context.is_self_addr(cg_member_added)? {
+            if join_vg
+                && !context
+                    .is_self_addr(cg_member_added)
+                    .map_err(|_| HandshakeError::NoSelfAddr)?
+            {
                 info!(context, "Message belongs to a different handshake (scaled up contact anyway to allow creation of group).");
-                return Ok(ret);
+                return Ok(abort_retval);
             }
             secure_connection_established(context, contact_chat_id);
             context.bob.write().unwrap().expects = 0;
             if join_vg {
+                // Bob -> Alice
                 send_handshake_msg(
                     context,
                     contact_chat_id,
@@ -620,48 +721,56 @@ pub(crate) fn handle_securejoin_handshake(
                     "",
                 );
             }
-            ret.stop_ongoing_process = true;
-            ret.bob_securejoin_success = Some(true);
+            context.bob.write().unwrap().status = 1;
+            context.stop_ongoing();
+            Ok(if join_vg {
+                HandshakeMessage::Propagate
+            } else {
+                HandshakeMessage::Done
+            })
         }
         "vg-member-added-received" => {
-            /* ============================================================
+            /*==========================================================
             ====              Alice - the inviter side              ====
             ====  Step 8 in "Out-of-band verified groups" protocol  ====
-            ============================================================ */
+            ==========================================================*/
+
             if let Ok(contact) = Contact::get_by_id(context, contact_id) {
                 if contact.is_verified(context) == VerifiedStatus::Unverified {
                     warn!(context, "vg-member-added-received invalid.",);
-                    return Ok(ret);
+                    return Ok(HandshakeMessage::Ignore);
                 }
                 inviter_progress!(context, contact_id, 800);
                 inviter_progress!(context, contact_id, 1000);
-                let field_grpid = mimeparser
+                let field_grpid = mime_message
                     .get(HeaderDef::SecureJoinGroup)
                     .map(|s| s.as_str())
                     .unwrap_or_else(|| "");
-                let (group_chat_id, _, _) = chat::get_chat_id_by_grpid(context, &field_grpid);
+                let (group_chat_id, _, _) = chat::get_chat_id_by_grpid(context, &field_grpid)
+                    .map_err(|err| {
+                        warn!(context, "Failed to lookup chat_id from grpid: {}", err);
+                        HandshakeError::ChatNotFound {
+                            group: field_grpid.to_string(),
+                        }
+                    })?;
                 context.call_cb(Event::SecurejoinMemberAdded {
                     chat_id: group_chat_id,
                     contact_id,
                 });
+                Ok(HandshakeMessage::Done)
             } else {
                 warn!(context, "vg-member-added-received invalid.",);
-                return Ok(ret);
+                Ok(HandshakeMessage::Ignore)
             }
         }
         _ => {
             warn!(context, "invalid step: {}", step);
+            Ok(HandshakeMessage::Ignore)
         }
     }
-
-    if ret.hide_this_msg {
-        ret.delete_this_msg = true;
-    }
-
-    Ok(ret)
 }
 
-fn secure_connection_established(context: &Context, contact_chat_id: u32) {
+fn secure_connection_established(context: &Context, contact_chat_id: ChatId) {
     let contact_id: u32 = chat_id_2_contact_id(context, contact_chat_id);
     let contact = Contact::get_by_id(context, contact_id);
     let addr = if let Ok(ref contact) = contact {
@@ -674,7 +783,11 @@ fn secure_connection_established(context: &Context, contact_chat_id: u32) {
     emit_event!(context, Event::ChatModified(contact_chat_id));
 }
 
-fn could_not_establish_secure_connection(context: &Context, contact_chat_id: u32, details: &str) {
+fn could_not_establish_secure_connection(
+    context: &Context,
+    contact_chat_id: ChatId,
+    details: &str,
+) {
     let contact_id = chat_id_2_contact_id(context, contact_chat_id);
     let contact = Contact::get_by_id(context, contact_id);
     let msg = context.stock_string_repl_str(
@@ -717,7 +830,7 @@ fn mark_peer_as_verified(context: &Context, fingerprint: impl AsRef<str>) -> Res
  * Tools: Misc.
  ******************************************************************************/
 
-fn encrypted_and_signed(mimeparser: &MimeParser, expected_fingerprint: impl AsRef<str>) -> bool {
+fn encrypted_and_signed(mimeparser: &MimeMessage, expected_fingerprint: impl AsRef<str>) -> bool {
     if !mimeparser.was_encrypted() {
         warn!(mimeparser.context, "Message not encrypted.",);
         false

src/simplify.rs

@@ -0,0 +1,251 @@
+/// Remove standard (RFC 3676, §4.3) footer if it is found.
+fn remove_message_footer<'a>(lines: &'a [&str]) -> &'a [&'a str] {
+    for (ix, &line) in lines.iter().enumerate() {
+        // quoted-printable may encode `-- ` to `-- =20` which is converted
+        // back to `--  `
+        match line {
+            "-- " | "--  " => return &lines[..ix],
+            _ => (),
+        }
+    }
+    lines
+}
+
+/// Remove nonstandard footer and a boolean indicating whether such
+/// footer was removed.
+fn remove_nonstandard_footer<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
+    for (ix, &line) in lines.iter().enumerate() {
+        if line == "--"
+            || line == "---"
+            || line == "----"
+            || line.starts_with("-----")
+            || line.starts_with("_____")
+            || line.starts_with("=====")
+            || line.starts_with("*****")
+            || line.starts_with("~~~~~")
+        {
+            return (&lines[..ix], true);
+        }
+    }
+    (lines, false)
+}
+
+fn split_lines(buf: &str) -> Vec<&str> {
+    buf.split('\n').collect()
+}
+
+/// Simplify message text for chat display.
+/// Remove quotes, signatures, trailing empty lines etc.
+pub fn simplify(mut input: String, is_chat_message: bool) -> (String, bool) {
+    input.retain(|c| c != '\r');
+    let lines = split_lines(&input);
+    let (lines, is_forwarded) = skip_forward_header(&lines);
+
+    let lines = remove_message_footer(lines);
+    let (lines, has_nonstandard_footer) = remove_nonstandard_footer(lines);
+    let (lines, has_bottom_quote) = if !is_chat_message {
+        remove_bottom_quote(lines)
+    } else {
+        (lines, false)
+    };
+    let (lines, has_top_quote) = if !is_chat_message {
+        remove_top_quote(lines)
+    } else {
+        (lines, false)
+    };
+
+    // re-create buffer from the remaining lines
+    let text = render_message(
+        lines,
+        has_top_quote,
+        has_nonstandard_footer || has_bottom_quote,
+    );
+    (text, is_forwarded)
+}
+
+/// Skips "forwarded message" header.
+/// Returns message body lines and a boolean indicating whether
+/// a message is forwarded or not.
+fn skip_forward_header<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
+    if lines.len() >= 3
+        && lines[0] == "---------- Forwarded message ----------"
+        && lines[1].starts_with("From: ")
+        && lines[2].is_empty()
+    {
+        (&lines[3..], true)
+    } else {
+        (lines, false)
+    }
+}
+
+fn remove_bottom_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
+    let mut last_quoted_line = None;
+    for (l, line) in lines.iter().enumerate().rev() {
+        if is_plain_quote(line) {
+            last_quoted_line = Some(l)
+        } else if !is_empty_line(line) {
+            break;
+        }
+    }
+    if let Some(mut l_last) = last_quoted_line {
+        if l_last > 1 && is_empty_line(lines[l_last - 1]) {
+            l_last -= 1
+        }
+        if l_last > 1 {
+            let line = lines[l_last - 1];
+            if is_quoted_headline(line) {
+                l_last -= 1
+            }
+        }
+        (&lines[..l_last], true)
+    } else {
+        (lines, false)
+    }
+}
+
+fn remove_top_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
+    let mut last_quoted_line = None;
+    let mut has_quoted_headline = false;
+    for (l, line) in lines.iter().enumerate() {
+        if is_plain_quote(line) {
+            last_quoted_line = Some(l)
+        } else if !is_empty_line(line) {
+            if is_quoted_headline(line) && !has_quoted_headline && last_quoted_line.is_none() {
+                has_quoted_headline = true
+            } else {
+                /* non-quoting line found */
+                break;
+            }
+        }
+    }
+    if let Some(last_quoted_line) = last_quoted_line {
+        (&lines[last_quoted_line + 1..], true)
+    } else {
+        (lines, false)
+    }
+}
+
+fn render_message(lines: &[&str], is_cut_at_begin: bool, is_cut_at_end: bool) -> String {
+    let mut ret = String::new();
+    if is_cut_at_begin {
+        ret += "[...]";
+    }
+    /* we write empty lines only in case and non-empty line follows */
+    let mut pending_linebreaks = 0;
+    let mut empty_body = true;
+    for line in lines {
+        if is_empty_line(line) {
+            pending_linebreaks += 1
+        } else {
+            if !empty_body {
+                if pending_linebreaks > 2 {
+                    pending_linebreaks = 2
+                }
+                while 0 != pending_linebreaks {
+                    ret += "\n";
+                    pending_linebreaks -= 1
+                }
+            }
+            // the incoming message might contain invalid UTF8
+            ret += line;
+            empty_body = false;
+            pending_linebreaks = 1
+        }
+    }
+    if is_cut_at_end && (!is_cut_at_begin || !empty_body) {
+        ret += " [...]";
+    }
+    ret
+}
+
+/**
+ * Tools
+ */
+fn is_empty_line(buf: &str) -> bool {
+    // XXX: can it be simplified to buf.chars().all(|c| c.is_whitespace())?
+    //
+    // Strictly speaking, it is not equivalent (^A is not whitespace, but less than ' '),
+    // but having control sequences in email body?!
+    //
+    // See discussion at: https://github.com/deltachat/deltachat-core-rust/pull/402#discussion_r317062392
+    for c in buf.chars() {
+        if c > ' ' {
+            return false;
+        }
+    }
+
+    true
+}
+
+fn is_quoted_headline(buf: &str) -> bool {
+    /* This function may be called for the line _directly_ before a quote.
+    The function checks if the line contains sth. like "On 01.02.2016, xy@z wrote:" in various languages.
+    - Currently, we simply check if the last character is a ':'.
+    - Checking for the existence of an email address may fail (headlines may show the user's name instead of the address) */
+
+    buf.len() <= 80 && buf.ends_with(':')
+}
+
+fn is_plain_quote(buf: &str) -> bool {
+    buf.starts_with('>')
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use proptest::prelude::*;
+
+    proptest! {
+        #[test]
+        // proptest does not support [[:graphical:][:space:]] regex.
+        fn test_simplify_plain_text_fuzzy(input in "[!-~\t \n]+") {
+            let (output, _is_forwarded) = simplify(input, true);
+            assert!(output.split('\n').all(|s| s != "-- "));
+        }
+    }
+
+    #[test]
+    fn test_simplify_trim() {
+        let input = "line1\n\r\r\rline2".to_string();
+        let (plain, is_forwarded) = simplify(input, false);
+
+        assert_eq!(plain, "line1\nline2");
+        assert!(!is_forwarded);
+    }
+
+    #[test]
+    fn test_simplify_forwarded_message() {
+        let input = "---------- Forwarded message ----------\r\nFrom: test@example.com\r\n\r\nForwarded message\r\n-- \r\nSignature goes here".to_string();
+        let (plain, is_forwarded) = simplify(input, false);
+
+        assert_eq!(plain, "Forwarded message");
+        assert!(is_forwarded);
+    }
+
+    #[test]
+    fn test_simplify_utilities() {
+        assert!(is_empty_line(" \t"));
+        assert!(is_empty_line(""));
+        assert!(is_empty_line(" \r"));
+        assert!(!is_empty_line(" x"));
+        assert!(is_plain_quote("> hello world"));
+        assert!(is_plain_quote(">>"));
+        assert!(!is_plain_quote("Life is pain"));
+        assert!(!is_plain_quote(""));
+    }
+
+    #[test]
+    fn test_remove_top_quote() {
+        let (lines, has_top_quote) = remove_top_quote(&["> first", "> second"]);
+        assert!(lines.is_empty());
+        assert!(has_top_quote);
+
+        let (lines, has_top_quote) = remove_top_quote(&["> first", "> second", "not a quote"]);
+        assert_eq!(lines, &["not a quote"]);
+        assert!(has_top_quote);
+
+        let (lines, has_top_quote) = remove_top_quote(&["not a quote", "> first", "> second"]);
+        assert_eq!(lines, &["not a quote", "> first", "> second"]);
+        assert!(!has_top_quote);
+    }
+}

src/smtp/mod.rs

@@ -2,39 +2,47 @@
 
 pub mod send;
 
+use std::time::Duration;
+
 use async_smtp::smtp::client::net::*;
 use async_smtp::*;
 
-use async_std::task;
-
 use crate::constants::*;
 use crate::context::Context;
 use crate::events::Event;
 use crate::login_param::{dc_build_tls, LoginParam};
 use crate::oauth2::*;
 
+/// SMTP write and read timeout in seconds.
+const SMTP_TIMEOUT: u64 = 30;
+
 #[derive(Debug, Fail)]
 pub enum Error {
     #[fail(display = "Bad parameters")]
     BadParameters,
+
     #[fail(display = "Invalid login address {}: {}", address, error)]
     InvalidLoginAddress {
         address: String,
         #[cause]
-        error: async_smtp::error::Error,
+        error: error::Error,
     },
+
     #[fail(display = "SMTP failed to connect: {:?}", _0)]
-    ConnectionFailure(#[cause] async_smtp::smtp::error::Error),
+    ConnectionFailure(#[cause] smtp::error::Error),
+
     #[fail(display = "SMTP: failed to setup connection {:?}", _0)]
-    ConnectionSetupFailure(#[cause] async_smtp::smtp::error::Error),
+    ConnectionSetupFailure(#[cause] smtp::error::Error),
+
     #[fail(display = "SMTP: oauth2 error {:?}", _0)]
     Oauth2Error { address: String },
+
     #[fail(display = "TLS error")]
-    Tls(#[cause] native_tls::Error),
+    Tls(#[cause] async_native_tls::Error),
 }
 
-impl From<native_tls::Error> for Error {
-    fn from(err: native_tls::Error) -> Error {
+impl From<async_native_tls::Error> for Error {
+    fn from(err: async_native_tls::Error) -> Error {
         Error::Tls(err)
     }
 }
@@ -44,7 +52,7 @@ pub type Result<T> = std::result::Result<T, Error>;
 #[derive(Default, DebugStub)]
 pub struct Smtp {
     #[debug_stub(some = "SmtpTransport")]
-    transport: Option<async_smtp::smtp::SmtpTransport>,
+    transport: Option<smtp::SmtpTransport>,
     /// Email address we are sending from.
     from: Option<EmailAddress>,
 }
@@ -57,18 +65,25 @@ impl Smtp {
 
     /// Disconnect the SMTP transport and drop it entirely.
     pub fn disconnect(&mut self) {
-        if let Some(ref mut transport) = self.transport.take() {
-            transport.close();
+        if let Some(mut transport) = self.transport.take() {
+            async_std::task::block_on(transport.close()).ok();
         }
     }
 
-    /// check whether we are connected
+    /// Check whether we are connected.
     pub fn is_connected(&self) -> bool {
-        self.transport.is_some()
+        self.transport
+            .as_ref()
+            .map(|t| t.is_connected())
+            .unwrap_or_default()
     }
 
-    /// Connect using the provided login params
+    /// Connect using the provided login params.
     pub fn connect(&mut self, context: &Context, lp: &LoginParam) -> Result<()> {
+        async_std::task::block_on(self.inner_connect(context, lp))
+    }
+
+    async fn inner_connect(&mut self, context: &Context, lp: &LoginParam) -> Result<()> {
         if self.is_connected() {
             warn!(context, "SMTP already connected.");
             return Ok(());
@@ -89,7 +104,7 @@ impl Smtp {
         let domain = &lp.send_server;
         let port = lp.send_port as u16;
 
-        let tls_config = dc_build_tls(lp.smtp_certificate_checks)?.into();
+        let tls_config = dc_build_tls(lp.smtp_certificate_checks);
         let tls_parameters = ClientTlsParameters::new(domain.to_string(), tls_config);
 
         let (creds, mechanism) = if 0 != lp.server_flags & (DC_LP_AUTH_OAUTH2 as i32) {
@@ -104,21 +119,21 @@ impl Smtp {
             }
             let user = &lp.send_user;
             (
-                async_smtp::smtp::authentication::Credentials::new(
+                smtp::authentication::Credentials::new(
                     user.to_string(),
                     access_token.unwrap_or_default(),
                 ),
-                vec![async_smtp::smtp::authentication::Mechanism::Xoauth2],
+                vec![smtp::authentication::Mechanism::Xoauth2],
             )
         } else {
             // plain
             let user = lp.send_user.clone();
             let pw = lp.send_pw.clone();
             (
-                async_smtp::smtp::authentication::Credentials::new(user, pw),
+                smtp::authentication::Credentials::new(user, pw),
                 vec![
-                    async_smtp::smtp::authentication::Mechanism::Plain,
-                    async_smtp::smtp::authentication::Mechanism::Login,
+                    smtp::authentication::Mechanism::Plain,
+                    smtp::authentication::Mechanism::Login,
                 ],
             )
         };
@@ -126,30 +141,31 @@ impl Smtp {
         let security = if 0
             != lp.server_flags & (DC_LP_SMTP_SOCKET_STARTTLS | DC_LP_SMTP_SOCKET_PLAIN) as i32
         {
-            async_smtp::smtp::ClientSecurity::Opportunistic(tls_parameters)
+            smtp::ClientSecurity::Opportunistic(tls_parameters)
         } else {
-            async_smtp::smtp::ClientSecurity::Wrapper(tls_parameters)
+            smtp::ClientSecurity::Wrapper(tls_parameters)
         };
 
-        let client = task::block_on(async_smtp::smtp::SmtpClient::with_security(
-            (domain.as_str(), port),
-            security,
-        ))
-        .map_err(Error::ConnectionSetupFailure)?;
+        let client = smtp::SmtpClient::with_security((domain.as_str(), port), security)
+            .await
+            .map_err(Error::ConnectionSetupFailure)?;
 
         let client = client
             .smtp_utf8(true)
             .credentials(creds)
             .authentication_mechanism(mechanism)
-            .connection_reuse(async_smtp::smtp::ConnectionReuseParameters::ReuseUnlimited);
+            .connection_reuse(smtp::ConnectionReuseParameters::ReuseUnlimited)
+            .timeout(Some(Duration::from_secs(SMTP_TIMEOUT)));
+
         let mut trans = client.into_transport();
-        task::block_on(trans.connect()).map_err(Error::ConnectionFailure)?;
+        trans.connect().await.map_err(Error::ConnectionFailure)?;
 
         self.transport = Some(trans);
         context.call_cb(Event::SmtpConnected(format!(
             "SMTP-LOGIN as {} ok",
             lp.send_user,
         )));
+
         Ok(())
     }
 }

src/smtp/send.rs

@@ -12,8 +12,10 @@ pub type Result<T> = std::result::Result<T, Error>;
 pub enum Error {
     #[fail(display = "Envelope error: {}", _0)]
     EnvelopeError(#[cause] async_smtp::error::Error),
+
     #[fail(display = "Send error: {}", _0)]
     SendError(#[cause] async_smtp::smtp::error::Error),
+
     #[fail(display = "SMTP has no transport")]
     NoTransport,
 }
@@ -43,6 +45,7 @@ impl Smtp {
             format!("{}", job_id), // only used for internal logging
             message,
         );
+
         if let Some(ref mut transport) = self.transport {
             transport.send(mail).await.map_err(Error::SendError)?;
 
@@ -50,7 +53,6 @@ impl Smtp {
                 "Message len={} was smtp-sent to {}",
                 message_len, recipients_display
             )));
-
             Ok(())
         } else {
             warn!(

src/sql.rs

@@ -365,6 +365,7 @@ fn table_exists(conn: &Connection, name: impl AsRef<str>) -> Result<bool> {
     Ok(exists)
 }
 
+#[allow(clippy::cognitive_complexity)]
 fn open(
     context: &Context,
     sql: &Sql,
@@ -544,7 +545,7 @@ fn open(
         // --------------------------------------------------------------------
 
         let mut dbversion = dbversion_before_update;
-        let mut recalc_fingerprints = 0;
+        let mut recalc_fingerprints = false;
         let mut update_icons = false;
 
         if dbversion < 1 {
@@ -687,7 +688,7 @@ fn open(
                 "CREATE INDEX acpeerstates_index4 ON acpeerstates (gossip_key_fingerprint);",
                 params![],
             )?;
-            recalc_fingerprints = 1;
+            recalc_fingerprints = true;
             dbversion = 34;
             sql.set_raw_config_int(context, "dbversion", 34)?;
         }
@@ -872,7 +873,7 @@ fn open(
         // (the structure is complete now and all objects are usable)
         // --------------------------------------------------------------------
 
-        if 0 != recalc_fingerprints {
+        if recalc_fingerprints {
             info!(context, "[migration] recalc fingerprints");
             sql.query_map(
                 "SELECT addr FROM acpeerstates;",

src/stock.rs

@@ -185,6 +185,9 @@ pub enum StockMessage {
                     Recipients don't need to install Delta Chat, visit websites or sign up anywhere - \
                     however, of course, if they like, you may point them to 👉 https://get.delta.chat"))]
     WelcomeMessage = 71,
+
+    #[strum(props(fallback = "Unknown Sender for this chat. See 'info' for more details."))]
+    UnknownSenderForChat = 72,
 }
 
 /*
@@ -544,8 +547,8 @@ mod tests {
         let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
         assert_eq!(chats.len(), 2);
 
-        chat::delete(&t.ctx, chats.get_chat_id(0)).ok();
-        chat::delete(&t.ctx, chats.get_chat_id(1)).ok();
+        chats.get_chat_id(0).delete(&t.ctx).ok();
+        chats.get_chat_id(1).delete(&t.ctx).ok();
         let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
         assert_eq!(chats.len(), 0);
 

src/test_utils.rs

@@ -2,7 +2,8 @@
 //!
 //! This module is only compiled for test runs.
 
-use libc::uintptr_t;
+use std::sync::Arc;
+
 use tempfile::{tempdir, TempDir};
 
 use crate::config::Config;
@@ -16,7 +17,7 @@ use crate::key;
 /// The temporary directory can be used to store the SQLite database,
 /// see e.g. [test_context] which does this.
 pub struct TestContext {
-    pub ctx: Context,
+    pub ctx: Arc<Context>,
     pub dir: TempDir,
 }
 
@@ -31,9 +32,15 @@ pub fn test_context(callback: Option<Box<ContextCallback>>) -> TestContext {
     let dbfile = dir.path().join("db.sqlite");
     let cb: Box<ContextCallback> = match callback {
         Some(cb) => cb,
-        None => Box::new(|_, _| 0),
+        None => Box::new(|_, _| ()),
     };
-    let ctx = Context::new(cb, "FakeOs".into(), dbfile).unwrap();
+    let ctx = Arc::new(Context::new("FakeOs".into(), dbfile).unwrap());
+    let ctx_1 = ctx.clone();
+
+    std::thread::spawn(move || {
+        ctx_1.run(|context, event| (*cb)(context, event));
+    });
+
     TestContext { ctx, dir }
 }
 
@@ -46,14 +53,13 @@ pub fn dummy_context() -> TestContext {
     test_context(None)
 }
 
-pub fn logging_cb(_ctx: &Context, evt: Event) -> uintptr_t {
+pub fn logging_cb(_ctx: &Context, evt: Event) {
     match evt {
         Event::Info(msg) => println!("I: {}", msg),
         Event::Warning(msg) => println!("W: {}", msg),
         Event::Error(msg) => println!("E: {}", msg),
         _ => (),
     }
-    0
 }
 
 /// Creates Alice with a pre-generated keypair.

src/token.rs

@@ -6,6 +6,7 @@
 
 use deltachat_derive::*;
 
+use crate::chat::ChatId;
 use crate::context::Context;
 use crate::dc_tools::*;
 use crate::sql;
@@ -27,28 +28,28 @@ impl Default for Namespace {
 
 /// Creates a new token and saves it into the database.
 /// Returns created token.
-pub fn save(context: &Context, namespace: Namespace, foreign_id: u32) -> String {
+pub fn save(context: &Context, namespace: Namespace, foreign_id: ChatId) -> String {
     // foreign_id may be 0
     let token = dc_create_id();
     sql::execute(
         context,
         &context.sql,
         "INSERT INTO tokens (namespc, foreign_id, token, timestamp) VALUES (?, ?, ?, ?);",
-        params![namespace, foreign_id as i32, &token, time()],
+        params![namespace, foreign_id, &token, time()],
     )
     .ok();
     token
 }
 
-pub fn lookup(context: &Context, namespace: Namespace, foreign_id: u32) -> Option<String> {
+pub fn lookup(context: &Context, namespace: Namespace, foreign_id: ChatId) -> Option<String> {
     context.sql.query_get_value::<_, String>(
         context,
         "SELECT token FROM tokens WHERE namespc=? AND foreign_id=?;",
-        params![namespace, foreign_id as i32],
+        params![namespace, foreign_id],
     )
 }
 
-pub fn lookup_or_new(context: &Context, namespace: Namespace, foreign_id: u32) -> String {
+pub fn lookup_or_new(context: &Context, namespace: Namespace, foreign_id: ChatId) -> String {
     lookup(context, namespace, foreign_id).unwrap_or_else(|| save(context, namespace, foreign_id))
 }
 

src/top_evil_rs.py

@@ -1,40 +0,0 @@
-#!/usr/bin/env python3
-
-
-from pathlib import Path
-import os
-import re
-
-if __name__ == "__main__":
-    if Path('src/top_evil_rs.py').exists():
-        os.chdir('src')
-    filestats = []
-    for fn in Path(".").glob("**/*.rs"):
-        s = fn.read_text()
-        s = re.sub(r"(?m)///.*$", "", s)  # remove comments
-        unsafe = s.count("unsafe")
-        free = s.count("free(")
-        unsafe_fn = s.count("unsafe fn")
-        chars = s.count("c_char") + s.count("CStr")
-        libc = s.count("libc")
-        filestats.append((fn, unsafe, free, unsafe_fn, chars, libc))
-
-    sum_unsafe, sum_free, sum_unsafe_fn, sum_chars, sum_libc = 0, 0, 0, 0, 0
-
-    for fn, unsafe, free, unsafe_fn, chars, libc in reversed(sorted(filestats, key=lambda x: sum(x[1:]))):
-        if unsafe + free + unsafe_fn + chars + libc == 0:
-            continue
-        print("{0: <25} unsafe: {1: >3} free: {2: >3} unsafe-fn: {3: >3} chars: {4: >3} libc: {5: >3}".format(str(fn), unsafe, free, unsafe_fn, chars, libc))
-        sum_unsafe += unsafe
-        sum_free += free
-        sum_unsafe_fn += unsafe_fn
-        sum_chars += chars
-        sum_libc += libc
-
-
-    print()
-    print("total unsafe:", sum_unsafe)
-    print("total free:", sum_free)
-    print("total unsafe-fn:", sum_unsafe_fn)
-    print("total c_chars:", sum_chars)
-    print("total libc:", sum_libc)

src/wrapmime.rs

@@ -1,59 +0,0 @@
-use mailparse::ParsedMail;
-
-use crate::error::Error;
-
-pub fn parse_message_id(message_id: &[u8]) -> Result<String, Error> {
-    let value = std::str::from_utf8(message_id)?;
-    let addrs = mailparse::addrparse(value)
-        .map_err(|err| format_err!("failed to parse message id {:?}", err))?;
-
-    if let Some(info) = addrs.extract_single_info() {
-        return Ok(info.addr);
-    }
-
-    bail!("could not parse message_id: {}", value);
-}
-
-/// Returns a reference to the encrypted payload and validates the autocrypt structure.
-pub fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Result<&'a ParsedMail<'b>, Error> {
-    ensure!(
-        mail.ctype.mimetype == "multipart/encrypted",
-        "Not a multipart/encrypted message: {}",
-        mail.ctype.mimetype
-    );
-    ensure!(
-        mail.subparts.len() == 2,
-        "Invalid Autocrypt Level 1 Mime Parts"
-    );
-
-    ensure!(
-        mail.subparts[0].ctype.mimetype == "application/pgp-encrypted",
-        "Invalid Autocrypt Level 1 version part: {:?}",
-        mail.subparts[0].ctype,
-    );
-
-    ensure!(
-        mail.subparts[1].ctype.mimetype == "application/octet-stream",
-        "Invalid Autocrypt Level 1 encrypted part: {:?}",
-        mail.subparts[1].ctype
-    );
-
-    Ok(&mail.subparts[1])
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_parse_message_id() {
-        assert_eq!(
-            parse_message_id(b"Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org").unwrap(),
-            "Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org"
-        );
-        assert_eq!(
-            parse_message_id(b"<Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org>").unwrap(),
-            "Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org"
-        );
-    }
-}

test-data/message/mail_with_cc.txt

@@ -9,6 +9,6 @@ message-id: <2dfdbde7@example.org>
 Date: Sat, 14 Sep 2019 19:00:13 +0200
 From: lmn <x@tux.org>
 To: abc <abc@bcd.com>
-CC: def <def@def.de>
+CC: def <Def@def.de>
 
 hi

tests/stress.rs

@@ -1,6 +1,7 @@
 //! Stress some functions for testing; if used as a lib, this file is obsolete.
 
 use std::collections::HashSet;
+use std::sync::Arc;
 
 use deltachat::config;
 use deltachat::context::*;
@@ -203,20 +204,24 @@ fn test_encryption_decryption() {
     assert_eq!(plain, original_text);
 }
 
-fn cb(_context: &Context, _event: Event) -> libc::uintptr_t {
-    0
-}
+fn cb(_context: &Context, _event: Event) {}
 
 #[allow(dead_code)]
 struct TestContext {
-    ctx: Context,
+    ctx: Arc<Context>,
     dir: TempDir,
 }
 
 fn create_test_context() -> TestContext {
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
-    let ctx = Context::new(Box::new(cb), "FakeOs".into(), dbfile).unwrap();
+    let ctx = Arc::new(Context::new("FakeOs".into(), dbfile).unwrap());
+    let ctx_1 = ctx.clone();
+
+    std::thread::spawn(move || {
+        ctx_1.run(cb);
+    });
+
     TestContext { ctx, dir }
 }