fix cast to libc::c_char, signed i8 fails eg. on android

This attempts to fix fetching when the inbox contains mail with non-utf8 subjects.

.circleci/config.yml

@@ -15,7 +15,7 @@ restore-workspace: &restore-workspace
 restore-cache: &restore-cache
   restore_cache:
     keys:
-      - cargo-v2-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+      - cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
       - repo-source-{{ .Branch }}-{{ .Revision }}
 
 commands:
@@ -44,7 +44,7 @@ jobs:
           command: cargo generate-lockfile
       - restore_cache:
           keys:
-            - cargo-v2-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+            - cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
       - run: rustup install $(cat rust-toolchain)
       - run: rustup default $(cat rust-toolchain)
       - run: rustup component add --toolchain $(cat rust-toolchain) rustfmt
@@ -60,7 +60,7 @@ jobs:
           paths:
             - crate
       - save_cache:
-          key: cargo-v2-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+          key: cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
           paths:
             - "~/.cargo"
             - "~/.rustup"
@@ -121,39 +121,54 @@ jobs:
     steps:
       - checkout
       - run: bash ci_scripts/run-doxygen.sh
-      - run: mkdir -p workspace/c-docs 
+      - run: mkdir -p workspace/c-docs
       - run: cp -av deltachat-ffi/{html,xml} workspace/c-docs/
       - persist_to_workspace:
           root: workspace
           paths:
             - c-docs
 
-  build_test_docs_wheel:
-    docker:
-      - image: deltachat/coredeps
-        environment:
-            TESTS: 1
-            DOCS: 1
-    working_directory: /mnt/crate
+  # build_test_docs_wheel:
+  #   docker:
+  #     - image: deltachat/coredeps
+  #       environment:
+  #           TESTS: 1
+  #           DOCS: 1
+  #   working_directory: /mnt/crate
+  #   steps:
+  #     - *restore-workspace
+  #     - *restore-cache
+  #     - run:
+  #         name: build docs, run tests and build wheels
+  #         command: ci_scripts/run-python.sh
+  #     - run:
+  #          name: copying docs and wheels to workspace
+  #          command: |
+  #            mkdir -p workspace/python
+  #            # cp -av docs workspace/c-docs
+  #            cp -av python/.docker-tox/wheelhouse workspace/
+  #            cp -av python/doc/_build/ workspace/py-docs
+  #     - persist_to_workspace:
+  #         root: workspace
+  #         paths:
+  #           # - c-docs
+  #           - py-docs
+  #           - wheelhouse
+
+  remote_tests_rust:
+    machine: true
     steps:
-      - *restore-workspace
-      - *restore-cache
-      - run:
-          name: build docs, run tests and build wheels
-          command: ci_scripts/run-python.sh
-      - run:
-           name: copying docs and wheels to workspace
-           command: |
-             mkdir -p workspace/python
-             # cp -av docs workspace/c-docs
-             cp -av python/.docker-tox/wheelhouse workspace/
-             cp -av python/doc/_build/ workspace/py-docs
-      - persist_to_workspace:
-          root: workspace
-          paths:
-            # - c-docs
-            - py-docs
-            - wheelhouse
+      - checkout
+      - run: ci_scripts/remote_tests_rust.sh
+    
+  remote_tests_python:
+    machine: true
+    steps:
+      - checkout
+      #- attach_workspace:
+      #    at: workspace
+      - run: ci_scripts/remote_tests_python.sh
+    # workspace/py-docs workspace/wheelhouse workspace/c-docs
 
   upload_docs_wheels:
     machine: true
@@ -181,15 +196,15 @@ workflows:
   test:
     jobs:
       - cargo_fetch
-      - build_doxygen
 
-      - build_test_docs_wheel:
-          requires:
-            - cargo_fetch
-      - upload_docs_wheels:
-          requires:
-            - build_test_docs_wheel
-            - build_doxygen 
+      - remote_tests_rust
+
+      - remote_tests_python
+
+      # - upload_docs_wheels:
+      #     requires:
+      #       - build_test_docs_wheel
+      #       - build_doxygen
       - rustfmt:
           requires:
             - cargo_fetch
@@ -197,10 +212,12 @@ workflows:
           requires:
            - cargo_fetch
 
+      - build_doxygen
+
       # Linux Desktop 64bit
-      - test_x86_64-unknown-linux-gnu:
-          requires:
-            - cargo_fetch
+      # - test_x86_64-unknown-linux-gnu:
+      #     requires:
+      #       - cargo_fetch
 
       # Linux Desktop 32bit
       # - test_i686-unknown-linux-gnu:

CHANGELOG.md

@@ -1,5 +1,51 @@
 # Changelog 
 
+## 1.0.0-beta.8
+
+- now uses async-email/async-imap as the new base 
+  which makes imap-idle interruptible and thus fixes
+  several issues around the imap thread being in zombie state . 
+  thanks @dignifiedquire, @hpk42 and @link2xt. 
+
+- fixes imap-protocol parsing bugs that lead to infinitely
+  repeated crashing while trying to receive messages with
+  a subjec that contained non-utf8. thanks @link2xt
+
+- fixed logic to find encryption subkey -- previously 
+  delta chat would use the primary key for encryption
+  (which works with RSA but not ECC). thanks @link2xt
+
+- introduce a new device chat where core and UIs can 
+  add "device" messages.  Android uses it for an initial
+  welcome message. thanks @r10s
+
+- fix time smearing (when two message are virtually send
+  in the same second, there would be misbehaviour because
+  we didn't persist smeared time). thanks @r10s
+
+- fix double-dotted extensions like .html.zip or .tar.gz  
+  to not mangle them when creating blobfiles.  thanks @flub
+
+- fix backup/exports where the wrong sql file would be modified,
+  leading to problems when exporting twice.  thanks @hpk42
+
+- several other little fixes and improvements 
+
+
+## 1.0.0-beta.7
+
+- fix location-streaming #782
+
+- fix display of messages that could not be decrypted #785
+ 
+- fix smtp MAILER-DAEMON bug #786 
+
+- fix a logging of durations #783
+
+- add more error logging #779
+
+- do not panic on some bad utf-8 mime #776
+
 ## 1.0.0-beta.6
 
 - fix chatlist.get_msg_id to return id, instead of wrongly erroring

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.0.0-beta.6"
+version = "1.0.0-beta.8"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL"
@@ -10,17 +10,18 @@ deltachat_derive = { path = "./deltachat_derive" }
 mmime = { version = "0.1.2", path = "./mmime" }
 
 libc = "0.2.51"
-pgp = { version = "0.2.3", default-features = false }
-hex = "0.3.2"
+pgp = { git = "https://github.com/rpgp/rpgp", branch = "master", default-features = false }
+hex = "0.4.0"
 sha2 = "0.8.0"
 rand = "0.6.5"
 smallvec = "0.6.9"
-reqwest = "0.9.15"
+reqwest = { version = "0.9.15", default-features = false, features = ["rustls-tls"] }
 num-derive = "0.2.5"
 num-traits = "0.2.6"
-native-tls = "0.2.3"
-lettre = { git = "https://github.com/deltachat/lettre", branch = "master" }
-imap = { git = "https://github.com/deltachat/rust-imap", branch = "master" }
+lettre = { git = "https://github.com/deltachat/lettre", branch = "feat/rustls" }
+async-imap = { git = "https://github.com/async-email/async-imap", branch="master" }
+async-tls = "0.6"
+async-std = { version = "1.0", features = ["unstable"] }
 base64 = "0.10"
 charset = "0.1"
 percent-encoding = "2.0"
@@ -49,6 +50,10 @@ bitflags = "1.1.0"
 jetscii = "0.4.4"
 debug_stub_derive = "0.3.0"
 sanitize-filename = "0.2.1"
+stop-token = { version = "0.1.1", features = ["unstable"] }
+rustls = "0.16.0"
+webpki-roots = "0.18.0"
+webpki = "0.21.0"
 
 [dev-dependencies]
 tempfile = "3.0"
@@ -74,6 +79,6 @@ path = "examples/repl/main.rs"
 
 [features]
 default = ["nightly", "ringbuf"]
-vendored = ["native-tls/vendored", "reqwest/default-tls-vendored"]
+vendored = []
 nightly = ["pgp/nightly"]
 ringbuf = ["pgp/ringbuf"]

README.md

@@ -1,11 +1,9 @@
 # Delta Chat Rust
 
-> Project porting deltachat-core to rust
+> Deltachat-core written in Rust 
 
 [![CircleCI build status][circle-shield]][circle] [![Appveyor build status][appveyor-shield]][appveyor]
 
-Current commit on deltachat/deltachat-core: `12ef73c8e76185f9b78e844ea673025f56a959ab`.
-
 ## Installing Rust and Cargo
 
 To download and install the official compiler for the Rust programming language, and the Cargo package manager, run the command in your user environment:
@@ -16,7 +14,7 @@ curl https://sh.rustup.rs -sSf | sh
 
 ## Using the CLI client
 
-Compile and run Delta Chat Core using `cargo`:
+Compile and run Delta Chat Core command line utility, using `cargo`:
 
 ```
 cargo run --example repl -- /path/to/db

assets/icon-device.svg

@@ -0,0 +1,110 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:xlink="http://www.w3.org/1999/xlink"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   inkscape:version="1.0beta1 (32d4812, 2019-09-19)"
+   sodipodi:docname="icon-device.svg"
+   id="svg4344"
+   preserveAspectRatio="xMidYMid meet"
+   viewBox="0 0 45 45"
+   height="60"
+   width="60"
+   version="1.0"
+   inkscape:export-filename="/Users/bpetersen/Documents/deltachat/icons/device4.png"
+   inkscape:export-xdpi="96"
+   inkscape:export-ydpi="96">
+  <defs
+     id="defs4348">
+    <linearGradient
+       inkscape:collect="always"
+       id="linearGradient865">
+      <stop
+         style="stop-color:#364e59;stop-opacity:1;"
+         offset="0"
+         id="stop861" />
+      <stop
+         style="stop-color:#ffffff;stop-opacity:1"
+         offset="1"
+         id="stop863" />
+    </linearGradient>
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient865"
+       id="linearGradient867"
+       x1="10.227795"
+       y1="43.5238"
+       x2="-25.458139"
+       y2="-7.4257693"
+       gradientUnits="userSpaceOnUse" />
+  </defs>
+  <sodipodi:namedview
+     inkscape:current-layer="svg4344"
+     inkscape:window-maximized="0"
+     inkscape:window-y="23"
+     inkscape:window-x="45"
+     inkscape:cy="28.471578"
+     inkscape:cx="41.455164"
+     inkscape:zoom="5.6"
+     units="px"
+     showgrid="false"
+     id="namedview4346"
+     inkscape:window-height="1035"
+     inkscape:window-width="1600"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="0"
+     guidetolerance="10"
+     gridtolerance="10"
+     objecttolerance="10"
+     borderopacity="1"
+     inkscape:document-rotation="0"
+     bordercolor="#666666"
+     pagecolor="#ffffff" />
+  <metadata
+     id="metadata4336">
+Created by potrace 1.15, written by Peter Selinger 2001-2017
+<rdf:RDF>
+  <cc:Work
+     rdf:about="">
+    <dc:format>image/svg+xml</dc:format>
+    <dc:type
+       rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+    <dc:title />
+  </cc:Work>
+</rdf:RDF>
+</metadata>
+  <circle
+     style="opacity:1;fill:#ffffff;fill-opacity:1;stroke-width:0.731071"
+     id="path859"
+     cx="22.5"
+     cy="22.5"
+     r="22.5" />
+  <circle
+     r="22.5"
+     cy="22.5"
+     cx="22.5"
+     id="path4917"
+     style="fill:url(#linearGradient867);fill-opacity:1;stroke-width:0.588554" />
+  <g
+     id="g4342"
+     stroke="none"
+     fill="#000000"
+     transform="matrix(0.00255113,0,0,-0.00255113,5.586152,38.200477)"
+     style="fill:#ffffff;fill-opacity:1">
+    <path
+       inkscape:connector-curvature="0"
+       id="path4338"
+       d="m 8175,12765 c -703,-114 -1248,-608 -1387,-1258 -17,-82 -21,-136 -22,-277 0,-202 15,-307 70,-470 149,-446 499,-733 1009,-828 142,-26 465,-23 619,6 691,131 1201,609 1328,1244 31,158 31,417 0,565 -114,533 -482,889 -1038,1004 -133,27 -448,35 -579,14 z"
+       style="fill:#ffffff;fill-opacity:1" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path4340"
+       d="m 7070,9203 c -212,-20 -275,-27 -397,-48 -691,-117 -1400,-444 -2038,-940 -182,-142 -328,-270 -585,-517 -595,-571 -911,-974 -927,-1181 -6,-76 11,-120 69,-184 75,-80 159,-108 245,-79 109,37 263,181 632,595 539,606 774,826 1035,969 135,75 231,105 341,106 82,1 94,-2 138,-27 116,-68 161,-209 122,-376 -9,-36 -349,-868 -757,-1850 -407,-982 -785,-1892 -838,-2021 -287,-694 -513,-1389 -615,-1889 -70,-342 -90,-683 -52,-874 88,-440 381,-703 882,-792 124,-23 401,-30 562,-16 783,69 1674,461 2561,1125 796,596 1492,1354 1607,1751 43,146 -33,308 -168,360 -61,23 -100,15 -173,-36 -105,-74 -202,-170 -539,-529 -515,-551 -762,-783 -982,-927 -251,-164 -437,-186 -543,-65 -56,64 -74,131 -67,247 13,179 91,434 249,815 135,324 1588,4102 1646,4280 106,325 151,561 159,826 9,281 -22,463 -112,652 -58,122 -114,199 -211,292 -245,233 -582,343 -1044,338 -91,-1 -181,-3 -200,-5 z"
+       style="fill:#ffffff;fill-opacity:1" />
+  </g>
+</svg>

assets/icon-saved-messages.svg

@@ -0,0 +1,99 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:xlink="http://www.w3.org/1999/xlink"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   inkscape:export-ydpi="204.8"
+   inkscape:export-xdpi="204.8"
+   inkscape:export-filename="/Users/bpetersen/Documents/deltachat/icons/saved-messages.png"
+   version="1.0"
+   width="60"
+   height="60"
+   viewBox="0 0 45 45"
+   preserveAspectRatio="xMidYMid meet"
+   id="svg4344"
+   sodipodi:docname="saved-messages.svg"
+   inkscape:version="1.0beta1 (32d4812, 2019-09-19)">
+  <defs
+     id="defs4348">
+    <linearGradient
+       inkscape:collect="always"
+       id="linearGradient906">
+      <stop
+         style="stop-color:#046cdb;stop-opacity:1"
+         offset="0"
+         id="stop902" />
+      <stop
+         id="stop845"
+         offset="1"
+         style="stop-color:#ffffff;stop-opacity:1" />
+    </linearGradient>
+    <linearGradient
+       y2="-11.049108"
+       x2="-3.1473193"
+       y1="31.473215"
+       x1="36.160713"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient847"
+       xlink:href="#linearGradient906"
+       inkscape:collect="always" />
+  </defs>
+  <sodipodi:namedview
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     inkscape:document-rotation="0"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="1600"
+     inkscape:window-height="1035"
+     id="namedview4346"
+     showgrid="false"
+     units="px"
+     inkscape:zoom="2.8"
+     inkscape:cx="48.214285"
+     inkscape:cy="55"
+     inkscape:window-x="45"
+     inkscape:window-y="23"
+     inkscape:window-maximized="0"
+     inkscape:current-layer="svg4344"
+     inkscape:lockguides="false" />
+  <metadata
+     id="metadata4336">
+Created by potrace 1.15, written by Peter Selinger 2001-2017
+<rdf:RDF>
+  <cc:Work
+     rdf:about="">
+    <dc:format>image/svg+xml</dc:format>
+    <dc:type
+       rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+    <dc:title />
+  </cc:Work>
+</rdf:RDF>
+</metadata>
+  <circle
+     r="22.5"
+     cy="22.5"
+     cx="22.5"
+     id="path859"
+     style="opacity:1;fill:#ffffff;fill-opacity:1;stroke-width:0.731071" />
+  <circle
+     style="fill:url(#linearGradient847);fill-opacity:1;stroke-width:0.588554"
+     id="path4917"
+     cx="22.5"
+     cy="22.5"
+     r="22.5" />
+  <path
+     id="rect846"
+     style="fill:#ffffff;stroke-width:0.84832"
+     d="M 17 9 L 17 55 L 17.125 55 L 30.048828 40.9375 L 42.974609 55 L 43 55 L 43 9 L 17 9 z M 30.005859 15 L 32.720703 22.121094 L 40 22.646484 L 34.396484 27.558594 L 36.169922 35 L 29.992188 30.916016 L 23.808594 34.988281 L 25.595703 27.550781 L 20 22.626953 L 27.28125 22.115234 L 30.005859 15 z "
+     transform="scale(0.75)" />
+</svg>

ci_scripts/README.md

@@ -1,52 +1,46 @@
-
 # Continuous Integration Scripts for Delta Chat
 
-Continuous Integration is run through CircleCI
-but is largely independent of it. 
+Continuous Integration, run through CircleCI and an own build machine.
+
+## Description of scripts 
+
+- `../.circleci/config.yml` describing the build jobs that are run
+  by Circle-CI
+
+- `remote_tests_python.sh` rsyncs to a build machine and runs
+  `run-python-test.sh` remotely on the build machine. 
+
+- `remote_tests_rust.sh` rsyncs to the build machine and runs
+  `run-rust-test.sh` remotely on the build machine. 
+
+- `doxygen/Dockerfile` specifies an image that contains
+  the doxygen tool which is used by `run-doxygen.sh` 
+  to generate C-docs which are then uploaded 
+  via `ci_upload.sh` to `https://c.delta.chat/_unofficial_unreleased_docs/<BRANCH>`
+  (and the master branch is linked to https://c.delta.chat proper). 
 
 
-## Generating docker containers for performing build step work 
+## Triggering runs on the build machine locally (fast!)
 
-All tests, docs and wheel building is run in docker containers:
+There is experimental support for triggering a remote Python or Rust test run 
+from your local checkout/branch. You will need to be authorized to login to 
+the build machine (ask your friendly sysadmin on #deltachat freenode) to type::
 
-- **coredeps/Dockerfile** specifies an image that contains all 
-  of Delta Chat's core dependencies as linkable libraries. 
-  It also serves to run python tests and build wheels 
-  (binary packages for Python). 
+    ci_scripts/manual_remote_tests.sh rust
+    ci_scripts/manual_remote_tests.sh python
 
-- **doxygen/Dockerfile** specifies an image that contains
-  the doxygen tool which is used to generate C-docs. 
+This will **rsync** your current checkout to the remote build machine 
+(no need to commit before) and then run either rust or python tests. 
 
-To run tests locally you can pull existing images from "docker.io",
-the hub for sharing Docker images::
+# Outdated files (for later re-use)
 
-    docker pull deltachat/coredeps
-    docker pull deltachat/doxygen 
+`coredeps/Dockerfile` specifies an image that contains all 
+of Delta Chat's core dependencies. It used to run 
+python tests and build wheels (binary packages for Python)
 
-or you can build the docker images yourself locally
+You can build the docker images yourself locally
 to avoid the relatively large download:: 
  
     cd ci_scripts  # where all CI things are 
     docker build -t deltachat/coredeps docker-coredeps
     docker build -t deltachat/doxygen docker-doxygen 
-
-## ci_run.sh (main entrypoint called by circle-ci)
-
-Once you have the docker images available 
-you can run python testing, documentation generation 
-and building binary wheels:: 
-
-    sh DOCS=1 TESTS=1 ci_scripts/ci_run.sh 
-    
-## ci_upload.sh (uploading artifacts on success) 
-
-- python docs to `https://py.delta.chat/_unofficial_unreleased_docs/<BRANCH>`
-
-- doxygen docs to `https://c.delta.chat/_unofficial_unreleased_docs/<BRANCH>`
-
-- python wheels to `https://m.devpi.net/dc/<BRANCH>`
-  so that you install fully self-contained wheels like this:
-  `pip install -U -i https://m.devpi.net/dc/<BRANCH> deltachat`
-
-
-

ci_scripts/ci_upload.sh

@@ -14,20 +14,24 @@ DOXYDOCDIR=${3:?directory where doxygen docs to be found}
 export BRANCH=${CIRCLE_BRANCH:?specify branch for uploading purposes}
 
 
-# python docs to py.delta.chat
-ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
-rsync -avz \
-  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
-  "$PYDOCDIR/html/" \
-  delta@py.delta.chat:build/${BRANCH}
+# DISABLED: python docs to py.delta.chat
+#ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
+#rsync -avz \
+#  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+#  "$PYDOCDIR/html/" \
+#  delta@py.delta.chat:build/${BRANCH}
 
 # C docs to c.delta.chat
-ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@c.delta.chat mkdir -p build-c/${BRANCH}
+ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@c.delta.chat mkdir -p build-c/${BRANCH}
 rsync -avz \
   -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
   "$DOXYDOCDIR/html/" \
   delta@c.delta.chat:build-c/${BRANCH}
 
+exit 0
+
+# OUTDATED -- for re-use from python release-scripts 
+
 echo -----------------------
 echo upload wheels 
 echo -----------------------

ci_scripts/docker-coredeps/Dockerfile

@@ -5,16 +5,6 @@ RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
     echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
 ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
 
-ENV PIP_DISABLE_PIP_VERSION_CHECK 1
-
-# Install python tools (auditwheels,tox, ...)
-ADD deps/build_python.sh /builder/build_python.sh
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
-
-# Install Rust nightly 
-ADD deps/build_rust.sh /builder/build_rust.sh
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1
-
 # Install a recent Perl, needed to install OpenSSL
 ADD deps/build_perl.sh /builder/build_perl.sh
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
@@ -23,3 +13,12 @@ RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
 ADD deps/build_openssl.sh /builder/build_openssl.sh
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_openssl.sh && cd .. && rm -r tmp1
 
+ENV PIP_DISABLE_PIP_VERSION_CHECK 1
+
+# Install python tools (auditwheels,tox, ...)
+ADD deps/build_python.sh /builder/build_python.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
+
+# Install Rust nightly
+ADD deps/build_rust.sh /builder/build_rust.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1

ci_scripts/docker-coredeps/deps/build_perl.sh

@@ -1,11 +1,11 @@
 #!/bin/bash
 
-PERL_VERSION=5.28.0
-PERL_SHA256=7e929f64d4cb0e9d1159d4a59fc89394e27fa1f7004d0836ca0d514685406ea8
+PERL_VERSION=5.30.0
+# PERL_SHA256=7e929f64d4cb0e9d1159d4a59fc89394e27fa1f7004d0836ca0d514685406ea8
 curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz
-echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
-tar xzf perl-${PERL_VERSION}.tar.gz
-cd perl-${PERL_VERSION} 
+# echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
+tar -xzf perl-${PERL_VERSION}.tar.gz
+cd perl-${PERL_VERSION}
 
 ./Configure -de
 make

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -1,11 +1,8 @@
 #!/bin/bash
 
-set -e -x 
+set -e -x
 
-# Install Rust 
-curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-07-10 -y
+# Install Rust
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-09-12 -y
 export PATH=/root/.cargo/bin:$PATH
 rustc --version
-
-# remove some 300-400 MB that we don't need for automated builds
-rm -rf /root/.rustup/toolchains/nightly-2019-07-10-x86_64-unknown-linux-gnu/share/

ci_scripts/manual_remote_tests.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -xe
+export CIRCLE_JOB=remote_tests_${1:?need to specify 'rust' or 'python'} 
+export CIRCLE_BUILD_NUM=$USER
+export CIRCLE_BRANCH=`git branch | grep \* | cut -d ' ' -f2`
+export CIRCLE_PROJECT_REPONAME=$(basename `git rev-parse --show-toplevel`)
+
+time bash ci_scripts/$CIRCLE_JOB.sh 

ci_scripts/old/gh-actions-rust.yml

@@ -0,0 +1,77 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+
+env:
+  RUSTFLAGS: -Dwarnings
+
+jobs:
+  build_and_test:
+    name: Build and test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macOS-latest]
+        rust: [nightly]
+
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Install ${{ matrix.rust }}
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: ${{ matrix.rust }}
+        override: true
+
+    - name: check
+      uses: actions-rs/cargo@v1
+      if: matrix.rust == 'nightly'
+      with:
+        command:  check
+        args: --all --bins --examples --tests
+
+    - name: tests
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --all
+
+    - name: tests ignored
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --all --release -- --ignored
+
+  check_fmt:
+    name: Checking fmt and docs
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@master
+
+    - uses: actions-rs/toolchain@v1
+      with:
+          profile: minimal
+          toolchain: nightly
+          override: true
+          components: rustfmt
+
+    - name: fmt
+      run: cargo fmt --all -- --check
+
+#  clippy_check:
+#    name: Clippy check
+#    runs-on: ubuntu-latest
+#
+#    steps:
+#    - uses: actions/checkout@v1
+#    - uses: actions-rs/toolchain@v1
+#      with:
+#          profile: minimal
+#          toolchain: nightly
+#          override: true
+#          components: clippy
+#
+#    - name: clippy
+#      run: cargo clippy --all

ci_scripts/old/run-python.sh

@@ -43,8 +43,8 @@ if [ -n "$TESTS" ]; then
     # we split out qr-tests run to minimize likelyness of flaky tests
     # (some qr tests are pretty heavy in terms of send/received
     # messages and rust's imap code likely has concurrency problems) 
-    tox --workdir "$TOXWORKDIR" -e py37 -- -k "not qr"
-    tox --workdir "$TOXWORKDIR" -e py37 -- -k "qr"
+    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "not qr"
+    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "qr"
     unset DCC_PY_LIVECONFIG
     tox --workdir "$TOXWORKDIR" -p4 -e lint,py35,py36,doc
     tox --workdir "$TOXWORKDIR" -e auditwheels

ci_scripts/remote_tests_python.sh

@@ -0,0 +1,46 @@
+#!/bin/bash 
+
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export SSHTARGET=ci@b1.delta.chat
+
+# we construct the BUILDDIR such that we can easily share the
+# CARGO_TARGET_DIR between runs ("..")
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+set -xe
+
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist 
+# we seem to need .git for setuptools_scm versioning 
+find .git >>.rsynclist
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+
+set +x
+
+echo "--- Running $CIRCLE_JOB remotely"
+
+ssh $SSHTARGET <<_HERE
+    set +x -e
+    cd $BUILDDIR
+    # let's share the target dir with our last run on this branch/job-type
+    # cargo will make sure to block/unblock us properly 
+    export CARGO_TARGET_DIR=\`pwd\`/../target
+    export TARGET=release
+    export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
+
+    #we rely on tox/virtualenv being available in the host
+    #rm -rf virtualenv venv
+    #virtualenv -q -p python3.7 venv 
+    #source venv/bin/activate
+    #pip install -q tox virtualenv
+
+    set -x
+    which python
+    source \$HOME/venv/bin/activate
+    which python
+
+    bash ci_scripts/run-python-test.sh 
+_HERE

ci_scripts/remote_tests_rust.sh

@@ -0,0 +1,32 @@
+#!/bin/bash 
+
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export SSHTARGET=ci@b1.delta.chat
+
+# we construct the BUILDDIR such that we can easily share the
+# CARGO_TARGET_DIR between runs ("..")
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+
+set -e
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist 
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+
+echo "--- Running $CIRCLE_JOB remotely"
+
+ssh $SSHTARGET <<_HERE
+    set +x -e
+    cd $BUILDDIR
+    # let's share the target dir with our last run on this branch/job-type
+    # cargo will make sure to block/unblock us properly 
+    export CARGO_TARGET_DIR=\`pwd\`/../target
+    export TARGET=x86_64-unknown-linux-gnu
+    export RUSTC_WRAPPER=sccache
+
+    bash ci_scripts/run-rust-test.sh
+_HERE
+

ci_scripts/run-python-test.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+#
+# Run functional tests for Delta Chat core using the python bindings 
+# and tox/pytest. 
+
+set -e -x
+
+# for core-building and python install step
+export DCC_RS_TARGET=release
+export DCC_RS_DEV=`pwd`
+
+cd python
+
+python install_python_bindings.py onlybuild
+
+# remove and inhibit writing PYC files 
+rm -rf tests/__pycache__
+rm -rf src/deltachat/__pycache__
+export PYTHONDONTWRITEBYTECODE=1
+
+# run python tests (tox invokes pytest to run tests in python/tests)
+#TOX_PARALLEL_NO_SPINNER=1 tox -e lint,doc
+tox -e lint
+tox -e doc,py37

ci_scripts/run-rust-test.sh

@@ -2,7 +2,7 @@
 
 set -ex
 
-export RUST_TEST_THREADS=1
+#export RUST_TEST_THREADS=1
 export RUST_BACKTRACE=1
 export RUSTFLAGS='--deny warnings'
 export OPT="--target=$TARGET"
@@ -37,7 +37,9 @@ else
     export OPT_RELEASE_IGNORED="${OPT_RELEASE} -- --ignored"
 fi
 
-# Run all the test configurations:
+# Run all the test configurations 
+# RUSTC_WRAPPER=SCCACHE seems to destroy parallelism / prolong the test
+unset RUSTC_WRAPPER
 $CARGO_CMD $CARGO_SUBCMD $OPT
 $CARGO_CMD $CARGO_SUBCMD $OPT_RELEASE
 $CARGO_CMD $CARGO_SUBCMD $OPT_RELEASE_IGNORED

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.0.0-beta.6"
+version = "1.0.0-beta.8"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"

deltachat-ffi/deltachat.h

@@ -338,6 +338,8 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `send_pw`      = SMTP-password, guessed if left out
  * - `send_port`    = SMTP-port, guessed if left out
  * - `server_flags` = IMAP-/SMTP-flags as a combination of @ref DC_LP flags, guessed if left out
+ * - `imap_certificate_checks` = how to check IMAP certificates, one of the @ref DC_CERTCK flags, defaults to #DC_CERTCK_AUTO (0)
+ * - `smtp_certificate_checks` = how to check SMTP certificates, one of the @ref DC_CERTCK flags, defaults to #DC_CERTCK_AUTO (0)
  * - `displayname`  = Own name to use when sending messages.  MUAs are allowed to spread this way eg. using CC, defaults to empty
  * - `selfstatus`   = Own status to display eg. in email footers, defaults to a standard text
  * - `selfavatar`   = File containing avatar. Will be copied to blob directory.
@@ -499,15 +501,9 @@ char*           dc_get_oauth2_url            (dc_context_t* context, const char*
  * To interrupt a configuration prematurely, use dc_stop_ongoing_process();
  * this is not needed if #DC_EVENT_CONFIGURE_PROGRESS reports success.
  *
- * On a successfull configuration,
- * the core makes a copy of the parameters mentioned above:
- * the original parameters as are never modified by the core.
- *
- * UI-implementors should keep this in mind -
- * eg. if the UI wants to prefill a configure-edit-dialog with these parameters,
- * the UI should reset them if the user cancels the dialog
- * after a configure-attempts has failed.
- * Otherwise the parameters may not reflect the current configuation.
+ * If #DC_EVENT_CONFIGURE_PROGRESS reports failure,
+ * the core continues to use the last working configuration
+ * and parameters as `addr`, `mail_pw` etc. are set to that.
  *
  * @memberof dc_context_t
  * @param context The context object as created by dc_context_new().
@@ -1096,6 +1092,66 @@ uint32_t        dc_send_text_msg             (dc_context_t* context, uint32_t ch
 void            dc_set_draft                 (dc_context_t* context, uint32_t chat_id, dc_msg_t* msg);
 
 
+/**
+ * Add a message to the device-chat.
+ * Device-messages usually contain update information
+ * and some hints that are added during the program runs, multi-device etc.
+ * The device-message may be defined by a label;
+ * if a message with the same label was added or skipped before,
+ * the message is not added again, even if the message was deleted in between.
+ * If needed, the device-chat is created before.
+ *
+ * Sends the event #DC_EVENT_MSGS_CHANGED on success.
+ * To check, if a given chat is a device-chat, see dc_chat_is_device_talk()
+ *
+ * @memberof dc_context_t
+ * @param context The context as created by dc_context_new().
+ * @param label A unique name for the message to add.
+ *     The label is typically not displayed to the user and
+ *     must be created from the characters `A-Z`, `a-z`, `0-9`, `_` or `-`.
+ *     If you pass NULL here, the message is added unconditionally.
+ * @param msg Message to be added to the device-chat.
+ *     The message appears to the user as an incoming message.
+ *     If you pass NULL here, only the given label will be added
+ *     and block adding messages with that label in the future.
+ * @return The ID of the just added message,
+ *     if the message was already added or no message to add is given, 0 is returned.
+ *
+ * Example:
+ * ~~~
+ * dc_msg_t* welcome_msg = dc_msg_new(DC_MSG_TEXT);
+ * dc_msg_set_text(welcome_msg, "great that you give this app a try!");
+ *
+ * dc_msg_t* changelog_msg = dc_msg_new(DC_MSG_TEXT);
+ * dc_msg_set_text(changelog_msg, "we have added 3 new emojis :)");
+ *
+ * if (dc_add_device_msg(context, "welcome", welcome_msg)) {
+ *     // do not add the changelog on a new installations -
+ *     // not now and not when this code is executed again
+ *     dc_add_device_msg(context, "update-123", NULL);
+ * } else {
+ *     // welcome message was not added now, this is an oder installation,
+ *     // add a changelog
+ *     dc_add_device_msg(context, "update-123", changelog_msg);
+ * }
+ * ~~~
+ */
+uint32_t        dc_add_device_msg            (dc_context_t* context, const char* label, dc_msg_t* msg);
+
+
+/**
+ * Check if a device-message with a given label was ever added.
+ * Device-messages can be added dc_add_device_msg().
+ *
+ * @memberof dc_context_t
+ * @param context The context as created by dc_context_new().
+ * @param label Label of the message to check.
+ * @return 1=A message with this label was added at some point,
+ *     0=A message with this label was never added.
+ */
+int             dc_was_device_msg_ever_added (dc_context_t* context, const char* label);
+
+
 /**
  * Get draft for a chat, if any.
  * See dc_set_draft() for more details about drafts.
@@ -2610,8 +2666,6 @@ int             dc_chat_get_type             (const dc_chat_t* chat);
  *
  * To change the name, use dc_set_chat_name()
  *
- * See also: dc_chat_get_subtitle()
- *
  * @memberof dc_chat_t
  * @param chat The chat object.
  * @return Chat name as a string. Must be released using dc_str_unref() after usage. Never NULL.
@@ -2619,13 +2673,13 @@ int             dc_chat_get_type             (const dc_chat_t* chat);
 char*           dc_chat_get_name             (const dc_chat_t* chat);
 
 
-/**
+/*
  * Get a subtitle for a chat.  The subtitle is eg. the email-address or the
  * number of group members.
  *
- * See also: dc_chat_get_name()
+ * Deprecated function. Subtitles should be created in the ui
+ * where plural forms and other specials can be handled more gracefully.
  *
- * @memberof dc_chat_t
  * @param chat The chat object to calulate the subtitle for.
  * @return Subtitle as a string. Must be released using dc_str_unref() after usage. Never NULL.
  */
@@ -2715,6 +2769,37 @@ int             dc_chat_is_unpromoted        (const dc_chat_t* chat);
 int             dc_chat_is_self_talk         (const dc_chat_t* chat);
 
 
+/**
+ * Check if a chat is a device-talk.
+ * Device-talks contain update information
+ * and some hints that are added during the program runs, multi-device etc.
+ *
+ * From the ui view, device-talks are not very special,
+ * the user can delete and forward messages, archive the chat, set notifications etc.
+ *
+ * Messages can be added to the device-talk using dc_add_device_msg()
+ *
+ * @memberof dc_chat_t
+ * @param chat The chat object.
+ * @return 1=chat is device-talk, 0=chat is no device-talk
+ */
+int             dc_chat_is_device_talk       (const dc_chat_t* chat);
+
+
+/**
+ * Check if messages can be sent to a give chat.
+ * This is not true eg. for the deaddrop or for the device-talk, cmp. dc_chat_is_device_talk().
+ *
+ * Calling dc_send_msg() for these chats will fail
+ * and the ui may decide to hide input controls therefore.
+ *
+ * @memberof dc_chat_t
+ * @param chat The chat object.
+ * @return 1=chat is writable, 0=chat is not writable
+ */
+int             dc_chat_can_send              (const dc_chat_t* chat);
+
+
 /**
  * Check if a chat is verified.  Verified chats contain only verified members
  * and encryption is alwasy enabled.  Verified chats are created using
@@ -3370,7 +3455,8 @@ void            dc_msg_latefiling_mediasize   (dc_msg_t* msg, int width, int hei
 
 
 #define         DC_CONTACT_ID_SELF           1
-#define         DC_CONTACT_ID_DEVICE         2
+#define         DC_CONTACT_ID_INFO           2 // centered messages as "member added", used in all chats
+#define         DC_CONTACT_ID_DEVICE         5 // messages "update info" in the device-chat
 #define         DC_CONTACT_ID_LAST_SPECIAL   9
 
 
@@ -3894,7 +3980,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * These constants configure TLS certificate checks for IMAP and SMTP connections.
  *
- * These constants are set via dc_set_config
+ * These constants are set via dc_set_config()
  * using keys "imap_certificate_checks" and "smtp_certificate_checks".
  *
  * @addtogroup DC_CERTCK
@@ -3907,8 +3993,8 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_CERTCK_AUTO 0
 
 /**
- * Strictly check TLS certificates.
- * Require that both the certificate and hostname are valid.
+ * Strictly check TLS certificates;
+ * require that both the certificate and hostname are valid.
  */
 #define DC_CERTCK_STRICT 1
 
@@ -4418,7 +4504,8 @@ void            dc_array_add_id              (dc_array_t*, uint32_t); // depreca
 #define DC_STR_MSGLOCATIONDISABLED        65
 #define DC_STR_LOCATION                   66
 #define DC_STR_STICKER                    67
-#define DC_STR_COUNT                      67
+#define DC_STR_DEVICE_MESSAGES            68
+#define DC_STR_COUNT                      68
 
 /*
  * @}

deltachat-ffi/src/lib.rs

@@ -461,7 +461,7 @@ pub unsafe extern "C" fn dc_perform_imap_jobs(context: *mut dc_context_t) {
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| job::perform_imap_jobs(ctx))
+        .with_inner(|ctx| job::perform_inbox_jobs(ctx))
         .unwrap_or(())
 }
 
@@ -473,19 +473,20 @@ pub unsafe extern "C" fn dc_perform_imap_fetch(context: *mut dc_context_t) {
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| job::perform_imap_fetch(ctx))
+        .with_inner(|ctx| job::perform_inbox_fetch(ctx))
         .unwrap_or(())
 }
 
 #[no_mangle]
 pub unsafe extern "C" fn dc_perform_imap_idle(context: *mut dc_context_t) {
+    // TODO rename function in co-ordination with UIs
     if context.is_null() {
         eprintln!("ignoring careless call to dc_perform_imap_idle()");
         return;
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| job::perform_imap_idle(ctx))
+        .with_inner(|ctx| job::perform_inbox_idle(ctx))
         .unwrap_or(())
 }
 
@@ -497,7 +498,7 @@ pub unsafe extern "C" fn dc_interrupt_imap_idle(context: *mut dc_context_t) {
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| job::interrupt_imap_idle(ctx))
+        .with_inner(|ctx| job::interrupt_inbox_idle(ctx, true))
         .unwrap_or(())
 }
 
@@ -811,6 +812,54 @@ pub unsafe extern "C" fn dc_set_draft(
         .unwrap_or(())
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_add_device_msg(
+    context: *mut dc_context_t,
+    label: *const libc::c_char,
+    msg: *mut dc_msg_t,
+) -> u32 {
+    if context.is_null() || (label.is_null() && msg.is_null()) {
+        eprintln!("ignoring careless call to dc_add_device_msg()");
+        return 0;
+    }
+    let ffi_context = &mut *context;
+    let msg = if msg.is_null() {
+        None
+    } else {
+        let ffi_msg: &mut MessageWrapper = &mut *msg;
+        Some(&mut ffi_msg.message)
+    };
+    ffi_context
+        .with_inner(|ctx| {
+            chat::add_device_msg(
+                ctx,
+                to_opt_string_lossy(label).as_ref().map(|x| x.as_str()),
+                msg,
+            )
+            .unwrap_or_log_default(ctx, "Failed to add device message")
+        })
+        .map(|msg_id| msg_id.to_u32())
+        .unwrap_or(0)
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_was_device_msg_ever_added(
+    context: *mut dc_context_t,
+    label: *const libc::c_char,
+) -> libc::c_int {
+    if context.is_null() || label.is_null() {
+        eprintln!("ignoring careless call to dc_was_device_msg_ever_added()");
+        return 0;
+    }
+    let ffi_context = &mut *context;
+    ffi_context
+        .with_inner(|ctx| {
+            chat::was_device_msg_ever_added(ctx, &to_string_lossy(label)).unwrap_or(false)
+                as libc::c_int
+        })
+        .unwrap_or(0)
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_draft(context: *mut dc_context_t, chat_id: u32) -> *mut dc_msg_t {
     if context.is_null() {
@@ -2264,6 +2313,26 @@ pub unsafe extern "C" fn dc_chat_is_self_talk(chat: *mut dc_chat_t) -> libc::c_i
     ffi_chat.chat.is_self_talk() as libc::c_int
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_chat_is_device_talk(chat: *mut dc_chat_t) -> libc::c_int {
+    if chat.is_null() {
+        eprintln!("ignoring careless call to dc_chat_is_device_talk()");
+        return 0;
+    }
+    let ffi_chat = &*chat;
+    ffi_chat.chat.is_device_talk() as libc::c_int
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_chat_can_send(chat: *mut dc_chat_t) -> libc::c_int {
+    if chat.is_null() {
+        eprintln!("ignoring careless call to dc_chat_can_send()");
+        return 0;
+    }
+    let ffi_chat = &*chat;
+    ffi_chat.chat.can_send() as libc::c_int
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_chat_is_verified(chat: *mut dc_chat_t) -> libc::c_int {
     if chat.is_null() {

deltachat-ffi/src/providers.rs

@@ -2,7 +2,7 @@ extern crate deltachat_provider_database;
 
 use std::ptr;
 
-use deltachat::dc_tools::{as_str, StrExt};
+use deltachat::dc_tools::{to_string_lossy, StrExt};
 use deltachat_provider_database::StatusState;
 
 #[no_mangle]
@@ -12,7 +12,7 @@ pub type dc_provider_t = deltachat_provider_database::Provider;
 pub unsafe extern "C" fn dc_provider_new_from_domain(
     domain: *const libc::c_char,
 ) -> *const dc_provider_t {
-    match deltachat_provider_database::get_provider_info(as_str(domain)) {
+    match deltachat_provider_database::get_provider_info(&to_string_lossy(domain)) {
         Some(provider) => provider,
         None => ptr::null(),
     }
@@ -22,7 +22,8 @@ pub unsafe extern "C" fn dc_provider_new_from_domain(
 pub unsafe extern "C" fn dc_provider_new_from_email(
     email: *const libc::c_char,
 ) -> *const dc_provider_t {
-    let domain = deltachat_provider_database::get_domain_from_email(as_str(email));
+    let email = to_string_lossy(email);
+    let domain = deltachat_provider_database::get_domain_from_email(&email);
     match deltachat_provider_database::get_provider_info(domain) {
         Some(provider) => provider,
         None => ptr::null(),

examples/repl/cmdline.rs

@@ -192,7 +192,7 @@ unsafe fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
     let msgtext = msg.get_text();
     info!(
         context,
-        "{}#{}{}{}: {} (Contact#{}): {} {}{}{}{}{} [{}]",
+        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{} [{}]",
         prefix.as_ref(),
         msg.get_id(),
         if msg.get_showpadlock() { "🔒" } else { "" },
@@ -240,7 +240,7 @@ unsafe fn log_msglist(context: &Context, msglist: &Vec<MsgId>) -> Result<(), Err
                 lines_out += 1
             }
             let msg = Message::load_from_db(context, msg_id)?;
-            log_msg(context, "Msg", &msg);
+            log_msg(context, "", &msg);
         }
     }
     if lines_out > 0 {
@@ -353,6 +353,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
                  configure\n\
                  connect\n\
                  disconnect\n\
+                 interrupt\n\
                  maybenetwork\n\
                  housekeeping\n\
                  help imex (Import/Export)\n\
@@ -378,6 +379,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
                  sendimage <file> [<text>]\n\
                  sendfile <file> [<text>]\n\
                  draft [<text>]\n\
+                 devicemsg <text>\n\
                  listmedia\n\
                  archive <chat-id>\n\
                  unarchive <chat-id>\n\
@@ -424,12 +426,12 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
             if msg.is_setupmessage() {
                 let setupcodebegin = msg.get_setupcodebegin(context);
                 println!(
-                    "The setup code for setup message Msg#{} starts with: {}",
+                    "The setup code for setup message {} starts with: {}",
                     msg_id,
                     setupcodebegin.unwrap_or_default(),
                 );
             } else {
-                bail!("Msg#{} is no setup message.", msg_id,);
+                bail!("{} is no setup message.", msg_id,);
             }
         }
         "continue-key-transfer" => {
@@ -493,6 +495,9 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
         "info" => {
             println!("{:#?}", context.get_info());
         }
+        "interrupt" => {
+            interrupt_inbox_idle(context, true);
+        }
         "maybenetwork" => {
             maybe_network(context);
         }
@@ -517,15 +522,12 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
 
                 for i in (0..cnt).rev() {
                     let chat = Chat::load_from_db(context, chatlist.get_chat_id(i))?;
-                    let temp_subtitle = chat.get_subtitle(context);
-                    let temp_name = chat.get_name();
                     info!(
                         context,
-                        "{}#{}: {} [{}] [{} fresh]",
+                        "{}#{}: {} [{} fresh]",
                         chat_prefix(&chat),
                         chat.get_id(),
-                        temp_name,
-                        temp_subtitle,
+                        chat.get_name(),
                         chat::get_fresh_msg_cnt(context, chat.get_id()),
                     );
                     let lot = chatlist.get_summary(context, i, Some(&chat));
@@ -583,20 +585,34 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
             let sel_chat = sel_chat.as_ref().unwrap();
 
             let msglist = chat::get_chat_msgs(context, sel_chat.get_id(), 0x1, None);
-            let temp2 = sel_chat.get_subtitle(context);
-            let temp_name = sel_chat.get_name();
+            let members = chat::get_chat_contacts(context, sel_chat.id);
+            let subtitle = if sel_chat.is_device_talk() {
+                "device-talk".to_string()
+            } else if sel_chat.get_type() == Chattype::Single && members.len() >= 1 {
+                let contact = Contact::get_by_id(context, members[0])?;
+                contact.get_addr().to_string()
+            } else {
+                format!("{} member(s)", members.len())
+            };
             info!(
                 context,
-                "{}#{}: {} [{}]{}",
+                "{}#{}: {} [{}]{}{}",
                 chat_prefix(sel_chat),
                 sel_chat.get_id(),
-                temp_name,
-                temp2,
+                sel_chat.get_name(),
+                subtitle,
                 if sel_chat.is_sending_locations() {
                     "📍"
                 } else {
                     ""
                 },
+                match sel_chat.get_profile_image(context) {
+                    Some(icon) => match icon.to_str() {
+                        Some(icon) => format!(" Icon: {}", icon),
+                        _ => " Icon: Err".to_string(),
+                    },
+                    _ => "".to_string(),
+                },
             );
             log_msglist(context, &msglist)?;
             if let Some(draft) = chat::get_draft(context, sel_chat.get_id())? {
@@ -710,7 +726,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
                 let marker = location.marker.as_ref().unwrap_or(&default_marker);
                 info!(
                     context,
-                    "Loc#{}: {}: lat={} lng={} acc={} Chat#{} Contact#{} Msg#{} {}",
+                    "Loc#{}: {}: lat={} lng={} acc={} Chat#{} Contact#{} {} {}",
                     location.location_id,
                     dc_timestamp_to_str(location.timestamp),
                     location.latitude,
@@ -814,6 +830,15 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
                 println!("Draft deleted.");
             }
         }
+        "devicemsg" => {
+            ensure!(
+                !arg1.is_empty(),
+                "Please specify text to add as device message."
+            );
+            let mut msg = Message::new(Viewtype::Text);
+            msg.set_text(Some(arg1.to_string()));
+            chat::add_device_msg(context, None, Some(&mut msg))?;
+        }
         "listmedia" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
 
@@ -827,9 +852,9 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
             println!("{} images or videos: ", images.len());
             for (i, data) in images.iter().enumerate() {
                 if 0 == i {
-                    print!("Msg#{}", data);
+                    print!("{}", data);
                 } else {
-                    print!(", Msg#{}", data);
+                    print!(", {}", data);
                 }
             }
             print!("\n");

examples/repl/main.rs

@@ -150,11 +150,11 @@ fn start_threads(c: Arc<RwLock<Context>>) {
     let ctx = c.clone();
     let handle_imap = std::thread::spawn(move || loop {
         while_running!({
-            perform_imap_jobs(&ctx.read().unwrap());
-            perform_imap_fetch(&ctx.read().unwrap());
+            perform_inbox_jobs(&ctx.read().unwrap());
+            perform_inbox_fetch(&ctx.read().unwrap());
             while_running!({
                 let context = ctx.read().unwrap();
-                perform_imap_idle(&context);
+                perform_inbox_idle(&context);
             });
         });
     });
@@ -202,7 +202,7 @@ fn stop_threads(context: &Context) {
         println!("Stopping threads");
         IS_RUNNING.store(false, Ordering::Relaxed);
 
-        interrupt_imap_idle(context);
+        interrupt_inbox_idle(context, true);
         interrupt_mvbox_idle(context);
         interrupt_sentbox_idle(context);
         interrupt_smtp_idle(context);
@@ -455,9 +455,9 @@ unsafe fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult
         }
         "imap-jobs" => {
             if HANDLE.clone().lock().unwrap().is_some() {
-                println!("imap-jobs are already running in a thread.");
+                println!("inbox-jobs are already running in a thread.");
             } else {
-                perform_imap_jobs(&ctx.read().unwrap());
+                perform_inbox_jobs(&ctx.read().unwrap());
             }
         }
         "configure" => {

examples/simple.rs

@@ -11,7 +11,8 @@ use deltachat::configure::*;
 use deltachat::contact::*;
 use deltachat::context::*;
 use deltachat::job::{
-    perform_imap_fetch, perform_imap_idle, perform_imap_jobs, perform_smtp_idle, perform_smtp_jobs,
+    perform_inbox_fetch, perform_inbox_idle, perform_inbox_jobs, perform_smtp_idle,
+    perform_smtp_jobs,
 };
 use deltachat::Event;
 
@@ -50,12 +51,12 @@ fn main() {
     let r1 = running.clone();
     let t1 = thread::spawn(move || {
         while *r1.read().unwrap() {
-            perform_imap_jobs(&ctx1);
+            perform_inbox_jobs(&ctx1);
             if *r1.read().unwrap() {
-                perform_imap_fetch(&ctx1);
+                perform_inbox_fetch(&ctx1);
 
                 if *r1.read().unwrap() {
-                    perform_imap_idle(&ctx1);
+                    perform_inbox_idle(&ctx1);
                 }
             }
         }
@@ -113,7 +114,7 @@ fn main() {
     println!("stopping threads");
 
     *running.clone().write().unwrap() = false;
-    deltachat::job::interrupt_imap_idle(&ctx);
+    deltachat::job::interrupt_inbox_idle(&ctx, true);
     deltachat::job::interrupt_smtp_idle(&ctx);
 
     println!("joining");

mmime/Cargo.toml

@@ -20,4 +20,4 @@ memmap = "0.7.0"
 lazy_static = "1.3.0"
 rand = "0.6.5"
 chrono = "0.4.6"
-hex = "0.3.2"
+hex = "0.4.0"

proptest-regressions/dc_mimeparser.txt

@@ -0,0 +1,7 @@
+# Seeds for failure cases proptest has generated in the past. It is
+# automatically read and these particular cases re-run before any
+# novel cases are generated.
+#
+# It is recommended to check this file in to source control so that
+# everyone who runs the test benefits from these saved cases.
+cc 03cab93c6d1f3a8245f63cf84dacb307944294fe6333c1e38f078a6600659c7a # shrinks to data = "a\t0aA\ta\t0 \ta\t0 \ta a\t\ta  A\tAA0a0a 0\t a\t aA   \t a\t  A0\t  AAa\taA0\taAAaA\t0\taa0a\ta Aa Aaaa A0A\t  a aA 0\t  A\t0\t0\t\t\t\t\t\tA \t\t a\tA Aa aAA0A0AA0aaA A\t\t  aa0\ta\t  \tAa\taA\t00  AA  A a\tA a aAAa \t 00\t0 \t\t  a A  0\t\t\t aAA Aa \taAAAA0a A0A\t\t1\\E$\t$R\tc\t^=\t\"\tQ<Uk\t\t>A\t\t&\t}v&\tM^\'|\tW5?dn\t\t+\t\tP\te`\t\t>:Brlq--?\t$#Q\tK=zJ\tb\"9*.\"\t`\tF&T*\tBs,\tg\'*\\\t:\t?l$\t\t|A\"HR:Hk\t\\KkV\t\t{=&!^e%:|_*0wV\t[$`\t:\t$%f\t\t[!\"Y. \tP\t\th\'?\'/?%:++NfQo#@\"+?\t(\\??{\t\'\'$Dzj\t0.?{s4?&?Y=/yj]Z=\t4n\t?Ja\"\t{I\t$\t;I}_8V\t&\t?N\'\tI2/\t9.\tFT%9%`\'\tz\to7Y\t|AXP&@G12g\t\'w\t\t%??\t\"h$?F\"\"6%q\\\\{\tT\t\"]$87$}h\'\t<\t$\tc%U:mT2:<v\t#Rl!;U\t\t\"^D\tRZ(BZ{n\t%[$fgH`\t{B}:*\t\t%%*F`%W\t//B}PQ\t\tsu2\tLz<1*!p-X\tnKv&&0\thm4<n\\.\\/.w\'\t<)E1g3#ood\t`?\t\\({N?,8_?h\ty\t0%\t*$A\t\t*w-ViQUj\tTiH\t%\t%&0p\'\'\tA%r**Fo\'Z\\\tNI*ely4=I?}$.o`\t$\ts\'&lt\t\",:~=Nv/0O%=+<LS\t%P\'\t$r%d.\t{G{/L:\t&I&8-`cy*\"{\t/%fP9.P<\t\t\'/`\t\t`\t\t`!t:\t::\t\tW\'X^\t@.uL&a\tN\t\t\t.\t?0*\tvUK>UJ\\\tQbj%w]P=Js\t\"R\t&s^?</=%\t\'VI:\" kT`{b*<\t\tF&?Y\t\t:/\t!C\'e0`\t\t\tx-\t*\\N\\wwu.I4\tb/y\t\"P:P%\"\t\tQ][\\\t^x\t\t):\t\t&s\t$1-\t\t\tXt`0\t;\t/UeCP*\"G\t\t\':\tk@91Hf&#\t(Uzz&`p2;\t{]\t\"I_%=\\%,XD\"\'06QfCd<;$:q^\t8:\"*\"H\t\to\t&xK/\t\ty0/<}<j<|*`~::b\t=S.=}=Ki\t<Y.\'{\tf\t{Ub*H%(_4*?*\tn2\t/\'\t\t\t/,4]\tt\t<y\t\t\tWi\t\tT&\"\t\t\t\t\t=/1Wu?\t\'A\"W-P\t$?e\\\t`\t6`vD\t8`\t\tccD J\tY&jI//?_\t\\j\t_\tsiq$\t?9\tQ\t.^%&..?%Jm??A%+<\tN&*\t.g\tS$W\"\"\tMpq\t\t:&\\\thTz&<iz%/%T&\'F\t\\]&\t\t}\t\t\tXr=dtp`:+<,\t%60Y*<(&K*kJ\todO\t=<V?&\tMU/\"\t= Y!)<\tV\t9\t)\t&v8q{\t\t&pT\t3\ttB,qcq\'i$;gv%j_%M_{[\"&;\t\t\t.B;6Y\\%\t\"\tY/a\t\\`:?\t<\t?]\taNwD;\\\t%l*74%5T?QS :~yR/E*R\t\t=u\t\\\t\t.Q<;\\\t_S/{&F$^\tw_>\'h=|%\t\t:W!\\<U2\'$\tb&`\t=|=L9\t\t\t\\WZ:! }{\t ;\t;\t\t 0.*\t.%\"\'r%{<Mh_t[[}\t-\tJo\"b/AC*-=`=T\tz$2\tC\t\t/k{47\"\t\t,q%\tZ\tT3\t\tf>%\t\'?%@^tx\t7\"1Bk{b{n\t\"Pj3\tHc\t\tt\tY<\t#?\tSh\\yk/N\\\t8 7=R4*9Cw&m\t\\-\'f\t|\'#t(Etu.Hdu(E\t%&v:\'aqW~A5\t\t w.s{J%lX<\"\t\'*%m<&:/B<&\':U}$&`.{)\t\t6S\t:/$*kQ-Z\t^\'t${/tH*\'v\t3\t=\t\tDyp:B\t`I_R&4SO\t\t&-j=*.\t87&\'e*( \t\t\t\'<$\\DJ<$p?{}\'&\tv\t\\Xk<Y:Y`!$K{\tF&\tzd\t\t*i$\tj\'\t<)R*\t%?\t!.\t=\"@#~:=*\t\tXO=_T,1\"\'.%%\"`{\\:\t\"\tfkeOb/\'$I~\ta\t|&\t[\\KK\"1&Z\t<k\t\t)%\'-~\"2n\tj\tW?*<@w{g%d\ta\\\'\'I\t;:ySR%ke:4\tc\t$=\t&9P]x4\tJ=\t6C6%a\t`0\tF\tm-\tTr\t}\t\tQum\t&@\typ|w2&\t\t3`i&t\t\tT5\"\t.&b&e*/==1.\'*\\[U*\tqPL%?$-0/}~|q`\t\t}\t$\tq==o+T$\'!H\t\ti&um\"?\"%%\t/\'p\tg>?{0{J{\t\t/\t\t{zKZ&>=\t[\"1h<H%z/8,/]s\tv{7\t\t:j*H,M//\t\t\td\'.)\t"

python/doc/api.rst

@@ -8,8 +8,8 @@ high level API reference
 
 - :class:`deltachat.account.Account` (your main entry point, creates the
   other classes)
-- :class:`deltachat.chatting.Contact`
-- :class:`deltachat.chatting.Chat`
+- :class:`deltachat.contact.Contact`
+- :class:`deltachat.chat.Chat`
 - :class:`deltachat.message.Message`
 
 Account
@@ -22,13 +22,13 @@ Account
 Contact
 -------
 
-.. autoclass:: deltachat.chatting.Contact
+.. autoclass:: deltachat.contact.Contact
     :members:
 
 Chat
 ----
 
-.. autoclass:: deltachat.chatting.Chat
+.. autoclass:: deltachat.chat.Chat
     :members:
 
 Message

python/install_python_bindings.py

@@ -9,17 +9,23 @@ import subprocess
 import sys
 
 if __name__ == "__main__":
-    os.environ["DCC_RS_TARGET"] = target = "release"
+    target = os.environ.get("DCC_RS_TARGET")
+    if target is None:
+        os.environ["DCC_RS_TARGET"] = target = "release"
     if "DCC_RS_DEV" not in os.environ:
         dn = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
         os.environ["DCC_RS_DEV"] = dn
 
+    # build the core library  in release + debug mode because
+    # as of Nov 2019 rPGP generates RSA keys which take
+    # prohibitively long for non-release installs
     os.environ["RUSTFLAGS"] = "-g"
     subprocess.check_call([
         "cargo", "build", "-p", "deltachat_ffi", "--" + target
     ])
     subprocess.check_call("rm -rf build/ src/deltachat/*.so" , shell=True)
 
-    subprocess.check_call([
-        sys.executable, "-m", "pip", "install", "-e", "."
-    ])
+    if len(sys.argv) <= 1 or sys.argv[1] != "onlybuild":
+        subprocess.check_call([
+            sys.executable, "-m", "pip", "install", "-e", "."
+        ])

python/src/deltachat/_build.py

@@ -29,7 +29,10 @@ def ffibuilder():
             extra_link_args = []
         else:
             raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
-        objs = [os.path.join(projdir, 'target', target, 'libdeltachat.a')]
+        target_dir = os.environ.get("CARGO_TARGET_DIR")
+        if target_dir is None:
+            target_dir = os.path.join(projdir, 'target')
+        objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
         assert os.path.exists(objs[0]), objs
         incs = [os.path.join(projdir, 'deltachat-ffi')]
     else:

python/src/deltachat/account.py

@@ -15,12 +15,14 @@ import deltachat
 from . import const
 from .capi import ffi, lib
 from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array, DCLot
-from .chatting import Contact, Chat, Message
+from .chat import Chat
+from .message import Message
+from .contact import Contact
 
 
 class Account(object):
     """ Each account is tied to a sqlite database file which is fully managed
-    by the underlying deltachat c-library.  All public Account methods are
+    by the underlying deltachat core library.  All public Account methods are
     meant to be memory-safe and return memory-safe objects.
     """
     def __init__(self, db_path, logid=None, eventlogging=True, debug=True):
@@ -151,6 +153,14 @@ class Account(object):
         self.check_is_configured()
         return from_dc_charpointer(lib.dc_get_info(self._dc_context))
 
+    def get_latest_backupfile(self, backupdir):
+        """ return the latest backup file in a given directory.
+        """
+        res = lib.dc_imex_has_backup(self._dc_context, as_dc_charpointer(backupdir))
+        if res == ffi.NULL:
+            return None
+        return from_dc_charpointer(res)
+
     def get_blobdir(self):
         """ return the directory for files.
 
@@ -160,9 +170,9 @@ class Account(object):
         return from_dc_charpointer(lib.dc_get_blobdir(self._dc_context))
 
     def get_self_contact(self):
-        """ return this account's identity as a :class:`deltachat.chatting.Contact`.
+        """ return this account's identity as a :class:`deltachat.contact.Contact`.
 
-        :returns: :class:`deltachat.chatting.Contact`
+        :returns: :class:`deltachat.contact.Contact`
         """
         self.check_is_configured()
         return Contact(self._dc_context, const.DC_CONTACT_ID_SELF)
@@ -174,7 +184,7 @@ class Account(object):
 
         :param email: email-address (text type)
         :param name: display name for this contact (optional)
-        :returns: :class:`deltachat.chatting.Contact` instance.
+        :returns: :class:`deltachat.contact.Contact` instance.
         """
         name = as_dc_charpointer(name)
         email = as_dc_charpointer(email)
@@ -200,7 +210,7 @@ class Account(object):
                       whose name or e-mail matches query.
         :param only_verified: if true only return verified contacts.
         :param with_self: if true the self-contact is also returned.
-        :returns: list of :class:`deltachat.chatting.Contact` objects.
+        :returns: list of :class:`deltachat.contact.Contact` objects.
         """
         flags = 0
         query = as_dc_charpointer(query)
@@ -218,7 +228,7 @@ class Account(object):
         """ create or get an existing 1:1 chat object for the specified contact or contact id.
 
         :param contact: chat_id (int) or contact object.
-        :returns: a :class:`deltachat.chatting.Chat` object.
+        :returns: a :class:`deltachat.chat.Chat` object.
         """
         if hasattr(contact, "id"):
             if contact._dc_context != self._dc_context:
@@ -235,7 +245,7 @@ class Account(object):
         the specified message.
 
         :param message: messsage id or message instance.
-        :returns: a :class:`deltachat.chatting.Chat` object.
+        :returns: a :class:`deltachat.chat.Chat` object.
         """
         if hasattr(message, "id"):
             if self._dc_context != message._dc_context:
@@ -253,7 +263,7 @@ class Account(object):
         Chats are unpromoted until the first message is sent.
 
         :param verified: if true only verified contacts can be added.
-        :returns: a :class:`deltachat.chatting.Chat` object.
+        :returns: a :class:`deltachat.chat.Chat` object.
         """
         bytes_name = name.encode("utf8")
         chat_id = lib.dc_create_group_chat(self._dc_context, int(verified), bytes_name)
@@ -262,7 +272,7 @@ class Account(object):
     def get_chats(self):
         """ return list of chats.
 
-        :returns: a list of :class:`deltachat.chatting.Chat` objects.
+        :returns: a list of :class:`deltachat.chat.Chat` objects.
         """
         dc_chatlist = ffi.gc(
             lib.dc_get_chatlist(self._dc_context, 0, ffi.NULL, 0),
@@ -280,9 +290,24 @@ class Account(object):
         return Chat(self, const.DC_CHAT_ID_DEADDROP)
 
     def get_message_by_id(self, msg_id):
-        """ return Message instance. """
+        """ return Message instance.
+        :param msg_id: integer id of this message.
+        :returns: :class:`deltachat.message.Message` instance.
+        """
         return Message.from_db(self, msg_id)
 
+    def get_chat_by_id(self, chat_id):
+        """ return Chat instance.
+        :param chat_id: integer id of this chat.
+        :returns: :class:`deltachat.chat.Chat` instance.
+        :raises: ValueError if chat does not exist.
+        """
+        res = lib.dc_get_chat(self._dc_context, chat_id)
+        if res == ffi.NULL:
+            raise ValueError("cannot get chat with id={}".format(chat_id))
+        lib.dc_chat_unref(res)
+        return Chat(self, chat_id)
+
     def mark_seen_messages(self, messages):
         """ mark the given set of messages as seen.
 
@@ -299,7 +324,7 @@ class Account(object):
         """ Forward list of messages to a chat.
 
         :param messages: list of :class:`deltachat.message.Message` object.
-        :param chat: :class:`deltachat.chatting.Chat` object.
+        :param chat: :class:`deltachat.chat.Chat` object.
         :returns: None
         """
         msg_ids = [msg.id for msg in messages]
@@ -411,7 +436,7 @@ class Account(object):
         """ setup contact and return a Chat after contact is established.
 
         Note that this function may block for a long time as messages are exchanged
-        with the emitter of the QR code.  On success a :class:`deltachat.chatting.Chat` instance
+        with the emitter of the QR code.  On success a :class:`deltachat.chat.Chat` instance
         is returned.
         :param qr: valid "setup contact" QR code (all other QR codes will result in an exception)
         """
@@ -425,7 +450,7 @@ class Account(object):
         """ join a chat group through a QR code.
 
         Note that this function may block for a long time as messages are exchanged
-        with the emitter of the QR code.  On success a :class:`deltachat.chatting.Chat` instance
+        with the emitter of the QR code.  On success a :class:`deltachat.chat.Chat` instance
         is returned which is the chat that we just joined.
 
         :param qr: valid "join-group" QR code (all other QR codes will result in an exception)
@@ -460,8 +485,9 @@ class Account(object):
 
     def stop_threads(self, wait=True):
         """ stop IMAP/SMTP threads. """
-        self.stop_ongoing()
-        self._threads.stop(wait=wait)
+        if self._threads.is_started():
+            self.stop_ongoing()
+            self._threads.stop(wait=wait)
 
     def shutdown(self, wait=True):
         """ stop threads and close and remove underlying dc_context and callbacks. """
@@ -492,6 +518,20 @@ class Account(object):
     def on_dc_event_imex_file_written(self, data1, data2):
         self._imex_events.put(data1)
 
+    def set_location(self, latitude=0.0, longitude=0.0, accuracy=0.0):
+        """set a new location. It effects all chats where we currently
+        have enabled location streaming.
+
+        :param latitude: float (use 0.0 if not known)
+        :param longitude: float (use 0.0 if not known)
+        :param accuracy: float (use 0.0 if not known)
+        :raises: ValueError if no chat is currently streaming locations
+        :returns: None
+        """
+        dc_res = lib.dc_set_location(self._dc_context, latitude, longitude, accuracy)
+        if dc_res == 0:
+            raise ValueError("no chat is streaming locations")
+
 
 class IOThreads:
     def __init__(self, dc_context, log_event=lambda *args: None):
@@ -533,8 +573,10 @@ class IOThreads:
         self._log_event("py-bindings-info", 0, "INBOX THREAD START")
         while not self._thread_quitflag:
             lib.dc_perform_imap_jobs(self._dc_context)
-            lib.dc_perform_imap_fetch(self._dc_context)
-            lib.dc_perform_imap_idle(self._dc_context)
+            if not self._thread_quitflag:
+                lib.dc_perform_imap_fetch(self._dc_context)
+            if not self._thread_quitflag:
+                lib.dc_perform_imap_idle(self._dc_context)
         self._log_event("py-bindings-info", 0, "INBOX THREAD FINISHED")
 
     def mvbox_thread_run(self):

python/src/deltachat/chat.py

@@ -1,58 +1,15 @@
-""" chatting related objects: Contact, Chat, Message. """
+""" Chat and Location related API. """
 
 import mimetypes
+import calendar
+from datetime import datetime
 import os
-from . import props
 from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array
 from .capi import lib, ffi
 from . import const
 from .message import Message
 
 
-class Contact(object):
-    """ Delta-Chat Contact.
-
-    You obtain instances of it through :class:`deltachat.account.Account`.
-    """
-    def __init__(self, dc_context, id):
-        self._dc_context = dc_context
-        self.id = id
-
-    def __eq__(self, other):
-        return self._dc_context == other._dc_context and self.id == other.id
-
-    def __ne__(self, other):
-        return not (self == other)
-
-    def __repr__(self):
-        return "<Contact id={} addr={} dc_context={}>".format(self.id, self.addr, self._dc_context)
-
-    @property
-    def _dc_contact(self):
-        return ffi.gc(
-            lib.dc_get_contact(self._dc_context, self.id),
-            lib.dc_contact_unref
-        )
-
-    @props.with_doc
-    def addr(self):
-        """ normalized e-mail address for this account. """
-        return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
-
-    @props.with_doc
-    def display_name(self):
-        """ display name for this contact. """
-        return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
-
-    def is_blocked(self):
-        """ Return True if the contact is blocked. """
-        return lib.dc_contact_is_blocked(self._dc_contact)
-
-    def is_verified(self):
-        """ Return True if the contact is verified. """
-        return lib.dc_contact_is_verified(self._dc_contact)
-
-
 class Chat(object):
     """ Chat object which manages members and through which you can send and retrieve messages.
 
@@ -312,9 +269,10 @@ class Chat(object):
     def get_contacts(self):
         """ get all contacts for this chat.
         :params: contact object.
-        :returns: list of :class:`deltachat.chatting.Contact` objects for this chat
+        :returns: list of :class:`deltachat.contact.Contact` objects for this chat
 
         """
+        from .contact import Contact
         dc_array = ffi.gc(
             lib.dc_get_chat_contacts(self._dc_context, self.id),
             lib.dc_array_unref
@@ -365,3 +323,62 @@ class Chat(object):
         if dc_res == ffi.NULL:
             return None
         return from_dc_charpointer(dc_res)
+
+    # ------  location streaming API ------------------------------
+
+    def is_sending_locations(self):
+        """return True if this chat has location-sending enabled currently.
+        :returns: True if location sending is enabled.
+        """
+        return lib.dc_is_sending_locations_to_chat(self._dc_context, self.id)
+
+    def enable_sending_locations(self, seconds):
+        """enable sending locations for this chat.
+
+        all subsequent messages will carry a location with them.
+        """
+        lib.dc_send_locations_to_chat(self._dc_context, self.id, seconds)
+
+    def get_locations(self, contact=None, timestamp_from=None, timestamp_to=None):
+        """return list of locations for the given contact in the given timespan.
+
+        :param contact: the contact for which locations shall be returned.
+        :param timespan_from: a datetime object or None (indicating "since beginning")
+        :param timespan_to: a datetime object or None (indicating up till now)
+        :returns: list of :class:`deltachat.chat.Location` objects.
+        """
+        if timestamp_from is None:
+            time_from = 0
+        else:
+            time_from = calendar.timegm(timestamp_from.utctimetuple())
+        if timestamp_to is None:
+            time_to = 0
+        else:
+            time_to = calendar.timegm(timestamp_to.utctimetuple())
+
+        if contact is None:
+            contact_id = 0
+        else:
+            contact_id = contact.id
+
+        dc_array = lib.dc_get_locations(self._dc_context, self.id, contact_id, time_from, time_to)
+        return [
+            Location(
+                latitude=lib.dc_array_get_latitude(dc_array, i),
+                longitude=lib.dc_array_get_longitude(dc_array, i),
+                accuracy=lib.dc_array_get_accuracy(dc_array, i),
+                timestamp=datetime.utcfromtimestamp(lib.dc_array_get_timestamp(dc_array, i)))
+            for i in range(lib.dc_array_get_cnt(dc_array))
+        ]
+
+
+class Location:
+    def __init__(self, latitude, longitude, accuracy, timestamp):
+        assert isinstance(timestamp, datetime)
+        self.latitude = latitude
+        self.longitude = longitude
+        self.accuracy = accuracy
+        self.timestamp = timestamp
+
+    def __eq__(self, other):
+        return self.__dict__ == other.__dict__

python/src/deltachat/const.py

@@ -47,7 +47,8 @@ DC_STATE_OUT_FAILED = 24
 DC_STATE_OUT_DELIVERED = 26
 DC_STATE_OUT_MDN_RCVD = 28
 DC_CONTACT_ID_SELF = 1
-DC_CONTACT_ID_DEVICE = 2
+DC_CONTACT_ID_INFO = 2
+DC_CONTACT_ID_DEVICE = 5
 DC_CONTACT_ID_LAST_SPECIAL = 9
 DC_MSG_TEXT = 10
 DC_MSG_IMAGE = 20

python/src/deltachat/contact.py

@@ -0,0 +1,49 @@
+""" Contact object. """
+
+from . import props
+from .cutil import from_dc_charpointer
+from .capi import lib, ffi
+
+
+class Contact(object):
+    """ Delta-Chat Contact.
+
+    You obtain instances of it through :class:`deltachat.account.Account`.
+    """
+    def __init__(self, dc_context, id):
+        self._dc_context = dc_context
+        self.id = id
+
+    def __eq__(self, other):
+        return self._dc_context == other._dc_context and self.id == other.id
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __repr__(self):
+        return "<Contact id={} addr={} dc_context={}>".format(self.id, self.addr, self._dc_context)
+
+    @property
+    def _dc_contact(self):
+        return ffi.gc(
+            lib.dc_get_contact(self._dc_context, self.id),
+            lib.dc_contact_unref
+        )
+
+    @props.with_doc
+    def addr(self):
+        """ normalized e-mail address for this account. """
+        return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
+
+    @props.with_doc
+    def display_name(self):
+        """ display name for this contact. """
+        return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
+
+    def is_blocked(self):
+        """ Return True if the contact is blocked. """
+        return lib.dc_contact_is_blocked(self._dc_contact)
+
+    def is_verified(self):
+        """ Return True if the contact is verified. """
+        return lib.dc_contact_is_verified(self._dc_contact)

python/src/deltachat/message.py

@@ -1,4 +1,4 @@
-""" chatting related objects: Contact, Chat, Message. """
+""" The Message object. """
 
 import os
 import shutil
@@ -13,7 +13,7 @@ class Message(object):
     """ Message object.
 
     You obtain instances of it through :class:`deltachat.account.Account` or
-    :class:`deltachat.chatting.Chat`.
+    :class:`deltachat.chat.Chat`.
     """
     def __init__(self, account, dc_msg):
         self.account = account
@@ -169,18 +169,18 @@ class Message(object):
     def chat(self):
         """chat this message was posted in.
 
-        :returns: :class:`deltachat.chatting.Chat` object
+        :returns: :class:`deltachat.chat.Chat` object
         """
-        from .chatting import Chat
+        from .chat import Chat
         chat_id = lib.dc_msg_get_chat_id(self._dc_msg)
         return Chat(self.account, chat_id)
 
     def get_sender_contact(self):
         """return the contact of who wrote the message.
 
-        :returns: :class:`deltachat.chatting.Contact` instance
+        :returns: :class:`deltachat.chat.Contact` instance
         """
-        from .chatting import Contact
+        from .contact import Contact
         contact_id = lib.dc_msg_get_from_id(self._dc_msg)
         return Contact(self._dc_context, contact_id)
 

python/tests/test_account.py

@@ -2,6 +2,7 @@ from __future__ import print_function
 import pytest
 import os
 import queue
+import time
 from deltachat import const, Account
 from deltachat.message import Message
 from datetime import datetime, timedelta
@@ -20,7 +21,7 @@ class TestOfflineAccountBasic:
         d = ac1.get_info()
         assert d["arch"]
         assert d["number_of_chats"] == "0"
-        assert d["bcc_self"] == "1"
+        assert d["bcc_self"] == "0"
 
     def test_is_not_configured(self, acfactory):
         ac1 = acfactory.get_unconfigured_account()
@@ -42,7 +43,7 @@ class TestOfflineAccountBasic:
     def test_has_bccself(self, acfactory):
         ac1 = acfactory.get_unconfigured_account()
         assert "bcc_self" in ac1.get_config("sys.config_keys").split()
-        assert ac1.get_config("bcc_self") == "1"
+        assert ac1.get_config("bcc_self") == "0"
 
     def test_selfcontact_if_unconfigured(self, acfactory):
         ac1 = acfactory.get_unconfigured_account()
@@ -121,6 +122,12 @@ class TestOfflineChat:
         str(chat1)
         repr(chat1)
 
+    def test_chat_by_id(self, chat1):
+        chat2 = chat1.account.get_chat_by_id(chat1.id)
+        assert chat2 == chat1
+        with pytest.raises(ValueError):
+            chat1.account.get_chat_by_id(123123)
+
     def test_chat_idempotent(self, chat1, ac1):
         contact1 = chat1.get_contacts()[0]
         chat2 = ac1.create_chat_by_contact(contact1.id)
@@ -398,6 +405,9 @@ class TestOnlineAccount:
         wait_successful_IMAP_SMTP_connection(ac1)
         wait_configuration_progress(ac1, 1000)
 
+        lp.sec("ac1: setting bcc_self=1")
+        ac1.set_config("bcc_self", "1")
+
         lp.sec("send out message with bcc to ourselves")
         msg_out = chat.send_text("message2")
         ev = ac1._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
@@ -420,15 +430,25 @@ class TestOnlineAccount:
         assert self_addr not in ev[2]
         ev = ac1._evlogger.get_matching("DC_EVENT_DELETED_BLOB_FILE")
 
-    def test_mvbox_sentbox_threads(self, acfactory):
+    def test_mvbox_sentbox_threads(self, acfactory, lp):
+        lp.sec("ac1: start with mvbox thread")
         ac1 = acfactory.get_online_configuring_account(mvbox=True, sentbox=True)
+
+        lp.sec("ac2: start without mvbox/sentbox threads")
         ac2 = acfactory.get_online_configuring_account()
+
+        lp.sec("ac2: waiting for configuration")
         wait_configuration_progress(ac2, 1000)
+
+        lp.sec("ac1: waiting for configuration")
         wait_configuration_progress(ac1, 1000)
+
+        lp.sec("ac1: send message and wait for ac2 to receive it")
         chat = self.get_chat(ac1, ac2)
         chat.send_text("message1")
         ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
         assert ev[2] > const.DC_CHAT_ID_LAST_SPECIAL
+        lp.sec("test finished")
 
     def test_move_works(self, acfactory):
         ac1 = acfactory.get_online_configuring_account()
@@ -635,18 +655,29 @@ class TestOnlineAccount:
         assert os.path.exists(msg_in.filename)
         assert os.stat(msg_in.filename).st_size == os.stat(path).st_size
 
-    def test_import_export_online_all(self, acfactory, tmpdir):
+    def test_import_export_online_all(self, acfactory, tmpdir, lp):
         ac1 = acfactory.get_online_configuring_account()
         wait_configuration_progress(ac1, 1000)
 
+        lp.sec("create some chat content")
         contact1 = ac1.create_contact("some1@hello.com", name="some1")
         chat = ac1.create_chat_by_contact(contact1)
         chat.send_text("msg1")
         backupdir = tmpdir.mkdir("backup")
+
+        lp.sec("export all to {}".format(backupdir))
         path = ac1.export_all(backupdir.strpath)
         assert os.path.exists(path)
+        t = time.time()
 
+        lp.sec("get fresh empty account")
         ac2 = acfactory.get_unconfigured_account()
+
+        lp.sec("get latest backup file")
+        path2 = ac2.get_latest_backupfile(backupdir.strpath)
+        assert path2 == path
+
+        lp.sec("import backup and check it's proper")
         ac2.import_all(path)
         contacts = ac2.get_contacts(query="some1")
         assert len(contacts) == 1
@@ -657,6 +688,17 @@ class TestOnlineAccount:
         assert len(messages) == 1
         assert messages[0].text == "msg1"
 
+        # wait until a second passed since last backup
+        # because get_latest_backupfile() shall return the latest backup
+        # from a UI it's unlikely anyone manages to export two
+        # backups in one second.
+        time.sleep(max(0, 1 - (time.time() - t)))
+        lp.sec("Second-time export all to {}".format(backupdir))
+        path2 = ac1.export_all(backupdir.strpath)
+        assert os.path.exists(path2)
+        assert path2 != path
+        assert ac2.get_latest_backupfile(backupdir.strpath) == path2
+
     def test_ac_setup_message(self, acfactory, lp):
         # note that the receiving account needs to be configured and running
         # before ther setup message is send. DC does not read old messages
@@ -688,6 +730,7 @@ class TestOnlineAccount:
         ac2._evlogger.set_timeout(30)
         wait_configuration_progress(ac2, 1000)
         wait_configuration_progress(ac1, 1000)
+
         lp.sec("trigger ac setup message but ignore")
         assert ac1.get_info()["fingerprint"] != ac2.get_info()["fingerprint"]
         ac1.initiate_key_transfer()
@@ -699,6 +742,7 @@ class TestOnlineAccount:
         msg = ac2.get_message_by_id(ev[2])
         assert msg.is_setup_message()
         assert msg.get_setupcodebegin() == setup_code2[:2]
+
         lp.sec("process second setup message")
         msg.continue_key_transfer(setup_code2)
         assert ac1.get_info()["fingerprint"] == ac2.get_info()["fingerprint"]
@@ -810,6 +854,54 @@ class TestOnlineAccount:
         assert chat1b.get_profile_image() is None
         assert chat.get_profile_image() is None
 
+    def test_send_receive_locations(self, acfactory, lp):
+        now = datetime.utcnow()
+        ac1, ac2 = acfactory.get_two_online_accounts()
+
+        lp.sec("ac1: create chat with ac2")
+        chat1 = self.get_chat(ac1, ac2)
+        chat2 = self.get_chat(ac2, ac1)
+
+        assert not chat1.is_sending_locations()
+        with pytest.raises(ValueError):
+            ac1.set_location(latitude=0.0, longitude=10.0)
+
+        ac1._evlogger.consume_events()
+        ac2._evlogger.consume_events()
+
+        lp.sec("ac1: enable location sending in chat")
+        chat1.enable_sending_locations(seconds=100)
+        assert chat1.is_sending_locations()
+        ac1._evlogger.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
+
+        ac1.set_location(latitude=2.0, longitude=3.0, accuracy=0.5)
+        ac1._evlogger.get_matching("DC_EVENT_LOCATION_CHANGED")
+        chat1.send_text("hello")
+        ac1._evlogger.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
+
+        lp.sec("ac2: wait for incoming location message")
+        ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG")  # "enabled-location streaming"
+
+        # currently core emits location changed before event_incoming message
+        ac2._evlogger.get_matching("DC_EVENT_LOCATION_CHANGED")
+        ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG")  # text message with location
+
+        locations = chat2.get_locations()
+        assert len(locations) == 1
+        assert locations[0].latitude == 2.0
+        assert locations[0].longitude == 3.0
+        assert locations[0].accuracy == 0.5
+        assert locations[0].timestamp > now
+
+        contact = ac2.create_contact(ac1.get_config("addr"))
+        locations2 = chat2.get_locations(contact=contact)
+        assert len(locations2) == 1
+        assert locations2 == locations
+
+        contact = ac2.create_contact("nonexisting@example.org")
+        locations3 = chat2.get_locations(contact=contact)
+        assert not locations3
+
 
 class TestOnlineConfigureFails:
     def test_invalid_password(self, acfactory):
@@ -818,7 +910,7 @@ class TestOnlineConfigureFails:
         ac1.start_threads()
         wait_configuration_progress(ac1, 500)
         ev1 = ac1._evlogger.get_matching("DC_EVENT_ERROR_NETWORK")
-        assert "authentication failed" in ev1[2].lower()
+        assert "cannot login" in ev1[2].lower()
         wait_configuration_progress(ac1, 0, 0)
 
     def test_invalid_user(self, acfactory):
@@ -827,7 +919,7 @@ class TestOnlineConfigureFails:
         ac1.start_threads()
         wait_configuration_progress(ac1, 500)
         ev1 = ac1._evlogger.get_matching("DC_EVENT_ERROR_NETWORK")
-        assert "authentication failed" in ev1[2].lower()
+        assert "cannot login" in ev1[2].lower()
         wait_configuration_progress(ac1, 0, 0)
 
     def test_invalid_domain(self, acfactory):

python/tests/test_lowlevel.py

@@ -41,7 +41,7 @@ def test_dc_close_events(tmpdir):
             else:
                 print("skipping event", *ev)
 
-    find("disconnecting INBOX-watch")
+    find("disconnecting inbox-thread")
     find("disconnecting sentbox-thread")
     find("disconnecting mvbox-thread")
     find("disconnecting SMTP")

python/tox.ini

@@ -1,19 +1,25 @@
 [tox]
 # make sure to update environment list in travis.yml and appveyor.yml
 envlist =
-    py35
+    py37
     lint
     auditwheels
 
 [testenv]
 commands = 
-    pytest -v -rsXx {posargs:tests}
-    python tests/package_wheels.py {toxworkdir}/wheelhouse
+    # (some qr tests are pretty heavy in terms of send/received
+    # messages and async-imap's likely has concurrency problems, 
+    # eg https://github.com/async-email/async-imap/issues/4 )
+    pytest -n6 --reruns 3 --reruns-delay 5 -v -rsXx -k "not qr" {posargs:tests} 
+    pytest -n6 --reruns 5 --reruns-delay 5 -v -rsXx -k "qr" {posargs:tests} 
+    # python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 
     DCC_RS_DEV
     DCC_RS_TARGET
     DCC_PY_LIVECONFIG
+    CARGO_TARGET_DIR
+    RUSTC_WRAPPER
 deps = 
     pytest
     pytest-rerunfailures
@@ -28,7 +34,6 @@ deps = auditwheel
 commands = 
     python tests/auditwheels.py {toxworkdir}/wheelhouse
 
-
 [testenv:lint]
 skipsdist = True
 usedevelop = True
@@ -43,18 +48,28 @@ commands =
     rst-lint --encoding 'utf-8' README.rst
 
 [testenv:doc]
-basepython = python3.5
+changedir=doc
 deps =
     sphinx==2.2.0
     breathe
-
-changedir = doc
 commands =
-    sphinx-build -w docker-toxdoc-warnings.log -b html . _build/html
+    sphinx-build -w toxdoc-warnings.log -b html . _build/html
+
+
+[testenv:lintdoc]
+skipsdist = True
+usedevelop = True
+deps =
+    {[testenv:lint]deps}
+    {[testenv:doc]deps}
+commands =
+    {[testenv:lint]commands}
+    {[testenv:doc]commands}
+
 
 
 [pytest]
-addopts = -v -rs --reruns 3 --reruns-delay 2
+addopts = -v -rs 
 python_files = tests/test_*.py 
 norecursedirs = .tox 
 xfail_strict=true

rust-toolchain

@@ -1 +1 @@
-nightly-2019-08-13
+nightly-2019-11-06

src/aheader.rs

@@ -1,3 +1,7 @@
+//! # Autocrypt header module
+//!
+//! Parse and create [Autocrypt-headers](https://autocrypt.org/en/latest/level1.html#the-autocrypt-header).
+
 use std::collections::BTreeMap;
 use std::ffi::CStr;
 use std::str::FromStr;
@@ -46,7 +50,7 @@ impl str::FromStr for EncryptPreference {
     }
 }
 
-/// Parse and create [Autocrypt-headers](https://autocrypt.org/en/latest/level1.html#the-autocrypt-header).
+/// Autocrypt header
 #[derive(Debug)]
 pub struct Aheader {
     pub addr: String,
@@ -55,6 +59,7 @@ pub struct Aheader {
 }
 
 impl Aheader {
+    /// Creates new autocrypt header
     pub fn new(addr: String, public_key: Key, prefer_encrypt: EncryptPreference) -> Self {
         Aheader {
             addr,

src/blob.rs

@@ -1,3 +1,6 @@
+//! # Blob directory management
+
+use std::ffi::OsStr;
 use std::fmt;
 use std::fs;
 use std::io::Write;
@@ -24,7 +27,8 @@ impl<'a> BlobObject<'a> {
     /// Creates a new file in the blob directory.  The name will be
     /// derived from the platform-agnostic basename of the suggested
     /// name, followed by a random number and followed by a possible
-    /// extension.  The `data` will be written into the file.
+    /// extension.  The `data` will be written into the file without
+    /// race-conditions.
     ///
     /// # Errors
     ///
@@ -41,29 +45,33 @@ impl<'a> BlobObject<'a> {
         data: &[u8],
     ) -> std::result::Result<BlobObject<'a>, BlobError> {
         let blobdir = context.get_blobdir();
-        let (stem, ext) = BlobObject::sanitise_name(suggested_name.as_ref().to_string());
-        let mut name = format!("{}{}", stem, ext);
+        let (stem, ext) = BlobObject::sanitise_name(suggested_name.as_ref());
+        let (name, mut file) = BlobObject::create_new_file(&blobdir, &stem, &ext)?;
+        file.write_all(data)
+            .map_err(|err| BlobError::new_write_failure(blobdir, &name, err))?;
+        let blob = BlobObject {
+            blobdir,
+            name: format!("$BLOBDIR/{}", name),
+        };
+        context.call_cb(Event::NewBlobFile(blob.as_name().to_string()));
+        Ok(blob)
+    }
+
+    // Creates a new file, returning a tuple of the name and the handle.
+    fn create_new_file(dir: &Path, stem: &str, ext: &str) -> Result<(String, fs::File), BlobError> {
         let max_attempt = 15;
+        let mut name = format!("{}{}", stem, ext);
         for attempt in 0..max_attempt {
-            let path = blobdir.join(&name);
+            let path = dir.join(&name);
             match fs::OpenOptions::new()
                 .create_new(true)
                 .write(true)
                 .open(&path)
             {
-                Ok(mut file) => {
-                    file.write_all(data)
-                        .map_err(|err| BlobError::new_write_failure(blobdir, &name, err))?;
-                    let blob = BlobObject {
-                        blobdir,
-                        name: format!("$BLOBDIR/{}", name),
-                    };
-                    context.call_cb(Event::NewBlobFile(blob.as_name().to_string()));
-                    return Ok(blob);
-                }
+                Ok(file) => return Ok((name, file)),
                 Err(err) => {
                     if attempt == max_attempt {
-                        return Err(BlobError::new_create_failure(blobdir, &name, err));
+                        return Err(BlobError::new_create_failure(dir, &name, err));
                     } else {
                         name = format!("{}-{}{}", stem, rand::random::<u32>(), ext);
                     }
@@ -71,7 +79,7 @@ impl<'a> BlobObject<'a> {
             }
         }
         Err(BlobError::new_create_failure(
-            blobdir,
+            dir,
             &name,
             format_err!("Unreachable code - supposedly"),
         ))
@@ -80,7 +88,9 @@ impl<'a> BlobObject<'a> {
     /// Creates a new blob object with unique name by copying an existing file.
     ///
     /// This creates a new blob as described in [BlobObject::create]
-    /// but also copies an existing file into it.
+    /// but also copies an existing file into it.  This is done in a
+    /// in way which avoids race-conditions when multiple files are
+    /// concurrently created.
     ///
     /// # Errors
     ///
@@ -91,11 +101,24 @@ impl<'a> BlobObject<'a> {
         context: &'a Context,
         src: impl AsRef<Path>,
     ) -> std::result::Result<BlobObject<'a>, BlobError> {
-        let blob = BlobObject::create(context, src.as_ref().to_string_lossy(), b"")?;
-        fs::copy(src.as_ref(), blob.to_abs_path()).map_err(|err| {
-            fs::remove_file(blob.to_abs_path()).ok();
-            BlobError::new_copy_failure(blob.blobdir, &blob.name, src.as_ref(), err)
+        let mut src_file = fs::File::open(src.as_ref()).map_err(|err| {
+            BlobError::new_copy_failure(context.get_blobdir(), "", src.as_ref(), err)
         })?;
+        let (stem, ext) = BlobObject::sanitise_name(&src.as_ref().to_string_lossy());
+        let (name, mut dst_file) = BlobObject::create_new_file(context.get_blobdir(), &stem, &ext)?;
+        std::io::copy(&mut src_file, &mut dst_file).map_err(|err| {
+            {
+                // Attempt to remove the failed file, swallow errors resulting from that.
+                let path = context.get_blobdir().join(&name);
+                fs::remove_file(path).ok();
+            }
+            BlobError::new_copy_failure(context.get_blobdir(), &name, src.as_ref(), err)
+        })?;
+        let blob = BlobObject {
+            blobdir: context.get_blobdir(),
+            name: format!("$BLOBDIR/{}", name),
+        };
+        context.call_cb(Event::NewBlobFile(blob.as_name().to_string()));
         Ok(blob)
     }
 
@@ -117,9 +140,10 @@ impl<'a> BlobObject<'a> {
         context: &Context,
         src: impl AsRef<Path>,
     ) -> std::result::Result<BlobObject, BlobError> {
-        match src.as_ref().starts_with(context.get_blobdir()) {
-            true => BlobObject::from_path(context, src),
-            false => BlobObject::create_and_copy(context, src),
+        if src.as_ref().starts_with(context.get_blobdir()) {
+            BlobObject::from_path(context, src)
+        } else {
+            BlobObject::create_and_copy(context, src)
         }
     }
 
@@ -145,6 +169,9 @@ impl<'a> BlobObject<'a> {
             .as_ref()
             .strip_prefix(context.get_blobdir())
             .map_err(|_| BlobError::new_wrong_blobdir(context.get_blobdir(), path.as_ref()))?;
+        if !BlobObject::is_acceptible_blob_name(&rel_path) {
+            return Err(BlobError::new_wrong_name(path.as_ref()));
+        }
         let name = rel_path
             .to_str()
             .ok_or_else(|| BlobError::new_wrong_name(path.as_ref()))?;
@@ -171,8 +198,7 @@ impl<'a> BlobObject<'a> {
             true => name.splitn(2, '/').last().unwrap().to_string(),
             false => name,
         };
-        let (stem, ext) = BlobObject::sanitise_name(name.clone());
-        if format!("{}{}", stem, ext) != name.as_ref() {
+        if !BlobObject::is_acceptible_blob_name(&name) {
             return Err(BlobError::new_wrong_name(name));
         }
         Ok(BlobObject {
@@ -236,15 +262,16 @@ impl<'a> BlobObject<'a> {
     /// ".txt")` while "bar" is returned as `("bar", "")`.
     ///
     /// The extension part will always be lowercased.
-    fn sanitise_name(mut name: String) -> (String, String) {
+    fn sanitise_name(name: &str) -> (String, String) {
+        let mut name = name.to_string();
         for part in name.rsplit('/') {
-            if part.len() > 0 {
+            if !part.is_empty() {
                 name = part.to_string();
                 break;
             }
         }
         for part in name.rsplit('\\') {
-            if part.len() > 0 {
+            if !part.is_empty() {
                 name = part.to_string();
                 break;
             }
@@ -256,16 +283,38 @@ impl<'a> BlobObject<'a> {
         };
 
         let clean = sanitize_filename::sanitize_with_options(name, opts);
-        let mut iter = clean.rsplitn(2, '.');
-        let mut ext = iter.next().unwrap_or_default().to_string();
+        let mut iter = clean.splitn(2, '.');
         let mut stem = iter.next().unwrap_or_default().to_string();
-        ext.truncate(32);
+        let mut ext = iter.next().unwrap_or_default().to_string();
         stem.truncate(64);
-        match stem.len() {
-            0 => (ext, "".to_string()),
+        ext.truncate(32);
+        match ext.len() {
+            0 => (stem, "".to_string()),
             _ => (stem, format!(".{}", ext).to_lowercase()),
         }
     }
+
+    /// Checks whether a name is a valid blob name.
+    ///
+    /// This is slightly less strict than stanitise_name, presumably
+    /// someone already created a file with such a name so we just
+    /// ensure it's not actually a path in disguise is actually utf-8.
+    fn is_acceptible_blob_name(name: impl AsRef<OsStr>) -> bool {
+        let uname = match name.as_ref().to_str() {
+            Some(name) => name,
+            None => return false,
+        };
+        if uname.find('/').is_some() {
+            return false;
+        }
+        if uname.find('\\').is_some() {
+            return false;
+        }
+        if uname.find('\0').is_some() {
+            return false;
+        }
+        true
+    }
 }
 
 impl<'a> fmt::Display for BlobObject<'a> {
@@ -561,6 +610,26 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_double_ext_preserved() {
+        let t = dummy_context();
+        BlobObject::create(&t.ctx, "foo.tar.gz", b"hello").unwrap();
+        let foo = t.ctx.get_blobdir().join("foo.tar.gz");
+        assert!(foo.exists());
+        BlobObject::create(&t.ctx, "foo.tar.gz", b"world").unwrap();
+        for dirent in fs::read_dir(t.ctx.get_blobdir()).unwrap() {
+            let fname = dirent.unwrap().file_name();
+            if fname == foo.file_name().unwrap() {
+                assert_eq!(fs::read(&foo).unwrap(), b"hello");
+            } else {
+                let name = fname.to_str().unwrap();
+                println!("{}", name);
+                assert!(name.starts_with("foo"));
+                assert!(name.ends_with(".tar.gz"));
+            }
+        }
+    }
+
     #[test]
     fn test_create_long_names() {
         let t = dummy_context();
@@ -615,4 +684,14 @@ mod tests {
             "$BLOBDIR/autocrypt-setup-message-4137848473.html"
         );
     }
+
+    #[test]
+    fn test_is_blob_name() {
+        assert!(BlobObject::is_acceptible_blob_name("foo"));
+        assert!(BlobObject::is_acceptible_blob_name("foo.txt"));
+        assert!(BlobObject::is_acceptible_blob_name("f".repeat(128)));
+        assert!(!BlobObject::is_acceptible_blob_name("foo/bar"));
+        assert!(!BlobObject::is_acceptible_blob_name("foo\\bar"));
+        assert!(!BlobObject::is_acceptible_blob_name("foo\x00bar"));
+    }
 }

src/chat.rs

@@ -1,6 +1,9 @@
+//! # Chat module
+
 use std::path::{Path, PathBuf};
 
 use itertools::Itertools;
+use num_traits::FromPrimitive;
 
 use crate::blob::{BlobErrorKind, BlobObject};
 use crate::chatlist::*;
@@ -36,6 +39,7 @@ pub struct Chat {
 }
 
 impl Chat {
+    /// Loads chat from the database by its ID.
     pub fn load_from_db(context: &Context, chat_id: u32) -> Result<Self, Error> {
         let res = context.sql.query_row(
             "SELECT c.id,c.type,c.name, c.grpid,c.param,c.archived, \
@@ -96,7 +100,9 @@ impl Chat {
                         }
 
                         if chat.param.exists(Param::Selftalk) {
-                            chat.name = context.stock_str(StockMessage::SelfMsg).into();
+                            chat.name = context.stock_str(StockMessage::SavedMessages).into();
+                        } else if chat.param.exists(Param::Devicetalk) {
+                            chat.name = context.stock_str(StockMessage::DeviceMessages).into();
                         }
                     }
                 }
@@ -109,6 +115,14 @@ impl Chat {
         self.param.exists(Param::Selftalk)
     }
 
+    pub fn is_device_talk(&self) -> bool {
+        self.param.exists(Param::Devicetalk)
+    }
+
+    pub fn can_send(&self) -> bool {
+        self.id > DC_CHAT_ID_LAST_SPECIAL && !self.is_device_talk()
+    }
+
     pub fn update_param(&mut self, context: &Context) -> Result<(), Error> {
         sql::execute(
             context,
@@ -215,6 +229,7 @@ impl Chat {
         color
     }
 
+    /// Returns true if the chat is archived.
     pub fn is_archived(&self) -> bool {
         self.archived
     }
@@ -258,7 +273,7 @@ impl Chat {
         }
 
         if (self.typ == Chattype::Group || self.typ == Chattype::VerifiedGroup)
-            && !is_contact_in_chat(context, self.id, 1 as u32)
+            && !is_contact_in_chat(context, self.id, DC_CONTACT_ID_SELF)
         {
             emit_event!(
                 context,
@@ -414,7 +429,7 @@ impl Chat {
                     &context.sql,
                     "INSERT INTO locations \
                      (timestamp,from_id,chat_id, latitude,longitude,independent)\
-                     VALUES (?,?,?, ?,?,1);",
+                     VALUES (?,?,?, ?,?,1);", // 1=DC_CONTACT_ID_SELF
                     params![
                         timestamp,
                         DC_CONTACT_ID_SELF,
@@ -446,7 +461,7 @@ impl Chat {
                         params![
                             new_rfc724_mid,
                             self.id as i32,
-                            1i32,
+                            DC_CONTACT_ID_SELF,
                             to_id as i32,
                             timestamp,
                             msg.type_0,
@@ -582,6 +597,26 @@ pub fn set_blocking(context: &Context, chat_id: u32, new_blocking: Blocked) -> b
     .is_ok()
 }
 
+fn copy_device_icon_to_blobs(context: &Context) -> Result<String, Error> {
+    let icon = include_bytes!("../assets/icon-device.png");
+    let blob = BlobObject::create(context, "icon-device.png".to_string(), icon)?;
+    Ok(blob.as_name().to_string())
+}
+
+pub fn update_saved_messages_icon(context: &Context) -> Result<(), Error> {
+    // if there is no saved-messages chat, there is nothing to update. this is no error.
+    if let Ok((chat_id, _)) = lookup_by_contact_id(context, DC_CONTACT_ID_SELF) {
+        let icon = include_bytes!("../assets/icon-saved-messages.png");
+        let blob = BlobObject::create(context, "icon-saved-messages.png".to_string(), icon)?;
+        let icon = blob.as_name().to_string();
+
+        let mut chat = Chat::load_from_db(context, chat_id)?;
+        chat.param.set(Param::ProfileImage, icon);
+        chat.update_param(context)?;
+    }
+    Ok(())
+}
+
 pub fn create_or_lookup_by_contact_id(
     context: &Context,
     contact_id: u32,
@@ -605,7 +640,14 @@ pub fn create_or_lookup_by_contact_id(
             "INSERT INTO chats (type, name, param, blocked, grpid) VALUES({}, '{}', '{}', {}, '{}')",
             100,
             chat_name,
-            if contact_id == DC_CONTACT_ID_SELF as u32 { "K=1" } else { "" },
+            match contact_id {
+                DC_CONTACT_ID_SELF => "K=1".to_string(), // K = Param::Selftalk
+                DC_CONTACT_ID_DEVICE => {
+                    let icon = copy_device_icon_to_blobs(context)?;
+                    format!("D=1\ni={}", icon) // D = Param::Devicetalk, i = Param::ProfileImage
+                },
+                _ => "".to_string()
+            },
             create_blocked as u8,
             contact.get_addr(),
         ),
@@ -624,6 +666,10 @@ pub fn create_or_lookup_by_contact_id(
         params![],
     )?;
 
+    if contact_id == DC_CONTACT_ID_SELF {
+        update_saved_messages_icon(context)?;
+    }
+
     Ok((chat_id, create_blocked))
 }
 
@@ -666,6 +712,8 @@ pub fn prepare_msg<'a>(
 
 pub fn msgtype_has_file(msgtype: Viewtype) -> bool {
     match msgtype {
+        Viewtype::Unknown => false,
+        Viewtype::Text => false,
         Viewtype::Image => true,
         Viewtype::Gif => true,
         Viewtype::Sticker => true,
@@ -673,12 +721,10 @@ pub fn msgtype_has_file(msgtype: Viewtype) -> bool {
         Viewtype::Voice => true,
         Viewtype::Video => true,
         Viewtype::File => true,
-        _ => false,
     }
 }
 
-fn prepare_msg_common(context: &Context, chat_id: u32, msg: &mut Message) -> Result<MsgId, Error> {
-    msg.id = MsgId::new_unset();
+fn prepare_msg_blob(context: &Context, msg: &mut Message) -> Result<(), Error> {
     if msg.type_0 == Viewtype::Text {
         // the caller should check if the message text is empty
     } else if msgtype_has_file(msg.type_0) {
@@ -714,10 +760,16 @@ fn prepare_msg_common(context: &Context, chat_id: u32, msg: &mut Message) -> Res
     } else {
         bail!("Cannot send messages of type #{}.", msg.type_0);
     }
+    Ok(())
+}
 
+fn prepare_msg_common(context: &Context, chat_id: u32, msg: &mut Message) -> Result<MsgId, Error> {
+    msg.id = MsgId::new_unset();
+    prepare_msg_blob(context, msg)?;
     unarchive(context, chat_id)?;
 
     let mut chat = Chat::load_from_db(context, chat_id)?;
+    ensure!(chat.can_send(), "cannot send to chat #{}", chat_id);
 
     // The OutPreparing state is set by dc_prepare_msg() before it
     // calls this function and the message is left in the OutPreparing
@@ -768,7 +820,8 @@ pub fn is_contact_in_chat(context: &Context, chat_id: u32, contact_id: u32) -> b
         .unwrap_or_default()
 }
 
-// Should return Result
+// note that unarchive() is not the same as archive(false) -
+// eg. unarchive() does not send events as done for archive(false).
 pub fn unarchive(context: &Context, chat_id: u32) -> Result<(), Error> {
     sql::execute(
         context,
@@ -844,7 +897,7 @@ pub fn send_text_msg(
 ) -> Result<MsgId, Error> {
     ensure!(
         chat_id > DC_CHAT_ID_LAST_SPECIAL,
-        "bad chat_id = {} <= 9",
+        "bad chat_id = {} <= DC_CHAT_ID_LAST_SPECIAL",
         chat_id
     );
 
@@ -914,7 +967,7 @@ fn do_set_draft(context: &Context, chat_id: u32, msg: &mut Message) -> Result<()
          VALUES (?,?,?, ?,?,?,?,?);",
         params![
             chat_id as i32,
-            1,
+            DC_CONTACT_ID_SELF,
             time(),
             msg.type_0,
             MessageState::OutDraft,
@@ -984,7 +1037,8 @@ pub fn get_chat_msgs(
         Ok(ret)
     };
     let success = if chat_id == DC_CHAT_ID_DEADDROP {
-        let show_emails = context.get_config_int(Config::ShowEmails);
+        let show_emails =
+            ShowEmails::from_i32(context.get_config_int(Config::ShowEmails)).unwrap_or_default();
         context.sql.query_map(
             concat!(
                 "SELECT m.id AS id, m.timestamp AS timestamp",
@@ -993,15 +1047,15 @@ pub fn get_chat_msgs(
                 "        ON m.chat_id=chats.id",
                 " LEFT JOIN contacts",
                 "        ON m.from_id=contacts.id",
-                " WHERE m.from_id!=1",
-                "   AND m.from_id!=2",
+                " WHERE m.from_id!=1", // 1=DC_CONTACT_ID_SELF
+                "   AND m.from_id!=2", // 2=DC_CONTACT_ID_INFO
                 "   AND m.hidden=0",
                 "   AND chats.blocked=2",
                 "   AND contacts.blocked=0",
                 "   AND m.msgrmsg>=?",
                 " ORDER BY m.timestamp,m.id;"
             ),
-            params![if show_emails == 2 { 0 } else { 1 }],
+            params![if show_emails == ShowEmails::All { 0 } else { 1 }],
             process_row,
             process_rows,
         )
@@ -1153,9 +1207,8 @@ pub fn get_chat_media(
             |ids| {
                 let mut ret = Vec::new();
                 for id in ids {
-                    match id {
-                        Ok(msg_id) => ret.push(msg_id),
-                        Err(_) => (),
+                    if let Ok(msg_id) = id {
+                        ret.push(msg_id)
                     }
                 }
                 Ok(ret)
@@ -1218,7 +1271,7 @@ pub fn get_next_media(
 pub fn archive(context: &Context, chat_id: u32, archive: bool) -> Result<(), Error> {
     ensure!(
         chat_id > DC_CHAT_ID_LAST_SPECIAL,
-        "bad chat_id = {} <= 9",
+        "bad chat_id = {} <= DC_CHAT_ID_LAST_SPECIAL",
         chat_id
     );
 
@@ -1253,7 +1306,7 @@ pub fn archive(context: &Context, chat_id: u32, archive: bool) -> Result<(), Err
 pub fn delete(context: &Context, chat_id: u32) -> Result<(), Error> {
     ensure!(
         chat_id > DC_CHAT_ID_LAST_SPECIAL,
-        "bad chat_id = {} <= 9",
+        "bad chat_id = {} <= DC_CHAT_ID_LAST_SPECIAL",
         chat_id
     );
     /* Up to 2017-11-02 deleting a group also implied leaving it, see above why we have changed this. */
@@ -1350,7 +1403,7 @@ pub fn create_group_chat(
     let chat_id = sql::get_rowid(context, &context.sql, "chats", "grpid", grpid);
 
     if chat_id != 0 {
-        if add_to_chat_contacts_table(context, chat_id, 1) {
+        if add_to_chat_contacts_table(context, chat_id, DC_CONTACT_ID_SELF) {
             let mut draft_msg = Message::new(Viewtype::Text);
             draft_msg.set_text(Some(draft_txt));
             set_draft_raw(context, chat_id, &mut draft_msg);
@@ -1476,7 +1529,7 @@ pub(crate) fn add_contact_to_chat_ex(
         msg.id = send_msg(context, chat_id, &mut msg)?;
         context.call_cb(Event::MsgsChanged {
             chat_id,
-            msg_id: MsgId::from(msg.id),
+            msg_id: msg.id,
         });
     }
     context.call_cb(Event::MsgsChanged {
@@ -1542,7 +1595,7 @@ pub fn remove_contact_from_chat(
 ) -> Result<(), Error> {
     ensure!(
         chat_id > DC_CHAT_ID_LAST_SPECIAL,
-        "bad chat_id = {} <= 9",
+        "bad chat_id = {} <= DC_CHAT_ID_LAST_SPECIAL",
         chat_id
     );
     ensure!(
@@ -1557,7 +1610,7 @@ pub fn remove_contact_from_chat(
     /* this allows to delete pending references to deleted contacts.  Of course, this should _not_ happen. */
     if let Ok(chat) = Chat::load_from_db(context, chat_id) {
         if real_group_exists(context, chat_id) {
-            if !is_contact_in_chat(context, chat_id, 1 as u32) {
+            if !is_contact_in_chat(context, chat_id, DC_CONTACT_ID_SELF) {
                 emit_event!(
                     context,
                     Event::ErrorSelfNotInGroup(
@@ -1653,7 +1706,7 @@ pub fn set_chat_name(
     if real_group_exists(context, chat_id) {
         if chat.name == new_name.as_ref() {
             success = true;
-        } else if !is_contact_in_chat(context, chat_id, 1) {
+        } else if !is_contact_in_chat(context, chat_id, DC_CONTACT_ID_SELF) {
             emit_event!(
                 context,
                 Event::ErrorSelfNotInGroup("Cannot set chat name; self not in group".into())
@@ -1898,15 +1951,87 @@ pub fn get_chat_id_by_grpid(context: &Context, grpid: impl AsRef<str>) -> (u32,
         .unwrap_or((0, false, Blocked::Not))
 }
 
-pub fn add_device_msg(context: &Context, chat_id: u32, text: impl AsRef<str>) {
+pub fn add_device_msg(
+    context: &Context,
+    label: Option<&str>,
+    msg: Option<&mut Message>,
+) -> Result<MsgId, Error> {
+    ensure!(
+        label.is_some() || msg.is_some(),
+        "device-messages need label, msg or both"
+    );
+    let (chat_id, _blocked) =
+        create_or_lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE, Blocked::Not)?;
+    let mut msg_id = MsgId::new_unset();
+
+    if let Some(label) = label {
+        if was_device_msg_ever_added(context, label)? {
+            info!(context, "device-message {} already added", label);
+            return Ok(msg_id);
+        }
+    }
+
+    if let Some(msg) = msg {
+        let rfc724_mid = dc_create_outgoing_rfc724_mid(None, "@device");
+        prepare_msg_blob(context, msg)?;
+        unarchive(context, chat_id)?;
+
+        context.sql.execute(
+            "INSERT INTO msgs (chat_id,from_id,to_id, timestamp,type,state, txt,param,rfc724_mid) \
+             VALUES (?,?,?, ?,?,?, ?,?,?);",
+            params![
+                chat_id,
+                DC_CONTACT_ID_DEVICE,
+                DC_CONTACT_ID_SELF,
+                dc_create_smeared_timestamp(context),
+                msg.type_0,
+                MessageState::InFresh,
+                msg.text.as_ref().map_or("", String::as_str),
+                msg.param.to_string(),
+                rfc724_mid,
+            ],
+        )?;
+
+        let row_id = sql::get_rowid(context, &context.sql, "msgs", "rfc724_mid", &rfc724_mid);
+        msg_id = MsgId::new(row_id);
+    }
+
+    if let Some(label) = label {
+        context.sql.execute(
+            "INSERT INTO devmsglabels (label) VALUES (?);",
+            params![label],
+        )?;
+    }
+
+    if !msg_id.is_unset() {
+        context.call_cb(Event::IncomingMsg { chat_id, msg_id });
+    }
+
+    Ok(msg_id)
+}
+
+pub fn was_device_msg_ever_added(context: &Context, label: &str) -> Result<bool, Error> {
+    ensure!(!label.is_empty(), "empty label");
+    if let Ok(()) = context.sql.query_row(
+        "SELECT label FROM devmsglabels WHERE label=?",
+        params![label],
+        |_| Ok(()),
+    ) {
+        return Ok(true);
+    }
+
+    Ok(false)
+}
+
+pub fn add_info_msg(context: &Context, chat_id: u32, text: impl AsRef<str>) {
     let rfc724_mid = dc_create_outgoing_rfc724_mid(None, "@device");
 
     if context.sql.execute(
         "INSERT INTO msgs (chat_id,from_id,to_id, timestamp,type,state, txt,rfc724_mid) VALUES (?,?,?, ?,?,?, ?,?);",
         params![
             chat_id as i32,
-            2,
-            2,
+            DC_CONTACT_ID_INFO,
+            DC_CONTACT_ID_INFO,
             dc_create_smeared_timestamp(context),
             Viewtype::Text,
             MessageState::InNoticed,
@@ -1975,4 +2100,210 @@ mod tests {
         let added = add_contact_to_chat_ex(&t.ctx, chat_id, DC_CONTACT_ID_SELF, false).unwrap();
         assert_eq!(added, false);
     }
+
+    #[test]
+    fn test_self_talk() {
+        let t = dummy_context();
+        let chat_id = create_by_contact_id(&t.ctx, DC_CONTACT_ID_SELF).unwrap();
+        assert_eq!(DC_CONTACT_ID_SELF, 1);
+        assert!(chat_id > DC_CHAT_ID_LAST_SPECIAL);
+        let chat = Chat::load_from_db(&t.ctx, chat_id).unwrap();
+        assert_eq!(chat.id, chat_id);
+        assert!(chat.is_self_talk());
+        assert!(!chat.archived);
+        assert!(!chat.is_device_talk());
+        assert!(chat.can_send());
+        assert_eq!(chat.name, t.ctx.stock_str(StockMessage::SavedMessages));
+        assert!(chat.get_profile_image(&t.ctx).is_some());
+    }
+
+    #[test]
+    fn test_deaddrop_chat() {
+        let t = dummy_context();
+        let chat = Chat::load_from_db(&t.ctx, DC_CHAT_ID_DEADDROP).unwrap();
+        assert_eq!(DC_CHAT_ID_DEADDROP, 1);
+        assert_eq!(chat.id, DC_CHAT_ID_DEADDROP);
+        assert!(!chat.is_self_talk());
+        assert!(!chat.archived);
+        assert!(!chat.is_device_talk());
+        assert!(!chat.can_send());
+        assert_eq!(chat.name, t.ctx.stock_str(StockMessage::DeadDrop));
+    }
+
+    #[test]
+    fn test_add_device_msg_unlabelled() {
+        let t = test_context(Some(Box::new(logging_cb)));
+
+        // add two device-messages
+        let mut msg1 = Message::new(Viewtype::Text);
+        msg1.text = Some("first message".to_string());
+        let msg1_id = add_device_msg(&t.ctx, None, Some(&mut msg1));
+        assert!(msg1_id.is_ok());
+
+        let mut msg2 = Message::new(Viewtype::Text);
+        msg2.text = Some("second message".to_string());
+        let msg2_id = add_device_msg(&t.ctx, None, Some(&mut msg2));
+        assert!(msg2_id.is_ok());
+        assert_ne!(msg1_id.as_ref().unwrap(), msg2_id.as_ref().unwrap());
+
+        // check added messages
+        let msg1 = message::Message::load_from_db(&t.ctx, msg1_id.unwrap());
+        assert!(msg1.is_ok());
+        let msg1 = msg1.unwrap();
+        assert_eq!(msg1.text.as_ref().unwrap(), "first message");
+        assert_eq!(msg1.from_id, DC_CONTACT_ID_DEVICE);
+        assert_eq!(msg1.to_id, DC_CONTACT_ID_SELF);
+        assert!(!msg1.is_info());
+        assert!(!msg1.is_setupmessage());
+
+        let msg2 = message::Message::load_from_db(&t.ctx, msg2_id.unwrap());
+        assert!(msg2.is_ok());
+        let msg2 = msg2.unwrap();
+        assert_eq!(msg2.text.as_ref().unwrap(), "second message");
+
+        // check device chat
+        assert_eq!(get_msg_cnt(&t.ctx, msg2.chat_id), 2);
+    }
+
+    #[test]
+    fn test_add_device_msg_labelled() {
+        let t = test_context(Some(Box::new(logging_cb)));
+
+        // add two device-messages with the same label (second attempt is not added)
+        let mut msg1 = Message::new(Viewtype::Text);
+        msg1.text = Some("first message".to_string());
+        let msg1_id = add_device_msg(&t.ctx, Some("any-label"), Some(&mut msg1));
+        assert!(msg1_id.is_ok());
+        assert!(!msg1_id.as_ref().unwrap().is_unset());
+
+        let mut msg2 = Message::new(Viewtype::Text);
+        msg2.text = Some("second message".to_string());
+        let msg2_id = add_device_msg(&t.ctx, Some("any-label"), Some(&mut msg2));
+        assert!(msg2_id.is_ok());
+        assert!(msg2_id.as_ref().unwrap().is_unset());
+
+        // check added message
+        let msg1 = message::Message::load_from_db(&t.ctx, *msg1_id.as_ref().unwrap());
+        assert!(msg1.is_ok());
+        let msg1 = msg1.unwrap();
+        assert_eq!(msg1_id.as_ref().unwrap(), &msg1.id);
+        assert_eq!(msg1.text.as_ref().unwrap(), "first message");
+        assert_eq!(msg1.from_id, DC_CONTACT_ID_DEVICE);
+        assert_eq!(msg1.to_id, DC_CONTACT_ID_SELF);
+        assert!(!msg1.is_info());
+        assert!(!msg1.is_setupmessage());
+
+        // check device chat
+        let chat_id = msg1.chat_id;
+        assert_eq!(get_msg_cnt(&t.ctx, chat_id), 1);
+        assert!(chat_id > DC_CHAT_ID_LAST_SPECIAL);
+        let chat = Chat::load_from_db(&t.ctx, chat_id);
+        assert!(chat.is_ok());
+        let chat = chat.unwrap();
+        assert!(chat.is_device_talk());
+        assert!(!chat.is_self_talk());
+        assert!(!chat.can_send());
+        assert_eq!(chat.name, t.ctx.stock_str(StockMessage::DeviceMessages));
+        assert!(chat.get_profile_image(&t.ctx).is_some());
+
+        // delete device message, make sure it is not added again
+        message::delete_msgs(&t.ctx, &[*msg1_id.as_ref().unwrap()]);
+        let msg1 = message::Message::load_from_db(&t.ctx, *msg1_id.as_ref().unwrap());
+        assert!(msg1.is_err() || msg1.unwrap().chat_id == DC_CHAT_ID_TRASH);
+        let msg3_id = add_device_msg(&t.ctx, Some("any-label"), Some(&mut msg2));
+        assert!(msg3_id.is_ok());
+        assert!(msg2_id.as_ref().unwrap().is_unset());
+    }
+
+    #[test]
+    fn test_add_device_msg_label_only() {
+        let t = test_context(Some(Box::new(logging_cb)));
+        let res = add_device_msg(&t.ctx, Some(""), None);
+        assert!(res.is_err());
+        let res = add_device_msg(&t.ctx, Some("some-label"), None);
+        assert!(res.is_ok());
+
+        let mut msg = Message::new(Viewtype::Text);
+        msg.text = Some("message text".to_string());
+
+        let msg_id = add_device_msg(&t.ctx, Some("some-label"), Some(&mut msg));
+        assert!(msg_id.is_ok());
+        assert!(msg_id.as_ref().unwrap().is_unset());
+
+        let msg_id = add_device_msg(&t.ctx, Some("unused-label"), Some(&mut msg));
+        assert!(msg_id.is_ok());
+        assert!(!msg_id.as_ref().unwrap().is_unset());
+    }
+
+    #[test]
+    fn test_was_device_msg_ever_added() {
+        let t = test_context(Some(Box::new(logging_cb)));
+        add_device_msg(&t.ctx, Some("some-label"), None).ok();
+        assert!(was_device_msg_ever_added(&t.ctx, "some-label").unwrap());
+
+        let mut msg = Message::new(Viewtype::Text);
+        msg.text = Some("message text".to_string());
+        add_device_msg(&t.ctx, Some("another-label"), Some(&mut msg)).ok();
+        assert!(was_device_msg_ever_added(&t.ctx, "another-label").unwrap());
+
+        assert!(!was_device_msg_ever_added(&t.ctx, "unused-label").unwrap());
+
+        assert!(was_device_msg_ever_added(&t.ctx, "").is_err());
+    }
+
+    fn chatlist_len(ctx: &Context, listflags: usize) -> usize {
+        Chatlist::try_load(ctx, listflags, None, None)
+            .unwrap()
+            .len()
+    }
+
+    #[test]
+    fn test_archive() {
+        // create two chats
+        let t = dummy_context();
+        let mut msg = Message::new(Viewtype::Text);
+        msg.text = Some("foo".to_string());
+        let msg_id = add_device_msg(&t.ctx, None, Some(&mut msg)).unwrap();
+        let chat_id1 = message::Message::load_from_db(&t.ctx, msg_id)
+            .unwrap()
+            .chat_id;
+        let chat_id2 = create_by_contact_id(&t.ctx, DC_CONTACT_ID_SELF).unwrap();
+        assert!(chat_id1 > DC_CHAT_ID_LAST_SPECIAL);
+        assert!(chat_id2 > DC_CHAT_ID_LAST_SPECIAL);
+        assert_eq!(get_chat_cnt(&t.ctx), 2);
+        assert_eq!(chatlist_len(&t.ctx, 0), 2);
+        assert_eq!(chatlist_len(&t.ctx, DC_GCL_NO_SPECIALS), 2);
+        assert_eq!(chatlist_len(&t.ctx, DC_GCL_ARCHIVED_ONLY), 0);
+        assert_eq!(DC_GCL_ARCHIVED_ONLY, 0x01);
+        assert_eq!(DC_GCL_NO_SPECIALS, 0x02);
+
+        // archive first chat
+        assert!(archive(&t.ctx, chat_id1, true).is_ok());
+        assert!(Chat::load_from_db(&t.ctx, chat_id1).unwrap().is_archived());
+        assert!(!Chat::load_from_db(&t.ctx, chat_id2).unwrap().is_archived());
+        assert_eq!(get_chat_cnt(&t.ctx), 2);
+        assert_eq!(chatlist_len(&t.ctx, 0), 2); // including DC_CHAT_ID_ARCHIVED_LINK now
+        assert_eq!(chatlist_len(&t.ctx, DC_GCL_NO_SPECIALS), 1);
+        assert_eq!(chatlist_len(&t.ctx, DC_GCL_ARCHIVED_ONLY), 1);
+
+        // archive second chat
+        assert!(archive(&t.ctx, chat_id2, true).is_ok());
+        assert!(Chat::load_from_db(&t.ctx, chat_id1).unwrap().is_archived());
+        assert!(Chat::load_from_db(&t.ctx, chat_id2).unwrap().is_archived());
+        assert_eq!(get_chat_cnt(&t.ctx), 2);
+        assert_eq!(chatlist_len(&t.ctx, 0), 1); // only DC_CHAT_ID_ARCHIVED_LINK now
+        assert_eq!(chatlist_len(&t.ctx, DC_GCL_NO_SPECIALS), 0);
+        assert_eq!(chatlist_len(&t.ctx, DC_GCL_ARCHIVED_ONLY), 2);
+
+        // archive already archived first chat, unarchive second chat two times
+        assert!(archive(&t.ctx, chat_id1, true).is_ok());
+        assert!(archive(&t.ctx, chat_id2, false).is_ok());
+        assert!(archive(&t.ctx, chat_id2, false).is_ok());
+        assert!(Chat::load_from_db(&t.ctx, chat_id1).unwrap().is_archived());
+        assert!(!Chat::load_from_db(&t.ctx, chat_id2).unwrap().is_archived());
+        assert_eq!(get_chat_cnt(&t.ctx), 2);
+        assert_eq!(chatlist_len(&t.ctx, 0), 2);
+        assert_eq!(chatlist_len(&t.ctx, DC_GCL_NO_SPECIALS), 1);
+        assert_eq!(chatlist_len(&t.ctx, DC_GCL_ARCHIVED_ONLY), 1);
+    }
 }

src/chatlist.rs

@@ -1,3 +1,5 @@
+//! # Chat list module
+
 use crate::chat::*;
 use crate::constants::*;
 use crate::contact::*;
@@ -126,7 +128,7 @@ impl Chatlist {
                     "               SELECT MAX(timestamp)",
                     "                 FROM msgs",
                     "                WHERE chat_id=c.id",
-                    "                  AND (hidden=0 OR (hidden=1 AND state=19)))",
+                    "                  AND hidden=0)",
                     " WHERE c.id>9",
                     "   AND c.blocked=0",
                     "   AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)",
@@ -149,7 +151,7 @@ impl Chatlist {
                     "               SELECT MAX(timestamp)",
                     "                 FROM msgs",
                     "                WHERE chat_id=c.id",
-                    "                  AND (hidden=0 OR (hidden=1 AND state=19)))",
+                    "                  AND hidden=0)",
                     " WHERE c.id>9",
                     "   AND c.blocked=0",
                     "   AND c.archived=1",
@@ -175,7 +177,7 @@ impl Chatlist {
                     "               SELECT MAX(timestamp)",
                     "                 FROM msgs",
                     "                WHERE chat_id=c.id",
-                    "                  AND (hidden=0 OR (hidden=1 AND state=19)))",
+                    "                  AND hidden=0)",
                     " WHERE c.id>9",
                     "   AND c.blocked=0",
                     "   AND c.name LIKE ?",
@@ -198,7 +200,7 @@ impl Chatlist {
                     "               SELECT MAX(timestamp)",
                     "                 FROM msgs",
                     "                WHERE chat_id=c.id",
-                    "                  AND (hidden=0 OR (hidden=1 AND state=19)))",
+                    "                  AND hidden=0)",
                     " WHERE c.id>9",
                     "   AND c.blocked=0",
                     "   AND c.archived=0",
@@ -233,6 +235,7 @@ impl Chatlist {
         self.ids.len()
     }
 
+    /// Returns true if chatlist is empty.
     pub fn is_empty(&self) -> bool {
         self.ids.is_empty()
     }
@@ -294,8 +297,8 @@ impl Chatlist {
         let lastmsg_id = self.ids[index].1;
         let mut lastcontact = None;
 
-        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id) {
-            if lastmsg.from_id != 1
+        let mut lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id) {
+            if lastmsg.from_id != DC_CONTACT_ID_SELF
                 && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup)
             {
                 lastcontact = Contact::load_from_db(context, lastmsg.from_id).ok();
@@ -306,6 +309,16 @@ impl Chatlist {
             None
         };
 
+        if let Ok(draft) = get_draft(context, chat.id) {
+            if draft.is_some()
+                && (lastmsg.is_none()
+                    || draft.as_ref().unwrap().timestamp_sort
+                        > lastmsg.as_ref().unwrap().timestamp_sort)
+            {
+                lastmsg = draft;
+            }
+        }
+
         if chat.id == DC_CHAT_ID_ARCHIVED_LINK {
             ret.text2 = None;
         } else if lastmsg.is_none() || lastmsg.as_ref().unwrap().from_id == DC_CONTACT_ID_UNDEFINED
@@ -319,6 +332,7 @@ impl Chatlist {
     }
 }
 
+/// Get the number of archived chats
 pub fn dc_get_archived_cnt(context: &Context) -> u32 {
     context
         .sql

src/config.rs

@@ -1,3 +1,5 @@
+//! # Key-value configuration management
+
 use strum::{EnumProperty, IntoEnumIterator};
 use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString};
 
@@ -31,7 +33,7 @@ pub enum Config {
     Displayname,
     Selfstatus,
     Selfavatar,
-    #[strum(props(default = "1"))]
+    #[strum(props(default = "0"))]
     BccSelf,
     #[strum(props(default = "1"))]
     E2eeEnabled,
@@ -119,7 +121,7 @@ impl Context {
             }
             Config::InboxWatch => {
                 let ret = self.sql.set_raw_config(self, key, value);
-                interrupt_imap_idle(self);
+                interrupt_inbox_idle(self, true);
                 ret
             }
             Config::SentboxWatch => {

src/configure/auto_mozilla.rs

@@ -1,3 +1,4 @@
+//! Thunderbird's Autoconfiguration implementation
 use quick_xml;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 

src/configure/mod.rs

@@ -1,3 +1,5 @@
+//! Email accounts autoconfiguration process module
+
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 
 use crate::config::Config;
@@ -5,7 +7,6 @@ use crate::constants::*;
 use crate::context::Context;
 use crate::dc_tools::*;
 use crate::e2ee;
-use crate::imap::*;
 use crate::job::*;
 use crate::login_param::LoginParam;
 use crate::oauth2::*;
@@ -62,7 +63,12 @@ pub fn dc_job_do_DC_JOB_CONFIGURE_IMAP(context: &Context) {
 
     let mut param_autoconfig: Option<LoginParam> = None;
 
-    context.inbox.read().unwrap().disconnect(context);
+    context
+        .inbox_thread
+        .read()
+        .unwrap()
+        .imap
+        .disconnect(context);
     context
         .sentbox_thread
         .read()
@@ -90,7 +96,7 @@ pub fn dc_job_do_DC_JOB_CONFIGURE_IMAP(context: &Context) {
     const STEP_3_INDEX: u8 = 13;
     let mut step_counter: u8 = 0;
     while !context.shall_stop_ongoing() {
-        step_counter = step_counter + 1;
+        step_counter += 1;
 
         let success = match step_counter {
             // Read login parameters from the database
@@ -356,9 +362,10 @@ pub fn dc_job_do_DC_JOB_CONFIGURE_IMAP(context: &Context) {
                     0
                 };
                 context
-                    .inbox
+                    .inbox_thread
                     .read()
                     .unwrap()
+                    .imap
                     .configure_folders(context, flags);
                 true
             }
@@ -398,7 +405,12 @@ pub fn dc_job_do_DC_JOB_CONFIGURE_IMAP(context: &Context) {
         }
     }
     if imap_connected_here {
-        context.inbox.read().unwrap().disconnect(context);
+        context
+            .inbox_thread
+            .read()
+            .unwrap()
+            .imap
+            .disconnect(context);
     }
     if smtp_connected_here {
         context.smtp.clone().lock().unwrap().disconnect();
@@ -421,6 +433,16 @@ pub fn dc_job_do_DC_JOB_CONFIGURE_IMAP(context: &Context) {
         );
     }
     */
+
+    // remember the entered parameters on success
+    // and restore to last-entered on failure.
+    // this way, the parameters visible to the ui are always in-sync with the current configuration.
+    if success {
+        LoginParam::from_database(context, "").save_to_database(context, "configured_raw_");
+    } else {
+        LoginParam::from_database(context, "configured_raw_").save_to_database(context, "");
+    }
+
     context.free_ongoing();
     progress!(context, if success { 1000 } else { 0 });
 }
@@ -484,7 +506,13 @@ fn try_imap_one_param(context: &Context, param: &LoginParam) -> Option<bool> {
         param.mail_user, param.mail_server, param.mail_port, param.server_flags
     );
     info!(context, "Trying: {}", inf);
-    if context.inbox.read().unwrap().connect(context, &param) {
+    if context
+        .inbox_thread
+        .read()
+        .unwrap()
+        .imap
+        .connect(context, &param)
+    {
         info!(context, "success: {}", inf);
         return Some(true);
     }
@@ -531,43 +559,26 @@ fn try_smtp_one_param(context: &Context, param: &LoginParam) -> Option<bool> {
         param.send_user, param.send_server, param.send_port, param.server_flags
     );
     info!(context, "Trying: {}", inf);
-    if context
+    match context
         .smtp
         .clone()
         .lock()
         .unwrap()
         .connect(context, &param)
     {
-        info!(context, "success: {}", inf);
-        return Some(true);
-    }
-    if context.shall_stop_ongoing() {
-        return Some(false);
-    }
-    info!(context, "could not connect: {}", inf);
-    None
-}
-
-/*******************************************************************************
- * Connect to configured account
- ******************************************************************************/
-pub fn dc_connect_to_configured_imap(context: &Context, imap: &Imap) -> libc::c_int {
-    let mut ret_connected = 0;
-
-    if imap.is_connected() {
-        ret_connected = 1
-    } else if !context.sql.get_raw_config_bool(context, "configured") {
-        warn!(context, "Not configured, cannot connect.",);
-    } else {
-        let param = LoginParam::from_database(context, "configured_");
-        // the trailing underscore is correct
-
-        if imap.connect(context, &param) {
-            ret_connected = 2;
+        Ok(()) => {
+            info!(context, "success: {}", inf);
+            Some(true)
+        }
+        Err(err) => {
+            if context.shall_stop_ongoing() {
+                Some(false)
+            } else {
+                warn!(context, "could not connect: {}", err);
+                None
+            }
         }
     }
-
-    ret_connected
 }
 
 /*******************************************************************************

src/constants.rs

@@ -1,4 +1,4 @@
-//! Constants
+//! # Constants
 #![allow(non_camel_case_types, dead_code)]
 
 use deltachat_derive::*;
@@ -130,7 +130,8 @@ const DC_MAX_GET_INFO_LEN: usize = 100000;
 
 pub const DC_CONTACT_ID_UNDEFINED: u32 = 0;
 pub const DC_CONTACT_ID_SELF: u32 = 1;
-pub const DC_CONTACT_ID_DEVICE: u32 = 2;
+pub const DC_CONTACT_ID_INFO: u32 = 2;
+pub const DC_CONTACT_ID_DEVICE: u32 = 5;
 pub const DC_CONTACT_ID_LAST_SPECIAL: u32 = 9;
 
 pub const DC_CREATE_MVBOX: usize = 1;

src/contact.rs

@@ -47,8 +47,8 @@ pub struct Contact {
     /// May be empty, initially set to `authname`.
     name: String,
     /// Name authorized by the contact himself. Only this name may be spread to others,
-    /// e.g. in To:-lists. May be empty. It is recommended to use `Contact::get_name`,
-    /// `Contact::get_display_name` or `Contact::get_name_n_addr` to access this field.
+    /// e.g. in To:-lists. May be empty. It is recommended to use `Contact::get_authname`,
+    /// to access this field.
     authname: String,
     /// E-Mail-Address of the contact. It is recommended to use `Contact::get_addr`` to access this field.
     addr: String,
@@ -153,7 +153,16 @@ impl Contact {
                 blocked: false,
                 origin: Origin::Unknown,
             };
-
+            return Ok(contact);
+        } else if contact_id == DC_CONTACT_ID_DEVICE {
+            let contact = Contact {
+                id: contact_id,
+                name: context.stock_str(StockMessage::DeviceMessages).into(),
+                authname: "".into(),
+                addr: "device@localhost".into(),
+                blocked: false,
+                origin: Origin::Unknown,
+            };
             return Ok(contact);
         }
 
@@ -264,7 +273,7 @@ impl Contact {
             .unwrap_or_default();
 
         if addr_normalized == addr_self {
-            return 1;
+            return DC_CONTACT_ID_SELF;
         }
 
         context.sql.query_get_value(
@@ -301,7 +310,7 @@ impl Contact {
             .unwrap_or_default();
 
         if addr == addr_self {
-            return Ok((1, sth_modified));
+            return Ok((DC_CONTACT_ID_SELF, sth_modified));
         }
 
         if !may_be_valid_addr(&addr) {
@@ -708,6 +717,7 @@ impl Contact {
         &self.addr
     }
 
+    /// Get name authorized by the contact.
     pub fn get_authname(&self) -> &str {
         &self.authname
     }
@@ -866,7 +876,7 @@ impl Contact {
     }
 
     pub fn real_exists_by_id(context: &Context, contact_id: u32) -> bool {
-        if !context.sql.is_open() || contact_id <= 9 {
+        if !context.sql.is_open() || contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
             return false;
         }
 
@@ -890,6 +900,7 @@ impl Contact {
     }
 }
 
+/// Extracts first name from full name.
 fn get_first_name<'a>(full_name: &'a str) -> &'a str {
     full_name.splitn(2, ' ').next().unwrap_or_default()
 }
@@ -900,6 +911,7 @@ pub fn may_be_valid_addr(addr: &str) -> bool {
     res.is_ok()
 }
 
+/// Returns address with whitespace trimmed and `mailto:` prefix removed.
 pub fn addr_normalize(addr: &str) -> &str {
     let norm = addr.trim();
 
@@ -911,7 +923,7 @@ pub fn addr_normalize(addr: &str) -> &str {
 }
 
 fn set_block_contact(context: &Context, contact_id: u32, new_blocking: bool) {
-    if contact_id <= 9 {
+    if contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
         return;
     }
 
@@ -1044,6 +1056,8 @@ fn split_address_book(book: &str) -> Vec<(&str, &str)> {
 mod tests {
     use super::*;
 
+    use crate::test_utils::*;
+
     #[test]
     fn test_may_be_valid_addr() {
         assert_eq!(may_be_valid_addr(""), false);
@@ -1085,4 +1099,109 @@ mod tests {
             vec![("Name one", "Address one"), ("Name two", "Address two")]
         )
     }
+
+    #[test]
+    fn test_get_contacts() {
+        let context = dummy_context();
+        let contacts = Contact::get_all(&context.ctx, 0, Some("some2")).unwrap();
+        assert_eq!(contacts.len(), 0);
+
+        let id = Contact::create(&context.ctx, "bob", "bob@mail.de").unwrap();
+        assert_ne!(id, 0);
+
+        let contacts = Contact::get_all(&context.ctx, 0, Some("bob")).unwrap();
+        assert_eq!(contacts.len(), 1);
+
+        let contacts = Contact::get_all(&context.ctx, 0, Some("alice")).unwrap();
+        assert_eq!(contacts.len(), 0);
+    }
+
+    #[test]
+    fn test_add_or_lookup() {
+        // add some contacts, this also tests add_address_book()
+        let t = dummy_context();
+        let book = concat!(
+            "  Name one  \n one@eins.org \n",
+            "Name two\ntwo@deux.net\n",
+            "\nthree@drei.sam\n",
+            "Name two\ntwo@deux.net\n" // should not be added again
+        );
+        assert_eq!(Contact::add_address_book(&t.ctx, book).unwrap(), 3);
+
+        // check first added contact, this does not modify because of lower origin
+        let (contact_id, sth_modified) =
+            Contact::add_or_lookup(&t.ctx, "bla foo", "one@eins.org", Origin::IncomingUnknownTo)
+                .unwrap();
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert_eq!(sth_modified, Modifier::None);
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_id(), contact_id);
+        assert_eq!(contact.get_name(), "Name one");
+        assert_eq!(contact.get_display_name(), "Name one");
+        assert_eq!(contact.get_addr(), "one@eins.org");
+        assert_eq!(contact.get_name_n_addr(), "Name one (one@eins.org)");
+
+        // modify first added contact
+        let (contact_id_test, sth_modified) = Contact::add_or_lookup(
+            &t.ctx,
+            "Real one",
+            " one@eins.org  ",
+            Origin::ManuallyCreated,
+        )
+        .unwrap();
+        assert_eq!(contact_id, contact_id_test);
+        assert_eq!(sth_modified, Modifier::Modified);
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_name(), "Real one");
+        assert_eq!(contact.get_addr(), "one@eins.org");
+        assert!(!contact.is_blocked());
+
+        // check third added contact (contact without name)
+        let (contact_id, sth_modified) =
+            Contact::add_or_lookup(&t.ctx, "", "three@drei.sam", Origin::IncomingUnknownTo)
+                .unwrap();
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert_eq!(sth_modified, Modifier::None);
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_name(), "");
+        assert_eq!(contact.get_display_name(), "three@drei.sam");
+        assert_eq!(contact.get_addr(), "three@drei.sam");
+        assert_eq!(contact.get_name_n_addr(), "three@drei.sam");
+
+        // add name to third contact from incoming message (this becomes authorized name)
+        let (contact_id_test, sth_modified) = Contact::add_or_lookup(
+            &t.ctx,
+            "m. serious",
+            "three@drei.sam",
+            Origin::IncomingUnknownFrom,
+        )
+        .unwrap();
+        assert_eq!(contact_id, contact_id_test);
+        assert_eq!(sth_modified, Modifier::Modified);
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_name_n_addr(), "m. serious (three@drei.sam)");
+        assert!(!contact.is_blocked());
+
+        // manually edit name of third contact (does not changed authorized name)
+        let (contact_id_test, sth_modified) = Contact::add_or_lookup(
+            &t.ctx,
+            "schnucki",
+            "three@drei.sam",
+            Origin::ManuallyCreated,
+        )
+        .unwrap();
+        assert_eq!(contact_id, contact_id_test);
+        assert_eq!(sth_modified, Modifier::Modified);
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_authname(), "m. serious");
+        assert_eq!(contact.get_name_n_addr(), "schnucki (three@drei.sam)");
+        assert!(!contact.is_blocked());
+
+        // check SELF
+        let contact = Contact::load_from_db(&t.ctx, DC_CONTACT_ID_SELF).unwrap();
+        assert_eq!(DC_CONTACT_ID_SELF, 1);
+        assert_eq!(contact.get_name(), t.ctx.stock_str(StockMessage::SelfMsg));
+        assert_eq!(contact.get_addr(), ""); // we're not configured
+        assert!(!contact.is_blocked());
+    }
 }

src/context.rs

@@ -39,12 +39,14 @@ pub type ContextCallback = dyn Fn(&Context, Event) -> uintptr_t + Send + Sync;
 
 #[derive(DebugStub)]
 pub struct Context {
+    /// Database file path
     dbfile: PathBuf,
+    /// Blob directory path
     blobdir: PathBuf,
     pub sql: Sql,
-    pub inbox: Arc<RwLock<Imap>>,
     pub perform_inbox_jobs_needed: Arc<RwLock<bool>>,
     pub probe_imap_network: Arc<RwLock<bool>>,
+    pub inbox_thread: Arc<RwLock<JobThread>>,
     pub sentbox_thread: Arc<RwLock<JobThread>>,
     pub mvbox_thread: Arc<RwLock<JobThread>>,
     pub smtp: Arc<Mutex<Smtp>>,
@@ -55,7 +57,7 @@ pub struct Context {
     pub os_name: Option<String>,
     pub cmdline_sel_chat_id: Arc<RwLock<u32>>,
     pub bob: Arc<RwLock<BobStatus>>,
-    pub last_smeared_timestamp: Arc<RwLock<i64>>,
+    pub last_smeared_timestamp: RwLock<i64>,
     pub running_state: Arc<RwLock<RunningState>>,
     /// Mutex to avoid generating the key for the user more than once.
     pub generating_key_mutex: Mutex<()>,
@@ -93,6 +95,7 @@ pub fn get_info() -> HashMap<&'static str, String> {
 }
 
 impl Context {
+    /// Creates new context.
     pub fn new(cb: Box<ContextCallback>, os_name: String, dbfile: PathBuf) -> Result<Context> {
         let mut blob_fname = OsString::new();
         blob_fname.push(dbfile.file_name().unwrap_or_default());
@@ -118,7 +121,6 @@ impl Context {
         let ctx = Context {
             blobdir,
             dbfile,
-            inbox: Arc::new(RwLock::new(Imap::new())),
             cb,
             os_name: Some(os_name),
             running_state: Arc::new(RwLock::new(Default::default())),
@@ -127,8 +129,13 @@ impl Context {
             smtp_state: Arc::new((Mutex::new(Default::default()), Condvar::new())),
             oauth2_critical: Arc::new(Mutex::new(())),
             bob: Arc::new(RwLock::new(Default::default())),
-            last_smeared_timestamp: Arc::new(RwLock::new(0)),
+            last_smeared_timestamp: RwLock::new(0),
             cmdline_sel_chat_id: Arc::new(RwLock::new(0)),
+            inbox_thread: Arc::new(RwLock::new(JobThread::new(
+                "INBOX",
+                "configured_inbox_folder",
+                Imap::new(),
+            ))),
             sentbox_thread: Arc::new(RwLock::new(JobThread::new(
                 "SENTBOX",
                 "configured_sentbox_folder",
@@ -153,10 +160,12 @@ impl Context {
         Ok(ctx)
     }
 
+    /// Returns database file path.
     pub fn get_dbfile(&self) -> &Path {
         self.dbfile.as_path()
     }
 
+    /// Returns blob directory path.
     pub fn get_blobdir(&self) -> &Path {
         self.blobdir.as_path()
     }
@@ -458,8 +467,8 @@ impl Context {
 
 impl Drop for Context {
     fn drop(&mut self) {
-        info!(self, "disconnecting INBOX-watch",);
-        self.inbox.read().unwrap().disconnect(self);
+        info!(self, "disconnecting inbox-thread",);
+        self.inbox_thread.read().unwrap().imap.disconnect(self);
         info!(self, "disconnecting sentbox-thread",);
         self.sentbox_thread.read().unwrap().imap.disconnect(self);
         info!(self, "disconnecting mvbox-thread",);

src/dc_mimeparser.rs

@@ -117,18 +117,30 @@ impl<'a> MimeParser<'a> {
         );
 
         if r == MAILIMF_NO_ERROR as libc::c_int && !self.mimeroot.is_null() {
-            let (encrypted, signatures, gossipped_addr) =
-                e2ee::try_decrypt(self.context, self.mimeroot)?;
-            self.encrypted = encrypted;
-            self.signatures = signatures;
-            self.gossipped_addr = gossipped_addr;
+            match e2ee::try_decrypt(self.context, self.mimeroot) {
+                Ok((encrypted, signatures, gossipped_addr)) => {
+                    self.encrypted = encrypted;
+                    self.signatures = signatures;
+                    self.gossipped_addr = gossipped_addr;
+                }
+                Err(err) => {
+                    // continue with the current, still encrypted, mime tree.
+                    // unencrypted parts will be replaced by an error message
+                    // that is added as "the message" to the chat then.
+                    //
+                    // if we just return here, the header is missing
+                    // and the caller cannot display the message
+                    // and try to assign the message to a chat
+                    warn!(self.context, "decryption failed: {}", err);
+                }
+            }
+
             self.parse_mime_recursive(self.mimeroot);
 
             if let Some(field) = self.lookup_field("Subject") {
                 if (*field).fld_type == MAILIMF_FIELD_SUBJECT as libc::c_int {
                     let subj = (*(*field).fld_data.fld_subject).sbj_value;
-
-                    self.subject = as_opt_str(subj).map(dc_decode_header_words);
+                    self.subject = to_opt_string_lossy(subj).map(|x| dc_decode_header_words(&x));
                 }
             }
 
@@ -599,7 +611,7 @@ impl<'a> MimeParser<'a> {
         let mut decoded_data = match wrapmime::mailmime_transfer_decode(mime) {
             Ok(decoded_data) => decoded_data,
             Err(_) => {
-                // Note that it's now always an error - might be no data
+                // Note that it's not always an error - might be no data
                 return false;
             }
         };
@@ -630,7 +642,7 @@ impl<'a> MimeParser<'a> {
                             self.context,
                             "Cannot convert {} bytes from \"{}\" to \"utf-8\".",
                             decoded_data.len(),
-                            as_str(charset),
+                            to_string_lossy(charset),
                         );
                     }
                 }
@@ -729,33 +741,21 @@ impl<'a> MimeParser<'a> {
                     if !(*mime).mm_content_type.is_null()
                         && !(*(*mime).mm_content_type).ct_subtype.is_null()
                     {
-                        desired_filename =
-                            format!("file.{}", as_str((*(*mime).mm_content_type).ct_subtype));
+                        desired_filename = format!(
+                            "file.{}",
+                            to_string_lossy((*(*mime).mm_content_type).ct_subtype)
+                        );
                     } else {
                         return false;
                     }
                 }
-                if desired_filename.starts_with("location") && desired_filename.ends_with(".kml") {
-                    if !decoded_data.is_empty() {
-                        let d = std::string::String::from_utf8_lossy(&decoded_data);
-                        self.location_kml = location::Kml::parse(self.context, &d).ok();
-                    }
-                } else if desired_filename.starts_with("message")
-                    && desired_filename.ends_with(".kml")
-                {
-                    if !decoded_data.is_empty() {
-                        let d = std::string::String::from_utf8_lossy(&decoded_data);
-                        self.message_kml = location::Kml::parse(self.context, &d).ok();
-                    }
-                } else if !decoded_data.is_empty() {
-                    self.do_add_single_file_part(
-                        msg_type,
-                        mime_type,
-                        raw_mime.as_ref(),
-                        &decoded_data,
-                        &desired_filename,
-                    );
-                }
+                self.do_add_single_file_part(
+                    msg_type,
+                    mime_type,
+                    raw_mime.as_ref(),
+                    &decoded_data,
+                    &desired_filename,
+                );
             }
             _ => {}
         }
@@ -769,20 +769,44 @@ impl<'a> MimeParser<'a> {
         mime_type: libc::c_int,
         raw_mime: Option<&String>,
         decoded_data: &[u8],
-        desired_filename: &str,
+        filename: &str,
     ) {
-        /* write decoded data to new blob file */
-        let blob = match BlobObject::create(self.context, desired_filename, decoded_data) {
+        if decoded_data.is_empty() {
+            return;
+        }
+        // treat location/message kml file attachments specially
+        if filename.ends_with(".kml") {
+            // XXX what if somebody sends eg an "location-highlights.kml"
+            // attachment unrelated to location streaming?
+            if filename.starts_with("location") || filename.starts_with("message") {
+                let parsed = location::Kml::parse(self.context, decoded_data)
+                    .map_err(|err| {
+                        warn!(self.context, "failed to parse kml part: {}", err);
+                    })
+                    .ok();
+                if filename.starts_with("location") {
+                    self.location_kml = parsed;
+                } else {
+                    self.message_kml = parsed;
+                }
+                return;
+            }
+        }
+        /* we have a regular file attachment,
+        write decoded data to new blob object */
+
+        let blob = match BlobObject::create(self.context, filename, decoded_data) {
             Ok(blob) => blob,
             Err(err) => {
                 error!(
                     self.context,
-                    "Could not add blob for mime part {}, error {}", desired_filename, err
+                    "Could not add blob for mime part {}, error {}", filename, err
                 );
                 return;
             }
         };
 
+        /* create and register Mime part referencing the new Blob object */
         let mut part = Part::default();
         part.typ = msg_type;
         part.mimetype = mime_type;
@@ -802,10 +826,15 @@ impl<'a> MimeParser<'a> {
     }
 
     fn do_add_single_part(&mut self, mut part: Part) {
-        if self.encrypted && self.signatures.len() > 0 {
-            part.param.set_int(Param::GuaranteeE2ee, 1);
-        } else if self.encrypted {
-            part.param.set_int(Param::ErroneousE2ee, 0x2);
+        if self.encrypted {
+            if self.signatures.len() > 0 {
+                part.param.set_int(Param::GuaranteeE2ee, 1);
+            } else {
+                // XXX if the message was encrypted but not signed
+                // it's not neccessarily an error we need to signal.
+                // we could just treat it as if it was not encrypted.
+                part.param.set_int(Param::ErroneousE2ee, 0x2);
+            }
         }
         self.parts.push(part);
     }
@@ -850,7 +879,8 @@ impl<'a> MimeParser<'a> {
             }) as *mut mailimf_mailbox;
 
             if !mb.is_null() {
-                let from_addr_norm = addr_normalize(as_str((*mb).mb_addr_spec));
+                let from_addr = to_string_lossy((*mb).mb_addr_spec);
+                let from_addr_norm = addr_normalize(&from_addr);
                 let recipients = wrapmime::mailimf_get_recipients(self.header_root);
                 if recipients.len() == 1 && recipients.contains(from_addr_norm) {
                     sender_equals_recipient = true;
@@ -989,15 +1019,16 @@ unsafe fn mailmime_get_mime_type(mime: *mut Mailmime) -> (libc::c_int, Viewtype,
                     }
                 }
 
-                let raw_mime = reconcat_mime(Some("text"), as_opt_str((*c).ct_subtype));
+                let raw_mime = reconcat_mime(Some("text"), to_opt_string_lossy((*c).ct_subtype));
                 (DC_MIMETYPE_FILE, Viewtype::File, Some(raw_mime))
             }
             MAILMIME_DISCRETE_TYPE_IMAGE => {
-                let subtype = as_opt_str((*c).ct_subtype);
-                let msg_type = match subtype {
+                let subtype = to_opt_string_lossy((*c).ct_subtype);
+                let msg_type = match subtype.as_ref().map(|x| x.as_str()) {
                     Some("gif") => Viewtype::Gif,
                     Some("svg+xml") => {
-                        let raw_mime = reconcat_mime(Some("image"), as_opt_str((*c).ct_subtype));
+                        let raw_mime =
+                            reconcat_mime(Some("image"), to_opt_string_lossy((*c).ct_subtype));
                         return (DC_MIMETYPE_FILE, Viewtype::File, Some(raw_mime));
                     }
                     _ => Viewtype::Image,
@@ -1007,11 +1038,11 @@ unsafe fn mailmime_get_mime_type(mime: *mut Mailmime) -> (libc::c_int, Viewtype,
                 (DC_MIMETYPE_IMAGE, msg_type, Some(raw_mime))
             }
             MAILMIME_DISCRETE_TYPE_AUDIO => {
-                let raw_mime = reconcat_mime(Some("audio"), as_opt_str((*c).ct_subtype));
+                let raw_mime = reconcat_mime(Some("audio"), to_opt_string_lossy((*c).ct_subtype));
                 (DC_MIMETYPE_AUDIO, Viewtype::Audio, Some(raw_mime))
             }
             MAILMIME_DISCRETE_TYPE_VIDEO => {
-                let raw_mime = reconcat_mime(Some("video"), as_opt_str((*c).ct_subtype));
+                let raw_mime = reconcat_mime(Some("video"), to_opt_string_lossy((*c).ct_subtype));
                 (DC_MIMETYPE_VIDEO, Viewtype::Video, Some(raw_mime))
             }
             _ => {
@@ -1022,13 +1053,15 @@ unsafe fn mailmime_get_mime_type(mime: *mut Mailmime) -> (libc::c_int, Viewtype,
                         b"autocrypt-setup\x00" as *const u8 as *const libc::c_char,
                     ) == 0i32
                 {
-                    let raw_mime = reconcat_mime(None, as_opt_str((*c).ct_subtype));
+                    let raw_mime = reconcat_mime(None, to_opt_string_lossy((*c).ct_subtype));
                     return (DC_MIMETYPE_AC_SETUP_FILE, Viewtype::File, Some(raw_mime));
                 }
 
                 let raw_mime = reconcat_mime(
-                    as_opt_str((*(*(*c).ct_type).tp_data.tp_discrete_type).dt_extension),
-                    as_opt_str((*c).ct_subtype),
+                    to_opt_string_lossy((*(*(*c).ct_type).tp_data.tp_discrete_type).dt_extension)
+                        .as_ref()
+                        .map(|x| x.as_str()),
+                    to_opt_string_lossy((*c).ct_subtype),
                 );
 
                 (DC_MIMETYPE_FILE, Viewtype::File, Some(raw_mime))
@@ -1038,9 +1071,9 @@ unsafe fn mailmime_get_mime_type(mime: *mut Mailmime) -> (libc::c_int, Viewtype,
             if (*(*(*c).ct_type).tp_data.tp_composite_type).ct_type
                 == MAILMIME_COMPOSITE_TYPE_MULTIPART as libc::c_int
             {
-                let subtype = as_opt_str((*c).ct_subtype);
+                let subtype = to_opt_string_lossy((*c).ct_subtype);
 
-                let mime_type = match subtype {
+                let mime_type = match subtype.as_ref().map(|x| x.as_str()) {
                     Some("alternative") => DC_MIMETYPE_MP_ALTERNATIVE,
                     Some("related") => DC_MIMETYPE_MP_RELATED,
                     Some("encrypted") => {
@@ -1075,9 +1108,9 @@ unsafe fn mailmime_get_mime_type(mime: *mut Mailmime) -> (libc::c_int, Viewtype,
     }
 }
 
-fn reconcat_mime(typ: Option<&str>, subtype: Option<&str>) -> String {
+fn reconcat_mime(typ: Option<&str>, subtype: Option<String>) -> String {
     let typ = typ.unwrap_or("application");
-    let subtype = subtype.unwrap_or("octet-stream");
+    let subtype = subtype.unwrap_or("octet-stream".to_string());
 
     format!("{}/{}", typ, subtype)
 }
@@ -1224,13 +1257,13 @@ mod tests {
         #[ignore]
         #[test]
         fn test_dc_mailmime_parse_crash_fuzzy(data in "[!-~\t ]{2000,}") {
-            // this test doesn't exercise much of dc_mimeparser anymore
-            // because a missing From-field early aborts parsing
             let context = dummy_context();
             let mut mimeparser = MimeParser::new(&context.ctx);
-            unsafe {
-                assert!(mimeparser.parse(data.as_bytes()).is_err());
-            }
+
+            // parsing should always succeed
+            // but the returned header will normally be empty on random data
+            assert!(unsafe {mimeparser.parse(data.as_bytes()).is_ok()});
+            assert!(mimeparser.header.is_empty());
         }
     }
 

src/dc_receive_imf.rs

@@ -65,12 +65,12 @@ pub unsafe fn dc_receive_imf(
 
     let mut mime_parser = MimeParser::new(context);
     if let Err(err) = mime_parser.parse(imf_raw) {
-        error!(context, "dc_receive_imf parse error: {}", err);
+        warn!(context, "dc_receive_imf parse error: {}", err);
     };
 
     if mime_parser.header.is_empty() {
         // Error - even adding an empty record won't help as we do not know the message ID
-        info!(context, "No header.");
+        warn!(context, "No header.");
         return;
     }
 
@@ -227,7 +227,7 @@ pub unsafe fn dc_receive_imf(
             &mut created_db_entries,
             &mut create_event_to_send,
         ) {
-            info!(context, "{}", err);
+            warn!(context, "{}", err);
 
             cleanup(
                 context,
@@ -257,7 +257,7 @@ pub unsafe fn dc_receive_imf(
         );
     }
 
-    if !mime_parser.message_kml.is_none() && chat_id > DC_CHAT_ID_LAST_SPECIAL {
+    if mime_parser.location_kml.is_some() || mime_parser.message_kml.is_some() {
         save_locations(
             context,
             &mime_parser,
@@ -391,7 +391,7 @@ unsafe fn add_parts(
         } else {
             MessageState::InFresh
         };
-        *to_id = 1;
+        *to_id = DC_CONTACT_ID_SELF;
         // handshake messages must be processed _before_ chats are created
         // (eg. contacs may be marked as verified)
         if mime_parser.lookup_field("Secure-Join").is_some() {
@@ -551,8 +551,9 @@ unsafe fn add_parts(
             if to_ids.is_empty() && 0 != to_self {
                 // from_id==to_id==DC_CONTACT_ID_SELF - this is a self-sent messages,
                 // maybe an Autocrypt Setup Messag
-                let (id, bl) = chat::create_or_lookup_by_contact_id(context, 1, Blocked::Not)
-                    .unwrap_or_default();
+                let (id, bl) =
+                    chat::create_or_lookup_by_contact_id(context, DC_CONTACT_ID_SELF, Blocked::Not)
+                        .unwrap_or_default();
                 *chat_id = id;
                 chat_id_blocked = bl;
 
@@ -604,6 +605,7 @@ unsafe fn add_parts(
     // into only one message; mails sent by other clients may result in several messages
     // (eg. one per attachment))
     let icnt = mime_parser.parts.len();
+
     let mut txt_raw = None;
 
     context.sql.prepare(
@@ -644,17 +646,6 @@ unsafe fn add_parts(
                         .set_int(Param::Cmd, mime_parser.is_system_message as i32);
                 }
 
-                /*
-                info!(
-                    context,
-                    "received mime message {:?}",
-                    String::from_utf8_lossy(std::slice::from_raw_parts(
-                        imf_raw_not_terminated as *const u8,
-                        imf_raw_bytes,
-                    ))
-                );
-                */
-
                 stmt.execute(params![
                     rfc724_mid,
                     server_folder.as_ref(),
@@ -808,9 +799,12 @@ unsafe fn handle_reports(
                                     && !of_org_msgid.is_null()
                                     && !(*of_org_msgid).fld_value.is_null()
                                 {
-                                    if let Ok(rfc724_mid) = wrapmime::parse_message_id(as_str(
-                                        (*of_org_msgid).fld_value,
-                                    )) {
+                                    if let Ok(rfc724_mid) =
+                                        wrapmime::parse_message_id(std::slice::from_raw_parts(
+                                            (*of_org_msgid).fld_value as *const u8,
+                                            libc::strlen((*of_org_msgid).fld_value),
+                                        ))
+                                    {
                                         if let Some((chat_id, msg_id)) = message::mdn_from_ext(
                                             context,
                                             from_id,
@@ -850,13 +844,16 @@ fn save_locations(
     insert_msg_id: MsgId,
     hidden: i32,
 ) {
+    if chat_id <= DC_CHAT_ID_LAST_SPECIAL {
+        return ();
+    }
     let mut location_id_written = false;
     let mut send_event = false;
 
-    if !mime_parser.message_kml.is_none() && chat_id > DC_CHAT_ID_LAST_SPECIAL as libc::c_uint {
+    if mime_parser.message_kml.is_some() {
         let locations = &mime_parser.message_kml.as_ref().unwrap().locations;
         let newest_location_id =
-            location::save(context, chat_id, from_id, locations, 1).unwrap_or_default();
+            location::save(context, chat_id, from_id, locations, true).unwrap_or_default();
         if 0 != newest_location_id && 0 == hidden {
             if location::set_msg_location_id(context, insert_msg_id, newest_location_id).is_ok() {
                 location_id_written = true;
@@ -865,15 +862,14 @@ fn save_locations(
         }
     }
 
-    if !mime_parser.location_kml.is_none() && chat_id > DC_CHAT_ID_LAST_SPECIAL as libc::c_uint {
+    if mime_parser.location_kml.is_some() {
         if let Some(ref addr) = mime_parser.location_kml.as_ref().unwrap().addr {
             if let Ok(contact) = Contact::get_by_id(context, from_id) {
-                if !contact.get_addr().is_empty()
-                    && contact.get_addr().to_lowercase() == addr.to_lowercase()
-                {
+                if contact.get_addr().to_lowercase() == addr.to_lowercase() {
                     let locations = &mime_parser.location_kml.as_ref().unwrap().locations;
                     let newest_location_id =
-                        location::save(context, chat_id, from_id, locations, 0).unwrap_or_default();
+                        location::save(context, chat_id, from_id, locations, false)
+                            .unwrap_or_default();
                     if newest_location_id != 0 && hidden == 0 && !location_id_written {
                         if let Err(err) = location::set_msg_location_id(
                             context,
@@ -993,7 +989,7 @@ unsafe fn create_or_lookup_group(
             let fld_message_id = (*field).fld_data.fld_message_id;
             if !fld_message_id.is_null() {
                 if let Some(extracted_grpid) =
-                    dc_extract_grpid_from_rfc724_mid(as_str((*fld_message_id).mid_value))
+                    dc_extract_grpid_from_rfc724_mid(&to_string_lossy((*fld_message_id).mid_value))
                 {
                     grpid = extracted_grpid.to_string();
                 } else {
@@ -1359,8 +1355,8 @@ unsafe fn create_or_lookup_adhoc_group(
     if !member_ids.contains(&from_id) {
         member_ids.push(from_id);
     }
-    if !member_ids.contains(&1) {
-        member_ids.push(1);
+    if !member_ids.contains(&DC_CONTACT_ID_SELF) {
+        member_ids.push(DC_CONTACT_ID_SELF);
     }
     if member_ids.len() < 3 {
         // too few contacts given
@@ -1480,7 +1476,7 @@ fn create_adhoc_grp_id(context: &Context, member_ids: &[u32]) -> String {
         .sql
         .query_map(
             format!(
-                "SELECT addr FROM contacts WHERE id IN({}) AND id!=1",
+                "SELECT addr FROM contacts WHERE id IN({}) AND id!=1", // 1=DC_CONTACT_ID_SELF
                 member_ids_str
             ),
             params![],
@@ -1534,7 +1530,7 @@ fn search_chat_ids_by_contact_ids(
                        WHERE cc.chat_id IN(SELECT chat_id FROM chats_contacts WHERE contact_id IN({})) \
                          AND c.type=120 \
                          AND cc.contact_id!=1 \
-                       ORDER BY cc.chat_id, cc.contact_id;",
+                       ORDER BY cc.chat_id, cc.contact_id;", // 1=DC_CONTACT_ID_SELF
                     contact_ids_str
                 ),
                 params![],
@@ -1731,7 +1727,7 @@ fn is_known_rfc724_mid(context: &Context, rfc724_mid: *const libc::c_char) -> li
              LEFT JOIN chats c ON m.chat_id=c.id  \
              WHERE m.rfc724_mid=?  \
              AND m.chat_id>9 AND c.blocked=0;",
-            params![as_str(rfc724_mid)],
+            params![to_string_lossy(rfc724_mid)],
         )
         .unwrap_or_default() as libc::c_int
 }
@@ -1783,11 +1779,7 @@ unsafe fn is_msgrmsg_rfc724_mid_in_list(context: &Context, mid_list: *const clis
         while !cur.is_null() {
             if 0 != is_msgrmsg_rfc724_mid(
                 context,
-                if !cur.is_null() {
-                    as_str((*cur).data as *const libc::c_char)
-                } else {
-                    ""
-                },
+                &to_string_lossy((*cur).data as *const libc::c_char),
             ) {
                 return 1;
             }
@@ -1923,7 +1915,7 @@ unsafe fn add_or_lookup_contact_by_addr(
         .get_config(Config::ConfiguredAddr)
         .unwrap_or_default();
 
-    if addr_cmp(self_addr, as_str(addr_spec)) {
+    if addr_cmp(self_addr, to_string_lossy(addr_spec)) {
         *check_self = 1;
     }
 
@@ -1933,13 +1925,18 @@ unsafe fn add_or_lookup_contact_by_addr(
     /* add addr_spec if missing, update otherwise */
     let mut display_name_dec = "".to_string();
     if !display_name_enc.is_null() {
-        let tmp = dc_decode_header_words(as_str(display_name_enc));
+        let tmp = dc_decode_header_words(&to_string_lossy(display_name_enc));
         display_name_dec = normalize_name(&tmp);
     }
     /*can be NULL*/
-    let row_id = Contact::add_or_lookup(context, display_name_dec, as_str(addr_spec), origin)
-        .map(|(id, _)| id)
-        .unwrap_or_default();
+    let row_id = Contact::add_or_lookup(
+        context,
+        display_name_dec,
+        to_string_lossy(addr_spec),
+        origin,
+    )
+    .map(|(id, _)| id)
+    .unwrap_or_default();
     if 0 != row_id && !ids.contains(&row_id) {
         ids.push(row_id);
     };

src/dc_strencode.rs

@@ -38,7 +38,7 @@ pub fn dc_encode_header_words(input: impl AsRef<str>) -> String {
 fn must_encode(byte: u8) -> bool {
     static SPECIALS: &[u8] = b",:!\"#$@[\\]^`{|}~=?_";
 
-    SPECIALS.into_iter().any(|b| *b == byte)
+    SPECIALS.iter().any(|b| *b == byte)
 }
 
 fn quote_word(word: &[u8]) -> String {

src/dc_tools.rs

@@ -1,7 +1,7 @@
 //! Some tools and enhancements to the used libraries, there should be
 //! no references to Context and other "larger" entities here.
 
-use core::cmp::max;
+use core::cmp::{max, min};
 use std::borrow::Cow;
 use std::ffi::{CStr, CString};
 use std::path::{Path, PathBuf};
@@ -82,7 +82,7 @@ pub(crate) fn dc_str_from_clist(list: *const clist, delimiter: &str) -> String {
             if !res.is_empty() {
                 res += delimiter;
             }
-            res += as_str(rfc724_mid as *const libc::c_char);
+            res += &to_string_lossy(rfc724_mid as *const libc::c_char);
         }
     }
     res
@@ -154,20 +154,37 @@ pub(crate) fn dc_timestamp_from_date(date_time: *mut mailimf_date_time) -> i64 {
  ******************************************************************************/
 
 pub fn dc_timestamp_to_str(wanted: i64) -> String {
-    let ts = chrono::Utc.timestamp(wanted, 0);
+    let ts = Local.timestamp(wanted, 0);
     ts.format("%Y.%m.%d %H:%M:%S").to_string()
 }
 
 pub(crate) fn dc_gm2local_offset() -> i64 {
+    /* returns the offset that must be _added_ to an UTC/GMT-time to create the localtime.
+    the function may return negative values. */
     let lt = Local::now();
-    ((lt.offset().local_minus_utc() / (60 * 60)) * 100) as i64
+    lt.offset().local_minus_utc() as i64
 }
 
-/* timesmearing */
+// timesmearing
+// - as e-mails typically only use a second-based-resolution for timestamps,
+//   the order of two mails sent withing one second is unclear.
+//   this is bad eg. when forwarding some messages from a chat -
+//   these messages will appear at the recipient easily out of order.
+// - we work around this issue by not sending out two mails with the same timestamp.
+// - for this purpose, in short, we track the last timestamp used in `last_smeared_timestamp`
+//   when another timestamp is needed in the same second, we use `last_smeared_timestamp+1`
+// - after some moments without messages sent out,
+//   `last_smeared_timestamp` is again in sync with the normal time.
+// - however, we do not do all this for the far future,
+//   but at max `MAX_SECONDS_TO_LEND_FROM_FUTURE`
+const MAX_SECONDS_TO_LEND_FROM_FUTURE: i64 = 5;
+
+// returns the currently smeared timestamp,
+// may be used to check if call to dc_create_smeared_timestamp() is needed or not.
+// the returned timestamp MUST NOT be used to be sent out or saved in the database!
 pub(crate) fn dc_smeared_time(context: &Context) -> i64 {
-    /* function returns a corrected time(NULL) */
     let mut now = time();
-    let ts = *context.last_smeared_timestamp.clone().read().unwrap();
+    let ts = *context.last_smeared_timestamp.read().unwrap();
     if ts >= now {
         now = ts + 1;
     }
@@ -175,32 +192,36 @@ pub(crate) fn dc_smeared_time(context: &Context) -> i64 {
     now
 }
 
+// returns a timestamp that is guaranteed to be unique.
 pub(crate) fn dc_create_smeared_timestamp(context: &Context) -> i64 {
     let now = time();
     let mut ret = now;
 
-    let ts = *context.last_smeared_timestamp.clone().write().unwrap();
-    if ret <= ts {
-        ret = ts + 1;
-        if ret - now > 5 {
-            ret = now + 5
+    let mut last_smeared_timestamp = context.last_smeared_timestamp.write().unwrap();
+    if ret <= *last_smeared_timestamp {
+        ret = *last_smeared_timestamp + 1;
+        if ret - now > MAX_SECONDS_TO_LEND_FROM_FUTURE {
+            ret = now + MAX_SECONDS_TO_LEND_FROM_FUTURE
         }
     }
 
+    *last_smeared_timestamp = ret;
     ret
 }
 
+// creates `count` timestamps that are guaranteed to be unique.
+// the frist created timestamps is returned directly,
+// get the other timestamps just by adding 1..count-1
 pub(crate) fn dc_create_smeared_timestamps(context: &Context, count: usize) -> i64 {
-    /* get a range to timestamps that can be used uniquely */
     let now = time();
-    let start = now + (if count < 5 { count } else { 5 }) as i64 - count as i64;
+    let count = count as i64;
+    let mut start = now + min(count, MAX_SECONDS_TO_LEND_FROM_FUTURE) - count;
 
-    let ts = *context.last_smeared_timestamp.clone().write().unwrap();
-    if ts + 1 > start {
-        ts + 1
-    } else {
-        start
-    }
+    let mut last_smeared_timestamp = context.last_smeared_timestamp.write().unwrap();
+    start = max(*last_smeared_timestamp + 1, start);
+
+    *last_smeared_timestamp = start + count - 1;
+    start
 }
 
 /* Message-ID tools */
@@ -249,12 +270,10 @@ pub(crate) fn dc_create_incoming_rfc724_mid(
     contact_id_from: u32,
     contact_ids_to: &[u32],
 ) -> Option<String> {
-    if contact_ids_to.is_empty() {
-        return None;
-    }
-    /* find out the largest receiver ID (we could also take the smallest, but it should be unique) */
-    let largest_id_to = contact_ids_to.iter().max().copied().unwrap_or_default();
+    /* create a deterministic rfc724_mid from input such that
+    repeatedly calling it with the same input results in the same Message-id */
 
+    let largest_id_to = contact_ids_to.iter().max().copied().unwrap_or_default();
     let result = format!(
         "{}-{}-{}@stub",
         message_timestamp, contact_id_from, largest_id_to
@@ -303,9 +322,9 @@ pub(crate) fn dc_extract_grpid_from_rfc724_mid_list(list: *const clist) -> *mut
     if !list.is_null() {
         unsafe {
             for cur in (*list).into_iter() {
-                let mid = as_str(cur as *const libc::c_char);
+                let mid = to_string_lossy(cur as *const libc::c_char);
 
-                if let Some(grpid) = dc_extract_grpid_from_rfc724_mid(mid) {
+                if let Some(grpid) = dc_extract_grpid_from_rfc724_mid(&mid) {
                     return grpid.strdup();
                 }
             }
@@ -420,8 +439,8 @@ pub(crate) fn dc_delete_file(context: &Context, path: impl AsRef<std::path::Path
             context.call_cb(Event::DeletedBlobFile(dpath));
             true
         }
-        Err(_err) => {
-            warn!(context, "Cannot delete \"{}\".", dpath);
+        Err(err) => {
+            warn!(context, "Cannot delete \"{}\": {}", dpath, err);
             false
         }
     }
@@ -429,20 +448,55 @@ pub(crate) fn dc_delete_file(context: &Context, path: impl AsRef<std::path::Path
 
 pub(crate) fn dc_copy_file(
     context: &Context,
-    src: impl AsRef<std::path::Path>,
-    dest: impl AsRef<std::path::Path>,
+    src_path: impl AsRef<std::path::Path>,
+    dest_path: impl AsRef<std::path::Path>,
 ) -> bool {
-    let src_abs = dc_get_abs_path(context, &src);
-    let dest_abs = dc_get_abs_path(context, &dest);
-    match fs::copy(&src_abs, &dest_abs) {
+    let src_abs = dc_get_abs_path(context, &src_path);
+    let mut src_file = match fs::File::open(&src_abs) {
+        Ok(file) => file,
+        Err(err) => {
+            warn!(
+                context,
+                "failed to open for read '{}': {}",
+                src_abs.display(),
+                err
+            );
+            return false;
+        }
+    };
+
+    let dest_abs = dc_get_abs_path(context, &dest_path);
+    let mut dest_file = match fs::OpenOptions::new()
+        .create_new(true)
+        .write(true)
+        .open(&dest_abs)
+    {
+        Ok(file) => file,
+        Err(err) => {
+            warn!(
+                context,
+                "failed to open for write '{}': {}",
+                dest_abs.display(),
+                err
+            );
+            return false;
+        }
+    };
+
+    match std::io::copy(&mut src_file, &mut dest_file) {
         Ok(_) => true,
-        Err(_) => {
+        Err(err) => {
             error!(
                 context,
-                "Cannot copy \"{}\" to \"{}\".",
-                src.as_ref().display(),
-                dest.as_ref().display(),
+                "Cannot copy \"{}\" to \"{}\": {}",
+                src_abs.display(),
+                dest_abs.display(),
+                err
             );
+            {
+                // Attempt to remove the failed file, swallow errors resulting from that.
+                fs::remove_file(dest_abs).ok();
+            }
             false
         }
     }
@@ -453,11 +507,12 @@ pub(crate) fn dc_create_folder(context: &Context, path: impl AsRef<std::path::Pa
     if !path_abs.exists() {
         match fs::create_dir_all(path_abs) {
             Ok(_) => true,
-            Err(_err) => {
+            Err(err) => {
                 warn!(
                     context,
-                    "Cannot create directory \"{}\".",
+                    "Cannot create directory \"{}\": {}",
                     path.as_ref().display(),
+                    err
                 );
                 false
             }
@@ -470,12 +525,13 @@ pub(crate) fn dc_create_folder(context: &Context, path: impl AsRef<std::path::Pa
 /// Write a the given content to provied file path.
 pub(crate) fn dc_write_file(context: &Context, path: impl AsRef<Path>, buf: &[u8]) -> bool {
     let path_abs = dc_get_abs_path(context, &path);
-    if let Err(_err) = fs::write(&path_abs, buf) {
+    if let Err(err) = fs::write(&path_abs, buf) {
         warn!(
             context,
-            "Cannot write {} bytes to \"{}\".",
+            "Cannot write {} bytes to \"{}\": {}",
             buf.len(),
             path.as_ref().display(),
+            err
         );
         false
     } else {
@@ -494,8 +550,9 @@ pub fn dc_read_file<P: AsRef<std::path::Path>>(
         Err(err) => {
             warn!(
                 context,
-                "Cannot read \"{}\" or file is empty.",
-                path.as_ref().display()
+                "Cannot read \"{}\" or file is empty: {}",
+                path.as_ref().display(),
+                err
             );
             Err(err.into())
         }
@@ -513,8 +570,9 @@ pub fn dc_open_file<P: AsRef<std::path::Path>>(
         Err(err) => {
             warn!(
                 context,
-                "Cannot read \"{}\" or file is empty.",
-                path.as_ref().display()
+                "Cannot read \"{}\" or file is empty: {}",
+                path.as_ref().display(),
+                err
             );
             Err(err.into())
         }
@@ -704,27 +762,6 @@ pub fn to_opt_string_lossy(s: *const libc::c_char) -> Option<String> {
     Some(to_string_lossy(s))
 }
 
-pub fn as_str<'a>(s: *const libc::c_char) -> &'a str {
-    as_str_safe(s).unwrap_or_else(|err| panic!("{}", err))
-}
-
-/// Converts a C string to either a Rust `&str` or `None` if  it is a null pointer.
-pub fn as_opt_str<'a>(s: *const libc::c_char) -> Option<&'a str> {
-    if s.is_null() {
-        return None;
-    }
-    Some(as_str(s))
-}
-
-fn as_str_safe<'a>(s: *const libc::c_char) -> Result<&'a str, Error> {
-    assert!(!s.is_null(), "cannot be used on null pointers");
-
-    let cstr = unsafe { CStr::from_ptr(s) };
-
-    cstr.to_str()
-        .map_err(|err| format_err!("Non utf8 string: '{:?}' ({:?})", cstr.to_bytes(), err))
-}
-
 /// Convert a C `*char` pointer to a [std::path::Path] slice.
 ///
 /// This converts a `*libc::c_char` pointer to a [Path] slice.  This
@@ -761,7 +798,11 @@ pub fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path {
 #[allow(dead_code)]
 fn as_path_unicode<'a>(s: *const libc::c_char) -> &'a std::path::Path {
     assert!(!s.is_null(), "cannot be used on null pointers");
-    std::path::Path::new(as_str(s))
+
+    let cstr = unsafe { CStr::from_ptr(s) };
+    let str = cstr.to_str().unwrap_or_else(|err| panic!("{}", err));
+
+    std::path::Path::new(str)
 }
 
 pub(crate) fn time() -> i64 {
@@ -1244,6 +1285,8 @@ mod tests {
     fn test_dc_create_incoming_rfc724_mid() {
         let res = dc_create_incoming_rfc724_mid(123, 45, &vec![6, 7]);
         assert_eq!(res, Some("123-45-7@stub".into()));
+        let res = dc_create_incoming_rfc724_mid(123, 45, &vec![]);
+        assert_eq!(res, Some("123-45-0@stub".into()));
     }
 
     #[test]
@@ -1295,6 +1338,9 @@ mod tests {
 
         assert!(dc_copy_file(context, "$BLOBDIR/foobar", "$BLOBDIR/dada",));
 
+        // attempting to copy a second time should fail
+        assert!(!dc_copy_file(context, "$BLOBDIR/foobar", "$BLOBDIR/dada",));
+
         assert_eq!(dc_get_filebytes(context, "$BLOBDIR/dada",), 7);
 
         let buf = dc_read_file(context, "$BLOBDIR/dada").unwrap();
@@ -1328,4 +1374,34 @@ mod tests {
         let listflags: u32 = DC_GCL_VERIFIED_ONLY.try_into().unwrap();
         assert!(listflags_has(listflags, DC_GCL_ADD_SELF) == false);
     }
+
+    #[test]
+    fn test_create_smeared_timestamp() {
+        let t = dummy_context();
+        assert_ne!(
+            dc_create_smeared_timestamp(&t.ctx),
+            dc_create_smeared_timestamp(&t.ctx)
+        );
+        assert!(
+            dc_create_smeared_timestamp(&t.ctx)
+                >= SystemTime::now()
+                    .duration_since(SystemTime::UNIX_EPOCH)
+                    .unwrap()
+                    .as_secs() as i64
+        );
+    }
+
+    #[test]
+    fn test_create_smeared_timestamps() {
+        let t = dummy_context();
+        let count = MAX_SECONDS_TO_LEND_FROM_FUTURE - 1;
+        let start = dc_create_smeared_timestamps(&t.ctx, count as usize);
+        let next = dc_smeared_time(&t.ctx);
+        assert!((start + count - 1) < next);
+
+        let count = MAX_SECONDS_TO_LEND_FROM_FUTURE + 30;
+        let start = dc_create_smeared_timestamps(&t.ctx, count as usize);
+        let next = dc_smeared_time(&t.ctx);
+        assert!((start + count - 1) < next);
+    }
 }

src/e2ee.rs

@@ -449,7 +449,7 @@ fn update_gossip_peerstates(
 
             let optional_field = unsafe { *optional_field };
             if !optional_field.fld_name.is_null()
-                && as_str(optional_field.fld_name) == "Autocrypt-Gossip"
+                && to_string_lossy(optional_field.fld_name) == "Autocrypt-Gossip"
             {
                 let value = to_string_lossy(optional_field.fld_value);
                 let gossip_header = Aheader::from_str(&value);
@@ -498,7 +498,7 @@ fn decrypt_if_autocrypt_message(
     public_keyring_for_validate: &Keyring,
     ret_valid_signatures: &mut HashSet<String>,
     ret_gossip_headers: *mut *mut mailimf_fields,
-) -> Result<(bool)> {
+) -> Result<bool> {
     /* The returned bool is true if we detected an Autocrypt-encrypted
     message and successfully decrypted it. Decryption then modifies the
     passed in mime structure in place. The returned bool is false
@@ -655,7 +655,7 @@ fn contains_report(mime: *mut Mailmime) -> bool {
 
         if tp_type == MAILMIME_TYPE_COMPOSITE_TYPE as libc::c_int
             && ct_type == MAILMIME_COMPOSITE_TYPE_MULTIPART as libc::c_int
-            && as_str(unsafe { (*mime.mm_content_type).ct_subtype }) == "report"
+            && to_string_lossy(unsafe { (*mime.mm_content_type).ct_subtype }) == "report"
         {
             return true;
         }

src/error.rs

@@ -34,6 +34,26 @@ pub enum Error {
     BlobError(#[cause] crate::blob::BlobError),
     #[fail(display = "Invalid Message ID.")]
     InvalidMsgId,
+    #[fail(display = "Watch folder not found {:?}", _0)]
+    WatchFolderNotFound(String),
+    #[fail(display = "Connection Failed params: {}", _0)]
+    ImapConnectionFailed(String),
+    #[fail(display = "Could not get OAUTH token")]
+    ImapOauthError,
+    #[fail(display = "Could not login as {}", _0)]
+    ImapLoginFailed(String),
+    #[fail(display = "Cannot idle")]
+    ImapMissesIdle,
+    #[fail(display = "Imap IDLE protocol failed to init/complete")]
+    ImapIdleProtocolFailed(String),
+    #[fail(display = "Imap IDLE failed to select folder {:?}", _0)]
+    ImapSelectFailed(String),
+    #[fail(display = "Connect without configured params")]
+    ConnectWithoutConfigure,
+    #[fail(display = "imap operation attempted while imap is torn down")]
+    ImapInTeardown,
+    #[fail(display = "No IMAP Connection established")]
+    ImapNoConnection,
 }
 
 pub type Result<T> = std::result::Result<T, Error>;

src/imap_client.rs

@@ -0,0 +1,294 @@
+use async_imap::{
+    error::{Error as ImapError, Result as ImapResult},
+    extensions::idle::Handle as ImapIdleHandle,
+    types::{Capabilities, Fetch, Mailbox, Name},
+    Client as ImapClient, Session as ImapSession,
+};
+use async_std::net::{self, TcpStream};
+use async_std::prelude::*;
+use async_std::sync::Arc;
+use async_tls::client::TlsStream;
+
+use crate::login_param::{dc_build_tls_config, CertificateChecks};
+
+const DCC_IMAP_DEBUG: &str = "DCC_IMAP_DEBUG";
+
+#[derive(Debug)]
+pub(crate) enum Client {
+    Secure(ImapClient<TlsStream<TcpStream>>),
+    Insecure(ImapClient<TcpStream>),
+}
+
+#[derive(Debug)]
+pub(crate) enum Session {
+    Secure(ImapSession<TlsStream<TcpStream>>),
+    Insecure(ImapSession<TcpStream>),
+}
+
+#[derive(Debug)]
+pub(crate) enum IdleHandle {
+    Secure(ImapIdleHandle<TlsStream<TcpStream>>),
+    Insecure(ImapIdleHandle<TcpStream>),
+}
+
+impl Client {
+    pub async fn connect_secure<A: net::ToSocketAddrs, S: AsRef<str>>(
+        addr: A,
+        domain: S,
+        certificate_checks: CertificateChecks,
+    ) -> ImapResult<Self> {
+        let stream = TcpStream::connect(addr).await?;
+        let tls_config = dc_build_tls_config(certificate_checks);
+        let tls_connector: async_tls::TlsConnector = Arc::new(tls_config).into();
+        let tls_stream = tls_connector.connect(domain.as_ref(), stream)?.await?;
+        let mut client = ImapClient::new(tls_stream);
+        if std::env::var(DCC_IMAP_DEBUG).is_ok() {
+            client.debug = true;
+        }
+
+        let _greeting = client
+            .read_response()
+            .await
+            .expect("failed to read greeting");
+
+        Ok(Client::Secure(client))
+    }
+
+    pub async fn connect_insecure<A: net::ToSocketAddrs>(addr: A) -> ImapResult<Self> {
+        let stream = TcpStream::connect(addr).await?;
+
+        let mut client = ImapClient::new(stream);
+        if std::env::var(DCC_IMAP_DEBUG).is_ok() {
+            client.debug = true;
+        }
+        let _greeting = client
+            .read_response()
+            .await
+            .expect("failed to read greeting");
+
+        Ok(Client::Insecure(client))
+    }
+
+    pub async fn secure<S: AsRef<str>>(
+        self,
+        domain: S,
+        _certificate_checks: CertificateChecks,
+    ) -> ImapResult<Client> {
+        match self {
+            Client::Insecure(client) => {
+                let tls = async_tls::TlsConnector::new();
+
+                let client_sec = client.secure(domain, &tls).await?;
+
+                Ok(Client::Secure(client_sec))
+            }
+            // Nothing to do
+            Client::Secure(_) => Ok(self),
+        }
+    }
+
+    pub async fn authenticate<A: async_imap::Authenticator, S: AsRef<str>>(
+        self,
+        auth_type: S,
+        authenticator: &A,
+    ) -> Result<Session, (ImapError, Client)> {
+        match self {
+            Client::Secure(i) => match i.authenticate(auth_type, authenticator).await {
+                Ok(session) => Ok(Session::Secure(session)),
+                Err((err, c)) => Err((err, Client::Secure(c))),
+            },
+            Client::Insecure(i) => match i.authenticate(auth_type, authenticator).await {
+                Ok(session) => Ok(Session::Insecure(session)),
+                Err((err, c)) => Err((err, Client::Insecure(c))),
+            },
+        }
+    }
+
+    pub async fn login<U: AsRef<str>, P: AsRef<str>>(
+        self,
+        username: U,
+        password: P,
+    ) -> Result<Session, (ImapError, Client)> {
+        match self {
+            Client::Secure(i) => match i.login(username, password).await {
+                Ok(session) => Ok(Session::Secure(session)),
+                Err((err, c)) => Err((err, Client::Secure(c))),
+            },
+            Client::Insecure(i) => match i.login(username, password).await {
+                Ok(session) => Ok(Session::Insecure(session)),
+                Err((err, c)) => Err((err, Client::Insecure(c))),
+            },
+        }
+    }
+}
+
+impl Session {
+    pub async fn capabilities(&mut self) -> ImapResult<Capabilities> {
+        let res = match self {
+            Session::Secure(i) => i.capabilities().await?,
+            Session::Insecure(i) => i.capabilities().await?,
+        };
+
+        Ok(res)
+    }
+
+    pub async fn list(
+        &mut self,
+        reference_name: Option<&str>,
+        mailbox_pattern: Option<&str>,
+    ) -> ImapResult<Vec<Name>> {
+        let res = match self {
+            Session::Secure(i) => {
+                i.list(reference_name, mailbox_pattern)
+                    .await?
+                    .collect::<ImapResult<_>>()
+                    .await?
+            }
+            Session::Insecure(i) => {
+                i.list(reference_name, mailbox_pattern)
+                    .await?
+                    .collect::<ImapResult<_>>()
+                    .await?
+            }
+        };
+        Ok(res)
+    }
+
+    pub async fn create<S: AsRef<str>>(&mut self, mailbox_name: S) -> ImapResult<()> {
+        match self {
+            Session::Secure(i) => i.create(mailbox_name).await?,
+            Session::Insecure(i) => i.create(mailbox_name).await?,
+        }
+        Ok(())
+    }
+
+    pub async fn subscribe<S: AsRef<str>>(&mut self, mailbox: S) -> ImapResult<()> {
+        match self {
+            Session::Secure(i) => i.subscribe(mailbox).await?,
+            Session::Insecure(i) => i.subscribe(mailbox).await?,
+        }
+        Ok(())
+    }
+
+    pub async fn close(&mut self) -> ImapResult<()> {
+        match self {
+            Session::Secure(i) => i.close().await?,
+            Session::Insecure(i) => i.close().await?,
+        }
+        Ok(())
+    }
+
+    pub async fn select<S: AsRef<str>>(&mut self, mailbox_name: S) -> ImapResult<Mailbox> {
+        let mbox = match self {
+            Session::Secure(i) => i.select(mailbox_name).await?,
+            Session::Insecure(i) => i.select(mailbox_name).await?,
+        };
+
+        Ok(mbox)
+    }
+
+    pub async fn fetch<S1, S2>(&mut self, sequence_set: S1, query: S2) -> ImapResult<Vec<Fetch>>
+    where
+        S1: AsRef<str>,
+        S2: AsRef<str>,
+    {
+        let res = match self {
+            Session::Secure(i) => {
+                i.fetch(sequence_set, query)
+                    .await?
+                    .collect::<ImapResult<_>>()
+                    .await?
+            }
+            Session::Insecure(i) => {
+                i.fetch(sequence_set, query)
+                    .await?
+                    .collect::<ImapResult<_>>()
+                    .await?
+            }
+        };
+        Ok(res)
+    }
+
+    pub async fn uid_fetch<S1, S2>(&mut self, uid_set: S1, query: S2) -> ImapResult<Vec<Fetch>>
+    where
+        S1: AsRef<str>,
+        S2: AsRef<str>,
+    {
+        let res = match self {
+            Session::Secure(i) => {
+                i.uid_fetch(uid_set, query)
+                    .await?
+                    .collect::<ImapResult<_>>()
+                    .await?
+            }
+            Session::Insecure(i) => {
+                i.uid_fetch(uid_set, query)
+                    .await?
+                    .collect::<ImapResult<_>>()
+                    .await?
+            }
+        };
+
+        Ok(res)
+    }
+
+    pub fn idle(self) -> IdleHandle {
+        match self {
+            Session::Secure(i) => {
+                let h = i.idle();
+                IdleHandle::Secure(h)
+            }
+            Session::Insecure(i) => {
+                let h = i.idle();
+                IdleHandle::Insecure(h)
+            }
+        }
+    }
+
+    pub async fn uid_store<S1, S2>(&mut self, uid_set: S1, query: S2) -> ImapResult<Vec<Fetch>>
+    where
+        S1: AsRef<str>,
+        S2: AsRef<str>,
+    {
+        let res = match self {
+            Session::Secure(i) => {
+                i.uid_store(uid_set, query)
+                    .await?
+                    .collect::<ImapResult<_>>()
+                    .await?
+            }
+            Session::Insecure(i) => {
+                i.uid_store(uid_set, query)
+                    .await?
+                    .collect::<ImapResult<_>>()
+                    .await?
+            }
+        };
+        Ok(res)
+    }
+
+    pub async fn uid_mv<S1: AsRef<str>, S2: AsRef<str>>(
+        &mut self,
+        uid_set: S1,
+        mailbox_name: S2,
+    ) -> ImapResult<()> {
+        match self {
+            Session::Secure(i) => i.uid_mv(uid_set, mailbox_name).await?,
+            Session::Insecure(i) => i.uid_mv(uid_set, mailbox_name).await?,
+        }
+        Ok(())
+    }
+
+    pub async fn uid_copy<S1: AsRef<str>, S2: AsRef<str>>(
+        &mut self,
+        uid_set: S1,
+        mailbox_name: S2,
+    ) -> ImapResult<()> {
+        match self {
+            Session::Secure(i) => i.uid_copy(uid_set, mailbox_name).await?,
+            Session::Insecure(i) => i.uid_copy(uid_set, mailbox_name).await?,
+        }
+
+        Ok(())
+    }
+}

src/imex.rs

@@ -1,5 +1,7 @@
+//! # Import/export module
+
 use core::cmp::{max, min};
-use std::path::{Path, PathBuf};
+use std::path::Path;
 
 use num_traits::FromPrimitive;
 use rand::{thread_rng, Rng};
@@ -49,7 +51,7 @@ pub enum ImexMode {
 
 /// Import/export things.
 /// For this purpose, the function creates a job that is executed in the IMAP-thread then;
-/// this requires to call dc_perform_imap_jobs() regularly.
+/// this requires to call dc_perform_inbox_jobs() regularly.
 ///
 /// What to do is defined by the _what_ parameter.
 ///
@@ -75,7 +77,7 @@ pub fn imex(context: &Context, what: ImexMode, param1: Option<impl AsRef<Path>>)
     job_add(context, Action::ImexImap, 0, param, 0);
 }
 
-/// Returns the filename of the backup if found, nullptr otherwise.
+/// Returns the filename of the backup found (otherwise an error)
 pub fn has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Result<String> {
     let dir_name = dir_name.as_ref();
     let dir_iter = std::fs::read_dir(dir_name)?;
@@ -90,13 +92,15 @@ pub fn has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Result<Strin
                 if name.starts_with("delta-chat") && name.ends_with(".bak") {
                     let sql = Sql::new();
                     if sql.open(context, &path, true) {
-                        let curr_backup_time =
-                            sql.get_raw_config_int(context, "backup_time")
-                                .unwrap_or_default() as u64;
+                        let curr_backup_time = sql
+                            .get_raw_config_int(context, "backup_time")
+                            .unwrap_or_default();
                         if curr_backup_time > newest_backup_time {
                             newest_backup_path = Some(path);
                             newest_backup_time = curr_backup_time;
                         }
+                        info!(context, "backup_time of {} is {}", name, curr_backup_time);
+                        sql.close(&context);
                     }
                 }
             }
@@ -105,7 +109,7 @@ pub fn has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Result<Strin
     }
     match newest_backup_path {
         Some(path) => Ok(path.to_string_lossy().into_owned()),
-        None => bail!("no backup found"),
+        None => bail!("no backup found in {}", dir_name.display()),
     }
 }
 
@@ -153,6 +157,11 @@ fn do_initiate_key_transfer(context: &Context) -> Result<String> {
             }
         }
     }
+    // no maybe_add_bcc_self_device_msg() here.
+    // the ui shows the dialog with the setup code on this device,
+    // it would be too much noise to have two things popping up at the same time.
+    // maybe_add_bcc_self_device_msg() is called on the other device
+    // once the transfer is completed.
     Ok(setup_code)
 }
 
@@ -228,6 +237,21 @@ pub fn create_setup_code(_context: &Context) -> String {
     ret
 }
 
+fn maybe_add_bcc_self_device_msg(context: &Context) -> Result<()> {
+    if !context.sql.get_raw_config_bool(context, "bcc_self") {
+        let mut msg = Message::new(Viewtype::Text);
+        // TODO: define this as a stockstring once the wording is settled.
+        msg.text = Some(
+            "It seems you are using multiple devices with Delta Chat. Great!\n\n\
+             If you also want to synchronize outgoing messages accross all devices, \
+             go to the settings and enable \"Send copy to self\"."
+                .to_string(),
+        );
+        chat::add_device_msg(context, Some("bcc-self-hint"), Some(&mut msg))?;
+    }
+    Ok(())
+}
+
 pub fn continue_key_transfer(context: &Context, msg_id: MsgId, setup_code: &str) -> Result<()> {
     ensure!(!msg_id.is_special(), "wrong id");
 
@@ -242,6 +266,7 @@ pub fn continue_key_transfer(context: &Context, msg_id: MsgId, setup_code: &str)
         let sc = normalize_setup_code(setup_code);
         let armored_key = decrypt_setup_file(context, &sc, file)?;
         set_self_key(context, &armored_key, true, true)?;
+        maybe_add_bcc_self_device_msg(context)?;
 
         Ok(())
     } else {
@@ -483,51 +508,56 @@ fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     // let dest_path_filename = dc_get_next_backup_file(context, dir, res);
     let now = time();
     let dest_path_filename = dc_get_next_backup_path(dir, now)?;
+    let dest_path_string = dest_path_filename.to_string_lossy().to_string();
 
     sql::housekeeping(context);
 
     sql::try_execute(context, &context.sql, "VACUUM;").ok();
+
+    // we close the database during the copy of the dbfile
     context.sql.close(context);
     info!(
         context,
-        "Backup \"{}\" to \"{}\".",
+        "Backup '{}' to '{}'.",
         context.get_dbfile().display(),
         dest_path_filename.display(),
     );
     let copied = dc_copy_file(context, context.get_dbfile(), &dest_path_filename);
     context.sql.open(&context, &context.get_dbfile(), false);
+
     if !copied {
-        let s = dest_path_filename.to_string_lossy().to_string();
         bail!(
-            "could not copy file from {:?} to {:?}",
-            context.get_dbfile(),
-            s
+            "could not copy file from '{}' to '{}'",
+            context.get_dbfile().display(),
+            dest_path_string
         );
     }
-    match add_files_to_export(context, &dest_path_filename) {
+    let dest_sql = Sql::new();
+    ensure!(
+        dest_sql.open(context, &dest_path_filename, false),
+        "could not open exported database {}",
+        dest_path_string
+    );
+    let res = match add_files_to_export(context, &dest_sql) {
         Err(err) => {
             dc_delete_file(context, &dest_path_filename);
             error!(context, "backup failed: {}", err);
             Err(err)
         }
         Ok(()) => {
-            context
-                .sql
-                .set_raw_config_int(context, "backup_time", now as i32)?;
+            dest_sql.set_raw_config_int(context, "backup_time", now as i32)?;
             context.call_cb(Event::ImexFileWritten(dest_path_filename.clone()));
             Ok(())
         }
-    }
+    };
+    dest_sql.close(context);
+
+    res
 }
 
-fn add_files_to_export(context: &Context, dest_path_filename: &PathBuf) -> Result<()> {
+fn add_files_to_export(context: &Context, sql: &Sql) -> Result<()> {
     // add all files as blobs to the database copy (this does not require
     // the source to be locked, neigher the destination as it is used only here)
-    let sql = Sql::new();
-    ensure!(
-        sql.open(context, &dest_path_filename, false),
-        "could not open db"
-    );
     if !sql.table_exists("backup_blobs") {
         sql::execute(
             context,

src/job.rs

@@ -135,8 +135,7 @@ impl Job {
         if !context.smtp.lock().unwrap().is_connected() {
             let loginparam = LoginParam::from_database(context, "configured_");
             let connected = context.smtp.lock().unwrap().connect(context, &loginparam);
-
-            if !connected {
+            if connected.is_err() {
                 self.try_again_later(3, None);
                 return;
             }
@@ -173,33 +172,17 @@ impl Job {
                     // its ok/error response processing. Note that if a message
                     // was sent we need to mark it in the database ASAP as we
                     // otherwise might send it twice.
-                    let mut sock = context.smtp.lock().unwrap();
-                    match sock.send(context, recipients_list, body) {
+                    let mut smtp = context.smtp.lock().unwrap();
+                    match smtp.send(context, recipients_list, body, self.job_id) {
                         Err(err) => {
-                            sock.disconnect();
+                            smtp.disconnect();
                             warn!(context, "smtp failed: {}", err);
                             self.try_again_later(-1, Some(err.to_string()));
                         }
                         Ok(()) => {
                             // smtp success, update db ASAP, then delete smtp file
                             if 0 != self.foreign_id {
-                                message::update_msg_state(
-                                    context,
-                                    MsgId::new(self.foreign_id),
-                                    MessageState::OutDelivered,
-                                );
-                                let chat_id: i32 = context
-                                    .sql
-                                    .query_get_value(
-                                        context,
-                                        "SELECT chat_id FROM msgs WHERE id=?",
-                                        params![self.foreign_id as i32],
-                                    )
-                                    .unwrap_or_default();
-                                context.call_cb(Event::MsgDelivered {
-                                    chat_id: chat_id as u32,
-                                    msg_id: MsgId::new(self.foreign_id),
-                                });
+                                set_delivered(context, MsgId::new(self.foreign_id));
                             }
                             // now also delete the generated file
                             dc_delete_file(context, filename);
@@ -220,7 +203,7 @@ impl Job {
 
     #[allow(non_snake_case)]
     fn do_DC_JOB_MOVE_MSG(&mut self, context: &Context) {
-        let inbox = context.inbox.read().unwrap();
+        let imap_inbox = &context.inbox_thread.read().unwrap().imap;
 
         if let Ok(msg) = Message::load_from_db(context, MsgId::new(self.foreign_id)) {
             if context
@@ -229,7 +212,7 @@ impl Job {
                 .unwrap_or_default()
                 < 3
             {
-                inbox.configure_folders(context, 0x1i32);
+                imap_inbox.configure_folders(context, 0x1i32);
             }
             let dest_folder = context
                 .sql
@@ -239,17 +222,17 @@ impl Job {
                 let server_folder = msg.server_folder.as_ref().unwrap();
                 let mut dest_uid = 0;
 
-                match inbox.mv(
+                match imap_inbox.mv(
                     context,
                     server_folder,
                     msg.server_uid,
                     &dest_folder,
                     &mut dest_uid,
                 ) {
-                    ImapResult::RetryLater => {
+                    ImapActionResult::RetryLater => {
                         self.try_again_later(3i32, None);
                     }
-                    ImapResult::Success => {
+                    ImapActionResult::Success => {
                         message::update_server_uid(
                             context,
                             &msg.rfc724_mid,
@@ -257,7 +240,7 @@ impl Job {
                             dest_uid,
                         );
                     }
-                    ImapResult::Failed | ImapResult::AlreadyDone => {}
+                    ImapActionResult::Failed | ImapActionResult::AlreadyDone => {}
                 }
             }
         }
@@ -265,7 +248,7 @@ impl Job {
 
     #[allow(non_snake_case)]
     fn do_DC_JOB_DELETE_MSG_ON_IMAP(&mut self, context: &Context) {
-        let inbox = context.inbox.read().unwrap();
+        let imap_inbox = &context.inbox_thread.read().unwrap().imap;
 
         if let Ok(mut msg) = Message::load_from_db(context, MsgId::new(self.foreign_id)) {
             if !msg.rfc724_mid.is_empty() {
@@ -280,8 +263,9 @@ impl Job {
                     we delete the message from the server */
                     let mid = msg.rfc724_mid;
                     let server_folder = msg.server_folder.as_ref().unwrap();
-                    let res = inbox.delete_msg(context, &mid, server_folder, &mut msg.server_uid);
-                    if res == ImapResult::RetryLater {
+                    let res =
+                        imap_inbox.delete_msg(context, &mid, server_folder, &mut msg.server_uid);
+                    if res == ImapActionResult::RetryLater {
                         self.try_again_later(-1i32, None);
                         return;
                     }
@@ -293,32 +277,32 @@ impl Job {
 
     #[allow(non_snake_case)]
     fn do_DC_JOB_EMPTY_SERVER(&mut self, context: &Context) {
-        let inbox = context.inbox.read().unwrap();
+        let imap_inbox = &context.inbox_thread.read().unwrap().imap;
         if self.foreign_id & DC_EMPTY_MVBOX > 0 {
             if let Some(mvbox_folder) = context
                 .sql
                 .get_raw_config(context, "configured_mvbox_folder")
             {
-                inbox.empty_folder(context, &mvbox_folder);
+                imap_inbox.empty_folder(context, &mvbox_folder);
             }
         }
         if self.foreign_id & DC_EMPTY_INBOX > 0 {
-            inbox.empty_folder(context, "INBOX");
+            imap_inbox.empty_folder(context, "INBOX");
         }
     }
 
     #[allow(non_snake_case)]
     fn do_DC_JOB_MARKSEEN_MSG_ON_IMAP(&mut self, context: &Context) {
-        let inbox = context.inbox.read().unwrap();
+        let imap_inbox = &context.inbox_thread.read().unwrap().imap;
 
         if let Ok(msg) = Message::load_from_db(context, MsgId::new(self.foreign_id)) {
             let folder = msg.server_folder.as_ref().unwrap();
-            match inbox.set_seen(context, folder, msg.server_uid) {
-                ImapResult::RetryLater => {
+            match imap_inbox.set_seen(context, folder, msg.server_uid) {
+                ImapActionResult::RetryLater => {
                     self.try_again_later(3i32, None);
                 }
-                ImapResult::AlreadyDone => {}
-                ImapResult::Success | ImapResult::Failed => {
+                ImapActionResult::AlreadyDone => {}
+                ImapActionResult::Success | ImapActionResult::Failed => {
                     // XXX the message might just have been moved
                     // we want to send out an MDN anyway
                     // The job will not be retried so locally
@@ -343,8 +327,8 @@ impl Job {
             .unwrap_or_default()
             .to_string();
         let uid = self.param.get_int(Param::ServerUid).unwrap_or_default() as u32;
-        let inbox = context.inbox.read().unwrap();
-        if inbox.set_seen(context, &folder, uid) == ImapResult::RetryLater {
+        let imap_inbox = &context.inbox_thread.read().unwrap().imap;
+        if imap_inbox.set_seen(context, &folder, uid) == ImapActionResult::RetryLater {
             self.try_again_later(3i32, None);
             return;
         }
@@ -355,15 +339,15 @@ impl Job {
                 .unwrap_or_default()
                 < 3
             {
-                inbox.configure_folders(context, 0x1i32);
+                imap_inbox.configure_folders(context, 0x1i32);
             }
             let dest_folder = context
                 .sql
                 .get_raw_config(context, "configured_mvbox_folder");
             if let Some(dest_folder) = dest_folder {
                 let mut dest_uid = 0;
-                if ImapResult::RetryLater
-                    == inbox.mv(context, &folder, uid, &dest_folder, &mut dest_uid)
+                if ImapActionResult::RetryLater
+                    == imap_inbox.mv(context, &folder, uid, &dest_folder, &mut dest_uid)
                 {
                     self.try_again_later(3, None);
                 }
@@ -383,45 +367,14 @@ pub fn job_kill_action(context: &Context, action: Action) -> bool {
     .is_ok()
 }
 
-pub fn perform_imap_fetch(context: &Context) {
-    let inbox = context.inbox.read().unwrap();
-    let start = std::time::Instant::now();
+pub fn perform_inbox_fetch(context: &Context) {
+    let use_network = context.get_config_bool(Config::InboxWatch);
 
-    if 0 == connect_to_inbox(context, &inbox) {
-        return;
-    }
-    if !context.get_config_bool(Config::InboxWatch) {
-        info!(context, "INBOX-watch disabled.",);
-        return;
-    }
-    info!(context, "INBOX-fetch started...",);
-    inbox.fetch(context);
-    if inbox.should_reconnect() {
-        info!(context, "INBOX-fetch aborted, starting over...",);
-        inbox.fetch(context);
-    }
-    info!(
-        context,
-        "INBOX-fetch done in {:.4} ms.",
-        start.elapsed().as_nanos() as f64 / 1000.0,
-    );
-}
-
-pub fn perform_imap_idle(context: &Context) {
-    let inbox = context.inbox.read().unwrap();
-
-    connect_to_inbox(context, &inbox);
-
-    if *context.perform_inbox_jobs_needed.clone().read().unwrap() {
-        info!(
-            context,
-            "INBOX-IDLE will not be started because of waiting jobs."
-        );
-        return;
-    }
-    info!(context, "INBOX-IDLE started...");
-    inbox.idle(context);
-    info!(context, "INBOX-IDLE ended.");
+    context
+        .inbox_thread
+        .write()
+        .unwrap()
+        .fetch(context, use_network);
 }
 
 pub fn perform_mvbox_fetch(context: &Context) {
@@ -434,20 +387,6 @@ pub fn perform_mvbox_fetch(context: &Context) {
         .fetch(context, use_network);
 }
 
-pub fn perform_mvbox_idle(context: &Context) {
-    let use_network = context.get_config_bool(Config::MvboxWatch);
-
-    context
-        .mvbox_thread
-        .read()
-        .unwrap()
-        .idle(context, use_network);
-}
-
-pub fn interrupt_mvbox_idle(context: &Context) {
-    context.mvbox_thread.read().unwrap().interrupt_idle(context);
-}
-
 pub fn perform_sentbox_fetch(context: &Context) {
     let use_network = context.get_config_bool(Config::SentboxWatch);
 
@@ -458,6 +397,33 @@ pub fn perform_sentbox_fetch(context: &Context) {
         .fetch(context, use_network);
 }
 
+pub fn perform_inbox_idle(context: &Context) {
+    if *context.perform_inbox_jobs_needed.clone().read().unwrap() {
+        info!(
+            context,
+            "INBOX-IDLE will not be started because of waiting jobs."
+        );
+        return;
+    }
+    let use_network = context.get_config_bool(Config::InboxWatch);
+
+    context
+        .inbox_thread
+        .read()
+        .unwrap()
+        .idle(context, use_network);
+}
+
+pub fn perform_mvbox_idle(context: &Context) {
+    let use_network = context.get_config_bool(Config::MvboxWatch);
+
+    context
+        .mvbox_thread
+        .read()
+        .unwrap()
+        .idle(context, use_network);
+}
+
 pub fn perform_sentbox_idle(context: &Context) {
     let use_network = context.get_config_bool(Config::SentboxWatch);
 
@@ -468,6 +434,27 @@ pub fn perform_sentbox_idle(context: &Context) {
         .idle(context, use_network);
 }
 
+pub fn interrupt_inbox_idle(context: &Context, block: bool) {
+    info!(context, "interrupt_inbox_idle called blocking={}", block);
+    if block {
+        context.inbox_thread.read().unwrap().interrupt_idle(context);
+    } else {
+        match context.inbox_thread.try_read() {
+            Ok(inbox_thread) => {
+                inbox_thread.interrupt_idle(context);
+            }
+            Err(err) => {
+                *context.perform_inbox_jobs_needed.write().unwrap() = true;
+                warn!(context, "could not interrupt idle: {}", err);
+            }
+        }
+    }
+}
+
+pub fn interrupt_mvbox_idle(context: &Context) {
+    context.mvbox_thread.read().unwrap().interrupt_idle(context);
+}
+
 pub fn interrupt_sentbox_idle(context: &Context) {
     context
         .sentbox_thread
@@ -523,7 +510,7 @@ pub fn perform_smtp_idle(context: &Context) {
                 let res = cvar.wait_timeout(state, dur).unwrap();
                 state = res.0;
 
-                if state.idle == true || res.1.timed_out() {
+                if state.idle || res.1.timed_out() {
                     // We received the notification and the value has been updated, we can leave.
                     break;
                 }
@@ -568,7 +555,7 @@ pub fn maybe_network(context: &Context) {
     }
 
     interrupt_smtp_idle(context);
-    interrupt_imap_idle(context);
+    interrupt_inbox_idle(context, true);
     interrupt_mvbox_idle(context);
     interrupt_sentbox_idle(context);
 }
@@ -580,6 +567,22 @@ pub fn job_action_exists(context: &Context, action: Action) -> bool {
         .unwrap_or_default()
 }
 
+fn set_delivered(context: &Context, msg_id: MsgId) {
+    message::update_msg_state(context, msg_id, MessageState::OutDelivered);
+    let chat_id: i32 = context
+        .sql
+        .query_get_value(
+            context,
+            "SELECT chat_id FROM msgs WHERE id=?",
+            params![msg_id],
+        )
+        .unwrap_or_default();
+    context.call_cb(Event::MsgDelivered {
+        chat_id: chat_id as u32,
+        msg_id: msg_id,
+    });
+}
+
 /* special case for DC_JOB_SEND_MSG_TO_SMTP */
 #[allow(non_snake_case)]
 pub fn job_send_msg(context: &Context, msg_id: MsgId) -> Result<(), Error> {
@@ -641,10 +644,12 @@ pub fn job_send_msg(context: &Context, msg_id: MsgId) -> Result<(), Error> {
     }
 
     if mimefactory.recipients_addr.is_empty() {
-        warn!(
+        // may happen eg. for groups with only SELF and bcc_self disabled
+        info!(
             context,
             "message {} has no recipient, skipping smtp-send", msg_id
         );
+        set_delivered(context, msg_id);
         return Ok(());
     }
 
@@ -682,15 +687,15 @@ pub fn job_send_msg(context: &Context, msg_id: MsgId) -> Result<(), Error> {
     Ok(())
 }
 
-pub fn perform_imap_jobs(context: &Context) {
-    info!(context, "dc_perform_imap_jobs starting.",);
+pub fn perform_inbox_jobs(context: &Context) {
+    info!(context, "dc_perform_inbox_jobs starting.",);
 
     let probe_imap_network = *context.probe_imap_network.clone().read().unwrap();
     *context.probe_imap_network.write().unwrap() = false;
     *context.perform_inbox_jobs_needed.write().unwrap() = false;
 
     job_perform(context, Thread::Imap, probe_imap_network);
-    info!(context, "dc_perform_imap_jobs ended.",);
+    info!(context, "dc_perform_inbox_jobs ended.",);
 }
 
 pub fn perform_mvbox_jobs(context: &Context) {
@@ -723,32 +728,31 @@ fn job_perform(context: &Context, thread: Thread, probe_network: bool) {
         params_probe
     };
 
-    let jobs: Result<Vec<Job>, _> = context.sql.query_map(
-        query,
-        params,
-        |row| {
-            let job = Job {
-                job_id: row.get(0)?,
-                action: row.get(1)?,
-                foreign_id: row.get(2)?,
-                desired_timestamp: row.get(5)?,
-                added_timestamp: row.get(4)?,
-                tries: row.get(6)?,
-                param: row.get::<_, String>(3)?.parse().unwrap_or_default(),
-                try_again: 0,
-                pending_error: None,
-            };
+    let jobs: Result<Vec<Job>, _> = context
+        .sql
+        .query_map(
+            query,
+            params,
+            |row| {
+                let job = Job {
+                    job_id: row.get(0)?,
+                    action: row.get(1)?,
+                    foreign_id: row.get(2)?,
+                    desired_timestamp: row.get(5)?,
+                    added_timestamp: row.get(4)?,
+                    tries: row.get(6)?,
+                    param: row.get::<_, String>(3)?.parse().unwrap_or_default(),
+                    try_again: 0,
+                    pending_error: None,
+                };
 
-            Ok(job)
-        },
-        |jobs| jobs.collect::<Result<Vec<Job>, _>>().map_err(Into::into),
-    );
-    match jobs {
-        Ok(ref _res) => {}
-        Err(ref err) => {
-            info!(context, "query failed: {:?}", err);
-        }
-    }
+                Ok(job)
+            },
+            |jobs| jobs.collect::<Result<Vec<Job>, _>>().map_err(Into::into),
+        )
+        .map_err(|err| {
+            warn!(context, "query failed: {:?}", err);
+        });
 
     for mut job in jobs.unwrap_or_default() {
         info!(
@@ -927,14 +931,6 @@ fn suspend_smtp_thread(context: &Context, suspend: bool) {
     }
 }
 
-pub fn connect_to_inbox(context: &Context, inbox: &Imap) -> libc::c_int {
-    let ret_connected = dc_connect_to_configured_imap(context, inbox);
-    if 0 != ret_connected {
-        inbox.set_watch_folder("INBOX".into());
-    }
-    ret_connected
-}
-
 fn send_mdn(context: &Context, msg_id: MsgId) -> Result<(), Error> {
     let mut mimefactory = MimeFactory::load_mdn(context, msg_id)?;
     unsafe { mimefactory.render()? };
@@ -1005,7 +1001,7 @@ pub fn job_add(
     ).ok();
 
     match thread {
-        Thread::Imap => interrupt_imap_idle(context),
+        Thread::Imap => interrupt_inbox_idle(context, false),
         Thread::Smtp => interrupt_smtp_idle(context),
         Thread::Unknown => {}
     }
@@ -1021,10 +1017,3 @@ pub fn interrupt_smtp_idle(context: &Context) {
     state.idle = true;
     cvar.notify_one();
 }
-
-pub fn interrupt_imap_idle(context: &Context) {
-    info!(context, "Interrupting IMAP-IDLE...",);
-
-    *context.perform_inbox_jobs_needed.write().unwrap() = true;
-    context.inbox.read().unwrap().interrupt_idle();
-}

src/job_thread.rs

@@ -1,7 +1,7 @@
 use std::sync::{Arc, Condvar, Mutex};
 
-use crate::configure::*;
 use crate::context::Context;
+use crate::error::{Error, Result};
 use crate::imap::Imap;
 
 #[derive(Debug)]
@@ -15,7 +15,7 @@ pub struct JobThread {
 #[derive(Clone, Debug, Default)]
 pub struct JobState {
     idle: bool,
-    jobs_needed: i32,
+    jobs_needed: bool,
     suspended: bool,
     using_handle: bool,
 }
@@ -58,7 +58,7 @@ impl JobThread {
 
     pub fn interrupt_idle(&self, context: &Context) {
         {
-            self.state.0.lock().unwrap().jobs_needed = 1;
+            self.state.0.lock().unwrap().jobs_needed = true;
         }
 
         info!(context, "Interrupting {}-IDLE...", self.name);
@@ -85,53 +85,49 @@ impl JobThread {
         }
 
         if use_network {
-            let start = std::time::Instant::now();
-            if self.connect_to_imap(context) {
-                info!(context, "{}-fetch started...", self.name);
-                self.imap.fetch(context);
-
-                if self.imap.should_reconnect() {
-                    info!(context, "{}-fetch aborted, starting over...", self.name,);
-                    self.imap.fetch(context);
+            if let Err(err) = self.connect_and_fetch(context) {
+                warn!(context, "connect+fetch failed: {}, reconnect & retry", err);
+                self.imap.trigger_reconnect();
+                if let Err(err) = self.connect_and_fetch(context) {
+                    warn!(context, "connect+fetch failed: {}", err);
                 }
-                info!(
-                    context,
-                    "{}-fetch done in {:.3} ms.",
-                    self.name,
-                    start.elapsed().as_millis(),
-                );
             }
         }
-
         self.state.0.lock().unwrap().using_handle = false;
     }
 
-    fn connect_to_imap(&self, context: &Context) -> bool {
-        if self.imap.is_connected() {
-            return true;
-        }
+    fn connect_and_fetch(&mut self, context: &Context) -> Result<()> {
+        let prefix = format!("{}-fetch", self.name);
+        match self.imap.connect_configured(context) {
+            Ok(()) => {
+                if let Some(watch_folder) = self.get_watch_folder(context) {
+                    let start = std::time::Instant::now();
+                    info!(context, "{} started...", prefix);
+                    let res = self.imap.fetch(context, &watch_folder);
+                    let elapsed = start.elapsed().as_millis();
+                    info!(context, "{} done in {:.3} ms.", prefix, elapsed);
 
-        let mut ret_connected = dc_connect_to_configured_imap(context, &self.imap) != 0;
-
-        if ret_connected {
-            if context
-                .sql
-                .get_raw_config_int(context, "folders_configured")
-                .unwrap_or_default()
-                < 3
-            {
-                self.imap.configure_folders(context, 0x1);
+                    res
+                } else {
+                    Err(Error::WatchFolderNotFound("not-set".to_string()))
+                }
             }
+            Err(err) => Err(err),
+        }
+    }
 
-            if let Some(mvbox_name) = context.sql.get_raw_config(context, self.folder_config_name) {
-                self.imap.set_watch_folder(mvbox_name);
-            } else {
-                self.imap.disconnect(context);
-                ret_connected = false;
+    fn get_watch_folder(&self, context: &Context) -> Option<String> {
+        match context.sql.get_raw_config(context, self.folder_config_name) {
+            Some(name) => Some(name),
+            None => {
+                if self.folder_config_name == "configured_inbox_folder" {
+                    // initialized with old version, so has not set configured_inbox_folder
+                    Some("INBOX".to_string())
+                } else {
+                    None
+                }
             }
         }
-
-        ret_connected
     }
 
     pub fn idle(&self, context: &Context, use_network: bool) {
@@ -139,13 +135,13 @@ impl JobThread {
             let &(ref lock, ref cvar) = &*self.state.clone();
             let mut state = lock.lock().unwrap();
 
-            if 0 != state.jobs_needed {
+            if state.jobs_needed {
                 info!(
                     context,
                     "{}-IDLE will not be started as it was interrupted while not ideling.",
                     self.name,
                 );
-                state.jobs_needed = 0;
+                state.jobs_needed = false;
                 return;
             }
 
@@ -170,10 +166,37 @@ impl JobThread {
             }
         }
 
-        self.connect_to_imap(context);
-        info!(context, "{}-IDLE started...", self.name,);
-        self.imap.idle(context);
-        info!(context, "{}-IDLE ended.", self.name);
+        let prefix = format!("{}-IDLE", self.name);
+        let do_fake_idle = match self.imap.connect_configured(context) {
+            Ok(()) => {
+                info!(context, "{} started...", prefix);
+                let watch_folder = self.get_watch_folder(context);
+                let res = self.imap.idle(context, watch_folder);
+                info!(context, "{} ended...", prefix);
+                match res {
+                    Ok(()) => false,
+                    Err(Error::ImapMissesIdle) => true, // we have to do fake_idle
+                    Err(err) => {
+                        warn!(context, "{} failed: {} -> reconnecting", prefix, err);
+                        // something is borked, let's start afresh on the next occassion
+                        self.imap.disconnect(context);
+
+                        false
+                    }
+                }
+            }
+            Err(err) => {
+                info!(context, "{}-IDLE connection fail: {:?}", self.name, err);
+                // if the connection fails, use fake_idle to retry periodically
+                // fake_idle() will be woken up by interrupt_idle() as
+                // well so will act on maybe_network events
+                true
+            }
+        };
+        if do_fake_idle {
+            let watch_folder = self.get_watch_folder(context);
+            self.imap.fake_idle(context, watch_folder);
+        }
 
         self.state.0.lock().unwrap().using_handle = false;
     }

src/key.rs

@@ -1,3 +1,5 @@
+//! Cryptographic key module
+
 use std::collections::BTreeMap;
 use std::io::Cursor;
 use std::path::Path;
@@ -11,6 +13,7 @@ use crate::context::Context;
 use crate::dc_tools::*;
 use crate::sql::{self, Sql};
 
+/// Cryptographic key
 #[derive(Debug, PartialEq, Eq, Clone)]
 pub enum Key {
     Public(SignedPublicKey),
@@ -29,44 +32,44 @@ impl From<SignedSecretKey> for Key {
     }
 }
 
-impl std::convert::TryInto<SignedSecretKey> for Key {
+impl std::convert::TryFrom<Key> for SignedSecretKey {
     type Error = ();
 
-    fn try_into(self) -> Result<SignedSecretKey, Self::Error> {
-        match self {
+    fn try_from(value: Key) -> Result<Self, Self::Error> {
+        match value {
             Key::Public(_) => Err(()),
             Key::Secret(key) => Ok(key),
         }
     }
 }
 
-impl<'a> std::convert::TryInto<&'a SignedSecretKey> for &'a Key {
+impl<'a> std::convert::TryFrom<&'a Key> for &'a SignedSecretKey {
     type Error = ();
 
-    fn try_into(self) -> Result<&'a SignedSecretKey, Self::Error> {
-        match self {
+    fn try_from(value: &'a Key) -> Result<Self, Self::Error> {
+        match value {
             Key::Public(_) => Err(()),
             Key::Secret(key) => Ok(key),
         }
     }
 }
 
-impl std::convert::TryInto<SignedPublicKey> for Key {
+impl std::convert::TryFrom<Key> for SignedPublicKey {
     type Error = ();
 
-    fn try_into(self) -> Result<SignedPublicKey, Self::Error> {
-        match self {
+    fn try_from(value: Key) -> Result<Self, Self::Error> {
+        match value {
             Key::Public(key) => Ok(key),
             Key::Secret(_) => Err(()),
         }
     }
 }
 
-impl<'a> std::convert::TryInto<&'a SignedPublicKey> for &'a Key {
+impl<'a> std::convert::TryFrom<&'a Key> for &'a SignedPublicKey {
     type Error = ();
 
-    fn try_into(self) -> Result<&'a SignedPublicKey, Self::Error> {
-        match self {
+    fn try_from(value: &'a Key) -> Result<Self, Self::Error> {
+        match value {
             Key::Public(key) => Ok(key),
             Key::Secret(_) => Err(()),
         }

src/lib.rs

@@ -40,6 +40,7 @@ pub mod contact;
 pub mod context;
 mod e2ee;
 mod imap;
+mod imap_client;
 pub mod imex;
 pub mod job;
 mod job_thread;

src/location.rs

@@ -1,3 +1,5 @@
+//! Location handling
+
 use bitflags::bitflags;
 use quick_xml;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
@@ -16,7 +18,7 @@ use crate::param::*;
 use crate::sql;
 use crate::stock::StockMessage;
 
-// location handling
+/// Location record
 #[derive(Debug, Clone, Default)]
 pub struct Location {
     pub location_id: u32,
@@ -62,14 +64,11 @@ impl Kml {
         Default::default()
     }
 
-    pub fn parse(context: &Context, content: impl AsRef<str>) -> Result<Self, Error> {
-        ensure!(
-            content.as_ref().len() <= (1024 * 1024),
-            "A kml-files with {} bytes is larger than reasonably expected.",
-            content.as_ref().len()
-        );
+    pub fn parse(context: &Context, content: &[u8]) -> Result<Self, Error> {
+        ensure!(content.len() <= 1024 * 1024, "kml-file is too large");
 
-        let mut reader = quick_xml::Reader::from_str(content.as_ref());
+        let to_parse = String::from_utf8_lossy(content);
+        let mut reader = quick_xml::Reader::from_str(&to_parse);
         reader.trim_text(true);
 
         let mut kml = Kml::new();
@@ -222,7 +221,7 @@ pub fn send_locations_to_chat(context: &Context, chat_id: u32, seconds: i64) {
             } else if 0 == seconds && is_sending_locations_before {
                 let stock_str =
                     context.stock_system_msg(StockMessage::MsgLocationDisabled, "", "", 0);
-                chat::add_device_msg(context, chat_id, stock_str);
+                chat::add_info_msg(context, chat_id, stock_str);
             }
             context.call_cb(Event::ChatModified(chat_id));
             if 0 != seconds {
@@ -278,7 +277,7 @@ pub fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64) -> b
                         accuracy,
                         time(),
                         chat_id,
-                        1,
+                        DC_CONTACT_ID_SELF,
                     ]
             ) {
                 warn!(context, "failed to store location {:?}", err);
@@ -305,16 +304,15 @@ pub fn get_range(
     if timestamp_to == 0 {
         timestamp_to = time() + 10;
     }
-
     context
         .sql
         .query_map(
             "SELECT l.id, l.latitude, l.longitude, l.accuracy, l.timestamp, l.independent, \
-             m.id, l.from_id, l.chat_id, m.txt \
+             COALESCE(m.id, 0) AS msg_id, l.from_id, l.chat_id, COALESCE(m.txt, '') AS txt \
              FROM locations l  LEFT JOIN msgs m ON l.id=m.location_id  WHERE (? OR l.chat_id=?) \
              AND (? OR l.from_id=?) \
              AND (l.independent=1 OR (l.timestamp>=? AND l.timestamp<=?)) \
-             ORDER BY l.timestamp DESC, l.id DESC, m.id DESC;",
+             ORDER BY l.timestamp DESC, l.id DESC, msg_id DESC;",
             params![
                 if chat_id == 0 { 1 } else { 0 },
                 chat_id as i32,
@@ -331,7 +329,6 @@ pub fn get_range(
                 } else {
                     None
                 };
-
                 let loc = Location {
                     location_id: row.get(0)?,
                     latitude: row.get(1)?,
@@ -369,9 +366,6 @@ pub fn delete_all(context: &Context) -> Result<(), Error> {
 }
 
 pub fn get_kml(context: &Context, chat_id: u32) -> Result<(String, u32), Error> {
-    let now = time();
-    let mut location_count = 0;
-    let mut ret = String::new();
     let mut last_added_location_id = 0;
 
     let self_addr = context
@@ -388,14 +382,17 @@ pub fn get_kml(context: &Context, chat_id: u32) -> Result<(String, u32), Error>
             Ok((send_begin, send_until, last_sent))
         })?;
 
-    if !(locations_send_begin == 0 || now > locations_send_until) {
+    let now = time();
+    let mut location_count = 0;
+    let mut ret = String::new();
+    if locations_send_begin != 0 && now <= locations_send_until {
         ret += &format!(
             "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<kml xmlns=\"http://www.opengis.net/kml/2.2\">\n<Document addr=\"{}\">\n",
             self_addr,
         );
 
         context.sql.query_map(
-            "SELECT id, latitude, longitude, accuracy, timestamp\
+            "SELECT id, latitude, longitude, accuracy, timestamp \
              FROM locations  WHERE from_id=? \
              AND timestamp>=? \
              AND (timestamp>=? OR timestamp=(SELECT MAX(timestamp) FROM locations WHERE from_id=?)) \
@@ -416,7 +413,7 @@ pub fn get_kml(context: &Context, chat_id: u32) -> Result<(String, u32), Error>
                 for row in rows {
                     let (location_id, latitude, longitude, accuracy, timestamp) = row?;
                     ret += &format!(
-                        "<Placemark><Timestamp><when>{}</when></Timestamp><Point><coordinates accuracy=\"{}\">{},{}</coordinates></Point></Placemark>\n\x00",
+                        "<Placemark><Timestamp><when>{}</when></Timestamp><Point><coordinates accuracy=\"{}\">{},{}</coordinates></Point></Placemark>\n",
                         timestamp,
                         accuracy,
                         longitude,
@@ -428,10 +425,10 @@ pub fn get_kml(context: &Context, chat_id: u32) -> Result<(String, u32), Error>
                 Ok(())
             }
         )?;
+        ret += "</Document>\n</kml>";
     }
 
     ensure!(location_count > 0, "No locations processed");
-    ret += "</Document>\n</kml>";
 
     Ok((ret, last_added_location_id))
 }
@@ -495,7 +492,7 @@ pub fn save(
     chat_id: u32,
     contact_id: u32,
     locations: &[Location],
-    independent: i32,
+    independent: bool,
 ) -> Result<u32, Error> {
     ensure!(chat_id > DC_CHAT_ID_LAST_SPECIAL, "Invalid chat id");
     context.sql.prepare2(
@@ -510,7 +507,7 @@ pub fn save(
             for location in locations {
                 let exists = stmt_test.exists(params![location.timestamp, contact_id as i32])?;
 
-                if 0 != independent || !exists {
+                if independent || !exists {
                     stmt_insert.execute(params![
                         location.timestamp,
                         contact_id as i32,
@@ -654,7 +651,7 @@ pub fn job_do_DC_JOB_MAYBE_SEND_LOC_ENDED(context: &Context, job: &mut Job) {
                     params![chat_id as i32],
                 ).is_ok() {
                     let stock_str = context.stock_system_msg(StockMessage::MsgLocationDisabled, "", "", 0);
-                    chat::add_device_msg(context, chat_id, stock_str);
+                    chat::add_info_msg(context, chat_id, stock_str);
                     context.call_cb(Event::ChatModified(chat_id));
                 }
             }
@@ -672,9 +669,9 @@ mod tests {
         let context = dummy_context();
 
         let xml =
-            "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<kml xmlns=\"http://www.opengis.net/kml/2.2\">\n<Document addr=\"user@example.org\">\n<Placemark><Timestamp><when>2019-03-06T21:09:57Z</when></Timestamp><Point><coordinates accuracy=\"32.000000\">9.423110,53.790302</coordinates></Point></Placemark>\n<PlaceMARK>\n<Timestamp><WHEN > \n\t2018-12-13T22:11:12Z\t</WHEN></Timestamp><Point><coordinates aCCuracy=\"2.500000\"> 19.423110 \t , \n 63.790302\n </coordinates></Point></PlaceMARK>\n</Document>\n</kml>";
+            b"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<kml xmlns=\"http://www.opengis.net/kml/2.2\">\n<Document addr=\"user@example.org\">\n<Placemark><Timestamp><when>2019-03-06T21:09:57Z</when></Timestamp><Point><coordinates accuracy=\"32.000000\">9.423110,53.790302</coordinates></Point></Placemark>\n<PlaceMARK>\n<Timestamp><WHEN > \n\t2018-12-13T22:11:12Z\t</WHEN></Timestamp><Point><coordinates aCCuracy=\"2.500000\"> 19.423110 \t , \n 63.790302\n </coordinates></Point></PlaceMARK>\n</Document>\n</kml>";
 
-        let kml = Kml::parse(&context.ctx, &xml).expect("parsing failed");
+        let kml = Kml::parse(&context.ctx, xml).expect("parsing failed");
 
         assert!(kml.addr.is_some());
         assert_eq!(kml.addr.as_ref().unwrap(), "user@example.org",);

src/log.rs

@@ -1,3 +1,5 @@
+//! # Logging macros
+
 #[macro_export]
 macro_rules! info {
     ($ctx:expr,  $msg:expr) => {

src/login_param.rs

@@ -3,6 +3,10 @@ use std::fmt;
 
 use crate::context::Context;
 use crate::error::Error;
+use async_std::sync::Arc;
+use rustls;
+use webpki;
+use webpki_roots;
 
 #[derive(Copy, Clone, Debug, Display, FromPrimitive)]
 #[repr(i32)]
@@ -251,27 +255,49 @@ fn get_readable_flags(flags: i32) -> String {
     res
 }
 
-pub fn dc_build_tls(
-    certificate_checks: CertificateChecks,
-) -> Result<native_tls::TlsConnector, native_tls::Error> {
-    let mut tls_builder = native_tls::TlsConnector::builder();
+pub struct NoCertificateVerification {}
+
+impl rustls::ServerCertVerifier for NoCertificateVerification {
+    fn verify_server_cert(
+        &self,
+        _roots: &rustls::RootCertStore,
+        _presented_certs: &[rustls::Certificate],
+        _dns_name: webpki::DNSNameRef<'_>,
+        _ocsp: &[u8],
+    ) -> Result<rustls::ServerCertVerified, rustls::TLSError> {
+        Ok(rustls::ServerCertVerified::assertion())
+    }
+}
+
+pub fn dc_build_tls_config(certificate_checks: CertificateChecks) -> rustls::ClientConfig {
+    let mut config = rustls::ClientConfig::new();
+    config
+        .root_store
+        .add_server_trust_anchors(&webpki_roots::TLS_SERVER_ROOTS);
+
     match certificate_checks {
+        CertificateChecks::Strict => {}
         CertificateChecks::Automatic => {
             // Same as AcceptInvalidCertificates for now.
             // TODO: use provider database when it becomes available
-            tls_builder
-                .danger_accept_invalid_hostnames(true)
-                .danger_accept_invalid_certs(true)
+            config
+                .dangerous()
+                .set_certificate_verifier(Arc::new(NoCertificateVerification {}));
+        }
+        CertificateChecks::AcceptInvalidCertificates => {
+            // TODO: only accept invalid certs
+            config
+                .dangerous()
+                .set_certificate_verifier(Arc::new(NoCertificateVerification {}));
         }
-        CertificateChecks::Strict => &mut tls_builder,
         CertificateChecks::AcceptInvalidHostnames => {
-            tls_builder.danger_accept_invalid_hostnames(true)
+            // TODO: only accept invalid hostnames
+            config
+                .dangerous()
+                .set_certificate_verifier(Arc::new(NoCertificateVerification {}));
         }
-        CertificateChecks::AcceptInvalidCertificates => tls_builder
-            .danger_accept_invalid_hostnames(true)
-            .danger_accept_invalid_certs(true),
     }
-    .build()
+    config
 }
 
 #[cfg(test)]

src/message.rs

@@ -476,8 +476,8 @@ impl Message {
 
     pub fn is_info(&self) -> bool {
         let cmd = self.param.get_cmd();
-        self.from_id == DC_CONTACT_ID_DEVICE as libc::c_uint
-            || self.to_id == DC_CONTACT_ID_DEVICE as libc::c_uint
+        self.from_id == DC_CONTACT_ID_INFO as libc::c_uint
+            || self.to_id == DC_CONTACT_ID_INFO as libc::c_uint
             || cmd != SystemMessage::Unknown && cmd != SystemMessage::AutocryptSetupMessage
     }
 
@@ -714,7 +714,7 @@ pub fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
         ret += "\n";
     }
 
-    if msg.from_id == 2 || msg.to_id == 2 {
+    if msg.from_id == DC_CONTACT_ID_INFO || msg.to_id == DC_CONTACT_ID_INFO {
         // device-internal message, no further details needed
         return ret;
     }
@@ -841,6 +841,11 @@ pub fn get_mime_headers(context: &Context, msg_id: MsgId) -> Option<String> {
 
 pub fn delete_msgs(context: &Context, msg_ids: &[MsgId]) {
     for msg_id in msg_ids.iter() {
+        if let Ok(msg) = Message::load_from_db(context, *msg_id) {
+            if msg.location_id > 0 {
+                delete_poi_location(context, msg.location_id);
+            }
+        }
         update_msg_chat_id(context, *msg_id, DC_CHAT_ID_TRASH);
         job_add(
             context,
@@ -871,6 +876,16 @@ fn update_msg_chat_id(context: &Context, msg_id: MsgId, chat_id: u32) -> bool {
     .is_ok()
 }
 
+fn delete_poi_location(context: &Context, location_id: u32) -> bool {
+    sql::execute(
+        context,
+        &context.sql,
+        "DELETE FROM locations WHERE independent = 1 AND id=?;",
+        params![location_id as i32],
+    )
+    .is_ok()
+}
+
 pub fn markseen_msgs(context: &Context, msg_ids: &[MsgId]) -> bool {
     if msg_ids.is_empty() {
         return false;
@@ -1104,7 +1119,7 @@ pub fn mdn_from_ext(
     rfc724_mid: &str,
     timestamp_sent: i64,
 ) -> Option<(u32, MsgId)> {
-    if from_id <= 9 || rfc724_mid.is_empty() {
+    if from_id <= DC_MSG_ID_LAST_SPECIAL || rfc724_mid.is_empty() {
         return None;
     }
 

src/mimefactory.rs

@@ -484,6 +484,7 @@ impl<'a> MimeFactory<'a> {
                 if !meta_part.is_null() {
                     mailmime_smart_add_part(message, meta_part);
                 }
+
                 if self.msg.param.exists(Param::SetLatitude) {
                     let param = &self.msg.param;
                     let kml_file = location::get_message_kml(
@@ -500,18 +501,21 @@ impl<'a> MimeFactory<'a> {
                 }
 
                 if location::is_sending_locations_to_chat(context, self.msg.chat_id) {
-                    if let Ok((kml_file, last_added_location_id)) =
-                        location::get_kml(context, self.msg.chat_id)
-                    {
-                        wrapmime::add_filename_part(
-                            message,
-                            "location.kml",
-                            "application/vnd.google-earth.kml+xml",
-                            &kml_file,
-                        )?;
-                        if !self.msg.param.exists(Param::SetLatitude) {
-                            // otherwise, the independent location is already filed
-                            self.out_last_added_location_id = last_added_location_id;
+                    match location::get_kml(context, self.msg.chat_id) {
+                        Ok((kml_content, last_added_location_id)) => {
+                            wrapmime::add_filename_part(
+                                message,
+                                "location.kml",
+                                "application/vnd.google-earth.kml+xml",
+                                &kml_content,
+                            )?;
+                            if !self.msg.param.exists(Param::SetLatitude) {
+                                // otherwise, the independent location is already filed
+                                self.out_last_added_location_id = last_added_location_id;
+                            }
+                        }
+                        Err(err) => {
+                            warn!(context, "mimefactory: could not get location: {}", err);
                         }
                     }
                 }

src/oauth2.rs

@@ -1,3 +1,5 @@
+//! OAuth 2 module
+
 use std::collections::HashMap;
 
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
@@ -33,12 +35,14 @@ struct Oauth2 {
     get_userinfo: Option<&'static str>,
 }
 
+/// OAuth 2 Access Token Response
 #[derive(Debug, Deserialize)]
 struct Response {
     // Should always be there according to: https://www.oauth.com/oauth2-servers/access-tokens/access-token-response/
     // but previous code handled its abscense.
     access_token: Option<String>,
     token_type: String,
+    /// Duration of time the token is granted for, in seconds
     expires_in: Option<u64>,
     refresh_token: Option<String>,
     scope: Option<String>,
@@ -205,7 +209,7 @@ pub fn dc_get_oauth2_access_token(
                 .ok();
             let expires_in = response
                 .expires_in
-                // refresh a bet before
+                // refresh a bit before
                 .map(|t| time() + t as i64 - 5)
                 .unwrap_or_else(|| 0);
             context

src/param.rs

@@ -76,6 +76,8 @@ pub enum Param {
     ProfileImage = b'i',
     // For Chats
     Selftalk = b'K',
+    // For Chats
+    Devicetalk = b'D',
     // For QR
     Auth = b's',
     // For QR

src/peerstate.rs

@@ -1,3 +1,4 @@
+//! # [Autocrypt Peer State](https://autocrypt.org/level1.html#peer-state-management) module
 use std::collections::HashSet;
 use std::fmt;
 
@@ -382,11 +383,11 @@ impl<'a> Peerstate<'a> {
             sql::execute(
                 self.context,
                 sql,
-		"UPDATE acpeerstates \
-		 SET last_seen=?, last_seen_autocrypt=?, prefer_encrypted=?, \
-		 public_key=?, gossip_timestamp=?, gossip_key=?, public_key_fingerprint=?, gossip_key_fingerprint=?, \
+                "UPDATE acpeerstates \
+                 SET last_seen=?, last_seen_autocrypt=?, prefer_encrypted=?, \
+                 public_key=?, gossip_timestamp=?, gossip_key=?, public_key_fingerprint=?, gossip_key_fingerprint=?, \
                  verified_key=?, verified_key_fingerprint=? \
-		 WHERE addr=?;",
+                 WHERE addr=?;",
                 params![
                     self.last_seen,
                     self.last_seen_autocrypt,

src/pgp.rs

@@ -1,15 +1,20 @@
+//! OpenPGP helper module using [rPGP facilities](https://github.com/rpgp/rpgp)
+
 use std::collections::{BTreeMap, HashSet};
 use std::convert::TryInto;
+use std::io;
 use std::io::Cursor;
 
 use pgp::armor::BlockType;
 use pgp::composed::{
     Deserializable, KeyType as PgpKeyType, Message, SecretKeyParamsBuilder, SignedPublicKey,
-    SignedSecretKey, SubkeyParamsBuilder,
+    SignedPublicSubKey, SignedSecretKey, SubkeyParamsBuilder,
 };
 use pgp::crypto::{HashAlgorithm, SymmetricKeyAlgorithm};
-use pgp::types::{CompressionAlgorithm, KeyTrait, SecretKeyTrait, StringToKey};
-use rand::thread_rng;
+use pgp::types::{
+    CompressionAlgorithm, KeyTrait, Mpi, PublicKeyTrait, SecretKeyTrait, StringToKey,
+};
+use rand::{thread_rng, CryptoRng, Rng};
 
 use crate::error::Error;
 use crate::key::*;
@@ -18,6 +23,68 @@ use crate::keyring::*;
 pub const HEADER_AUTOCRYPT: &str = "autocrypt-prefer-encrypt";
 pub const HEADER_SETUPCODE: &str = "passphrase-begin";
 
+/// A wrapper for rPGP public key types
+#[derive(Debug)]
+enum SignedPublicKeyOrSubkey<'a> {
+    Key(&'a SignedPublicKey),
+    Subkey(&'a SignedPublicSubKey),
+}
+
+impl<'a> KeyTrait for SignedPublicKeyOrSubkey<'a> {
+    fn fingerprint(&self) -> Vec<u8> {
+        match self {
+            Self::Key(k) => k.fingerprint(),
+            Self::Subkey(k) => k.fingerprint(),
+        }
+    }
+
+    fn key_id(&self) -> pgp::types::KeyId {
+        match self {
+            Self::Key(k) => k.key_id(),
+            Self::Subkey(k) => k.key_id(),
+        }
+    }
+
+    fn algorithm(&self) -> pgp::crypto::PublicKeyAlgorithm {
+        match self {
+            Self::Key(k) => k.algorithm(),
+            Self::Subkey(k) => k.algorithm(),
+        }
+    }
+}
+
+impl<'a> PublicKeyTrait for SignedPublicKeyOrSubkey<'a> {
+    fn verify_signature(
+        &self,
+        hash: HashAlgorithm,
+        data: &[u8],
+        sig: &[Mpi],
+    ) -> pgp::errors::Result<()> {
+        match self {
+            Self::Key(k) => k.verify_signature(hash, data, sig),
+            Self::Subkey(k) => k.verify_signature(hash, data, sig),
+        }
+    }
+
+    fn encrypt<R: Rng + CryptoRng>(
+        &self,
+        rng: &mut R,
+        plain: &[u8],
+    ) -> pgp::errors::Result<Vec<Mpi>> {
+        match self {
+            Self::Key(k) => k.encrypt(rng, plain),
+            Self::Subkey(k) => k.encrypt(rng, plain),
+        }
+    }
+
+    fn to_writer_old(&self, writer: &mut impl io::Write) -> pgp::errors::Result<()> {
+        match self {
+            Self::Key(k) => k.to_writer_old(writer),
+            Self::Subkey(k) => k.to_writer_old(writer),
+        }
+    }
+}
+
 /// Split data from PGP Armored Data as defined in https://tools.ietf.org/html/rfc4880#section-6.2.
 ///
 /// Returns (type, headers, base64 encoded body).
@@ -97,20 +164,49 @@ pub fn create_keypair(addr: impl AsRef<str>) -> Option<(Key, Key)> {
     Some((Key::Public(public_key), Key::Secret(private_key)))
 }
 
+/// Select public key or subkey to use for encryption.
+///
+/// First, tries to use subkeys. If none of the subkeys are suitable
+/// for encryption, tries to use primary key. Returns `None` if the public
+/// key cannot be used for encryption.
+///
+/// TODO: take key flags and expiration dates into account
+fn select_pk_for_encryption(key: &SignedPublicKey) -> Option<SignedPublicKeyOrSubkey> {
+    key.public_subkeys
+        .iter()
+        .find(|subkey| subkey.is_encryption_key())
+        .map_or_else(
+            || {
+                // No usable subkey found, try primary key
+                if key.is_encryption_key() {
+                    Some(SignedPublicKeyOrSubkey::Key(key))
+                } else {
+                    None
+                }
+            },
+            |subkey| Some(SignedPublicKeyOrSubkey::Subkey(subkey)),
+        )
+}
+
+/// Encrypts `plain` text using `public_keys_for_encryption`
+/// and signs it using `private_key_for_signing`.
 pub fn pk_encrypt(
     plain: &[u8],
     public_keys_for_encryption: &Keyring,
     private_key_for_signing: Option<&Key>,
 ) -> Result<String, Error> {
     let lit_msg = Message::new_literal_bytes("", plain);
-    let pkeys: Vec<&SignedPublicKey> = public_keys_for_encryption
+    let pkeys: Vec<SignedPublicKeyOrSubkey> = public_keys_for_encryption
         .keys()
         .iter()
         .filter_map(|key| {
-            let k: &Key = &key;
-            k.try_into().ok()
+            key.as_ref()
+                .try_into()
+                .ok()
+                .and_then(select_pk_for_encryption)
         })
         .collect();
+    let pkeys_refs: Vec<&SignedPublicKeyOrSubkey> = pkeys.iter().collect();
 
     let mut rng = thread_rng();
 
@@ -123,9 +219,9 @@ pub fn pk_encrypt(
         lit_msg
             .sign(skey, || "".into(), Default::default())
             .and_then(|msg| msg.compress(CompressionAlgorithm::ZLIB))
-            .and_then(|msg| msg.encrypt_to_keys(&mut rng, Default::default(), &pkeys))
+            .and_then(|msg| msg.encrypt_to_keys(&mut rng, Default::default(), &pkeys_refs))
     } else {
-        lit_msg.encrypt_to_keys(&mut rng, Default::default(), &pkeys)
+        lit_msg.encrypt_to_keys(&mut rng, Default::default(), &pkeys_refs)
     };
 
     let msg = encrypted_msg?;

src/qr.rs

@@ -1,3 +1,5 @@
+//! # QR code module
+
 use lazy_static::lazy_static;
 use percent_encoding::percent_decode_str;
 
@@ -150,7 +152,7 @@ fn decode_openpgp(context: &Context, qr: &str) -> Lot {
             let (id, _) = chat::create_or_lookup_by_contact_id(context, lot.id, Blocked::Deaddrop)
                 .unwrap_or_default();
 
-            chat::add_device_msg(
+            chat::add_info_msg(
                 context,
                 id,
                 format!("{} verified.", peerstate.addr.unwrap_or_default()),

src/securejoin.rs

@@ -1,3 +1,5 @@
+//! Verified contact protocol implementation as [specified by countermitm project](https://countermitm.readthedocs.io/en/stable/new.html#setup-contact-protocol)
+
 use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC};
 
 use crate::aheader::EncryptPreference;
@@ -638,7 +640,7 @@ fn secure_connection_established(context: &Context, contact_chat_id: u32) {
         "?"
     };
     let msg = context.stock_string_repl_str(StockMessage::ContactVerified, addr);
-    chat::add_device_msg(context, contact_chat_id, msg);
+    chat::add_info_msg(context, contact_chat_id, msg);
     emit_event!(context, Event::ChatModified(contact_chat_id));
 }
 
@@ -654,7 +656,7 @@ fn could_not_establish_secure_connection(context: &Context, contact_chat_id: u32
         },
     );
 
-    chat::add_device_msg(context, contact_chat_id, &msg);
+    chat::add_info_msg(context, contact_chat_id, &msg);
     error!(context, "{} ({})", &msg, details);
 }
 
@@ -735,7 +737,7 @@ pub fn handle_degrade_event(context: &Context, peerstate: &Peerstate) -> Result<
             };
             let msg = context.stock_string_repl_str(StockMessage::ContactSetupChanged, peeraddr);
 
-            chat::add_device_msg(context, contact_chat_id, msg);
+            chat::add_info_msg(context, contact_chat_id, msg);
             emit_event!(context, Event::ChatModified(contact_chat_id));
         }
     }

src/smtp.rs

@@ -5,7 +5,7 @@ use crate::constants::*;
 use crate::context::Context;
 use crate::error::Error;
 use crate::events::Event;
-use crate::login_param::{dc_build_tls, LoginParam};
+use crate::login_param::{dc_build_tls_config, LoginParam};
 use crate::oauth2::*;
 
 #[derive(DebugStub)]
@@ -44,43 +44,41 @@ impl Smtp {
     }
 
     /// Connect using the provided login params
-    pub fn connect(&mut self, context: &Context, lp: &LoginParam) -> bool {
+    pub fn connect(&mut self, context: &Context, lp: &LoginParam) -> Result<(), Error> {
         if self.is_connected() {
             warn!(context, "SMTP already connected.");
-            return true;
+            return Ok(());
         }
 
         if lp.send_server.is_empty() || lp.send_port == 0 {
             context.call_cb(Event::ErrorNetwork("SMTP bad parameters.".into()));
+            bail!("SMTP Bad parameters");
         }
 
-        self.from = if let Ok(addr) = EmailAddress::new(lp.addr.clone()) {
-            Some(addr)
-        } else {
-            None
+        self.from = match EmailAddress::new(lp.addr.clone()) {
+            Ok(addr) => Some(addr),
+            Err(err) => {
+                bail!("invalid login address {}: {}", lp.addr, err);
+            }
         };
 
-        if self.from.is_none() {
-            // TODO: print error
-            return false;
-        }
-
         let domain = &lp.send_server;
         let port = lp.send_port as u16;
 
-        let tls = dc_build_tls(lp.smtp_certificate_checks).unwrap();
-        let tls_parameters = ClientTlsParameters::new(domain.to_string(), tls);
+        let tls_config = dc_build_tls_config(lp.smtp_certificate_checks);
+        let tls_parameters = ClientTlsParameters::new(domain.to_string(), tls_config);
 
         let (creds, mechanism) = if 0 != lp.server_flags & (DC_LP_AUTH_OAUTH2 as i32) {
             // oauth2
             let addr = &lp.addr;
             let send_pw = &lp.send_pw;
             let access_token = dc_get_oauth2_access_token(context, addr, send_pw, false);
-            if access_token.is_none() {
-                return false;
-            }
+            ensure!(
+                access_token.is_some(),
+                "could not get oaut2_access token addr={}",
+                addr
+            );
             let user = &lp.send_user;
-
             (
                 lettre::smtp::authentication::Credentials::new(
                     user.to_string(),
@@ -125,27 +123,27 @@ impl Smtp {
                             "SMTP-LOGIN as {} ok",
                             lp.send_user,
                         )));
-                        return true;
+                        return Ok(());
                     }
                     Err(err) => {
-                        warn!(context, "SMTP: failed to connect {:?}", err);
+                        bail!("SMTP: failed to connect {:?}", err);
                     }
                 }
             }
             Err(err) => {
-                warn!(context, "SMTP: failed to setup connection {:?}", err);
+                bail!("SMTP: failed to setup connection {:?}", err);
             }
         }
-        false
     }
 
     /// SMTP-Send a prepared mail to recipients.
-    /// returns boolean whether send was successful.
-    pub fn send<'a>(
+    /// on successful send out Ok() is returned.
+    pub fn send(
         &mut self,
         context: &Context,
         recipients: Vec<EmailAddress>,
         message: Vec<u8>,
+        job_id: u32,
     ) -> Result<(), Error> {
         let message_len = message.len();
 
@@ -156,12 +154,15 @@ impl Smtp {
             .join(",");
 
         if let Some(ref mut transport) = self.transport {
-            let envelope = Envelope::new(self.from.clone(), recipients);
-            ensure!(envelope.is_ok(), "internal smtp-message construction fail");
-            let envelope = envelope.unwrap();
+            let envelope = match Envelope::new(self.from.clone(), recipients) {
+                Ok(env) => env,
+                Err(err) => {
+                    bail!("{}", err);
+                }
+            };
             let mail = SendableEmail::new(
                 envelope,
-                "mail-id".into(), // TODO: random id
+                format!("{}", job_id), // only used for internal logging
                 message,
             );
 

src/sql.rs

@@ -1,3 +1,5 @@
+//! # SQLite wrapper
+
 use std::collections::HashSet;
 use std::sync::{Arc, RwLock};
 use std::time::Duration;
@@ -5,6 +7,8 @@ use std::time::Duration;
 use rusqlite::{Connection, OpenFlags, Statement, NO_PARAMS};
 use thread_local_object::ThreadLocal;
 
+use crate::chat::update_saved_messages_icon;
+use crate::constants::ShowEmails;
 use crate::context::Context;
 use crate::dc_tools::*;
 use crate::error::{Error, Result};
@@ -92,8 +96,7 @@ impl Sql {
         self.start_stmt(sql.to_string());
         self.with_conn(|conn| {
             let stmt = conn.prepare(sql)?;
-            let res = g(stmt, conn)?;
-            Ok(res)
+            g(stmt, conn)
         })
     }
 
@@ -106,8 +109,7 @@ impl Sql {
             let stmt1 = conn.prepare(sql1)?;
             let stmt2 = conn.prepare(sql2)?;
 
-            let res = g(stmt1, stmt2, conn)?;
-            Ok(res)
+            g(stmt1, stmt2, conn)
         })
     }
 
@@ -350,7 +352,7 @@ fn open(
     }
 
     if !readonly {
-        let mut exists_before_update = 0;
+        let mut exists_before_update = false;
         let mut dbversion_before_update = 0;
         /* Init tables to dbversion=0 */
         if !sql.table_exists("config") {
@@ -385,8 +387,8 @@ fn open(
             )?;
             sql.execute(
                 "INSERT INTO contacts (id,name,origin) VALUES \
-                 (1,'self',262144), (2,'device',262144), (3,'rsvd',262144), \
-                 (4,'rsvd',262144), (5,'rsvd',262144), (6,'rsvd',262144), \
+                 (1,'self',262144), (2,'info',262144), (3,'rsvd',262144), \
+                 (4,'rsvd',262144), (5,'device',262144), (6,'rsvd',262144), \
                  (7,'rsvd',262144), (8,'rsvd',262144), (9,'rsvd',262144);",
                 params![],
             )?;
@@ -480,7 +482,7 @@ fn open(
                 sql.set_raw_config_int(context, "dbversion", 0)?;
             }
         } else {
-            exists_before_update = 1;
+            exists_before_update = true;
             dbversion_before_update = sql
                 .get_raw_config_int(context, "dbversion")
                 .unwrap_or_default();
@@ -490,9 +492,11 @@ fn open(
         // this should be done before updates that use high-level objects that
         // rely themselves on the low-level structure.
         // --------------------------------------------------------------------
+
         let mut dbversion = dbversion_before_update;
         let mut recalc_fingerprints = 0;
         let mut update_file_paths = 0;
+        let mut update_icons = false;
 
         if dbversion < 1 {
             info!(context, "[migration] v1");
@@ -590,6 +594,8 @@ fn open(
         }
         if dbversion < 27 {
             info!(context, "[migration] v27");
+            // chat.id=1 and chat.id=2 are the old deaddrops,
+            // the current ones are defined by chats.blocked=2
             sql.execute("DELETE FROM msgs WHERE chat_id=1 OR chat_id=2;", params![])?;
             sql.execute(
                 "CREATE INDEX chats_contacts_index2 ON chats_contacts (contact_id);",
@@ -655,6 +661,9 @@ fn open(
                 params![],
             )?;
             if dbversion_before_update == 34 {
+                // migrate database from the use of verified-flags to verified_key,
+                // _only_ version 34 (0.17.0) has the fields public_key_verified and gossip_key_verified
+                // this block can be deleted in half a year or so (created 5/2018)
                 sql.execute(
                     "UPDATE acpeerstates SET verified_key=gossip_key, verified_key_fingerprint=gossip_key_fingerprint WHERE gossip_key_verified=2;",
                     params![]
@@ -684,6 +693,8 @@ fn open(
         }
         if dbversion < 42 {
             info!(context, "[migration] v42");
+            // older versions set the txt-field to the filenames, for debugging and fulltext search.
+            // to allow text+attachment compound messages, we need to reset these fields.
             sql.execute("UPDATE msgs SET txt='' WHERE type!=10", params![])?;
             dbversion = 42;
             sql.set_raw_config_int(context, "dbversion", 42)?;
@@ -737,14 +748,19 @@ fn open(
         }
         if dbversion < 50 {
             info!(context, "[migration] v50");
-            if 0 != exists_before_update {
-                sql.set_raw_config_int(context, "show_emails", 2)?;
+            // installations <= 0.100.1 used DC_SHOW_EMAILS_ALL implicitly;
+            // keep this default and use DC_SHOW_EMAILS_NO
+            // only for new installations
+            if exists_before_update {
+                sql.set_raw_config_int(context, "show_emails", ShowEmails::All as i32)?;
             }
             dbversion = 50;
             sql.set_raw_config_int(context, "dbversion", 50)?;
         }
         if dbversion < 53 {
             info!(context, "[migration] v53");
+            // the messages containing _only_ locations
+            // are also added to the database as _hidden_.
             sql.execute(
                 "CREATE TABLE locations ( id INTEGER PRIMARY KEY AUTOINCREMENT, latitude REAL DEFAULT 0.0, longitude REAL DEFAULT 0.0, accuracy REAL DEFAULT 0.0, timestamp INTEGER DEFAULT 0, chat_id INTEGER DEFAULT 0, from_id INTEGER DEFAULT 0);",
                 params![]
@@ -792,9 +808,30 @@ fn open(
                 "ALTER TABLE locations ADD COLUMN independent INTEGER DEFAULT 0;",
                 params![],
             )?;
-
             sql.set_raw_config_int(context, "dbversion", 55)?;
         }
+        if dbversion < 59 {
+            info!(context, "[migration] v59");
+            // records in the devmsglabels are kept when the message is deleted.
+            // so, msg_id may or may not exist.
+            sql.execute(
+                "CREATE TABLE devmsglabels (id INTEGER PRIMARY KEY AUTOINCREMENT, label TEXT, msg_id INTEGER DEFAULT 0);",
+                NO_PARAMS,
+            )?;
+            sql.execute(
+                "CREATE INDEX devmsglabels_index1 ON devmsglabels (label);",
+                NO_PARAMS,
+            )?;
+            if exists_before_update && sql.get_raw_config_int(context, "bcc_self").is_none() {
+                sql.set_raw_config_int(context, "bcc_self", 1)?;
+            }
+            update_icons = true;
+            sql.set_raw_config_int(context, "dbversion", 59)?;
+        }
+
+        // (2) updates that require high-level objects
+        // (the structure is complete now and all objects are usable)
+        // --------------------------------------------------------------------
 
         if 0 != recalc_fingerprints {
             info!(context, "[migration] recalc fingerprints");
@@ -842,6 +879,9 @@ fn open(
 
             sql.set_raw_config(context, "backup_for", None)?;
         }
+        if update_icons {
+            update_saved_messages_icon(context)?;
+        }
     }
 
     info!(context, "Opened {:?}.", dbfile.as_ref(),);

src/stock.rs

@@ -1,3 +1,5 @@
+//! Module to work with translatable stock strings
+
 use std::borrow::Cow;
 
 use strum::EnumProperty;
@@ -110,6 +112,10 @@ pub enum StockMessage {
     Location = 66,
     #[strum(props(fallback = "Sticker"))]
     Sticker = 67,
+    #[strum(props(fallback = "Device messages"))]
+    DeviceMessages = 68,
+    #[strum(props(fallback = "Saved messages"))]
+    SavedMessages = 69,
 }
 
 impl StockMessage {

src/token.rs

@@ -1,4 +1,8 @@
+//! # Token module
+//!
 //! Functions to read/write token from/to the database. A token is any string associated with a key.
+//!
+//! Tokens are used in countermitm verification protocols.
 
 use deltachat_derive::*;
 
@@ -6,7 +10,7 @@ use crate::context::Context;
 use crate::dc_tools::*;
 use crate::sql;
 
-// Token namespaces
+/// Token namespace
 #[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql)]
 #[repr(i32)]
 pub enum Namespace {
@@ -21,6 +25,8 @@ impl Default for Namespace {
     }
 }
 
+/// Creates a new token and saves it into the database.
+/// Returns created token.
 pub fn save(context: &Context, namespace: Namespace, foreign_id: u32) -> String {
     // foreign_id may be 0
     let token = dc_create_id();

src/wrapmime.rs

@@ -49,20 +49,27 @@ pub fn get_ct_subtype(mime: *mut Mailmime) -> Option<String> {
     }
 }
 
-pub fn parse_message_id(message_id: &str) -> Result<String, Error> {
+pub fn parse_message_id(message_id: &[u8]) -> Result<String, Error> {
     let mut dummy = 0;
-    let c_message_id = CString::new(message_id).unwrap_or_default();
-    let c_ptr = c_message_id.as_ptr();
     let mut rfc724_mid_c = std::ptr::null_mut();
-    if unsafe { mailimf_msg_id_parse(c_ptr, libc::strlen(c_ptr), &mut dummy, &mut rfc724_mid_c) }
-        == MAIL_NO_ERROR as libc::c_int
+    if unsafe {
+        mailimf_msg_id_parse(
+            message_id.as_ptr() as *const libc::c_char,
+            message_id.len(),
+            &mut dummy,
+            &mut rfc724_mid_c,
+        )
+    } == MAIL_NO_ERROR as libc::c_int
         && !rfc724_mid_c.is_null()
     {
         let res = to_string_lossy(rfc724_mid_c);
         unsafe { libc::free(rfc724_mid_c.cast()) };
         Ok(res)
     } else {
-        bail!("could not parse message_id: {}", message_id);
+        bail!(
+            "could not parse message_id: {}",
+            String::from_utf8_lossy(message_id)
+        );
     }
 }
 
@@ -143,7 +150,8 @@ pub fn get_field_date(imffields: *mut mailimf_fields) -> Result<i64, Error> {
 
 fn mailimf_get_recipients_add_addr(recipients: &mut HashSet<String>, mb: *mut mailimf_mailbox) {
     if !mb.is_null() {
-        let addr_norm = addr_normalize(as_str(unsafe { (*mb).mb_addr_spec }));
+        let addr = to_string_lossy(unsafe { (*mb).mb_addr_spec });
+        let addr_norm = addr_normalize(&addr);
         recipients.insert(addr_norm.into());
     }
 }
@@ -382,8 +390,8 @@ pub fn mailimf_find_first_addr(mb_list: *const mailimf_mailbox_list) -> Option<S
     for cur in unsafe { (*(*mb_list).mb_list).into_iter() } {
         let mb = cur as *mut mailimf_mailbox;
         if !mb.is_null() && !unsafe { (*mb).mb_addr_spec.is_null() } {
-            let addr = unsafe { as_str((*mb).mb_addr_spec) };
-            return Some(addr_normalize(addr).to_string());
+            let addr = unsafe { to_string_lossy((*mb).mb_addr_spec) };
+            return Some(addr_normalize(&addr).to_string());
         }
     }
 
@@ -489,7 +497,9 @@ pub fn content_type_needs_encoding(content: *const mailmime_content) -> bool {
         if (*(*content).ct_type).tp_type == MAILMIME_TYPE_COMPOSITE_TYPE as libc::c_int {
             let composite = (*(*content).ct_type).tp_data.tp_composite_type;
             match (*composite).ct_type as u32 {
-                MAILMIME_COMPOSITE_TYPE_MESSAGE => as_str((*content).ct_subtype) != "rfc822",
+                MAILMIME_COMPOSITE_TYPE_MESSAGE => {
+                    to_string_lossy((*content).ct_subtype) != "rfc822"
+                }
                 MAILMIME_COMPOSITE_TYPE_MULTIPART => false,
                 _ => false,
             }
@@ -540,11 +550,11 @@ mod tests {
     #[test]
     fn test_parse_message_id() {
         assert_eq!(
-            parse_message_id("Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org").unwrap(),
+            parse_message_id(b"Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org").unwrap(),
             "Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org"
         );
         assert_eq!(
-            parse_message_id("<Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org>").unwrap(),
+            parse_message_id(b"<Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org>").unwrap(),
             "Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org"
         );
     }

tests/stress.rs

@@ -261,22 +261,6 @@ fn test_stress_tests() {
     }
 }
 
-#[test]
-fn test_get_contacts() {
-    let context = create_test_context();
-    let contacts = Contact::get_all(&context.ctx, 0, Some("some2")).unwrap();
-    assert_eq!(contacts.len(), 0);
-
-    let id = Contact::create(&context.ctx, "bob", "bob@mail.de").unwrap();
-    assert_ne!(id, 0);
-
-    let contacts = Contact::get_all(&context.ctx, 0, Some("bob")).unwrap();
-    assert_eq!(contacts.len(), 1);
-
-    let contacts = Contact::get_all(&context.ctx, 0, Some("alice")).unwrap();
-    assert_eq!(contacts.len(), 0);
-}
-
 #[test]
 fn test_chat() {
     let context = create_test_context();