address @link2xt comment

and otherwise properly fetch the last messages to determine the last seen uid.
also add some tracing.

.circleci/config.yml

@@ -4,6 +4,9 @@ executors:
     docker:
       - image: filecoin/rust:latest
     working_directory: /mnt/crate
+  doxygen:
+    docker:
+      - image: hrektts/doxygen
 
 restore-workspace: &restore-workspace
   attach_workspace:
@@ -12,7 +15,7 @@ restore-workspace: &restore-workspace
 restore-cache: &restore-cache
   restore_cache:
     keys:
-      - cargo-v2-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+      - cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
       - repo-source-{{ .Branch }}-{{ .Revision }}
 
 commands:
@@ -41,7 +44,7 @@ jobs:
           command: cargo generate-lockfile
       - restore_cache:
           keys:
-            - cargo-v2-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+            - cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
       - run: rustup install $(cat rust-toolchain)
       - run: rustup default $(cat rust-toolchain)
       - run: rustup component add --toolchain $(cat rust-toolchain) rustfmt
@@ -57,7 +60,7 @@ jobs:
           paths:
             - crate
       - save_cache:
-          key: cargo-v2-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+          key: cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
           paths:
             - "~/.cargo"
             - "~/.rustup"
@@ -113,32 +116,59 @@ jobs:
           target: "aarch64-linux-android"
 
 
-  build_test_docs_wheel:
-    docker:
-      - image: deltachat/coredeps
-        environment:
-            TESTS: 1
-            DOCS: 1
-    working_directory: /mnt/crate
+  build_doxygen:
+    executor: doxygen
     steps:
-      - *restore-workspace
-      - *restore-cache
-      - run:
-          name: build docs, run tests and build wheels
-          command: ci_scripts/run-python.sh
-      - run:
-           name: copying docs and wheels to workspace
-           command: |
-             mkdir -p workspace/python
-             # cp -av docs workspace/c-docs
-             cp -av python/.docker-tox/wheelhouse workspace/
-             cp -av python/doc/_build/ workspace/py-docs
+      - checkout
+      - run: bash ci_scripts/run-doxygen.sh
+      - run: mkdir -p workspace/c-docs
+      - run: cp -av deltachat-ffi/{html,xml} workspace/c-docs/
       - persist_to_workspace:
           root: workspace
           paths:
-            # - c-docs
-            - py-docs
-            - wheelhouse
+            - c-docs
+
+  # build_test_docs_wheel:
+  #   docker:
+  #     - image: deltachat/coredeps
+  #       environment:
+  #           TESTS: 1
+  #           DOCS: 1
+  #   working_directory: /mnt/crate
+  #   steps:
+  #     - *restore-workspace
+  #     - *restore-cache
+  #     - run:
+  #         name: build docs, run tests and build wheels
+  #         command: ci_scripts/run-python.sh
+  #     - run:
+  #          name: copying docs and wheels to workspace
+  #          command: |
+  #            mkdir -p workspace/python
+  #            # cp -av docs workspace/c-docs
+  #            cp -av python/.docker-tox/wheelhouse workspace/
+  #            cp -av python/doc/_build/ workspace/py-docs
+  #     - persist_to_workspace:
+  #         root: workspace
+  #         paths:
+  #           # - c-docs
+  #           - py-docs
+  #           - wheelhouse
+
+  remote_tests_rust:
+    machine: true
+    steps:
+      - checkout
+      - run: ci_scripts/remote_tests_rust.sh
+    
+  remote_tests_python:
+    machine: true
+    steps:
+      - checkout
+      #- attach_workspace:
+      #    at: workspace
+      - run: ci_scripts/remote_tests_python.sh
+    # workspace/py-docs workspace/wheelhouse workspace/c-docs
 
   upload_docs_wheels:
     machine: true
@@ -148,7 +178,7 @@ jobs:
           at: workspace
       - run: pyenv global 3.5.2
       - run: ls -laR workspace
-      - run: ci_scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse
+      - run: ci_scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
 
   clippy:
     executor: default
@@ -166,23 +196,28 @@ workflows:
   test:
     jobs:
       - cargo_fetch
-      - build_test_docs_wheel:
-          requires:
-            - cargo_fetch
-      - upload_docs_wheels:
-          requires:
-            - build_test_docs_wheel
+
+      - remote_tests_rust
+
+      - remote_tests_python
+
+      # - upload_docs_wheels:
+      #     requires:
+      #       - build_test_docs_wheel
+      #       - build_doxygen
       - rustfmt:
           requires:
             - cargo_fetch
       - clippy:
           requires:
-            - cargo_fetch
+           - cargo_fetch
+
+      - build_doxygen
 
       # Linux Desktop 64bit
-      - test_x86_64-unknown-linux-gnu:
-          requires:
-            - cargo_fetch
+      # - test_x86_64-unknown-linux-gnu:
+      #     requires:
+      #       - cargo_fetch
 
       # Linux Desktop 32bit
       # - test_i686-unknown-linux-gnu:

.gitignore

@@ -23,3 +23,5 @@ python/liveconfig*
 # ignore doxgen generated files
 deltachat-ffi/html
 deltachat-ffi/xml
+
+.rsynclist

CHANGELOG.md

@@ -0,0 +1,120 @@
+# Changelog 
+
+## 1.0.0-beta.8
+
+- now uses async-email/async-imap as the new base 
+  which makes imap-idle interruptible and thus fixes
+  several issues around the imap thread being in zombie state . 
+  thanks @dignifiedquire, @hpk42 and @link2xt. 
+
+- fixes imap-protocol parsing bugs that lead to infinitely
+  repeated crashing while trying to receive messages with
+  a subjec that contained non-utf8. thanks @link2xt
+
+- fixed logic to find encryption subkey -- previously 
+  delta chat would use the primary key for encryption
+  (which works with RSA but not ECC). thanks @link2xt
+
+- introduce a new device chat where core and UIs can 
+  add "device" messages.  Android uses it for an initial
+  welcome message. thanks @r10s
+
+- fix time smearing (when two message are virtually send
+  in the same second, there would be misbehaviour because
+  we didn't persist smeared time). thanks @r10s
+
+- fix double-dotted extensions like .html.zip or .tar.gz  
+  to not mangle them when creating blobfiles.  thanks @flub
+
+- fix backup/exports where the wrong sql file would be modified,
+  leading to problems when exporting twice.  thanks @hpk42
+
+- several other little fixes and improvements 
+
+
+## 1.0.0-beta.7
+
+- fix location-streaming #782
+
+- fix display of messages that could not be decrypted #785
+ 
+- fix smtp MAILER-DAEMON bug #786 
+
+- fix a logging of durations #783
+
+- add more error logging #779
+
+- do not panic on some bad utf-8 mime #776
+
+## 1.0.0-beta.6
+
+- fix chatlist.get_msg_id to return id, instead of wrongly erroring
+
+## 1.0.0-beta.5
+
+- fix dc_get_msg() to return empty messages when asked for special ones 
+
+## 1.0.0-beta.4
+
+- fix more than one sending of autocrypt setup message
+
+- fix recognition of mailto-address-qr-codes, add tests
+
+- tune down error to warning when adding self to chat
+
+## 1.0.0-beta.3
+
+- add back `dc_empty_server()` #682
+
+- if `show_emails` is set to `DC_SHOW_EMAILS_ALL`,
+  email-based contact requests are added to the chatlist directly
+
+- fix IMAP hangs #717 and cleanups
+
+- several rPGP fixes
+
+- code streamlining and rustifications
+
+
+## 1.0.0-beta.2
+
+- https://c.delta.chat docs are now regenerated again through our CI 
+
+- several rPGP cleanups, security fixes and better multi-platform support 
+
+- reconnect on io errors and broken pipes (imap)
+
+- probe SMTP with real connection not just setup
+
+- various imap/smtp related fixes
+
+- use to_string_lossy in most places instead of relying on valid utf-8
+  encodings
+ 
+- rework, rustify and test autoconfig-reading and parsing 
+
+- some rustifications/boolifications of c-ints 
+
+
+## 1.0.0-beta.1 
+
+- first beta of the Delta Chat Rust core library. many fixes of crashes
+  and other issues compared to 1.0.0-alpha.5.
+
+- Most code is now "rustified" and does not do manual memory allocation anymore. 
+
+- The `DC_EVENT_GET_STRING` event is not used anymore, removing the last
+  event where the core requested a return value from the event callback. 
+
+  Please now use `dc_set_stock_translation()` API for core messages
+  to be properly localized. 
+
+- Deltachat FFI docs are automatically generated and available here: 
+  https://c.delta.chat 
+
+- New events ImapMessageMoved and ImapMessageDeleted
+
+For a full list of changes, please see our closed Pull Requests: 
+
+https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
+

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "deltachat"
-version = "1.0.0-alpha.5"
-authors = ["dignifiedquire <dignifiedquire@gmail.com>"]
+version = "1.0.0-beta.8"
+authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL"
 
@@ -10,17 +10,18 @@ deltachat_derive = { path = "./deltachat_derive" }
 mmime = { version = "0.1.2", path = "./mmime" }
 
 libc = "0.2.51"
-pgp = { version = "0.2", default-features = false }
-hex = "0.3.2"
+pgp = { git = "https://github.com/rpgp/rpgp", branch = "master", default-features = false }
+hex = "0.4.0"
 sha2 = "0.8.0"
 rand = "0.6.5"
 smallvec = "0.6.9"
-reqwest = "0.9.15"
+reqwest = { version = "0.9.15", default-features = false, features = ["rustls-tls"] }
 num-derive = "0.2.5"
 num-traits = "0.2.6"
-native-tls = "0.2.3"
-lettre = "0.9.0"
-imap = { git = "https://github.com/jonhoo/rust-imap", rev = "281d2eb8ab50dc656ceff2ae749ca5045f334e15" }
+lettre = { git = "https://github.com/deltachat/lettre", branch = "feat/rustls" }
+async-imap = { git = "https://github.com/async-email/async-imap", branch="master" }
+async-tls = "0.6"
+async-std = { version = "1.0", features = ["unstable"] }
 base64 = "0.10"
 charset = "0.1"
 percent-encoding = "2.0"
@@ -48,6 +49,11 @@ escaper = "0.1.0"
 bitflags = "1.1.0"
 jetscii = "0.4.4"
 debug_stub_derive = "0.3.0"
+sanitize-filename = "0.2.1"
+stop-token = { version = "0.1.1", features = ["unstable"] }
+rustls = "0.16.0"
+webpki-roots = "0.18.0"
+webpki = "0.21.0"
 
 [dev-dependencies]
 tempfile = "3.0"
@@ -73,6 +79,6 @@ path = "examples/repl/main.rs"
 
 [features]
 default = ["nightly", "ringbuf"]
-vendored = ["native-tls/vendored", "reqwest/default-tls-vendored"]
+vendored = []
 nightly = ["pgp/nightly"]
 ringbuf = ["pgp/ringbuf"]

README.md

@@ -1,11 +1,9 @@
 # Delta Chat Rust
 
-> Project porting deltachat-core to rust
+> Deltachat-core written in Rust 
 
 [![CircleCI build status][circle-shield]][circle] [![Appveyor build status][appveyor-shield]][appveyor]
 
-Current commit on deltachat/deltachat-core: `12ef73c8e76185f9b78e844ea673025f56a959ab`.
-
 ## Installing Rust and Cargo
 
 To download and install the official compiler for the Rust programming language, and the Cargo package manager, run the command in your user environment:
@@ -16,7 +14,7 @@ curl https://sh.rustup.rs -sSf | sh
 
 ## Using the CLI client
 
-Compile and run Delta Chat Core using `cargo`:
+Compile and run Delta Chat Core command line utility, using `cargo`:
 
 ```
 cargo run --example repl -- /path/to/db

Xargo.toml

@@ -1,6 +0,0 @@
-[dependencies.std]
-features = ["panic-unwind"]
-
-# if using `cargo test`
-[dependencies.test]
-stage = 1

assets/icon-device.svg

@@ -0,0 +1,110 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:xlink="http://www.w3.org/1999/xlink"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   inkscape:version="1.0beta1 (32d4812, 2019-09-19)"
+   sodipodi:docname="icon-device.svg"
+   id="svg4344"
+   preserveAspectRatio="xMidYMid meet"
+   viewBox="0 0 45 45"
+   height="60"
+   width="60"
+   version="1.0"
+   inkscape:export-filename="/Users/bpetersen/Documents/deltachat/icons/device4.png"
+   inkscape:export-xdpi="96"
+   inkscape:export-ydpi="96">
+  <defs
+     id="defs4348">
+    <linearGradient
+       inkscape:collect="always"
+       id="linearGradient865">
+      <stop
+         style="stop-color:#364e59;stop-opacity:1;"
+         offset="0"
+         id="stop861" />
+      <stop
+         style="stop-color:#ffffff;stop-opacity:1"
+         offset="1"
+         id="stop863" />
+    </linearGradient>
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient865"
+       id="linearGradient867"
+       x1="10.227795"
+       y1="43.5238"
+       x2="-25.458139"
+       y2="-7.4257693"
+       gradientUnits="userSpaceOnUse" />
+  </defs>
+  <sodipodi:namedview
+     inkscape:current-layer="svg4344"
+     inkscape:window-maximized="0"
+     inkscape:window-y="23"
+     inkscape:window-x="45"
+     inkscape:cy="28.471578"
+     inkscape:cx="41.455164"
+     inkscape:zoom="5.6"
+     units="px"
+     showgrid="false"
+     id="namedview4346"
+     inkscape:window-height="1035"
+     inkscape:window-width="1600"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="0"
+     guidetolerance="10"
+     gridtolerance="10"
+     objecttolerance="10"
+     borderopacity="1"
+     inkscape:document-rotation="0"
+     bordercolor="#666666"
+     pagecolor="#ffffff" />
+  <metadata
+     id="metadata4336">
+Created by potrace 1.15, written by Peter Selinger 2001-2017
+<rdf:RDF>
+  <cc:Work
+     rdf:about="">
+    <dc:format>image/svg+xml</dc:format>
+    <dc:type
+       rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+    <dc:title />
+  </cc:Work>
+</rdf:RDF>
+</metadata>
+  <circle
+     style="opacity:1;fill:#ffffff;fill-opacity:1;stroke-width:0.731071"
+     id="path859"
+     cx="22.5"
+     cy="22.5"
+     r="22.5" />
+  <circle
+     r="22.5"
+     cy="22.5"
+     cx="22.5"
+     id="path4917"
+     style="fill:url(#linearGradient867);fill-opacity:1;stroke-width:0.588554" />
+  <g
+     id="g4342"
+     stroke="none"
+     fill="#000000"
+     transform="matrix(0.00255113,0,0,-0.00255113,5.586152,38.200477)"
+     style="fill:#ffffff;fill-opacity:1">
+    <path
+       inkscape:connector-curvature="0"
+       id="path4338"
+       d="m 8175,12765 c -703,-114 -1248,-608 -1387,-1258 -17,-82 -21,-136 -22,-277 0,-202 15,-307 70,-470 149,-446 499,-733 1009,-828 142,-26 465,-23 619,6 691,131 1201,609 1328,1244 31,158 31,417 0,565 -114,533 -482,889 -1038,1004 -133,27 -448,35 -579,14 z"
+       style="fill:#ffffff;fill-opacity:1" />
+    <path
+       inkscape:connector-curvature="0"
+       id="path4340"
+       d="m 7070,9203 c -212,-20 -275,-27 -397,-48 -691,-117 -1400,-444 -2038,-940 -182,-142 -328,-270 -585,-517 -595,-571 -911,-974 -927,-1181 -6,-76 11,-120 69,-184 75,-80 159,-108 245,-79 109,37 263,181 632,595 539,606 774,826 1035,969 135,75 231,105 341,106 82,1 94,-2 138,-27 116,-68 161,-209 122,-376 -9,-36 -349,-868 -757,-1850 -407,-982 -785,-1892 -838,-2021 -287,-694 -513,-1389 -615,-1889 -70,-342 -90,-683 -52,-874 88,-440 381,-703 882,-792 124,-23 401,-30 562,-16 783,69 1674,461 2561,1125 796,596 1492,1354 1607,1751 43,146 -33,308 -168,360 -61,23 -100,15 -173,-36 -105,-74 -202,-170 -539,-529 -515,-551 -762,-783 -982,-927 -251,-164 -437,-186 -543,-65 -56,64 -74,131 -67,247 13,179 91,434 249,815 135,324 1588,4102 1646,4280 106,325 151,561 159,826 9,281 -22,463 -112,652 -58,122 -114,199 -211,292 -245,233 -582,343 -1044,338 -91,-1 -181,-3 -200,-5 z"
+       style="fill:#ffffff;fill-opacity:1" />
+  </g>
+</svg>

assets/icon-saved-messages.svg

@@ -0,0 +1,99 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:xlink="http://www.w3.org/1999/xlink"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   inkscape:export-ydpi="204.8"
+   inkscape:export-xdpi="204.8"
+   inkscape:export-filename="/Users/bpetersen/Documents/deltachat/icons/saved-messages.png"
+   version="1.0"
+   width="60"
+   height="60"
+   viewBox="0 0 45 45"
+   preserveAspectRatio="xMidYMid meet"
+   id="svg4344"
+   sodipodi:docname="saved-messages.svg"
+   inkscape:version="1.0beta1 (32d4812, 2019-09-19)">
+  <defs
+     id="defs4348">
+    <linearGradient
+       inkscape:collect="always"
+       id="linearGradient906">
+      <stop
+         style="stop-color:#046cdb;stop-opacity:1"
+         offset="0"
+         id="stop902" />
+      <stop
+         id="stop845"
+         offset="1"
+         style="stop-color:#ffffff;stop-opacity:1" />
+    </linearGradient>
+    <linearGradient
+       y2="-11.049108"
+       x2="-3.1473193"
+       y1="31.473215"
+       x1="36.160713"
+       gradientUnits="userSpaceOnUse"
+       id="linearGradient847"
+       xlink:href="#linearGradient906"
+       inkscape:collect="always" />
+  </defs>
+  <sodipodi:namedview
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     inkscape:document-rotation="0"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="1600"
+     inkscape:window-height="1035"
+     id="namedview4346"
+     showgrid="false"
+     units="px"
+     inkscape:zoom="2.8"
+     inkscape:cx="48.214285"
+     inkscape:cy="55"
+     inkscape:window-x="45"
+     inkscape:window-y="23"
+     inkscape:window-maximized="0"
+     inkscape:current-layer="svg4344"
+     inkscape:lockguides="false" />
+  <metadata
+     id="metadata4336">
+Created by potrace 1.15, written by Peter Selinger 2001-2017
+<rdf:RDF>
+  <cc:Work
+     rdf:about="">
+    <dc:format>image/svg+xml</dc:format>
+    <dc:type
+       rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+    <dc:title />
+  </cc:Work>
+</rdf:RDF>
+</metadata>
+  <circle
+     r="22.5"
+     cy="22.5"
+     cx="22.5"
+     id="path859"
+     style="opacity:1;fill:#ffffff;fill-opacity:1;stroke-width:0.731071" />
+  <circle
+     style="fill:url(#linearGradient847);fill-opacity:1;stroke-width:0.588554"
+     id="path4917"
+     cx="22.5"
+     cy="22.5"
+     r="22.5" />
+  <path
+     id="rect846"
+     style="fill:#ffffff;stroke-width:0.84832"
+     d="M 17 9 L 17 55 L 17.125 55 L 30.048828 40.9375 L 42.974609 55 L 43 55 L 43 9 L 17 9 z M 30.005859 15 L 32.720703 22.121094 L 40 22.646484 L 34.396484 27.558594 L 36.169922 35 L 29.992188 30.916016 L 23.808594 34.988281 L 25.595703 27.550781 L 20 22.626953 L 27.28125 22.115234 L 30.005859 15 z "
+     transform="scale(0.75)" />
+</svg>

ci_scripts/README.md

@@ -1,52 +1,46 @@
-
 # Continuous Integration Scripts for Delta Chat
 
-Continuous Integration is run through CircleCI
-but is largely independent of it. 
+Continuous Integration, run through CircleCI and an own build machine.
+
+## Description of scripts 
+
+- `../.circleci/config.yml` describing the build jobs that are run
+  by Circle-CI
+
+- `remote_tests_python.sh` rsyncs to a build machine and runs
+  `run-python-test.sh` remotely on the build machine. 
+
+- `remote_tests_rust.sh` rsyncs to the build machine and runs
+  `run-rust-test.sh` remotely on the build machine. 
+
+- `doxygen/Dockerfile` specifies an image that contains
+  the doxygen tool which is used by `run-doxygen.sh` 
+  to generate C-docs which are then uploaded 
+  via `ci_upload.sh` to `https://c.delta.chat/_unofficial_unreleased_docs/<BRANCH>`
+  (and the master branch is linked to https://c.delta.chat proper). 
 
 
-## Generating docker containers for performing build step work 
+## Triggering runs on the build machine locally (fast!)
 
-All tests, docs and wheel building is run in docker containers:
+There is experimental support for triggering a remote Python or Rust test run 
+from your local checkout/branch. You will need to be authorized to login to 
+the build machine (ask your friendly sysadmin on #deltachat freenode) to type::
 
-- **coredeps/Dockerfile** specifies an image that contains all 
-  of Delta Chat's core dependencies as linkable libraries. 
-  It also serves to run python tests and build wheels 
-  (binary packages for Python). 
+    ci_scripts/manual_remote_tests.sh rust
+    ci_scripts/manual_remote_tests.sh python
 
-- **doxygen/Dockerfile** specifies an image that contains
-  the doxygen tool which is used to generate C-docs. 
+This will **rsync** your current checkout to the remote build machine 
+(no need to commit before) and then run either rust or python tests. 
 
-To run tests locally you can pull existing images from "docker.io",
-the hub for sharing Docker images::
+# Outdated files (for later re-use)
 
-    docker pull deltachat/coredeps
-    docker pull deltachat/doxygen 
+`coredeps/Dockerfile` specifies an image that contains all 
+of Delta Chat's core dependencies. It used to run 
+python tests and build wheels (binary packages for Python)
 
-or you can build the docker images yourself locally
+You can build the docker images yourself locally
 to avoid the relatively large download:: 
  
     cd ci_scripts  # where all CI things are 
     docker build -t deltachat/coredeps docker-coredeps
     docker build -t deltachat/doxygen docker-doxygen 
-
-## ci_run.sh (main entrypoint called by circle-ci)
-
-Once you have the docker images available 
-you can run python testing, documentation generation 
-and building binary wheels:: 
-
-    sh DOCS=1 TESTS=1 ci_scripts/ci_run.sh 
-    
-## ci_upload.sh (uploading artifacts on success) 
-
-- python docs to `https://py.delta.chat/_unofficial_unreleased_docs/<BRANCH>`
-
-- doxygen docs to `https://c.delta.chat/_unofficial_unreleased_docs/<BRANCH>`
-
-- python wheels to `https://m.devpi.net/dc/<BRANCH>`
-  so that you install fully self-contained wheels like this:
-  `pip install -U -i https://m.devpi.net/dc/<BRANCH> deltachat`
-
-
-

ci_scripts/ci_upload.sh

@@ -7,25 +7,30 @@ fi
 
 set -xe
 
-#DOXYDOCDIR=${1:?directory where doxygen docs to be found}
 PYDOCDIR=${1:?directory with python docs}
 WHEELHOUSEDIR=${2:?directory with pre-built wheels}
+DOXYDOCDIR=${3:?directory where doxygen docs to be found}
 
 export BRANCH=${CIRCLE_BRANCH:?specify branch for uploading purposes}
 
 
-# python docs to py.delta.chat
-ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
-rsync -avz \
-  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
-  "$PYDOCDIR/html/" \
-  delta@py.delta.chat:build/${BRANCH}
-
-# C docs to c.delta.chat
+# DISABLED: python docs to py.delta.chat
+#ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
 #rsync -avz \
 #  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
-#  "$DOXYDOCDIR/html/" \
-#  delta@py.delta.chat:build-c/${BRANCH}
+#  "$PYDOCDIR/html/" \
+#  delta@py.delta.chat:build/${BRANCH}
+
+# C docs to c.delta.chat
+ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@c.delta.chat mkdir -p build-c/${BRANCH}
+rsync -avz \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  "$DOXYDOCDIR/html/" \
+  delta@c.delta.chat:build-c/${BRANCH}
+
+exit 0
+
+# OUTDATED -- for re-use from python release-scripts 
 
 echo -----------------------
 echo upload wheels 

ci_scripts/docker-coredeps/Dockerfile

@@ -5,16 +5,6 @@ RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
     echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
 ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
 
-ENV PIP_DISABLE_PIP_VERSION_CHECK 1
-
-# Install python tools (auditwheels,tox, ...)
-ADD deps/build_python.sh /builder/build_python.sh
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
-
-# Install Rust nightly 
-ADD deps/build_rust.sh /builder/build_rust.sh
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1
-
 # Install a recent Perl, needed to install OpenSSL
 ADD deps/build_perl.sh /builder/build_perl.sh
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
@@ -23,3 +13,12 @@ RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
 ADD deps/build_openssl.sh /builder/build_openssl.sh
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_openssl.sh && cd .. && rm -r tmp1
 
+ENV PIP_DISABLE_PIP_VERSION_CHECK 1
+
+# Install python tools (auditwheels,tox, ...)
+ADD deps/build_python.sh /builder/build_python.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
+
+# Install Rust nightly
+ADD deps/build_rust.sh /builder/build_rust.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1

ci_scripts/docker-coredeps/deps/build_perl.sh

@@ -1,11 +1,11 @@
 #!/bin/bash
 
-PERL_VERSION=5.28.0
-PERL_SHA256=7e929f64d4cb0e9d1159d4a59fc89394e27fa1f7004d0836ca0d514685406ea8
+PERL_VERSION=5.30.0
+# PERL_SHA256=7e929f64d4cb0e9d1159d4a59fc89394e27fa1f7004d0836ca0d514685406ea8
 curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz
-echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
-tar xzf perl-${PERL_VERSION}.tar.gz
-cd perl-${PERL_VERSION} 
+# echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
+tar -xzf perl-${PERL_VERSION}.tar.gz
+cd perl-${PERL_VERSION}
 
 ./Configure -de
 make

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -1,11 +1,8 @@
 #!/bin/bash
 
-set -e -x 
+set -e -x
 
-# Install Rust 
-curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-07-10 -y
+# Install Rust
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-09-12 -y
 export PATH=/root/.cargo/bin:$PATH
 rustc --version
-
-# remove some 300-400 MB that we don't need for automated builds
-rm -rf /root/.rustup/toolchains/nightly-2019-07-10-x86_64-unknown-linux-gnu/share/

ci_scripts/manual_remote_tests.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -xe
+export CIRCLE_JOB=remote_tests_${1:?need to specify 'rust' or 'python'} 
+export CIRCLE_BUILD_NUM=$USER
+export CIRCLE_BRANCH=`git branch | grep \* | cut -d ' ' -f2`
+export CIRCLE_PROJECT_REPONAME=$(basename `git rev-parse --show-toplevel`)
+
+time bash ci_scripts/$CIRCLE_JOB.sh 

ci_scripts/old/gh-actions-rust.yml

@@ -0,0 +1,77 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+
+env:
+  RUSTFLAGS: -Dwarnings
+
+jobs:
+  build_and_test:
+    name: Build and test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macOS-latest]
+        rust: [nightly]
+
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Install ${{ matrix.rust }}
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: ${{ matrix.rust }}
+        override: true
+
+    - name: check
+      uses: actions-rs/cargo@v1
+      if: matrix.rust == 'nightly'
+      with:
+        command:  check
+        args: --all --bins --examples --tests
+
+    - name: tests
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --all
+
+    - name: tests ignored
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --all --release -- --ignored
+
+  check_fmt:
+    name: Checking fmt and docs
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@master
+
+    - uses: actions-rs/toolchain@v1
+      with:
+          profile: minimal
+          toolchain: nightly
+          override: true
+          components: rustfmt
+
+    - name: fmt
+      run: cargo fmt --all -- --check
+
+#  clippy_check:
+#    name: Clippy check
+#    runs-on: ubuntu-latest
+#
+#    steps:
+#    - uses: actions/checkout@v1
+#    - uses: actions-rs/toolchain@v1
+#      with:
+#          profile: minimal
+#          toolchain: nightly
+#          override: true
+#          components: clippy
+#
+#    - name: clippy
+#      run: cargo clippy --all

ci_scripts/old/run-python.sh

@@ -43,8 +43,8 @@ if [ -n "$TESTS" ]; then
     # we split out qr-tests run to minimize likelyness of flaky tests
     # (some qr tests are pretty heavy in terms of send/received
     # messages and rust's imap code likely has concurrency problems) 
-    tox --workdir "$TOXWORKDIR" -e py37 -- -k "not qr"
-    tox --workdir "$TOXWORKDIR" -e py37 -- -k "qr"
+    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "not qr"
+    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "qr"
     unset DCC_PY_LIVECONFIG
     tox --workdir "$TOXWORKDIR" -p4 -e lint,py35,py36,doc
     tox --workdir "$TOXWORKDIR" -e auditwheels

ci_scripts/remote_tests_python.sh

@@ -0,0 +1,46 @@
+#!/bin/bash 
+
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export SSHTARGET=ci@b1.delta.chat
+
+# we construct the BUILDDIR such that we can easily share the
+# CARGO_TARGET_DIR between runs ("..")
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+set -xe
+
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist 
+# we seem to need .git for setuptools_scm versioning 
+find .git >>.rsynclist
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+
+set +x
+
+echo "--- Running $CIRCLE_JOB remotely"
+
+ssh $SSHTARGET <<_HERE
+    set +x -e
+    cd $BUILDDIR
+    # let's share the target dir with our last run on this branch/job-type
+    # cargo will make sure to block/unblock us properly 
+    export CARGO_TARGET_DIR=\`pwd\`/../target
+    export TARGET=release
+    export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
+
+    #we rely on tox/virtualenv being available in the host
+    #rm -rf virtualenv venv
+    #virtualenv -q -p python3.7 venv 
+    #source venv/bin/activate
+    #pip install -q tox virtualenv
+
+    set -x
+    which python
+    source \$HOME/venv/bin/activate
+    which python
+
+    bash ci_scripts/run-python-test.sh 
+_HERE

ci_scripts/remote_tests_rust.sh

@@ -0,0 +1,32 @@
+#!/bin/bash 
+
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export SSHTARGET=ci@b1.delta.chat
+
+# we construct the BUILDDIR such that we can easily share the
+# CARGO_TARGET_DIR between runs ("..")
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+
+set -e
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist 
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+
+echo "--- Running $CIRCLE_JOB remotely"
+
+ssh $SSHTARGET <<_HERE
+    set +x -e
+    cd $BUILDDIR
+    # let's share the target dir with our last run on this branch/job-type
+    # cargo will make sure to block/unblock us properly 
+    export CARGO_TARGET_DIR=\`pwd\`/../target
+    export TARGET=x86_64-unknown-linux-gnu
+    export RUSTC_WRAPPER=sccache
+
+    bash ci_scripts/run-rust-test.sh
+_HERE
+

ci_scripts/run-doxygen.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -ex
+
+cd deltachat-ffi 
+doxygen 
+

ci_scripts/run-python-test.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+#
+# Run functional tests for Delta Chat core using the python bindings 
+# and tox/pytest. 
+
+set -e -x
+
+# for core-building and python install step
+export DCC_RS_TARGET=release
+export DCC_RS_DEV=`pwd`
+
+cd python
+
+python install_python_bindings.py onlybuild
+
+# remove and inhibit writing PYC files 
+rm -rf tests/__pycache__
+rm -rf src/deltachat/__pycache__
+export PYTHONDONTWRITEBYTECODE=1
+
+# run python tests (tox invokes pytest to run tests in python/tests)
+#TOX_PARALLEL_NO_SPINNER=1 tox -e lint,doc
+tox -e lint
+tox -e doc,py37

ci_scripts/run-rust-test.sh

@@ -2,7 +2,7 @@
 
 set -ex
 
-export RUST_TEST_THREADS=1
+#export RUST_TEST_THREADS=1
 export RUST_BACKTRACE=1
 export RUSTFLAGS='--deny warnings'
 export OPT="--target=$TARGET"
@@ -37,7 +37,9 @@ else
     export OPT_RELEASE_IGNORED="${OPT_RELEASE} -- --ignored"
 fi
 
-# Run all the test configurations:
+# Run all the test configurations 
+# RUSTC_WRAPPER=SCCACHE seems to destroy parallelism / prolong the test
+unset RUSTC_WRAPPER
 $CARGO_CMD $CARGO_SUBCMD $OPT
 $CARGO_CMD $CARGO_SUBCMD $OPT_RELEASE
 $CARGO_CMD $CARGO_SUBCMD $OPT_RELEASE_IGNORED

deltachat-ffi/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
 name = "deltachat_ffi"
-version = "1.0.0-alpha.5"
+version = "1.0.0-beta.8"
 description = "Deltachat FFI"
-authors = ["dignifiedquire <dignifiedquire@gmail.com>"]
+authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 readme = "README.md"
 license = "MIT OR Apache-2.0"

deltachat-ffi/deltachat.h

@@ -78,8 +78,7 @@ typedef struct _dc_provider dc_provider_t;
  *
  * The example above uses "pthreads",
  * however, you can also use anything else for thread handling.
- * NB: The deltachat-core library itself does not create any threads on its own,
- * however, functions, unless stated otherwise, are thread-safe.
+ * All deltachat-core-functions, unless stated otherwise, are thread-safe.
  *
  * After that you can  **define and open a database.**
  * The database is a normal sqlite-file and is created as needed:
@@ -135,7 +134,7 @@ typedef struct _dc_provider dc_provider_t;
  *
  *     printf("Message %i: %s\n", i+1, text);
  *
- *     free(text);
+ *     dc_str_unref(text);
  *     dc_msg_unref(msg);
  * }
  * dc_array_unref(msglist);
@@ -321,7 +320,7 @@ int             dc_is_open                   (const dc_context_t* context);
  * @memberof dc_context_t
  * @param context The context object as created by dc_context_new().
  * @return Blob directory associated with the context object, empty string if unset or on errors. NULL is never returned.
- *     The returned string must be free()'d.
+ *     The returned string must be released using dc_str_unref().
  */
 char*           dc_get_blobdir               (const dc_context_t* context);
 
@@ -339,6 +338,8 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `send_pw`      = SMTP-password, guessed if left out
  * - `send_port`    = SMTP-port, guessed if left out
  * - `server_flags` = IMAP-/SMTP-flags as a combination of @ref DC_LP flags, guessed if left out
+ * - `imap_certificate_checks` = how to check IMAP certificates, one of the @ref DC_CERTCK flags, defaults to #DC_CERTCK_AUTO (0)
+ * - `smtp_certificate_checks` = how to check SMTP certificates, one of the @ref DC_CERTCK flags, defaults to #DC_CERTCK_AUTO (0)
  * - `displayname`  = Own name to use when sending messages.  MUAs are allowed to spread this way eg. using CC, defaults to empty
  * - `selfstatus`   = Own status to display eg. in email footers, defaults to a standard text
  * - `selfavatar`   = File containing avatar. Will be copied to blob directory.
@@ -397,11 +398,24 @@ int             dc_set_config                (dc_context_t* context, const char*
  * @param context The context object as created by dc_context_new(). For querying system values, this can be NULL.
  * @param key The key to query.
  * @return Returns current value of "key", if "key" is unset, the default
- *     value is returned.  The returned value must be free()'d, NULL is never
+ *     value is returned.  The returned value must be released using dc_str_unref(), NULL is never
  *     returned.  If there is an error an empty string will be returned.
  */
 char*           dc_get_config                (dc_context_t* context, const char* key);
 
+/**
+ * Set stock string translation. 
+ *
+ * The function will emit warnings if it returns an error state. 
+ *
+ * @memberof dc_context_t
+ * @param context The context object
+ * @param stock_id   the integer id of the stock message (DC_STR_*)
+ * @param stock_msg  the message to be used 
+ * @return int (==0 on error, 1 on success)
+ */
+int             dc_set_stock_translation(dc_context_t* context, uint32_t stock_id, const char* stock_msg);
+
 
 /**
  * Get information about the context.
@@ -415,7 +429,7 @@ char*           dc_get_config                (dc_context_t* context, const char*
  *
  * @memberof dc_context_t
  * @param context The context as created by dc_context_new().
- * @return String which must be free()'d after usage.  Never returns NULL.
+ * @return String which must be released using dc_str_unref() after usage.  Never returns NULL.
  */
 char*           dc_get_info                  (dc_context_t* context);
 
@@ -446,6 +460,7 @@ char*           dc_get_info                  (dc_context_t* context);
  *     `https://localhost:PORT/PATH`, `urn:ietf:wg:oauth:2.0:oob`
  *     (the latter just displays the code the user can copy+paste then)
  * @return URL that can be opened in the browser to start OAuth2.
+ *     Returned strings must be released using dc_str_unref().
  *     If OAuth2 is not possible for the given e-mail-address, NULL is returned.
  */
 char*           dc_get_oauth2_url            (dc_context_t* context, const char* addr, const char* redirect_uri);
@@ -486,15 +501,9 @@ char*           dc_get_oauth2_url            (dc_context_t* context, const char*
  * To interrupt a configuration prematurely, use dc_stop_ongoing_process();
  * this is not needed if #DC_EVENT_CONFIGURE_PROGRESS reports success.
  *
- * On a successfull configuration,
- * the core makes a copy of the parameters mentioned above:
- * the original parameters as are never modified by the core.
- *
- * UI-implementors should keep this in mind -
- * eg. if the UI wants to prefill a configure-edit-dialog with these parameters,
- * the UI should reset them if the user cancels the dialog
- * after a configure-attempts has failed.
- * Otherwise the parameters may not reflect the current configuation.
+ * If #DC_EVENT_CONFIGURE_PROGRESS reports failure,
+ * the core continues to use the last working configuration
+ * and parameters as `addr`, `mail_pw` etc. are set to that.
  *
  * @memberof dc_context_t
  * @param context The context object as created by dc_context_new().
@@ -694,6 +703,7 @@ void            dc_perform_mvbox_idle        (dc_context_t* context);
  */
 void            dc_interrupt_mvbox_idle      (dc_context_t* context);
 
+
 /**
  * Execute pending sentbox-jobs.
  * This function and dc_perform_sentbox_fetch() and dc_perform_sentbox_idle()
@@ -1082,6 +1092,66 @@ uint32_t        dc_send_text_msg             (dc_context_t* context, uint32_t ch
 void            dc_set_draft                 (dc_context_t* context, uint32_t chat_id, dc_msg_t* msg);
 
 
+/**
+ * Add a message to the device-chat.
+ * Device-messages usually contain update information
+ * and some hints that are added during the program runs, multi-device etc.
+ * The device-message may be defined by a label;
+ * if a message with the same label was added or skipped before,
+ * the message is not added again, even if the message was deleted in between.
+ * If needed, the device-chat is created before.
+ *
+ * Sends the event #DC_EVENT_MSGS_CHANGED on success.
+ * To check, if a given chat is a device-chat, see dc_chat_is_device_talk()
+ *
+ * @memberof dc_context_t
+ * @param context The context as created by dc_context_new().
+ * @param label A unique name for the message to add.
+ *     The label is typically not displayed to the user and
+ *     must be created from the characters `A-Z`, `a-z`, `0-9`, `_` or `-`.
+ *     If you pass NULL here, the message is added unconditionally.
+ * @param msg Message to be added to the device-chat.
+ *     The message appears to the user as an incoming message.
+ *     If you pass NULL here, only the given label will be added
+ *     and block adding messages with that label in the future.
+ * @return The ID of the just added message,
+ *     if the message was already added or no message to add is given, 0 is returned.
+ *
+ * Example:
+ * ~~~
+ * dc_msg_t* welcome_msg = dc_msg_new(DC_MSG_TEXT);
+ * dc_msg_set_text(welcome_msg, "great that you give this app a try!");
+ *
+ * dc_msg_t* changelog_msg = dc_msg_new(DC_MSG_TEXT);
+ * dc_msg_set_text(changelog_msg, "we have added 3 new emojis :)");
+ *
+ * if (dc_add_device_msg(context, "welcome", welcome_msg)) {
+ *     // do not add the changelog on a new installations -
+ *     // not now and not when this code is executed again
+ *     dc_add_device_msg(context, "update-123", NULL);
+ * } else {
+ *     // welcome message was not added now, this is an oder installation,
+ *     // add a changelog
+ *     dc_add_device_msg(context, "update-123", changelog_msg);
+ * }
+ * ~~~
+ */
+uint32_t        dc_add_device_msg            (dc_context_t* context, const char* label, dc_msg_t* msg);
+
+
+/**
+ * Check if a device-message with a given label was ever added.
+ * Device-messages can be added dc_add_device_msg().
+ *
+ * @memberof dc_context_t
+ * @param context The context as created by dc_context_new().
+ * @param label Label of the message to check.
+ * @return 1=A message with this label was added at some point,
+ *     0=A message with this label was never added.
+ */
+int             dc_was_device_msg_ever_added (dc_context_t* context, const char* label);
+
+
 /**
  * Get draft for a chat, if any.
  * See dc_set_draft() for more details about drafts.
@@ -1197,7 +1267,7 @@ void            dc_marknoticed_all_chats     (dc_context_t* context);
  * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
  * @param chat_id The chat ID to get all messages with media from.
- * @param msg_type Specify a message type to query here, one of the DC_MSG_* constats.
+ * @param msg_type Specify a message type to query here, one of the @ref DC_MSG constants.
  * @param msg_type2 Alternative message type to search for. 0 to skip.
  * @param msg_type3 Alternative message type to search for. 0 to skip.
  * @return An array with messages from the given chat ID that have the wanted message types.
@@ -1465,7 +1535,7 @@ int             dc_set_chat_profile_image    (dc_context_t* context, uint32_t ch
  * @memberof dc_context_t
  * @param context The context object as created by dc_context_new().
  * @param msg_id The message id for which information should be generated
- * @return Text string, must be free()'d after usage
+ * @return Text string, must be released using dc_str_unref() after usage
  */
 char*           dc_get_msg_info              (dc_context_t* context, uint32_t msg_id);
 
@@ -1479,7 +1549,7 @@ char*           dc_get_msg_info              (dc_context_t* context, uint32_t ms
  * @memberof dc_context_t
  * @param context The context object as created by dc_context_new().
  * @param msg_id The message id, must be the id of an incoming message.
- * @return Raw headers as a multi-line string, must be free()'d after usage.
+ * @return Raw headers as a multi-line string, must be released using dc_str_unref() after usage.
  *     Returns NULL if there are no headers saved for the given message,
  *     eg. because of save_mime_headers is not set
  *     or the message is not incoming.
@@ -1499,6 +1569,16 @@ char*           dc_get_mime_headers          (dc_context_t* context, uint32_t ms
  */
 void            dc_delete_msgs               (dc_context_t* context, const uint32_t* msg_ids, int msg_cnt);
 
+/**
+ * Empty IMAP server folder: delete all messages. 
+ *
+ * @memberof dc_context_t
+ * @param context The context object as created by dc_context_new()
+ * @param flags What to delete, a combination of the @ref DC_EMPTY flags
+ * @return None.
+ */
+void            dc_empty_server              (dc_context_t* context, uint32_t flags);
+
 
 /**
  * Forward messages to another chat.
@@ -1720,7 +1800,7 @@ void            dc_block_contact             (dc_context_t* context, uint32_t co
  * @memberof dc_context_t
  * @param context The context object as created by dc_context_new().
  * @param contact_id ID of the contact to get the encryption info for.
- * @return Multi-line text, must be free()'d after usage.
+ * @return Multi-line text, must be released using dc_str_unref() after usage.
  */
 char*           dc_get_contact_encrinfo      (dc_context_t* context, uint32_t contact_id);
 
@@ -1856,7 +1936,8 @@ void            dc_imex                      (dc_context_t* context, int what, c
  * @memberof dc_context_t
  * @param context The context as created by dc_context_new().
  * @param dir Directory to search backups in.
- * @return String with the backup file, typically given to dc_imex(), returned strings must be free()'d.
+ * @return String with the backup file, typically given to dc_imex(),
+ *     returned strings must be released using dc_str_unref().
  *     The function returns NULL if no backup was found.
  */
 char*           dc_imex_has_backup           (dc_context_t* context, const char* dir);
@@ -1904,7 +1985,7 @@ char*           dc_imex_has_backup           (dc_context_t* context, const char*
  *
  * @memberof dc_context_t
  * @param context The context object.
- * @return The setup code. Must be free()'d after usage.
+ * @return The setup code. Must be released using dc_str_unref() after usage.
  *     On errors, eg. if the message could not be sent, NULL is returned.
  */
 char*           dc_initiate_key_transfer     (dc_context_t* context);
@@ -2009,7 +2090,7 @@ dc_lot_t*       dc_check_qr                  (dc_context_t* context, const char*
  *     If set to 0, the setup-Verified-contact-protocol is offered in the QR code.
  * @return Text that should go to the QR code,
  *     On errors, an empty QR code is returned, NULL is never returned.
- *     The returned string must be free()'d after usage.
+ *     The returned string must be released using dc_str_unref() after usage.
  */
 char*           dc_get_securejoin_qr         (dc_context_t* context, uint32_t chat_id);
 
@@ -2179,6 +2260,21 @@ dc_array_t* dc_get_locations                (dc_context_t* context, uint32_t cha
 void        dc_delete_all_locations         (dc_context_t* context);
 
 
+/**
+ * Release a string returned by another deltachat-core function.
+ * - Strings returned by any deltachat-core-function
+ *   MUST NOT be released by the standard free() function;
+ *   always use dc_str_unref() for this purpose.
+ * - dc_str_unref() MUST NOT be called for strings not returned by deltachat-core.
+ * - dc_str_unref() MUST NOT be called for other objectes returned by deltachat-core.
+ *
+ * @memberof dc_context_t
+ * @param str The string to release.
+ * @return None.
+ */
+void dc_str_unref (char* str);
+
+
 /**
  * @class dc_array_t
  *
@@ -2319,7 +2415,7 @@ uint32_t         dc_array_get_msg_id         (const dc_array_t* array, size_t in
  * @param index Index of the item. Must be between 0 and dc_array_get_cnt()-1.
  * @return Marker-character of the item at the given index.
  *     NULL if there is no marker-character bound to the given item.
- *     The returned value must be free()'d after usage.
+ *     The returned value must be released using dc_str_unref() after usage.
  */
 char*            dc_array_get_marker         (const dc_array_t* array, size_t index);
 
@@ -2570,24 +2666,22 @@ int             dc_chat_get_type             (const dc_chat_t* chat);
  *
  * To change the name, use dc_set_chat_name()
  *
- * See also: dc_chat_get_subtitle()
- *
  * @memberof dc_chat_t
  * @param chat The chat object.
- * @return Chat name as a string. Must be free()'d after usage. Never NULL.
+ * @return Chat name as a string. Must be released using dc_str_unref() after usage. Never NULL.
  */
 char*           dc_chat_get_name             (const dc_chat_t* chat);
 
 
-/**
+/*
  * Get a subtitle for a chat.  The subtitle is eg. the email-address or the
  * number of group members.
  *
- * See also: dc_chat_get_name()
+ * Deprecated function. Subtitles should be created in the ui
+ * where plural forms and other specials can be handled more gracefully.
  *
- * @memberof dc_chat_t
  * @param chat The chat object to calulate the subtitle for.
- * @return Subtitle as a string. Must be free()'d after usage. Never NULL.
+ * @return Subtitle as a string. Must be released using dc_str_unref() after usage. Never NULL.
  */
 char*           dc_chat_get_subtitle         (const dc_chat_t* chat);
 
@@ -2603,7 +2697,7 @@ char*           dc_chat_get_subtitle         (const dc_chat_t* chat);
  * @param chat The chat object.
  * @return Path and file if the profile image, if any.
  *     NULL otherwise.
- *     Must be free()'d after usage.
+ *     Must be released using dc_str_unref() after usage.
  */
 char*           dc_chat_get_profile_image    (const dc_chat_t* chat);
 
@@ -2675,6 +2769,37 @@ int             dc_chat_is_unpromoted        (const dc_chat_t* chat);
 int             dc_chat_is_self_talk         (const dc_chat_t* chat);
 
 
+/**
+ * Check if a chat is a device-talk.
+ * Device-talks contain update information
+ * and some hints that are added during the program runs, multi-device etc.
+ *
+ * From the ui view, device-talks are not very special,
+ * the user can delete and forward messages, archive the chat, set notifications etc.
+ *
+ * Messages can be added to the device-talk using dc_add_device_msg()
+ *
+ * @memberof dc_chat_t
+ * @param chat The chat object.
+ * @return 1=chat is device-talk, 0=chat is no device-talk
+ */
+int             dc_chat_is_device_talk       (const dc_chat_t* chat);
+
+
+/**
+ * Check if messages can be sent to a give chat.
+ * This is not true eg. for the deaddrop or for the device-talk, cmp. dc_chat_is_device_talk().
+ *
+ * Calling dc_send_msg() for these chats will fail
+ * and the ui may decide to hide input controls therefore.
+ *
+ * @memberof dc_chat_t
+ * @param chat The chat object.
+ * @return 1=chat is writable, 0=chat is not writable
+ */
+int             dc_chat_can_send              (const dc_chat_t* chat);
+
+
 /**
  * Check if a chat is verified.  Verified chats contain only verified members
  * and encryption is alwasy enabled.  Verified chats are created using
@@ -2907,7 +3032,7 @@ int64_t          dc_msg_get_sort_timestamp     (const dc_msg_t* msg);
  *
  * @memberof dc_msg_t
  * @param msg The message object.
- * @return Message text. The result must be free()'d. Never returns NULL.
+ * @return Message text. The result must be released using dc_str_unref(). Never returns NULL.
  */
 char*           dc_msg_get_text               (const dc_msg_t* msg);
 
@@ -2921,9 +3046,9 @@ char*           dc_msg_get_text               (const dc_msg_t* msg);
  *
  * @memberof dc_msg_t
  * @param msg The message object.
- * @return Full path, file name and extension of the file associated with the
- *     message.  If there is no file associated with the message, an emtpy
- *     string is returned.  NULL is never returned and the returned value must be free()'d.
+ * @return Full path, file name and extension of the file associated with the message.
+ *     If there is no file associated with the message, an emtpy string is returned.
+ *     NULL is never returned and the returned value must be released using dc_str_unref().
  */
 char*           dc_msg_get_file               (const dc_msg_t* msg);
 
@@ -2936,7 +3061,7 @@ char*           dc_msg_get_file               (const dc_msg_t* msg);
  * @param msg The message object.
  * @return Base file name plus extension without part.  If there is no file
  *     associated with the message, an empty string is returned.  The returned
- *     value must be free()'d.
+ *     value must be released using dc_str_unref().
  */
 char*           dc_msg_get_filename           (const dc_msg_t* msg);
 
@@ -2948,7 +3073,8 @@ char*           dc_msg_get_filename           (const dc_msg_t* msg);
  *
  * @memberof dc_msg_t
  * @param msg The message object.
- * @return String containing the mime type. Must be free()'d after usage. NULL is never returned.
+ * @return String containing the mime type.
+ *     Must be released using dc_str_unref() after usage. NULL is never returned.
  */
 char*           dc_msg_get_filemime           (const dc_msg_t* msg);
 
@@ -3056,7 +3182,8 @@ dc_lot_t*       dc_msg_get_summary            (const dc_msg_t* msg, const dc_cha
  * @memberof dc_msg_t
  * @param msg The message object.
  * @param approx_characters Rough length of the expected string.
- * @return A summary for the given messages. The returned string must be free()'d.
+ * @return A summary for the given messages.
+ *     The returned string must be released using dc_str_unref().
  *     Returns an empty string on errors, never returns NULL.
  */
 char*           dc_msg_get_summarytext        (const dc_msg_t* msg, int approx_characters);
@@ -3201,7 +3328,7 @@ int             dc_msg_is_setupmessage        (const dc_msg_t* msg);
  * @memberof dc_msg_t
  * @param msg The message object.
  * @return Typically, the first two digits of the setup code or an empty string if unknown.
- *     NULL is never returned. Must be free()'d when done.
+ *     NULL is never returned. Must be released using dc_str_unref() when done.
  */
 char*           dc_msg_get_setupcodebegin     (const dc_msg_t* msg);
 
@@ -3328,7 +3455,8 @@ void            dc_msg_latefiling_mediasize   (dc_msg_t* msg, int width, int hei
 
 
 #define         DC_CONTACT_ID_SELF           1
-#define         DC_CONTACT_ID_DEVICE         2
+#define         DC_CONTACT_ID_INFO           2 // centered messages as "member added", used in all chats
+#define         DC_CONTACT_ID_DEVICE         5 // messages "update info" in the device-chat
 #define         DC_CONTACT_ID_LAST_SPECIAL   9
 
 
@@ -3358,7 +3486,8 @@ uint32_t        dc_contact_get_id            (const dc_contact_t* contact);
  *
  * @memberof dc_contact_t
  * @param contact The contact object.
- * @return String with the email address, must be free()'d. Never returns NULL.
+ * @return String with the email address,
+ *     must be released using dc_str_unref(). Never returns NULL.
  */
 char*           dc_contact_get_addr          (const dc_contact_t* contact);
 
@@ -3372,7 +3501,8 @@ char*           dc_contact_get_addr          (const dc_contact_t* contact);
  *
  * @memberof dc_contact_t
  * @param contact The contact object.
- * @return String with the name to display, must be free()'d. Empty string if unset, never returns NULL.
+ * @return String with the name to display, must be released using dc_str_unref().
+ *     Empty string if unset, never returns NULL.
  */
 char*           dc_contact_get_name          (const dc_contact_t* contact);
 
@@ -3386,7 +3516,8 @@ char*           dc_contact_get_name          (const dc_contact_t* contact);
  *
  * @memberof dc_contact_t
  * @param contact The contact object.
- * @return String with the name to display, must be free()'d. Never returns NULL.
+ * @return String with the name to display, must be released using dc_str_unref().
+ *     Never returns NULL.
  */
 char*           dc_contact_get_display_name  (const dc_contact_t* contact);
 
@@ -3402,7 +3533,8 @@ char*           dc_contact_get_display_name  (const dc_contact_t* contact);
  *
  * @memberof dc_contact_t
  * @param contact The contact object.
- * @return Summary string, must be free()'d. Never returns NULL.
+ * @return Summary string, must be released using dc_str_unref().
+ *     Never returns NULL.
  */
 char*           dc_contact_get_name_n_addr   (const dc_contact_t* contact);
 
@@ -3414,7 +3546,8 @@ char*           dc_contact_get_name_n_addr   (const dc_contact_t* contact);
  *
  * @memberof dc_contact_t
  * @param contact The contact object.
- * @return String with the name to display, must be free()'d. Never returns NULL.
+ * @return String with the name to display, must be released using dc_str_unref().
+ *     Never returns NULL.
  */
 char*           dc_contact_get_first_name    (const dc_contact_t* contact);
 
@@ -3428,7 +3561,7 @@ char*           dc_contact_get_first_name    (const dc_contact_t* contact);
  * @param contact The contact object.
  * @return Path and file if the profile image, if any.
  *     NULL otherwise.
- *     Must be free()'d after usage.
+ *     Must be released using dc_str_unref() after usage.
  */
 char*           dc_contact_get_profile_image (const dc_contact_t* contact);
 
@@ -3509,7 +3642,7 @@ dc_provider_t*  dc_provider_new_from_email            (const char* email);
  *
  * @memberof dc_provider_t
  * @param provider The dc_provider_t struct.
- * @return A string which must be free()d.
+ * @return A string which must be released using dc_str_unref().
  */
 char*           dc_provider_get_overview_page         (const dc_provider_t* provider);
 
@@ -3521,7 +3654,7 @@ char*           dc_provider_get_overview_page         (const dc_provider_t* prov
  *
  * @memberof dc_provider_t
  * @param provider The dc_provider_t struct.
- * @return A string which must be free()d.
+ * @return A string which must be released using dc_str_unref().
  */
 char*           dc_provider_get_name                  (const dc_provider_t* provider);
 
@@ -3534,7 +3667,7 @@ char*           dc_provider_get_name                  (const dc_provider_t* prov
  *
  * @memberof dc_provider_t
  * @param provider The dc_provider_t struct.
- * @return A string which must be free()d.
+ * @return A string which must be released using dc_str_unref().
  */
 char*           dc_provider_get_markdown              (const dc_provider_t* provider);
 
@@ -3546,7 +3679,7 @@ char*           dc_provider_get_markdown              (const dc_provider_t* prov
  *
  * @memberof dc_provider_t
  * @param provider The dc_provider_t struct.
- * @return A string which must be free()d.
+ * @return A string which must be released using dc_str_unref().
  */
 char*           dc_provider_get_status_date           (const dc_provider_t* provider);
 
@@ -3608,7 +3741,9 @@ void            dc_lot_unref             (dc_lot_t* lot);
  *
  * @memberof dc_lot_t
  * @param lot The lot object.
- * @return A string, the string may be empty and the returned value must be free()'d. NULL if there is no such string.
+ * @return A string, the string may be empty
+ *     and the returned value must be released using dc_str_unref().
+ *     NULL if there is no such string.
  */
 char*           dc_lot_get_text1         (const dc_lot_t* lot);
 
@@ -3617,10 +3752,10 @@ char*           dc_lot_get_text1         (const dc_lot_t* lot);
  * Get second string. The meaning of the string is defined by the creator of the object.
  *
  * @memberof dc_lot_t
- *
  * @param lot The lot object.
- *
- * @return A string, the string may be empty and the returned value must be free()'d	. NULL if there is no such string.
+ * @return A string, the string may be empty
+ *     and the returned value must be released using dc_str_unref().
+ *     NULL if there is no such string.
  */
 char*           dc_lot_get_text2         (const dc_lot_t* lot);
 
@@ -3641,9 +3776,7 @@ int             dc_lot_get_text1_meaning (const dc_lot_t* lot);
  * Get the associated state. The meaning of the state is defined by the creator of the object.
  *
  * @memberof dc_lot_t
- *
  * @param lot The lot object.
- *
  * @return The state as defined by the creator of the object. 0 if there is not state or on errors.
  */
 int             dc_lot_get_state         (const dc_lot_t* lot);
@@ -3665,9 +3798,7 @@ uint32_t        dc_lot_get_id            (const dc_lot_t* lot);
  * The meaning of the timestamp is defined by the creator of the object.
  *
  * @memberof dc_lot_t
- *
  * @param lot The lot object.
- *
  * @return The timestamp as defined by the creator of the object. 0 if there is not timestamp or on errors.
  */
 int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
@@ -3719,6 +3850,14 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_MSG_GIF       21
 
 
+/**
+ * Message containing a sticker, similar to image.
+ * If possible, the ui should display the image without borders in a transparent way.
+ * A click on a sticker will offer to install the sticker set in some future.
+ */
+#define DC_MSG_STICKER     23
+
+
 /**
  * Message containing an Audio file.
  * File and duration are set via dc_msg_set_file(), dc_msg_set_duration()
@@ -3836,6 +3975,67 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_LP_IMAP_SOCKET_FLAGS (DC_LP_IMAP_SOCKET_STARTTLS|DC_LP_IMAP_SOCKET_SSL|DC_LP_IMAP_SOCKET_PLAIN) // if none of these flags are set, the default is chosen
 #define DC_LP_SMTP_SOCKET_FLAGS (DC_LP_SMTP_SOCKET_STARTTLS|DC_LP_SMTP_SOCKET_SSL|DC_LP_SMTP_SOCKET_PLAIN) // if none of these flags are set, the default is chosen
 
+/**
+ * @defgroup DC_CERTCK DC_CERTCK
+ *
+ * These constants configure TLS certificate checks for IMAP and SMTP connections.
+ *
+ * These constants are set via dc_set_config()
+ * using keys "imap_certificate_checks" and "smtp_certificate_checks".
+ *
+ * @addtogroup DC_CERTCK
+ * @{
+ */
+
+/**
+ * Configure certificate checks automatically.
+ */
+#define DC_CERTCK_AUTO 0
+
+/**
+ * Strictly check TLS certificates;
+ * require that both the certificate and hostname are valid.
+ */
+#define DC_CERTCK_STRICT 1
+
+/**
+ * Accept invalid hostnames, but not invalid certificates.
+ */
+#define DC_CERTCK_ACCEPT_INVALID_HOSTNAMES 2
+
+/**
+ * Accept invalid certificates, including self-signed ones
+ * or having incorrect hostname.
+ */
+#define DC_CERTCK_ACCEPT_INVALID_CERTIFICATES 3
+
+/**
+ * @}
+ */
+
+
+/**
+ * @defgroup DC_EMPTY DC_EMPTY
+ *
+ * These constants configure emptying imap folders with dc_empty_server()
+ *
+ * @addtogroup DC_EMPTY
+ * @{
+ */
+
+/**
+ * Clear all mvbox messages.
+ */
+#define DC_EMPTY_MVBOX 0x01
+
+/**
+ * Clear all INBOX messages.
+ */
+#define DC_EMPTY_INBOX 0x02
+
+/**
+ * @}
+ */
 
 
 /**
@@ -3850,7 +4050,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @{
  */
 
-
 /**
  * The library-user may write an informational string to the log.
  * Passed to the callback given to dc_context_new().
@@ -3859,7 +4058,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
- *     Must not be free()'d or modified and is valid only until the callback returns.
+ *     Must not be unref'd or modified and is valid only until the callback returns.
  * @return 0
  */
 #define DC_EVENT_INFO                     100
@@ -3870,7 +4069,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
- *     Must not be free()'d or modified and is valid only until the callback returns.
+ *     Must not be unref'd or modified and is valid only until the callback returns.
  * @return 0
  */
 #define DC_EVENT_SMTP_CONNECTED           101
@@ -3881,7 +4080,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
- *     Must not be free()'d or modified and is valid only until the callback returns.
+ *     Must not be unref'd or modified and is valid only until the callback returns.
  * @return 0
  */
 #define DC_EVENT_IMAP_CONNECTED           102
@@ -3891,11 +4090,60 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
- *     Must not be free()'d or modified and is valid only until the callback returns.
+ *     Must not be unref'd or modified and is valid only until the callback returns.
  * @return 0
  */
 #define DC_EVENT_SMTP_MESSAGE_SENT        103
 
+/**
+ * Emitted when a message was successfully marked as deleted on the IMAP server.
+ *
+ * @param data1 0
+ * @param data2 (const char*) Info string in english language.
+ *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
+ */
+#define DC_EVENT_IMAP_MESSAGE_DELETED   104
+
+/**
+ * Emitted when a message was successfully moved on IMAP.
+ *
+ * @param data1 0
+ * @param data2 (const char*) Info string in english language.
+ *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
+ */
+#define DC_EVENT_IMAP_MESSAGE_MOVED   105
+
+/**
+ * Emitted when an IMAP folder was emptied. 
+ *
+ * @param data1 0
+ * @param data2 (const char*) folder name. 
+ *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
+ */
+#define DC_EVENT_IMAP_FOLDER_EMPTIED  106
+
+/**
+ * Emitted when a new blob file was successfully written 
+ *
+ * @param data1 0
+ * @param data2 (const char*) path name 
+ *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
+ */
+#define DC_EVENT_NEW_BLOB_FILE 150
+
+/**
+ * Emitted when a blob file was successfully deleted 
+ *
+ * @param data1 0
+ * @param data2 (const char*) path name 
+ *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
+ */
+#define DC_EVENT_DELETED_BLOB_FILE 151
 
 /**
  * The library-user should write a warning string to the log.
@@ -3905,7 +4153,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 0
  * @param data2 (const char*) Warning string in english language.
- *     Must not be free()'d or modified and is valid only until the callback returns.
+ *     Must not be unref'd or modified and is valid only until the callback returns.
  * @return 0
  */
 #define DC_EVENT_WARNING                  300
@@ -3925,9 +4173,10 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * in a messasge box then.
  *
  * @param data1 0
- * @param data2 (const char*) Error string, always set, never NULL. Frequent error strings are
- *     localized using #DC_EVENT_GET_STRING, however, most error strings will be in english language.
- *     Must not be free()'d or modified and is valid only until the callback returns.
+ * @param data2 (const char*) Error string, always set, never NULL.
+ *     Some error strings are taken from dc_set_stock_translation(),
+ *     however, most error strings will be in english language.
+ *     Must not be unref'd or modified and is valid only until the callback returns.
  * @return 0
  */
 #define DC_EVENT_ERROR                    400
@@ -3951,7 +4200,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 (int) 1=first/new network error, should be reported the user;
  *     0=subsequent network error, should be logged only
  * @param data2 (const char*) Error string, always set, never NULL.
- *     Must not be free()'d or modified and is valid only until the callback returns.
+ *     Must not be unref'd or modified and is valid only until the callback returns.
  * @return 0
  */
 #define DC_EVENT_ERROR_NETWORK            401
@@ -3966,7 +4215,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
- *     Must not be free()'d or modified
+ *     Must not be unref'd or modified
  *     and is valid only until the callback returns.
  * @return 0
  */
@@ -4097,7 +4346,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * services.
  *
  * @param data1 (const char*) Path and file name.
- *     Must not be free()'d or modified and is valid only until the callback returns.
+ *     Must not be unref'd or modified and is valid only until the callback returns.
  * @param data2 0
  * @return 0
  */
@@ -4138,36 +4387,31 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_EVENT_SECUREJOIN_JOINER_PROGRESS       2061
 
 
-// the following events are functions that should be provided by the frontends
-
-
 /**
- * Requeste a localized string from the frontend.
+ * This event is sent out to the inviter when a joiner successfully joined a group.
  *
- * @param data1 (int) ID of the string to request, one of the DC_STR_* constants.
- * @param data2 (int) The count. If the requested string contains a placeholder for a numeric value,
- *     the ui may use this value to return different strings on different plural forms.
- * @return (const char*) Null-terminated UTF-8 string.
- *     The string will be free()'d by the core,
- *     so it must be allocated using malloc() or a compatible function.
- *     Return 0 if the ui cannot provide the requested string
- *     the core will use a default string in english language then.
+ * @param data1 (int) chat_id
+ * @param data2 (int) contact_id
+ * @return 0
  */
-#define DC_EVENT_GET_STRING               2091
+#define DC_EVENT_SECUREJOIN_MEMBER_ADDED 2062
+
 
 /**
  * @}
  */
 
-#define DC_EVENT_FILE_COPIED         2055 // deprecated
-#define DC_EVENT_IS_OFFLINE          2081 // deprecated
-#define DC_ERROR_SEE_STRING          0    // deprecated
-#define DC_ERROR_SELF_NOT_IN_GROUP   1    // deprecated
-#define DC_STR_SELFNOTINGRP          21   // deprecated
+
+#define DC_EVENT_FILE_COPIED         2055 // not used anymore
+#define DC_EVENT_IS_OFFLINE          2081 // not used anymore
+#define DC_EVENT_GET_STRING          2091 // not used anymore, use dc_set_stock_translation()
+#define DC_ERROR_SEE_STRING          0    // not used anymore
+#define DC_ERROR_SELF_NOT_IN_GROUP   1    // not used anymore
+#define DC_STR_SELFNOTINGRP          21   // not used anymore
 #define DC_EVENT_DATA1_IS_STRING(e)  ((e)==DC_EVENT_IMEX_FILE_WRITTEN || (e)==DC_EVENT_FILE_COPIED)
 #define DC_EVENT_DATA2_IS_STRING(e)  ((e)>=100 && (e)<=499)
-#define DC_EVENT_RETURNS_INT(e)      ((e)==DC_EVENT_IS_OFFLINE)
-#define DC_EVENT_RETURNS_STRING(e)   ((e)==DC_EVENT_GET_STRING)
+#define DC_EVENT_RETURNS_INT(e)      ((e)==DC_EVENT_IS_OFFLINE) // not used anymore
+#define DC_EVENT_RETURNS_STRING(e)   ((e)==DC_EVENT_GET_STRING) // not used anymore
 char*           dc_get_version_str           (void); // deprecated
 void            dc_array_add_id              (dc_array_t*, uint32_t); // deprecated
 
@@ -4269,10 +4513,9 @@ void            dc_array_add_id              (dc_array_t*, uint32_t); // depreca
 #define DC_STR_MSGLOCATIONENABLED         64
 #define DC_STR_MSGLOCATIONDISABLED        65
 #define DC_STR_LOCATION                   66
-#define DC_STR_COUNT                      66
-
-void dc_str_unref (char*);
-
+#define DC_STR_STICKER                    67
+#define DC_STR_DEVICE_MESSAGES            68
+#define DC_STR_COUNT                      68
 
 /*
  * @}

deltachat-ffi/src/lib.rs

@@ -22,11 +22,14 @@ use std::sync::RwLock;
 use libc::uintptr_t;
 use num_traits::{FromPrimitive, ToPrimitive};
 
+use deltachat::constants::DC_MSG_ID_LAST_SPECIAL;
 use deltachat::contact::Contact;
 use deltachat::context::Context;
 use deltachat::dc_tools::{
-    as_path, as_str, dc_strdup, to_string, to_string_lossy, OsStrExt, StrExt,
+    as_path, dc_strdup, to_opt_string_lossy, to_string_lossy, OsStrExt, StrExt,
 };
+use deltachat::message::MsgId;
+use deltachat::stock::StockMessage;
 use deltachat::*;
 
 // as C lacks a good and portable error handling,
@@ -125,20 +128,28 @@ impl ContextWrapper {
                     | Event::SmtpConnected(msg)
                     | Event::ImapConnected(msg)
                     | Event::SmtpMessageSent(msg)
+                    | Event::ImapMessageDeleted(msg)
+                    | Event::ImapMessageMoved(msg)
+                    | Event::ImapFolderEmptied(msg)
+                    | Event::NewBlobFile(msg)
+                    | Event::DeletedBlobFile(msg)
                     | Event::Warning(msg)
                     | Event::Error(msg)
                     | Event::ErrorNetwork(msg)
                     | Event::ErrorSelfNotInGroup(msg) => {
-                        let data2 = CString::new(msg).unwrap();
+                        let data2 = CString::new(msg).unwrap_or_default();
                         ffi_cb(self, event_id, 0, data2.as_ptr() as uintptr_t)
                     }
                     Event::MsgsChanged { chat_id, msg_id }
                     | Event::IncomingMsg { chat_id, msg_id }
                     | Event::MsgDelivered { chat_id, msg_id }
                     | Event::MsgFailed { chat_id, msg_id }
-                    | Event::MsgRead { chat_id, msg_id } => {
-                        ffi_cb(self, event_id, chat_id as uintptr_t, msg_id as uintptr_t)
-                    }
+                    | Event::MsgRead { chat_id, msg_id } => ffi_cb(
+                        self,
+                        event_id,
+                        chat_id as uintptr_t,
+                        msg_id.to_u32() as uintptr_t,
+                    ),
                     Event::ChatModified(chat_id) => ffi_cb(self, event_id, chat_id as uintptr_t, 0),
                     Event::ContactsChanged(id) | Event::LocationChanged(id) => {
                         let id = id.unwrap_or_default();
@@ -148,7 +159,7 @@ impl ContextWrapper {
                         ffi_cb(self, event_id, progress as uintptr_t, 0)
                     }
                     Event::ImexFileWritten(file) => {
-                        let data1 = file.to_c_string().unwrap();
+                        let data1 = file.to_c_string().unwrap_or_default();
                         ffi_cb(self, event_id, data1.as_ptr() as uintptr_t, 0)
                     }
                     Event::SecurejoinInviterProgress {
@@ -164,11 +175,14 @@ impl ContextWrapper {
                         contact_id as uintptr_t,
                         progress as uintptr_t,
                     ),
-                    Event::GetString { id, count } => ffi_cb(
+                    Event::SecurejoinMemberAdded {
+                        chat_id,
+                        contact_id,
+                    } => ffi_cb(
                         self,
                         event_id,
-                        id.to_u32().unwrap_or_default() as uintptr_t,
-                        count as uintptr_t,
+                        chat_id as uintptr_t,
+                        contact_id as uintptr_t,
                     ),
                 }
             }
@@ -306,11 +320,14 @@ pub unsafe extern "C" fn dc_set_config(
         return 0;
     }
     let ffi_context = &*context;
-    match config::Config::from_str(as_str(key)) {
+    match config::Config::from_str(&to_string_lossy(key)) {
         // When ctx.set_config() fails it already logged the error.
         // TODO: Context::set_config() should not log this
         Ok(key) => ffi_context
-            .with_inner(|ctx| ctx.set_config(key, as_opt_str(value)).is_ok() as libc::c_int)
+            .with_inner(|ctx| {
+                ctx.set_config(key, to_opt_string_lossy(value).as_ref().map(|x| x.as_str()))
+                    .is_ok() as libc::c_int
+            })
             .unwrap_or(0),
         Err(_) => {
             ffi_context.error("dc_set_config(): invalid key");
@@ -329,7 +346,7 @@ pub unsafe extern "C" fn dc_get_config(
         return "".strdup();
     }
     let ffi_context = &*context;
-    match config::Config::from_str(as_str(key)) {
+    match config::Config::from_str(&to_string_lossy(key)) {
         Ok(key) => ffi_context
             .with_inner(|ctx| ctx.get_config(key).unwrap_or_default().strdup())
             .unwrap_or_else(|_| "".strdup()),
@@ -340,6 +357,35 @@ pub unsafe extern "C" fn dc_get_config(
     }
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_set_stock_translation(
+    context: *mut dc_context_t,
+    stock_id: u32,
+    stock_msg: *mut libc::c_char,
+) -> libc::c_int {
+    if context.is_null() || stock_msg.is_null() {
+        eprintln!("ignoring careless call to dc_set_stock_string");
+        return 0;
+    }
+    let msg = to_string_lossy(stock_msg);
+    let ffi_context = &*context;
+    ffi_context
+        .with_inner(|ctx| match StockMessage::from_u32(stock_id) {
+            Some(id) => match ctx.set_stock_translation(id, msg) {
+                Ok(()) => 1,
+                Err(err) => {
+                    warn!(ctx, "set_stock_translation failed: {}", err);
+                    0
+                }
+            },
+            None => {
+                warn!(ctx, "invalid stock message id {}", stock_id);
+                0
+            }
+        })
+        .unwrap_or(0)
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_info(context: *mut dc_context_t) -> *mut libc::c_char {
     if context.is_null() {
@@ -377,8 +423,8 @@ pub unsafe extern "C" fn dc_get_oauth2_url(
         return ptr::null_mut(); // NULL explicitly defined as "unknown"
     }
     let ffi_context = &*context;
-    let addr = to_string(addr);
-    let redirect = to_string(redirect);
+    let addr = to_string_lossy(addr);
+    let redirect = to_string_lossy(redirect);
     ffi_context
         .with_inner(|ctx| match oauth2::dc_get_oauth2_url(ctx, addr, redirect) {
             Some(res) => res.strdup(),
@@ -424,7 +470,7 @@ pub unsafe extern "C" fn dc_perform_imap_jobs(context: *mut dc_context_t) {
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| job::perform_imap_jobs(ctx))
+        .with_inner(|ctx| job::perform_inbox_jobs(ctx))
         .unwrap_or(())
 }
 
@@ -436,19 +482,20 @@ pub unsafe extern "C" fn dc_perform_imap_fetch(context: *mut dc_context_t) {
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| job::perform_imap_fetch(ctx))
+        .with_inner(|ctx| job::perform_inbox_fetch(ctx))
         .unwrap_or(())
 }
 
 #[no_mangle]
 pub unsafe extern "C" fn dc_perform_imap_idle(context: *mut dc_context_t) {
+    // TODO rename function in co-ordination with UIs
     if context.is_null() {
         eprintln!("ignoring careless call to dc_perform_imap_idle()");
         return;
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| job::perform_imap_idle(ctx))
+        .with_inner(|ctx| job::perform_inbox_idle(ctx))
         .unwrap_or(())
 }
 
@@ -460,7 +507,7 @@ pub unsafe extern "C" fn dc_interrupt_imap_idle(context: *mut dc_context_t) {
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| job::interrupt_imap_idle(ctx))
+        .with_inner(|ctx| job::interrupt_inbox_idle(ctx, true))
         .unwrap_or(())
 }
 
@@ -620,22 +667,24 @@ pub unsafe extern "C" fn dc_get_chatlist(
         return ptr::null_mut();
     }
     let ffi_context = &*context;
-    let qs = if query_str.is_null() {
-        None
-    } else {
-        Some(as_str(query_str))
-    };
+    let qs = to_opt_string_lossy(query_str);
+
     let qi = if query_id == 0 { None } else { Some(query_id) };
     ffi_context
-        .with_inner(
-            |ctx| match chatlist::Chatlist::try_load(ctx, flags as usize, qs, qi) {
+        .with_inner(|ctx| {
+            match chatlist::Chatlist::try_load(
+                ctx,
+                flags as usize,
+                qs.as_ref().map(|x| x.as_str()),
+                qi,
+            ) {
                 Ok(list) => {
                     let ffi_list = ChatlistWrapper { context, list };
                     Box::into_raw(Box::new(ffi_list))
                 }
                 Err(_) => ptr::null_mut(),
-            },
-        )
+            }
+        })
         .unwrap_or_else(|_| ptr::null_mut())
 }
 
@@ -648,7 +697,8 @@ pub unsafe extern "C" fn dc_create_chat_by_msg_id(context: *mut dc_context_t, ms
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            chat::create_by_msg_id(ctx, msg_id).unwrap_or_log_default(ctx, "Failed to create chat")
+            chat::create_by_msg_id(ctx, MsgId::new(msg_id))
+                .unwrap_or_log_default(ctx, "Failed to create chat")
         })
         .unwrap_or(0)
 }
@@ -682,10 +732,7 @@ pub unsafe extern "C" fn dc_get_chat_id_by_contact_id(
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| {
-            chat::get_by_contact_id(ctx, contact_id)
-                .unwrap_or_log_default(ctx, "Failed to get chat")
-        })
+        .with_inner(|ctx| chat::get_by_contact_id(ctx, contact_id).unwrap_or(0))
         .unwrap_or(0)
 }
 
@@ -706,6 +753,7 @@ pub unsafe extern "C" fn dc_prepare_msg(
             chat::prepare_msg(ctx, chat_id, &mut ffi_msg.message)
                 .unwrap_or_log_default(ctx, "Failed to prepare message")
         })
+        .map(|msg_id| msg_id.to_u32())
         .unwrap_or(0)
 }
 
@@ -726,6 +774,7 @@ pub unsafe extern "C" fn dc_send_msg(
             chat::send_msg(ctx, chat_id, &mut ffi_msg.message)
                 .unwrap_or_log_default(ctx, "Failed to send message")
         })
+        .map(|msg_id| msg_id.to_u32())
         .unwrap_or(0)
 }
 
@@ -744,6 +793,7 @@ pub unsafe extern "C" fn dc_send_text_msg(
     ffi_context
         .with_inner(|ctx| {
             chat::send_text_msg(ctx, chat_id, text_to_send)
+                .map(|msg_id| msg_id.to_u32())
                 .unwrap_or_log_default(ctx, "Failed to send text message")
         })
         .unwrap_or(0)
@@ -771,6 +821,54 @@ pub unsafe extern "C" fn dc_set_draft(
         .unwrap_or(())
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_add_device_msg(
+    context: *mut dc_context_t,
+    label: *const libc::c_char,
+    msg: *mut dc_msg_t,
+) -> u32 {
+    if context.is_null() || (label.is_null() && msg.is_null()) {
+        eprintln!("ignoring careless call to dc_add_device_msg()");
+        return 0;
+    }
+    let ffi_context = &mut *context;
+    let msg = if msg.is_null() {
+        None
+    } else {
+        let ffi_msg: &mut MessageWrapper = &mut *msg;
+        Some(&mut ffi_msg.message)
+    };
+    ffi_context
+        .with_inner(|ctx| {
+            chat::add_device_msg(
+                ctx,
+                to_opt_string_lossy(label).as_ref().map(|x| x.as_str()),
+                msg,
+            )
+            .unwrap_or_log_default(ctx, "Failed to add device message")
+        })
+        .map(|msg_id| msg_id.to_u32())
+        .unwrap_or(0)
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_was_device_msg_ever_added(
+    context: *mut dc_context_t,
+    label: *const libc::c_char,
+) -> libc::c_int {
+    if context.is_null() || label.is_null() {
+        eprintln!("ignoring careless call to dc_was_device_msg_ever_added()");
+        return 0;
+    }
+    let ffi_context = &mut *context;
+    ffi_context
+        .with_inner(|ctx| {
+            chat::was_device_msg_ever_added(ctx, &to_string_lossy(label)).unwrap_or(false)
+                as libc::c_int
+        })
+        .unwrap_or(0)
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_draft(context: *mut dc_context_t, chat_id: u32) -> *mut dc_msg_t {
     if context.is_null() {
@@ -808,9 +906,19 @@ pub unsafe extern "C" fn dc_get_chat_msgs(
         return ptr::null_mut();
     }
     let ffi_context = &*context;
+    let marker_flag = if marker1before <= DC_MSG_ID_LAST_SPECIAL {
+        None
+    } else {
+        Some(MsgId::new(marker1before))
+    };
     ffi_context
         .with_inner(|ctx| {
-            let arr = dc_array_t::from(chat::get_chat_msgs(ctx, chat_id, flags, marker1before));
+            let arr = dc_array_t::from(
+                chat::get_chat_msgs(ctx, chat_id, flags, marker_flag)
+                    .iter()
+                    .map(|msg_id| msg_id.to_u32())
+                    .collect::<Vec<u32>>(),
+            );
             Box::into_raw(Box::new(arr))
         })
         .unwrap_or_else(|_| ptr::null_mut())
@@ -854,7 +962,12 @@ pub unsafe extern "C" fn dc_get_fresh_msgs(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            let arr = dc_array_t::from(ctx.get_fresh_msgs());
+            let arr = dc_array_t::from(
+                ctx.get_fresh_msgs()
+                    .iter()
+                    .map(|msg_id| msg_id.to_u32())
+                    .collect::<Vec<u32>>(),
+            );
             Box::into_raw(Box::new(arr))
         })
         .unwrap_or_else(|_| ptr::null_mut())
@@ -916,13 +1029,12 @@ pub unsafe extern "C" fn dc_get_chat_media(
         from_prim(or_msg_type3).expect(&format!("incorrect or_msg_type3 = {}", or_msg_type3));
     ffi_context
         .with_inner(|ctx| {
-            let arr = dc_array_t::from(chat::get_chat_media(
-                ctx,
-                chat_id,
-                msg_type,
-                or_msg_type2,
-                or_msg_type3,
-            ));
+            let arr = dc_array_t::from(
+                chat::get_chat_media(ctx, chat_id, msg_type, or_msg_type2, or_msg_type3)
+                    .iter()
+                    .map(|msg_id| msg_id.to_u32())
+                    .collect::<Vec<u32>>(),
+            );
             Box::into_raw(Box::new(arr))
         })
         .unwrap_or_else(|_| ptr::null_mut())
@@ -955,7 +1067,16 @@ pub unsafe extern "C" fn dc_get_next_media(
         from_prim(or_msg_type3).expect(&format!("incorrect or_msg_type3 = {}", or_msg_type3));
     ffi_context
         .with_inner(|ctx| {
-            chat::get_next_media(ctx, msg_id, direction, msg_type, or_msg_type2, or_msg_type3)
+            chat::get_next_media(
+                ctx,
+                MsgId::new(msg_id),
+                direction,
+                msg_type,
+                or_msg_type2,
+                or_msg_type3,
+            )
+            .map(|msg_id| msg_id.to_u32())
+            .unwrap_or(0)
         })
         .unwrap_or(0)
 }
@@ -1026,7 +1147,12 @@ pub unsafe extern "C" fn dc_search_msgs(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            let arr = dc_array_t::from(ctx.search_msgs(chat_id, as_str(query)));
+            let arr = dc_array_t::from(
+                ctx.search_msgs(chat_id, to_string_lossy(query))
+                    .iter()
+                    .map(|msg_id| msg_id.to_u32())
+                    .collect::<Vec<u32>>(),
+            );
             Box::into_raw(Box::new(arr))
         })
         .unwrap_or_else(|_| ptr::null_mut())
@@ -1068,7 +1194,7 @@ pub unsafe extern "C" fn dc_create_group_chat(
     };
     ffi_context
         .with_inner(|ctx| {
-            chat::create_group_chat(ctx, verified, as_str(name))
+            chat::create_group_chat(ctx, verified, to_string_lossy(name))
                 .unwrap_or_log_default(ctx, "Failed to create group chat")
         })
         .unwrap_or(0)
@@ -1140,7 +1266,7 @@ pub unsafe extern "C" fn dc_set_chat_name(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            chat::set_chat_name(ctx, chat_id, as_str(name))
+            chat::set_chat_name(ctx, chat_id, to_string_lossy(name))
                 .map(|_| 1)
                 .unwrap_or_log_default(ctx, "Failed to set chat name")
         })
@@ -1160,15 +1286,9 @@ pub unsafe extern "C" fn dc_set_chat_profile_image(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            chat::set_chat_profile_image(ctx, chat_id, {
-                if image.is_null() {
-                    ""
-                } else {
-                    as_str(image)
-                }
-            })
-            .map(|_| 1)
-            .unwrap_or_log_default(ctx, "Failed to set profile image")
+            chat::set_chat_profile_image(ctx, chat_id, to_string_lossy(image))
+                .map(|_| 1)
+                .unwrap_or_log_default(ctx, "Failed to set profile image")
         })
         .unwrap_or(0)
 }
@@ -1184,7 +1304,7 @@ pub unsafe extern "C" fn dc_get_msg_info(
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| message::get_msg_info(ctx, msg_id).strdup())
+        .with_inner(|ctx| message::get_msg_info(ctx, MsgId::new(msg_id)).strdup())
         .unwrap_or_else(|_| ptr::null_mut())
 }
 
@@ -1200,7 +1320,7 @@ pub unsafe extern "C" fn dc_get_mime_headers(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            message::get_mime_headers(ctx, msg_id)
+            message::get_mime_headers(ctx, MsgId::new(msg_id))
                 .map(|s| s.strdup())
                 .unwrap_or_else(|| ptr::null_mut())
         })
@@ -1218,11 +1338,21 @@ pub unsafe extern "C" fn dc_delete_msgs(
         return;
     }
     let ffi_context = &*context;
-
-    let ids = std::slice::from_raw_parts(msg_ids, msg_cnt as usize);
-
+    let msg_ids = convert_and_prune_message_ids(msg_ids, msg_cnt);
     ffi_context
-        .with_inner(|ctx| message::delete_msgs(ctx, ids))
+        .with_inner(|ctx| message::delete_msgs(ctx, &msg_ids[..]))
+        .unwrap_or(())
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_empty_server(context: *mut dc_context_t, flags: u32) {
+    if context.is_null() || flags == 0 {
+        eprintln!("ignoring careless call to dc_empty_server()");
+        return;
+    }
+    let ffi_context = &*context;
+    ffi_context
+        .with_inner(|ctx| message::dc_empty_server(ctx, flags))
         .unwrap_or(())
 }
 
@@ -1241,12 +1371,14 @@ pub unsafe extern "C" fn dc_forward_msgs(
         eprintln!("ignoring careless call to dc_forward_msgs()");
         return;
     }
-    let ids = std::slice::from_raw_parts(msg_ids, msg_cnt as usize);
-
+    let msg_ids = convert_and_prune_message_ids(msg_ids, msg_cnt);
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| chat::forward_msgs(ctx, ids, chat_id))
-        .unwrap_or(())
+        .with_inner(|ctx| {
+            chat::forward_msgs(ctx, &msg_ids[..], chat_id)
+                .unwrap_or_log_default(ctx, "Failed to forward message")
+        })
+        .unwrap_or_default()
 }
 
 #[no_mangle]
@@ -1271,11 +1403,10 @@ pub unsafe extern "C" fn dc_markseen_msgs(
         eprintln!("ignoring careless call to dc_markseen_msgs()");
         return;
     }
-    let ids = std::slice::from_raw_parts(msg_ids, msg_cnt as usize);
-
+    let msg_ids = convert_and_prune_message_ids(msg_ids, msg_cnt);
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| message::markseen_msgs(ctx, ids))
+        .with_inner(|ctx| message::markseen_msgs(ctx, &msg_ids[..]))
         .ok();
 }
 
@@ -1290,12 +1421,10 @@ pub unsafe extern "C" fn dc_star_msgs(
         eprintln!("ignoring careless call to dc_star_msgs()");
         return;
     }
-
-    let ids = std::slice::from_raw_parts(msg_ids, msg_cnt as usize);
-
+    let msg_ids = convert_and_prune_message_ids(msg_ids, msg_cnt);
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| message::star_msgs(ctx, ids, star == 1))
+        .with_inner(|ctx| message::star_msgs(ctx, &msg_ids[..], star == 1))
         .ok();
 }
 
@@ -1308,11 +1437,23 @@ pub unsafe extern "C" fn dc_get_msg(context: *mut dc_context_t, msg_id: u32) ->
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            let message = match message::Message::load_from_db(ctx, msg_id) {
+            let message = match message::Message::load_from_db(ctx, MsgId::new(msg_id)) {
                 Ok(msg) => msg,
                 Err(e) => {
-                    error!(ctx, "Error getting msg #{}: {}", msg_id, e);
-                    return ptr::null_mut();
+                    if msg_id <= constants::DC_MSG_ID_LAST_SPECIAL {
+                        // C-core API returns empty messages, do the same
+                        warn!(
+                            ctx,
+                            "dc_get_msg called with special msg_id={}, returning empty msg", msg_id
+                        );
+                        message::Message::default()
+                    } else {
+                        error!(
+                            ctx,
+                            "dc_get_msg could not retrieve msg_id {}: {}", msg_id, e
+                        );
+                        return ptr::null_mut();
+                    }
                 }
             };
             let ffi_msg = MessageWrapper { context, message };
@@ -1328,7 +1469,7 @@ pub unsafe extern "C" fn dc_may_be_valid_addr(addr: *const libc::c_char) -> libc
         return 0;
     }
 
-    contact::may_be_valid_addr(as_str(addr)) as libc::c_int
+    contact::may_be_valid_addr(&to_string_lossy(addr)) as libc::c_int
 }
 
 #[no_mangle]
@@ -1342,7 +1483,7 @@ pub unsafe extern "C" fn dc_lookup_contact_id_by_addr(
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| Contact::lookup_id_by_addr(ctx, as_str(addr)))
+        .with_inner(|ctx| Contact::lookup_id_by_addr(ctx, to_string_lossy(addr)))
         .unwrap_or(0)
 }
 
@@ -1357,12 +1498,14 @@ pub unsafe extern "C" fn dc_create_contact(
         return 0;
     }
     let ffi_context = &*context;
-    let name = if name.is_null() { "" } else { as_str(name) };
+    let name = to_string_lossy(name);
     ffi_context
-        .with_inner(|ctx| match Contact::create(ctx, name, as_str(addr)) {
-            Ok(id) => id,
-            Err(_) => 0,
-        })
+        .with_inner(
+            |ctx| match Contact::create(ctx, name, to_string_lossy(addr)) {
+                Ok(id) => id,
+                Err(_) => 0,
+            },
+        )
         .unwrap_or(0)
 }
 
@@ -1378,7 +1521,7 @@ pub unsafe extern "C" fn dc_add_address_book(
     let ffi_context = &*context;
     ffi_context
         .with_inner(
-            |ctx| match Contact::add_address_book(ctx, as_str(addr_book)) {
+            |ctx| match Contact::add_address_book(ctx, to_string_lossy(addr_book)) {
                 Ok(cnt) => cnt as libc::c_int,
                 Err(_) => 0,
             },
@@ -1397,11 +1540,7 @@ pub unsafe extern "C" fn dc_get_contacts(
         return ptr::null_mut();
     }
     let ffi_context = &*context;
-    let query = if query.is_null() {
-        None
-    } else {
-        Some(as_str(query))
-    };
+    let query = to_opt_string_lossy(query);
     ffi_context
         .with_inner(|ctx| match Contact::get_all(ctx, flags, query) {
             Ok(contacts) => Box::into_raw(Box::new(dc_array_t::from(contacts))),
@@ -1528,9 +1667,17 @@ pub unsafe extern "C" fn dc_imex(
         eprintln!("ignoring careless call to dc_imex()");
         return;
     }
+    let what = match imex::ImexMode::from_i32(what as i32) {
+        Some(what) => what,
+        None => {
+            eprintln!("ignoring invalid argument {} to dc_imex", what);
+            return;
+        }
+    };
+
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| dc_imex::dc_imex(ctx, what, as_opt_str(param1)))
+        .with_inner(|ctx| imex::imex(ctx, what, to_opt_string_lossy(param1)))
         .ok();
 }
 
@@ -1545,7 +1692,7 @@ pub unsafe extern "C" fn dc_imex_has_backup(
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| match dc_imex::dc_imex_has_backup(ctx, as_str(dir)) {
+        .with_inner(|ctx| match imex::has_backup(ctx, to_string_lossy(dir)) {
             Ok(res) => res.strdup(),
             Err(err) => {
                 error!(ctx, "dc_imex_has_backup: {}", err);
@@ -1563,7 +1710,7 @@ pub unsafe extern "C" fn dc_initiate_key_transfer(context: *mut dc_context_t) ->
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| match dc_imex::dc_initiate_key_transfer(ctx) {
+        .with_inner(|ctx| match imex::initiate_key_transfer(ctx) {
             Ok(res) => res.strdup(),
             Err(err) => {
                 error!(ctx, "dc_initiate_key_transfer(): {}", err);
@@ -1589,7 +1736,8 @@ pub unsafe extern "C" fn dc_continue_key_transfer(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            match dc_imex::dc_continue_key_transfer(ctx, msg_id, as_str(setup_code)) {
+            match imex::continue_key_transfer(ctx, MsgId::new(msg_id), &to_string_lossy(setup_code))
+            {
                 Ok(()) => 1,
                 Err(err) => {
                     error!(ctx, "dc_continue_key_transfer: {}", err);
@@ -1607,9 +1755,7 @@ pub unsafe extern "C" fn dc_stop_ongoing_process(context: *mut dc_context_t) {
         return;
     }
     let ffi_context = &*context;
-    ffi_context
-        .with_inner(|ctx| configure::dc_stop_ongoing_process(ctx))
-        .ok();
+    ffi_context.with_inner(|ctx| ctx.stop_ongoing()).ok();
 }
 
 #[no_mangle]
@@ -1624,7 +1770,7 @@ pub unsafe extern "C" fn dc_check_qr(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| {
-            let lot = qr::check_qr(ctx, as_str(qr));
+            let lot = qr::check_qr(ctx, to_string_lossy(qr));
             Box::into_raw(Box::new(lot))
         })
         .unwrap_or_else(|_| ptr::null_mut())
@@ -1660,7 +1806,7 @@ pub unsafe extern "C" fn dc_join_securejoin(
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| securejoin::dc_join_securejoin(ctx, as_str(qr)))
+        .with_inner(|ctx| securejoin::dc_join_securejoin(ctx, &to_string_lossy(qr)))
         .unwrap_or(0)
 }
 
@@ -1709,7 +1855,7 @@ pub unsafe extern "C" fn dc_set_location(
     let ffi_context = &*context;
     ffi_context
         .with_inner(|ctx| location::set(ctx, latitude, longitude, accuracy))
-        .unwrap_or(0)
+        .unwrap_or(false) as _
 }
 
 #[no_mangle]
@@ -2002,7 +2148,11 @@ pub unsafe extern "C" fn dc_chatlist_get_msg_id(
         return 0;
     }
     let ffi_list = &*chatlist;
-    ffi_list.list.get_msg_id(index as usize)
+    ffi_list
+        .list
+        .get_msg_id(index as usize)
+        .map(|msg_id| msg_id.to_u32())
+        .unwrap_or(0)
 }
 
 #[no_mangle]
@@ -2123,7 +2273,7 @@ pub unsafe extern "C" fn dc_chat_get_profile_image(chat: *mut dc_chat_t) -> *mut
     let ffi_context = &*ffi_chat.context;
     ffi_context
         .with_inner(|ctx| match ffi_chat.chat.get_profile_image(ctx) {
-            Some(p) => p.to_str().unwrap().to_string().strdup(),
+            Some(p) => p.to_string_lossy().strdup(),
             None => ptr::null_mut(),
         })
         .unwrap_or_else(|_| ptr::null_mut())
@@ -2172,6 +2322,26 @@ pub unsafe extern "C" fn dc_chat_is_self_talk(chat: *mut dc_chat_t) -> libc::c_i
     ffi_chat.chat.is_self_talk() as libc::c_int
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_chat_is_device_talk(chat: *mut dc_chat_t) -> libc::c_int {
+    if chat.is_null() {
+        eprintln!("ignoring careless call to dc_chat_is_device_talk()");
+        return 0;
+    }
+    let ffi_chat = &*chat;
+    ffi_chat.chat.is_device_talk() as libc::c_int
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_chat_can_send(chat: *mut dc_chat_t) -> libc::c_int {
+    if chat.is_null() {
+        eprintln!("ignoring careless call to dc_chat_can_send()");
+        return 0;
+    }
+    let ffi_chat = &*chat;
+    ffi_chat.chat.can_send() as libc::c_int
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_chat_is_verified(chat: *mut dc_chat_t) -> libc::c_int {
     if chat.is_null() {
@@ -2244,7 +2414,7 @@ pub unsafe extern "C" fn dc_msg_get_id(msg: *mut dc_msg_t) -> u32 {
         return 0;
     }
     let ffi_msg = &*msg;
-    ffi_msg.message.get_id()
+    ffi_msg.message.get_id().to_u32()
 }
 
 #[no_mangle]
@@ -2468,7 +2638,7 @@ pub unsafe extern "C" fn dc_msg_get_summarytext(
         .with_inner(|ctx| {
             ffi_msg
                 .message
-                .get_summarytext(ctx, approx_characters.try_into().unwrap())
+                .get_summarytext(ctx, approx_characters.try_into().unwrap_or_default())
         })
         .unwrap_or_default()
         .strdup()
@@ -2575,8 +2745,7 @@ pub unsafe extern "C" fn dc_msg_set_text(msg: *mut dc_msg_t, text: *const libc::
         return;
     }
     let ffi_msg = &mut *msg;
-    // TODO: {text} equal to NULL is treated as "", which is strange. Does anyone rely on it?
-    ffi_msg.message.set_text(as_opt_str(text).map(Into::into))
+    ffi_msg.message.set_text(to_opt_string_lossy(text))
 }
 
 #[no_mangle]
@@ -2590,7 +2759,10 @@ pub unsafe extern "C" fn dc_msg_set_file(
         return;
     }
     let ffi_msg = &mut *msg;
-    ffi_msg.message.set_file(as_str(file), as_opt_str(filemime))
+    ffi_msg.message.set_file(
+        to_string_lossy(file),
+        to_opt_string_lossy(filemime).as_ref().map(|x| x.as_str()),
+    )
 }
 
 #[no_mangle]
@@ -2760,7 +2932,7 @@ pub unsafe extern "C" fn dc_contact_get_profile_image(
             ffi_contact
                 .contact
                 .get_profile_image(ctx)
-                .map(|p| p.to_str().unwrap().to_string().strdup())
+                .map(|p| p.to_string_lossy().strdup())
                 .unwrap_or_else(|| std::ptr::null_mut())
         })
         .unwrap_or_else(|_| ptr::null_mut())
@@ -2885,14 +3057,6 @@ pub unsafe extern "C" fn dc_str_unref(s: *mut libc::c_char) {
     libc::free(s as *mut _)
 }
 
-fn as_opt_str<'a>(s: *const libc::c_char) -> Option<&'a str> {
-    if s.is_null() {
-        return None;
-    }
-
-    Some(as_str(s))
-}
-
 pub mod providers;
 
 pub trait ResultExt<T> {
@@ -2937,3 +3101,14 @@ impl<T, E> ResultNullableExt<T> for Result<T, E> {
         }
     }
 }
+
+fn convert_and_prune_message_ids(msg_ids: *const u32, msg_cnt: libc::c_int) -> Vec<MsgId> {
+    let ids = unsafe { std::slice::from_raw_parts(msg_ids, msg_cnt as usize) };
+    let msg_ids: Vec<MsgId> = ids
+        .iter()
+        .filter(|id| **id > DC_MSG_ID_LAST_SPECIAL)
+        .map(|id| MsgId::new(*id))
+        .collect();
+
+    msg_ids
+}

deltachat-ffi/src/providers.rs

@@ -2,7 +2,7 @@ extern crate deltachat_provider_database;
 
 use std::ptr;
 
-use deltachat::dc_tools::{as_str, StrExt};
+use deltachat::dc_tools::{to_string_lossy, StrExt};
 use deltachat_provider_database::StatusState;
 
 #[no_mangle]
@@ -12,7 +12,7 @@ pub type dc_provider_t = deltachat_provider_database::Provider;
 pub unsafe extern "C" fn dc_provider_new_from_domain(
     domain: *const libc::c_char,
 ) -> *const dc_provider_t {
-    match deltachat_provider_database::get_provider_info(as_str(domain)) {
+    match deltachat_provider_database::get_provider_info(&to_string_lossy(domain)) {
         Some(provider) => provider,
         None => ptr::null(),
     }
@@ -22,7 +22,8 @@ pub unsafe extern "C" fn dc_provider_new_from_domain(
 pub unsafe extern "C" fn dc_provider_new_from_email(
     email: *const libc::c_char,
 ) -> *const dc_provider_t {
-    let domain = deltachat_provider_database::get_domain_from_email(as_str(email));
+    let email = to_string_lossy(email);
+    let domain = deltachat_provider_database::get_domain_from_email(&email);
     match deltachat_provider_database::get_provider_info(domain) {
         Some(provider) => provider,
         None => ptr::null(),

examples/repl/cmdline.rs

@@ -4,18 +4,17 @@ use std::str::FromStr;
 use deltachat::chat::{self, Chat};
 use deltachat::chatlist::*;
 use deltachat::config;
-use deltachat::configure::*;
 use deltachat::constants::*;
 use deltachat::contact::*;
 use deltachat::context::*;
-use deltachat::dc_imex::*;
 use deltachat::dc_receive_imf::*;
 use deltachat::dc_tools::*;
 use deltachat::error::Error;
+use deltachat::imex::*;
 use deltachat::job::*;
 use deltachat::location;
 use deltachat::lot::LotState;
-use deltachat::message::{self, Message, MessageState};
+use deltachat::message::{self, Message, MessageState, MsgId};
 use deltachat::peerstate::*;
 use deltachat::qr::*;
 use deltachat::sql;
@@ -87,7 +86,7 @@ pub unsafe fn dc_reset_tables(context: &Context, bits: i32) -> i32 {
 
     context.call_cb(Event::MsgsChanged {
         chat_id: 0,
-        msg_id: 0,
+        msg_id: MsgId::new(0),
     });
 
     1
@@ -102,7 +101,7 @@ fn dc_poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<(),
 
 /// Import a file to the database.
 /// For testing, import a folder with eml-files, a single eml-file, e-mail plus public key and so on.
-/// For normal importing, use dc_imex().
+/// For normal importing, use imex().
 ///
 /// @private @memberof Context
 /// @param context The context as created by dc_context_new().
@@ -119,13 +118,13 @@ fn poke_spec(context: &Context, spec: *const libc::c_char) -> libc::c_int {
 
     /* if `spec` is given, remember it for later usage; if it is not given, try to use the last one */
     if !spec.is_null() {
-        real_spec = to_string(spec);
+        real_spec = to_string_lossy(spec);
         context
             .sql
-            .set_config(context, "import_spec", Some(&real_spec))
+            .set_raw_config(context, "import_spec", Some(&real_spec))
             .unwrap();
     } else {
-        let rs = context.sql.get_config(context, "import_spec");
+        let rs = context.sql.get_raw_config(context, "import_spec");
         if rs.is_none() {
             error!(context, "Import: No file or folder given.");
             return 0;
@@ -133,10 +132,8 @@ fn poke_spec(context: &Context, spec: *const libc::c_char) -> libc::c_int {
         real_spec = rs.unwrap();
     }
     if let Some(suffix) = dc_get_filesuffix_lc(&real_spec) {
-        if suffix == "eml" {
-            if dc_poke_eml_file(context, &real_spec).is_ok() {
-                read_cnt += 1
-            }
+        if suffix == "eml" && dc_poke_eml_file(context, &real_spec).is_ok() {
+            read_cnt += 1
         }
     } else {
         /* import a directory */
@@ -171,7 +168,7 @@ fn poke_spec(context: &Context, spec: *const libc::c_char) -> libc::c_int {
     if read_cnt > 0 {
         context.call_cb(Event::MsgsChanged {
             chat_id: 0,
-            msg_id: 0,
+            msg_id: MsgId::new(0),
         });
     }
     1
@@ -193,9 +190,9 @@ unsafe fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
     let msgtext = msg.get_text();
     info!(
         context,
-        "{}#{}{}{}: {} (Contact#{}): {} {}{}{}{} [{}]",
+        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{} [{}]",
         prefix.as_ref(),
-        msg.get_id() as libc::c_int,
+        msg.get_id(),
         if msg.get_showpadlock() { "🔒" } else { "" },
         if msg.has_location() { "📍" } else { "" },
         &contact_name,
@@ -212,22 +209,27 @@ unsafe fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
             "[FRESH]"
         },
         if msg.is_info() { "[INFO]" } else { "" },
+        if msg.is_forwarded() {
+            "[FORWARDED]"
+        } else {
+            ""
+        },
         statestr,
         &temp2,
     );
 }
 
-unsafe fn log_msglist(context: &Context, msglist: &Vec<u32>) -> Result<(), Error> {
+unsafe fn log_msglist(context: &Context, msglist: &Vec<MsgId>) -> Result<(), Error> {
     let mut lines_out = 0;
     for &msg_id in msglist {
-        if msg_id == 9 as libc::c_uint {
+        if msg_id.is_daymarker() {
             info!(
                 context,
                 "--------------------------------------------------------------------------------"
             );
 
             lines_out += 1
-        } else if msg_id > 0 {
+        } else if !msg_id.is_special() {
             if lines_out == 0 {
                 info!(
                     context,
@@ -236,7 +238,7 @@ unsafe fn log_msglist(context: &Context, msglist: &Vec<u32>) -> Result<(), Error
                 lines_out += 1
             }
             let msg = Message::load_from_db(context, msg_id)?;
-            log_msg(context, "Msg", &msg);
+            log_msg(context, "", &msg);
         }
     }
     if lines_out > 0 {
@@ -349,6 +351,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
                  configure\n\
                  connect\n\
                  disconnect\n\
+                 interrupt\n\
                  maybenetwork\n\
                  housekeeping\n\
                  help imex (Import/Export)\n\
@@ -374,6 +377,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
                  sendimage <file> [<text>]\n\
                  sendfile <file> [<text>]\n\
                  draft [<text>]\n\
+                 devicemsg <text>\n\
                  listmedia\n\
                  archive <chat-id>\n\
                  unarchive <chat-id>\n\
@@ -400,12 +404,13 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
                  checkqr <qr-content>\n\
                  event <event-id to test>\n\
                  fileinfo <file>\n\
+                 emptyserver <flags> (1=MVBOX 2=INBOX)\n\
                  clear -- clear screen\n\
                  exit or quit\n\
                  ============================================="
             ),
         },
-        "initiate-key-transfer" => match dc_initiate_key_transfer(context) {
+        "initiate-key-transfer" => match initiate_key_transfer(context) {
             Ok(setup_code) => println!(
                 "Setup code for the transferred setup message: {}",
                 setup_code,
@@ -414,17 +419,17 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
         },
         "get-setupcodebegin" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let msg_id: u32 = arg1.parse()?;
+            let msg_id: MsgId = MsgId::new(arg1.parse()?);
             let msg = Message::load_from_db(context, msg_id)?;
             if msg.is_setupmessage() {
                 let setupcodebegin = msg.get_setupcodebegin(context);
                 println!(
-                    "The setup code for setup message Msg#{} starts with: {}",
+                    "The setup code for setup message {} starts with: {}",
                     msg_id,
                     setupcodebegin.unwrap_or_default(),
                 );
             } else {
-                bail!("Msg#{} is no setup message.", msg_id,);
+                bail!("{} is no setup message.", msg_id,);
             }
         }
         "continue-key-transfer" => {
@@ -432,28 +437,28 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
                 !arg1.is_empty() && !arg2.is_empty(),
                 "Arguments <msg-id> <setup-code> expected"
             );
-            dc_continue_key_transfer(context, arg1.parse()?, &arg2)?;
+            continue_key_transfer(context, MsgId::new(arg1.parse()?), &arg2)?;
         }
         "has-backup" => {
-            dc_imex_has_backup(context, blobdir)?;
+            has_backup(context, blobdir)?;
         }
         "export-backup" => {
-            dc_imex(context, 11, Some(blobdir));
+            imex(context, ImexMode::ExportBackup, Some(blobdir));
         }
         "import-backup" => {
             ensure!(!arg1.is_empty(), "Argument <backup-file> missing.");
-            dc_imex(context, 12, Some(arg1));
+            imex(context, ImexMode::ImportBackup, Some(arg1));
         }
         "export-keys" => {
-            dc_imex(context, 1, Some(blobdir));
+            imex(context, ImexMode::ExportSelfKeys, Some(blobdir));
         }
         "import-keys" => {
-            dc_imex(context, 2, Some(blobdir));
+            imex(context, ImexMode::ImportSelfKeys, Some(blobdir));
         }
         "export-setup" => {
-            let setup_code = dc_create_setup_code(context);
+            let setup_code = create_setup_code(context);
             let file_name = blobdir.join("autocrypt-setup-message.html");
-            let file_content = dc_render_setup_file(context, &setup_code)?;
+            let file_content = render_setup_file(context, &setup_code)?;
             std::fs::write(&file_name, file_content)?;
             println!(
                 "Setup message written to: {}\nSetup code: {}",
@@ -471,7 +476,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
             ensure!(0 != dc_reset_tables(context, bits), "Reset failed");
         }
         "stop" => {
-            dc_stop_ongoing_process(context);
+            context.stop_ongoing();
         }
         "set" => {
             ensure!(!arg1.is_empty(), "Argument <key> missing.");
@@ -488,6 +493,9 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
         "info" => {
             println!("{:#?}", context.get_info());
         }
+        "interrupt" => {
+            interrupt_inbox_idle(context, true);
+        }
         "maybenetwork" => {
             maybe_network(context);
         }
@@ -512,15 +520,12 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
 
                 for i in (0..cnt).rev() {
                     let chat = Chat::load_from_db(context, chatlist.get_chat_id(i))?;
-                    let temp_subtitle = chat.get_subtitle(context);
-                    let temp_name = chat.get_name();
                     info!(
                         context,
-                        "{}#{}: {} [{}] [{} fresh]",
+                        "{}#{}: {} [{} fresh]",
                         chat_prefix(&chat),
                         chat.get_id(),
-                        temp_name,
-                        temp_subtitle,
+                        chat.get_name(),
                         chat::get_fresh_msg_cnt(context, chat.get_id()),
                     );
                     let lot = chatlist.get_summary(context, i, Some(&chat));
@@ -577,21 +582,35 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
             ensure!(sel_chat.is_some(), "Failed to select chat");
             let sel_chat = sel_chat.as_ref().unwrap();
 
-            let msglist = chat::get_chat_msgs(context, sel_chat.get_id(), 0x1, 0);
-            let temp2 = sel_chat.get_subtitle(context);
-            let temp_name = sel_chat.get_name();
+            let msglist = chat::get_chat_msgs(context, sel_chat.get_id(), 0x1, None);
+            let members = chat::get_chat_contacts(context, sel_chat.id);
+            let subtitle = if sel_chat.is_device_talk() {
+                "device-talk".to_string()
+            } else if sel_chat.get_type() == Chattype::Single && !members.is_empty() {
+                let contact = Contact::get_by_id(context, members[0])?;
+                contact.get_addr().to_string()
+            } else {
+                format!("{} member(s)", members.len())
+            };
             info!(
                 context,
-                "{}#{}: {} [{}]{}",
+                "{}#{}: {} [{}]{}{}",
                 chat_prefix(sel_chat),
                 sel_chat.get_id(),
-                temp_name,
-                temp2,
+                sel_chat.get_name(),
+                subtitle,
                 if sel_chat.is_sending_locations() {
                     "📍"
                 } else {
                     ""
                 },
+                match sel_chat.get_profile_image(context) {
+                    Some(icon) => match icon.to_str() {
+                        Some(icon) => format!(" Icon: {}", icon),
+                        _ => " Icon: Err".to_string(),
+                    },
+                    _ => "".to_string(),
+                },
             );
             log_msglist(context, &msglist)?;
             if let Some(draft) = chat::get_draft(context, sel_chat.get_id())? {
@@ -613,7 +632,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
         }
         "createchatbymsg" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing");
-            let msg_id: u32 = arg1.parse()?;
+            let msg_id = MsgId::new(arg1.parse()?);
             let chat_id = chat::create_by_msg_id(context, msg_id)?;
             let chat = Chat::load_from_db(context, chat_id)?;
 
@@ -705,7 +724,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
                 let marker = location.marker.as_ref().unwrap_or(&default_marker);
                 info!(
                     context,
-                    "Loc#{}: {}: lat={} lng={} acc={} Chat#{} Contact#{} Msg#{} {}",
+                    "Loc#{}: {}: lat={} lng={} acc={} Chat#{} Contact#{} {} {}",
                     location.location_id,
                     dc_timestamp_to_str(location.timestamp),
                     location.latitude,
@@ -742,7 +761,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
             let longitude = arg2.parse()?;
 
             let continue_streaming = location::set(context, latitude, longitude, 0.);
-            if 0 != continue_streaming {
+            if continue_streaming {
                 println!("Success, streaming should be continued.");
             } else {
                 println!("Success, streaming can be stoppped.");
@@ -809,6 +828,15 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
                 println!("Draft deleted.");
             }
         }
+        "devicemsg" => {
+            ensure!(
+                !arg1.is_empty(),
+                "Please specify text to add as device message."
+            );
+            let mut msg = Message::new(Viewtype::Text);
+            msg.set_text(Some(arg1.to_string()));
+            chat::add_device_msg(context, None, Some(&mut msg))?;
+        }
         "listmedia" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
 
@@ -822,9 +850,9 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
             println!("{} images or videos: ", images.len());
             for (i, data) in images.iter().enumerate() {
                 if 0 == i {
-                    print!("Msg#{}", data);
+                    print!("{}", data);
                 } else {
-                    print!(", Msg#{}", data);
+                    print!(", {}", data);
                 }
             }
             print!("\n");
@@ -832,11 +860,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
         "archive" | "unarchive" => {
             ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
             let chat_id = arg1.parse()?;
-            chat::archive(
-                context,
-                chat_id,
-                if arg0 == "archive" { true } else { false },
-            )?;
+            chat::archive(context, chat_id, arg0 == "archive")?;
         }
         "delchat" => {
             ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
@@ -845,7 +869,7 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
         }
         "msginfo" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let id = arg1.parse()?;
+            let id = MsgId::new(arg1.parse()?);
             let res = message::get_msg_info(context, id);
             println!("{}", res);
         }
@@ -861,27 +885,27 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
                 "Arguments <msg-id> <chat-id> expected"
             );
 
-            let mut msg_ids = [0; 1];
+            let mut msg_ids = [MsgId::new(0); 1];
             let chat_id = arg2.parse()?;
-            msg_ids[0] = arg1.parse()?;
-            chat::forward_msgs(context, &msg_ids, chat_id);
+            msg_ids[0] = MsgId::new(arg1.parse()?);
+            chat::forward_msgs(context, &msg_ids, chat_id)?;
         }
         "markseen" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let mut msg_ids = [0; 1];
-            msg_ids[0] = arg1.parse()?;
+            let mut msg_ids = [MsgId::new(0); 1];
+            msg_ids[0] = MsgId::new(arg1.parse()?);
             message::markseen_msgs(context, &msg_ids);
         }
         "star" | "unstar" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let mut msg_ids = [0; 1];
-            msg_ids[0] = arg1.parse()?;
+            let mut msg_ids = [MsgId::new(0); 1];
+            msg_ids[0] = MsgId::new(arg1.parse()?);
             message::star_msgs(context, &msg_ids, arg0 == "star");
         }
         "delmsg" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let mut ids = [0; 1];
-            ids[0] = arg1.parse()?;
+            let mut ids = [MsgId::new(0); 1];
+            ids[0] = MsgId::new(arg1.parse()?);
             message::delete_msgs(context, &ids);
         }
         "listcontacts" | "contacts" | "listverified" => {
@@ -972,6 +996,11 @@ pub unsafe fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::E
                 bail!("Command failed.");
             }
         }
+        "emptyserver" => {
+            ensure!(!arg1.is_empty(), "Argument <flags> missing");
+
+            message::dc_empty_server(context, arg1.parse()?);
+        }
         "" => (),
         _ => bail!("Unknown command: \"{}\" type ? for help.", arg0),
     }

examples/repl/main.rs

@@ -43,7 +43,6 @@ use self::cmdline::*;
 
 fn receive_event(_context: &Context, event: Event) -> libc::uintptr_t {
     match event {
-        Event::GetString { .. } => {}
         Event::Info(msg) => {
             /* do not show the event as this would fill the screen */
             println!("{}", msg);
@@ -151,11 +150,11 @@ fn start_threads(c: Arc<RwLock<Context>>) {
     let ctx = c.clone();
     let handle_imap = std::thread::spawn(move || loop {
         while_running!({
-            perform_imap_jobs(&ctx.read().unwrap());
-            perform_imap_fetch(&ctx.read().unwrap());
+            perform_inbox_jobs(&ctx.read().unwrap());
+            perform_inbox_fetch(&ctx.read().unwrap());
             while_running!({
                 let context = ctx.read().unwrap();
-                perform_imap_idle(&context);
+                perform_inbox_idle(&context);
             });
         });
     });
@@ -203,7 +202,7 @@ fn stop_threads(context: &Context) {
         println!("Stopping threads");
         IS_RUNNING.store(false, Ordering::Relaxed);
 
-        interrupt_imap_idle(context);
+        interrupt_inbox_idle(context, true);
         interrupt_mvbox_idle(context);
         interrupt_sentbox_idle(context);
         interrupt_smtp_idle(context);
@@ -236,7 +235,7 @@ impl Completer for DcHelper {
     }
 }
 
-const IMEX_COMMANDS: [&'static str; 12] = [
+const IMEX_COMMANDS: [&str; 12] = [
     "initiate-key-transfer",
     "get-setupcodebegin",
     "continue-key-transfer",
@@ -251,7 +250,7 @@ const IMEX_COMMANDS: [&'static str; 12] = [
     "stop",
 ];
 
-const DB_COMMANDS: [&'static str; 11] = [
+const DB_COMMANDS: [&str; 11] = [
     "info",
     "open",
     "close",
@@ -265,7 +264,7 @@ const DB_COMMANDS: [&'static str; 11] = [
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&'static str; 24] = [
+const CHAT_COMMANDS: [&str; 24] = [
     "listchats",
     "listarchived",
     "chat",
@@ -291,7 +290,7 @@ const CHAT_COMMANDS: [&'static str; 24] = [
     "unarchive",
     "delchat",
 ];
-const MESSAGE_COMMANDS: [&'static str; 8] = [
+const MESSAGE_COMMANDS: [&str; 8] = [
     "listmsgs",
     "msginfo",
     "listfresh",
@@ -301,7 +300,7 @@ const MESSAGE_COMMANDS: [&'static str; 8] = [
     "unstar",
     "delmsg",
 ];
-const CONTACT_COMMANDS: [&'static str; 6] = [
+const CONTACT_COMMANDS: [&str; 6] = [
     "listcontacts",
     "listverified",
     "addcontact",
@@ -309,7 +308,7 @@ const CONTACT_COMMANDS: [&'static str; 6] = [
     "delcontact",
     "cleanupcontacts",
 ];
-const MISC_COMMANDS: [&'static str; 9] = [
+const MISC_COMMANDS: [&str; 9] = [
     "getqr", "getbadqr", "checkqr", "event", "fileinfo", "clear", "exit", "quit", "help",
 ];
 
@@ -335,8 +334,8 @@ impl Hinter for DcHelper {
     }
 }
 
-static COLORED_PROMPT: &'static str = "\x1b[1;32m> \x1b[0m";
-static PROMPT: &'static str = "> ";
+static COLORED_PROMPT: &str = "\x1b[1;32m> \x1b[0m";
+static PROMPT: &str = "> ";
 
 impl Highlighter for DcHelper {
     fn highlight_prompt<'p>(&self, prompt: &'p str) -> Cow<'p, str> {
@@ -456,9 +455,9 @@ unsafe fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult
         }
         "imap-jobs" => {
             if HANDLE.clone().lock().unwrap().is_some() {
-                println!("imap-jobs are already running in a thread.");
+                println!("inbox-jobs are already running in a thread.");
             } else {
-                perform_imap_jobs(&ctx.read().unwrap());
+                perform_inbox_jobs(&ctx.read().unwrap());
             }
         }
         "configure" => {

examples/simple.rs

@@ -11,7 +11,8 @@ use deltachat::configure::*;
 use deltachat::contact::*;
 use deltachat::context::*;
 use deltachat::job::{
-    perform_imap_fetch, perform_imap_idle, perform_imap_jobs, perform_smtp_idle, perform_smtp_jobs,
+    perform_inbox_fetch, perform_inbox_idle, perform_inbox_jobs, perform_smtp_idle,
+    perform_smtp_jobs,
 };
 use deltachat::Event;
 
@@ -35,93 +36,90 @@ fn cb(_ctx: &Context, event: Event) -> usize {
 }
 
 fn main() {
-    unsafe {
-        let dir = tempdir().unwrap();
-        let dbfile = dir.path().join("db.sqlite");
-        println!("creating database {:?}", dbfile);
-        let ctx =
-            Context::new(Box::new(cb), "FakeOs".into(), dbfile).expect("Failed to create context");
-        let running = Arc::new(RwLock::new(true));
-        let info = ctx.get_info();
-        let duration = time::Duration::from_millis(4000);
-        println!("info: {:#?}", info);
+    let dir = tempdir().unwrap();
+    let dbfile = dir.path().join("db.sqlite");
+    println!("creating database {:?}", dbfile);
+    let ctx =
+        Context::new(Box::new(cb), "FakeOs".into(), dbfile).expect("Failed to create context");
+    let running = Arc::new(RwLock::new(true));
+    let info = ctx.get_info();
+    let duration = time::Duration::from_millis(4000);
+    println!("info: {:#?}", info);
+
+    let ctx = Arc::new(ctx);
+    let ctx1 = ctx.clone();
+    let r1 = running.clone();
+    let t1 = thread::spawn(move || {
+        while *r1.read().unwrap() {
+            perform_inbox_jobs(&ctx1);
+            if *r1.read().unwrap() {
+                perform_inbox_fetch(&ctx1);
 
-        let ctx = Arc::new(ctx);
-        let ctx1 = ctx.clone();
-        let r1 = running.clone();
-        let t1 = thread::spawn(move || {
-            while *r1.read().unwrap() {
-                perform_imap_jobs(&ctx1);
                 if *r1.read().unwrap() {
-                    perform_imap_fetch(&ctx1);
-
-                    if *r1.read().unwrap() {
-                        perform_imap_idle(&ctx1);
-                    }
+                    perform_inbox_idle(&ctx1);
                 }
             }
-        });
-
-        let ctx1 = ctx.clone();
-        let r1 = running.clone();
-        let t2 = thread::spawn(move || {
-            while *r1.read().unwrap() {
-                perform_smtp_jobs(&ctx1);
-                if *r1.read().unwrap() {
-                    perform_smtp_idle(&ctx1);
-                }
-            }
-        });
-
-        println!("configuring");
-        let args = std::env::args().collect::<Vec<String>>();
-        assert_eq!(args.len(), 2, "missing password");
-        let pw = args[1].clone();
-        ctx.set_config(config::Config::Addr, Some("d@testrun.org"))
-            .unwrap();
-        ctx.set_config(config::Config::MailPw, Some(&pw)).unwrap();
-        configure(&ctx);
-
-        thread::sleep(duration);
-
-        println!("sending a message");
-        let contact_id =
-            Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com").unwrap();
-        let chat_id = chat::create_by_contact_id(&ctx, contact_id).unwrap();
-        chat::send_text_msg(&ctx, chat_id, "Hi, here is my first message!".into()).unwrap();
-
-        println!("fetching chats..");
-        let chats = Chatlist::try_load(&ctx, 0, None, None).unwrap();
-
-        for i in 0..chats.len() {
-            let summary = chats.get_summary(&ctx, 0, None);
-            let text1 = summary.get_text1();
-            let text2 = summary.get_text2();
-            println!("chat: {} - {:?} - {:?}", i, text1, text2,);
         }
+    });
 
-        thread::sleep(duration);
+    let ctx1 = ctx.clone();
+    let r1 = running.clone();
+    let t2 = thread::spawn(move || {
+        while *r1.read().unwrap() {
+            perform_smtp_jobs(&ctx1);
+            if *r1.read().unwrap() {
+                perform_smtp_idle(&ctx1);
+            }
+        }
+    });
 
-        // let msglist = dc_get_chat_msgs(&ctx, chat_id, 0, 0);
-        // for i in 0..dc_array_get_cnt(msglist) {
-        //     let msg_id = dc_array_get_id(msglist, i);
-        //     let msg = dc_get_msg(context, msg_id);
-        //     let text = CStr::from_ptr(dc_msg_get_text(msg)).unwrap();
-        //     println!("Message {}: {}\n", i + 1, text.to_str().unwrap());
-        //     dc_msg_unref(msg);
-        // }
-        // dc_array_unref(msglist);
+    println!("configuring");
+    let args = std::env::args().collect::<Vec<String>>();
+    assert_eq!(args.len(), 2, "missing password");
+    let pw = args[1].clone();
+    ctx.set_config(config::Config::Addr, Some("d@testrun.org"))
+        .unwrap();
+    ctx.set_config(config::Config::MailPw, Some(&pw)).unwrap();
+    configure(&ctx);
 
-        println!("stopping threads");
+    thread::sleep(duration);
 
-        *running.clone().write().unwrap() = false;
-        deltachat::job::interrupt_imap_idle(&ctx);
-        deltachat::job::interrupt_smtp_idle(&ctx);
+    println!("sending a message");
+    let contact_id = Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com").unwrap();
+    let chat_id = chat::create_by_contact_id(&ctx, contact_id).unwrap();
+    chat::send_text_msg(&ctx, chat_id, "Hi, here is my first message!".into()).unwrap();
 
-        println!("joining");
-        t1.join().unwrap();
-        t2.join().unwrap();
+    println!("fetching chats..");
+    let chats = Chatlist::try_load(&ctx, 0, None, None).unwrap();
 
-        println!("closing");
+    for i in 0..chats.len() {
+        let summary = chats.get_summary(&ctx, 0, None);
+        let text1 = summary.get_text1();
+        let text2 = summary.get_text2();
+        println!("chat: {} - {:?} - {:?}", i, text1, text2,);
     }
+
+    thread::sleep(duration);
+
+    // let msglist = dc_get_chat_msgs(&ctx, chat_id, 0, 0);
+    // for i in 0..dc_array_get_cnt(msglist) {
+    //     let msg_id = dc_array_get_id(msglist, i);
+    //     let msg = dc_get_msg(context, msg_id);
+    //     let text = CStr::from_ptr(dc_msg_get_text(msg)).unwrap();
+    //     println!("Message {}: {}\n", i + 1, text.to_str().unwrap());
+    //     dc_msg_unref(msg);
+    // }
+    // dc_array_unref(msglist);
+
+    println!("stopping threads");
+
+    *running.write().unwrap() = false;
+    deltachat::job::interrupt_inbox_idle(&ctx, true);
+    deltachat::job::interrupt_smtp_idle(&ctx);
+
+    println!("joining");
+    t1.join().unwrap();
+    t2.join().unwrap();
+
+    println!("closing");
 }

mmime/Cargo.toml

@@ -20,4 +20,4 @@ memmap = "0.7.0"
 lazy_static = "1.3.0"
 rand = "0.6.5"
 chrono = "0.4.6"
-hex = "0.3.2"
+hex = "0.4.0"

mmime/src/charconv.rs

@@ -17,12 +17,12 @@ pub unsafe fn charconv(
     assert!(!fromcode.is_null(), "invalid fromcode");
     assert!(!s.is_null(), "invalid input string");
     if let Some(encoding) =
-        charset::Charset::for_label(CStr::from_ptr(fromcode).to_str().unwrap().as_bytes())
+        charset::Charset::for_label(CStr::from_ptr(fromcode).to_string_lossy().as_bytes())
     {
         let data = std::slice::from_raw_parts(s as *const u8, strlen(s));
 
         let (res, _, _) = encoding.decode(data);
-        let res_c = CString::new(res.as_bytes()).unwrap();
+        let res_c = CString::new(res.as_bytes()).unwrap_or_default();
         *result = strdup(res_c.as_ptr()) as *mut _;
 
         MAIL_CHARCONV_NO_ERROR as libc::c_int

mmime/src/display.rs

@@ -123,7 +123,7 @@ unsafe fn display_mime_discrete_type(mut discrete_type: *mut mailmime_discrete_t
         _ => {}
     };
 }
-unsafe fn display_mime_data(mut data: *mut mailmime_data) {
+pub unsafe fn display_mime_data(mut data: *mut mailmime_data) {
     match (*data).dt_type {
         0 => {
             println!(

mmime/src/mailimf/types.rs

@@ -888,6 +888,7 @@ pub unsafe fn mailimf_fields_new(mut fld_list: *mut clist) -> *mut mailimf_field
     (*fields).fld_list = fld_list;
     return fields;
 }
+
 #[no_mangle]
 pub unsafe fn mailimf_field_new(
     mut fld_type: libc::c_int,
@@ -947,6 +948,20 @@ pub unsafe fn mailimf_field_new(
     }
     return field;
 }
+
+#[no_mangle]
+pub unsafe fn mailimf_field_new_subject(fld_subject: *mut mailimf_subject) -> *mut mailimf_field {
+    let mut field: *mut mailimf_field = 0 as *mut mailimf_field;
+    field = malloc(::std::mem::size_of::<mailimf_field>() as libc::size_t) as *mut mailimf_field;
+    if field.is_null() {
+        return 0 as *mut mailimf_field;
+    }
+    (*field).fld_type = MAILIMF_FIELD_SUBJECT as libc::c_int;
+    (*field).fld_data.fld_subject = fld_subject;
+
+    field
+}
+
 #[no_mangle]
 pub unsafe fn mailimf_orig_date_new(
     mut dt_date_time: *mut mailimf_date_time,

mmime/src/mailimf/write_generic.rs

@@ -1598,7 +1598,7 @@ unsafe fn mailimf_date_time_write_driver(
         (*date_time).dt_sec,
         (*date_time).dt_zone,
     );
-    let date_str_c = std::ffi::CString::new(date_str).unwrap();
+    let date_str_c = std::ffi::CString::new(date_str).unwrap_or_default();
     let r = mailimf_string_write_driver(
         do_write,
         data,

mmime/src/mailmime/types.rs

@@ -708,6 +708,28 @@ pub unsafe fn mailmime_new(
     return mime;
 }
 
+pub unsafe fn mailmime_new_simple(
+    mut mm_type: libc::c_int,
+    mut mm_mime_fields: *mut mailmime_fields,
+    mut mm_content_type: *mut mailmime_content,
+    mut mm_fields: *mut mailimf_fields,
+    mut mm_msg_mime: *mut Mailmime,
+) -> *mut Mailmime {
+    mailmime_new(
+        mm_type,
+        std::ptr::null(),
+        0,
+        mm_mime_fields,
+        mm_content_type,
+        std::ptr::null_mut(),
+        std::ptr::null_mut(),
+        std::ptr::null_mut(),
+        std::ptr::null_mut(),
+        mm_fields,
+        mm_msg_mime,
+    )
+}
+
 pub unsafe fn mailmime_free(mut mime: *mut Mailmime) {
     match (*mime).mm_type {
         1 => {

mmime/src/mailmime/types_helper.rs

@@ -848,7 +848,7 @@ pub unsafe fn mailmime_generate_boundary() -> *mut libc::c_char {
         hex::encode(&std::process::id().to_le_bytes()[..2])
     );
 
-    let c = std::ffi::CString::new(raw).unwrap();
+    let c = std::ffi::CString::new(raw).unwrap_or_default();
     strdup(c.as_ptr())
 }
 

mmime/src/mailmime/write_generic.rs

@@ -338,7 +338,7 @@ unsafe fn mailmime_disposition_param_write_driver(
         4 => {
             let value = (*param).pa_data.pa_size as u32;
             let raw = format!("{}", value);
-            let raw_c = std::ffi::CString::new(raw).unwrap();
+            let raw_c = std::ffi::CString::new(raw).unwrap_or_default();
             sizestr = strdup(raw_c.as_ptr());
             len = strlen(b"size=\x00" as *const u8 as *const libc::c_char)
                 .wrapping_add(strlen(sizestr))
@@ -542,7 +542,7 @@ unsafe fn mailmime_version_write_driver(
     }
 
     let raw = format!("{}.{}", (version >> 16) as i32, (version & 0xffff) as i32);
-    let raw_c = std::ffi::CString::new(raw).unwrap();
+    let raw_c = std::ffi::CString::new(raw).unwrap_or_default();
     let mut versionstr = strdup(raw_c.as_ptr());
     r = mailimf_string_write_driver(do_write, data, col, versionstr, strlen(versionstr));
     if r != MAILIMF_NO_ERROR as libc::c_int {
@@ -1515,8 +1515,8 @@ pub unsafe fn mailmime_data_write_driver(
         }
         1 => {
             let filename = CStr::from_ptr((*mime_data).dt_data.dt_filename)
-                .to_str()
-                .unwrap();
+                .to_string_lossy()
+                .into_owned();
             if let Ok(file) = std::fs::File::open(filename) {
                 if let Ok(mut text) = memmap::MmapOptions::new().map_copy(&file) {
                     if 0 != (*mime_data).dt_encoded {
@@ -1797,7 +1797,7 @@ pub unsafe fn mailmime_quoted_printable_write_driver(
                             start = text.offset(i as isize).offset(1isize);
 
                             let raw = format!("={:02X}", (ch as libc::c_int));
-                            let raw_c = std::ffi::CString::new(raw).unwrap();
+                            let raw_c = std::ffi::CString::new(raw).unwrap_or_default();
                             let mut hexstr = strdup(raw_c.as_ptr());
                             r = mailimf_string_write_driver(
                                 do_write,
@@ -1822,7 +1822,7 @@ pub unsafe fn mailmime_quoted_printable_write_driver(
                         }
                         start = text.offset(i as isize).offset(1isize);
                         let raw = format!("={:02X}", ch as libc::c_int);
-                        let raw_c = std::ffi::CString::new(raw).unwrap();
+                        let raw_c = std::ffi::CString::new(raw).unwrap_or_default();
                         let mut hexstr = strdup(raw_c.as_ptr());
                         r = mailimf_string_write_driver(
                             do_write,
@@ -1866,7 +1866,7 @@ pub unsafe fn mailmime_quoted_printable_write_driver(
                     }
                     start = text.offset(i as isize);
                     let raw = format!("={:02X}", b'\r' as i32);
-                    let raw_c = std::ffi::CString::new(raw).unwrap();
+                    let raw_c = std::ffi::CString::new(raw).unwrap_or_default();
                     let mut hexstr = strdup(raw_c.as_ptr());
                     r = mailimf_string_write_driver(do_write, data, col, hexstr, 3i32 as size_t);
                     if r != MAILIMF_NO_ERROR as libc::c_int {
@@ -1890,7 +1890,7 @@ pub unsafe fn mailmime_quoted_printable_write_driver(
                         "={:02X}\r\n",
                         *text.offset(i.wrapping_sub(1i32 as libc::size_t) as isize) as libc::c_int
                     );
-                    let raw_c = std::ffi::CString::new(raw).unwrap();
+                    let raw_c = std::ffi::CString::new(raw).unwrap_or_default();
                     let mut hexstr = strdup(raw_c.as_ptr());
 
                     r = mailimf_string_write_driver(do_write, data, col, hexstr, strlen(hexstr));
@@ -1917,7 +1917,7 @@ pub unsafe fn mailmime_quoted_printable_write_driver(
                         "={:02X}\r\n",
                         *text.offset(i.wrapping_sub(2i32 as libc::size_t) as isize) as libc::c_int
                     );
-                    let raw_c = std::ffi::CString::new(raw).unwrap();
+                    let raw_c = std::ffi::CString::new(raw).unwrap_or_default();
                     let mut hexstr = strdup(raw_c.as_ptr());
 
                     r = mailimf_string_write_driver(do_write, data, col, hexstr, strlen(hexstr));
@@ -1938,7 +1938,7 @@ pub unsafe fn mailmime_quoted_printable_write_driver(
                         (*text.offset(i.wrapping_sub(2i32 as libc::size_t) as isize) as u8 as char),
                         b'\r' as i32
                     );
-                    let raw_c = std::ffi::CString::new(raw).unwrap();
+                    let raw_c = std::ffi::CString::new(raw).unwrap_or_default();
                     let mut hexstr = strdup(raw_c.as_ptr());
 
                     r = mailimf_string_write_driver(do_write, data, col, hexstr, strlen(hexstr));

mmime/src/other.rs

@@ -8,10 +8,14 @@ use crate::mailmime::types_helper::*;
 
 pub(crate) use libc::{
     calloc, close, free, isalpha, isdigit, malloc, memcmp, memcpy, memmove, memset, realloc,
-    strcpy, strlen, strncmp, strncpy,
+    strcpy, strlen, strncmp, strncpy, strnlen,
 };
 
 pub(crate) unsafe fn strcasecmp(s1: *const libc::c_char, s2: *const libc::c_char) -> libc::c_int {
+    if s1.is_null() || s2.is_null() {
+        return 1;
+    }
+
     let s1 = std::ffi::CStr::from_ptr(s1)
         .to_string_lossy()
         .to_lowercase();
@@ -30,16 +34,25 @@ pub(crate) unsafe fn strncasecmp(
     s2: *const libc::c_char,
     n: libc::size_t,
 ) -> libc::c_int {
-    let s1 = std::ffi::CStr::from_ptr(s1)
-        .to_string_lossy()
-        .to_lowercase();
-    let s2 = std::ffi::CStr::from_ptr(s2)
-        .to_string_lossy()
-        .to_lowercase();
-    let m1 = std::cmp::min(n, s1.len());
-    let m2 = std::cmp::min(n, s2.len());
+    if s1.is_null() || s2.is_null() {
+        return 1;
+    }
 
-    if s1[..m1] == s2[..m2] {
+    // s1 and s2 might not be null terminated.
+
+    let s1_slice =
+        std::slice::from_raw_parts(s1 as *const u8, strnlen(s1 as *const libc::c_char, n));
+    let s2_slice =
+        std::slice::from_raw_parts(s2 as *const u8, strnlen(s2 as *const libc::c_char, n));
+
+    let s1 = std::ffi::CStr::from_bytes_with_nul_unchecked(s1_slice)
+        .to_string_lossy()
+        .to_lowercase();
+    let s2 = std::ffi::CStr::from_bytes_with_nul_unchecked(s2_slice)
+        .to_string_lossy()
+        .to_lowercase();
+
+    if s1 == s2 {
         0
     } else {
         1
@@ -1682,6 +1695,14 @@ mod tests {
                 4,
             )
         });
+
+        assert_eq!(0, unsafe {
+            strncasecmp(
+                CString::new("hell").unwrap().as_ptr(),
+                CString::new("Hell").unwrap().as_ptr(),
+                100_000_000,
+            )
+        });
     }
 
     #[test]

proptest-regressions/dc_mimeparser.txt

@@ -0,0 +1,7 @@
+# Seeds for failure cases proptest has generated in the past. It is
+# automatically read and these particular cases re-run before any
+# novel cases are generated.
+#
+# It is recommended to check this file in to source control so that
+# everyone who runs the test benefits from these saved cases.
+cc 03cab93c6d1f3a8245f63cf84dacb307944294fe6333c1e38f078a6600659c7a # shrinks to data = "a\t0aA\ta\t0 \ta\t0 \ta a\t\ta  A\tAA0a0a 0\t a\t aA   \t a\t  A0\t  AAa\taA0\taAAaA\t0\taa0a\ta Aa Aaaa A0A\t  a aA 0\t  A\t0\t0\t\t\t\t\t\tA \t\t a\tA Aa aAA0A0AA0aaA A\t\t  aa0\ta\t  \tAa\taA\t00  AA  A a\tA a aAAa \t 00\t0 \t\t  a A  0\t\t\t aAA Aa \taAAAA0a A0A\t\t1\\E$\t$R\tc\t^=\t\"\tQ<Uk\t\t>A\t\t&\t}v&\tM^\'|\tW5?dn\t\t+\t\tP\te`\t\t>:Brlq--?\t$#Q\tK=zJ\tb\"9*.\"\t`\tF&T*\tBs,\tg\'*\\\t:\t?l$\t\t|A\"HR:Hk\t\\KkV\t\t{=&!^e%:|_*0wV\t[$`\t:\t$%f\t\t[!\"Y. \tP\t\th\'?\'/?%:++NfQo#@\"+?\t(\\??{\t\'\'$Dzj\t0.?{s4?&?Y=/yj]Z=\t4n\t?Ja\"\t{I\t$\t;I}_8V\t&\t?N\'\tI2/\t9.\tFT%9%`\'\tz\to7Y\t|AXP&@G12g\t\'w\t\t%??\t\"h$?F\"\"6%q\\\\{\tT\t\"]$87$}h\'\t<\t$\tc%U:mT2:<v\t#Rl!;U\t\t\"^D\tRZ(BZ{n\t%[$fgH`\t{B}:*\t\t%%*F`%W\t//B}PQ\t\tsu2\tLz<1*!p-X\tnKv&&0\thm4<n\\.\\/.w\'\t<)E1g3#ood\t`?\t\\({N?,8_?h\ty\t0%\t*$A\t\t*w-ViQUj\tTiH\t%\t%&0p\'\'\tA%r**Fo\'Z\\\tNI*ely4=I?}$.o`\t$\ts\'&lt\t\",:~=Nv/0O%=+<LS\t%P\'\t$r%d.\t{G{/L:\t&I&8-`cy*\"{\t/%fP9.P<\t\t\'/`\t\t`\t\t`!t:\t::\t\tW\'X^\t@.uL&a\tN\t\t\t.\t?0*\tvUK>UJ\\\tQbj%w]P=Js\t\"R\t&s^?</=%\t\'VI:\" kT`{b*<\t\tF&?Y\t\t:/\t!C\'e0`\t\t\tx-\t*\\N\\wwu.I4\tb/y\t\"P:P%\"\t\tQ][\\\t^x\t\t):\t\t&s\t$1-\t\t\tXt`0\t;\t/UeCP*\"G\t\t\':\tk@91Hf&#\t(Uzz&`p2;\t{]\t\"I_%=\\%,XD\"\'06QfCd<;$:q^\t8:\"*\"H\t\to\t&xK/\t\ty0/<}<j<|*`~::b\t=S.=}=Ki\t<Y.\'{\tf\t{Ub*H%(_4*?*\tn2\t/\'\t\t\t/,4]\tt\t<y\t\t\tWi\t\tT&\"\t\t\t\t\t=/1Wu?\t\'A\"W-P\t$?e\\\t`\t6`vD\t8`\t\tccD J\tY&jI//?_\t\\j\t_\tsiq$\t?9\tQ\t.^%&..?%Jm??A%+<\tN&*\t.g\tS$W\"\"\tMpq\t\t:&\\\thTz&<iz%/%T&\'F\t\\]&\t\t}\t\t\tXr=dtp`:+<,\t%60Y*<(&K*kJ\todO\t=<V?&\tMU/\"\t= Y!)<\tV\t9\t)\t&v8q{\t\t&pT\t3\ttB,qcq\'i$;gv%j_%M_{[\"&;\t\t\t.B;6Y\\%\t\"\tY/a\t\\`:?\t<\t?]\taNwD;\\\t%l*74%5T?QS :~yR/E*R\t\t=u\t\\\t\t.Q<;\\\t_S/{&F$^\tw_>\'h=|%\t\t:W!\\<U2\'$\tb&`\t=|=L9\t\t\t\\WZ:! }{\t ;\t;\t\t 0.*\t.%\"\'r%{<Mh_t[[}\t-\tJo\"b/AC*-=`=T\tz$2\tC\t\t/k{47\"\t\t,q%\tZ\tT3\t\tf>%\t\'?%@^tx\t7\"1Bk{b{n\t\"Pj3\tHc\t\tt\tY<\t#?\tSh\\yk/N\\\t8 7=R4*9Cw&m\t\\-\'f\t|\'#t(Etu.Hdu(E\t%&v:\'aqW~A5\t\t w.s{J%lX<\"\t\'*%m<&:/B<&\':U}$&`.{)\t\t6S\t:/$*kQ-Z\t^\'t${/tH*\'v\t3\t=\t\tDyp:B\t`I_R&4SO\t\t&-j=*.\t87&\'e*( \t\t\t\'<$\\DJ<$p?{}\'&\tv\t\\Xk<Y:Y`!$K{\tF&\tzd\t\t*i$\tj\'\t<)R*\t%?\t!.\t=\"@#~:=*\t\tXO=_T,1\"\'.%%\"`{\\:\t\"\tfkeOb/\'$I~\ta\t|&\t[\\KK\"1&Z\t<k\t\t)%\'-~\"2n\tj\tW?*<@w{g%d\ta\\\'\'I\t;:ySR%ke:4\tc\t$=\t&9P]x4\tJ=\t6C6%a\t`0\tF\tm-\tTr\t}\t\tQum\t&@\typ|w2&\t\t3`i&t\t\tT5\"\t.&b&e*/==1.\'*\\[U*\tqPL%?$-0/}~|q`\t\t}\t$\tq==o+T$\'!H\t\ti&um\"?\"%%\t/\'p\tg>?{0{J{\t\t/\t\t{zKZ&>=\t[\"1h<H%z/8,/]s\tv{7\t\t:j*H,M//\t\t\td\'.)\t"

python/README.rst

@@ -97,7 +97,7 @@ If you want to run "liveconfig" functional tests you can set
   chat devs.
 
 - or the path of a file that contains two lines, each describing
-  via "addr=... mail_pwd=..." a test account login that will
+  via "addr=... mail_pw=..." a test account login that will
   be used for the live tests.
 
 With ``DCC_PY_LIVECONFIG`` set pytest invocations will use real

python/doc/_templates/globaltoc.html

@@ -6,7 +6,6 @@
   <li><a href="{{ pathto('install') }}">install</a></li>
   <li><a href="{{ pathto('api') }}">high level API</a></li>
   <li><a href="{{ pathto('lapi') }}">low level API</a></li>
-  <li><a href="{{ pathto('capi') }}">C deltachat.h</a></li>
 </ul>
 <b>external links:</b>
 <ul>

python/doc/api.rst

@@ -8,8 +8,8 @@ high level API reference
 
 - :class:`deltachat.account.Account` (your main entry point, creates the
   other classes)
-- :class:`deltachat.chatting.Contact`
-- :class:`deltachat.chatting.Chat`
+- :class:`deltachat.contact.Contact`
+- :class:`deltachat.chat.Chat`
 - :class:`deltachat.message.Message`
 
 Account
@@ -22,13 +22,13 @@ Account
 Contact
 -------
 
-.. autoclass:: deltachat.chatting.Contact
+.. autoclass:: deltachat.contact.Contact
     :members:
 
 Chat
 ----
 
-.. autoclass:: deltachat.chatting.Chat
+.. autoclass:: deltachat.chat.Chat
     :members:
 
 Message

python/doc/lapi.rst

@@ -2,7 +2,13 @@
 low level API reference
 ===================================
 
-for full C-docs, defines and function checkout :doc:`capi`
+for full doxygen-generated C-docs, defines and functions please checkout
+
+https://c.delta.chat
+
+
+Python low-level capi calls
+---------------------------
 
 
 .. automodule:: deltachat.capi.lib

python/install_python_bindings.py

@@ -9,17 +9,23 @@ import subprocess
 import sys
 
 if __name__ == "__main__":
-    os.environ["DCC_RS_TARGET"] = target = "release"
+    target = os.environ.get("DCC_RS_TARGET")
+    if target is None:
+        os.environ["DCC_RS_TARGET"] = target = "release"
     if "DCC_RS_DEV" not in os.environ:
         dn = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
         os.environ["DCC_RS_DEV"] = dn
 
+    # build the core library  in release + debug mode because
+    # as of Nov 2019 rPGP generates RSA keys which take
+    # prohibitively long for non-release installs
     os.environ["RUSTFLAGS"] = "-g"
     subprocess.check_call([
         "cargo", "build", "-p", "deltachat_ffi", "--" + target
     ])
     subprocess.check_call("rm -rf build/ src/deltachat/*.so" , shell=True)
 
-    subprocess.check_call([
-        sys.executable, "-m", "pip", "install", "-e", "."
-    ])
+    if len(sys.argv) <= 1 or sys.argv[1] != "onlybuild":
+        subprocess.check_call([
+            sys.executable, "-m", "pip", "install", "-e", "."
+        ])

python/src/deltachat/_build.py

@@ -29,7 +29,10 @@ def ffibuilder():
             extra_link_args = []
         else:
             raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
-        objs = [os.path.join(projdir, 'target', target, 'libdeltachat.a')]
+        target_dir = os.environ.get("CARGO_TARGET_DIR")
+        if target_dir is None:
+            target_dir = os.path.join(projdir, 'target')
+        objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
         assert os.path.exists(objs[0]), objs
         incs = [os.path.join(projdir, 'deltachat-ffi')]
     else:

python/src/deltachat/account.py

@@ -1,6 +1,7 @@
 """ Account class implementation. """
 
 from __future__ import print_function
+import atexit
 import threading
 import re
 import time
@@ -14,12 +15,14 @@ import deltachat
 from . import const
 from .capi import ffi, lib
 from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array, DCLot
-from .chatting import Contact, Chat, Message
+from .chat import Chat
+from .message import Message
+from .contact import Contact
 
 
 class Account(object):
     """ Each account is tied to a sqlite database file which is fully managed
-    by the underlying deltachat c-library.  All public Account methods are
+    by the underlying deltachat core library.  All public Account methods are
     meant to be memory-safe and return memory-safe objects.
     """
     def __init__(self, db_path, logid=None, eventlogging=True, debug=True):
@@ -49,9 +52,10 @@ class Account(object):
             raise ValueError("Could not dc_open: {}".format(db_path))
         self._configkeys = self.get_config("sys.config_keys").split()
         self._imex_events = Queue()
+        atexit.register(self.shutdown)
 
-    def __del__(self):
-        self.shutdown()
+    # def __del__(self):
+    #    self.shutdown()
 
     def _check_config_key(self, name):
         if name not in self._configkeys:
@@ -69,6 +73,18 @@ class Account(object):
             d[key.lower()] = value
         return d
 
+    def set_stock_translation(self, id, string):
+        """ set stock translation string.
+
+        :param id: id of stock string (const.DC_STR_*)
+        :param value: string to set as new transalation
+        :returns: None
+        """
+        string = string.encode("utf8")
+        res = lib.dc_set_stock_translation(self._dc_context, id, string)
+        if res == 0:
+            raise ValueError("could not set translation string")
+
     def set_config(self, name, value):
         """ set configuration values.
 
@@ -121,11 +137,30 @@ class Account(object):
         if not self.is_configured():
             raise ValueError("need to configure first")
 
+    def empty_server_folders(self, inbox=False, mvbox=False):
+        """ empty server folders. """
+        flags = 0
+        if inbox:
+            flags |= const.DC_EMPTY_INBOX
+        if mvbox:
+            flags |= const.DC_EMPTY_MVBOX
+        if not flags:
+            raise ValueError("no flags set")
+        lib.dc_empty_server(self._dc_context, flags)
+
     def get_infostring(self):
         """ return info of the configured account. """
         self.check_is_configured()
         return from_dc_charpointer(lib.dc_get_info(self._dc_context))
 
+    def get_latest_backupfile(self, backupdir):
+        """ return the latest backup file in a given directory.
+        """
+        res = lib.dc_imex_has_backup(self._dc_context, as_dc_charpointer(backupdir))
+        if res == ffi.NULL:
+            return None
+        return from_dc_charpointer(res)
+
     def get_blobdir(self):
         """ return the directory for files.
 
@@ -135,9 +170,9 @@ class Account(object):
         return from_dc_charpointer(lib.dc_get_blobdir(self._dc_context))
 
     def get_self_contact(self):
-        """ return this account's identity as a :class:`deltachat.chatting.Contact`.
+        """ return this account's identity as a :class:`deltachat.contact.Contact`.
 
-        :returns: :class:`deltachat.chatting.Contact`
+        :returns: :class:`deltachat.contact.Contact`
         """
         self.check_is_configured()
         return Contact(self._dc_context, const.DC_CONTACT_ID_SELF)
@@ -149,7 +184,7 @@ class Account(object):
 
         :param email: email-address (text type)
         :param name: display name for this contact (optional)
-        :returns: :class:`deltachat.chatting.Contact` instance.
+        :returns: :class:`deltachat.contact.Contact` instance.
         """
         name = as_dc_charpointer(name)
         email = as_dc_charpointer(email)
@@ -175,7 +210,7 @@ class Account(object):
                       whose name or e-mail matches query.
         :param only_verified: if true only return verified contacts.
         :param with_self: if true the self-contact is also returned.
-        :returns: list of :class:`deltachat.chatting.Contact` objects.
+        :returns: list of :class:`deltachat.contact.Contact` objects.
         """
         flags = 0
         query = as_dc_charpointer(query)
@@ -193,7 +228,7 @@ class Account(object):
         """ create or get an existing 1:1 chat object for the specified contact or contact id.
 
         :param contact: chat_id (int) or contact object.
-        :returns: a :class:`deltachat.chatting.Chat` object.
+        :returns: a :class:`deltachat.chat.Chat` object.
         """
         if hasattr(contact, "id"):
             if contact._dc_context != self._dc_context:
@@ -210,7 +245,7 @@ class Account(object):
         the specified message.
 
         :param message: messsage id or message instance.
-        :returns: a :class:`deltachat.chatting.Chat` object.
+        :returns: a :class:`deltachat.chat.Chat` object.
         """
         if hasattr(message, "id"):
             if self._dc_context != message._dc_context:
@@ -228,7 +263,7 @@ class Account(object):
         Chats are unpromoted until the first message is sent.
 
         :param verified: if true only verified contacts can be added.
-        :returns: a :class:`deltachat.chatting.Chat` object.
+        :returns: a :class:`deltachat.chat.Chat` object.
         """
         bytes_name = name.encode("utf8")
         chat_id = lib.dc_create_group_chat(self._dc_context, int(verified), bytes_name)
@@ -237,7 +272,7 @@ class Account(object):
     def get_chats(self):
         """ return list of chats.
 
-        :returns: a list of :class:`deltachat.chatting.Chat` objects.
+        :returns: a list of :class:`deltachat.chat.Chat` objects.
         """
         dc_chatlist = ffi.gc(
             lib.dc_get_chatlist(self._dc_context, 0, ffi.NULL, 0),
@@ -255,9 +290,24 @@ class Account(object):
         return Chat(self, const.DC_CHAT_ID_DEADDROP)
 
     def get_message_by_id(self, msg_id):
-        """ return Message instance. """
+        """ return Message instance.
+        :param msg_id: integer id of this message.
+        :returns: :class:`deltachat.message.Message` instance.
+        """
         return Message.from_db(self, msg_id)
 
+    def get_chat_by_id(self, chat_id):
+        """ return Chat instance.
+        :param chat_id: integer id of this chat.
+        :returns: :class:`deltachat.chat.Chat` instance.
+        :raises: ValueError if chat does not exist.
+        """
+        res = lib.dc_get_chat(self._dc_context, chat_id)
+        if res == ffi.NULL:
+            raise ValueError("cannot get chat with id={}".format(chat_id))
+        lib.dc_chat_unref(res)
+        return Chat(self, chat_id)
+
     def mark_seen_messages(self, messages):
         """ mark the given set of messages as seen.
 
@@ -274,7 +324,7 @@ class Account(object):
         """ Forward list of messages to a chat.
 
         :param messages: list of :class:`deltachat.message.Message` object.
-        :param chat: :class:`deltachat.chatting.Chat` object.
+        :param chat: :class:`deltachat.chat.Chat` object.
         :returns: None
         """
         msg_ids = [msg.id for msg in messages]
@@ -386,7 +436,7 @@ class Account(object):
         """ setup contact and return a Chat after contact is established.
 
         Note that this function may block for a long time as messages are exchanged
-        with the emitter of the QR code.  On success a :class:`deltachat.chatting.Chat` instance
+        with the emitter of the QR code.  On success a :class:`deltachat.chat.Chat` instance
         is returned.
         :param qr: valid "setup contact" QR code (all other QR codes will result in an exception)
         """
@@ -400,7 +450,7 @@ class Account(object):
         """ join a chat group through a QR code.
 
         Note that this function may block for a long time as messages are exchanged
-        with the emitter of the QR code.  On success a :class:`deltachat.chatting.Chat` instance
+        with the emitter of the QR code.  On success a :class:`deltachat.chat.Chat` instance
         is returned which is the chat that we just joined.
 
         :param qr: valid "join-group" QR code (all other QR codes will result in an exception)
@@ -411,6 +461,9 @@ class Account(object):
             raise ValueError("could not join group")
         return Chat(self, chat_id)
 
+    def stop_ongoing(self):
+        lib.dc_stop_ongoing_process(self._dc_context)
+
     #
     # meta API for start/stop and event based processing
     #
@@ -432,8 +485,9 @@ class Account(object):
 
     def stop_threads(self, wait=True):
         """ stop IMAP/SMTP threads. """
-        lib.dc_stop_ongoing_process(self._dc_context)
-        self._threads.stop(wait=wait)
+        if self._threads.is_started():
+            self.stop_ongoing()
+            self._threads.stop(wait=wait)
 
     def shutdown(self, wait=True):
         """ stop threads and close and remove underlying dc_context and callbacks. """
@@ -444,6 +498,7 @@ class Account(object):
             self.stop_threads(wait=wait)  # to wait for threads
             deltachat.clear_context_callback(self._dc_context)
             del self._dc_context
+            atexit.unregister(self.shutdown)
 
     def _process_event(self, ctx, evt_name, data1, data2):
         assert ctx == self._dc_context
@@ -463,6 +518,20 @@ class Account(object):
     def on_dc_event_imex_file_written(self, data1, data2):
         self._imex_events.put(data1)
 
+    def set_location(self, latitude=0.0, longitude=0.0, accuracy=0.0):
+        """set a new location. It effects all chats where we currently
+        have enabled location streaming.
+
+        :param latitude: float (use 0.0 if not known)
+        :param longitude: float (use 0.0 if not known)
+        :param accuracy: float (use 0.0 if not known)
+        :raises: ValueError if no chat is currently streaming locations
+        :returns: None
+        """
+        dc_res = lib.dc_set_location(self._dc_context, latitude, longitude, accuracy)
+        if dc_res == 0:
+            raise ValueError("no chat is streaming locations")
+
 
 class IOThreads:
     def __init__(self, dc_context, log_event=lambda *args: None):
@@ -504,8 +573,10 @@ class IOThreads:
         self._log_event("py-bindings-info", 0, "INBOX THREAD START")
         while not self._thread_quitflag:
             lib.dc_perform_imap_jobs(self._dc_context)
-            lib.dc_perform_imap_fetch(self._dc_context)
-            lib.dc_perform_imap_idle(self._dc_context)
+            if not self._thread_quitflag:
+                lib.dc_perform_imap_fetch(self._dc_context)
+            if not self._thread_quitflag:
+                lib.dc_perform_imap_idle(self._dc_context)
         self._log_event("py-bindings-info", 0, "INBOX THREAD FINISHED")
 
     def mvbox_thread_run(self):
@@ -574,11 +645,11 @@ class EventLogger:
             else:
                 assert not rex.match(ev[0]), "event found {}".format(ev)
 
-    def get_matching(self, event_name_regex, check_error=True):
+    def get_matching(self, event_name_regex, check_error=True, timeout=None):
         self._log("-- waiting for event with regex: {} --".format(event_name_regex))
         rex = re.compile("(?:{}).*".format(event_name_regex))
         while 1:
-            ev = self.get()
+            ev = self.get(timeout=timeout, check_error=check_error)
             if rex.match(ev[0]):
                 return ev
 

python/src/deltachat/chat.py

@@ -1,58 +1,15 @@
-""" chatting related objects: Contact, Chat, Message. """
+""" Chat and Location related API. """
 
 import mimetypes
+import calendar
+from datetime import datetime
 import os
-from . import props
 from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array
 from .capi import lib, ffi
 from . import const
 from .message import Message
 
 
-class Contact(object):
-    """ Delta-Chat Contact.
-
-    You obtain instances of it through :class:`deltachat.account.Account`.
-    """
-    def __init__(self, dc_context, id):
-        self._dc_context = dc_context
-        self.id = id
-
-    def __eq__(self, other):
-        return self._dc_context == other._dc_context and self.id == other.id
-
-    def __ne__(self, other):
-        return not (self == other)
-
-    def __repr__(self):
-        return "<Contact id={} addr={} dc_context={}>".format(self.id, self.addr, self._dc_context)
-
-    @property
-    def _dc_contact(self):
-        return ffi.gc(
-            lib.dc_get_contact(self._dc_context, self.id),
-            lib.dc_contact_unref
-        )
-
-    @props.with_doc
-    def addr(self):
-        """ normalized e-mail address for this account. """
-        return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
-
-    @props.with_doc
-    def display_name(self):
-        """ display name for this contact. """
-        return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
-
-    def is_blocked(self):
-        """ Return True if the contact is blocked. """
-        return lib.dc_contact_is_blocked(self._dc_contact)
-
-    def is_verified(self):
-        """ Return True if the contact is verified. """
-        return lib.dc_contact_is_verified(self._dc_contact)
-
-
 class Chat(object):
     """ Chat object which manages members and through which you can send and retrieve messages.
 
@@ -312,9 +269,10 @@ class Chat(object):
     def get_contacts(self):
         """ get all contacts for this chat.
         :params: contact object.
-        :returns: list of :class:`deltachat.chatting.Contact` objects for this chat
+        :returns: list of :class:`deltachat.contact.Contact` objects for this chat
 
         """
+        from .contact import Contact
         dc_array = ffi.gc(
             lib.dc_get_chat_contacts(self._dc_context, self.id),
             lib.dc_array_unref
@@ -365,3 +323,62 @@ class Chat(object):
         if dc_res == ffi.NULL:
             return None
         return from_dc_charpointer(dc_res)
+
+    # ------  location streaming API ------------------------------
+
+    def is_sending_locations(self):
+        """return True if this chat has location-sending enabled currently.
+        :returns: True if location sending is enabled.
+        """
+        return lib.dc_is_sending_locations_to_chat(self._dc_context, self.id)
+
+    def enable_sending_locations(self, seconds):
+        """enable sending locations for this chat.
+
+        all subsequent messages will carry a location with them.
+        """
+        lib.dc_send_locations_to_chat(self._dc_context, self.id, seconds)
+
+    def get_locations(self, contact=None, timestamp_from=None, timestamp_to=None):
+        """return list of locations for the given contact in the given timespan.
+
+        :param contact: the contact for which locations shall be returned.
+        :param timespan_from: a datetime object or None (indicating "since beginning")
+        :param timespan_to: a datetime object or None (indicating up till now)
+        :returns: list of :class:`deltachat.chat.Location` objects.
+        """
+        if timestamp_from is None:
+            time_from = 0
+        else:
+            time_from = calendar.timegm(timestamp_from.utctimetuple())
+        if timestamp_to is None:
+            time_to = 0
+        else:
+            time_to = calendar.timegm(timestamp_to.utctimetuple())
+
+        if contact is None:
+            contact_id = 0
+        else:
+            contact_id = contact.id
+
+        dc_array = lib.dc_get_locations(self._dc_context, self.id, contact_id, time_from, time_to)
+        return [
+            Location(
+                latitude=lib.dc_array_get_latitude(dc_array, i),
+                longitude=lib.dc_array_get_longitude(dc_array, i),
+                accuracy=lib.dc_array_get_accuracy(dc_array, i),
+                timestamp=datetime.utcfromtimestamp(lib.dc_array_get_timestamp(dc_array, i)))
+            for i in range(lib.dc_array_get_cnt(dc_array))
+        ]
+
+
+class Location:
+    def __init__(self, latitude, longitude, accuracy, timestamp):
+        assert isinstance(timestamp, datetime)
+        self.latitude = latitude
+        self.longitude = longitude
+        self.accuracy = accuracy
+        self.timestamp = timestamp
+
+    def __eq__(self, other):
+        return self.__dict__ == other.__dict__

python/src/deltachat/const.py

@@ -8,9 +8,6 @@ from os.path import join as joinpath
 # this works well when you in a git-checkout
 # run "python deltachat/const.py" to regenerate events
 # begin const generated
-DC_PROVIDER_STATUS_OK = 1
-DC_PROVIDER_STATUS_PREPARATION = 2
-DC_PROVIDER_STATUS_BROKEN = 3
 DC_GCL_ARCHIVED_ONLY = 0x01
 DC_GCL_NO_SPECIALS = 0x02
 DC_GCL_ADD_ALLDONE_HINT = 0x04
@@ -50,19 +47,40 @@ DC_STATE_OUT_FAILED = 24
 DC_STATE_OUT_DELIVERED = 26
 DC_STATE_OUT_MDN_RCVD = 28
 DC_CONTACT_ID_SELF = 1
-DC_CONTACT_ID_DEVICE = 2
+DC_CONTACT_ID_INFO = 2
+DC_CONTACT_ID_DEVICE = 5
 DC_CONTACT_ID_LAST_SPECIAL = 9
 DC_MSG_TEXT = 10
 DC_MSG_IMAGE = 20
 DC_MSG_GIF = 21
+DC_MSG_STICKER = 23
 DC_MSG_AUDIO = 40
 DC_MSG_VOICE = 41
 DC_MSG_VIDEO = 50
 DC_MSG_FILE = 60
+DC_LP_AUTH_OAUTH2 = 0x2
+DC_LP_AUTH_NORMAL = 0x4
+DC_LP_IMAP_SOCKET_STARTTLS = 0x100
+DC_LP_IMAP_SOCKET_SSL = 0x200
+DC_LP_IMAP_SOCKET_PLAIN = 0x400
+DC_LP_SMTP_SOCKET_STARTTLS = 0x10000
+DC_LP_SMTP_SOCKET_SSL = 0x20000
+DC_LP_SMTP_SOCKET_PLAIN = 0x40000
+DC_CERTCK_AUTO = 0
+DC_CERTCK_STRICT = 1
+DC_CERTCK_ACCEPT_INVALID_HOSTNAMES = 2
+DC_CERTCK_ACCEPT_INVALID_CERTIFICATES = 3
+DC_EMPTY_MVBOX = 0x01
+DC_EMPTY_INBOX = 0x02
 DC_EVENT_INFO = 100
 DC_EVENT_SMTP_CONNECTED = 101
 DC_EVENT_IMAP_CONNECTED = 102
 DC_EVENT_SMTP_MESSAGE_SENT = 103
+DC_EVENT_IMAP_MESSAGE_DELETED = 104
+DC_EVENT_IMAP_MESSAGE_MOVED = 105
+DC_EVENT_IMAP_FOLDER_EMPTIED = 106
+DC_EVENT_NEW_BLOB_FILE = 150
+DC_EVENT_DELETED_BLOB_FILE = 151
 DC_EVENT_WARNING = 300
 DC_EVENT_ERROR = 400
 DC_EVENT_ERROR_NETWORK = 401
@@ -80,15 +98,67 @@ DC_EVENT_IMEX_PROGRESS = 2051
 DC_EVENT_IMEX_FILE_WRITTEN = 2052
 DC_EVENT_SECUREJOIN_INVITER_PROGRESS = 2060
 DC_EVENT_SECUREJOIN_JOINER_PROGRESS = 2061
-DC_EVENT_GET_STRING = 2091
+DC_EVENT_SECUREJOIN_MEMBER_ADDED = 2062
 DC_EVENT_FILE_COPIED = 2055
 DC_EVENT_IS_OFFLINE = 2081
+DC_EVENT_GET_STRING = 2091
+DC_STR_SELFNOTINGRP = 21
+DC_PROVIDER_STATUS_OK = 1
+DC_PROVIDER_STATUS_PREPARATION = 2
+DC_PROVIDER_STATUS_BROKEN = 3
+DC_STR_NOMESSAGES = 1
+DC_STR_SELF = 2
+DC_STR_DRAFT = 3
+DC_STR_MEMBER = 4
+DC_STR_CONTACT = 6
+DC_STR_VOICEMESSAGE = 7
+DC_STR_DEADDROP = 8
+DC_STR_IMAGE = 9
+DC_STR_VIDEO = 10
+DC_STR_AUDIO = 11
+DC_STR_FILE = 12
+DC_STR_STATUSLINE = 13
+DC_STR_NEWGROUPDRAFT = 14
+DC_STR_MSGGRPNAME = 15
+DC_STR_MSGGRPIMGCHANGED = 16
+DC_STR_MSGADDMEMBER = 17
+DC_STR_MSGDELMEMBER = 18
+DC_STR_MSGGROUPLEFT = 19
+DC_STR_GIF = 23
+DC_STR_ENCRYPTEDMSG = 24
+DC_STR_E2E_AVAILABLE = 25
+DC_STR_ENCR_TRANSP = 27
+DC_STR_ENCR_NONE = 28
+DC_STR_CANTDECRYPT_MSG_BODY = 29
+DC_STR_FINGERPRINTS = 30
+DC_STR_READRCPT = 31
+DC_STR_READRCPT_MAILBODY = 32
+DC_STR_MSGGRPIMGDELETED = 33
+DC_STR_E2E_PREFERRED = 34
+DC_STR_CONTACT_VERIFIED = 35
+DC_STR_CONTACT_NOT_VERIFIED = 36
+DC_STR_CONTACT_SETUP_CHANGED = 37
+DC_STR_ARCHIVEDCHATS = 40
+DC_STR_STARREDMSGS = 41
+DC_STR_AC_SETUP_MSG_SUBJECT = 42
+DC_STR_AC_SETUP_MSG_BODY = 43
+DC_STR_SELFTALK_SUBTITLE = 50
+DC_STR_CANNOT_LOGIN = 60
+DC_STR_SERVER_RESPONSE = 61
+DC_STR_MSGACTIONBYUSER = 62
+DC_STR_MSGACTIONBYME = 63
+DC_STR_MSGLOCATIONENABLED = 64
+DC_STR_MSGLOCATIONDISABLED = 65
+DC_STR_LOCATION = 66
+DC_STR_STICKER = 67
+DC_STR_DEVICE_MESSAGES = 68
+DC_STR_COUNT = 68
 # end const generated
 
 
 def read_event_defines(f):
-    rex = re.compile(r'#define\s+((?:DC_EVENT_|DC_QR|DC_MSG|DC_STATE_|'
-                     r'DC_CONTACT_ID_|DC_GCL|DC_CHAT|DC_PROVIDER)\S+)\s+([x\d]+).*')
+    rex = re.compile(r'#define\s+((?:DC_EVENT|DC_QR|DC_MSG|DC_LP|DC_EMPTY|DC_CERTCK|DC_STATE|DC_STR|'
+                     r'DC_CONTACT_ID|DC_GCL|DC_CHAT|DC_PROVIDER)_\S+)\s+([x\d]+).*')
     for line in f:
         m = rex.match(line)
         if m:

python/src/deltachat/contact.py

@@ -0,0 +1,49 @@
+""" Contact object. """
+
+from . import props
+from .cutil import from_dc_charpointer
+from .capi import lib, ffi
+
+
+class Contact(object):
+    """ Delta-Chat Contact.
+
+    You obtain instances of it through :class:`deltachat.account.Account`.
+    """
+    def __init__(self, dc_context, id):
+        self._dc_context = dc_context
+        self.id = id
+
+    def __eq__(self, other):
+        return self._dc_context == other._dc_context and self.id == other.id
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __repr__(self):
+        return "<Contact id={} addr={} dc_context={}>".format(self.id, self.addr, self._dc_context)
+
+    @property
+    def _dc_contact(self):
+        return ffi.gc(
+            lib.dc_get_contact(self._dc_context, self.id),
+            lib.dc_contact_unref
+        )
+
+    @props.with_doc
+    def addr(self):
+        """ normalized e-mail address for this account. """
+        return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
+
+    @props.with_doc
+    def display_name(self):
+        """ display name for this contact. """
+        return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
+
+    def is_blocked(self):
+        """ Return True if the contact is blocked. """
+        return lib.dc_contact_is_blocked(self._dc_contact)
+
+    def is_verified(self):
+        """ Return True if the contact is verified. """
+        return lib.dc_contact_is_verified(self._dc_contact)

python/src/deltachat/message.py

@@ -1,4 +1,4 @@
-""" chatting related objects: Contact, Chat, Message. """
+""" The Message object. """
 
 import os
 import shutil
@@ -13,7 +13,7 @@ class Message(object):
     """ Message object.
 
     You obtain instances of it through :class:`deltachat.account.Account` or
-    :class:`deltachat.chatting.Chat`.
+    :class:`deltachat.chat.Chat`.
     """
     def __init__(self, account, dc_msg):
         self.account = account
@@ -109,6 +109,10 @@ class Message(object):
         """ return True if this message was encrypted. """
         return bool(lib.dc_msg_get_showpadlock(self._dc_msg))
 
+    def is_forwarded(self):
+        """ return True if this message was forwarded. """
+        return bool(lib.dc_msg_is_forwarded(self._dc_msg))
+
     def get_message_info(self):
         """ Return informational text for a single message.
 
@@ -165,18 +169,18 @@ class Message(object):
     def chat(self):
         """chat this message was posted in.
 
-        :returns: :class:`deltachat.chatting.Chat` object
+        :returns: :class:`deltachat.chat.Chat` object
         """
-        from .chatting import Chat
+        from .chat import Chat
         chat_id = lib.dc_msg_get_chat_id(self._dc_msg)
         return Chat(self.account, chat_id)
 
     def get_sender_contact(self):
         """return the contact of who wrote the message.
 
-        :returns: :class:`deltachat.chatting.Contact` instance
+        :returns: :class:`deltachat.chat.Contact` instance
         """
-        from .chatting import Contact
+        from .contact import Contact
         contact_id = lib.dc_msg_get_from_id(self._dc_msg)
         return Contact(self._dc_context, contact_id)
 

python/tests/conftest.py

@@ -4,6 +4,7 @@ import pytest
 import requests
 import time
 from deltachat import Account
+from deltachat import const
 from deltachat.capi import lib
 import tempfile
 
@@ -150,6 +151,11 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig):
             lib.dc_set_config(ac._dc_context, b"configured", b"1")
             return ac
 
+        def peek_online_config(self):
+            if not session_liveconfig:
+                pytest.skip("specify DCC_PY_LIVECONFIG or --liveconfig")
+            return session_liveconfig.get(self.live_count)
+
         def get_online_config(self):
             if not session_liveconfig:
                 pytest.skip("specify DCC_PY_LIVECONFIG or --liveconfig")
@@ -157,6 +163,11 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig):
             self.live_count += 1
             if "e2ee_enabled" not in configdict:
                 configdict["e2ee_enabled"] = "1"
+
+            # Enable strict certificate checks for online accounts
+            configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+            configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+
             tmpdb = tmpdir.join("livedb%d" % self.live_count)
             ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
             ac._evlogger.init_time = self.init_time
@@ -169,6 +180,12 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig):
             ac.start_threads(mvbox=mvbox, sentbox=sentbox)
             return ac
 
+        def get_one_online_account(self):
+            ac1 = self.get_online_configuring_account()
+            wait_successful_IMAP_SMTP_connection(ac1)
+            wait_configuration_progress(ac1, 1000)
+            return ac1
+
         def get_two_online_accounts(self):
             ac1 = self.get_online_configuring_account()
             ac2 = self.get_online_configuring_account()

python/tests/test_account.py

@@ -1,6 +1,8 @@
 from __future__ import print_function
 import pytest
 import os
+import queue
+import time
 from deltachat import const, Account
 from deltachat.message import Message
 from datetime import datetime, timedelta
@@ -19,6 +21,7 @@ class TestOfflineAccountBasic:
         d = ac1.get_info()
         assert d["arch"]
         assert d["number_of_chats"] == "0"
+        assert d["bcc_self"] == "0"
 
     def test_is_not_configured(self, acfactory):
         ac1 = acfactory.get_unconfigured_account()
@@ -37,6 +40,11 @@ class TestOfflineAccountBasic:
         ac1 = acfactory.get_unconfigured_account()
         assert "save_mime_headers" in ac1.get_config("sys.config_keys").split()
 
+    def test_has_bccself(self, acfactory):
+        ac1 = acfactory.get_unconfigured_account()
+        assert "bcc_self" in ac1.get_config("sys.config_keys").split()
+        assert ac1.get_config("bcc_self") == "0"
+
     def test_selfcontact_if_unconfigured(self, acfactory):
         ac1 = acfactory.get_unconfigured_account()
         with pytest.raises(ValueError):
@@ -93,7 +101,7 @@ class TestOfflineContact:
         ac1 = acfactory.get_configured_offline_account()
         contact1 = ac1.create_contact(email="some1@example.com", name="some1")
         chat = ac1.create_chat_by_contact(contact1)
-        msg = chat.send_text("one messae")
+        msg = chat.send_text("one message")
         assert not ac1.delete_contact(contact1)
         assert not msg.filemime
 
@@ -114,6 +122,12 @@ class TestOfflineChat:
         str(chat1)
         repr(chat1)
 
+    def test_chat_by_id(self, chat1):
+        chat2 = chat1.account.get_chat_by_id(chat1.id)
+        assert chat2 == chat1
+        with pytest.raises(ValueError):
+            chat1.account.get_chat_by_id(123123)
+
     def test_chat_idempotent(self, chat1, ac1):
         contact1 = chat1.get_contacts()[0]
         chat2 = ac1.create_chat_by_contact(contact1.id)
@@ -141,6 +155,27 @@ class TestOfflineChat:
         chat.set_name("title2")
         assert chat.get_name() == "title2"
 
+    def test_group_chat_creation_with_translation(self, ac1):
+        ac1.set_stock_translation(const.DC_STR_NEWGROUPDRAFT, "xyz %1$s")
+        ac1._evlogger.consume_events()
+        with pytest.raises(ValueError):
+            ac1.set_stock_translation(const.DC_STR_NEWGROUPDRAFT, "xyz %2$s")
+        ac1._evlogger.get_matching("DC_EVENT_WARNING")
+        with pytest.raises(ValueError):
+            ac1.set_stock_translation(500, "xyz %1$s")
+        ac1._evlogger.get_matching("DC_EVENT_WARNING")
+        contact1 = ac1.create_contact("some1@hello.com", name="some1")
+        contact2 = ac1.create_contact("some2@hello.com", name="some2")
+        chat = ac1.create_group_chat(name="title1")
+        chat.add_contact(contact1)
+        chat.add_contact(contact2)
+        assert chat.get_name() == "title1"
+        assert contact1 in chat.get_contacts()
+        assert contact2 in chat.get_contacts()
+        assert not chat.is_promoted()
+        msg = chat.get_draft()
+        assert msg.text == "xyz title1"
+
     @pytest.mark.parametrize("verified", [True, False])
     def test_group_chat_qr(self, acfactory, ac1, verified):
         ac2 = acfactory.get_configured_offline_account()
@@ -222,7 +257,9 @@ class TestOfflineChat:
             chat1.send_image(path="notexists")
         fn = data.get_path("d.png")
         lp.sec("sending image")
+        chat1.account._evlogger.consume_events()
         msg = chat1.send_image(fn)
+        chat1.account._evlogger.get_matching("DC_EVENT_NEW_BLOB_FILE")
         assert msg.is_image()
         assert msg
         assert msg.id > 0
@@ -335,12 +372,20 @@ class TestOfflineChat:
 
 
 class TestOnlineAccount:
-    def get_chat(self, ac1, ac2):
+    def get_chat(self, ac1, ac2, both_created=False):
         c2 = ac1.create_contact(email=ac2.get_config("addr"))
         chat = ac1.create_chat_by_contact(c2)
         assert chat.id > const.DC_CHAT_ID_LAST_SPECIAL
+        if both_created:
+            ac2.create_chat_by_contact(ac2.create_contact(email=ac1.get_config("addr")))
         return chat
 
+    def test_configure_canceled(self, acfactory):
+        ac1 = acfactory.get_online_configuring_account()
+        wait_configuration_progress(ac1, 200)
+        ac1.stop_ongoing()
+        wait_configuration_progress(ac1, 0, 0)
+
     def test_export_import_self_keys(self, acfactory, tmpdir):
         ac1, ac2 = acfactory.get_two_online_accounts()
         dir = tmpdir.mkdir("exportdir")
@@ -351,28 +396,84 @@ class TestOnlineAccount:
         ac1._evlogger.consume_events()
         ac1.import_self_keys(dir.strpath)
 
-    def test_one_account_send(self, acfactory):
+    def test_one_account_send_bcc_setting(self, acfactory, lp):
         ac1 = acfactory.get_online_configuring_account()
-        c2 = ac1.create_contact(email=ac1.get_config("addr"))
+        ac2_config = acfactory.peek_online_config()
+        c2 = ac1.create_contact(email=ac2_config["addr"])
         chat = ac1.create_chat_by_contact(c2)
         assert chat.id > const.DC_CHAT_ID_LAST_SPECIAL
         wait_successful_IMAP_SMTP_connection(ac1)
         wait_configuration_progress(ac1, 1000)
 
-        msg_out = chat.send_text("message2")
-        # wait for own account to receive
-        ev = ac1._evlogger.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
-        assert ev[1] == msg_out.id
+        lp.sec("ac1: setting bcc_self=1")
+        ac1.set_config("bcc_self", "1")
 
-    def test_mvbox_sentbox_threads(self, acfactory):
+        lp.sec("send out message with bcc to ourselves")
+        msg_out = chat.send_text("message2")
+        ev = ac1._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
+        assert ev[2] == msg_out.id
+        # wait for send out (BCC)
+        assert ac1.get_config("bcc_self") == "1"
+        self_addr = ac1.get_config("addr")
+        ev = ac1._evlogger.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
+        assert self_addr in ev[2]
+        ev = ac1._evlogger.get_matching("DC_EVENT_DELETED_BLOB_FILE")
+
+        ac1._evlogger.consume_events()
+        lp.sec("send out message without bcc")
+        ac1.set_config("bcc_self", "0")
+        msg_out = chat.send_text("message3")
+        assert not msg_out.is_forwarded()
+        ev = ac1._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
+        assert ev[2] == msg_out.id
+        ev = ac1._evlogger.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
+        assert self_addr not in ev[2]
+        ev = ac1._evlogger.get_matching("DC_EVENT_DELETED_BLOB_FILE")
+
+    def test_mvbox_sentbox_threads(self, acfactory, lp):
+        lp.sec("ac1: start with mvbox thread")
         ac1 = acfactory.get_online_configuring_account(mvbox=True, sentbox=True)
+
+        lp.sec("ac2: start without mvbox/sentbox threads")
         ac2 = acfactory.get_online_configuring_account()
+
+        lp.sec("ac2: waiting for configuration")
+        wait_configuration_progress(ac2, 1000)
+
+        lp.sec("ac1: waiting for configuration")
+        wait_configuration_progress(ac1, 1000)
+
+        lp.sec("ac1: send message and wait for ac2 to receive it")
+        chat = self.get_chat(ac1, ac2)
+        chat.send_text("message1")
+        ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
+        assert ev[2] > const.DC_CHAT_ID_LAST_SPECIAL
+        lp.sec("test finished")
+
+    def test_move_works(self, acfactory):
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.get_online_configuring_account(mvbox=True)
         wait_configuration_progress(ac2, 1000)
         wait_configuration_progress(ac1, 1000)
         chat = self.get_chat(ac1, ac2)
         chat.send_text("message1")
         ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
         assert ev[2] > const.DC_CHAT_ID_LAST_SPECIAL
+        ev = ac2._evlogger.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
+
+    def test_move_works_on_self_sent(self, acfactory):
+        ac1 = acfactory.get_online_configuring_account(mvbox=True)
+        ac1.set_config("bcc_self", "1")
+        ac2 = acfactory.get_online_configuring_account()
+        wait_configuration_progress(ac2, 1000)
+        wait_configuration_progress(ac1, 1000)
+        chat = self.get_chat(ac1, ac2)
+        chat.send_text("message1")
+        chat.send_text("message2")
+        chat.send_text("message3")
+        ac1._evlogger.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
+        ac1._evlogger.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
+        ac1._evlogger.get_matching("DC_EVENT_IMAP_MESSAGE_MOVED")
 
     def test_forward_messages(self, acfactory):
         ac1, ac2 = acfactory.get_two_online_accounts()
@@ -389,6 +490,7 @@ class TestOnlineAccount:
         # check the message arrived in contact-requests/deaddrop
         chat2 = msg_in.chat
         assert msg_in in chat2.get_messages()
+        assert not msg_in.is_forwarded()
         assert chat2.is_deaddrop()
         assert chat2 == ac2.get_deaddrop_chat()
         chat3 = ac2.create_group_chat("newgroup")
@@ -396,9 +498,47 @@ class TestOnlineAccount:
         ac2.forward_messages([msg_in], chat3)
         assert chat3.is_promoted()
         messages = chat3.get_messages()
+        msg = messages[-1]
+        assert msg.is_forwarded()
         ac2.delete_messages(messages)
         assert not chat3.get_messages()
 
+    def test_forward_own_message(self, acfactory, lp):
+        ac1, ac2 = acfactory.get_two_online_accounts()
+        chat = self.get_chat(ac1, ac2, both_created=True)
+
+        lp.sec("sending message")
+        msg_out = chat.send_text("message2")
+
+        lp.sec("receiving message")
+        ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG")
+        msg_in = ac2.get_message_by_id(ev[2])
+        assert msg_in.text == "message2"
+        assert not msg_in.is_forwarded()
+
+        lp.sec("ac1: creating group chat, and forward own message")
+        group = ac1.create_group_chat("newgroup2")
+        group.add_contact(ac1.create_contact(ac2.get_config("addr")))
+        ac1.forward_messages([msg_out], group)
+
+        # wait for other account to receive
+        ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG")
+        msg_in = ac2.get_message_by_id(ev[2])
+        assert msg_in.text == "message2"
+        assert msg_in.is_forwarded()
+
+    def test_send_self_message_and_empty_folder(self, acfactory, lp):
+        ac1 = acfactory.get_one_online_account()
+        lp.sec("ac1: create self chat")
+        chat = ac1.create_chat_by_contact(ac1.get_self_contact())
+        chat.send_text("hello")
+        ac1._evlogger.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
+        ac1.empty_server_folders(inbox=True, mvbox=True)
+        ev = ac1._evlogger.get_matching("DC_EVENT_IMAP_FOLDER_EMPTIED")
+        assert ev[2] == "DeltaChat"
+        ev = ac1._evlogger.get_matching("DC_EVENT_IMAP_FOLDER_EMPTIED")
+        assert ev[2] == "INBOX"
+
     def test_send_and_receive_message_markseen(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
@@ -418,6 +558,7 @@ class TestOnlineAccount:
         assert ev[2] == msg_out.id
         msg_in = ac2.get_message_by_id(msg_out.id)
         assert msg_in.text == "message1"
+        assert not msg_in.is_forwarded()
 
         lp.sec("check the message arrived in contact-requets/deaddrop")
         chat2 = msg_in.chat
@@ -444,6 +585,14 @@ class TestOnlineAccount:
         lp.step("2")
         assert msg_out.is_out_mdn_received()
 
+        lp.sec("check that a second call to mark_seen does not create change or smtp job")
+        ac2._evlogger.consume_events()
+        ac2.mark_seen_messages([msg_in])
+        try:
+            ac2._evlogger.get_matching("DC_EVENT_MSG_READ", timeout=0.01)
+        except queue.Empty:
+            pass  # mark_seen_messages() has generated events before it returns
+
     def test_send_and_receive_will_encrypt_decrypt(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
@@ -472,6 +621,14 @@ class TestOnlineAccount:
         assert msg_back.text == "message-back"
         assert msg_back.is_encrypted()
 
+        lp.sec("create group chat with two members, one of which has no encrypt state")
+        chat = ac1.create_group_chat("encryption test")
+        chat.add_contact(ac1.create_contact(ac2.get_config("addr")))
+        chat.add_contact(ac1.create_contact("notexisting@testrun.org"))
+        msg = chat.send_text("test not encrypt")
+        ev = ac1._evlogger.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
+        assert not msg.is_encrypted()
+
     def test_saved_mime_on_received_message(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
@@ -512,18 +669,29 @@ class TestOnlineAccount:
         assert os.path.exists(msg_in.filename)
         assert os.stat(msg_in.filename).st_size == os.stat(path).st_size
 
-    def test_import_export_online_all(self, acfactory, tmpdir):
+    def test_import_export_online_all(self, acfactory, tmpdir, lp):
         ac1 = acfactory.get_online_configuring_account()
         wait_configuration_progress(ac1, 1000)
 
+        lp.sec("create some chat content")
         contact1 = ac1.create_contact("some1@hello.com", name="some1")
         chat = ac1.create_chat_by_contact(contact1)
         chat.send_text("msg1")
         backupdir = tmpdir.mkdir("backup")
+
+        lp.sec("export all to {}".format(backupdir))
         path = ac1.export_all(backupdir.strpath)
         assert os.path.exists(path)
+        t = time.time()
 
+        lp.sec("get fresh empty account")
         ac2 = acfactory.get_unconfigured_account()
+
+        lp.sec("get latest backup file")
+        path2 = ac2.get_latest_backupfile(backupdir.strpath)
+        assert path2 == path
+
+        lp.sec("import backup and check it's proper")
         ac2.import_all(path)
         contacts = ac2.get_contacts(query="some1")
         assert len(contacts) == 1
@@ -534,6 +702,17 @@ class TestOnlineAccount:
         assert len(messages) == 1
         assert messages[0].text == "msg1"
 
+        # wait until a second passed since last backup
+        # because get_latest_backupfile() shall return the latest backup
+        # from a UI it's unlikely anyone manages to export two
+        # backups in one second.
+        time.sleep(max(0, 1 - (time.time() - t)))
+        lp.sec("Second-time export all to {}".format(backupdir))
+        path2 = ac1.export_all(backupdir.strpath)
+        assert os.path.exists(path2)
+        assert path2 != path
+        assert ac2.get_latest_backupfile(backupdir.strpath) == path2
+
     def test_ac_setup_message(self, acfactory, lp):
         # note that the receiving account needs to be configured and running
         # before ther setup message is send. DC does not read old messages
@@ -559,6 +738,29 @@ class TestOnlineAccount:
         msg.continue_key_transfer(setup_code)
         assert ac1.get_info()["fingerprint"] == ac2.get_info()["fingerprint"]
 
+    def test_ac_setup_message_twice(self, acfactory, lp):
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.clone_online_account(ac1)
+        ac2._evlogger.set_timeout(30)
+        wait_configuration_progress(ac2, 1000)
+        wait_configuration_progress(ac1, 1000)
+
+        lp.sec("trigger ac setup message but ignore")
+        assert ac1.get_info()["fingerprint"] != ac2.get_info()["fingerprint"]
+        ac1.initiate_key_transfer()
+        ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
+
+        lp.sec("trigger second ac setup message, wait for receive ")
+        setup_code2 = ac1.initiate_key_transfer()
+        ev = ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG|DC_EVENT_MSGS_CHANGED")
+        msg = ac2.get_message_by_id(ev[2])
+        assert msg.is_setup_message()
+        assert msg.get_setupcodebegin() == setup_code2[:2]
+
+        lp.sec("process second setup message")
+        msg.continue_key_transfer(setup_code2)
+        assert ac1.get_info()["fingerprint"] == ac2.get_info()["fingerprint"]
+
     def test_qr_setup_contact(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
         lp.sec("ac1: create QR code and let ac2 scan it, starting the securejoin")
@@ -576,7 +778,11 @@ class TestOnlineAccount:
         lp.sec("ac2: start QR-code based join-group protocol")
         ch = ac2.qr_join_chat(qr)
         assert ch.id >= 10
+        # check that at least some of the handshake messages are deleted
+        ac1._evlogger.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
+        ac2._evlogger.get_matching("DC_EVENT_IMAP_MESSAGE_DELETED")
         wait_securejoin_inviter_progress(ac1, 1000)
+        ac1._evlogger.get_matching("DC_EVENT_SECUREJOIN_MEMBER_ADDED")
 
     def test_qr_verified_group_and_chatting(self, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
@@ -588,6 +794,7 @@ class TestOnlineAccount:
         chat2 = ac2.qr_join_chat(qr)
         assert chat2.id >= 10
         wait_securejoin_inviter_progress(ac1, 1000)
+        ac1._evlogger.get_matching("DC_EVENT_SECUREJOIN_MEMBER_ADDED")
 
         lp.sec("ac2: read member added message")
         msg = ac2.wait_next_incoming_message()
@@ -663,6 +870,54 @@ class TestOnlineAccount:
         assert chat1b.get_profile_image() is None
         assert chat.get_profile_image() is None
 
+    def test_send_receive_locations(self, acfactory, lp):
+        now = datetime.utcnow()
+        ac1, ac2 = acfactory.get_two_online_accounts()
+
+        lp.sec("ac1: create chat with ac2")
+        chat1 = self.get_chat(ac1, ac2)
+        chat2 = self.get_chat(ac2, ac1)
+
+        assert not chat1.is_sending_locations()
+        with pytest.raises(ValueError):
+            ac1.set_location(latitude=0.0, longitude=10.0)
+
+        ac1._evlogger.consume_events()
+        ac2._evlogger.consume_events()
+
+        lp.sec("ac1: enable location sending in chat")
+        chat1.enable_sending_locations(seconds=100)
+        assert chat1.is_sending_locations()
+        ac1._evlogger.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
+
+        ac1.set_location(latitude=2.0, longitude=3.0, accuracy=0.5)
+        ac1._evlogger.get_matching("DC_EVENT_LOCATION_CHANGED")
+        chat1.send_text("hello")
+        ac1._evlogger.get_matching("DC_EVENT_SMTP_MESSAGE_SENT")
+
+        lp.sec("ac2: wait for incoming location message")
+        ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG")  # "enabled-location streaming"
+
+        # currently core emits location changed before event_incoming message
+        ac2._evlogger.get_matching("DC_EVENT_LOCATION_CHANGED")
+        ac2._evlogger.get_matching("DC_EVENT_INCOMING_MSG")  # text message with location
+
+        locations = chat2.get_locations()
+        assert len(locations) == 1
+        assert locations[0].latitude == 2.0
+        assert locations[0].longitude == 3.0
+        assert locations[0].accuracy == 0.5
+        assert locations[0].timestamp > now
+
+        contact = ac2.create_contact(ac1.get_config("addr"))
+        locations2 = chat2.get_locations(contact=contact)
+        assert len(locations2) == 1
+        assert locations2 == locations
+
+        contact = ac2.create_contact("nonexisting@example.org")
+        locations3 = chat2.get_locations(contact=contact)
+        assert not locations3
+
 
 class TestOnlineConfigureFails:
     def test_invalid_password(self, acfactory):
@@ -671,7 +926,7 @@ class TestOnlineConfigureFails:
         ac1.start_threads()
         wait_configuration_progress(ac1, 500)
         ev1 = ac1._evlogger.get_matching("DC_EVENT_ERROR_NETWORK")
-        assert "authentication failed" in ev1[2].lower()
+        assert "cannot login" in ev1[2].lower()
         wait_configuration_progress(ac1, 0, 0)
 
     def test_invalid_user(self, acfactory):
@@ -680,7 +935,7 @@ class TestOnlineConfigureFails:
         ac1.start_threads()
         wait_configuration_progress(ac1, 500)
         ev1 = ac1._evlogger.get_matching("DC_EVENT_ERROR_NETWORK")
-        assert "authentication failed" in ev1[2].lower()
+        assert "cannot login" in ev1[2].lower()
         wait_configuration_progress(ac1, 0, 0)
 
     def test_invalid_domain(self, acfactory):

python/tests/test_lowlevel.py

@@ -41,7 +41,7 @@ def test_dc_close_events(tmpdir):
             else:
                 print("skipping event", *ev)
 
-    find("disconnecting INBOX-watch")
+    find("disconnecting inbox-thread")
     find("disconnecting sentbox-thread")
     find("disconnecting mvbox-thread")
     find("disconnecting SMTP")
@@ -95,6 +95,13 @@ def test_markseen_invalid_message_ids(acfactory):
     ac1._evlogger.ensure_event_not_queued("DC_EVENT_WARNING|DC_EVENT_ERROR")
 
 
+def test_get_special_message_id_returns_empty_message(acfactory):
+    ac1 = acfactory.get_configured_offline_account()
+    for i in range(1, 10):
+        msg = ac1.get_message_by_id(i)
+        assert msg.id == 0
+
+
 def test_provider_info():
     provider = lib.dc_provider_new_from_email(cutil.as_dc_charpointer("ex@example.com"))
     assert cutil.from_dc_charpointer(

python/tox.ini

@@ -1,19 +1,25 @@
 [tox]
 # make sure to update environment list in travis.yml and appveyor.yml
 envlist =
-    py35
+    py37
     lint
     auditwheels
 
 [testenv]
 commands = 
-    pytest -v -rsXx {posargs:tests}
-    python tests/package_wheels.py {toxworkdir}/wheelhouse
+    # (some qr tests are pretty heavy in terms of send/received
+    # messages and async-imap's likely has concurrency problems, 
+    # eg https://github.com/async-email/async-imap/issues/4 )
+    pytest -n6 --reruns 3 --reruns-delay 5 -v -rsXx -k "not qr" {posargs:tests} 
+    pytest -n6 --reruns 5 --reruns-delay 5 -v -rsXx -k "qr" {posargs:tests} 
+    # python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 
     DCC_RS_DEV
     DCC_RS_TARGET
     DCC_PY_LIVECONFIG
+    CARGO_TARGET_DIR
+    RUSTC_WRAPPER
 deps = 
     pytest
     pytest-rerunfailures
@@ -28,7 +34,6 @@ deps = auditwheel
 commands = 
     python tests/auditwheels.py {toxworkdir}/wheelhouse
 
-
 [testenv:lint]
 skipsdist = True
 usedevelop = True
@@ -43,18 +48,28 @@ commands =
     rst-lint --encoding 'utf-8' README.rst
 
 [testenv:doc]
-basepython = python3.5
+changedir=doc
 deps =
     sphinx==2.2.0
     breathe
-
-changedir = doc
 commands =
-    sphinx-build -w docker-toxdoc-warnings.log -b html . _build/html
+    sphinx-build -Q -w toxdoc-warnings.log -b html . _build/html
+
+
+[testenv:lintdoc]
+skipsdist = True
+usedevelop = True
+deps =
+    {[testenv:lint]deps}
+    {[testenv:doc]deps}
+commands =
+    {[testenv:lint]commands}
+    {[testenv:doc]commands}
+
 
 
 [pytest]
-addopts = -v -rs --reruns 3 --reruns-delay 2
+addopts = -v -rs 
 python_files = tests/test_*.py 
 norecursedirs = .tox 
 xfail_strict=true

run-integration-tests.sh

@@ -23,7 +23,7 @@ if [ $? != 0 ]; then
 fi
 
 pushd python
-if [ -e "./liveconfig" && -z "$DCC_PY_LIVECONFIG" ]; then
+if [ -e "./liveconfig" -a -z "$DCC_PY_LIVECONFIG" ]; then
     export DCC_PY_LIVECONFIG=liveconfig
 fi
 tox "$@"

rust-toolchain

@@ -1 +1 @@
-nightly-2019-08-13
+nightly-2019-11-06

set_core_version.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import re
+import pathlib
+import subprocess
+
+rex = re.compile(r'version = "(\S+)"')
+
+def read_toml_version(relpath):
+    p = pathlib.Path(relpath)
+    assert p.exists()
+    for line in open(str(p)):
+        m = rex.match(line)
+        if m is not None:
+            return m.group(1)
+    raise ValueError("no version found in {}".format(relpath))
+
+def replace_toml_version(relpath, newversion):
+    p = pathlib.Path(relpath)
+    assert p.exists()
+    tmp_path = str(p) + "_tmp"
+    with open(tmp_path, "w") as f:
+        for line in open(str(p)):
+            m = rex.match(line)
+            if m is not None:
+                f.write('version = "{}"\n'.format(newversion))
+            else:
+                f.write(line)
+    os.rename(tmp_path, str(p))
+
+if __name__ == "__main__":
+
+    if len(sys.argv) < 2:
+        raise SystemExit("need argument: new version, example 1.0.0-beta.27")
+    newversion = sys.argv[1]
+    if newversion.count(".") < 2:
+        raise SystemExit("need at least two dots in version")
+
+    core_toml = read_toml_version("Cargo.toml")
+    ffi_toml = read_toml_version("deltachat-ffi/Cargo.toml")
+    assert core_toml == ffi_toml, (core_toml, ffi_toml)
+
+    for line in open("CHANGELOG.md"):
+        ## 1.0.0-beta5
+        if line.startswith("## "):
+            if line[2:].strip().startswith(newversion):
+                break
+    else:
+        raise SystemExit("CHANGELOG.md contains no entry for version: {}".format(newversion))
+
+    replace_toml_version("Cargo.toml", newversion)
+    replace_toml_version("deltachat-ffi/Cargo.toml", newversion)
+
+    subprocess.call(["cargo", "update", "-p", "deltachat"])
+
+    print("after commit make sure to: ")
+    print("")
+    print("   git tag {}".format(newversion))
+    print("")

spec.md

@@ -334,9 +334,7 @@ only on image changes.
 
 # Miscellaneous
 
-Messengers SHOULD use the header `Chat-Predecessor`
-instead of `In-Reply-To` as the latter one results
-in infinite threads on typical MUAs.
+Messengers SHOULD use the header `In-Reply-To` as usual.
 
 Messengers SHOULD add a `Chat-Voice-message: 1` header
 if an attached audio file is a voice message.
@@ -346,7 +344,7 @@ to specify the duration of attached audio or video files.
 The value MUST be the duration in milliseconds.
 This allows the receiver to show the time without knowing the file format.
 
-    Chat-Predecessor: foo123@domain
+    In-Reply-To: Gr.12345uvwxyZ.0005@domain
     Chat-Voice-Message: 1
     Chat-Duration: 10000
 

src/aheader.rs

@@ -1,3 +1,7 @@
+//! # Autocrypt header module
+//!
+//! Parse and create [Autocrypt-headers](https://autocrypt.org/en/latest/level1.html#the-autocrypt-header).
+
 use std::collections::BTreeMap;
 use std::ffi::CStr;
 use std::str::FromStr;
@@ -40,13 +44,13 @@ impl str::FromStr for EncryptPreference {
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         match s {
             "mutual" => Ok(EncryptPreference::Mutual),
-            "reset" => Ok(EncryptPreference::Reset),
-            _ => Ok(EncryptPreference::NoPreference),
+            "nopreference" => Ok(EncryptPreference::NoPreference),
+            _ => Err(()),
         }
     }
 }
 
-/// Parse and create [Autocrypt-headers](https://autocrypt.org/en/latest/level1.html#the-autocrypt-header).
+/// Autocrypt header
 #[derive(Debug)]
 pub struct Aheader {
     pub addr: String,
@@ -55,6 +59,7 @@ pub struct Aheader {
 }
 
 impl Aheader {
+    /// Creates new autocrypt header
     pub fn new(addr: String, public_key: Key, prefer_encrypt: EncryptPreference) -> Self {
         Aheader {
             addr,
@@ -79,27 +84,21 @@ impl Aheader {
                 let optional_field = unsafe { (*field).fld_data.fld_optional_field };
                 if !optional_field.is_null()
                     && unsafe { !(*optional_field).fld_name.is_null() }
-                    && unsafe { CStr::from_ptr((*optional_field).fld_name).to_str().unwrap() }
+                    && unsafe { CStr::from_ptr((*optional_field).fld_name).to_string_lossy() }
                         == "Autocrypt"
                 {
-                    let value = unsafe {
-                        CStr::from_ptr((*optional_field).fld_value)
-                            .to_str()
-                            .unwrap()
-                    };
+                    let value =
+                        unsafe { CStr::from_ptr((*optional_field).fld_value).to_string_lossy() };
 
-                    match Self::from_str(value) {
-                        Ok(test) => {
-                            if addr_cmp(&test.addr, wanted_from) {
-                                if fine_header.is_none() {
-                                    fine_header = Some(test);
-                                } else {
-                                    // TODO: figure out what kind of error case this is
-                                    return None;
-                                }
+                    if let Ok(test) = Self::from_str(&value) {
+                        if addr_cmp(&test.addr, wanted_from) {
+                            if fine_header.is_none() {
+                                fine_header = Some(test);
+                            } else {
+                                // TODO: figure out what kind of error case this is
+                                return None;
                             }
                         }
-                        _ => {}
                     }
                 }
             }
@@ -131,9 +130,9 @@ impl str::FromStr for Aheader {
 
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         let mut attributes: BTreeMap<String, String> = s
-            .split(";")
+            .split(';')
             .filter_map(|a| {
-                let attribute: Vec<&str> = a.trim().splitn(2, "=").collect();
+                let attribute: Vec<&str> = a.trim().splitn(2, '=').collect();
                 if attribute.len() < 2 {
                     return None;
                 }
@@ -178,7 +177,7 @@ impl str::FromStr for Aheader {
 
         // Autocrypt-Level0: unknown attributes starting with an underscore can be safely ignored
         // Autocrypt-Level0: unknown attribute, treat the header as invalid
-        if attributes.keys().find(|k| !k.starts_with("_")).is_some() {
+        if attributes.keys().any(|k| !k.starts_with('_')) {
             return Err(());
         }
 

src/blob.rs

@@ -0,0 +1,595 @@
+//! # Blob directory management
+
+use std::ffi::OsStr;
+use std::fmt;
+use std::fs;
+use std::io::Write;
+use std::path::{Path, PathBuf};
+
+use crate::context::Context;
+use crate::events::Event;
+
+/// Represents a file in the blob directory.
+///
+/// The object has a name, which will always be valid UTF-8.  Having a
+/// blob object does not imply the respective file exists, however
+/// when using one of the `create*()` methods a unique file is
+/// created.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct BlobObject<'a> {
+    blobdir: &'a Path,
+    name: String,
+}
+
+impl<'a> BlobObject<'a> {
+    /// Creates a new blob object with a unique name.
+    ///
+    /// Creates a new file in the blob directory.  The name will be
+    /// derived from the platform-agnostic basename of the suggested
+    /// name, followed by a random number and followed by a possible
+    /// extension.  The `data` will be written into the file without
+    /// race-conditions.
+    ///
+    /// # Errors
+    ///
+    /// [BlobError::CreateFailure] is used when the file could not
+    /// be created.  You can expect [BlobError.cause] to contain an
+    /// underlying error.
+    ///
+    /// [BlobError::WriteFailure] is used when the file could not
+    /// be written to.  You can expect [BlobError.cause] to contain an
+    /// underlying error.
+    pub fn create(
+        context: &'a Context,
+        suggested_name: impl AsRef<str>,
+        data: &[u8],
+    ) -> std::result::Result<BlobObject<'a>, BlobError> {
+        let blobdir = context.get_blobdir();
+        let (stem, ext) = BlobObject::sanitise_name(suggested_name.as_ref());
+        let (name, mut file) = BlobObject::create_new_file(&blobdir, &stem, &ext)?;
+        file.write_all(data)
+            .map_err(|err| BlobError::WriteFailure {
+                blobdir: blobdir.to_path_buf(),
+                blobname: name.clone(),
+                cause: err,
+                backtrace: failure::Backtrace::new(),
+            })?;
+        let blob = BlobObject {
+            blobdir,
+            name: format!("$BLOBDIR/{}", name),
+        };
+        context.call_cb(Event::NewBlobFile(blob.as_name().to_string()));
+        Ok(blob)
+    }
+
+    // Creates a new file, returning a tuple of the name and the handle.
+    fn create_new_file(dir: &Path, stem: &str, ext: &str) -> Result<(String, fs::File), BlobError> {
+        let max_attempt = 15;
+        let mut name = format!("{}{}", stem, ext);
+        for attempt in 0..max_attempt {
+            let path = dir.join(&name);
+            match fs::OpenOptions::new()
+                .create_new(true)
+                .write(true)
+                .open(&path)
+            {
+                Ok(file) => return Ok((name, file)),
+                Err(err) => {
+                    if attempt == max_attempt {
+                        return Err(BlobError::CreateFailure {
+                            blobdir: dir.to_path_buf(),
+                            blobname: name,
+                            cause: err,
+                            backtrace: failure::Backtrace::new(),
+                        });
+                    } else {
+                        name = format!("{}-{}{}", stem, rand::random::<u32>(), ext);
+                    }
+                }
+            }
+        }
+        // This is supposed to be unreachable, but the compiler doesn't know.
+        Err(BlobError::CreateFailure {
+            blobdir: dir.to_path_buf(),
+            blobname: name,
+            cause: std::io::Error::new(std::io::ErrorKind::Other, "supposedly unreachable"),
+            backtrace: failure::Backtrace::new(),
+        })
+    }
+
+    /// Creates a new blob object with unique name by copying an existing file.
+    ///
+    /// This creates a new blob as described in [BlobObject::create]
+    /// but also copies an existing file into it.  This is done in a
+    /// in way which avoids race-conditions when multiple files are
+    /// concurrently created.
+    ///
+    /// # Errors
+    ///
+    /// In addition to the errors in [BlobObject::create] the
+    /// [BlobError::CopyFailure] is used when the data can not be
+    /// copied.
+    pub fn create_and_copy(
+        context: &'a Context,
+        src: impl AsRef<Path>,
+    ) -> std::result::Result<BlobObject<'a>, BlobError> {
+        let mut src_file = fs::File::open(src.as_ref()).map_err(|err| BlobError::CopyFailure {
+            blobdir: context.get_blobdir().to_path_buf(),
+            blobname: String::from(""),
+            src: src.as_ref().to_path_buf(),
+            cause: err,
+            backtrace: failure::Backtrace::new(),
+        })?;
+        let (stem, ext) = BlobObject::sanitise_name(&src.as_ref().to_string_lossy());
+        let (name, mut dst_file) = BlobObject::create_new_file(context.get_blobdir(), &stem, &ext)?;
+        let name_for_err = name.clone();
+        std::io::copy(&mut src_file, &mut dst_file).map_err(|err| {
+            {
+                // Attempt to remove the failed file, swallow errors resulting from that.
+                let path = context.get_blobdir().join(&name_for_err);
+                fs::remove_file(path).ok();
+            }
+            BlobError::CopyFailure {
+                blobdir: context.get_blobdir().to_path_buf(),
+                blobname: name_for_err,
+                src: src.as_ref().to_path_buf(),
+                cause: err,
+                backtrace: failure::Backtrace::new(),
+            }
+        })?;
+        let blob = BlobObject {
+            blobdir: context.get_blobdir(),
+            name: format!("$BLOBDIR/{}", name),
+        };
+        context.call_cb(Event::NewBlobFile(blob.as_name().to_string()));
+        Ok(blob)
+    }
+
+    /// Creates a blob from a file, possibly copying it to the blobdir.
+    ///
+    /// If the source file is not a path to into the blob directory
+    /// the file will be copied into the blob directory first.  If the
+    /// source file is already in the blobdir it will not be copied
+    /// and only be created if it is a valid blobname, that is no
+    /// subdirectory is used and [BlobObject::sanitise_name] does not
+    /// modify the filename.
+    ///
+    /// # Errors
+    ///
+    /// This merely delegates to the [BlobObject::create_and_copy] and
+    /// the [BlobObject::from_path] methods.  See those for possible
+    /// errors.
+    pub fn create_from_path(
+        context: &Context,
+        src: impl AsRef<Path>,
+    ) -> std::result::Result<BlobObject, BlobError> {
+        if src.as_ref().starts_with(context.get_blobdir()) {
+            BlobObject::from_path(context, src)
+        } else {
+            BlobObject::create_and_copy(context, src)
+        }
+    }
+
+    /// Returns a [BlobObject] for an existing blob from a path.
+    ///
+    /// The path must designate a file directly in the blobdir and
+    /// must use a valid blob name.  That is after sanitisation the
+    /// name must still be the same, that means it must be valid UTF-8
+    /// and not have any special characters in it.
+    ///
+    /// # Errors
+    ///
+    /// [BlobError::WrongBlobdir] is used if the path is not in
+    /// the blob directory.
+    ///
+    /// [BlobError::WrongName] is used if the file name does not
+    /// remain identical after sanitisation.
+    pub fn from_path(
+        context: &Context,
+        path: impl AsRef<Path>,
+    ) -> std::result::Result<BlobObject, BlobError> {
+        let rel_path = path
+            .as_ref()
+            .strip_prefix(context.get_blobdir())
+            .map_err(|_| BlobError::WrongBlobdir {
+                blobdir: context.get_blobdir().to_path_buf(),
+                src: path.as_ref().to_path_buf(),
+                backtrace: failure::Backtrace::new(),
+            })?;
+        if !BlobObject::is_acceptible_blob_name(&rel_path) {
+            return Err(BlobError::WrongName {
+                blobname: path.as_ref().to_path_buf(),
+                backtrace: failure::Backtrace::new(),
+            });
+        }
+        let name = rel_path.to_str().ok_or_else(|| BlobError::WrongName {
+            blobname: path.as_ref().to_path_buf(),
+            backtrace: failure::Backtrace::new(),
+        })?;
+        BlobObject::from_name(context, name.to_string())
+    }
+
+    /// Returns a [BlobObject] for an existing blob.
+    ///
+    /// The `name` may optionally be prefixed with the `$BLOBDIR/`
+    /// prefixed, as returned by [BlobObject::as_name].  This is how
+    /// you want to create a [BlobObject] for a filename read from the
+    /// database.
+    ///
+    /// # Errors
+    ///
+    /// [BlobError::WrongName] is used if the name is not a valid
+    /// blobname, i.e. if [BlobObject::sanitise_name] does modify the
+    /// provided name.
+    pub fn from_name(
+        context: &'a Context,
+        name: String,
+    ) -> std::result::Result<BlobObject<'a>, BlobError> {
+        let name: String = match name.starts_with("$BLOBDIR/") {
+            true => name.splitn(2, '/').last().unwrap().to_string(),
+            false => name,
+        };
+        if !BlobObject::is_acceptible_blob_name(&name) {
+            return Err(BlobError::WrongName {
+                blobname: PathBuf::from(name),
+                backtrace: failure::Backtrace::new(),
+            });
+        }
+        Ok(BlobObject {
+            blobdir: context.get_blobdir(),
+            name: format!("$BLOBDIR/{}", name),
+        })
+    }
+
+    /// Returns the absolute path to the blob in the filesystem.
+    pub fn to_abs_path(&self) -> PathBuf {
+        let fname = Path::new(&self.name).strip_prefix("$BLOBDIR/").unwrap();
+        self.blobdir.join(fname)
+    }
+
+    /// Returns the blob name, as stored in the database.
+    ///
+    /// This returns the blob in the `$BLOBDIR/<name>` format used in
+    /// the database.  Do not use this unless you're about to store
+    /// this string in the database or [Params].  Eventually even
+    /// those conversions should be handled by the type system.
+    ///
+    /// [Params]: crate::param::Params
+    pub fn as_name(&self) -> &str {
+        &self.name
+    }
+
+    /// Returns the filename of the blob.
+    pub fn as_file_name(&self) -> &str {
+        self.name.rsplitn(2, '/').next().unwrap()
+    }
+
+    /// The path relative in the blob directory.
+    pub fn as_rel_path(&self) -> &Path {
+        Path::new(self.as_file_name())
+    }
+
+    /// Returns the extension of the blob.
+    ///
+    /// If a blob's filename has an extension, it is always guaranteed
+    /// to be lowercase.
+    pub fn suffix(&self) -> Option<&str> {
+        let ext = self.name.rsplitn(2, '.').next();
+        if ext == Some(&self.name) {
+            None
+        } else {
+            ext
+        }
+    }
+
+    /// Create a safe name based on a messy input string.
+    ///
+    /// The safe name will be a valid filename on Unix and Windows and
+    /// not contain any path separators.  The input can contain path
+    /// segments separated by either Unix or Windows path separators,
+    /// the rightmost non-empty segment will be used as name,
+    /// sanitised for special characters.
+    ///
+    /// The resulting name is returned as a tuple, the first part
+    /// being the stem or basename and the second being an extension,
+    /// including the dot.  E.g. "foo.txt" is returned as `("foo",
+    /// ".txt")` while "bar" is returned as `("bar", "")`.
+    ///
+    /// The extension part will always be lowercased.
+    fn sanitise_name(name: &str) -> (String, String) {
+        let mut name = name.to_string();
+        for part in name.rsplit('/') {
+            if !part.is_empty() {
+                name = part.to_string();
+                break;
+            }
+        }
+        for part in name.rsplit('\\') {
+            if !part.is_empty() {
+                name = part.to_string();
+                break;
+            }
+        }
+        let opts = sanitize_filename::Options {
+            truncate: true,
+            windows: true,
+            replacement: "",
+        };
+
+        let clean = sanitize_filename::sanitize_with_options(name, opts);
+        let mut iter = clean.splitn(2, '.');
+        let mut stem = iter.next().unwrap_or_default().to_string();
+        let mut ext = iter.next().unwrap_or_default().to_string();
+        stem.truncate(64);
+        ext.truncate(32);
+        match ext.len() {
+            0 => (stem, "".to_string()),
+            _ => (stem, format!(".{}", ext).to_lowercase()),
+        }
+    }
+
+    /// Checks whether a name is a valid blob name.
+    ///
+    /// This is slightly less strict than stanitise_name, presumably
+    /// someone already created a file with such a name so we just
+    /// ensure it's not actually a path in disguise is actually utf-8.
+    fn is_acceptible_blob_name(name: impl AsRef<OsStr>) -> bool {
+        let uname = match name.as_ref().to_str() {
+            Some(name) => name,
+            None => return false,
+        };
+        if uname.find('/').is_some() {
+            return false;
+        }
+        if uname.find('\\').is_some() {
+            return false;
+        }
+        if uname.find('\0').is_some() {
+            return false;
+        }
+        true
+    }
+}
+
+impl<'a> fmt::Display for BlobObject<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "$BLOBDIR/{}", self.name)
+    }
+}
+
+/// Errors for the [BlobObject].
+#[derive(Fail, Debug)]
+pub enum BlobError {
+    CreateFailure {
+        blobdir: PathBuf,
+        blobname: String,
+        #[cause]
+        cause: std::io::Error,
+        backtrace: failure::Backtrace,
+    },
+    WriteFailure {
+        blobdir: PathBuf,
+        blobname: String,
+        #[cause]
+        cause: std::io::Error,
+        backtrace: failure::Backtrace,
+    },
+    CopyFailure {
+        blobdir: PathBuf,
+        blobname: String,
+        src: PathBuf,
+        #[cause]
+        cause: std::io::Error,
+        backtrace: failure::Backtrace,
+    },
+    WrongBlobdir {
+        blobdir: PathBuf,
+        src: PathBuf,
+        backtrace: failure::Backtrace,
+    },
+    WrongName {
+        blobname: PathBuf,
+        backtrace: failure::Backtrace,
+    },
+}
+
+// Implementing Display is done by hand because the failure
+// #[fail(display = "...")] syntax does not allow using
+// `blobdir.display()`.
+impl fmt::Display for BlobError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        // Match on the data rather than kind, they are equivalent for
+        // identifying purposes but contain the actual data we need.
+        match &self {
+            BlobError::CreateFailure {
+                blobdir, blobname, ..
+            } => write!(
+                f,
+                "Failed to create blob {} in {}",
+                blobname,
+                blobdir.display()
+            ),
+            BlobError::WriteFailure {
+                blobdir, blobname, ..
+            } => write!(
+                f,
+                "Failed to write data to blob {} in {}",
+                blobname,
+                blobdir.display()
+            ),
+            BlobError::CopyFailure {
+                blobdir,
+                blobname,
+                src,
+                ..
+            } => write!(
+                f,
+                "Failed to copy data from {} to blob {} in {}",
+                src.display(),
+                blobname,
+                blobdir.display(),
+            ),
+            BlobError::WrongBlobdir { blobdir, src, .. } => write!(
+                f,
+                "File path {} is not in blobdir {}",
+                src.display(),
+                blobdir.display(),
+            ),
+            BlobError::WrongName { blobname, .. } => {
+                write!(f, "Blob has a bad name: {}", blobname.display(),)
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use crate::test_utils::*;
+
+    #[test]
+    fn test_create() {
+        let t = dummy_context();
+        let blob = BlobObject::create(&t.ctx, "foo", b"hello").unwrap();
+        let fname = t.ctx.get_blobdir().join("foo");
+        let data = fs::read(fname).unwrap();
+        assert_eq!(data, b"hello");
+        assert_eq!(blob.as_name(), "$BLOBDIR/foo");
+        assert_eq!(blob.to_abs_path(), t.ctx.get_blobdir().join("foo"));
+    }
+
+    #[test]
+    fn test_lowercase_ext() {
+        let t = dummy_context();
+        let blob = BlobObject::create(&t.ctx, "foo.TXT", b"hello").unwrap();
+        assert_eq!(blob.as_name(), "$BLOBDIR/foo.txt");
+    }
+
+    #[test]
+    fn test_as_file_name() {
+        let t = dummy_context();
+        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
+        assert_eq!(blob.as_file_name(), "foo.txt");
+    }
+
+    #[test]
+    fn test_as_rel_path() {
+        let t = dummy_context();
+        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
+        assert_eq!(blob.as_rel_path(), Path::new("foo.txt"));
+    }
+
+    #[test]
+    fn test_suffix() {
+        let t = dummy_context();
+        let foo = BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
+        assert_eq!(foo.suffix(), Some("txt"));
+        let bar = BlobObject::create(&t.ctx, "bar", b"world").unwrap();
+        assert_eq!(bar.suffix(), None);
+    }
+
+    #[test]
+    fn test_create_dup() {
+        let t = dummy_context();
+        BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
+        let foo = t.ctx.get_blobdir().join("foo.txt");
+        assert!(foo.exists());
+        BlobObject::create(&t.ctx, "foo.txt", b"world").unwrap();
+        for dirent in fs::read_dir(t.ctx.get_blobdir()).unwrap() {
+            let fname = dirent.unwrap().file_name();
+            if fname == foo.file_name().unwrap() {
+                assert_eq!(fs::read(&foo).unwrap(), b"hello");
+            } else {
+                let name = fname.to_str().unwrap();
+                assert!(name.starts_with("foo"));
+                assert!(name.ends_with(".txt"));
+            }
+        }
+    }
+
+    #[test]
+    fn test_double_ext_preserved() {
+        let t = dummy_context();
+        BlobObject::create(&t.ctx, "foo.tar.gz", b"hello").unwrap();
+        let foo = t.ctx.get_blobdir().join("foo.tar.gz");
+        assert!(foo.exists());
+        BlobObject::create(&t.ctx, "foo.tar.gz", b"world").unwrap();
+        for dirent in fs::read_dir(t.ctx.get_blobdir()).unwrap() {
+            let fname = dirent.unwrap().file_name();
+            if fname == foo.file_name().unwrap() {
+                assert_eq!(fs::read(&foo).unwrap(), b"hello");
+            } else {
+                let name = fname.to_str().unwrap();
+                println!("{}", name);
+                assert!(name.starts_with("foo"));
+                assert!(name.ends_with(".tar.gz"));
+            }
+        }
+    }
+
+    #[test]
+    fn test_create_long_names() {
+        let t = dummy_context();
+        let s = "1".repeat(150);
+        let blob = BlobObject::create(&t.ctx, &s, b"data").unwrap();
+        let blobname = blob.as_name().split('/').last().unwrap();
+        assert!(blobname.len() < 128);
+    }
+
+    #[test]
+    fn test_create_and_copy() {
+        let t = dummy_context();
+        let src = t.dir.path().join("src");
+        fs::write(&src, b"boo").unwrap();
+        let blob = BlobObject::create_and_copy(&t.ctx, &src).unwrap();
+        assert_eq!(blob.as_name(), "$BLOBDIR/src");
+        let data = fs::read(blob.to_abs_path()).unwrap();
+        assert_eq!(data, b"boo");
+
+        let whoops = t.dir.path().join("whoops");
+        assert!(BlobObject::create_and_copy(&t.ctx, &whoops).is_err());
+        let whoops = t.ctx.get_blobdir().join("whoops");
+        assert!(!whoops.exists());
+    }
+
+    #[test]
+    fn test_create_from_path() {
+        let t = dummy_context();
+
+        let src_ext = t.dir.path().join("external");
+        fs::write(&src_ext, b"boo").unwrap();
+        let blob = BlobObject::create_from_path(&t.ctx, &src_ext).unwrap();
+        assert_eq!(blob.as_name(), "$BLOBDIR/external");
+        let data = fs::read(blob.to_abs_path()).unwrap();
+        assert_eq!(data, b"boo");
+
+        let src_int = t.ctx.get_blobdir().join("internal");
+        fs::write(&src_int, b"boo").unwrap();
+        let blob = BlobObject::create_from_path(&t.ctx, &src_int).unwrap();
+        assert_eq!(blob.as_name(), "$BLOBDIR/internal");
+        let data = fs::read(blob.to_abs_path()).unwrap();
+        assert_eq!(data, b"boo");
+    }
+    #[test]
+    fn test_create_from_name_long() {
+        let t = dummy_context();
+        let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
+        fs::write(&src_ext, b"boo").unwrap();
+        let blob = BlobObject::create_from_path(&t.ctx, &src_ext).unwrap();
+        assert_eq!(
+            blob.as_name(),
+            "$BLOBDIR/autocrypt-setup-message-4137848473.html"
+        );
+    }
+
+    #[test]
+    fn test_is_blob_name() {
+        assert!(BlobObject::is_acceptible_blob_name("foo"));
+        assert!(BlobObject::is_acceptible_blob_name("foo.txt"));
+        assert!(BlobObject::is_acceptible_blob_name("f".repeat(128)));
+        assert!(!BlobObject::is_acceptible_blob_name("foo/bar"));
+        assert!(!BlobObject::is_acceptible_blob_name("foo\\bar"));
+        assert!(!BlobObject::is_acceptible_blob_name("foo\x00bar"));
+    }
+}

src/chatlist.rs

@@ -1,10 +1,12 @@
+//! # Chat list module
+
 use crate::chat::*;
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::*;
 use crate::error::Result;
 use crate::lot::Lot;
-use crate::message::Message;
+use crate::message::{Message, MessageState, MsgId};
 use crate::stock::StockMessage;
 
 /// An object representing a single chatlist in memory.
@@ -34,7 +36,7 @@ use crate::stock::StockMessage;
 #[derive(Debug)]
 pub struct Chatlist {
     /// Stores pairs of `chat_id, message_id`
-    ids: Vec<(u32, u32)>,
+    ids: Vec<(u32, MsgId)>,
 }
 
 impl Chatlist {
@@ -86,25 +88,12 @@ impl Chatlist {
         query: Option<&str>,
         query_contact_id: Option<u32>,
     ) -> Result<Self> {
-        let mut add_archived_link_item = 0;
-
-        // select with left join and minimum:
-        // - the inner select must use `hidden` and _not_ `m.hidden`
-        //   which would refer the outer select and take a lot of time
-        // - `GROUP BY` is needed several messages may have the same timestamp
-        // - the list starts with the newest chats
-        // nb: the query currently shows messages from blocked contacts in groups.
-        // however, for normal-groups, this is okay as the message is also returned by dc_get_chat_msgs()
-        // (otherwise it would be hard to follow conversations, wa and tg do the same)
-        // for the deaddrop, however, they should really be hidden, however, _currently_ the deaddrop is not
-        // shown at all permanent in the chatlist.
+        let mut add_archived_link_item = false;
 
         let process_row = |row: &rusqlite::Row| {
-            let chat_id: i32 = row.get(0)?;
-            // TODO: verify that it is okay for this to be Null
-            let msg_id: i32 = row.get(1).unwrap_or_default();
-
-            Ok((chat_id as u32, msg_id as u32))
+            let chat_id: u32 = row.get(0)?;
+            let msg_id: MsgId = row.get(1).unwrap_or_default();
+            Ok((chat_id, msg_id))
         };
 
         let process_rows = |rows: rusqlite::MappedRows<_>| {
@@ -112,37 +101,60 @@ impl Chatlist {
                 .map_err(Into::into)
         };
 
-        // nb: the query currently shows messages from blocked contacts in groups.
-        // however, for normal-groups, this is okay as the message is also returned by dc_get_chat_msgs()
-        // (otherwise it would be hard to follow conversations, wa and tg do the same)
-        // for the deaddrop, however, they should really be hidden, however, _currently_ the deaddrop is not
+        // select with left join and minimum:
+        //
+        // - the inner select must use `hidden` and _not_ `m.hidden`
+        //   which would refer the outer select and take a lot of time
+        // - `GROUP BY` is needed several messages may have the same
+        //   timestamp
+        // - the list starts with the newest chats
+        //
+        // nb: the query currently shows messages from blocked
+        // contacts in groups.  however, for normal-groups, this is
+        // okay as the message is also returned by dc_get_chat_msgs()
+        // (otherwise it would be hard to follow conversations, wa and
+        // tg do the same) for the deaddrop, however, they should
+        // really be hidden, however, _currently_ the deaddrop is not
         // shown at all permanent in the chatlist.
-
         let mut ids = if let Some(query_contact_id) = query_contact_id {
             // show chats shared with a given contact
             context.sql.query_map(
-            "SELECT c.id, m.id FROM chats c  LEFT JOIN msgs m         \
-             ON c.id=m.chat_id        \
-             AND m.timestamp=( SELECT MAX(timestamp)   \
-             FROM msgs  WHERE chat_id=c.id    \
-             AND (hidden=0 OR (hidden=1 AND state=19))) WHERE c.id>9   \
-             AND c.blocked=0 AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)  \
-             GROUP BY c.id  ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;",
-            params![query_contact_id as i32],
-            process_row,
-            process_rows,
-        )?
+                "SELECT c.id, m.id
+                 FROM chats c
+                 LEFT JOIN msgs m
+                        ON c.id=m.chat_id
+                       AND m.timestamp=(
+                               SELECT MAX(timestamp)
+                                 FROM msgs
+                                WHERE chat_id=c.id
+                                  AND (hidden=0 OR state=?))
+                 WHERE c.id>9
+                   AND c.blocked=0
+                   AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)
+                 GROUP BY c.id
+                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                params![MessageState::OutDraft, query_contact_id as i32],
+                process_row,
+                process_rows,
+            )?
         } else if 0 != listflags & DC_GCL_ARCHIVED_ONLY {
             // show archived chats
             context.sql.query_map(
-                "SELECT c.id, m.id FROM chats c  LEFT JOIN msgs m         \
-                 ON c.id=m.chat_id        \
-                 AND m.timestamp=( SELECT MAX(timestamp)   \
-                 FROM msgs  WHERE chat_id=c.id    \
-                 AND (hidden=0 OR (hidden=1 AND state=19))) WHERE c.id>9   \
-                 AND c.blocked=0 AND c.archived=1  GROUP BY c.id  \
-                 ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;",
-                params![],
+                "SELECT c.id, m.id
+                 FROM chats c
+                 LEFT JOIN msgs m
+                        ON c.id=m.chat_id
+                       AND m.timestamp=(
+                               SELECT MAX(timestamp)
+                                 FROM msgs
+                                WHERE chat_id=c.id
+                                  AND (hidden=0 OR state=?))
+                 WHERE c.id>9
+                   AND c.blocked=0
+                   AND c.archived=1
+                 GROUP BY c.id
+                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                params![MessageState::OutDraft],
                 process_row,
                 process_rows,
             )?
@@ -152,48 +164,59 @@ impl Chatlist {
 
             let str_like_cmd = format!("%{}%", query);
             context.sql.query_map(
-                "SELECT c.id, m.id FROM chats c  LEFT JOIN msgs m         \
-                 ON c.id=m.chat_id        \
-                 AND m.timestamp=( SELECT MAX(timestamp)   \
-                 FROM msgs  WHERE chat_id=c.id    \
-                 AND (hidden=0 OR (hidden=1 AND state=19))) WHERE c.id>9   \
-                 AND c.blocked=0 AND c.name LIKE ?  \
-                 GROUP BY c.id  ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;",
-                params![str_like_cmd],
+                "SELECT c.id, m.id
+                 FROM chats c
+                 LEFT JOIN msgs m
+                        ON c.id=m.chat_id
+                       AND m.timestamp=(
+                               SELECT MAX(timestamp)
+                                 FROM msgs
+                                WHERE chat_id=c.id
+                                  AND (hidden=0 OR state=?))
+                 WHERE c.id>9
+                   AND c.blocked=0
+                   AND c.name LIKE ?
+                 GROUP BY c.id
+                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                params![MessageState::OutDraft, str_like_cmd],
                 process_row,
                 process_rows,
             )?
         } else {
             //  show normal chatlist
             let mut ids = context.sql.query_map(
-                "SELECT c.id, m.id FROM chats c  \
-                 LEFT JOIN msgs m         \
-                 ON c.id=m.chat_id        \
-                 AND m.timestamp=( SELECT MAX(timestamp)   \
-                 FROM msgs  WHERE chat_id=c.id    \
-                 AND (hidden=0 OR (hidden=1 AND state=19))) WHERE c.id>9   \
-                 AND c.blocked=0 AND c.archived=0  \
-                 GROUP BY c.id  \
-                 ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;",
-                params![],
+                "SELECT c.id, m.id
+                 FROM chats c
+                 LEFT JOIN msgs m
+                        ON c.id=m.chat_id
+                       AND m.timestamp=(
+                               SELECT MAX(timestamp)
+                                 FROM msgs
+                                WHERE chat_id=c.id
+                                  AND (hidden=0 OR state=?))
+                 WHERE c.id>9
+                   AND c.blocked=0
+                   AND c.archived=0
+                 GROUP BY c.id
+                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                params![MessageState::OutDraft],
                 process_row,
                 process_rows,
             )?;
             if 0 == listflags & DC_GCL_NO_SPECIALS {
-                let last_deaddrop_fresh_msg_id = get_last_deaddrop_fresh_msg(context);
-                if last_deaddrop_fresh_msg_id > 0 {
-                    ids.push((1, last_deaddrop_fresh_msg_id));
+                if let Some(last_deaddrop_fresh_msg_id) = get_last_deaddrop_fresh_msg(context) {
+                    ids.insert(0, (DC_CHAT_ID_DEADDROP, last_deaddrop_fresh_msg_id));
                 }
-                add_archived_link_item = 1;
+                add_archived_link_item = true;
             }
             ids
         };
 
-        if 0 != add_archived_link_item && dc_get_archived_cnt(context) > 0 {
+        if add_archived_link_item && dc_get_archived_cnt(context) > 0 {
             if ids.is_empty() && 0 != listflags & DC_GCL_ADD_ALLDONE_HINT {
-                ids.push((DC_CHAT_ID_ALLDONE_HINT, 0));
+                ids.push((DC_CHAT_ID_ALLDONE_HINT, MsgId::new(0)));
             }
-            ids.push((DC_CHAT_ID_ARCHIVED_LINK, 0));
+            ids.push((DC_CHAT_ID_ARCHIVED_LINK, MsgId::new(0)));
         }
 
         Ok(Chatlist { ids })
@@ -204,6 +227,7 @@ impl Chatlist {
         self.ids.len()
     }
 
+    /// Returns true if chatlist is empty.
     pub fn is_empty(&self) -> bool {
         self.ids.is_empty()
     }
@@ -221,12 +245,9 @@ impl Chatlist {
     /// Get a single message ID of a chatlist.
     ///
     /// To get the message object from the message ID, use dc_get_msg().
-    pub fn get_msg_id(&self, index: usize) -> u32 {
-        if index >= self.ids.len() {
-            return 0;
-        }
-
-        self.ids[index].1
+    pub fn get_msg_id(&self, index: usize) -> Result<MsgId> {
+        ensure!(index < self.ids.len(), "Chatlist index out of range");
+        Ok(self.ids[index].1)
     }
 
     /// Get a summary for a chatlist index.
@@ -258,30 +279,24 @@ impl Chatlist {
         let chat_loaded: Chat;
         let chat = if let Some(chat) = chat {
             chat
+        } else if let Ok(chat) = Chat::load_from_db(context, self.ids[index].0) {
+            chat_loaded = chat;
+            &chat_loaded
         } else {
-            if let Ok(chat) = Chat::load_from_db(context, self.ids[index].0) {
-                chat_loaded = chat;
-                &chat_loaded
-            } else {
-                return ret;
-            }
+            return ret;
         };
 
         let lastmsg_id = self.ids[index].1;
         let mut lastcontact = None;
 
-        let lastmsg = if 0 != lastmsg_id {
-            if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id) {
-                if lastmsg.from_id != 1 as libc::c_uint
-                    && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup)
-                {
-                    lastcontact = Contact::load_from_db(context, lastmsg.from_id).ok();
-                }
-
-                Some(lastmsg)
-            } else {
-                None
+        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id) {
+            if lastmsg.from_id != DC_CONTACT_ID_SELF
+                && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup)
+            {
+                lastcontact = Contact::load_from_db(context, lastmsg.from_id).ok();
             }
+
+            Some(lastmsg)
         } else {
             None
         };
@@ -299,6 +314,7 @@ impl Chatlist {
     }
 }
 
+/// Get the number of archived chats
 pub fn dc_get_archived_cnt(context: &Context) -> u32 {
     context
         .sql
@@ -310,19 +326,62 @@ pub fn dc_get_archived_cnt(context: &Context) -> u32 {
         .unwrap_or_default()
 }
 
-fn get_last_deaddrop_fresh_msg(context: &Context) -> u32 {
-    // We have an index over the state-column, this should be sufficient as there are typically
-    // only few fresh messages.
-    context
-        .sql
-        .query_get_value(
-            context,
-            "SELECT m.id  FROM msgs m  LEFT JOIN chats c ON c.id=m.chat_id  \
-             WHERE m.state=10   \
-             AND m.hidden=0    \
-             AND c.blocked=2 \
-             ORDER BY m.timestamp DESC, m.id DESC;",
-            params![],
-        )
-        .unwrap_or_default()
+fn get_last_deaddrop_fresh_msg(context: &Context) -> Option<MsgId> {
+    // We have an index over the state-column, this should be
+    // sufficient as there are typically only few fresh messages.
+    context.sql.query_get_value(
+        context,
+        concat!(
+            "SELECT m.id",
+            " FROM msgs m",
+            " LEFT JOIN chats c",
+            "        ON c.id=m.chat_id",
+            " WHERE m.state=10",
+            "   AND m.hidden=0",
+            "   AND c.blocked=2",
+            " ORDER BY m.timestamp DESC, m.id DESC;"
+        ),
+        params![],
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use crate::chat;
+    use crate::test_utils::*;
+
+    #[test]
+    fn test_try_load() {
+        let t = dummy_context();
+        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat").unwrap();
+        let chat_id2 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "b chat").unwrap();
+        let chat_id3 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "c chat").unwrap();
+
+        // check that the chatlist starts with the most recent message
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
+        assert_eq!(chats.len(), 3);
+        assert_eq!(chats.get_chat_id(0), chat_id3);
+        assert_eq!(chats.get_chat_id(1), chat_id2);
+        assert_eq!(chats.get_chat_id(2), chat_id1);
+
+        // drafts are sorted to the top
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some("hello".to_string()));
+        set_draft(&t.ctx, chat_id2, Some(&mut msg));
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
+        assert_eq!(chats.get_chat_id(0), chat_id2);
+
+        // check chatlist query and archive functionality
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("b"), None).unwrap();
+        assert_eq!(chats.len(), 1);
+
+        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None).unwrap();
+        assert_eq!(chats.len(), 0);
+
+        chat::archive(&t.ctx, chat_id1, true).ok();
+        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None).unwrap();
+        assert_eq!(chats.len(), 1);
+    }
 }

src/config.rs

@@ -1,6 +1,9 @@
+//! # Key-value configuration management
+
 use strum::{EnumProperty, IntoEnumIterator};
 use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString};
 
+use crate::blob::BlobObject;
 use crate::constants::DC_VERSION_STR;
 use crate::context::Context;
 use crate::dc_tools::*;
@@ -19,16 +22,20 @@ pub enum Config {
     MailUser,
     MailPw,
     MailPort,
+    ImapCertificateChecks,
     SendServer,
     SendUser,
     SendPw,
     SendPort,
+    SmtpCertificateChecks,
     ServerFlags,
     #[strum(props(default = "INBOX"))]
     ImapFolder,
     Displayname,
     Selfstatus,
     Selfavatar,
+    #[strum(props(default = "0"))]
+    BccSelf,
     #[strum(props(default = "1"))]
     E2eeEnabled,
     #[strum(props(default = "1"))]
@@ -41,7 +48,7 @@ pub enum Config {
     MvboxWatch,
     #[strum(props(default = "1"))]
     MvboxMove,
-    #[strum(props(default = "0"))]
+    #[strum(props(default = "0"))] // also change ShowEmails.default() on changes
     ShowEmails,
     SaveMimeHeaders,
     ConfiguredAddr,
@@ -50,10 +57,12 @@ pub enum Config {
     ConfiguredMailPw,
     ConfiguredMailPort,
     ConfiguredMailSecurity,
+    ConfiguredImapCertificateChecks,
     ConfiguredSendServer,
     ConfiguredSendUser,
     ConfiguredSendPw,
     ConfiguredSendPort,
+    ConfiguredSmtpCertificateChecks,
     ConfiguredServerFlags,
     ConfiguredSendSecurity,
     ConfiguredE2EEEnabled,
@@ -72,13 +81,13 @@ impl Context {
     pub fn get_config(&self, key: Config) -> Option<String> {
         let value = match key {
             Config::Selfavatar => {
-                let rel_path = self.sql.get_config(self, key);
-                rel_path.map(|p| dc_get_abs_path(self, &p).to_str().unwrap().to_string())
+                let rel_path = self.sql.get_raw_config(self, key);
+                rel_path.map(|p| dc_get_abs_path(self, &p).to_string_lossy().into_owned())
             }
             Config::SysVersion => Some((&*DC_VERSION_STR).clone()),
             Config::SysMsgsizeMaxRecommended => Some(format!("{}", 24 * 1024 * 1024 / 4 * 3)),
             Config::SysConfigKeys => Some(get_config_keys_string()),
-            _ => self.sql.get_config(self, key),
+            _ => self.sql.get_raw_config(self, key),
         };
 
         if value.is_some() {
@@ -92,27 +101,36 @@ impl Context {
         }
     }
 
+    pub fn get_config_int(&self, key: Config) -> i32 {
+        self.get_config(key)
+            .and_then(|s| s.parse().ok())
+            .unwrap_or_default()
+    }
+
+    pub fn get_config_bool(&self, key: Config) -> bool {
+        self.get_config_int(key) != 0
+    }
+
     /// Set the given config key.
     /// If `None` is passed as a value the value is cleared and set to the default if there is one.
     pub fn set_config(&self, key: Config, value: Option<&str>) -> Result<(), Error> {
         match key {
             Config::Selfavatar if value.is_some() => {
-                let rel_path = std::fs::canonicalize(value.unwrap())?;
-                self.sql
-                    .set_config(self, key, Some(&rel_path.to_string_lossy()))
+                let blob = BlobObject::create_from_path(&self, value.unwrap())?;
+                self.sql.set_raw_config(self, key, Some(blob.as_name()))
             }
             Config::InboxWatch => {
-                let ret = self.sql.set_config(self, key, value);
-                interrupt_imap_idle(self);
+                let ret = self.sql.set_raw_config(self, key, value);
+                interrupt_inbox_idle(self, true);
                 ret
             }
             Config::SentboxWatch => {
-                let ret = self.sql.set_config(self, key, value);
+                let ret = self.sql.set_raw_config(self, key, value);
                 interrupt_sentbox_idle(self);
                 ret
             }
             Config::MvboxWatch => {
-                let ret = self.sql.set_config(self, key, value);
+                let ret = self.sql.set_raw_config(self, key, value);
                 interrupt_mvbox_idle(self);
                 ret
             }
@@ -124,9 +142,9 @@ impl Context {
                     value
                 };
 
-                self.sql.set_config(self, key, val)
+                self.sql.set_raw_config(self, key, val)
             }
-            _ => self.sql.set_config(self, key, value),
+            _ => self.sql.set_raw_config(self, key, value),
         }
     }
 }
@@ -149,6 +167,8 @@ mod tests {
     use std::str::FromStr;
     use std::string::ToString;
 
+    use crate::test_utils::*;
+
     #[test]
     fn test_to_string() {
         assert_eq!(Config::MailServer.to_string(), "mail_server");
@@ -165,4 +185,32 @@ mod tests {
     fn test_default_prop() {
         assert_eq!(Config::ImapFolder.get_str("default"), Some("INBOX"));
     }
+
+    #[test]
+    fn test_selfavatar() -> failure::Fallible<()> {
+        let t = dummy_context();
+        let avatar_src = t.dir.path().join("avatar.jpg");
+        std::fs::write(&avatar_src, b"avatar")?;
+        let avatar_blob = t.ctx.get_blobdir().join("avatar.jpg");
+        assert!(!avatar_blob.exists());
+        t.ctx
+            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))?;
+        assert!(avatar_blob.exists());
+        assert_eq!(std::fs::read(&avatar_blob)?, b"avatar");
+        let avatar_cfg = t.ctx.get_config(Config::Selfavatar);
+        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
+        Ok(())
+    }
+
+    #[test]
+    fn test_selfavatar_in_blobdir() -> failure::Fallible<()> {
+        let t = dummy_context();
+        let avatar_src = t.ctx.get_blobdir().join("avatar.jpg");
+        std::fs::write(&avatar_src, b"avatar")?;
+        t.ctx
+            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))?;
+        let avatar_cfg = t.ctx.get_config(Config::Selfavatar);
+        assert_eq!(avatar_cfg, avatar_src.to_str().map(|s| s.to_string()));
+        Ok(())
+    }
 }

src/configure/auto_mozilla.rs

@@ -1,10 +1,10 @@
-use libc::free;
+//! Thunderbird's Autoconfiguration implementation
 use quick_xml;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
 use crate::constants::*;
 use crate::context::Context;
-use crate::dc_tools::*;
+use crate::error::Error;
 use crate::login_param::LoginParam;
 
 use super::read_autoconf_file;
@@ -12,52 +12,56 @@ use super::read_autoconf_file;
  * Thunderbird's Autoconfigure
  ******************************************************************************/
 /* documentation: https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration */
-#[repr(C)]
-struct moz_autoconfigure_t<'a> {
-    pub in_0: &'a LoginParam,
+struct MozAutoconfigure<'a> {
+    pub in_emailaddr: &'a str,
     pub in_emaildomain: &'a str,
     pub in_emaillocalpart: &'a str,
     pub out: LoginParam,
-    pub out_imap_set: libc::c_int,
-    pub out_smtp_set: libc::c_int,
-    pub tag_server: libc::c_int,
-    pub tag_config: libc::c_int,
+    pub out_imap_set: bool,
+    pub out_smtp_set: bool,
+    pub tag_server: MozServer,
+    pub tag_config: MozConfigTag,
 }
 
-pub unsafe fn moz_autoconfigure(
-    context: &Context,
-    url: &str,
-    param_in: &LoginParam,
-) -> Option<LoginParam> {
-    let xml_raw = read_autoconf_file(context, url);
-    if xml_raw.is_null() {
-        return None;
-    }
+#[derive(PartialEq)]
+enum MozServer {
+    Undefined,
+    Imap,
+    Smtp,
+}
 
-    // Split address into local part and domain part.
-    let p = param_in.addr.find("@");
-    if p.is_none() {
-        free(xml_raw as *mut libc::c_void);
-        return None;
-    }
-    let (in_emaillocalpart, in_emaildomain) = param_in.addr.split_at(p.unwrap());
-    let in_emaildomain = &in_emaildomain[1..];
+enum MozConfigTag {
+    Undefined,
+    Hostname,
+    Port,
+    Sockettype,
+    Username,
+}
 
-    let mut reader = quick_xml::Reader::from_str(as_str(xml_raw));
+pub fn moz_parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result<LoginParam, Error> {
+    let mut reader = quick_xml::Reader::from_str(xml_raw);
     reader.trim_text(true);
 
-    let mut buf = Vec::new();
+    // Split address into local part and domain part.
+    let p = match in_emailaddr.find('@') {
+        Some(i) => i,
+        None => bail!("Email address {} does not contain @", in_emailaddr),
+    };
+    let (in_emaillocalpart, in_emaildomain) = in_emailaddr.split_at(p);
+    let in_emaildomain = &in_emaildomain[1..];
 
-    let mut moz_ac = moz_autoconfigure_t {
-        in_0: param_in,
+    let mut moz_ac = MozAutoconfigure {
+        in_emailaddr,
         in_emaildomain,
         in_emaillocalpart,
         out: LoginParam::new(),
-        out_imap_set: 0,
-        out_smtp_set: 0,
-        tag_server: 0,
-        tag_config: 0,
+        out_imap_set: false,
+        out_smtp_set: false,
+        tag_server: MozServer::Undefined,
+        tag_config: MozConfigTag::Undefined,
     };
+
+    let mut buf = Vec::new();
     loop {
         match reader.read_event(&mut buf) {
             Ok(quick_xml::events::Event::Start(ref e)) => {
@@ -68,8 +72,7 @@ pub unsafe fn moz_autoconfigure(
                 moz_autoconfigure_text_cb(e, &mut moz_ac, &reader)
             }
             Err(e) => {
-                error!(
-                    context,
+                bail!(
                     "Configure xml: Error at position {}: {:?}",
                     reader.buffer_position(),
                     e
@@ -87,23 +90,36 @@ pub unsafe fn moz_autoconfigure(
         || moz_ac.out.send_port == 0
     {
         let r = moz_ac.out.to_string();
-        warn!(context, "Bad or incomplete autoconfig: {}", r,);
-        free(xml_raw as *mut libc::c_void);
-        return None;
+        bail!("Bad or incomplete autoconfig: {}", r,);
     }
 
-    free(xml_raw as *mut libc::c_void);
-    Some(moz_ac.out)
+    Ok(moz_ac.out)
+}
+
+pub fn moz_autoconfigure(
+    context: &Context,
+    url: &str,
+    param_in: &LoginParam,
+) -> Option<LoginParam> {
+    let xml_raw = read_autoconf_file(context, url)?;
+
+    match moz_parse_xml(&param_in.addr, &xml_raw) {
+        Err(err) => {
+            warn!(context, "{}", err);
+            None
+        }
+        Ok(lp) => Some(lp),
+    }
 }
 
 fn moz_autoconfigure_text_cb<B: std::io::BufRead>(
     event: &BytesText,
-    moz_ac: &mut moz_autoconfigure_t,
+    moz_ac: &mut MozAutoconfigure,
     reader: &quick_xml::Reader<B>,
 ) {
     let val = event.unescape_and_decode(reader).unwrap_or_default();
 
-    let addr = &moz_ac.in_0.addr;
+    let addr = moz_ac.in_emailaddr;
     let email_local = moz_ac.in_emaillocalpart;
     let email_domain = moz_ac.in_emaildomain;
 
@@ -113,12 +129,12 @@ fn moz_autoconfigure_text_cb<B: std::io::BufRead>(
         .replace("%EMAILLOCALPART%", email_local)
         .replace("%EMAILDOMAIN%", email_domain);
 
-    if moz_ac.tag_server == 1 {
-        match moz_ac.tag_config {
-            10 => moz_ac.out.mail_server = val,
-            11 => moz_ac.out.mail_port = val.parse().unwrap_or_default(),
-            12 => moz_ac.out.mail_user = val,
-            13 => {
+    match moz_ac.tag_server {
+        MozServer::Imap => match moz_ac.tag_config {
+            MozConfigTag::Hostname => moz_ac.out.mail_server = val,
+            MozConfigTag::Port => moz_ac.out.mail_port = val.parse().unwrap_or_default(),
+            MozConfigTag::Username => moz_ac.out.mail_user = val,
+            MozConfigTag::Sockettype => {
                 let val_lower = val.to_lowercase();
                 if val_lower == "ssl" {
                     moz_ac.out.server_flags |= DC_LP_IMAP_SOCKET_SSL as i32
@@ -131,13 +147,12 @@ fn moz_autoconfigure_text_cb<B: std::io::BufRead>(
                 }
             }
             _ => {}
-        }
-    } else if moz_ac.tag_server == 2 {
-        match moz_ac.tag_config {
-            10 => moz_ac.out.send_server = val,
-            11 => moz_ac.out.send_port = val.parse().unwrap_or_default(),
-            12 => moz_ac.out.send_user = val,
-            13 => {
+        },
+        MozServer::Smtp => match moz_ac.tag_config {
+            MozConfigTag::Hostname => moz_ac.out.send_server = val,
+            MozConfigTag::Port => moz_ac.out.send_port = val.parse().unwrap_or_default(),
+            MozConfigTag::Username => moz_ac.out.send_user = val,
+            MozConfigTag::Sockettype => {
                 let val_lower = val.to_lowercase();
                 if val_lower == "ssl" {
                     moz_ac.out.server_flags |= DC_LP_SMTP_SOCKET_SSL as i32
@@ -150,29 +165,34 @@ fn moz_autoconfigure_text_cb<B: std::io::BufRead>(
                 }
             }
             _ => {}
-        }
+        },
+        MozServer::Undefined => {}
     }
 }
 
-fn moz_autoconfigure_endtag_cb(event: &BytesEnd, moz_ac: &mut moz_autoconfigure_t) {
+fn moz_autoconfigure_endtag_cb(event: &BytesEnd, moz_ac: &mut MozAutoconfigure) {
     let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
 
     if tag == "incomingserver" {
-        moz_ac.tag_server = 0;
-        moz_ac.tag_config = 0;
-        moz_ac.out_imap_set = 1;
+        if moz_ac.tag_server == MozServer::Imap {
+            moz_ac.out_imap_set = true;
+        }
+        moz_ac.tag_server = MozServer::Undefined;
+        moz_ac.tag_config = MozConfigTag::Undefined;
     } else if tag == "outgoingserver" {
-        moz_ac.tag_server = 0;
-        moz_ac.tag_config = 0;
-        moz_ac.out_smtp_set = 1;
+        if moz_ac.tag_server == MozServer::Smtp {
+            moz_ac.out_smtp_set = true;
+        }
+        moz_ac.tag_server = MozServer::Undefined;
+        moz_ac.tag_config = MozConfigTag::Undefined;
     } else {
-        moz_ac.tag_config = 0;
+        moz_ac.tag_config = MozConfigTag::Undefined;
     }
 }
 
 fn moz_autoconfigure_starttag_cb<B: std::io::BufRead>(
     event: &BytesStart,
-    moz_ac: &mut moz_autoconfigure_t,
+    moz_ac: &mut MozAutoconfigure,
     reader: &quick_xml::Reader<B>,
 ) {
     let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
@@ -189,25 +209,115 @@ fn moz_autoconfigure_starttag_cb<B: std::io::BufRead>(
                 .unwrap_or_default()
                 .to_lowercase();
 
-            if typ == "imap" && moz_ac.out_imap_set == 0 {
-                1
+            if typ == "imap" && !moz_ac.out_imap_set {
+                MozServer::Imap
             } else {
-                0
+                MozServer::Undefined
             }
         } else {
-            0
+            MozServer::Undefined
         };
-        moz_ac.tag_config = 0;
+        moz_ac.tag_config = MozConfigTag::Undefined;
     } else if tag == "outgoingserver" {
-        moz_ac.tag_server = if moz_ac.out_smtp_set == 0 { 2 } else { 0 };
-        moz_ac.tag_config = 0;
+        moz_ac.tag_server = if !moz_ac.out_smtp_set {
+            MozServer::Smtp
+        } else {
+            MozServer::Undefined
+        };
+        moz_ac.tag_config = MozConfigTag::Undefined;
     } else if tag == "hostname" {
-        moz_ac.tag_config = 10;
+        moz_ac.tag_config = MozConfigTag::Hostname;
     } else if tag == "port" {
-        moz_ac.tag_config = 11;
+        moz_ac.tag_config = MozConfigTag::Port;
     } else if tag == "sockettype" {
-        moz_ac.tag_config = 13;
+        moz_ac.tag_config = MozConfigTag::Sockettype;
     } else if tag == "username" {
-        moz_ac.tag_config = 12;
+        moz_ac.tag_config = MozConfigTag::Username;
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_outlook_autoconfig() {
+        // Copied from https://autoconfig.thunderbird.net/v1.1/outlook.com on 2019-10-11
+        let xml_raw =
+"<clientConfig version=\"1.1\">
+  <emailProvider id=\"outlook.com\">
+    <domain>hotmail.com</domain>
+    <domain>hotmail.co.uk</domain>
+    <domain>hotmail.co.jp</domain>
+    <domain>hotmail.com.br</domain>
+    <domain>hotmail.de</domain>
+    <domain>hotmail.fr</domain>
+    <domain>hotmail.it</domain>
+    <domain>hotmail.es</domain>
+    <domain>live.com</domain>
+    <domain>live.co.uk</domain>
+    <domain>live.co.jp</domain>
+    <domain>live.de</domain>
+    <domain>live.fr</domain>
+    <domain>live.it</domain>
+    <domain>live.jp</domain>
+    <domain>msn.com</domain>
+    <domain>outlook.com</domain>
+    <displayName>Outlook.com (Microsoft)</displayName>
+    <displayShortName>Outlook</displayShortName>
+    <incomingServer type=\"exchange\">
+      <hostname>outlook.office365.com</hostname>
+      <port>443</port>
+      <username>%EMAILADDRESS%</username>
+      <socketType>SSL</socketType>
+      <authentication>OAuth2</authentication>
+      <owaURL>https://outlook.office365.com/owa/</owaURL>
+      <ewsURL>https://outlook.office365.com/ews/exchange.asmx</ewsURL>
+      <useGlobalPreferredServer>true</useGlobalPreferredServer>
+    </incomingServer>
+    <incomingServer type=\"imap\">
+      <hostname>outlook.office365.com</hostname>
+      <port>993</port>
+      <socketType>SSL</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </incomingServer>
+    <incomingServer type=\"pop3\">
+      <hostname>outlook.office365.com</hostname>
+      <port>995</port>
+      <socketType>SSL</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+      <pop3>
+        <leaveMessagesOnServer>true</leaveMessagesOnServer>
+        <!-- Outlook.com docs specifically mention that POP3 deletes have effect on the main inbox on webmail and IMAP -->
+      </pop3>
+    </incomingServer>
+    <outgoingServer type=\"smtp\">
+      <hostname>smtp.office365.com</hostname>
+      <port>587</port>
+      <socketType>STARTTLS</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </outgoingServer>
+    <documentation url=\"http://windows.microsoft.com/en-US/windows/outlook/send-receive-from-app\">
+      <descr lang=\"en\">Set up an email app with Outlook.com</descr>
+    </documentation>
+  </emailProvider>
+  <webMail>
+    <loginPage url=\"https://www.outlook.com/\"/>
+    <loginPageInfo url=\"https://www.outlook.com/\">
+      <username>%EMAILADDRESS%</username>
+      <usernameField id=\"i0116\" name=\"login\"/>
+      <passwordField id=\"i0118\" name=\"passwd\"/>
+      <loginButton id=\"idSIButton9\" name=\"SI\"/>
+    </loginPageInfo>
+  </webMail>
+</clientConfig>";
+        let res = moz_parse_xml("example@outlook.com", xml_raw).expect("XML parsing failed");
+        assert_eq!(res.mail_server, "outlook.office365.com");
+        assert_eq!(res.mail_port, 993);
+        assert_eq!(res.send_server, "smtp.office365.com");
+        assert_eq!(res.send_port, 587);
     }
 }

src/configure/auto_outlook.rs

@@ -1,214 +1,249 @@
-use std::ptr;
-
-use libc::free;
 use quick_xml;
-use quick_xml::events::{BytesEnd, BytesStart, BytesText};
+use quick_xml::events::BytesEnd;
 
 use crate::constants::*;
 use crate::context::Context;
-use crate::dc_tools::*;
+use crate::error::Error;
 use crate::login_param::LoginParam;
 
 use super::read_autoconf_file;
-/* ******************************************************************************
- * Outlook's Autodiscover
- ******************************************************************************/
-#[repr(C)]
-struct outlk_autodiscover_t<'a> {
-    pub in_0: &'a LoginParam,
+
+/// Outlook's Autodiscover
+struct OutlookAutodiscover {
     pub out: LoginParam,
-    pub out_imap_set: libc::c_int,
-    pub out_smtp_set: libc::c_int,
-    pub tag_config: libc::c_int,
-    pub config: [*mut libc::c_char; 6],
-    pub redirect: *mut libc::c_char,
+    pub out_imap_set: bool,
+    pub out_smtp_set: bool,
+    pub config_type: Option<String>,
+    pub config_server: String,
+    pub config_port: i32,
+    pub config_ssl: String,
+    pub config_redirecturl: Option<String>,
 }
 
-pub unsafe fn outlk_autodiscover(
-    context: &Context,
-    url__: &str,
-    param_in: &LoginParam,
-) -> Option<LoginParam> {
-    let mut xml_raw: *mut libc::c_char = ptr::null_mut();
-    let mut url = url__.strdup();
-    let mut outlk_ad = outlk_autodiscover_t {
-        in_0: param_in,
+enum ParsingResult {
+    LoginParam(LoginParam),
+    RedirectUrl(String),
+}
+
+fn outlk_parse_xml(xml_raw: &str) -> Result<ParsingResult, Error> {
+    let mut outlk_ad = OutlookAutodiscover {
         out: LoginParam::new(),
-        out_imap_set: 0,
-        out_smtp_set: 0,
-        tag_config: 0,
-        config: [ptr::null_mut(); 6],
-        redirect: ptr::null_mut(),
+        out_imap_set: false,
+        out_smtp_set: false,
+        config_type: None,
+        config_server: String::new(),
+        config_port: 0,
+        config_ssl: String::new(),
+        config_redirecturl: None,
     };
-    let ok_to_continue;
-    let mut i = 0;
+
+    let mut reader = quick_xml::Reader::from_str(&xml_raw);
+    reader.trim_text(true);
+
+    let mut buf = Vec::new();
+
+    let mut current_tag: Option<String> = None;
+
     loop {
-        if !(i < 10) {
-            ok_to_continue = true;
-            break;
-        }
-        libc::memset(
-            &mut outlk_ad as *mut outlk_autodiscover_t as *mut libc::c_void,
-            0,
-            ::std::mem::size_of::<outlk_autodiscover_t>(),
-        );
-        xml_raw = read_autoconf_file(context, as_str(url));
-        if xml_raw.is_null() {
-            ok_to_continue = false;
-            break;
-        }
+        match reader.read_event(&mut buf) {
+            Ok(quick_xml::events::Event::Start(ref e)) => {
+                let tag = String::from_utf8_lossy(e.name()).trim().to_lowercase();
 
-        let mut reader = quick_xml::Reader::from_str(as_str(xml_raw));
-        reader.trim_text(true);
+                if tag == "protocol" {
+                    outlk_ad.config_type = None;
+                    outlk_ad.config_server = String::new();
+                    outlk_ad.config_port = 0;
+                    outlk_ad.config_ssl = String::new();
+                    outlk_ad.config_redirecturl = None;
 
-        let mut buf = Vec::new();
-
-        loop {
-            match reader.read_event(&mut buf) {
-                Ok(quick_xml::events::Event::Start(ref e)) => {
-                    outlk_autodiscover_starttag_cb(e, &mut outlk_ad)
+                    current_tag = None;
+                } else {
+                    current_tag = Some(tag);
                 }
-                Ok(quick_xml::events::Event::End(ref e)) => {
-                    outlk_autodiscover_endtag_cb(e, &mut outlk_ad)
-                }
-                Ok(quick_xml::events::Event::Text(ref e)) => {
-                    outlk_autodiscover_text_cb(e, &mut outlk_ad, &reader)
-                }
-                Err(e) => {
-                    error!(
-                        context,
-                        "Configure xml: Error at position {}: {:?}",
-                        reader.buffer_position(),
-                        e
-                    );
-                }
-                Ok(quick_xml::events::Event::Eof) => break,
-                _ => (),
             }
-            buf.clear();
-        }
+            Ok(quick_xml::events::Event::End(ref e)) => {
+                outlk_autodiscover_endtag_cb(e, &mut outlk_ad);
+                current_tag = None;
+            }
+            Ok(quick_xml::events::Event::Text(ref e)) => {
+                let val = e.unescape_and_decode(&reader).unwrap_or_default();
 
-        if !(!outlk_ad.config[5].is_null()
-            && 0 != *outlk_ad.config[5usize].offset(0isize) as libc::c_int)
-        {
-            ok_to_continue = true;
-            break;
+                if let Some(ref tag) = current_tag {
+                    match tag.as_str() {
+                        "type" => {
+                            outlk_ad.config_type = Some(val.trim().to_lowercase().to_string())
+                        }
+                        "server" => outlk_ad.config_server = val.trim().to_string(),
+                        "port" => outlk_ad.config_port = val.trim().parse().unwrap_or_default(),
+                        "ssl" => outlk_ad.config_ssl = val.trim().to_string(),
+                        "redirecturl" => outlk_ad.config_redirecturl = Some(val.trim().to_string()),
+                        _ => {}
+                    };
+                }
+            }
+            Err(e) => {
+                bail!(
+                    "Configure xml: Error at position {}: {:?}",
+                    reader.buffer_position(),
+                    e
+                );
+            }
+            Ok(quick_xml::events::Event::Eof) => break,
+            _ => (),
         }
-        free(url as *mut libc::c_void);
-        url = dc_strdup(outlk_ad.config[5usize]);
-
-        outlk_clean_config(&mut outlk_ad);
-        free(xml_raw as *mut libc::c_void);
-        xml_raw = ptr::null_mut();
-        i += 1;
+        buf.clear();
     }
 
-    if ok_to_continue {
+    // XML redirect via redirecturl
+    if outlk_ad.config_redirecturl.is_none()
+        || outlk_ad.config_redirecturl.as_ref().unwrap().is_empty()
+    {
         if outlk_ad.out.mail_server.is_empty()
             || outlk_ad.out.mail_port == 0
             || outlk_ad.out.send_server.is_empty()
             || outlk_ad.out.send_port == 0
         {
             let r = outlk_ad.out.to_string();
-            warn!(context, "Bad or incomplete autoconfig: {}", r,);
-            free(url as *mut libc::c_void);
-            free(xml_raw as *mut libc::c_void);
-            outlk_clean_config(&mut outlk_ad);
+            bail!("Bad or incomplete autoconfig: {}", r,);
+        }
+        Ok(ParsingResult::LoginParam(outlk_ad.out))
+    } else {
+        Ok(ParsingResult::RedirectUrl(
+            outlk_ad.config_redirecturl.unwrap(),
+        ))
+    }
+}
 
+pub fn outlk_autodiscover(
+    context: &Context,
+    url: &str,
+    _param_in: &LoginParam,
+) -> Option<LoginParam> {
+    let mut url = url.to_string();
+    /* Follow up to 10 xml-redirects (http-redirects are followed in read_autoconf_file() */
+    for _i in 0..10 {
+        if let Some(xml_raw) = read_autoconf_file(context, &url) {
+            match outlk_parse_xml(&xml_raw) {
+                Err(err) => {
+                    warn!(context, "{}", err);
+                    return None;
+                }
+                Ok(res) => match res {
+                    ParsingResult::RedirectUrl(redirect_url) => url = redirect_url,
+                    ParsingResult::LoginParam(login_param) => return Some(login_param),
+                },
+            }
+        } else {
             return None;
         }
     }
-    free(url as *mut libc::c_void);
-    free(xml_raw as *mut libc::c_void);
-    outlk_clean_config(&mut outlk_ad);
-    Some(outlk_ad.out)
+    None
 }
 
-unsafe fn outlk_clean_config(mut outlk_ad: *mut outlk_autodiscover_t) {
-    for i in 0..6 {
-        free((*outlk_ad).config[i] as *mut libc::c_void);
-        (*outlk_ad).config[i] = ptr::null_mut();
-    }
-}
-
-fn outlk_autodiscover_text_cb<B: std::io::BufRead>(
-    event: &BytesText,
-    outlk_ad: &mut outlk_autodiscover_t,
-    reader: &quick_xml::Reader<B>,
-) {
-    let val = event.unescape_and_decode(reader).unwrap_or_default();
-
-    unsafe {
-        free(outlk_ad.config[outlk_ad.tag_config as usize].cast());
-        outlk_ad.config[outlk_ad.tag_config as usize] = val.trim().strdup();
-    }
-}
-
-unsafe fn outlk_autodiscover_endtag_cb(event: &BytesEnd, outlk_ad: &mut outlk_autodiscover_t) {
+fn outlk_autodiscover_endtag_cb(event: &BytesEnd, outlk_ad: &mut OutlookAutodiscover) {
     let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
 
     if tag == "protocol" {
-        if !outlk_ad.config[1].is_null() {
-            let port = dc_atoi_null_is_0(outlk_ad.config[3]);
-            let ssl_on = (!outlk_ad.config[4].is_null()
-                && strcasecmp(
-                    outlk_ad.config[4],
-                    b"on\x00" as *const u8 as *const libc::c_char,
-                ) == 0) as libc::c_int;
-            let ssl_off = (!outlk_ad.config[4].is_null()
-                && strcasecmp(
-                    outlk_ad.config[4],
-                    b"off\x00" as *const u8 as *const libc::c_char,
-                ) == 0) as libc::c_int;
-            if strcasecmp(
-                outlk_ad.config[1],
-                b"imap\x00" as *const u8 as *const libc::c_char,
-            ) == 0
-                && outlk_ad.out_imap_set == 0
-            {
-                outlk_ad.out.mail_server = to_string(outlk_ad.config[2]);
+        if let Some(type_) = &outlk_ad.config_type {
+            let port = outlk_ad.config_port;
+            let ssl_on = outlk_ad.config_ssl == "on";
+            let ssl_off = outlk_ad.config_ssl == "off";
+            if type_ == "imap" && !outlk_ad.out_imap_set {
+                outlk_ad.out.mail_server =
+                    std::mem::replace(&mut outlk_ad.config_server, String::new());
                 outlk_ad.out.mail_port = port;
-                if 0 != ssl_on {
+                if ssl_on {
                     outlk_ad.out.server_flags |= DC_LP_IMAP_SOCKET_SSL as i32
-                } else if 0 != ssl_off {
+                } else if ssl_off {
                     outlk_ad.out.server_flags |= DC_LP_IMAP_SOCKET_PLAIN as i32
                 }
-                outlk_ad.out_imap_set = 1
-            } else if strcasecmp(
-                outlk_ad.config[1usize],
-                b"smtp\x00" as *const u8 as *const libc::c_char,
-            ) == 0
-                && outlk_ad.out_smtp_set == 0
-            {
-                outlk_ad.out.send_server = to_string(outlk_ad.config[2]);
-                outlk_ad.out.send_port = port;
-                if 0 != ssl_on {
+                outlk_ad.out_imap_set = true
+            } else if type_ == "smtp" && !outlk_ad.out_smtp_set {
+                outlk_ad.out.send_server =
+                    std::mem::replace(&mut outlk_ad.config_server, String::new());
+                outlk_ad.out.send_port = outlk_ad.config_port;
+                if ssl_on {
                     outlk_ad.out.server_flags |= DC_LP_SMTP_SOCKET_SSL as i32
-                } else if 0 != ssl_off {
+                } else if ssl_off {
                     outlk_ad.out.server_flags |= DC_LP_SMTP_SOCKET_PLAIN as i32
                 }
-                outlk_ad.out_smtp_set = 1
+                outlk_ad.out_smtp_set = true
             }
         }
-        outlk_clean_config(outlk_ad);
     }
-    outlk_ad.tag_config = 0;
 }
 
-fn outlk_autodiscover_starttag_cb(event: &BytesStart, outlk_ad: &mut outlk_autodiscover_t) {
-    let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
+#[cfg(test)]
+mod tests {
+    use super::*;
 
-    if tag == "protocol" {
-        unsafe { outlk_clean_config(outlk_ad) };
-    } else if tag == "type" {
-        outlk_ad.tag_config = 1
-    } else if tag == "server" {
-        outlk_ad.tag_config = 2
-    } else if tag == "port" {
-        outlk_ad.tag_config = 3
-    } else if tag == "ssl" {
-        outlk_ad.tag_config = 4
-    } else if tag == "redirecturl" {
-        outlk_ad.tag_config = 5
-    };
+    #[test]
+    fn test_parse_redirect() {
+        let res = outlk_parse_xml("
+<?xml version=\"1.0\" encoding=\"utf-8\"?>
+  <Autodiscover xmlns=\"http://schemas.microsoft.com/exchange/autodiscover/responseschema/2006\">
+    <Response xmlns=\"http://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a\">
+      <Account>
+        <AccountType>email</AccountType>
+        <Action>redirectUrl</Action>
+        <RedirectUrl>https://mail.example.com/autodiscover/autodiscover.xml</RedirectUrl>
+      </Account>
+    </Response>
+  </Autodiscover>
+ ").expect("XML is not parsed successfully");
+        match res {
+            ParsingResult::LoginParam(_lp) => {
+                panic!("redirecturl is not found");
+            }
+            ParsingResult::RedirectUrl(url) => {
+                assert_eq!(
+                    url,
+                    "https://mail.example.com/autodiscover/autodiscover.xml"
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn test_parse_loginparam() {
+        let res = outlk_parse_xml(
+            "\
+<?xml version=\"1.0\" encoding=\"utf-8\"?>
+<Autodiscover xmlns=\"http://schemas.microsoft.com/exchange/autodiscover/responseschema/2006\">
+  <Response xmlns=\"http://schemas.microsoft.com/exchange/autodiscover/outlook/responseschema/2006a\">
+    <Account>
+      <AccountType>email</AccountType>
+      <Action>settings</Action>
+      <Protocol>
+        <Type>IMAP</Type>
+        <Server>example.com</Server>
+        <Port>993</Port>
+        <SSL>on</SSL>
+        <AuthRequired>on</AuthRequired>
+      </Protocol>
+      <Protocol>
+        <Type>SMTP</Type>
+        <Server>smtp.example.com</Server>
+        <Port>25</Port>
+        <SSL>off</SSL>
+        <AuthRequired>on</AuthRequired>
+      </Protocol>
+    </Account>
+  </Response>
+</Autodiscover>",
+        )
+        .expect("XML is not parsed successfully");
+
+        match res {
+            ParsingResult::LoginParam(lp) => {
+                assert_eq!(lp.mail_server, "example.com");
+                assert_eq!(lp.mail_port, 993);
+                assert_eq!(lp.send_server, "smtp.example.com");
+                assert_eq!(lp.send_port, 25);
+            }
+            ParsingResult::RedirectUrl(_) => {
+                panic!("RedirectUrl is not expected");
+            }
+        }
+    }
 }

src/constants.rs

@@ -1,4 +1,4 @@
-//! Constants
+//! # Constants
 #![allow(non_camel_case_types, dead_code)]
 
 use deltachat_derive::*;
@@ -8,24 +8,8 @@ lazy_static! {
     pub static ref DC_VERSION_STR: String = env!("CARGO_PKG_VERSION").to_string();
 }
 
-#[repr(u8)]
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql)]
-pub enum MoveState {
-    Undefined = 0,
-    Pending = 1,
-    Stay = 2,
-    Moving = 3,
-}
-
-impl Default for MoveState {
-    fn default() -> Self {
-        MoveState::Undefined
-    }
-}
-
 // some defaults
 const DC_E2EE_DEFAULT_ENABLED: i32 = 1;
-pub const DC_MDNS_DEFAULT_ENABLED: i32 = 1;
 const DC_INBOX_WATCH_DEFAULT: i32 = 1;
 const DC_SENTBOX_WATCH_DEFAULT: i32 = 1;
 const DC_MVBOX_WATCH_DEFAULT: i32 = 1;
@@ -45,6 +29,20 @@ impl Default for Blocked {
     }
 }
 
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[repr(u8)]
+pub enum ShowEmails {
+    Off = 0,
+    AcceptedContacts = 1,
+    All = 2,
+}
+
+impl Default for ShowEmails {
+    fn default() -> Self {
+        ShowEmails::Off // also change Config.ShowEmails props(default) on changes
+    }
+}
+
 pub const DC_IMAP_SEEN: u32 = 0x1;
 
 pub const DC_HANDSHAKE_CONTINUE_NORMAL_PROCESSING: i32 = 0x01;
@@ -55,7 +53,7 @@ pub const DC_GCL_ARCHIVED_ONLY: usize = 0x01;
 pub const DC_GCL_NO_SPECIALS: usize = 0x02;
 pub const DC_GCL_ADD_ALLDONE_HINT: usize = 0x04;
 
-const DC_GCM_ADDDAYMARKER: usize = 0x01;
+pub const DC_GCM_ADDDAYMARKER: u32 = 0x01;
 
 pub const DC_GCL_VERIFIED_ONLY: usize = 0x01;
 pub const DC_GCL_ADD_SELF: usize = 0x02;
@@ -64,15 +62,6 @@ pub const DC_GCL_ADD_SELF: usize = 0x02;
 pub(crate) const DC_FP_NO_AUTOCRYPT_HEADER: i32 = 2;
 pub(crate) const DC_FP_ADD_AUTOCRYPT_HEADER: i32 = 1;
 
-/// param1 is a directory where the keys are written to
-pub const DC_IMEX_EXPORT_SELF_KEYS: i32 = 1;
-/// param1 is a directory where the keys are searched in and read from
-pub const DC_IMEX_IMPORT_SELF_KEYS: i32 = 2;
-/// param1 is a directory where the backup is written to
-pub const DC_IMEX_EXPORT_BACKUP: i32 = 11;
-/// param1 is the file with the backup to import
-pub const DC_IMEX_IMPORT_BACKUP: i32 = 12;
-
 /// virtual chat showing all messages belonging to chats flagged with chats.blocked=2
 pub(crate) const DC_CHAT_ID_DEADDROP: u32 = 1;
 /// messages that should be deleted get this chat_id; the messages are deleted from the working thread later then. This is also needed as rfc724_mid should be preset as long as the message is not deleted on the server (otherwise it is downloaded again)
@@ -116,7 +105,7 @@ impl Default for Chattype {
 }
 
 pub const DC_MSG_ID_MARKER1: u32 = 1;
-const DC_MSG_ID_DAYMARKER: u32 = 9;
+pub const DC_MSG_ID_DAYMARKER: u32 = 9;
 pub const DC_MSG_ID_LAST_SPECIAL: u32 = 9;
 
 /// approx. max. length returned by dc_msg_get_text()
@@ -126,10 +115,14 @@ const DC_MAX_GET_INFO_LEN: usize = 100000;
 
 pub const DC_CONTACT_ID_UNDEFINED: u32 = 0;
 pub const DC_CONTACT_ID_SELF: u32 = 1;
-pub const DC_CONTACT_ID_DEVICE: u32 = 2;
+pub const DC_CONTACT_ID_INFO: u32 = 2;
+pub const DC_CONTACT_ID_DEVICE: u32 = 5;
 pub const DC_CONTACT_ID_LAST_SPECIAL: u32 = 9;
 
-pub const DC_CREATE_MVBOX: usize = 1;
+// Flags for empty server job
+
+pub const DC_EMPTY_MVBOX: u32 = 0x01;
+pub const DC_EMPTY_INBOX: u32 = 0x02;
 
 // Flags for configuring IMAP and SMTP servers.
 // These flags are optional
@@ -139,23 +132,23 @@ pub const DC_CREATE_MVBOX: usize = 1;
 /// Force OAuth2 authorization. This flag does not skip automatic configuration.
 /// Before calling configure() with DC_LP_AUTH_OAUTH2 set,
 /// the user has to confirm access at the URL returned by dc_get_oauth2_url().
-pub const DC_LP_AUTH_OAUTH2: usize = 0x2;
+pub const DC_LP_AUTH_OAUTH2: i32 = 0x2;
 
 /// Force NORMAL authorization, this is the default.
 /// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_AUTH_NORMAL: usize = 0x4;
+pub const DC_LP_AUTH_NORMAL: i32 = 0x4;
 
 /// Connect to IMAP via STARTTLS.
 /// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_IMAP_SOCKET_STARTTLS: usize = 0x100;
+pub const DC_LP_IMAP_SOCKET_STARTTLS: i32 = 0x100;
 
 /// Connect to IMAP via SSL.
 /// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_IMAP_SOCKET_SSL: usize = 0x200;
+pub const DC_LP_IMAP_SOCKET_SSL: i32 = 0x200;
 
 /// Connect to IMAP unencrypted, this should not be used.
 /// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_IMAP_SOCKET_PLAIN: usize = 0x400;
+pub const DC_LP_IMAP_SOCKET_PLAIN: i32 = 0x400;
 
 /// Connect to SMTP via STARTTLS.
 /// If this flag is set, automatic configuration is skipped.
@@ -170,9 +163,9 @@ pub const DC_LP_SMTP_SOCKET_SSL: usize = 0x20000;
 pub const DC_LP_SMTP_SOCKET_PLAIN: usize = 0x40000;
 
 /// if none of these flags are set, the default is chosen
-pub const DC_LP_AUTH_FLAGS: usize = (DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL);
+pub const DC_LP_AUTH_FLAGS: i32 = (DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL);
 /// if none of these flags are set, the default is chosen
-pub const DC_LP_IMAP_SOCKET_FLAGS: usize =
+pub const DC_LP_IMAP_SOCKET_FLAGS: i32 =
     (DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_SSL | DC_LP_IMAP_SOCKET_PLAIN);
 /// if none of these flags are set, the default is chosen
 pub const DC_LP_SMTP_SOCKET_FLAGS: usize =
@@ -204,6 +197,11 @@ pub enum Viewtype {
     /// and retrieved via dc_msg_get_file(), dc_msg_get_width(), dc_msg_get_height().
     Gif = 21,
 
+    /// Message containing a sticker, similar to image.
+    /// If possible, the ui should display the image without borders in a transparent way.
+    /// A click on a sticker will offer to install the sticker set in some future.
+    Sticker = 23,
+
     /// Message containing an Audio file.
     /// File and duration are set via dc_msg_set_file(), dc_msg_set_duration()
     /// and retrieved via dc_msg_get_file(), dc_msg_get_duration().
@@ -256,13 +254,8 @@ const DC_ERROR_SEE_STRING: usize = 0; // deprecated;
 const DC_ERROR_SELF_NOT_IN_GROUP: usize = 1; // deprecated;
 const DC_STR_SELFNOTINGRP: usize = 21; // deprecated;
 
-/// Values for dc_get|set_config("show_emails")
-const DC_SHOW_EMAILS_OFF: usize = 0;
-const DC_SHOW_EMAILS_ACCEPTED_CONTACTS: usize = 1;
-const DC_SHOW_EMAILS_ALL: usize = 2;
-
 // TODO: Strings need some doumentation about used placeholders.
-// These constants are used to request strings using #DC_EVENT_GET_STRING.
+// These constants are used to set stock translation strings
 
 const DC_STR_NOMESSAGES: usize = 1;
 const DC_STR_SELF: usize = 2;
@@ -308,7 +301,8 @@ const DC_STR_MSGACTIONBYME: usize = 63;
 const DC_STR_MSGLOCATIONENABLED: usize = 64;
 const DC_STR_MSGLOCATIONDISABLED: usize = 65;
 const DC_STR_LOCATION: usize = 66;
-const DC_STR_COUNT: usize = 66;
+const DC_STR_STICKER: usize = 67;
+const DC_STR_COUNT: usize = 67;
 
 pub const DC_JOB_DELETE_MSG_ON_IMAP: i32 = 110;
 

src/contact.rs

@@ -14,7 +14,7 @@ use crate::error::Result;
 use crate::events::Event;
 use crate::key::*;
 use crate::login_param::LoginParam;
-use crate::message::MessageState;
+use crate::message::{MessageState, MsgId};
 use crate::peerstate::*;
 use crate::sql;
 use crate::stock::StockMessage;
@@ -47,8 +47,8 @@ pub struct Contact {
     /// May be empty, initially set to `authname`.
     name: String,
     /// Name authorized by the contact himself. Only this name may be spread to others,
-    /// e.g. in To:-lists. May be empty. It is recommended to use `Contact::get_name`,
-    /// `Contact::get_display_name` or `Contact::get_name_n_addr` to access this field.
+    /// e.g. in To:-lists. May be empty. It is recommended to use `Contact::get_authname`,
+    /// to access this field.
     authname: String,
     /// E-Mail-Address of the contact. It is recommended to use `Contact::get_addr`` to access this field.
     addr: String,
@@ -153,7 +153,16 @@ impl Contact {
                 blocked: false,
                 origin: Origin::Unknown,
             };
-
+            return Ok(contact);
+        } else if contact_id == DC_CONTACT_ID_DEVICE {
+            let contact = Contact {
+                id: contact_id,
+                name: context.stock_str(StockMessage::DeviceMessages).into(),
+                authname: "".into(),
+                addr: "device@localhost".into(),
+                blocked: false,
+                origin: Origin::Unknown,
+            };
             return Ok(contact);
         }
 
@@ -243,7 +252,7 @@ impl Contact {
         {
             context.call_cb(Event::MsgsChanged {
                 chat_id: 0,
-                msg_id: 0,
+                msg_id: MsgId::new(0),
             });
         }
     }
@@ -263,8 +272,8 @@ impl Contact {
             .get_config(Config::ConfiguredAddr)
             .unwrap_or_default();
 
-        if addr_normalized == addr_self {
-            return 1;
+        if addr_cmp(addr_normalized, addr_self) {
+            return DC_CONTACT_ID_SELF;
         }
 
         context.sql.query_get_value(
@@ -300,8 +309,8 @@ impl Contact {
             .get_config(Config::ConfiguredAddr)
             .unwrap_or_default();
 
-        if addr == addr_self {
-            return Ok((1, sth_modified));
+        if addr_cmp(addr, addr_self) {
+            return Ok((DC_CONTACT_ID_SELF, sth_modified));
         }
 
         if !may_be_valid_addr(&addr) {
@@ -390,20 +399,18 @@ impl Contact {
                 }
                 sth_modified = Modifier::Modified;
             }
+        } else if sql::execute(
+            context,
+            &context.sql,
+            "INSERT INTO contacts (name, addr, origin) VALUES(?, ?, ?);",
+            params![name.as_ref(), addr, origin,],
+        )
+        .is_ok()
+        {
+            row_id = sql::get_rowid(context, &context.sql, "contacts", "addr", addr);
+            sth_modified = Modifier::Created;
         } else {
-            if sql::execute(
-                context,
-                &context.sql,
-                "INSERT INTO contacts (name, addr, origin) VALUES(?, ?, ?);",
-                params![name.as_ref(), addr, origin,],
-            )
-            .is_ok()
-            {
-                row_id = sql::get_rowid(context, &context.sql, "contacts", "addr", addr);
-                sth_modified = Modifier::Created;
-            } else {
-                error!(context, "Cannot add contact.");
-            }
+            error!(context, "Cannot add contact.");
         }
 
         Ok((row_id, sth_modified))
@@ -578,7 +585,12 @@ impl Contact {
 
             let mut self_key = Key::from_self_public(context, &loginparam.addr, &context.sql);
 
-            if peerstate.is_some() && peerstate.as_ref().and_then(|p| p.peek_key(0)).is_some() {
+            if peerstate.is_some()
+                && peerstate
+                    .as_ref()
+                    .and_then(|p| p.peek_key(PeerstateVerifiedStatus::Unverified))
+                    .is_some()
+            {
                 let peerstate = peerstate.as_ref().unwrap();
                 let p =
                     context.stock_str(if peerstate.prefer_encrypt == EncryptPreference::Mutual {
@@ -598,11 +610,11 @@ impl Contact {
                     .map(|k| k.formatted_fingerprint())
                     .unwrap_or_default();
                 let fingerprint_other_verified = peerstate
-                    .peek_key(2)
+                    .peek_key(PeerstateVerifiedStatus::BidirectVerified)
                     .map(|k| k.formatted_fingerprint())
                     .unwrap_or_default();
                 let fingerprint_other_unverified = peerstate
-                    .peek_key(0)
+                    .peek_key(PeerstateVerifiedStatus::Unverified)
                     .map(|k| k.formatted_fingerprint())
                     .unwrap_or_default();
                 if peerstate.addr.is_some() && &loginparam.addr < peerstate.addr.as_ref().unwrap() {
@@ -710,6 +722,7 @@ impl Contact {
         &self.addr
     }
 
+    /// Get name authorized by the contact.
     pub fn get_authname(&self) -> &str {
         &self.authname
     }
@@ -827,7 +840,7 @@ impl Contact {
         if let Ok(contact) = Contact::load_from_db(context, contact_id) {
             if !contact.addr.is_empty() {
                 let normalized_addr = addr_normalize(addr.as_ref());
-                if &contact.addr == &normalized_addr {
+                if contact.addr == normalized_addr {
                     return true;
                 }
             }
@@ -868,7 +881,7 @@ impl Contact {
     }
 
     pub fn real_exists_by_id(context: &Context, contact_id: u32) -> bool {
-        if !context.sql.is_open() || contact_id <= 9 {
+        if !context.sql.is_open() || contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
             return false;
         }
 
@@ -892,6 +905,7 @@ impl Contact {
     }
 }
 
+/// Extracts first name from full name.
 fn get_first_name<'a>(full_name: &'a str) -> &'a str {
     full_name.splitn(2, ' ').next().unwrap_or_default()
 }
@@ -902,6 +916,7 @@ pub fn may_be_valid_addr(addr: &str) -> bool {
     res.is_ok()
 }
 
+/// Returns address with whitespace trimmed and `mailto:` prefix removed.
 pub fn addr_normalize(addr: &str) -> &str {
     let norm = addr.trim();
 
@@ -913,7 +928,7 @@ pub fn addr_normalize(addr: &str) -> &str {
 }
 
 fn set_block_contact(context: &Context, contact_id: u32, new_blocking: bool) {
-    if contact_id <= 9 {
+    if contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
         return;
     }
 
@@ -963,9 +978,9 @@ pub fn normalize_name(full_name: impl AsRef<str>) -> String {
     if len > 0 {
         let firstchar = full_name.as_bytes()[0];
         let lastchar = full_name.as_bytes()[len - 1];
-        if firstchar == '\'' as u8 && lastchar == '\'' as u8
-            || firstchar == '\"' as u8 && lastchar == '\"' as u8
-            || firstchar == '<' as u8 && lastchar == '>' as u8
+        if firstchar == b'\'' && lastchar == b'\''
+            || firstchar == b'\"' && lastchar == b'\"'
+            || firstchar == b'<' && lastchar == b'>'
         {
             full_name = &full_name[1..len - 1];
         }
@@ -1011,17 +1026,16 @@ fn cat_fingerprint(
 }
 
 pub fn addr_cmp(addr1: impl AsRef<str>, addr2: impl AsRef<str>) -> bool {
-    let norm1 = addr_normalize(addr1.as_ref());
-    let norm2 = addr_normalize(addr2.as_ref());
+    let norm1 = addr_normalize(addr1.as_ref()).to_lowercase();
+    let norm2 = addr_normalize(addr2.as_ref()).to_lowercase();
 
     norm1 == norm2
 }
 
 pub fn addr_equals_self(context: &Context, addr: impl AsRef<str>) -> bool {
     if !addr.as_ref().is_empty() {
-        let normalized_addr = addr_normalize(addr.as_ref());
         if let Some(self_addr) = context.get_config(Config::ConfiguredAddr) {
-            return normalized_addr == self_addr;
+            return addr_cmp(addr, self_addr);
         }
     }
     false
@@ -1046,6 +1060,8 @@ fn split_address_book(book: &str) -> Vec<(&str, &str)> {
 mod tests {
     use super::*;
 
+    use crate::test_utils::*;
+
     #[test]
     fn test_may_be_valid_addr() {
         assert_eq!(may_be_valid_addr(""), false);
@@ -1071,6 +1087,10 @@ mod tests {
     fn test_normalize_addr() {
         assert_eq!(addr_normalize("mailto:john@doe.com"), "john@doe.com");
         assert_eq!(addr_normalize("  hello@world.com   "), "hello@world.com");
+
+        // normalisation preserves case to allow user-defined spelling.
+        // however, case is ignored on addr_cmp()
+        assert_ne!(addr_normalize("John@Doe.com"), "john@doe.com");
     }
 
     #[test]
@@ -1087,4 +1107,116 @@ mod tests {
             vec![("Name one", "Address one"), ("Name two", "Address two")]
         )
     }
+
+    #[test]
+    fn test_get_contacts() {
+        let context = dummy_context();
+        let contacts = Contact::get_all(&context.ctx, 0, Some("some2")).unwrap();
+        assert_eq!(contacts.len(), 0);
+
+        let id = Contact::create(&context.ctx, "bob", "bob@mail.de").unwrap();
+        assert_ne!(id, 0);
+
+        let contacts = Contact::get_all(&context.ctx, 0, Some("bob")).unwrap();
+        assert_eq!(contacts.len(), 1);
+
+        let contacts = Contact::get_all(&context.ctx, 0, Some("alice")).unwrap();
+        assert_eq!(contacts.len(), 0);
+    }
+
+    #[test]
+    fn test_add_or_lookup() {
+        // add some contacts, this also tests add_address_book()
+        let t = dummy_context();
+        let book = concat!(
+            "  Name one  \n one@eins.org \n",
+            "Name two\ntwo@deux.net\n",
+            "\nthree@drei.sam\n",
+            "Name two\ntwo@deux.net\n" // should not be added again
+        );
+        assert_eq!(Contact::add_address_book(&t.ctx, book).unwrap(), 3);
+
+        // check first added contact, this does not modify because of lower origin
+        let (contact_id, sth_modified) =
+            Contact::add_or_lookup(&t.ctx, "bla foo", "one@eins.org", Origin::IncomingUnknownTo)
+                .unwrap();
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert_eq!(sth_modified, Modifier::None);
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_id(), contact_id);
+        assert_eq!(contact.get_name(), "Name one");
+        assert_eq!(contact.get_display_name(), "Name one");
+        assert_eq!(contact.get_addr(), "one@eins.org");
+        assert_eq!(contact.get_name_n_addr(), "Name one (one@eins.org)");
+
+        // modify first added contact
+        let (contact_id_test, sth_modified) = Contact::add_or_lookup(
+            &t.ctx,
+            "Real one",
+            " one@eins.org  ",
+            Origin::ManuallyCreated,
+        )
+        .unwrap();
+        assert_eq!(contact_id, contact_id_test);
+        assert_eq!(sth_modified, Modifier::Modified);
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_name(), "Real one");
+        assert_eq!(contact.get_addr(), "one@eins.org");
+        assert!(!contact.is_blocked());
+
+        // check third added contact (contact without name)
+        let (contact_id, sth_modified) =
+            Contact::add_or_lookup(&t.ctx, "", "three@drei.sam", Origin::IncomingUnknownTo)
+                .unwrap();
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert_eq!(sth_modified, Modifier::None);
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_name(), "");
+        assert_eq!(contact.get_display_name(), "three@drei.sam");
+        assert_eq!(contact.get_addr(), "three@drei.sam");
+        assert_eq!(contact.get_name_n_addr(), "three@drei.sam");
+
+        // add name to third contact from incoming message (this becomes authorized name)
+        let (contact_id_test, sth_modified) = Contact::add_or_lookup(
+            &t.ctx,
+            "m. serious",
+            "three@drei.sam",
+            Origin::IncomingUnknownFrom,
+        )
+        .unwrap();
+        assert_eq!(contact_id, contact_id_test);
+        assert_eq!(sth_modified, Modifier::Modified);
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_name_n_addr(), "m. serious (three@drei.sam)");
+        assert!(!contact.is_blocked());
+
+        // manually edit name of third contact (does not changed authorized name)
+        let (contact_id_test, sth_modified) = Contact::add_or_lookup(
+            &t.ctx,
+            "schnucki",
+            "three@drei.sam",
+            Origin::ManuallyCreated,
+        )
+        .unwrap();
+        assert_eq!(contact_id, contact_id_test);
+        assert_eq!(sth_modified, Modifier::Modified);
+        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
+        assert_eq!(contact.get_authname(), "m. serious");
+        assert_eq!(contact.get_name_n_addr(), "schnucki (three@drei.sam)");
+        assert!(!contact.is_blocked());
+
+        // check SELF
+        let contact = Contact::load_from_db(&t.ctx, DC_CONTACT_ID_SELF).unwrap();
+        assert_eq!(DC_CONTACT_ID_SELF, 1);
+        assert_eq!(contact.get_name(), t.ctx.stock_str(StockMessage::SelfMsg));
+        assert_eq!(contact.get_addr(), ""); // we're not configured
+        assert!(!contact.is_blocked());
+    }
+
+    #[test]
+    fn test_addr_cmp() {
+        assert!(addr_cmp("AA@AA.ORG", "aa@aa.ORG"));
+        assert!(addr_cmp(" aa@aa.ORG ", "AA@AA.ORG"));
+        assert!(addr_cmp(" mailto:AA@AA.ORG", "Aa@Aa.orG"));
+    }
 }

src/context.rs

@@ -6,6 +6,7 @@ use std::sync::{Arc, Condvar, Mutex, RwLock};
 use libc::uintptr_t;
 
 use crate::chat::*;
+use crate::config::Config;
 use crate::constants::*;
 use crate::contact::*;
 use crate::error::*;
@@ -16,7 +17,7 @@ use crate::job_thread::JobThread;
 use crate::key::*;
 use crate::login_param::LoginParam;
 use crate::lot::Lot;
-use crate::message::{self, Message};
+use crate::message::{self, Message, MsgId};
 use crate::param::Params;
 use crate::smtp::*;
 use crate::sql::Sql;
@@ -38,12 +39,14 @@ pub type ContextCallback = dyn Fn(&Context, Event) -> uintptr_t + Send + Sync;
 
 #[derive(DebugStub)]
 pub struct Context {
+    /// Database file path
     dbfile: PathBuf,
+    /// Blob directory path
     blobdir: PathBuf,
     pub sql: Sql,
-    pub inbox: Arc<RwLock<Imap>>,
     pub perform_inbox_jobs_needed: Arc<RwLock<bool>>,
     pub probe_imap_network: Arc<RwLock<bool>>,
+    pub inbox_thread: Arc<RwLock<JobThread>>,
     pub sentbox_thread: Arc<RwLock<JobThread>>,
     pub mvbox_thread: Arc<RwLock<JobThread>>,
     pub smtp: Arc<Mutex<Smtp>>,
@@ -54,16 +57,17 @@ pub struct Context {
     pub os_name: Option<String>,
     pub cmdline_sel_chat_id: Arc<RwLock<u32>>,
     pub bob: Arc<RwLock<BobStatus>>,
-    pub last_smeared_timestamp: Arc<RwLock<i64>>,
+    pub last_smeared_timestamp: RwLock<i64>,
     pub running_state: Arc<RwLock<RunningState>>,
     /// Mutex to avoid generating the key for the user more than once.
     pub generating_key_mutex: Mutex<()>,
+    pub translated_stockstrings: RwLock<HashMap<usize, String>>,
 }
 
 #[derive(Debug, PartialEq, Eq)]
 pub struct RunningState {
     pub ongoing_running: bool,
-    pub shall_stop_ongoing: bool,
+    shall_stop_ongoing: bool,
 }
 
 /// Return some info about deltachat-core
@@ -91,6 +95,7 @@ pub fn get_info() -> HashMap<&'static str, String> {
 }
 
 impl Context {
+    /// Creates new context.
     pub fn new(cb: Box<ContextCallback>, os_name: String, dbfile: PathBuf) -> Result<Context> {
         let mut blob_fname = OsString::new();
         blob_fname.push(dbfile.file_name().unwrap_or_default());
@@ -116,7 +121,6 @@ impl Context {
         let ctx = Context {
             blobdir,
             dbfile,
-            inbox: Arc::new(RwLock::new(Imap::new())),
             cb,
             os_name: Some(os_name),
             running_state: Arc::new(RwLock::new(Default::default())),
@@ -125,8 +129,13 @@ impl Context {
             smtp_state: Arc::new((Mutex::new(Default::default()), Condvar::new())),
             oauth2_critical: Arc::new(Mutex::new(())),
             bob: Arc::new(RwLock::new(Default::default())),
-            last_smeared_timestamp: Arc::new(RwLock::new(0)),
+            last_smeared_timestamp: RwLock::new(0),
             cmdline_sel_chat_id: Arc::new(RwLock::new(0)),
+            inbox_thread: Arc::new(RwLock::new(JobThread::new(
+                "INBOX",
+                "configured_inbox_folder",
+                Imap::new(),
+            ))),
             sentbox_thread: Arc::new(RwLock::new(JobThread::new(
                 "SENTBOX",
                 "configured_sentbox_folder",
@@ -140,20 +149,23 @@ impl Context {
             probe_imap_network: Arc::new(RwLock::new(false)),
             perform_inbox_jobs_needed: Arc::new(RwLock::new(false)),
             generating_key_mutex: Mutex::new(()),
+            translated_stockstrings: RwLock::new(HashMap::new()),
         };
 
         ensure!(
-            ctx.sql.open(&ctx, &ctx.dbfile, 0),
+            ctx.sql.open(&ctx, &ctx.dbfile, false),
             "Failed opening sqlite database"
         );
 
         Ok(ctx)
     }
 
+    /// Returns database file path.
     pub fn get_dbfile(&self) -> &Path {
         self.dbfile.as_path()
     }
 
+    /// Returns blob directory path.
     pub fn get_blobdir(&self) -> &Path {
         self.blobdir.as_path()
     }
@@ -162,31 +174,84 @@ impl Context {
         (*self.cb)(self, event)
     }
 
+    /*******************************************************************************
+     * Ongoing process allocation/free/check
+     ******************************************************************************/
+
+    pub fn alloc_ongoing(&self) -> bool {
+        if self.has_ongoing() {
+            warn!(self, "There is already another ongoing process running.",);
+
+            false
+        } else {
+            let s_a = self.running_state.clone();
+            let mut s = s_a.write().unwrap();
+
+            s.ongoing_running = true;
+            s.shall_stop_ongoing = false;
+
+            true
+        }
+    }
+
+    pub fn free_ongoing(&self) {
+        let s_a = self.running_state.clone();
+        let mut s = s_a.write().unwrap();
+
+        s.ongoing_running = false;
+        s.shall_stop_ongoing = true;
+    }
+
+    pub fn has_ongoing(&self) -> bool {
+        let s_a = self.running_state.clone();
+        let s = s_a.read().unwrap();
+
+        s.ongoing_running || !s.shall_stop_ongoing
+    }
+
+    /// Signal an ongoing process to stop.
+    pub fn stop_ongoing(&self) {
+        let s_a = self.running_state.clone();
+        let mut s = s_a.write().unwrap();
+
+        if s.ongoing_running && !s.shall_stop_ongoing {
+            info!(self, "Signaling the ongoing process to stop ASAP.",);
+            s.shall_stop_ongoing = true;
+        } else {
+            info!(self, "No ongoing process to stop.",);
+        };
+    }
+
+    pub fn shall_stop_ongoing(&self) -> bool {
+        self.running_state
+            .clone()
+            .read()
+            .unwrap()
+            .shall_stop_ongoing
+    }
+
+    /*******************************************************************************
+     * UI chat/message related API
+     ******************************************************************************/
+
     pub fn get_info(&self) -> HashMap<&'static str, String> {
         let unset = "0";
         let l = LoginParam::from_database(self, "");
         let l2 = LoginParam::from_database(self, "configured_");
-        let displayname = self.sql.get_config(self, "displayname");
+        let displayname = self.get_config(Config::Displayname);
         let chats = get_chat_cnt(self) as usize;
         let real_msgs = message::get_real_msg_cnt(self) as usize;
         let deaddrop_msgs = message::get_deaddrop_msg_cnt(self) as usize;
         let contacts = Contact::get_real_cnt(self) as usize;
-        let is_configured = self
-            .sql
-            .get_config_int(self, "configured")
-            .unwrap_or_default();
+        let is_configured = self.get_config_int(Config::Configured);
         let dbversion = self
             .sql
-            .get_config_int(self, "dbversion")
+            .get_raw_config_int(self, "dbversion")
             .unwrap_or_default();
-        let e2ee_enabled = self
-            .sql
-            .get_config_int(self, "e2ee_enabled")
-            .unwrap_or_else(|| 1);
-        let mdns_enabled = self
-            .sql
-            .get_config_int(self, "mdns_enabled")
-            .unwrap_or_else(|| 1);
+
+        let e2ee_enabled = self.get_config_int(Config::E2eeEnabled);
+        let mdns_enabled = self.get_config_int(Config::MdnsEnabled);
+        let bcc_self = self.get_config_int(Config::BccSelf);
 
         let prv_key_cnt: Option<isize> =
             self.sql
@@ -204,33 +269,22 @@ impl Context {
             "<Not yet calculated>".into()
         };
 
-        let inbox_watch = self
-            .sql
-            .get_config_int(self, "inbox_watch")
-            .unwrap_or_else(|| 1);
-        let sentbox_watch = self
-            .sql
-            .get_config_int(self, "sentbox_watch")
-            .unwrap_or_else(|| 1);
-        let mvbox_watch = self
-            .sql
-            .get_config_int(self, "mvbox_watch")
-            .unwrap_or_else(|| 1);
-        let mvbox_move = self
-            .sql
-            .get_config_int(self, "mvbox_move")
-            .unwrap_or_else(|| 1);
+        let inbox_watch = self.get_config_int(Config::InboxWatch);
+        let sentbox_watch = self.get_config_int(Config::SentboxWatch);
+        let mvbox_watch = self.get_config_int(Config::MvboxWatch);
+        let mvbox_move = self.get_config_int(Config::MvboxMove);
         let folders_configured = self
             .sql
-            .get_config_int(self, "folders_configured")
+            .get_raw_config_int(self, "folders_configured")
             .unwrap_or_default();
+
         let configured_sentbox_folder = self
             .sql
-            .get_config(self, "configured_sentbox_folder")
+            .get_raw_config(self, "configured_sentbox_folder")
             .unwrap_or_else(|| "<unset>".to_string());
         let configured_mvbox_folder = self
             .sql
-            .get_config(self, "configured_mvbox_folder")
+            .get_raw_config(self, "configured_mvbox_folder")
             .unwrap_or_else(|| "<unset>".to_string());
 
         let mut res = get_info();
@@ -254,6 +308,7 @@ impl Context {
         res.insert("configured_mvbox_folder", configured_mvbox_folder);
         res.insert("mdns_enabled", mdns_enabled.to_string());
         res.insert("e2ee_enabled", e2ee_enabled.to_string());
+        res.insert("bcc_self", bcc_self.to_string());
         res.insert(
             "private_key_count",
             prv_key_cnt.unwrap_or_default().to_string(),
@@ -267,33 +322,39 @@ impl Context {
         res
     }
 
-    pub fn get_fresh_msgs(&self) -> Vec<u32> {
+    pub fn get_fresh_msgs(&self) -> Vec<MsgId> {
         let show_deaddrop = 0;
-
         self.sql
             .query_map(
-                "SELECT m.id FROM msgs m LEFT JOIN contacts ct \
-                 ON m.from_id=ct.id LEFT JOIN chats c ON m.chat_id=c.id WHERE m.state=?   \
-                 AND m.hidden=0   \
-                 AND m.chat_id>?   \
-                 AND ct.blocked=0   \
-                 AND (c.blocked=0 OR c.blocked=?) ORDER BY m.timestamp DESC,m.id DESC;",
+                concat!(
+                    "SELECT m.id",
+                    " FROM msgs m",
+                    " LEFT JOIN contacts ct",
+                    "        ON m.from_id=ct.id",
+                    " LEFT JOIN chats c",
+                    "        ON m.chat_id=c.id",
+                    " WHERE m.state=?",
+                    "   AND m.hidden=0",
+                    "   AND m.chat_id>?",
+                    "   AND ct.blocked=0",
+                    "   AND (c.blocked=0 OR c.blocked=?)",
+                    " ORDER BY m.timestamp DESC,m.id DESC;"
+                ),
                 &[10, 9, if 0 != show_deaddrop { 2 } else { 0 }],
-                |row| row.get(0),
+                |row| row.get::<_, MsgId>(0),
                 |rows| {
                     let mut ret = Vec::new();
                     for row in rows {
-                        let id: u32 = row?;
-                        ret.push(id);
+                        ret.push(row?);
                     }
                     Ok(ret)
                 },
             )
-            .unwrap()
+            .unwrap_or_default()
     }
 
     #[allow(non_snake_case)]
-    pub fn search_msgs(&self, chat_id: u32, query: impl AsRef<str>) -> Vec<u32> {
+    pub fn search_msgs(&self, chat_id: u32, query: impl AsRef<str>) -> Vec<MsgId> {
         let real_query = query.as_ref().trim();
         if real_query.is_empty() {
             return Vec::new();
@@ -302,25 +363,43 @@ impl Context {
         let strLikeBeg = format!("{}%", real_query);
 
         let query = if 0 != chat_id {
-            "SELECT m.id, m.timestamp FROM msgs m LEFT JOIN contacts ct ON m.from_id=ct.id WHERE m.chat_id=?  \
-         AND m.hidden=0  \
-         AND ct.blocked=0 AND (txt LIKE ? OR ct.name LIKE ?) ORDER BY m.timestamp,m.id;"
+            concat!(
+                "SELECT m.id AS id, m.timestamp AS timestamp",
+                " FROM msgs m",
+                " LEFT JOIN contacts ct",
+                "        ON m.from_id=ct.id",
+                " WHERE m.chat_id=?",
+                "   AND m.hidden=0",
+                "   AND ct.blocked=0",
+                "   AND (txt LIKE ? OR ct.name LIKE ?)",
+                " ORDER BY m.timestamp,m.id;"
+            )
         } else {
-            "SELECT m.id, m.timestamp FROM msgs m LEFT JOIN contacts ct ON m.from_id=ct.id \
-         LEFT JOIN chats c ON m.chat_id=c.id WHERE m.chat_id>9 AND m.hidden=0  \
-         AND (c.blocked=0 OR c.blocked=?) \
-         AND ct.blocked=0 AND (m.txt LIKE ? OR ct.name LIKE ?) ORDER BY m.timestamp DESC,m.id DESC;"
+            concat!(
+                "SELECT m.id AS id, m.timestamp AS timestamp",
+                " FROM msgs m",
+                " LEFT JOIN contacts ct",
+                "        ON m.from_id=ct.id",
+                " LEFT JOIN chats c",
+                "        ON m.chat_id=c.id",
+                " WHERE m.chat_id>9",
+                "   AND m.hidden=0",
+                "   AND (c.blocked=0 OR c.blocked=?)",
+                "   AND ct.blocked=0",
+                "   AND (m.txt LIKE ? OR ct.name LIKE ?)",
+                " ORDER BY m.timestamp DESC,m.id DESC;"
+            )
         };
 
         self.sql
             .query_map(
                 query,
                 params![chat_id as i32, &strLikeInText, &strLikeBeg],
-                |row| row.get::<_, i32>(0),
+                |row| row.get::<_, MsgId>("id"),
                 |rows| {
                     let mut ret = Vec::new();
                     for id in rows {
-                        ret.push(id? as u32);
+                        ret.push(id?);
                     }
                     Ok(ret)
                 },
@@ -333,7 +412,7 @@ impl Context {
     }
 
     pub fn is_sentbox(&self, folder_name: impl AsRef<str>) -> bool {
-        let sentbox_name = self.sql.get_config(self, "configured_sentbox_folder");
+        let sentbox_name = self.sql.get_raw_config(self, "configured_sentbox_folder");
         if let Some(name) = sentbox_name {
             name == folder_name.as_ref()
         } else {
@@ -342,7 +421,7 @@ impl Context {
     }
 
     pub fn is_mvbox(&self, folder_name: impl AsRef<str>) -> bool {
-        let mvbox_name = self.sql.get_config(self, "configured_mvbox_folder");
+        let mvbox_name = self.sql.get_raw_config(self, "configured_mvbox_folder");
 
         if let Some(name) = mvbox_name {
             name == folder_name.as_ref()
@@ -351,20 +430,14 @@ impl Context {
         }
     }
 
-    pub fn do_heuristics_moves(&self, folder: &str, msg_id: u32) {
-        if self
-            .sql
-            .get_config_int(self, "mvbox_move")
-            .unwrap_or_else(|| 1)
-            == 0
-        {
+    pub fn do_heuristics_moves(&self, folder: &str, msg_id: MsgId) {
+        if !self.get_config_bool(Config::MvboxMove) {
             return;
         }
 
-        if !self.is_inbox(folder) && !self.is_sentbox(folder) {
+        if self.is_mvbox(folder) {
             return;
         }
-
         if let Ok(msg) = Message::load_from_db(self, msg_id) {
             if msg.is_setupmessage() {
                 // do not move setup messages;
@@ -372,20 +445,15 @@ impl Context {
                 return;
             }
 
-            if self.is_mvbox(folder) {
-                message::update_msg_move_state(self, &msg.rfc724_mid, MoveState::Stay);
-            }
-
             // 1 = dc message, 2 = reply to dc message
             if 0 != msg.is_dc_message {
                 job_add(
                     self,
                     Action::MoveMsg,
-                    msg.id as libc::c_int,
+                    msg.id.to_u32() as i32,
                     Params::new(),
                     0,
                 );
-                message::update_msg_move_state(self, &msg.rfc724_mid, MoveState::Moving);
             }
         }
     }
@@ -393,8 +461,8 @@ impl Context {
 
 impl Drop for Context {
     fn drop(&mut self) {
-        info!(self, "disconnecting INBOX-watch",);
-        self.inbox.read().unwrap().disconnect(self);
+        info!(self, "disconnecting inbox-thread",);
+        self.inbox_thread.read().unwrap().imap.disconnect(self);
         info!(self, "disconnecting sentbox-thread",);
         self.sentbox_thread.read().unwrap().imap.disconnect(self);
         info!(self, "disconnecting mvbox-thread",);
@@ -421,12 +489,25 @@ pub struct BobStatus {
     pub qr_scan: Option<Lot>,
 }
 
+#[derive(Debug, PartialEq)]
+pub enum PerformJobsNeeded {
+    Not,
+    AtOnce,
+    AvoidDos,
+}
+
+impl Default for PerformJobsNeeded {
+    fn default() -> Self {
+        Self::Not
+    }
+}
+
 #[derive(Default, Debug)]
 pub struct SmtpState {
     pub idle: bool,
     pub suspended: bool,
     pub doing_jobs: bool,
-    pub perform_jobs_needed: i32,
+    pub perform_jobs_needed: PerformJobsNeeded,
     pub probe_network: bool,
 }
 

src/dc_imex.rs

@@ -1,960 +0,0 @@
-use std::ffi::CString;
-use std::path::Path;
-use std::ptr;
-
-use libc::{free, strlen};
-use mmime::mailmime::content::*;
-use mmime::mmapstring::*;
-use mmime::other::*;
-use rand::{thread_rng, Rng};
-
-use crate::chat;
-use crate::config::Config;
-use crate::configure::*;
-use crate::constants::*;
-use crate::context::Context;
-use crate::dc_tools::*;
-use crate::e2ee;
-use crate::error::*;
-use crate::events::Event;
-use crate::job::*;
-use crate::key::*;
-use crate::message::Message;
-use crate::param::*;
-use crate::pgp::*;
-use crate::sql::{self, Sql};
-use crate::stock::StockMessage;
-
-// import/export and tools
-// param1 is a directory where the keys are written to
-// param1 is a directory where the keys are searched in and read from
-// param1 is a directory where the backup is written to
-// param1 is the file with the backup to import
-pub fn dc_imex(context: &Context, what: libc::c_int, param1: Option<impl AsRef<Path>>) {
-    let mut param = Params::new();
-    param.set_int(Param::Cmd, what as i32);
-    if let Some(param1) = param1 {
-        param.set(Param::Arg, param1.as_ref().to_string_lossy());
-    }
-
-    job_kill_action(context, Action::ImexImap);
-    job_add(context, Action::ImexImap, 0, param, 0);
-}
-
-/// Returns the filename of the backup if found, nullptr otherwise.
-pub fn dc_imex_has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Result<String> {
-    let dir_name = dir_name.as_ref();
-    let dir_iter = std::fs::read_dir(dir_name)?;
-    let mut newest_backup_time = 0;
-    let mut newest_backup_path: Option<std::path::PathBuf> = None;
-    for dirent in dir_iter {
-        match dirent {
-            Ok(dirent) => {
-                let path = dirent.path();
-                let name = dirent.file_name();
-                let name = name.to_string_lossy();
-                if name.starts_with("delta-chat") && name.ends_with(".bak") {
-                    let sql = Sql::new();
-                    if sql.open(context, &path, 0x1) {
-                        let curr_backup_time =
-                            sql.get_config_int(context, "backup_time")
-                                .unwrap_or_default() as u64;
-                        if curr_backup_time > newest_backup_time {
-                            newest_backup_path = Some(path);
-                            newest_backup_time = curr_backup_time;
-                        }
-                    }
-                }
-            }
-            Err(_) => (),
-        }
-    }
-    match newest_backup_path {
-        Some(path) => Ok(path.to_string_lossy().into_owned()),
-        None => bail!("no backup found"),
-    }
-}
-
-pub fn dc_initiate_key_transfer(context: &Context) -> Result<String> {
-    let mut msg: Message;
-    ensure!(dc_alloc_ongoing(context), "could not allocate ongoing");
-    let setup_code = dc_create_setup_code(context);
-    /* this may require a keypair to be created. this may take a second ... */
-    if !context
-        .running_state
-        .clone()
-        .read()
-        .unwrap()
-        .shall_stop_ongoing
-    {
-        if let Ok(ref setup_file_content) = dc_render_setup_file(context, &setup_code) {
-            /* encrypting may also take a while ... */
-            if !context
-                .running_state
-                .clone()
-                .read()
-                .unwrap()
-                .shall_stop_ongoing
-            {
-                let setup_file_name =
-                    dc_get_fine_path_filename(context, "$BLOBDIR", "autocrypt-setup-message.html");
-                if dc_write_file(context, &setup_file_name, setup_file_content.as_bytes()) {
-                    if let Ok(chat_id) = chat::create_by_contact_id(context, 1) {
-                        msg = Message::default();
-                        msg.type_0 = Viewtype::File;
-                        msg.param
-                            .set(Param::File, setup_file_name.to_string_lossy());
-
-                        msg.param
-                            .set(Param::MimeType, "application/autocrypt-setup");
-                        msg.param.set_int(Param::Cmd, 6);
-                        msg.param
-                            .set_int(Param::ForcePlaintext, DC_FP_NO_AUTOCRYPT_HEADER);
-
-                        if !context
-                            .running_state
-                            .clone()
-                            .read()
-                            .unwrap()
-                            .shall_stop_ongoing
-                        {
-                            if let Ok(msg_id) = chat::send_msg(context, chat_id, &mut msg) {
-                                info!(context, "Wait for setup message being sent ...",);
-                                loop {
-                                    if context
-                                        .running_state
-                                        .clone()
-                                        .read()
-                                        .unwrap()
-                                        .shall_stop_ongoing
-                                    {
-                                        break;
-                                    }
-                                    std::thread::sleep(std::time::Duration::from_secs(1));
-                                    if let Ok(msg) = Message::load_from_db(context, msg_id) {
-                                        if msg.is_sent() {
-                                            info!(context, "... setup message sent.",);
-                                            break;
-                                        }
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-    dc_free_ongoing(context);
-
-    Ok(setup_code)
-}
-
-/// Renders HTML body of a setup file message.
-///
-/// The `passphrase` must be at least 2 characters long.
-pub fn dc_render_setup_file(context: &Context, passphrase: &str) -> Result<String> {
-    ensure!(
-        passphrase.len() >= 2,
-        "Passphrase must be at least 2 chars long."
-    );
-    let self_addr = e2ee::ensure_secret_key_exists(context)?;
-    let private_key = Key::from_self_private(context, self_addr, &context.sql)
-        .ok_or(format_err!("Failed to get private key."))?;
-    let ac_headers = match context
-        .sql
-        .get_config_int(context, Config::E2eeEnabled)
-        .unwrap_or(1)
-    {
-        0 => None,
-        _ => Some(("Autocrypt-Prefer-Encrypt", "mutual")),
-    };
-    let private_key_asc = private_key.to_asc(ac_headers);
-    let encr = dc_pgp_symm_encrypt(&passphrase, private_key_asc.as_bytes())?;
-
-    let replacement = format!(
-        concat!(
-            "-----BEGIN PGP MESSAGE-----\r\n",
-            "Passphrase-Format: numeric9x4\r\n",
-            "Passphrase-Begin: {}"
-        ),
-        &passphrase[..2]
-    );
-    let pgp_msg = encr.replace("-----BEGIN PGP MESSAGE-----", &replacement);
-
-    let msg_subj = context.stock_str(StockMessage::AcSetupMsgSubject);
-    let msg_body = context.stock_str(StockMessage::AcSetupMsgBody);
-    let msg_body_html = msg_body.replace("\r", "").replace("\n", "<br>");
-    Ok(format!(
-        concat!(
-            "<!DOCTYPE html>\r\n",
-            "<html>\r\n",
-            "  <head>\r\n",
-            "    <title>{}</title>\r\n",
-            "  </head>\r\n",
-            "  <body>\r\n",
-            "    <h1>{}</h1>\r\n",
-            "    <p>{}</p>\r\n",
-            "    <pre>\r\n{}\r\n</pre>\r\n",
-            "  </body>\r\n",
-            "</html>\r\n"
-        ),
-        msg_subj, msg_subj, msg_body_html, pgp_msg
-    ))
-}
-
-pub fn dc_create_setup_code(_context: &Context) -> String {
-    let mut random_val: u16;
-    let mut rng = thread_rng();
-    let mut ret = String::new();
-
-    for i in 0..9 {
-        loop {
-            random_val = rng.gen();
-            if !(random_val as libc::c_int > 60000) {
-                break;
-            }
-        }
-        random_val = (random_val as libc::c_int % 10000) as u16;
-        ret += &format!(
-            "{}{:04}",
-            if 0 != i { "-" } else { "" },
-            random_val as libc::c_int,
-        );
-    }
-
-    ret
-}
-
-pub fn dc_continue_key_transfer(context: &Context, msg_id: u32, setup_code: &str) -> Result<()> {
-    ensure!(msg_id > DC_MSG_ID_LAST_SPECIAL, "wrong id");
-
-    let msg = Message::load_from_db(context, msg_id);
-    if msg.is_err() {
-        bail!("Message is no Autocrypt Setup Message.");
-    }
-    let msg = msg.unwrap();
-    ensure!(
-        msg.is_setupmessage(),
-        "Message is no Autocrypt Setup Message."
-    );
-
-    if let Some(filename) = msg.get_file(context) {
-        if let Ok(buf) = dc_read_file(context, filename) {
-            let norm_sc = CString::yolo(dc_normalize_setup_code(setup_code));
-            unsafe {
-                if let Ok(armored_key) =
-                    dc_decrypt_setup_file(context, norm_sc.as_ptr(), buf.as_ptr().cast())
-                {
-                    set_self_key(context, &armored_key, true, true)?;
-                } else {
-                    bail!("Bad setup code.")
-                }
-            }
-            Ok(())
-        } else {
-            bail!("Cannot read Autocrypt Setup Message file.");
-        }
-    } else {
-        bail!("Message is no Autocrypt Setup Message.");
-    }
-}
-
-fn set_self_key(
-    context: &Context,
-    armored: &str,
-    set_default: bool,
-    prefer_encrypt_required: bool,
-) -> Result<()> {
-    // try hard to only modify key-state
-    let keys = Key::from_armored_string(armored, KeyType::Private)
-        .and_then(|(k, h)| if k.verify() { Some((k, h)) } else { None })
-        .and_then(|(k, h)| k.split_key().map(|pub_key| (k, pub_key, h)));
-
-    ensure!(keys.is_some(), "Not a valid private key");
-
-    let (private_key, public_key, header) = keys.unwrap();
-    let preferencrypt = header.get("Autocrypt-Prefer-Encrypt");
-    match preferencrypt.map(|s| s.as_str()) {
-        Some(headerval) => {
-            let e2ee_enabled = match headerval {
-                "nopreference" => 0,
-                "mutual" => 1,
-                _ => {
-                    bail!("invalid Autocrypt-Prefer-Encrypt header: {:?}", header);
-                }
-            };
-            context
-                .sql
-                .set_config_int(context, "e2ee_enabled", e2ee_enabled)?;
-        }
-        None => {
-            if prefer_encrypt_required {
-                bail!("missing Autocrypt-Prefer-Encrypt header");
-            }
-        }
-    };
-
-    let self_addr = context.get_config(Config::ConfiguredAddr);
-    ensure!(self_addr.is_some(), "Missing self addr");
-
-    // XXX maybe better make dc_key_save_self_keypair delete things
-    sql::execute(
-        context,
-        &context.sql,
-        "DELETE FROM keypairs WHERE public_key=? OR private_key=?;",
-        params![public_key.to_bytes(), private_key.to_bytes()],
-    )?;
-
-    if set_default {
-        sql::execute(
-            context,
-            &context.sql,
-            "UPDATE keypairs SET is_default=0;",
-            params![],
-        )?;
-    }
-
-    if !dc_key_save_self_keypair(
-        context,
-        &public_key,
-        &private_key,
-        self_addr.unwrap(),
-        set_default as libc::c_int,
-        &context.sql,
-    ) {
-        bail!("Cannot save keypair, internal key-state possibly corrupted now!");
-    }
-    Ok(())
-}
-
-pub unsafe fn dc_decrypt_setup_file(
-    context: &Context,
-    passphrase: *const libc::c_char,
-    filecontent: *const libc::c_char,
-) -> Result<String> {
-    let fc_buf: *mut libc::c_char;
-    let mut fc_headerline = String::default();
-    let mut fc_base64: *const libc::c_char = ptr::null();
-    let mut binary: *mut libc::c_char = ptr::null_mut();
-    let mut binary_bytes: libc::size_t = 0;
-    let mut indx: libc::size_t = 0;
-
-    let mut payload: Result<String> = Err(format_err!("Failed to decrypt"));
-
-    fc_buf = dc_strdup(filecontent);
-    if dc_split_armored_data(
-        fc_buf,
-        &mut fc_headerline,
-        ptr::null_mut(),
-        ptr::null_mut(),
-        &mut fc_base64,
-    ) && fc_headerline == "-----BEGIN PGP MESSAGE-----"
-        && !fc_base64.is_null()
-    {
-        /* convert base64 to binary */
-        /*must be freed using mmap_string_unref()*/
-        if !(mailmime_base64_body_parse(
-            fc_base64,
-            strlen(fc_base64),
-            &mut indx,
-            &mut binary,
-            &mut binary_bytes,
-        ) != MAILIMF_NO_ERROR as libc::c_int
-            || binary.is_null()
-            || binary_bytes == 0)
-        {
-            /* decrypt symmetrically */
-            match dc_pgp_symm_decrypt(
-                as_str(passphrase),
-                std::slice::from_raw_parts(binary as *const u8, binary_bytes),
-            ) {
-                Ok(plain) => payload = Ok(String::from_utf8(plain).unwrap()),
-                Err(err) => {
-                    error!(context, "Failed to decrypt message: {:?}", err);
-                    payload = Err(err);
-                }
-            }
-        }
-    }
-
-    free(fc_buf as *mut libc::c_void);
-    if !binary.is_null() {
-        mmap_string_unref(binary);
-    }
-
-    payload
-}
-
-pub fn dc_normalize_setup_code(s: &str) -> String {
-    let mut out = String::new();
-    for c in s.chars() {
-        if c >= '0' && c <= '9' {
-            out.push(c);
-            if let 4 | 9 | 14 | 19 | 24 | 29 | 34 | 39 = out.len() {
-                out += "-"
-            }
-        }
-    }
-    out
-}
-
-#[allow(non_snake_case)]
-pub fn dc_job_do_DC_JOB_IMEX_IMAP(context: &Context, job: &Job) -> Result<()> {
-    ensure!(dc_alloc_ongoing(context), "could not allocate ongoing");
-    let what = job.param.get_int(Param::Cmd).unwrap_or_default();
-    let param = job.param.get(Param::Arg).unwrap_or_default();
-
-    ensure!(!param.is_empty(), "No Import/export dir/file given.");
-    info!(context, "Import/export process started.");
-    context.call_cb(Event::ImexProgress(10));
-
-    ensure!(context.sql.is_open(), "Database not opened.");
-    if what == DC_IMEX_EXPORT_BACKUP || what == DC_IMEX_EXPORT_SELF_KEYS {
-        /* before we export anything, make sure the private key exists */
-        if e2ee::ensure_secret_key_exists(context).is_err() {
-            dc_free_ongoing(context);
-            bail!("Cannot create private key or private key not available.");
-        } else {
-            dc_create_folder(context, &param);
-        }
-    }
-    let path = Path::new(param);
-    let success = match what {
-        DC_IMEX_EXPORT_SELF_KEYS => export_self_keys(context, path),
-        DC_IMEX_IMPORT_SELF_KEYS => import_self_keys(context, path),
-        DC_IMEX_EXPORT_BACKUP => unsafe { export_backup(context, path) },
-        DC_IMEX_IMPORT_BACKUP => import_backup(context, path),
-        _ => {
-            bail!("unknown IMEX type: {}", what);
-        }
-    };
-    dc_free_ongoing(context);
-    match success {
-        Ok(()) => {
-            info!(context, "IMEX successfully completed");
-            context.call_cb(Event::ImexProgress(1000));
-            Ok(())
-        }
-        Err(err) => {
-            context.call_cb(Event::ImexProgress(0));
-            bail!("IMEX FAILED to complete: {}", err);
-        }
-    }
-}
-
-/*******************************************************************************
- * Import backup
- ******************************************************************************/
-
-#[allow(non_snake_case)]
-fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Result<()> {
-    info!(
-        context,
-        "Import \"{}\" to \"{}\".",
-        backup_to_import.as_ref().display(),
-        context.get_dbfile().display()
-    );
-
-    ensure!(
-        !dc_is_configured(context),
-        "Cannot import backups to accounts in use."
-    );
-    &context.sql.close(&context);
-    dc_delete_file(context, context.get_dbfile());
-    ensure!(
-        !dc_file_exist(context, context.get_dbfile()),
-        "Cannot delete old database."
-    );
-
-    ensure!(
-        dc_copy_file(context, backup_to_import.as_ref(), context.get_dbfile()),
-        "could not copy file"
-    );
-    /* error already logged */
-    /* re-open copied database file */
-    ensure!(
-        context.sql.open(&context, &context.get_dbfile(), 0),
-        "could not re-open db"
-    );
-
-    let total_files_cnt = context
-        .sql
-        .query_get_value::<_, isize>(context, "SELECT COUNT(*) FROM backup_blobs;", params![])
-        .unwrap_or_default() as usize;
-    info!(
-        context,
-        "***IMPORT-in-progress: total_files_cnt={:?}", total_files_cnt,
-    );
-
-    let res = context.sql.query_map(
-        "SELECT file_name, file_content FROM backup_blobs ORDER BY id;",
-        params![],
-        |row| {
-            let name: String = row.get(0)?;
-            let blob: Vec<u8> = row.get(1)?;
-
-            Ok((name, blob))
-        },
-        |files| {
-            let mut processed_files_cnt = 0;
-
-            for file in files {
-                let (file_name, file_blob) = file?;
-                if context
-                    .running_state
-                    .clone()
-                    .read()
-                    .unwrap()
-                    .shall_stop_ongoing
-                {
-                    bail!("received stop signal");
-                }
-                processed_files_cnt += 1;
-                let mut permille = processed_files_cnt * 1000 / total_files_cnt;
-                if permille < 10 {
-                    permille = 10
-                }
-                if permille > 990 {
-                    permille = 990
-                }
-                context.call_cb(Event::ImexProgress(permille));
-                if file_blob.is_empty() {
-                    continue;
-                }
-
-                let pathNfilename = context.get_blobdir().join(file_name);
-                if dc_write_file(context, &pathNfilename, &file_blob) {
-                    continue;
-                }
-                bail!(
-                    "Storage full? Cannot write file {} with {} bytes.",
-                    pathNfilename.display(),
-                    file_blob.len(),
-                );
-            }
-            Ok(())
-        },
-    );
-
-    res.and_then(|_| {
-        // only delete backup_blobs if all files were successfully extracted
-        sql::execute(context, &context.sql, "DROP TABLE backup_blobs;", params![])?;
-        sql::try_execute(context, &context.sql, "VACUUM;").ok();
-        Ok(())
-    })
-}
-
-/*******************************************************************************
- * Export backup
- ******************************************************************************/
-/* the FILE_PROGRESS macro calls the callback with the permille of files processed.
-The macro avoids weird values of 0% or 100% while still working. */
-#[allow(non_snake_case)]
-unsafe fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
-    let mut ok_to_continue = true;
-    let mut success = false;
-
-    let mut delete_dest_file: libc::c_int = 0;
-    // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
-    // FIXME: we should write to a temporary file first and rename it on success. this would guarantee the backup is complete. however, currently it is not clear it the import exists in the long run (may be replaced by a restore-from-imap)
-    let now = time();
-    let res = chrono::NaiveDateTime::from_timestamp(now as i64, 0)
-        .format("delta-chat-%Y-%m-%d.bak")
-        .to_string();
-
-    let dest_path_filename = dc_get_fine_path_filename(context, dir, res);
-
-    sql::housekeeping(context);
-
-    sql::try_execute(context, &context.sql, "VACUUM;").ok();
-    context.sql.close(context);
-    info!(
-        context,
-        "Backup \"{}\" to \"{}\".",
-        context.get_dbfile().display(),
-        dest_path_filename.display(),
-    );
-    let copied = dc_copy_file(context, context.get_dbfile(), &dest_path_filename);
-    context.sql.open(&context, &context.get_dbfile(), 0);
-    if !copied {
-        let s = dest_path_filename.to_string_lossy().to_string();
-        bail!(
-            "could not copy file from {:?} to {:?}",
-            context.get_dbfile(),
-            s
-        );
-    }
-    /* add all files as blobs to the database copy (this does not require the source to be locked, neigher the destination as it is used only here) */
-    /*for logging only*/
-    let sql = Sql::new();
-    if sql.open(context, &dest_path_filename, 0) {
-        if !sql.table_exists("backup_blobs") {
-            if sql::execute(
-                context,
-                &sql,
-                "CREATE TABLE backup_blobs (id INTEGER PRIMARY KEY, file_name, file_content);",
-                params![],
-            )
-            .is_err()
-            {
-                /* error already logged */
-                ok_to_continue = false;
-            }
-        }
-        if ok_to_continue {
-            let mut total_files_cnt = 0;
-            let dir = context.get_blobdir();
-            if let Ok(dir_handle) = std::fs::read_dir(&dir) {
-                total_files_cnt += dir_handle.filter(|r| r.is_ok()).count();
-
-                info!(context, "EXPORT: total_files_cnt={}", total_files_cnt);
-                if total_files_cnt > 0 {
-                    // scan directory, pass 2: copy files
-                    if let Ok(dir_handle) = std::fs::read_dir(&dir) {
-                        sql.prepare(
-                                    "INSERT INTO backup_blobs (file_name, file_content) VALUES (?, ?);",
-                                    move |mut stmt, _| {
-                                        let mut processed_files_cnt = 0;
-                                        for entry in dir_handle {
-                                            if entry.is_err() {
-                                                break;
-                                            }
-                                            let entry = entry.unwrap();
-                                            if context
-                                                .running_state
-                                                .clone()
-                                                .read()
-                                                .unwrap()
-                                                .shall_stop_ongoing
-                                            {
-                                                delete_dest_file = 1;
-                                                ok_to_continue = false;
-                                                break;
-                                            } else {
-                                                processed_files_cnt += 1;
-                                                let mut permille =
-                                                    processed_files_cnt * 1000 / total_files_cnt;
-                                                if permille < 10 {
-                                                    permille = 10;
-                                                }
-                                                if permille > 990 {
-                                                    permille = 990;
-                                                }
-                                                context.call_cb(Event::ImexProgress(permille));
-
-                                                let name_f = entry.file_name();
-                                                let name = name_f.to_string_lossy();
-                                                if name.starts_with("delta-chat") && name.ends_with(".bak")
-                                                {
-                                                    continue;
-                                                } else {
-                                                    info!(context, "EXPORTing filename={}", name);
-                                                    let curr_pathNfilename = context.get_blobdir().join(entry.file_name());
-                                                    if let Ok(buf) =
-                                                        dc_read_file(context, &curr_pathNfilename)
-                                                    {
-                                                        if buf.is_empty() {
-                                                            continue;
-                                                        }
-                                                        if stmt.execute(params![name, buf]).is_err() {
-                                                            error!(
-                                                                context,
-                                                                "Disk full? Cannot add file \"{}\" to backup.",
-                                                                curr_pathNfilename.display(),
-                                                            );
-                                                            /* this is not recoverable! writing to the sqlite database should work! */
-                                                            ok_to_continue = false;
-                                                            break;
-                                                        }
-                                                    } else {
-                                                        continue;
-                                                    }
-                                                }
-                                            }
-                                        }
-                                        Ok(())
-                                    }
-                                ).unwrap();
-                    } else {
-                        error!(
-                            context,
-                            "Backup: Cannot copy from blob-directory \"{}\".",
-                            context.get_blobdir().display(),
-                        );
-                    }
-                } else {
-                    info!(context, "Backup: No files to copy.",);
-                }
-                if ok_to_continue {
-                    if sql
-                        .set_config_int(context, "backup_time", now as i32)
-                        .is_ok()
-                    {
-                        context.call_cb(Event::ImexFileWritten(dest_path_filename.clone()));
-                        success = true;
-                    }
-                }
-            } else {
-                error!(
-                    context,
-                    "Backup: Cannot get info for blob-directory \"{}\".",
-                    context.get_blobdir().display(),
-                );
-            };
-        }
-    }
-    if 0 != delete_dest_file {
-        dc_delete_file(context, &dest_path_filename);
-    }
-
-    ensure!(success, "failed");
-    Ok(())
-}
-
-/*******************************************************************************
- * Classic key import
- ******************************************************************************/
-fn import_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
-    /* hint: even if we switch to import Autocrypt Setup Files, we should leave the possibility to import
-    plain ASC keys, at least keys without a password, if we do not want to implement a password entry function.
-    Importing ASC keys is useful to use keys in Delta Chat used by any other non-Autocrypt-PGP implementation.
-
-    Maybe we should make the "default" key handlong also a little bit smarter
-    (currently, the last imported key is the standard key unless it contains the string "legacy" in its name) */
-    let mut set_default: bool;
-    let mut key: String;
-    let mut imported_cnt = 0;
-
-    let dir_name = dir.as_ref().to_string_lossy();
-    if let Ok(dir_handle) = std::fs::read_dir(&dir) {
-        for entry in dir_handle {
-            let entry_fn = match entry {
-                Err(err) => {
-                    info!(context, "file-dir error: {}", err);
-                    break;
-                }
-                Ok(res) => res.file_name(),
-            };
-            let name_f = entry_fn.to_string_lossy();
-            let path_plus_name = dir.as_ref().join(&entry_fn);
-            match dc_get_filesuffix_lc(&name_f) {
-                Some(suffix) => {
-                    if suffix != "asc" {
-                        continue;
-                    }
-                    set_default = if name_f.contains("legacy") {
-                        info!(context, "found legacy key '{}'", path_plus_name.display());
-                        false
-                    } else {
-                        true
-                    }
-                }
-                None => {
-                    continue;
-                }
-            }
-            let ccontent = if let Ok(content) = dc_read_file(context, &path_plus_name) {
-                key = String::from_utf8_lossy(&content).to_string();
-                CString::new(content).unwrap()
-            } else {
-                continue;
-            };
-
-            /* only import if we have a private key */
-            let mut buf2_headerline = String::default();
-            let split_res: bool;
-            unsafe {
-                let buf2 = dc_strdup(ccontent.as_ptr());
-                split_res = dc_split_armored_data(
-                    buf2,
-                    &mut buf2_headerline,
-                    ptr::null_mut(),
-                    ptr::null_mut(),
-                    ptr::null_mut(),
-                );
-                free(buf2 as *mut libc::c_void);
-            }
-            if split_res
-                && buf2_headerline.contains("-----BEGIN PGP PUBLIC KEY BLOCK-----")
-                && !key.contains("-----BEGIN PGP PRIVATE KEY BLOCK")
-            {
-                info!(context, "ignoring public key file '{}", name_f);
-                // it's fine: DC exports public with private
-                continue;
-            }
-            if let Err(err) = set_self_key(context, &key, set_default, false) {
-                error!(context, "set_self_key: {}", err);
-                continue;
-            }
-            imported_cnt += 1
-        }
-        ensure!(
-            imported_cnt > 0,
-            "No private keys found in \"{}\".",
-            dir_name
-        );
-        return Ok(());
-    } else {
-        bail!("Import: Cannot open directory \"{}\".", dir_name);
-    }
-}
-
-fn export_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
-    let mut export_errors = 0;
-
-    context.sql.query_map(
-        "SELECT id, public_key, private_key, is_default FROM keypairs;",
-        params![],
-        |row| {
-            let id = row.get(0)?;
-            let public_key_blob: Vec<u8> = row.get(1)?;
-            let public_key = Key::from_slice(&public_key_blob, KeyType::Public);
-            let private_key_blob: Vec<u8> = row.get(2)?;
-            let private_key = Key::from_slice(&private_key_blob, KeyType::Private);
-            let is_default: i32 = row.get(3)?;
-
-            Ok((id, public_key, private_key, is_default))
-        },
-        |keys| {
-            for key_pair in keys {
-                let (id, public_key, private_key, is_default) = key_pair?;
-                let id = Some(id).filter(|_| is_default != 0);
-                if let Some(key) = public_key {
-                    if !export_key_to_asc_file(context, &dir, id, &key) {
-                        export_errors += 1;
-                    }
-                } else {
-                    export_errors += 1;
-                }
-                if let Some(key) = private_key {
-                    if !export_key_to_asc_file(context, &dir, id, &key) {
-                        export_errors += 1;
-                    }
-                } else {
-                    export_errors += 1;
-                }
-            }
-
-            Ok(())
-        },
-    )?;
-
-    ensure!(export_errors == 0, "errors while importing");
-    Ok(())
-}
-
-/*******************************************************************************
- * Classic key export
- ******************************************************************************/
-fn export_key_to_asc_file(
-    context: &Context,
-    dir: impl AsRef<Path>,
-    id: Option<i64>,
-    key: &Key,
-) -> bool {
-    let mut success = false;
-    let file_name = {
-        let kind = if key.is_public() { "public" } else { "private" };
-        let id = id.map_or("default".into(), |i| i.to_string());
-
-        dir.as_ref().join(format!("{}-key-{}.asc", kind, &id))
-    };
-    info!(context, "Exporting key {}", file_name.display());
-    dc_delete_file(context, &file_name);
-
-    if !key.write_asc_to_file(&file_name, context) {
-        error!(context, "Cannot write key to {}", file_name.display());
-    } else {
-        context.call_cb(Event::ImexFileWritten(file_name));
-        success = true;
-    }
-
-    success
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    use crate::test_utils::*;
-
-    #[test]
-    fn test_render_setup_file() {
-        let t = test_context(Some(Box::new(logging_cb)));
-
-        configure_alice_keypair(&t.ctx);
-        let msg = dc_render_setup_file(&t.ctx, "hello").unwrap();
-        println!("{}", &msg);
-        // Check some substrings, indicating things got substituted.
-        // In particular note the mixing of `\r\n` and `\n` depending
-        // on who generated the stings.
-        assert!(msg.contains("<title>Autocrypt Setup Message</title"));
-        assert!(msg.contains("<h1>Autocrypt Setup Message</h1>"));
-        assert!(msg.contains("<p>This is the Autocrypt Setup Message used to"));
-        assert!(msg.contains("-----BEGIN PGP MESSAGE-----\r\n"));
-        assert!(msg.contains("Passphrase-Format: numeric9x4\r\n"));
-        assert!(msg.contains("Passphrase-Begin: he\n"));
-        assert!(msg.contains("==\n"));
-        assert!(msg.contains("-----END PGP MESSAGE-----\n"));
-    }
-
-    fn ac_setup_msg_cb(ctx: &Context, evt: Event) -> libc::uintptr_t {
-        match evt {
-            Event::GetString {
-                id: StockMessage::AcSetupMsgBody,
-                ..
-            } => unsafe { "hello\r\nthere".strdup() as usize },
-            _ => logging_cb(ctx, evt),
-        }
-    }
-
-    #[test]
-    fn test_render_setup_file_newline_replace() {
-        let t = test_context(Some(Box::new(ac_setup_msg_cb)));
-        configure_alice_keypair(&t.ctx);
-        let msg = dc_render_setup_file(&t.ctx, "pw").unwrap();
-        println!("{}", &msg);
-        assert!(msg.contains("<p>hello<br>there</p>"));
-    }
-
-    #[test]
-    fn test_create_setup_code() {
-        let t = dummy_context();
-        let setupcode = dc_create_setup_code(&t.ctx);
-        assert_eq!(setupcode.len(), 44);
-        assert_eq!(setupcode.chars().nth(4).unwrap(), '-');
-        assert_eq!(setupcode.chars().nth(9).unwrap(), '-');
-        assert_eq!(setupcode.chars().nth(14).unwrap(), '-');
-        assert_eq!(setupcode.chars().nth(19).unwrap(), '-');
-        assert_eq!(setupcode.chars().nth(24).unwrap(), '-');
-        assert_eq!(setupcode.chars().nth(29).unwrap(), '-');
-        assert_eq!(setupcode.chars().nth(34).unwrap(), '-');
-        assert_eq!(setupcode.chars().nth(39).unwrap(), '-');
-    }
-
-    #[test]
-    fn test_export_key_to_asc_file() {
-        let context = dummy_context();
-        let base64 = include_str!("../test-data/key/public.asc");
-        let key = Key::from_base64(base64, KeyType::Public).unwrap();
-        let blobdir = "$BLOBDIR";
-        assert!(export_key_to_asc_file(&context.ctx, blobdir, None, &key));
-        let blobdir = context.ctx.get_blobdir().to_str().unwrap();
-        let filename = format!("{}/public-key-default.asc", blobdir);
-        let bytes = std::fs::read(&filename).unwrap();
-
-        assert_eq!(bytes, key.to_asc(None).into_bytes());
-    }
-
-    #[test]
-    fn test_normalize_setup_code() {
-        let norm = dc_normalize_setup_code("123422343234423452346234723482349234");
-        assert_eq!(norm, "1234-2234-3234-4234-5234-6234-7234-8234-9234");
-
-        let norm = dc_normalize_setup_code(
-            "\t1 2 3422343234- foo bar-- 423-45 2 34 6234723482349234      ",
-        );
-        assert_eq!(norm, "1234-2234-3234-4234-5234-6234-7234-8234-9234");
-    }
-}

src/dc_mimeparser.rs

@@ -5,27 +5,27 @@ use std::ptr;
 use charset::Charset;
 use deltachat_derive::{FromSql, ToSql};
 use libc::{strcmp, strlen, strncmp};
-use mmime::clist::*;
 use mmime::mailimf::types::*;
 use mmime::mailimf::*;
 use mmime::mailmime::content::*;
 use mmime::mailmime::disposition::*;
 use mmime::mailmime::types::*;
 use mmime::mailmime::*;
-use mmime::mmapstring::*;
 use mmime::other::*;
 
+use crate::blob::BlobObject;
 use crate::constants::Viewtype;
 use crate::contact::*;
 use crate::context::Context;
 use crate::dc_simplify::*;
 use crate::dc_strencode::*;
 use crate::dc_tools::*;
-use crate::e2ee::*;
+use crate::e2ee;
 use crate::error::Error;
 use crate::location;
 use crate::param::*;
 use crate::stock::StockMessage;
+use crate::wrapmime;
 
 #[derive(Debug)]
 pub struct MimeParser<'a> {
@@ -38,7 +38,9 @@ pub struct MimeParser<'a> {
     pub subject: Option<String>,
     pub is_send_by_messenger: bool,
     pub decrypting_failed: bool,
-    pub e2ee_helper: E2eeHelper,
+    pub encrypted: bool,
+    pub signatures: HashSet<String>,
+    pub gossipped_addr: HashSet<String>,
     pub is_forwarded: bool,
     pub reports: Vec<*mut Mailmime>,
     pub is_system_message: SystemMessage,
@@ -92,7 +94,9 @@ impl<'a> MimeParser<'a> {
             subject: None,
             is_send_by_messenger: false,
             decrypting_failed: false,
-            e2ee_helper: Default::default(),
+            encrypted: false,
+            signatures: Default::default(),
+            gossipped_addr: Default::default(),
             is_forwarded: false,
             context,
             reports: Vec::new(),
@@ -113,14 +117,30 @@ impl<'a> MimeParser<'a> {
         );
 
         if r == MAILIMF_NO_ERROR as libc::c_int && !self.mimeroot.is_null() {
-            self.e2ee_helper.try_decrypt(self.context, self.mimeroot)?;
+            match e2ee::try_decrypt(self.context, self.mimeroot) {
+                Ok((encrypted, signatures, gossipped_addr)) => {
+                    self.encrypted = encrypted;
+                    self.signatures = signatures;
+                    self.gossipped_addr = gossipped_addr;
+                }
+                Err(err) => {
+                    // continue with the current, still encrypted, mime tree.
+                    // unencrypted parts will be replaced by an error message
+                    // that is added as "the message" to the chat then.
+                    //
+                    // if we just return here, the header is missing
+                    // and the caller cannot display the message
+                    // and try to assign the message to a chat
+                    warn!(self.context, "decryption failed: {}", err);
+                }
+            }
+
             self.parse_mime_recursive(self.mimeroot);
 
             if let Some(field) = self.lookup_field("Subject") {
                 if (*field).fld_type == MAILIMF_FIELD_SUBJECT as libc::c_int {
                     let subj = (*(*field).fld_data.fld_subject).sbj_value;
-
-                    self.subject = as_opt_str(subj).map(dc_decode_header_words_safe);
+                    self.subject = to_opt_string_lossy(subj).map(|x| dc_decode_header_words(&x));
                 }
             }
 
@@ -157,21 +177,17 @@ impl<'a> MimeParser<'a> {
                         }
                     }
                 }
-            } else {
-                if let Some(optional_field) = self.lookup_optional_field("Chat-Content") {
-                    if optional_field == "location-streaming-enabled" {
-                        self.is_system_message = SystemMessage::LocationStreamingEnabled;
-                    }
+            } else if let Some(optional_field) = self.lookup_optional_field("Chat-Content") {
+                if optional_field == "location-streaming-enabled" {
+                    self.is_system_message = SystemMessage::LocationStreamingEnabled;
                 }
             }
             if self.lookup_field("Chat-Group-Image").is_some() && !self.parts.is_empty() {
                 let textpart = &self.parts[0];
-                if textpart.typ == Viewtype::Text {
-                    if self.parts.len() >= 2 {
-                        let imgpart = &mut self.parts[1];
-                        if imgpart.typ == Viewtype::Image {
-                            imgpart.is_meta = true;
-                        }
+                if textpart.typ == Viewtype::Text && self.parts.len() >= 2 {
+                    let imgpart = &mut self.parts[1];
+                    if imgpart.typ == Viewtype::Image {
+                        imgpart.is_meta = true;
                     }
                 }
             }
@@ -182,6 +198,7 @@ impl<'a> MimeParser<'a> {
                     textpart.typ == Viewtype::Text
                         && (filepart.typ == Viewtype::Image
                             || filepart.typ == Viewtype::Gif
+                            || filepart.typ == Viewtype::Sticker
                             || filepart.typ == Viewtype::Audio
                             || filepart.typ == Viewtype::Voice
                             || filepart.typ == Viewtype::Video
@@ -249,6 +266,14 @@ impl<'a> MimeParser<'a> {
                         part_mut.typ = Viewtype::Voice;
                     }
                 }
+                if self.parts[0].typ == Viewtype::Image {
+                    if let Some(content_type) = self.lookup_optional_field("Chat-Content") {
+                        if content_type == "sticker" {
+                            let part_mut = &mut self.parts[0];
+                            part_mut.typ = Viewtype::Sticker;
+                        }
+                    }
+                }
                 let part = &self.parts[0];
                 if part.typ == Viewtype::Audio
                     || part.typ == Viewtype::Voice
@@ -270,7 +295,7 @@ impl<'a> MimeParser<'a> {
                     if self.get_last_nonmeta().is_some() {
                         let mut mb_list: *mut mailimf_mailbox_list = ptr::null_mut();
                         let mut index_0 = 0;
-                        let dn_field_c = CString::new(dn_field).unwrap();
+                        let dn_field_c = CString::new(dn_field).unwrap_or_default();
 
                         if mailimf_mailbox_list_parse(
                             dn_field_c.as_ptr(),
@@ -280,12 +305,12 @@ impl<'a> MimeParser<'a> {
                         ) == MAILIMF_NO_ERROR as libc::c_int
                             && !mb_list.is_null()
                         {
-                            if let Some(dn_to_addr) = mailimf_find_first_addr(mb_list) {
+                            if let Some(dn_to_addr) = wrapmime::mailimf_find_first_addr(mb_list) {
                                 if let Some(from_field) = self.lookup_field("From") {
                                     if (*from_field).fld_type == MAILIMF_FIELD_FROM as libc::c_int
                                         && !(*from_field).fld_data.fld_from.is_null()
                                     {
-                                        let from_addr = mailimf_find_first_addr(
+                                        let from_addr = wrapmime::mailimf_find_first_addr(
                                             (*(*from_field).fld_data.fld_from).frm_mb_list,
                                         );
                                         if let Some(from_addr) = from_addr {
@@ -583,10 +608,10 @@ impl<'a> MimeParser<'a> {
             return false;
         }
 
-        let mut decoded_data = match mailmime_transfer_decode(mime) {
+        let mut decoded_data = match wrapmime::mailmime_transfer_decode(mime) {
             Ok(decoded_data) => decoded_data,
             Err(_) => {
-                // Note that it's now always an error - might be no data
+                // Note that it's not always an error - might be no data
                 return false;
             }
         };
@@ -594,14 +619,9 @@ impl<'a> MimeParser<'a> {
         let old_part_count = self.parts.len();
 
         /* regard `Content-Transfer-Encoding:` */
-        let mut ok_to_continue = true;
         let mut desired_filename = String::default();
-        let mut simplifier: Option<Simplify> = None;
         match mime_type {
             DC_MIMETYPE_TEXT_PLAIN | DC_MIMETYPE_TEXT_HTML => {
-                if simplifier.is_none() {
-                    simplifier = Some(Simplify::new());
-                }
                 /* get from `Content-Type: text/...; charset=utf-8`; must not be free()'d */
                 let charset = mailmime_content_charset_get((*mime).mm_content_type);
                 if !charset.is_null()
@@ -609,49 +629,47 @@ impl<'a> MimeParser<'a> {
                     && strcmp(charset, b"UTF-8\x00" as *const u8 as *const libc::c_char) != 0i32
                 {
                     if let Some(encoding) =
-                        Charset::for_label(CStr::from_ptr(charset).to_str().unwrap().as_bytes())
+                        Charset::for_label(CStr::from_ptr(charset).to_string_lossy().as_bytes())
                     {
                         let (res, _, _) = encoding.decode(&decoded_data);
                         if res.is_empty() {
                             /* no error - but nothing to add */
-                            ok_to_continue = false;
-                        } else {
-                            decoded_data = res.as_bytes().to_vec()
+                            return false;
                         }
+                        decoded_data = res.as_bytes().to_vec()
                     } else {
                         warn!(
                             self.context,
                             "Cannot convert {} bytes from \"{}\" to \"utf-8\".",
                             decoded_data.len(),
-                            as_str(charset),
+                            to_string_lossy(charset),
                         );
                     }
                 }
-                if ok_to_continue {
-                    /* check header directly as is_send_by_messenger is not yet set up */
-                    let is_msgrmsg = self.lookup_optional_field("Chat-Version").is_some();
+                /* check header directly as is_send_by_messenger is not yet set up */
+                let is_msgrmsg = self.lookup_optional_field("Chat-Version").is_some();
 
-                    let simplified_txt = if decoded_data.is_empty() {
-                        "".into()
-                    } else {
-                        let input = std::string::String::from_utf8_lossy(&decoded_data);
-                        let is_html = mime_type == 70;
+                let mut simplifier = Simplify::new();
+                let simplified_txt = if decoded_data.is_empty() {
+                    "".into()
+                } else {
+                    let input = std::string::String::from_utf8_lossy(&decoded_data);
+                    let is_html = mime_type == 70;
 
-                        simplifier.unwrap().simplify(&input, is_html, is_msgrmsg)
-                    };
-                    if !simplified_txt.is_empty() {
-                        let mut part = Part::default();
-                        part.typ = Viewtype::Text;
-                        part.mimetype = mime_type;
-                        part.msg = Some(simplified_txt);
-                        part.msg_raw =
-                            Some(std::string::String::from_utf8_lossy(&decoded_data).to_string());
-                        self.do_add_single_part(part);
-                    }
+                    simplifier.simplify(&input, is_html, is_msgrmsg)
+                };
+                if !simplified_txt.is_empty() {
+                    let mut part = Part::default();
+                    part.typ = Viewtype::Text;
+                    part.mimetype = mime_type;
+                    part.msg = Some(simplified_txt);
+                    part.msg_raw =
+                        Some(std::string::String::from_utf8_lossy(&decoded_data).to_string());
+                    self.do_add_single_part(part);
+                }
 
-                    if simplifier.unwrap().is_forwarded {
-                        self.is_forwarded = true;
-                    }
+                if simplifier.is_forwarded {
+                    self.is_forwarded = true;
                 }
             }
             DC_MIMETYPE_IMAGE
@@ -697,7 +715,7 @@ impl<'a> MimeParser<'a> {
                                         // might be a wrongly encoded filename
                                         let s = to_string_lossy((*dsp_param).pa_data.pa_filename);
                                         // this is used only if the parts buffer stays empty
-                                        desired_filename = dc_decode_header_words_safe(&s)
+                                        desired_filename = dc_decode_header_words(&s)
                                     }
                                 }
                             }
@@ -723,37 +741,21 @@ impl<'a> MimeParser<'a> {
                     if !(*mime).mm_content_type.is_null()
                         && !(*(*mime).mm_content_type).ct_subtype.is_null()
                     {
-                        desired_filename =
-                            format!("file.{}", as_str((*(*mime).mm_content_type).ct_subtype));
-                    } else {
-                        ok_to_continue = false;
-                    }
-                }
-                if ok_to_continue {
-                    if desired_filename.starts_with("location")
-                        && desired_filename.ends_with(".kml")
-                    {
-                        if !decoded_data.is_empty() {
-                            let d = std::string::String::from_utf8_lossy(&decoded_data);
-                            self.location_kml = location::Kml::parse(self.context, &d).ok();
-                        }
-                    } else if desired_filename.starts_with("message")
-                        && desired_filename.ends_with(".kml")
-                    {
-                        if !decoded_data.is_empty() {
-                            let d = std::string::String::from_utf8_lossy(&decoded_data);
-                            self.message_kml = location::Kml::parse(self.context, &d).ok();
-                        }
-                    } else if !decoded_data.is_empty() {
-                        self.do_add_single_file_part(
-                            msg_type,
-                            mime_type,
-                            raw_mime.as_ref(),
-                            &decoded_data,
-                            &desired_filename,
+                        desired_filename = format!(
+                            "file.{}",
+                            to_string_lossy((*(*mime).mm_content_type).ct_subtype)
                         );
+                    } else {
+                        return false;
                     }
                 }
+                self.do_add_single_file_part(
+                    msg_type,
+                    mime_type,
+                    raw_mime.as_ref(),
+                    &decoded_data,
+                    &desired_filename,
+                );
             }
             _ => {}
         }
@@ -767,37 +769,72 @@ impl<'a> MimeParser<'a> {
         mime_type: libc::c_int,
         raw_mime: Option<&String>,
         decoded_data: &[u8],
-        desired_filename: &str,
+        filename: &str,
     ) {
-        /* create a free file name to use */
-        let path_filename = dc_get_fine_path_filename(self.context, "$BLOBDIR", desired_filename);
-
-        /* copy data to file */
-        if dc_write_file(self.context, &path_filename, decoded_data) {
-            let mut part = Part::default();
-            part.typ = msg_type;
-            part.mimetype = mime_type;
-            part.bytes = decoded_data.len() as libc::c_int;
-            part.param.set(Param::File, path_filename.to_string_lossy());
-            if let Some(raw_mime) = raw_mime {
-                part.param.set(Param::MimeType, raw_mime);
-            }
-
-            if mime_type == DC_MIMETYPE_IMAGE {
-                if let Ok((width, height)) = dc_get_filemeta(decoded_data) {
-                    part.param.set_int(Param::Width, width as i32);
-                    part.param.set_int(Param::Height, height as i32);
-                }
-            }
-            self.do_add_single_part(part);
+        if decoded_data.is_empty() {
+            return;
         }
+        // treat location/message kml file attachments specially
+        if filename.ends_with(".kml") {
+            // XXX what if somebody sends eg an "location-highlights.kml"
+            // attachment unrelated to location streaming?
+            if filename.starts_with("location") || filename.starts_with("message") {
+                let parsed = location::Kml::parse(self.context, decoded_data)
+                    .map_err(|err| {
+                        warn!(self.context, "failed to parse kml part: {}", err);
+                    })
+                    .ok();
+                if filename.starts_with("location") {
+                    self.location_kml = parsed;
+                } else {
+                    self.message_kml = parsed;
+                }
+                return;
+            }
+        }
+        /* we have a regular file attachment,
+        write decoded data to new blob object */
+
+        let blob = match BlobObject::create(self.context, filename, decoded_data) {
+            Ok(blob) => blob,
+            Err(err) => {
+                error!(
+                    self.context,
+                    "Could not add blob for mime part {}, error {}", filename, err
+                );
+                return;
+            }
+        };
+
+        /* create and register Mime part referencing the new Blob object */
+        let mut part = Part::default();
+        part.typ = msg_type;
+        part.mimetype = mime_type;
+        part.bytes = decoded_data.len() as libc::c_int;
+        part.param.set(Param::File, blob.as_name());
+        if let Some(raw_mime) = raw_mime {
+            part.param.set(Param::MimeType, raw_mime);
+        }
+
+        if mime_type == DC_MIMETYPE_IMAGE {
+            if let Ok((width, height)) = dc_get_filemeta(decoded_data) {
+                part.param.set_int(Param::Width, width as i32);
+                part.param.set_int(Param::Height, height as i32);
+            }
+        }
+        self.do_add_single_part(part);
     }
 
     fn do_add_single_part(&mut self, mut part: Part) {
-        if self.e2ee_helper.encrypted && self.e2ee_helper.signatures.len() > 0 {
-            part.param.set_int(Param::GuranteeE2ee, 1);
-        } else if self.e2ee_helper.encrypted {
-            part.param.set_int(Param::ErroneousE2ee, 0x2);
+        if self.encrypted {
+            if self.signatures.len() > 0 {
+                part.param.set_int(Param::GuaranteeE2ee, 1);
+            } else {
+                // XXX if the message was encrypted but not signed
+                // it's not neccessarily an error we need to signal.
+                // we could just treat it as if it was not encrypted.
+                part.param.set_int(Param::ErroneousE2ee, 0x2);
+            }
         }
         self.parts.push(part);
     }
@@ -825,7 +862,7 @@ impl<'a> MimeParser<'a> {
         let mut fld_from: *const mailimf_from = ptr::null();
 
         /* get From: and check there is exactly one sender */
-        let fld = mailimf_find_field(self.header_root, MAILIMF_FIELD_FROM as libc::c_int);
+        let fld = wrapmime::mailimf_find_field(self.header_root, MAILIMF_FIELD_FROM as libc::c_int);
         if !(fld.is_null()
             || {
                 fld_from = (*fld).fld_data.fld_from;
@@ -842,12 +879,11 @@ impl<'a> MimeParser<'a> {
             }) as *mut mailimf_mailbox;
 
             if !mb.is_null() {
-                let from_addr_norm = addr_normalize(as_str((*mb).mb_addr_spec));
-                let recipients = mailimf_get_recipients(self.header_root);
-                if recipients.len() == 1 {
-                    if recipients.contains(from_addr_norm) {
-                        sender_equals_recipient = true;
-                    }
+                let from_addr = to_string_lossy((*mb).mb_addr_spec);
+                let from_addr_norm = addr_normalize(&from_addr);
+                let recipients = wrapmime::mailimf_get_recipients(self.header_root);
+                if recipients.len() == 1 && recipients.contains(from_addr_norm) {
+                    sender_equals_recipient = true;
                 }
             }
         }
@@ -874,7 +910,7 @@ impl<'a> MimeParser<'a> {
             unsafe {
                 let fld_message_id = (*field).fld_data.fld_message_id;
                 if !fld_message_id.is_null() {
-                    return Some(to_string((*fld_message_id).mid_value));
+                    return Some(to_string_lossy((*fld_message_id).mid_value));
                 }
             }
         }
@@ -904,22 +940,6 @@ pub struct Part {
     pub param: Params,
 }
 
-pub fn mailimf_find_first_addr(mb_list: *const mailimf_mailbox_list) -> Option<String> {
-    if mb_list.is_null() {
-        return None;
-    }
-
-    for cur in unsafe { (*(*mb_list).mb_list).into_iter() } {
-        let mb = cur as *mut mailimf_mailbox;
-        if !mb.is_null() && !unsafe { (*mb).mb_addr_spec.is_null() } {
-            let addr = unsafe { as_str((*mb).mb_addr_spec) };
-            return Some(addr_normalize(addr).to_string());
-        }
-    }
-
-    None
-}
-
 unsafe fn hash_header(out: &mut HashMap<String, *mut mailimf_field>, in_0: *const mailimf_fields) {
     if in_0.is_null() {
         return;
@@ -990,26 +1010,25 @@ unsafe fn mailmime_get_mime_type(mime: *mut Mailmime) -> (libc::c_int, Viewtype,
                     ) == 0i32
                     {
                         return (DC_MIMETYPE_TEXT_PLAIN, Viewtype::Text, None);
-                    } else {
-                        if strcmp(
-                            (*c).ct_subtype,
-                            b"html\x00" as *const u8 as *const libc::c_char,
-                        ) == 0i32
-                        {
-                            return (DC_MIMETYPE_TEXT_HTML, Viewtype::Text, None);
-                        }
+                    } else if strcmp(
+                        (*c).ct_subtype,
+                        b"html\x00" as *const u8 as *const libc::c_char,
+                    ) == 0i32
+                    {
+                        return (DC_MIMETYPE_TEXT_HTML, Viewtype::Text, None);
                     }
                 }
 
-                let raw_mime = reconcat_mime(Some("text"), as_opt_str((*c).ct_subtype));
+                let raw_mime = reconcat_mime(Some("text"), to_opt_string_lossy((*c).ct_subtype));
                 (DC_MIMETYPE_FILE, Viewtype::File, Some(raw_mime))
             }
             MAILMIME_DISCRETE_TYPE_IMAGE => {
-                let subtype = as_opt_str((*c).ct_subtype);
-                let msg_type = match subtype {
+                let subtype = to_opt_string_lossy((*c).ct_subtype);
+                let msg_type = match subtype.as_ref().map(|x| x.as_str()) {
                     Some("gif") => Viewtype::Gif,
                     Some("svg+xml") => {
-                        let raw_mime = reconcat_mime(Some("image"), as_opt_str((*c).ct_subtype));
+                        let raw_mime =
+                            reconcat_mime(Some("image"), to_opt_string_lossy((*c).ct_subtype));
                         return (DC_MIMETYPE_FILE, Viewtype::File, Some(raw_mime));
                     }
                     _ => Viewtype::Image,
@@ -1019,11 +1038,11 @@ unsafe fn mailmime_get_mime_type(mime: *mut Mailmime) -> (libc::c_int, Viewtype,
                 (DC_MIMETYPE_IMAGE, msg_type, Some(raw_mime))
             }
             MAILMIME_DISCRETE_TYPE_AUDIO => {
-                let raw_mime = reconcat_mime(Some("audio"), as_opt_str((*c).ct_subtype));
+                let raw_mime = reconcat_mime(Some("audio"), to_opt_string_lossy((*c).ct_subtype));
                 (DC_MIMETYPE_AUDIO, Viewtype::Audio, Some(raw_mime))
             }
             MAILMIME_DISCRETE_TYPE_VIDEO => {
-                let raw_mime = reconcat_mime(Some("video"), as_opt_str((*c).ct_subtype));
+                let raw_mime = reconcat_mime(Some("video"), to_opt_string_lossy((*c).ct_subtype));
                 (DC_MIMETYPE_VIDEO, Viewtype::Video, Some(raw_mime))
             }
             _ => {
@@ -1034,13 +1053,15 @@ unsafe fn mailmime_get_mime_type(mime: *mut Mailmime) -> (libc::c_int, Viewtype,
                         b"autocrypt-setup\x00" as *const u8 as *const libc::c_char,
                     ) == 0i32
                 {
-                    let raw_mime = reconcat_mime(None, as_opt_str((*c).ct_subtype));
+                    let raw_mime = reconcat_mime(None, to_opt_string_lossy((*c).ct_subtype));
                     return (DC_MIMETYPE_AC_SETUP_FILE, Viewtype::File, Some(raw_mime));
                 }
 
                 let raw_mime = reconcat_mime(
-                    as_opt_str((*(*(*c).ct_type).tp_data.tp_discrete_type).dt_extension),
-                    as_opt_str((*c).ct_subtype),
+                    to_opt_string_lossy((*(*(*c).ct_type).tp_data.tp_discrete_type).dt_extension)
+                        .as_ref()
+                        .map(|x| x.as_str()),
+                    to_opt_string_lossy((*c).ct_subtype),
                 );
 
                 (DC_MIMETYPE_FILE, Viewtype::File, Some(raw_mime))
@@ -1050,9 +1071,9 @@ unsafe fn mailmime_get_mime_type(mime: *mut Mailmime) -> (libc::c_int, Viewtype,
             if (*(*(*c).ct_type).tp_data.tp_composite_type).ct_type
                 == MAILMIME_COMPOSITE_TYPE_MULTIPART as libc::c_int
             {
-                let subtype = as_opt_str((*c).ct_subtype);
+                let subtype = to_opt_string_lossy((*c).ct_subtype);
 
-                let mime_type = match subtype {
+                let mime_type = match subtype.as_ref().map(|x| x.as_str()) {
                     Some("alternative") => DC_MIMETYPE_MP_ALTERNATIVE,
                     Some("related") => DC_MIMETYPE_MP_RELATED,
                     Some("encrypted") => {
@@ -1087,9 +1108,9 @@ unsafe fn mailmime_get_mime_type(mime: *mut Mailmime) -> (libc::c_int, Viewtype,
     }
 }
 
-fn reconcat_mime(typ: Option<&str>, subtype: Option<&str>) -> String {
+fn reconcat_mime(typ: Option<&str>, subtype: Option<String>) -> String {
     let typ = typ.unwrap_or("application");
-    let subtype = subtype.unwrap_or("octet-stream");
+    let subtype = subtype.unwrap_or("octet-stream".to_string());
 
     format!("{}/{}", typ, subtype)
 }
@@ -1141,207 +1162,6 @@ pub unsafe fn mailmime_find_ct_parameter(
     ptr::null_mut()
 }
 
-pub unsafe fn mailmime_transfer_decode(mime: *mut Mailmime) -> Result<Vec<u8>, Error> {
-    ensure!(!mime.is_null(), "invalid inputs");
-
-    let mut mime_transfer_encoding = MAILMIME_MECHANISM_BINARY as libc::c_int;
-
-    let mime_data = (*mime).mm_data.mm_single;
-    if !(*mime).mm_mime_fields.is_null() {
-        for cur in (*(*(*mime).mm_mime_fields).fld_list).into_iter() {
-            let field = cur as *mut mailmime_field;
-
-            if !field.is_null()
-                && (*field).fld_type == MAILMIME_FIELD_TRANSFER_ENCODING as libc::c_int
-                && !(*field).fld_data.fld_encoding.is_null()
-            {
-                mime_transfer_encoding = (*(*field).fld_data.fld_encoding).enc_type;
-                break;
-            }
-        }
-    }
-
-    if mime_transfer_encoding == MAILMIME_MECHANISM_7BIT as libc::c_int
-        || mime_transfer_encoding == MAILMIME_MECHANISM_8BIT as libc::c_int
-        || mime_transfer_encoding == MAILMIME_MECHANISM_BINARY as libc::c_int
-    {
-        let decoded_data = (*mime_data).dt_data.dt_text.dt_data;
-        let decoded_data_bytes = (*mime_data).dt_data.dt_text.dt_length;
-
-        if decoded_data.is_null() || decoded_data_bytes <= 0 {
-            bail!("No data to decode found");
-        } else {
-            let result = std::slice::from_raw_parts(decoded_data as *const u8, decoded_data_bytes);
-            return Ok(result.to_vec());
-        }
-    }
-
-    let mut current_index = 0;
-    let mut transfer_decoding_buffer = ptr::null_mut();
-    let mut decoded_data_bytes = 0;
-
-    let r = mailmime_part_parse(
-        (*mime_data).dt_data.dt_text.dt_data,
-        (*mime_data).dt_data.dt_text.dt_length,
-        &mut current_index,
-        mime_transfer_encoding,
-        &mut transfer_decoding_buffer,
-        &mut decoded_data_bytes,
-    );
-
-    if r == MAILIMF_NO_ERROR as libc::c_int
-        && !transfer_decoding_buffer.is_null()
-        && decoded_data_bytes > 0
-    {
-        let result =
-            std::slice::from_raw_parts(transfer_decoding_buffer as *const u8, decoded_data_bytes)
-                .to_vec();
-        mmap_string_unref(transfer_decoding_buffer);
-
-        return Ok(result);
-    }
-
-    Err(format_err!("Failed to to decode"))
-}
-
-pub unsafe fn mailimf_get_recipients(imffields: *mut mailimf_fields) -> HashSet<String> {
-    /* returned addresses are normalized. */
-    let mut recipients: HashSet<String> = Default::default();
-
-    for cur in (*(*imffields).fld_list).into_iter() {
-        let fld = cur as *mut mailimf_field;
-
-        let fld_to: *mut mailimf_to;
-        let fld_cc: *mut mailimf_cc;
-
-        let mut addr_list: *mut mailimf_address_list = ptr::null_mut();
-        // TODO match on enums /rtn
-        match (*fld).fld_type {
-            13 => {
-                fld_to = (*fld).fld_data.fld_to;
-                if !fld_to.is_null() {
-                    addr_list = (*fld_to).to_addr_list
-                }
-            }
-            14 => {
-                fld_cc = (*fld).fld_data.fld_cc;
-                if !fld_cc.is_null() {
-                    addr_list = (*fld_cc).cc_addr_list
-                }
-            }
-            _ => {}
-        }
-
-        if !addr_list.is_null() {
-            for cur2 in (*(*addr_list).ad_list).into_iter() {
-                let adr = cur2 as *mut mailimf_address;
-
-                if !adr.is_null() {
-                    if (*adr).ad_type == MAILIMF_ADDRESS_MAILBOX as libc::c_int {
-                        mailimf_get_recipients_add_addr(&mut recipients, (*adr).ad_data.ad_mailbox);
-                    } else if (*adr).ad_type == MAILIMF_ADDRESS_GROUP as libc::c_int {
-                        let group: *mut mailimf_group = (*adr).ad_data.ad_group;
-                        if !group.is_null() && !(*group).grp_mb_list.is_null() {
-                            for cur3 in (*(*(*group).grp_mb_list).mb_list).into_iter() {
-                                mailimf_get_recipients_add_addr(
-                                    &mut recipients,
-                                    cur3 as *mut mailimf_mailbox,
-                                );
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-    recipients
-}
-
-fn mailimf_get_recipients_add_addr(recipients: &mut HashSet<String>, mb: *mut mailimf_mailbox) {
-    if !mb.is_null() {
-        let addr_norm = addr_normalize(as_str(unsafe { (*mb).mb_addr_spec }));
-        recipients.insert(addr_norm.into());
-    }
-}
-
-/*the result is a pointer to mime, must not be freed*/
-pub unsafe fn mailimf_find_field(
-    header: *mut mailimf_fields,
-    wanted_fld_type: libc::c_int,
-) -> *mut mailimf_field {
-    if header.is_null() || (*header).fld_list.is_null() {
-        return ptr::null_mut();
-    }
-    let mut cur1: *mut clistiter = (*(*header).fld_list).first;
-    while !cur1.is_null() {
-        let field: *mut mailimf_field = (if !cur1.is_null() {
-            (*cur1).data
-        } else {
-            ptr::null_mut()
-        }) as *mut mailimf_field;
-        if !field.is_null() {
-            if (*field).fld_type == wanted_fld_type {
-                return field;
-            }
-        }
-        cur1 = if !cur1.is_null() {
-            (*cur1).next
-        } else {
-            ptr::null_mut()
-        }
-    }
-
-    ptr::null_mut()
-}
-
-/*the result is a pointer to mime, must not be freed*/
-pub unsafe fn mailmime_find_mailimf_fields(mime: *mut Mailmime) -> *mut mailimf_fields {
-    if mime.is_null() {
-        return ptr::null_mut();
-    }
-
-    match (*mime).mm_type as _ {
-        MAILMIME_MULTIPLE => {
-            for cur_data in (*(*mime).mm_data.mm_multipart.mm_mp_list).into_iter() {
-                let header = mailmime_find_mailimf_fields(cur_data as *mut _);
-                if !header.is_null() {
-                    return header;
-                }
-            }
-        }
-        MAILMIME_MESSAGE => return (*mime).mm_data.mm_message.mm_fields,
-        _ => {}
-    }
-
-    ptr::null_mut()
-}
-
-pub unsafe fn mailimf_find_optional_field(
-    header: *mut mailimf_fields,
-    wanted_fld_name: *const libc::c_char,
-) -> *mut mailimf_optional_field {
-    if header.is_null() || (*header).fld_list.is_null() {
-        return ptr::null_mut();
-    }
-    for cur_data in (*(*header).fld_list).into_iter() {
-        let field: *mut mailimf_field = cur_data as *mut _;
-
-        if (*field).fld_type == MAILIMF_FIELD_OPTIONAL_FIELD as libc::c_int {
-            let optional_field: *mut mailimf_optional_field = (*field).fld_data.fld_optional_field;
-            if !optional_field.is_null()
-                && !(*optional_field).fld_name.is_null()
-                && !(*optional_field).fld_value.is_null()
-                && strcasecmp((*optional_field).fld_name, wanted_fld_name) == 0i32
-            {
-                return optional_field;
-            }
-        }
-    }
-
-    ptr::null_mut()
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -1357,14 +1177,13 @@ mod tests {
             let mut mime: *mut Mailmime = ptr::null_mut();
             let mut dummy = 0;
             let res = mailmime_parse(txt, strlen(txt), &mut dummy, &mut mime);
-
             assert_eq!(res, MAIL_NO_ERROR as libc::c_int);
             assert!(!mime.is_null());
 
-            let fields: *mut mailimf_fields = mailmime_find_mailimf_fields(mime);
+            let fields: *mut mailimf_fields = wrapmime::mailmime_find_mailimf_fields(mime);
             assert!(!fields.is_null());
 
-            let mut of_a: *mut mailimf_optional_field = mailimf_find_optional_field(
+            let mut of_a: *mut mailimf_optional_field = wrapmime::mailimf_find_optional_field(
                 fields,
                 b"fielda\x00" as *const u8 as *const libc::c_char,
             );
@@ -1384,7 +1203,7 @@ mod tests {
                 "ValueA",
             );
 
-            of_a = mailimf_find_optional_field(
+            of_a = wrapmime::mailimf_find_optional_field(
                 fields,
                 b"FIELDA\x00" as *const u8 as *const libc::c_char,
             );
@@ -1404,7 +1223,7 @@ mod tests {
                 "ValueA",
             );
 
-            let of_b: *mut mailimf_optional_field = mailimf_find_optional_field(
+            let of_b: *mut mailimf_optional_field = wrapmime::mailimf_find_optional_field(
                 fields,
                 b"FieldB\x00" as *const u8 as *const libc::c_char,
             );
@@ -1427,17 +1246,24 @@ mod tests {
         let context = dummy_context();
         let raw = include_bytes!("../test-data/message/issue_523.txt");
         let mut mimeparser = MimeParser::new(&context.ctx);
-        unsafe { mimeparser.parse(&raw[..]).unwrap() };
+        unsafe {
+            mimeparser.parse(&raw[..]).unwrap();
+        };
         assert_eq!(mimeparser.subject, None);
         assert_eq!(mimeparser.parts.len(), 1);
     }
 
     proptest! {
+        #[ignore]
         #[test]
         fn test_dc_mailmime_parse_crash_fuzzy(data in "[!-~\t ]{2000,}") {
             let context = dummy_context();
             let mut mimeparser = MimeParser::new(&context.ctx);
-            unsafe { mimeparser.parse(data.as_bytes()).unwrap() };
+
+            // parsing should always succeed
+            // but the returned header will normally be empty on random data
+            assert!(unsafe {mimeparser.parse(data.as_bytes()).is_ok()});
+            assert!(mimeparser.header.is_empty());
         }
     }
 
@@ -1466,7 +1292,7 @@ mod tests {
     fn test_mimeparser_with_context() {
         unsafe {
             let context = dummy_context();
-            let raw = b"Content-Type: multipart/mixed; boundary=\"==break==\";\nSubject: outer-subject\nX-Special-A: special-a\nFoo: Bar\nChat-Version: 0.0\n\n--==break==\nContent-Type: text/plain; protected-headers=\"v1\";\nSubject: inner-subject\nX-Special-B: special-b\nFoo: Xy\nChat-Version: 1.0\n\ntest1\n\n--==break==--\n\n\x00";
+            let raw = b"From: hello\nContent-Type: multipart/mixed; boundary=\"==break==\";\nSubject: outer-subject\nX-Special-A: special-a\nFoo: Bar\nChat-Version: 0.0\n\n--==break==\nContent-Type: text/plain; protected-headers=\"v1\";\nSubject: inner-subject\nX-Special-B: special-b\nFoo: Xy\nChat-Version: 1.0\n\ntest1\n\n--==break==--\n\n\x00";
             let mut mimeparser = MimeParser::new(&context.ctx);
             mimeparser.parse(&raw[..]).unwrap();
 

src/dc_simplify.rs

@@ -10,18 +10,16 @@ pub struct Simplify {
 ///
 /// Also return whether not-standard (rfc3676, §4.3) footer is found.
 fn find_message_footer(lines: &[&str]) -> (usize, bool) {
-    for ix in 0..lines.len() {
-        let line = lines[ix];
-
+    for (ix, &line) in lines.iter().enumerate() {
         // quoted-printable may encode `-- ` to `-- =20` which is converted
         // back to `--  `
-        match line.as_ref() {
+        match line {
             "-- " | "--  " => return (ix, false),
             "--" | "---" | "----" => return (ix, true),
             _ => (),
         }
     }
-    return (lines.len(), false);
+    (lines.len(), false)
 }
 
 impl Simplify {
@@ -103,10 +101,8 @@ impl Simplify {
             if let Some(last_quoted_line) = l_lastQuotedLine {
                 l_last = last_quoted_line;
                 is_cut_at_end = true;
-                if l_last > 1 {
-                    if is_empty_line(lines[l_last - 1]) {
-                        l_last -= 1
-                    }
+                if l_last > 1 && is_empty_line(lines[l_last - 1]) {
+                    l_last -= 1
                 }
                 if l_last > 1 {
                     let line = lines[l_last - 1];
@@ -146,8 +142,8 @@ impl Simplify {
             ret += "[...]";
         }
         /* we write empty lines only in case and non-empty line follows */
-        let mut pending_linebreaks: libc::c_int = 0i32;
-        let mut content_lines_added: libc::c_int = 0i32;
+        let mut pending_linebreaks = 0;
+        let mut content_lines_added = 0;
         for l in l_first..l_last {
             let line = lines[l];
             if is_empty_line(line) {
@@ -205,7 +201,7 @@ fn is_quoted_headline(buf: &str) -> bool {
 }
 
 fn is_plain_quote(buf: &str) -> bool {
-    buf.starts_with(">")
+    buf.starts_with('>')
 }
 
 #[cfg(test)]

src/dc_strencode.rs

@@ -4,7 +4,7 @@ use std::ffi::CString;
 use std::ptr;
 
 use charset::Charset;
-use libc::{free, strlen};
+use libc::free;
 use mmime::mailmime::decode::mailmime_encoded_phrase_parse;
 use mmime::other::*;
 use percent_encoding::{percent_decode, utf8_percent_encode, AsciiSet, CONTROLS};
@@ -38,7 +38,7 @@ pub fn dc_encode_header_words(input: impl AsRef<str>) -> String {
 fn must_encode(byte: u8) -> bool {
     static SPECIALS: &[u8] = b",:!\"#$@[\\]^`{|}~=?_";
 
-    SPECIALS.into_iter().any(|b| *b == byte)
+    SPECIALS.iter().any(|b| *b == byte)
 }
 
 fn quote_word(word: &[u8]) -> String {
@@ -68,28 +68,7 @@ fn quote_word(word: &[u8]) -> String {
  * Encode/decode header words, RFC 2047
  ******************************************************************************/
 
-pub unsafe fn dc_decode_header_words(in_0: *const libc::c_char) -> *mut libc::c_char {
-    if in_0.is_null() {
-        return ptr::null_mut();
-    }
-    let mut out: *mut libc::c_char = ptr::null_mut();
-    let mut cur_token = 0;
-    let r: libc::c_int = mailmime_encoded_phrase_parse(
-        b"iso-8859-1\x00" as *const u8 as *const libc::c_char,
-        in_0,
-        strlen(in_0),
-        &mut cur_token,
-        b"utf-8\x00" as *const u8 as *const libc::c_char,
-        &mut out,
-    );
-    if r != MAILIMF_NO_ERROR as libc::c_int || out.is_null() {
-        out = dc_strdup(in_0)
-    }
-
-    out
-}
-
-pub fn dc_decode_header_words_safe(input: &str) -> String {
+pub(crate) fn dc_decode_header_words(input: &str) -> String {
     static FROM_ENCODING: &[u8] = b"iso-8859-1\x00";
     static TO_ENCODING: &[u8] = b"utf-8\x00";
     let mut out = ptr::null_mut();
@@ -107,7 +86,7 @@ pub fn dc_decode_header_words_safe(input: &str) -> String {
         if r as u32 != MAILIMF_NO_ERROR || out.is_null() {
             input.to_string()
         } else {
-            let res = to_string(out);
+            let res = to_string_lossy(out);
             free(out.cast());
             res
         }
@@ -177,60 +156,31 @@ pub fn dc_decode_ext_header(to_decode: &[u8]) -> Cow<str> {
 mod tests {
     use super::*;
 
-    use libc::strcmp;
-    use std::ffi::CStr;
-
     #[test]
     fn test_dc_decode_header_words() {
-        unsafe {
-            let mut buf1: *mut libc::c_char = dc_decode_header_words(
-                b"=?utf-8?B?dGVzdMOkw7bDvC50eHQ=?=\x00" as *const u8 as *const libc::c_char,
-            );
-            assert_eq!(
-                strcmp(
-                    buf1,
-                    b"test\xc3\xa4\xc3\xb6\xc3\xbc.txt\x00" as *const u8 as *const libc::c_char
-                ),
-                0
-            );
-            free(buf1 as *mut libc::c_void);
+        assert_eq!(
+            dc_decode_header_words("=?utf-8?B?dGVzdMOkw7bDvC50eHQ=?="),
+            std::string::String::from_utf8(b"test\xc3\xa4\xc3\xb6\xc3\xbc.txt".to_vec()).unwrap(),
+        );
 
-            buf1 =
-                dc_decode_header_words(b"just ascii test\x00" as *const u8 as *const libc::c_char);
-            assert_eq!(CStr::from_ptr(buf1).to_str().unwrap(), "just ascii test");
-            free(buf1 as *mut libc::c_void);
+        assert_eq!(dc_decode_header_words("just ascii test"), "just ascii test");
 
-            assert_eq!(dc_encode_header_words("abcdef"), "abcdef");
+        assert_eq!(dc_encode_header_words("abcdef"), "abcdef");
 
-            let r = dc_encode_header_words(
-                std::string::String::from_utf8(b"test\xc3\xa4\xc3\xb6\xc3\xbc.txt".to_vec())
-                    .unwrap(),
+        let r = dc_encode_header_words(
+            std::string::String::from_utf8(b"test\xc3\xa4\xc3\xb6\xc3\xbc.txt".to_vec()).unwrap(),
+        );
+        assert!(r.starts_with("=?utf-8"));
+
+        assert_eq!(
+            dc_decode_header_words(&r),
+            std::string::String::from_utf8(b"test\xc3\xa4\xc3\xb6\xc3\xbc.txt".to_vec()).unwrap(),
+        );
+
+        assert_eq!(
+                dc_decode_header_words("=?ISO-8859-1?Q?attachment=3B=0D=0A_filename=3D?= =?ISO-8859-1?Q?=22test=E4=F6=FC=2Etxt=22=3B=0D=0A_size=3D39?="),
+                std::string::String::from_utf8(b"attachment;\r\n filename=\"test\xc3\xa4\xc3\xb6\xc3\xbc.txt\";\r\n size=39".to_vec()).unwrap(),
             );
-            assert!(r.starts_with("=?utf-8"));
-
-            buf1 = r.strdup();
-            let buf2: *mut libc::c_char = dc_decode_header_words(buf1);
-            assert_eq!(
-                strcmp(
-                    buf2,
-                    b"test\xc3\xa4\xc3\xb6\xc3\xbc.txt\x00" as *const u8 as *const libc::c_char
-                ),
-                0
-            );
-            free(buf2 as *mut libc::c_void);
-
-            buf1 = dc_decode_header_words(
-                b"=?ISO-8859-1?Q?attachment=3B=0D=0A_filename=3D?= =?ISO-8859-1?Q?=22test=E4=F6=FC=2Etxt=22=3B=0D=0A_size=3D39?=\x00" as *const u8 as *const libc::c_char
-            );
-            assert_eq!(
-                strcmp(
-                    buf1,
-                    b"attachment;\r\n filename=\"test\xc3\xa4\xc3\xb6\xc3\xbc.txt\";\r\n size=39\x00" as *const u8 as *const libc::c_char,
-
-                ),
-                0
-            );
-        }
     }
 
     #[test]
@@ -293,7 +243,7 @@ mod tests {
         #[test]
         fn test_dc_header_roundtrip(input: String) {
             let encoded = dc_encode_header_words(&input);
-            let decoded = dc_decode_header_words_safe(&encoded);
+            let decoded = dc_decode_header_words(&encoded);
 
             assert_eq!(input, decoded);
         }

src/dc_tools.rs

@@ -1,8 +1,9 @@
 //! Some tools and enhancements to the used libraries, there should be
 //! no references to Context and other "larger" entities here.
 
+use core::cmp::{max, min};
 use std::borrow::Cow;
-use std::ffi::{CStr, CString, OsString};
+use std::ffi::{CStr, CString};
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 use std::time::SystemTime;
@@ -16,6 +17,7 @@ use rand::{thread_rng, Rng};
 
 use crate::context::Context;
 use crate::error::Error;
+use crate::events::Event;
 
 pub(crate) fn dc_exactly_one_bit_set(v: libc::c_int) -> bool {
     0 != v && 0 == v & (v - 1)
@@ -28,11 +30,11 @@ pub(crate) fn dc_exactly_one_bit_set(v: libc::c_int) -> bool {
 /// # Examples
 ///
 /// ```
-/// use deltachat::dc_tools::{dc_strdup, to_string};
+/// use deltachat::dc_tools::{dc_strdup, to_string_lossy};
 /// unsafe {
 ///     let str_a = b"foobar\x00" as *const u8 as *const libc::c_char;
 ///     let str_a_copy = dc_strdup(str_a);
-///     assert_eq!(to_string(str_a_copy), "foobar");
+///     assert_eq!(to_string_lossy(str_a_copy), "foobar");
 ///     assert_ne!(str_a, str_a_copy);
 /// }
 /// ```
@@ -49,83 +51,6 @@ pub unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char {
     ret
 }
 
-pub(crate) fn dc_atoi_null_is_0(s: *const libc::c_char) -> libc::c_int {
-    if !s.is_null() {
-        as_str(s).parse().unwrap_or_default()
-    } else {
-        0
-    }
-}
-
-unsafe fn dc_ltrim(buf: *mut libc::c_char) {
-    let mut len: libc::size_t;
-    let mut cur: *const libc::c_uchar;
-    if !buf.is_null() && 0 != *buf as libc::c_int {
-        len = strlen(buf);
-        cur = buf as *const libc::c_uchar;
-        while 0 != *cur as libc::c_int && 0 != libc::isspace(*cur as libc::c_int) {
-            cur = cur.offset(1isize);
-            len = len.wrapping_sub(1)
-        }
-        if buf as *const libc::c_uchar != cur {
-            libc::memmove(
-                buf as *mut libc::c_void,
-                cur as *const libc::c_void,
-                len.wrapping_add(1),
-            );
-        }
-    };
-}
-
-unsafe fn dc_rtrim(buf: *mut libc::c_char) {
-    let mut len: libc::size_t;
-    let mut cur: *mut libc::c_uchar;
-    if !buf.is_null() && 0 != *buf as libc::c_int {
-        len = strlen(buf);
-        cur = (buf as *mut libc::c_uchar)
-            .offset(len as isize)
-            .offset(-1isize);
-        while cur != buf as *mut libc::c_uchar && 0 != libc::isspace(*cur as libc::c_int) {
-            cur = cur.offset(-1isize);
-            len = len.wrapping_sub(1)
-        }
-        *cur.offset(
-            (if 0 != libc::isspace(*cur as libc::c_int) {
-                0
-            } else {
-                1
-            }) as isize,
-        ) = '\u{0}' as i32 as libc::c_uchar
-    };
-}
-
-pub(crate) unsafe fn dc_trim(buf: *mut libc::c_char) {
-    dc_ltrim(buf);
-    dc_rtrim(buf);
-}
-
-/* remove all \r characters from string */
-pub(crate) unsafe fn dc_remove_cr_chars(buf: *mut libc::c_char) {
-    /* search for first `\r` */
-    let mut p1: *const libc::c_char = buf;
-    while 0 != *p1 {
-        if *p1 as libc::c_int == '\r' as i32 {
-            break;
-        }
-        p1 = p1.offset(1isize)
-    }
-    /* p1 is `\r` or null-byte; start removing `\r` */
-    let mut p2: *mut libc::c_char = p1 as *mut libc::c_char;
-    while 0 != *p1 {
-        if *p1 as libc::c_int != '\r' as i32 {
-            *p2 = *p1;
-            p2 = p2.offset(1isize)
-        }
-        p1 = p1.offset(1isize)
-    }
-    *p2 = 0 as libc::c_char;
-}
-
 /// Shortens a string to a specified length and adds "..." or "[...]" to the end of
 /// the shortened string.
 pub(crate) fn dc_truncate(buf: &str, approx_chars: usize, do_unwrap: bool) -> Cow<str> {
@@ -140,7 +65,7 @@ pub(crate) fn dc_truncate(buf: &str, approx_chars: usize, do_unwrap: bool) -> Co
             .unwrap_or_default();
 
         if let Some(index) = buf[..end_pos].rfind(|c| c == ' ' || c == '\n') {
-            Cow::Owned(format!("{}{}", &buf[..index + 1], ellipse))
+            Cow::Owned(format!("{}{}", &buf[..=index], ellipse))
         } else {
             Cow::Owned(format!("{}{}", &buf[..end_pos], ellipse))
         }
@@ -149,36 +74,18 @@ pub(crate) fn dc_truncate(buf: &str, approx_chars: usize, do_unwrap: bool) -> Co
     }
 }
 
-pub(crate) unsafe fn dc_str_from_clist(
-    list: *const clist,
-    delimiter: *const libc::c_char,
-) -> *mut libc::c_char {
+pub(crate) fn dc_str_from_clist(list: *const clist, delimiter: &str) -> String {
     let mut res = String::new();
 
     if !list.is_null() {
-        let mut cur: *mut clistiter = (*list).first;
-        while !cur.is_null() {
-            let rfc724_mid = (if !cur.is_null() {
-                (*cur).data
-            } else {
-                ptr::null_mut()
-            }) as *const libc::c_char;
-
-            if !rfc724_mid.is_null() {
-                if !res.is_empty() && !delimiter.is_null() {
-                    res += as_str(delimiter);
-                }
-                res += as_str(rfc724_mid);
-            }
-            cur = if !cur.is_null() {
-                (*cur).next
-            } else {
-                ptr::null_mut()
+        for rfc724_mid in unsafe { (*list).into_iter() } {
+            if !res.is_empty() {
+                res += delimiter;
             }
+            res += &to_string_lossy(rfc724_mid as *const libc::c_char);
         }
     }
-
-    res.strdup()
+    res
 }
 
 pub(crate) fn dc_str_to_clist(str: &str, delimiter: &str) -> *mut clist {
@@ -217,26 +124,26 @@ pub(crate) fn dc_str_to_color(s: impl AsRef<str>) -> u32 {
 
 /* date/time tools */
 /* the result is UTC or DC_INVALID_TIMESTAMP */
-pub(crate) unsafe fn dc_timestamp_from_date(date_time: *mut mailimf_date_time) -> i64 {
-    let sec = (*date_time).dt_sec;
-    let min = (*date_time).dt_min;
-    let hour = (*date_time).dt_hour;
-    let day = (*date_time).dt_day;
-    let month = (*date_time).dt_month;
-    let year = (*date_time).dt_year;
+pub(crate) fn dc_timestamp_from_date(date_time: *mut mailimf_date_time) -> i64 {
+    assert!(!date_time.is_null());
+    let dt = unsafe { *date_time };
+
+    let sec = dt.dt_sec;
+    let min = dt.dt_min;
+    let hour = dt.dt_hour;
+    let day = dt.dt_day;
+    let month = dt.dt_month;
+    let year = dt.dt_year;
 
     let ts = chrono::NaiveDateTime::new(
         chrono::NaiveDate::from_ymd(year, month as u32, day as u32),
         chrono::NaiveTime::from_hms(hour as u32, min as u32, sec as u32),
     );
 
-    let (zone_hour, zone_min) = if (*date_time).dt_zone >= 0 {
-        ((*date_time).dt_zone / 100, (*date_time).dt_zone % 100)
+    let (zone_hour, zone_min) = if dt.dt_zone >= 0 {
+        (dt.dt_zone / 100, dt.dt_zone % 100)
     } else {
-        (
-            -(-(*date_time).dt_zone / 100),
-            -(-(*date_time).dt_zone % 100),
-        )
+        (-(-dt.dt_zone / 100), -(-dt.dt_zone % 100))
     };
 
     ts.timestamp() - (zone_hour * 3600 + zone_min * 60) as i64
@@ -247,20 +154,37 @@ pub(crate) unsafe fn dc_timestamp_from_date(date_time: *mut mailimf_date_time) -
  ******************************************************************************/
 
 pub fn dc_timestamp_to_str(wanted: i64) -> String {
-    let ts = chrono::Utc.timestamp(wanted, 0);
+    let ts = Local.timestamp(wanted, 0);
     ts.format("%Y.%m.%d %H:%M:%S").to_string()
 }
 
 pub(crate) fn dc_gm2local_offset() -> i64 {
+    /* returns the offset that must be _added_ to an UTC/GMT-time to create the localtime.
+    the function may return negative values. */
     let lt = Local::now();
-    ((lt.offset().local_minus_utc() / (60 * 60)) * 100) as i64
+    lt.offset().local_minus_utc() as i64
 }
 
-/* timesmearing */
+// timesmearing
+// - as e-mails typically only use a second-based-resolution for timestamps,
+//   the order of two mails sent withing one second is unclear.
+//   this is bad eg. when forwarding some messages from a chat -
+//   these messages will appear at the recipient easily out of order.
+// - we work around this issue by not sending out two mails with the same timestamp.
+// - for this purpose, in short, we track the last timestamp used in `last_smeared_timestamp`
+//   when another timestamp is needed in the same second, we use `last_smeared_timestamp+1`
+// - after some moments without messages sent out,
+//   `last_smeared_timestamp` is again in sync with the normal time.
+// - however, we do not do all this for the far future,
+//   but at max `MAX_SECONDS_TO_LEND_FROM_FUTURE`
+const MAX_SECONDS_TO_LEND_FROM_FUTURE: i64 = 5;
+
+// returns the currently smeared timestamp,
+// may be used to check if call to dc_create_smeared_timestamp() is needed or not.
+// the returned timestamp MUST NOT be used to be sent out or saved in the database!
 pub(crate) fn dc_smeared_time(context: &Context) -> i64 {
-    /* function returns a corrected time(NULL) */
     let mut now = time();
-    let ts = *context.last_smeared_timestamp.clone().read().unwrap();
+    let ts = *context.last_smeared_timestamp.read().unwrap();
     if ts >= now {
         now = ts + 1;
     }
@@ -268,32 +192,36 @@ pub(crate) fn dc_smeared_time(context: &Context) -> i64 {
     now
 }
 
+// returns a timestamp that is guaranteed to be unique.
 pub(crate) fn dc_create_smeared_timestamp(context: &Context) -> i64 {
     let now = time();
     let mut ret = now;
 
-    let ts = *context.last_smeared_timestamp.clone().write().unwrap();
-    if ret <= ts {
-        ret = ts + 1;
-        if ret - now > 5 {
-            ret = now + 5
+    let mut last_smeared_timestamp = context.last_smeared_timestamp.write().unwrap();
+    if ret <= *last_smeared_timestamp {
+        ret = *last_smeared_timestamp + 1;
+        if ret - now > MAX_SECONDS_TO_LEND_FROM_FUTURE {
+            ret = now + MAX_SECONDS_TO_LEND_FROM_FUTURE
         }
     }
 
+    *last_smeared_timestamp = ret;
     ret
 }
 
+// creates `count` timestamps that are guaranteed to be unique.
+// the frist created timestamps is returned directly,
+// get the other timestamps just by adding 1..count-1
 pub(crate) fn dc_create_smeared_timestamps(context: &Context, count: usize) -> i64 {
-    /* get a range to timestamps that can be used uniquely */
     let now = time();
-    let start = now + (if count < 5 { count } else { 5 }) as i64 - count as i64;
+    let count = count as i64;
+    let mut start = now + min(count, MAX_SECONDS_TO_LEND_FROM_FUTURE) - count;
 
-    let ts = *context.last_smeared_timestamp.clone().write().unwrap();
-    if ts + 1 > start {
-        ts + 1
-    } else {
-        start
-    }
+    let mut last_smeared_timestamp = context.last_smeared_timestamp.write().unwrap();
+    start = max(*last_smeared_timestamp + 1, start);
+
+    *last_smeared_timestamp = start + count - 1;
+    start
 }
 
 /* Message-ID tools */
@@ -333,21 +261,19 @@ fn encode_66bits_as_base64(v1: u32, v2: u32, fill: u32) -> String {
         enc.write_u8(((fill & 0x3) as u8) << 6).unwrap();
         enc.finish().unwrap();
     }
-    assert_eq!(wrapped_writer.pop(), Some('A' as u8)); // Remove last "A"
+    assert_eq!(wrapped_writer.pop(), Some(b'A')); // Remove last "A"
     String::from_utf8(wrapped_writer).unwrap()
 }
 
 pub(crate) fn dc_create_incoming_rfc724_mid(
     message_timestamp: i64,
     contact_id_from: u32,
-    contact_ids_to: &Vec<u32>,
+    contact_ids_to: &[u32],
 ) -> Option<String> {
-    if contact_ids_to.is_empty() {
-        return None;
-    }
-    /* find out the largest receiver ID (we could also take the smallest, but it should be unique) */
-    let largest_id_to = contact_ids_to.iter().max().copied().unwrap_or_default();
+    /* create a deterministic rfc724_mid from input such that
+    repeatedly calling it with the same input results in the same Message-id */
 
+    let largest_id_to = contact_ids_to.iter().max().copied().unwrap_or_default();
     let result = format!(
         "{}-{}-{}@stub",
         message_timestamp, contact_id_from, largest_id_to
@@ -396,9 +322,9 @@ pub(crate) fn dc_extract_grpid_from_rfc724_mid_list(list: *const clist) -> *mut
     if !list.is_null() {
         unsafe {
             for cur in (*list).into_iter() {
-                let mid = as_str(cur as *const libc::c_char);
+                let mid = to_string_lossy(cur as *const libc::c_char);
 
-                if let Some(grpid) = dc_extract_grpid_from_rfc724_mid(mid) {
+                if let Some(grpid) = dc_extract_grpid_from_rfc724_mid(&mid) {
                     return grpid.strdup();
                 }
             }
@@ -415,12 +341,41 @@ pub(crate) fn dc_ensure_no_slash_safe(path: &str) -> &str {
     path
 }
 
-/// Function modifies the given buffer and replaces all characters not valid in filenames by a "-".
-fn validate_filename(filename: &str) -> String {
-    filename
-        .replace('/', "-")
-        .replace('\\', "-")
-        .replace(':', "-")
+// Function returns a sanitized basename that does not contain
+// win/linux path separators and also not any non-ascii chars
+fn get_safe_basename(filename: &str) -> String {
+    // return the (potentially mangled) basename of the input filename
+    // this might be a path that comes in from another operating system
+    let mut index: usize = 0;
+
+    if let Some(unix_index) = filename.rfind('/') {
+        index = unix_index + 1;
+    }
+    if let Some(win_index) = filename.rfind('\\') {
+        index = max(index, win_index + 1);
+    }
+    if index >= filename.len() {
+        "nobasename".to_string()
+    } else {
+        // we don't allow any non-ascii to be super-safe
+        filename[index..].replace(|c: char| !c.is_ascii() || c == ':', "-")
+    }
+}
+
+pub fn dc_derive_safe_stem_ext(filename: &str) -> (String, String) {
+    let basename = get_safe_basename(&filename);
+    let (mut stem, mut ext) = if let Some(index) = basename.rfind('.') {
+        (
+            basename[0..index].to_string(),
+            basename[index..].to_string(),
+        )
+    } else {
+        (basename, "".to_string())
+    };
+    // limit length of stem and ext
+    stem.truncate(32);
+    ext.truncate(32);
+    (stem, ext)
 }
 
 // the returned suffix is lower-case
@@ -456,10 +411,6 @@ pub(crate) fn dc_get_abs_path<P: AsRef<std::path::Path>>(
     }
 }
 
-pub(crate) fn dc_file_exist(context: &Context, path: impl AsRef<std::path::Path>) -> bool {
-    dc_get_abs_path(context, &path).exists()
-}
-
 pub(crate) fn dc_get_filebytes(context: &Context, path: impl AsRef<std::path::Path>) -> u64 {
     let path_abs = dc_get_abs_path(context, &path);
     match fs::metadata(&path_abs) {
@@ -482,10 +433,14 @@ pub(crate) fn dc_delete_file(context: &Context, path: impl AsRef<std::path::Path
         return false;
     }
 
+    let dpath = format!("{}", path.as_ref().to_string_lossy());
     match fs::remove_file(path_abs) {
-        Ok(_) => true,
-        Err(_err) => {
-            warn!(context, "Cannot delete \"{}\".", path.as_ref().display());
+        Ok(_) => {
+            context.call_cb(Event::DeletedBlobFile(dpath));
+            true
+        }
+        Err(err) => {
+            warn!(context, "Cannot delete \"{}\": {}", dpath, err);
             false
         }
     }
@@ -493,20 +448,55 @@ pub(crate) fn dc_delete_file(context: &Context, path: impl AsRef<std::path::Path
 
 pub(crate) fn dc_copy_file(
     context: &Context,
-    src: impl AsRef<std::path::Path>,
-    dest: impl AsRef<std::path::Path>,
+    src_path: impl AsRef<std::path::Path>,
+    dest_path: impl AsRef<std::path::Path>,
 ) -> bool {
-    let src_abs = dc_get_abs_path(context, &src);
-    let dest_abs = dc_get_abs_path(context, &dest);
-    match fs::copy(&src_abs, &dest_abs) {
+    let src_abs = dc_get_abs_path(context, &src_path);
+    let mut src_file = match fs::File::open(&src_abs) {
+        Ok(file) => file,
+        Err(err) => {
+            warn!(
+                context,
+                "failed to open for read '{}': {}",
+                src_abs.display(),
+                err
+            );
+            return false;
+        }
+    };
+
+    let dest_abs = dc_get_abs_path(context, &dest_path);
+    let mut dest_file = match fs::OpenOptions::new()
+        .create_new(true)
+        .write(true)
+        .open(&dest_abs)
+    {
+        Ok(file) => file,
+        Err(err) => {
+            warn!(
+                context,
+                "failed to open for write '{}': {}",
+                dest_abs.display(),
+                err
+            );
+            return false;
+        }
+    };
+
+    match std::io::copy(&mut src_file, &mut dest_file) {
         Ok(_) => true,
-        Err(_) => {
+        Err(err) => {
             error!(
                 context,
-                "Cannot copy \"{}\" to \"{}\".",
-                src.as_ref().display(),
-                dest.as_ref().display(),
+                "Cannot copy \"{}\" to \"{}\": {}",
+                src_abs.display(),
+                dest_abs.display(),
+                err
             );
+            {
+                // Attempt to remove the failed file, swallow errors resulting from that.
+                fs::remove_file(dest_abs).ok();
+            }
             false
         }
     }
@@ -517,11 +507,12 @@ pub(crate) fn dc_create_folder(context: &Context, path: impl AsRef<std::path::Pa
     if !path_abs.exists() {
         match fs::create_dir_all(path_abs) {
             Ok(_) => true,
-            Err(_err) => {
+            Err(err) => {
                 warn!(
                     context,
-                    "Cannot create directory \"{}\".",
+                    "Cannot create directory \"{}\": {}",
                     path.as_ref().display(),
+                    err
                 );
                 false
             }
@@ -532,19 +523,22 @@ pub(crate) fn dc_create_folder(context: &Context, path: impl AsRef<std::path::Pa
 }
 
 /// Write a the given content to provied file path.
-pub(crate) fn dc_write_file(context: &Context, path: impl AsRef<Path>, buf: &[u8]) -> bool {
+pub(crate) fn dc_write_file(
+    context: &Context,
+    path: impl AsRef<Path>,
+    buf: &[u8],
+) -> Result<(), std::io::Error> {
     let path_abs = dc_get_abs_path(context, &path);
-    if let Err(_err) = fs::write(&path_abs, buf) {
+    fs::write(&path_abs, buf).map_err(|err| {
         warn!(
             context,
-            "Cannot write {} bytes to \"{}\".",
+            "Cannot write {} bytes to \"{}\": {}",
             buf.len(),
             path.as_ref().display(),
+            err
         );
-        false
-    } else {
-        true
-    }
+        err
+    })
 }
 
 pub fn dc_read_file<P: AsRef<std::path::Path>>(
@@ -558,92 +552,53 @@ pub fn dc_read_file<P: AsRef<std::path::Path>>(
         Err(err) => {
             warn!(
                 context,
-                "Cannot read \"{}\" or file is empty.",
-                path.as_ref().display()
+                "Cannot read \"{}\" or file is empty: {}",
+                path.as_ref().display(),
+                err
             );
             Err(err.into())
         }
     }
 }
 
-pub(crate) fn dc_get_fine_path_filename(
+pub fn dc_open_file<P: AsRef<std::path::Path>>(
     context: &Context,
-    folder: impl AsRef<Path>,
-    desired_filename_suffix: impl AsRef<str>,
-) -> PathBuf {
-    let now = time();
+    path: P,
+) -> Result<std::fs::File, Error> {
+    let path_abs = dc_get_abs_path(context, &path);
 
-    let folder = PathBuf::from(folder.as_ref());
-    // XXX sanitize desired_filename eg using
-    // https://github.com/kardeiz/sanitize-filename/blob/master/src/lib.rs
-    let suffix = validate_filename(desired_filename_suffix.as_ref());
-    let file_name = PathBuf::from(suffix);
-    let extension = file_name.extension().map(|c| c.clone());
-
-    for i in 0..100_000 {
-        let ret = if i == 0 {
-            let mut folder = folder.clone();
-            folder.push(&file_name);
-            folder
-        } else {
-            let idx = if i < 100 { i } else { now + i };
-            let file_name = if let Some(stem) = file_name.file_stem() {
-                let mut stem = stem.to_os_string();
-                stem.push(format!("-{}", idx));
-                stem
-            } else {
-                OsString::from(idx.to_string())
-            };
-            let mut folder = folder.clone();
-            folder.push(file_name);
-            if let Some(ext) = extension {
-                folder.set_extension(&ext);
-            }
-            folder
-        };
-
-        if !dc_file_exist(context, &ret) {
-            // fine filename found
-            return ret;
+    match fs::File::open(&path_abs) {
+        Ok(bytes) => Ok(bytes),
+        Err(err) => {
+            warn!(
+                context,
+                "Cannot read \"{}\" or file is empty: {}",
+                path.as_ref().display(),
+                err
+            );
+            Err(err.into())
         }
     }
-
-    panic!("Something is really wrong, you need to clean up your disk");
 }
 
-pub(crate) fn dc_is_blobdir_path(context: &Context, path: impl AsRef<str>) -> bool {
-    context
-        .get_blobdir()
-        .to_str()
-        .map(|s| path.as_ref().starts_with(s))
-        .unwrap_or_default()
-        || path.as_ref().starts_with("$BLOBDIR")
-}
+pub(crate) fn dc_get_next_backup_path(
+    folder: impl AsRef<Path>,
+    backup_time: i64,
+) -> Result<PathBuf, Error> {
+    let folder = PathBuf::from(folder.as_ref());
+    let stem = chrono::NaiveDateTime::from_timestamp(backup_time, 0)
+        .format("delta-chat-%Y-%m-%d")
+        .to_string();
 
-fn dc_make_rel_path(context: &Context, path: &mut String) {
-    if context
-        .get_blobdir()
-        .to_str()
-        .map(|s| path.starts_with(s))
-        .unwrap_or_default()
-    {
-        *path = path.replace(context.get_blobdir().to_str().unwrap(), "$BLOBDIR");
+    // 64 backup files per day should be enough for everyone
+    for i in 0..64 {
+        let mut path = folder.clone();
+        path.push(format!("{}-{}.bak", stem, i));
+        if !path.exists() {
+            return Ok(path);
+        }
     }
-}
-
-pub(crate) fn dc_make_rel_and_copy(context: &Context, path: &mut String) -> bool {
-    if dc_is_blobdir_path(context, &path) {
-        dc_make_rel_path(context, path);
-        return true;
-    }
-    let blobdir_path = dc_get_fine_path_filename(context, "$BLOBDIR", &path);
-    if dc_copy_file(context, &path, &blobdir_path) {
-        *path = blobdir_path.to_string_lossy().to_string();
-        dc_make_rel_path(context, path);
-        return true;
-    }
-
-    false
+    bail!("could not create backup file, disk full?");
 }
 
 /// Error type for the [OsStrExt] trait
@@ -791,22 +746,6 @@ impl<T: AsRef<str>> StrExt for T {
     }
 }
 
-pub fn to_string(s: *const libc::c_char) -> String {
-    if s.is_null() {
-        return "".into();
-    }
-
-    let cstr = unsafe { CStr::from_ptr(s) };
-
-    cstr.to_str().map(|s| s.to_string()).unwrap_or_else(|err| {
-        panic!(
-            "Non utf8 string: '{:?}' ({:?})",
-            cstr.to_string_lossy(),
-            err
-        );
-    })
-}
-
 pub fn to_string_lossy(s: *const libc::c_char) -> String {
     if s.is_null() {
         return "".into();
@@ -814,30 +753,15 @@ pub fn to_string_lossy(s: *const libc::c_char) -> String {
 
     let cstr = unsafe { CStr::from_ptr(s) };
 
-    cstr.to_str()
-        .map(|s| s.to_string())
-        .unwrap_or_else(|_| cstr.to_string_lossy().to_string())
+    cstr.to_string_lossy().to_string()
 }
 
-pub fn as_str<'a>(s: *const libc::c_char) -> &'a str {
-    as_str_safe(s).unwrap_or_else(|err| panic!("{}", err))
-}
-
-/// Converts a C string to either a Rust `&str` or `None` if  it is a null pointer.
-pub fn as_opt_str<'a>(s: *const libc::c_char) -> Option<&'a str> {
+pub fn to_opt_string_lossy(s: *const libc::c_char) -> Option<String> {
     if s.is_null() {
         return None;
     }
-    Some(as_str(s))
-}
 
-fn as_str_safe<'a>(s: *const libc::c_char) -> Result<&'a str, Error> {
-    assert!(!s.is_null(), "cannot be used on null pointers");
-
-    let cstr = unsafe { CStr::from_ptr(s) };
-
-    cstr.to_str()
-        .map_err(|err| format_err!("Non utf8 string: '{:?}' ({:?})", cstr.to_bytes(), err))
+    Some(to_string_lossy(s))
 }
 
 /// Convert a C `*char` pointer to a [std::path::Path] slice.
@@ -876,7 +800,11 @@ pub fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path {
 #[allow(dead_code)]
 fn as_path_unicode<'a>(s: *const libc::c_char) -> &'a std::path::Path {
     assert!(!s.is_null(), "cannot be used on null pointers");
-    std::path::Path::new(as_str(s))
+
+    let cstr = unsafe { CStr::from_ptr(s) };
+    let str = cstr.to_str().unwrap_or_else(|err| panic!("{}", err));
+
+    std::path::Path::new(str)
 }
 
 pub(crate) fn time() -> i64 {
@@ -1021,54 +949,6 @@ mod tests {
         }
     }
 
-    #[test]
-    fn test_dc_ltrim() {
-        unsafe {
-            let html: *const libc::c_char =
-                b"\r\r\nline1<br>\r\n\r\n\r\rline2\n\r\x00" as *const u8 as *const libc::c_char;
-            let out: *mut libc::c_char = strndup(html, strlen(html) as libc::c_ulong);
-
-            dc_ltrim(out);
-
-            assert_eq!(
-                CStr::from_ptr(out as *const libc::c_char).to_str().unwrap(),
-                "line1<br>\r\n\r\n\r\rline2\n\r"
-            );
-        }
-    }
-
-    #[test]
-    fn test_dc_rtrim() {
-        unsafe {
-            let html: *const libc::c_char =
-                b"\r\r\nline1<br>\r\n\r\n\r\rline2\n\r\x00" as *const u8 as *const libc::c_char;
-            let out: *mut libc::c_char = strndup(html, strlen(html) as libc::c_ulong);
-
-            dc_rtrim(out);
-
-            assert_eq!(
-                CStr::from_ptr(out as *const libc::c_char).to_str().unwrap(),
-                "\r\r\nline1<br>\r\n\r\n\r\rline2"
-            );
-        }
-    }
-
-    #[test]
-    fn test_dc_trim() {
-        unsafe {
-            let html: *const libc::c_char =
-                b"\r\r\nline1<br>\r\n\r\n\r\rline2\n\r\x00" as *const u8 as *const libc::c_char;
-            let out: *mut libc::c_char = strndup(html, strlen(html) as libc::c_ulong);
-
-            dc_trim(out);
-
-            assert_eq!(
-                CStr::from_ptr(out as *const libc::c_char).to_str().unwrap(),
-                "line1<br>\r\n\r\n\r\rline2"
-            );
-        }
-    }
-
     #[test]
     fn test_rust_ftoa() {
         assert_eq!("1.22", format!("{}", 1.22));
@@ -1144,21 +1024,6 @@ mod tests {
         }
     }
 
-    fn strndup(s: *const libc::c_char, n: libc::c_ulong) -> *mut libc::c_char {
-        if s.is_null() {
-            return std::ptr::null_mut();
-        }
-
-        let end = std::cmp::min(n as usize, unsafe { strlen(s) });
-        unsafe {
-            let result = libc::malloc(end + 1);
-            memcpy(result, s as *const _, end);
-            std::ptr::write_bytes(result.offset(end as isize), b'\x00', 1);
-
-            result as *mut _
-        }
-    }
-
     #[test]
     fn test_dc_str_to_clist_1() {
         unsafe {
@@ -1174,17 +1039,11 @@ mod tests {
         unsafe {
             let list: *mut clist = dc_str_to_clist("foo bar test", " ");
             assert_eq!((*list).count, 3);
-            let str: *mut libc::c_char =
-                dc_str_from_clist(list, b" \x00" as *const u8 as *const libc::c_char);
-
-            assert_eq!(
-                CStr::from_ptr(str as *const libc::c_char).to_str().unwrap(),
-                "foo bar test"
-            );
+            let str = dc_str_from_clist(list, " ");
+            assert_eq!(str, "foo bar test");
 
             clist_free_content(list);
             clist_free(list);
-            free(str as *mut libc::c_void);
         }
     }
 
@@ -1345,6 +1204,25 @@ mod tests {
         assert_eq!(grpid, Some("1234567890123456"));
     }
 
+    #[test]
+    fn test_dc_create_outgoing_rfc724_mid() {
+        // create a normal message-id
+        let mid = dc_create_outgoing_rfc724_mid(None, "foo@bar.de");
+        assert!(mid.starts_with("Mr."));
+        assert!(mid.ends_with("bar.de"));
+        assert!(dc_extract_grpid_from_rfc724_mid(mid.as_str()).is_none());
+
+        // create a message-id containing a group-id
+        let grpid = dc_create_id();
+        let mid = dc_create_outgoing_rfc724_mid(Some(&grpid), "foo@bar.de");
+        assert!(mid.starts_with("Gr."));
+        assert!(mid.ends_with("bar.de"));
+        assert_eq!(
+            dc_extract_grpid_from_rfc724_mid(mid.as_str()),
+            Some(grpid.as_str())
+        );
+    }
+
     #[test]
     fn test_emailaddress_parse() {
         assert_eq!(EmailAddress::new("").is_ok(), false);
@@ -1409,38 +1287,32 @@ mod tests {
     fn test_dc_create_incoming_rfc724_mid() {
         let res = dc_create_incoming_rfc724_mid(123, 45, &vec![6, 7]);
         assert_eq!(res, Some("123-45-7@stub".into()));
+        let res = dc_create_incoming_rfc724_mid(123, 45, &vec![]);
+        assert_eq!(res, Some("123-45-0@stub".into()));
     }
 
     #[test]
-    fn test_dc_make_rel_path() {
-        let t = dummy_context();
-        let mut foo: String = t
-            .ctx
-            .get_blobdir()
-            .join("foo")
-            .to_string_lossy()
-            .into_owned();
-        dc_make_rel_path(&t.ctx, &mut foo);
-        assert_eq!(foo, format!("$BLOBDIR{}foo", std::path::MAIN_SEPARATOR));
-    }
-
-    #[test]
-    fn test_strndup() {
-        unsafe {
-            let res = strndup(b"helloworld\x00" as *const u8 as *const libc::c_char, 4);
-            assert_eq!(
-                to_string(res),
-                to_string(b"hell\x00" as *const u8 as *const libc::c_char)
-            );
-            assert_eq!(strlen(res), 4);
-            free(res as *mut _);
-        }
+    fn test_file_get_safe_basename() {
+        assert_eq!(get_safe_basename("12312/hello"), "hello");
+        assert_eq!(get_safe_basename("12312\\hello"), "hello");
+        assert_eq!(get_safe_basename("//12312\\hello"), "hello");
+        assert_eq!(get_safe_basename("//123:12\\hello"), "hello");
+        assert_eq!(get_safe_basename("//123:12/\\\\hello"), "hello");
+        assert_eq!(get_safe_basename("//123:12//hello"), "hello");
+        assert_eq!(get_safe_basename("//123:12//"), "nobasename");
+        assert_eq!(get_safe_basename("//123:12/"), "nobasename");
+        assert!(get_safe_basename("123\x012.hello").ends_with(".hello"));
     }
 
     #[test]
     fn test_file_handling() {
         let t = dummy_context();
         let context = &t.ctx;
+        let dc_file_exist = |ctx: &Context, fname: &str| {
+            ctx.get_blobdir()
+                .join(Path::new(fname).file_name().unwrap())
+                .exists()
+        };
 
         assert!(!dc_delete_file(context, "$BLOBDIR/lkqwjelqkwlje"));
         if dc_file_exist(context, "$BLOBDIR/foobar")
@@ -1453,7 +1325,7 @@ mod tests {
             dc_delete_file(context, "$BLOBDIR/foobar.dadada");
             dc_delete_file(context, "$BLOBDIR/foobar-folder");
         }
-        assert!(dc_write_file(context, "$BLOBDIR/foobar", b"content"));
+        assert!(dc_write_file(context, "$BLOBDIR/foobar", b"content").is_ok());
         assert!(dc_file_exist(context, "$BLOBDIR/foobar",));
         assert!(!dc_file_exist(context, "$BLOBDIR/foobarx"));
         assert_eq!(dc_get_filebytes(context, "$BLOBDIR/foobar"), 7);
@@ -1464,13 +1336,13 @@ mod tests {
             .to_string_lossy()
             .to_string();
 
-        assert!(dc_is_blobdir_path(context, &abs_path));
-
-        assert!(dc_is_blobdir_path(context, "$BLOBDIR/fofo",));
-        assert!(!dc_is_blobdir_path(context, "/BLOBDIR/fofo",));
         assert!(dc_file_exist(context, &abs_path));
 
         assert!(dc_copy_file(context, "$BLOBDIR/foobar", "$BLOBDIR/dada",));
+
+        // attempting to copy a second time should fail
+        assert!(!dc_copy_file(context, "$BLOBDIR/foobar", "$BLOBDIR/dada",));
+
         assert_eq!(dc_get_filebytes(context, "$BLOBDIR/dada",), 7);
 
         let buf = dc_read_file(context, "$BLOBDIR/dada").unwrap();
@@ -1483,14 +1355,12 @@ mod tests {
         assert!(dc_create_folder(context, "$BLOBDIR/foobar-folder"));
         assert!(dc_file_exist(context, "$BLOBDIR/foobar-folder",));
         assert!(!dc_delete_file(context, "$BLOBDIR/foobar-folder"));
-        let fn0 = dc_get_fine_path_filename(context, "$BLOBDIR", "foobar.dadada");
-        assert_eq!(fn0, PathBuf::from("$BLOBDIR/foobar.dadada"));
 
-        assert!(dc_write_file(context, &fn0, b"content"));
-        let fn1 = dc_get_fine_path_filename(context, "$BLOBDIR", "foobar.dadada");
-        assert_eq!(fn1, PathBuf::from("$BLOBDIR/foobar-1.dadada"));
+        let fn0 = "$BLOBDIR/data.data";
+        assert!(dc_write_file(context, &fn0, b"content").is_ok());
 
         assert!(dc_delete_file(context, &fn0));
+        assert!(!dc_file_exist(context, &fn0));
     }
 
     #[test]
@@ -1508,17 +1378,32 @@ mod tests {
     }
 
     #[test]
-    fn test_dc_remove_cr_chars() {
-        unsafe {
-            let input = "foo\r\nbar".strdup();
-            dc_remove_cr_chars(input);
-            assert_eq!("foo\nbar", to_string(input));
-            free(input.cast());
+    fn test_create_smeared_timestamp() {
+        let t = dummy_context();
+        assert_ne!(
+            dc_create_smeared_timestamp(&t.ctx),
+            dc_create_smeared_timestamp(&t.ctx)
+        );
+        assert!(
+            dc_create_smeared_timestamp(&t.ctx)
+                >= SystemTime::now()
+                    .duration_since(SystemTime::UNIX_EPOCH)
+                    .unwrap()
+                    .as_secs() as i64
+        );
+    }
 
-            let input = "\rfoo\r\rbar\r".strdup();
-            dc_remove_cr_chars(input);
-            assert_eq!("foobar", to_string(input));
-            free(input.cast());
-        }
+    #[test]
+    fn test_create_smeared_timestamps() {
+        let t = dummy_context();
+        let count = MAX_SECONDS_TO_LEND_FROM_FUTURE - 1;
+        let start = dc_create_smeared_timestamps(&t.ctx, count as usize);
+        let next = dc_smeared_time(&t.ctx);
+        assert!((start + count - 1) < next);
+
+        let count = MAX_SECONDS_TO_LEND_FROM_FUTURE + 30;
+        let start = dc_create_smeared_timestamps(&t.ctx, count as usize);
+        let next = dc_smeared_time(&t.ctx);
+        assert!((start + count - 1) < next);
     }
 }

src/e2ee.rs

@@ -1,11 +1,10 @@
 //! End-to-end encryption support.
 
 use std::collections::HashSet;
-use std::ffi::CStr;
 use std::ptr;
 use std::str::FromStr;
 
-use libc::{strcmp, strlen, strncmp};
+use libc::strlen;
 use mmime::clist::*;
 use mmime::mailimf::types::*;
 use mmime::mailimf::types_helper::*;
@@ -18,18 +17,18 @@ use mmime::mailmime::*;
 use mmime::mailprivacy_prepare_mime;
 use mmime::mmapstring::*;
 use mmime::{mailmime_substitute, MAILIMF_NO_ERROR, MAIL_NO_ERROR};
+use num_traits::FromPrimitive;
 
 use crate::aheader::*;
 use crate::config::Config;
 use crate::context::Context;
-use crate::dc_mimeparser::*;
 use crate::dc_tools::*;
 use crate::error::*;
 use crate::key::*;
 use crate::keyring::*;
 use crate::mimefactory::MimeFactory;
 use crate::peerstate::*;
-use crate::pgp::*;
+use crate::pgp;
 use crate::securejoin::handle_degrade_event;
 use crate::wrapmime;
 use crate::wrapmime::*;
@@ -47,12 +46,9 @@ pub struct EncryptHelper {
 
 impl EncryptHelper {
     pub fn new(context: &Context) -> Result<EncryptHelper> {
-        let e2ee = context.sql.get_config_int(&context, "e2ee_enabled");
-        let prefer_encrypt = if 0 != e2ee.unwrap_or_default() {
-            EncryptPreference::Mutual
-        } else {
-            EncryptPreference::NoPreference
-        };
+        let prefer_encrypt =
+            EncryptPreference::from_i32(context.get_config_int(Config::E2eeEnabled))
+                .unwrap_or_default();
         let addr = match context.get_config(Config::ConfiguredAddr) {
             None => {
                 bail!("addr not configured!");
@@ -61,6 +57,7 @@ impl EncryptHelper {
         };
 
         let public_key = load_or_generate_self_public_key(context, &addr)?;
+
         Ok(EncryptHelper {
             prefer_encrypt,
             addr,
@@ -78,16 +75,18 @@ impl EncryptHelper {
         &mut self,
         factory: &mut MimeFactory,
         e2ee_guaranteed: bool,
-        min_verified: libc::c_int,
+        min_verified: PeerstateVerifiedStatus,
         do_gossip: bool,
         mut in_out_message: *mut Mailmime,
         imffields_unprotected: *mut mailimf_fields,
     ) -> Result<bool> {
-        /* libEtPan's pgp_encrypt_mime() takes the parent as the new root.
-        We just expect the root as being given to this function. */
-        if in_out_message.is_null() || unsafe { !(*in_out_message).mm_parent.is_null() } {
-            bail!("corrupted inputs");
-        }
+        // libEtPan's pgp_encrypt_mime() takes the parent as the new root.
+        // We just expect the root as being given to this function.
+        ensure!(
+            !in_out_message.is_null() && unsafe { (*in_out_message).mm_parent.is_null() },
+            "corrupted inputs"
+        );
+
         if !(self.prefer_encrypt == EncryptPreference::Mutual || e2ee_guaranteed) {
             return Ok(false);
         }
@@ -106,22 +105,23 @@ impl EncryptHelper {
                 None => {
                     let msg = format!("peerstate for {} missing, cannot encrypt", recipient_addr);
                     if e2ee_guaranteed {
-                        bail!("{}", msg);
+                        return Err(format_err!("{}", msg));
                     } else {
                         info!(context, "{}", msg);
                         return Ok(false);
                     }
                 }
             };
+
             if peerstate.prefer_encrypt != EncryptPreference::Mutual && !e2ee_guaranteed {
                 info!(context, "peerstate for {} is no-encrypt", recipient_addr);
                 return Ok(false);
             }
 
-            if let Some(key) = peerstate.peek_key(min_verified as usize) {
+            if let Some(key) = peerstate.peek_key(min_verified) {
                 keyring.add_owned(key.clone());
                 if do_gossip {
-                    if let Some(header) = peerstate.render_gossip_header(min_verified as usize) {
+                    if let Some(header) = peerstate.render_gossip_header(min_verified) {
                         gossip_headers.push(header.to_string());
                     }
                 }
@@ -136,30 +136,24 @@ impl EncryptHelper {
         let sign_key = {
             keyring.add_ref(&self.public_key);
             let key = Key::from_self_private(context, self.addr.clone(), &context.sql);
-            if key.is_none() {
-                bail!("no own private key found")
-            }
+            ensure!(key.is_some(), "no own private key found");
+
             key
         };
 
-        /* encrypt message */
+        // encrypt message
         unsafe {
             mailprivacy_prepare_mime(in_out_message);
-            let mut part_to_encrypt: *mut Mailmime =
-                (*in_out_message).mm_data.mm_message.mm_msg_mime;
+            let mut part_to_encrypt = (*in_out_message).mm_data.mm_message.mm_msg_mime;
             (*part_to_encrypt).mm_parent = ptr::null_mut();
-            let imffields_encrypted: *mut mailimf_fields = mailimf_fields_new_empty();
-            /* mailmime_new_message_data() calls mailmime_fields_new_with_version() which would add the unwanted MIME-Version:-header */
-            let message_to_encrypt: *mut Mailmime = mailmime_new(
+            let imffields_encrypted = mailimf_fields_new_empty();
+
+            // mailmime_new_message_data() calls mailmime_fields_new_with_version()
+            // which would add the unwanted MIME-Version:-header
+            let message_to_encrypt = mailmime_new_simple(
                 MAILMIME_MESSAGE as libc::c_int,
-                ptr::null(),
-                0 as libc::size_t,
                 mailmime_fields_new_empty(),
                 mailmime_get_content_message(),
-                ptr::null_mut(),
-                ptr::null_mut(),
-                ptr::null_mut(),
-                ptr::null_mut(),
                 imffields_encrypted,
                 part_to_encrypt,
             );
@@ -168,12 +162,13 @@ impl EncryptHelper {
                 wrapmime::new_custom_field(imffields_encrypted, "Autocrypt-Gossip", &header)
             }
 
-            /* memoryhole headers: move some headers into encrypted part */
+            // memoryhole headers: move some headers into encrypted part
             // XXX note we can't use clist's into_iter() because the loop body also removes items
-            let mut cur: *mut clistiter = (*(*imffields_unprotected).fld_list).first;
+            let mut cur = (*(*imffields_unprotected).fld_list).first;
             while !cur.is_null() {
-                let field: *mut mailimf_field = (*cur).data as *mut mailimf_field;
+                let field = (*cur).data as *mut mailimf_field;
                 let mut move_to_encrypted = false;
+
                 if !field.is_null() {
                     if (*field).fld_type == MAILIMF_FIELD_SUBJECT as libc::c_int {
                         move_to_encrypted = true;
@@ -189,6 +184,7 @@ impl EncryptHelper {
                         }
                     }
                 }
+
                 if move_to_encrypted {
                     mailimf_fields_add(imffields_encrypted, field);
                     cur = clist_delete((*imffields_unprotected).fld_list, cur);
@@ -196,215 +192,151 @@ impl EncryptHelper {
                     cur = (*cur).next;
                 }
             }
-            let subject: *mut mailimf_subject = mailimf_subject_new("...".strdup());
-            mailimf_fields_add(
-                imffields_unprotected,
-                mailimf_field_new(
-                    MAILIMF_FIELD_SUBJECT as libc::c_int,
-                    ptr::null_mut(),
-                    ptr::null_mut(),
-                    ptr::null_mut(),
-                    ptr::null_mut(),
-                    ptr::null_mut(),
-                    ptr::null_mut(),
-                    ptr::null_mut(),
-                    ptr::null_mut(),
-                    ptr::null_mut(),
-                    ptr::null_mut(),
-                    ptr::null_mut(),
-                    ptr::null_mut(),
-                    ptr::null_mut(),
-                    ptr::null_mut(),
-                    ptr::null_mut(),
-                    ptr::null_mut(),
-                    ptr::null_mut(),
-                    ptr::null_mut(),
-                    subject,
-                    ptr::null_mut(),
-                    ptr::null_mut(),
-                    ptr::null_mut(),
-                ),
-            );
+
+            let subject = mailimf_subject_new("...".strdup());
+            mailimf_fields_add(imffields_unprotected, mailimf_field_new_subject(subject));
+
             wrapmime::append_ct_param(
                 (*part_to_encrypt).mm_content_type,
                 "protected-headers",
                 "v1",
             )?;
-            let plain: *mut MMAPString =
-                mmap_string_new(b"\x00" as *const u8 as *const libc::c_char);
-            let mut col: libc::c_int = 0i32;
+            let plain = mmap_string_new(b"\x00" as *const u8 as *const libc::c_char);
+            let mut col = 0;
             mailmime_write_mem(plain, &mut col, message_to_encrypt);
             mailmime_free(message_to_encrypt);
-            if (*plain).str_0.is_null() || (*plain).len <= 0 {
-                bail!("could not write/allocate");
-            }
 
-            let ctext = dc_pgp_pk_encrypt(
+            ensure!(
+                !(*plain).str_0.is_null() && (*plain).len > 0,
+                "could not write/allocate"
+            );
+
+            let ctext = pgp::pk_encrypt(
                 std::slice::from_raw_parts((*plain).str_0 as *const u8, (*plain).len),
                 &keyring,
                 sign_key.as_ref(),
             );
             mmap_string_free(plain);
 
-            if let Ok(ctext_v) = ctext {
-                /* create MIME-structure that will contain the encrypted text */
-                let mut encrypted_part: *mut Mailmime = new_data_part(
-                    ptr::null_mut(),
-                    0 as libc::size_t,
-                    "multipart/encrypted",
-                    MAILMIME_MECHANISM_BASE64,
-                )?;
-                let content: *mut mailmime_content = (*encrypted_part).mm_content_type;
-                wrapmime::append_ct_param(content, "protocol", "application/pgp-encrypted")?;
-                let version_mime: *mut Mailmime = new_data_part(
-                    VERSION_CONTENT.as_mut_ptr() as *mut libc::c_void,
-                    strlen(VERSION_CONTENT.as_mut_ptr()),
-                    "application/pgp-encrypted",
-                    MAILMIME_MECHANISM_7BIT,
-                )?;
-                mailmime_smart_add_part(encrypted_part, version_mime);
+            let ctext_v = ctext?;
 
-                // we assume that ctext_v is not dropped until the end
-                // of this if-scope
-                let ctext_part: *mut Mailmime = new_data_part(
-                    ctext_v.as_ptr() as *mut libc::c_void,
-                    ctext_v.len(),
-                    "application/octet-stream",
-                    MAILMIME_MECHANISM_7BIT,
-                )?;
-                mailmime_smart_add_part(encrypted_part, ctext_part);
-                (*in_out_message).mm_data.mm_message.mm_msg_mime = encrypted_part;
-                (*encrypted_part).mm_parent = in_out_message;
-                let gossiped = !&gossip_headers.is_empty();
-                factory.finalize_mime_message(in_out_message, true, gossiped)?;
-                Ok(true)
-            } else {
-                bail!("encryption failed")
-            }
+            // create MIME-structure that will contain the encrypted text
+            let mut encrypted_part = new_data_part(
+                ptr::null_mut(),
+                0 as libc::size_t,
+                "multipart/encrypted",
+                MAILMIME_MECHANISM_BASE64,
+            )?;
+            let content = (*encrypted_part).mm_content_type;
+            wrapmime::append_ct_param(content, "protocol", "application/pgp-encrypted")?;
+
+            let version_mime = new_data_part(
+                VERSION_CONTENT.as_mut_ptr() as *mut libc::c_void,
+                strlen(VERSION_CONTENT.as_mut_ptr()),
+                "application/pgp-encrypted",
+                MAILMIME_MECHANISM_7BIT,
+            )?;
+            mailmime_smart_add_part(encrypted_part, version_mime);
+
+            // we assume that ctext_v is not dropped until the end
+            // of this if-scope
+            let ctext_part = new_data_part(
+                ctext_v.as_ptr() as *mut libc::c_void,
+                ctext_v.len(),
+                "application/octet-stream",
+                MAILMIME_MECHANISM_7BIT,
+            )?;
+
+            mailmime_smart_add_part(encrypted_part, ctext_part);
+            (*in_out_message).mm_data.mm_message.mm_msg_mime = encrypted_part;
+            (*encrypted_part).mm_parent = in_out_message;
+            let gossiped = !&gossip_headers.is_empty();
+            factory.finalize_mime_message(in_out_message, true, gossiped)?;
+
+            Ok(true)
         }
     }
 }
 
-#[derive(Debug, Default)]
-pub struct E2eeHelper {
-    // for decrypting only
-    pub encrypted: bool,
-    pub signatures: HashSet<String>,
-    pub gossipped_addr: HashSet<String>,
-}
+pub fn try_decrypt(
+    context: &Context,
+    in_out_message: *mut Mailmime,
+) -> Result<(bool, HashSet<String>, HashSet<String>)> {
+    // just a pointer into mailmime structure, must not be freed
+    let imffields = mailmime_find_mailimf_fields(in_out_message);
+    ensure!(
+        !in_out_message.is_null() && !imffields.is_null(),
+        "corrupt invalid mime inputs"
+    );
 
-impl E2eeHelper {
-    pub unsafe fn try_decrypt(
-        &mut self,
-        context: &Context,
-        in_out_message: *mut Mailmime,
-    ) -> Result<()> {
-        /*just a pointer into mailmime structure, must not be freed*/
-        let imffields: *mut mailimf_fields = mailmime_find_mailimf_fields(in_out_message);
-        let mut message_time = 0;
-        let mut from = None;
-        let mut private_keyring = Keyring::default();
-        let mut public_keyring_for_validate = Keyring::default();
-        let mut gossip_headers: *mut mailimf_fields = ptr::null_mut();
+    let from = wrapmime::get_field_from(imffields)?;
+    let message_time = wrapmime::get_field_date(imffields)?;
 
-        // XXX do wrapmime:: helper for the next block
-        if !(in_out_message.is_null() || imffields.is_null()) {
-            let mut field = mailimf_find_field(imffields, MAILIMF_FIELD_FROM as libc::c_int);
+    let mut peerstate = None;
+    let autocryptheader = Aheader::from_imffields(&from, imffields);
 
-            if !field.is_null() && !(*field).fld_data.fld_from.is_null() {
-                from = mailimf_find_first_addr((*(*field).fld_data.fld_from).frm_mb_list)
-            }
-
-            field = mailimf_find_field(imffields, MAILIMF_FIELD_ORIG_DATE as libc::c_int);
-            if !field.is_null() && !(*field).fld_data.fld_orig_date.is_null() {
-                let orig_date: *mut mailimf_orig_date = (*field).fld_data.fld_orig_date;
-                if !orig_date.is_null() {
-                    message_time = dc_timestamp_from_date((*orig_date).dt_date_time);
-                    if message_time != 0 && message_time > time() {
-                        message_time = time()
-                    }
-                }
-            }
-            let mut peerstate = None;
-            let autocryptheader = from
-                .as_ref()
-                .and_then(|from| Aheader::from_imffields(from, imffields));
-            if message_time > 0 {
-                if let Some(ref from) = from {
-                    peerstate = Peerstate::from_addr(context, &context.sql, from);
-
-                    if let Some(ref mut peerstate) = peerstate {
-                        if let Some(ref header) = autocryptheader {
-                            peerstate.apply_header(&header, message_time);
-                            peerstate.save_to_db(&context.sql, false).unwrap();
-                        } else if message_time > peerstate.last_seen_autocrypt
-                            && !contains_report(in_out_message)
-                        {
-                            peerstate.degrade_encryption(message_time);
-                            peerstate.save_to_db(&context.sql, false).unwrap();
-                        }
-                    } else if let Some(ref header) = autocryptheader {
-                        let p = Peerstate::from_header(context, header, message_time);
-                        p.save_to_db(&context.sql, true).unwrap();
-                        peerstate = Some(p);
-                    }
-                }
-            }
-            /* load private key for decryption */
-            let self_addr = context.get_config(Config::ConfiguredAddr);
-            if let Some(self_addr) = self_addr {
-                if private_keyring.load_self_private_for_decrypting(
-                    context,
-                    self_addr,
-                    &context.sql,
-                ) {
-                    if peerstate.as_ref().map(|p| p.last_seen).unwrap_or_else(|| 0) == 0 {
-                        peerstate =
-                            Peerstate::from_addr(&context, &context.sql, &from.unwrap_or_default());
-                    }
-                    if let Some(ref peerstate) = peerstate {
-                        if peerstate.degrade_event.is_some() {
-                            handle_degrade_event(context, &peerstate);
-                        }
-                        if let Some(ref key) = peerstate.gossip_key {
-                            public_keyring_for_validate.add_ref(key);
-                        }
-                        if let Some(ref key) = peerstate.public_key {
-                            public_keyring_for_validate.add_ref(key);
-                        }
-                    }
-
-                    match decrypt_if_autocrypt_message(
-                        context,
-                        in_out_message,
-                        &private_keyring,
-                        &public_keyring_for_validate,
-                        &mut self.signatures,
-                        &mut gossip_headers,
-                    ) {
-                        Ok(res) => {
-                            self.encrypted = res;
-                        }
-                        Err(err) => {
-                            bail!("failed to decrypt: {}", err);
-                        }
-                    }
-                    if !gossip_headers.is_null() {
-                        self.gossipped_addr = update_gossip_peerstates(
-                            context,
-                            message_time,
-                            imffields,
-                            gossip_headers,
-                        )
-                    }
-                }
+    if message_time > 0 {
+        peerstate = Peerstate::from_addr(context, &context.sql, &from);
+
+        if let Some(ref mut peerstate) = peerstate {
+            if let Some(ref header) = autocryptheader {
+                peerstate.apply_header(&header, message_time);
+                peerstate.save_to_db(&context.sql, false)?;
+            } else if message_time > peerstate.last_seen_autocrypt
+                && !contains_report(in_out_message)
+            {
+                peerstate.degrade_encryption(message_time);
+                peerstate.save_to_db(&context.sql, false)?;
             }
+        } else if let Some(ref header) = autocryptheader {
+            let p = Peerstate::from_header(context, header, message_time);
+            p.save_to_db(&context.sql, true)?;
+            peerstate = Some(p);
         }
-        if !gossip_headers.is_null() {
-            mailimf_fields_free(gossip_headers);
-        }
-        Ok(())
     }
+    /* possibly perform decryption */
+    let mut private_keyring = Keyring::default();
+    let mut public_keyring_for_validate = Keyring::default();
+    let mut encrypted = false;
+    let mut signatures = HashSet::default();
+    let mut gossipped_addr = HashSet::default();
+
+    let self_addr = context.get_config(Config::ConfiguredAddr);
+
+    if let Some(self_addr) = self_addr {
+        if private_keyring.load_self_private_for_decrypting(context, self_addr, &context.sql) {
+            if peerstate.as_ref().map(|p| p.last_seen).unwrap_or_else(|| 0) == 0 {
+                peerstate = Peerstate::from_addr(&context, &context.sql, &from);
+            }
+            if let Some(ref peerstate) = peerstate {
+                if peerstate.degrade_event.is_some() {
+                    handle_degrade_event(context, &peerstate)?;
+                }
+                if let Some(ref key) = peerstate.gossip_key {
+                    public_keyring_for_validate.add_ref(key);
+                }
+                if let Some(ref key) = peerstate.public_key {
+                    public_keyring_for_validate.add_ref(key);
+                }
+            }
+
+            let mut gossip_headers = ptr::null_mut();
+            encrypted = decrypt_if_autocrypt_message(
+                context,
+                in_out_message,
+                &private_keyring,
+                &public_keyring_for_validate,
+                &mut signatures,
+                &mut gossip_headers,
+            )?;
+            if !gossip_headers.is_null() {
+                gossipped_addr =
+                    update_gossip_peerstates(context, message_time, imffields, gossip_headers)?;
+                unsafe { mailimf_fields_free(gossip_headers) };
+            }
+        }
+    }
+    Ok((encrypted, signatures, gossipped_addr))
 }
 
 fn new_data_part(
@@ -414,31 +346,34 @@ fn new_data_part(
     default_encoding: u32,
 ) -> Result<*mut Mailmime> {
     let content = new_content_type(&content_type)?;
-    unsafe {
-        let mut encoding: *mut mailmime_mechanism = ptr::null_mut();
-        if wrapmime::content_type_needs_encoding(content) {
-            encoding = mailmime_mechanism_new(default_encoding as i32, ptr::null_mut());
-            ensure!(!encoding.is_null(), "failed to create encoding");
-        }
-        let mime_fields = mailmime_fields_new_with_data(
-            encoding,
-            ptr::null_mut(),
-            ptr::null_mut(),
-            ptr::null_mut(),
-            ptr::null_mut(),
-        );
-        ensure!(!mime_fields.is_null(), "internal mime error");
-
-        let mime = mailmime_new_empty(content, mime_fields);
-        ensure!(!mime.is_null(), "internal mime error");
-
-        if (*mime).mm_type == MAILMIME_SINGLE as libc::c_int {
-            if !data.is_null() && data_bytes > 0 {
-                mailmime_set_body_text(mime, data as *mut libc::c_char, data_bytes);
-            }
-        }
-        return Ok(mime);
+    let mut encoding = ptr::null_mut();
+    if wrapmime::content_type_needs_encoding(content) {
+        encoding = unsafe { mailmime_mechanism_new(default_encoding as i32, ptr::null_mut()) };
+        ensure!(!encoding.is_null(), "failed to create encoding");
     }
+    let mime_fields = {
+        unsafe {
+            mailmime_fields_new_with_data(
+                encoding,
+                ptr::null_mut(),
+                ptr::null_mut(),
+                ptr::null_mut(),
+                ptr::null_mut(),
+            )
+        }
+    };
+    ensure!(!mime_fields.is_null(), "internal mime error");
+
+    let mime = unsafe { mailmime_new_empty(content, mime_fields) };
+    ensure!(!mime.is_null(), "internal mime error");
+
+    if unsafe { (*mime).mm_type } == MAILMIME_SINGLE as libc::c_int {
+        if !data.is_null() && data_bytes > 0 {
+            unsafe { mailmime_set_body_text(mime, data as *mut libc::c_char, data_bytes) };
+        }
+    }
+
+    Ok(mime)
 }
 
 /// Load public key from database or generate a new one.
@@ -463,14 +398,14 @@ fn load_or_generate_self_public_key(context: &Context, self_addr: impl AsRef<str
         context,
         "Generating keypair with {} bits, e={} ...", 2048, 65537,
     );
-    match dc_pgp_create_keypair(&self_addr) {
+    match pgp::create_keypair(&self_addr) {
         Some((public_key, private_key)) => {
             match dc_key_save_self_keypair(
                 context,
                 &public_key,
                 &private_key,
                 &self_addr,
-                1,
+                true,
                 &context.sql,
             ) {
                 true => {
@@ -488,31 +423,37 @@ fn load_or_generate_self_public_key(context: &Context, self_addr: impl AsRef<str
     }
 }
 
-unsafe fn update_gossip_peerstates(
+fn update_gossip_peerstates(
     context: &Context,
     message_time: i64,
     imffields: *mut mailimf_fields,
     gossip_headers: *const mailimf_fields,
-) -> HashSet<String> {
+) -> Result<HashSet<String>> {
     // XXX split the parsing from the modification part
     let mut recipients: Option<HashSet<String>> = None;
     let mut gossipped_addr: HashSet<String> = Default::default();
 
-    for cur_data in (*(*gossip_headers).fld_list).into_iter() {
-        let field: *mut mailimf_field = cur_data as *mut _;
-        if (*field).fld_type == MAILIMF_FIELD_OPTIONAL_FIELD as libc::c_int {
-            let optional_field = (*field).fld_data.fld_optional_field;
-            if !optional_field.is_null()
-                && !(*optional_field).fld_name.is_null()
-                && strcasecmp(
-                    (*optional_field).fld_name,
-                    b"Autocrypt-Gossip\x00" as *const u8 as *const libc::c_char,
-                ) == 0i32
+    for cur_data in unsafe { (*(*gossip_headers).fld_list).into_iter() } {
+        let field = cur_data as *mut mailimf_field;
+        if field.is_null() {
+            continue;
+        }
+
+        let field = unsafe { *field };
+
+        if field.fld_type == MAILIMF_FIELD_OPTIONAL_FIELD as libc::c_int {
+            let optional_field = unsafe { field.fld_data.fld_optional_field };
+            if optional_field.is_null() {
+                continue;
+            }
+
+            let optional_field = unsafe { *optional_field };
+            if !optional_field.fld_name.is_null()
+                && to_string_lossy(optional_field.fld_name) == "Autocrypt-Gossip"
             {
-                let value = CStr::from_ptr((*optional_field).fld_value)
-                    .to_str()
-                    .unwrap();
-                let gossip_header = Aheader::from_str(value);
+                let value = to_string_lossy(optional_field.fld_value);
+                let gossip_header = Aheader::from_str(&value);
+
                 if let Ok(ref header) = gossip_header {
                     if recipients.is_none() {
                         recipients = Some(mailimf_get_recipients(imffields));
@@ -522,15 +463,15 @@ unsafe fn update_gossip_peerstates(
                             Peerstate::from_addr(context, &context.sql, &header.addr);
                         if let Some(ref mut peerstate) = peerstate {
                             peerstate.apply_gossip(header, message_time);
-                            peerstate.save_to_db(&context.sql, false).unwrap();
+                            peerstate.save_to_db(&context.sql, false)?;
                         } else {
                             let p = Peerstate::from_gossip(context, header, message_time);
-                            p.save_to_db(&context.sql, true).unwrap();
+                            p.save_to_db(&context.sql, true)?;
                             peerstate = Some(p);
                         }
                         if let Some(peerstate) = peerstate {
                             if peerstate.degrade_event.is_some() {
-                                handle_degrade_event(context, &peerstate);
+                                handle_degrade_event(context, &peerstate)?;
                             }
                         }
 
@@ -547,7 +488,7 @@ unsafe fn update_gossip_peerstates(
         }
     }
 
-    gossipped_addr
+    Ok(gossipped_addr)
 }
 
 fn decrypt_if_autocrypt_message(
@@ -557,7 +498,7 @@ fn decrypt_if_autocrypt_message(
     public_keyring_for_validate: &Keyring,
     ret_valid_signatures: &mut HashSet<String>,
     ret_gossip_headers: *mut *mut mailimf_fields,
-) -> Result<(bool)> {
+) -> Result<bool> {
     /* The returned bool is true if we detected an Autocrypt-encrypted
     message and successfully decrypted it. Decryption then modifies the
     passed in mime structure in place. The returned bool is false
@@ -582,18 +523,17 @@ fn decrypt_if_autocrypt_message(
         public_keyring_for_validate,
         ret_valid_signatures,
     )?;
-    /* decrypted_mime is a dangling pointer which we now put into
-    mailmime's Ownership */
+    // decrypted_mime is a dangling pointer which we now put into mailmime's Ownership
     unsafe {
         mailmime_substitute(mime, decrypted_mime);
         mailmime_free(mime);
     }
 
-    /* finally, let's also return gossip headers
-    XXX better return parsed headers so that upstream
-    does not need to dive into mmime-stuff again. */
+    // finally, let's also return gossip headers
+    // XXX better return parsed headers so that upstream
+    // does not need to dive into mmime-stuff again.
     unsafe {
-        if (*ret_gossip_headers).is_null() && ret_valid_signatures.len() > 0 {
+        if (*ret_gossip_headers).is_null() && !ret_valid_signatures.is_empty() {
             let mut dummy: libc::size_t = 0;
             let mut test: *mut mailimf_fields = ptr::null_mut();
             if mailimf_envelope_and_optional_fields_parse(
@@ -604,11 +544,12 @@ fn decrypt_if_autocrypt_message(
             ) == MAILIMF_NO_ERROR as libc::c_int
                 && !test.is_null()
             {
-                *ret_gossip_headers = test
+                *ret_gossip_headers = test;
             }
         }
     }
-    return Ok(true);
+
+    Ok(true)
 }
 
 fn decrypt_part(
@@ -632,76 +573,67 @@ fn decrypt_part(
         mime_transfer_encoding = enc;
     }
 
-    let (decoded_data, decoded_data_bytes) =
-        wrapmime::decode_dt_data(mime_data, mime_transfer_encoding)?;
-    /* encrypted, non-NULL decoded data in decoded_data now ...
-    Note that we need to take care of freeing decoded_data ourself,
-    after encryption has been attempted.
-    */
+    let data: Vec<u8> = wrapmime::decode_dt_data(mime_data, mime_transfer_encoding)?;
+
     let mut ret_decrypted_mime = ptr::null_mut();
 
-    unsafe {
-        if has_decrypted_pgp_armor(decoded_data, decoded_data_bytes as libc::c_int) {
-            /* we should only have one decryption happening */
-            ensure!(ret_valid_signatures.is_empty(), "corrupt signatures");
+    if has_decrypted_pgp_armor(&data) {
+        // we should only have one decryption happening
+        ensure!(ret_valid_signatures.is_empty(), "corrupt signatures");
 
-            let plain = match dc_pgp_pk_decrypt(
-                std::slice::from_raw_parts(decoded_data as *const u8, decoded_data_bytes),
-                &private_keyring,
-                &public_keyring_for_validate,
-                Some(ret_valid_signatures),
-            ) {
-                Ok(plain) => {
-                    ensure!(!ret_valid_signatures.is_empty(), "no valid signatures");
-                    plain
-                }
-                Err(err) => {
-                    mmap_string_unref(decoded_data);
-                    bail!("could not decrypt: {}", err)
-                }
-            };
-            let plain_bytes = plain.len();
-            let plain_buf = plain.as_ptr() as *const libc::c_char;
+        let plain = match pgp::pk_decrypt(
+            &data,
+            &private_keyring,
+            &public_keyring_for_validate,
+            Some(ret_valid_signatures),
+        ) {
+            Ok(plain) => {
+                ensure!(!ret_valid_signatures.is_empty(), "no valid signatures");
+                plain
+            }
+            Err(err) => bail!("could not decrypt: {}", err),
+        };
+        let plain_bytes = plain.len();
+        let plain_buf = plain.as_ptr() as *const libc::c_char;
 
-            let mut index: libc::size_t = 0;
-            let mut decrypted_mime: *mut Mailmime = ptr::null_mut();
-            if mailmime_parse(
+        let mut index = 0;
+        let mut decrypted_mime = ptr::null_mut();
+        if unsafe {
+            mailmime_parse(
                 plain_buf as *const _,
                 plain_bytes,
                 &mut index,
                 &mut decrypted_mime,
-            ) != MAIL_NO_ERROR as libc::c_int
-                || decrypted_mime.is_null()
-            {
-                if !decrypted_mime.is_null() {
-                    mailmime_free(decrypted_mime);
-                }
-            } else {
-                ret_decrypted_mime = decrypted_mime;
+            )
+        } != MAIL_NO_ERROR as libc::c_int
+            || decrypted_mime.is_null()
+        {
+            if !decrypted_mime.is_null() {
+                unsafe { mailmime_free(decrypted_mime) };
             }
+        } else {
+            // decrypted_mime points into `plain`.
+            // FIXME(@dignifiedquire): this still leaks memory I believe, as mailmime_free
+            // does not free the underlying buffer. But for now we have to live with it
             std::mem::forget(plain);
+            ret_decrypted_mime = decrypted_mime;
         }
-        mmap_string_unref(decoded_data);
     }
+
     Ok(ret_decrypted_mime)
 }
 
-unsafe fn has_decrypted_pgp_armor(str__: *const libc::c_char, mut str_bytes: libc::c_int) -> bool {
-    let str_end: *const libc::c_uchar = (str__ as *const libc::c_uchar).offset(str_bytes as isize);
-    let mut p: *const libc::c_uchar = str__ as *const libc::c_uchar;
-    while p < str_end {
-        if *p as libc::c_int > ' ' as i32 {
-            break;
+fn has_decrypted_pgp_armor(input: &[u8]) -> bool {
+    if let Some(index) = input.iter().position(|b| *b > b' ') {
+        if input.len() - index > 26 {
+            let start = index;
+            let end = start + 27;
+
+            return &input[start..end] == b"-----BEGIN PGP MESSAGE-----";
         }
-        p = p.offset(1isize);
-        str_bytes -= 1
     }
-    str_bytes > 27i32
-        && strncmp(
-            p as *const libc::c_char,
-            b"-----BEGIN PGP MESSAGE-----\x00" as *const u8 as *const libc::c_char,
-            27,
-        ) == 0
+
+    false
 }
 
 /// Check if a MIME structure contains a multipart/report part.
@@ -712,29 +644,31 @@ unsafe fn has_decrypted_pgp_armor(str__: *const libc::c_char, mut str_bytes: lib
 /// However, Delta Chat itself has no problem with encrypted multipart/report
 /// parts and MUAs should be encouraged to encrpyt multipart/reports as well so
 /// that we could use the normal Autocrypt processing.
-unsafe fn contains_report(mime: *mut Mailmime) -> bool {
-    if (*mime).mm_type == MAILMIME_MULTIPLE as libc::c_int {
-        if (*(*(*mime).mm_content_type).ct_type).tp_type
-            == MAILMIME_TYPE_COMPOSITE_TYPE as libc::c_int
-            && (*(*(*(*mime).mm_content_type).ct_type)
-                .tp_data
-                .tp_composite_type)
-                .ct_type
-                == MAILMIME_COMPOSITE_TYPE_MULTIPART as libc::c_int
-            && strcmp(
-                (*(*mime).mm_content_type).ct_subtype,
-                b"report\x00" as *const u8 as *const libc::c_char,
-            ) == 0i32
+fn contains_report(mime: *mut Mailmime) -> bool {
+    assert!(!mime.is_null());
+    let mime = unsafe { *mime };
+
+    if mime.mm_type == MAILMIME_MULTIPLE as libc::c_int {
+        let tp_type = unsafe { (*(*mime.mm_content_type).ct_type).tp_type };
+        let ct_type =
+            unsafe { (*(*(*mime.mm_content_type).ct_type).tp_data.tp_composite_type).ct_type };
+
+        if tp_type == MAILMIME_TYPE_COMPOSITE_TYPE as libc::c_int
+            && ct_type == MAILMIME_COMPOSITE_TYPE_MULTIPART as libc::c_int
+            && to_string_lossy(unsafe { (*mime.mm_content_type).ct_subtype }) == "report"
         {
             return true;
         }
-        for cur_data in (*(*(*mime).mm_mime_fields).fld_list).into_iter() {
+
+        for cur_data in unsafe { (*(*mime.mm_mime_fields).fld_list).into_iter() } {
             if contains_report(cur_data as *mut Mailmime) {
                 return true;
             }
         }
-    } else if (*mime).mm_type == MAILMIME_MESSAGE as libc::c_int {
-        if contains_report((*mime).mm_data.mm_message.mm_msg_mime) {
+    } else if mime.mm_type == MAILMIME_MESSAGE as libc::c_int {
+        let m = unsafe { mime.mm_data.mm_message.mm_msg_mime };
+
+        if contains_report(m) {
             return true;
         }
     }
@@ -752,12 +686,12 @@ unsafe fn contains_report(mime: *mut Mailmime) -> bool {
 /// If this succeeds you are also guaranteed that the
 /// [Config::ConfiguredAddr] is configured, this address is returned.
 pub fn ensure_secret_key_exists(context: &Context) -> Result<String> {
-    let self_addr = context
-        .get_config(Config::ConfiguredAddr)
-        .ok_or(format_err!(concat!(
+    let self_addr = context.get_config(Config::ConfiguredAddr).ok_or_else(|| {
+        format_err!(concat!(
             "Failed to get self address, ",
             "cannot ensure secret key if not configured."
-        )))?;
+        ))
+    })?;
     load_or_generate_self_public_key(context, &self_addr)?;
     Ok(self_addr)
 }
@@ -869,4 +803,22 @@ Sent with my Delta Chat Messenger: https://delta.chat";
             assert_eq!(res0.unwrap(), res1.unwrap());
         }
     }
+
+    #[test]
+    fn test_has_decrypted_pgp_armor() {
+        let data = b" -----BEGIN PGP MESSAGE-----";
+        assert_eq!(has_decrypted_pgp_armor(data), true);
+
+        let data = b"    \n-----BEGIN PGP MESSAGE-----";
+        assert_eq!(has_decrypted_pgp_armor(data), true);
+
+        let data = b"    -----BEGIN PGP MESSAGE---";
+        assert_eq!(has_decrypted_pgp_armor(data), false);
+
+        let data = b" -----BEGIN PGP MESSAGE-----";
+        assert_eq!(has_decrypted_pgp_armor(data), true);
+
+        let data = b"blas";
+        assert_eq!(has_decrypted_pgp_armor(data), false);
+    }
 }

src/error.rs

@@ -26,6 +26,16 @@ pub enum Error {
     CStringError(crate::dc_tools::CStringError),
     #[fail(display = "PGP: {:?}", _0)]
     Pgp(pgp::errors::Error),
+    #[fail(display = "Base64Decode: {:?}", _0)]
+    Base64Decode(base64::DecodeError),
+    #[fail(display = "{:?}", _0)]
+    FromUtf8(std::string::FromUtf8Error),
+    #[fail(display = "{}", _0)]
+    BlobError(#[cause] crate::blob::BlobError),
+    #[fail(display = "Invalid Message ID.")]
+    InvalidMsgId,
+    #[fail(display = "Watch folder not found {:?}", _0)]
+    WatchFolderNotFound(String),
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -36,6 +46,12 @@ impl From<rusqlite::Error> for Error {
     }
 }
 
+impl From<base64::DecodeError> for Error {
+    fn from(err: base64::DecodeError) -> Error {
+        Error::Base64Decode(err)
+    }
+}
+
 impl From<failure::Error> for Error {
     fn from(err: failure::Error) -> Error {
         Error::Failure(err)
@@ -78,6 +94,24 @@ impl From<pgp::errors::Error> for Error {
     }
 }
 
+impl From<std::string::FromUtf8Error> for Error {
+    fn from(err: std::string::FromUtf8Error) -> Error {
+        Error::FromUtf8(err)
+    }
+}
+
+impl From<crate::blob::BlobError> for Error {
+    fn from(err: crate::blob::BlobError) -> Error {
+        Error::BlobError(err)
+    }
+}
+
+impl From<crate::message::InvalidMsgId> for Error {
+    fn from(_err: crate::message::InvalidMsgId) -> Error {
+        Error::InvalidMsgId
+    }
+}
+
 #[macro_export]
 macro_rules! bail {
     ($e:expr) => {

src/events.rs

@@ -2,7 +2,7 @@ use std::path::PathBuf;
 
 use strum::EnumProperty;
 
-use crate::stock::StockMessage;
+use crate::message::MsgId;
 
 impl Event {
     /// Returns the corresponding Event id.
@@ -42,6 +42,36 @@ pub enum Event {
     #[strum(props(id = "103"))]
     SmtpMessageSent(String),
 
+    /// Emitted when an IMAP message has been marked as deleted
+    ///
+    /// @return 0
+    #[strum(props(id = "104"))]
+    ImapMessageDeleted(String),
+
+    /// Emitted when an IMAP message has been moved
+    ///
+    /// @return 0
+    #[strum(props(id = "105"))]
+    ImapMessageMoved(String),
+
+    /// Emitted when an IMAP folder was emptied
+    ///
+    /// @return 0
+    #[strum(props(id = "106"))]
+    ImapFolderEmptied(String),
+
+    /// Emitted when an new file in the $BLOBDIR was created
+    ///
+    /// @return 0
+    #[strum(props(id = "150"))]
+    NewBlobFile(String),
+
+    /// Emitted when an new file in the $BLOBDIR was created
+    ///
+    /// @return 0
+    #[strum(props(id = "151"))]
+    DeletedBlobFile(String),
+
     /// The library-user should write a warning string to the log.
     /// Passed to the callback given to dc_context_new().
     ///
@@ -103,7 +133,7 @@ pub enum Event {
     ///
     /// @return 0
     #[strum(props(id = "2000"))]
-    MsgsChanged { chat_id: u32, msg_id: u32 },
+    MsgsChanged { chat_id: u32, msg_id: MsgId },
 
     /// There is a fresh message. Typically, the user will show an notification
     /// when receiving this message.
@@ -112,28 +142,28 @@ pub enum Event {
     ///
     /// @return 0
     #[strum(props(id = "2005"))]
-    IncomingMsg { chat_id: u32, msg_id: u32 },
+    IncomingMsg { chat_id: u32, msg_id: MsgId },
 
     /// A single message is sent successfully. State changed from  DC_STATE_OUT_PENDING to
     /// DC_STATE_OUT_DELIVERED, see dc_msg_get_state().
     ///
     /// @return 0
     #[strum(props(id = "2010"))]
-    MsgDelivered { chat_id: u32, msg_id: u32 },
+    MsgDelivered { chat_id: u32, msg_id: MsgId },
 
     /// A single message could not be sent. State changed from DC_STATE_OUT_PENDING or DC_STATE_OUT_DELIVERED to
     /// DC_STATE_OUT_FAILED, see dc_msg_get_state().
     ///
     /// @return 0
     #[strum(props(id = "2012"))]
-    MsgFailed { chat_id: u32, msg_id: u32 },
+    MsgFailed { chat_id: u32, msg_id: MsgId },
 
     /// A single message is read by the receiver. State changed from DC_STATE_OUT_DELIVERED to
     /// DC_STATE_OUT_MDN_RCVD, see dc_msg_get_state().
     ///
     /// @return 0
     #[strum(props(id = "2015"))]
-    MsgRead { chat_id: u32, msg_id: u32 },
+    MsgRead { chat_id: u32, msg_id: MsgId },
 
     /// Chat changed.  The name or the image of a chat group was changed or members were added or removed.
     /// Or the verify state of a chat has changed.
@@ -167,7 +197,7 @@ pub enum Event {
     #[strum(props(id = "2041"))]
     ConfigureProgress(usize),
 
-    /// Inform about the import/export progress started by dc_imex().
+    /// Inform about the import/export progress started by imex().
     ///
     /// @param data1 (usize) 0=error, 1-999=progress in permille, 1000=success and done
     /// @param data2 0
@@ -175,8 +205,8 @@ pub enum Event {
     #[strum(props(id = "2051"))]
     ImexProgress(usize),
 
-    /// A file has been exported. A file has been written by dc_imex().
-    /// This event may be sent multiple times by a single call to dc_imex().
+    /// A file has been exported. A file has been written by imex().
+    /// This event may be sent multiple times by a single call to imex().
     ///
     /// A typical purpose for a handler of this event may be to make the file public to some system
     /// services.
@@ -214,16 +244,10 @@ pub enum Event {
     #[strum(props(id = "2061"))]
     SecurejoinJoinerProgress { contact_id: u32, progress: usize },
 
-    // the following events are functions that should be provided by the frontends
-    /// Requeste a localized string from the frontend.
-    /// @param data1 (int) ID of the string to request, one of the DC_STR_/// constants.
-    /// @param data2 (int) The count. If the requested string contains a placeholder for a numeric value,
-    ///     the ui may use this value to return different strings on different plural forms.
-    /// @return (const char*) Null-terminated UTF-8 string.
-    ///     The string will be free()'d by the core,
-    ///     so it must be allocated using malloc() or a compatible function.
-    ///     Return 0 if the ui cannot provide the requested string
-    ///     the core will use a default string in english language then.
-    #[strum(props(id = "2091"))]
-    GetString { id: StockMessage, count: usize },
+    /// This event is sent out to the inviter when a joiner successfully joined a group.
+    /// @param data1 (int) chat_id
+    /// @param data2 (int) contact_id
+    /// @return 0
+    #[strum(props(id = "2062"))]
+    SecurejoinMemberAdded { chat_id: u32, contact_id: u32 },
 }

src/imap_client.rs

@@ -0,0 +1,295 @@
+use async_imap::{
+    error::{Error as ImapError, Result as ImapResult},
+    extensions::idle::Handle as ImapIdleHandle,
+    types::{Capabilities, Fetch, Mailbox, Name},
+    Client as ImapClient, Session as ImapSession,
+};
+use async_std::net::{self, TcpStream};
+use async_std::prelude::*;
+use async_std::sync::Arc;
+use async_tls::client::TlsStream;
+
+use crate::login_param::{dc_build_tls_config, CertificateChecks};
+
+const DCC_IMAP_DEBUG: &str = "DCC_IMAP_DEBUG";
+
+#[derive(Debug)]
+pub(crate) enum Client {
+    Secure(ImapClient<TlsStream<TcpStream>>),
+    Insecure(ImapClient<TcpStream>),
+}
+
+#[derive(Debug)]
+pub(crate) enum Session {
+    Secure(ImapSession<TlsStream<TcpStream>>),
+    Insecure(ImapSession<TcpStream>),
+}
+
+#[derive(Debug)]
+pub(crate) enum IdleHandle {
+    Secure(ImapIdleHandle<TlsStream<TcpStream>>),
+    Insecure(ImapIdleHandle<TcpStream>),
+}
+
+impl Client {
+    pub async fn connect_secure<A: net::ToSocketAddrs, S: AsRef<str>>(
+        addr: A,
+        domain: S,
+        certificate_checks: CertificateChecks,
+    ) -> ImapResult<Self> {
+        let stream = TcpStream::connect(addr).await?;
+        let tls_config = dc_build_tls_config(certificate_checks);
+        let tls_connector: async_tls::TlsConnector = Arc::new(tls_config).into();
+        let tls_stream = tls_connector.connect(domain.as_ref(), stream)?.await?;
+        let mut client = ImapClient::new(tls_stream);
+        if std::env::var(DCC_IMAP_DEBUG).is_ok() {
+            client.debug = true;
+        }
+
+        let _greeting = client
+            .read_response()
+            .await
+            .expect("failed to read greeting");
+
+        Ok(Client::Secure(client))
+    }
+
+    pub async fn connect_insecure<A: net::ToSocketAddrs>(addr: A) -> ImapResult<Self> {
+        let stream = TcpStream::connect(addr).await?;
+
+        let mut client = ImapClient::new(stream);
+        if std::env::var(DCC_IMAP_DEBUG).is_ok() {
+            client.debug = true;
+        }
+        let _greeting = client
+            .read_response()
+            .await
+            .expect("failed to read greeting");
+
+        Ok(Client::Insecure(client))
+    }
+
+    pub async fn secure<S: AsRef<str>>(
+        self,
+        domain: S,
+        certificate_checks: CertificateChecks,
+    ) -> ImapResult<Client> {
+        match self {
+            Client::Insecure(client) => {
+                let tls_config = dc_build_tls_config(certificate_checks);
+                let tls: async_tls::TlsConnector = Arc::new(tls_config).into();
+
+                let client_sec = client.secure(domain, &tls).await?;
+
+                Ok(Client::Secure(client_sec))
+            }
+            // Nothing to do
+            Client::Secure(_) => Ok(self),
+        }
+    }
+
+    pub async fn authenticate<A: async_imap::Authenticator, S: AsRef<str>>(
+        self,
+        auth_type: S,
+        authenticator: &A,
+    ) -> Result<Session, (ImapError, Client)> {
+        match self {
+            Client::Secure(i) => match i.authenticate(auth_type, authenticator).await {
+                Ok(session) => Ok(Session::Secure(session)),
+                Err((err, c)) => Err((err, Client::Secure(c))),
+            },
+            Client::Insecure(i) => match i.authenticate(auth_type, authenticator).await {
+                Ok(session) => Ok(Session::Insecure(session)),
+                Err((err, c)) => Err((err, Client::Insecure(c))),
+            },
+        }
+    }
+
+    pub async fn login<U: AsRef<str>, P: AsRef<str>>(
+        self,
+        username: U,
+        password: P,
+    ) -> Result<Session, (ImapError, Client)> {
+        match self {
+            Client::Secure(i) => match i.login(username, password).await {
+                Ok(session) => Ok(Session::Secure(session)),
+                Err((err, c)) => Err((err, Client::Secure(c))),
+            },
+            Client::Insecure(i) => match i.login(username, password).await {
+                Ok(session) => Ok(Session::Insecure(session)),
+                Err((err, c)) => Err((err, Client::Insecure(c))),
+            },
+        }
+    }
+}
+
+impl Session {
+    pub async fn capabilities(&mut self) -> ImapResult<Capabilities> {
+        let res = match self {
+            Session::Secure(i) => i.capabilities().await?,
+            Session::Insecure(i) => i.capabilities().await?,
+        };
+
+        Ok(res)
+    }
+
+    pub async fn list(
+        &mut self,
+        reference_name: Option<&str>,
+        mailbox_pattern: Option<&str>,
+    ) -> ImapResult<Vec<Name>> {
+        let res = match self {
+            Session::Secure(i) => {
+                i.list(reference_name, mailbox_pattern)
+                    .await?
+                    .collect::<ImapResult<_>>()
+                    .await?
+            }
+            Session::Insecure(i) => {
+                i.list(reference_name, mailbox_pattern)
+                    .await?
+                    .collect::<ImapResult<_>>()
+                    .await?
+            }
+        };
+        Ok(res)
+    }
+
+    pub async fn create<S: AsRef<str>>(&mut self, mailbox_name: S) -> ImapResult<()> {
+        match self {
+            Session::Secure(i) => i.create(mailbox_name).await?,
+            Session::Insecure(i) => i.create(mailbox_name).await?,
+        }
+        Ok(())
+    }
+
+    pub async fn subscribe<S: AsRef<str>>(&mut self, mailbox: S) -> ImapResult<()> {
+        match self {
+            Session::Secure(i) => i.subscribe(mailbox).await?,
+            Session::Insecure(i) => i.subscribe(mailbox).await?,
+        }
+        Ok(())
+    }
+
+    pub async fn close(&mut self) -> ImapResult<()> {
+        match self {
+            Session::Secure(i) => i.close().await?,
+            Session::Insecure(i) => i.close().await?,
+        }
+        Ok(())
+    }
+
+    pub async fn select<S: AsRef<str>>(&mut self, mailbox_name: S) -> ImapResult<Mailbox> {
+        let mbox = match self {
+            Session::Secure(i) => i.select(mailbox_name).await?,
+            Session::Insecure(i) => i.select(mailbox_name).await?,
+        };
+
+        Ok(mbox)
+    }
+
+    pub async fn fetch<S1, S2>(&mut self, sequence_set: S1, query: S2) -> ImapResult<Vec<Fetch>>
+    where
+        S1: AsRef<str>,
+        S2: AsRef<str>,
+    {
+        let res = match self {
+            Session::Secure(i) => {
+                i.fetch(sequence_set, query)
+                    .await?
+                    .collect::<ImapResult<_>>()
+                    .await?
+            }
+            Session::Insecure(i) => {
+                i.fetch(sequence_set, query)
+                    .await?
+                    .collect::<ImapResult<_>>()
+                    .await?
+            }
+        };
+        Ok(res)
+    }
+
+    pub async fn uid_fetch<S1, S2>(&mut self, uid_set: S1, query: S2) -> ImapResult<Vec<Fetch>>
+    where
+        S1: AsRef<str>,
+        S2: AsRef<str>,
+    {
+        let res = match self {
+            Session::Secure(i) => {
+                i.uid_fetch(uid_set, query)
+                    .await?
+                    .collect::<ImapResult<_>>()
+                    .await?
+            }
+            Session::Insecure(i) => {
+                i.uid_fetch(uid_set, query)
+                    .await?
+                    .collect::<ImapResult<_>>()
+                    .await?
+            }
+        };
+
+        Ok(res)
+    }
+
+    pub fn idle(self) -> IdleHandle {
+        match self {
+            Session::Secure(i) => {
+                let h = i.idle();
+                IdleHandle::Secure(h)
+            }
+            Session::Insecure(i) => {
+                let h = i.idle();
+                IdleHandle::Insecure(h)
+            }
+        }
+    }
+
+    pub async fn uid_store<S1, S2>(&mut self, uid_set: S1, query: S2) -> ImapResult<Vec<Fetch>>
+    where
+        S1: AsRef<str>,
+        S2: AsRef<str>,
+    {
+        let res = match self {
+            Session::Secure(i) => {
+                i.uid_store(uid_set, query)
+                    .await?
+                    .collect::<ImapResult<_>>()
+                    .await?
+            }
+            Session::Insecure(i) => {
+                i.uid_store(uid_set, query)
+                    .await?
+                    .collect::<ImapResult<_>>()
+                    .await?
+            }
+        };
+        Ok(res)
+    }
+
+    pub async fn uid_mv<S1: AsRef<str>, S2: AsRef<str>>(
+        &mut self,
+        uid_set: S1,
+        mailbox_name: S2,
+    ) -> ImapResult<()> {
+        match self {
+            Session::Secure(i) => i.uid_mv(uid_set, mailbox_name).await?,
+            Session::Insecure(i) => i.uid_mv(uid_set, mailbox_name).await?,
+        }
+        Ok(())
+    }
+
+    pub async fn uid_copy<S1: AsRef<str>, S2: AsRef<str>>(
+        &mut self,
+        uid_set: S1,
+        mailbox_name: S2,
+    ) -> ImapResult<()> {
+        match self {
+            Session::Secure(i) => i.uid_copy(uid_set, mailbox_name).await?,
+            Session::Insecure(i) => i.uid_copy(uid_set, mailbox_name).await?,
+        }
+
+        Ok(())
+    }
+}

src/imex.rs

@@ -0,0 +1,851 @@
+//! # Import/export module
+
+use core::cmp::{max, min};
+use std::path::Path;
+
+use num_traits::FromPrimitive;
+use rand::{thread_rng, Rng};
+
+use crate::blob::BlobObject;
+use crate::chat;
+use crate::config::Config;
+use crate::configure::*;
+use crate::constants::*;
+use crate::context::Context;
+use crate::dc_mimeparser::SystemMessage;
+use crate::dc_tools::*;
+use crate::e2ee;
+use crate::error::*;
+use crate::events::Event;
+use crate::job::*;
+use crate::key::*;
+use crate::message::{Message, MsgId};
+use crate::param::*;
+use crate::pgp;
+use crate::sql::{self, Sql};
+use crate::stock::StockMessage;
+
+#[derive(Debug, Display, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive)]
+#[repr(i32)]
+pub enum ImexMode {
+    /// Export all private keys and all public keys of the user to the
+    /// directory given as `param1`.  The default key is written to the files `public-key-default.asc`
+    /// and `private-key-default.asc`, if there are more keys, they are written to files as
+    /// `public-key-<id>.asc` and `private-key-<id>.asc`
+    ExportSelfKeys = 1,
+    /// Import private keys found in the directory given as `param1`.
+    /// The last imported key is made the default keys unless its name contains the string `legacy`.
+    /// Public keys are not imported.
+    ImportSelfKeys = 2,
+    /// Export a backup to the directory given as `param1`.
+    /// The backup contains all contacts, chats, images and other data and device independent settings.
+    /// The backup does not contain device dependent settings as ringtones or LED notification settings.
+    /// The name of the backup is typically `delta-chat.<day>.bak`, if more than one backup is create on a day,
+    /// the format is `delta-chat.<day>-<number>.bak`
+    ExportBackup = 11,
+    /// `param1` is the file (not: directory) to import. The file is normally
+    /// created by DC_IMEX_EXPORT_BACKUP and detected by dc_imex_has_backup(). Importing a backup
+    /// is only possible as long as the context is not configured or used in another way.
+    ImportBackup = 12,
+}
+
+/// Import/export things.
+/// For this purpose, the function creates a job that is executed in the IMAP-thread then;
+/// this requires to call dc_perform_inbox_jobs() regularly.
+///
+/// What to do is defined by the _what_ parameter.
+///
+/// While dc_imex() returns immediately, the started job may take a while,
+/// you can stop it using dc_stop_ongoing_process(). During execution of the job,
+/// some events are sent out:
+///
+/// - A number of #DC_EVENT_IMEX_PROGRESS events are sent and may be used to create
+///   a progress bar or stuff like that. Moreover, you'll be informed when the imex-job is done.
+///
+/// - For each file written on export, the function sends #DC_EVENT_IMEX_FILE_WRITTEN
+///
+/// Only one import-/export-progress can run at the same time.
+/// To cancel an import-/export-progress, use dc_stop_ongoing_process().
+pub fn imex(context: &Context, what: ImexMode, param1: Option<impl AsRef<Path>>) {
+    let mut param = Params::new();
+    param.set_int(Param::Cmd, what as i32);
+    if let Some(param1) = param1 {
+        param.set(Param::Arg, param1.as_ref().to_string_lossy());
+    }
+
+    job_kill_action(context, Action::ImexImap);
+    job_add(context, Action::ImexImap, 0, param, 0);
+}
+
+/// Returns the filename of the backup found (otherwise an error)
+pub fn has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Result<String> {
+    let dir_name = dir_name.as_ref();
+    let dir_iter = std::fs::read_dir(dir_name)?;
+    let mut newest_backup_time = 0;
+    let mut newest_backup_path: Option<std::path::PathBuf> = None;
+    for dirent in dir_iter {
+        match dirent {
+            Ok(dirent) => {
+                let path = dirent.path();
+                let name = dirent.file_name();
+                let name = name.to_string_lossy();
+                if name.starts_with("delta-chat") && name.ends_with(".bak") {
+                    let sql = Sql::new();
+                    if sql.open(context, &path, true) {
+                        let curr_backup_time = sql
+                            .get_raw_config_int(context, "backup_time")
+                            .unwrap_or_default();
+                        if curr_backup_time > newest_backup_time {
+                            newest_backup_path = Some(path);
+                            newest_backup_time = curr_backup_time;
+                        }
+                        info!(context, "backup_time of {} is {}", name, curr_backup_time);
+                        sql.close(&context);
+                    }
+                }
+            }
+            Err(_) => (),
+        }
+    }
+    match newest_backup_path {
+        Some(path) => Ok(path.to_string_lossy().into_owned()),
+        None => bail!("no backup found in {}", dir_name.display()),
+    }
+}
+
+pub fn initiate_key_transfer(context: &Context) -> Result<String> {
+    ensure!(context.alloc_ongoing(), "could not allocate ongoing");
+    let res = do_initiate_key_transfer(context);
+    context.free_ongoing();
+    res
+}
+
+fn do_initiate_key_transfer(context: &Context) -> Result<String> {
+    let mut msg: Message;
+    let setup_code = create_setup_code(context);
+    /* this may require a keypair to be created. this may take a second ... */
+    ensure!(!context.shall_stop_ongoing(), "canceled");
+    let setup_file_content = render_setup_file(context, &setup_code)?;
+    /* encrypting may also take a while ... */
+    ensure!(!context.shall_stop_ongoing(), "canceled");
+    let setup_file_blob = BlobObject::create(
+        context,
+        "autocrypt-setup-message.html",
+        setup_file_content.as_bytes(),
+    )?;
+
+    let chat_id = chat::create_by_contact_id(context, DC_CONTACT_ID_SELF)?;
+    msg = Message::default();
+    msg.type_0 = Viewtype::File;
+    msg.param.set(Param::File, setup_file_blob.as_name());
+
+    msg.param
+        .set(Param::MimeType, "application/autocrypt-setup");
+    msg.param.set_cmd(SystemMessage::AutocryptSetupMessage);
+    msg.param
+        .set_int(Param::ForcePlaintext, DC_FP_NO_AUTOCRYPT_HEADER);
+
+    ensure!(!context.shall_stop_ongoing(), "canceled");
+    let msg_id = chat::send_msg(context, chat_id, &mut msg)?;
+    info!(context, "Wait for setup message being sent ...",);
+    while !context.shall_stop_ongoing() {
+        std::thread::sleep(std::time::Duration::from_secs(1));
+        if let Ok(msg) = Message::load_from_db(context, msg_id) {
+            if msg.is_sent() {
+                info!(context, "... setup message sent.",);
+                break;
+            }
+        }
+    }
+    // no maybe_add_bcc_self_device_msg() here.
+    // the ui shows the dialog with the setup code on this device,
+    // it would be too much noise to have two things popping up at the same time.
+    // maybe_add_bcc_self_device_msg() is called on the other device
+    // once the transfer is completed.
+    Ok(setup_code)
+}
+
+/// Renders HTML body of a setup file message.
+///
+/// The `passphrase` must be at least 2 characters long.
+pub fn render_setup_file(context: &Context, passphrase: &str) -> Result<String> {
+    ensure!(
+        passphrase.len() >= 2,
+        "Passphrase must be at least 2 chars long."
+    );
+    let self_addr = e2ee::ensure_secret_key_exists(context)?;
+    let private_key = Key::from_self_private(context, self_addr, &context.sql)
+        .ok_or(format_err!("Failed to get private key."))?;
+    let ac_headers = match context.get_config_bool(Config::E2eeEnabled) {
+        false => None,
+        true => Some(("Autocrypt-Prefer-Encrypt", "mutual")),
+    };
+    let private_key_asc = private_key.to_asc(ac_headers);
+    let encr = pgp::symm_encrypt(&passphrase, private_key_asc.as_bytes())?;
+
+    let replacement = format!(
+        concat!(
+            "-----BEGIN PGP MESSAGE-----\r\n",
+            "Passphrase-Format: numeric9x4\r\n",
+            "Passphrase-Begin: {}"
+        ),
+        &passphrase[..2]
+    );
+    let pgp_msg = encr.replace("-----BEGIN PGP MESSAGE-----", &replacement);
+
+    let msg_subj = context.stock_str(StockMessage::AcSetupMsgSubject);
+    let msg_body = context.stock_str(StockMessage::AcSetupMsgBody);
+    let msg_body_html = msg_body.replace("\r", "").replace("\n", "<br>");
+    Ok(format!(
+        concat!(
+            "<!DOCTYPE html>\r\n",
+            "<html>\r\n",
+            "  <head>\r\n",
+            "    <title>{}</title>\r\n",
+            "  </head>\r\n",
+            "  <body>\r\n",
+            "    <h1>{}</h1>\r\n",
+            "    <p>{}</p>\r\n",
+            "    <pre>\r\n{}\r\n</pre>\r\n",
+            "  </body>\r\n",
+            "</html>\r\n"
+        ),
+        msg_subj, msg_subj, msg_body_html, pgp_msg
+    ))
+}
+
+pub fn create_setup_code(_context: &Context) -> String {
+    let mut random_val: u16;
+    let mut rng = thread_rng();
+    let mut ret = String::new();
+
+    for i in 0..9 {
+        loop {
+            random_val = rng.gen();
+            if !(random_val as usize > 60000) {
+                break;
+            }
+        }
+        random_val = (random_val as usize % 10000) as u16;
+        ret += &format!(
+            "{}{:04}",
+            if 0 != i { "-" } else { "" },
+            random_val as usize
+        );
+    }
+
+    ret
+}
+
+fn maybe_add_bcc_self_device_msg(context: &Context) -> Result<()> {
+    if !context.sql.get_raw_config_bool(context, "bcc_self") {
+        let mut msg = Message::new(Viewtype::Text);
+        // TODO: define this as a stockstring once the wording is settled.
+        msg.text = Some(
+            "It seems you are using multiple devices with Delta Chat. Great!\n\n\
+             If you also want to synchronize outgoing messages accross all devices, \
+             go to the settings and enable \"Send copy to self\"."
+                .to_string(),
+        );
+        chat::add_device_msg(context, Some("bcc-self-hint"), Some(&mut msg))?;
+    }
+    Ok(())
+}
+
+pub fn continue_key_transfer(context: &Context, msg_id: MsgId, setup_code: &str) -> Result<()> {
+    ensure!(!msg_id.is_special(), "wrong id");
+
+    let msg = Message::load_from_db(context, msg_id)?;
+    ensure!(
+        msg.is_setupmessage(),
+        "Message is no Autocrypt Setup Message."
+    );
+
+    if let Some(filename) = msg.get_file(context) {
+        let file = dc_open_file(context, filename)?;
+        let sc = normalize_setup_code(setup_code);
+        let armored_key = decrypt_setup_file(context, &sc, file)?;
+        set_self_key(context, &armored_key, true, true)?;
+        maybe_add_bcc_self_device_msg(context)?;
+
+        Ok(())
+    } else {
+        bail!("Message is no Autocrypt Setup Message.");
+    }
+}
+
+fn set_self_key(
+    context: &Context,
+    armored: &str,
+    set_default: bool,
+    prefer_encrypt_required: bool,
+) -> Result<()> {
+    // try hard to only modify key-state
+    let keys = Key::from_armored_string(armored, KeyType::Private)
+        .and_then(|(k, h)| if k.verify() { Some((k, h)) } else { None })
+        .and_then(|(k, h)| k.split_key().map(|pub_key| (k, pub_key, h)));
+
+    ensure!(keys.is_some(), "Not a valid private key");
+
+    let (private_key, public_key, header) = keys.unwrap();
+    let preferencrypt = header.get("Autocrypt-Prefer-Encrypt");
+    match preferencrypt.map(|s| s.as_str()) {
+        Some(headerval) => {
+            let e2ee_enabled = match headerval {
+                "nopreference" => 0,
+                "mutual" => 1,
+                _ => {
+                    bail!("invalid Autocrypt-Prefer-Encrypt header: {:?}", header);
+                }
+            };
+            context
+                .sql
+                .set_raw_config_int(context, "e2ee_enabled", e2ee_enabled)?;
+        }
+        None => {
+            if prefer_encrypt_required {
+                bail!("missing Autocrypt-Prefer-Encrypt header");
+            }
+        }
+    };
+
+    let self_addr = context.get_config(Config::ConfiguredAddr);
+    ensure!(self_addr.is_some(), "Missing self addr");
+
+    // XXX maybe better make dc_key_save_self_keypair delete things
+    sql::execute(
+        context,
+        &context.sql,
+        "DELETE FROM keypairs WHERE public_key=? OR private_key=?;",
+        params![public_key.to_bytes(), private_key.to_bytes()],
+    )?;
+
+    if set_default {
+        sql::execute(
+            context,
+            &context.sql,
+            "UPDATE keypairs SET is_default=0;",
+            params![],
+        )?;
+    }
+
+    if !dc_key_save_self_keypair(
+        context,
+        &public_key,
+        &private_key,
+        self_addr.unwrap_or_default(),
+        set_default,
+        &context.sql,
+    ) {
+        bail!("Cannot save keypair, internal key-state possibly corrupted now!");
+    }
+    Ok(())
+}
+
+fn decrypt_setup_file<T: std::io::Read + std::io::Seek>(
+    _context: &Context,
+    passphrase: &str,
+    file: T,
+) -> Result<String> {
+    let plain_bytes = pgp::symm_decrypt(passphrase, file)?;
+    let plain_text = std::string::String::from_utf8(plain_bytes)?;
+
+    Ok(plain_text)
+}
+
+pub fn normalize_setup_code(s: &str) -> String {
+    let mut out = String::new();
+    for c in s.chars() {
+        if c >= '0' && c <= '9' {
+            out.push(c);
+            if let 4 | 9 | 14 | 19 | 24 | 29 | 34 | 39 = out.len() {
+                out += "-"
+            }
+        }
+    }
+    out
+}
+
+#[allow(non_snake_case)]
+pub fn JobImexImap(context: &Context, job: &Job) -> Result<()> {
+    ensure!(context.alloc_ongoing(), "could not allocate ongoing");
+    let what: Option<ImexMode> = job.param.get_int(Param::Cmd).and_then(ImexMode::from_i32);
+    let param = job.param.get(Param::Arg).unwrap_or_default();
+
+    ensure!(!param.is_empty(), "No Import/export dir/file given.");
+    info!(context, "Import/export process started.");
+    context.call_cb(Event::ImexProgress(10));
+
+    ensure!(context.sql.is_open(), "Database not opened.");
+    if what == Some(ImexMode::ExportBackup) || what == Some(ImexMode::ExportSelfKeys) {
+        // before we export anything, make sure the private key exists
+        if e2ee::ensure_secret_key_exists(context).is_err() {
+            context.free_ongoing();
+            bail!("Cannot create private key or private key not available.");
+        } else {
+            dc_create_folder(context, &param);
+        }
+    }
+    let path = Path::new(param);
+    let success = match what {
+        Some(ImexMode::ExportSelfKeys) => export_self_keys(context, path),
+        Some(ImexMode::ImportSelfKeys) => import_self_keys(context, path),
+        Some(ImexMode::ExportBackup) => export_backup(context, path),
+        Some(ImexMode::ImportBackup) => import_backup(context, path),
+        None => {
+            bail!("unknown IMEX type");
+        }
+    };
+    context.free_ongoing();
+    match success {
+        Ok(()) => {
+            info!(context, "IMEX successfully completed");
+            context.call_cb(Event::ImexProgress(1000));
+            Ok(())
+        }
+        Err(err) => {
+            context.call_cb(Event::ImexProgress(0));
+            bail!("IMEX FAILED to complete: {}", err);
+        }
+    }
+}
+
+/// Import Backup
+fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Result<()> {
+    info!(
+        context,
+        "Import \"{}\" to \"{}\".",
+        backup_to_import.as_ref().display(),
+        context.get_dbfile().display()
+    );
+
+    ensure!(
+        !dc_is_configured(context),
+        "Cannot import backups to accounts in use."
+    );
+    context.sql.close(&context);
+    dc_delete_file(context, context.get_dbfile());
+    ensure!(
+        !context.get_dbfile().exists(),
+        "Cannot delete old database."
+    );
+
+    ensure!(
+        dc_copy_file(context, backup_to_import.as_ref(), context.get_dbfile()),
+        "could not copy file"
+    );
+    /* error already logged */
+    /* re-open copied database file */
+    ensure!(
+        context.sql.open(&context, &context.get_dbfile(), false),
+        "could not re-open db"
+    );
+
+    let total_files_cnt = context
+        .sql
+        .query_get_value::<_, isize>(context, "SELECT COUNT(*) FROM backup_blobs;", params![])
+        .unwrap_or_default() as usize;
+    info!(
+        context,
+        "***IMPORT-in-progress: total_files_cnt={:?}", total_files_cnt,
+    );
+
+    let res = context.sql.query_map(
+        "SELECT file_name, file_content FROM backup_blobs ORDER BY id;",
+        params![],
+        |row| {
+            let name: String = row.get(0)?;
+            let blob: Vec<u8> = row.get(1)?;
+
+            Ok((name, blob))
+        },
+        |files| {
+            for (processed_files_cnt, file) in files.enumerate() {
+                let (file_name, file_blob) = file?;
+                ensure!(!context.shall_stop_ongoing(), "received stop signal");
+                let mut permille = processed_files_cnt * 1000 / total_files_cnt;
+                if permille < 10 {
+                    permille = 10
+                }
+                if permille > 990 {
+                    permille = 990
+                }
+                context.call_cb(Event::ImexProgress(permille));
+                if file_blob.is_empty() {
+                    continue;
+                }
+
+                let path_filename = context.get_blobdir().join(file_name);
+                if dc_write_file(context, &path_filename, &file_blob).is_err() {
+                    bail!(
+                        "Storage full? Cannot write file {} with {} bytes.",
+                        path_filename.display(),
+                        file_blob.len(),
+                    );
+                } else {
+                    continue;
+                }
+            }
+            Ok(())
+        },
+    );
+
+    res.and_then(|_| {
+        // only delete backup_blobs if all files were successfully extracted
+        sql::execute(context, &context.sql, "DROP TABLE backup_blobs;", params![])?;
+        sql::try_execute(context, &context.sql, "VACUUM;").ok();
+        Ok(())
+    })
+}
+
+/*******************************************************************************
+ * Export backup
+ ******************************************************************************/
+/* the FILE_PROGRESS macro calls the callback with the permille of files processed.
+The macro avoids weird values of 0% or 100% while still working. */
+fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
+    // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
+    // FIXME: we should write to a temporary file first and rename it on success. this would guarantee the backup is complete.
+    // let dest_path_filename = dc_get_next_backup_file(context, dir, res);
+    let now = time();
+    let dest_path_filename = dc_get_next_backup_path(dir, now)?;
+    let dest_path_string = dest_path_filename.to_string_lossy().to_string();
+
+    sql::housekeeping(context);
+
+    sql::try_execute(context, &context.sql, "VACUUM;").ok();
+
+    // we close the database during the copy of the dbfile
+    context.sql.close(context);
+    info!(
+        context,
+        "Backup '{}' to '{}'.",
+        context.get_dbfile().display(),
+        dest_path_filename.display(),
+    );
+    let copied = dc_copy_file(context, context.get_dbfile(), &dest_path_filename);
+    context.sql.open(&context, &context.get_dbfile(), false);
+
+    if !copied {
+        bail!(
+            "could not copy file from '{}' to '{}'",
+            context.get_dbfile().display(),
+            dest_path_string
+        );
+    }
+    let dest_sql = Sql::new();
+    ensure!(
+        dest_sql.open(context, &dest_path_filename, false),
+        "could not open exported database {}",
+        dest_path_string
+    );
+    let res = match add_files_to_export(context, &dest_sql) {
+        Err(err) => {
+            dc_delete_file(context, &dest_path_filename);
+            error!(context, "backup failed: {}", err);
+            Err(err)
+        }
+        Ok(()) => {
+            dest_sql.set_raw_config_int(context, "backup_time", now as i32)?;
+            context.call_cb(Event::ImexFileWritten(dest_path_filename.clone()));
+            Ok(())
+        }
+    };
+    dest_sql.close(context);
+
+    res
+}
+
+fn add_files_to_export(context: &Context, sql: &Sql) -> Result<()> {
+    // add all files as blobs to the database copy (this does not require
+    // the source to be locked, neigher the destination as it is used only here)
+    if !sql.table_exists("backup_blobs") {
+        sql::execute(
+            context,
+            &sql,
+            "CREATE TABLE backup_blobs (id INTEGER PRIMARY KEY, file_name, file_content);",
+            params![],
+        )?
+    }
+    // copy all files from BLOBDIR into backup-db
+    let mut total_files_cnt = 0;
+    let dir = context.get_blobdir();
+    let dir_handle = std::fs::read_dir(&dir)?;
+    total_files_cnt += dir_handle.filter(|r| r.is_ok()).count();
+
+    info!(context, "EXPORT: total_files_cnt={}", total_files_cnt);
+    // scan directory, pass 2: copy files
+    let dir_handle = std::fs::read_dir(&dir)?;
+    sql.prepare(
+        "INSERT INTO backup_blobs (file_name, file_content) VALUES (?, ?);",
+        |mut stmt, _| {
+            let mut processed_files_cnt = 0;
+            for entry in dir_handle {
+                let entry = entry?;
+                ensure!(
+                    !context.shall_stop_ongoing(),
+                    "canceled during export-files"
+                );
+                processed_files_cnt += 1;
+                let permille = max(min(processed_files_cnt * 1000 / total_files_cnt, 990), 10);
+                context.call_cb(Event::ImexProgress(permille));
+
+                let name_f = entry.file_name();
+                let name = name_f.to_string_lossy();
+                if name.starts_with("delta-chat") && name.ends_with(".bak") {
+                    continue;
+                }
+                info!(context, "EXPORT: copying filename={}", name);
+                let curr_path_filename = context.get_blobdir().join(entry.file_name());
+                if let Ok(buf) = dc_read_file(context, &curr_path_filename) {
+                    if buf.is_empty() {
+                        continue;
+                    }
+                    // bail out if we can't insert
+                    stmt.execute(params![name, buf])?;
+                }
+            }
+            Ok(())
+        },
+    )?;
+    Ok(())
+}
+
+/*******************************************************************************
+ * Classic key import
+ ******************************************************************************/
+fn import_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
+    /* hint: even if we switch to import Autocrypt Setup Files, we should leave the possibility to import
+    plain ASC keys, at least keys without a password, if we do not want to implement a password entry function.
+    Importing ASC keys is useful to use keys in Delta Chat used by any other non-Autocrypt-PGP implementation.
+
+    Maybe we should make the "default" key handlong also a little bit smarter
+    (currently, the last imported key is the standard key unless it contains the string "legacy" in its name) */
+    let mut set_default: bool;
+    let mut imported_cnt = 0;
+
+    let dir_name = dir.as_ref().to_string_lossy();
+    let dir_handle = std::fs::read_dir(&dir)?;
+    for entry in dir_handle {
+        let entry_fn = entry?.file_name();
+        let name_f = entry_fn.to_string_lossy();
+        let path_plus_name = dir.as_ref().join(&entry_fn);
+        match dc_get_filesuffix_lc(&name_f) {
+            Some(suffix) => {
+                if suffix != "asc" {
+                    continue;
+                }
+                set_default = if name_f.contains("legacy") {
+                    info!(context, "found legacy key '{}'", path_plus_name.display());
+                    false
+                } else {
+                    true
+                }
+            }
+            None => {
+                continue;
+            }
+        }
+        match dc_read_file(context, &path_plus_name) {
+            Ok(buf) => {
+                let armored = std::string::String::from_utf8_lossy(&buf);
+                if let Err(err) = set_self_key(context, &armored, set_default, false) {
+                    error!(context, "set_self_key: {}", err);
+                    continue;
+                }
+            }
+            Err(_) => continue,
+        }
+        imported_cnt += 1;
+    }
+    ensure!(
+        imported_cnt > 0,
+        "No private keys found in \"{}\".",
+        dir_name
+    );
+    Ok(())
+}
+
+fn export_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
+    let mut export_errors = 0;
+
+    context.sql.query_map(
+        "SELECT id, public_key, private_key, is_default FROM keypairs;",
+        params![],
+        |row| {
+            let id = row.get(0)?;
+            let public_key_blob: Vec<u8> = row.get(1)?;
+            let public_key = Key::from_slice(&public_key_blob, KeyType::Public);
+            let private_key_blob: Vec<u8> = row.get(2)?;
+            let private_key = Key::from_slice(&private_key_blob, KeyType::Private);
+            let is_default: i32 = row.get(3)?;
+
+            Ok((id, public_key, private_key, is_default))
+        },
+        |keys| {
+            for key_pair in keys {
+                let (id, public_key, private_key, is_default) = key_pair?;
+                let id = Some(id).filter(|_| is_default != 0);
+                if let Some(key) = public_key {
+                    if export_key_to_asc_file(context, &dir, id, &key).is_err() {
+                        export_errors += 1;
+                    }
+                } else {
+                    export_errors += 1;
+                }
+                if let Some(key) = private_key {
+                    if export_key_to_asc_file(context, &dir, id, &key).is_err() {
+                        export_errors += 1;
+                    }
+                } else {
+                    export_errors += 1;
+                }
+            }
+
+            Ok(())
+        },
+    )?;
+
+    ensure!(export_errors == 0, "errors while exporting keys");
+    Ok(())
+}
+
+/*******************************************************************************
+ * Classic key export
+ ******************************************************************************/
+fn export_key_to_asc_file(
+    context: &Context,
+    dir: impl AsRef<Path>,
+    id: Option<i64>,
+    key: &Key,
+) -> std::io::Result<()> {
+    let file_name = {
+        let kind = if key.is_public() { "public" } else { "private" };
+        let id = id.map_or("default".into(), |i| i.to_string());
+
+        dir.as_ref().join(format!("{}-key-{}.asc", kind, &id))
+    };
+    info!(context, "Exporting key {}", file_name.display());
+    dc_delete_file(context, &file_name);
+
+    let res = key.write_asc_to_file(&file_name, context);
+    if res.is_err() {
+        error!(context, "Cannot write key to {}", file_name.display());
+    } else {
+        context.call_cb(Event::ImexFileWritten(file_name));
+    }
+    res
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use crate::pgp::{split_armored_data, HEADER_AUTOCRYPT, HEADER_SETUPCODE};
+    use crate::test_utils::*;
+    use ::pgp::armor::BlockType;
+
+    #[test]
+    fn test_render_setup_file() {
+        let t = test_context(Some(Box::new(logging_cb)));
+
+        configure_alice_keypair(&t.ctx);
+        let msg = render_setup_file(&t.ctx, "hello").unwrap();
+        println!("{}", &msg);
+        // Check some substrings, indicating things got substituted.
+        // In particular note the mixing of `\r\n` and `\n` depending
+        // on who generated the stings.
+        assert!(msg.contains("<title>Autocrypt Setup Message</title"));
+        assert!(msg.contains("<h1>Autocrypt Setup Message</h1>"));
+        assert!(msg.contains("<p>This is the Autocrypt Setup Message used to"));
+        assert!(msg.contains("-----BEGIN PGP MESSAGE-----\r\n"));
+        assert!(msg.contains("Passphrase-Format: numeric9x4\r\n"));
+        assert!(msg.contains("Passphrase-Begin: he\n"));
+        assert!(msg.contains("==\n"));
+        assert!(msg.contains("-----END PGP MESSAGE-----\n"));
+    }
+
+    #[test]
+    fn test_render_setup_file_newline_replace() {
+        let t = dummy_context();
+        t.ctx
+            .set_stock_translation(StockMessage::AcSetupMsgBody, "hello\r\nthere".to_string())
+            .unwrap();
+        configure_alice_keypair(&t.ctx);
+        let msg = render_setup_file(&t.ctx, "pw").unwrap();
+        println!("{}", &msg);
+        assert!(msg.contains("<p>hello<br>there</p>"));
+    }
+
+    #[test]
+    fn test_create_setup_code() {
+        let t = dummy_context();
+        let setupcode = create_setup_code(&t.ctx);
+        assert_eq!(setupcode.len(), 44);
+        assert_eq!(setupcode.chars().nth(4).unwrap(), '-');
+        assert_eq!(setupcode.chars().nth(9).unwrap(), '-');
+        assert_eq!(setupcode.chars().nth(14).unwrap(), '-');
+        assert_eq!(setupcode.chars().nth(19).unwrap(), '-');
+        assert_eq!(setupcode.chars().nth(24).unwrap(), '-');
+        assert_eq!(setupcode.chars().nth(29).unwrap(), '-');
+        assert_eq!(setupcode.chars().nth(34).unwrap(), '-');
+        assert_eq!(setupcode.chars().nth(39).unwrap(), '-');
+    }
+
+    #[test]
+    fn test_export_key_to_asc_file() {
+        let context = dummy_context();
+        let base64 = include_str!("../test-data/key/public.asc");
+        let key = Key::from_base64(base64, KeyType::Public).unwrap();
+        let blobdir = "$BLOBDIR";
+        assert!(export_key_to_asc_file(&context.ctx, blobdir, None, &key).is_ok());
+        let blobdir = context.ctx.get_blobdir().to_str().unwrap();
+        let filename = format!("{}/public-key-default.asc", blobdir);
+        let bytes = std::fs::read(&filename).unwrap();
+
+        assert_eq!(bytes, key.to_asc(None).into_bytes());
+    }
+
+    #[test]
+    fn test_normalize_setup_code() {
+        let norm = normalize_setup_code("123422343234423452346234723482349234");
+        assert_eq!(norm, "1234-2234-3234-4234-5234-6234-7234-8234-9234");
+
+        let norm =
+            normalize_setup_code("\t1 2 3422343234- foo bar-- 423-45 2 34 6234723482349234      ");
+        assert_eq!(norm, "1234-2234-3234-4234-5234-6234-7234-8234-9234");
+    }
+
+    /* S_EM_SETUPFILE is a AES-256 symm. encrypted setup message created by Enigmail
+    with an "encrypted session key", see RFC 4880.  The code is in S_EM_SETUPCODE */
+    const S_EM_SETUPCODE: &str = "1742-0185-6197-1303-7016-8412-3581-4441-0597";
+    const S_EM_SETUPFILE: &str = include_str!("../test-data/message/stress.txt");
+
+    #[test]
+    fn test_split_and_decrypt() {
+        let ctx = dummy_context();
+        let context = &ctx.ctx;
+
+        let buf_1 = S_EM_SETUPFILE.as_bytes().to_vec();
+        let (typ, headers, base64) = split_armored_data(&buf_1).unwrap();
+        assert_eq!(typ, BlockType::Message);
+        assert!(S_EM_SETUPCODE.starts_with(headers.get(HEADER_SETUPCODE).unwrap()));
+        assert!(headers.get(HEADER_AUTOCRYPT).is_none());
+
+        assert!(!base64.is_empty());
+
+        let setup_file = S_EM_SETUPFILE.to_string();
+        let decrypted = decrypt_setup_file(
+            context,
+            S_EM_SETUPCODE,
+            std::io::Cursor::new(setup_file.as_bytes()),
+        )
+        .unwrap();
+
+        let (typ, headers, _base64) = split_armored_data(decrypted.as_bytes()).unwrap();
+
+        assert_eq!(typ, BlockType::PrivateKey);
+        assert_eq!(headers.get(HEADER_AUTOCRYPT), Some(&"mutual".to_string()));
+        assert!(headers.get(HEADER_SETUPCODE).is_none());
+    }
+}

src/job_thread.rs

@@ -1,7 +1,7 @@
 use std::sync::{Arc, Condvar, Mutex};
 
-use crate::configure::*;
 use crate::context::Context;
+use crate::error::{Error, Result};
 use crate::imap::Imap;
 
 #[derive(Debug)]
@@ -15,7 +15,7 @@ pub struct JobThread {
 #[derive(Clone, Debug, Default)]
 pub struct JobState {
     idle: bool,
-    jobs_needed: i32,
+    jobs_needed: bool,
     suspended: bool,
     using_handle: bool,
 }
@@ -58,7 +58,7 @@ impl JobThread {
 
     pub fn interrupt_idle(&self, context: &Context) {
         {
-            self.state.0.lock().unwrap().jobs_needed = 1;
+            self.state.0.lock().unwrap().jobs_needed = true;
         }
 
         info!(context, "Interrupting {}-IDLE...", self.name);
@@ -70,6 +70,7 @@ impl JobThread {
 
         state.idle = true;
         cvar.notify_one();
+        info!(context, "Interrupting {}-IDLE... finished", self.name);
     }
 
     pub fn fetch(&mut self, context: &Context, use_network: bool) {
@@ -85,53 +86,49 @@ impl JobThread {
         }
 
         if use_network {
-            let start = std::time::Instant::now();
-            if self.connect_to_imap(context) {
-                info!(context, "{}-fetch started...", self.name);
-                self.imap.fetch(context);
-
-                if self.imap.should_reconnect() {
-                    info!(context, "{}-fetch aborted, starting over...", self.name,);
-                    self.imap.fetch(context);
+            if let Err(err) = self.connect_and_fetch(context) {
+                warn!(context, "connect+fetch failed: {}, reconnect & retry", err);
+                self.imap.trigger_reconnect();
+                if let Err(err) = self.connect_and_fetch(context) {
+                    warn!(context, "connect+fetch failed: {}", err);
                 }
-                info!(
-                    context,
-                    "{}-fetch done in {:.3} ms.",
-                    self.name,
-                    start.elapsed().as_millis(),
-                );
             }
         }
-
         self.state.0.lock().unwrap().using_handle = false;
     }
 
-    fn connect_to_imap(&self, context: &Context) -> bool {
-        if self.imap.is_connected() {
-            return true;
-        }
+    fn connect_and_fetch(&mut self, context: &Context) -> Result<()> {
+        let prefix = format!("{}-fetch", self.name);
+        match self.imap.connect_configured(context) {
+            Ok(()) => {
+                if let Some(watch_folder) = self.get_watch_folder(context) {
+                    let start = std::time::Instant::now();
+                    info!(context, "{} started...", prefix);
+                    let res = self.imap.fetch(context, &watch_folder).map_err(Into::into);
+                    let elapsed = start.elapsed().as_millis();
+                    info!(context, "{} done in {:.3} ms.", prefix, elapsed);
 
-        let mut ret_connected = dc_connect_to_configured_imap(context, &self.imap) != 0;
-
-        if ret_connected {
-            if context
-                .sql
-                .get_config_int(context, "folders_configured")
-                .unwrap_or_default()
-                < 3
-            {
-                self.imap.configure_folders(context, 0x1);
+                    res
+                } else {
+                    Err(Error::WatchFolderNotFound("not-set".to_string()))
+                }
             }
+            Err(err) => Err(crate::error::Error::Message(err.to_string())),
+        }
+    }
 
-            if let Some(mvbox_name) = context.sql.get_config(context, self.folder_config_name) {
-                self.imap.set_watch_folder(mvbox_name);
-            } else {
-                self.imap.disconnect(context);
-                ret_connected = false;
+    fn get_watch_folder(&self, context: &Context) -> Option<String> {
+        match context.sql.get_raw_config(context, self.folder_config_name) {
+            Some(name) => Some(name),
+            None => {
+                if self.folder_config_name == "configured_inbox_folder" {
+                    // initialized with old version, so has not set configured_inbox_folder
+                    Some("INBOX".to_string())
+                } else {
+                    None
+                }
             }
         }
-
-        ret_connected
     }
 
     pub fn idle(&self, context: &Context, use_network: bool) {
@@ -139,13 +136,13 @@ impl JobThread {
             let &(ref lock, ref cvar) = &*self.state.clone();
             let mut state = lock.lock().unwrap();
 
-            if 0 != state.jobs_needed {
+            if state.jobs_needed {
                 info!(
                     context,
                     "{}-IDLE will not be started as it was interrupted while not ideling.",
                     self.name,
                 );
-                state.jobs_needed = 0;
+                state.jobs_needed = false;
                 return;
             }
 
@@ -170,10 +167,37 @@ impl JobThread {
             }
         }
 
-        self.connect_to_imap(context);
-        info!(context, "{}-IDLE started...", self.name,);
-        self.imap.idle(context);
-        info!(context, "{}-IDLE ended.", self.name);
+        let prefix = format!("{}-IDLE", self.name);
+        let do_fake_idle = match self.imap.connect_configured(context) {
+            Ok(()) => {
+                info!(context, "{} started...", prefix);
+                let watch_folder = self.get_watch_folder(context);
+                let res = self.imap.idle(context, watch_folder);
+                info!(context, "{} ended...", prefix);
+                match res {
+                    Ok(()) => false,
+                    Err(crate::imap::Error::IdleAbilityMissing) => true, // we have to do fake_idle
+                    Err(err) => {
+                        warn!(context, "{} failed: {} -> reconnecting", prefix, err);
+                        // something is borked, let's start afresh on the next occassion
+                        self.imap.disconnect(context);
+
+                        false
+                    }
+                }
+            }
+            Err(err) => {
+                info!(context, "{}-IDLE connection fail: {:?}", self.name, err);
+                // if the connection fails, use fake_idle to retry periodically
+                // fake_idle() will be woken up by interrupt_idle() as
+                // well so will act on maybe_network events
+                true
+            }
+        };
+        if do_fake_idle {
+            let watch_folder = self.get_watch_folder(context);
+            self.imap.fake_idle(context, watch_folder);
+        }
 
         self.state.0.lock().unwrap().using_handle = false;
     }

src/key.rs

@@ -1,8 +1,9 @@
+//! Cryptographic key module
+
 use std::collections::BTreeMap;
 use std::io::Cursor;
 use std::path::Path;
 
-use libc;
 use pgp::composed::{Deserializable, SignedPublicKey, SignedSecretKey};
 use pgp::ser::Serialize;
 use pgp::types::{KeyTrait, SecretKeyTrait};
@@ -12,6 +13,7 @@ use crate::context::Context;
 use crate::dc_tools::*;
 use crate::sql::{self, Sql};
 
+/// Cryptographic key
 #[derive(Debug, PartialEq, Eq, Clone)]
 pub enum Key {
     Public(SignedPublicKey),
@@ -30,44 +32,44 @@ impl From<SignedSecretKey> for Key {
     }
 }
 
-impl std::convert::TryInto<SignedSecretKey> for Key {
+impl std::convert::TryFrom<Key> for SignedSecretKey {
     type Error = ();
 
-    fn try_into(self) -> Result<SignedSecretKey, Self::Error> {
-        match self {
+    fn try_from(value: Key) -> Result<Self, Self::Error> {
+        match value {
             Key::Public(_) => Err(()),
             Key::Secret(key) => Ok(key),
         }
     }
 }
 
-impl<'a> std::convert::TryInto<&'a SignedSecretKey> for &'a Key {
+impl<'a> std::convert::TryFrom<&'a Key> for &'a SignedSecretKey {
     type Error = ();
 
-    fn try_into(self) -> Result<&'a SignedSecretKey, Self::Error> {
-        match self {
+    fn try_from(value: &'a Key) -> Result<Self, Self::Error> {
+        match value {
             Key::Public(_) => Err(()),
             Key::Secret(key) => Ok(key),
         }
     }
 }
 
-impl std::convert::TryInto<SignedPublicKey> for Key {
+impl std::convert::TryFrom<Key> for SignedPublicKey {
     type Error = ();
 
-    fn try_into(self) -> Result<SignedPublicKey, Self::Error> {
-        match self {
+    fn try_from(value: Key) -> Result<Self, Self::Error> {
+        match value {
             Key::Public(key) => Ok(key),
             Key::Secret(_) => Err(()),
         }
     }
 }
 
-impl<'a> std::convert::TryInto<&'a SignedPublicKey> for &'a Key {
+impl<'a> std::convert::TryFrom<&'a Key> for &'a SignedPublicKey {
     type Error = ();
 
-    fn try_into(self) -> Result<&'a SignedPublicKey, Self::Error> {
-        match self {
+    fn try_from(value: &'a Key) -> Result<Self, Self::Error> {
+        match value {
             Key::Public(key) => Ok(key),
             Key::Secret(_) => Err(()),
         }
@@ -164,8 +166,8 @@ impl Key {
 
     pub fn to_bytes(&self) -> Vec<u8> {
         match self {
-            Key::Public(k) => k.to_bytes().unwrap(),
-            Key::Secret(k) => k.to_bytes().unwrap(),
+            Key::Public(k) => k.to_bytes().unwrap_or_default(),
+            Key::Secret(k) => k.to_bytes().unwrap_or_default(),
         }
     }
 
@@ -181,15 +183,15 @@ impl Key {
 
         let encoded = base64::encode(&buf);
         encoded
-            .as_bytes()
-            .chunks(break_every)
-            .fold(String::new(), |mut res, buf| {
-                // safe because we are using a base64 encoded string
-                res += unsafe { std::str::from_utf8_unchecked(buf) };
-                res += " ";
+            .chars()
+            .enumerate()
+            .fold(String::new(), |mut res, (i, c)| {
+                if i > 0 && i % break_every == 0 {
+                    res.push(' ')
+                }
+                res.push(c);
                 res
             })
-            .trim()
             .to_string()
     }
 
@@ -215,15 +217,18 @@ impl Key {
             .expect("failed to serialize key")
     }
 
-    pub fn write_asc_to_file(&self, file: impl AsRef<Path>, context: &Context) -> bool {
+    pub fn write_asc_to_file(
+        &self,
+        file: impl AsRef<Path>,
+        context: &Context,
+    ) -> std::io::Result<()> {
         let file_content = self.to_asc(None).into_bytes();
 
-        if dc_write_file(context, &file, &file_content) {
-            return true;
-        } else {
+        let res = dc_write_file(context, &file, &file_content);
+        if res.is_err() {
             error!(context, "Cannot write key to {}", file.as_ref().display());
-            return false;
         }
+        res
     }
 
     pub fn fingerprint(&self) -> String {
@@ -254,14 +259,14 @@ pub fn dc_key_save_self_keypair(
     public_key: &Key,
     private_key: &Key,
     addr: impl AsRef<str>,
-    is_default: libc::c_int,
+    is_default: bool,
     sql: &Sql,
 ) -> bool {
     sql::execute(
         context,
         sql,
         "INSERT INTO keypairs (addr, is_default, public_key, private_key, created) VALUES (?,?,?,?,?);",
-        params![addr.as_ref(), is_default, public_key.to_bytes(), private_key.to_bytes(), time()],
+        params![addr.as_ref(), is_default as i32, public_key.to_bytes(), private_key.to_bytes(), time()],
     ).is_ok()
 }
 
@@ -382,7 +387,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     #[test]
     #[ignore] // is too expensive
     fn test_from_slice_roundtrip() {
-        let (public_key, private_key) = crate::pgp::dc_pgp_create_keypair("hello").unwrap();
+        let (public_key, private_key) = crate::pgp::create_keypair("hello").unwrap();
 
         let binary = public_key.to_bytes();
         let public_key2 = Key::from_slice(&binary, KeyType::Public).expect("invalid public key");
@@ -417,7 +422,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     #[test]
     #[ignore] // is too expensive
     fn test_ascii_roundtrip() {
-        let (public_key, private_key) = crate::pgp::dc_pgp_create_keypair("hello").unwrap();
+        let (public_key, private_key) = crate::pgp::create_keypair("hello").unwrap();
 
         let s = public_key.to_armored_string(None).unwrap();
         let (public_key2, _) =

src/lib.rs

@@ -30,6 +30,7 @@ pub(crate) mod events;
 pub use events::*;
 
 mod aheader;
+pub mod blob;
 pub mod chat;
 pub mod chatlist;
 pub mod config;
@@ -39,36 +40,37 @@ pub mod contact;
 pub mod context;
 mod e2ee;
 mod imap;
+mod imap_client;
+pub mod imex;
 pub mod job;
 mod job_thread;
 pub mod key;
 pub mod keyring;
 pub mod location;
+mod login_param;
 pub mod lot;
 pub mod message;
+mod mimefactory;
 pub mod oauth2;
 mod param;
 pub mod peerstate;
 pub mod pgp;
 pub mod qr;
+pub mod securejoin;
 mod smtp;
 pub mod sql;
-mod stock;
+pub mod stock;
+mod token;
+#[macro_use]
+mod wrapmime;
 
 pub mod dc_array;
 mod dc_dehtml;
-pub mod dc_imex;
 pub mod dc_mimeparser;
 pub mod dc_receive_imf;
 mod dc_simplify;
 mod dc_strencode;
 pub mod dc_tools;
-mod login_param;
-mod mimefactory;
-pub mod securejoin;
-mod token;
-#[macro_use]
-mod wrapmime;
 
 #[cfg(test)]
 mod test_utils;

src/location.rs

@@ -1,20 +1,24 @@
+//! Location handling
+
 use bitflags::bitflags;
 use quick_xml;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
 use crate::chat;
+use crate::config::Config;
 use crate::constants::*;
 use crate::context::*;
+use crate::dc_mimeparser::SystemMessage;
 use crate::dc_tools::*;
 use crate::error::Error;
 use crate::events::Event;
 use crate::job::*;
-use crate::message::Message;
+use crate::message::{Message, MsgId};
 use crate::param::*;
 use crate::sql;
 use crate::stock::StockMessage;
 
-// location handling
+/// Location record
 #[derive(Debug, Clone, Default)]
 pub struct Location {
     pub location_id: u32,
@@ -60,14 +64,11 @@ impl Kml {
         Default::default()
     }
 
-    pub fn parse(context: &Context, content: impl AsRef<str>) -> Result<Self, Error> {
-        ensure!(
-            content.as_ref().len() <= (1 * 1024 * 1024),
-            "A kml-files with {} bytes is larger than reasonably expected.",
-            content.as_ref().len()
-        );
+    pub fn parse(context: &Context, content: &[u8]) -> Result<Self, Error> {
+        ensure!(content.len() <= 1024 * 1024, "kml-file is too large");
 
-        let mut reader = quick_xml::Reader::from_str(content.as_ref());
+        let to_parse = String::from_utf8_lossy(content);
+        let mut reader = quick_xml::Reader::from_str(&to_parse);
         reader.trim_text(true);
 
         let mut kml = Kml::new();
@@ -194,10 +195,8 @@ impl Kml {
 // location streaming
 pub fn send_locations_to_chat(context: &Context, chat_id: u32, seconds: i64) {
     let now = time();
-    let mut msg: Message;
-    let is_sending_locations_before: bool;
-    if !(seconds < 0 || chat_id <= 9i32 as libc::c_uint) {
-        is_sending_locations_before = is_sending_locations_to_chat(context, chat_id);
+    if !(seconds < 0 || chat_id <= DC_CHAT_ID_LAST_SPECIAL) {
+        let is_sending_locations_before = is_sending_locations_to_chat(context, chat_id);
         if sql::execute(
             context,
             &context.sql,
@@ -214,19 +213,19 @@ pub fn send_locations_to_chat(context: &Context, chat_id: u32, seconds: i64) {
         .is_ok()
         {
             if 0 != seconds && !is_sending_locations_before {
-                msg = Message::new(Viewtype::Text);
+                let mut msg = Message::new(Viewtype::Text);
                 msg.text =
                     Some(context.stock_system_msg(StockMessage::MsgLocationEnabled, "", "", 0));
-                msg.param.set_int(Param::Cmd, 8);
-                chat::send_msg(context, chat_id, &mut msg).unwrap();
+                msg.param.set_cmd(SystemMessage::LocationStreamingEnabled);
+                chat::send_msg(context, chat_id, &mut msg).unwrap_or_default();
             } else if 0 == seconds && is_sending_locations_before {
                 let stock_str =
                     context.stock_system_msg(StockMessage::MsgLocationDisabled, "", "", 0);
-                chat::add_device_msg(context, chat_id, stock_str);
+                chat::add_info_msg(context, chat_id, stock_str);
             }
             context.call_cb(Event::ChatModified(chat_id));
             if 0 != seconds {
-                schedule_MAYBE_SEND_LOCATIONS(context, 0i32);
+                schedule_MAYBE_SEND_LOCATIONS(context, false);
                 job_add(
                     context,
                     Action::MaybeSendLocationsEnded,
@@ -240,8 +239,8 @@ pub fn send_locations_to_chat(context: &Context, chat_id: u32, seconds: i64) {
 }
 
 #[allow(non_snake_case)]
-fn schedule_MAYBE_SEND_LOCATIONS(context: &Context, flags: i32) {
-    if 0 != flags & 0x1 || !job_action_exists(context, Action::MaybeSendLocations) {
+fn schedule_MAYBE_SEND_LOCATIONS(context: &Context, force_schedule: bool) {
+    if force_schedule || !job_action_exists(context, Action::MaybeSendLocations) {
         job_add(context, Action::MaybeSendLocations, 0, Params::new(), 60);
     };
 }
@@ -256,9 +255,9 @@ pub fn is_sending_locations_to_chat(context: &Context, chat_id: u32) -> bool {
         .unwrap_or_default()
 }
 
-pub fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64) -> libc::c_int {
+pub fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64) -> bool {
     if latitude == 0.0 && longitude == 0.0 {
-        return 1;
+        return true;
     }
     let mut continue_streaming = false;
 
@@ -278,7 +277,7 @@ pub fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64) -> l
                         accuracy,
                         time(),
                         chat_id,
-                        1,
+                        DC_CONTACT_ID_SELF,
                     ]
             ) {
                 warn!(context, "failed to store location {:?}", err);
@@ -287,12 +286,12 @@ pub fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64) -> l
             }
         }
         if continue_streaming {
-            context.call_cb(Event::LocationChanged(Some(1)));
+            context.call_cb(Event::LocationChanged(Some(DC_CONTACT_ID_SELF)));
         };
-        schedule_MAYBE_SEND_LOCATIONS(context, 0);
+        schedule_MAYBE_SEND_LOCATIONS(context, false);
     }
 
-    continue_streaming as libc::c_int
+    continue_streaming
 }
 
 pub fn get_range(
@@ -305,16 +304,15 @@ pub fn get_range(
     if timestamp_to == 0 {
         timestamp_to = time() + 10;
     }
-
     context
         .sql
         .query_map(
             "SELECT l.id, l.latitude, l.longitude, l.accuracy, l.timestamp, l.independent, \
-             m.id, l.from_id, l.chat_id, m.txt \
+             COALESCE(m.id, 0) AS msg_id, l.from_id, l.chat_id, COALESCE(m.txt, '') AS txt \
              FROM locations l  LEFT JOIN msgs m ON l.id=m.location_id  WHERE (? OR l.chat_id=?) \
              AND (? OR l.from_id=?) \
              AND (l.independent=1 OR (l.timestamp>=? AND l.timestamp<=?)) \
-             ORDER BY l.timestamp DESC, l.id DESC, m.id DESC;",
+             ORDER BY l.timestamp DESC, l.id DESC, msg_id DESC;",
             params![
                 if chat_id == 0 { 1 } else { 0 },
                 chat_id as i32,
@@ -331,7 +329,6 @@ pub fn get_range(
                 } else {
                     None
                 };
-
                 let loc = Location {
                     location_id: row.get(0)?,
                     latitude: row.get(1)?,
@@ -359,7 +356,7 @@ pub fn get_range(
 }
 
 fn is_marker(txt: &str) -> bool {
-    txt.len() == 1 && txt.chars().next().unwrap() != ' '
+    txt.len() == 1 && !txt.starts_with(' ')
 }
 
 pub fn delete_all(context: &Context) -> Result<(), Error> {
@@ -369,14 +366,10 @@ pub fn delete_all(context: &Context) -> Result<(), Error> {
 }
 
 pub fn get_kml(context: &Context, chat_id: u32) -> Result<(String, u32), Error> {
-    let now = time();
-    let mut location_count = 0;
-    let mut ret = String::new();
     let mut last_added_location_id = 0;
 
     let self_addr = context
-        .sql
-        .get_config(context, "configured_addr")
+        .get_config(Config::ConfiguredAddr)
         .unwrap_or_default();
 
     let (locations_send_begin, locations_send_until, locations_last_sent) = context.sql.query_row(
@@ -389,21 +382,24 @@ pub fn get_kml(context: &Context, chat_id: u32) -> Result<(String, u32), Error>
             Ok((send_begin, send_until, last_sent))
         })?;
 
-    if !(locations_send_begin == 0 || now > locations_send_until) {
+    let now = time();
+    let mut location_count = 0;
+    let mut ret = String::new();
+    if locations_send_begin != 0 && now <= locations_send_until {
         ret += &format!(
             "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<kml xmlns=\"http://www.opengis.net/kml/2.2\">\n<Document addr=\"{}\">\n",
             self_addr,
         );
 
         context.sql.query_map(
-            "SELECT id, latitude, longitude, accuracy, timestamp\
+            "SELECT id, latitude, longitude, accuracy, timestamp \
              FROM locations  WHERE from_id=? \
              AND timestamp>=? \
              AND (timestamp>=? OR timestamp=(SELECT MAX(timestamp) FROM locations WHERE from_id=?)) \
              AND independent=0 \
              GROUP BY timestamp \
              ORDER BY timestamp;",
-            params![1, locations_send_begin, locations_last_sent, 1],
+            params![DC_CONTACT_ID_SELF, locations_send_begin, locations_last_sent, DC_CONTACT_ID_SELF],
             |row| {
                 let location_id: i32 = row.get(0)?;
                 let latitude: f64 = row.get(1)?;
@@ -417,7 +413,7 @@ pub fn get_kml(context: &Context, chat_id: u32) -> Result<(String, u32), Error>
                 for row in rows {
                     let (location_id, latitude, longitude, accuracy, timestamp) = row?;
                     ret += &format!(
-                        "<Placemark><Timestamp><when>{}</when></Timestamp><Point><coordinates accuracy=\"{}\">{},{}</coordinates></Point></Placemark>\n\x00",
+                        "<Placemark><Timestamp><when>{}</when></Timestamp><Point><coordinates accuracy=\"{}\">{},{}</coordinates></Point></Placemark>\n",
                         timestamp,
                         accuracy,
                         longitude,
@@ -429,10 +425,10 @@ pub fn get_kml(context: &Context, chat_id: u32) -> Result<(String, u32), Error>
                 Ok(())
             }
         )?;
+        ret += "</Document>\n</kml>";
     }
 
     ensure!(location_count > 0, "No locations processed");
-    ret += "</Document>\n</kml>";
 
     Ok((ret, last_added_location_id))
 }
@@ -476,12 +472,16 @@ pub fn set_kml_sent_timestamp(
     Ok(())
 }
 
-pub fn set_msg_location_id(context: &Context, msg_id: u32, location_id: u32) -> Result<(), Error> {
+pub fn set_msg_location_id(
+    context: &Context,
+    msg_id: MsgId,
+    location_id: u32,
+) -> Result<(), Error> {
     sql::execute(
         context,
         &context.sql,
         "UPDATE msgs SET location_id=? WHERE id=?;",
-        params![location_id, msg_id as i32],
+        params![location_id, msg_id],
     )?;
 
     Ok(())
@@ -492,9 +492,9 @@ pub fn save(
     chat_id: u32,
     contact_id: u32,
     locations: &[Location],
-    independent: i32,
+    independent: bool,
 ) -> Result<u32, Error> {
-    ensure!(chat_id > 9, "Invalid chat id");
+    ensure!(chat_id > DC_CHAT_ID_LAST_SPECIAL, "Invalid chat id");
     context.sql.prepare2(
         "SELECT id FROM locations WHERE timestamp=? AND from_id=?",
         "INSERT INTO locations\
@@ -507,7 +507,7 @@ pub fn save(
             for location in locations {
                 let exists = stmt_test.exists(params![location.timestamp, contact_id as i32])?;
 
-                if 0 != independent || !exists {
+                if independent || !exists {
                     stmt_insert.execute(params![
                         location.timestamp,
                         contact_id as i32,
@@ -538,9 +538,9 @@ pub fn save(
 }
 
 #[allow(non_snake_case)]
-pub fn job_do_DC_JOB_MAYBE_SEND_LOCATIONS(context: &Context, _job: &Job) {
+pub fn JobMaybeSendLocations(context: &Context, _job: &Job) {
     let now = time();
-    let mut continue_streaming: libc::c_int = 1;
+    let mut continue_streaming = false;
     info!(
         context,
         " ----------------- MAYBE_SEND_LOCATIONS -------------- ",
@@ -555,7 +555,7 @@ pub fn job_do_DC_JOB_MAYBE_SEND_LOCATIONS(context: &Context, _job: &Job) {
             let chat_id: i32 = row.get(0)?;
             let locations_send_begin: i64 = row.get(1)?;
             let locations_last_sent: i64 = row.get(2)?;
-            continue_streaming = 1;
+            continue_streaming = true;
 
             // be a bit tolerant as the timer may not align exactly with time(NULL)
             if now - locations_last_sent < (60 - 3) {
@@ -585,7 +585,11 @@ pub fn job_do_DC_JOB_MAYBE_SEND_LOCATIONS(context: &Context, _job: &Job) {
                         .into_iter()
                         .filter_map(|(chat_id, locations_send_begin, locations_last_sent)| {
                             if !stmt_locations
-                                .exists(params![1, locations_send_begin, locations_last_sent,])
+                                .exists(params![
+                                    DC_CONTACT_ID_SELF,
+                                    locations_send_begin,
+                                    locations_last_sent,
+                                ])
                                 .unwrap_or_default()
                             {
                                 // if there is no new location, there's nothing to send.
@@ -603,7 +607,7 @@ pub fn job_do_DC_JOB_MAYBE_SEND_LOCATIONS(context: &Context, _job: &Job) {
                                 // and dc_set_location() is typically called periodically, this is ok)
                                 let mut msg = Message::new(Viewtype::Text);
                                 msg.hidden = true;
-                                msg.param.set_int(Param::Cmd, 9);
+                                msg.param.set_cmd(SystemMessage::LocationOnly);
                                 Some((chat_id, msg))
                             }
                         })
@@ -615,16 +619,16 @@ pub fn job_do_DC_JOB_MAYBE_SEND_LOCATIONS(context: &Context, _job: &Job) {
 
         for (chat_id, mut msg) in msgs.into_iter() {
             // TODO: better error handling
-            chat::send_msg(context, chat_id as u32, &mut msg).unwrap();
+            chat::send_msg(context, chat_id as u32, &mut msg).unwrap_or_default();
         }
     }
-    if 0 != continue_streaming {
-        schedule_MAYBE_SEND_LOCATIONS(context, 0x1);
+    if continue_streaming {
+        schedule_MAYBE_SEND_LOCATIONS(context, true);
     }
 }
 
 #[allow(non_snake_case)]
-pub fn job_do_DC_JOB_MAYBE_SEND_LOC_ENDED(context: &Context, job: &mut Job) {
+pub fn JobMaybeSendLocationsEnded(context: &Context, job: &mut Job) {
     // this function is called when location-streaming _might_ have ended for a chat.
     // the function checks, if location-streaming is really ended;
     // if so, a device-message is added if not yet done.
@@ -647,7 +651,7 @@ pub fn job_do_DC_JOB_MAYBE_SEND_LOC_ENDED(context: &Context, job: &mut Job) {
                     params![chat_id as i32],
                 ).is_ok() {
                     let stock_str = context.stock_system_msg(StockMessage::MsgLocationDisabled, "", "", 0);
-                    chat::add_device_msg(context, chat_id, stock_str);
+                    chat::add_info_msg(context, chat_id, stock_str);
                     context.call_cb(Event::ChatModified(chat_id));
                 }
             }
@@ -665,9 +669,9 @@ mod tests {
         let context = dummy_context();
 
         let xml =
-            "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<kml xmlns=\"http://www.opengis.net/kml/2.2\">\n<Document addr=\"user@example.org\">\n<Placemark><Timestamp><when>2019-03-06T21:09:57Z</when></Timestamp><Point><coordinates accuracy=\"32.000000\">9.423110,53.790302</coordinates></Point></Placemark>\n<PlaceMARK>\n<Timestamp><WHEN > \n\t2018-12-13T22:11:12Z\t</WHEN></Timestamp><Point><coordinates aCCuracy=\"2.500000\"> 19.423110 \t , \n 63.790302\n </coordinates></Point></PlaceMARK>\n</Document>\n</kml>";
+            b"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<kml xmlns=\"http://www.opengis.net/kml/2.2\">\n<Document addr=\"user@example.org\">\n<Placemark><Timestamp><when>2019-03-06T21:09:57Z</when></Timestamp><Point><coordinates accuracy=\"32.000000\">9.423110,53.790302</coordinates></Point></Placemark>\n<PlaceMARK>\n<Timestamp><WHEN > \n\t2018-12-13T22:11:12Z\t</WHEN></Timestamp><Point><coordinates aCCuracy=\"2.500000\"> 19.423110 \t , \n 63.790302\n </coordinates></Point></PlaceMARK>\n</Document>\n</kml>";
 
-        let kml = Kml::parse(&context.ctx, &xml).expect("parsing failed");
+        let kml = Kml::parse(&context.ctx, xml).expect("parsing failed");
 
         assert!(kml.addr.is_some());
         assert_eq!(kml.addr.as_ref().unwrap(), "user@example.org",);

src/log.rs

@@ -1,12 +1,14 @@
+//! # Logging macros
+
 #[macro_export]
 macro_rules! info {
     ($ctx:expr,  $msg:expr) => {
         info!($ctx, $msg,)
     };
-    ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {
+    ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
         let formatted = format!($msg, $($args),*);
         emit_event!($ctx, $crate::Event::Info(formatted));
-    };
+    }};
 }
 
 #[macro_export]
@@ -14,10 +16,10 @@ macro_rules! warn {
     ($ctx:expr, $msg:expr) => {
         warn!($ctx, $msg,)
     };
-    ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {
+    ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
         let formatted = format!($msg, $($args),*);
         emit_event!($ctx, $crate::Event::Warning(formatted));
-    };
+    }};
 }
 
 #[macro_export]
@@ -25,10 +27,10 @@ macro_rules! error {
     ($ctx:expr, $msg:expr) => {
         error!($ctx, $msg,)
     };
-    ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {
+    ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
         let formatted = format!($msg, $($args),*);
         emit_event!($ctx, $crate::Event::Error(formatted));
-    };
+    }};
 }
 
 #[macro_export]

src/login_param.rs

@@ -3,6 +3,26 @@ use std::fmt;
 
 use crate::context::Context;
 use crate::error::Error;
+use async_std::sync::Arc;
+use rustls;
+use webpki;
+use webpki_roots;
+
+#[derive(Copy, Clone, Debug, Display, FromPrimitive)]
+#[repr(i32)]
+#[strum(serialize_all = "snake_case")]
+pub enum CertificateChecks {
+    Automatic = 0,
+    Strict = 1,
+    AcceptInvalidHostnames = 2,
+    AcceptInvalidCertificates = 3,
+}
+
+impl Default for CertificateChecks {
+    fn default() -> Self {
+        Self::Automatic
+    }
+}
 
 #[derive(Default, Debug)]
 pub struct LoginParam {
@@ -11,10 +31,14 @@ pub struct LoginParam {
     pub mail_user: String,
     pub mail_pw: String,
     pub mail_port: i32,
+    /// IMAP TLS options: whether to allow invalid certificates and/or invalid hostnames
+    pub imap_certificate_checks: CertificateChecks,
     pub send_server: String,
     pub send_user: String,
     pub send_pw: String,
     pub send_port: i32,
+    /// SMTP TLS options: whether to allow invalid certificates and/or invalid hostnames
+    pub smtp_certificate_checks: CertificateChecks,
     pub server_flags: i32,
 }
 
@@ -31,37 +55,53 @@ impl LoginParam {
 
         let key = format!("{}addr", prefix);
         let addr = sql
-            .get_config(context, key)
+            .get_raw_config(context, key)
             .unwrap_or_default()
             .trim()
             .to_string();
 
         let key = format!("{}mail_server", prefix);
-        let mail_server = sql.get_config(context, key).unwrap_or_default();
+        let mail_server = sql.get_raw_config(context, key).unwrap_or_default();
 
         let key = format!("{}mail_port", prefix);
-        let mail_port = sql.get_config_int(context, key).unwrap_or_default();
+        let mail_port = sql.get_raw_config_int(context, key).unwrap_or_default();
 
         let key = format!("{}mail_user", prefix);
-        let mail_user = sql.get_config(context, key).unwrap_or_default();
+        let mail_user = sql.get_raw_config(context, key).unwrap_or_default();
 
         let key = format!("{}mail_pw", prefix);
-        let mail_pw = sql.get_config(context, key).unwrap_or_default();
+        let mail_pw = sql.get_raw_config(context, key).unwrap_or_default();
+
+        let key = format!("{}imap_certificate_checks", prefix);
+        let imap_certificate_checks =
+            if let Some(certificate_checks) = sql.get_raw_config_int(context, key) {
+                num_traits::FromPrimitive::from_i32(certificate_checks).unwrap()
+            } else {
+                Default::default()
+            };
 
         let key = format!("{}send_server", prefix);
-        let send_server = sql.get_config(context, key).unwrap_or_default();
+        let send_server = sql.get_raw_config(context, key).unwrap_or_default();
 
         let key = format!("{}send_port", prefix);
-        let send_port = sql.get_config_int(context, key).unwrap_or_default();
+        let send_port = sql.get_raw_config_int(context, key).unwrap_or_default();
 
         let key = format!("{}send_user", prefix);
-        let send_user = sql.get_config(context, key).unwrap_or_default();
+        let send_user = sql.get_raw_config(context, key).unwrap_or_default();
 
         let key = format!("{}send_pw", prefix);
-        let send_pw = sql.get_config(context, key).unwrap_or_default();
+        let send_pw = sql.get_raw_config(context, key).unwrap_or_default();
+
+        let key = format!("{}smtp_certificate_checks", prefix);
+        let smtp_certificate_checks =
+            if let Some(certificate_checks) = sql.get_raw_config_int(context, key) {
+                num_traits::FromPrimitive::from_i32(certificate_checks).unwrap()
+            } else {
+                Default::default()
+            };
 
         let key = format!("{}server_flags", prefix);
-        let server_flags = sql.get_config_int(context, key).unwrap_or_default();
+        let server_flags = sql.get_raw_config_int(context, key).unwrap_or_default();
 
         LoginParam {
             addr: addr.to_string(),
@@ -69,10 +109,12 @@ impl LoginParam {
             mail_user,
             mail_pw,
             mail_port,
+            imap_certificate_checks,
             send_server,
             send_user,
             send_pw,
             send_port,
+            smtp_certificate_checks,
             server_flags,
         }
     }
@@ -91,34 +133,40 @@ impl LoginParam {
         let sql = &context.sql;
 
         let key = format!("{}addr", prefix);
-        sql.set_config(context, key, Some(&self.addr))?;
+        sql.set_raw_config(context, key, Some(&self.addr))?;
 
         let key = format!("{}mail_server", prefix);
-        sql.set_config(context, key, Some(&self.mail_server))?;
+        sql.set_raw_config(context, key, Some(&self.mail_server))?;
 
         let key = format!("{}mail_port", prefix);
-        sql.set_config_int(context, key, self.mail_port)?;
+        sql.set_raw_config_int(context, key, self.mail_port)?;
 
         let key = format!("{}mail_user", prefix);
-        sql.set_config(context, key, Some(&self.mail_user))?;
+        sql.set_raw_config(context, key, Some(&self.mail_user))?;
 
         let key = format!("{}mail_pw", prefix);
-        sql.set_config(context, key, Some(&self.mail_pw))?;
+        sql.set_raw_config(context, key, Some(&self.mail_pw))?;
+
+        let key = format!("{}imap_certificate_checks", prefix);
+        sql.set_raw_config_int(context, key, self.imap_certificate_checks as i32)?;
 
         let key = format!("{}send_server", prefix);
-        sql.set_config(context, key, Some(&self.send_server))?;
+        sql.set_raw_config(context, key, Some(&self.send_server))?;
 
         let key = format!("{}send_port", prefix);
-        sql.set_config_int(context, key, self.send_port)?;
+        sql.set_raw_config_int(context, key, self.send_port)?;
 
         let key = format!("{}send_user", prefix);
-        sql.set_config(context, key, Some(&self.send_user))?;
+        sql.set_raw_config(context, key, Some(&self.send_user))?;
 
         let key = format!("{}send_pw", prefix);
-        sql.set_config(context, key, Some(&self.send_pw))?;
+        sql.set_raw_config(context, key, Some(&self.send_pw))?;
+
+        let key = format!("{}smtp_certificate_checks", prefix);
+        sql.set_raw_config_int(context, key, self.smtp_certificate_checks as i32)?;
 
         let key = format!("{}server_flags", prefix);
-        sql.set_config_int(context, key, self.server_flags)?;
+        sql.set_raw_config_int(context, key, self.server_flags)?;
 
         Ok(())
     }
@@ -133,16 +181,18 @@ impl fmt::Display for LoginParam {
 
         write!(
             f,
-            "{} {}:{}:{}:{} {}:{}:{}:{} {}",
+            "{} imap:{}:{}:{}:{}:cert_{} smtp:{}:{}:{}:{}:cert_{} {}",
             unset_empty(&self.addr),
             unset_empty(&self.mail_user),
             if !self.mail_pw.is_empty() { pw } else { unset },
             unset_empty(&self.mail_server),
             self.mail_port,
+            self.imap_certificate_checks,
             unset_empty(&self.send_user),
             if !self.send_pw.is_empty() { pw } else { unset },
             unset_empty(&self.send_server),
             self.send_port,
+            self.smtp_certificate_checks,
             flags_readable,
         )
     }
@@ -204,3 +254,63 @@ fn get_readable_flags(flags: i32) -> String {
 
     res
 }
+
+pub struct NoCertificateVerification {}
+
+impl rustls::ServerCertVerifier for NoCertificateVerification {
+    fn verify_server_cert(
+        &self,
+        _roots: &rustls::RootCertStore,
+        _presented_certs: &[rustls::Certificate],
+        _dns_name: webpki::DNSNameRef<'_>,
+        _ocsp: &[u8],
+    ) -> Result<rustls::ServerCertVerified, rustls::TLSError> {
+        Ok(rustls::ServerCertVerified::assertion())
+    }
+}
+
+pub fn dc_build_tls_config(certificate_checks: CertificateChecks) -> rustls::ClientConfig {
+    let mut config = rustls::ClientConfig::new();
+    config
+        .root_store
+        .add_server_trust_anchors(&webpki_roots::TLS_SERVER_ROOTS);
+
+    match certificate_checks {
+        CertificateChecks::Strict => {}
+        CertificateChecks::Automatic => {
+            // Same as AcceptInvalidCertificates for now.
+            // TODO: use provider database when it becomes available
+            config
+                .dangerous()
+                .set_certificate_verifier(Arc::new(NoCertificateVerification {}));
+        }
+        CertificateChecks::AcceptInvalidCertificates => {
+            // TODO: only accept invalid certs
+            config
+                .dangerous()
+                .set_certificate_verifier(Arc::new(NoCertificateVerification {}));
+        }
+        CertificateChecks::AcceptInvalidHostnames => {
+            // TODO: only accept invalid hostnames
+            config
+                .dangerous()
+                .set_certificate_verifier(Arc::new(NoCertificateVerification {}));
+        }
+    }
+    config
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_certificate_checks_display() {
+        use std::string::ToString;
+
+        assert_eq!(
+            "accept_invalid_hostnames".to_string(),
+            CertificateChecks::AcceptInvalidHostnames.to_string()
+        );
+    }
+}

src/message.rs

@@ -1,7 +1,7 @@
 use std::path::{Path, PathBuf};
-use std::ptr;
 
 use deltachat_derive::{FromSql, ToSql};
+use failure::Fail;
 
 use crate::chat::{self, Chat};
 use crate::constants::*;
@@ -18,9 +18,134 @@ use crate::pgp::*;
 use crate::sql;
 use crate::stock::StockMessage;
 
-/// In practice, the user additionally cuts the string himself pixel-accurate.
+// In practice, the user additionally cuts the string themselves
+// pixel-accurate.
 const SUMMARY_CHARACTERS: usize = 160;
 
+/// Message ID, including reserved IDs.
+///
+/// Some message IDs are reserved to identify special message types.
+/// This type can represent both the special as well as normal
+/// messages.
+#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)]
+pub struct MsgId(u32);
+
+impl MsgId {
+    /// Create a new [MsgId].
+    pub fn new(id: u32) -> MsgId {
+        MsgId(id)
+    }
+
+    /// Create a new unset [MsgId].
+    pub fn new_unset() -> MsgId {
+        MsgId(0)
+    }
+
+    /// Whether the message ID signifies a special message.
+    ///
+    /// This kind of message ID can not be used for real messages.
+    pub fn is_special(&self) -> bool {
+        match self.0 {
+            0..=DC_MSG_ID_LAST_SPECIAL => true,
+            _ => false,
+        }
+    }
+
+    /// Whether the message ID is unset.
+    ///
+    /// When a message is created it initially has a ID of `0`, which
+    /// is filled in by a real message ID once the message is saved in
+    /// the database.  This returns true while the message has not
+    /// been saved and thus not yet been given an actual message ID.
+    ///
+    /// When this is `true`, [MsgId::is_special] will also always be
+    /// `true`.
+    pub fn is_unset(&self) -> bool {
+        self.0 == 0
+    }
+
+    /// Whether the message ID is the special marker1 marker.
+    ///
+    /// See the docs of the `dc_get_chat_msgs` C API for details.
+    pub fn is_marker1(&self) -> bool {
+        self.0 == DC_MSG_ID_MARKER1
+    }
+
+    /// Whether the message ID is the special day marker.
+    ///
+    /// See the docs of the `dc_get_chat_msgs` C API for details.
+    pub fn is_daymarker(&self) -> bool {
+        self.0 == DC_MSG_ID_DAYMARKER
+    }
+
+    /// Bad evil escape hatch.
+    ///
+    /// Avoid using this, eventually types should be cleaned up enough
+    /// that it is no longer necessary.
+    pub fn to_u32(&self) -> u32 {
+        self.0
+    }
+}
+
+impl std::fmt::Display for MsgId {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        // Would be nice if we could use match here, but no computed values in ranges.
+        if self.0 == DC_MSG_ID_MARKER1 {
+            write!(f, "Msg#Marker1")
+        } else if self.0 == DC_MSG_ID_DAYMARKER {
+            write!(f, "Msg#DayMarker")
+        } else if self.0 <= DC_MSG_ID_LAST_SPECIAL {
+            write!(f, "Msg#UnknownSpecial")
+        } else {
+            write!(f, "Msg#{}", self.0)
+        }
+    }
+}
+
+/// Allow converting [MsgId] to an SQLite type.
+///
+/// This allows you to directly store [MsgId] into the database.
+///
+/// # Errors
+///
+/// This **does** ensure that no special message IDs are written into
+/// the database and the conversion will fail if this is not the case.
+impl rusqlite::types::ToSql for MsgId {
+    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
+        if self.0 <= DC_MSG_ID_LAST_SPECIAL {
+            return Err(rusqlite::Error::ToSqlConversionFailure(Box::new(
+                InvalidMsgId.compat(),
+            )));
+        }
+        let val = rusqlite::types::Value::Integer(self.0 as i64);
+        let out = rusqlite::types::ToSqlOutput::Owned(val);
+        Ok(out)
+    }
+}
+
+/// Allow converting an SQLite integer directly into [MsgId].
+impl rusqlite::types::FromSql for MsgId {
+    fn column_result(value: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
+        // Would be nice if we could use match here, but alas.
+        i64::column_result(value).and_then(|val| {
+            if 0 <= val && val <= std::u32::MAX as i64 {
+                Ok(MsgId::new(val as u32))
+            } else {
+                Err(rusqlite::types::FromSqlError::OutOfRange(val))
+            }
+        })
+    }
+}
+
+/// Message ID was invalid.
+///
+/// This usually occurs when trying to use a message ID of
+/// [DC_MSG_ID_LAST_SPECIAL] or below in a situation where this is not
+/// possible.
+#[derive(Debug, Fail)]
+#[fail(display = "Invalid Message ID.")]
+pub struct InvalidMsgId;
+
 /// An object representing a single message in memory.
 /// The message object is not updated.
 /// If you want an update, you have to recreate the object.
@@ -30,11 +155,10 @@ const SUMMARY_CHARACTERS: usize = 160;
 /// approx. max. length returned by dc_get_msg_info()
 #[derive(Debug, Clone, Default)]
 pub struct Message {
-    pub(crate) id: u32,
+    pub(crate) id: MsgId,
     pub(crate) from_id: u32,
     pub(crate) to_id: u32,
     pub(crate) chat_id: u32,
-    pub(crate) move_state: MoveState,
     pub(crate) type_0: Viewtype,
     pub(crate) state: MessageState,
     pub(crate) hidden: bool,
@@ -62,69 +186,103 @@ impl Message {
         msg
     }
 
-    pub fn load_from_db(context: &Context, id: u32) -> Result<Message, Error> {
+    pub fn load_from_db(context: &Context, id: MsgId) -> Result<Message, Error> {
+        ensure!(
+            !id.is_special(),
+            "Can not load special message IDs from DB."
+        );
         context.sql.query_row(
-        "SELECT  \
-         m.id,rfc724_mid,m.mime_in_reply_to,m.server_folder,m.server_uid,m.move_state,m.chat_id,  \
-         m.from_id,m.to_id,m.timestamp,m.timestamp_sent,m.timestamp_rcvd, m.type,m.state,m.msgrmsg,m.txt,  \
-         m.param,m.starred,m.hidden,m.location_id, c.blocked  \
-         FROM msgs m \
-         LEFT JOIN chats c ON c.id=m.chat_id WHERE m.id=?;",
-        params![id as i32],
-        |row| {
-            let mut msg = Message::default();
-            msg.id = row.get::<_, i32>(0)? as u32;
-            msg.rfc724_mid = row.get::<_, String>(1)?;
-            msg.in_reply_to = row.get::<_, Option<String>>(2)?;
-            msg.server_folder = row.get::<_, Option<String>>(3)?;
-            msg.server_uid = row.get(4)?;
-            msg.move_state = row.get(5)?;
-            msg.chat_id = row.get(6)?;
-            msg.from_id = row.get(7)?;
-            msg.to_id = row.get(8)?;
-            msg.timestamp_sort = row.get(9)?;
-            msg.timestamp_sent = row.get(10)?;
-            msg.timestamp_rcvd = row.get(11)?;
-            msg.type_0 = row.get(12)?;
-            msg.state = row.get(13)?;
-            msg.is_dc_message = row.get(14)?;
+            concat!(
+                "SELECT",
+                "    m.id AS id,",
+                "    rfc724_mid AS rfc724mid,",
+                "    m.mime_in_reply_to AS mime_in_reply_to,",
+                "    m.server_folder AS server_folder,",
+                "    m.server_uid AS server_uid,",
+                "    m.chat_id AS chat_id,",
+                "    m.from_id AS from_id,",
+                "    m.to_id AS to_id,",
+                "    m.timestamp AS timestamp,",
+                "    m.timestamp_sent AS timestamp_sent,",
+                "    m.timestamp_rcvd AS timestamp_rcvd,",
+                "    m.type AS type,",
+                "    m.state AS state,",
+                "    m.msgrmsg AS msgrmsg,",
+                "    m.txt AS txt,",
+                "    m.param AS param,",
+                "    m.starred AS starred,",
+                "    m.hidden AS hidden,",
+                "    m.location_id AS location,",
+                "    c.blocked AS blocked",
+                " FROM msgs m LEFT JOIN chats c ON c.id=m.chat_id",
+                " WHERE m.id=?;"
+            ),
+            params![id],
+            |row| {
+                let mut msg = Message::default();
+                // msg.id = row.get::<_, AnyMsgId>("id")?;
+                msg.id = row.get("id")?;
+                msg.rfc724_mid = row.get::<_, String>("rfc724mid")?;
+                msg.in_reply_to = row.get::<_, Option<String>>("mime_in_reply_to")?;
+                msg.server_folder = row.get::<_, Option<String>>("server_folder")?;
+                msg.server_uid = row.get("server_uid")?;
+                msg.chat_id = row.get("chat_id")?;
+                msg.from_id = row.get("from_id")?;
+                msg.to_id = row.get("to_id")?;
+                msg.timestamp_sort = row.get("timestamp")?;
+                msg.timestamp_sent = row.get("timestamp_sent")?;
+                msg.timestamp_rcvd = row.get("timestamp_rcvd")?;
+                msg.type_0 = row.get("type")?;
+                msg.state = row.get("state")?;
+                msg.is_dc_message = row.get("msgrmsg")?;
 
-            let text;
-            if let rusqlite::types::ValueRef::Text(buf) = row.get_raw(15) {
-                if let Ok(t) = String::from_utf8(buf.to_vec()) {
-                    text = t;
+                let text;
+                if let rusqlite::types::ValueRef::Text(buf) = row.get_raw("txt") {
+                    if let Ok(t) = String::from_utf8(buf.to_vec()) {
+                        text = t;
+                    } else {
+                        warn!(
+                            context,
+                            concat!(
+                                "dc_msg_load_from_db: could not get ",
+                                "text column as non-lossy utf8 id {}"
+                            ),
+                            id
+                        );
+                        text = String::from_utf8_lossy(buf).into_owned();
+                    }
                 } else {
-                    warn!(context, "dc_msg_load_from_db: could not get text column as non-lossy utf8 id {}", id);
-                    text = String::from_utf8_lossy(buf).into_owned();
+                    text = "".to_string();
                 }
-            } else {
-                text = "".to_string();
-            }
-            msg.text = Some(text);
+                msg.text = Some(text);
 
-            msg.param = row.get::<_, String>(16)?.parse().unwrap_or_default();
-            msg.starred = row.get(17)?;
-            msg.hidden = row.get(18)?;
-            msg.location_id = row.get(19)?;
-            msg.chat_blocked = row.get::<_, Option<Blocked>>(20)?.unwrap_or_default();
-            Ok(msg)
-        })
+                msg.param = row.get::<_, String>("param")?.parse().unwrap_or_default();
+                msg.starred = row.get("starred")?;
+                msg.hidden = row.get("hidden")?;
+                msg.location_id = row.get("location")?;
+                msg.chat_blocked = row
+                    .get::<_, Option<Blocked>>("blocked")?
+                    .unwrap_or_default();
+
+                Ok(msg)
+            },
+        )
     }
 
-    pub fn delete_from_db(context: &Context, msg_id: u32) {
+    pub fn delete_from_db(context: &Context, msg_id: MsgId) {
         if let Ok(msg) = Message::load_from_db(context, msg_id) {
             sql::execute(
                 context,
                 &context.sql,
                 "DELETE FROM msgs WHERE id=?;",
-                params![msg.id as i32],
+                params![msg.id],
             )
             .ok();
             sql::execute(
                 context,
                 &context.sql,
                 "DELETE FROM msgs_mdns WHERE msg_id=?;",
-                params![msg.id as i32],
+                params![msg.id],
             )
             .ok();
         }
@@ -145,9 +303,7 @@ impl Message {
     }
 
     pub fn get_file(&self, context: &Context) -> Option<PathBuf> {
-        self.param
-            .get(Param::File)
-            .map(|f| dc_get_abs_path(context, f))
+        self.param.get_path(Param::File, context).unwrap_or(None)
     }
 
     /// Check if a message has a location bound to it.
@@ -190,7 +346,7 @@ impl Message {
         }
     }
 
-    pub fn get_id(&self) -> u32 {
+    pub fn get_id(&self) -> MsgId {
         self.id
     }
 
@@ -237,8 +393,9 @@ impl Message {
 
     pub fn get_filebytes(&self, context: &Context) -> u64 {
         self.param
-            .get(Param::File)
-            .map(|file| dc_get_filebytes(context, &file))
+            .get_path(Param::File, context)
+            .unwrap_or(None)
+            .map(|path| dc_get_filebytes(context, &path))
             .unwrap_or_default()
     }
 
@@ -255,7 +412,7 @@ impl Message {
     }
 
     pub fn get_showpadlock(&self) -> bool {
-        self.param.get_int(Param::GuranteeE2ee).unwrap_or_default() != 0
+        self.param.get_int(Param::GuaranteeE2ee).unwrap_or_default() != 0
     }
 
     pub fn get_summary(&mut self, context: &Context, chat: Option<&Chat>) -> Lot {
@@ -316,11 +473,19 @@ impl Message {
 
     pub fn is_info(&self) -> bool {
         let cmd = self.param.get_cmd();
-        self.from_id == DC_CONTACT_ID_DEVICE as libc::c_uint
-            || self.to_id == DC_CONTACT_ID_DEVICE as libc::c_uint
+        self.from_id == DC_CONTACT_ID_INFO as libc::c_uint
+            || self.to_id == DC_CONTACT_ID_INFO as libc::c_uint
             || cmd != SystemMessage::Unknown && cmd != SystemMessage::AutocryptSetupMessage
     }
 
+    /// Whether the message is still being created.
+    ///
+    /// Messages with attachments might be created before the
+    /// attachment is ready.  In this case some more restrictions on
+    /// the attachment apply, e.g. if the file to be attached is still
+    /// being written to or otherwise will still change it can not be
+    /// copied to the blobdir.  Thus those attachments need to be
+    /// created immediately in the blobdir with a valid filename.
     pub fn is_increation(&self) -> bool {
         chat::msgtype_has_file(self.type_0) && self.state == MessageState::OutPreparing
     }
@@ -339,23 +504,10 @@ impl Message {
         }
 
         if let Some(filename) = self.get_file(context) {
-            if let Ok(mut buf) = dc_read_file(context, filename) {
-                unsafe {
-                    // just a pointer inside buf, MUST NOT be free()'d
-                    let mut buf_headerline = String::default();
-                    // just a pointer inside buf, MUST NOT be free()'d
-                    let mut buf_setupcodebegin = ptr::null();
-
-                    if dc_split_armored_data(
-                        buf.as_mut_ptr().cast(),
-                        &mut buf_headerline,
-                        &mut buf_setupcodebegin,
-                        ptr::null_mut(),
-                        ptr::null_mut(),
-                    ) && buf_headerline == "-----BEGIN PGP MESSAGE-----"
-                        && !buf_setupcodebegin.is_null()
-                    {
-                        return Some(to_string(buf_setupcodebegin));
+            if let Ok(ref buf) = dc_read_file(context, filename) {
+                if let Ok((typ, headers, _)) = split_armored_data(buf) {
+                    if typ == pgp::armor::BlockType::Message {
+                        return headers.get(crate::pgp::HEADER_SETUPCODE).cloned();
                     }
                 }
             }
@@ -406,7 +558,7 @@ impl Message {
             context,
             &context.sql,
             "UPDATE msgs SET param=? WHERE id=?;",
-            params![self.param.to_string(), self.id as i32],
+            params![self.param.to_string(), self.id],
         )
         .is_ok()
     }
@@ -494,12 +646,10 @@ impl Lot {
                     } else {
                         self.text1 = None;
                     }
+                } else if let Some(contact) = contact {
+                    self.text1 = Some(contact.get_first_name().into());
                 } else {
-                    if let Some(contact) = contact {
-                        self.text1 = Some(contact.get_first_name().into());
-                    } else {
-                        self.text1 = None;
-                    }
+                    self.text1 = None;
                 }
                 self.text1_meaning = Meaning::Text1Username;
             }
@@ -518,7 +668,7 @@ impl Lot {
     }
 }
 
-pub fn get_msg_info(context: &Context, msg_id: u32) -> String {
+pub fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
     let mut ret = String::new();
 
     let msg = Message::load_from_db(context, msg_id);
@@ -526,19 +676,19 @@ pub fn get_msg_info(context: &Context, msg_id: u32) -> String {
         return ret;
     }
 
-    let msg = msg.unwrap();
+    let msg = msg.unwrap_or_default();
 
     let rawtxt: Option<String> = context.sql.query_get_value(
         context,
         "SELECT txt_raw FROM msgs WHERE id=?;",
-        params![msg_id as i32],
+        params![msg_id],
     );
 
     if rawtxt.is_none() {
-        ret += &format!("Cannot load message #{}.", msg_id as usize);
+        ret += &format!("Cannot load message {}.", msg_id);
         return ret;
     }
-    let rawtxt = rawtxt.unwrap();
+    let rawtxt = rawtxt.unwrap_or_default();
     let rawtxt = dc_truncate(rawtxt.trim(), 100000, false);
 
     let fts = dc_timestamp_to_str(msg.get_timestamp());
@@ -561,14 +711,14 @@ pub fn get_msg_info(context: &Context, msg_id: u32) -> String {
         ret += "\n";
     }
 
-    if msg.from_id == 2 || msg.to_id == 2 {
+    if msg.from_id == DC_CONTACT_ID_INFO || msg.to_id == DC_CONTACT_ID_INFO {
         // device-internal message, no further details needed
         return ret;
     }
 
     if let Ok(rows) = context.sql.query_map(
         "SELECT contact_id, timestamp_sent FROM msgs_mdns WHERE msg_id=?;",
-        params![msg_id as i32],
+        params![msg_id],
         |row| {
             let contact_id: i32 = row.get(0)?;
             let ts: i64 = row.get(1)?;
@@ -613,14 +763,13 @@ pub fn get_msg_info(context: &Context, msg_id: u32) -> String {
         if 0 != e2ee_errors & 0x2 {
             ret += ", Encrypted, no valid signature";
         }
-    } else if 0 != msg.param.get_int(Param::GuranteeE2ee).unwrap_or_default() {
+    } else if 0 != msg.param.get_int(Param::GuaranteeE2ee).unwrap_or_default() {
         ret += ", Encrypted";
     }
 
     ret += "\n";
-    match msg.param.get(Param::Error) {
-        Some(err) => ret += &format!("Error: {}", err),
-        _ => {}
+    if let Some(err) = msg.param.get(Param::Error) {
+        ret += &format!("Error: {}", err)
     }
 
     if let Some(path) = msg.get_file(context) {
@@ -679,21 +828,26 @@ pub fn guess_msgtype_from_suffix(path: &Path) -> Option<(Viewtype, &str)> {
     Some(info)
 }
 
-pub fn get_mime_headers(context: &Context, msg_id: u32) -> Option<String> {
+pub fn get_mime_headers(context: &Context, msg_id: MsgId) -> Option<String> {
     context.sql.query_get_value(
         context,
         "SELECT mime_headers FROM msgs WHERE id=?;",
-        params![msg_id as i32],
+        params![msg_id],
     )
 }
 
-pub fn delete_msgs(context: &Context, msg_ids: &[u32]) {
-    for msg_id in msg_ids.into_iter() {
+pub fn delete_msgs(context: &Context, msg_ids: &[MsgId]) {
+    for msg_id in msg_ids.iter() {
+        if let Ok(msg) = Message::load_from_db(context, *msg_id) {
+            if msg.location_id > 0 {
+                delete_poi_location(context, msg.location_id);
+            }
+        }
         update_msg_chat_id(context, *msg_id, DC_CHAT_ID_TRASH);
         job_add(
             context,
             Action::DeleteMsgOnImap,
-            *msg_id as libc::c_int,
+            msg_id.to_u32() as i32,
             Params::new(),
             0,
         );
@@ -702,35 +856,55 @@ pub fn delete_msgs(context: &Context, msg_ids: &[u32]) {
     if !msg_ids.is_empty() {
         context.call_cb(Event::MsgsChanged {
             chat_id: 0,
-            msg_id: 0,
+            msg_id: MsgId::new(0),
         });
         job_kill_action(context, Action::Housekeeping);
         job_add(context, Action::Housekeeping, 0, Params::new(), 10);
     };
 }
 
-fn update_msg_chat_id(context: &Context, msg_id: u32, chat_id: u32) -> bool {
+fn update_msg_chat_id(context: &Context, msg_id: MsgId, chat_id: u32) -> bool {
     sql::execute(
         context,
         &context.sql,
         "UPDATE msgs SET chat_id=? WHERE id=?;",
-        params![chat_id as i32, msg_id as i32],
+        params![chat_id as i32, msg_id],
     )
     .is_ok()
 }
 
-pub fn markseen_msgs(context: &Context, msg_ids: &[u32]) -> bool {
+fn delete_poi_location(context: &Context, location_id: u32) -> bool {
+    sql::execute(
+        context,
+        &context.sql,
+        "DELETE FROM locations WHERE independent = 1 AND id=?;",
+        params![location_id as i32],
+    )
+    .is_ok()
+}
+
+pub fn markseen_msgs(context: &Context, msg_ids: &[MsgId]) -> bool {
     if msg_ids.is_empty() {
         return false;
     }
 
     let msgs = context.sql.prepare(
-        "SELECT m.state, c.blocked  FROM msgs m  LEFT JOIN chats c ON c.id=m.chat_id  WHERE m.id=? AND m.chat_id>9",
+        concat!(
+            "SELECT",
+            "    m.state AS state,",
+            "    c.blocked AS blocked",
+            " FROM msgs m LEFT JOIN chats c ON c.id=m.chat_id",
+            " WHERE m.id=? AND m.chat_id>9"
+        ),
         |mut stmt, _| {
             let mut res = Vec::with_capacity(msg_ids.len());
-            for id in msg_ids.into_iter() {
-                let query_res = stmt.query_row(params![*id as i32], |row| {
-                    Ok((row.get::<_, MessageState>(0)?, row.get::<_, Option<Blocked>>(1)?.unwrap_or_default()))
+            for id in msg_ids.iter() {
+                let query_res = stmt.query_row(params![*id], |row| {
+                    Ok((
+                        row.get::<_, MessageState>("state")?,
+                        row.get::<_, Option<Blocked>>("blocked")?
+                            .unwrap_or_default(),
+                    ))
                 });
                 if let Err(rusqlite::Error::QueryReturnedNoRows) = query_res {
                     continue;
@@ -740,7 +914,7 @@ pub fn markseen_msgs(context: &Context, msg_ids: &[u32]) -> bool {
             }
 
             Ok(res)
-        }
+        },
     );
 
     if msgs.is_err() {
@@ -748,18 +922,18 @@ pub fn markseen_msgs(context: &Context, msg_ids: &[u32]) -> bool {
         return false;
     }
     let mut send_event = false;
-    let msgs = msgs.unwrap();
+    let msgs = msgs.unwrap_or_default();
 
     for (id, curr_state, curr_blocked) in msgs.into_iter() {
         if curr_blocked == Blocked::Not {
             if curr_state == MessageState::InFresh || curr_state == MessageState::InNoticed {
                 update_msg_state(context, *id, MessageState::InSeen);
-                info!(context, "Seen message #{}.", id);
+                info!(context, "Seen message {}.", id);
 
                 job_add(
                     context,
                     Action::MarkseenMsgOnImap,
-                    *id as i32,
+                    id.to_u32() as i32,
                     Params::new(),
                     0,
                 );
@@ -774,32 +948,32 @@ pub fn markseen_msgs(context: &Context, msg_ids: &[u32]) -> bool {
     if send_event {
         context.call_cb(Event::MsgsChanged {
             chat_id: 0,
-            msg_id: 0,
+            msg_id: MsgId::new(0),
         });
     }
 
     true
 }
 
-pub fn update_msg_state(context: &Context, msg_id: u32, state: MessageState) -> bool {
+pub fn update_msg_state(context: &Context, msg_id: MsgId, state: MessageState) -> bool {
     sql::execute(
         context,
         &context.sql,
         "UPDATE msgs SET state=? WHERE id=?;",
-        params![state, msg_id as i32],
+        params![state, msg_id],
     )
     .is_ok()
 }
 
-pub fn star_msgs(context: &Context, msg_ids: &[u32], star: bool) -> bool {
+pub fn star_msgs(context: &Context, msg_ids: &[MsgId], star: bool) -> bool {
     if msg_ids.is_empty() {
         return false;
     }
     context
         .sql
         .prepare("UPDATE msgs SET starred=? WHERE id=?;", |mut stmt, _| {
-            for msg_id in msg_ids.into_iter() {
-                stmt.execute(params![star as i32, *msg_id as i32])?;
+            for msg_id in msg_ids.iter() {
+                stmt.execute(params![star as i32, *msg_id])?;
             }
             Ok(())
         })
@@ -818,6 +992,7 @@ pub fn get_summarytext_by_raw(
     let prefix = match viewtype {
         Viewtype::Image => context.stock_str(StockMessage::Image).into_owned(),
         Viewtype::Gif => context.stock_str(StockMessage::Gif).into_owned(),
+        Viewtype::Sticker => context.stock_str(StockMessage::Sticker).into_owned(),
         Viewtype::Video => context.stock_str(StockMessage::Video).into_owned(),
         Viewtype::Voice => context.stock_str(StockMessage::VoiceMessage).into_owned(),
         Viewtype::Audio | Viewtype::File => {
@@ -827,17 +1002,14 @@ pub fn get_summarytext_by_raw(
                     .stock_str(StockMessage::AcSetupMsgSubject)
                     .to_string()
             } else {
-                let file_name: String = if let Some(file_path) = param.get(Param::File) {
-                    if let Some(file_name) = Path::new(file_path).file_name() {
-                        Some(file_name.to_string_lossy().into_owned())
-                    } else {
-                        None
-                    }
-                } else {
-                    None
-                }
-                .unwrap_or("ErrFileName".to_string());
-
+                let file_name: String = param
+                    .get_path(Param::File, context)
+                    .unwrap_or(None)
+                    .and_then(|path| {
+                        path.file_name()
+                            .map(|fname| fname.to_string_lossy().into_owned())
+                    })
+                    .unwrap_or_else(|| String::from("ErrFileName"));
                 let label = context.stock_str(if viewtype == Viewtype::Audio {
                     StockMessage::Audio
                 } else {
@@ -861,7 +1033,9 @@ pub fn get_summarytext_by_raw(
     }
 
     if let Some(text) = text {
-        if prefix.is_empty() {
+        if text.as_ref().is_empty() {
+            prefix
+        } else if prefix.is_empty() {
             dc_truncate(text.as_ref(), approx_characters, true).to_string()
         } else {
             let tmp = format!("{} – {}", prefix, text.as_ref());
@@ -879,8 +1053,8 @@ pub fn get_summarytext_by_raw(
 
 // Context functions to work with messages
 
-pub fn exists(context: &Context, msg_id: u32) -> bool {
-    if msg_id <= DC_CHAT_ID_LAST_SPECIAL {
+pub fn exists(context: &Context, msg_id: MsgId) -> bool {
+    if msg_id.is_special() {
         return false;
     }
 
@@ -897,19 +1071,7 @@ pub fn exists(context: &Context, msg_id: u32) -> bool {
     }
 }
 
-pub fn update_msg_move_state(context: &Context, rfc724_mid: &str, state: MoveState) -> bool {
-    // we update the move_state for all messages belonging to a given Message-ID
-    // so that the state stay intact when parts are deleted
-    sql::execute(
-        context,
-        &context.sql,
-        "UPDATE msgs SET move_state=? WHERE rfc724_mid=?;",
-        params![state as i32, rfc724_mid],
-    )
-    .is_ok()
-}
-
-pub fn set_msg_failed(context: &Context, msg_id: u32, error: Option<impl AsRef<str>>) {
+pub fn set_msg_failed(context: &Context, msg_id: MsgId, error: Option<impl AsRef<str>>) {
     if let Ok(mut msg) = Message::load_from_db(context, msg_id) {
         if msg.state.can_fail() {
             msg.state = MessageState::OutFailed;
@@ -923,7 +1085,7 @@ pub fn set_msg_failed(context: &Context, msg_id: u32, error: Option<impl AsRef<s
             context,
             &context.sql,
             "UPDATE msgs SET state=?, param=? WHERE id=?;",
-            params![msg.state, msg.param.to_string(), msg_id as i32],
+            params![msg.state, msg.param.to_string(), msg_id],
         )
         .is_ok()
         {
@@ -935,38 +1097,39 @@ pub fn set_msg_failed(context: &Context, msg_id: u32, error: Option<impl AsRef<s
     }
 }
 
-/// returns true if an event should be send
+/// returns Some if an event should be send
 pub fn mdn_from_ext(
     context: &Context,
     from_id: u32,
     rfc724_mid: &str,
     timestamp_sent: i64,
-    ret_chat_id: &mut u32,
-    ret_msg_id: &mut u32,
-) -> bool {
-    if from_id <= 9 || rfc724_mid.is_empty() || *ret_chat_id != 0 || *ret_msg_id != 0 {
-        return false;
+) -> Option<(u32, MsgId)> {
+    if from_id <= DC_MSG_ID_LAST_SPECIAL || rfc724_mid.is_empty() {
+        return None;
     }
 
-    let mut read_by_all = false;
-
     if let Ok((msg_id, chat_id, chat_type, msg_state)) = context.sql.query_row(
-        "SELECT m.id, c.id, c.type, m.state FROM msgs m  \
-         LEFT JOIN chats c ON m.chat_id=c.id  \
-         WHERE rfc724_mid=? AND from_id=1  \
-         ORDER BY m.id;",
+        concat!(
+            "SELECT",
+            "    m.id AS msg_id,",
+            "    c.id AS chat_id,",
+            "    c.type AS type,",
+            "    m.state AS state",
+            " FROM msgs m LEFT JOIN chats c ON m.chat_id=c.id",
+            " WHERE rfc724_mid=? AND from_id=1",
+            " ORDER BY m.id;"
+        ),
         params![rfc724_mid],
         |row| {
             Ok((
-                row.get::<_, i32>(0)?,
-                row.get::<_, i32>(1)?,
-                row.get::<_, Chattype>(2)?,
-                row.get::<_, MessageState>(3)?,
+                row.get::<_, MsgId>("msg_id")?,
+                row.get::<_, u32>("chat_id")?,
+                row.get::<_, Chattype>("type")?,
+                row.get::<_, MessageState>("state")?,
             ))
         },
     ) {
-        *ret_msg_id = msg_id as u32;
-        *ret_chat_id = chat_id as u32;
+        let mut read_by_all = false;
 
         // if already marked as MDNS_RCVD msgstate_can_fail() returns false.
         // however, it is important, that ret_msg_id is set above as this
@@ -976,20 +1139,20 @@ pub fn mdn_from_ext(
                 .sql
                 .exists(
                     "SELECT contact_id FROM msgs_mdns WHERE msg_id=? AND contact_id=?;",
-                    params![*ret_msg_id as i32, from_id as i32,],
+                    params![msg_id, from_id as i32,],
                 )
                 .unwrap_or_default();
 
             if !mdn_already_in_table {
                 context.sql.execute(
                     "INSERT INTO msgs_mdns (msg_id, contact_id, timestamp_sent) VALUES (?, ?, ?);",
-                    params![*ret_msg_id as i32, from_id as i32, timestamp_sent],
-                ).unwrap(); // TODO: better error handling
+                    params![msg_id, from_id as i32, timestamp_sent],
+                ).unwrap_or_default(); // TODO: better error handling
             }
 
             // Normal chat? that's quite easy.
             if chat_type == Chattype::Single {
-                update_msg_state(context, *ret_msg_id, MessageState::OutMdnRcvd);
+                update_msg_state(context, msg_id, MessageState::OutMdnRcvd);
                 read_by_all = true;
             } else {
                 // send event about new state
@@ -998,7 +1161,7 @@ pub fn mdn_from_ext(
                     .query_get_value::<_, isize>(
                         context,
                         "SELECT COUNT(*) FROM msgs_mdns WHERE msg_id=?;",
-                        params![*ret_msg_id as i32],
+                        params![msg_id],
                     )
                     .unwrap_or_default() as usize;
                 /*
@@ -1014,16 +1177,19 @@ pub fn mdn_from_ext(
                 (S=Sender, R=Recipient)
                  */
                 // for rounding, SELF is already included!
-                let soll_cnt = (chat::get_chat_contact_cnt(context, *ret_chat_id) + 1) / 2;
+                let soll_cnt = (chat::get_chat_contact_cnt(context, chat_id) + 1) / 2;
                 if ist_cnt >= soll_cnt {
-                    update_msg_state(context, *ret_msg_id, MessageState::OutMdnRcvd);
+                    update_msg_state(context, msg_id, MessageState::OutMdnRcvd);
                     read_by_all = true;
                 } // else wait for more receipts
             }
         }
+        return match read_by_all {
+            true => Some((chat_id, msg_id)),
+            false => None,
+        };
     }
-
-    read_by_all
+    None
 }
 
 /// The number of messages assigned to real chat (!=deaddrop, !=trash)
@@ -1077,7 +1243,7 @@ pub fn rfc724_mid_cnt(context: &Context, rfc724_mid: &str) -> libc::c_int {
 pub(crate) fn rfc724_mid_exists(
     context: &Context,
     rfc724_mid: &str,
-) -> Result<(String, u32, u32), Error> {
+) -> Result<(String, u32, MsgId), Error> {
     ensure!(!rfc724_mid.is_empty(), "empty rfc724_mid");
 
     context.sql.query_row(
@@ -1086,7 +1252,7 @@ pub(crate) fn rfc724_mid_exists(
         |row| {
             let server_folder = row.get::<_, Option<String>>(0)?.unwrap_or_default();
             let server_uid = row.get(1)?;
-            let msg_id = row.get(2)?;
+            let msg_id: MsgId = row.get(2)?;
 
             Ok((server_folder, server_uid, msg_id))
         },
@@ -1110,6 +1276,12 @@ pub fn update_server_uid(
     }
 }
 
+#[allow(dead_code)]
+pub fn dc_empty_server(context: &Context, flags: u32) {
+    job_kill_action(context, Action::EmptyServer);
+    job_add(context, Action::EmptyServer, flags as i32, Params::new(), 0);
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -1145,4 +1317,121 @@ mod tests {
         let _msg2 = Message::load_from_db(ctx, msg_id).unwrap();
         assert_eq!(_msg2.get_filemime(), None);
     }
+
+    #[test]
+    pub fn test_get_summarytext_by_raw() {
+        let d = test::dummy_context();
+        let ctx = &d.ctx;
+
+        let some_text = Some("bla bla".to_string());
+        let empty_text = Some("".to_string());
+        let no_text: Option<String> = None;
+
+        let mut some_file = Params::new();
+        some_file.set(Param::File, "foo.bar");
+
+        assert_eq!(
+            get_summarytext_by_raw(
+                Viewtype::Text,
+                some_text.as_ref(),
+                &mut Params::new(),
+                50,
+                &ctx
+            ),
+            "bla bla" // for simple text, the type is not added to the summary
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(Viewtype::Image, no_text.as_ref(), &mut some_file, 50, &ctx,),
+            "Image" // file names are not added for images
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(Viewtype::Video, no_text.as_ref(), &mut some_file, 50, &ctx,),
+            "Video" // file names are not added for videos
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(Viewtype::Gif, no_text.as_ref(), &mut some_file, 50, &ctx,),
+            "GIF" // file names are not added for GIFs
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(
+                Viewtype::Sticker,
+                no_text.as_ref(),
+                &mut some_file,
+                50,
+                &ctx,
+            ),
+            "Sticker" // file names are not added for stickers
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(
+                Viewtype::Voice,
+                empty_text.as_ref(),
+                &mut some_file,
+                50,
+                &ctx,
+            ),
+            "Voice message" // file names are not added for voice messages, empty text is skipped
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(Viewtype::Voice, no_text.as_ref(), &mut some_file, 50, &ctx),
+            "Voice message" // file names are not added for voice messages
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(
+                Viewtype::Voice,
+                some_text.as_ref(),
+                &mut some_file,
+                50,
+                &ctx
+            ),
+            "Voice message \u{2013} bla bla" // `\u{2013}` explicitly checks for "EN DASH"
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(Viewtype::Audio, no_text.as_ref(), &mut some_file, 50, &ctx),
+            "Audio \u{2013} foo.bar" // file name is added for audio
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(
+                Viewtype::Audio,
+                empty_text.as_ref(),
+                &mut some_file,
+                50,
+                &ctx,
+            ),
+            "Audio \u{2013} foo.bar" // file name is added for audio, empty text is not added
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(
+                Viewtype::Audio,
+                some_text.as_ref(),
+                &mut some_file,
+                50,
+                &ctx
+            ),
+            "Audio \u{2013} foo.bar \u{2013} bla bla" // file name and text added for audio
+        );
+
+        assert_eq!(
+            get_summarytext_by_raw(Viewtype::File, some_text.as_ref(), &mut some_file, 50, &ctx),
+            "File \u{2013} foo.bar \u{2013} bla bla" // file name is added for files
+        );
+
+        let mut asm_file = Params::new();
+        asm_file.set(Param::File, "foo.bar");
+        asm_file.set_cmd(SystemMessage::AutocryptSetupMessage);
+        assert_eq!(
+            get_summarytext_by_raw(Viewtype::File, no_text.as_ref(), &mut asm_file, 50, &ctx),
+            "Autocrypt Setup Message" // file name is not added for autocrypt setup messages
+        );
+    }
 }

src/mimefactory.rs

@@ -1,4 +1,3 @@
-use std::path::Path;
 use std::ptr;
 
 use chrono::TimeZone;
@@ -13,15 +12,17 @@ use mmime::mmapstring::*;
 use mmime::other::*;
 
 use crate::chat::{self, Chat};
+use crate::config::Config;
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::{get_version_str, Context};
-use crate::dc_mimeparser::{mailmime_find_mailimf_fields, SystemMessage};
+use crate::dc_mimeparser::SystemMessage;
 use crate::dc_strencode::*;
 use crate::dc_tools::*;
 use crate::e2ee::*;
 use crate::error::Error;
 use crate::location;
+use crate::message::MsgId;
 use crate::message::{self, Message};
 use crate::param::*;
 use crate::stock::StockMessage;
@@ -59,11 +60,13 @@ pub struct MimeFactory<'a> {
 
 impl<'a> MimeFactory<'a> {
     fn new(context: &'a Context, msg: Message) -> Self {
-        let cget = |context: &Context, name: &str| context.sql.get_config(context, name);
         MimeFactory {
-            from_addr: cget(&context, "configured_addr").unwrap_or_default(),
-            from_displayname: cget(&context, "displayname").unwrap_or_default(),
-            selfstatus: cget(&context, "selfstatus")
+            from_addr: context
+                .get_config(Config::ConfiguredAddr)
+                .unwrap_or_default(),
+            from_displayname: context.get_config(Config::Displayname).unwrap_or_default(),
+            selfstatus: context
+                .get_config(Config::Selfstatus)
                 .unwrap_or_else(|| context.stock_str(StockMessage::StatusLine).to_string()),
             recipients_names: Vec::with_capacity(5),
             recipients_addr: Vec::with_capacity(5),
@@ -105,16 +108,11 @@ impl<'a> MimeFactory<'a> {
         Ok(())
     }
 
-    pub fn load_mdn(context: &'a Context, msg_id: u32) -> Result<MimeFactory, Error> {
-        if 0 == context
-            .sql
-            .get_config_int(context, "mdns_enabled")
-            .unwrap_or_else(|| 1)
-        {
-            // MDNs not enabled - check this is late, in the job. the use may have changed its
-            // choice while offline ...
-
-            bail!("MDNs disabled ")
+    pub fn load_mdn(context: &'a Context, msg_id: MsgId) -> Result<MimeFactory, Error> {
+        if !context.get_config_bool(Config::MdnsEnabled) {
+            // MDNs not enabled - check this is late, in the job. the
+            // user may have changed its choice while offline ...
+            bail!("MDNs meanwhile disabled")
         }
 
         let msg = Message::load_from_db(context, msg_id)?;
@@ -141,52 +139,35 @@ impl<'a> MimeFactory<'a> {
     }
 
     /*******************************************************************************
-     * Render
+     * Render a basic email
      ******************************************************************************/
-    // restrict unsafe to parts, introduce wrapmime helpers where appropriate
+    // XXX restrict unsafe to parts, introduce wrapmime helpers where appropriate
     pub unsafe fn render(&mut self) -> Result<(), Error> {
         if self.loaded == Loaded::Nothing || !self.out.is_null() {
             bail!("Invalid use of mimefactory-object.");
         }
         let context = &self.context;
+        let from = wrapmime::new_mailbox_list(&self.from_displayname, &self.from_addr);
 
-        /* create basic mail
-         *************************************************************************/
-
-        let from: *mut mailimf_mailbox_list = mailimf_mailbox_list_new_empty();
-        mailimf_mailbox_list_add(
-            from,
-            mailimf_mailbox_new(
-                if !self.from_displayname.is_empty() {
-                    dc_encode_header_words(&self.from_displayname).strdup()
-                } else {
-                    ptr::null_mut()
-                },
-                self.from_addr.strdup(),
-            ),
-        );
-        let mut to: *mut mailimf_address_list = ptr::null_mut();
-        if !self.recipients_names.is_empty() && !self.recipients_addr.is_empty() {
-            to = mailimf_address_list_new_empty();
-            let name_iter = self.recipients_names.iter();
-            let addr_iter = self.recipients_addr.iter();
-            for (name, addr) in name_iter.zip(addr_iter) {
-                mailimf_address_list_add(
-                    to,
-                    mailimf_address_new(
-                        MAILIMF_ADDRESS_MAILBOX as libc::c_int,
-                        mailimf_mailbox_new(
-                            if !name.is_empty() {
-                                dc_encode_header_words(&name).strdup()
-                            } else {
-                                ptr::null_mut()
-                            },
-                            addr.strdup(),
-                        ),
-                        ptr::null_mut(),
+        let to = mailimf_address_list_new_empty();
+        let name_iter = self.recipients_names.iter();
+        let addr_iter = self.recipients_addr.iter();
+        for (name, addr) in name_iter.zip(addr_iter) {
+            mailimf_address_list_add(
+                to,
+                mailimf_address_new(
+                    MAILIMF_ADDRESS_MAILBOX as libc::c_int,
+                    mailimf_mailbox_new(
+                        if !name.is_empty() {
+                            dc_encode_header_words(&name).strdup()
+                        } else {
+                            ptr::null_mut()
+                        },
+                        addr.strdup(),
                     ),
-                );
-            }
+                    ptr::null_mut(),
+                ),
+            );
         }
         let references_list = if !self.references.is_empty() {
             dc_str_to_clist(&self.references, " ")
@@ -198,6 +179,7 @@ impl<'a> MimeFactory<'a> {
         } else {
             ptr::null_mut()
         };
+
         let imf_fields = mailimf_fields_new_with_data_all(
             mailimf_get_date(self.timestamp as i64),
             from,
@@ -248,7 +230,7 @@ impl<'a> MimeFactory<'a> {
 
         // 1=add Autocrypt-header (needed eg. for handshaking), 2=no Autocrypte-header (used for MDN)
         let mut e2ee_guaranteed = false;
-        let mut min_verified: libc::c_int = 0;
+        let mut min_verified = crate::peerstate::PeerstateVerifiedStatus::Unverified;
         let mut do_gossip = false;
         let mut grpimage = None;
         let force_plaintext: libc::c_int;
@@ -264,7 +246,7 @@ impl<'a> MimeFactory<'a> {
                     wrapmime::new_custom_field(imf_fields, "Chat-Verified", "1");
                     force_plaintext = 0;
                     e2ee_guaranteed = true;
-                    min_verified = 2
+                    min_verified = crate::peerstate::PeerstateVerifiedStatus::BidirectVerified;
                 } else {
                     force_plaintext = self
                         .msg
@@ -275,7 +257,7 @@ impl<'a> MimeFactory<'a> {
                         e2ee_guaranteed = self
                             .msg
                             .param
-                            .get_int(Param::GuranteeE2ee)
+                            .get_int(Param::GuaranteeE2ee)
                             .unwrap_or_default()
                             != 0;
                     }
@@ -402,15 +384,8 @@ impl<'a> MimeFactory<'a> {
                                     &fingerprint,
                                 );
                             }
-                            match msg.param.get(Param::Arg4) {
-                                Some(id) => {
-                                    wrapmime::new_custom_field(
-                                        imf_fields,
-                                        "Secure-Join-Group",
-                                        &id,
-                                    );
-                                }
-                                None => {}
+                            if let Some(id) = msg.param.get(Param::Arg4) {
+                                wrapmime::new_custom_field(imf_fields, "Secure-Join-Group", &id);
                             };
                         }
                     }
@@ -435,6 +410,10 @@ impl<'a> MimeFactory<'a> {
                     }
                 }
 
+                if self.msg.type_0 == Viewtype::Sticker {
+                    wrapmime::new_custom_field(imf_fields, "Chat-Content", "sticker");
+                }
+
                 if self.msg.type_0 == Viewtype::Voice
                     || self.msg.type_0 == Viewtype::Audio
                     || self.msg.type_0 == Viewtype::Video
@@ -505,6 +484,7 @@ impl<'a> MimeFactory<'a> {
                 if !meta_part.is_null() {
                     mailmime_smart_add_part(message, meta_part);
                 }
+
                 if self.msg.param.exists(Param::SetLatitude) {
                     let param = &self.msg.param;
                     let kml_file = location::get_message_kml(
@@ -521,18 +501,21 @@ impl<'a> MimeFactory<'a> {
                 }
 
                 if location::is_sending_locations_to_chat(context, self.msg.chat_id) {
-                    if let Ok((kml_file, last_added_location_id)) =
-                        location::get_kml(context, self.msg.chat_id)
-                    {
-                        wrapmime::add_filename_part(
-                            message,
-                            "location.kml",
-                            "application/vnd.google-earth.kml+xml",
-                            &kml_file,
-                        )?;
-                        if !self.msg.param.exists(Param::SetLatitude) {
-                            // otherwise, the independent location is already filed
-                            self.out_last_added_location_id = last_added_location_id;
+                    match location::get_kml(context, self.msg.chat_id) {
+                        Ok((kml_content, last_added_location_id)) => {
+                            wrapmime::add_filename_part(
+                                message,
+                                "location.kml",
+                                "application/vnd.google-earth.kml+xml",
+                                &kml_content,
+                            )?;
+                            if !self.msg.param.exists(Param::SetLatitude) {
+                                // otherwise, the independent location is already filed
+                                self.out_last_added_location_id = last_added_location_id;
+                            }
+                        }
+                        Err(err) => {
+                            warn!(context, "mimefactory: could not get location: {}", err);
                         }
                     }
                 }
@@ -559,7 +542,7 @@ impl<'a> MimeFactory<'a> {
                     != self
                         .msg
                         .param
-                        .get_int(Param::GuranteeE2ee)
+                        .get_int(Param::GuaranteeE2ee)
                         .unwrap_or_default()
                 {
                     self.context
@@ -594,7 +577,6 @@ impl<'a> MimeFactory<'a> {
                 wrapmime::set_body_text(mach_mime_part, &message_text2)?;
                 mailmime_add_part(multipart, mach_mime_part);
                 force_plaintext = DC_FP_NO_AUTOCRYPT_HEADER;
-                info!(context, "sending MDM {:?}", message_text2);
                 /* currently, we do not send MDNs encrypted:
                 - in a multi-device-setup that is not set up properly, MDNs would disturb the communication as they
                   are send automatically which may lead to spreading outdated Autocrypt headers.
@@ -645,7 +627,7 @@ impl<'a> MimeFactory<'a> {
         );
 
         /*just a pointer into mailmime structure, must not be freed*/
-        let imffields_unprotected = mailmime_find_mailimf_fields(message);
+        let imffields_unprotected = wrapmime::mailmime_find_mailimf_fields(message);
         ensure!(
             !imffields_unprotected.is_null(),
             "could not find mime fields"
@@ -657,17 +639,18 @@ impl<'a> MimeFactory<'a> {
             let aheader = encrypt_helper.get_aheader().to_string();
             wrapmime::new_custom_field(imffields_unprotected, "Autocrypt", &aheader);
         }
-        let mut finalized = false;
-        if force_plaintext == 0 {
-            finalized = encrypt_helper.try_encrypt(
+        let finalized = if force_plaintext == 0 {
+            encrypt_helper.try_encrypt(
                 self,
                 e2ee_guaranteed,
                 min_verified,
                 do_gossip,
                 message,
                 imffields_unprotected,
-            )?;
-        }
+            )?
+        } else {
+            false
+        };
         if !finalized {
             self.finalize_mime_message(message, false, false)?;
         }
@@ -675,9 +658,7 @@ impl<'a> MimeFactory<'a> {
         Ok(())
     }
 
-    pub fn load_msg(context: &Context, msg_id: u32) -> Result<MimeFactory, Error> {
-        ensure!(msg_id > DC_CHAT_ID_LAST_SPECIAL, "Invalid chat id");
-
+    pub fn load_msg(context: &Context, msg_id: MsgId) -> Result<MimeFactory, Error> {
         let msg = Message::load_from_db(context, msg_id)?;
         let chat = Chat::load_from_db(context, msg.chat_id)?;
         let mut factory = MimeFactory::new(context, msg);
@@ -692,31 +673,28 @@ impl<'a> MimeFactory<'a> {
                 .push(factory.from_displayname.to_string());
             factory.recipients_addr.push(factory.from_addr.to_string());
         } else {
-            context
-                .sql
-                .query_map(
-                    "SELECT c.authname, c.addr  \
-                     FROM chats_contacts cc  \
-                     LEFT JOIN contacts c ON cc.contact_id=c.id  \
-                     WHERE cc.chat_id=? AND cc.contact_id>9;",
-                    params![factory.msg.chat_id as i32],
-                    |row| {
-                        let authname: String = row.get(0)?;
-                        let addr: String = row.get(1)?;
-                        Ok((authname, addr))
-                    },
-                    |rows| {
-                        for row in rows {
-                            let (authname, addr) = row?;
-                            if !vec_contains_lowercase(&factory.recipients_addr, &addr) {
-                                factory.recipients_addr.push(addr);
-                                factory.recipients_names.push(authname);
-                            }
+            context.sql.query_map(
+                "SELECT c.authname, c.addr  \
+                 FROM chats_contacts cc  \
+                 LEFT JOIN contacts c ON cc.contact_id=c.id  \
+                 WHERE cc.chat_id=? AND cc.contact_id>9;",
+                params![factory.msg.chat_id as i32],
+                |row| {
+                    let authname: String = row.get(0)?;
+                    let addr: String = row.get(1)?;
+                    Ok((authname, addr))
+                },
+                |rows| {
+                    for row in rows {
+                        let (authname, addr) = row?;
+                        if !vec_contains_lowercase(&factory.recipients_addr, &addr) {
+                            factory.recipients_addr.push(addr);
+                            factory.recipients_names.push(authname);
                         }
-                        Ok(())
-                    },
-                )
-                .unwrap();
+                    }
+                    Ok(())
+                },
+            )?;
 
             let command = factory.msg.param.get_cmd();
             let msg = &factory.msg;
@@ -726,11 +704,10 @@ impl<'a> MimeFactory<'a> {
                 let email_to_remove = msg.param.get(Param::Arg).unwrap_or_default();
 
                 let self_addr = context
-                    .sql
-                    .get_config(context, "configured_addr")
+                    .get_config(Config::ConfiguredAddr)
                     .unwrap_or_default();
 
-                if !email_to_remove.is_empty() && email_to_remove != self_addr {
+                if !email_to_remove.is_empty() && !addr_cmp(email_to_remove, self_addr) {
                     if !vec_contains_lowercase(&factory.recipients_addr, &email_to_remove) {
                         factory.recipients_names.push("".to_string());
                         factory.recipients_addr.push(email_to_remove.to_string());
@@ -739,17 +716,14 @@ impl<'a> MimeFactory<'a> {
             }
             if command != SystemMessage::AutocryptSetupMessage
                 && command != SystemMessage::SecurejoinMessage
-                && 0 != context
-                    .sql
-                    .get_config_int(context, "mdns_enabled")
-                    .unwrap_or_else(|| 1)
+                && context.get_config_bool(Config::MdnsEnabled)
             {
                 factory.req_mdn = true;
             }
         }
         let row = context.sql.query_row(
             "SELECT mime_in_reply_to, mime_references FROM msgs WHERE id=?",
-            params![factory.msg.id as i32],
+            params![factory.msg.id],
             |row| {
                 let in_reply_to: String = row.get(0)?;
                 let references: String = row.get(1)?;
@@ -822,26 +796,21 @@ fn build_body_file(
     msg: &Message,
     base_name: &str,
 ) -> Result<(*mut Mailmime, String), Error> {
-    let path_filename = match msg.param.get(Param::File) {
-        None => {
-            bail!("msg has no filename");
-        }
-        Some(path) => path,
-    };
-    let suffix = dc_get_filesuffix_lc(path_filename).unwrap_or_else(|| "dat".into());
+    let blob = msg
+        .param
+        .get_blob(Param::File, context, true)?
+        .ok_or_else(|| format_err!("msg has no filename"))?;
+    let suffix = blob.suffix().unwrap_or("dat");
 
-    /* get file name to use for sending
-    (for privacy purposes, we do not transfer the original filenames eg. for images;
-    these names are normally not needed and contain timestamps, running numbers etc.) */
-    let filename_to_send = match msg.type_0 {
+    // Get file name to use for sending.  For privacy purposes, we do
+    // not transfer the original filenames eg. for images; these names
+    // are normally not needed and contain timestamps, running numbers
+    // etc.
+    let filename_to_send: String = match msg.type_0 {
         Viewtype::Voice => chrono::Utc
             .timestamp(msg.timestamp_sort as i64, 0)
-            .format(&format!("voice-message_%Y-%m-%d_%H-%M-%S.{}", suffix))
+            .format(&format!("voice-message_%Y-%m-%d_%H-%M-%S.{}", &suffix))
             .to_string(),
-        Viewtype::Audio => Path::new(path_filename)
-            .file_name()
-            .map(|c| c.to_string_lossy().to_string())
-            .unwrap_or_default(),
         Viewtype::Image | Viewtype::Gif => format!(
             "{}.{}",
             if base_name.is_empty() {
@@ -852,18 +821,14 @@ fn build_body_file(
             &suffix,
         ),
         Viewtype::Video => format!("video.{}", &suffix),
-        _ => Path::new(path_filename)
-            .file_name()
-            .map(|c| c.to_string_lossy().to_string())
-            .unwrap_or_default(),
+        _ => blob.as_file_name().to_string(),
     };
 
     /* check mimetype */
     let mimetype = match msg.param.get(Param::MimeType) {
         Some(mtype) => mtype,
         None => {
-            let path = Path::new(path_filename);
-            if let Some(res) = message::guess_msgtype_from_suffix(&path) {
+            if let Some(res) = message::guess_msgtype_from_suffix(blob.as_rel_path()) {
                 res.1
             } else {
                 "application/octet-stream"
@@ -923,15 +888,13 @@ fn build_body_file(
         wrapmime::append_ct_param(content, "name", &filename_encoded)?;
 
         let mime_sub = mailmime_new_empty(content, mime_fields);
-        let abs_path = dc_get_abs_path(context, path_filename)
-            .to_c_string()
-            .unwrap();
+        let abs_path = blob.to_abs_path().to_c_string()?;
         mailmime_set_body_file(mime_sub, dc_strdup(abs_path.as_ptr()));
         Ok((mime_sub, filename_to_send))
     }
 }
 
-pub(crate) fn vec_contains_lowercase(vec: &Vec<String>, part: &str) -> bool {
+pub(crate) fn vec_contains_lowercase(vec: &[String], part: &str) -> bool {
     let partlc = part.to_lowercase();
     for cur in vec.iter() {
         if cur.to_lowercase() == partlc {
@@ -942,13 +905,11 @@ pub(crate) fn vec_contains_lowercase(vec: &Vec<String>, part: &str) -> bool {
 }
 
 fn is_file_size_okay(context: &Context, msg: &Message) -> bool {
-    let mut file_size_okay = true;
-    let path = msg.param.get(Param::File).unwrap_or_default();
-    let bytes = dc_get_filebytes(context, &path);
-
-    if bytes > (49 * 1024 * 1024 / 4 * 3) {
-        file_size_okay = false;
+    match msg.param.get_path(Param::File, context).unwrap_or(None) {
+        Some(path) => {
+            let bytes = dc_get_filebytes(context, &path);
+            bytes <= (49 * 1024 * 1024 / 4 * 3)
+        }
+        None => false,
     }
-
-    file_size_okay
 }

src/oauth2.rs

@@ -1,3 +1,5 @@
+//! OAuth 2 module
+
 use std::collections::HashMap;
 
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
@@ -7,6 +9,7 @@ use crate::context::Context;
 use crate::dc_tools::*;
 
 const OAUTH2_GMAIL: Oauth2 = Oauth2 {
+    // see https://developers.google.com/identity/protocols/OAuth2InstalledApp
     client_id: "959970109878-4mvtgf6feshskf7695nfln6002mom908.apps.googleusercontent.com",
     get_code: "https://accounts.google.com/o/oauth2/auth?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&response_type=code&scope=https%3A%2F%2Fmail.google.com%2F%20email&access_type=offline",
     init_token: "https://accounts.google.com/o/oauth2/token?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&code=$CODE&grant_type=authorization_code",
@@ -15,6 +18,7 @@ const OAUTH2_GMAIL: Oauth2 = Oauth2 {
 };
 
 const OAUTH2_YANDEX: Oauth2 = Oauth2 {
+    // see https://tech.yandex.com/oauth/doc/dg/reference/auto-code-client-docpage/
     client_id: "c4d0b6735fc8420a816d7e1303469341",
     get_code: "https://oauth.yandex.com/authorize?client_id=$CLIENT_ID&response_type=code&scope=mail%3Aimap_full%20mail%3Asmtp&force_confirm=true",
     init_token: "https://oauth.yandex.com/token?grant_type=authorization_code&code=$CODE&client_id=$CLIENT_ID&client_secret=58b8c6e94cf44fbe952da8511955dacf",
@@ -31,12 +35,14 @@ struct Oauth2 {
     get_userinfo: Option<&'static str>,
 }
 
+/// OAuth 2 Access Token Response
 #[derive(Debug, Deserialize)]
 struct Response {
     // Should always be there according to: https://www.oauth.com/oauth2-servers/access-tokens/access-token-response/
     // but previous code handled its abscense.
     access_token: Option<String>,
     token_type: String,
+    /// Duration of time the token is granted for, in seconds
     expires_in: Option<u64>,
     refresh_token: Option<String>,
     scope: Option<String>,
@@ -50,7 +56,7 @@ pub fn dc_get_oauth2_url(
     if let Some(oauth2) = Oauth2::from_address(addr) {
         if context
             .sql
-            .set_config(
+            .set_raw_config(
                 context,
                 "oauth2_pending_redirect_uri",
                 Some(redirect_uri.as_ref()),
@@ -82,17 +88,18 @@ pub fn dc_get_oauth2_access_token(
 
         // read generated token
         if !regenerate && !is_expired(context) {
-            let access_token = context.sql.get_config(context, "oauth2_access_token");
+            let access_token = context.sql.get_raw_config(context, "oauth2_access_token");
             if access_token.is_some() {
                 // success
                 return access_token;
             }
         }
 
-        let refresh_token = context.sql.get_config(context, "oauth2_refresh_token");
+        // generate new token: build & call auth url
+        let refresh_token = context.sql.get_raw_config(context, "oauth2_refresh_token");
         let refresh_token_for = context
             .sql
-            .get_config(context, "oauth2_refresh_token_for")
+            .get_raw_config(context, "oauth2_refresh_token_for")
             .unwrap_or_else(|| "unset".into());
 
         let (redirect_uri, token_url, update_redirect_uri_on_success) =
@@ -101,7 +108,7 @@ pub fn dc_get_oauth2_access_token(
                 (
                     context
                         .sql
-                        .get_config(context, "oauth2_pending_redirect_uri")
+                        .get_raw_config(context, "oauth2_pending_redirect_uri")
                         .unwrap_or_else(|| "unset".into()),
                     oauth2.init_token,
                     true,
@@ -114,20 +121,43 @@ pub fn dc_get_oauth2_access_token(
                 (
                     context
                         .sql
-                        .get_config(context, "oauth2_redirect_uri")
+                        .get_raw_config(context, "oauth2_redirect_uri")
                         .unwrap_or_else(|| "unset".into()),
                     oauth2.refresh_token,
                     false,
                 )
             };
-        let mut token_url = replace_in_uri(&token_url, "$CLIENT_ID", oauth2.client_id);
-        token_url = replace_in_uri(&token_url, "$REDIRECT_URI", &redirect_uri);
-        token_url = replace_in_uri(&token_url, "$CODE", code.as_ref());
-        if let Some(ref token) = refresh_token {
-            token_url = replace_in_uri(&token_url, "$REFRESH_TOKEN", token);
+
+        // to allow easier specification of different configurations,
+        // token_url is in GET-method-format, sth. as https://domain?param1=val1&param2=val2 -
+        // convert this to POST-format ...
+        let mut parts = token_url.splitn(2, '?');
+        let post_url = parts.next().unwrap_or_default();
+        let post_args = parts.next().unwrap_or_default();
+        let mut post_param = HashMap::new();
+        for key_value_pair in post_args.split('&') {
+            let mut parts = key_value_pair.splitn(2, '=');
+            let key = parts.next().unwrap_or_default();
+            let mut value = parts.next().unwrap_or_default();
+
+            if value == "$CLIENT_ID" {
+                value = oauth2.client_id;
+            } else if value == "$REDIRECT_URI" {
+                value = &redirect_uri;
+            } else if value == "$CODE" {
+                value = code.as_ref();
+            } else if value == "$REFRESH_TOKEN" && refresh_token.is_some() {
+                value = refresh_token.as_ref().unwrap();
+            }
+
+            post_param.insert(key, value);
         }
 
-        let response = reqwest::Client::new().post(&token_url).send();
+        // ... and POST
+        let response = reqwest::Client::new()
+            .post(post_url)
+            .form(&post_param)
+            .send();
         if response.is_err() {
             warn!(
                 context,
@@ -139,13 +169,14 @@ pub fn dc_get_oauth2_access_token(
         if !response.status().is_success() {
             warn!(
                 context,
-                "Error calling OAuth2 at {}: {:?}",
+                "Unsuccessful response when calling OAuth2 at {}: {:?}",
                 token_url,
                 response.status()
             );
             return None;
         }
 
+        // generate new token: parse returned json
         let parsed: reqwest::Result<Response> = response.json();
         if parsed.is_err() {
             warn!(
@@ -155,15 +186,17 @@ pub fn dc_get_oauth2_access_token(
             return None;
         }
         println!("response: {:?}", &parsed);
+
+        // update refresh_token if given, typically on the first round, but we update it later as well.
         let response = parsed.unwrap();
         if let Some(ref token) = response.refresh_token {
             context
                 .sql
-                .set_config(context, "oauth2_refresh_token", Some(token))
+                .set_raw_config(context, "oauth2_refresh_token", Some(token))
                 .ok();
             context
                 .sql
-                .set_config(context, "oauth2_refresh_token_for", Some(code.as_ref()))
+                .set_raw_config(context, "oauth2_refresh_token_for", Some(code.as_ref()))
                 .ok();
         }
 
@@ -172,22 +205,22 @@ pub fn dc_get_oauth2_access_token(
         if let Some(ref token) = response.access_token {
             context
                 .sql
-                .set_config(context, "oauth2_access_token", Some(token))
+                .set_raw_config(context, "oauth2_access_token", Some(token))
                 .ok();
             let expires_in = response
                 .expires_in
-                // refresh a bet before
+                // refresh a bit before
                 .map(|t| time() + t as i64 - 5)
                 .unwrap_or_else(|| 0);
             context
                 .sql
-                .set_config_int64(context, "oauth2_timestamp_expires", expires_in)
+                .set_raw_config_int64(context, "oauth2_timestamp_expires", expires_in)
                 .ok();
 
             if update_redirect_uri_on_success {
                 context
                     .sql
-                    .set_config(context, "oauth2_redirect_uri", Some(redirect_uri.as_ref()))
+                    .set_raw_config(context, "oauth2_redirect_uri", Some(redirect_uri.as_ref()))
                     .ok();
             }
         } else {
@@ -207,14 +240,8 @@ pub fn dc_get_oauth2_addr(
     addr: impl AsRef<str>,
     code: impl AsRef<str>,
 ) -> Option<String> {
-    let oauth2 = Oauth2::from_address(addr.as_ref());
-    if oauth2.is_none() {
-        return None;
-    }
-    let oauth2 = oauth2.unwrap();
-    if oauth2.get_userinfo.is_none() {
-        return None;
-    }
+    let oauth2 = Oauth2::from_address(addr.as_ref())?;
+    oauth2.get_userinfo?;
 
     if let Some(access_token) =
         dc_get_oauth2_access_token(context, addr.as_ref(), code.as_ref(), false)
@@ -274,7 +301,7 @@ impl Oauth2 {
             return None;
         }
 
-        let parsed: reqwest::Result<HashMap<String, String>> = response.json();
+        let parsed: reqwest::Result<HashMap<String, serde_json::Value>> = response.json();
         if parsed.is_err() {
             warn!(
                 context,
@@ -283,11 +310,13 @@ impl Oauth2 {
             return None;
         }
         if let Ok(response) = parsed {
+            // serde_json::Value.as_str() removes the quotes of json-strings
             let addr = response.get("email");
             if addr.is_none() {
                 warn!(context, "E-mail missing in userinfo.");
+                return None;
             }
-
+            let addr = addr.unwrap().as_str();
             addr.map(|addr| addr.to_string())
         } else {
             warn!(context, "Failed to parse userinfo.");
@@ -299,7 +328,7 @@ impl Oauth2 {
 fn is_expired(context: &Context) -> bool {
     let expire_timestamp = context
         .sql
-        .get_config_int64(context, "oauth2_timestamp_expires")
+        .get_raw_config_int64(context, "oauth2_timestamp_expires")
         .unwrap_or_default();
 
     if expire_timestamp <= 0 {
@@ -326,6 +355,8 @@ fn normalize_addr(addr: &str) -> &str {
 mod tests {
     use super::*;
 
+    use crate::test_utils::*;
+
     #[test]
     fn test_normalize_addr() {
         assert_eq!(normalize_addr(" hello@mail.de  "), "hello@mail.de");
@@ -355,4 +386,24 @@ mod tests {
 
         assert_eq!(Oauth2::from_address("hello@web.de"), None);
     }
+
+    #[test]
+    fn test_dc_get_oauth2_addr() {
+        let ctx = dummy_context();
+        let addr = "dignifiedquire@gmail.com";
+        let code = "fail";
+        let res = dc_get_oauth2_addr(&ctx.ctx, addr, code);
+        // this should fail as it is an invalid password
+        assert_eq!(res, None);
+    }
+
+    #[test]
+    fn test_dc_get_oauth2_token() {
+        let ctx = dummy_context();
+        let addr = "dignifiedquire@gmail.com";
+        let code = "fail";
+        let res = dc_get_oauth2_access_token(&ctx.ctx, addr, code, false);
+        // this should fail as it is an invalid password
+        assert_eq!(res, None);
+    }
 }

src/param.rs

@@ -1,9 +1,12 @@
 use std::collections::BTreeMap;
 use std::fmt;
+use std::path::PathBuf;
 use std::str;
 
 use num_traits::FromPrimitive;
 
+use crate::blob::{BlobError, BlobObject};
+use crate::context::Context;
 use crate::dc_mimeparser::SystemMessage;
 use crate::error;
 
@@ -12,65 +15,75 @@ use crate::error;
 #[repr(u8)]
 pub enum Param {
     /// For messages and jobs
-    File = 'f' as u8,
+    File = b'f',
     /// For Messages
-    Width = 'w' as u8,
+    Width = b'w',
     /// For Messages
-    Height = 'h' as u8,
+    Height = b'h',
     /// For Messages
-    Duration = 'd' as u8,
+    Duration = b'd',
     /// For Messages
-    MimeType = 'm' as u8,
+    MimeType = b'm',
     /// For Messages: message is encryoted, outgoing: guarantee E2EE or the message is not send
-    GuranteeE2ee = 'c' as u8,
+    GuaranteeE2ee = b'c',
     /// For Messages: decrypted with validation errors or without mutual set, if neither
     /// 'c' nor 'e' are preset, the messages is only transport encrypted.
-    ErroneousE2ee = 'e' as u8,
+    ErroneousE2ee = b'e',
     /// For Messages: force unencrypted message, either `ForcePlaintext::AddAutocryptHeader` (1),
     /// `ForcePlaintext::NoAutocryptHeader` (2) or 0.
-    ForcePlaintext = 'u' as u8,
+    ForcePlaintext = b'u',
     /// For Messages
-    WantsMdn = 'r' as u8,
+    WantsMdn = b'r',
     /// For Messages
-    Forwarded = 'a' as u8,
+    Forwarded = b'a',
     /// For Messages
-    Cmd = 'S' as u8,
+    Cmd = b'S',
     /// For Messages
-    Arg = 'E' as u8,
+    Arg = b'E',
     /// For Messages
-    Arg2 = 'F' as u8,
+    Arg2 = b'F',
     /// For Messages
-    Arg3 = 'G' as u8,
+    Arg3 = b'G',
     /// For Messages
-    Arg4 = 'H' as u8,
+    Arg4 = b'H',
     /// For Messages
-    Error = 'L' as u8,
+    Error = b'L',
     /// For Messages: space-separated list of messaged IDs of forwarded copies.
-    PrepForwards = 'P' as u8,
+    ///
+    /// This is used when a [Message] is in the
+    /// [MessageState::OutPending] state but is already forwarded.
+    /// In this case the forwarded messages are written to the
+    /// database and their message IDs are added to this parameter of
+    /// the original message, which is also saved in the database.
+    /// When the original message is then finally sent this parameter
+    /// is used to also send all the forwarded messages.
+    PrepForwards = b'P',
     /// For Jobs
-    SetLatitude = 'l' as u8,
+    SetLatitude = b'l',
     /// For Jobs
-    SetLongitude = 'n' as u8,
+    SetLongitude = b'n',
     /// For Jobs
-    ServerFolder = 'Z' as u8,
+    ServerFolder = b'Z',
     /// For Jobs
-    ServerUid = 'z' as u8,
+    ServerUid = b'z',
     /// For Jobs
-    AlsoMove = 'M' as u8,
+    AlsoMove = b'M',
     /// For Jobs: space-separated list of message recipients
-    Recipients = 'R' as u8,
+    Recipients = b'R',
     // For Groups
-    Unpromoted = 'U' as u8,
+    Unpromoted = b'U',
     // For Groups and Contacts
-    ProfileImage = 'i' as u8,
+    ProfileImage = b'i',
     // For Chats
-    Selftalk = 'K' as u8,
+    Selftalk = b'K',
+    // For Chats
+    Devicetalk = b'D',
     // For QR
-    Auth = 's' as u8,
+    Auth = b's',
     // For QR
-    GroupId = 'x' as u8,
+    GroupId = b'x',
     // For QR
-    GroupName = 'g' as u8,
+    GroupName = b'g',
 }
 
 /// Possible values for `Param::ForcePlaintext`.
@@ -122,8 +135,8 @@ impl str::FromStr for Params {
             ensure!(key.is_some(), "Missing key");
             ensure!(value.is_some(), "Missing value");
 
-            let key = key.unwrap().trim();
-            let value = value.unwrap().trim();
+            let key = key.unwrap_or_default().trim();
+            let value = value.unwrap_or_default().trim();
 
             if let Some(key) = Param::from_u8(key.as_bytes()[0]) {
                 inner.insert(key, value.to_string());
@@ -186,11 +199,82 @@ impl Params {
             .unwrap_or_default()
     }
 
+    /// Set the parameter behind `Param::Cmd`.
+    pub fn set_cmd(&mut self, value: SystemMessage) {
+        self.set_int(Param::Cmd, value as i32);
+    }
+
     /// Get the given parameter and parse as `f64`.
     pub fn get_float(&self, key: Param) -> Option<f64> {
         self.get(key).and_then(|s| s.parse().ok())
     }
 
+    /// Gets the given parameter and parse as [ParamsFile].
+    ///
+    /// See also [Params::get_blob] and [Params::get_path] which may
+    /// be more convenient.
+    pub fn get_file<'a>(
+        &self,
+        key: Param,
+        context: &'a Context,
+    ) -> Result<Option<ParamsFile<'a>>, BlobError> {
+        let val = match self.get(key) {
+            Some(val) => val,
+            None => return Ok(None),
+        };
+        ParamsFile::from_param(context, val).map(|file| Some(file))
+    }
+
+    /// Gets the parameter and returns a [BlobObject] for it.
+    ///
+    /// This parses the parameter value as a [ParamsFile] and than
+    /// tries to return a [BlobObject] for that file.  If the file is
+    /// not yet a valid blob, one will be created by copying the file
+    /// only if `create` is set to `true`, otherwise the a [BlobError]
+    /// will result.
+    ///
+    /// Note that in the [ParamsFile::FsPath] case the blob can be
+    /// created without copying if the path already referes to a valid
+    /// blob.  If so a [BlobObject] will be returned regardless of the
+    /// `create` argument.
+    pub fn get_blob<'a>(
+        &self,
+        key: Param,
+        context: &'a Context,
+        create: bool,
+    ) -> Result<Option<BlobObject<'a>>, BlobError> {
+        let val = match self.get(key) {
+            Some(val) => val,
+            None => return Ok(None),
+        };
+        let file = ParamsFile::from_param(context, val)?;
+        let blob = match file {
+            ParamsFile::FsPath(path) => match create {
+                true => BlobObject::create_from_path(context, path)?,
+                false => BlobObject::from_path(context, path)?,
+            },
+            ParamsFile::Blob(blob) => blob,
+        };
+        Ok(Some(blob))
+    }
+
+    /// Gets the parameter and returns a [PathBuf] for it.
+    ///
+    /// This parses the parameter value as a [ParamsFile] and returns
+    /// a [PathBuf] to the file.
+    pub fn get_path(&self, key: Param, context: &Context) -> Result<Option<PathBuf>, BlobError> {
+        let val = match self.get(key) {
+            Some(val) => val,
+            None => return Ok(None),
+        };
+        let file = ParamsFile::from_param(context, val)?;
+        let path = match file {
+            ParamsFile::FsPath(path) => path,
+            ParamsFile::Blob(blob) => blob.to_abs_path(),
+        };
+        Ok(Some(path))
+    }
+
     /// Set the given paramter to the passed in `i32`.
     pub fn set_int(&mut self, key: Param, value: i32) -> &mut Self {
         self.set(key, format!("{}", value));
@@ -204,10 +288,41 @@ impl Params {
     }
 }
 
+/// The value contained in [Param::File].
+///
+/// Because the only way to construct this object is from a valid
+/// UTF-8 string it is always safe to convert the value contained
+/// within the [ParamsFile::FsPath] back to a [String] or [&str].
+/// Despite the type itself does not guarantee this.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum ParamsFile<'a> {
+    FsPath(PathBuf),
+    Blob(BlobObject<'a>),
+}
+
+impl<'a> ParamsFile<'a> {
+    /// Parse the [Param::File] value into an object.
+    ///
+    /// If the value was stored into the [Params] correctly this
+    /// should not fail.
+    pub fn from_param(context: &'a Context, src: &str) -> Result<ParamsFile<'a>, BlobError> {
+        let param = match src.starts_with("$BLOBDIR/") {
+            true => ParamsFile::Blob(BlobObject::from_name(context, src.to_string())?),
+            false => ParamsFile::FsPath(PathBuf::from(src)),
+        };
+        Ok(param)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
 
+    use std::fs;
+    use std::path::Path;
+
+    use crate::test_utils::*;
+
     #[test]
     fn test_dc_param() {
         let mut p1: Params = "\r\n\r\na=1\nf=2\n\nc = 3 ".parse().unwrap();
@@ -225,7 +340,7 @@ mod tests {
 
         p1.set(Param::Forwarded, "foo")
             .set_int(Param::File, 2)
-            .remove(Param::GuranteeE2ee)
+            .remove(Param::GuaranteeE2ee)
             .set_int(Param::Duration, 4);
 
         assert_eq!(p1.to_string(), "a=foo\nd=4\nf=2");
@@ -251,4 +366,67 @@ mod tests {
             .unwrap();
         assert_eq!(p1.get(Param::Forwarded).unwrap(), "cli%40deltachat.de");
     }
+
+    #[test]
+    fn test_params_file_fs_path() {
+        let t = dummy_context();
+        if let ParamsFile::FsPath(p) = ParamsFile::from_param(&t.ctx, "/foo/bar/baz").unwrap() {
+            assert_eq!(p, Path::new("/foo/bar/baz"));
+        } else {
+            assert!(false, "Wrong enum variant");
+        }
+    }
+
+    #[test]
+    fn test_params_file_blob() {
+        let t = dummy_context();
+        if let ParamsFile::Blob(b) = ParamsFile::from_param(&t.ctx, "$BLOBDIR/foo").unwrap() {
+            assert_eq!(b.as_name(), "$BLOBDIR/foo");
+        } else {
+            assert!(false, "Wrong enum variant");
+        }
+    }
+
+    // Tests for Params::get_file(), Params::get_path() and Params::get_blob().
+    #[test]
+    fn test_params_get_fileparam() {
+        let t = dummy_context();
+        let fname = t.dir.path().join("foo");
+        let mut p = Params::new();
+        p.set(Param::File, fname.to_str().unwrap());
+
+        let file = p.get_file(Param::File, &t.ctx).unwrap().unwrap();
+        assert_eq!(file, ParamsFile::FsPath(fname.clone()));
+
+        let path = p.get_path(Param::File, &t.ctx).unwrap().unwrap();
+        assert_eq!(path, fname);
+
+        // Blob does not exist yet, expect BlobError.
+        let err = p.get_blob(Param::File, &t.ctx, false).unwrap_err();
+        match err {
+            BlobError::WrongBlobdir { .. } => (),
+            _ => panic!("wrong error type/variant: {:?}", err),
+        }
+
+        fs::write(fname, b"boo").unwrap();
+        let blob = p.get_blob(Param::File, &t.ctx, true).unwrap().unwrap();
+        assert_eq!(
+            blob,
+            BlobObject::from_name(&t.ctx, "foo".to_string()).unwrap()
+        );
+
+        // Blob in blobdir, expect blob.
+        let bar = t.ctx.get_blobdir().join("bar");
+        p.set(Param::File, bar.to_str().unwrap());
+        let blob = p.get_blob(Param::File, &t.ctx, false).unwrap().unwrap();
+        assert_eq!(
+            blob,
+            BlobObject::from_name(&t.ctx, "bar".to_string()).unwrap()
+        );
+
+        p.remove(Param::File);
+        assert!(p.get_file(Param::File, &t.ctx).unwrap().is_none());
+        assert!(p.get_path(Param::File, &t.ctx).unwrap().is_none());
+        assert!(p.get_blob(Param::File, &t.ctx, false).unwrap().is_none());
+    }
 }

src/peerstate.rs

@@ -1,3 +1,4 @@
+//! # [Autocrypt Peer State](https://autocrypt.org/level1.html#peer-state-management) module
 use std::collections::HashSet;
 use std::fmt;
 
@@ -11,6 +12,17 @@ use crate::error::*;
 use crate::key::*;
 use crate::sql::{self, Sql};
 
+pub const DC_PS_GOSSIP_KEY: u32 = 0;
+pub const DC_PS_PUBLIC_KEY: u32 = 1;
+
+#[derive(Debug, PartialEq, Eq, Clone, Copy, FromPrimitive)]
+#[repr(u8)]
+pub enum PeerstateVerifiedStatus {
+    Unverified = 0,
+    //Verified = 1, // not used
+    BidirectVerified = 2,
+}
+
 /// Peerstate represents the state of an Autocrypt peer.
 pub struct Peerstate<'a> {
     pub context: &'a Context,
@@ -310,7 +322,7 @@ impl<'a> Peerstate<'a> {
         };
     }
 
-    pub fn render_gossip_header(&self, min_verified: usize) -> Option<String> {
+    pub fn render_gossip_header(&self, min_verified: PeerstateVerifiedStatus) -> Option<String> {
         if let Some(ref addr) = self.addr {
             if let Some(key) = self.peek_key(min_verified) {
                 // TODO: avoid cloning
@@ -326,12 +338,12 @@ impl<'a> Peerstate<'a> {
         None
     }
 
-    pub fn peek_key(&self, min_verified: usize) -> Option<&Key> {
+    pub fn peek_key(&self, min_verified: PeerstateVerifiedStatus) -> Option<&Key> {
         if self.public_key.is_none() && self.gossip_key.is_none() && self.verified_key.is_none() {
             return None;
         }
 
-        if 0 != min_verified {
+        if min_verified != PeerstateVerifiedStatus::Unverified {
             return self.verified_key.as_ref();
         }
         if self.public_key.is_some() {
@@ -341,10 +353,17 @@ impl<'a> Peerstate<'a> {
         self.gossip_key.as_ref()
     }
 
-    pub fn set_verified(&mut self, which_key: usize, fingerprint: &str, verified: usize) -> bool {
+    pub fn set_verified(
+        &mut self,
+        which_key: u32,
+        fingerprint: &str,
+        verified: PeerstateVerifiedStatus,
+    ) -> bool {
         let mut success = false;
-        if !(which_key != 0 && which_key != 1 || verified != 2) {
-            if which_key == 1
+        if !(which_key != DC_PS_GOSSIP_KEY && which_key != DC_PS_PUBLIC_KEY
+            || verified != PeerstateVerifiedStatus::BidirectVerified)
+        {
+            if which_key == DC_PS_PUBLIC_KEY
                 && self.public_key_fingerprint.is_some()
                 && self.public_key_fingerprint.as_ref().unwrap() == fingerprint
             {
@@ -353,7 +372,7 @@ impl<'a> Peerstate<'a> {
                 self.verified_key_fingerprint = self.public_key_fingerprint.clone();
                 success = true;
             }
-            if which_key == 0
+            if which_key == DC_PS_GOSSIP_KEY
                 && self.gossip_key_fingerprint.is_some()
                 && self.gossip_key_fingerprint.as_ref().unwrap() == fingerprint
             {
@@ -382,11 +401,11 @@ impl<'a> Peerstate<'a> {
             sql::execute(
                 self.context,
                 sql,
-		"UPDATE acpeerstates \
-		 SET last_seen=?, last_seen_autocrypt=?, prefer_encrypted=?, \
-		 public_key=?, gossip_timestamp=?, gossip_key=?, public_key_fingerprint=?, gossip_key_fingerprint=?, \
+                "UPDATE acpeerstates \
+                 SET last_seen=?, last_seen_autocrypt=?, prefer_encrypted=?, \
+                 public_key=?, gossip_timestamp=?, gossip_key=?, public_key_fingerprint=?, gossip_key_fingerprint=?, \
                  verified_key=?, verified_key_fingerprint=? \
-		 WHERE addr=?;",
+                 WHERE addr=?;",
                 params![
                     self.last_seen,
                     self.last_seen_autocrypt,
@@ -400,7 +419,8 @@ impl<'a> Peerstate<'a> {
                     &self.verified_key_fingerprint,
                     &self.addr,
                 ],
-            )?
+            )?;
+            reset_gossiped_timestamp(self.context, 0);
         } else if self.to_save == Some(ToSave::Timestamps) {
             sql::execute(
                 self.context,
@@ -416,10 +436,6 @@ impl<'a> Peerstate<'a> {
             )?;
         }
 
-        if self.to_save == Some(ToSave::All) || create {
-            reset_gossiped_timestamp(self.context, 0);
-        }
-
         Ok(())
     }
 
@@ -475,7 +491,7 @@ mod tests {
             "failed to save to db"
         );
 
-        let peerstate_new = Peerstate::from_addr(&ctx.ctx, &ctx.ctx.sql, addr.into())
+        let peerstate_new = Peerstate::from_addr(&ctx.ctx, &ctx.ctx.sql, addr)
             .expect("failed to load peerstate from db");
 
         // clear to_save, as that is not persissted
@@ -487,6 +503,44 @@ mod tests {
         assert_eq!(peerstate, peerstate_new2);
     }
 
+    #[test]
+    fn test_peerstate_double_create() {
+        let ctx = crate::test_utils::dummy_context();
+        let addr = "hello@mail.com";
+
+        let pub_key = crate::key::Key::from_base64(
+            include_str!("../test-data/key/public.asc"),
+            KeyType::Public,
+        )
+        .unwrap();
+
+        let peerstate = Peerstate {
+            context: &ctx.ctx,
+            addr: Some(addr.into()),
+            last_seen: 10,
+            last_seen_autocrypt: 11,
+            prefer_encrypt: EncryptPreference::Mutual,
+            public_key: Some(pub_key.clone()),
+            public_key_fingerprint: Some(pub_key.fingerprint()),
+            gossip_key: None,
+            gossip_timestamp: 12,
+            gossip_key_fingerprint: None,
+            verified_key: None,
+            verified_key_fingerprint: None,
+            to_save: Some(ToSave::All),
+            degrade_event: None,
+        };
+
+        assert!(
+            peerstate.save_to_db(&ctx.ctx.sql, true).is_ok(),
+            "failed to save"
+        );
+        assert!(
+            peerstate.save_to_db(&ctx.ctx.sql, true).is_ok(),
+            "double-call with create failed"
+        );
+    }
+
     #[test]
     fn test_peerstate_with_empty_gossip_key_save_to_db() {
         let ctx = crate::test_utils::dummy_context();
@@ -520,7 +574,7 @@ mod tests {
             "failed to save"
         );
 
-        let peerstate_new = Peerstate::from_addr(&ctx.ctx, &ctx.ctx.sql, addr.into())
+        let peerstate_new = Peerstate::from_addr(&ctx.ctx, &ctx.ctx.sql, addr)
             .expect("failed to load peerstate from db");
 
         // clear to_save, as that is not persissted

src/pgp.rs

@@ -1,137 +1,120 @@
-use std::collections::HashSet;
-use std::convert::TryInto;
-use std::io::Cursor;
-use std::ptr;
+//! OpenPGP helper module using [rPGP facilities](https://github.com/rpgp/rpgp)
 
-use libc::{strchr, strlen, strncmp, strspn, strstr};
+use std::collections::{BTreeMap, HashSet};
+use std::convert::TryInto;
+use std::io;
+use std::io::Cursor;
+
+use pgp::armor::BlockType;
 use pgp::composed::{
     Deserializable, KeyType as PgpKeyType, Message, SecretKeyParamsBuilder, SignedPublicKey,
-    SignedSecretKey, SubkeyParamsBuilder,
+    SignedPublicSubKey, SignedSecretKey, SubkeyParamsBuilder,
 };
 use pgp::crypto::{HashAlgorithm, SymmetricKeyAlgorithm};
-use pgp::types::{CompressionAlgorithm, KeyTrait, SecretKeyTrait, StringToKey};
-use rand::thread_rng;
+use pgp::types::{
+    CompressionAlgorithm, KeyTrait, Mpi, PublicKeyTrait, SecretKeyTrait, StringToKey,
+};
+use rand::{thread_rng, CryptoRng, Rng};
 
-use crate::dc_tools::*;
 use crate::error::Error;
 use crate::key::*;
 use crate::keyring::*;
 
-pub unsafe fn dc_split_armored_data(
-    buf: *mut libc::c_char,
-    ret_headerline: *mut String,
-    ret_setupcodebegin: *mut *const libc::c_char,
-    ret_preferencrypt: *mut *const libc::c_char,
-    ret_base64: *mut *const libc::c_char,
-) -> bool {
-    let mut success = false;
-    let mut line_chars: libc::size_t = 0;
-    let mut line: *mut libc::c_char = buf;
-    let mut p1: *mut libc::c_char = buf;
-    let mut p2: *mut libc::c_char;
-    let mut headerline: *mut libc::c_char = ptr::null_mut();
-    let mut base64: *mut libc::c_char = ptr::null_mut();
-    if !ret_setupcodebegin.is_null() {
-        *ret_setupcodebegin = ptr::null_mut();
-    }
-    if !ret_preferencrypt.is_null() {
-        *ret_preferencrypt = ptr::null();
-    }
-    if !ret_base64.is_null() {
-        *ret_base64 = ptr::null();
-    }
-    if !buf.is_null() {
-        dc_remove_cr_chars(buf);
-        while 0 != *p1 {
-            if *p1 as libc::c_int == '\n' as i32 {
-                *line.offset(line_chars as isize) = 0i32 as libc::c_char;
-                if headerline.is_null() {
-                    dc_trim(line);
-                    if strncmp(
-                        line,
-                        b"-----BEGIN \x00" as *const u8 as *const libc::c_char,
-                        1,
-                    ) == 0i32
-                        && strncmp(
-                            &mut *line.offset(strlen(line).wrapping_sub(5) as isize),
-                            b"-----\x00" as *const u8 as *const libc::c_char,
-                            5,
-                        ) == 0i32
-                    {
-                        headerline = line;
-                        *ret_headerline = as_str(headerline).to_string();
-                    }
-                } else if strspn(line, b"\t\r\n \x00" as *const u8 as *const libc::c_char)
-                    == strlen(line)
-                {
-                    base64 = p1.offset(1isize);
-                    break;
-                } else {
-                    p2 = strchr(line, ':' as i32);
-                    if p2.is_null() {
-                        *line.offset(line_chars as isize) = '\n' as i32 as libc::c_char;
-                        base64 = line;
-                        break;
-                    } else {
-                        *p2 = 0i32 as libc::c_char;
-                        dc_trim(line);
-                        if strcasecmp(
-                            line,
-                            b"Passphrase-Begin\x00" as *const u8 as *const libc::c_char,
-                        ) == 0i32
-                        {
-                            p2 = p2.offset(1isize);
-                            dc_trim(p2);
-                            if !ret_setupcodebegin.is_null() {
-                                *ret_setupcodebegin = p2
-                            }
-                        } else if strcasecmp(
-                            line,
-                            b"Autocrypt-Prefer-Encrypt\x00" as *const u8 as *const libc::c_char,
-                        ) == 0i32
-                        {
-                            p2 = p2.offset(1isize);
-                            dc_trim(p2);
-                            if !ret_preferencrypt.is_null() {
-                                *ret_preferencrypt = p2
-                            }
-                        }
-                    }
-                }
-                p1 = p1.offset(1isize);
-                line = p1;
-                line_chars = 0;
-            } else {
-                p1 = p1.offset(1isize);
-                line_chars = line_chars.wrapping_add(1)
-            }
-        }
-        if !(headerline.is_null() || base64.is_null()) {
-            /* now, line points to beginning of base64 data, search end */
-            /*the trailing space makes sure, this is not a normal base64 sequence*/
-            p1 = strstr(base64, b"-----END \x00" as *const u8 as *const libc::c_char);
-            if !(p1.is_null()
-                || strncmp(
-                    p1.offset(9isize),
-                    headerline.offset(11isize),
-                    strlen(headerline.offset(11isize)),
-                ) != 0i32)
-            {
-                *p1 = 0i32 as libc::c_char;
-                dc_trim(base64);
-                if !ret_base64.is_null() {
-                    *ret_base64 = base64
-                }
-                success = true;
-            }
+pub const HEADER_AUTOCRYPT: &str = "autocrypt-prefer-encrypt";
+pub const HEADER_SETUPCODE: &str = "passphrase-begin";
+
+/// A wrapper for rPGP public key types
+#[derive(Debug)]
+enum SignedPublicKeyOrSubkey<'a> {
+    Key(&'a SignedPublicKey),
+    Subkey(&'a SignedPublicSubKey),
+}
+
+impl<'a> KeyTrait for SignedPublicKeyOrSubkey<'a> {
+    fn fingerprint(&self) -> Vec<u8> {
+        match self {
+            Self::Key(k) => k.fingerprint(),
+            Self::Subkey(k) => k.fingerprint(),
         }
     }
 
-    success
+    fn key_id(&self) -> pgp::types::KeyId {
+        match self {
+            Self::Key(k) => k.key_id(),
+            Self::Subkey(k) => k.key_id(),
+        }
+    }
+
+    fn algorithm(&self) -> pgp::crypto::PublicKeyAlgorithm {
+        match self {
+            Self::Key(k) => k.algorithm(),
+            Self::Subkey(k) => k.algorithm(),
+        }
+    }
+}
+
+impl<'a> PublicKeyTrait for SignedPublicKeyOrSubkey<'a> {
+    fn verify_signature(
+        &self,
+        hash: HashAlgorithm,
+        data: &[u8],
+        sig: &[Mpi],
+    ) -> pgp::errors::Result<()> {
+        match self {
+            Self::Key(k) => k.verify_signature(hash, data, sig),
+            Self::Subkey(k) => k.verify_signature(hash, data, sig),
+        }
+    }
+
+    fn encrypt<R: Rng + CryptoRng>(
+        &self,
+        rng: &mut R,
+        plain: &[u8],
+    ) -> pgp::errors::Result<Vec<Mpi>> {
+        match self {
+            Self::Key(k) => k.encrypt(rng, plain),
+            Self::Subkey(k) => k.encrypt(rng, plain),
+        }
+    }
+
+    fn to_writer_old(&self, writer: &mut impl io::Write) -> pgp::errors::Result<()> {
+        match self {
+            Self::Key(k) => k.to_writer_old(writer),
+            Self::Subkey(k) => k.to_writer_old(writer),
+        }
+    }
+}
+
+/// Split data from PGP Armored Data as defined in https://tools.ietf.org/html/rfc4880#section-6.2.
+///
+/// Returns (type, headers, base64 encoded body).
+pub fn split_armored_data(
+    buf: &[u8],
+) -> Result<(BlockType, BTreeMap<String, String>, Vec<u8>), Error> {
+    use std::io::Read;
+
+    let cursor = Cursor::new(buf);
+    let mut dearmor = pgp::armor::Dearmor::new(cursor);
+
+    let mut bytes = Vec::with_capacity(buf.len());
+
+    dearmor.read_to_end(&mut bytes)?;
+    ensure!(dearmor.typ.is_some(), "Failed to parse type");
+
+    let typ = dearmor.typ.unwrap();
+
+    // normalize headers
+    let headers = dearmor
+        .headers
+        .into_iter()
+        .map(|(key, value)| (key.trim().to_lowercase(), value.trim().to_string()))
+        .collect();
+
+    Ok((typ, headers, bytes))
 }
 
 /// Create a new key pair.
-pub fn dc_pgp_create_keypair(addr: impl AsRef<str>) -> Option<(Key, Key)> {
+pub fn create_keypair(addr: impl AsRef<str>) -> Option<(Key, Key)> {
     let user_id = format!("<{}>", addr.as_ref());
 
     let key_params = SecretKeyParamsBuilder::default()
@@ -181,20 +164,49 @@ pub fn dc_pgp_create_keypair(addr: impl AsRef<str>) -> Option<(Key, Key)> {
     Some((Key::Public(public_key), Key::Secret(private_key)))
 }
 
-pub fn dc_pgp_pk_encrypt(
+/// Select public key or subkey to use for encryption.
+///
+/// First, tries to use subkeys. If none of the subkeys are suitable
+/// for encryption, tries to use primary key. Returns `None` if the public
+/// key cannot be used for encryption.
+///
+/// TODO: take key flags and expiration dates into account
+fn select_pk_for_encryption(key: &SignedPublicKey) -> Option<SignedPublicKeyOrSubkey> {
+    key.public_subkeys
+        .iter()
+        .find(|subkey| subkey.is_encryption_key())
+        .map_or_else(
+            || {
+                // No usable subkey found, try primary key
+                if key.is_encryption_key() {
+                    Some(SignedPublicKeyOrSubkey::Key(key))
+                } else {
+                    None
+                }
+            },
+            |subkey| Some(SignedPublicKeyOrSubkey::Subkey(subkey)),
+        )
+}
+
+/// Encrypts `plain` text using `public_keys_for_encryption`
+/// and signs it using `private_key_for_signing`.
+pub fn pk_encrypt(
     plain: &[u8],
     public_keys_for_encryption: &Keyring,
     private_key_for_signing: Option<&Key>,
 ) -> Result<String, Error> {
     let lit_msg = Message::new_literal_bytes("", plain);
-    let pkeys: Vec<&SignedPublicKey> = public_keys_for_encryption
+    let pkeys: Vec<SignedPublicKeyOrSubkey> = public_keys_for_encryption
         .keys()
-        .into_iter()
+        .iter()
         .filter_map(|key| {
-            let k: &Key = &key;
-            k.try_into().ok()
+            key.as_ref()
+                .try_into()
+                .ok()
+                .and_then(select_pk_for_encryption)
         })
         .collect();
+    let pkeys_refs: Vec<&SignedPublicKeyOrSubkey> = pkeys.iter().collect();
 
     let mut rng = thread_rng();
 
@@ -207,9 +219,9 @@ pub fn dc_pgp_pk_encrypt(
         lit_msg
             .sign(skey, || "".into(), Default::default())
             .and_then(|msg| msg.compress(CompressionAlgorithm::ZLIB))
-            .and_then(|msg| msg.encrypt_to_keys(&mut rng, Default::default(), &pkeys))
+            .and_then(|msg| msg.encrypt_to_keys(&mut rng, Default::default(), &pkeys_refs))
     } else {
-        lit_msg.encrypt_to_keys(&mut rng, Default::default(), &pkeys)
+        lit_msg.encrypt_to_keys(&mut rng, Default::default(), &pkeys_refs)
     };
 
     let msg = encrypted_msg?;
@@ -218,7 +230,7 @@ pub fn dc_pgp_pk_encrypt(
     Ok(encoded_msg)
 }
 
-pub fn dc_pgp_pk_decrypt(
+pub fn pk_decrypt(
     ctext: &[u8],
     private_keys_for_decryption: &Keyring,
     public_keys_for_validation: &Keyring,
@@ -267,7 +279,7 @@ pub fn dc_pgp_pk_decrypt(
 }
 
 /// Symmetric encryption.
-pub fn dc_pgp_symm_encrypt(passphrase: &str, plain: &[u8]) -> Result<String, Error> {
+pub fn symm_encrypt(passphrase: &str, plain: &[u8]) -> Result<String, Error> {
     let mut rng = thread_rng();
     let lit_msg = Message::new_literal_bytes("", plain);
 
@@ -281,8 +293,11 @@ pub fn dc_pgp_symm_encrypt(passphrase: &str, plain: &[u8]) -> Result<String, Err
 }
 
 /// Symmetric decryption.
-pub fn dc_pgp_symm_decrypt(passphrase: &str, ctext: &[u8]) -> Result<Vec<u8>, Error> {
-    let enc_msg = Message::from_bytes(Cursor::new(ctext))?;
+pub fn symm_decrypt<T: std::io::Read + std::io::Seek>(
+    passphrase: &str,
+    ctext: T,
+) -> Result<Vec<u8>, Error> {
+    let (enc_msg, _) = Message::from_armor_single(ctext)?;
     let decryptor = enc_msg.decrypt_with_password(|| passphrase.into())?;
 
     let msgs = decryptor.collect::<Result<Vec<_>, _>>()?;
@@ -293,3 +308,35 @@ pub fn dc_pgp_symm_decrypt(passphrase: &str, ctext: &[u8]) -> Result<Vec<u8>, Er
         None => bail!("Decrypted message is empty"),
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_split_armored_data_1() {
+        let (typ, _headers, base64) = split_armored_data(
+            b"-----BEGIN PGP MESSAGE-----\nNoVal:\n\naGVsbG8gd29ybGQ=\n-----END PGP MESSAGE----",
+        )
+        .unwrap();
+
+        assert_eq!(typ, BlockType::Message);
+        assert!(!base64.is_empty());
+        assert_eq!(
+            std::string::String::from_utf8(base64).unwrap(),
+            "hello world"
+        );
+    }
+
+    #[test]
+    fn test_split_armored_data_2() {
+        let (typ, headers, base64) = split_armored_data(
+            b"-----BEGIN PGP PRIVATE KEY BLOCK-----\nAutocrypt-Prefer-Encrypt: mutual \n\naGVsbG8gd29ybGQ=\n-----END PGP PRIVATE KEY BLOCK-----"
+        )
+            .unwrap();
+
+        assert_eq!(typ, BlockType::PrivateKey);
+        assert!(!base64.is_empty());
+        assert_eq!(headers.get(HEADER_AUTOCRYPT), Some(&"mutual".to_string()));
+    }
+}

src/qr.rs

@@ -1,3 +1,5 @@
+//! # QR code module
+
 use lazy_static::lazy_static;
 use percent_encoding::percent_decode_str;
 
@@ -90,7 +92,8 @@ fn decode_openpgp(context: &Context, qr: &str) -> Lot {
 
     // what is up with that param name?
     let name = if let Some(encoded_name) = param.get(Param::SetLongitude) {
-        match percent_decode_str(encoded_name).decode_utf8() {
+        let encoded_name = encoded_name.replace("+", "%20"); // sometimes spaces are encoded as `+`
+        match percent_decode_str(&encoded_name).decode_utf8() {
             Ok(name) => name.to_string(),
             Err(err) => return format_err!("Invalid name: {}", err).into(),
         }
@@ -104,7 +107,8 @@ fn decode_openpgp(context: &Context, qr: &str) -> Lot {
 
     let grpname = if grpid.is_some() {
         if let Some(encoded_name) = param.get(Param::GroupName) {
-            match percent_decode_str(encoded_name).decode_utf8() {
+            let encoded_name = encoded_name.replace("+", "%20"); // sometimes spaces are encoded as `+`
+            match percent_decode_str(&encoded_name).decode_utf8() {
                 Ok(name) => Some(name.to_string()),
                 Err(err) => return format_err!("Invalid group name: {}", err).into(),
             }
@@ -148,7 +152,7 @@ fn decode_openpgp(context: &Context, qr: &str) -> Lot {
             let (id, _) = chat::create_or_lookup_by_contact_id(context, lot.id, Blocked::Deaddrop)
                 .unwrap_or_default();
 
-            chat::add_device_msg(
+            chat::add_info_msg(
                 context,
                 id,
                 format!("{} verified.", peerstate.addr.unwrap_or_default()),
@@ -186,7 +190,7 @@ fn decode_mailto(context: &Context, qr: &str) -> Lot {
     let addr = if let Some(query_index) = payload.find('?') {
         &payload[..query_index]
     } else {
-        return format_err!("Invalid mailto found").into();
+        payload
     };
 
     let addr = match normalize_address(addr) {
@@ -405,13 +409,21 @@ mod tests {
             &ctx.ctx,
             "mailto:stress@test.local?subject=hello&body=world",
         );
-
         println!("{:?}", res);
         assert_eq!(res.get_state(), LotState::QrAddr);
         assert_ne!(res.get_id(), 0);
-
         let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
         assert_eq!(contact.get_addr(), "stress@test.local");
+
+        let res = check_qr(&ctx.ctx, "mailto:no-questionmark@example.org");
+        assert_eq!(res.get_state(), LotState::QrAddr);
+        assert_ne!(res.get_id(), 0);
+        let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
+        assert_eq!(contact.get_addr(), "no-questionmark@example.org");
+
+        let res = check_qr(&ctx.ctx, "mailto:no-addr");
+        assert_eq!(res.get_state(), LotState::QrError);
+        assert!(res.get_text1().is_some());
     }
 
     #[test]
@@ -434,12 +446,13 @@ mod tests {
 
         let res = check_qr(
             &ctx.ctx,
-            "OPENPGP4FPR:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&g=testtesttest&x=h-0oKQf2CDK&i=9JEXlxAqGM0&s=0V7LzL9cxRL"
+            "OPENPGP4FPR:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&g=test%20%3F+test%20%21&x=h-0oKQf2CDK&i=9JEXlxAqGM0&s=0V7LzL9cxRL"
         );
 
         println!("{:?}", res);
         assert_eq!(res.get_state(), LotState::QrAskVerifyGroup);
         assert_ne!(res.get_id(), 0);
+        assert_eq!(res.get_text1().unwrap(), "test ? test !");
 
         let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
         assert_eq!(contact.get_addr(), "cli@deltachat.de");
@@ -451,7 +464,7 @@ mod tests {
 
         let res = check_qr(
             &ctx.ctx,
-            "OPENPGP4FPR:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&n=&i=TbnwJ6lSvD5&s=0ejvbdFSQxB"
+            "OPENPGP4FPR:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&n=J%C3%B6rn%20P.+P.&i=TbnwJ6lSvD5&s=0ejvbdFSQxB"
         );
 
         println!("{:?}", res);
@@ -460,5 +473,6 @@ mod tests {
 
         let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
         assert_eq!(contact.get_addr(), "cli@deltachat.de");
+        assert_eq!(contact.get_name(), "Jörn P. P.");
     }
 }

src/securejoin.rs

@@ -1,9 +1,10 @@
+//! Verified contact protocol implementation as [specified by countermitm project](https://countermitm.readthedocs.io/en/stable/new.html#setup-contact-protocol)
+
 use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC};
 
 use crate::aheader::EncryptPreference;
 use crate::chat::{self, Chat};
 use crate::config::*;
-use crate::configure::*;
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::Context;
@@ -82,10 +83,7 @@ pub fn dc_get_securejoin_qr(context: &Context, group_chat_id: u32) -> Option<Str
         }
     };
 
-    let self_name = context
-        .sql
-        .get_config(context, "displayname")
-        .unwrap_or_default();
+    let self_name = context.get_config(Config::Displayname).unwrap_or_default();
 
     fingerprint = match get_self_fingerprint(context) {
         Some(fp) => fp,
@@ -160,7 +158,7 @@ pub fn dc_join_securejoin(context: &Context, qr: &str) -> u32 {
             bob.qr_scan = None;
 
             if ongoing_allocated {
-                dc_free_ongoing(context);
+                context.free_ongoing();
             }
             ret_chat_id as u32
         };
@@ -173,7 +171,7 @@ pub fn dc_join_securejoin(context: &Context, qr: &str) -> u32 {
 
     info!(context, "Requesting secure-join ...",);
     ensure_secret_key_exists(context).ok();
-    if !dc_alloc_ongoing(context) {
+    if !context.alloc_ongoing() {
         return cleanup(&context, contact_chat_id, false, join_vg);
     }
     let qr_scan = check_qr(context, &qr);
@@ -187,7 +185,7 @@ pub fn dc_join_securejoin(context: &Context, qr: &str) -> u32 {
         error!(context, "Unknown contact.",);
         return cleanup(&context, contact_chat_id, true, join_vg);
     }
-    if check_exit(context) {
+    if context.shall_stop_ongoing() {
         return cleanup(&context, contact_chat_id, true, join_vg);
     }
     join_vg = qr_scan.get_state() == LotState::QrAskVerifyGroup;
@@ -213,7 +211,7 @@ pub fn dc_join_securejoin(context: &Context, qr: &str) -> u32 {
         info!(context, "Taking protocol shortcut.");
         context.bob.write().unwrap().expects = DC_VC_CONTACT_CONFIRM;
         joiner_progress!(context, chat_id_2_contact_id(context, contact_chat_id), 400);
-        let own_fingerprint = get_self_fingerprint(context).unwrap();
+        let own_fingerprint = get_self_fingerprint(context).unwrap_or_default();
         send_handshake_msg(
             context,
             contact_chat_id,
@@ -243,21 +241,13 @@ pub fn dc_join_securejoin(context: &Context, qr: &str) -> u32 {
     }
 
     // Bob -> Alice
-    while !check_exit(&context) {
-        std::thread::sleep(std::time::Duration::new(0, 3_000_000));
+    while !context.shall_stop_ongoing() {
+        // Don't sleep too long, the user is waiting.
+        std::thread::sleep(std::time::Duration::from_millis(200));
     }
     cleanup(&context, contact_chat_id, true, join_vg)
 }
 
-fn check_exit(context: &Context) -> bool {
-    context
-        .running_state
-        .clone()
-        .read()
-        .unwrap()
-        .shall_stop_ongoing
-}
-
 fn send_handshake_msg(
     context: &Context,
     contact_chat_id: u32,
@@ -270,7 +260,7 @@ fn send_handshake_msg(
     msg.type_0 = Viewtype::Text;
     msg.text = Some(format!("Secure-Join: {}", step));
     msg.hidden = true;
-    msg.param.set_int(Param::Cmd, 7);
+    msg.param.set_cmd(SystemMessage::SecurejoinMessage);
     if step.is_empty() {
         msg.param.remove(Param::Arg);
     } else {
@@ -291,10 +281,10 @@ fn send_handshake_msg(
             ForcePlaintext::AddAutocryptHeader as i32,
         );
     } else {
-        msg.param.set_int(Param::GuranteeE2ee, 1);
+        msg.param.set_int(Param::GuaranteeE2ee, 1);
     }
     // TODO. handle cleanup on error
-    chat::send_msg(context, contact_chat_id, &mut msg).unwrap();
+    chat::send_msg(context, contact_chat_id, &mut msg).unwrap_or_default();
 }
 
 fn chat_id_2_contact_id(context: &Context, contact_chat_id: u32) -> u32 {
@@ -329,21 +319,40 @@ fn fingerprint_equals_sender(
     false
 }
 
+pub(crate) struct HandshakeMessageStatus {
+    pub(crate) hide_this_msg: bool,
+    pub(crate) delete_this_msg: bool,
+    pub(crate) stop_ongoing_process: bool,
+    pub(crate) bob_securejoin_success: Option<bool>,
+}
+
+impl Default for HandshakeMessageStatus {
+    fn default() -> Self {
+        Self {
+            hide_this_msg: true,
+            delete_this_msg: false,
+            stop_ongoing_process: false,
+            bob_securejoin_success: None,
+        }
+    }
+}
+
 /* library private: secure-join */
-pub fn handle_securejoin_handshake(
+pub(crate) fn handle_securejoin_handshake(
     context: &Context,
     mimeparser: &MimeParser,
     contact_id: u32,
-) -> libc::c_int {
+) -> Result<HandshakeMessageStatus, Error> {
     let own_fingerprint: String;
 
-    if contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
-        return 0;
-    }
+    ensure!(
+        contact_id > DC_CONTACT_ID_LAST_SPECIAL,
+        "handle_securejoin_handshake(): called with special contact id"
+    );
     let step = match mimeparser.lookup_optional_field("Secure-Join") {
         Some(s) => s,
         None => {
-            return 0;
+            bail!("This message is not a Secure-Join message");
         }
     };
     info!(
@@ -356,8 +365,8 @@ pub fn handle_securejoin_handshake(
     if contact_chat_id_blocked != Blocked::Not {
         chat::unblock(context, contact_chat_id);
     }
-    let mut ret: libc::c_int = DC_HANDSHAKE_STOP_NORMAL_PROCESSING;
     let join_vg = step.starts_with("vg-");
+    let mut ret = HandshakeMessageStatus::default();
 
     match step.as_str() {
         "vg-request" | "vc-request" => {
@@ -373,12 +382,12 @@ pub fn handle_securejoin_handshake(
                 Some(n) => n,
                 None => {
                     warn!(context, "Secure-join denied (invitenumber missing).",);
-                    return ret;
+                    return Ok(ret);
                 }
             };
             if !token::exists(context, token::Namespace::InviteNumber, &invitenumber) {
                 warn!(context, "Secure-join denied (bad invitenumber).",);
-                return ret;
+                return Ok(ret);
             }
             info!(context, "Secure-join requested.",);
 
@@ -404,7 +413,7 @@ pub fn handle_securejoin_handshake(
             if cond {
                 warn!(context, "auth-required message out of sync.",);
                 // no error, just aborted somehow or a mail from another handshake
-                return ret;
+                return Ok(ret);
             }
             let scanned_fingerprint_of_alice = get_qr_attr!(context, fingerprint).to_string();
             let auth = get_qr_attr!(context, auth).to_string();
@@ -413,14 +422,15 @@ pub fn handle_securejoin_handshake(
                 could_not_establish_secure_connection(
                     context,
                     contact_chat_id,
-                    if mimeparser.e2ee_helper.encrypted {
+                    if mimeparser.encrypted {
                         "No valid signature."
                     } else {
                         "Not encrypted."
                     },
                 );
-                end_bobs_joining(context, DC_BOB_ERROR);
-                return ret;
+                ret.stop_ongoing_process = true;
+                ret.bob_securejoin_success = Some(false);
+                return Ok(ret);
             }
             if !fingerprint_equals_sender(context, &scanned_fingerprint_of_alice, contact_chat_id) {
                 could_not_establish_secure_connection(
@@ -428,8 +438,9 @@ pub fn handle_securejoin_handshake(
                     contact_chat_id,
                     "Fingerprint mismatch on joiner-side.",
                 );
-                end_bobs_joining(context, DC_BOB_ERROR);
-                return ret;
+                ret.stop_ongoing_process = true;
+                ret.bob_securejoin_success = Some(false);
+                return Ok(ret);
             }
             info!(context, "Fingerprint verified.",);
             own_fingerprint = get_self_fingerprint(context).unwrap();
@@ -464,7 +475,7 @@ pub fn handle_securejoin_handshake(
                         contact_chat_id,
                         "Fingerprint not provided.",
                     );
-                    return ret;
+                    return Ok(ret);
                 }
             };
             if !encrypted_and_signed(mimeparser, &fingerprint) {
@@ -473,7 +484,7 @@ pub fn handle_securejoin_handshake(
                     contact_chat_id,
                     "Auth not encrypted.",
                 );
-                return ret;
+                return Ok(ret);
             }
             if !fingerprint_equals_sender(context, &fingerprint, contact_chat_id) {
                 could_not_establish_secure_connection(
@@ -481,7 +492,7 @@ pub fn handle_securejoin_handshake(
                     contact_chat_id,
                     "Fingerprint mismatch on inviter-side.",
                 );
-                return ret;
+                return Ok(ret);
             }
             info!(context, "Fingerprint verified.",);
             // verify that the `Secure-Join-Auth:`-header matches the secret written to the QR code
@@ -493,12 +504,12 @@ pub fn handle_securejoin_handshake(
                         contact_chat_id,
                         "Auth not provided.",
                     );
-                    return ret;
+                    return Ok(ret);
                 }
             };
             if !token::exists(context, token::Namespace::Auth, &auth_0) {
                 could_not_establish_secure_connection(context, contact_chat_id, "Auth invalid.");
-                return ret;
+                return Ok(ret);
             }
             if mark_peer_as_verified(context, fingerprint).is_err() {
                 could_not_establish_secure_connection(
@@ -506,7 +517,7 @@ pub fn handle_securejoin_handshake(
                     contact_chat_id,
                     "Fingerprint mismatch on inviter-side.",
                 );
-                return ret;
+                return Ok(ret);
             }
             Contact::scaleup_origin_by_id(context, contact_id, Origin::SecurejoinInvited);
             info!(context, "Auth verified.",);
@@ -520,7 +531,7 @@ pub fn handle_securejoin_handshake(
                 let (group_chat_id, _, _) = chat::get_chat_id_by_grpid(context, &field_grpid);
                 if group_chat_id == 0 {
                     error!(context, "Chat {} not found.", &field_grpid);
-                    return ret;
+                    return Ok(ret);
                 } else {
                     if let Err(err) =
                         chat::add_contact_to_chat_ex(context, group_chat_id, contact_id, true)
@@ -535,11 +546,11 @@ pub fn handle_securejoin_handshake(
         }
         "vg-member-added" | "vc-contact-confirm" => {
             if join_vg {
-                ret = DC_HANDSHAKE_CONTINUE_NORMAL_PROCESSING;
+                ret.hide_this_msg = false;
             }
             if context.bob.read().unwrap().expects != DC_VC_CONTACT_CONFIRM {
                 info!(context, "Message belongs to a different handshake.",);
-                return ret;
+                return Ok(ret);
             }
             let cond = {
                 let bob = context.bob.read().unwrap();
@@ -551,7 +562,7 @@ pub fn handle_securejoin_handshake(
                     context,
                     "Message out of sync or belongs to a different handshake.",
                 );
-                return ret;
+                return Ok(ret);
             }
             let scanned_fingerprint_of_alice = get_qr_attr!(context, fingerprint).to_string();
 
@@ -575,8 +586,8 @@ pub fn handle_securejoin_handshake(
                     contact_chat_id,
                     "Contact confirm message not encrypted.",
                 );
-                end_bobs_joining(context, DC_BOB_ERROR);
-                return ret;
+                ret.bob_securejoin_success = Some(false);
+                return Ok(ret);
             }
 
             if mark_peer_as_verified(context, &scanned_fingerprint_of_alice).is_err() {
@@ -585,7 +596,7 @@ pub fn handle_securejoin_handshake(
                     contact_chat_id,
                     "Fingerprint mismatch on joiner-side.",
                 );
-                return ret;
+                return Ok(ret);
             }
             Contact::scaleup_origin_by_id(context, contact_id, Origin::SecurejoinJoined);
             emit_event!(context, Event::ContactsChanged(None));
@@ -594,7 +605,7 @@ pub fn handle_securejoin_handshake(
                 .unwrap_or_default();
             if join_vg && !addr_equals_self(context, cg_member_added) {
                 info!(context, "Message belongs to a different handshake (scaled up contact anyway to allow creation of group).");
-                return ret;
+                return Ok(ret);
             }
             secure_connection_established(context, contact_chat_id);
             context.bob.write().unwrap().expects = 0;
@@ -608,7 +619,8 @@ pub fn handle_securejoin_handshake(
                     "",
                 );
             }
-            end_bobs_joining(context, DC_BOB_SUCCESS);
+            ret.stop_ongoing_process = true;
+            ret.bob_securejoin_success = Some(true);
         }
         "vg-member-added-received" => {
             /* ============================================================
@@ -618,28 +630,31 @@ pub fn handle_securejoin_handshake(
             if let Ok(contact) = Contact::get_by_id(context, contact_id) {
                 if contact.is_verified(context) == VerifiedStatus::Unverified {
                     warn!(context, "vg-member-added-received invalid.",);
-                    return ret;
+                    return Ok(ret);
                 }
                 inviter_progress!(context, contact_id, 800);
                 inviter_progress!(context, contact_id, 1000);
+                let field_grpid = mimeparser
+                    .lookup_optional_field("Secure-Join-Group")
+                    .unwrap_or_default();
+                let (group_chat_id, _, _) = chat::get_chat_id_by_grpid(context, &field_grpid);
+                context.call_cb(Event::SecurejoinMemberAdded {
+                    chat_id: group_chat_id,
+                    contact_id: contact_id,
+                });
             } else {
                 warn!(context, "vg-member-added-received invalid.",);
-                return ret;
+                return Ok(ret);
             }
         }
         _ => {
             warn!(context, "invalid step: {}", step);
         }
     }
-    if ret == DC_HANDSHAKE_STOP_NORMAL_PROCESSING {
-        ret |= DC_HANDSHAKE_ADD_DELETE_JOB;
+    if ret.hide_this_msg {
+        ret.delete_this_msg = true;
     }
-    ret
-}
-
-fn end_bobs_joining(context: &Context, status: libc::c_int) {
-    context.bob.write().unwrap().status = status;
-    dc_stop_ongoing_process(context);
+    Ok(ret)
 }
 
 fn secure_connection_established(context: &Context, contact_chat_id: u32) {
@@ -651,7 +666,7 @@ fn secure_connection_established(context: &Context, contact_chat_id: u32) {
         "?"
     };
     let msg = context.stock_string_repl_str(StockMessage::ContactVerified, addr);
-    chat::add_device_msg(context, contact_chat_id, msg);
+    chat::add_info_msg(context, contact_chat_id, msg);
     emit_event!(context, Event::ChatModified(contact_chat_id));
 }
 
@@ -667,7 +682,7 @@ fn could_not_establish_secure_connection(context: &Context, contact_chat_id: u32
         },
     );
 
-    chat::add_device_msg(context, contact_chat_id, &msg);
+    chat::add_info_msg(context, contact_chat_id, &msg);
     error!(context, "{} ({})", &msg, details);
 }
 
@@ -675,10 +690,16 @@ fn mark_peer_as_verified(context: &Context, fingerprint: impl AsRef<str>) -> Res
     if let Some(ref mut peerstate) =
         Peerstate::from_fingerprint(context, &context.sql, fingerprint.as_ref())
     {
-        if peerstate.set_verified(1, fingerprint.as_ref(), 2) {
+        if peerstate.set_verified(
+            DC_PS_PUBLIC_KEY,
+            fingerprint.as_ref(),
+            PeerstateVerifiedStatus::BidirectVerified,
+        ) {
             peerstate.prefer_encrypt = EncryptPreference::Mutual;
             peerstate.to_save = Some(ToSave::All);
-            peerstate.save_to_db(&context.sql, false).unwrap();
+            peerstate
+                .save_to_db(&context.sql, false)
+                .unwrap_or_default();
             return Ok(());
         }
     }
@@ -693,17 +714,16 @@ fn mark_peer_as_verified(context: &Context, fingerprint: impl AsRef<str>) -> Res
  ******************************************************************************/
 
 fn encrypted_and_signed(mimeparser: &MimeParser, expected_fingerprint: impl AsRef<str>) -> bool {
-    if !mimeparser.e2ee_helper.encrypted {
+    if !mimeparser.encrypted {
         warn!(mimeparser.context, "Message not encrypted.",);
         false
-    } else if mimeparser.e2ee_helper.signatures.len() <= 0 {
+    } else if mimeparser.signatures.is_empty() {
         warn!(mimeparser.context, "Message not signed.",);
         false
     } else if expected_fingerprint.as_ref().is_empty() {
         warn!(mimeparser.context, "Fingerprint for comparison missing.",);
         false
     } else if !mimeparser
-        .e2ee_helper
         .signatures
         .contains(expected_fingerprint.as_ref())
     {
@@ -718,21 +738,24 @@ fn encrypted_and_signed(mimeparser: &MimeParser, expected_fingerprint: impl AsRe
     }
 }
 
-pub fn handle_degrade_event(context: &Context, peerstate: &Peerstate) {
+pub fn handle_degrade_event(context: &Context, peerstate: &Peerstate) -> Result<(), Error> {
     // - we do not issue an warning for DC_DE_ENCRYPTION_PAUSED as this is quite normal
     // - currently, we do not issue an extra warning for DC_DE_VERIFICATION_LOST - this always comes
     //   together with DC_DE_FINGERPRINT_CHANGED which is logged, the idea is not to bother
     //   with things they cannot fix, so the user is just kicked from the verified group
     //   (and he will know this and can fix this)
     if Some(DegradeEvent::FingerprintChanged) == peerstate.degrade_event {
-        let contact_id: i32 = context
-            .sql
-            .query_get_value(
-                context,
-                "SELECT id FROM contacts WHERE addr=?;",
-                params![&peerstate.addr],
-            )
-            .unwrap_or_default();
+        let contact_id: i32 = match context.sql.query_get_value(
+            context,
+            "SELECT id FROM contacts WHERE addr=?;",
+            params![&peerstate.addr],
+        ) {
+            None => bail!(
+                "contact with peerstate.addr {:?} not found",
+                &peerstate.addr
+            ),
+            Some(contact_id) => contact_id,
+        };
         if contact_id > 0 {
             let (contact_chat_id, _) =
                 chat::create_or_lookup_by_contact_id(context, contact_id as u32, Blocked::Deaddrop)
@@ -744,8 +767,9 @@ pub fn handle_degrade_event(context: &Context, peerstate: &Peerstate) {
             };
             let msg = context.stock_string_repl_str(StockMessage::ContactSetupChanged, peeraddr);
 
-            chat::add_device_msg(context, contact_chat_id, msg);
+            chat::add_info_msg(context, contact_chat_id, msg);
             emit_event!(context, Event::ChatModified(contact_chat_id));
         }
     }
+    Ok(())
 }

src/smtp.rs

@@ -1,10 +1,13 @@
 use lettre::smtp::client::net::*;
 use lettre::*;
 
+use failure::Fail;
+
 use crate::constants::*;
 use crate::context::Context;
+use crate::error::Error;
 use crate::events::Event;
-use crate::login_param::LoginParam;
+use crate::login_param::{dc_build_tls_config, LoginParam};
 use crate::oauth2::*;
 
 #[derive(DebugStub)]
@@ -14,7 +17,16 @@ pub struct Smtp {
     transport_connected: bool,
     /// Email address we are sending from.
     from: Option<EmailAddress>,
-    pub error: Option<String>,
+}
+
+#[derive(Debug, Fail)]
+pub enum SmtpError {
+    #[fail(display = "Envelope error: {}", _0)]
+    EnvelopeError(#[cause] lettre::error::Error),
+    #[fail(display = "Send error: {}", _0)]
+    SendError(#[cause] lettre::smtp::error::Error),
+    #[fail(display = "SMTP has no transport")]
+    NoTransport,
 }
 
 impl Smtp {
@@ -24,7 +36,6 @@ impl Smtp {
             transport: None,
             transport_connected: false,
             from: None,
-            error: None,
         }
     }
 
@@ -45,56 +56,59 @@ impl Smtp {
     }
 
     /// Connect using the provided login params
-    pub fn connect(&mut self, context: &Context, lp: &LoginParam) -> bool {
+    pub fn connect(&mut self, context: &Context, lp: &LoginParam) -> Result<(), Error> {
         if self.is_connected() {
             warn!(context, "SMTP already connected.");
-            return true;
+            return Ok(());
         }
 
         if lp.send_server.is_empty() || lp.send_port == 0 {
             context.call_cb(Event::ErrorNetwork("SMTP bad parameters.".into()));
+            bail!("SMTP Bad parameters");
         }
 
-        self.from = if let Ok(addr) = EmailAddress::new(lp.addr.clone()) {
-            Some(addr)
-        } else {
-            None
+        self.from = match EmailAddress::new(lp.addr.clone()) {
+            Ok(addr) => Some(addr),
+            Err(err) => {
+                bail!("invalid login address {}: {}", lp.addr, err);
+            }
         };
 
-        if self.from.is_none() {
-            // TODO: print error
-            return false;
-        }
-
         let domain = &lp.send_server;
         let port = lp.send_port as u16;
 
-        let tls = native_tls::TlsConnector::builder()
-            // see also: https://github.com/deltachat/deltachat-core-rust/issues/203
-            .danger_accept_invalid_hostnames(true)
-            .danger_accept_invalid_certs(true)
-            .min_protocol_version(Some(DEFAULT_TLS_PROTOCOLS[0]))
-            .build()
-            .unwrap();
+        let tls_config = dc_build_tls_config(lp.smtp_certificate_checks);
+        let tls_parameters = ClientTlsParameters::new(domain.to_string(), tls_config);
 
-        let tls_parameters = ClientTlsParameters::new(domain.to_string(), tls);
-
-        let creds = if 0 != lp.server_flags & (DC_LP_AUTH_OAUTH2 as i32) {
+        let (creds, mechanism) = if 0 != lp.server_flags & (DC_LP_AUTH_OAUTH2 as i32) {
             // oauth2
             let addr = &lp.addr;
             let send_pw = &lp.send_pw;
             let access_token = dc_get_oauth2_access_token(context, addr, send_pw, false);
-            if access_token.is_none() {
-                return false;
-            }
+            ensure!(
+                access_token.is_some(),
+                "could not get oaut2_access token addr={}",
+                addr
+            );
             let user = &lp.send_user;
-
-            lettre::smtp::authentication::Credentials::new(user.to_string(), access_token.unwrap())
+            (
+                lettre::smtp::authentication::Credentials::new(
+                    user.to_string(),
+                    access_token.unwrap_or_default(),
+                ),
+                vec![lettre::smtp::authentication::Mechanism::Xoauth2],
+            )
         } else {
             // plain
             let user = lp.send_user.clone();
             let pw = lp.send_pw.clone();
-            lettre::smtp::authentication::Credentials::new(user, pw)
+            (
+                lettre::smtp::authentication::Credentials::new(user, pw),
+                vec![
+                    lettre::smtp::authentication::Mechanism::Plain,
+                    lettre::smtp::authentication::Mechanism::Login,
+                ],
+            )
         };
 
         let security = if 0
@@ -110,52 +124,70 @@ impl Smtp {
                 let client = client
                     .smtp_utf8(true)
                     .credentials(creds)
+                    .authentication_mechanism(mechanism)
                     .connection_reuse(lettre::smtp::ConnectionReuseParameters::ReuseUnlimited);
-                self.transport = Some(client.transport());
-                context.call_cb(Event::SmtpConnected(format!(
-                    "SMTP-LOGIN as {} ok",
-                    lp.send_user,
-                )));
-                true
+                let mut trans = client.transport();
+                match trans.connect() {
+                    Ok(()) => {
+                        self.transport = Some(trans);
+                        self.transport_connected = true;
+                        context.call_cb(Event::SmtpConnected(format!(
+                            "SMTP-LOGIN as {} ok",
+                            lp.send_user,
+                        )));
+                        return Ok(());
+                    }
+                    Err(err) => {
+                        bail!("SMTP: failed to connect {:?}", err);
+                    }
+                }
             }
             Err(err) => {
-                warn!(context, "SMTP: failed to establish connection {:?}", err);
-                false
+                bail!("SMTP: failed to setup connection {:?}", err);
             }
         }
     }
 
-    pub fn send<'a>(
+    /// SMTP-Send a prepared mail to recipients.
+    /// on successful send out Ok() is returned.
+    pub fn send(
         &mut self,
         context: &Context,
         recipients: Vec<EmailAddress>,
-        body: Vec<u8>,
-    ) -> usize {
+        message: Vec<u8>,
+        job_id: u32,
+    ) -> Result<(), SmtpError> {
+        let message_len = message.len();
+
+        let recipients_display = recipients
+            .iter()
+            .map(|x| format!("{}", x))
+            .collect::<Vec<String>>()
+            .join(",");
+
         if let Some(ref mut transport) = self.transport {
-            let envelope = Envelope::new(self.from.clone(), recipients).expect("invalid envelope");
+            let envelope =
+                Envelope::new(self.from.clone(), recipients).map_err(SmtpError::EnvelopeError)?;
             let mail = SendableEmail::new(
                 envelope,
-                "mail-id".into(), // TODO: random id
-                body,
+                format!("{}", job_id), // only used for internal logging
+                message,
             );
 
-            match transport.send(mail) {
-                Ok(_) => {
-                    context.call_cb(Event::SmtpMessageSent(
-                        "Message was sent to SMTP server".into(),
-                    ));
-                    self.transport_connected = true;
-                    1
-                }
-                Err(err) => {
-                    warn!(context, "SMTP failed to send message: {}", err);
-                    self.error = Some(format!("{}", err));
-                    0
-                }
-            }
+            transport.send(mail).map_err(SmtpError::SendError)?;
+
+            context.call_cb(Event::SmtpMessageSent(format!(
+                "Message len={} was smtp-sent to {}",
+                message_len, recipients_display
+            )));
+            self.transport_connected = true;
+            Ok(())
         } else {
-            // TODO: log error
-            0
+            warn!(
+                context,
+                "uh? SMTP has no transport, failed to send to {}", recipients_display
+            );
+            return Err(SmtpError::NoTransport);
         }
     }
 }

src/sql.rs

@@ -1,17 +1,20 @@
+//! # SQLite wrapper
+
 use std::collections::HashSet;
 use std::sync::{Arc, RwLock};
+use std::time::Duration;
 
 use rusqlite::{Connection, OpenFlags, Statement, NO_PARAMS};
 use thread_local_object::ThreadLocal;
 
+use crate::chat::update_saved_messages_icon;
+use crate::constants::ShowEmails;
 use crate::context::Context;
 use crate::dc_tools::*;
 use crate::error::{Error, Result};
 use crate::param::*;
 use crate::peerstate::*;
 
-const DC_OPEN_READONLY: usize = 0x01;
-
 /// A wrapper around the underlying Sqlite3 object.
 #[derive(DebugStub)]
 pub struct Sql {
@@ -41,8 +44,8 @@ impl Sql {
     }
 
     // return true on success, false on failure
-    pub fn open(&self, context: &Context, dbfile: &std::path::Path, flags: libc::c_int) -> bool {
-        match open(context, self, dbfile, flags) {
+    pub fn open(&self, context: &Context, dbfile: &std::path::Path, readonly: bool) -> bool {
+        match open(context, self, dbfile, readonly) {
             Ok(_) => true,
             Err(Error::SqlAlreadyOpen) => false,
             Err(_) => {
@@ -68,6 +71,16 @@ impl Sql {
         let res = match &*self.pool.read().unwrap() {
             Some(pool) => {
                 let conn = pool.get()?;
+
+                // Only one process can make changes to the database at one time.
+                // busy_timeout defines, that if a seconds process wants write access,
+                // this second process will wait some milliseconds
+                // and try over until it gets write access or the given timeout is elapsed.
+                // If the second process does not get write access within the given timeout,
+                // sqlite3_step() will return the error SQLITE_BUSY.
+                // (without a busy_timeout, sqlite3_step() would return SQLITE_BUSY _at once_)
+                conn.busy_timeout(Duration::from_secs(10))?;
+
                 g(&conn)
             }
             None => Err(Error::SqlNoConnection),
@@ -83,8 +96,7 @@ impl Sql {
         self.start_stmt(sql.to_string());
         self.with_conn(|conn| {
             let stmt = conn.prepare(sql)?;
-            let res = g(stmt, conn)?;
-            Ok(res)
+            g(stmt, conn)
         })
     }
 
@@ -97,8 +109,7 @@ impl Sql {
             let stmt1 = conn.prepare(sql1)?;
             let stmt2 = conn.prepare(sql2)?;
 
-            let res = g(stmt1, stmt2, conn)?;
-            Ok(res)
+            g(stmt1, stmt2, conn)
         })
     }
 
@@ -182,14 +193,14 @@ impl Sql {
     ///
     /// Setting `None` deletes the value.  On failure an error message
     /// will already have been logged.
-    pub fn set_config(
+    pub fn set_raw_config(
         &self,
         context: &Context,
         key: impl AsRef<str>,
         value: Option<&str>,
     ) -> Result<()> {
         if !self.is_open() {
-            error!(context, "set_config(): Database not ready.");
+            error!(context, "set_raw_config(): Database not ready.");
             return Err(Error::SqlNoConnection);
         }
 
@@ -223,14 +234,14 @@ impl Sql {
         match res {
             Ok(_) => Ok(()),
             Err(err) => {
-                error!(context, "set_config(): Cannot change value. {:?}", &err);
-                Err(err.into())
+                error!(context, "set_raw_config(): Cannot change value. {:?}", &err);
+                Err(err)
             }
         }
     }
 
     /// Get configuration options from the database.
-    pub fn get_config(&self, context: &Context, key: impl AsRef<str>) -> Option<String> {
+    pub fn get_raw_config(&self, context: &Context, key: impl AsRef<str>) -> Option<String> {
         if !self.is_open() || key.as_ref().is_empty() {
             return None;
         }
@@ -241,44 +252,46 @@ impl Sql {
         )
     }
 
-    pub fn set_config_int(
+    pub fn set_raw_config_int(
         &self,
         context: &Context,
         key: impl AsRef<str>,
         value: i32,
     ) -> Result<()> {
-        self.set_config(context, key, Some(&format!("{}", value)))
+        self.set_raw_config(context, key, Some(&format!("{}", value)))
     }
 
-    pub fn get_config_int(&self, context: &Context, key: impl AsRef<str>) -> Option<i32> {
-        self.get_config(context, key).and_then(|s| s.parse().ok())
+    pub fn get_raw_config_int(&self, context: &Context, key: impl AsRef<str>) -> Option<i32> {
+        self.get_raw_config(context, key)
+            .and_then(|s| s.parse().ok())
     }
 
-    pub fn get_config_bool(&self, context: &Context, key: impl AsRef<str>) -> bool {
+    pub fn get_raw_config_bool(&self, context: &Context, key: impl AsRef<str>) -> bool {
         // Not the most obvious way to encode bool as string, but it is matter
         // of backward compatibility.
-        self.get_config_int(context, key).unwrap_or_default() > 0
+        self.get_raw_config_int(context, key).unwrap_or_default() > 0
     }
 
-    pub fn set_config_bool<T>(&self, context: &Context, key: T, value: bool) -> Result<()>
+    pub fn set_raw_config_bool<T>(&self, context: &Context, key: T, value: bool) -> Result<()>
     where
         T: AsRef<str>,
     {
         let value = if value { Some("1") } else { None };
-        self.set_config(context, key, value)
+        self.set_raw_config(context, key, value)
     }
 
-    pub fn set_config_int64(
+    pub fn set_raw_config_int64(
         &self,
         context: &Context,
         key: impl AsRef<str>,
         value: i64,
     ) -> Result<()> {
-        self.set_config(context, key, Some(&format!("{}", value)))
+        self.set_raw_config(context, key, Some(&format!("{}", value)))
     }
 
-    pub fn get_config_int64(&self, context: &Context, key: impl AsRef<str>) -> Option<i64> {
-        self.get_config(context, key).and_then(|r| r.parse().ok())
+    pub fn get_raw_config_int64(&self, context: &Context, key: impl AsRef<str>) -> Option<i64> {
+        self.get_raw_config(context, key)
+            .and_then(|r| r.parse().ok())
     }
 
     fn start_stmt(&self, stmt: impl AsRef<str>) {
@@ -307,7 +320,7 @@ fn open(
     context: &Context,
     sql: &Sql,
     dbfile: impl AsRef<std::path::Path>,
-    flags: libc::c_int,
+    readonly: bool,
 ) -> Result<()> {
     if sql.is_open() {
         error!(
@@ -319,7 +332,7 @@ fn open(
     }
 
     let mut open_flags = OpenFlags::SQLITE_OPEN_NO_MUTEX;
-    if 0 != (flags & DC_OPEN_READONLY as i32) {
+    if readonly {
         open_flags.insert(OpenFlags::SQLITE_OPEN_READ_ONLY);
     } else {
         open_flags.insert(OpenFlags::SQLITE_OPEN_READ_WRITE);
@@ -330,7 +343,7 @@ fn open(
         .with_init(|c| c.execute_batch("PRAGMA secure_delete=on;"));
     let pool = r2d2::Pool::builder()
         .min_idle(Some(2))
-        .max_size(4)
+        .max_size(10)
         .connection_timeout(std::time::Duration::new(60, 0))
         .build(mgr)?;
 
@@ -338,8 +351,8 @@ fn open(
         *sql.pool.write().unwrap() = Some(pool);
     }
 
-    if 0 == flags & DC_OPEN_READONLY as i32 {
-        let mut exists_before_update = 0;
+    if !readonly {
+        let mut exists_before_update = false;
         let mut dbversion_before_update = 0;
         /* Init tables to dbversion=0 */
         if !sql.table_exists("config") {
@@ -374,8 +387,8 @@ fn open(
             )?;
             sql.execute(
                 "INSERT INTO contacts (id,name,origin) VALUES \
-                 (1,'self',262144), (2,'device',262144), (3,'rsvd',262144), \
-                 (4,'rsvd',262144), (5,'rsvd',262144), (6,'rsvd',262144), \
+                 (1,'self',262144), (2,'info',262144), (3,'rsvd',262144), \
+                 (4,'rsvd',262144), (5,'device',262144), (6,'rsvd',262144), \
                  (7,'rsvd',262144), (8,'rsvd',262144), (9,'rsvd',262144);",
                 params![],
             )?;
@@ -466,22 +479,27 @@ fn open(
                 // cannot create the tables - maybe we cannot write?
                 return Err(Error::SqlFailedToOpen);
             } else {
-                sql.set_config_int(context, "dbversion", 0)?;
+                sql.set_raw_config_int(context, "dbversion", 0)?;
             }
         } else {
-            exists_before_update = 1;
-            dbversion_before_update = sql.get_config_int(context, "dbversion").unwrap_or_default();
+            exists_before_update = true;
+            dbversion_before_update = sql
+                .get_raw_config_int(context, "dbversion")
+                .unwrap_or_default();
         }
 
         // (1) update low-level database structure.
         // this should be done before updates that use high-level objects that
         // rely themselves on the low-level structure.
         // --------------------------------------------------------------------
+
         let mut dbversion = dbversion_before_update;
         let mut recalc_fingerprints = 0;
         let mut update_file_paths = 0;
+        let mut update_icons = false;
 
         if dbversion < 1 {
+            info!(context, "[migration] v1");
             sql.execute(
                 "CREATE TABLE leftgrps ( id INTEGER PRIMARY KEY, grpid TEXT DEFAULT '');",
                 params![],
@@ -491,17 +509,19 @@ fn open(
                 params![],
             )?;
             dbversion = 1;
-            sql.set_config_int(context, "dbversion", 1)?;
+            sql.set_raw_config_int(context, "dbversion", 1)?;
         }
         if dbversion < 2 {
+            info!(context, "[migration] v2");
             sql.execute(
                 "ALTER TABLE contacts ADD COLUMN authname TEXT DEFAULT '';",
                 params![],
             )?;
             dbversion = 2;
-            sql.set_config_int(context, "dbversion", 2)?;
+            sql.set_raw_config_int(context, "dbversion", 2)?;
         }
         if dbversion < 7 {
+            info!(context, "[migration] v7");
             sql.execute(
                 "CREATE TABLE keypairs (\
                  id INTEGER PRIMARY KEY, \
@@ -513,9 +533,10 @@ fn open(
                 params![],
             )?;
             dbversion = 7;
-            sql.set_config_int(context, "dbversion", 7)?;
+            sql.set_raw_config_int(context, "dbversion", 7)?;
         }
         if dbversion < 10 {
+            info!(context, "[migration] v10");
             sql.execute(
                 "CREATE TABLE acpeerstates (\
                  id INTEGER PRIMARY KEY, \
@@ -531,9 +552,10 @@ fn open(
                 params![],
             )?;
             dbversion = 10;
-            sql.set_config_int(context, "dbversion", 10)?;
+            sql.set_raw_config_int(context, "dbversion", 10)?;
         }
         if dbversion < 12 {
+            info!(context, "[migration] v12");
             sql.execute(
                 "CREATE TABLE msgs_mdns ( msg_id INTEGER,  contact_id INTEGER);",
                 params![],
@@ -543,9 +565,10 @@ fn open(
                 params![],
             )?;
             dbversion = 12;
-            sql.set_config_int(context, "dbversion", 12)?;
+            sql.set_raw_config_int(context, "dbversion", 12)?;
         }
         if dbversion < 17 {
+            info!(context, "[migration] v17");
             sql.execute(
                 "ALTER TABLE chats ADD COLUMN archived INTEGER DEFAULT 0;",
                 params![],
@@ -557,18 +580,22 @@ fn open(
             )?;
             sql.execute("CREATE INDEX msgs_index5 ON msgs (starred);", params![])?;
             dbversion = 17;
-            sql.set_config_int(context, "dbversion", 17)?;
+            sql.set_raw_config_int(context, "dbversion", 17)?;
         }
         if dbversion < 18 {
+            info!(context, "[migration] v18");
             sql.execute(
                 "ALTER TABLE acpeerstates ADD COLUMN gossip_timestamp INTEGER DEFAULT 0;",
                 params![],
             )?;
             sql.execute("ALTER TABLE acpeerstates ADD COLUMN gossip_key;", params![])?;
             dbversion = 18;
-            sql.set_config_int(context, "dbversion", 18)?;
+            sql.set_raw_config_int(context, "dbversion", 18)?;
         }
         if dbversion < 27 {
+            info!(context, "[migration] v27");
+            // chat.id=1 and chat.id=2 are the old deaddrops,
+            // the current ones are defined by chats.blocked=2
             sql.execute("DELETE FROM msgs WHERE chat_id=1 OR chat_id=2;", params![])?;
             sql.execute(
                 "CREATE INDEX chats_contacts_index2 ON chats_contacts (contact_id);",
@@ -583,9 +610,10 @@ fn open(
                 params![],
             )?;
             dbversion = 27;
-            sql.set_config_int(context, "dbversion", 27)?;
+            sql.set_raw_config_int(context, "dbversion", 27)?;
         }
         if dbversion < 34 {
+            info!(context, "[migration] v34");
             sql.execute(
                 "ALTER TABLE msgs ADD COLUMN hidden INTEGER DEFAULT 0;",
                 params![],
@@ -612,9 +640,10 @@ fn open(
             )?;
             recalc_fingerprints = 1;
             dbversion = 34;
-            sql.set_config_int(context, "dbversion", 34)?;
+            sql.set_raw_config_int(context, "dbversion", 34)?;
         }
         if dbversion < 39 {
+            info!(context, "[migration] v39");
             sql.execute(
                 "CREATE TABLE tokens ( id INTEGER PRIMARY KEY, namespc INTEGER DEFAULT 0, foreign_id INTEGER DEFAULT 0, token TEXT DEFAULT '', timestamp INTEGER DEFAULT 0);",
                 params![]
@@ -632,6 +661,9 @@ fn open(
                 params![],
             )?;
             if dbversion_before_update == 34 {
+                // migrate database from the use of verified-flags to verified_key,
+                // _only_ version 34 (0.17.0) has the fields public_key_verified and gossip_key_verified
+                // this block can be deleted in half a year or so (created 5/2018)
                 sql.execute(
                     "UPDATE acpeerstates SET verified_key=gossip_key, verified_key_fingerprint=gossip_key_fingerprint WHERE gossip_key_verified=2;",
                     params![]
@@ -642,32 +674,39 @@ fn open(
                 )?;
             }
             dbversion = 39;
-            sql.set_config_int(context, "dbversion", 39)?;
+            sql.set_raw_config_int(context, "dbversion", 39)?;
         }
         if dbversion < 40 {
+            info!(context, "[migration] v40");
             sql.execute(
                 "ALTER TABLE jobs ADD COLUMN thread INTEGER DEFAULT 0;",
                 params![],
             )?;
             dbversion = 40;
-            sql.set_config_int(context, "dbversion", 40)?;
+            sql.set_raw_config_int(context, "dbversion", 40)?;
         }
         if dbversion < 41 {
+            info!(context, "[migration] v41");
             update_file_paths = 1;
             dbversion = 41;
-            sql.set_config_int(context, "dbversion", 41)?;
+            sql.set_raw_config_int(context, "dbversion", 41)?;
         }
         if dbversion < 42 {
+            info!(context, "[migration] v42");
+            // older versions set the txt-field to the filenames, for debugging and fulltext search.
+            // to allow text+attachment compound messages, we need to reset these fields.
             sql.execute("UPDATE msgs SET txt='' WHERE type!=10", params![])?;
             dbversion = 42;
-            sql.set_config_int(context, "dbversion", 42)?;
+            sql.set_raw_config_int(context, "dbversion", 42)?;
         }
         if dbversion < 44 {
+            info!(context, "[migration] v44");
             sql.execute("ALTER TABLE msgs ADD COLUMN mime_headers TEXT;", params![])?;
             dbversion = 44;
-            sql.set_config_int(context, "dbversion", 44)?;
+            sql.set_raw_config_int(context, "dbversion", 44)?;
         }
         if dbversion < 46 {
+            info!(context, "[migration] v46");
             sql.execute(
                 "ALTER TABLE msgs ADD COLUMN mime_in_reply_to TEXT;",
                 params![],
@@ -677,7 +716,7 @@ fn open(
                 params![],
             )?;
             dbversion = 46;
-            sql.set_config_int(context, "dbversion", 46)?;
+            sql.set_raw_config_int(context, "dbversion", 46)?;
         }
         if dbversion < 47 {
             info!(context, "[migration] v47");
@@ -686,17 +725,18 @@ fn open(
                 params![],
             )?;
             dbversion = 47;
-            sql.set_config_int(context, "dbversion", 47)?;
+            sql.set_raw_config_int(context, "dbversion", 47)?;
         }
         if dbversion < 48 {
             info!(context, "[migration] v48");
+            // NOTE: move_state is not used anymore
             sql.execute(
                 "ALTER TABLE msgs ADD COLUMN move_state INTEGER DEFAULT 1;",
                 params![],
             )?;
 
             dbversion = 48;
-            sql.set_config_int(context, "dbversion", 48)?;
+            sql.set_raw_config_int(context, "dbversion", 48)?;
         }
         if dbversion < 49 {
             info!(context, "[migration] v49");
@@ -705,18 +745,23 @@ fn open(
                 params![],
             )?;
             dbversion = 49;
-            sql.set_config_int(context, "dbversion", 49)?;
+            sql.set_raw_config_int(context, "dbversion", 49)?;
         }
         if dbversion < 50 {
             info!(context, "[migration] v50");
-            if 0 != exists_before_update {
-                sql.set_config_int(context, "show_emails", 2)?;
+            // installations <= 0.100.1 used DC_SHOW_EMAILS_ALL implicitly;
+            // keep this default and use DC_SHOW_EMAILS_NO
+            // only for new installations
+            if exists_before_update {
+                sql.set_raw_config_int(context, "show_emails", ShowEmails::All as i32)?;
             }
             dbversion = 50;
-            sql.set_config_int(context, "dbversion", 50)?;
+            sql.set_raw_config_int(context, "dbversion", 50)?;
         }
         if dbversion < 53 {
             info!(context, "[migration] v53");
+            // the messages containing _only_ locations
+            // are also added to the database as _hidden_.
             sql.execute(
                 "CREATE TABLE locations ( id INTEGER PRIMARY KEY AUTOINCREMENT, latitude REAL DEFAULT 0.0, longitude REAL DEFAULT 0.0, accuracy REAL DEFAULT 0.0, timestamp INTEGER DEFAULT 0, chat_id INTEGER DEFAULT 0, from_id INTEGER DEFAULT 0);",
                 params![]
@@ -746,7 +791,7 @@ fn open(
                 params![],
             )?;
             dbversion = 53;
-            sql.set_config_int(context, "dbversion", 53)?;
+            sql.set_raw_config_int(context, "dbversion", 53)?;
         }
         if dbversion < 54 {
             info!(context, "[migration] v54");
@@ -756,18 +801,49 @@ fn open(
             )?;
             sql.execute("CREATE INDEX msgs_index6 ON msgs (location_id);", params![])?;
             dbversion = 54;
-            sql.set_config_int(context, "dbversion", 54)?;
+            sql.set_raw_config_int(context, "dbversion", 54)?;
         }
         if dbversion < 55 {
+            info!(context, "[migration] v55");
             sql.execute(
                 "ALTER TABLE locations ADD COLUMN independent INTEGER DEFAULT 0;",
                 params![],
             )?;
-
-            sql.set_config_int(context, "dbversion", 55)?;
+            sql.set_raw_config_int(context, "dbversion", 55)?;
+        }
+        if dbversion < 59 {
+            info!(context, "[migration] v59");
+            // records in the devmsglabels are kept when the message is deleted.
+            // so, msg_id may or may not exist.
+            sql.execute(
+                "CREATE TABLE devmsglabels (id INTEGER PRIMARY KEY AUTOINCREMENT, label TEXT, msg_id INTEGER DEFAULT 0);",
+                NO_PARAMS,
+            )?;
+            sql.execute(
+                "CREATE INDEX devmsglabels_index1 ON devmsglabels (label);",
+                NO_PARAMS,
+            )?;
+            if exists_before_update && sql.get_raw_config_int(context, "bcc_self").is_none() {
+                sql.set_raw_config_int(context, "bcc_self", 1)?;
+            }
+            update_icons = true;
+            sql.set_raw_config_int(context, "dbversion", 59)?;
+        }
+        if dbversion < 60 {
+            info!(context, "[migration] v60");
+            sql.execute(
+                "ALTER TABLE chats ADD COLUMN created_timestamp INTEGER DEFAULT 0;",
+                NO_PARAMS,
+            )?;
+            sql.set_raw_config_int(context, "dbversion", 60)?;
         }
 
+        // (2) updates that require high-level objects
+        // (the structure is complete now and all objects are usable)
+        // --------------------------------------------------------------------
+
         if 0 != recalc_fingerprints {
+            info!(context, "[migration] recalc fingerprints");
             sql.query_map(
                 "SELECT addr FROM acpeerstates;",
                 params![],
@@ -777,7 +853,7 @@ fn open(
                         if let Some(ref mut peerstate) = Peerstate::from_addr(context, sql, &addr?)
                         {
                             peerstate.recalc_fingerprint();
-                            peerstate.save_to_db(sql, false).unwrap();
+                            peerstate.save_to_db(sql, false)?;
                         }
                     }
                     Ok(())
@@ -788,11 +864,9 @@ fn open(
             // versions before 2018-08 save the absolute paths in the database files at "param.f=";
             // for newer versions, we copy files always to the blob directory and store relative paths.
             // this snippet converts older databases and can be removed after some time.
-
-            info!(context, "[open] update file paths");
-
+            info!(context, "[migration] update file paths");
             let repl_from = sql
-                .get_config(context, "backup_for")
+                .get_raw_config(context, "backup_for")
                 .unwrap_or_else(|| context.get_blobdir().to_string_lossy().into());
 
             let repl_from = dc_ensure_no_slash_safe(&repl_from);
@@ -812,7 +886,10 @@ fn open(
                 NO_PARAMS,
             )?;
 
-            sql.set_config(context, "backup_for", None)?;
+            sql.set_raw_config(context, "backup_for", None)?;
+        }
+        if update_icons {
+            update_saved_messages_icon(context)?;
         }
     }
 
@@ -1133,12 +1210,8 @@ mod test {
         maybe_add_file(&mut files, "$BLOBDIR/world.txt");
         maybe_add_file(&mut files, "world2.txt");
 
-        assert!(is_file_in_use(&mut files, None, "hello"));
-        assert!(!is_file_in_use(&mut files, Some(".txt"), "hello"));
-        assert!(is_file_in_use(
-            &mut files,
-            Some("-suffix"),
-            "world.txt-suffix"
-        ));
+        assert!(is_file_in_use(&files, None, "hello"));
+        assert!(!is_file_in_use(&files, Some(".txt"), "hello"));
+        assert!(is_file_in_use(&files, Some("-suffix"), "world.txt-suffix"));
     }
 }