Add some information about mailing list managers.

Remove additional chat type MailingList, make them GroupLists with a special param, make can_send() return false, code style and deduplication

.circleci/config.yml

@@ -4,6 +4,9 @@ executors:
     docker:
       - image: filecoin/rust:latest
     working_directory: /mnt/crate
+  doxygen:
+    docker:
+      - image: hrektts/doxygen
 
 restore-workspace: &restore-workspace
   attach_workspace:
@@ -12,7 +15,7 @@ restore-workspace: &restore-workspace
 restore-cache: &restore-cache
   restore_cache:
     keys:
-      - cargo-v1-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+      - cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
       - repo-source-{{ .Branch }}-{{ .Revision }}
 
 commands:
@@ -23,20 +26,9 @@ commands:
     steps:
       - *restore-workspace
       - *restore-cache
-      - setup_remote_docker:
-          docker_layer_caching: true
-      # TODO: move into image
-      - run:
-          name: Install Docker client
-          command: |
-            set -x
-            VER="18.09.2"
-            curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$VER.tgz
-            tar -xz -C /tmp -f /tmp/docker-$VER.tgz
-            mv /tmp/docker/* /usr/bin
       - run:
           name: Test (<< parameters.target >>)
-          command: TARGET=<< parameters.target >> ci/run.sh
+          command: TARGET=<< parameters.target >> ci_scripts/run-rust-test.sh
           no_output_timeout: 15m
 
 jobs:
@@ -44,30 +36,24 @@ jobs:
     executor: default
     steps:
       - checkout
-      - run:
-          name: Update submodules
-          command: git submodule update --init --recursive
-      - run:
-          name: Calculate dependencies
-          command: cargo generate-lockfile
       - restore_cache:
           keys:
-            - cargo-v1-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+            - cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
       - run: rustup install $(cat rust-toolchain)
       - run: rustup default $(cat rust-toolchain)
       - run: rustup component add --toolchain $(cat rust-toolchain) rustfmt
       - run: rustup component add --toolchain $(cat rust-toolchain) clippy-preview
-      - run: cargo update
       - run: cargo fetch
       - run: rustc +stable --version
       - run: rustc +$(cat rust-toolchain) --version
-      - run: rm -rf .git
+      # make sure this git repo doesn't grow too big
+      - run: git gc
       - persist_to_workspace:
           root: /mnt
           paths:
             - crate
       - save_cache:
-          key: cargo-v1-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
+          key: cargo-v3-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }}-{{ arch }}
           paths:
             - "~/.cargo"
             - "~/.rustup"
@@ -98,11 +84,10 @@ jobs:
             curl https://sh.rustup.rs -sSf | sh -s -- -y
       - run: rustup install $(cat rust-toolchain)
       - run: rustup default $(cat rust-toolchain)
-      - run: cargo update
       - run: cargo fetch
       - run:
           name: Test
-          command: TARGET=x86_64-apple-darwin ci/run.sh
+          command: TARGET=x86_64-apple-darwin ci_scripts/run-rust-test.sh
 
   test_x86_64-unknown-linux-gnu:
     executor: default
@@ -123,27 +108,25 @@ jobs:
           target: "aarch64-linux-android"
 
 
-  build_test_docs_wheel:
-    machine: True
+  build_doxygen:
+    executor: doxygen
     steps:
       - checkout
-      # - run: docker pull deltachat/doxygen
-      - run: docker pull deltachat/coredeps
-      - run:
-          name: build docs, run tests and build wheels
-          command: ci_scripts/ci_run.sh
-          environment:
-            TESTS: 1
-            DOCS: 1
-
-      - run:
-           name: copying docs and wheels to workspace
-           command: |
-             mkdir -p workspace/python
-             # cp -av docs workspace/c-docs
-             cp -av python/.docker-tox/wheelhouse workspace/
-             cp -av python/doc/_build/ workspace/py-docs
+      - run: bash ci_scripts/run-doxygen.sh
+      - run: mkdir -p workspace/c-docs
+      - run: cp -av deltachat-ffi/{html,xml} workspace/c-docs/
+      - persist_to_workspace:
+          root: workspace
+          paths:
+            - c-docs
 
+  remote_python_packaging:
+    machine: true
+    steps:
+      - checkout
+      # the following commands on success produces
+      # workspace/{wheelhouse,py-docs} as artefact directories
+      - run: bash ci_scripts/remote_python_packaging.sh 
       - persist_to_workspace:
           root: workspace
           paths:
@@ -151,15 +134,28 @@ jobs:
             - py-docs
             - wheelhouse
 
+  remote_tests_rust:
+    machine: true
+    steps:
+      - checkout
+      - run: ci_scripts/remote_tests_rust.sh
+    
+  remote_tests_python:
+    machine: true
+    steps:
+      - checkout
+      - run: ci_scripts/remote_tests_python.sh
+
   upload_docs_wheels:
     machine: true
     steps:
       - checkout
       - attach_workspace:
           at: workspace
+      - run: pyenv versions
       - run: pyenv global 3.5.2
       - run: ls -laR workspace
-      - run: ci_scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse
+      - run: ci_scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
 
   clippy:
     executor: default
@@ -168,7 +164,7 @@ jobs:
       - *restore-cache
       - run:
           name: Run cargo clippy
-          command: cargo clippy --all
+          command: cargo clippy 
 
 
 workflows:
@@ -176,23 +172,49 @@ workflows:
 
   test:
     jobs:
-      - build_test_docs_wheel
+      # - cargo_fetch
+
+      - remote_tests_rust:
+          filters:
+            tags:
+              only: /.*/
+
+      - remote_tests_python:
+          filters:
+            tags:
+              only: /.*/
+
+      - remote_python_packaging:
+          requires: 
+             - remote_tests_python
+             - remote_tests_rust 
+          filters:
+            tags:
+              only: /.*/
+
       - upload_docs_wheels:
           requires:
-            - build_test_docs_wheel
-      - cargo_fetch
-      - rustfmt:
-          requires:
-            - cargo_fetch
+             - remote_python_packaging
+             - build_doxygen
+          filters:
+            tags:
+              only: /.*/
+      # - rustfmt:
+      #     requires:
+      #       - cargo_fetch
+      # - clippy:
+      #     requires:
+      #      - cargo_fetch
 
-      - clippy:
-          requires:
-            - cargo_fetch
+      - build_doxygen:
+          filters:
+            tags:
+              only: /.*/
 
       # Linux Desktop 64bit
-      - test_x86_64-unknown-linux-gnu:
-          requires:
-            - cargo_fetch
+      # - test_x86_64-unknown-linux-gnu:
+      #     requires:
+      #       - cargo_fetch
 
       # Linux Desktop 32bit
       # - test_i686-unknown-linux-gnu:

.gitattributes

@@ -2,6 +2,10 @@
 # ensures this even if the user has not set core.autocrlf.
 *  text=auto
 
+# This directory contains email messages verbatim, and changing CRLF to
+# LF will corrupt them.
+test-data/* text=false
+
 # binary files should be detected by git, however, to be sure, you can add them here explicitly
 *.png  binary
 *.jpg  binary

.github/workflows/code-quality.yml

@@ -0,0 +1,48 @@
+on: push
+name: Code Quality
+jobs:
+
+  check:
+    name: Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly-2020-03-12
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          command: check
+          args: --workspace --examples --tests --all-features
+
+  fmt:
+    name: Rustfmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly-2020-03-12
+          override: true
+      - run: rustup component add rustfmt
+      - uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
+  run_clippy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+            toolchain: nightly-2020-03-12
+            components: clippy
+            override: true
+      - uses: actions-rs/clippy-check@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          args: --all-features

.gitignore

@@ -16,9 +16,12 @@ python/.tox
 *.egg-info
 __pycache__
 python/src/deltachat/capi*.so
+python/.venv/
 
 python/liveconfig*
 
 # ignore doxgen generated files
 deltachat-ffi/html
 deltachat-ffi/xml
+
+.rsynclist

CHANGELOG.md

@@ -0,0 +1,424 @@
+# Changelog 
+
+## 1.28.0
+
+- new flag DC_GCL_FOR_FORWARDING for dc_get_chatlist()
+  that will sort the "saved messages" chat to the top of the chatlist #1336
+- mark mails as being deleted from server in dc_empty_server() #1333
+- fix interaction with servers that do not allow folder creation on root-level;
+  use path separator as defined by the email server #1359
+- fix group creation if group was created by non-delta clients #1357
+- fix showing replies from non-delta clients #1353
+- fix member list on rejoining left groups #1343
+- fix crash when using empty groups #1354
+- fix potential crash on special names #1350
+
+
+## 1.27.0
+
+- handle keys reliably on armv7 #1327
+
+
+## 1.26.0
+
+- change generated key type back to RSA as shipped versions
+  have problems to encrypt to Ed25519 keys
+
+- update rPGP to encrypt reliably to Ed25519 keys;
+  one of the next versions can finally use Ed25519 keys then
+
+
+## 1.25.0
+
+- save traffic by downloading only messages that are really displayed #1236
+
+- change generated key type to Ed25519, these keys are much shorter
+  than RSA keys, which results in saving traffic and speed improvements #1287
+
+- improve key handling #1237 #1240 #1242 #1247
+
+- mute handling, apis are dc_set_chat_mute_duration()
+  dc_chat_is_muted() and dc_chat_get_remaining_mute_duration() #1143
+
+- pinning chats, new apis are dc_set_chat_visibility() and
+  dc_chat_get_visibility() #1248
+
+- add dc_provider_new_from_email() api that queries the new, integrated
+  provider-database #1207
+
+- account creation by scanning a qr code
+  in the DCACCOUNT scheme (https://mailadm.readthedocs.io),
+  new api is dc_set_config_from_qr() #1249
+
+- if possible, dc_join_securejoin(), returns the new chat-id immediately
+  and does the handshake in background #1225
+
+- update imap and smtp dependencies #1115
+
+- check for MOVE capability before using MOVE command #1263
+
+- allow inline attachments from RFC 2183 #1280
+
+- fix updating names from incoming mails #1298
+
+- fix error messages shown on import #1234
+
+- directly attempt to re-connect if the smtp connection is maybe stale #1296
+
+- improve adding group members #1291
+
+- improve rust-api #1261
+
+- cleanup #1302 #1283 #1282 #1276 #1270-#1274 #1267 #1258-#1260
+  #1257 #1239 #1231 #1224
+
+- update spec #1286 #1291
+
+
+## 1.0.0-beta.24
+
+- fix oauth2/gmail bug introduced in beta23 (not used in releases) #1219
+
+- fix panic when receiving eg. cyrillic filenames #1216
+
+- delete all consumed secure-join handshake messagess #1209 #1212
+
+- rust-level cleanups #1218 #1217 #1210 #1205
+
+- python-level cleanups #1204 #1202 #1201
+
+
+## 1.0.0-beta.23
+
+- #1197 fix imap-deletion of messages 
+
+- #1171 Combine multiple MDNs into a single mail, reducing traffic 
+
+- #1155 fix to not send out gossip always, reducing traffic
+
+- #1160 fix reply-to-encrypted determination 
+
+- #1182 Add "Auto-Submitted: auto-replied" header to MDNs
+
+- #1194 produce python wheels again, fix c/py.delta.chat
+  master-deployment 
+
+- rust-level housekeeping and improvements #1161 #1186 #1185 #1190 #1194 #1199 #1191 #1190 #1184 and more
+
+- #1063 clarify licensing 
+
+- #1147 use mailparse 0.10.2 
+
+
+## 1.0.0-beta.22
+
+- #1095 normalize email lineends to CRLF
+
+- #1095 enable link-time-optimization, saves eg. on android 11 mb
+
+- #1099 fix import regarding devicechats
+
+- #1092 improve logging
+
+- #1096 #1097 #1094 #1090 #1091 internal cleanups
+
+## 1.0.0-beta.21
+
+- #1078 #1082 ensure RFC compliance by producing 78 column lines for
+  encoded attachments. 
+
+- #1080 don't recreate and thus break group membership if an unknown 
+  sender (or mailer-daemon) sends a message referencing the group chat 
+
+- #1081 #1079 some internal cleanups 
+
+- update imap-proto dependency, to fix yandex/oauth 
+
+## 1.0.0-beta.20
+
+- #1074 fix OAUTH2/gmail
+- #1072 fix group members not appearing in contact list
+- #1071 never block interrupt_idle (thus hopefully also not on maybe_network())
+- #1069 reduce smtp-timeout to 30 seconds
+- #1066 #1065 avoid unwrap in dehtml, make literals more readable
+
+## 1.0.0-beta.19
+
+- #1058 timeout smtp-send if it doesn't complete in 15 minutes 
+
+- #1059 trim down logging
+
+## 1.0.0-beta.18
+
+- #1056 avoid panicking when we couldn't read imap-server's greeting
+  message 
+
+- #1055 avoid panicking when we don't have a selected folder
+
+- #1052 #1049 #1051 improve logging to add thread-id/name and
+  file/lineno to each info/warn message.
+
+- #1050 allow python bindings to initialize Account with "os_name".
+
+
+## 1.0.0-beta.17
+
+- #1044 implement avatar recoding to 192x192 in core to keep file sizes small. 
+
+- #1024 fix #1021 SQL/injection malformed Chat-Group-Name breakage
+
+- #1036 fix smtp crash by pulling in a fixed async-smtp 
+
+- #1039 fix read-receipts appearing as normal messages when you change
+  MDN settings 
+
+- #1040 do not panic on SystemTimeDifference
+
+- #1043 avoid potential crashes in malformed From/Chat-Disposition... headers  
+
+- #1045 #1041 #1038 #1035 #1034 #1029 #1025 various cleanups and doc
+  improvments 
+
+## 1.0.0-beta.16
+
+- alleviate login problems with providers which only
+  support RSA1024 keys by switching back from Rustls 
+  to native-tls, by using the new async-email/async-native-tls 
+  crate from @dignifiedquire. thanks @link2xt. 
+
+- introduce per-contact profile images to send out 
+  own profile image heuristically, and fix sending
+  out of profile images in "in-prepare" groups. 
+  this also extends the Chat-spec that is maintained
+  in core to specify Chat-Group-Image and Chat-Group-Avatar
+  headers. thanks @r10s and @hpk42.
+
+- fix merging of protected headers from the encrypted
+  to the unencrypted parts, now not happening recursively
+  anymore.  thanks @hpk and @r10s
+
+- fix/optimize autocrypt gossip headers to only get 
+  sent when there are more than 2 people in a chat. 
+  thanks @link2xt
+
+- fix displayname to use the authenticated name 
+  when available (displayname as coming from contacts 
+  themselves). thanks @simon-laux
+
+- introduce preliminary support for offline autoconfig 
+  for nauta provider. thanks @hpk42 @r10s
+
+## 1.0.0-beta.15
+
+- fix #994 attachment appeared doubled in chats (and where actually
+  downloaded after smtp-send). @hpk42
+
+## 1.0.0-beta.14
+
+- fix packaging issue with our rust-email fork, now we are tracking
+  master again there. hpk42
+
+## 1.0.0-beta.13
+
+- fix #976 -- unicode-issues in display-name of email addresses. @hpk42
+
+- fix #985 group add/remove member bugs resulting in broken groups.  @hpk42
+
+- fix hanging IMAP connections -- we now detect with a 15second timeout
+  if we cannot terminate the IDLE IMAP protocol. @hpk42 @link2xt
+
+- fix incoming multipart/mixed containing html, to show up as
+  attachments again.  Fixes usage for simplebot which sends html
+  files for users to interact with the bot. @adbenitez @hpk42 
+
+- refinements to internal autocrypt-handling code, do not send
+  prefer-encrypt=nopreference as it is the default if no attribute
+  is present.  @linkxt 
+
+- simplify, modularize and rustify several parts 
+  of dc-core (general WIP). @link2xt @flub @hpk42 @r10s
+
+- use async-email/async-smtp to handle SMTP connections, might
+  fix connection/reconnection issues. @link2xt 
+
+- more tests and refinements for dealing with blobstorage @flub @hpk42 
+
+- use a dedicated build-server for CI testing of core PRs
+
+
+## 1.0.0-beta.12
+
+- fix python bindings to use core for copying attachments to blobdir
+  and fix core to actually do it. @hpk42
+
+## 1.0.0-beta.11
+
+- trigger reconnect more often on imap error states.  Should fix an 
+  issue observed when trying to empty a folder.  @hpk42
+
+- un-split qr tests: we fixed qr-securejoin protocol flakyness 
+  last weeks. @hpk42
+
+## 1.0.0-beta.10
+
+- fix grpid-determination from in-reply-to and references headers. @hpk42
+
+- only send Autocrypt-gossip headers on encrypted messages. @dignifiedquire
+
+- fix reply-to-encrypted message to also be encrypted. @hpk42
+
+- remove last unsafe code from dc_receive_imf :) @hpk42
+
+- add experimental new dc_chat_get_info_json FFI/API so that desktop devs
+  can play with using it. @jikstra
+
+- fix encoding of subjects and attachment-filenames @hpk42
+  @dignifiedquire . 
+
+## 1.0.0-beta.9
+
+- historic: we now use the mailparse crate and lettre-email to generate mime
+  messages.  This got rid of mmime completely, the C2rust generated port of the libetpan 
+  mime-parse -- IOW 22KLocs of cumbersome code removed! see 
+  https://github.com/deltachat/deltachat-core-rust/pull/904#issuecomment-561163330
+  many thanks @dignifiedquire for making everybody's life easier 
+  and @jonhoo (from rust-imap fame) for suggesting to use the mailparse crate :) 
+
+- lots of improvements and better error handling in many rust modules 
+  thanks @link2xt @flub @r10s, @hpk42 and @dignifiedquire 
+
+- @r10s introduced a new device chat which has an initial
+  welcome message.  See 
+  https://c.delta.chat/classdc__context__t.html#a1a2aad98bd23c1d21ee42374e241f389
+  for the main new FFI-API.
+
+- fix moving self-sent messages, thanks @r10s, @flub, @hpk42
+
+- fix flakyness/sometimes-failing verified/join-protocols, 
+  thanks @flub, @r10s, @hpk42
+
+- fix reply-to-encrypted message to keep encryption 
+
+- new DC_EVENT_SECUREJOIN_MEMBER_ADDED event 
+
+- many little fixes and rustifications (@link2xt, @flub, @hpk42)
+
+
+## 1.0.0-beta.8
+
+- now uses async-email/async-imap as the new base 
+  which makes imap-idle interruptible and thus fixes
+  several issues around the imap thread being in zombie state . 
+  thanks @dignifiedquire, @hpk42 and @link2xt. 
+
+- fixes imap-protocol parsing bugs that lead to infinitely
+  repeated crashing while trying to receive messages with
+  a subjec that contained non-utf8. thanks @link2xt
+
+- fixed logic to find encryption subkey -- previously 
+  delta chat would use the primary key for encryption
+  (which works with RSA but not ECC). thanks @link2xt
+
+- introduce a new device chat where core and UIs can 
+  add "device" messages.  Android uses it for an initial
+  welcome message. thanks @r10s
+
+- fix time smearing (when two message are virtually send
+  in the same second, there would be misbehaviour because
+  we didn't persist smeared time). thanks @r10s
+
+- fix double-dotted extensions like .html.zip or .tar.gz  
+  to not mangle them when creating blobfiles.  thanks @flub
+
+- fix backup/exports where the wrong sql file would be modified,
+  leading to problems when exporting twice.  thanks @hpk42
+
+- several other little fixes and improvements 
+
+
+## 1.0.0-beta.7
+
+- fix location-streaming #782
+
+- fix display of messages that could not be decrypted #785
+ 
+- fix smtp MAILER-DAEMON bug #786 
+
+- fix a logging of durations #783
+
+- add more error logging #779
+
+- do not panic on some bad utf-8 mime #776
+
+## 1.0.0-beta.6
+
+- fix chatlist.get_msg_id to return id, instead of wrongly erroring
+
+## 1.0.0-beta.5
+
+- fix dc_get_msg() to return empty messages when asked for special ones 
+
+## 1.0.0-beta.4
+
+- fix more than one sending of autocrypt setup message
+
+- fix recognition of mailto-address-qr-codes, add tests
+
+- tune down error to warning when adding self to chat
+
+## 1.0.0-beta.3
+
+- add back `dc_empty_server()` #682
+
+- if `show_emails` is set to `DC_SHOW_EMAILS_ALL`,
+  email-based contact requests are added to the chatlist directly
+
+- fix IMAP hangs #717 and cleanups
+
+- several rPGP fixes
+
+- code streamlining and rustifications
+
+
+## 1.0.0-beta.2
+
+- https://c.delta.chat docs are now regenerated again through our CI 
+
+- several rPGP cleanups, security fixes and better multi-platform support 
+
+- reconnect on io errors and broken pipes (imap)
+
+- probe SMTP with real connection not just setup
+
+- various imap/smtp related fixes
+
+- use to_string_lossy in most places instead of relying on valid utf-8
+  encodings
+ 
+- rework, rustify and test autoconfig-reading and parsing 
+
+- some rustifications/boolifications of c-ints 
+
+
+## 1.0.0-beta.1 
+
+- first beta of the Delta Chat Rust core library. many fixes of crashes
+  and other issues compared to 1.0.0-alpha.5.
+
+- Most code is now "rustified" and does not do manual memory allocation anymore. 
+
+- The `DC_EVENT_GET_STRING` event is not used anymore, removing the last
+  event where the core requested a return value from the event callback. 
+
+  Please now use `dc_set_stock_translation()` API for core messages
+  to be properly localized. 
+
+- Deltachat FFI docs are automatically generated and available here: 
+  https://c.delta.chat 
+
+- New events ImapMessageMoved and ImapMessageDeleted
+
+For a full list of changes, please see our closed Pull Requests: 
+
+https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
+

Cargo.toml

@@ -1,56 +1,65 @@
 [package]
 name = "deltachat"
-version = "1.0.0-alpha.4"
-authors = ["dignifiedquire <dignifiedquire@gmail.com>"]
+version = "1.28.0"
+authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
-license = "MPL"
+license = "MPL-2.0"
 
-[build-dependencies]
-cc = "1.0.35"
-pkg-config = "0.3"
+[profile.release]
+lto = true
 
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
+
 libc = "0.2.51"
-pgp = { version = "0.2", default-features = false }
-hex = "0.3.2"
+pgp = { version = "0.5.1", default-features = false } 
+hex = "0.4.0"
 sha2 = "0.8.0"
-rand = "0.6.5"
-phf = { git = "https://github.com/sfackler/rust-phf", rev = "0d00821", features = ["macros"] }
-smallvec = "0.6.9"
-reqwest = "0.9.15"
-num-derive = "0.2.5"
+rand = "0.7.0"
+smallvec = "1.0.0"
+reqwest = { version = "0.10.0", features = ["blocking", "json"] }
+num-derive = "0.3.0"
 num-traits = "0.2.6"
-native-tls = "0.2.3"
-lettre = "0.9.0"
-imap = { git = "https://github.com/jonhoo/rust-imap", rev = "281d2eb8ab50dc656ceff2ae749ca5045f334e15" }
-mmime = { git = "https://github.com/dignifiedquire/mmime", rev = "bccd2c2" }
-base64 = "0.10"
+async-smtp = "0.2"
+email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
+lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
+async-imap = "0.2"
+async-native-tls = "0.3.1"
+async-std = { version = "1.4", features = ["unstable"] }
+base64 = "0.11"
 charset = "0.1"
 percent-encoding = "2.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 chrono = "0.4.6"
-failure = "0.1.5"
-failure_derive = "0.1.5"
-# TODO: make optional
-rustyline = "4.1.0"
+indexmap = "1.3.0"
 lazy_static = "1.4.0"
 regex = "1.1.6"
-rusqlite = { version = "0.20", features = ["bundled"] }
-r2d2_sqlite = "0.12.0"
+rusqlite = { version = "0.21", features = ["bundled"] }
+r2d2_sqlite = "0.13.0"
 r2d2 = "0.8.5"
-strum = "0.15.0"
-strum_macros = "0.15.0"
+strum = "0.16.0"
+strum_macros = "0.16.0"
 thread-local-object = "0.1.0"
 backtrace = "0.3.33"
 byteorder = "1.3.1"
 itertools = "0.8.0"
 image-meta = "0.1.0"
-quick-xml = "0.15.0"
+quick-xml = "0.17.1"
 escaper = "0.1.0"
 bitflags = "1.1.0"
-jetscii = "0.4.4"
+debug_stub_derive = "0.3.0"
+sanitize-filename = "0.2.1"
+stop-token = { version = "0.1.1", features = ["unstable"] }
+mailparse = "0.12.0"
+encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
+native-tls = "0.2.3"
+image = { version = "0.22.4", default-features=false, features = ["gif_codec", "jpeg", "ico", "png_codec", "pnm", "webp", "bmp"] }
+pretty_env_logger = "0.3.1"
+
+rustyline = { version = "4.1.0", optional = true }
+thiserror = "1.0.14"
+anyhow = "1.0.28"
 
 [dev-dependencies]
 tempfile = "3.0"
@@ -71,10 +80,11 @@ path = "examples/simple.rs"
 [[example]]
 name = "repl"
 path = "examples/repl/main.rs"
+required-features = ["rustyline"]
 
 
 [features]
-default = ["nightly", "ringbuf"]
-vendored = ["native-tls/vendored", "reqwest/default-tls-vendored"]
+default = ["nightly"]
+vendored = ["async-native-tls/vendored", "reqwest/native-tls-vendored", "async-smtp/native-tls-vendored"]
 nightly = ["pgp/nightly"]
-ringbuf = ["pgp/ringbuf"]
+

LICENSE

@@ -2,9 +2,6 @@ The files in this directory and under its subdirectories
 are (c) 2019 by Bjoern Petersen and contributors and released under the
 Mozilla Public License Version 2.0, see below for a copy.
 
-NOTE that the files in the "libs" directory are copyrighted by third parties
-and come with their own respective licenses.
-
 Mozilla Public License Version 2.0
 ==================================
 

README.md

@@ -1,11 +1,9 @@
 # Delta Chat Rust
 
-> Project porting deltachat-core to rust
+> Deltachat-core written in Rust 
 
 [![CircleCI build status][circle-shield]][circle] [![Appveyor build status][appveyor-shield]][appveyor]
 
-Current commit on deltachat/deltachat-core: `12ef73c8e76185f9b78e844ea673025f56a959ab`.
-
 ## Installing Rust and Cargo
 
 To download and install the official compiler for the Rust programming language, and the Cargo package manager, run the command in your user environment:
@@ -16,7 +14,7 @@ curl https://sh.rustup.rs -sSf | sh
 
 ## Using the CLI client
 
-Compile and run Delta Chat Core using `cargo`:
+Compile and run Delta Chat Core command line utility, using `cargo`:
 
 ```
 cargo run --example repl -- /path/to/db
@@ -89,6 +87,15 @@ $ cargo test --all
 $ cargo build -p deltachat_ffi --release
 ```
 
+## Debugging environment variables 
+
+- `DCC_IMAP_DEBUG`: if set IMAP protocol commands and responses will be
+  printed
+
+- `DCC_MIME_DEBUG`: if set outgoing and incoming message will be printed 
+
+
+
 ### Expensive tests
 
 Some tests are expensive and marked with `#[ignore]`, to run these
@@ -101,9 +108,27 @@ $ cargo test -- --ignored
 
 - `vendored`: When using Openssl for TLS, this bundles a vendored version.
 - `nightly`: Enable nightly only performance and security related features.
-- `ringbuf`: Enable the use of [`slice_deque`](https://github.com/gnzlbg/slice_deque) in pgp.
 
 [circle-shield]: https://img.shields.io/circleci/project/github/deltachat/deltachat-core-rust/master.svg?style=flat-square
 [circle]: https://circleci.com/gh/deltachat/deltachat-core-rust/
 [appveyor-shield]: https://ci.appveyor.com/api/projects/status/lqpegel3ld4ipxj8/branch/master?style=flat-square
 [appveyor]: https://ci.appveyor.com/project/dignifiedquire/deltachat-core-rust/branch/master
+
+## Language bindings and frontend projects
+
+Language bindings are available for:
+
+- [C](https://c.delta.chat)
+- [Node.js](https://www.npmjs.com/package/deltachat-node)
+- [Python](https://py.delta.chat)
+- [Go](https://github.com/hugot/go-deltachat/)
+- **Java** and **Swift** (contained in the Android/iOS repos)
+
+The following "frontend" projects make use of the Rust-library
+or its language bindings:
+
+- [Android](https://github.com/deltachat/deltachat-android)
+- [iOS](https://github.com/deltachat/deltachat-ios)
+- [Desktop](https://github.com/deltachat/deltachat-desktop)
+- [Pidgin](https://code.ur.gs/lupine/purple-plugin-delta/)
+- several **Bots**

Xargo.toml

@@ -1,6 +0,0 @@
-[dependencies.std]
-features = ["panic-unwind"]
-
-# if using `cargo test`
-[dependencies.test]
-stage = 1

appveyor.yml

@@ -4,16 +4,15 @@ environment:
 
 install:
   - appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
-  - rustup-init -yv --default-toolchain nightly-2019-07-10
+  - rustup-init -yv --default-toolchain nightly-2020-03-12
   - set PATH=%PATH%;%USERPROFILE%\.cargo\bin
   - rustc -vV
   - cargo -vV
-  - cargo update
 
 build: false
 
 test_script:
-  - cargo test --release
+  - cargo test --release --all
 
 cache:
   - target

assets/icon-device.svg

@@ -0,0 +1,83 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   inkscape:export-ydpi="409.60001"
+   inkscape:export-xdpi="409.60001"
+   inkscape:export-filename="/Users/bpetersen/projects/deltachat-core-rust/assets/icon-device.png"
+   version="1.0"
+   width="60"
+   height="60"
+   viewBox="0 0 45 45"
+   preserveAspectRatio="xMidYMid meet"
+   id="svg4344"
+   sodipodi:docname="icon-device.svg"
+   inkscape:version="1.0beta1 (32d4812, 2019-09-19)">
+  <defs
+     id="defs4348" />
+  <sodipodi:namedview
+     inkscape:snap-global="false"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     inkscape:document-rotation="0"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="1600"
+     inkscape:window-height="1035"
+     id="namedview4346"
+     showgrid="false"
+     units="px"
+     inkscape:zoom="3.959798"
+     inkscape:cx="28.322498"
+     inkscape:cy="24.898474"
+     inkscape:window-x="45"
+     inkscape:window-y="23"
+     inkscape:window-maximized="0"
+     inkscape:current-layer="svg4344" />
+  <metadata
+     id="metadata4336">
+Created by potrace 1.15, written by Peter Selinger 2001-2017
+<rdf:RDF>
+  <cc:Work
+     rdf:about="">
+    <dc:format>image/svg+xml</dc:format>
+    <dc:type
+       rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+    <dc:title />
+  </cc:Work>
+</rdf:RDF>
+</metadata>
+  <rect
+     y="-4.4408921e-16"
+     x="0"
+     height="45"
+     width="45"
+     id="rect860"
+     style="opacity:1;fill:#76868b;fill-opacity:1;stroke-width:0.819271" />
+  <g
+     fill="#000000"
+     stroke="none"
+     style="fill:#ffffff;fill-opacity:1"
+     transform="matrix(0.00255113,0,0,-0.00255113,5.586152,38.200477)"
+     id="g4342">
+    <path
+       style="fill:#ffffff;fill-opacity:1"
+       d="m 8175,12765 c -703,-114 -1248,-608 -1387,-1258 -17,-82 -21,-136 -22,-277 0,-202 15,-307 70,-470 149,-446 499,-733 1009,-828 142,-26 465,-23 619,6 691,131 1201,609 1328,1244 31,158 31,417 0,565 -114,533 -482,889 -1038,1004 -133,27 -448,35 -579,14 z"
+       id="path4338"
+       inkscape:connector-curvature="0" />
+    <path
+       style="fill:#ffffff;fill-opacity:1"
+       d="m 7070,9203 c -212,-20 -275,-27 -397,-48 -691,-117 -1400,-444 -2038,-940 -182,-142 -328,-270 -585,-517 -595,-571 -911,-974 -927,-1181 -6,-76 11,-120 69,-184 75,-80 159,-108 245,-79 109,37 263,181 632,595 539,606 774,826 1035,969 135,75 231,105 341,106 82,1 94,-2 138,-27 116,-68 161,-209 122,-376 -9,-36 -349,-868 -757,-1850 -407,-982 -785,-1892 -838,-2021 -287,-694 -513,-1389 -615,-1889 -70,-342 -90,-683 -52,-874 88,-440 381,-703 882,-792 124,-23 401,-30 562,-16 783,69 1674,461 2561,1125 796,596 1492,1354 1607,1751 43,146 -33,308 -168,360 -61,23 -100,15 -173,-36 -105,-74 -202,-170 -539,-529 -515,-551 -762,-783 -982,-927 -251,-164 -437,-186 -543,-65 -56,64 -74,131 -67,247 13,179 91,434 249,815 135,324 1588,4102 1646,4280 106,325 151,561 159,826 9,281 -22,463 -112,652 -58,122 -114,199 -211,292 -245,233 -582,343 -1044,338 -91,-1 -181,-3 -200,-5 z"
+       id="path4340"
+       inkscape:connector-curvature="0" />
+  </g>
+</svg>

assets/icon-saved-messages.svg

@@ -0,0 +1,71 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   inkscape:export-ydpi="409.60001"
+   inkscape:export-xdpi="409.60001"
+   inkscape:export-filename="/home/kerle/test-icon.png"
+   version="1.0"
+   width="60"
+   height="60"
+   viewBox="0 0 45 45"
+   preserveAspectRatio="xMidYMid meet"
+   id="svg4344"
+   sodipodi:docname="icon-saved-messages.svg"
+   inkscape:version="1.0beta1 (32d4812, 2019-09-19)">
+  <defs
+     id="defs4348" />
+  <sodipodi:namedview
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     inkscape:document-rotation="0"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="1395"
+     inkscape:window-height="855"
+     id="namedview4346"
+     showgrid="false"
+     units="px"
+     inkscape:zoom="4"
+     inkscape:cx="29.308676"
+     inkscape:cy="49.03624"
+     inkscape:window-x="89"
+     inkscape:window-y="108"
+     inkscape:window-maximized="0"
+     inkscape:current-layer="svg4344"
+     inkscape:lockguides="false" />
+  <metadata
+     id="metadata4336">
+Created by potrace 1.15, written by Peter Selinger 2001-2017
+<rdf:RDF>
+  <cc:Work
+     rdf:about="">
+    <dc:format>image/svg+xml</dc:format>
+    <dc:type
+       rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+    <dc:title />
+  </cc:Work>
+</rdf:RDF>
+</metadata>
+  <rect
+     y="0"
+     x="0"
+     height="45"
+     width="45"
+     id="rect1420"
+     style="fill:#87aade;fill-opacity:1;stroke:none;stroke-width:0.968078" />
+  <path
+     id="rect846"
+     style="fill:#ffffff;stroke-width:0.58409804"
+     d="M 13.5,7.5 V 39 h 0.08654 L 22.533801,29.370239 31.482419,39 h 0.01758 V 7.5 Z m 9.004056,4.108698 1.879508,4.876388 5.039514,0.359779 -3.879358,3.363728 1.227764,5.095749 -4.276893,-2.796643 -4.280949,2.788618 1.237229,-5.093073 -3.873949,-3.371754 5.040866,-0.350417 z"
+     inkscape:connector-curvature="0" />
+</svg>

build.rs

@@ -1,38 +0,0 @@
-extern crate cc;
-
-fn link_static(lib: &str) {
-    println!("cargo:rustc-link-lib=static={}", lib);
-}
-
-fn link_framework(fw: &str) {
-    println!("cargo:rustc-link-lib=framework={}", fw);
-}
-
-fn add_search_path(p: &str) {
-    println!("cargo:rustc-link-search={}", p);
-}
-
-fn build_tools() {
-    let mut config = cc::Build::new();
-    config.file("misc.c").compile("libtools.a");
-
-    println!("rerun-if-changed=build.rs");
-    println!("rerun-if-changed=misc.h");
-    println!("rerun-if-changed=misc.c");
-}
-
-fn main() {
-    build_tools();
-
-    add_search_path("/usr/local/lib");
-
-    let target = std::env::var("TARGET").unwrap();
-    if target.contains("-apple") || target.contains("-darwin") {
-        link_framework("CoreFoundation");
-        link_framework("CoreServices");
-        link_framework("Security");
-    }
-
-    // local tools
-    link_static("tools");
-}

ci_scripts/README.md

@@ -1,52 +1,46 @@
-
 # Continuous Integration Scripts for Delta Chat
 
-Continuous Integration is run through CircleCI
-but is largely independent of it. 
+Continuous Integration, run through CircleCI and an own build machine.
+
+## Description of scripts 
+
+- `../.circleci/config.yml` describing the build jobs that are run
+  by Circle-CI
+
+- `remote_tests_python.sh` rsyncs to a build machine and runs
+  `run-python-test.sh` remotely on the build machine. 
+
+- `remote_tests_rust.sh` rsyncs to the build machine and runs
+  `run-rust-test.sh` remotely on the build machine. 
+
+- `doxygen/Dockerfile` specifies an image that contains
+  the doxygen tool which is used by `run-doxygen.sh` 
+  to generate C-docs which are then uploaded 
+  via `ci_upload.sh` to `https://c.delta.chat/_unofficial_unreleased_docs/<BRANCH>`
+  (and the master branch is linked to https://c.delta.chat proper). 
 
 
-## Generating docker containers for performing build step work 
+## Triggering runs on the build machine locally (fast!)
 
-All tests, docs and wheel building is run in docker containers:
+There is experimental support for triggering a remote Python or Rust test run 
+from your local checkout/branch. You will need to be authorized to login to 
+the build machine (ask your friendly sysadmin on #deltachat freenode) to type::
 
-- **coredeps/Dockerfile** specifies an image that contains all 
-  of Delta Chat's core dependencies as linkable libraries. 
-  It also serves to run python tests and build wheels 
-  (binary packages for Python). 
+    ci_scripts/manual_remote_tests.sh rust
+    ci_scripts/manual_remote_tests.sh python
 
-- **doxygen/Dockerfile** specifies an image that contains
-  the doxygen tool which is used to generate C-docs. 
+This will **rsync** your current checkout to the remote build machine 
+(no need to commit before) and then run either rust or python tests. 
 
-To run tests locally you can pull existing images from "docker.io",
-the hub for sharing Docker images::
+# Outdated files (for later re-use)
 
-    docker pull deltachat/coredeps
-    docker pull deltachat/doxygen 
+`coredeps/Dockerfile` specifies an image that contains all 
+of Delta Chat's core dependencies. It used to run 
+python tests and build wheels (binary packages for Python)
 
-or you can build the docker images yourself locally
+You can build the docker images yourself locally
 to avoid the relatively large download:: 
  
     cd ci_scripts  # where all CI things are 
     docker build -t deltachat/coredeps docker-coredeps
     docker build -t deltachat/doxygen docker-doxygen 
-
-## ci_run.sh (main entrypoint called by circle-ci)
-
-Once you have the docker images available 
-you can run python testing, documentation generation 
-and building binary wheels:: 
-
-    sh DOCS=1 TESTS=1 ci_scripts/ci_run.sh 
-    
-## ci_upload.sh (uploading artifacts on success) 
-
-- python docs to `https://py.delta.chat/_unofficial_unreleased_docs/<BRANCH>`
-
-- doxygen docs to `https://c.delta.chat/_unofficial_unreleased_docs/<BRANCH>`
-
-- python wheels to `https://m.devpi.net/dc/<BRANCH>`
-  so that you install fully self-contained wheels like this:
-  `pip install -U -i https://m.devpi.net/dc/<BRANCH> deltachat`
-
-
-

ci_scripts/ci_run.sh

@@ -1,22 +0,0 @@
-# perform CI jobs on PRs and after merges to master.
-# triggered from .circleci/config.yml
-
-set -e -x
-
-export BRANCH=${CIRCLE_BRANCH:-test7}
-    
-# run doxygen on c-source (needed by later doc-generation steps).
-# XXX modifies the host filesystem docs/xml and docs/html directories
-# XXX which you can then only remove with sudo as they belong to root
-
-# XXX we don't do doxygen doc generation with Rust anymore, needs to be
-# substituted with rust-docs 
-#if [ -n "$DOCS" ] ; then 
-#    docker run --rm -it -v $PWD:/mnt -w /mnt/docs deltachat/doxygen doxygen
-#fi
-
-# run everything else inside docker (TESTS, DOCS, WHEELS) 
-docker run -e DCC_PY_LIVECONFIG -e BRANCH -e TESTS -e DOCS \
-           --rm -it -v $(pwd):/mnt -w /mnt \
-           deltachat/coredeps ci_scripts/run_all.sh
-

ci_scripts/ci_upload.sh

@@ -7,9 +7,9 @@ fi
 
 set -xe
 
-#DOXYDOCDIR=${1:?directory where doxygen docs to be found}
 PYDOCDIR=${1:?directory with python docs}
 WHEELHOUSEDIR=${2:?directory with pre-built wheels}
+DOXYDOCDIR=${3:?directory where doxygen docs to be found}
 
 export BRANCH=${CIRCLE_BRANCH:?specify branch for uploading purposes}
 
@@ -17,15 +17,18 @@ export BRANCH=${CIRCLE_BRANCH:?specify branch for uploading purposes}
 # python docs to py.delta.chat
 ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
 rsync -avz \
+  --delete \
   -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
   "$PYDOCDIR/html/" \
   delta@py.delta.chat:build/${BRANCH}
 
 # C docs to c.delta.chat
-#rsync -avz \
-#  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
-#  "$DOXYDOCDIR/html/" \
-#  delta@py.delta.chat:build-c/${BRANCH}
+ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@c.delta.chat mkdir -p build-c/${BRANCH}
+rsync -avz \
+  --delete \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  "$DOXYDOCDIR/html/" \
+  delta@c.delta.chat:build-c/${BRANCH}
 
 echo -----------------------
 echo upload wheels 
@@ -34,6 +37,7 @@ echo -----------------------
 # Bundle external shared libraries into the wheels
 pushd $WHEELHOUSEDIR
 
+pip3 install -U pip
 pip3 install devpi-client
 devpi use https://m.devpi.net
 devpi login dc --password $DEVPI_LOGIN
@@ -45,6 +49,9 @@ devpi use dc/$N_BRANCH || {
     devpi use dc/$N_BRANCH
 }
 devpi index $N_BRANCH bases=/root/pypi
-devpi upload deltachat*.whl
+devpi upload deltachat*
 
 popd
+
+# remove devpi non-master dc indices if thy are too old
+python ci_scripts/cleanup_devpi_indices.py

ci_scripts/cleanup_devpi_indices.py

@@ -0,0 +1,69 @@
+"""
+Remove old "dc" indices except for master which always stays.
+
+"""
+from requests import Session
+import datetime
+import sys
+import subprocess
+
+MAXDAYS=7
+
+session = Session()
+session.headers["Accept"] = "application/json"
+
+
+def get_indexes(baseurl, username):
+    response = session.get(baseurl + username)
+    assert response.status_code == 200
+    result = response.json()["result"]
+    return result["indexes"]
+
+
+def get_projectnames(baseurl, username, indexname):
+    response = session.get(baseurl + username + "/" + indexname)
+    assert response.status_code == 200
+    result = response.json()["result"]
+    return result["projects"]
+
+
+def get_release_dates(baseurl, username, indexname, projectname):
+    response = session.get(baseurl + username + "/" + indexname + "/" + projectname)
+    assert response.status_code == 200
+    result = response.json()["result"]
+    dates = set()
+    for value in result.values():
+        if "+links" not in value:
+            continue
+        for link in value["+links"]:
+            for log in link["log"]:
+                dates.add(tuple(log["when"]))
+    return dates
+
+
+def run():
+    baseurl = "https://m.devpi.net/"
+    username = "dc"
+    for indexname in get_indexes(baseurl, username):
+        projectnames = get_projectnames(baseurl, username, indexname)
+        if indexname == "master" or not indexname:
+            continue
+        assert projectnames == ["deltachat"]
+        for projectname in projectnames:
+            dates = get_release_dates(baseurl, username, indexname, projectname)
+            if not dates:
+                print(
+                    "%s has no releases" % (baseurl + username + "/" + indexname),
+                    file=sys.stderr)
+                date = datetime.datetime.now()
+            else:
+                date = datetime.datetime(*max(dates))
+            if (datetime.datetime.now() - date) > datetime.timedelta(days=MAXDAYS):
+                assert username and indexname
+                url = baseurl + username + "/" + indexname
+                subprocess.check_call(["devpi", "index", "-y", "--delete", url])
+
+
+
+if __name__ == '__main__':
+    run()

ci_scripts/docker-coredeps/Dockerfile

@@ -5,21 +5,17 @@ RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
     echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
 ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
 
+# Install a recent Perl, needed to install the openssl crate 
+ADD deps/build_perl.sh /builder/build_perl.sh
+RUN rm /usr/bin/perl
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
+
 ENV PIP_DISABLE_PIP_VERSION_CHECK 1
 
 # Install python tools (auditwheels,tox, ...)
 ADD deps/build_python.sh /builder/build_python.sh
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
 
-# Install Rust nightly 
+# Install Rust nightly
 ADD deps/build_rust.sh /builder/build_rust.sh
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1
-
-# Install a recent Perl, needed to install OpenSSL
-ADD deps/build_perl.sh /builder/build_perl.sh
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
-
-# Install OpenSSL
-ADD deps/build_openssl.sh /builder/build_openssl.sh
-RUN mkdir tmp1 && cd tmp1 && bash /builder/build_openssl.sh && cd .. && rm -r tmp1
-

ci_scripts/docker-coredeps/deps/build_perl.sh

@@ -1,11 +1,11 @@
 #!/bin/bash
 
-PERL_VERSION=5.28.0
-PERL_SHA256=7e929f64d4cb0e9d1159d4a59fc89394e27fa1f7004d0836ca0d514685406ea8
+PERL_VERSION=5.30.0
+# PERL_SHA256=7e929f64d4cb0e9d1159d4a59fc89394e27fa1f7004d0836ca0d514685406ea8
 curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz
-echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
-tar xzf perl-${PERL_VERSION}.tar.gz
-cd perl-${PERL_VERSION} 
+# echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
+tar -xzf perl-${PERL_VERSION}.tar.gz
+cd perl-${PERL_VERSION}
 
 ./Configure -de
 make

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -1,11 +1,11 @@
 #!/bin/bash
 
-set -e -x 
+set -e -x
 
-# Install Rust 
-curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-07-10 -y
+# Install Rust
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2020-03-12 -y
 export PATH=/root/.cargo/bin:$PATH
 rustc --version
 
 # remove some 300-400 MB that we don't need for automated builds
-rm -rf /root/.rustup/toolchains/nightly-2019-07-10-x86_64-unknown-linux-gnu/share/
+rm -rf /root/.rustup/toolchains/nightly-2020-03-12-x86_64-unknown-linux-gnu/share/

ci_scripts/docker-coredeps/deps/run_all.sh

@@ -1,49 +0,0 @@
-#!/bin/bash
-#
-# Build the Delta Chat C/Rust library
-#
-set -e -x
-
-# perform clean build of core and install 
-export TOXWORKDIR=.docker-tox
-
-# build core library
-
-cargo build --release -p deltachat_ffi
-
-# configure access to a base python and 
-# to several python interpreters needed by tox below
-export PATH=$PATH:/opt/python/cp35-cp35m/bin
-export PYTHONDONTWRITEBYTECODE=1
-pushd /bin
-ln -s /opt/python/cp27-cp27m/bin/python2.7
-ln -s /opt/python/cp36-cp36m/bin/python3.6
-ln -s /opt/python/cp37-cp37m/bin/python3.7
-popd
-
-#
-# run python tests
-#
-
-if [ -n "$TESTS" ]; then 
-
-    echo ----------------
-    echo run python tests
-    echo ----------------
-
-    pushd python 
-    # first run all tests ...
-    rm -rf tests/__pycache__
-    rm -rf src/deltachat/__pycache__
-    export PYTHONDONTWRITEBYTECODE=1
-    tox --workdir "$TOXWORKDIR" -e py27,py35,py36,py37
-    popd
-fi
-
-
-if [ -n "$DOCS" ]; then 
-    echo -----------------------
-    echo generating python docs
-    echo -----------------------
-    (cd python && tox --workdir "$TOXWORKDIR" -e doc) 
-fi

ci_scripts/manual_remote_tests.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -xe
+export CIRCLE_JOB=remote_tests_${1:?need to specify 'rust' or 'python'} 
+export CIRCLE_BUILD_NUM=$USER
+export CIRCLE_BRANCH=`git branch | grep \* | cut -d ' ' -f2`
+export CIRCLE_PROJECT_REPONAME=$(basename `git rev-parse --show-toplevel`)
+
+time bash ci_scripts/$CIRCLE_JOB.sh 

ci_scripts/old/gh-actions-rust.yml

@@ -0,0 +1,77 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+
+env:
+  RUSTFLAGS: -Dwarnings
+
+jobs:
+  build_and_test:
+    name: Build and test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macOS-latest]
+        rust: [nightly]
+
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Install ${{ matrix.rust }}
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: ${{ matrix.rust }}
+        override: true
+
+    - name: check
+      uses: actions-rs/cargo@v1
+      if: matrix.rust == 'nightly'
+      with:
+        command:  check
+        args: --all --bins --examples --tests
+
+    - name: tests
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --all
+
+    - name: tests ignored
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --all --release -- --ignored
+
+  check_fmt:
+    name: Checking fmt and docs
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@master
+
+    - uses: actions-rs/toolchain@v1
+      with:
+          profile: minimal
+          toolchain: nightly
+          override: true
+          components: rustfmt
+
+    - name: fmt
+      run: cargo fmt --all -- --check
+
+#  clippy_check:
+#    name: Clippy check
+#    runs-on: ubuntu-latest
+#
+#    steps:
+#    - uses: actions/checkout@v1
+#    - uses: actions-rs/toolchain@v1
+#      with:
+#          profile: minimal
+#          toolchain: nightly
+#          override: true
+#          components: clippy
+#
+#    - name: clippy
+#      run: cargo clippy --all

ci_scripts/old/run-python.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+#
+# Build the Delta Chat C/Rust library typically run in a docker
+# container that contains all library deps but should also work
+# outside if you have the dependencies installed on your system.
+
+set -e -x
+
+# Perform clean build of core and install.
+export TOXWORKDIR=.docker-tox
+
+# install core lib
+
+export PATH=/root/.cargo/bin:$PATH
+cargo build --release -p deltachat_ffi
+# cargo test --all --all-features
+
+# Statically link against libdeltachat.a.
+export DCC_RS_DEV=$(pwd)
+
+# Configure access to a base python and to several python interpreters
+# needed by tox below.
+export PATH=$PATH:/opt/python/cp35-cp35m/bin
+export PYTHONDONTWRITEBYTECODE=1
+pushd /bin
+ln -s /opt/python/cp27-cp27m/bin/python2.7
+ln -s /opt/python/cp36-cp36m/bin/python3.6
+ln -s /opt/python/cp37-cp37m/bin/python3.7
+popd
+
+if [ -n "$TESTS" ]; then
+
+    pushd python
+    # prepare a clean tox run
+    rm -rf tests/__pycache__
+    rm -rf src/deltachat/__pycache__
+    export PYTHONDONTWRITEBYTECODE=1
+
+    # run tox. The circle-ci project env-var-setting DCC_PY_LIVECONFIG 
+    # allows running of "liveconfig" tests but for speed reasons
+    # we run them only for the highest python version we support
+    
+    # we split out qr-tests run to minimize likelyness of flaky tests
+    # (some qr tests are pretty heavy in terms of send/received
+    # messages and rust's imap code likely has concurrency problems) 
+    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "not qr"
+    tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "qr"
+    unset DCC_PY_LIVECONFIG
+    unset DCC_NEW_TMP_EMAIL
+    tox --workdir "$TOXWORKDIR" -p4 -e lint,py35,py36,doc
+    tox --workdir "$TOXWORKDIR" -e auditwheels
+    popd
+fi
+
+
+# if [ -n "$DOCS" ]; then
+#     echo -----------------------
+#     echo generating python docs
+#     echo -----------------------
+#     (cd python && tox --workdir "$TOXWORKDIR" -e doc)
+# fi

ci_scripts/remote_python_packaging.sh

@@ -0,0 +1,51 @@
+#!/bin/bash 
+
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+
+# we construct the BUILDDIR such that we can easily share the
+# CARGO_TARGET_DIR between runs ("..")
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+set -xe
+
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist 
+# we seem to need .git for setuptools_scm versioning 
+find .git >>.rsynclist
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+
+set +x
+
+# we have to create a remote file for the remote-docker run 
+# so we can do a simple ssh command with a TTY
+# so that when our job dies, all container-runs are aborted.
+# sidenote: the circle-ci machinery will kill ongoing jobs
+# if there are new commits and we want to ensure that 
+# everything is terminated/cleaned up and we have no orphaned
+# useless still-running docker-containers consuming resources. 
+
+ssh $SSHTARGET bash -c "cat >$BUILDDIR/exec_docker_run" <<_HERE
+    set +x -e
+    cd $BUILDDIR
+    export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
+    export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
+    set -x
+
+    # run everything else inside docker 
+    docker run -e DCC_NEW_TMP_EMAIL -e DCC_PY_LIVECONFIG \
+       --rm -it -v \$(pwd):/mnt -w /mnt \
+       deltachat/coredeps ci_scripts/run_all.sh
+
+_HERE
+
+echo "--- Running $CIRCLE_JOB remotely"
+
+ssh -t $SSHTARGET bash "$BUILDDIR/exec_docker_run"
+mkdir -p workspace 
+rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/wheelhouse/*manylinux1*" workspace/wheelhouse/
+rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/dist/*" workspace/wheelhouse/
+rsync -avz "$SSHTARGET:$BUILDDIR/python/doc/_build/" workspace/py-docs

ci_scripts/remote_tests_python.sh

@@ -0,0 +1,47 @@
+#!/bin/bash 
+
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+
+# we construct the BUILDDIR such that we can easily share the
+# CARGO_TARGET_DIR between runs ("..")
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+set -xe
+
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist 
+# we seem to need .git for setuptools_scm versioning 
+find .git >>.rsynclist
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+
+set +x
+
+echo "--- Running $CIRCLE_JOB remotely"
+
+ssh $SSHTARGET <<_HERE
+    set +x -e
+    cd $BUILDDIR
+    # let's share the target dir with our last run on this branch/job-type
+    # cargo will make sure to block/unblock us properly 
+    export CARGO_TARGET_DIR=\`pwd\`/../target
+    export TARGET=release
+    export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
+    export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
+
+    #we rely on tox/virtualenv being available in the host
+    #rm -rf virtualenv venv
+    #virtualenv -q -p python3.7 venv 
+    #source venv/bin/activate
+    #pip install -q tox virtualenv
+
+    set -x
+    which python
+    source \$HOME/venv/bin/activate
+    which python
+
+    bash ci_scripts/run-python-test.sh 
+_HERE

ci_scripts/remote_tests_rust.sh

@@ -0,0 +1,32 @@
+#!/bin/bash 
+
+export BRANCH=${CIRCLE_BRANCH:?branch to build}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+
+# we construct the BUILDDIR such that we can easily share the
+# CARGO_TARGET_DIR between runs ("..")
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+
+set -e
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist 
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+
+echo "--- Running $CIRCLE_JOB remotely"
+
+ssh $SSHTARGET <<_HERE
+    set +x -e
+    cd $BUILDDIR
+    # let's share the target dir with our last run on this branch/job-type
+    # cargo will make sure to block/unblock us properly 
+    export CARGO_TARGET_DIR=\`pwd\`/../target
+    export TARGET=x86_64-unknown-linux-gnu
+    export RUSTC_WRAPPER=sccache
+
+    bash ci_scripts/run-rust-test.sh
+_HERE
+

ci_scripts/run-doxygen.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -ex
+
+cd deltachat-ffi 
+PROJECT_NUMBER=$(git log -1 --format "%h (%cd)") doxygen 

ci_scripts/run-python-test.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+#
+# Run functional tests for Delta Chat core using the python bindings 
+# and tox/pytest. 
+
+set -e -x
+
+# for core-building and python install step
+export DCC_RS_TARGET=debug
+export DCC_RS_DEV=`pwd`
+
+cd python
+
+python install_python_bindings.py onlybuild
+
+# remove and inhibit writing PYC files 
+rm -rf tests/__pycache__
+rm -rf src/deltachat/__pycache__
+export PYTHONDONTWRITEBYTECODE=1
+
+# run python tests (tox invokes pytest to run tests in python/tests)
+#TOX_PARALLEL_NO_SPINNER=1 tox -e lint,doc
+tox -e lint
+tox -e doc,py37

ci_scripts/run-rust-test.sh

@@ -2,7 +2,7 @@
 
 set -ex
 
-export RUST_TEST_THREADS=1
+#export RUST_TEST_THREADS=1
 export RUST_BACKTRACE=1
 export RUSTFLAGS='--deny warnings'
 export OPT="--target=$TARGET"
@@ -31,16 +31,15 @@ fi
 if [[ $NORUN == "1" ]]; then
     export CARGO_SUBCMD="build"
 else
-    export CARGO_SUBCMD="test"
+    export CARGO_SUBCMD="test --all"
     export OPT="${OPT} "
     export OPT_RELEASE="${OPT_RELEASE} "
     export OPT_RELEASE_IGNORED="${OPT_RELEASE} -- --ignored"
 fi
 
-# Run all the test configurations:
+# Run all the test configurations 
+# RUSTC_WRAPPER=SCCACHE seems to destroy parallelism / prolong the test
+unset RUSTC_WRAPPER
 $CARGO_CMD $CARGO_SUBCMD $OPT
 $CARGO_CMD $CARGO_SUBCMD $OPT_RELEASE
 $CARGO_CMD $CARGO_SUBCMD $OPT_RELEASE_IGNORED
-
-# Build the ffi lib
-$CARGO_CMD $CARGO_SUBCMD $OPT_FFI_RELEASE

ci_scripts/run_all.sh

@@ -1,15 +1,13 @@
 #!/bin/bash
 #
-# Build the Delta Chat C/Rust library typically run in a docker
-# container that contains all library deps but should also work
-# outside if you have the dependencies installed on your system.
+# Build the Delta Chat Core Rust library, Python wheels and docs 
 
 set -e -x
 
 # Perform clean build of core and install.
 export TOXWORKDIR=.docker-tox
 
-# install core lib
+# compile core lib
 
 export PATH=/root/.cargo/bin:$PATH
 cargo build --release -p deltachat_ffi
@@ -26,29 +24,26 @@ pushd /bin
 ln -s /opt/python/cp27-cp27m/bin/python2.7
 ln -s /opt/python/cp36-cp36m/bin/python3.6
 ln -s /opt/python/cp37-cp37m/bin/python3.7
+ln -s /opt/python/cp38-cp38/bin/python3.8
 popd
 
-if [ -n "$TESTS" ]; then
+pushd python
+# prepare a clean tox run
+rm -rf tests/__pycache__
+rm -rf src/deltachat/__pycache__
+mkdir -p $TOXWORKDIR
 
-    pushd python
-    # prepare a clean tox run
-    rm -rf tests/__pycache__
-    rm -rf src/deltachat/__pycache__
-    export PYTHONDONTWRITEBYTECODE=1
-
-    # run tox
-    # XXX we don't run liveconfig tests because they hang sometimes
-    # see https://github.com/deltachat/deltachat-core-rust/issues/331
-    # unset DCC_PY_LIVECONFIG
-
-    tox --workdir "$TOXWORKDIR" -e lint,py35,py36,py37,auditwheels
-    popd
-fi
+# disable live-account testing to speed up test runs and wheel building
+# XXX we may switch on some live-tests on for better ensurances 
+# Note that the independent remote_tests_python step does all kinds of
+# live-testing already. 
+unset DCC_PY_LIVECONFIG
+unset DCC_NEW_TMP_EMAIL
+tox --workdir "$TOXWORKDIR" -e py35,py36,py37,py38,auditwheels
+popd
 
 
-if [ -n "$DOCS" ]; then
-    echo -----------------------
-    echo generating python docs
-    echo -----------------------
-    (cd python && tox --workdir "$TOXWORKDIR" -e doc)
-fi
+echo -----------------------
+echo generating python docs
+echo -----------------------
+(cd python && tox --workdir "$TOXWORKDIR" -e doc)

deltachat-ffi/Cargo.toml

@@ -1,11 +1,11 @@
 [package]
 name = "deltachat_ffi"
-version = "1.0.0-alpha.4"
+version = "1.28.0"
 description = "Deltachat FFI"
-authors = ["dignifiedquire <dignifiedquire@gmail.com>"]
+authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 readme = "README.md"
-license = "MIT OR Apache-2.0"
+license = "MPL-2.0"
 
 keywords = ["deltachat", "chat", "openpgp", "email", "encryption"]
 categories = ["cryptography", "std", "email"]
@@ -19,9 +19,11 @@ deltachat = { path = "../", default-features = false }
 libc = "0.2"
 human-panic = "1.0.1"
 num-traits = "0.2.6"
+serde_json = "1.0"
+anyhow = "1.0.28"
+thiserror = "1.0.14"
 
 [features]
-default = ["vendored", "nightly", "ringbuf"]
+default = ["vendored", "nightly"]
 vendored = ["deltachat/vendored"]
 nightly = ["deltachat/nightly"]
-ringbuf = ["deltachat/ringbuf"]

deltachat-ffi/src/dc_array.rs

@@ -1,9 +1,7 @@
 use crate::location::Location;
-use crate::types::*;
 
 /* * the structure behind dc_array_t */
-#[derive(Clone)]
-#[allow(non_camel_case_types)]
+#[derive(Debug, Clone)]
 pub enum dc_array_t {
     Locations(Vec<Location>),
     Uint(Vec<u32>),
@@ -19,7 +17,7 @@ impl dc_array_t {
         dc_array_t::Locations(Vec::with_capacity(capacity))
     }
 
-    pub fn add_id(&mut self, item: uint32_t) {
+    pub fn add_id(&mut self, item: u32) {
         if let Self::Uint(array) = self {
             array.push(item);
         } else {
@@ -35,10 +33,10 @@ impl dc_array_t {
         }
     }
 
-    pub fn get_id(&self, index: usize) -> uint32_t {
+    pub fn get_id(&self, index: usize) -> u32 {
         match self {
             Self::Locations(array) => array[index].location_id,
-            Self::Uint(array) => array[index] as uint32_t,
+            Self::Uint(array) => array[index] as u32,
         }
     }
 

deltachat-ffi/src/string.rs

@@ -0,0 +1,410 @@
+use std::ffi::{CStr, CString};
+use std::ptr;
+
+/// Duplicates a string
+///
+/// returns an empty string if NULL is given, never returns NULL (exits on errors)
+///
+/// # Examples
+///
+/// ```rust,norun
+/// use crate::string::{dc_strdup, to_string_lossy};
+/// unsafe {
+///     let str_a = b"foobar\x00" as *const u8 as *const libc::c_char;
+///     let str_a_copy = dc_strdup(str_a);
+///     assert_eq!(to_string_lossy(str_a_copy), "foobar");
+///     assert_ne!(str_a, str_a_copy);
+/// }
+/// ```
+unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char {
+    let ret: *mut libc::c_char;
+    if !s.is_null() {
+        ret = libc::strdup(s);
+        assert!(!ret.is_null());
+    } else {
+        ret = libc::calloc(1, 1) as *mut libc::c_char;
+        assert!(!ret.is_null());
+    }
+
+    ret
+}
+
+/// Error type for the [OsStrExt] trait
+#[derive(Debug, PartialEq, thiserror::Error)]
+pub(crate) enum CStringError {
+    /// The string contains an interior null byte
+    #[error("String contains an interior null byte")]
+    InteriorNullByte,
+    /// The string is not valid Unicode
+    #[error("String is not valid unicode")]
+    NotUnicode,
+}
+
+/// Extra convenience methods on [std::ffi::OsStr] to work with `*libc::c_char`.
+///
+/// The primary function of this trait is to more easily convert
+/// [OsStr], [OsString] or [Path] into pointers to C strings.  This always
+/// allocates a new string since it is very common for the source
+/// string not to have the required terminal null byte.
+///
+/// It is implemented for `AsRef<std::ffi::OsStr>>` trait, which
+/// allows any type which implements this trait to transparently use
+/// this.  This is how the conversion for [Path] works.
+///
+/// [OsStr]: std::ffi::OsStr
+/// [OsString]: std::ffi::OsString
+/// [Path]: std::path::Path
+///
+/// # Example
+///
+/// ```
+/// use deltachat::dc_tools::{dc_strdup, OsStrExt};
+/// let path = std::path::Path::new("/some/path");
+/// let path_c = path.to_c_string().unwrap();
+/// unsafe {
+///     let mut c_ptr: *mut libc::c_char = dc_strdup(path_c.as_ptr());
+/// }
+/// ```
+pub(crate) trait OsStrExt {
+    /// Convert a  [std::ffi::OsStr] to an [std::ffi::CString]
+    ///
+    /// This is useful to convert e.g. a [std::path::Path] to
+    /// [*libc::c_char] by using
+    /// [Path::as_os_str()](std::path::Path::as_os_str) and
+    /// [CStr::as_ptr()](std::ffi::CStr::as_ptr).
+    ///
+    /// This returns [CString] and not [&CStr] because not all [OsStr]
+    /// slices end with a null byte, particularly those coming from
+    /// [Path] do not have a null byte and having to handle this as
+    /// the caller would defeat the point of this function.
+    ///
+    /// On Windows this requires that the [OsStr] contains valid
+    /// unicode, which should normally be the case for a [Path].
+    ///
+    /// [CString]: std::ffi::CString
+    /// [CStr]: std::ffi::CStr
+    /// [OsStr]: std::ffi::OsStr
+    /// [Path]: std::path::Path
+    ///
+    /// # Errors
+    ///
+    /// Since a C `*char` is terminated by a NULL byte this conversion
+    /// will fail, when the [OsStr] has an interior null byte.  The
+    /// function will return
+    /// `[Err]([CStringError::InteriorNullByte])`.  When converting
+    /// from a [Path] it should be safe to
+    /// [`.unwrap()`](std::result::Result::unwrap) this anyway since a
+    /// [Path] should not contain interior null bytes.
+    ///
+    /// On windows when the string contains invalid Unicode
+    /// `[Err]([CStringError::NotUnicode])` is returned.
+    fn to_c_string(&self) -> Result<CString, CStringError>;
+}
+
+impl<T: AsRef<std::ffi::OsStr>> OsStrExt for T {
+    #[cfg(not(target_os = "windows"))]
+    fn to_c_string(&self) -> Result<CString, CStringError> {
+        use std::os::unix::ffi::OsStrExt;
+        CString::new(self.as_ref().as_bytes()).map_err(|err| match err {
+            std::ffi::NulError { .. } => CStringError::InteriorNullByte,
+        })
+    }
+
+    #[cfg(target_os = "windows")]
+    fn to_c_string(&self) -> Result<CString, CStringError> {
+        os_str_to_c_string_unicode(&self)
+    }
+}
+
+// Implementation for os_str_to_c_string on windows.
+#[allow(dead_code)]
+fn os_str_to_c_string_unicode(
+    os_str: &dyn AsRef<std::ffi::OsStr>,
+) -> Result<CString, CStringError> {
+    match os_str.as_ref().to_str() {
+        Some(val) => CString::new(val.as_bytes()).map_err(|err| match err {
+            std::ffi::NulError { .. } => CStringError::InteriorNullByte,
+        }),
+        None => Err(CStringError::NotUnicode),
+    }
+}
+
+/// Convenience methods/associated functions for working with [CString]
+trait CStringExt {
+    /// Create a new [CString], best effort
+    ///
+    /// Like the [to_string_lossy] this doesn't give up in the face of
+    /// bad input (embedded null bytes in this case) instead it does
+    /// the best it can by stripping the embedded null bytes.
+    fn new_lossy<T: Into<Vec<u8>>>(t: T) -> CString {
+        let mut s = t.into();
+        s.retain(|&c| c != 0);
+        CString::new(s).unwrap_or_default()
+    }
+}
+
+impl CStringExt for CString {}
+
+/// Convenience methods to turn strings into C strings.
+///
+/// To interact with (legacy) C APIs we often need to convert from
+/// Rust strings to raw C strings.  This can be clumsy to do correctly
+/// and the compiler sometimes allows it in an unsafe way.  These
+/// methods make it more succinct and help you get it right.
+pub(crate) trait Strdup {
+    /// Allocate a new raw C `*char` version of this string.
+    ///
+    /// This allocates a new raw C string which must be freed using
+    /// `free`.  It takes care of some common pitfalls with using
+    /// [CString.as_ptr].
+    ///
+    /// [CString.as_ptr]: std::ffi::CString.as_ptr
+    ///
+    /// # Panics
+    ///
+    /// This function will panic when the original string contains an
+    /// interior null byte as this can not be represented in raw C
+    /// strings.
+    unsafe fn strdup(&self) -> *mut libc::c_char;
+}
+
+impl<T: AsRef<str>> Strdup for T {
+    unsafe fn strdup(&self) -> *mut libc::c_char {
+        let tmp = CString::new_lossy(self.as_ref());
+        dc_strdup(tmp.as_ptr())
+    }
+}
+
+// We can not implement for AsRef<OsStr> because we already implement
+// AsRev<str> and this conflicts.  So implement for Path directly.
+impl Strdup for std::path::Path {
+    unsafe fn strdup(&self) -> *mut libc::c_char {
+        let tmp = self.to_c_string().unwrap_or_else(|_| CString::default());
+        dc_strdup(tmp.as_ptr())
+    }
+}
+
+/// Convenience methods to turn optional strings into C strings.
+///
+/// This is the same as the [Strdup] trait but a different trait name
+/// to work around the type system not allowing to implement [Strdup]
+/// for `Option<impl Strdup>` When we already have an [Strdup] impl
+/// for `AsRef<&str>`.
+///
+/// When the [Option] is [Option::Some] this behaves just like
+/// [Strdup::strdup], when it is [Option::None] a null pointer is
+/// returned.
+pub(crate) trait OptStrdup {
+    /// Allocate a new raw C `*char` version of this string, or NULL.
+    ///
+    /// See [Strdup::strdup] for details.
+    unsafe fn strdup(&self) -> *mut libc::c_char;
+}
+
+impl<T: AsRef<str>> OptStrdup for Option<T> {
+    unsafe fn strdup(&self) -> *mut libc::c_char {
+        match self {
+            Some(s) => {
+                let tmp = CString::new_lossy(s.as_ref());
+                dc_strdup(tmp.as_ptr())
+            }
+            None => ptr::null_mut(),
+        }
+    }
+}
+
+pub(crate) fn to_string_lossy(s: *const libc::c_char) -> String {
+    if s.is_null() {
+        return "".into();
+    }
+
+    let cstr = unsafe { CStr::from_ptr(s) };
+
+    cstr.to_string_lossy().to_string()
+}
+
+pub(crate) fn to_opt_string_lossy(s: *const libc::c_char) -> Option<String> {
+    if s.is_null() {
+        return None;
+    }
+
+    Some(to_string_lossy(s))
+}
+
+/// Convert a C `*char` pointer to a [std::path::Path] slice.
+///
+/// This converts a `*libc::c_char` pointer to a [Path] slice.  This
+/// essentially has to convert the pointer to [std::ffi::OsStr] to do
+/// so and thus is the inverse of [OsStrExt::to_c_string].  Just like
+/// [OsStrExt::to_c_string] requires valid Unicode on Windows, this
+/// requires that the pointer contains valid UTF-8 on Windows.
+///
+/// Because this returns a reference the [Path] silce can not outlive
+/// the original pointer.
+///
+/// [Path]: std::path::Path
+#[cfg(not(target_os = "windows"))]
+pub(crate) fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path {
+    assert!(!s.is_null(), "cannot be used on null pointers");
+    use std::os::unix::ffi::OsStrExt;
+    unsafe {
+        let c_str = std::ffi::CStr::from_ptr(s).to_bytes();
+        let os_str = std::ffi::OsStr::from_bytes(c_str);
+        std::path::Path::new(os_str)
+    }
+}
+
+// as_path() implementation for windows, documented above.
+#[cfg(target_os = "windows")]
+pub(crate) fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path {
+    as_path_unicode(s)
+}
+
+// Implementation for as_path() on Windows.
+//
+// Having this as a separate function means it can be tested on unix
+// too.
+#[allow(dead_code)]
+fn as_path_unicode<'a>(s: *const libc::c_char) -> &'a std::path::Path {
+    assert!(!s.is_null(), "cannot be used on null pointers");
+
+    let cstr = unsafe { CStr::from_ptr(s) };
+    let str = cstr.to_str().unwrap_or_else(|err| panic!("{}", err));
+
+    std::path::Path::new(str)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use libc::{free, strcmp};
+
+    #[test]
+    fn test_os_str_to_c_string_cwd() {
+        let some_dir = std::env::current_dir().unwrap();
+        some_dir.as_os_str().to_c_string().unwrap();
+    }
+
+    #[test]
+    fn test_os_str_to_c_string_unicode() {
+        let some_str = String::from("/some/valid/utf8");
+        let some_dir = std::path::Path::new(&some_str);
+        assert_eq!(
+            some_dir.as_os_str().to_c_string().unwrap(),
+            CString::new("/some/valid/utf8").unwrap()
+        );
+    }
+
+    #[test]
+    fn test_os_str_to_c_string_nul() {
+        let some_str = std::ffi::OsString::from("foo\x00bar");
+        assert_eq!(
+            some_str.to_c_string().err().unwrap(),
+            CStringError::InteriorNullByte
+        )
+    }
+
+    #[test]
+    fn test_path_to_c_string_cwd() {
+        let some_dir = std::env::current_dir().unwrap();
+        some_dir.to_c_string().unwrap();
+    }
+
+    #[test]
+    fn test_path_to_c_string_unicode() {
+        let some_str = String::from("/some/valid/utf8");
+        let some_dir = std::path::Path::new(&some_str);
+        assert_eq!(
+            some_dir.as_os_str().to_c_string().unwrap(),
+            CString::new("/some/valid/utf8").unwrap()
+        );
+    }
+
+    #[test]
+    fn test_os_str_to_c_string_unicode_fn() {
+        let some_str = std::ffi::OsString::from("foo");
+        assert_eq!(
+            os_str_to_c_string_unicode(&some_str).unwrap(),
+            CString::new("foo").unwrap()
+        );
+    }
+
+    #[test]
+    fn test_path_to_c_string_unicode_fn() {
+        let some_str = String::from("/some/path");
+        let some_path = std::path::Path::new(&some_str);
+        assert_eq!(
+            os_str_to_c_string_unicode(&some_path).unwrap(),
+            CString::new("/some/path").unwrap()
+        );
+    }
+
+    #[test]
+    fn test_os_str_to_c_string_unicode_fn_nul() {
+        let some_str = std::ffi::OsString::from("fooz\x00bar");
+        assert_eq!(
+            os_str_to_c_string_unicode(&some_str).err().unwrap(),
+            CStringError::InteriorNullByte
+        );
+    }
+
+    #[test]
+    fn test_as_path() {
+        let some_path = CString::new("/some/path").unwrap();
+        let ptr = some_path.as_ptr();
+        assert_eq!(as_path(ptr), std::ffi::OsString::from("/some/path"))
+    }
+
+    #[test]
+    fn test_as_path_unicode_fn() {
+        let some_path = CString::new("/some/path").unwrap();
+        let ptr = some_path.as_ptr();
+        assert_eq!(as_path_unicode(ptr), std::ffi::OsString::from("/some/path"));
+    }
+
+    #[test]
+    fn test_cstring_new_lossy() {
+        assert!(CString::new("hel\x00lo").is_err());
+        assert!(CString::new(String::from("hel\x00o")).is_err());
+        let r = CString::new("hello").unwrap();
+        assert_eq!(CString::new_lossy("hello"), r);
+        assert_eq!(CString::new_lossy("hel\x00lo"), r);
+        assert_eq!(CString::new_lossy(String::from("hello")), r);
+        assert_eq!(CString::new_lossy(String::from("hel\x00lo")), r);
+    }
+
+    #[test]
+    fn test_strdup_str() {
+        unsafe {
+            let s = "hello".strdup();
+            let cmp = strcmp(s, b"hello\x00" as *const u8 as *const libc::c_char);
+            free(s as *mut libc::c_void);
+            assert_eq!(cmp, 0);
+        }
+    }
+
+    #[test]
+    fn test_strdup_string() {
+        unsafe {
+            let s = String::from("hello").strdup();
+            let cmp = strcmp(s, b"hello\x00" as *const u8 as *const libc::c_char);
+            free(s as *mut libc::c_void);
+            assert_eq!(cmp, 0);
+        }
+    }
+
+    #[test]
+    fn test_strdup_opt_string() {
+        unsafe {
+            let s = Some("hello");
+            let c = s.strdup();
+            let cmp = strcmp(c, b"hello\x00" as *const u8 as *const libc::c_char);
+            free(c as *mut libc::c_void);
+            assert_eq!(cmp, 0);
+
+            let s: Option<&str> = None;
+            let c = s.strdup();
+            assert_eq!(c, ptr::null_mut());
+        }
+    }
+}

deltachat_derive/Cargo.toml

@@ -1,12 +1,13 @@
 [package]
 name = "deltachat_derive"
-version = "0.1.0"
-authors = ["Dmitry Bogatov <KAction@debian.org>"]
+version = "2.0.0"
+authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
+license = "MPL-2.0"
 
 [lib]
 proc-macro = true
 
 [dependencies]
-syn = "0.14.4"
-quote = "0.6.3"
+syn = "1.0.13"
+quote = "1.0.2"

deltachat_derive/src/lib.rs

@@ -3,7 +3,6 @@ extern crate proc_macro;
 
 use crate::proc_macro::TokenStream;
 use quote::quote;
-use syn;
 
 // For now, assume (not check) that these macroses are applied to enum without
 // data.  If this assumption is violated, compiler error will point to

draft/group-sync.rst

@@ -0,0 +1,126 @@
+
+Problem: missing eventual group consistency
+--------------------------------------------
+
+If group members are concurrently adding new members,
+the new members will miss each other's additions, example:
+
+- Alice and Bob are in a two-member group
+
+- Alice adds Carol, concurrently Bob adds Doris
+
+- Carol will see a three-member group (Alice, Bob, Carol),
+  Doris will see a different three-member group (Alice, Bob, Doris),
+  and only Alice and Bob will have all four members.
+
+Note that for verified groups any mitigation mechanism likely
+needs to make all clients to know who originally added a member.
+
+
+solution: memorize+attach (possible encrypted) chat-meta mime messages
+----------------------------------------------------------------------
+
+For reference, please see https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#add-and-remove-members how MemberAdded/Removed messages are shaped.
+
+
+- All Chat-Group-Member-Added/Removed messages are recorded in their
+  full raw (signed and encrypted) mime-format in the DB
+
+- If an incoming member-add/member-delete messages has a member list
+  which is, apart from the added/removed member, not consistent
+  with our own view, broadcast a "Chat-Group-Member-Correction" message to
+  all members, attaching the original added/removed mime-message for all mismatching
+  contacts.  If we have no relevant add/del information, don't send a
+  correction message out.
+
+- Upong receiving added/removed attachments we don't do the
+  check_consistency+correction message dance.
+  This avoids recursion problems and hard-to-reason-about chatter.
+
+Notes:
+
+- mechanism works for both encrypted and unencrypted add/del messages
+
+- we already have a "mime_headers" column in the DB for each incoming message.
+  We could extend it to also include the payload and store mime unconditionally
+  for member-added/removed messages.
+
+- multiple member-added/removed messages can be attached in a single
+  correction message
+
+- it is minimal on the number of overall messages to reach group consistency
+  (best-case: no extra messages, the ABCD case above: max two extra messages)
+
+- somewhat backward compatible: older clients will probably ignore
+  messages which are signed by someone who is not the outer From-address.
+
+- the correction-protocol also helps with dropped messages.  If a member
+  did not see a member-added/removed message, the next member add/removed
+  message in the group will likely heal group consistency for this member.
+
+- we can quite easily extend the mechanism to also provide the group-avatar or
+  other meta-information.
+
+Discussions of variants
+++++++++++++++++++++++++
+
+- instead of acting on MemberAdded/Removed message we could send
+  corrections for any received message that addresses inconsistent group members but
+  a) this would delay group-membership healing
+  b) could lead to a lot of members sending corrections
+
+- instead of broadcasting correction messages we could only send it to
+  the sender of the inconsistent member-added/removed message.
+  A receiver of such a correction message would then need to forward
+  the message to the members it thinks also have an inconsistent view.
+  This sounds complicated and error-prone.  Concretely, if Alice
+  receives Bob's "Member-added: Doris" message, then Alice
+  broadcasting the correction message with "Member-added: Carol"
+  would reach all four members, healing group consistency in one step.
+  If Bob meanwhile receives Alice's "Member-Added: Carol" message,
+  Bob would broadcast a correction message to all four members as well.
+  (Imagine a situation where Alice/Bob added Carol/Doris
+  while both being in an offline or bad-connection situation).
+
+
+solution2: repeat member-added/removed messages
+---------------------------------------------------
+
+Introduce a new Chat-Group-Member-Changed header and deprecate Chat-Group-Member-Added/Removed
+but keep sending out the old headers until the new protocol is sufficiently deployed.
+
+The new Chat-Group-Member-Changed header contains a Time-to-Live number (TTL)
+which controls repetition of the signed "add/del e-mail address" payload.
+
+Example::
+
+    Chat-Group-Member-Changed: TTL add "somedisplayname" someone@example.org
+        owEBYQGe/pANAwACAY47A6J5t3LWAcsxYgBeTQypYWRkICJzb21lZGlzcGxheW5h
+        bWUiIHNvbWVvbmVAZXhhbXBsZS5vcmcgCokBHAQAAQIABgUCXk0MqQAKCRCOOwOi
+        ebdy1hfRB/wJ74tgFQulicthcv9n+ZsqzwOtBKMEVIHqJCzzDB/Hg/2z8ogYoZNR
+        iUKKrv3Y1XuFvdKyOC+wC/unXAWKFHYzY6Tv6qDp6r+amt+ad+8Z02q53h9E55IP
+        FUBdq2rbS8hLGjQB+mVRowYrUACrOqGgNbXMZjQfuV7fSc7y813OsCQgi3tjstup
+        b+uduVzxCp3PChGhcZPs3iOGCnQvSB8VAaLGMWE2d7nTo/yMQ0Jx69x5qwfXogTk
+        mTt5rOJyrosbtf09TMKFzGgtqBcEqHLp3+mQpZQ+WHUKAbsRa8Jc9DOUOSKJ8SNM
+        clKdskprY+4LY0EBwLD3SQ7dPkTITCRD
+        =P6GG
+
+TTL is set to "2" on an initial Chat-Group-Member-Changed add/del message.
+Receivers will apply the add/del change to the group-membership,
+decrease the TTL by 1, and if TTL>0 re-sent the header.
+
+The "add|del e-mail address" payload is pgp-signed and repeated verbatim.
+This allows to propagate, in a cryptographically secured way,
+who added a member. This is particularly important for allowing
+to show in verified groups who added a member (planned).
+
+Disadvantage to solution 1:
+
+- requires to specify encoding and precise rules for what/how is signed.
+
+- causes O(N^2) extra messages
+
+- Not easily extendable for other things (without introducing a new
+  header / encoding)
+
+

draft/mailing_list.rst

@@ -0,0 +1,19 @@
+2. Mailing lists are now only detected by the ListId header, because how should DC sort mailing lists if the ListId header is unset and only the precedence header says it's a mailing list (?). We could just hide these emails (with "junk" or "list" precedence), as we did before, if there is a reason to do so. 
+
+   I do not actually understand why such emails were hidden in the first place, though; if there is an automatic answer stating that someone is out of office or if an email could not be delivered (these are the usual reasons why such emails are sent), I would want to know about this as a user.
+   
+  Björn: to the ListId-header: it is already an improvement to use only that and to keep ignoring mails with Precedence-header. historically, i ignored all these mails to reduce noise that is created from them (eg. contacts should not be added to the suggestion list...) and to be able to concentrate on other things first.
+   
+   
+3. for `List-Id: foo <bla>`, bla now is the id and foo is the name. For GitHub and GitLab notifications this is bad because all notifications will go into one chat. Maybe we should instead take the "in-reply-to" header to find out what messages belong together. For normal mailing lists, of course, this will create a second chat if someone does not use the "Reply" button to write to that mailing list.
+
+  Björn: well, having all notifications in one chat is not that bad. i would just follow the guidelines for now, use the ID and not the Name.
+
+
+4. Currently a mailing list is shown as an empty group (ChatType `Group`) ("0 members"). 
+
+   Maybe we should change the ChatType to `Single` because this way, the UI would fit better. Disadvantage: We can't show the different senders of the messages. 
+   
+   Or we introduce a `ChatType` `MailingList`. Very big disadvantage: We would have to adapt all UI project and it would be nice if we could keep the changes within the core.
+
+5. Contacts from mailings lists stay "unknown" now and are not shown in the contacts suggestion list..

draft/mailing_list_current_state.rst

@@ -0,0 +1,13 @@
+1. Mailing lists with Precedence-header and without List-Id are shown as normal messages.
+   
+2. for `List-Id: foo <bla>`, bla now is the id and foo is the name. If foo is not present, bla is taken as the name as well. For GitHub and GitLab notifications all notifications will go into one chat. To distinguish, all subjects are shown in mailing lists.
+
+3. Currently a mailing list is shown as a group with SELF and the List-Id. I could not find any stable way to get a mail address that could be called the "mailing list address". To get this to work, I disabled the may_be_valid_addr check. (to be discussed...)
+
+4. Contacts from mailings lists stay "unknown" and are not shown in the contacts suggestion list.
+
+5. In general, only the From: address is taken as the author. If the domain matches with the List-Id domain (like `deltachat.github.com` and `Hocuri <notifications@notification.github.com>`) then the display name is added to the email address -> `Hocuri - notifications@notification.github.com`. This is not really nice, but as the UIs distinguish bettween contacts only based on the address this was the only way I could find without changing the API. 
+
+   TODO: Prevent the user from sending emails to such contacts (like Hocuri - notifications@notification.github.com)
+
+6. You can't send to mailing lists.

draft/mailing_list_managers.md

@@ -0,0 +1,71 @@
+# Mailing list integration of Delta Chat
+
+A collection of information to help with the integration of mailing lists into Delta Chat.
+
+## Chat name
+
+How should the chat be titled?
+
+It could be taken from the email header `List-Id`:
+
+### Mailman 
+
+* Schema: `${list-description} <${list-local-part}.${list-domain}>`
+* $list-description could be many words, should be truncated. Could also be empty, though.
+
+### Schleuder
+* Schema: `<${list-local-part}.${list-domain}>`
+* No name available.
+* Could be absent: <https://0xacab.org/schleuder/schleuder/-/blob/master/lib/schleuder/mail/message.rb#L357-358>.
+
+
+### Sympa
+* Schema: `<${list-local-part}.${list-domain}>`
+* No name available.
+* Always present <https://github.com/sympa-community/sympa/blob/sympa-6.2/src/lib/Sympa/Spindle/TransformOutgoing.pm#L110-L118>, <https://github.com/sympa-community/sympa/blob/sympa-6.2/src/lib/Sympa/List.pm#L6832-L6833>.
+
+
+## Sender name
+
+What's the name of the person that actually sent the message?
+
+Some MLM change the sender information to avoid problems with DMARC.
+
+### Mailman
+
+* Since v2.1.16 the `From` depends on the list's configuration and possibly on the sender-domain's DMARC-configuration — i.e. messages from the same list can arrive one of the Variants A-D!
+* Documentation of config options:
+ * <https://wiki.list.org/DOC/Mailman%202.1%20List%20Administrators%20Manual#line-544>,
+ * <https://wiki.list.org/DOC/Mailman%202.1%20List%20Administrators%20Manual#line-163>,
+ * <https://wiki.list.org/DEV/DMARC>
+* Variant A: `From` is unchanged.
+* Variant B. `From` is mangled like this: `${sender-name} via ${list-name} <${list-addr-spec}>`. The original sender is put into `Reply-To`.
+* Variant C. `From` is mangled like this: `${sender-name} <${encoded-sender-addr-spec}@${list-domain}>`. The original sender is put into `Reply-To`.
+* Variant D: `From` is set to ${list-name-addr}, the originally sent message is included as mime-part.
+* Variant E: `From` is set to ${list-name-addr}, original sender information is removed.
+
+### Schleuder
+
+* Visible only in mime-body (which is possibly encrypted), or not at all.
+* The first `text/plain` mime-part may include the information, as taken from the original message: `From: ${sender-name-addr}`.
+
+### Sympa
+* Depends on the list's configuration.
+* Documentation:
+ * <https://sympa-community.github.io/manual/customize/dmarc-protection.html>,
+ * <https://sympa-community.github.io/gpldoc/man/sympa.conf.5.html#dmarc-protection>.
+* Variant A: `From` is unchanged.
+* Variant B: `From` is mangled like Mailman Variant B, but the original sender information is put into `X-Original-From`.
+
+
+## Autocrypt
+
+### Mailman
+* Not supported.
+
+### Schleuder
+* A list's key is included in sent messages (as of version 3.5.0) (optional, by default active): <https://0xacab.org/schleuder/schleuder/-/blob/master/lib/schleuder/mail/message.rb#L349-355>.
+* Incoming keys are not yet looked at (that feature is planned: <https://0xacab.org/schleuder/schleuder/issues/435>).
+
+### Sympa
+* Not supported.

examples/repl/main.rs

@@ -7,29 +7,25 @@
 #[macro_use]
 extern crate deltachat;
 #[macro_use]
-extern crate failure;
-#[macro_use]
 extern crate lazy_static;
 #[macro_use]
 extern crate rusqlite;
 
 use std::borrow::Cow::{self, Borrowed, Owned};
 use std::io::{self, Write};
+use std::path::Path;
 use std::process::Command;
-use std::ptr;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex, RwLock};
 
+use anyhow::{bail, Error};
+use deltachat::chat::ChatId;
 use deltachat::config;
-use deltachat::configure::*;
-use deltachat::constants::*;
 use deltachat::context::*;
-use deltachat::dc_tools::*;
 use deltachat::job::*;
 use deltachat::oauth2::*;
 use deltachat::securejoin::*;
-use deltachat::types::*;
-use deltachat::x::*;
+use deltachat::Event;
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
 use rustyline::config::OutputStreamType;
 use rustyline::error::ReadlineError;
@@ -44,100 +40,76 @@ use self::cmdline::*;
 
 // Event Handler
 
-unsafe extern "C" fn receive_event(
-    _context: &Context,
-    event: Event,
-    data1: uintptr_t,
-    data2: uintptr_t,
-) -> uintptr_t {
+fn receive_event(_context: &Context, event: Event) {
     match event {
-        Event::GET_STRING => {}
-        Event::INFO => {
+        Event::Info(msg) => {
             /* do not show the event as this would fill the screen */
-            println!("{}", to_string(data2 as *const _),);
+            println!("{}", msg);
         }
-        Event::SMTP_CONNECTED => {
-            println!("[DC_EVENT_SMTP_CONNECTED] {}", to_string(data2 as *const _));
+        Event::SmtpConnected(msg) => {
+            println!("[DC_EVENT_SMTP_CONNECTED] {}", msg);
         }
-        Event::IMAP_CONNECTED => {
-            println!("[DC_EVENT_IMAP_CONNECTED] {}", to_string(data2 as *const _),);
+        Event::ImapConnected(msg) => {
+            println!("[DC_EVENT_IMAP_CONNECTED] {}", msg);
         }
-        Event::SMTP_MESSAGE_SENT => {
-            println!(
-                "[DC_EVENT_SMTP_MESSAGE_SENT] {}",
-                to_string(data2 as *const _),
-            );
+        Event::SmtpMessageSent(msg) => {
+            println!("[DC_EVENT_SMTP_MESSAGE_SENT] {}", msg);
         }
-        Event::WARNING => {
-            println!("[Warning] {}", to_string(data2 as *const _),);
+        Event::Warning(msg) => {
+            println!("[Warning] {}", msg);
         }
-        Event::ERROR => {
-            println!(
-                "\x1b[31m[DC_EVENT_ERROR] {}\x1b[0m",
-                to_string(data2 as *const _),
-            );
+        Event::Error(msg) => {
+            println!("\x1b[31m[DC_EVENT_ERROR] {}\x1b[0m", msg);
         }
-        Event::ERROR_NETWORK => {
-            println!(
-                "\x1b[31m[DC_EVENT_ERROR_NETWORK] first={}, msg={}\x1b[0m",
-                data1 as usize,
-                to_string(data2 as *const _),
-            );
+        Event::ErrorNetwork(msg) => {
+            println!("\x1b[31m[DC_EVENT_ERROR_NETWORK] msg={}\x1b[0m", msg);
         }
-        Event::ERROR_SELF_NOT_IN_GROUP => {
-            println!(
-                "\x1b[31m[DC_EVENT_ERROR_SELF_NOT_IN_GROUP] {}\x1b[0m",
-                to_string(data2 as *const _),
-            );
+        Event::ErrorSelfNotInGroup(msg) => {
+            println!("\x1b[31m[DC_EVENT_ERROR_SELF_NOT_IN_GROUP] {}\x1b[0m", msg);
         }
-        Event::MSGS_CHANGED => {
+        Event::MsgsChanged { chat_id, msg_id } => {
             print!(
-                "\x1b[33m{{Received DC_EVENT_MSGS_CHANGED({}, {})}}\n\x1b[0m",
-                data1 as usize, data2 as usize,
+                "\x1b[33m{{Received DC_EVENT_MSGS_CHANGED(chat_id={}, msg_id={})}}\n\x1b[0m",
+                chat_id, msg_id,
             );
         }
-        Event::CONTACTS_CHANGED => {
+        Event::ContactsChanged(_) => {
             print!("\x1b[33m{{Received DC_EVENT_CONTACTS_CHANGED()}}\n\x1b[0m");
         }
-        Event::LOCATION_CHANGED => {
+        Event::LocationChanged(contact) => {
             print!(
-                "\x1b[33m{{Received DC_EVENT_LOCATION_CHANGED(contact={})}}\n\x1b[0m",
-                data1 as usize,
+                "\x1b[33m{{Received DC_EVENT_LOCATION_CHANGED(contact={:?})}}\n\x1b[0m",
+                contact,
             );
         }
-        Event::CONFIGURE_PROGRESS => {
+        Event::ConfigureProgress(progress) => {
             print!(
                 "\x1b[33m{{Received DC_EVENT_CONFIGURE_PROGRESS({} ‰)}}\n\x1b[0m",
-                data1 as usize,
+                progress,
             );
         }
-        Event::IMEX_PROGRESS => {
+        Event::ImexProgress(progress) => {
             print!(
                 "\x1b[33m{{Received DC_EVENT_IMEX_PROGRESS({} ‰)}}\n\x1b[0m",
-                data1 as usize,
+                progress,
             );
         }
-        Event::IMEX_FILE_WRITTEN => {
+        Event::ImexFileWritten(file) => {
             print!(
                 "\x1b[33m{{Received DC_EVENT_IMEX_FILE_WRITTEN({})}}\n\x1b[0m",
-                to_string(data1 as *const _)
+                file.display()
             );
         }
-        Event::CHAT_MODIFIED => {
+        Event::ChatModified(chat) => {
             print!(
                 "\x1b[33m{{Received DC_EVENT_CHAT_MODIFIED({})}}\n\x1b[0m",
-                data1 as usize,
+                chat
             );
         }
         _ => {
-            print!(
-                "\x1b[33m{{Received {:?}({}, {})}}\n\x1b[0m",
-                event, data1 as usize, data2 as usize,
-            );
+            print!("\x1b[33m{{Received {:?}}}\n\x1b[0m", event);
         }
     }
-
-    0
 }
 
 // Threads for waiting for messages and for jobs
@@ -175,11 +147,11 @@ fn start_threads(c: Arc<RwLock<Context>>) {
     let ctx = c.clone();
     let handle_imap = std::thread::spawn(move || loop {
         while_running!({
-            perform_imap_jobs(&ctx.read().unwrap());
-            perform_imap_fetch(&ctx.read().unwrap());
+            perform_inbox_jobs(&ctx.read().unwrap());
+            perform_inbox_fetch(&ctx.read().unwrap());
             while_running!({
                 let context = ctx.read().unwrap();
-                perform_imap_idle(&context);
+                perform_inbox_idle(&context);
             });
         });
     });
@@ -227,7 +199,7 @@ fn stop_threads(context: &Context) {
         println!("Stopping threads");
         IS_RUNNING.store(false, Ordering::Relaxed);
 
-        interrupt_imap_idle(context);
+        interrupt_inbox_idle(context);
         interrupt_mvbox_idle(context);
         interrupt_sentbox_idle(context);
         interrupt_smtp_idle(context);
@@ -260,7 +232,7 @@ impl Completer for DcHelper {
     }
 }
 
-const IMEX_COMMANDS: [&'static str; 12] = [
+const IMEX_COMMANDS: [&str; 12] = [
     "initiate-key-transfer",
     "get-setupcodebegin",
     "continue-key-transfer",
@@ -275,7 +247,7 @@ const IMEX_COMMANDS: [&'static str; 12] = [
     "stop",
 ];
 
-const DB_COMMANDS: [&'static str; 11] = [
+const DB_COMMANDS: [&str; 11] = [
     "info",
     "open",
     "close",
@@ -289,7 +261,7 @@ const DB_COMMANDS: [&'static str; 11] = [
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&'static str; 24] = [
+const CHAT_COMMANDS: [&str; 26] = [
     "listchats",
     "listarchived",
     "chat",
@@ -313,9 +285,11 @@ const CHAT_COMMANDS: [&'static str; 24] = [
     "listmedia",
     "archive",
     "unarchive",
+    "pin",
+    "unpin",
     "delchat",
 ];
-const MESSAGE_COMMANDS: [&'static str; 8] = [
+const MESSAGE_COMMANDS: [&str; 8] = [
     "listmsgs",
     "msginfo",
     "listfresh",
@@ -325,7 +299,7 @@ const MESSAGE_COMMANDS: [&'static str; 8] = [
     "unstar",
     "delmsg",
 ];
-const CONTACT_COMMANDS: [&'static str; 6] = [
+const CONTACT_COMMANDS: [&str; 6] = [
     "listcontacts",
     "listverified",
     "addcontact",
@@ -333,8 +307,17 @@ const CONTACT_COMMANDS: [&'static str; 6] = [
     "delcontact",
     "cleanupcontacts",
 ];
-const MISC_COMMANDS: [&'static str; 9] = [
-    "getqr", "getbadqr", "checkqr", "event", "fileinfo", "clear", "exit", "quit", "help",
+const MISC_COMMANDS: [&str; 10] = [
+    "getqr",
+    "getbadqr",
+    "checkqr",
+    "event",
+    "fileinfo",
+    "clear",
+    "exit",
+    "quit",
+    "help",
+    "estimatedeletion",
 ];
 
 impl Hinter for DcHelper {
@@ -359,8 +342,8 @@ impl Hinter for DcHelper {
     }
 }
 
-static COLORED_PROMPT: &'static str = "\x1b[1;32m> \x1b[0m";
-static PROMPT: &'static str = "> ";
+static COLORED_PROMPT: &str = "\x1b[1;32m> \x1b[0m";
+static PROMPT: &str = "> ";
 
 impl Highlighter for DcHelper {
     fn highlight_prompt<'p>(&self, prompt: &'p str) -> Cow<'p, str> {
@@ -386,16 +369,16 @@ impl Highlighter for DcHelper {
 
 impl Helper for DcHelper {}
 
-fn main_0(args: Vec<String>) -> Result<(), failure::Error> {
-    let mut context = dc_context_new(Some(receive_event), ptr::null_mut(), Some("CLI".into()));
-
-    if args.len() == 2 {
-        if unsafe { !dc_open(&mut context, &args[1], None) } {
-            println!("Error: Cannot open {}.", args[0],);
-        }
-    } else if args.len() != 1 {
+fn main_0(args: Vec<String>) -> Result<(), Error> {
+    if args.len() < 2 {
         println!("Error: Bad arguments, expected [db-name].");
+        bail!("No db-name specified");
     }
+    let context = Context::new(
+        Box::new(receive_event),
+        "CLI".into(),
+        Path::new(&args[1]).to_path_buf(),
+    )?;
 
     println!("Delta Chat Core is awaiting your commands.");
 
@@ -428,7 +411,7 @@ fn main_0(args: Vec<String>) -> Result<(), failure::Error> {
                 // TODO: ignore "set mail_pw"
                 rl.add_history_entry(line.as_str());
                 let ctx = ctx.clone();
-                match unsafe { handle_cmd(line.trim(), ctx) } {
+                match handle_cmd(line.trim(), ctx) {
                     Ok(ExitResult::Continue) => {}
                     Ok(ExitResult::Exit) => break,
                     Err(err) => println!("Error: {}", err),
@@ -459,15 +442,10 @@ enum ExitResult {
     Exit,
 }
 
-unsafe fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, failure::Error> {
+fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, Error> {
     let mut args = line.splitn(2, ' ');
     let arg0 = args.next().unwrap_or_default();
     let arg1 = args.next().unwrap_or_default();
-    let arg1_c = if arg1.is_empty() {
-        std::ptr::null()
-    } else {
-        arg1.strdup()
-    };
 
     match arg0 {
         "connect" => {
@@ -485,14 +463,14 @@ unsafe fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult
         }
         "imap-jobs" => {
             if HANDLE.clone().lock().unwrap().is_some() {
-                println!("imap-jobs are already running in a thread.");
+                println!("inbox-jobs are already running in a thread.");
             } else {
-                perform_imap_jobs(&ctx.read().unwrap());
+                perform_inbox_jobs(&ctx.read().unwrap());
             }
         }
         "configure" => {
             start_threads(ctx.clone());
-            configure(&ctx.read().unwrap());
+            ctx.read().unwrap().configure();
         }
         "oauth2" => {
             if let Some(addr) = ctx.read().unwrap().get_config(config::Config::Addr) {
@@ -516,9 +494,10 @@ unsafe fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult
         }
         "getqr" | "getbadqr" => {
             start_threads(ctx.clone());
-            if let Some(mut qr) =
-                dc_get_securejoin_qr(&ctx.read().unwrap(), arg1.parse().unwrap_or_default())
-            {
+            if let Some(mut qr) = dc_get_securejoin_qr(
+                &ctx.read().unwrap(),
+                ChatId::new(arg1.parse().unwrap_or_default()),
+            ) {
                 if !qr.is_empty() {
                     if arg0 == "getbadqr" && qr.len() > 40 {
                         qr.replace_range(12..22, "0000000000")
@@ -543,12 +522,10 @@ unsafe fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult
         _ => dc_cmdline(&ctx.read().unwrap(), line)?,
     }
 
-    free(arg1_c as *mut _);
-
     Ok(ExitResult::Continue)
 }
 
-pub fn main() -> Result<(), failure::Error> {
+pub fn main() -> Result<(), Error> {
     let _ = pretty_env_logger::try_init();
 
     let args: Vec<String> = std::env::args().collect();

examples/simple.rs

@@ -1,6 +1,5 @@
 extern crate deltachat;
 
-use std::ffi::CStr;
 use std::sync::{Arc, RwLock};
 use std::{thread, time};
 use tempfile::tempdir;
@@ -8,128 +7,105 @@ use tempfile::tempdir;
 use deltachat::chat;
 use deltachat::chatlist::*;
 use deltachat::config;
-use deltachat::configure::*;
-use deltachat::constants::Event;
 use deltachat::contact::*;
 use deltachat::context::*;
 use deltachat::job::{
-    perform_imap_fetch, perform_imap_idle, perform_imap_jobs, perform_smtp_idle, perform_smtp_jobs,
+    perform_inbox_fetch, perform_inbox_idle, perform_inbox_jobs, perform_smtp_idle,
+    perform_smtp_jobs,
 };
+use deltachat::Event;
 
-extern "C" fn cb(_ctx: &Context, event: Event, data1: usize, data2: usize) -> usize {
-    println!("[{:?}]", event);
+fn cb(_ctx: &Context, event: Event) {
+    print!("[{:?}]", event);
 
     match event {
-        Event::CONFIGURE_PROGRESS => {
-            println!("  progress: {}", data1);
-            0
+        Event::ConfigureProgress(progress) => {
+            println!("  progress: {}", progress);
         }
-        Event::INFO | Event::WARNING | Event::ERROR | Event::ERROR_NETWORK => {
-            println!(
-                "  {}",
-                unsafe { CStr::from_ptr(data2 as *const _) }
-                    .to_str()
-                    .unwrap()
-            );
-            0
+        Event::Info(msg) | Event::Warning(msg) | Event::Error(msg) | Event::ErrorNetwork(msg) => {
+            println!("  {}", msg);
+        }
+        _ => {
+            println!();
         }
-        _ => 0,
     }
 }
 
 fn main() {
-    unsafe {
-        let ctx = dc_context_new(Some(cb), std::ptr::null_mut(), None);
-        let running = Arc::new(RwLock::new(true));
-        let info = dc_get_info(&ctx);
-        let info_s = CStr::from_ptr(info);
-        let duration = time::Duration::from_millis(4000);
-        println!("info: {}", info_s.to_str().unwrap());
+    let dir = tempdir().unwrap();
+    let dbfile = dir.path().join("db.sqlite");
+    println!("creating database {:?}", dbfile);
+    let ctx =
+        Context::new(Box::new(cb), "FakeOs".into(), dbfile).expect("Failed to create context");
+    let running = Arc::new(RwLock::new(true));
+    let info = ctx.get_info();
+    let duration = time::Duration::from_millis(4000);
+    println!("info: {:#?}", info);
+
+    let ctx = Arc::new(ctx);
+    let ctx1 = ctx.clone();
+    let r1 = running.clone();
+    let t1 = thread::spawn(move || {
+        while *r1.read().unwrap() {
+            perform_inbox_jobs(&ctx1);
+            if *r1.read().unwrap() {
+                perform_inbox_fetch(&ctx1);
 
-        let ctx = Arc::new(ctx);
-        let ctx1 = ctx.clone();
-        let r1 = running.clone();
-        let t1 = thread::spawn(move || {
-            while *r1.read().unwrap() {
-                perform_imap_jobs(&ctx1);
                 if *r1.read().unwrap() {
-                    perform_imap_fetch(&ctx1);
-
-                    if *r1.read().unwrap() {
-                        perform_imap_idle(&ctx1);
-                    }
+                    perform_inbox_idle(&ctx1);
                 }
             }
-        });
-
-        let ctx1 = ctx.clone();
-        let r1 = running.clone();
-        let t2 = thread::spawn(move || {
-            while *r1.read().unwrap() {
-                perform_smtp_jobs(&ctx1);
-                if *r1.read().unwrap() {
-                    perform_smtp_idle(&ctx1);
-                }
-            }
-        });
-
-        let dir = tempdir().unwrap();
-        let dbfile = dir.path().join("db.sqlite");
-
-        println!("opening database {:?}", dbfile);
-
-        assert!(dc_open(&ctx, dbfile.to_str().unwrap(), None));
-
-        println!("configuring");
-        let args = std::env::args().collect::<Vec<String>>();
-        assert_eq!(args.len(), 2, "missing password");
-        let pw = args[1].clone();
-        ctx.set_config(config::Config::Addr, Some("d@testrun.org"))
-            .unwrap();
-        ctx.set_config(config::Config::MailPw, Some(&pw)).unwrap();
-        configure(&ctx);
-
-        thread::sleep(duration);
-
-        println!("sending a message");
-        let contact_id =
-            Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com").unwrap();
-        let chat_id = chat::create_by_contact_id(&ctx, contact_id).unwrap();
-        chat::send_text_msg(&ctx, chat_id, "Hi, here is my first message!".into()).unwrap();
-
-        println!("fetching chats..");
-        let chats = Chatlist::try_load(&ctx, 0, None, None).unwrap();
-
-        for i in 0..chats.len() {
-            let summary = chats.get_summary(0, None);
-            let text1 = summary.get_text1();
-            let text2 = summary.get_text2();
-            println!("chat: {} - {:?} - {:?}", i, text1, text2,);
         }
+    });
 
-        thread::sleep(duration);
+    let ctx1 = ctx.clone();
+    let r1 = running.clone();
+    let t2 = thread::spawn(move || {
+        while *r1.read().unwrap() {
+            perform_smtp_jobs(&ctx1);
+            if *r1.read().unwrap() {
+                perform_smtp_idle(&ctx1);
+            }
+        }
+    });
 
-        // let msglist = dc_get_chat_msgs(&ctx, chat_id, 0, 0);
-        // for i in 0..dc_array_get_cnt(msglist) {
-        //     let msg_id = dc_array_get_id(msglist, i);
-        //     let msg = dc_get_msg(context, msg_id);
-        //     let text = CStr::from_ptr(dc_msg_get_text(msg)).unwrap();
-        //     println!("Message {}: {}\n", i + 1, text.to_str().unwrap());
-        //     dc_msg_unref(msg);
-        // }
-        // dc_array_unref(msglist);
+    println!("configuring");
+    let args = std::env::args().collect::<Vec<String>>();
+    assert_eq!(args.len(), 2, "missing password");
+    let pw = args[1].clone();
+    ctx.set_config(config::Config::Addr, Some("d@testrun.org"))
+        .unwrap();
+    ctx.set_config(config::Config::MailPw, Some(&pw)).unwrap();
+    ctx.configure();
 
-        println!("stopping threads");
+    thread::sleep(duration);
 
-        *running.clone().write().unwrap() = false;
-        deltachat::job::interrupt_imap_idle(&ctx);
-        deltachat::job::interrupt_smtp_idle(&ctx);
+    println!("sending a message");
+    let contact_id = Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com").unwrap();
+    let chat_id = chat::create_by_contact_id(&ctx, contact_id).unwrap();
+    chat::send_text_msg(&ctx, chat_id, "Hi, here is my first message!".into()).unwrap();
 
-        println!("joining");
-        t1.join().unwrap();
-        t2.join().unwrap();
+    println!("fetching chats..");
+    let chats = Chatlist::try_load(&ctx, 0, None, None).unwrap();
 
-        println!("closing");
-        dc_close(&ctx);
+    for i in 0..chats.len() {
+        let summary = chats.get_summary(&ctx, 0, None);
+        let text1 = summary.get_text1();
+        let text2 = summary.get_text2();
+        println!("chat: {} - {:?} - {:?}", i, text1, text2,);
     }
+
+    thread::sleep(duration);
+
+    println!("stopping threads");
+
+    *running.write().unwrap() = false;
+    deltachat::job::interrupt_inbox_idle(&ctx);
+    deltachat::job::interrupt_smtp_idle(&ctx);
+
+    println!("joining");
+    t1.join().unwrap();
+    t2.join().unwrap();
+
+    println!("closing");
 }

misc.c

@@ -1,52 +0,0 @@
-#include <stdlib.h>
-#include <stdarg.h>
-#include <ctype.h>
-#include <string.h>
-#include <stdio.h>
-#include "misc.h"
-
-
-static char* internal_dc_strdup(const char* s) /* strdup(NULL) is undefined, save_strdup(NULL) returns an empty string in this case */
-{
-	char* ret = NULL;
-	if (s) {
-		if ((ret=strdup(s))==NULL) {
-			exit(16); /* cannot allocate (little) memory, unrecoverable error */
-		}
-	}
-	else {
-		if ((ret=(char*)calloc(1, 1))==NULL) {
-			exit(17); /* cannot allocate little memory, unrecoverable error */
-		}
-	}
-	return ret;
-}
-
-char* dc_mprintf(const char* format, ...)
-{
-	char  testbuf[1];
-	char* buf = NULL;
-	int   char_cnt_without_zero = 0;
-
-	va_list argp;
-	va_list argp_copy;
-	va_start(argp, format);
-	va_copy(argp_copy, argp);
-
-	char_cnt_without_zero = vsnprintf(testbuf, 0, format, argp);
-	va_end(argp);
-	if (char_cnt_without_zero < 0) {
-		va_end(argp_copy);
-		return internal_dc_strdup("ErrFmt");
-	}
-
-	buf = malloc(char_cnt_without_zero+2 /* +1 would be enough, however, protect against off-by-one-errors */);
-	if (buf==NULL) {
-		va_end(argp_copy);
-		return internal_dc_strdup("ErrMem");
-	}
-
-	vsnprintf(buf, char_cnt_without_zero+1, format, argp_copy);
-	va_end(argp_copy);
-	return buf;
-}

misc.h

@@ -1 +0,0 @@
-char* dc_mprintf                 (const char* format, ...); /* The result must be free()'d. */

proptest-regressions/dc_mimeparser.txt

@@ -0,0 +1,7 @@
+# Seeds for failure cases proptest has generated in the past. It is
+# automatically read and these particular cases re-run before any
+# novel cases are generated.
+#
+# It is recommended to check this file in to source control so that
+# everyone who runs the test benefits from these saved cases.
+cc 03cab93c6d1f3a8245f63cf84dacb307944294fe6333c1e38f078a6600659c7a # shrinks to data = "a\t0aA\ta\t0 \ta\t0 \ta a\t\ta  A\tAA0a0a 0\t a\t aA   \t a\t  A0\t  AAa\taA0\taAAaA\t0\taa0a\ta Aa Aaaa A0A\t  a aA 0\t  A\t0\t0\t\t\t\t\t\tA \t\t a\tA Aa aAA0A0AA0aaA A\t\t  aa0\ta\t  \tAa\taA\t00  AA  A a\tA a aAAa \t 00\t0 \t\t  a A  0\t\t\t aAA Aa \taAAAA0a A0A\t\t1\\E$\t$R\tc\t^=\t\"\tQ<Uk\t\t>A\t\t&\t}v&\tM^\'|\tW5?dn\t\t+\t\tP\te`\t\t>:Brlq--?\t$#Q\tK=zJ\tb\"9*.\"\t`\tF&T*\tBs,\tg\'*\\\t:\t?l$\t\t|A\"HR:Hk\t\\KkV\t\t{=&!^e%:|_*0wV\t[$`\t:\t$%f\t\t[!\"Y. \tP\t\th\'?\'/?%:++NfQo#@\"+?\t(\\??{\t\'\'$Dzj\t0.?{s4?&?Y=/yj]Z=\t4n\t?Ja\"\t{I\t$\t;I}_8V\t&\t?N\'\tI2/\t9.\tFT%9%`\'\tz\to7Y\t|AXP&@G12g\t\'w\t\t%??\t\"h$?F\"\"6%q\\\\{\tT\t\"]$87$}h\'\t<\t$\tc%U:mT2:<v\t#Rl!;U\t\t\"^D\tRZ(BZ{n\t%[$fgH`\t{B}:*\t\t%%*F`%W\t//B}PQ\t\tsu2\tLz<1*!p-X\tnKv&&0\thm4<n\\.\\/.w\'\t<)E1g3#ood\t`?\t\\({N?,8_?h\ty\t0%\t*$A\t\t*w-ViQUj\tTiH\t%\t%&0p\'\'\tA%r**Fo\'Z\\\tNI*ely4=I?}$.o`\t$\ts\'&lt\t\",:~=Nv/0O%=+<LS\t%P\'\t$r%d.\t{G{/L:\t&I&8-`cy*\"{\t/%fP9.P<\t\t\'/`\t\t`\t\t`!t:\t::\t\tW\'X^\t@.uL&a\tN\t\t\t.\t?0*\tvUK>UJ\\\tQbj%w]P=Js\t\"R\t&s^?</=%\t\'VI:\" kT`{b*<\t\tF&?Y\t\t:/\t!C\'e0`\t\t\tx-\t*\\N\\wwu.I4\tb/y\t\"P:P%\"\t\tQ][\\\t^x\t\t):\t\t&s\t$1-\t\t\tXt`0\t;\t/UeCP*\"G\t\t\':\tk@91Hf&#\t(Uzz&`p2;\t{]\t\"I_%=\\%,XD\"\'06QfCd<;$:q^\t8:\"*\"H\t\to\t&xK/\t\ty0/<}<j<|*`~::b\t=S.=}=Ki\t<Y.\'{\tf\t{Ub*H%(_4*?*\tn2\t/\'\t\t\t/,4]\tt\t<y\t\t\tWi\t\tT&\"\t\t\t\t\t=/1Wu?\t\'A\"W-P\t$?e\\\t`\t6`vD\t8`\t\tccD J\tY&jI//?_\t\\j\t_\tsiq$\t?9\tQ\t.^%&..?%Jm??A%+<\tN&*\t.g\tS$W\"\"\tMpq\t\t:&\\\thTz&<iz%/%T&\'F\t\\]&\t\t}\t\t\tXr=dtp`:+<,\t%60Y*<(&K*kJ\todO\t=<V?&\tMU/\"\t= Y!)<\tV\t9\t)\t&v8q{\t\t&pT\t3\ttB,qcq\'i$;gv%j_%M_{[\"&;\t\t\t.B;6Y\\%\t\"\tY/a\t\\`:?\t<\t?]\taNwD;\\\t%l*74%5T?QS :~yR/E*R\t\t=u\t\\\t\t.Q<;\\\t_S/{&F$^\tw_>\'h=|%\t\t:W!\\<U2\'$\tb&`\t=|=L9\t\t\t\\WZ:! }{\t ;\t;\t\t 0.*\t.%\"\'r%{<Mh_t[[}\t-\tJo\"b/AC*-=`=T\tz$2\tC\t\t/k{47\"\t\t,q%\tZ\tT3\t\tf>%\t\'?%@^tx\t7\"1Bk{b{n\t\"Pj3\tHc\t\tt\tY<\t#?\tSh\\yk/N\\\t8 7=R4*9Cw&m\t\\-\'f\t|\'#t(Etu.Hdu(E\t%&v:\'aqW~A5\t\t w.s{J%lX<\"\t\'*%m<&:/B<&\':U}$&`.{)\t\t6S\t:/$*kQ-Z\t^\'t${/tH*\'v\t3\t=\t\tDyp:B\t`I_R&4SO\t\t&-j=*.\t87&\'e*( \t\t\t\'<$\\DJ<$p?{}\'&\tv\t\\Xk<Y:Y`!$K{\tF&\tzd\t\t*i$\tj\'\t<)R*\t%?\t!.\t=\"@#~:=*\t\tXO=_T,1\"\'.%%\"`{\\:\t\"\tfkeOb/\'$I~\ta\t|&\t[\\KK\"1&Z\t<k\t\t)%\'-~\"2n\tj\tW?*<@w{g%d\ta\\\'\'I\t;:ySR%ke:4\tc\t$=\t&9P]x4\tJ=\t6C6%a\t`0\tF\tm-\tTr\t}\t\tQum\t&@\typ|w2&\t\t3`i&t\t\tT5\"\t.&b&e*/==1.\'*\\[U*\tqPL%?$-0/}~|q`\t\t}\t$\tq==o+T$\'!H\t\ti&um\"?\"%%\t/\'p\tg>?{0{J{\t\t/\t\t{zKZ&>=\t[\"1h<H%z/8,/]s\tv{7\t\t:j*H,M//\t\t\td\'.)\t"

proptest-regressions/dc_strencode.txt

@@ -5,3 +5,4 @@
 # It is recommended to check this file in to source control so that
 # everyone who runs the test benefits from these saved cases.
 cc 679506fe9ac59df773f8cfa800fdab5f0a32fe49d2ab370394000a1aa5bc2a72 # shrinks to buf = "%0A"
+cc e34960438edb2426904b44fb4215154e7e2880f2fd1c3183b98bfcc76fec4882 # shrinks to input = " 0"

python/CHANGELOG

@@ -1,6 +1,29 @@
-0.600.1
+0.900.0 (DRAFT)
+---------------
+
+- refactored internals to use plugin-approach 
+
+- introduced PerAccount and Global hooks that plugins can implement 
+
+- introduced `ac_member_added()` and `ac_member_removed()` plugin events. 
+
+- introduced two documented examples for an echo and a group-membership
+  tracking plugin. 
+
+0.800.0
+-------
+
+- use latest core 1.25.0 
+
+- refine tests and some internal changes to core bindings
+
+0.700.0
 ---------
 
+- lots of new Python APIs
+
+- use rust core-beta23 
+
 - introduce automatic versioning via setuptools_scm, 
   based on py-X.Y.Z tags 
 

python/README.rst

@@ -3,41 +3,98 @@ deltachat python bindings
 =========================
 
 This package provides bindings to the deltachat-core_ Rust -library
-which provides imap/smtp/crypto handling as well as chat/group/messages
-handling to Android, Desktop and IO user interfaces.
+which implements IMAP/SMTP/MIME/PGP e-mail standards and offers
+a low-level Chat/Contact/Message API to user interfaces and bots.
 
-Installing pre-built packages (linux-only)
-==========================================
 
-If you have a linux system you may install the ``deltachat`` binary "wheel" package
+Installing bindings from source (Updated: 20-Jan-2020)
+=========================================================
+
+Install Rust and Cargo first.  Deltachat needs a specific nightly
+version, the easiest is probably to first install Rust stable from
+rustup and then use this to install the correct nightly version.
+
+Bootstrap Rust and Cargo by using rustup::
+
+   curl https://sh.rustup.rs -sSf | sh
+
+Then GIT clone the deltachat-core-rust repo and get the actual
+rust- and cargo-toolchain needed by deltachat::
+
+   git clone https://github.com/deltachat/deltachat-core-rust
+   cd deltachat-core-rust
+   rustup show
+
+To install the Delta Chat Python bindings make sure you have Python3 installed.
+E.g. on Debian-based systems `apt install python3 python3-pip
+python3-venv` should give you a usable python installation.
+
+Ensure you are in the deltachat-core-rust/python directory, create the
+virtual environment and activate it in your shell::
+
+   cd python
+   python3 -m venv venv  # or: virtualenv venv
+   source venv/bin/activate
+
+You should now be able to build the python bindings using the supplied script::
+
+   ./install_python_bindings.py
+
+The installation might take a while, depending on your machine.
+The bindings will be installed in release mode but with debug symbols.
+The release mode is currently necessary because some tests generate RSA keys
+which is prohibitively slow in non-release mode.
+
+After successful binding installation you can install a few more
+Python packages before running the tests::
+
+    python -m pip install pytest pytest-timeout pytest-rerunfailures requests
+    pytest -v tests
+
+
+running "live" tests with temporary accounts
+---------------------------------------------
+
+If you want to run "liveconfig" functional tests you can set
+``DCC_NEW_TMP_EMAIL`` to:
+
+- a particular https-url that you can ask for from the delta
+  chat devs. This is implemented on the server side via
+  the [mailadm](https://github.com/deltachat/mailadm) command line tool.
+
+- or the path of a file that contains two lines, each describing
+  via "addr=... mail_pw=..." a test account login that will
+  be used for the live tests.
+
+With ``DCC_NEW_TMP_EMAIL`` set pytest invocations will use real
+e-mail accounts and run through all functional "liveconfig" tests.
+
+
+Installing pre-built packages (Linux-only)
+========================================================
+
+If you have a Linux system you may try to install the ``deltachat`` binary "wheel" packages
 without any "build-from-source" steps.
 
-1. `Install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
-   then create a fresh python environment and activate it in your shell::
+We suggest to `Install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
+then create a fresh Python virtual environment and activate it in your shell::
 
         virtualenv venv  # or: python -m venv
         source venv/bin/activate
 
-   Afterwards, invoking ``python`` or ``pip install`` will only
-   modify files in your ``venv`` directory and leave your system installation
-   alone.
-
-2. Install the wheel for linux::
-
-        pip install deltachat
-
-    Verify it worked by typing::
-
-        python -c "import deltachat"
-
-
-Installing a wheel from a PR/branch
----------------------------------------
+Afterwards, invoking ``python`` or ``pip install`` only
+modifies files in your ``venv`` directory and leaves
+your system installation alone.
 
 For Linux, we automatically build wheels for all github PR branches
-and push them to a python package index. To install the latest github ``master`` branch::
+and push them to a python package index. To install the latest
+github ``master`` branch::
 
-    pip install -i https://m.devpi.net/dc/master deltachat
+    pip install --pre -i https://m.devpi.net/dc/master deltachat
+
+To verify it worked::
+
+    python -c "import deltachat"
 
 .. note::
 
@@ -46,65 +103,6 @@ and push them to a python package index. To install the latest github ``master``
     `in contact with us <https://delta.chat/en/contribute>`_.
 
 
-Installing bindings from source
-===============================
-
-If you can't use "binary" method above then you need to compile
-to core deltachat library::
-
-    git clone https://github.com/deltachat/deltachat-core-rust
-    cd deltachat-core-rust
-    cd python
-
-If you don't have one active, create and activate a python "virtualenv":
-
-    python virtualenv venv  # or python -m venv
-    source venv/bin/activate
-
-Afterwards ``which python`` tells you that it comes out of the "venv"
-directory that contains all python install artifacts. Let's first
-install test tools::
-
-    pip install pytest pytest-timeout requests
-
-then cargo-build and install the deltachat bindings::
-
-    python install_python_bindings.py
-
-The bindings will be installed in release mode but with debug symbols.
-The release mode is necessary because some tests generate RSA keys
-which is prohibitively slow in debug mode.
-
-After successful binding installation you can finally run the tests::
-
-    pytest -v tests
-
-.. note::
-
-    Some tests are sometimes failing/hanging because of
-    https://github.com/deltachat/deltachat-core-rust/issues/331
-    and
-    https://github.com/deltachat/deltachat-core-rust/issues/326
-
-
-running "live" tests (experimental)
------------------------------------
-
-If you want to run "liveconfig" functional tests you can set
-``DCC_PY_LIVECONFIG`` to:
-
-- a particular https-url that you can ask for from the delta
-  chat devs.
-
-- or the path of a file that contains two lines, each describing
-  via "addr=... mail_pwd=..." a test account login that will
-  be used for the live tests.
-
-With ``DCC_PY_LIVECONFIG`` set pytest invocations will use real
-e-mail accounts and run through all functional "liveconfig" tests.
-
-
-
 Code examples
 =============
 
@@ -115,15 +113,11 @@ You may look at `examples <https://py.delta.chat/examples.html>`_.
 .. _`deltachat-core`: https://github.com/deltachat/deltachat-core-rust
 
 
-Building manylinux1 wheels
-==========================
-
-.. note::
-
-   This section may not fully work.
+Building manylinux1 based wheels
+================================
 
 Building portable manylinux1 wheels which come with libdeltachat.so
-and all it's dependencies is easy using the provided docker tooling.
+can be done with docker-tooling.
 
 using docker pull / premade images
 ------------------------------------
@@ -136,9 +130,9 @@ organization::
 
 This docker image can be used to run tests and build Python wheels for all interpreters::
 
-    $ bash ci_scripts/ci_run.sh
-
-This command runs tests and build-wheel scripts in a docker container.
+    $ docker run -e DCC_NEW_TMP_EMAIL \
+       --rm -it -v \$(pwd):/mnt -w /mnt \
+       deltachat/coredeps ci_scripts/run_all.sh
 
 
 Optionally build your own docker image

python/doc/_static/custom.css

@@ -15,3 +15,7 @@ div.globaltoc {
 img.logo {
     height: 120px;
 }
+
+div.footer {
+    display: none;
+}

python/doc/_templates/globaltoc.html

@@ -6,7 +6,6 @@
   <li><a href="{{ pathto('install') }}">install</a></li>
   <li><a href="{{ pathto('api') }}">high level API</a></li>
   <li><a href="{{ pathto('lapi') }}">low level API</a></li>
-  <li><a href="{{ pathto('capi') }}">C deltachat.h</a></li>
 </ul>
 <b>external links:</b>
 <ul>

python/doc/api.rst

@@ -2,14 +2,10 @@
 high level API reference
 ========================
 
-.. note::
-
-    This API is work in progress and may change in versions prior to 1.0.
-
 - :class:`deltachat.account.Account` (your main entry point, creates the
   other classes)
-- :class:`deltachat.chatting.Contact`
-- :class:`deltachat.chatting.Chat`
+- :class:`deltachat.contact.Contact`
+- :class:`deltachat.chat.Chat`
 - :class:`deltachat.message.Message`
 
 Account
@@ -22,13 +18,13 @@ Account
 Contact
 -------
 
-.. autoclass:: deltachat.chatting.Contact
+.. autoclass:: deltachat.contact.Contact
     :members:
 
 Chat
 ----
 
-.. autoclass:: deltachat.chatting.Chat
+.. autoclass:: deltachat.chat.Chat
     :members:
 
 Message

python/doc/capi.rst

@@ -1,7 +0,0 @@
-
-C deltachat interface
-=====================
-
-See :doc:`lapi` for accessing many of the below functions
-through the ``deltachat.capi.lib`` namespace.
-

python/doc/conf.py

@@ -55,7 +55,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'deltachat'
-copyright = u'2018, holger krekel and contributors'
+copyright = u'2020, holger krekel and contributors'
 
 
 # The language for content autogenerated by Sphinx. Refer to documentation

python/doc/examples.rst

@@ -1,37 +1,60 @@
 
-
 examples
 ========
 
-
-Playing around on the commandline
-----------------------------------
-
 Once you have :doc:`installed deltachat bindings <install>`
-you can start playing from the python interpreter commandline.
-For example you can type ``python`` and then::
+you need email/password credentials for an IMAP/SMTP account.
+Delta Chat developers and the CI system use a special URL to create
+temporary e-mail accounts on [testrun.org](https://testrun.org) for testing.
 
-    # instantiate and configure deltachat account
-    import deltachat
-    ac = deltachat.Account("/tmp/db")
+Receiving a Chat message from the command line
+----------------------------------------------
 
-    # start configuration activity and smtp/imap threads
-    ac.start_threads()
-    ac.configure(addr="test2@hq5.merlinux.eu", mail_pw="********")
+Here is a simple bot that:
 
-    # create a contact and send a message
-    contact = ac.create_contact("someother@email.address")
-    chat = ac.create_chat_by_contact(contact)
-    chat.send_text("hi from the python interpreter command line")
+- receives a message and sends back ("echoes") a message
 
-Checkout our :doc:`api` for the various high-level things you can do
-to send/receive messages, create contacts and chats.
+- terminates the bot if the message `/quit` is sent
 
+.. include:: ../examples/echo_and_quit.py
+    :literal:
 
-Looking at a real example
+With this file in your working directory you can run the bot
+by specifying a database path, an e-mail address and password of
+a SMTP-IMAP account::
+
+    $ cd examples
+    $ python echo_and_quit.py /tmp/db --email ADDRESS --password PASSWORD
+
+While this process is running you can start sending chat messages
+to `ADDRESS`.
+
+Track member additions and removals in a group
+----------------------------------------------
+
+Here is a simple bot that:
+
+- echoes messages sent to it
+
+- tracks if configuration completed
+
+- tracks member additions and removals for all chat groups
+
+.. include:: ../examples/group_tracking.py
+    :literal:
+
+With this file in your working directory you can run the bot
+by specifying a database path, an e-mail address and password of
+a SMTP-IMAP account::
+
+    python group_tracking.py --email ADDRESS --password PASSWORD /tmp/db
+
+When this process is running you can start sending chat messages
+to `ADDRESS`.
+
+Writing bots for real
 -------------------------
 
 The `deltabot repository <https://github.com/deltachat/deltabot#deltachat-example-bot>`_
-contains a real-life example of Python bindings usage.
-
+contains a little framework for writing deltachat bots in Python.
 

python/doc/index.rst

@@ -1,15 +1,15 @@
 deltachat python bindings
 =========================
 
-The ``deltachat`` Python package provides two bindings for the core Rust-library
-of the https://delta.chat messaging ecosystem:
+The ``deltachat`` Python package provides two layers of bindings for the
+core Rust-library of the https://delta.chat messaging ecosystem:
 
-- :doc:`api` is a high level interface to deltachat-core which aims
-  to be memory safe and thoroughly tested through continous tox/pytest runs.
+- :doc:`api` is a high level interface to deltachat-core.
 
-- :doc:`capi` is a lowlevel CFFI-binding to the previous
-  `deltachat-core C-API <https://c.delta.chat>`_ (so far the Rust library
-  replicates exactly the same C-level API).
+- :doc:`plugins` is a brief introduction into implementing plugin hooks.
+
+- :doc:`lapi` is a lowlevel CFFI-binding to the `Rust Core
+  <https://github.com/deltachat/deltachat-core-rust>`_.
 
 
 
@@ -28,8 +28,8 @@ getting started
    links
    changelog
    api
-   capi
    lapi
+   plugins
 
 ..
     Indices and tables

python/doc/lapi.rst

@@ -2,7 +2,13 @@
 low level API reference
 ===================================
 
-for full C-docs, defines and function checkout :doc:`capi`
+for full doxygen-generated C-docs, defines and functions please checkout
+
+https://c.delta.chat
+
+
+Python low-level capi calls
+---------------------------
 
 
 .. automodule:: deltachat.capi.lib

python/doc/plugins.rst

@@ -0,0 +1,38 @@
+
+Implementing Plugin Hooks
+==========================
+
+The Delta Chat Python bindings use `pluggy <https://pluggy.readthedocs.io>`_
+for managing global and per-account plugin registration, and performing
+hook calls. There are two kinds of plugins:
+
+- Global plugins that are active for all accounts; they can implement
+  hooks at account-creation and account-shutdown time.
+
+- Account plugins that are only active during the lifetime of a
+  single Account instance.
+
+
+Registering a plugin
+--------------------
+
+.. autofunction:: deltachat.register_global_plugin
+    :noindex:
+
+.. automethod:: deltachat.account.Account.add_account_plugin
+    :noindex:
+
+
+Per-Account Hook specifications
+-------------------------------
+
+.. autoclass:: deltachat.hookspec.PerAccount
+    :members:
+
+
+Global Hook specifications
+--------------------------
+
+.. autoclass:: deltachat.hookspec.Global
+    :members:
+

python/examples/echo_and_quit.py

@@ -0,0 +1,30 @@
+
+# content of echo_and_quit.py
+
+from deltachat import account_hookimpl, run_cmdline
+
+
+class EchoPlugin:
+    @account_hookimpl
+    def ac_incoming_message(self, message):
+        print("process_incoming message", message)
+        if message.text.strip() == "/quit":
+            message.account.shutdown()
+        else:
+            # unconditionally accept the chat
+            message.accept_sender_contact()
+            addr = message.get_sender_contact().addr
+            text = message.text
+            message.chat.send_text("echoing from {}:\n{}".format(addr, text))
+
+    @account_hookimpl
+    def ac_message_delivered(self, message):
+        print("ac_message_delivered", message)
+
+
+def main(argv=None):
+    run_cmdline(argv=argv, account_plugins=[EchoPlugin()])
+
+
+if __name__ == "__main__":
+    main()

python/examples/group_tracking.py

@@ -0,0 +1,40 @@
+
+# content of group_tracking.py
+
+from deltachat import account_hookimpl, run_cmdline
+
+
+class GroupTrackingPlugin:
+    @account_hookimpl
+    def ac_incoming_message(self, message):
+        print("process_incoming message", message)
+        if message.text.strip() == "/quit":
+            message.account.shutdown()
+        else:
+            # unconditionally accept the chat
+            message.accept_sender_contact()
+            addr = message.get_sender_contact().addr
+            text = message.text
+            message.chat.send_text("echoing from {}:\n{}".format(addr, text))
+
+    @account_hookimpl
+    def ac_configure_completed(self, success):
+        print("*** ac_configure_completed:", success)
+
+    @account_hookimpl
+    def ac_member_added(self, chat, contact):
+        print("*** ac_member_added", contact.addr, "from", chat)
+        for member in chat.get_contacts():
+            print("chat member: {}".format(member.addr))
+
+    @account_hookimpl
+    def ac_member_removed(self, chat, contact):
+        print("*** ac_member_removed", contact.addr, "from", chat)
+
+
+def main(argv=None):
+    run_cmdline(argv=argv, account_plugins=[GroupTrackingPlugin()])
+
+
+if __name__ == "__main__":
+    main()

python/examples/test_examples.py

@@ -0,0 +1,72 @@
+
+import pytest
+import py
+import echo_and_quit
+import group_tracking
+from deltachat.eventlogger import FFIEventLogger
+
+
+@pytest.fixture(scope='session')
+def datadir():
+    """The py.path.local object of the test-data/ directory."""
+    for path in reversed(py.path.local(__file__).parts()):
+        datadir = path.join('test-data')
+        if datadir.isdir():
+            return datadir
+    else:
+        pytest.skip('test-data directory not found')
+
+
+def test_echo_quit_plugin(acfactory):
+    botproc = acfactory.run_bot_process(echo_and_quit)
+
+    ac1 = acfactory.get_one_online_account()
+    bot_contact = ac1.create_contact(botproc.addr)
+    ch1 = ac1.create_chat_by_contact(bot_contact)
+    ch1.send_text("hello")
+    reply = ac1._evtracker.wait_next_incoming_message()
+    assert "hello" in reply.text
+    assert reply.chat == ch1
+    ch1.send_text("/quit")
+    botproc.wait()
+
+
+def test_group_tracking_plugin(acfactory, lp):
+    lp.sec("creating one group-tracking bot and two temp accounts")
+    botproc = acfactory.run_bot_process(group_tracking)
+
+    ac1, ac2 = acfactory.get_two_online_accounts(quiet=True)
+
+    botproc.fnmatch_lines("""
+        *ac_configure_completed: True*
+    """)
+    ac1.add_account_plugin(FFIEventLogger(ac1, "ac1"))
+    ac2.add_account_plugin(FFIEventLogger(ac2, "ac2"))
+
+    lp.sec("creating bot test group with all three")
+    bot_contact = ac1.create_contact(botproc.addr)
+    ch = ac1.create_group_chat("bot test group")
+    ch.add_contact(bot_contact)
+    ch.send_text("hello")
+
+    botproc.fnmatch_lines("""
+        *ac_member_added {}*
+    """.format(ac1.get_config("addr")))
+
+    lp.sec("adding third member {}".format(ac2.get_config("addr")))
+    contact3 = ac1.create_contact(ac2.get_config("addr"))
+    ch.add_contact(contact3)
+
+    reply = ac1._evtracker.wait_next_incoming_message()
+    assert "hello" in reply.text
+
+    lp.sec("now looking at what the bot received")
+    botproc.fnmatch_lines("""
+        *ac_member_added {}*
+    """.format(contact3.addr))
+
+    lp.sec("contact successfully added, now removing")
+    ch.remove_contact(contact3)
+    botproc.fnmatch_lines("""
+        *ac_member_removed {}*
+    """.format(contact3.addr))

python/install_python_bindings.py

@@ -6,20 +6,23 @@
 
 import os
 import subprocess
-import os
+import sys
 
 if __name__ == "__main__":
-    os.environ["DCC_RS_TARGET"] = target = "release"
+    target = os.environ.get("DCC_RS_TARGET")
+    if target is None:
+        os.environ["DCC_RS_TARGET"] = target = "debug"
     if "DCC_RS_DEV" not in os.environ:
         dn = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
         os.environ["DCC_RS_DEV"] = dn
 
-    os.environ["RUSTFLAGS"] = "-g"
-    subprocess.check_call([
-        "cargo", "build", "-p", "deltachat_ffi", "--" + target
-    ])
+    cmd = ["cargo", "build", "-p", "deltachat_ffi"]
+    if target == 'release':
+        cmd.append("--release")
+    subprocess.check_call(cmd)
     subprocess.check_call("rm -rf build/ src/deltachat/*.so" , shell=True)
 
-    subprocess.check_call([
-        "pip", "install", "-e", "."
-    ])
+    if len(sys.argv) <= 1 or sys.argv[1] != "onlybuild":
+        subprocess.check_call([
+            sys.executable, "-m", "pip", "install", "-e", "."
+        ])

python/setup.py

@@ -13,14 +13,20 @@ def main():
             "root": "..",
             "relative_to": __file__,
             'tag_regex': r'^(?P<prefix>py-)?(?P<version>[^\+]+)(?P<suffix>.*)?$',
+            'git_describe_command': "git describe --dirty --tags --long --match py-*.*",
         },
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'six'],
+        install_requires=['cffi>=1.0.0', 'pluggy'],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],
+        entry_points = {
+            'pytest11': [
+                'deltachat.testplugin = deltachat.testplugin',
+            ],
+        },
         classifiers=[
             'Development Status :: 4 - Beta',
             'Intended Audience :: Developers',

python/src/deltachat/__init__.py

@@ -1,6 +1,13 @@
-from deltachat import capi, const
-from deltachat.capi import ffi
-from deltachat.account import Account  # noqa
+import sys
+
+from . import capi, const, hookspec
+from .capi import ffi
+from .account import Account  # noqa
+from .message import Message  # noqa
+from .contact import Contact  # noqa
+from .chat import Chat        # noqa
+from .hookspec import account_hookimpl, global_hookimpl  # noqa
+from . import eventlogger
 
 from pkg_resources import get_distribution, DistributionNotFound
 try:
@@ -74,3 +81,62 @@ def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}):
             if name.startswith("DC_EVENT_"):
                 _DC_EVENTNAME_MAP[val] = name
     return _DC_EVENTNAME_MAP[integer]
+
+
+def register_global_plugin(plugin):
+    """ Register a global plugin which implements one or more
+    of the :class:`deltachat.hookspec.Global` hooks.
+    """
+    gm = hookspec.Global._get_plugin_manager()
+    gm.register(plugin)
+    gm.check_pending()
+
+
+def unregister_global_plugin(plugin):
+    gm = hookspec.Global._get_plugin_manager()
+    gm.unregister(plugin)
+
+
+register_global_plugin(eventlogger)
+
+
+def run_cmdline(argv=None, account_plugins=None):
+    """ Run a simple default command line app, registering the specified
+    account plugins. """
+    import argparse
+    if argv is None:
+        argv = sys.argv
+
+    parser = argparse.ArgumentParser(prog=argv[0] if argv else None)
+    parser.add_argument("db", action="store", help="database file")
+    parser.add_argument("--show-ffi", action="store_true", help="show low level ffi events")
+    parser.add_argument("--email", action="store", help="email address")
+    parser.add_argument("--password", action="store", help="password")
+
+    args = parser.parse_args(argv[1:])
+
+    ac = Account(args.db)
+
+    if args.show_ffi:
+        log = eventlogger.FFIEventLogger(ac, "bot")
+        ac.add_account_plugin(log)
+
+    if not ac.is_configured():
+        assert args.email and args.password, (
+            "you must specify --email and --password once to configure this database/account"
+        )
+        ac.set_config("addr", args.email)
+        ac.set_config("mail_pw", args.password)
+        ac.set_config("mvbox_move", "0")
+        ac.set_config("mvbox_watch", "0")
+        ac.set_config("sentbox_watch", "0")
+
+    for plugin in account_plugins or []:
+        ac.add_account_plugin(plugin)
+
+    # start IO threads and configure if neccessary
+    ac.start()
+
+    print("{}: waiting for message".format(ac.get_config("addr")))
+
+    ac.wait_shutdown()

python/src/deltachat/_build.py

@@ -29,7 +29,10 @@ def ffibuilder():
             extra_link_args = []
         else:
             raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
-        objs = [os.path.join(projdir, 'target', target, 'libdeltachat.a')]
+        target_dir = os.environ.get("CARGO_TARGET_DIR")
+        if target_dir is None:
+            target_dir = os.path.join(projdir, 'target')
+        objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
         assert os.path.exists(objs[0]), objs
         incs = [os.path.join(projdir, 'deltachat-ffi')]
     else:

python/src/deltachat/account.py

@@ -1,59 +1,81 @@
 """ Account class implementation. """
 
 from __future__ import print_function
-import threading
+import atexit
+from contextlib import contextmanager
+from email.utils import parseaddr
+import queue
+from threading import Event
 import os
-import re
-import time
 from array import array
-try:
-    from queue import Queue, Empty
-except ImportError:
-    from Queue import Queue, Empty
-
 import deltachat
 from . import const
 from .capi import ffi, lib
 from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array, DCLot
-from .chatting import Contact, Chat, Message
+from .chat import Chat
+from .message import Message
+from .contact import Contact
+from .tracker import ImexTracker
+from . import hookspec, iothreads
+
+
+class MissingCredentials(ValueError):
+    """ Account is missing `addr` and `mail_pw` config values. """
 
 
 class Account(object):
     """ Each account is tied to a sqlite database file which is fully managed
-    by the underlying deltachat c-library.  All public Account methods are
+    by the underlying deltachat core library.  All public Account methods are
     meant to be memory-safe and return memory-safe objects.
     """
-    def __init__(self, db_path, logid=None, eventlogging=True):
+    MissingCredentials = MissingCredentials
+
+    def __init__(self, db_path, os_name=None):
         """ initialize account object.
 
         :param db_path: a path to the account database. The database
                         will be created if it doesn't exist.
-        :param logid: an optional logging prefix that should be used with
-                      the default internal logging.
-        :param eventlogging: if False no eventlogging and no context callback will be configured
+        :param os_name: this will be put to the X-Mailer header in outgoing messages
         """
+        # initialize per-account plugin system
+        self._pm = hookspec.PerAccount._make_plugin_manager()
+        self.add_account_plugin(self)
+
         self._dc_context = ffi.gc(
-            lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+            lib.dc_context_new(lib.py_dc_callback, ffi.NULL, as_dc_charpointer(os_name)),
             _destroy_dc_context,
         )
-        if eventlogging:
-            self._evlogger = EventLogger(self._dc_context, logid)
-            deltachat.set_context_callback(self._dc_context, self._process_event)
-            self._threads = IOThreads(self._dc_context, self._evlogger._log_event)
-        else:
-            self._threads = IOThreads(self._dc_context)
 
+        hook = hookspec.Global._get_plugin_manager().hook
+
+        self._threads = iothreads.IOThreads(self)
+        self._hook_event_queue = queue.Queue()
+        self._in_use_iter_events = False
+        self._shutdown_event = Event()
+
+        # open database
+        self.db_path = db_path
         if hasattr(db_path, "encode"):
             db_path = db_path.encode("utf8")
         if not lib.dc_open(self._dc_context, db_path, ffi.NULL):
             raise ValueError("Could not dc_open: {}".format(db_path))
         self._configkeys = self.get_config("sys.config_keys").split()
-        self._imex_completed = threading.Event()
+        atexit.register(self.shutdown)
+        hook.dc_account_init(account=self)
+
+    @hookspec.account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        name, kwargs = self._map_ffi_event(ffi_event)
+        if name is not None:
+            ev = HookEvent(self, name=name, kwargs=kwargs)
+            self._hook_event_queue.put(ev)
 
-    # XXX this can cause "illegal instructions" at test ends so we omit it for now
     # def __del__(self):
     #    self.shutdown()
 
+    def ac_log_line(self, msg):
+        self._pm.hook.ac_log_line(message=msg)
+
     def _check_config_key(self, name):
         if name not in self._configkeys:
             raise KeyError("{!r} not a valid config key, existing keys: {!r}".format(
@@ -70,6 +92,18 @@ class Account(object):
             d[key.lower()] = value
         return d
 
+    def set_stock_translation(self, id, string):
+        """ set stock translation string.
+
+        :param id: id of stock string (const.DC_STR_*)
+        :param value: string to set as new transalation
+        :returns: None
+        """
+        string = string.encode("utf8")
+        res = lib.dc_set_stock_translation(self._dc_context, id, string)
+        if res == 0:
+            raise ValueError("could not set translation string")
+
     def set_config(self, name, value):
         """ set configuration values.
 
@@ -79,9 +113,12 @@ class Account(object):
         """
         self._check_config_key(name)
         name = name.encode("utf8")
-        value = value.encode("utf8")
         if name == b"addr" and self.is_configured():
             raise ValueError("can not change 'addr' after account is configured.")
+        if value is not None:
+            value = value.encode("utf8")
+        else:
+            value = ffi.NULL
         lib.dc_set_config(self._dc_context, name, value)
 
     def get_config(self, name):
@@ -98,16 +135,27 @@ class Account(object):
         assert res != ffi.NULL, "config value not found for: {!r}".format(name)
         return from_dc_charpointer(res)
 
-    def configure(self, **kwargs):
-        """ set config values and configure this account.
+    def _preconfigure_keypair(self, addr, public, secret):
+        """See dc_preconfigure_keypair() in deltachat.h.
+
+        In other words, you don't need this.
+        """
+        res = lib.dc_preconfigure_keypair(self._dc_context,
+                                          as_dc_charpointer(addr),
+                                          as_dc_charpointer(public),
+                                          as_dc_charpointer(secret))
+        if res == 0:
+            raise Exception("Failed to set key")
+
+    def update_config(self, kwargs):
+        """ update config values.
 
         :param kwargs: name=value config settings for this account.
                        values need to be unicode.
         :returns: None
         """
-        for name, value in kwargs.items():
-            self.set_config(name, value)
-        lib.dc_configure(self._dc_context)
+        for key, value in kwargs.items():
+            self.set_config(key, str(value))
 
     def is_configured(self):
         """ determine if the account is configured already; an initial connection
@@ -115,17 +163,43 @@ class Account(object):
 
         :returns: True if account is configured.
         """
-        return lib.dc_is_configured(self._dc_context)
+        return bool(lib.dc_is_configured(self._dc_context))
+
+    def set_avatar(self, img_path):
+        """Set self avatar.
+
+        :raises ValueError: if profile image could not be set
+        :returns: None
+        """
+        if img_path is None:
+            self.set_config("selfavatar", None)
+        else:
+            assert os.path.exists(img_path), img_path
+            self.set_config("selfavatar", img_path)
 
     def check_is_configured(self):
         """ Raise ValueError if this account is not configured. """
         if not self.is_configured():
             raise ValueError("need to configure first")
 
-    def get_infostring(self):
-        """ return info of the configured account. """
-        self.check_is_configured()
-        return from_dc_charpointer(lib.dc_get_info(self._dc_context))
+    def empty_server_folders(self, inbox=False, mvbox=False):
+        """ empty server folders. """
+        flags = 0
+        if inbox:
+            flags |= const.DC_EMPTY_INBOX
+        if mvbox:
+            flags |= const.DC_EMPTY_MVBOX
+        if not flags:
+            raise ValueError("no flags set")
+        lib.dc_empty_server(self._dc_context, flags)
+
+    def get_latest_backupfile(self, backupdir):
+        """ return the latest backup file in a given directory.
+        """
+        res = lib.dc_imex_has_backup(self._dc_context, as_dc_charpointer(backupdir))
+        if res == ffi.NULL:
+            return None
+        return from_dc_charpointer(res)
 
     def get_blobdir(self):
         """ return the directory for files.
@@ -136,12 +210,11 @@ class Account(object):
         return from_dc_charpointer(lib.dc_get_blobdir(self._dc_context))
 
     def get_self_contact(self):
-        """ return this account's identity as a :class:`deltachat.chatting.Contact`.
+        """ return this account's identity as a :class:`deltachat.contact.Contact`.
 
-        :returns: :class:`deltachat.chatting.Contact`
+        :returns: :class:`deltachat.contact.Contact`
         """
-        self.check_is_configured()
-        return Contact(self._dc_context, const.DC_CONTACT_ID_SELF)
+        return Contact(self, const.DC_CONTACT_ID_SELF)
 
     def create_contact(self, email, name=None):
         """ create a (new) Contact. If there already is a Contact
@@ -150,13 +223,16 @@ class Account(object):
 
         :param email: email-address (text type)
         :param name: display name for this contact (optional)
-        :returns: :class:`deltachat.chatting.Contact` instance.
+        :returns: :class:`deltachat.contact.Contact` instance.
         """
-        name = as_dc_charpointer(name)
-        email = as_dc_charpointer(email)
-        contact_id = lib.dc_create_contact(self._dc_context, name, email)
+        realname, addr = parseaddr(email)
+        if name:
+            realname = name
+        realname = as_dc_charpointer(realname)
+        addr = as_dc_charpointer(addr)
+        contact_id = lib.dc_create_contact(self._dc_context, realname, addr)
         assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
-        return Contact(self._dc_context, contact_id)
+        return Contact(self, contact_id)
 
     def delete_contact(self, contact):
         """ delete a Contact.
@@ -176,7 +252,7 @@ class Account(object):
                       whose name or e-mail matches query.
         :param only_verified: if true only return verified contacts.
         :param with_self: if true the self-contact is also returned.
-        :returns: list of :class:`deltachat.chatting.Contact` objects.
+        :returns: list of :class:`deltachat.contact.Contact` objects.
         """
         flags = 0
         query = as_dc_charpointer(query)
@@ -188,13 +264,13 @@ class Account(object):
             lib.dc_get_contacts(self._dc_context, flags, query),
             lib.dc_array_unref
         )
-        return list(iter_array(dc_array, lambda x: Contact(self._dc_context, x)))
+        return list(iter_array(dc_array, lambda x: Contact(self, x)))
 
     def create_chat_by_contact(self, contact):
         """ create or get an existing 1:1 chat object for the specified contact or contact id.
 
         :param contact: chat_id (int) or contact object.
-        :returns: a :class:`deltachat.chatting.Chat` object.
+        :returns: a :class:`deltachat.chat.Chat` object.
         """
         if hasattr(contact, "id"):
             if contact._dc_context != self._dc_context:
@@ -210,8 +286,11 @@ class Account(object):
         """ create or get an existing chat object for the
         the specified message.
 
+        If this message is in the deaddrop chat then
+        the sender will become an accepted contact.
+
         :param message: messsage id or message instance.
-        :returns: a :class:`deltachat.chatting.Chat` object.
+        :returns: a :class:`deltachat.chat.Chat` object.
         """
         if hasattr(message, "id"):
             if self._dc_context != message._dc_context:
@@ -229,16 +308,16 @@ class Account(object):
         Chats are unpromoted until the first message is sent.
 
         :param verified: if true only verified contacts can be added.
-        :returns: a :class:`deltachat.chatting.Chat` object.
+        :returns: a :class:`deltachat.chat.Chat` object.
         """
         bytes_name = name.encode("utf8")
-        chat_id = lib.dc_create_group_chat(self._dc_context, verified, bytes_name)
+        chat_id = lib.dc_create_group_chat(self._dc_context, int(verified), bytes_name)
         return Chat(self, chat_id)
 
     def get_chats(self):
         """ return list of chats.
 
-        :returns: a list of :class:`deltachat.chatting.Chat` objects.
+        :returns: a list of :class:`deltachat.chat.Chat` objects.
         """
         dc_chatlist = ffi.gc(
             lib.dc_get_chatlist(self._dc_context, 0, ffi.NULL, 0),
@@ -256,9 +335,31 @@ class Account(object):
         return Chat(self, const.DC_CHAT_ID_DEADDROP)
 
     def get_message_by_id(self, msg_id):
-        """ return Message instance. """
+        """ return Message instance.
+        :param msg_id: integer id of this message.
+        :returns: :class:`deltachat.message.Message` instance.
+        """
         return Message.from_db(self, msg_id)
 
+    def get_contact_by_id(self, contact_id):
+        """ return Contact instance or None.
+        :param contact_id: integer id of this contact.
+        :returns: None or :class:`deltachat.contact.Contact` instance.
+        """
+        return Contact(self, contact_id)
+
+    def get_chat_by_id(self, chat_id):
+        """ return Chat instance.
+        :param chat_id: integer id of this chat.
+        :returns: :class:`deltachat.chat.Chat` instance.
+        :raises: ValueError if chat does not exist.
+        """
+        res = lib.dc_get_chat(self._dc_context, chat_id)
+        if res == ffi.NULL:
+            raise ValueError("cannot get chat with id={}".format(chat_id))
+        lib.dc_chat_unref(res)
+        return Chat(self, chat_id)
+
     def mark_seen_messages(self, messages):
         """ mark the given set of messages as seen.
 
@@ -275,7 +376,7 @@ class Account(object):
         """ Forward list of messages to a chat.
 
         :param messages: list of :class:`deltachat.message.Message` object.
-        :param chat: :class:`deltachat.chatting.Chat` object.
+        :param chat: :class:`deltachat.chat.Chat` object.
         :returns: None
         """
         msg_ids = [msg.id for msg in messages]
@@ -290,31 +391,55 @@ class Account(object):
         msg_ids = [msg.id for msg in messages]
         lib.dc_delete_msgs(self._dc_context, msg_ids, len(msg_ids))
 
-    def export_to_dir(self, backupdir):
-        """return after all delta chat state is exported to a new file in
-        the specified directory.
-        """
-        snap_files = os.listdir(backupdir)
-        self._imex_completed.clear()
-        lib.dc_imex(self._dc_context, 11, as_dc_charpointer(backupdir), ffi.NULL)
-        if not self._threads.is_started():
-            lib.dc_perform_imap_jobs(self._dc_context)
-        self._imex_completed.wait()
-        for x in os.listdir(backupdir):
-            if x not in snap_files:
-                return os.path.join(backupdir, x)
+    def export_self_keys(self, path):
+        """ export public and private keys to the specified directory.
 
-    def import_from_file(self, path):
-        """import delta chat state from the specified backup file.
+        Note that the account does not have to be started.
+        """
+        return self._export(path, imex_cmd=1)
+
+    def export_all(self, path):
+        """return new file containing a backup of all database state
+        (chats, contacts, keys, media, ...). The file is created in the
+        the `path` directory.
+
+        Note that the account does not have to be started.
+        """
+        export_files = self._export(path, 11)
+        if len(export_files) != 1:
+            raise RuntimeError("found more than one new file")
+        return export_files[0]
+
+    def _export(self, path, imex_cmd):
+        with self.temp_plugin(ImexTracker()) as imex_tracker:
+            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
+            if not self._threads.is_started():
+                lib.dc_perform_imap_jobs(self._dc_context)
+            return imex_tracker.wait_finish()
+
+    def import_self_keys(self, path):
+        """ Import private keys found in the `path` directory.
+        The last imported key is made the default keys unless its name
+        contains the string legacy. Public keys are not imported.
+
+        Note that the account does not have to be started.
+        """
+        self._import(path, imex_cmd=2)
+
+    def import_all(self, path):
+        """import delta chat state from the specified backup `path` (a file).
 
         The account must be in unconfigured state for import to attempted.
         """
         assert not self.is_configured(), "cannot import into configured account"
-        self._imex_completed.clear()
-        lib.dc_imex(self._dc_context, 12, as_dc_charpointer(path), ffi.NULL)
-        if not self._threads.is_started():
-            lib.dc_perform_imap_jobs(self._dc_context)
-        self._imex_completed.wait()
+        self._import(path, imex_cmd=12)
+
+    def _import(self, path, imex_cmd):
+        with self.temp_plugin(ImexTracker()) as imex_tracker:
+            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
+            if not self._threads.is_started():
+                lib.dc_perform_imap_jobs(self._dc_context)
+            imex_tracker.wait_finish()
 
     def initiate_key_transfer(self):
         """return setup code after a Autocrypt setup message
@@ -354,7 +479,7 @@ class Account(object):
         """ setup contact and return a Chat after contact is established.
 
         Note that this function may block for a long time as messages are exchanged
-        with the emitter of the QR code.  On success a :class:`deltachat.chatting.Chat` instance
+        with the emitter of the QR code.  On success a :class:`deltachat.chat.Chat` instance
         is returned.
         :param qr: valid "setup contact" QR code (all other QR codes will result in an exception)
         """
@@ -368,7 +493,7 @@ class Account(object):
         """ join a chat group through a QR code.
 
         Note that this function may block for a long time as messages are exchanged
-        with the emitter of the QR code.  On success a :class:`deltachat.chatting.Chat` instance
+        with the emitter of the QR code.  On success a :class:`deltachat.chat.Chat` instance
         is returned which is the chat that we just joined.
 
         :param qr: valid "join-group" QR code (all other QR codes will result in an exception)
@@ -379,162 +504,137 @@ class Account(object):
             raise ValueError("could not join group")
         return Chat(self, chat_id)
 
-    def start_threads(self):
-        """ start IMAP/SMTP threads (and configure account if it hasn't happened).
+    def set_location(self, latitude=0.0, longitude=0.0, accuracy=0.0):
+        """set a new location. It effects all chats where we currently
+        have enabled location streaming.
+
+        :param latitude: float (use 0.0 if not known)
+        :param longitude: float (use 0.0 if not known)
+        :param accuracy: float (use 0.0 if not known)
+        :raises: ValueError if no chat is currently streaming locations
+        :returns: None
+        """
+        dc_res = lib.dc_set_location(self._dc_context, latitude, longitude, accuracy)
+        if dc_res == 0:
+            raise ValueError("no chat is streaming locations")
+
+    #
+    # meta API for start/stop and event based processing
+    #
+
+    def add_account_plugin(self, plugin, name=None):
+        """ add an account plugin which implements one or more of
+        the :class:`deltachat.hookspec.PerAccount` hooks.
+        """
+        self._pm.register(plugin, name=name)
+        self._pm.check_pending()
+        return plugin
+
+    @contextmanager
+    def temp_plugin(self, plugin):
+        """ run a with-block with the given plugin temporarily registered. """
+        self._pm.register(plugin)
+        yield plugin
+        self._pm.unregister(plugin)
+
+    def stop_ongoing(self):
+        """ Stop ongoing securejoin, configuration or other core jobs. """
+        lib.dc_stop_ongoing_process(self._dc_context)
+
+    def start(self, callback_thread=True):
+        """ start this account (activate imap/smtp threads etc.)
+        and return immediately.
+
+        If this account is not configured, an internal configuration
+        job will be scheduled if config values are sufficiently specified.
+
+        You may call `wait_shutdown` or `shutdown` after the
+        account is in started mode.
+
+        :raises MissingCredentials: if `addr` and `mail_pw` values are not set.
 
-        :raises: ValueError if 'addr' or 'mail_pw' are not configured.
         :returns: None
         """
         if not self.is_configured():
-            self.configure()
-        self._threads.start()
+            if not self.get_config("addr") or not self.get_config("mail_pw"):
+                raise MissingCredentials("addr or mail_pwd not set in config")
+            lib.dc_configure(self._dc_context)
+        self._threads.start(callback_thread=callback_thread)
 
-    def stop_threads(self, wait=True):
-        """ stop IMAP/SMTP threads. """
-        lib.dc_stop_ongoing_process(self._dc_context)
-        self._threads.stop(wait=wait)
+    def wait_shutdown(self):
+        """ wait until shutdown of this account has completed. """
+        self._shutdown_event.wait()
 
     def shutdown(self, wait=True):
-        """ stop threads and close and remove underlying dc_context and callbacks. """
-        if hasattr(self, "_dc_context") and hasattr(self, "_threads"):
-            self.stop_threads(wait=False)  # to interrupt idle and tell python threads to stop
-            lib.dc_close(self._dc_context)
-            self.stop_threads(wait=wait)  # to wait for threads
-            deltachat.clear_context_callback(self._dc_context)
-            del self._dc_context
+        """ shutdown account, stop threads and close and remove
+        underlying dc_context and callbacks. """
+        dc_context = self._dc_context
+        if dc_context is None:
+            return
 
-    def _process_event(self, ctx, evt_name, data1, data2):
-        assert ctx == self._dc_context
-        if hasattr(self, "_evlogger"):
-            self._evlogger(evt_name, data1, data2)
-            method = getattr(self, "on_" + evt_name.lower(), None)
-            if method is not None:
-                method(data1, data2)
-        return 0
+        if self._threads.is_started():
+            self.stop_ongoing()
+            self._threads.stop(wait=False)
+        lib.dc_close(dc_context)
+        self._hook_event_queue.put(None)
+        self._threads.stop(wait=wait)  # to wait for threads
+        self._dc_context = None
+        atexit.unregister(self.shutdown)
+        self._shutdown_event.set()
+        hook = hookspec.Global._get_plugin_manager().hook
+        hook.dc_account_after_shutdown(account=self, dc_context=dc_context)
 
-    def on_dc_event_imex_progress(self, data1, data2):
-        if data1 == 1000:
-            self._imex_completed.set()
-
-
-class IOThreads:
-    def __init__(self, dc_context, log_event=lambda *args: None):
-        self._dc_context = dc_context
-        self._thread_quitflag = False
-        self._name2thread = {}
-        self._log_event = log_event
-
-    def is_started(self):
-        return len(self._name2thread) > 0
-
-    def start(self, imap=True, smtp=True):
-        assert not self.is_started()
-        if imap:
-            self._start_one_thread("imap", self.imap_thread_run)
-        if smtp:
-            self._start_one_thread("smtp", self.smtp_thread_run)
-
-    def _start_one_thread(self, name, func):
-        self._name2thread[name] = t = threading.Thread(target=func, name=name)
-        t.setDaemon(1)
-        t.start()
-
-    def stop(self, wait=False):
-        self._thread_quitflag = True
-        lib.dc_interrupt_imap_idle(self._dc_context)
-        lib.dc_interrupt_smtp_idle(self._dc_context)
-        if wait:
-            for name, thread in self._name2thread.items():
-                thread.join()
-
-    def imap_thread_run(self):
-        self._log_event("py-bindings-info", 0, "IMAP THREAD START")
-        while not self._thread_quitflag:
-            lib.dc_perform_imap_jobs(self._dc_context)
-            lib.dc_perform_imap_fetch(self._dc_context)
-            lib.dc_perform_imap_idle(self._dc_context)
-        self._log_event("py-bindings-info", 0, "IMAP THREAD FINISHED")
-
-    def smtp_thread_run(self):
-        self._log_event("py-bindings-info", 0, "SMTP THREAD START")
-        while not self._thread_quitflag:
-            lib.dc_perform_smtp_jobs(self._dc_context)
-            lib.dc_perform_smtp_idle(self._dc_context)
-        self._log_event("py-bindings-info", 0, "SMTP THREAD FINISHED")
-
-
-class EventLogger:
-    _loglock = threading.RLock()
-
-    def __init__(self, dc_context, logid=None, debug=True):
-        self._dc_context = dc_context
-        self._event_queue = Queue()
-        self._debug = debug
-        if logid is None:
-            logid = str(self._dc_context).strip(">").split()[-1]
-        self.logid = logid
-        self._timeout = None
-        self.init_time = time.time()
-
-    def __call__(self, evt_name, data1, data2):
-        self._log_event(evt_name, data1, data2)
-        self._event_queue.put((evt_name, data1, data2))
-
-    def set_timeout(self, timeout):
-        self._timeout = timeout
-
-    def consume_events(self, check_error=True):
-        while not self._event_queue.empty():
-            self.get()
-
-    def get(self, timeout=None, check_error=True):
-        timeout = timeout or self._timeout
-        ev = self._event_queue.get(timeout=timeout)
-        if check_error and ev[0] == "DC_EVENT_ERROR":
-            raise ValueError("{}({!r},{!r})".format(*ev))
-        return ev
-
-    def ensure_event_not_queued(self, event_name_regex):
-        __tracebackhide__ = True
-        rex = re.compile("(?:{}).*".format(event_name_regex))
+    def _handle_current_events(self):
+        """ handle all currently queued events and then return. """
         while 1:
             try:
-                ev = self._event_queue.get(False)
-            except Empty:
+                event = self._hook_event_queue.get(block=False)
+            except queue.Empty:
                 break
             else:
-                assert not rex.match(ev[0]), "event found {}".format(ev)
+                event.call_hook()
 
-    def get_matching(self, event_name_regex, check_error=True):
-        self._log("-- waiting for event with regex: {} --".format(event_name_regex))
-        rex = re.compile("(?:{}).*".format(event_name_regex))
+    def iter_events(self, timeout=None):
+        """ yield hook events until shutdown.
+
+        It is not allowed to call iter_events() from multiple threads.
+        """
+        if self._in_use_iter_events:
+            raise RuntimeError("can only call iter_events() from one thread")
+        self._in_use_iter_events = True
         while 1:
-            ev = self.get()
-            if rex.match(ev[0]):
-                return ev
+            event = self._hook_event_queue.get(timeout=timeout)
+            if event is None:
+                break
+            yield event
 
-    def get_info_matching(self, regex):
-        rex = re.compile("(?:{}).*".format(regex))
-        while 1:
-            ev = self.get_matching("DC_EVENT_INFO")
-            if rex.match(ev[2]):
-                return ev
-
-    def _log_event(self, evt_name, data1, data2):
-        # don't show events that are anyway empty impls now
-        if evt_name == "DC_EVENT_GET_STRING":
-            return
-        if self._debug:
-            evpart = "{}({!r},{!r})".format(evt_name, data1, data2)
-            self._log(evpart)
-
-    def _log(self, msg):
-        t = threading.currentThread()
-        tname = getattr(t, "name", t)
-        if tname == "MainThread":
-            tname = "MAIN"
-        with self._loglock:
-            print("{:2.2f} [{}-{}] {}".format(time.time() - self.init_time, tname, self.logid, msg))
+    def _map_ffi_event(self, ffi_event):
+        name = ffi_event.name
+        if name == "DC_EVENT_CONFIGURE_PROGRESS":
+            data1 = ffi_event.data1
+            if data1 == 0 or data1 == 1000:
+                success = data1 == 1000
+                return "ac_configure_completed", dict(success=success)
+        elif name == "DC_EVENT_INCOMING_MSG":
+            msg = self.get_message_by_id(ffi_event.data2)
+            return "ac_incoming_message", dict(message=msg)
+        elif name == "DC_EVENT_MSGS_CHANGED":
+            if ffi_event.data2 != 0:
+                msg = self.get_message_by_id(ffi_event.data2)
+                if msg.is_in_fresh():
+                    return "ac_incoming_message", dict(message=msg)
+        elif name == "DC_EVENT_MSG_DELIVERED":
+            msg = self.get_message_by_id(ffi_event.data2)
+            return "ac_message_delivered", dict(message=msg)
+        elif name == "DC_EVENT_MEMBER_ADDED":
+            chat = self.get_chat_by_id(ffi_event.data1)
+            contact = self.get_contact_by_id(ffi_event.data2)
+            return "ac_member_added", dict(chat=chat, contact=contact)
+        elif name == "DC_EVENT_MEMBER_REMOVED":
+            chat = self.get_chat_by_id(ffi_event.data1)
+            contact = self.get_contact_by_id(ffi_event.data2)
+            return "ac_member_removed", dict(chat=chat, contact=contact)
+        return None, {}
 
 
 def _destroy_dc_context(dc_context, dc_context_unref=lib.dc_context_unref):
@@ -561,3 +661,17 @@ class ScannedQRCode:
     @property
     def contact_id(self):
         return self._dc_lot.id()
+
+
+class HookEvent:
+    def __init__(self, account, name, kwargs):
+        assert hasattr(account._pm.hook, name), name
+        self.account = account
+        self.name = name
+        self.kwargs = kwargs
+
+    def call_hook(self):
+        hook = getattr(self.account._pm.hook, self.name, None)
+        if hook is None:
+            raise ValueError("event_name {} unknown".format(self.name))
+        return hook(**self.kwargs)

python/src/deltachat/chat.py

@@ -1,58 +1,16 @@
-""" chatting related objects: Contact, Chat, Message. """
+""" Chat and Location related API. """
 
 import mimetypes
+import calendar
+import json
+from datetime import datetime
 import os
-from . import props
 from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array
 from .capi import lib, ffi
 from . import const
 from .message import Message
 
 
-class Contact(object):
-    """ Delta-Chat Contact.
-
-    You obtain instances of it through :class:`deltachat.account.Account`.
-    """
-    def __init__(self, dc_context, id):
-        self._dc_context = dc_context
-        self.id = id
-
-    def __eq__(self, other):
-        return self._dc_context == other._dc_context and self.id == other.id
-
-    def __ne__(self, other):
-        return not (self == other)
-
-    def __repr__(self):
-        return "<Contact id={} addr={} dc_context={}>".format(self.id, self.addr, self._dc_context)
-
-    @property
-    def _dc_contact(self):
-        return ffi.gc(
-            lib.dc_get_contact(self._dc_context, self.id),
-            lib.dc_contact_unref
-        )
-
-    @props.with_doc
-    def addr(self):
-        """ normalized e-mail address for this account. """
-        return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
-
-    @props.with_doc
-    def display_name(self):
-        """ display name for this contact. """
-        return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
-
-    def is_blocked(self):
-        """ Return True if the contact is blocked. """
-        return lib.dc_contact_is_blocked(self._dc_contact)
-
-    def is_verified(self):
-        """ Return True if the contact is verified. """
-        return lib.dc_contact_is_verified(self._dc_contact)
-
-
 class Chat(object):
     """ Chat object which manages members and through which you can send and retrieve messages.
 
@@ -93,6 +51,16 @@ class Chat(object):
 
     # ------  chat status/metadata API ------------------------------
 
+    def is_group(self):
+        """ return true if this chat is a group chat.
+
+        :returns: True if chat is a group-chat, false if it's a contact 1:1 chat.
+        """
+        return lib.dc_chat_get_type(self._dc_chat) in (
+            const.DC_CHAT_TYPE_GROUP,
+            const.DC_CHAT_TYPE_VERIFIED_GROUP
+        )
+
     def is_deaddrop(self):
         """ return true if this chat is a deaddrop chat.
 
@@ -100,6 +68,13 @@ class Chat(object):
         """
         return self.id == const.DC_CHAT_ID_DEADDROP
 
+    def is_muted(self):
+        """ return true if this chat is muted.
+
+        :returns: True if chat is muted, False otherwise.
+        """
+        return lib.dc_chat_is_muted(self._dc_chat)
+
     def is_promoted(self):
         """ return True if this chat is promoted, i.e.
         the member contacts are aware of their membership,
@@ -109,6 +84,13 @@ class Chat(object):
         """
         return not lib.dc_chat_is_unpromoted(self._dc_chat)
 
+    def is_verified(self):
+        """ return True if this chat is a verified group.
+
+        :returns: True if chat is verified, False otherwise.
+        """
+        return lib.dc_chat_is_verified(self._dc_chat)
+
     def get_name(self):
         """ return name of this chat.
 
@@ -119,14 +101,45 @@ class Chat(object):
     def set_name(self, name):
         """ set name of this chat.
 
-        :param: name as a unicode string.
+        :param name: as a unicode string.
         :returns: None
         """
         name = as_dc_charpointer(name)
         return lib.dc_set_chat_name(self._dc_context, self.id, name)
 
+    def mute(self, duration=None):
+        """ mutes the chat
+
+        :param duration: Number of seconds to mute the chat for. None to mute until unmuted again.
+        :returns: None
+        """
+        if duration is None:
+            mute_duration = -1
+        else:
+            mute_duration = duration
+        ret = lib.dc_set_chat_mute_duration(self._dc_context, self.id, mute_duration)
+        if not bool(ret):
+            raise ValueError("Call to dc_set_chat_mute_duration failed")
+
+    def unmute(self):
+        """ unmutes the chat
+
+        :returns: None
+        """
+        ret = lib.dc_set_chat_mute_duration(self._dc_context, self.id, 0)
+        if not bool(ret):
+            raise ValueError("Failed to unmute chat")
+
+    def get_mute_duration(self):
+        """ Returns the number of seconds until the mute of this chat is lifted.
+
+        :param duration:
+        :returns: Returns the number of seconds the chat is still muted for. (0 for not muted, -1 forever muted)
+        """
+        return bool(lib.dc_chat_get_remaining_mute_duration(self.id))
+
     def get_type(self):
-        """ return type of this chat.
+        """ (deprecated) return type of this chat.
 
         :returns: one of const.DC_CHAT_TYPE_*
         """
@@ -144,6 +157,30 @@ class Chat(object):
 
     # ------  chat messaging API ------------------------------
 
+    def send_msg(self, msg):
+        """send a message by using a ready Message object.
+
+        :param msg: a :class:`deltachat.message.Message` instance
+           previously returned by
+           e.g. :meth:`deltachat.message.Message.new_empty` or
+           :meth:`prepare_file`.
+        :raises ValueError: if message can not be sent.
+
+        :returns: a :class:`deltachat.message.Message` instance as
+           sent out.  This is the same object as was passed in, which
+           has been modified with the new state of the core.
+        """
+        if msg.is_out_preparing():
+            assert msg.id != 0
+            # get a fresh copy of dc_msg, the core needs it
+            msg = Message.from_db(self.account, msg.id)
+        sent_id = lib.dc_send_msg(self._dc_context, self.id, msg._dc_msg)
+        if sent_id == 0:
+            raise ValueError("message could not be sent")
+        # modify message in place to avoid bad state for the caller
+        msg._dc_msg = Message.from_db(self.account, sent_id)._dc_msg
+        return msg
+
     def send_text(self, text):
         """ send a text message and return the resulting Message instance.
 
@@ -165,9 +202,12 @@ class Chat(object):
         :raises ValueError: if message can not be send/chat does not exist.
         :returns: the resulting :class:`deltachat.message.Message` instance
         """
-        msg = self.prepare_message_file(path=path, mime_type=mime_type)
-        self.send_prepared(msg)
-        return msg
+        msg = Message.new_empty(self.account, view_type="file")
+        msg.set_file(path, mime_type)
+        sent_id = lib.dc_send_msg(self._dc_context, self.id, msg._dc_msg)
+        if sent_id == 0:
+            raise ValueError("message could not be sent")
+        return Message.from_db(self.account, sent_id)
 
     def send_image(self, path):
         """ send an image message and return the resulting Message instance.
@@ -177,9 +217,12 @@ class Chat(object):
         :returns: the resulting :class:`deltachat.message.Message` instance
         """
         mime_type = mimetypes.guess_type(path)[0]
-        msg = self.prepare_message_file(path=path, mime_type=mime_type, view_type="image")
-        self.send_prepared(msg)
-        return msg
+        msg = Message.new_empty(self.account, view_type="image")
+        msg.set_file(path, mime_type)
+        sent_id = lib.dc_send_msg(self._dc_context, self.id, msg._dc_msg)
+        if sent_id == 0:
+            raise ValueError("message could not be sent")
+        return Message.from_db(self.account, sent_id)
 
     def prepare_message(self, msg):
         """ create a new prepared message.
@@ -278,6 +321,12 @@ class Chat(object):
         """
         return lib.dc_marknoticed_chat(self._dc_context, self.id)
 
+    def get_summary(self):
+        """ return dictionary with summary information. """
+        dc_res = lib.dc_chat_get_info_json(self._dc_context, self.id)
+        s = from_dc_charpointer(dc_res)
+        return json.loads(s)
+
     # ------  group management API ------------------------------
 
     def add_contact(self, contact):
@@ -305,16 +354,16 @@ class Chat(object):
     def get_contacts(self):
         """ get all contacts for this chat.
         :params: contact object.
-        :raises ValueError: if contact could not be added
-        :returns: list of :class:`deltachat.chatting.Contact` objects for this chat
+        :returns: list of :class:`deltachat.contact.Contact` objects for this chat
 
         """
+        from .contact import Contact
         dc_array = ffi.gc(
             lib.dc_get_chat_contacts(self._dc_context, self.id),
             lib.dc_array_unref
         )
         return list(iter_array(
-            dc_array, lambda id: Contact(self._dc_context, id))
+            dc_array, lambda id: Contact(self.account, id))
         )
 
     def set_profile_image(self, img_path):
@@ -359,3 +408,80 @@ class Chat(object):
         if dc_res == ffi.NULL:
             return None
         return from_dc_charpointer(dc_res)
+
+    def get_color(self):
+        """return the color of the chat.
+        :returns: color as 0x00rrggbb
+        """
+        return lib.dc_chat_get_color(self._dc_chat)
+
+    def get_subtitle(self):
+        """return the subtitle of the chat
+        :returns: the subtitle
+        """
+        return from_dc_charpointer(lib.dc_chat_get_subtitle(self._dc_chat))
+
+    # ------  location streaming API ------------------------------
+
+    def is_sending_locations(self):
+        """return True if this chat has location-sending enabled currently.
+        :returns: True if location sending is enabled.
+        """
+        return lib.dc_is_sending_locations_to_chat(self._dc_context, self.id)
+
+    def is_archived(self):
+        """return True if this chat is archived.
+        :returns: True if archived.
+        """
+        return lib.dc_chat_get_visibility(self._dc_chat) == const.DC_CHAT_VISIBILITY_ARCHIVED
+
+    def enable_sending_locations(self, seconds):
+        """enable sending locations for this chat.
+
+        all subsequent messages will carry a location with them.
+        """
+        lib.dc_send_locations_to_chat(self._dc_context, self.id, seconds)
+
+    def get_locations(self, contact=None, timestamp_from=None, timestamp_to=None):
+        """return list of locations for the given contact in the given timespan.
+
+        :param contact: the contact for which locations shall be returned.
+        :param timespan_from: a datetime object or None (indicating "since beginning")
+        :param timespan_to: a datetime object or None (indicating up till now)
+        :returns: list of :class:`deltachat.chat.Location` objects.
+        """
+        if timestamp_from is None:
+            time_from = 0
+        else:
+            time_from = calendar.timegm(timestamp_from.utctimetuple())
+        if timestamp_to is None:
+            time_to = 0
+        else:
+            time_to = calendar.timegm(timestamp_to.utctimetuple())
+
+        if contact is None:
+            contact_id = 0
+        else:
+            contact_id = contact.id
+
+        dc_array = lib.dc_get_locations(self._dc_context, self.id, contact_id, time_from, time_to)
+        return [
+            Location(
+                latitude=lib.dc_array_get_latitude(dc_array, i),
+                longitude=lib.dc_array_get_longitude(dc_array, i),
+                accuracy=lib.dc_array_get_accuracy(dc_array, i),
+                timestamp=datetime.utcfromtimestamp(lib.dc_array_get_timestamp(dc_array, i)))
+            for i in range(lib.dc_array_get_cnt(dc_array))
+        ]
+
+
+class Location:
+    def __init__(self, latitude, longitude, accuracy, timestamp):
+        assert isinstance(timestamp, datetime)
+        self.latitude = latitude
+        self.longitude = longitude
+        self.accuracy = accuracy
+        self.timestamp = timestamp
+
+    def __eq__(self, other):
+        return self.__dict__ == other.__dict__

python/src/deltachat/const.py

@@ -18,6 +18,7 @@ DC_QR_ASK_VERIFYGROUP = 202
 DC_QR_FPR_OK = 210
 DC_QR_FPR_MISMATCH = 220
 DC_QR_FPR_WITHOUT_ADDR = 230
+DC_QR_ACCOUNT = 250
 DC_QR_ADDR = 320
 DC_QR_TEXT = 330
 DC_QR_URL = 332
@@ -47,19 +48,39 @@ DC_STATE_OUT_FAILED = 24
 DC_STATE_OUT_DELIVERED = 26
 DC_STATE_OUT_MDN_RCVD = 28
 DC_CONTACT_ID_SELF = 1
-DC_CONTACT_ID_DEVICE = 2
+DC_CONTACT_ID_INFO = 2
+DC_CONTACT_ID_DEVICE = 5
 DC_CONTACT_ID_LAST_SPECIAL = 9
 DC_MSG_TEXT = 10
 DC_MSG_IMAGE = 20
 DC_MSG_GIF = 21
+DC_MSG_STICKER = 23
 DC_MSG_AUDIO = 40
 DC_MSG_VOICE = 41
 DC_MSG_VIDEO = 50
 DC_MSG_FILE = 60
+DC_LP_AUTH_OAUTH2 = 0x2
+DC_LP_AUTH_NORMAL = 0x4
+DC_LP_IMAP_SOCKET_STARTTLS = 0x100
+DC_LP_IMAP_SOCKET_SSL = 0x200
+DC_LP_IMAP_SOCKET_PLAIN = 0x400
+DC_LP_SMTP_SOCKET_STARTTLS = 0x10000
+DC_LP_SMTP_SOCKET_SSL = 0x20000
+DC_LP_SMTP_SOCKET_PLAIN = 0x40000
+DC_CERTCK_AUTO = 0
+DC_CERTCK_STRICT = 1
+DC_CERTCK_ACCEPT_INVALID_CERTIFICATES = 3
+DC_EMPTY_MVBOX = 0x01
+DC_EMPTY_INBOX = 0x02
 DC_EVENT_INFO = 100
 DC_EVENT_SMTP_CONNECTED = 101
 DC_EVENT_IMAP_CONNECTED = 102
 DC_EVENT_SMTP_MESSAGE_SENT = 103
+DC_EVENT_IMAP_MESSAGE_DELETED = 104
+DC_EVENT_IMAP_MESSAGE_MOVED = 105
+DC_EVENT_IMAP_FOLDER_EMPTIED = 106
+DC_EVENT_NEW_BLOB_FILE = 150
+DC_EVENT_DELETED_BLOB_FILE = 151
 DC_EVENT_WARNING = 300
 DC_EVENT_ERROR = 400
 DC_EVENT_ERROR_NETWORK = 401
@@ -77,14 +98,75 @@ DC_EVENT_IMEX_PROGRESS = 2051
 DC_EVENT_IMEX_FILE_WRITTEN = 2052
 DC_EVENT_SECUREJOIN_INVITER_PROGRESS = 2060
 DC_EVENT_SECUREJOIN_JOINER_PROGRESS = 2061
-DC_EVENT_GET_STRING = 2091
+DC_EVENT_SECUREJOIN_MEMBER_ADDED = 2062
+DC_EVENT_MEMBER_ADDED = 2063
+DC_EVENT_MEMBER_REMOVED = 2064
 DC_EVENT_FILE_COPIED = 2055
 DC_EVENT_IS_OFFLINE = 2081
+DC_EVENT_GET_STRING = 2091
+DC_STR_SELFNOTINGRP = 21
+DC_KEY_GEN_DEFAULT = 0
+DC_KEY_GEN_RSA2048 = 1
+DC_KEY_GEN_ED25519 = 2
+DC_PROVIDER_STATUS_OK = 1
+DC_PROVIDER_STATUS_PREPARATION = 2
+DC_PROVIDER_STATUS_BROKEN = 3
+DC_CHAT_VISIBILITY_NORMAL = 0
+DC_CHAT_VISIBILITY_ARCHIVED = 1
+DC_CHAT_VISIBILITY_PINNED = 2
+DC_STR_NOMESSAGES = 1
+DC_STR_SELF = 2
+DC_STR_DRAFT = 3
+DC_STR_MEMBER = 4
+DC_STR_CONTACT = 6
+DC_STR_VOICEMESSAGE = 7
+DC_STR_DEADDROP = 8
+DC_STR_IMAGE = 9
+DC_STR_VIDEO = 10
+DC_STR_AUDIO = 11
+DC_STR_FILE = 12
+DC_STR_STATUSLINE = 13
+DC_STR_NEWGROUPDRAFT = 14
+DC_STR_MSGGRPNAME = 15
+DC_STR_MSGGRPIMGCHANGED = 16
+DC_STR_MSGADDMEMBER = 17
+DC_STR_MSGDELMEMBER = 18
+DC_STR_MSGGROUPLEFT = 19
+DC_STR_GIF = 23
+DC_STR_ENCRYPTEDMSG = 24
+DC_STR_E2E_AVAILABLE = 25
+DC_STR_ENCR_TRANSP = 27
+DC_STR_ENCR_NONE = 28
+DC_STR_CANTDECRYPT_MSG_BODY = 29
+DC_STR_FINGERPRINTS = 30
+DC_STR_READRCPT = 31
+DC_STR_READRCPT_MAILBODY = 32
+DC_STR_MSGGRPIMGDELETED = 33
+DC_STR_E2E_PREFERRED = 34
+DC_STR_CONTACT_VERIFIED = 35
+DC_STR_CONTACT_NOT_VERIFIED = 36
+DC_STR_CONTACT_SETUP_CHANGED = 37
+DC_STR_ARCHIVEDCHATS = 40
+DC_STR_STARREDMSGS = 41
+DC_STR_AC_SETUP_MSG_SUBJECT = 42
+DC_STR_AC_SETUP_MSG_BODY = 43
+DC_STR_SELFTALK_SUBTITLE = 50
+DC_STR_CANNOT_LOGIN = 60
+DC_STR_SERVER_RESPONSE = 61
+DC_STR_MSGACTIONBYUSER = 62
+DC_STR_MSGACTIONBYME = 63
+DC_STR_MSGLOCATIONENABLED = 64
+DC_STR_MSGLOCATIONDISABLED = 65
+DC_STR_LOCATION = 66
+DC_STR_STICKER = 67
+DC_STR_DEVICE_MESSAGES = 68
+DC_STR_COUNT = 68
 # end const generated
 
 
 def read_event_defines(f):
-    rex = re.compile(r'#define\s+((?:DC_EVENT_|DC_QR|DC_MSG|DC_STATE_|DC_CONTACT_ID_|DC_GCL|DC_CHAT)\S+)\s+([x\d]+).*')
+    rex = re.compile(r'#define\s+((?:DC_EVENT|DC_QR|DC_MSG|DC_LP|DC_EMPTY|DC_CERTCK|DC_STATE|DC_STR|'
+                     r'DC_CONTACT_ID|DC_GCL|DC_CHAT|DC_PROVIDER|DC_KEY_GEN)_\S+)\s+([x\d]+).*')
     for line in f:
         m = rex.match(line)
         if m:

python/src/deltachat/contact.py

@@ -0,0 +1,64 @@
+""" Contact object. """
+
+from . import props
+from .cutil import from_dc_charpointer
+from .capi import lib, ffi
+
+
+class Contact(object):
+    """ Delta-Chat Contact.
+
+    You obtain instances of it through :class:`deltachat.account.Account`.
+    """
+    def __init__(self, account, id):
+        self.account = account
+        self._dc_context = account._dc_context
+        self.id = id
+
+    def __eq__(self, other):
+        return self._dc_context == other._dc_context and self.id == other.id
+
+    def __ne__(self, other):
+        return not (self == other)
+
+    def __repr__(self):
+        return "<Contact id={} addr={} dc_context={}>".format(self.id, self.addr, self._dc_context)
+
+    @property
+    def _dc_contact(self):
+        return ffi.gc(
+            lib.dc_get_contact(self._dc_context, self.id),
+            lib.dc_contact_unref
+        )
+
+    @props.with_doc
+    def addr(self):
+        """ normalized e-mail address for this account. """
+        return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
+
+    @props.with_doc
+    def display_name(self):
+        """ display name for this contact. """
+        return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
+
+    def is_blocked(self):
+        """ Return True if the contact is blocked. """
+        return lib.dc_contact_is_blocked(self._dc_contact)
+
+    def is_verified(self):
+        """ Return True if the contact is verified. """
+        return lib.dc_contact_is_verified(self._dc_contact)
+
+    def get_profile_image(self):
+        """Get contact profile image.
+
+        :returns: path to profile image, None if no profile image exists.
+        """
+        dc_res = lib.dc_contact_get_profile_image(self._dc_contact)
+        if dc_res == ffi.NULL:
+            return None
+        return from_dc_charpointer(dc_res)
+
+    def get_chat(self):
+        """return 1:1 chat for this contact. """
+        return self.account.create_chat_by_contact(self)

python/src/deltachat/eventlogger.py

@@ -0,0 +1,137 @@
+import deltachat
+import threading
+import time
+import re
+from queue import Queue, Empty
+from .hookspec import account_hookimpl, global_hookimpl
+
+
+@global_hookimpl
+def dc_account_init(account):
+    # send all FFI events for this account to a plugin hook
+    def _ll_event(ctx, evt_name, data1, data2):
+        assert ctx == account._dc_context
+        ffi_event = FFIEvent(name=evt_name, data1=data1, data2=data2)
+        account._pm.hook.ac_process_ffi_event(
+            account=account, ffi_event=ffi_event
+        )
+    deltachat.set_context_callback(account._dc_context, _ll_event)
+
+
+@global_hookimpl
+def dc_account_after_shutdown(dc_context):
+    deltachat.clear_context_callback(dc_context)
+
+
+class FFIEvent:
+    def __init__(self, name, data1, data2):
+        self.name = name
+        self.data1 = data1
+        self.data2 = data2
+
+    def __str__(self):
+        return "{name} data1={data1} data2={data2}".format(**self.__dict__)
+
+
+class FFIEventLogger:
+    """ If you register an instance of this logger with an Account
+    you'll get all ffi-events printed.
+    """
+    # to prevent garbled logging
+    _loglock = threading.RLock()
+
+    def __init__(self, account, logid):
+        """
+        :param logid: an optional logging prefix that should be used with
+                      the default internal logging.
+        """
+        self.account = account
+        self.logid = logid
+        self.init_time = time.time()
+
+    @account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        self._log_event(ffi_event)
+
+    def _log_event(self, ffi_event):
+        # don't show events that are anyway empty impls now
+        if ffi_event.name == "DC_EVENT_GET_STRING":
+            return
+        self.account.ac_log_line(str(ffi_event))
+
+    @account_hookimpl
+    def ac_log_line(self, message):
+        t = threading.currentThread()
+        tname = getattr(t, "name", t)
+        if tname == "MainThread":
+            tname = "MAIN"
+        elapsed = time.time() - self.init_time
+        locname = tname
+        if self.logid:
+            locname += "-" + self.logid
+        s = "{:2.2f} [{}] {}".format(elapsed, locname, message)
+        with self._loglock:
+            print(s, flush=True)
+
+
+class FFIEventTracker:
+    def __init__(self, account, timeout=None):
+        self.account = account
+        self._timeout = timeout
+        self._event_queue = Queue()
+
+    @account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        self._event_queue.put(ffi_event)
+
+    def set_timeout(self, timeout):
+        self._timeout = timeout
+
+    def consume_events(self, check_error=True):
+        while not self._event_queue.empty():
+            self.get(check_error=check_error)
+
+    def get(self, timeout=None, check_error=True):
+        timeout = timeout if timeout is not None else self._timeout
+        ev = self._event_queue.get(timeout=timeout)
+        if check_error and ev.name == "DC_EVENT_ERROR":
+            raise ValueError(str(ev))
+        return ev
+
+    def ensure_event_not_queued(self, event_name_regex):
+        __tracebackhide__ = True
+        rex = re.compile("(?:{}).*".format(event_name_regex))
+        while 1:
+            try:
+                ev = self._event_queue.get(False)
+            except Empty:
+                break
+            else:
+                assert not rex.match(ev.name), "event found {}".format(ev)
+
+    def get_matching(self, event_name_regex, check_error=True, timeout=None):
+        self.account.ac_log_line("-- waiting for event with regex: {} --".format(event_name_regex))
+        rex = re.compile("(?:{}).*".format(event_name_regex))
+        while 1:
+            ev = self.get(timeout=timeout, check_error=check_error)
+            if rex.match(ev.name):
+                return ev
+
+    def get_info_matching(self, regex):
+        rex = re.compile("(?:{}).*".format(regex))
+        while 1:
+            ev = self.get_matching("DC_EVENT_INFO")
+            if rex.match(ev.data2):
+                return ev
+
+    def wait_next_incoming_message(self):
+        """ wait for and return next incoming message. """
+        ev = self.get_matching("DC_EVENT_INCOMING_MSG")
+        return self.account.get_message_by_id(ev.data2)
+
+    def wait_next_messages_changed(self):
+        """ wait for and return next message-changed message or None
+        if the event contains no msgid"""
+        ev = self.get_matching("DC_EVENT_MSGS_CHANGED")
+        if ev.data2 > 0:
+            return self.account.get_message_by_id(ev.data2)

python/src/deltachat/hookspec.py

@@ -0,0 +1,84 @@
+""" Hooks for Python bindings to Delta Chat Core Rust CFFI"""
+
+import pluggy
+
+
+account_spec_name = "deltachat-account"
+account_hookspec = pluggy.HookspecMarker(account_spec_name)
+account_hookimpl = pluggy.HookimplMarker(account_spec_name)
+
+global_spec_name = "deltachat-global"
+global_hookspec = pluggy.HookspecMarker(global_spec_name)
+global_hookimpl = pluggy.HookimplMarker(global_spec_name)
+
+
+class PerAccount:
+    """ per-Account-instance hook specifications.
+
+    Except for ac_process_ffi_event all hooks are executed
+    in the thread which calls Account.wait_shutdown().
+    """
+    @classmethod
+    def _make_plugin_manager(cls):
+        pm = pluggy.PluginManager(account_spec_name)
+        pm.add_hookspecs(cls)
+        return pm
+
+    @account_hookspec
+    def ac_process_ffi_event(self, ffi_event):
+        """ process a CFFI low level events for a given account.
+
+        ffi_event has "name", "data1", "data2" values as specified
+        with `DC_EVENT_* <https://c.delta.chat/group__DC__EVENT.html>`_.
+
+        DANGER: this hook is executed from the callback invoked by core.
+        Hook implementations need to be short running and can typically
+        not call back into core because this would easily cause recursion issues.
+        """
+
+    @account_hookspec
+    def ac_log_line(self, message):
+        """ log a message related to the account. """
+
+    @account_hookspec
+    def ac_configure_completed(self, success):
+        """ Called when a configure process completed. """
+
+    @account_hookspec
+    def ac_incoming_message(self, message):
+        """ Called on any incoming message (to deaddrop or chat). """
+
+    @account_hookspec
+    def ac_message_delivered(self, message):
+        """ Called when an outgoing message has been delivered to SMTP. """
+
+    @account_hookspec
+    def ac_member_added(self, chat, contact):
+        """ Called for each contact added to a chat. """
+
+    @account_hookspec
+    def ac_member_removed(self, chat, contact):
+        """ Called for each contact removed from a chat. """
+
+
+class Global:
+    """ global hook specifications using a per-process singleton
+    plugin manager instance.
+
+    """
+    _plugin_manager = None
+
+    @classmethod
+    def _get_plugin_manager(cls):
+        if cls._plugin_manager is None:
+            cls._plugin_manager = pm = pluggy.PluginManager(global_spec_name)
+            pm.add_hookspecs(cls)
+        return cls._plugin_manager
+
+    @global_hookspec
+    def dc_account_init(self, account):
+        """ called when `Account::__init__()` function starts executing. """
+
+    @global_hookspec
+    def dc_account_after_shutdown(self, account, dc_context):
+        """ Called after the account has been shutdown. """

python/src/deltachat/iothreads.py

@@ -0,0 +1,106 @@
+
+import threading
+import time
+
+from contextlib import contextmanager
+
+from .capi import lib
+
+
+class IOThreads:
+    def __init__(self, account):
+        self.account = account
+        self._dc_context = account._dc_context
+        self._thread_quitflag = False
+        self._name2thread = {}
+
+    def is_started(self):
+        return len(self._name2thread) > 0
+
+    def start(self, callback_thread):
+        assert not self.is_started()
+        self._start_one_thread("inbox", self.imap_thread_run)
+        self._start_one_thread("smtp", self.smtp_thread_run)
+
+        if callback_thread:
+            self._start_one_thread("cb", self.cb_thread_run)
+
+        if int(self.account.get_config("mvbox_watch")):
+            self._start_one_thread("mvbox", self.mvbox_thread_run)
+
+        if int(self.account.get_config("sentbox_watch")):
+            self._start_one_thread("sentbox", self.sentbox_thread_run)
+
+    def _start_one_thread(self, name, func):
+        self._name2thread[name] = t = threading.Thread(target=func, name=name)
+        t.setDaemon(1)
+        t.start()
+
+    @contextmanager
+    def log_execution(self, message):
+        self.account.ac_log_line(message + " START")
+        yield
+        self.account.ac_log_line(message + " FINISHED")
+
+    def stop(self, wait=False):
+        self._thread_quitflag = True
+
+        # Workaround for a race condition. Make sure that thread is
+        # not in between checking for quitflag and entering idle.
+        time.sleep(0.5)
+
+        lib.dc_interrupt_imap_idle(self._dc_context)
+        lib.dc_interrupt_smtp_idle(self._dc_context)
+        if "mvbox" in self._name2thread:
+            lib.dc_interrupt_mvbox_idle(self._dc_context)
+        if "sentbox" in self._name2thread:
+            lib.dc_interrupt_sentbox_idle(self._dc_context)
+        if wait:
+            for name, thread in self._name2thread.items():
+                if thread != threading.currentThread():
+                    thread.join()
+
+    def cb_thread_run(self):
+        with self.log_execution("CALLBACK THREAD START"):
+            it = self.account.iter_events()
+            while not self._thread_quitflag:
+                try:
+                    ev = next(it)
+                except StopIteration:
+                    break
+                self.account.ac_log_line("calling hook name={} kwargs={}".format(ev.name, ev.kwargs))
+                ev.call_hook()
+
+    def imap_thread_run(self):
+        with self.log_execution("INBOX THREAD START"):
+            while not self._thread_quitflag:
+                lib.dc_perform_imap_jobs(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_imap_fetch(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_imap_idle(self._dc_context)
+
+    def mvbox_thread_run(self):
+        with self.log_execution("MVBOX THREAD"):
+            while not self._thread_quitflag:
+                lib.dc_perform_mvbox_jobs(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_mvbox_fetch(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_mvbox_idle(self._dc_context)
+
+    def sentbox_thread_run(self):
+        with self.log_execution("SENTBOX THREAD"):
+            while not self._thread_quitflag:
+                lib.dc_perform_sentbox_jobs(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_sentbox_fetch(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_sentbox_idle(self._dc_context)
+
+    def smtp_thread_run(self):
+        with self.log_execution("SMTP THREAD"):
+            while not self._thread_quitflag:
+                lib.dc_perform_smtp_jobs(self._dc_context)
+                if not self._thread_quitflag:
+                    lib.dc_perform_smtp_idle(self._dc_context)

python/src/deltachat/message.py

@@ -1,7 +1,6 @@
-""" chatting related objects: Contact, Chat, Message. """
+""" The Message object. """
 
 import os
-import shutil
 from . import props
 from .cutil import from_dc_charpointer, as_dc_charpointer
 from .capi import lib, ffi
@@ -13,7 +12,7 @@ class Message(object):
     """ Message object.
 
     You obtain instances of it through :class:`deltachat.account.Account` or
-    :class:`deltachat.chatting.Chat`.
+    :class:`deltachat.chat.Chat`.
     """
     def __init__(self, account, dc_msg):
         self.account = account
@@ -51,6 +50,16 @@ class Message(object):
             lib.dc_msg_unref
         ))
 
+    def accept_sender_contact(self):
+        """ ensure that the sender is an accepted contact
+        and that the message has a non-deaddrop chat object.
+        """
+        self.account.create_chat_by_message(self)
+        self._dc_msg = ffi.gc(
+                lib.dc_get_msg(self._dc_context, self.id),
+                lib.dc_msg_unref
+        )
+
     @props.with_doc
     def text(self):
         """unicode text of this messages (might be empty if not a text message). """
@@ -58,8 +67,6 @@ class Message(object):
 
     def set_text(self, text):
         """set text of this message. """
-        assert self.id > 0, "message not prepared"
-        assert self.is_out_preparing()
         lib.dc_msg_set_text(self._dc_msg, as_dc_charpointer(text))
 
     @props.with_doc
@@ -72,19 +79,6 @@ class Message(object):
         mtype = ffi.NULL if mime_type is None else as_dc_charpointer(mime_type)
         if not os.path.exists(path):
             raise ValueError("path does not exist: {!r}".format(path))
-        blobdir = self.account.get_blobdir()
-        if not path.startswith(blobdir):
-            for i in range(50):
-                ext = "" if i == 0 else "-" + str(i)
-                dest = os.path.join(blobdir, os.path.basename(path) + ext)
-                if os.path.exists(dest):
-                    continue
-                shutil.copyfile(path, dest)
-                break
-            else:
-                raise ValueError("could not create blobdir-path for {}".format(path))
-            path = dest
-        assert path.startswith(blobdir), path
         lib.dc_msg_set_file(self._dc_msg, as_dc_charpointer(path), mtype)
 
     @props.with_doc
@@ -101,6 +95,18 @@ class Message(object):
         """ return True if this message is a setup message. """
         return lib.dc_msg_is_setupmessage(self._dc_msg)
 
+    def get_setupcodebegin(self):
+        """ return the first characters of a setup code in a setup message. """
+        return from_dc_charpointer(lib.dc_msg_get_setupcodebegin(self._dc_msg))
+
+    def is_encrypted(self):
+        """ return True if this message was encrypted. """
+        return bool(lib.dc_msg_get_showpadlock(self._dc_msg))
+
+    def is_forwarded(self):
+        """ return True if this message was forwarded. """
+        return bool(lib.dc_msg_is_forwarded(self._dc_msg))
+
     def get_message_info(self):
         """ Return informational text for a single message.
 
@@ -110,7 +116,13 @@ class Message(object):
 
     def continue_key_transfer(self, setup_code):
         """ extract key and use it as primary key for this account. """
-        lib.dc_continue_key_transfer(self._dc_context, self.id, as_dc_charpointer(setup_code))
+        res = lib.dc_continue_key_transfer(
+                self._dc_context,
+                self.id,
+                as_dc_charpointer(setup_code)
+        )
+        if res == 0:
+            raise ValueError("could not decrypt")
 
     @props.with_doc
     def time_sent(self):
@@ -144,27 +156,34 @@ class Message(object):
         if mime_headers:
             s = ffi.string(ffi.gc(mime_headers, lib.dc_str_unref))
             if isinstance(s, bytes):
-                s = s.decode("ascii")
+                return email.message_from_bytes(s)
             return email.message_from_string(s)
 
     @property
     def chat(self):
         """chat this message was posted in.
 
-        :returns: :class:`deltachat.chatting.Chat` object
+        :returns: :class:`deltachat.chat.Chat` object
         """
-        from .chatting import Chat
+        from .chat import Chat
         chat_id = lib.dc_msg_get_chat_id(self._dc_msg)
         return Chat(self.account, chat_id)
 
+    def get_sender_chat(self):
+        """return the 1:1 chat with the sender of this message.
+
+        :returns: :class:`deltachat.chat.Chat` instance
+        """
+        return self.get_sender_contact().get_chat()
+
     def get_sender_contact(self):
         """return the contact of who wrote the message.
 
-        :returns: :class:`deltachat.chatting.Contact` instance
+        :returns: :class:`deltachat.chat.Contact` instance
         """
-        from .chatting import Contact
+        from .contact import Contact
         contact_id = lib.dc_msg_get_from_id(self._dc_msg)
-        return Contact(self._dc_context, contact_id)
+        return Contact(self.account, contact_id)
 
     #
     # Message State query methods
@@ -172,7 +191,7 @@ class Message(object):
     @property
     def _msgstate(self):
         if self.id == 0:
-            dc_msg = self.message._dc_msg
+            dc_msg = self._dc_msg
         else:
             # load message from db to get a fresh/current state
             dc_msg = ffi.gc(
@@ -267,6 +286,10 @@ class Message(object):
         """ return True if it's a file message. """
         return self._view_type == const.DC_MSG_FILE
 
+    def mark_seen(self):
+        """ mark this message as seen. """
+        self.account.mark_seen_messages([self.id])
+
 
 # some code for handling DC_MSG_* view types
 

python/src/deltachat/provider.py

@@ -0,0 +1,47 @@
+"""Provider info class."""
+
+from .capi import ffi, lib
+from .cutil import as_dc_charpointer, from_dc_charpointer
+
+
+class ProviderNotFoundError(Exception):
+    """The provider information was not found."""
+
+
+class Provider(object):
+    """Provider information.
+
+    :param domain: The email to get the provider info for.
+    """
+
+    def __init__(self, account, addr):
+        provider = ffi.gc(
+            lib.dc_provider_new_from_email(account._dc_context, as_dc_charpointer(addr)),
+            lib.dc_provider_unref,
+        )
+        if provider == ffi.NULL:
+            raise ProviderNotFoundError("Provider not found")
+        self._provider = provider
+
+    @property
+    def overview_page(self):
+        """URL to the overview page of the provider on providers.delta.chat."""
+        return from_dc_charpointer(
+            lib.dc_provider_get_overview_page(self._provider))
+
+    @property
+    def get_before_login_hints(self):
+        """Should be shown to the user on login."""
+        return from_dc_charpointer(
+            lib.dc_provider_get_before_login_hints(self._provider))
+
+    @property
+    def status(self):
+        """The status of the provider information.
+
+        This is one of the
+        :attr:`deltachat.const.DC_PROVIDER_STATUS_OK`,
+        :attr:`deltachat.const.DC_PROVIDER_STATUS_PREPARATION` or
+        :attr:`deltachat.const.DC_PROVIDER_STATUS_BROKEN` constants.
+        """
+        return lib.dc_provider_get_status(self._provider)

python/src/deltachat/testplugin.py

@@ -0,0 +1,373 @@
+from __future__ import print_function
+import os
+import sys
+import subprocess
+import queue
+import threading
+import fnmatch
+import pytest
+import requests
+import time
+from . import Account, const
+from .tracker import ConfigureTracker
+from .capi import lib
+from .eventlogger import FFIEventLogger, FFIEventTracker
+from _pytest.monkeypatch import MonkeyPatch
+from _pytest._code import Source
+
+import tempfile
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--liveconfig", action="store", default=None,
+        help="a file with >=2 lines where each line "
+             "contains NAME=VALUE config settings for one account"
+    )
+    parser.addoption(
+        "--ignored", action="store_true",
+        help="Also run tests marked with the ignored marker",
+    )
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers", "ignored: Mark test as bing slow, skipped unless --ignored is used."
+    )
+    cfg = config.getoption('--liveconfig')
+    if not cfg:
+        cfg = os.getenv('DCC_NEW_TMP_EMAIL')
+        if cfg:
+            config.option.liveconfig = cfg
+
+
+def pytest_runtest_setup(item):
+    if (list(item.iter_markers(name="ignored"))
+            and not item.config.getoption("ignored")):
+        pytest.skip("Ignored tests not requested, use --ignored")
+
+
+def pytest_report_header(config, startdir):
+    summary = []
+
+    t = tempfile.mktemp()
+    m = MonkeyPatch()
+    try:
+        m.setattr(sys.stdout, "write", lambda x: len(x))
+        ac = Account(t)
+        info = ac.get_info()
+        ac.shutdown()
+    finally:
+        m.undo()
+        os.remove(t)
+    summary.extend(['Deltachat core={} sqlite={}'.format(
+         info['deltachat_core_version'],
+         info['sqlite_version'],
+     )])
+
+    cfg = config.option.liveconfig
+    if cfg:
+        if "?" in cfg:
+            url, token = cfg.split("?", 1)
+            summary.append('Liveconfig provider: {}?<token ommitted>'.format(url))
+        else:
+            summary.append('Liveconfig file: {}'.format(cfg))
+    return summary
+
+
+class SessionLiveConfigFromFile:
+    def __init__(self, fn):
+        self.fn = fn
+        self.configlist = []
+        for line in open(fn):
+            if line.strip() and not line.strip().startswith('#'):
+                d = {}
+                for part in line.split():
+                    name, value = part.split("=")
+                    d[name] = value
+                self.configlist.append(d)
+
+    def get(self, index):
+        return self.configlist[index]
+
+    def exists(self):
+        return bool(self.configlist)
+
+
+class SessionLiveConfigFromURL:
+    def __init__(self, url):
+        self.configlist = []
+        self.url = url
+
+    def get(self, index):
+        try:
+            return self.configlist[index]
+        except IndexError:
+            assert index == len(self.configlist), index
+            res = requests.post(self.url)
+            if res.status_code != 200:
+                pytest.skip("creating newtmpuser failed {!r}".format(res))
+            d = res.json()
+            config = dict(addr=d["email"], mail_pw=d["password"])
+            self.configlist.append(config)
+            return config
+
+    def exists(self):
+        return bool(self.configlist)
+
+
+@pytest.fixture(scope="session")
+def session_liveconfig(request):
+    liveconfig_opt = request.config.option.liveconfig
+    if liveconfig_opt:
+        if liveconfig_opt.startswith("http"):
+            return SessionLiveConfigFromURL(liveconfig_opt)
+        else:
+            return SessionLiveConfigFromFile(liveconfig_opt)
+
+
+@pytest.fixture
+def data(request):
+    class Data:
+        def __init__(self):
+            # trying to find test data heuristically
+            # because we are run from a dev-setup with pytest direct,
+            # through tox, and then maybe also from deltachat-binding
+            # users like "deltabot".
+            self.paths = [os.path.normpath(x) for x in [
+                os.path.join(os.path.dirname(request.fspath.strpath), "data"),
+                os.path.join(os.path.dirname(__file__), "..", "..", "..", "test-data")
+            ]]
+
+        def get_path(self, bn):
+            """ return path of file or None if it doesn't exist. """
+            for path in self.paths:
+                fn = os.path.join(path, *bn.split("/"))
+                if os.path.exists(fn):
+                    return fn
+            print("WARNING: path does not exist: {!r}".format(fn))
+
+        def read_path(self, bn, mode="r"):
+            fn = self.get_path(bn)
+            if fn is not None:
+                with open(fn, mode) as f:
+                    return f.read()
+
+    return Data()
+
+
+@pytest.fixture
+def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
+
+    class AccountMaker:
+        def __init__(self):
+            self.live_count = 0
+            self.offline_count = 0
+            self._finalizers = []
+            self.init_time = time.time()
+            self._generated_keys = ["alice", "bob", "charlie",
+                                    "dom", "elena", "fiona"]
+
+        def finalize(self):
+            while self._finalizers:
+                fin = self._finalizers.pop()
+                fin()
+
+        def make_account(self, path, logid, quiet=False):
+            ac = Account(path)
+            ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
+            ac._configtracker = ac.add_account_plugin(ConfigureTracker())
+            if not quiet:
+                ac.add_account_plugin(FFIEventLogger(ac, logid=logid))
+            self._finalizers.append(ac.shutdown)
+            return ac
+
+        def get_unconfigured_account(self):
+            self.offline_count += 1
+            tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
+            ac._evtracker.init_time = self.init_time
+            ac._evtracker.set_timeout(2)
+            return ac
+
+        def _preconfigure_key(self, account, addr):
+            # Only set a key if we haven't used it yet for another account.
+            if self._generated_keys:
+                keyname = self._generated_keys.pop(0)
+                fname_pub = data.read_path("key/{name}-public.asc".format(name=keyname))
+                fname_sec = data.read_path("key/{name}-secret.asc".format(name=keyname))
+                if fname_pub and fname_sec:
+                    account._preconfigure_keypair(addr, fname_pub, fname_sec)
+                    return True
+                else:
+                    print("WARN: could not use preconfigured keys for {!r}".format(addr))
+
+        def get_configured_offline_account(self):
+            ac = self.get_unconfigured_account()
+
+            # do a pseudo-configured account
+            addr = "addr{}@offline.org".format(self.offline_count)
+            ac.set_config("addr", addr)
+            self._preconfigure_key(ac, addr)
+            lib.dc_set_config(ac._dc_context, b"configured_addr", addr.encode("ascii"))
+            ac.set_config("mail_pw", "123")
+            lib.dc_set_config(ac._dc_context, b"configured_mail_pw", b"123")
+            lib.dc_set_config(ac._dc_context, b"configured", b"1")
+            return ac
+
+        def get_online_config(self, pre_generated_key=True, quiet=False):
+            if not session_liveconfig:
+                pytest.skip("specify DCC_NEW_TMP_EMAIL or --liveconfig")
+            configdict = session_liveconfig.get(self.live_count)
+            self.live_count += 1
+            if "e2ee_enabled" not in configdict:
+                configdict["e2ee_enabled"] = "1"
+
+            # Enable strict certificate checks for online accounts
+            configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+            configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+
+            tmpdb = tmpdir.join("livedb%d" % self.live_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count), quiet=quiet)
+            if pre_generated_key:
+                self._preconfigure_key(ac, configdict['addr'])
+            ac._evtracker.init_time = self.init_time
+            ac._evtracker.set_timeout(30)
+            return ac, dict(configdict)
+
+        def get_online_configuring_account(self, mvbox=False, sentbox=False, move=False,
+                                           pre_generated_key=True, quiet=False, config={}):
+            ac, configdict = self.get_online_config(
+                pre_generated_key=pre_generated_key, quiet=quiet)
+            configdict.update(config)
+            configdict["mvbox_watch"] = str(int(mvbox))
+            configdict["mvbox_move"] = str(int(move))
+            configdict["sentbox_watch"] = str(int(sentbox))
+            ac.update_config(configdict)
+            ac.start()
+            return ac
+
+        def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
+            ac1 = self.get_online_configuring_account(
+                pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
+            ac1._configtracker.wait_imap_connected()
+            ac1._configtracker.wait_smtp_connected()
+            ac1._configtracker.wait_finish()
+            return ac1
+
+        def get_two_online_accounts(self, move=False, quiet=False):
+            ac1 = self.get_online_configuring_account(move=True, quiet=quiet)
+            ac2 = self.get_online_configuring_account(quiet=quiet)
+            ac1._configtracker.wait_finish()
+            ac2._configtracker.wait_finish()
+            return ac1, ac2
+
+        def clone_online_account(self, account, pre_generated_key=True):
+            self.live_count += 1
+            tmpdb = tmpdir.join("livedb%d" % self.live_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
+            if pre_generated_key:
+                self._preconfigure_key(ac, account.get_config("addr"))
+            ac._evtracker.init_time = self.init_time
+            ac._evtracker.set_timeout(30)
+            ac.update_config(dict(
+                addr=account.get_config("addr"),
+                mail_pw=account.get_config("mail_pw"),
+                mvbox_watch=account.get_config("mvbox_watch"),
+                mvbox_move=account.get_config("mvbox_move"),
+                sentbox_watch=account.get_config("sentbox_watch"),
+            ))
+            ac.start()
+            return ac
+
+        def run_bot_process(self, module):
+            fn = module.__file__
+
+            bot_ac, bot_cfg = self.get_online_config()
+
+            args = [
+                sys.executable,
+                fn,
+                "--show-ffi",
+                "--email", bot_cfg["addr"],
+                "--password", bot_cfg["mail_pw"],
+                bot_ac.db_path,
+            ]
+            print("$", " ".join(args))
+            popen = subprocess.Popen(
+                args=args,
+                stdin=subprocess.DEVNULL,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,  # combine stdout/stderr in one stream
+                bufsize=1,                 # line buffering
+                close_fds=True,            # close all FDs other than 0/1/2
+                universal_newlines=True    # give back text
+            )
+            bot = BotProcess(popen, bot_cfg)
+            self._finalizers.append(bot.kill)
+            return bot
+
+    am = AccountMaker()
+    request.addfinalizer(am.finalize)
+    return am
+
+
+class BotProcess:
+    def __init__(self, popen, bot_cfg):
+        self.popen = popen
+        self.addr = bot_cfg["addr"]
+
+        # we read stdout as quickly as we can in a thread and make
+        # the (unicode) lines available for readers through a queue.
+        self.stdout_queue = queue.Queue()
+        self.stdout_thread = t = threading.Thread(target=self._run_stdout_thread, name="bot-stdout-thread")
+        t.setDaemon(1)
+        t.start()
+
+    def _run_stdout_thread(self):
+        try:
+            while 1:
+                line = self.popen.stdout.readline()
+                if not line:
+                    break
+                line = line.strip()
+                self.stdout_queue.put(line)
+        finally:
+            self.stdout_queue.put(None)
+
+    def kill(self):
+        self.popen.kill()
+
+    def wait(self, timeout=30):
+        self.popen.wait(timeout=timeout)
+
+    def fnmatch_lines(self, pattern_lines):
+        patterns = [x.strip() for x in Source(pattern_lines.rstrip()).lines if x.strip()]
+        for next_pattern in patterns:
+            print("+++FNMATCH:", next_pattern)
+            while 1:
+                line = self.stdout_queue.get(timeout=15)
+                if line is None:
+                    raise IOError("BOT stdout-thread terminated")
+                if fnmatch.fnmatch(line, next_pattern):
+                    print("+++MATCHED:", line)
+                    break
+                else:
+                    print("+++IGN:", line)
+
+
+@pytest.fixture
+def tmp_db_path(tmpdir):
+    return tmpdir.join("test.db").strpath
+
+
+@pytest.fixture
+def lp():
+    class Printer:
+        def sec(self, msg):
+            print()
+            print("=" * 10, msg, "=" * 10)
+
+        def step(self, msg):
+            print("-" * 5, "step " + msg, "-" * 5)
+    return Printer()

python/src/deltachat/tracker.py

@@ -0,0 +1,76 @@
+
+from queue import Queue
+from threading import Event
+
+from .hookspec import account_hookimpl
+
+
+class ImexFailed(RuntimeError):
+    """ Exception for signalling that import/export operations failed."""
+
+
+class ImexTracker:
+    def __init__(self):
+        self._imex_events = Queue()
+
+    @account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        if ffi_event.name == "DC_EVENT_IMEX_PROGRESS":
+            self._imex_events.put(ffi_event.data1)
+        elif ffi_event.name == "DC_EVENT_IMEX_FILE_WRITTEN":
+            self._imex_events.put(ffi_event.data1)
+
+    def wait_finish(self, progress_timeout=60):
+        """ Return list of written files, raise ValueError if ExportFailed. """
+        files_written = []
+        while True:
+            ev = self._imex_events.get(timeout=progress_timeout)
+            if isinstance(ev, str):
+                files_written.append(ev)
+            elif ev == 0:
+                raise ImexFailed("export failed, exp-files: {}".format(files_written))
+            elif ev == 1000:
+                return files_written
+
+
+class ConfigureFailed(RuntimeError):
+    """ Exception for signalling that configuration failed."""
+
+
+class ConfigureTracker:
+    ConfigureFailed = ConfigureFailed
+
+    def __init__(self):
+        self._configure_events = Queue()
+        self._smtp_finished = Event()
+        self._imap_finished = Event()
+        self._ffi_events = []
+
+    @account_hookimpl
+    def ac_process_ffi_event(self, ffi_event):
+        self._ffi_events.append(ffi_event)
+        if ffi_event.name == "DC_EVENT_SMTP_CONNECTED":
+            self._smtp_finished.set()
+        elif ffi_event.name == "DC_EVENT_IMAP_CONNECTED":
+            self._imap_finished.set()
+
+    @account_hookimpl
+    def ac_configure_completed(self, success):
+        self._configure_events.put(success)
+
+    def wait_smtp_connected(self):
+        """ wait until smtp is configured. """
+        self._smtp_finished.wait()
+
+    def wait_imap_connected(self):
+        """ wait until smtp is configured. """
+        self._imap_finished.wait()
+
+    def wait_finish(self):
+        """ wait until configure is completed.
+
+        Raise Exception if Configure failed
+        """
+        if not self._configure_events.get():
+            content = "\n".join(map(str, self._ffi_events))
+            raise ConfigureFailed(content)

python/tests/conftest.py

@@ -1,244 +1,18 @@
 from __future__ import print_function
-import os
-import pytest
-import requests
-import time
-from deltachat import Account
-from deltachat.capi import lib
-import tempfile
 
 
-def pytest_addoption(parser):
-    parser.addoption(
-        "--liveconfig", action="store", default=None,
-        help="a file with >=2 lines where each line "
-             "contains NAME=VALUE config settings for one account"
-    )
-
-
-def pytest_configure(config):
-    cfg = config.getoption('--liveconfig')
-    if not cfg:
-        cfg = os.getenv('DCC_PY_LIVECONFIG')
-        if cfg:
-            config.option.liveconfig = cfg
-
-
-@pytest.hookimpl(trylast=True)
-def pytest_runtest_call(item):
-    # perform early finalization because we otherwise get cloberred
-    # output from concurrent threads printing between execution
-    # of the test function and the teardown phase of that test function
-    if "acfactory" in item.funcargs:
-        print("*"*30, "finalizing", "*"*30)
-        acfactory = item.funcargs["acfactory"]
-        acfactory.finalize()
-
-
-def pytest_report_header(config, startdir):
-    summary = []
-
-    t = tempfile.mktemp()
-    try:
-        ac = Account(t, eventlogging=False)
-        info = ac.get_info()
-        ac.shutdown()
-    finally:
-        os.remove(t)
-    summary.extend(['Deltachat core={} sqlite={}'.format(
-         info['deltachat_core_version'],
-         info['sqlite_version'],
-     )])
-
-    cfg = config.option.liveconfig
-    if cfg:
-        if "#" in cfg:
-            url, token = cfg.split("#", 1)
-            summary.append('Liveconfig provider: {}#<token ommitted>'.format(url))
-        else:
-            summary.append('Liveconfig file: {}'.format(cfg))
-    return summary
-
-
-@pytest.fixture(scope="session")
-def data():
-    class Data:
-        def __init__(self):
-            self.path = os.path.join(os.path.dirname(__file__), "data")
-
-        def get_path(self, bn):
-            fn = os.path.join(self.path, bn)
-            assert os.path.exists(fn)
-            return fn
-    return Data()
-
-
-class SessionLiveConfigFromFile:
-    def __init__(self, fn):
-        self.fn = fn
-        self.configlist = []
-        for line in open(fn):
-            if line.strip() and not line.strip().startswith('#'):
-                d = {}
-                for part in line.split():
-                    name, value = part.split("=")
-                    d[name] = value
-                self.configlist.append(d)
-
-    def get(self, index):
-        return self.configlist[index]
-
-    def exists(self):
-        return bool(self.configlist)
-
-
-class SessionLiveConfigFromURL:
-    def __init__(self, url, create_token):
-        self.configlist = []
-        for i in range(2):
-            res = requests.post(url, json={"token_create_user": int(create_token)})
-            if res.status_code != 200:
-                pytest.skip("creating newtmpuser failed {!r}".format(res))
-            d = res.json()
-            config = dict(addr=d["email"], mail_pw=d["password"])
-            self.configlist.append(config)
-
-    def get(self, index):
-        return self.configlist[index]
-
-    def exists(self):
-        return bool(self.configlist)
-
-
-@pytest.fixture(scope="session")
-def session_liveconfig(request):
-    liveconfig_opt = request.config.option.liveconfig
-    if liveconfig_opt:
-        if liveconfig_opt.startswith("http"):
-            url, create_token = liveconfig_opt.split("#", 1)
-            return SessionLiveConfigFromURL(url, create_token)
-        else:
-            return SessionLiveConfigFromFile(liveconfig_opt)
-
-
-@pytest.fixture
-def acfactory(pytestconfig, tmpdir, request, session_liveconfig):
-
-    class AccountMaker:
-        def __init__(self):
-            self.live_count = 0
-            self.offline_count = 0
-            self._finalizers = []
-            self.init_time = time.time()
-
-        def finalize(self):
-            while self._finalizers:
-                fin = self._finalizers.pop()
-                fin()
-
-        def get_unconfigured_account(self):
-            self.offline_count += 1
-            tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
-            ac = Account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
-            ac._evlogger.init_time = self.init_time
-            ac._evlogger.set_timeout(2)
-            self._finalizers.append(ac.shutdown)
-            return ac
-
-        def get_configured_offline_account(self):
-            ac = self.get_unconfigured_account()
-
-            # do a pseudo-configured account
-            addr = "addr{}@offline.org".format(self.offline_count)
-            ac.set_config("addr", addr)
-            lib.dc_set_config(ac._dc_context, b"configured_addr", addr.encode("ascii"))
-            ac.set_config("mail_pw", "123")
-            lib.dc_set_config(ac._dc_context, b"configured_mail_pw", b"123")
-            lib.dc_set_config(ac._dc_context, b"configured", b"1")
-            return ac
-
-        def get_online_configuring_account(self):
-            if not session_liveconfig:
-                pytest.skip("specify DCC_PY_LIVECONFIG or --liveconfig")
-            configdict = session_liveconfig.get(self.live_count)
-            self.live_count += 1
-            if "e2ee_enabled" not in configdict:
-                configdict["e2ee_enabled"] = "1"
-            tmpdb = tmpdir.join("livedb%d" % self.live_count)
-            ac = Account(tmpdb.strpath, logid="ac{}".format(self.live_count))
-            ac._evlogger.init_time = self.init_time
-            ac._evlogger.set_timeout(30)
-            ac.configure(**configdict)
-            ac.start_threads()
-            self._finalizers.append(ac.shutdown)
-            return ac
-
-        def clone_online_account(self, account):
-            self.live_count += 1
-            tmpdb = tmpdir.join("livedb%d" % self.live_count)
-            ac = Account(tmpdb.strpath, logid="ac{}".format(self.live_count))
-            ac._evlogger.init_time = self.init_time
-            ac._evlogger.set_timeout(30)
-            ac.configure(addr=account.get_config("addr"), mail_pw=account.get_config("mail_pw"))
-            ac.start_threads()
-            self._finalizers.append(ac.shutdown)
-            return ac
-
-    return AccountMaker()
-
-
-@pytest.fixture
-def tmp_db_path(tmpdir):
-    return tmpdir.join("test.db").strpath
-
-
-@pytest.fixture
-def lp():
-    class Printer:
-        def sec(self, msg):
-            print()
-            print("=" * 10, msg, "=" * 10)
-
-        def step(self, msg):
-            print("-" * 5, "step " + msg, "-" * 5)
-    return Printer()
-
-
-def wait_configuration_progress(account, target):
+def wait_configuration_progress(account, min_target, max_target=1001):
+    min_target = min(min_target, max_target)
     while 1:
-        evt_name, data1, data2 = \
-            account._evlogger.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
-        if data1 >= target:
-            print("** CONFIG PROGRESS {}".format(target), account)
+        event = account._evtracker.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
+        if event.data1 >= min_target and event.data1 <= max_target:
+            print("** CONFIG PROGRESS {}".format(min_target), account)
             break
 
 
 def wait_securejoin_inviter_progress(account, target):
     while 1:
-        evt_name, data1, data2 = \
-            account._evlogger.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS")
-        if data2 >= target:
+        event = account._evtracker.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS")
+        if event.data2 >= target:
             print("** SECUREJOINT-INVITER PROGRESS {}".format(target), account)
             break
-
-
-def wait_successful_IMAP_SMTP_connection(account):
-    imap_ok = smtp_ok = False
-    while not imap_ok or not smtp_ok:
-        evt_name, data1, data2 = \
-            account._evlogger.get_matching("DC_EVENT_(IMAP|SMTP)_CONNECTED")
-        if evt_name == "DC_EVENT_IMAP_CONNECTED":
-            imap_ok = True
-            print("** IMAP OK", account)
-        if evt_name == "DC_EVENT_SMTP_CONNECTED":
-            smtp_ok = True
-            print("** SMTP OK", account)
-    print("** IMAP and SMTP logins successful", account)
-
-
-def wait_msgs_changed(account, chat_id, msg_id=None):
-    ev = account._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
-    assert ev[1] == chat_id
-    if msg_id is not None:
-        assert ev[2] == msg_id
-    return ev[2]

python/tests/data/key

@@ -0,0 +1 @@
+../../../test-data/key

python/tests/test_increation.py

@@ -1,10 +1,57 @@
 from __future__ import print_function
+
+import os.path
+import shutil
+
+import pytest
 from filecmp import cmp
+
+from conftest import wait_configuration_progress
 from deltachat import const
-from conftest import wait_configuration_progress, wait_msgs_changed
 
 
-class TestInCreation:
+def wait_msgs_changed(account, chat_id, msg_id=None):
+    ev = account._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+    assert ev.data1 == chat_id
+    if msg_id is not None:
+        assert ev.data2 == msg_id
+    return ev.data2
+
+
+class TestOnlineInCreation:
+    def test_increation_not_blobdir(self, tmpdir, acfactory, lp):
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.get_online_configuring_account()
+        wait_configuration_progress(ac1, 1000)
+        wait_configuration_progress(ac2, 1000)
+
+        c2 = ac1.create_contact(email=ac2.get_config("addr"))
+        chat = ac1.create_chat_by_contact(c2)
+
+        lp.sec("Creating in-creation file outside of blobdir")
+        assert tmpdir.strpath != ac1.get_blobdir()
+        src = tmpdir.join('file.txt').ensure(file=1)
+        with pytest.raises(Exception):
+            chat.prepare_message_file(src.strpath)
+
+    def test_no_increation_copies_to_blobdir(self, tmpdir, acfactory, lp):
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.get_online_configuring_account()
+        wait_configuration_progress(ac1, 1000)
+        wait_configuration_progress(ac2, 1000)
+
+        c2 = ac1.create_contact(email=ac2.get_config("addr"))
+        chat = ac1.create_chat_by_contact(c2)
+
+        lp.sec("Creating file outside of blobdir")
+        assert tmpdir.strpath != ac1.get_blobdir()
+        src = tmpdir.join('file.txt')
+        src.write("hello there\n")
+        chat.send_file(src.strpath)
+
+        blob_src = os.path.join(ac1.get_blobdir(), 'file.txt')
+        assert os.path.exists(blob_src), "file.txt not copied to blobdir"
+
     def test_forward_increation(self, acfactory, data, lp):
         ac1 = acfactory.get_online_configuring_account()
         ac2 = acfactory.get_online_configuring_account()
@@ -17,7 +64,10 @@ class TestInCreation:
         wait_msgs_changed(ac1, 0, 0)  # why no chat id?
 
         lp.sec("create a message with a file in creation")
-        path = data.get_path("d.png")
+        orig = data.get_path("d.png")
+        path = os.path.join(ac1.get_blobdir(), 'd.png')
+        with open(path, "x") as fp:
+            fp.write("preparing")
         prepared_original = chat.prepare_message_file(path)
         assert prepared_original.is_out_preparing()
         wait_msgs_changed(ac1, chat.id, prepared_original.id)
@@ -38,6 +88,7 @@ class TestInCreation:
 
         lp.sec("finish creating the file and send it")
         assert prepared_original.is_out_preparing()
+        shutil.copyfile(orig, path)
         chat.send_prepared(prepared_original)
         assert prepared_original.is_out_pending() or prepared_original.is_out_delivered()
         wait_msgs_changed(ac1, chat.id, prepared_original.id)
@@ -48,22 +99,22 @@ class TestInCreation:
         assert fwd_msg.is_out_pending() or fwd_msg.is_out_delivered()
 
         lp.sec("wait for the messages to be delivered to SMTP")
-        ev = ac1._evlogger.get_matching("DC_EVENT_MSG_DELIVERED")
-        assert ev[1] == chat.id
-        assert ev[2] == prepared_original.id
-        ev = ac1._evlogger.get_matching("DC_EVENT_MSG_DELIVERED")
-        assert ev[1] == chat2.id
-        assert ev[2] == forwarded_id
+        ev = ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
+        assert ev.data1 == chat.id
+        assert ev.data2 == prepared_original.id
+        ev = ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
+        assert ev.data1 == chat2.id
+        assert ev.data2 == forwarded_id
 
         lp.sec("wait1 for original or forwarded messages to arrive")
-        ev1 = ac2._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
-        assert ev1[1] >= const.DC_CHAT_ID_LAST_SPECIAL
-        received_original = ac2.get_message_by_id(ev1[2])
-        assert cmp(received_original.filename, path, False)
+        ev1 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+        assert ev1.data1 > const.DC_CHAT_ID_LAST_SPECIAL
+        received_original = ac2.get_message_by_id(ev1.data2)
+        assert cmp(received_original.filename, orig, shallow=False)
 
         lp.sec("wait2 for original or forwarded messages to arrive")
-        ev2 = ac2._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
-        assert ev2[1] >= const.DC_CHAT_ID_LAST_SPECIAL
-        assert ev2[1] != ev1[1]
-        received_copy = ac2.get_message_by_id(ev2[2])
-        assert cmp(received_copy.filename, path, False)
+        ev2 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+        assert ev2.data1 > const.DC_CHAT_ID_LAST_SPECIAL
+        assert ev2.data1 != ev1.data1
+        received_copy = ac2.get_message_by_id(ev2.data2)
+        assert cmp(received_copy.filename, orig, shallow=False)

python/tests/test_lowlevel.py

@@ -1,8 +1,9 @@
 from __future__ import print_function
-from deltachat import capi, const, set_context_callback, clear_context_callback
+from deltachat import capi, cutil, const, set_context_callback, clear_context_callback
+from deltachat import register_global_plugin
+from deltachat.hookspec import global_hookimpl
 from deltachat.capi import ffi
 from deltachat.capi import lib
-from deltachat.account import EventLogger
 
 
 def test_empty_context():
@@ -17,23 +18,33 @@ def test_callback_None2int():
     clear_context_callback(ctx)
 
 
-def test_dc_close_events():
-    ctx = capi.lib.dc_context_new(capi.lib.py_dc_callback, ffi.NULL, ffi.NULL)
-    evlog = EventLogger(ctx)
-    evlog.set_timeout(5)
-    set_context_callback(ctx, lambda ctx, evt_name, data1, data2: evlog(evt_name, data1, data2))
-    capi.lib.dc_close(ctx)
+def test_dc_close_events(tmpdir, acfactory):
+    ac1 = acfactory.get_unconfigured_account()
+
+    # register after_shutdown function
+    shutdowns = []
+
+    class ShutdownPlugin:
+        @global_hookimpl
+        def dc_account_after_shutdown(self, account):
+            assert account._dc_context is None
+            shutdowns.append(account)
+    register_global_plugin(ShutdownPlugin())
+    assert hasattr(ac1, "_dc_context")
+    ac1.shutdown()
+    assert shutdowns == [ac1]
 
     def find(info_string):
+        evlog = ac1._evtracker
         while 1:
             ev = evlog.get_matching("DC_EVENT_INFO", check_error=False)
-            data2 = ev[2]
+            data2 = ev.data2
             if info_string in data2:
                 return
             else:
-                print("skipping event", *ev)
+                print("skipping event", ev)
 
-    find("disconnecting INBOX-watch")
+    find("disconnecting inbox-thread")
     find("disconnecting sentbox-thread")
     find("disconnecting mvbox-thread")
     find("disconnecting SMTP")
@@ -51,6 +62,16 @@ def test_wrong_db(tmpdir):
     assert not lib.dc_open(dc_context, p.strpath.encode("ascii"), ffi.NULL)
 
 
+def test_empty_blobdir(tmpdir):
+    # Apparently some client code expects this to be the same as passing NULL.
+    ctx = ffi.gc(
+        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_unref,
+    )
+    db_fname = tmpdir.join("hello.db")
+    assert lib.dc_open(ctx, db_fname.strpath.encode("ascii"), b"")
+
+
 def test_event_defines():
     assert const.DC_EVENT_INFO == 100
     assert const.DC_CONTACT_ID_SELF
@@ -71,7 +92,62 @@ def test_markseen_invalid_message_ids(acfactory):
     contact1 = ac1.create_contact(email="some1@example.com", name="some1")
     chat = ac1.create_chat_by_contact(contact1)
     chat.send_text("one messae")
-    ac1._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
+    ac1._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
     msg_ids = [9]
     lib.dc_markseen_msgs(ac1._dc_context, msg_ids, len(msg_ids))
-    ac1._evlogger.ensure_event_not_queued("DC_EVENT_WARNING|DC_EVENT_ERROR")
+    ac1._evtracker.ensure_event_not_queued("DC_EVENT_WARNING|DC_EVENT_ERROR")
+
+
+def test_get_special_message_id_returns_empty_message(acfactory):
+    ac1 = acfactory.get_configured_offline_account()
+    for i in range(1, 10):
+        msg = ac1.get_message_by_id(i)
+        assert msg.id == 0
+
+
+def test_provider_info_none():
+    ctx = ffi.gc(
+        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_unref,
+    )
+    assert lib.dc_provider_new_from_email(ctx, cutil.as_dc_charpointer("email@unexistent.no")) == ffi.NULL
+
+
+def test_get_info_closed():
+    ctx = ffi.gc(
+        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_unref,
+    )
+    info = cutil.from_dc_charpointer(lib.dc_get_info(ctx))
+    assert 'deltachat_core_version' in info
+    assert 'database_dir' not in info
+
+
+def test_get_info_open(tmpdir):
+    ctx = ffi.gc(
+        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_unref,
+    )
+    db_fname = tmpdir.join("test.db")
+    lib.dc_open(ctx, db_fname.strpath.encode("ascii"), ffi.NULL)
+    info = cutil.from_dc_charpointer(lib.dc_get_info(ctx))
+    assert 'deltachat_core_version' in info
+    assert 'database_dir' in info
+
+
+def test_is_open_closed():
+    ctx = ffi.gc(
+        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_unref,
+    )
+    assert lib.dc_is_open(ctx) == 0
+
+
+def test_is_open_actually_open(tmpdir):
+    ctx = ffi.gc(
+        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_unref,
+    )
+    db_fname = tmpdir.join("test.db")
+    lib.dc_open(ctx, db_fname.strpath.encode("ascii"), ffi.NULL)
+    assert lib.dc_is_open(ctx) == 1

python/tox.ini

@@ -1,34 +1,39 @@
 [tox]
 # make sure to update environment list in travis.yml and appveyor.yml
 envlist =
-    py35
+    py37
     lint
     auditwheels
 
 [testenv]
 commands = 
-    pytest -v -rsXx {posargs:tests}
+    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored {posargs: tests examples}
     python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 
     DCC_RS_DEV
     DCC_RS_TARGET
     DCC_PY_LIVECONFIG
+    DCC_NEW_TMP_EMAIL 
+    CARGO_TARGET_DIR
+    RUSTC_WRAPPER
 deps = 
     pytest
-    pytest-faulthandler
+    pytest-rerunfailures
+    pytest-timeout
+    pytest-xdist
     pdbpp
     requests
 
 [testenv:auditwheels]
 skipsdist = True
+deps = auditwheel
 commands = 
     python tests/auditwheels.py {toxworkdir}/wheelhouse
 
-
 [testenv:lint]
 skipsdist = True
-usedevelop = True
+skip_install = True
 deps =
     flake8
     # pygments required by rst-lint
@@ -36,26 +41,36 @@ deps =
     restructuredtext_lint 
 commands =
     flake8 src/deltachat
-    flake8 tests/
+    flake8 tests/ examples/
     rst-lint --encoding 'utf-8' README.rst
 
 [testenv:doc]
-basepython = python3.5
+changedir=doc
 deps =
-    sphinx==2.0.1
+    sphinx
     breathe
-
-changedir = doc
 commands =
-    sphinx-build -w docker-toxdoc-warnings.log -b html . _build/html
+    sphinx-build -Q -w toxdoc-warnings.log -b html . _build/html
+
+
+[testenv:lintdoc]
+skipsdist = True
+usedevelop = True
+deps =
+    {[testenv:lint]deps}
+    {[testenv:doc]deps}
+commands =
+    {[testenv:lint]commands}
+    {[testenv:doc]commands}
+
 
 
 [pytest]
-addopts = -v -rs
-python_files = tests/test_*.py 
+addopts = -v -ra --strict-markers
 norecursedirs = .tox 
 xfail_strict=true
-timeout = 60
+timeout = 90
+timeout_method = thread
 
 [flake8]
 max-line-length = 120

run-integration-tests.sh

@@ -23,7 +23,7 @@ if [ $? != 0 ]; then
 fi
 
 pushd python
-if [ -e "./liveconfig" && -z "$DCC_PY_LIVECONFIG" ]; then
+if [ -e "./liveconfig" -a -z "$DCC_PY_LIVECONFIG" ]; then
     export DCC_PY_LIVECONFIG=liveconfig
 fi
 tox "$@"

rust-toolchain

@@ -1 +1 @@
-nightly-2019-08-13
+nightly-2020-03-12

scripts/proxy.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+# Examples:
+#
+# Original server that doesn't use SSL:
+# ./proxy.py 8080 imap.nauta.cu 143
+# ./proxy.py 8081 smtp.nauta.cu 25
+#
+# Original server that uses SSL:
+# ./proxy.py 8080 testrun.org 993 --ssl
+# ./proxy.py 8081 testrun.org 465 --ssl
+
+from datetime import datetime
+import argparse
+import selectors
+import ssl
+import socket
+import socketserver
+
+
+class Proxy(socketserver.ThreadingTCPServer):
+    allow_reuse_address = True
+
+    def __init__(self, proxy_host, proxy_port, real_host, real_port, use_ssl):
+        self.real_host = real_host
+        self.real_port = real_port
+        self.use_ssl = use_ssl
+        super().__init__((proxy_host, proxy_port), RequestHandler)
+
+
+class RequestHandler(socketserver.BaseRequestHandler):
+
+    def handle(self):
+        print('{} - {} CONNECTED.'.format(datetime.now(), self.client_address))
+
+        total = 0
+        real_server = (self.server.real_host, self.server.real_port)
+        with socket.create_connection(real_server) as sock:
+            if self.server.use_ssl:
+                context = ssl.create_default_context()
+                sock = context.wrap_socket(
+                    sock, server_hostname=real_server[0])
+
+            forward = {self.request: sock, sock: self.request}
+
+            sel = selectors.DefaultSelector()
+            sel.register(self.request, selectors.EVENT_READ,
+                         self.client_address)
+            sel.register(sock, selectors.EVENT_READ, real_server)
+
+            active = True
+            while active:
+                events = sel.select()
+                for key, mask in events:
+                    print('\n{} - {} wrote:'.format(datetime.now(), key.data))
+                    data = key.fileobj.recv(1024)
+                    received = len(data)
+                    total += received
+                    print(data)
+                    print('{} Bytes\nTotal: {} Bytes'.format(received, total))
+                    if data:
+                        forward[key.fileobj].sendall(data)
+                    else:
+                        print('\nCLOSING CONNECTION.\n\n')
+                        forward[key.fileobj].close()
+                        key.fileobj.close()
+                        active = False
+
+
+if __name__ == '__main__':
+    p = argparse.ArgumentParser(description='Simple Python Proxy')
+    p.add_argument(
+        "proxy_port", help="the port where the proxy will listen", type=int)
+    p.add_argument('host', help="the real host")
+    p.add_argument('port', help="the port of the real host", type=int)
+    p.add_argument("--ssl", help="use ssl to connect to the real host",
+                   action="store_true")
+    args = p.parse_args()
+
+    with Proxy('', args.proxy_port, args.host, args.port, args.ssl) as proxy:
+        proxy.serve_forever()

set_core_version.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import re
+import pathlib
+import subprocess
+
+rex = re.compile(r'version = "(\S+)"')
+
+def read_toml_version(relpath):
+    p = pathlib.Path(relpath)
+    assert p.exists()
+    for line in open(str(p)):
+        m = rex.match(line)
+        if m is not None:
+            return m.group(1)
+    raise ValueError("no version found in {}".format(relpath))
+
+def replace_toml_version(relpath, newversion):
+    p = pathlib.Path(relpath)
+    assert p.exists()
+    tmp_path = str(p) + "_tmp"
+    with open(tmp_path, "w") as f:
+        for line in open(str(p)):
+            m = rex.match(line)
+            if m is not None:
+                f.write('version = "{}"\n'.format(newversion))
+            else:
+                f.write(line)
+    os.rename(tmp_path, str(p))
+
+if __name__ == "__main__":
+
+    if len(sys.argv) < 2:
+        for x in ("Cargo.toml", "deltachat-ffi/Cargo.toml"):
+            print("{}: {}".format(x, read_toml_version(x)))
+        raise SystemExit("need argument: new version, example: 1.25.0")
+    newversion = sys.argv[1]
+    if newversion.count(".") < 2:
+        raise SystemExit("need at least two dots in version")
+
+    core_toml = read_toml_version("Cargo.toml")
+    ffi_toml = read_toml_version("deltachat-ffi/Cargo.toml")
+    assert core_toml == ffi_toml, (core_toml, ffi_toml)
+
+    for line in open("CHANGELOG.md"):
+        ## 1.25.0
+        if line.startswith("## "):
+            if line[2:].strip().startswith(newversion):
+                break
+    else:
+        raise SystemExit("CHANGELOG.md contains no entry for version: {}".format(newversion))
+
+    replace_toml_version("Cargo.toml", newversion)
+    replace_toml_version("deltachat-ffi/Cargo.toml", newversion)
+
+    subprocess.call(["cargo", "check"])
+    subprocess.call(["git", "add", "-u"])
+    # subprocess.call(["cargo", "update", "-p", "deltachat"])
+
+    print("after commit make sure to: ")
+    print("")
+    print("   git tag {}".format(newversion))
+    print("")

spec.md

@@ -0,0 +1,441 @@
+# Chat-over-Email specification
+
+Version 0.30.0
+
+This document describes how emails can be used
+to implement typical messenger functions
+while staying compatible to existing MUAs.
+
+- [Encryption](#encryption)
+- [Outgoing messages](#outgoing-messages)
+- [Incoming messages](#incoming-messages)
+- [Forwarded messages](#forwarded-messages)
+- [Groups](#groups)
+    - [Outgoing group messages](#outgoing-group-messages)
+    - [Incoming group messages](#incoming-group-messages)
+    - [Add and remove members](#add-and-remove-members)
+    - [Change group name](#change-group-name)
+    - [Set group image](#set-group-image)
+- [Set profile image](#set-profile-image)
+- [Locations](#locations)
+    - [User locations](#user-locations)
+    - [Points of interest](#points-of-interest)
+- [Miscellaneous](#miscellaneous)
+
+
+# Encryption
+
+Messages SHOULD be encrypted by the
+[Autocrypt](https://autocrypt.org/level1.html) standard;
+`prefer-encrypt=mutual` MAY be set by default.
+
+Meta data (at least the subject and all chat-headers) SHOULD be encrypted
+by the [Memoryhole](https://github.com/autocrypt/memoryhole) standard.
+If Memoryhole is not used,
+the subject of encrypted messages SHOULD be replaced by the string `...`.
+
+
+# Outgoing messages
+
+Messengers MUST add a `Chat-Version: 1.0` header to outgoing messages.
+For filtering and smart appearance of the messages in normal MUAs,
+the `Subject` header SHOULD start with the characters `Chat:`
+and SHOULD be an excerpt of the message.
+Replies to messages MAY follow the typical `Re:`-format.
+
+The body MAY contain text which MUST have the content type `text/plain`
+or `mulipart/alternative` containing `text/plain`.
+
+The text MAY be divided into a user-text-part and a footer-part using the
+line `-- ` (minus, minus, space, lineend).
+
+The user-text-part MUST contain only user generated content.
+User generated content are eg. texts a user has actually typed
+or pasted or forwarded from another user.
+Full quotes, footers or sth. like that MUST NOT go to the user-text-part.
+
+    From: sender@domain
+    To: rcpt@domain
+    Chat-Version: 1.0
+    Content-Type: text/plain
+    Subject: Chat: Hello ...
+
+    Hello world!
+
+
+# Incoming messages
+
+The `Chat-Version` header MAY be used
+to detect if a messages comes from a compatible messenger.
+
+The `Subject` header MUST NOT be used
+to detect compatible messengers, groups or whatever.
+
+Messenger SHOULD show the `Subject`
+if the message comes from a normal MUA together with the email-body.
+The email-body SHOULD be converted
+to plain text, full-quotes and similar regions SHOULD be cut.
+
+Attachments SHOULD be shown where possible.
+If an attachment cannot be shown, a non-distracting warning SHOULD be printed.
+
+
+# Forwarded messages
+
+Forwarded messages are outgoing messages that contain a forwarded-header
+before the user generated content.
+
+The forwarded header MUST contain two lines: 
+The first line contains the text
+`---------- Forwarded message ----------`
+(10 minus, space, text `Forwarded message`, space, 10 minus). 
+The second line starts with `From: ` followed by the original sender
+which SHOULD be anonymized or just a placeholder.
+
+    From: sender@domain
+    To: rcpt@domain
+    Chat-Version: 1.0
+    Content-Type: text/plain
+    Subject: Chat: Forwarded message
+
+    ---------- Forwarded message ----------
+    From: Messenger
+
+    Hello world!
+
+Incoming forwarded messages are detected by the header.
+The messenger SHOULD mark these messages in a way that
+it becomes obvious that the message is not created by the sender.
+Note that most messengers do not show the original sender of forwarded messages
+but MUAs typically expose the sender in the UI.
+
+
+# Groups
+
+Groups are chats with usually more than one recipient,
+each defined by an email-address.
+The sender plus the recipients are the group members.
+All group members form the member list.
+
+To allow different groups with the same members,
+groups are identified by a group-id.
+The group-id MUST be created only from the characters
+`0`-`9`, `A`-`Z`, `a`-`z` `_` and `-`
+and MUST have a length of at least 11 characters.
+
+Groups MUST have a group-name.
+The group-name is any non-zero-length UTF-8 string.
+
+Groups MAY have a group-image.
+
+
+## Outgoing groups messages
+
+All group members MUST be added to the `From`/`To` headers.
+The group-id MUST be written to the `Chat-Group-ID` header.
+The group-name MUST be written to `Chat-Group-Name` header
+(the forced presence of this header makes it easier
+to join a group chat on a second device any time).
+
+The `Subject` header of outgoing group messages
+SHOULD be set to the group-name.
+
+To identify the group-id on replies from normal MUAs,
+the group-id MUST also be added to the message-id of outgoing messages.
+The message-id MUST have the format `Gr.<group-id>.<unique data>`.
+
+    From: member1@domain
+    To: member2@domain, member3@domain
+    Chat-Version: 1.0
+    Chat-Group-ID: 12345uvwxyZ
+    Chat-Group-Name: My Group
+    Message-ID: Gr.12345uvwxyZ.0001@domain
+    Subject: Chat: My Group: Hello group ...
+
+    Hello group - this group contains three members
+
+Messengers adding the member list in the form `Name <email-address>`
+MUST take care only to distribute the names authorized by the contacts themselves.
+Otherwise, names as _Daddy_ or _Honey_ may be distributed
+(this issue is also true for normal MUAs, however,
+for more contact- and chat-centralized apps
+such situations happen more frequently).
+
+
+## Incoming group messages
+
+The messenger MUST search incoming messages for the group-id
+in the following headers: `Chat-Group-ID`,
+`Message-ID`, `In-Reply-To` and `References` (in this order).
+
+If the messenger finds a valid and existent group-id,
+the message SHOULD be assigned to the given group.
+If the messenger finds a valid but not existent group-id,
+the messenger MAY create a new group.
+If no group-id is found,
+the message MAY be assigned
+to a normal single-user chat with the email-address given in `From`.
+
+
+## Add and remove members
+
+Messenger clients MUST init the member list
+from the `From`/`To` headers on the first group message.
+
+When a member is added later,
+a `Chat-Group-Member-Added` action header must be sent
+with the value set to the email-address of the added member.
+When receiving a `Chat-Group-Member-Added` header, however,
+_all missing_ members  the `From`/`To` headers has to be added.
+This is to mitigate problems when receiving messages
+in different orders, esp. on creating new groups.
+
+To remove a member, a `Chat-Group-Member-Removed` header must be sent
+with the value set to the email-address of the member to remove.
+When receiving a `Chat-Group-Member-Removed` header,
+only exaxtly the given member has to be removed from the member list.
+
+Messenger clients MUST NOT construct the member list
+on other group messages
+(this is to avoid accidentally altered To-lists in normal MUAs;
+the user does not expect adding a user to a _message_
+will also add him to the _group_ "forever").
+
+The messenger SHOULD send an explicit mail for each added or removed member.
+The body of the message SHOULD contain
+a localized description about what happened
+and the message SHOULD appear as a message or action from the sender.
+
+    From: member1@domain
+    To: member2@domain, member3@domain, member4@domain
+    Chat-Version: 1.0
+    Chat-Group-ID: 12345uvwxyZ
+    Chat-Group-Name: My Group
+    Chat-Group-Member-Added: member4@domain
+    Message-ID: Gr.12345uvwxyZ.0002@domain
+    Subject: Chat: My Group: Hello, ...
+
+    Hello, I've added member4@domain to our group.  Now we have 4 members.
+
+To remove a member:
+
+    From: member1@domain
+    To: member2@domain, member3@domain
+    Chat-Version: 1.0
+    Chat-Group-ID: 12345uvwxyZ
+    Chat-Group-Name: My Group
+    Chat-Group-Member-Removed: member4@domain
+    Message-ID: Gr.12345uvwxyZ.0003@domain
+    Subject: Chat: My Group: Hello, ...
+
+    Hello, I've removed member4@domain from our group.  Now we have 3 members.
+
+
+## Change group name
+
+To change the group-name,
+the messenger MUST send the action header `Chat-Group-Name-Changed`
+with the value set to the old group name to all group members.
+The new group name goes to the header `Chat-Group-Name`.
+
+The messenger SHOULD send an explicit mail for each name change.
+The body of the message SHOULD contain
+a localized description about what happened
+and the message SHOULD appear as a message or action from the sender.
+
+    From: member1@domain
+    To: member2@domain, member3@domain
+    Chat-Version: 1.0
+    Chat-Group-ID: 12345uvwxyZ
+    Chat-Group-Name: Our Group
+    Chat-Group-Name-Changed: My Group
+    Message-ID: Gr.12345uvwxyZ.0004@domain
+    Subject: Chat: Our Group: Hello, ...
+
+    Hello, I've changed the group name from "My Group" to "Our Group".
+
+
+## Set group image
+
+A group MAY have a group-image.
+To change or set the group-image,
+the messenger MUST attach an image file to a message
+and MUST add the header `Chat-Group-Avatar`
+with the value set to the image name.
+
+To remove the group-image,
+the messenger MUST add the header `Chat-Group-Avatar: 0`.
+
+The messenger SHOULD send an explicit mail for each group image change.
+The body of the message SHOULD contain
+a localized description about what happened
+and the message SHOULD appear as a message or action from the sender.
+
+
+    From: member1@domain
+    To: member2@domain, member3@domain
+    Chat-Version: 1.0
+    Chat-Group-ID: 12345uvwxyZ
+    Chat-Group-Name: Our Group
+    Chat-Group-Avatar: image.jpg
+    Message-ID: Gr.12345uvwxyZ.0005@domain
+    Subject: Chat: Our Group: Hello, ...
+    Content-Type: multipart/mixed; boundary="==break=="
+
+    --==break==
+    Content-Type: text/plain
+
+    Hello, I've changed the group image.
+    --==break==
+    Content-Type: image/jpeg
+    Content-Disposition: attachment; filename="image.jpg"
+
+    /9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAYEBQYFBAYGBQYHBw ...
+    --==break==--
+
+The image format SHOULD be image/jpeg or image/png.
+To save data, it is RECOMMENDED
+to add a `Chat-Group-Avatar` only on image changes.
+
+
+# Set profile image
+
+A user MAY have a profile-image that MAY be distributed to their contacts.
+To change or set the profile-image,
+the messenger MUST attach an image file to a message
+and MUST add the header `Chat-User-Avatar`
+with the value set to the image name.
+
+To remove the profile-image,
+the messenger MUST add the header `Chat-User-Avatar: 0`.
+
+To distribute the image,
+the messenger MAY send the profile image
+together with the next mail to a given contact
+(to do this only once,
+the messenger has to keep a `user_avatar_update_state` somewhere).
+Alternatively, the messenger MAY send an explicit mail
+for each profile-image change to all contacts using a compatible messenger.
+The messenger SHOULD NOT send an explicit mail to normal MUAs.
+
+    From: sender@domain
+    To: rcpt@domain
+    Chat-Version: 1.0
+    Chat-User-Avatar: photo.jpg
+    Subject: Chat: Hello, ...
+    Content-Type: multipart/mixed; boundary="==break=="
+
+    --==break==
+    Content-Type: text/plain
+
+    Hello, I've changed my profile image.
+    --==break==
+    Content-Type: image/jpeg
+    Content-Disposition: attachment; filename="photo.jpg"
+
+    AKCgkJi3j4l5kjoldfUAKCgkJi3j4lldfHjgWICwgIEBQYFBA ...
+    --==break==--
+
+The image format SHOULD be image/jpeg or image/png.
+Note that `Chat-User-Avatar` may appear together with all other headers,
+eg. there may be a `Chat-User-Avatar` and a `Chat-Group-Avatar` header
+in the same message.
+To save data, it is RECOMMENDED to add a `Chat-User-Avatar` header
+only on image changes.
+
+
+# Locations
+
+Locations can be attachted to messages using
+[standard kml-files](https://www.opengeospatial.org/standards/kml/)
+with well-known names.
+
+
+## User locations
+
+To send the location of the sender,
+the app can attach a file with the name `location.kml`.
+The file can contain one or more locations.
+Apps that support location streaming will typically collect some location events
+and send them together in one file.
+As each location has an independent timestamp,
+the apps can show the location as a track.
+
+Note that the `addr` attribute inside the  `location.kml` file
+MUST match the users email-address.
+Otherwise, the file is discarded silently;
+this is to protect against getting wrong locations,
+eg. forwarded from a normal MUA.
+
+    <?xml version="1.0" encoding="UTF-8"?>
+    <kml xmlns="http://www.opengis.net/kml/2.2">
+      <Document addr="ndh@deltachat.de">
+        <Placemark>
+          <Timestamp><when>2020-01-11T20:40:19Z</when></Timestamp>
+          <Point><coordinates accuracy="1.2">1.234,5.678</coordinates></Point>
+        </Placemark>
+        <Placemark>
+          <Timestamp><when>2020-01-11T20:40:25Z</when></Timestamp>
+          <Point><coordinates accuracy="5.4">7.654,3.21</coordinates></Point>
+        </Placemark>
+      </Document>
+    </kml>
+
+
+## Points of interest
+
+To send an "Point of interest", a POI,
+use a normal message and attach a file with the name  `message.kml`.
+In contrast to user locations, this file should contain only one location
+and an address-attribute is not needed -
+as the location belongs to the message content,
+it is fine if the location is detected on forwarding etc.
+
+    <?xml version="1.0" encoding="UTF-8"?>
+    <kml xmlns="http://www.opengis.net/kml/2.2">
+      <Document>
+        <Placemark>
+          <Timestamp><when>2020-01-01T20:40:19Z</when></Timestamp>
+          <Point><coordinates accuracy="1.2">1.234,5.678</coordinates></Point>
+        </Placemark>
+      </Document>
+    </kml>
+
+
+# Miscellaneous
+
+Messengers SHOULD use the header `In-Reply-To` as usual.
+
+Messengers SHOULD add a `Chat-Voice-message: 1` header
+if an attached audio file is a voice message.
+
+Messengers MAY add a `Chat-Duration` header
+to specify the duration of attached audio or video files.
+The value MUST be the duration in milliseconds.
+This allows the receiver to show the time without knowing the file format.
+
+    In-Reply-To: Gr.12345uvwxyZ.0005@domain
+    Chat-Voice-Message: 1
+    Chat-Duration: 10000
+
+Messengers MAY send and receive Message Disposition Notifications
+(MDNs, [RFC 8098](https://tools.ietf.org/html/rfc8098),
+[RFC 3503](https://tools.ietf.org/html/rfc3503))
+using the `Chat-Disposition-Notification-To` header
+instead of the `Disposition-Notification-To`
+(which unfortunately forces many other MUAs
+to send weird mails not following any standard).
+
+
+## Sync messages
+
+If some action is required by a message header,
+the action should only be performed if the _effective date_ is newer
+than the date the last action was performed.
+
+We define the effective date of a message
+as the sending time of the message as indicated by its Date header,
+or the time of first receipt if that date is in the future or unavailable.
+
+
+Copyright © 2017-2020 Delta Chat contributors.

src/aheader.rs

@@ -1,14 +1,15 @@
+//! # Autocrypt header module
+//!
+//! Parse and create [Autocrypt-headers](https://autocrypt.org/en/latest/level1.html#the-autocrypt-header).
+
 use std::collections::BTreeMap;
-use std::ffi::CStr;
 use std::str::FromStr;
 use std::{fmt, str};
 
-use mmime::mailimf_types::*;
-
-use crate::constants::*;
 use crate::contact::*;
-use crate::dc_tools::as_str;
-use crate::key::*;
+use crate::context::Context;
+use crate::headerdef::{HeaderDef, HeaderDefMap};
+use crate::key::{DcKey, SignedPublicKey};
 
 /// Possible values for encryption preference
 #[derive(PartialEq, Eq, Debug, Clone, Copy, FromPrimitive, ToPrimitive)]
@@ -41,22 +42,27 @@ impl str::FromStr for EncryptPreference {
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         match s {
             "mutual" => Ok(EncryptPreference::Mutual),
-            "reset" => Ok(EncryptPreference::Reset),
-            _ => Ok(EncryptPreference::NoPreference),
+            "nopreference" => Ok(EncryptPreference::NoPreference),
+            _ => Err(()),
         }
     }
 }
 
-/// Parse and create [Autocrypt-headers](https://autocrypt.org/en/latest/level1.html#the-autocrypt-header).
+/// Autocrypt header
 #[derive(Debug)]
 pub struct Aheader {
     pub addr: String,
-    pub public_key: Key,
+    pub public_key: SignedPublicKey,
     pub prefer_encrypt: EncryptPreference,
 }
 
 impl Aheader {
-    pub fn new(addr: String, public_key: Key, prefer_encrypt: EncryptPreference) -> Self {
+    /// Creates new autocrypt header
+    pub fn new(
+        addr: String,
+        public_key: SignedPublicKey,
+        prefer_encrypt: EncryptPreference,
+    ) -> Self {
         Aheader {
             addr,
             public_key,
@@ -64,69 +70,52 @@ impl Aheader {
         }
     }
 
-    pub fn from_imffields(
-        wanted_from: *const libc::c_char,
-        header: *const mailimf_fields,
+    pub fn from_headers(
+        context: &Context,
+        wanted_from: &str,
+        headers: &[mailparse::MailHeader<'_>],
     ) -> Option<Self> {
-        if wanted_from.is_null() || header.is_null() {
-            return None;
-        }
-
-        let mut fine_header = None;
-        let mut cur = unsafe { (*(*header).fld_list).first };
-
-        while !cur.is_null() {
-            let field = unsafe { (*cur).data as *mut mailimf_field };
-            if !field.is_null()
-                && unsafe { (*field).fld_type } == MAILIMF_FIELD_OPTIONAL_FIELD as libc::c_int
-            {
-                let optional_field = unsafe { (*field).fld_data.fld_optional_field };
-                if !optional_field.is_null()
-                    && unsafe { !(*optional_field).fld_name.is_null() }
-                    && unsafe { CStr::from_ptr((*optional_field).fld_name).to_str().unwrap() }
-                        == "Autocrypt"
-                {
-                    let value = unsafe {
-                        CStr::from_ptr((*optional_field).fld_value)
-                            .to_str()
-                            .unwrap()
-                    };
-
-                    match Self::from_str(value) {
-                        Ok(test) => {
-                            if addr_cmp(&test.addr, as_str(wanted_from)) {
-                                if fine_header.is_none() {
-                                    fine_header = Some(test);
-                                } else {
-                                    // TODO: figure out what kind of error case this is
-                                    return None;
-                                }
-                            }
-                        }
-                        _ => {}
+        if let Some(value) = headers.get_header_value(HeaderDef::Autocrypt) {
+            match Self::from_str(&value) {
+                Ok(header) => {
+                    if addr_cmp(&header.addr, wanted_from) {
+                        return Some(header);
                     }
                 }
+                Err(err) => {
+                    warn!(
+                        context,
+                        "found invalid autocrypt header {}: {:?}", value, err
+                    );
+                }
             }
-
-            cur = unsafe { (*cur).next };
         }
 
-        fine_header
+        None
     }
 }
 
 impl fmt::Display for Aheader {
     fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
-        // TODO replace 78 with enum /rtn
-        // adds a whitespace every 78 characters, this allows libEtPan to
-        // wrap the lines according to RFC 5322
+        write!(fmt, "addr={};", self.addr)?;
+        if self.prefer_encrypt == EncryptPreference::Mutual {
+            write!(fmt, " prefer-encrypt=mutual;")?;
+        }
+
+        // adds a whitespace every 78 characters, this allows
+        // email crate to wrap the lines according to RFC 5322
         // (which may insert a linebreak before every whitespace)
-        let keydata = self.public_key.to_base64(78);
-        write!(
-            fmt,
-            "addr={}; prefer-encrypt={}; keydata={}",
-            self.addr, self.prefer_encrypt, keydata
-        )
+        let keydata = self.public_key.to_base64().chars().enumerate().fold(
+            String::new(),
+            |mut res, (i, c)| {
+                if i % 78 == 78 - "keydata=".len() {
+                    res.push(' ')
+                }
+                res.push(c);
+                res
+            },
+        );
+        write!(fmt, " keydata={}", keydata)
     }
 }
 
@@ -135,17 +124,13 @@ impl str::FromStr for Aheader {
 
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         let mut attributes: BTreeMap<String, String> = s
-            .split(";")
+            .split(';')
             .filter_map(|a| {
-                let attribute: Vec<&str> = a.trim().splitn(2, "=").collect();
-                if attribute.len() < 2 {
-                    return None;
+                let attribute: Vec<&str> = a.trim().splitn(2, '=').collect();
+                match &attribute[..] {
+                    [key, value] => Some((key.trim().to_string(), value.trim().to_string())),
+                    _ => None,
                 }
-
-                Some((
-                    attribute[0].trim().to_string(),
-                    attribute[1].trim().to_string(),
-                ))
             })
             .collect();
 
@@ -155,34 +140,20 @@ impl str::FromStr for Aheader {
                 return Err(());
             }
         };
-
-        let public_key = match attributes
+        let public_key: SignedPublicKey = attributes
             .remove("keydata")
-            .and_then(|raw| Key::from_base64(&raw, KeyType::Public))
-        {
-            Some(key) => {
-                if key.verify() {
-                    key
-                } else {
-                    return Err(());
-                }
-            }
-            None => {
-                return Err(());
-            }
-        };
+            .ok_or(())
+            .and_then(|raw| SignedPublicKey::from_base64(&raw).or(Err(())))
+            .and_then(|key| key.verify().and(Ok(key)).or(Err(())))?;
 
-        let prefer_encrypt = match attributes
+        let prefer_encrypt = attributes
             .remove("prefer-encrypt")
             .and_then(|raw| raw.parse().ok())
-        {
-            Some(pref) => pref,
-            None => EncryptPreference::NoPreference,
-        };
+            .unwrap_or_default();
 
         // Autocrypt-Level0: unknown attributes starting with an underscore can be safely ignored
         // Autocrypt-Level0: unknown attribute, treat the header as invalid
-        if attributes.keys().find(|k| !k.starts_with("_")).is_some() {
+        if attributes.keys().any(|k| !k.starts_with('_')) {
             return Err(());
         }
 
@@ -198,15 +169,13 @@ impl str::FromStr for Aheader {
 mod tests {
     use super::*;
 
-    fn rawkey() -> String {
-        "xsBNBFzG3j0BCAC6iNhT8zydvCXi8LI/gFnkadMbfmSE/rTJskRRra/utGbLyDta/yTrJgWL7O3y/g4HdDW/dN2z26Y6W13IMzx9gLInn1KQZChtqWAcr/ReUucXcymwcfg1mdkBGk3TSLeLihN6CJx8Wsv8ig+kgAzte4f5rqEEAJVQ9WZHuti7UiYs6oRzqTo06CRe9owVXxzdMf0VDQtf7ZFm9dpzKKbhH7Lu8880iiotQ9/yRCkDGp9fNThsrLdZiK6OIAcIBAqi2rI89aS1dAmnRbktQieCx5izzyYkR1KvVL3gTTllHOzfKVEC2asmtWu2e4se/+O4WMIS1eGrn7GeWVb0Vwc5ABEBAAHNETxhQEBiLmV4YW1wbGUuZGU+wsCJBBABCAAzAhkBBQJcxt5FAhsDBAsJCAcGFQgJCgsCAxYCARYhBI4xxYKBgH3ANh5cufaKrc9mtiMLAAoJEPaKrc9mtiML938H/18F+3Wf9/JaAy/8hCO1v4S2PVBhxaKCokaNFtkfaMRne2l087LscCFPiFNyb4mv6Z3YeK8Xpxlp2sI0ecvdiqLUOGfnxS6tQrj+83EjtIrZ/hXOk1h121QFWH9Zg2VNHtODXjAgdLDC0NWUrclR0ZOqEDQHeo0ibTILdokVfXFN25wakPmGaYJP2y729cb1ve7RzvIvwn+Dddfxo3ao72rBfLi7l4NQ4S0KsY4cw+/6l5bRCKYCP77wZtvCwUvfVVosLdT43agtSiBI49+ayqvZ8OCvSJa61i+v81brTiEy9GBod4eAp45Ibsuemkw+gon4ZOvUXHTjwFB+h63MrozOwE0EXMbePQEIAL/vauf1zK8JgCu3V+G+SOX0iWw5xUlCPX+ERpBbWfwu3uAqn4wYXD3JDE/fVAF668xiV4eTPtlSUd5h0mn+G7uXMMOtkb+20SoEt50f8zw8TrL9t+ZsV11GKZWJpCar5AhXWsn6EEi8I2hLL5vn55ZZmHuGgN4jjmkRl3ToKCLhaXwTBjCJem7N5EH7F75wErEITa55v4Lb4Nfca7vnvtYrI1OA446xa8gHra0SINelTD09/JM/Fw4sWVPBaRZmJK/Tnu79N23No9XBUubmFPv1pNexZsQclicnTpt/BEWhiun7d6lfGB63K1aoHRTR1pcrWvBuALuuz0gqar2zlI0AEQEAAcLAdgQYAQgAIAUCXMbeRQIbDBYhBI4xxYKBgH3ANh5cufaKrc9mtiMLAAoJEPaKrc9mtiMLKSEIAIyLCRO2OyZ0IYRvRPpMn4p7E+7Pfcz/0mSkOy+1hshgJnqivXurm8zwGrwdMqeV4eslKR9H1RUdWGUQJNbtwmmjrt5DHpIhYHl5t3FpCBaGbV20Omo00Q38lBl9MtrmZkZw+ktEk6X+0xCKssMF+2MADkSOIufbR5HrDVB89VZOHCO9DeXvCUUAw2hyJiL/LHmLzJ40zYoTmb+F//f0k0j+tRdbkefyRoCmwG7YGiT+2hnCdgcezswnzah5J3ZKlrg7jOGo1LxtbvNUzxNBbC6S/aNgwm6qxo7xegRhmEl5uZ16zwyj4qz+xkjGy25Of5mWfUDoNw7OT7sjUbHOOMc=".into()
-    }
+    const RAWKEY: &str = "xsBNBFzG3j0BCAC6iNhT8zydvCXi8LI/gFnkadMbfmSE/rTJskRRra/utGbLyDta/yTrJgWL7O3y/g4HdDW/dN2z26Y6W13IMzx9gLInn1KQZChtqWAcr/ReUucXcymwcfg1mdkBGk3TSLeLihN6CJx8Wsv8ig+kgAzte4f5rqEEAJVQ9WZHuti7UiYs6oRzqTo06CRe9owVXxzdMf0VDQtf7ZFm9dpzKKbhH7Lu8880iiotQ9/yRCkDGp9fNThsrLdZiK6OIAcIBAqi2rI89aS1dAmnRbktQieCx5izzyYkR1KvVL3gTTllHOzfKVEC2asmtWu2e4se/+O4WMIS1eGrn7GeWVb0Vwc5ABEBAAHNETxhQEBiLmV4YW1wbGUuZGU+wsCJBBABCAAzAhkBBQJcxt5FAhsDBAsJCAcGFQgJCgsCAxYCARYhBI4xxYKBgH3ANh5cufaKrc9mtiMLAAoJEPaKrc9mtiML938H/18F+3Wf9/JaAy/8hCO1v4S2PVBhxaKCokaNFtkfaMRne2l087LscCFPiFNyb4mv6Z3YeK8Xpxlp2sI0ecvdiqLUOGfnxS6tQrj+83EjtIrZ/hXOk1h121QFWH9Zg2VNHtODXjAgdLDC0NWUrclR0ZOqEDQHeo0ibTILdokVfXFN25wakPmGaYJP2y729cb1ve7RzvIvwn+Dddfxo3ao72rBfLi7l4NQ4S0KsY4cw+/6l5bRCKYCP77wZtvCwUvfVVosLdT43agtSiBI49+ayqvZ8OCvSJa61i+v81brTiEy9GBod4eAp45Ibsuemkw+gon4ZOvUXHTjwFB+h63MrozOwE0EXMbePQEIAL/vauf1zK8JgCu3V+G+SOX0iWw5xUlCPX+ERpBbWfwu3uAqn4wYXD3JDE/fVAF668xiV4eTPtlSUd5h0mn+G7uXMMOtkb+20SoEt50f8zw8TrL9t+ZsV11GKZWJpCar5AhXWsn6EEi8I2hLL5vn55ZZmHuGgN4jjmkRl3ToKCLhaXwTBjCJem7N5EH7F75wErEITa55v4Lb4Nfca7vnvtYrI1OA446xa8gHra0SINelTD09/JM/Fw4sWVPBaRZmJK/Tnu79N23No9XBUubmFPv1pNexZsQclicnTpt/BEWhiun7d6lfGB63K1aoHRTR1pcrWvBuALuuz0gqar2zlI0AEQEAAcLAdgQYAQgAIAUCXMbeRQIbDBYhBI4xxYKBgH3ANh5cufaKrc9mtiMLAAoJEPaKrc9mtiMLKSEIAIyLCRO2OyZ0IYRvRPpMn4p7E+7Pfcz/0mSkOy+1hshgJnqivXurm8zwGrwdMqeV4eslKR9H1RUdWGUQJNbtwmmjrt5DHpIhYHl5t3FpCBaGbV20Omo00Q38lBl9MtrmZkZw+ktEk6X+0xCKssMF+2MADkSOIufbR5HrDVB89VZOHCO9DeXvCUUAw2hyJiL/LHmLzJ40zYoTmb+F//f0k0j+tRdbkefyRoCmwG7YGiT+2hnCdgcezswnzah5J3ZKlrg7jOGo1LxtbvNUzxNBbC6S/aNgwm6qxo7xegRhmEl5uZ16zwyj4qz+xkjGy25Of5mWfUDoNw7OT7sjUbHOOMc=";
 
     #[test]
     fn test_from_str() {
         let h: Aheader = format!(
             "addr=me@mail.com; prefer-encrypt=mutual; keydata={}",
-            rawkey()
+            RAWKEY
         )
         .parse()
         .expect("failed to parse");
@@ -215,9 +184,22 @@ mod tests {
         assert_eq!(h.prefer_encrypt, EncryptPreference::Mutual);
     }
 
+    // EncryptPreference::Reset is an internal value, parser should never return it
+    #[test]
+    fn test_from_str_reset() {
+        let raw = format!(
+            "addr=reset@example.com; prefer-encrypt=reset; keydata={}",
+            RAWKEY
+        );
+        let h: Aheader = raw.parse().expect("failed to parse");
+
+        assert_eq!(h.addr, "reset@example.com");
+        assert_eq!(h.prefer_encrypt, EncryptPreference::NoPreference);
+    }
+
     #[test]
     fn test_from_str_non_critical() {
-        let raw = format!("addr=me@mail.com; _foo=one; _bar=two; keydata={}", rawkey());
+        let raw = format!("addr=me@mail.com; _foo=one; _bar=two; keydata={}", RAWKEY);
         let h: Aheader = raw.parse().expect("failed to parse");
 
         assert_eq!(h.addr, "me@mail.com");
@@ -228,33 +210,57 @@ mod tests {
     fn test_from_str_superflous_critical() {
         let raw = format!(
             "addr=me@mail.com; _foo=one; _bar=two; other=me; keydata={}",
-            rawkey()
+            RAWKEY
         );
         assert!(raw.parse::<Aheader>().is_err());
     }
 
     #[test]
     fn test_good_headers() {
-        let fixed_header = "addr=a@b.example.org; prefer-encrypt=mutual; keydata=xsBNBFzG3j0BCAC6iNhT8zydvCXi8LI/gFnkadMbfmSE/rTJskRRra/utGbLyDta/yTrJgWL7O3y/g 4HdDW/dN2z26Y6W13IMzx9gLInn1KQZChtqWAcr/ReUucXcymwcfg1mdkBGk3TSLeLihN6CJx8Wsv8 ig+kgAzte4f5rqEEAJVQ9WZHuti7UiYs6oRzqTo06CRe9owVXxzdMf0VDQtf7ZFm9dpzKKbhH7Lu88 80iiotQ9/yRCkDGp9fNThsrLdZiK6OIAcIBAqi2rI89aS1dAmnRbktQieCx5izzyYkR1KvVL3gTTll HOzfKVEC2asmtWu2e4se/+O4WMIS1eGrn7GeWVb0Vwc5ABEBAAHNETxhQEBiLmV4YW1wbGUuZGU+ws CJBBABCAAzAhkBBQJcxt5FAhsDBAsJCAcGFQgJCgsCAxYCARYhBI4xxYKBgH3ANh5cufaKrc9mtiML AAoJEPaKrc9mtiML938H/18F+3Wf9/JaAy/8hCO1v4S2PVBhxaKCokaNFtkfaMRne2l087LscCFPiF Nyb4mv6Z3YeK8Xpxlp2sI0ecvdiqLUOGfnxS6tQrj+83EjtIrZ/hXOk1h121QFWH9Zg2VNHtODXjAg dLDC0NWUrclR0ZOqEDQHeo0ibTILdokVfXFN25wakPmGaYJP2y729cb1ve7RzvIvwn+Dddfxo3ao72 rBfLi7l4NQ4S0KsY4cw+/6l5bRCKYCP77wZtvCwUvfVVosLdT43agtSiBI49+ayqvZ8OCvSJa61i+v 81brTiEy9GBod4eAp45Ibsuemkw+gon4ZOvUXHTjwFB+h63MrozOwE0EXMbePQEIAL/vauf1zK8JgC u3V+G+SOX0iWw5xUlCPX+ERpBbWfwu3uAqn4wYXD3JDE/fVAF668xiV4eTPtlSUd5h0mn+G7uXMMOt kb+20SoEt50f8zw8TrL9t+ZsV11GKZWJpCar5AhXWsn6EEi8I2hLL5vn55ZZmHuGgN4jjmkRl3ToKC LhaXwTBjCJem7N5EH7F75wErEITa55v4Lb4Nfca7vnvtYrI1OA446xa8gHra0SINelTD09/JM/Fw4s WVPBaRZmJK/Tnu79N23No9XBUubmFPv1pNexZsQclicnTpt/BEWhiun7d6lfGB63K1aoHRTR1pcrWv BuALuuz0gqar2zlI0AEQEAAcLAdgQYAQgAIAUCXMbeRQIbDBYhBI4xxYKBgH3ANh5cufaKrc9mtiML AAoJEPaKrc9mtiMLKSEIAIyLCRO2OyZ0IYRvRPpMn4p7E+7Pfcz/0mSkOy+1hshgJnqivXurm8zwGr wdMqeV4eslKR9H1RUdWGUQJNbtwmmjrt5DHpIhYHl5t3FpCBaGbV20Omo00Q38lBl9MtrmZkZw+ktE k6X+0xCKssMF+2MADkSOIufbR5HrDVB89VZOHCO9DeXvCUUAw2hyJiL/LHmLzJ40zYoTmb+F//f0k0 j+tRdbkefyRoCmwG7YGiT+2hnCdgcezswnzah5J3ZKlrg7jOGo1LxtbvNUzxNBbC6S/aNgwm6qxo7x egRhmEl5uZ16zwyj4qz+xkjGy25Of5mWfUDoNw7OT7sjUbHOOMc=";
+        let fixed_header = concat!(
+            "addr=a@b.example.org; prefer-encrypt=mutual; ",
+            "keydata=xsBNBFzG3j0BCAC6iNhT8zydvCXi8LI/gFnkadMbfmSE/rTJskRRra/utGbLyDta/yTrJg",
+            " WL7O3y/g4HdDW/dN2z26Y6W13IMzx9gLInn1KQZChtqWAcr/ReUucXcymwcfg1mdkBGk3TSLeLihN6",
+            " CJx8Wsv8ig+kgAzte4f5rqEEAJVQ9WZHuti7UiYs6oRzqTo06CRe9owVXxzdMf0VDQtf7ZFm9dpzKK",
+            " bhH7Lu8880iiotQ9/yRCkDGp9fNThsrLdZiK6OIAcIBAqi2rI89aS1dAmnRbktQieCx5izzyYkR1Kv",
+            " VL3gTTllHOzfKVEC2asmtWu2e4se/+O4WMIS1eGrn7GeWVb0Vwc5ABEBAAHNETxhQEBiLmV4YW1wbG",
+            " UuZGU+wsCJBBABCAAzAhkBBQJcxt5FAhsDBAsJCAcGFQgJCgsCAxYCARYhBI4xxYKBgH3ANh5cufaK",
+            " rc9mtiMLAAoJEPaKrc9mtiML938H/18F+3Wf9/JaAy/8hCO1v4S2PVBhxaKCokaNFtkfaMRne2l087",
+            " LscCFPiFNyb4mv6Z3YeK8Xpxlp2sI0ecvdiqLUOGfnxS6tQrj+83EjtIrZ/hXOk1h121QFWH9Zg2VN",
+            " HtODXjAgdLDC0NWUrclR0ZOqEDQHeo0ibTILdokVfXFN25wakPmGaYJP2y729cb1ve7RzvIvwn+Ddd",
+            " fxo3ao72rBfLi7l4NQ4S0KsY4cw+/6l5bRCKYCP77wZtvCwUvfVVosLdT43agtSiBI49+ayqvZ8OCv",
+            " SJa61i+v81brTiEy9GBod4eAp45Ibsuemkw+gon4ZOvUXHTjwFB+h63MrozOwE0EXMbePQEIAL/vau",
+            " f1zK8JgCu3V+G+SOX0iWw5xUlCPX+ERpBbWfwu3uAqn4wYXD3JDE/fVAF668xiV4eTPtlSUd5h0mn+",
+            " G7uXMMOtkb+20SoEt50f8zw8TrL9t+ZsV11GKZWJpCar5AhXWsn6EEi8I2hLL5vn55ZZmHuGgN4jjm",
+            " kRl3ToKCLhaXwTBjCJem7N5EH7F75wErEITa55v4Lb4Nfca7vnvtYrI1OA446xa8gHra0SINelTD09",
+            " /JM/Fw4sWVPBaRZmJK/Tnu79N23No9XBUubmFPv1pNexZsQclicnTpt/BEWhiun7d6lfGB63K1aoHR",
+            " TR1pcrWvBuALuuz0gqar2zlI0AEQEAAcLAdgQYAQgAIAUCXMbeRQIbDBYhBI4xxYKBgH3ANh5cufaK",
+            " rc9mtiMLAAoJEPaKrc9mtiMLKSEIAIyLCRO2OyZ0IYRvRPpMn4p7E+7Pfcz/0mSkOy+1hshgJnqivX",
+            " urm8zwGrwdMqeV4eslKR9H1RUdWGUQJNbtwmmjrt5DHpIhYHl5t3FpCBaGbV20Omo00Q38lBl9Mtrm",
+            " ZkZw+ktEk6X+0xCKssMF+2MADkSOIufbR5HrDVB89VZOHCO9DeXvCUUAw2hyJiL/LHmLzJ40zYoTmb",
+            " +F//f0k0j+tRdbkefyRoCmwG7YGiT+2hnCdgcezswnzah5J3ZKlrg7jOGo1LxtbvNUzxNBbC6S/aNg",
+            " wm6qxo7xegRhmEl5uZ16zwyj4qz+xkjGy25Of5mWfUDoNw7OT7sjUbHOOMc="
+        );
 
         let ah = Aheader::from_str(fixed_header).expect("failed to parse");
         assert_eq!(ah.addr, "a@b.example.org");
         assert_eq!(ah.prefer_encrypt, EncryptPreference::Mutual);
+        assert_eq!(format!("{}", ah), fixed_header);
 
         let rendered = ah.to_string();
         assert_eq!(rendered, fixed_header);
 
-        let ah = Aheader::from_str(&format!(" _foo; __FOO=BAR ;;; addr = a@b.example.org ;\r\n   prefer-encrypt = mutual ; keydata = {}", rawkey())).expect("failed to parse");
+        let ah = Aheader::from_str(&format!(" _foo; __FOO=BAR ;;; addr = a@b.example.org ;\r\n   prefer-encrypt = mutual ; keydata = {}", RAWKEY)).expect("failed to parse");
         assert_eq!(ah.addr, "a@b.example.org");
         assert_eq!(ah.prefer_encrypt, EncryptPreference::Mutual);
 
         Aheader::from_str(&format!(
             "addr=a@b.example.org; prefer-encrypt=ignoreUnknownValues; keydata={}",
-            rawkey()
+            RAWKEY
         ))
         .expect("failed to parse");
 
-        Aheader::from_str(&format!("addr=a@b.example.org; keydata={}", rawkey()))
+        Aheader::from_str(&format!("addr=a@b.example.org; keydata={}", RAWKEY))
             .expect("failed to parse");
     }
 
@@ -266,4 +272,30 @@ mod tests {
         assert!(Aheader::from_str(" ;;").is_err());
         assert!(Aheader::from_str("addr=a@t.de; unknwon=1; keydata=jau").is_err());
     }
+
+    #[test]
+    fn test_display_aheader() {
+        assert!(format!(
+            "{}",
+            Aheader::new(
+                "test@example.com".to_string(),
+                SignedPublicKey::from_base64(RAWKEY).unwrap(),
+                EncryptPreference::Mutual
+            )
+        )
+        .contains("prefer-encrypt=mutual;"));
+
+        // According to Autocrypt Level 1 specification,
+        // only "prefer-encrypt=mutual;" can be used.
+        // If the setting is nopreference, the whole attribute is omitted.
+        assert!(!format!(
+            "{}",
+            Aheader::new(
+                "test@example.com".to_string(),
+                SignedPublicKey::from_base64(RAWKEY).unwrap(),
+                EncryptPreference::NoPreference
+            )
+        )
+        .contains("prefer-encrypt"));
+    }
 }

src/blob.rs

@@ -0,0 +1,572 @@
+//! # Blob directory management
+
+use std::ffi::OsStr;
+use std::fmt;
+use std::fs;
+use std::io::Write;
+use std::path::{Path, PathBuf};
+
+use image::GenericImageView;
+use thiserror::Error;
+
+use crate::constants::AVATAR_SIZE;
+use crate::context::Context;
+use crate::events::Event;
+
+/// Represents a file in the blob directory.
+///
+/// The object has a name, which will always be valid UTF-8.  Having a
+/// blob object does not imply the respective file exists, however
+/// when using one of the `create*()` methods a unique file is
+/// created.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct BlobObject<'a> {
+    blobdir: &'a Path,
+    name: String,
+}
+
+impl<'a> BlobObject<'a> {
+    /// Creates a new blob object with a unique name.
+    ///
+    /// Creates a new file in the blob directory.  The name will be
+    /// derived from the platform-agnostic basename of the suggested
+    /// name, followed by a random number and followed by a possible
+    /// extension.  The `data` will be written into the file without
+    /// race-conditions.
+    ///
+    /// # Errors
+    ///
+    /// [BlobError::CreateFailure] is used when the file could not
+    /// be created.  You can expect [BlobError.cause] to contain an
+    /// underlying error.
+    ///
+    /// [BlobError::WriteFailure] is used when the file could not
+    /// be written to.  You can expect [BlobError.cause] to contain an
+    /// underlying error.
+    pub fn create(
+        context: &'a Context,
+        suggested_name: impl AsRef<str>,
+        data: &[u8],
+    ) -> std::result::Result<BlobObject<'a>, BlobError> {
+        let blobdir = context.get_blobdir();
+        let (stem, ext) = BlobObject::sanitise_name(suggested_name.as_ref());
+        let (name, mut file) = BlobObject::create_new_file(&blobdir, &stem, &ext)?;
+        file.write_all(data)
+            .map_err(|err| BlobError::WriteFailure {
+                blobdir: blobdir.to_path_buf(),
+                blobname: name.clone(),
+                cause: err,
+            })?;
+        let blob = BlobObject {
+            blobdir,
+            name: format!("$BLOBDIR/{}", name),
+        };
+        context.call_cb(Event::NewBlobFile(blob.as_name().to_string()));
+        Ok(blob)
+    }
+
+    // Creates a new file, returning a tuple of the name and the handle.
+    fn create_new_file(dir: &Path, stem: &str, ext: &str) -> Result<(String, fs::File), BlobError> {
+        let max_attempt = 15;
+        let mut name = format!("{}{}", stem, ext);
+        for attempt in 0..max_attempt {
+            let path = dir.join(&name);
+            match fs::OpenOptions::new()
+                .create_new(true)
+                .write(true)
+                .open(&path)
+            {
+                Ok(file) => return Ok((name, file)),
+                Err(err) => {
+                    if attempt == max_attempt {
+                        return Err(BlobError::CreateFailure {
+                            blobdir: dir.to_path_buf(),
+                            blobname: name,
+                            cause: err,
+                        });
+                    } else {
+                        name = format!("{}-{}{}", stem, rand::random::<u32>(), ext);
+                    }
+                }
+            }
+        }
+        // This is supposed to be unreachable, but the compiler doesn't know.
+        Err(BlobError::CreateFailure {
+            blobdir: dir.to_path_buf(),
+            blobname: name,
+            cause: std::io::Error::new(std::io::ErrorKind::Other, "supposedly unreachable"),
+        })
+    }
+
+    /// Creates a new blob object with unique name by copying an existing file.
+    ///
+    /// This creates a new blob as described in [BlobObject::create]
+    /// but also copies an existing file into it.  This is done in a
+    /// in way which avoids race-conditions when multiple files are
+    /// concurrently created.
+    ///
+    /// # Errors
+    ///
+    /// In addition to the errors in [BlobObject::create] the
+    /// [BlobError::CopyFailure] is used when the data can not be
+    /// copied.
+    pub fn create_and_copy(
+        context: &'a Context,
+        src: impl AsRef<Path>,
+    ) -> std::result::Result<BlobObject<'a>, BlobError> {
+        let mut src_file = fs::File::open(src.as_ref()).map_err(|err| BlobError::CopyFailure {
+            blobdir: context.get_blobdir().to_path_buf(),
+            blobname: String::from(""),
+            src: src.as_ref().to_path_buf(),
+            cause: err,
+        })?;
+        let (stem, ext) = BlobObject::sanitise_name(&src.as_ref().to_string_lossy());
+        let (name, mut dst_file) = BlobObject::create_new_file(context.get_blobdir(), &stem, &ext)?;
+        let name_for_err = name.clone();
+        std::io::copy(&mut src_file, &mut dst_file).map_err(|err| {
+            {
+                // Attempt to remove the failed file, swallow errors resulting from that.
+                let path = context.get_blobdir().join(&name_for_err);
+                fs::remove_file(path).ok();
+            }
+            BlobError::CopyFailure {
+                blobdir: context.get_blobdir().to_path_buf(),
+                blobname: name_for_err,
+                src: src.as_ref().to_path_buf(),
+                cause: err,
+            }
+        })?;
+        let blob = BlobObject {
+            blobdir: context.get_blobdir(),
+            name: format!("$BLOBDIR/{}", name),
+        };
+        context.call_cb(Event::NewBlobFile(blob.as_name().to_string()));
+        Ok(blob)
+    }
+
+    /// Creates a blob from a file, possibly copying it to the blobdir.
+    ///
+    /// If the source file is not a path to into the blob directory
+    /// the file will be copied into the blob directory first.  If the
+    /// source file is already in the blobdir it will not be copied
+    /// and only be created if it is a valid blobname, that is no
+    /// subdirectory is used and [BlobObject::sanitise_name] does not
+    /// modify the filename.
+    ///
+    /// # Errors
+    ///
+    /// This merely delegates to the [BlobObject::create_and_copy] and
+    /// the [BlobObject::from_path] methods.  See those for possible
+    /// errors.
+    pub fn new_from_path(
+        context: &Context,
+        src: impl AsRef<Path>,
+    ) -> std::result::Result<BlobObject, BlobError> {
+        if src.as_ref().starts_with(context.get_blobdir()) {
+            BlobObject::from_path(context, src)
+        } else {
+            BlobObject::create_and_copy(context, src)
+        }
+    }
+
+    /// Returns a [BlobObject] for an existing blob from a path.
+    ///
+    /// The path must designate a file directly in the blobdir and
+    /// must use a valid blob name.  That is after sanitisation the
+    /// name must still be the same, that means it must be valid UTF-8
+    /// and not have any special characters in it.
+    ///
+    /// # Errors
+    ///
+    /// [BlobError::WrongBlobdir] is used if the path is not in
+    /// the blob directory.
+    ///
+    /// [BlobError::WrongName] is used if the file name does not
+    /// remain identical after sanitisation.
+    pub fn from_path(
+        context: &Context,
+        path: impl AsRef<Path>,
+    ) -> std::result::Result<BlobObject, BlobError> {
+        let rel_path = path
+            .as_ref()
+            .strip_prefix(context.get_blobdir())
+            .map_err(|_| BlobError::WrongBlobdir {
+                blobdir: context.get_blobdir().to_path_buf(),
+                src: path.as_ref().to_path_buf(),
+            })?;
+        if !BlobObject::is_acceptible_blob_name(&rel_path) {
+            return Err(BlobError::WrongName {
+                blobname: path.as_ref().to_path_buf(),
+            });
+        }
+        let name = rel_path.to_str().ok_or_else(|| BlobError::WrongName {
+            blobname: path.as_ref().to_path_buf(),
+        })?;
+        BlobObject::from_name(context, name.to_string())
+    }
+
+    /// Returns a [BlobObject] for an existing blob.
+    ///
+    /// The `name` may optionally be prefixed with the `$BLOBDIR/`
+    /// prefixed, as returned by [BlobObject::as_name].  This is how
+    /// you want to create a [BlobObject] for a filename read from the
+    /// database.
+    ///
+    /// # Errors
+    ///
+    /// [BlobError::WrongName] is used if the name is not a valid
+    /// blobname, i.e. if [BlobObject::sanitise_name] does modify the
+    /// provided name.
+    pub fn from_name(
+        context: &'a Context,
+        name: String,
+    ) -> std::result::Result<BlobObject<'a>, BlobError> {
+        let name: String = match name.starts_with("$BLOBDIR/") {
+            true => name.splitn(2, '/').last().unwrap().to_string(),
+            false => name,
+        };
+        if !BlobObject::is_acceptible_blob_name(&name) {
+            return Err(BlobError::WrongName {
+                blobname: PathBuf::from(name),
+            });
+        }
+        Ok(BlobObject {
+            blobdir: context.get_blobdir(),
+            name: format!("$BLOBDIR/{}", name),
+        })
+    }
+
+    /// Returns the absolute path to the blob in the filesystem.
+    pub fn to_abs_path(&self) -> PathBuf {
+        let fname = Path::new(&self.name).strip_prefix("$BLOBDIR/").unwrap();
+        self.blobdir.join(fname)
+    }
+
+    /// Returns the blob name, as stored in the database.
+    ///
+    /// This returns the blob in the `$BLOBDIR/<name>` format used in
+    /// the database.  Do not use this unless you're about to store
+    /// this string in the database or [Params].  Eventually even
+    /// those conversions should be handled by the type system.
+    ///
+    /// [Params]: crate::param::Params
+    pub fn as_name(&self) -> &str {
+        &self.name
+    }
+
+    /// Returns the filename of the blob.
+    pub fn as_file_name(&self) -> &str {
+        self.name.rsplitn(2, '/').next().unwrap()
+    }
+
+    /// The path relative in the blob directory.
+    pub fn as_rel_path(&self) -> &Path {
+        Path::new(self.as_file_name())
+    }
+
+    /// Returns the extension of the blob.
+    ///
+    /// If a blob's filename has an extension, it is always guaranteed
+    /// to be lowercase.
+    pub fn suffix(&self) -> Option<&str> {
+        let ext = self.name.rsplitn(2, '.').next();
+        if ext == Some(&self.name) {
+            None
+        } else {
+            ext
+        }
+    }
+
+    /// Create a safe name based on a messy input string.
+    ///
+    /// The safe name will be a valid filename on Unix and Windows and
+    /// not contain any path separators.  The input can contain path
+    /// segments separated by either Unix or Windows path separators,
+    /// the rightmost non-empty segment will be used as name,
+    /// sanitised for special characters.
+    ///
+    /// The resulting name is returned as a tuple, the first part
+    /// being the stem or basename and the second being an extension,
+    /// including the dot.  E.g. "foo.txt" is returned as `("foo",
+    /// ".txt")` while "bar" is returned as `("bar", "")`.
+    ///
+    /// The extension part will always be lowercased.
+    fn sanitise_name(name: &str) -> (String, String) {
+        let mut name = name.to_string();
+        for part in name.rsplit('/') {
+            if !part.is_empty() {
+                name = part.to_string();
+                break;
+            }
+        }
+        for part in name.rsplit('\\') {
+            if !part.is_empty() {
+                name = part.to_string();
+                break;
+            }
+        }
+        let opts = sanitize_filename::Options {
+            truncate: true,
+            windows: true,
+            replacement: "",
+        };
+
+        let clean = sanitize_filename::sanitize_with_options(name, opts);
+        let mut iter = clean.splitn(2, '.');
+        let stem: String = iter.next().unwrap_or_default().chars().take(64).collect();
+        let ext: String = iter.next().unwrap_or_default().chars().take(32).collect();
+        if ext.is_empty() {
+            (stem, "".to_string())
+        } else {
+            (stem, format!(".{}", ext).to_lowercase())
+        }
+    }
+
+    /// Checks whether a name is a valid blob name.
+    ///
+    /// This is slightly less strict than stanitise_name, presumably
+    /// someone already created a file with such a name so we just
+    /// ensure it's not actually a path in disguise is actually utf-8.
+    fn is_acceptible_blob_name(name: impl AsRef<OsStr>) -> bool {
+        let uname = match name.as_ref().to_str() {
+            Some(name) => name,
+            None => return false,
+        };
+        if uname.find('/').is_some() {
+            return false;
+        }
+        if uname.find('\\').is_some() {
+            return false;
+        }
+        if uname.find('\0').is_some() {
+            return false;
+        }
+        true
+    }
+
+    pub fn recode_to_avatar_size(&self, context: &Context) -> Result<(), BlobError> {
+        let blob_abs = self.to_abs_path();
+        let img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
+            blobdir: context.get_blobdir().to_path_buf(),
+            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+            cause: err,
+        })?;
+
+        if img.width() <= AVATAR_SIZE && img.height() <= AVATAR_SIZE {
+            return Ok(());
+        }
+
+        let img = img.thumbnail(AVATAR_SIZE, AVATAR_SIZE);
+
+        img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
+            blobdir: context.get_blobdir().to_path_buf(),
+            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+            cause: err,
+        })?;
+
+        Ok(())
+    }
+}
+
+impl<'a> fmt::Display for BlobObject<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "$BLOBDIR/{}", self.name)
+    }
+}
+
+/// Errors for the [BlobObject].
+#[derive(Debug, Error)]
+pub enum BlobError {
+    #[error("Failed to create blob {blobname} in {}", .blobdir.display())]
+    CreateFailure {
+        blobdir: PathBuf,
+        blobname: String,
+        #[source]
+        cause: std::io::Error,
+    },
+    #[error("Failed to write data to blob {blobname} in {}", .blobdir.display())]
+    WriteFailure {
+        blobdir: PathBuf,
+        blobname: String,
+        #[source]
+        cause: std::io::Error,
+    },
+    #[error("Failed to copy data from {} to blob {blobname} in {}", .src.display(), .blobdir.display())]
+    CopyFailure {
+        blobdir: PathBuf,
+        blobname: String,
+        src: PathBuf,
+        #[source]
+        cause: std::io::Error,
+    },
+    #[error("Failed to recode to blob {blobname} in {}", .blobdir.display())]
+    RecodeFailure {
+        blobdir: PathBuf,
+        blobname: String,
+        #[source]
+        cause: image::ImageError,
+    },
+    #[error("File path {} is not in {}", .src.display(), .blobdir.display())]
+    WrongBlobdir { blobdir: PathBuf, src: PathBuf },
+    #[error("Blob has a badname {}", .blobname.display())]
+    WrongName { blobname: PathBuf },
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use crate::test_utils::*;
+
+    #[test]
+    fn test_create() {
+        let t = dummy_context();
+        let blob = BlobObject::create(&t.ctx, "foo", b"hello").unwrap();
+        let fname = t.ctx.get_blobdir().join("foo");
+        let data = fs::read(fname).unwrap();
+        assert_eq!(data, b"hello");
+        assert_eq!(blob.as_name(), "$BLOBDIR/foo");
+        assert_eq!(blob.to_abs_path(), t.ctx.get_blobdir().join("foo"));
+    }
+
+    #[test]
+    fn test_lowercase_ext() {
+        let t = dummy_context();
+        let blob = BlobObject::create(&t.ctx, "foo.TXT", b"hello").unwrap();
+        assert_eq!(blob.as_name(), "$BLOBDIR/foo.txt");
+    }
+
+    #[test]
+    fn test_as_file_name() {
+        let t = dummy_context();
+        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
+        assert_eq!(blob.as_file_name(), "foo.txt");
+    }
+
+    #[test]
+    fn test_as_rel_path() {
+        let t = dummy_context();
+        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
+        assert_eq!(blob.as_rel_path(), Path::new("foo.txt"));
+    }
+
+    #[test]
+    fn test_suffix() {
+        let t = dummy_context();
+        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
+        assert_eq!(blob.suffix(), Some("txt"));
+        let blob = BlobObject::create(&t.ctx, "bar", b"world").unwrap();
+        assert_eq!(blob.suffix(), None);
+    }
+
+    #[test]
+    fn test_create_dup() {
+        let t = dummy_context();
+        BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
+        let foo_path = t.ctx.get_blobdir().join("foo.txt");
+        assert!(foo_path.exists());
+        BlobObject::create(&t.ctx, "foo.txt", b"world").unwrap();
+        for dirent in fs::read_dir(t.ctx.get_blobdir()).unwrap() {
+            let fname = dirent.unwrap().file_name();
+            if fname == foo_path.file_name().unwrap() {
+                assert_eq!(fs::read(&foo_path).unwrap(), b"hello");
+            } else {
+                let name = fname.to_str().unwrap();
+                assert!(name.starts_with("foo"));
+                assert!(name.ends_with(".txt"));
+            }
+        }
+    }
+
+    #[test]
+    fn test_double_ext_preserved() {
+        let t = dummy_context();
+        BlobObject::create(&t.ctx, "foo.tar.gz", b"hello").unwrap();
+        let foo_path = t.ctx.get_blobdir().join("foo.tar.gz");
+        assert!(foo_path.exists());
+        BlobObject::create(&t.ctx, "foo.tar.gz", b"world").unwrap();
+        for dirent in fs::read_dir(t.ctx.get_blobdir()).unwrap() {
+            let fname = dirent.unwrap().file_name();
+            if fname == foo_path.file_name().unwrap() {
+                assert_eq!(fs::read(&foo_path).unwrap(), b"hello");
+            } else {
+                let name = fname.to_str().unwrap();
+                println!("{}", name);
+                assert!(name.starts_with("foo"));
+                assert!(name.ends_with(".tar.gz"));
+            }
+        }
+    }
+
+    #[test]
+    fn test_create_long_names() {
+        let t = dummy_context();
+        let s = "1".repeat(150);
+        let blob = BlobObject::create(&t.ctx, &s, b"data").unwrap();
+        let blobname = blob.as_name().split('/').last().unwrap();
+        assert!(blobname.len() < 128);
+    }
+
+    #[test]
+    fn test_create_and_copy() {
+        let t = dummy_context();
+        let src = t.dir.path().join("src");
+        fs::write(&src, b"boo").unwrap();
+        let blob = BlobObject::create_and_copy(&t.ctx, &src).unwrap();
+        assert_eq!(blob.as_name(), "$BLOBDIR/src");
+        let data = fs::read(blob.to_abs_path()).unwrap();
+        assert_eq!(data, b"boo");
+
+        let whoops = t.dir.path().join("whoops");
+        assert!(BlobObject::create_and_copy(&t.ctx, &whoops).is_err());
+        let whoops = t.ctx.get_blobdir().join("whoops");
+        assert!(!whoops.exists());
+    }
+
+    #[test]
+    fn test_create_from_path() {
+        let t = dummy_context();
+
+        let src_ext = t.dir.path().join("external");
+        fs::write(&src_ext, b"boo").unwrap();
+        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).unwrap();
+        assert_eq!(blob.as_name(), "$BLOBDIR/external");
+        let data = fs::read(blob.to_abs_path()).unwrap();
+        assert_eq!(data, b"boo");
+
+        let src_int = t.ctx.get_blobdir().join("internal");
+        fs::write(&src_int, b"boo").unwrap();
+        let blob = BlobObject::new_from_path(&t.ctx, &src_int).unwrap();
+        assert_eq!(blob.as_name(), "$BLOBDIR/internal");
+        let data = fs::read(blob.to_abs_path()).unwrap();
+        assert_eq!(data, b"boo");
+    }
+    #[test]
+    fn test_create_from_name_long() {
+        let t = dummy_context();
+        let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
+        fs::write(&src_ext, b"boo").unwrap();
+        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).unwrap();
+        assert_eq!(
+            blob.as_name(),
+            "$BLOBDIR/autocrypt-setup-message-4137848473.html"
+        );
+    }
+
+    #[test]
+    fn test_is_blob_name() {
+        assert!(BlobObject::is_acceptible_blob_name("foo"));
+        assert!(BlobObject::is_acceptible_blob_name("foo.txt"));
+        assert!(BlobObject::is_acceptible_blob_name("f".repeat(128)));
+        assert!(!BlobObject::is_acceptible_blob_name("foo/bar"));
+        assert!(!BlobObject::is_acceptible_blob_name("foo\\bar"));
+        assert!(!BlobObject::is_acceptible_blob_name("foo\x00bar"));
+    }
+
+    #[test]
+    fn test_sanitise_name() {
+        let (_, ext) =
+            BlobObject::sanitise_name("Я ЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯ.txt");
+        assert_eq!(ext, ".txt");
+    }
+}

src/chatlist.rs

@@ -1,10 +1,13 @@
+//! # Chat list module
+
+use crate::chat;
 use crate::chat::*;
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::*;
-use crate::error::Result;
+use crate::error::{bail, ensure, Result};
 use crate::lot::Lot;
-use crate::message::*;
+use crate::message::{Message, MessageState, MsgId};
 use crate::stock::StockMessage;
 
 /// An object representing a single chatlist in memory.
@@ -31,17 +34,13 @@ use crate::stock::StockMessage;
 /// first entry and only present on new messages, there is the rough idea that it can be optionally always
 /// present and sorted into the list by date. Rendering the deaddrop in the described way
 /// would not add extra work in the UI then.
-pub struct Chatlist<'a> {
-    context: &'a Context,
+#[derive(Debug)]
+pub struct Chatlist {
     /// Stores pairs of `chat_id, message_id`
-    ids: Vec<(u32, u32)>,
+    ids: Vec<(ChatId, MsgId)>,
 }
 
-impl<'a> Chatlist<'a> {
-    pub fn get_context(&self) -> &Context {
-        self.context
-    }
-
+impl Chatlist {
     /// Get a list of chats.
     /// The list can be filtered by query parameters.
     ///
@@ -62,7 +61,7 @@ impl<'a> Chatlist<'a> {
     ///   or "Not now".
     ///   The UI can also offer a "Close" button that calls dc_marknoticed_contact() then.
     /// - DC_CHAT_ID_ARCHIVED_LINK (6) - this special chat is present if the user has
-    ///   archived _any_ chat using dc_archive_chat(). The UI should show a link as
+    ///   archived *any* chat using dc_set_chat_visibility(). The UI should show a link as
     ///   "Show archived chats", if the user clicks this item, the UI should show a
     ///   list of all archived chats that can be created by this function hen using
     ///   the DC_GCL_ARCHIVED_ONLY flag.
@@ -73,8 +72,11 @@ impl<'a> Chatlist<'a> {
     /// The `listflags` is a combination of flags:
     /// - if the flag DC_GCL_ARCHIVED_ONLY is set, only archived chats are returned.
     ///   if DC_GCL_ARCHIVED_ONLY is not set, only unarchived chats are returned and
-    ///   the pseudo-chat DC_CHAT_ID_ARCHIVED_LINK is added if there are _any_ archived
+    ///   the pseudo-chat DC_CHAT_ID_ARCHIVED_LINK is added if there are *any* archived
     ///   chats
+    /// - the flag DC_GCL_FOR_FORWARDING sorts "Saved messages" to the top of the chatlist
+    ///   and hides the device-chat,
+    //    typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS
     /// - if the flag DC_GCL_NO_SPECIALS is set, deaddrop and archive link are not added
     ///   to the list (may be used eg. for selecting chats on forwarding, the flag is
     ///   not needed when DC_GCL_ARCHIVED_ONLY is already set)
@@ -85,30 +87,23 @@ impl<'a> Chatlist<'a> {
     /// `query_contact_id`: An optional contact ID for filtering the list. Only chats including this contact ID
     ///     are returned.
     pub fn try_load(
-        context: &'a Context,
+        context: &Context,
         listflags: usize,
         query: Option<&str>,
         query_contact_id: Option<u32>,
     ) -> Result<Self> {
-        let mut add_archived_link_item = 0;
+        // Note that we do not emit DC_EVENT_MSGS_MODIFIED here even if some
+        // messages get hidden to avoid reloading the same chatlist.
+        if let Err(err) = hide_device_expired_messages(context) {
+            warn!(context, "Failed to hide expired messages: {}", err);
+        }
 
-        // select with left join and minimum:
-        // - the inner select must use `hidden` and _not_ `m.hidden`
-        //   which would refer the outer select and take a lot of time
-        // - `GROUP BY` is needed several messages may have the same timestamp
-        // - the list starts with the newest chats
-        // nb: the query currently shows messages from blocked contacts in groups.
-        // however, for normal-groups, this is okay as the message is also returned by dc_get_chat_msgs()
-        // (otherwise it would be hard to follow conversations, wa and tg do the same)
-        // for the deaddrop, however, they should really be hidden, however, _currently_ the deaddrop is not
-        // shown at all permanent in the chatlist.
+        let mut add_archived_link_item = false;
 
         let process_row = |row: &rusqlite::Row| {
-            let chat_id: i32 = row.get(0)?;
-            // TODO: verify that it is okay for this to be Null
-            let msg_id: i32 = row.get(1).unwrap_or_default();
-
-            Ok((chat_id as u32, msg_id as u32))
+            let chat_id: ChatId = row.get(0)?;
+            let msg_id: MsgId = row.get(1).unwrap_or_default();
+            Ok((chat_id, msg_id))
         };
 
         let process_rows = |rows: rusqlite::MappedRows<_>| {
@@ -116,37 +111,71 @@ impl<'a> Chatlist<'a> {
                 .map_err(Into::into)
         };
 
-        // nb: the query currently shows messages from blocked contacts in groups.
-        // however, for normal-groups, this is okay as the message is also returned by dc_get_chat_msgs()
-        // (otherwise it would be hard to follow conversations, wa and tg do the same)
-        // for the deaddrop, however, they should really be hidden, however, _currently_ the deaddrop is not
-        // shown at all permanent in the chatlist.
+        let skip_id = if 0 != listflags & DC_GCL_FOR_FORWARDING {
+            chat::lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
+                .unwrap_or_default()
+                .0
+        } else {
+            ChatId::new(0)
+        };
 
+        // select with left join and minimum:
+        //
+        // - the inner select must use `hidden` and _not_ `m.hidden`
+        //   which would refer the outer select and take a lot of time
+        // - `GROUP BY` is needed several messages may have the same
+        //   timestamp
+        // - the list starts with the newest chats
+        //
+        // nb: the query currently shows messages from blocked
+        // contacts in groups.  however, for normal-groups, this is
+        // okay as the message is also returned by dc_get_chat_msgs()
+        // (otherwise it would be hard to follow conversations, wa and
+        // tg do the same) for the deaddrop, however, they should
+        // really be hidden, however, _currently_ the deaddrop is not
+        // shown at all permanent in the chatlist.
         let mut ids = if let Some(query_contact_id) = query_contact_id {
             // show chats shared with a given contact
             context.sql.query_map(
-            "SELECT c.id, m.id FROM chats c  LEFT JOIN msgs m         \
-             ON c.id=m.chat_id        \
-             AND m.timestamp=( SELECT MAX(timestamp)   \
-             FROM msgs  WHERE chat_id=c.id    \
-             AND (hidden=0 OR (hidden=1 AND state=19))) WHERE c.id>9   \
-             AND c.blocked=0 AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)  \
-             GROUP BY c.id  ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;",
-            params![query_contact_id as i32],
-            process_row,
-            process_rows,
-        )?
+                "SELECT c.id, m.id
+                 FROM chats c
+                 LEFT JOIN msgs m
+                        ON c.id=m.chat_id
+                       AND m.timestamp=(
+                               SELECT MAX(timestamp)
+                                 FROM msgs
+                                WHERE chat_id=c.id
+                                  AND (hidden=0 OR state=?1))
+                 WHERE c.id>9
+                   AND c.blocked=0
+                   AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?2)
+                 GROUP BY c.id
+                 ORDER BY c.archived=?3 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                params![MessageState::OutDraft, query_contact_id as i32, ChatVisibility::Pinned],
+                process_row,
+                process_rows,
+            )?
         } else if 0 != listflags & DC_GCL_ARCHIVED_ONLY {
             // show archived chats
+            // (this includes the archived device-chat; we could skip it,
+            // however, then the number of archived chats do not match, which might be even more irritating.
+            // and adapting the number requires larger refactorings and seems not to be worth the effort)
             context.sql.query_map(
-                "SELECT c.id, m.id FROM chats c  LEFT JOIN msgs m         \
-                 ON c.id=m.chat_id        \
-                 AND m.timestamp=( SELECT MAX(timestamp)   \
-                 FROM msgs  WHERE chat_id=c.id    \
-                 AND (hidden=0 OR (hidden=1 AND state=19))) WHERE c.id>9   \
-                 AND c.blocked=0 AND c.archived=1  GROUP BY c.id  \
-                 ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;",
-                params![],
+                "SELECT c.id, m.id
+                 FROM chats c
+                 LEFT JOIN msgs m
+                        ON c.id=m.chat_id
+                       AND m.timestamp=(
+                               SELECT MAX(timestamp)
+                                 FROM msgs
+                                WHERE chat_id=c.id
+                                  AND (hidden=0 OR state=?))
+                 WHERE c.id>9
+                   AND c.blocked=0
+                   AND c.archived=1
+                 GROUP BY c.id
+                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                params![MessageState::OutDraft],
                 process_row,
                 process_rows,
             )?
@@ -154,53 +183,82 @@ impl<'a> Chatlist<'a> {
             let query = query.trim().to_string();
             ensure!(!query.is_empty(), "missing query");
 
+            // allow searching over special names that may change at any time
+            // when the ui calls set_stock_translation()
+            if let Err(err) = update_special_chat_names(context) {
+                warn!(context, "cannot update special chat names: {:?}", err)
+            }
+
             let str_like_cmd = format!("%{}%", query);
             context.sql.query_map(
-                "SELECT c.id, m.id FROM chats c  LEFT JOIN msgs m         \
-                 ON c.id=m.chat_id        \
-                 AND m.timestamp=( SELECT MAX(timestamp)   \
-                 FROM msgs  WHERE chat_id=c.id    \
-                 AND (hidden=0 OR (hidden=1 AND state=19))) WHERE c.id>9   \
-                 AND c.blocked=0 AND c.name LIKE ?  \
-                 GROUP BY c.id  ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;",
-                params![str_like_cmd],
+                "SELECT c.id, m.id
+                 FROM chats c
+                 LEFT JOIN msgs m
+                        ON c.id=m.chat_id
+                       AND m.timestamp=(
+                               SELECT MAX(timestamp)
+                                 FROM msgs
+                                WHERE chat_id=c.id
+                                  AND (hidden=0 OR state=?1))
+                 WHERE c.id>9 AND c.id!=?2
+                   AND c.blocked=0
+                   AND c.name LIKE ?3
+                 GROUP BY c.id
+                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                params![MessageState::OutDraft, skip_id, str_like_cmd],
                 process_row,
                 process_rows,
             )?
         } else {
             //  show normal chatlist
+            let sort_id_up = if 0 != listflags & DC_GCL_FOR_FORWARDING {
+                chat::lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
+                    .unwrap_or_default()
+                    .0
+            } else {
+                ChatId::new(0)
+            };
             let mut ids = context.sql.query_map(
-                "SELECT c.id, m.id FROM chats c  \
-                 LEFT JOIN msgs m         \
-                 ON c.id=m.chat_id        \
-                 AND m.timestamp=( SELECT MAX(timestamp)   \
-                 FROM msgs  WHERE chat_id=c.id    \
-                 AND (hidden=0 OR (hidden=1 AND state=19))) WHERE c.id>9   \
-                 AND c.blocked=0 AND c.archived=0  \
-                 GROUP BY c.id  \
-                 ORDER BY IFNULL(m.timestamp,0) DESC, m.id DESC;",
-                params![],
+                "SELECT c.id, m.id
+                 FROM chats c
+                 LEFT JOIN msgs m
+                        ON c.id=m.chat_id
+                       AND m.timestamp=(
+                               SELECT MAX(timestamp)
+                                 FROM msgs
+                                WHERE chat_id=c.id
+                                  AND (hidden=0 OR state=?1))
+                 WHERE c.id>9 AND c.id!=?2
+                   AND c.blocked=0
+                   AND NOT c.archived=?3
+                 GROUP BY c.id
+                 ORDER BY c.id=?4 DESC, c.archived=?5 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                params![MessageState::OutDraft, skip_id, ChatVisibility::Archived, sort_id_up, ChatVisibility::Pinned],
                 process_row,
                 process_rows,
             )?;
             if 0 == listflags & DC_GCL_NO_SPECIALS {
-                let last_deaddrop_fresh_msg_id = get_last_deaddrop_fresh_msg(context);
-                if last_deaddrop_fresh_msg_id > 0 {
-                    ids.push((1, last_deaddrop_fresh_msg_id));
+                if let Some(last_deaddrop_fresh_msg_id) = get_last_deaddrop_fresh_msg(context) {
+                    if 0 == listflags & DC_GCL_FOR_FORWARDING {
+                        ids.insert(
+                            0,
+                            (ChatId::new(DC_CHAT_ID_DEADDROP), last_deaddrop_fresh_msg_id),
+                        );
+                    }
                 }
-                add_archived_link_item = 1;
+                add_archived_link_item = true;
             }
             ids
         };
 
-        if 0 != add_archived_link_item && dc_get_archived_cnt(context) > 0 {
+        if add_archived_link_item && dc_get_archived_cnt(context) > 0 {
             if ids.is_empty() && 0 != listflags & DC_GCL_ADD_ALLDONE_HINT {
-                ids.push((DC_CHAT_ID_ALLDONE_HINT, 0));
+                ids.push((ChatId::new(DC_CHAT_ID_ALLDONE_HINT), MsgId::new(0)));
             }
-            ids.push((DC_CHAT_ID_ARCHIVED_LINK, 0));
+            ids.push((ChatId::new(DC_CHAT_ID_ARCHIVED_LINK), MsgId::new(0)));
         }
 
-        Ok(Chatlist { context, ids })
+        Ok(Chatlist { ids })
     }
 
     /// Find out the number of chats.
@@ -208,6 +266,7 @@ impl<'a> Chatlist<'a> {
         self.ids.len()
     }
 
+    /// Returns true if chatlist is empty.
     pub fn is_empty(&self) -> bool {
         self.ids.is_empty()
     }
@@ -215,22 +274,21 @@ impl<'a> Chatlist<'a> {
     /// Get a single chat ID of a chatlist.
     ///
     /// To get the message object from the message ID, use dc_get_chat().
-    pub fn get_chat_id(&self, index: usize) -> u32 {
-        if index >= self.ids.len() {
-            return 0;
+    pub fn get_chat_id(&self, index: usize) -> ChatId {
+        match self.ids.get(index) {
+            Some((chat_id, _msg_id)) => *chat_id,
+            None => ChatId::new(0),
         }
-        self.ids[index].0
     }
 
     /// Get a single message ID of a chatlist.
     ///
     /// To get the message object from the message ID, use dc_get_msg().
-    pub fn get_msg_id(&self, index: usize) -> u32 {
-        if index >= self.ids.len() {
-            return 0;
+    pub fn get_msg_id(&self, index: usize) -> Result<MsgId> {
+        match self.ids.get(index) {
+            Some((_chat_id, msg_id)) => Ok(*msg_id),
+            None => bail!("Chatlist index out of range"),
         }
-
-        self.ids[index].1
     }
 
     /// Get a summary for a chatlist index.
@@ -247,93 +305,187 @@ impl<'a> Chatlist<'a> {
     /// - dc_lot_t::timestamp: the timestamp of the message.  0 if not applicable.
     /// - dc_lot_t::state: The state of the message as one of the DC_STATE_* constants (see #dc_msg_get_state()).
     //    0 if not applicable.
-    pub fn get_summary(&self, index: usize, chat: Option<&Chat<'a>>) -> Lot {
+    pub fn get_summary(&self, context: &Context, index: usize, chat: Option<&Chat>) -> Lot {
         // The summary is created by the chat, not by the last message.
         // This is because we may want to display drafts here or stuff as
         // "is typing".
         // Also, sth. as "No messages" would not work if the summary comes from a message.
-
         let mut ret = Lot::new();
-        if index >= self.ids.len() {
-            ret.text2 = Some("ErrBadChatlistIndex".to_string());
-            return ret;
-        }
 
-        let chat_loaded: Chat;
-        let chat = if let Some(chat) = chat {
-            chat
-        } else {
-            if let Ok(chat) = Chat::load_from_db(self.context, self.ids[index].0) {
-                chat_loaded = chat;
-                &chat_loaded
-            } else {
+        let (chat_id, lastmsg_id) = match self.ids.get(index) {
+            Some(ids) => ids,
+            None => {
+                ret.text2 = Some("ErrBadChatlistIndex".to_string());
                 return ret;
             }
         };
 
-        let lastmsg_id = self.ids[index].1;
+        let chat_loaded: Chat;
+        let chat = if let Some(chat) = chat {
+            chat
+        } else if let Ok(chat) = Chat::load_from_db(context, *chat_id) {
+            chat_loaded = chat;
+            &chat_loaded
+        } else {
+            return ret;
+        };
+
         let mut lastcontact = None;
 
-        let lastmsg = if 0 != lastmsg_id {
-            if let Ok(lastmsg) = dc_msg_load_from_db(self.context, lastmsg_id) {
-                if lastmsg.from_id != 1 as libc::c_uint
-                    && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup)
-                {
-                    lastcontact = Contact::load_from_db(self.context, lastmsg.from_id).ok();
-                }
-
-                Some(lastmsg)
-            } else {
-                None
+        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, *lastmsg_id) {
+            if lastmsg.from_id != DC_CONTACT_ID_SELF
+                && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup)
+            {
+                lastcontact = Contact::load_from_db(context, lastmsg.from_id).ok();
             }
+
+            Some(lastmsg)
         } else {
             None
         };
 
-        if chat.id == DC_CHAT_ID_ARCHIVED_LINK {
+        if chat.id.is_archived_link() {
             ret.text2 = None;
         } else if lastmsg.is_none() || lastmsg.as_ref().unwrap().from_id == DC_CONTACT_ID_UNDEFINED
         {
-            ret.text2 = Some(self.context.stock_str(StockMessage::NoMessages).to_string());
+            ret.text2 = Some(context.stock_str(StockMessage::NoMessages).to_string());
         } else {
-            ret.fill(
-                &mut lastmsg.unwrap(),
-                chat,
-                lastcontact.as_ref(),
-                self.context,
-            );
+            ret.fill(&mut lastmsg.unwrap(), chat, lastcontact.as_ref(), context);
         }
 
         ret
     }
+
+    pub fn get_index_for_id(&self, id: ChatId) -> Option<usize> {
+        self.ids.iter().position(|(chat_id, _)| chat_id == &id)
+    }
 }
 
+/// Returns the number of archived chats
 pub fn dc_get_archived_cnt(context: &Context) -> u32 {
     context
         .sql
-        .query_row_col(
+        .query_get_value(
             context,
             "SELECT COUNT(*) FROM chats WHERE blocked=0 AND archived=1;",
             params![],
-            0,
         )
         .unwrap_or_default()
 }
 
-fn get_last_deaddrop_fresh_msg(context: &Context) -> u32 {
-    // We have an index over the state-column, this should be sufficient as there are typically
-    // only few fresh messages.
-    context
-        .sql
-        .query_row_col(
-            context,
-            "SELECT m.id  FROM msgs m  LEFT JOIN chats c ON c.id=m.chat_id  \
-             WHERE m.state=10   \
-             AND m.hidden=0    \
-             AND c.blocked=2 \
-             ORDER BY m.timestamp DESC, m.id DESC;",
-            params![],
-            0,
-        )
-        .unwrap_or_default()
+fn get_last_deaddrop_fresh_msg(context: &Context) -> Option<MsgId> {
+    // We have an index over the state-column, this should be
+    // sufficient as there are typically only few fresh messages.
+    context.sql.query_get_value(
+        context,
+        concat!(
+            "SELECT m.id",
+            " FROM msgs m",
+            " LEFT JOIN chats c",
+            "        ON c.id=m.chat_id",
+            " WHERE m.state=10",
+            "   AND m.hidden=0",
+            "   AND c.blocked=2",
+            " ORDER BY m.timestamp DESC, m.id DESC;"
+        ),
+        params![],
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    use crate::test_utils::*;
+
+    #[test]
+    fn test_try_load() {
+        let t = dummy_context();
+        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat").unwrap();
+        let chat_id2 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "b chat").unwrap();
+        let chat_id3 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "c chat").unwrap();
+
+        // check that the chatlist starts with the most recent message
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
+        assert_eq!(chats.len(), 3);
+        assert_eq!(chats.get_chat_id(0), chat_id3);
+        assert_eq!(chats.get_chat_id(1), chat_id2);
+        assert_eq!(chats.get_chat_id(2), chat_id1);
+
+        // drafts are sorted to the top
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some("hello".to_string()));
+        chat_id2.set_draft(&t.ctx, Some(&mut msg));
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
+        assert_eq!(chats.get_chat_id(0), chat_id2);
+
+        // check chatlist query and archive functionality
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("b"), None).unwrap();
+        assert_eq!(chats.len(), 1);
+
+        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None).unwrap();
+        assert_eq!(chats.len(), 0);
+
+        chat_id1
+            .set_visibility(&t.ctx, ChatVisibility::Archived)
+            .ok();
+        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None).unwrap();
+        assert_eq!(chats.len(), 1);
+    }
+
+    #[test]
+    fn test_sort_self_talk_up_on_forward() {
+        let t = dummy_context();
+        t.ctx.update_device_chats().unwrap();
+        create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat").unwrap();
+
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
+        assert!(chats.len() == 3);
+        assert!(!Chat::load_from_db(&t.ctx, chats.get_chat_id(0))
+            .unwrap()
+            .is_self_talk());
+
+        let chats = Chatlist::try_load(&t.ctx, DC_GCL_FOR_FORWARDING, None, None).unwrap();
+        assert!(chats.len() == 2); // device chat cannot be written and is skipped on forwarding
+        assert!(Chat::load_from_db(&t.ctx, chats.get_chat_id(0))
+            .unwrap()
+            .is_self_talk());
+    }
+
+    #[test]
+    fn test_search_special_chat_names() {
+        let t = dummy_context();
+        t.ctx.update_device_chats().unwrap();
+
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None).unwrap();
+        assert_eq!(chats.len(), 0);
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None).unwrap();
+        assert_eq!(chats.len(), 0);
+
+        t.ctx
+            .set_stock_translation(StockMessage::SavedMessages, "test-1234-save".to_string())
+            .unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None).unwrap();
+        assert_eq!(chats.len(), 1);
+
+        t.ctx
+            .set_stock_translation(StockMessage::DeviceMessages, "test-5678-babbel".to_string())
+            .unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None).unwrap();
+        assert_eq!(chats.len(), 1);
+    }
+
+    #[test]
+    fn test_get_summary_unwrap() {
+        let t = dummy_context();
+        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat").unwrap();
+
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some("foo:\nbar \r\n test".to_string()));
+        chat_id1.set_draft(&t.ctx, Some(&mut msg));
+
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
+        let summary = chats.get_summary(&t.ctx, 0, None);
+        assert_eq!(summary.get_text2().unwrap(), "foo: bar test"); // the linebreak should be removed from summary
+    }
 }

src/config.rs

@@ -1,12 +1,19 @@
+//! # Key-value configuration management
+
 use strum::{EnumProperty, IntoEnumIterator};
 use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString};
 
+use crate::blob::BlobObject;
+use crate::chat::ChatId;
 use crate::constants::DC_VERSION_STR;
 use crate::context::Context;
 use crate::dc_tools::*;
-use crate::error::Error;
+use crate::events::Event;
 use crate::job::*;
+use crate::message::MsgId;
+use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::stock::StockMessage;
+use rusqlite::NO_PARAMS;
 
 /// The available configuration keys.
 #[derive(
@@ -19,30 +26,67 @@ pub enum Config {
     MailUser,
     MailPw,
     MailPort,
+    ImapCertificateChecks,
     SendServer,
     SendUser,
     SendPw,
     SendPort,
+    SmtpCertificateChecks,
     ServerFlags,
+
     #[strum(props(default = "INBOX"))]
     ImapFolder,
+
     Displayname,
     Selfstatus,
     Selfavatar,
+
+    #[strum(props(default = "0"))]
+    BccSelf,
+
     #[strum(props(default = "1"))]
     E2eeEnabled,
+
     #[strum(props(default = "1"))]
     MdnsEnabled,
+
     #[strum(props(default = "1"))]
     InboxWatch,
+
     #[strum(props(default = "1"))]
     SentboxWatch,
+
     #[strum(props(default = "1"))]
     MvboxWatch,
+
     #[strum(props(default = "1"))]
     MvboxMove,
-    #[strum(props(default = "0"))]
+
+    #[strum(props(default = "0"))] // also change ShowEmails.default() on changes
     ShowEmails,
+
+    #[strum(props(default = "0"))]
+    KeyGenType,
+
+    /// Timer in seconds after which the message is deleted from the
+    /// server.
+    ///
+    /// Equals to 0 by default, which means the message is never
+    /// deleted.
+    ///
+    /// Value 1 is treated as "delete at once": messages are deleted
+    /// immediately, without moving to DeltaChat folder.
+    #[strum(props(default = "0"))]
+    DeleteServerAfter,
+
+    /// Timer in seconds after which the message is deleted from the
+    /// device.
+    ///
+    /// Equals to 0 by default, which means the message is never
+    /// deleted.
+    #[strum(props(default = "0"))]
+    DeleteDeviceAfter,
+
     SaveMimeHeaders,
     ConfiguredAddr,
     ConfiguredMailServer,
@@ -50,19 +94,23 @@ pub enum Config {
     ConfiguredMailPw,
     ConfiguredMailPort,
     ConfiguredMailSecurity,
+    ConfiguredImapCertificateChecks,
     ConfiguredSendServer,
     ConfiguredSendUser,
     ConfiguredSendPw,
     ConfiguredSendPort,
+    ConfiguredSmtpCertificateChecks,
     ConfiguredServerFlags,
     ConfiguredSendSecurity,
     ConfiguredE2EEEnabled,
     Configured,
-    // Deprecated
+
     #[strum(serialize = "sys.version")]
     SysVersion,
+
     #[strum(serialize = "sys.msgsize_max_recommended")]
     SysMsgsizeMaxRecommended,
+
     #[strum(serialize = "sys.config_keys")]
     SysConfigKeys,
 }
@@ -72,13 +120,13 @@ impl Context {
     pub fn get_config(&self, key: Config) -> Option<String> {
         let value = match key {
             Config::Selfavatar => {
-                let rel_path = self.sql.get_config(self, key);
-                rel_path.map(|p| dc_get_abs_path_safe(self, &p).to_str().unwrap().to_string())
+                let rel_path = self.sql.get_raw_config(self, key);
+                rel_path.map(|p| dc_get_abs_path(self, &p).to_string_lossy().into_owned())
             }
             Config::SysVersion => Some((&*DC_VERSION_STR).clone()),
-            Config::SysMsgsizeMaxRecommended => Some(format!("{}", 24 * 1024 * 1024 / 4 * 3)),
+            Config::SysMsgsizeMaxRecommended => Some(format!("{}", RECOMMENDED_FILE_SIZE)),
             Config::SysConfigKeys => Some(get_config_keys_string()),
-            _ => self.sql.get_config(self, key),
+            _ => self.sql.get_raw_config(self, key),
         };
 
         if value.is_some() {
@@ -92,27 +140,69 @@ impl Context {
         }
     }
 
+    pub fn get_config_int(&self, key: Config) -> i32 {
+        self.get_config(key)
+            .and_then(|s| s.parse().ok())
+            .unwrap_or_default()
+    }
+
+    pub fn get_config_bool(&self, key: Config) -> bool {
+        self.get_config_int(key) != 0
+    }
+
+    /// Gets configured "delete_server_after" value.
+    ///
+    /// `None` means never delete the message, `Some(0)` means delete
+    /// at once, `Some(x)` means delete after `x` seconds.
+    pub fn get_config_delete_server_after(&self) -> Option<i64> {
+        match self.get_config_int(Config::DeleteServerAfter) {
+            0 => None,
+            1 => Some(0),
+            x => Some(x as i64),
+        }
+    }
+
+    /// Gets configured "delete_device_after" value.
+    ///
+    /// `None` means never delete the message, `Some(x)` means delete
+    /// after `x` seconds.
+    pub fn get_config_delete_device_after(&self) -> Option<i64> {
+        match self.get_config_int(Config::DeleteDeviceAfter) {
+            0 => None,
+            x => Some(x as i64),
+        }
+    }
+
     /// Set the given config key.
     /// If `None` is passed as a value the value is cleared and set to the default if there is one.
-    pub fn set_config(&self, key: Config, value: Option<&str>) -> Result<(), Error> {
+    pub fn set_config(&self, key: Config, value: Option<&str>) -> crate::sql::Result<()> {
         match key {
-            Config::Selfavatar if value.is_some() => {
-                let rel_path = std::fs::canonicalize(value.unwrap())?;
+            Config::Selfavatar => {
                 self.sql
-                    .set_config(self, key, Some(&rel_path.to_string_lossy()))
+                    .execute("UPDATE contacts SET selfavatar_sent=0;", NO_PARAMS)?;
+                self.sql
+                    .set_raw_config_bool(self, "attach_selfavatar", true)?;
+                match value {
+                    Some(value) => {
+                        let blob = BlobObject::new_from_path(&self, value)?;
+                        blob.recode_to_avatar_size(self)?;
+                        self.sql.set_raw_config(self, key, Some(blob.as_name()))
+                    }
+                    None => self.sql.set_raw_config(self, key, None),
+                }
             }
             Config::InboxWatch => {
-                let ret = self.sql.set_config(self, key, value);
-                interrupt_imap_idle(self);
+                let ret = self.sql.set_raw_config(self, key, value);
+                interrupt_inbox_idle(self);
                 ret
             }
             Config::SentboxWatch => {
-                let ret = self.sql.set_config(self, key, value);
+                let ret = self.sql.set_raw_config(self, key, value);
                 interrupt_sentbox_idle(self);
                 ret
             }
             Config::MvboxWatch => {
-                let ret = self.sql.set_config(self, key, value);
+                let ret = self.sql.set_raw_config(self, key, value);
                 interrupt_mvbox_idle(self);
                 ret
             }
@@ -124,9 +214,18 @@ impl Context {
                     value
                 };
 
-                self.sql.set_config(self, key, val)
+                self.sql.set_raw_config(self, key, val)
             }
-            _ => self.sql.set_config(self, key, value),
+            Config::DeleteDeviceAfter => {
+                let ret = self.sql.set_raw_config(self, key, value);
+                // Force chatlist reload to delete old messages immediately.
+                self.call_cb(Event::MsgsChanged {
+                    msg_id: MsgId::new(0),
+                    chat_id: ChatId::new(0),
+                });
+                ret
+            }
+            _ => self.sql.set_raw_config(self, key, value),
         }
     }
 }
@@ -149,6 +248,12 @@ mod tests {
     use std::str::FromStr;
     use std::string::ToString;
 
+    use crate::constants::AVATAR_SIZE;
+    use crate::test_utils::*;
+    use image::GenericImageView;
+    use std::fs::File;
+    use std::io::Write;
+
     #[test]
     fn test_to_string() {
         assert_eq!(Config::MailServer.to_string(), "mail_server");
@@ -165,4 +270,57 @@ mod tests {
     fn test_default_prop() {
         assert_eq!(Config::ImapFolder.get_str("default"), Some("INBOX"));
     }
+
+    #[test]
+    fn test_selfavatar_outside_blobdir() {
+        let t = dummy_context();
+        let avatar_src = t.dir.path().join("avatar.jpg");
+        let avatar_bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
+        File::create(&avatar_src)
+            .unwrap()
+            .write_all(avatar_bytes)
+            .unwrap();
+        let avatar_blob = t.ctx.get_blobdir().join("avatar.jpg");
+        assert!(!avatar_blob.exists());
+        t.ctx
+            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+            .unwrap();
+        assert!(avatar_blob.exists());
+        assert!(std::fs::metadata(&avatar_blob).unwrap().len() < avatar_bytes.len() as u64);
+        let avatar_cfg = t.ctx.get_config(Config::Selfavatar);
+        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
+
+        let img = image::open(avatar_src).unwrap();
+        assert_eq!(img.width(), 1000);
+        assert_eq!(img.height(), 1000);
+
+        let img = image::open(avatar_blob).unwrap();
+        assert_eq!(img.width(), AVATAR_SIZE);
+        assert_eq!(img.height(), AVATAR_SIZE);
+    }
+
+    #[test]
+    fn test_selfavatar_in_blobdir() {
+        let t = dummy_context();
+        let avatar_src = t.ctx.get_blobdir().join("avatar.png");
+        let avatar_bytes = include_bytes!("../test-data/image/avatar900x900.png");
+        File::create(&avatar_src)
+            .unwrap()
+            .write_all(avatar_bytes)
+            .unwrap();
+
+        let img = image::open(&avatar_src).unwrap();
+        assert_eq!(img.width(), 900);
+        assert_eq!(img.height(), 900);
+
+        t.ctx
+            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
+            .unwrap();
+        let avatar_cfg = t.ctx.get_config(Config::Selfavatar);
+        assert_eq!(avatar_cfg, avatar_src.to_str().map(|s| s.to_string()));
+
+        let img = image::open(avatar_src).unwrap();
+        assert_eq!(img.width(), AVATAR_SIZE);
+        assert_eq!(img.height(), AVATAR_SIZE);
+    }
 }

src/configure/auto_mozilla.rs

@@ -1,81 +1,83 @@
-use quick_xml;
+//! # Thunderbird's Autoconfiguration implementation
+//!
+//! Documentation: https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration */
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
+use crate::constants::*;
 use crate::context::Context;
-use crate::dc_tools::*;
 use crate::login_param::LoginParam;
-use crate::x::*;
 
-use super::read_autoconf_file;
-/* ******************************************************************************
- * Thunderbird's Autoconfigure
- ******************************************************************************/
-/* documentation: https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration */
-#[repr(C)]
-struct moz_autoconfigure_t<'a> {
-    pub in_0: &'a LoginParam,
+use super::read_url::read_url;
+use super::Error;
+
+#[derive(Debug)]
+struct MozAutoconfigure<'a> {
+    pub in_emailaddr: &'a str,
     pub in_emaildomain: &'a str,
     pub in_emaillocalpart: &'a str,
     pub out: LoginParam,
-    pub out_imap_set: libc::c_int,
-    pub out_smtp_set: libc::c_int,
-    pub tag_server: libc::c_int,
-    pub tag_config: libc::c_int,
+    pub out_imap_set: bool,
+    pub out_smtp_set: bool,
+    pub tag_server: MozServer,
+    pub tag_config: MozConfigTag,
 }
 
-pub unsafe fn moz_autoconfigure(
-    context: &Context,
-    url: &str,
-    param_in: &LoginParam,
-) -> Option<LoginParam> {
-    let xml_raw = read_autoconf_file(context, url);
-    if xml_raw.is_null() {
-        return None;
-    }
+#[derive(Debug, PartialEq)]
+enum MozServer {
+    Undefined,
+    Imap,
+    Smtp,
+}
 
-    // Split address into local part and domain part.
-    let p = param_in.addr.find("@");
-    if p.is_none() {
-        free(xml_raw as *mut libc::c_void);
-        return None;
-    }
-    let (in_emaillocalpart, in_emaildomain) = param_in.addr.split_at(p.unwrap());
-    let in_emaildomain = &in_emaildomain[1..];
+#[derive(Debug)]
+enum MozConfigTag {
+    Undefined,
+    Hostname,
+    Port,
+    Sockettype,
+    Username,
+}
 
-    let mut reader = quick_xml::Reader::from_str(as_str(xml_raw));
+fn parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result<LoginParam, Error> {
+    let mut reader = quick_xml::Reader::from_str(xml_raw);
     reader.trim_text(true);
 
-    let mut buf = Vec::new();
+    // Split address into local part and domain part.
+    let parts: Vec<&str> = in_emailaddr.rsplitn(2, '@').collect();
+    let (in_emaillocalpart, in_emaildomain) = match &parts[..] {
+        [domain, local] => (local, domain),
+        _ => return Err(Error::InvalidEmailAddress(in_emailaddr.to_string())),
+    };
 
-    let mut moz_ac = moz_autoconfigure_t {
-        in_0: param_in,
+    let mut moz_ac = MozAutoconfigure {
+        in_emailaddr,
         in_emaildomain,
         in_emaillocalpart,
         out: LoginParam::new(),
-        out_imap_set: 0,
-        out_smtp_set: 0,
-        tag_server: 0,
-        tag_config: 0,
+        out_imap_set: false,
+        out_smtp_set: false,
+        tag_server: MozServer::Undefined,
+        tag_config: MozConfigTag::Undefined,
     };
+
+    let mut buf = Vec::new();
     loop {
-        match reader.read_event(&mut buf) {
-            Ok(quick_xml::events::Event::Start(ref e)) => {
+        let event = reader
+            .read_event(&mut buf)
+            .map_err(|error| Error::InvalidXml {
+                position: reader.buffer_position(),
+                error,
+            })?;
+
+        match event {
+            quick_xml::events::Event::Start(ref e) => {
                 moz_autoconfigure_starttag_cb(e, &mut moz_ac, &reader)
             }
-            Ok(quick_xml::events::Event::End(ref e)) => moz_autoconfigure_endtag_cb(e, &mut moz_ac),
-            Ok(quick_xml::events::Event::Text(ref e)) => {
+            quick_xml::events::Event::End(ref e) => moz_autoconfigure_endtag_cb(e, &mut moz_ac),
+            quick_xml::events::Event::Text(ref e) => {
                 moz_autoconfigure_text_cb(e, &mut moz_ac, &reader)
             }
-            Err(e) => {
-                error!(
-                    context,
-                    0,
-                    "Configure xml: Error at position {}: {:?}",
-                    reader.buffer_position(),
-                    e
-                );
-            }
-            Ok(quick_xml::events::Event::Eof) => break,
+            quick_xml::events::Event::Eof => break,
             _ => (),
         }
         buf.clear();
@@ -86,24 +88,37 @@ pub unsafe fn moz_autoconfigure(
         || moz_ac.out.send_server.is_empty()
         || moz_ac.out.send_port == 0
     {
-        let r = moz_ac.out.to_string();
-        warn!(context, 0, "Bad or incomplete autoconfig: {}", r,);
-        free(xml_raw as *mut libc::c_void);
-        return None;
+        Err(Error::IncompleteAutoconfig(moz_ac.out))
+    } else {
+        Ok(moz_ac.out)
     }
+}
 
-    free(xml_raw as *mut libc::c_void);
-    Some(moz_ac.out)
+pub fn moz_autoconfigure(
+    context: &Context,
+    url: &str,
+    param_in: &LoginParam,
+) -> Result<LoginParam, Error> {
+    let xml_raw = read_url(context, url)?;
+
+    let res = parse_xml(&param_in.addr, &xml_raw);
+    if let Err(err) = &res {
+        warn!(
+            context,
+            "Failed to parse Thunderbird autoconfiguration XML: {}", err
+        );
+    }
+    res
 }
 
 fn moz_autoconfigure_text_cb<B: std::io::BufRead>(
     event: &BytesText,
-    moz_ac: &mut moz_autoconfigure_t,
+    moz_ac: &mut MozAutoconfigure,
     reader: &quick_xml::Reader<B>,
 ) {
     let val = event.unescape_and_decode(reader).unwrap_or_default();
 
-    let addr = &moz_ac.in_0.addr;
+    let addr = moz_ac.in_emailaddr;
     let email_local = moz_ac.in_emaillocalpart;
     let email_domain = moz_ac.in_emaildomain;
 
@@ -113,66 +128,70 @@ fn moz_autoconfigure_text_cb<B: std::io::BufRead>(
         .replace("%EMAILLOCALPART%", email_local)
         .replace("%EMAILDOMAIN%", email_domain);
 
-    if moz_ac.tag_server == 1 {
-        match moz_ac.tag_config {
-            10 => moz_ac.out.mail_server = val,
-            11 => moz_ac.out.mail_port = val.parse().unwrap_or_default(),
-            12 => moz_ac.out.mail_user = val,
-            13 => {
+    match moz_ac.tag_server {
+        MozServer::Imap => match moz_ac.tag_config {
+            MozConfigTag::Hostname => moz_ac.out.mail_server = val,
+            MozConfigTag::Port => moz_ac.out.mail_port = val.parse().unwrap_or_default(),
+            MozConfigTag::Username => moz_ac.out.mail_user = val,
+            MozConfigTag::Sockettype => {
                 let val_lower = val.to_lowercase();
                 if val_lower == "ssl" {
-                    moz_ac.out.server_flags |= 0x200
+                    moz_ac.out.server_flags |= DC_LP_IMAP_SOCKET_SSL as i32
                 }
                 if val_lower == "starttls" {
-                    moz_ac.out.server_flags |= 0x100
+                    moz_ac.out.server_flags |= DC_LP_IMAP_SOCKET_STARTTLS as i32
                 }
                 if val_lower == "plain" {
-                    moz_ac.out.server_flags |= 0x400
+                    moz_ac.out.server_flags |= DC_LP_IMAP_SOCKET_PLAIN as i32
                 }
             }
             _ => {}
-        }
-    } else if moz_ac.tag_server == 2 {
-        match moz_ac.tag_config {
-            10 => moz_ac.out.send_server = val,
-            11 => moz_ac.out.send_port = val.parse().unwrap_or_default(),
-            12 => moz_ac.out.send_user = val,
-            13 => {
+        },
+        MozServer::Smtp => match moz_ac.tag_config {
+            MozConfigTag::Hostname => moz_ac.out.send_server = val,
+            MozConfigTag::Port => moz_ac.out.send_port = val.parse().unwrap_or_default(),
+            MozConfigTag::Username => moz_ac.out.send_user = val,
+            MozConfigTag::Sockettype => {
                 let val_lower = val.to_lowercase();
                 if val_lower == "ssl" {
-                    moz_ac.out.server_flags |= 0x20000
+                    moz_ac.out.server_flags |= DC_LP_SMTP_SOCKET_SSL as i32
                 }
                 if val_lower == "starttls" {
-                    moz_ac.out.server_flags |= 0x10000
+                    moz_ac.out.server_flags |= DC_LP_SMTP_SOCKET_STARTTLS as i32
                 }
                 if val_lower == "plain" {
-                    moz_ac.out.server_flags |= 0x40000
+                    moz_ac.out.server_flags |= DC_LP_SMTP_SOCKET_PLAIN as i32
                 }
             }
             _ => {}
-        }
+        },
+        MozServer::Undefined => {}
     }
 }
 
-fn moz_autoconfigure_endtag_cb(event: &BytesEnd, moz_ac: &mut moz_autoconfigure_t) {
+fn moz_autoconfigure_endtag_cb(event: &BytesEnd, moz_ac: &mut MozAutoconfigure) {
     let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
 
     if tag == "incomingserver" {
-        moz_ac.tag_server = 0;
-        moz_ac.tag_config = 0;
-        moz_ac.out_imap_set = 1;
+        if moz_ac.tag_server == MozServer::Imap {
+            moz_ac.out_imap_set = true;
+        }
+        moz_ac.tag_server = MozServer::Undefined;
+        moz_ac.tag_config = MozConfigTag::Undefined;
     } else if tag == "outgoingserver" {
-        moz_ac.tag_server = 0;
-        moz_ac.tag_config = 0;
-        moz_ac.out_smtp_set = 1;
+        if moz_ac.tag_server == MozServer::Smtp {
+            moz_ac.out_smtp_set = true;
+        }
+        moz_ac.tag_server = MozServer::Undefined;
+        moz_ac.tag_config = MozConfigTag::Undefined;
     } else {
-        moz_ac.tag_config = 0;
+        moz_ac.tag_config = MozConfigTag::Undefined;
     }
 }
 
 fn moz_autoconfigure_starttag_cb<B: std::io::BufRead>(
     event: &BytesStart,
-    moz_ac: &mut moz_autoconfigure_t,
+    moz_ac: &mut MozAutoconfigure,
     reader: &quick_xml::Reader<B>,
 ) {
     let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
@@ -189,25 +208,115 @@ fn moz_autoconfigure_starttag_cb<B: std::io::BufRead>(
                 .unwrap_or_default()
                 .to_lowercase();
 
-            if typ == "imap" && moz_ac.out_imap_set == 0 {
-                1
+            if typ == "imap" && !moz_ac.out_imap_set {
+                MozServer::Imap
             } else {
-                0
+                MozServer::Undefined
             }
         } else {
-            0
+            MozServer::Undefined
         };
-        moz_ac.tag_config = 0;
+        moz_ac.tag_config = MozConfigTag::Undefined;
     } else if tag == "outgoingserver" {
-        moz_ac.tag_server = if moz_ac.out_smtp_set == 0 { 2 } else { 0 };
-        moz_ac.tag_config = 0;
+        moz_ac.tag_server = if !moz_ac.out_smtp_set {
+            MozServer::Smtp
+        } else {
+            MozServer::Undefined
+        };
+        moz_ac.tag_config = MozConfigTag::Undefined;
     } else if tag == "hostname" {
-        moz_ac.tag_config = 10;
+        moz_ac.tag_config = MozConfigTag::Hostname;
     } else if tag == "port" {
-        moz_ac.tag_config = 11;
+        moz_ac.tag_config = MozConfigTag::Port;
     } else if tag == "sockettype" {
-        moz_ac.tag_config = 13;
+        moz_ac.tag_config = MozConfigTag::Sockettype;
     } else if tag == "username" {
-        moz_ac.tag_config = 12;
+        moz_ac.tag_config = MozConfigTag::Username;
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_outlook_autoconfig() {
+        // Copied from https://autoconfig.thunderbird.net/v1.1/outlook.com on 2019-10-11
+        let xml_raw =
+"<clientConfig version=\"1.1\">
+  <emailProvider id=\"outlook.com\">
+    <domain>hotmail.com</domain>
+    <domain>hotmail.co.uk</domain>
+    <domain>hotmail.co.jp</domain>
+    <domain>hotmail.com.br</domain>
+    <domain>hotmail.de</domain>
+    <domain>hotmail.fr</domain>
+    <domain>hotmail.it</domain>
+    <domain>hotmail.es</domain>
+    <domain>live.com</domain>
+    <domain>live.co.uk</domain>
+    <domain>live.co.jp</domain>
+    <domain>live.de</domain>
+    <domain>live.fr</domain>
+    <domain>live.it</domain>
+    <domain>live.jp</domain>
+    <domain>msn.com</domain>
+    <domain>outlook.com</domain>
+    <displayName>Outlook.com (Microsoft)</displayName>
+    <displayShortName>Outlook</displayShortName>
+    <incomingServer type=\"exchange\">
+      <hostname>outlook.office365.com</hostname>
+      <port>443</port>
+      <username>%EMAILADDRESS%</username>
+      <socketType>SSL</socketType>
+      <authentication>OAuth2</authentication>
+      <owaURL>https://outlook.office365.com/owa/</owaURL>
+      <ewsURL>https://outlook.office365.com/ews/exchange.asmx</ewsURL>
+      <useGlobalPreferredServer>true</useGlobalPreferredServer>
+    </incomingServer>
+    <incomingServer type=\"imap\">
+      <hostname>outlook.office365.com</hostname>
+      <port>993</port>
+      <socketType>SSL</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </incomingServer>
+    <incomingServer type=\"pop3\">
+      <hostname>outlook.office365.com</hostname>
+      <port>995</port>
+      <socketType>SSL</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+      <pop3>
+        <leaveMessagesOnServer>true</leaveMessagesOnServer>
+        <!-- Outlook.com docs specifically mention that POP3 deletes have effect on the main inbox on webmail and IMAP -->
+      </pop3>
+    </incomingServer>
+    <outgoingServer type=\"smtp\">
+      <hostname>smtp.office365.com</hostname>
+      <port>587</port>
+      <socketType>STARTTLS</socketType>
+      <authentication>password-cleartext</authentication>
+      <username>%EMAILADDRESS%</username>
+    </outgoingServer>
+    <documentation url=\"http://windows.microsoft.com/en-US/windows/outlook/send-receive-from-app\">
+      <descr lang=\"en\">Set up an email app with Outlook.com</descr>
+    </documentation>
+  </emailProvider>
+  <webMail>
+    <loginPage url=\"https://www.outlook.com/\"/>
+    <loginPageInfo url=\"https://www.outlook.com/\">
+      <username>%EMAILADDRESS%</username>
+      <usernameField id=\"i0116\" name=\"login\"/>
+      <passwordField id=\"i0118\" name=\"passwd\"/>
+      <loginButton id=\"idSIButton9\" name=\"SI\"/>
+    </loginPageInfo>
+  </webMail>
+</clientConfig>";
+        let res = parse_xml("example@outlook.com", xml_raw).expect("XML parsing failed");
+        assert_eq!(res.mail_server, "outlook.office365.com");
+        assert_eq!(res.mail_port, 993);
+        assert_eq!(res.send_server, "smtp.office365.com");
+        assert_eq!(res.send_port, 587);
     }
 }