Do not panic on failure to read IMAP server greeting

.circleci/config.yml

@@ -7,10 +7,6 @@ executors:
   doxygen:
     docker:
       - image: hrektts/doxygen
-  python:
-    docker:
-      - image: 3.7.7-stretch
-            
 
 restore-workspace: &restore-workspace
   attach_workspace:
@@ -124,25 +120,47 @@ jobs:
           paths:
             - c-docs
 
-  remote_python_packaging:
+  # build_test_docs_wheel:
+  #   docker:
+  #     - image: deltachat/coredeps
+  #       environment:
+  #           TESTS: 1
+  #           DOCS: 1
+  #   working_directory: /mnt/crate
+  #   steps:
+  #     - *restore-workspace
+  #     - *restore-cache
+  #     - run:
+  #         name: build docs, run tests and build wheels
+  #         command: ci_scripts/run-python.sh
+  #     - run:
+  #          name: copying docs and wheels to workspace
+  #          command: |
+  #            mkdir -p workspace/python
+  #            # cp -av docs workspace/c-docs
+  #            cp -av python/.docker-tox/wheelhouse workspace/
+  #            cp -av python/doc/_build/ workspace/py-docs
+  #     - persist_to_workspace:
+  #         root: workspace
+  #         paths:
+  #           # - c-docs
+  #           - py-docs
+  #           - wheelhouse
+
+  remote_tests_rust:
     machine: true
     steps:
       - checkout
-      # the following commands on success produces
-      # workspace/{wheelhouse,py-docs} as artefact directories
-      - run: bash ci_scripts/remote_python_packaging.sh 
-      - persist_to_workspace:
-          root: workspace
-          paths:
-            # - c-docs
-            - py-docs
-            - wheelhouse
-
+      - run: ci_scripts/remote_tests_rust.sh
+    
   remote_tests_python:
     machine: true
     steps:
       - checkout
+      #- attach_workspace:
+      #    at: workspace
       - run: ci_scripts/remote_tests_python.sh
+    # workspace/py-docs workspace/wheelhouse workspace/c-docs
 
   upload_docs_wheels:
     machine: true
@@ -150,7 +168,6 @@ jobs:
       - checkout
       - attach_workspace:
           at: workspace
-      - run: pyenv versions
       - run: pyenv global 3.5.2
       - run: ls -laR workspace
       - run: ci_scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
@@ -172,29 +189,14 @@ workflows:
     jobs:
       # - cargo_fetch
 
-      - remote_tests_python:
-          filters:
-            tags:
-              only: /.*/
+      - remote_tests_rust
 
-      - remote_python_packaging:
-          requires: 
-             - remote_tests_python
-          filters:
-            branches:
-              only: master
-            tags:
-              only: /.*/
+      - remote_tests_python
 
-      - upload_docs_wheels:
-          requires:
-             - remote_python_packaging
-             - build_doxygen
-          filters:
-            branches:
-              only: master
-            tags:
-              only: /.*/
+      # - upload_docs_wheels:
+      #     requires:
+      #       - build_test_docs_wheel
+      #       - build_doxygen
       # - rustfmt:
       #     requires:
       #       - cargo_fetch
@@ -202,12 +204,7 @@ workflows:
       #     requires:
       #      - cargo_fetch
 
-      - build_doxygen:
-          filters:
-            branches:
-              only: master
-            tags:
-              only: /.*/
+      - build_doxygen
 
       # Linux Desktop 64bit
       # - test_x86_64-unknown-linux-gnu:

.github/workflows/ci.yml

@@ -1,89 +0,0 @@
-name: Rust CI
-
-on:
-  pull_request:
-  push:
-    branches:
-      - master
-      - staging
-      - trying
-
-jobs:
-
-  fmt:
-    name: Rustfmt
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: 1.43.1
-          override: true
-      - run: rustup component add rustfmt
-      - uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
-
-  run_clippy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-            toolchain: 1.43.1
-            components: clippy
-            override: true
-      - uses: actions-rs/clippy-check@v1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-
-
-  build_and_test:
-    name: Build and test
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-latest, windows-latest, macOS-latest]
-        rust: [nightly, 1.43.1]
-
-    steps:
-    - uses: actions/checkout@master
-
-    - name: Install ${{ matrix.rust }}
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: ${{ matrix.rust }}
-        override: true
-
-    - name: Cache cargo registry
-      uses: actions/cache@v2
-      with:
-        path: ~/.cargo/registry
-        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}
-
-    - name: Cache cargo index
-      uses: actions/cache@v2
-      with:
-        path: ~/.cargo/git
-        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}
-
-    - name: Cache cargo build
-      uses: actions/cache@v2
-      with:
-        path: target
-        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-build-target-${{ hashFiles('**/Cargo.toml') }}
-
-    - name: check
-      uses: actions-rs/cargo@v1
-      with:
-        command:  check
-        args: --workspace --all --bins --examples --tests
-
-    - name: tests
-      uses: actions-rs/cargo@v1
-      with:
-        command: test
-        args: --workspace
-          

.github/workflows/code-quality.yml

@@ -0,0 +1,47 @@
+on: push
+name: Code Quality
+jobs:
+
+  check:
+    name: Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly-2019-11-06
+          override: true
+      - uses: actions-rs/cargo@v1
+        with:
+          command: check
+
+  fmt:
+    name: Rustfmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly-2019-11-06
+          override: true
+      - run: rustup component add rustfmt
+      - uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
+  run_clippy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+            toolchain: nightly-2019-11-06
+            components: clippy
+            override: true
+      - uses: actions-rs/clippy-check@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          args: --all-features

ASYNC-API-TODO.txt

@@ -1,54 +0,0 @@
-
-Delta Chat ASYNC  (friedel, bjoern, floris, friedel)
-
-- smtp fake-idle/load jobs gerade noch alle fuenf sekunden , sollte alle zehn minuten (oder gar nicht)
-
-APIs:
-    dc_context_new                 # opens the database
-    dc_open                             # FFI only
-      -> drop it and move parameters to dc_context_new()
-
-    dc_configure                      # note: dc_start_jobs() is NOT allowed to run concurrently
-    dc_imex NEVER goes through the job system
-    dc_imex import_backup needs to ensure dc_stop_jobs()
-
-    dc_start_io                # start smtp/imap and job handling subsystems
-    dc_stop_io                 # stop smtp/imap and job handling subsystems
-    dc_is_io_running           # return 1 if smtp/imap/jobs susbystem is running
-
-    dc_close                            # FFI only
-      -> can be dropped
-    dc_context_unref
-
-    for ios share-extension:
-    Int dc_direct_send()  -> try send out without going through jobs system, but queue a job in db if it needs to be retried on failure
-        0: message was sent
-        1: message failed to go out, is queued as a job to be retried later
-        2: message permanently failed?
-
-EVENT handling:
-    start a callback thread and call get_next_event() which is BLOCKING
-    it's fine to start this callback thread later, it will see all events.
-    Note that the core infinitely fills the internal queue if you never drain it.
-
-    FFI-get_next_event() returns NULL if the context is unrefed already?
-
-   sidenote: how python's callback thread does it currently:
-         CB-thread runs this while loop:
-         while not QUITFLAG:
-             ev = context.get_next_event( )
-             ...
-         So in order to shutdown properly one has to set QUITFLAG
-         before calling dc_stop_jobs() and dc_context_unref
-
-    event API:
-        get_data1_int
-        get_data2_int
-        get_data3_str
-
-
-- userdata likely only used for the callbacks, likely can be dropped, needs verification
-
-
-- iOS needs for the share app to call "try_send_smtp" wihtout a full dc_context_run and without going
-

CHANGELOG.md

@@ -1,328 +1,5 @@
 # Changelog 
 
-## 1.39.0
-
-- fix handling of `mvbox_watch`, `sentbox_watch`, `inbox_watch` #1654 #1658
-
-- fix potential panics, update dependencies #1650 #1655
-
-
-## 1.38.0
-
-- fix sorting, esp. for multi-device
-
-
-## 1.37.0
-
-- improve ndn heuristics #1630
-
-- get oauth2 authorizer from provider-db #1641
-
-- removed linebreaks and spaces from generated qr-code #1631
-
-- more fixes #1633 #1635 #1636 #1637
-
-
-## 1.36.0
-
-- parse ndn (network delivery notification) reports
-  and report failed messages as such #1552 #1622 #1630
-
-- add oauth2 support for gsuite domains #1626
-
-- read image orientation from exif before recoding #1619
-
-- improve logging #1593 #1598
-
-- improve python and bot bindings #1583 #1609
-
-- improve imap logout #1595
-
-- fix sorting #1600 #1604
-
-- fix qr code generation #1631
-
-- update rustcrypto releases #1603
-
-- refactorings #1617
-
-
-## 1.35.0
-
-- enable strict-tls from a new provider-db setting #1587
-
-- new subject 'Message from USER' for one-to-one chats #1395
-
-- recode images #1563
-
-- improve reconnect handling #1549 #1580
-
-- improve importing addresses #1544
-
-- improve configure and folder detection #1539 #1548
-
-- improve test suite #1559 #1564 #1580 #1581 #1582 #1584 #1588:
-
-- fix ad-hoc groups #1566
-
-- preventions against being marked as spam #1575
-
-- refactorings #1542 #1569
-
-
-## 1.34.0
-
-- new api for io, thread and event handling #1356,
-  see the example atop of `deltachat.h` to get an overview
-
-- LOTS of speed improvements due to async processing #1356
-
-- enable WAL mode for sqlite #1492
-
-- process incoming messages in bulk #1527
-
-- improve finding out the sent-folder #1488
-
-- several bug fixes
-
-
-## 1.33.0
-
-- let `dc_set_muted()` also mute one-to-one chats #1470
-
-- fix a bug that led to load and traffic if the server does not use sent-folder
-  #1472
-
-
-## 1.32.0
-
-- fix endless loop when trying to download messages with bad RFC Message-ID,
-  also be more reliable on similar errors #1463 #1466 #1462
-
-- fix bug with comma in contact request #1438
-
-- do not refer to hidden messages on replies #1459
-
-- improve error handling #1468 #1465 #1464
-
-
-## 1.31.0
-
-- always describe the context of the displayed error #1451
-
-- do not emit `DC_EVENT_ERROR` when message sending fails;
-  `dc_msg_get_state()` and `dc_get_msg_info()` are sufficient #1451
-
-- new config-option `media_quality` #1449
-
-- try over if writing message to database fails #1447
-
-
-## 1.30.0
-
-- expunge deleted messages #1440
-
-- do not send `DC_EVENT_MSGS_CHANGED|INCOMING_MSG` on hidden messages #1439
-
-
-## 1.29.0
-
-- new config options `delete_device_after` and `delete_server_after`,
-  each taking an amount of seconds after which messages
-  are deleted from the device and/or the server #1310 #1335 #1411 #1417 #1423
-
-- new api `dc_estimate_deletion_cnt()` to estimate the effect
-  of `delete_device_after` and `delete_server_after`
-
-- use Ed25519 keys by default, these keys are much shorter
-  than RSA keys, which results in saving traffic and speed improvements #1362
-
-- improve message ellipsizing #1397 #1430
-
-- emit `DC_EVENT_ERROR_NETWORK` also on smtp-errors #1378
-
-- do not show badly formatted non-delta-messages as empty #1384
-
-- try over SMTP on potentially recoverable error 5.5.0 #1379
-
-- remove device-chat from forward-to-chat-list #1367
-
-- improve group-handling #1368
-
-- `dc_get_info()` returns uptime (how long the context is in use)
-
-- python improvements and adaptions #1408 #1415
-
-- log to the stdout and stderr in tests #1416
-
-- refactoring, code improvements #1363 #1365 #1366 #1370 #1375 #1389 #1390 #1418 #1419
-
-- removed api: `dc_chat_get_subtitle()`, `dc_get_version_str()`, `dc_array_add_id()`
-
-- removed events: `DC_EVENT_MEMBER_ADDED`, `DC_EVENT_MEMBER_REMOVED`
-
-
-## 1.28.0
-
-- new flag DC_GCL_FOR_FORWARDING for dc_get_chatlist()
-  that will sort the "saved messages" chat to the top of the chatlist #1336
-- mark mails as being deleted from server in dc_empty_server() #1333
-- fix interaction with servers that do not allow folder creation on root-level;
-  use path separator as defined by the email server #1359
-- fix group creation if group was created by non-delta clients #1357
-- fix showing replies from non-delta clients #1353
-- fix member list on rejoining left groups #1343
-- fix crash when using empty groups #1354
-- fix potential crash on special names #1350
-
-
-## 1.27.0
-
-- handle keys reliably on armv7 #1327
-
-
-## 1.26.0
-
-- change generated key type back to RSA as shipped versions
-  have problems to encrypt to Ed25519 keys
-
-- update rPGP to encrypt reliably to Ed25519 keys;
-  one of the next versions can finally use Ed25519 keys then
-
-
-## 1.25.0
-
-- save traffic by downloading only messages that are really displayed #1236
-
-- change generated key type to Ed25519, these keys are much shorter
-  than RSA keys, which results in saving traffic and speed improvements #1287
-
-- improve key handling #1237 #1240 #1242 #1247
-
-- mute handling, apis are dc_set_chat_mute_duration()
-  dc_chat_is_muted() and dc_chat_get_remaining_mute_duration() #1143
-
-- pinning chats, new apis are dc_set_chat_visibility() and
-  dc_chat_get_visibility() #1248
-
-- add dc_provider_new_from_email() api that queries the new, integrated
-  provider-database #1207
-
-- account creation by scanning a qr code
-  in the DCACCOUNT scheme (https://mailadm.readthedocs.io),
-  new api is dc_set_config_from_qr() #1249
-
-- if possible, dc_join_securejoin(), returns the new chat-id immediately
-  and does the handshake in background #1225
-
-- update imap and smtp dependencies #1115
-
-- check for MOVE capability before using MOVE command #1263
-
-- allow inline attachments from RFC 2183 #1280
-
-- fix updating names from incoming mails #1298
-
-- fix error messages shown on import #1234
-
-- directly attempt to re-connect if the smtp connection is maybe stale #1296
-
-- improve adding group members #1291
-
-- improve rust-api #1261
-
-- cleanup #1302 #1283 #1282 #1276 #1270-#1274 #1267 #1258-#1260
-  #1257 #1239 #1231 #1224
-
-- update spec #1286 #1291
-
-
-## 1.0.0-beta.24
-
-- fix oauth2/gmail bug introduced in beta23 (not used in releases) #1219
-
-- fix panic when receiving eg. cyrillic filenames #1216
-
-- delete all consumed secure-join handshake messagess #1209 #1212
-
-- rust-level cleanups #1218 #1217 #1210 #1205
-
-- python-level cleanups #1204 #1202 #1201
-
-
-## 1.0.0-beta.23
-
-- #1197 fix imap-deletion of messages 
-
-- #1171 Combine multiple MDNs into a single mail, reducing traffic 
-
-- #1155 fix to not send out gossip always, reducing traffic
-
-- #1160 fix reply-to-encrypted determination 
-
-- #1182 Add "Auto-Submitted: auto-replied" header to MDNs
-
-- #1194 produce python wheels again, fix c/py.delta.chat
-  master-deployment 
-
-- rust-level housekeeping and improvements #1161 #1186 #1185 #1190 #1194 #1199 #1191 #1190 #1184 and more
-
-- #1063 clarify licensing 
-
-- #1147 use mailparse 0.10.2 
-
-
-## 1.0.0-beta.22
-
-- #1095 normalize email lineends to CRLF
-
-- #1095 enable link-time-optimization, saves eg. on android 11 mb
-
-- #1099 fix import regarding devicechats
-
-- #1092 improve logging
-
-- #1096 #1097 #1094 #1090 #1091 internal cleanups
-
-## 1.0.0-beta.21
-
-- #1078 #1082 ensure RFC compliance by producing 78 column lines for
-  encoded attachments. 
-
-- #1080 don't recreate and thus break group membership if an unknown 
-  sender (or mailer-daemon) sends a message referencing the group chat 
-
-- #1081 #1079 some internal cleanups 
-
-- update imap-proto dependency, to fix yandex/oauth 
-
-## 1.0.0-beta.20
-
-- #1074 fix OAUTH2/gmail
-- #1072 fix group members not appearing in contact list
-- #1071 never block interrupt_idle (thus hopefully also not on maybe_network())
-- #1069 reduce smtp-timeout to 30 seconds
-- #1066 #1065 avoid unwrap in dehtml, make literals more readable
-
-## 1.0.0-beta.19
-
-- #1058 timeout smtp-send if it doesn't complete in 15 minutes 
-
-- #1059 trim down logging
-
-## 1.0.0-beta.18
-
-- #1056 avoid panicking when we couldn't read imap-server's greeting
-  message 
-
-- #1055 avoid panicking when we don't have a selected folder
-
-- #1052 #1049 #1051 improve logging to add thread-id/name and
-  file/lineno to each info/warn message.
-
-- #1050 allow python bindings to initialize Account with "os_name".
-
-
 ## 1.0.0-beta.17
 
 - #1044 implement avatar recoding to 192x192 in core to keep file sizes small. 

Cargo.toml

@@ -1,78 +1,70 @@
 [package]
 name = "deltachat"
-version = "1.39.0"
+version = "1.0.0-beta.17"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
-license = "MPL-2.0"
-
-[profile.release]
-lto = true
+license = "MPL"
 
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
 
 libc = "0.2.51"
-pgp = { version = "0.6.0", default-features = false } 
+pgp = { version = "0.4.0", default-features = false }
 hex = "0.4.0"
-sha2 = "0.9.0"
+sha2 = "0.8.0"
 rand = "0.7.0"
 smallvec = "1.0.0"
-surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
+reqwest = { version = "0.9.15" }
 num-derive = "0.3.0"
 num-traits = "0.2.6"
-async-smtp = { version = "0.3" }
+async-smtp = { git = "https://github.com/async-email/async-smtp", branch = "master" }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
-lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
-async-imap = "0.3.1"
-async-native-tls = { version = "0.3.3" }
-async-std = { version = "1.6.1", features = ["unstable"] }
-base64 = "0.12"
+lettre_email = { git = "https://github.com/deltachat/lettre", branch = "native_tls" }
+
+# XXX newer commits of async-imap lead to import-export tests hanging
+async-imap = { git = "https://github.com/async-email/async-imap", branch = "dcc-stable" }
+
+async-native-tls = "0.1.1"
+async-std = { version = "1.0", features = ["unstable"] }
+base64 = "0.11"
 charset = "0.1"
 percent-encoding = "2.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 chrono = "0.4.6"
+failure = "0.1.5"
+failure_derive = "0.1.5"
 indexmap = "1.3.0"
-kamadak-exif = "0.5"
+# TODO: make optional
+rustyline = "4.1.0"
 lazy_static = "1.4.0"
 regex = "1.1.6"
-strum = "0.18.0"
-strum_macros = "0.18.0"
+rusqlite = { version = "0.20", features = ["bundled"] }
+r2d2_sqlite = "0.12.0"
+r2d2 = "0.8.5"
+strum = "0.16.0"
+strum_macros = "0.16.0"
+thread-local-object = "0.1.0"
 backtrace = "0.3.33"
 byteorder = "1.3.1"
 itertools = "0.8.0"
 image-meta = "0.1.0"
-quick-xml = "0.18.1"
+quick-xml = "0.17.1"
 escaper = "0.1.0"
 bitflags = "1.1.0"
+debug_stub_derive = "0.3.0"
 sanitize-filename = "0.2.1"
 stop-token = { version = "0.1.1", features = ["unstable"] }
-mailparse = "0.12.1"
+mailparse = "0.10.1"
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
 native-tls = "0.2.3"
-image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
-futures = "0.3.4"
-thiserror = "1.0.14"
-anyhow = "1.0.28"
-async-trait = "0.1.31"
-url = "2.1.1"
-async-std-resolver = "0.19.5"
-sqlx = { git = "https://github.com/launchbadge/sqlx", branch = "master", features = ["runtime-async-std", "sqlite", "macros"] }
-libsqlite3-sys = { version = "0.18", features = ["bundled", "min_sqlite_version_3_7_16"] }
-
-pretty_env_logger = { version = "0.4.0", optional = true }
-log = { version = "0.4.8", optional = true }
-rustyline = { version = "4.1.0", optional = true }
-ansi_term = { version = "0.12.1", optional = true }
+image = "0.22.3"
 
 [dev-dependencies]
 tempfile = "3.0"
 pretty_assertions = "0.6.1"
-pretty_env_logger = "0.4.0"
-proptest = "0.10"
-async-std = { version = "1.6.0", features = ["unstable", "attributes"] }
-smol = "0.1.11"
-log = "0.4.8"
+pretty_env_logger = "0.3.0"
+proptest = "0.9.4"
 
 [workspace]
 members = [
@@ -83,17 +75,14 @@ members = [
 [[example]]
 name = "simple"
 path = "examples/simple.rs"
-required-features = ["repl"]
 
 [[example]]
 name = "repl"
 path = "examples/repl/main.rs"
-required-features = ["repl"]
 
 
 [features]
-default = []
-internals = []
-repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term"]
-vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored"]
+default = ["nightly", "ringbuf"]
+vendored = ["native-tls/vendored", "reqwest/default-tls-vendored"]
 nightly = ["pgp/nightly"]
+ringbuf = ["pgp/ringbuf"]

LICENSE

@@ -2,6 +2,9 @@ The files in this directory and under its subdirectories
 are (c) 2019 by Bjoern Petersen and contributors and released under the
 Mozilla Public License Version 2.0, see below for a copy.
 
+NOTE that the files in the "libs" directory are copyrighted by third parties
+and come with their own respective licenses.
+
 Mozilla Public License Version 2.0
 ==================================
 

README.md

@@ -9,7 +9,7 @@
 To download and install the official compiler for the Rust programming language, and the Cargo package manager, run the command in your user environment:
 
 ```
-$ curl https://sh.rustup.rs -sSf | sh
+curl https://sh.rustup.rs -sSf | sh
 ```
 
 ## Using the CLI client
@@ -17,9 +17,8 @@ $ curl https://sh.rustup.rs -sSf | sh
 Compile and run Delta Chat Core command line utility, using `cargo`:
 
 ```
-$ RUST_LOG=info cargo run --example repl --features repl -- ~/deltachat-db
+cargo run --example repl -- /path/to/db
 ```
-where ~/deltachat-db is the database file. Delta Chat will create it if it does not exist.
 
 Configure your account (if not already configured):
 
@@ -109,29 +108,9 @@ $ cargo test -- --ignored
 
 - `vendored`: When using Openssl for TLS, this bundles a vendored version.
 - `nightly`: Enable nightly only performance and security related features.
+- `ringbuf`: Enable the use of [`slice_deque`](https://github.com/gnzlbg/slice_deque) in pgp.
 
 [circle-shield]: https://img.shields.io/circleci/project/github/deltachat/deltachat-core-rust/master.svg?style=flat-square
 [circle]: https://circleci.com/gh/deltachat/deltachat-core-rust/
 [appveyor-shield]: https://ci.appveyor.com/api/projects/status/lqpegel3ld4ipxj8/branch/master?style=flat-square
 [appveyor]: https://ci.appveyor.com/project/dignifiedquire/deltachat-core-rust/branch/master
-
-## Language bindings and frontend projects
-
-Language bindings are available for:
-
-- [C](https://c.delta.chat)
-- [Node.js](https://www.npmjs.com/package/deltachat-node)
-- [Python](https://py.delta.chat)
-- [Go](https://github.com/hugot/go-deltachat/)
-- [Free Pascal](https://github.com/deltachat/deltachat-fp/)
-- **Java** and **Swift** (contained in the Android/iOS repos)
-
-The following "frontend" projects make use of the Rust-library
-or its language bindings:
-
-- [Android](https://github.com/deltachat/deltachat-android)
-- [iOS](https://github.com/deltachat/deltachat-ios)
-- [Desktop](https://github.com/deltachat/deltachat-desktop)
-- [Pidgin](https://code.ur.gs/lupine/purple-plugin-delta/)
-- [Telepathy](https://code.ur.gs/lupine/telepathy-padfoot/)
-- several **Bots**

appveyor.yml

@@ -0,0 +1,19 @@
+environment:
+  matrix:
+    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
+
+install:
+  - appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
+  - rustup-init -yv --default-toolchain nightly-2019-07-10
+  - set PATH=%PATH%;%USERPROFILE%\.cargo\bin
+  - rustc -vV
+  - cargo -vV
+
+build: false
+
+test_script:
+  - cargo test --release --all
+
+cache:
+  - target
+  - C:\Users\appveyor\.cargo\registry

ci_scripts/ci_upload.sh

@@ -14,22 +14,24 @@ DOXYDOCDIR=${3:?directory where doxygen docs to be found}
 export BRANCH=${CIRCLE_BRANCH:?specify branch for uploading purposes}
 
 
-# python docs to py.delta.chat
-ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
-rsync -avz \
-  --delete \
-  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
-  "$PYDOCDIR/html/" \
-  delta@py.delta.chat:build/${BRANCH}
+# DISABLED: python docs to py.delta.chat
+#ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
+#rsync -avz \
+#  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+#  "$PYDOCDIR/html/" \
+#  delta@py.delta.chat:build/${BRANCH}
 
 # C docs to c.delta.chat
 ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@c.delta.chat mkdir -p build-c/${BRANCH}
 rsync -avz \
-  --delete \
   -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
   "$DOXYDOCDIR/html/" \
   delta@c.delta.chat:build-c/${BRANCH}
 
+exit 0
+
+# OUTDATED -- for re-use from python release-scripts 
+
 echo -----------------------
 echo upload wheels 
 echo -----------------------
@@ -37,7 +39,6 @@ echo -----------------------
 # Bundle external shared libraries into the wheels
 pushd $WHEELHOUSEDIR
 
-pip3 install -U pip setuptools
 pip3 install devpi-client
 devpi use https://m.devpi.net
 devpi login dc --password $DEVPI_LOGIN
@@ -49,9 +50,6 @@ devpi use dc/$N_BRANCH || {
     devpi use dc/$N_BRANCH
 }
 devpi index $N_BRANCH bases=/root/pypi
-devpi upload deltachat*
+devpi upload deltachat*.whl
 
 popd
-
-# remove devpi non-master dc indices if thy are too old
-python ci_scripts/cleanup_devpi_indices.py

ci_scripts/cleanup_devpi_indices.py

@@ -1,72 +0,0 @@
-"""
-Remove old "dc" indices except for master which always stays.
-
-"""
-from requests import Session
-import datetime
-import sys
-import subprocess
-
-MAXDAYS=7
-
-session = Session()
-session.headers["Accept"] = "application/json"
-
-
-def get_indexes(baseurl, username):
-    response = session.get(baseurl + username)
-    assert response.status_code == 200
-    result = response.json()["result"]
-    return result["indexes"]
-
-
-def get_projectnames(baseurl, username, indexname):
-    response = session.get(baseurl + username + "/" + indexname)
-    assert response.status_code == 200
-    result = response.json()["result"]
-    return result["projects"]
-
-
-def get_release_dates(baseurl, username, indexname, projectname):
-    response = session.get(baseurl + username + "/" + indexname + "/" + projectname)
-    assert response.status_code == 200
-    result = response.json()["result"]
-    dates = set()
-    for value in result.values():
-        if "+links" not in value:
-            continue
-        for link in value["+links"]:
-            for log in link["log"]:
-                dates.add(tuple(log["when"]))
-    return dates
-
-
-def run():
-    baseurl = "https://m.devpi.net/"
-    username = "dc"
-    for indexname in get_indexes(baseurl, username):
-        projectnames = get_projectnames(baseurl, username, indexname)
-        if indexname == "master" or not indexname:
-            continue
-        clear_index = not projectnames
-        for projectname in projectnames:
-            dates = get_release_dates(baseurl, username, indexname, projectname)
-            if not dates:
-                print(
-                    "%s has no releases" % (baseurl + username + "/" + indexname),
-                    file=sys.stderr)
-                date = datetime.datetime.now()
-            else:
-                date = datetime.datetime(*max(dates))
-            if (datetime.datetime.now() - date) > datetime.timedelta(days=MAXDAYS):
-                assert username and indexname
-                clear_index = True
-                break
-        if clear_index:
-            url = baseurl + username + "/" + indexname
-            subprocess.check_call(["devpi", "index", "-y", "--delete", url])
-
-
-
-if __name__ == '__main__':
-    run()

ci_scripts/docker-coredeps/Dockerfile

@@ -1,15 +1,18 @@
-FROM quay.io/pypa/manylinux2010_x86_64
+FROM quay.io/pypa/manylinux1_x86_64
 
 # Configure ld.so/ldconfig and pkg-config
 RUN echo /usr/local/lib64 > /etc/ld.so.conf.d/local.conf && \
     echo /usr/local/lib >> /etc/ld.so.conf.d/local.conf
 ENV PKG_CONFIG_PATH /usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
 
-# Install a recent Perl, needed to install the openssl crate 
+# Install a recent Perl, needed to install OpenSSL
 ADD deps/build_perl.sh /builder/build_perl.sh
-RUN rm /usr/bin/perl
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_perl.sh && cd .. && rm -r tmp1
 
+# Install OpenSSL
+ADD deps/build_openssl.sh /builder/build_openssl.sh
+RUN mkdir tmp1 && cd tmp1 && bash /builder/build_openssl.sh && cd .. && rm -r tmp1
+
 ENV PIP_DISABLE_PIP_VERSION_CHECK 1
 
 # Install python tools (auditwheels,tox, ...)

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -3,9 +3,6 @@
 set -e -x
 
 # Install Rust
-curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.43.1-x86_64-unknown-linux-gnu -y
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-09-12 -y
 export PATH=/root/.cargo/bin:$PATH
 rustc --version
-
-# remove some 300-400 MB that we don't need for automated builds
-rm -rf /root/.rustup/toolchains/1.43.1-x86_64-unknown-linux-gnu/share

ci_scripts/docker-coredeps/deps/run_all.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+#
+# Build the Delta Chat C/Rust library
+#
+set -e -x
+
+# perform clean build of core and install 
+export TOXWORKDIR=.docker-tox
+
+# build core library
+
+cargo build --release -p deltachat_ffi
+
+# configure access to a base python and 
+# to several python interpreters needed by tox below
+export PATH=$PATH:/opt/python/cp35-cp35m/bin
+export PYTHONDONTWRITEBYTECODE=1
+pushd /bin
+ln -s /opt/python/cp27-cp27m/bin/python2.7
+ln -s /opt/python/cp36-cp36m/bin/python3.6
+ln -s /opt/python/cp37-cp37m/bin/python3.7
+popd
+
+#
+# run python tests
+#
+
+if [ -n "$TESTS" ]; then 
+
+    echo ----------------
+    echo run python tests
+    echo ----------------
+
+    pushd python 
+    # first run all tests ...
+    rm -rf tests/__pycache__
+    rm -rf src/deltachat/__pycache__
+    export PYTHONDONTWRITEBYTECODE=1
+    tox --workdir "$TOXWORKDIR" -e py27,py35,py36,py37
+    popd
+fi
+
+
+if [ -n "$DOCS" ]; then 
+    echo -----------------------
+    echo generating python docs
+    echo -----------------------
+    (cd python && tox --workdir "$TOXWORKDIR" -e doc) 
+fi

ci_scripts/old/run-python.sh

@@ -46,7 +46,6 @@ if [ -n "$TESTS" ]; then
     tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "not qr"
     tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "qr"
     unset DCC_PY_LIVECONFIG
-    unset DCC_NEW_TMP_EMAIL
     tox --workdir "$TOXWORKDIR" -p4 -e lint,py35,py36,doc
     tox --workdir "$TOXWORKDIR" -e auditwheels
     popd

ci_scripts/remote_python_packaging.sh

@@ -1,51 +0,0 @@
-#!/bin/bash 
-
-export BRANCH=${CIRCLE_BRANCH:?branch to build}
-export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
-export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
-
-# we construct the BUILDDIR such that we can easily share the
-# CARGO_TARGET_DIR between runs ("..")
-export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
-
-echo "--- Copying files to $SSHTARGET:$BUILDDIR"
-
-set -xe
-
-ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
-git ls-files >.rsynclist 
-# we seem to need .git for setuptools_scm versioning 
-find .git >>.rsynclist
-rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
-
-set +x
-
-# we have to create a remote file for the remote-docker run 
-# so we can do a simple ssh command with a TTY
-# so that when our job dies, all container-runs are aborted.
-# sidenote: the circle-ci machinery will kill ongoing jobs
-# if there are new commits and we want to ensure that 
-# everything is terminated/cleaned up and we have no orphaned
-# useless still-running docker-containers consuming resources. 
-
-ssh $SSHTARGET bash -c "cat >$BUILDDIR/exec_docker_run" <<_HERE
-    set +x -e
-    cd $BUILDDIR
-    export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
-    export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
-    set -x
-
-    # run everything else inside docker 
-    docker run -e DCC_NEW_TMP_EMAIL -e DCC_PY_LIVECONFIG \
-       --rm -it -v \$(pwd):/mnt -w /mnt \
-       deltachat/coredeps ci_scripts/run_all.sh
-
-_HERE
-
-echo "--- Running $CIRCLE_JOB remotely"
-
-ssh -t $SSHTARGET bash "$BUILDDIR/exec_docker_run"
-mkdir -p workspace 
-rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/wheelhouse/*manylinux201*" workspace/wheelhouse/
-rsync -avz "$SSHTARGET:$BUILDDIR/python/.docker-tox/dist/*" workspace/wheelhouse/
-rsync -avz "$SSHTARGET:$BUILDDIR/python/doc/_build/" workspace/py-docs

ci_scripts/remote_tests_python.sh

@@ -30,7 +30,6 @@ ssh $SSHTARGET <<_HERE
     export CARGO_TARGET_DIR=\`pwd\`/../target
     export TARGET=release
     export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
-    export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
 
     #we rely on tox/virtualenv being available in the host
     #rm -rf virtualenv venv

ci_scripts/run-doxygen.sh

@@ -3,4 +3,5 @@
 set -ex
 
 cd deltachat-ffi 
-PROJECT_NUMBER=$(git log -1 --format "%h (%cd)") doxygen 
+doxygen 
+

ci_scripts/run-python-test.sh

@@ -6,7 +6,7 @@
 set -e -x
 
 # for core-building and python install step
-export DCC_RS_TARGET=debug
+export DCC_RS_TARGET=release
 export DCC_RS_DEV=`pwd`
 
 cd python

ci_scripts/run_all.sh

@@ -1,50 +0,0 @@
-#!/bin/bash
-#
-# Build the Delta Chat Core Rust library, Python wheels and docs 
-
-set -e -x
-
-# Perform clean build of core and install.
-export TOXWORKDIR=.docker-tox
-
-# compile core lib
-
-export PATH=/root/.cargo/bin:$PATH
-cargo build --release -p deltachat_ffi
-# cargo test --all --all-features
-
-# Statically link against libdeltachat.a.
-export DCC_RS_DEV=$(pwd)
-
-# Configure access to a base python and to several python interpreters
-# needed by tox below.
-export PATH=$PATH:/opt/python/cp35-cp35m/bin
-export PYTHONDONTWRITEBYTECODE=1
-pushd /bin
-rm -f python3.5
-ln -s /opt/python/cp35-cp35m/bin/python3.5
-ln -s /opt/python/cp36-cp36m/bin/python3.6
-ln -s /opt/python/cp37-cp37m/bin/python3.7
-ln -s /opt/python/cp38-cp38/bin/python3.8
-popd
-
-pushd python
-# prepare a clean tox run
-rm -rf tests/__pycache__
-rm -rf src/deltachat/__pycache__
-mkdir -p $TOXWORKDIR
-
-# disable live-account testing to speed up test runs and wheel building
-# XXX we may switch on some live-tests on for better ensurances 
-# Note that the independent remote_tests_python step does all kinds of
-# live-testing already. 
-unset DCC_PY_LIVECONFIG
-unset DCC_NEW_TMP_EMAIL
-tox --workdir "$TOXWORKDIR" -e py35,py36,py37,py38,auditwheels
-popd
-
-
-echo -----------------------
-echo generating python docs
-echo -----------------------
-(cd python && tox --workdir "$TOXWORKDIR" -e doc)

deltachat-ffi/Cargo.toml

@@ -1,11 +1,11 @@
 [package]
 name = "deltachat_ffi"
-version = "1.39.0"
+version = "1.0.0-beta.17"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 readme = "README.md"
-license = "MPL-2.0"
+license = "MIT OR Apache-2.0"
 
 keywords = ["deltachat", "chat", "openpgp", "email", "encryption"]
 categories = ["cryptography", "std", "email"]
@@ -16,16 +16,14 @@ crate-type = ["cdylib", "staticlib"]
 
 [dependencies]
 deltachat = { path = "../", default-features = false }
+deltachat-provider-database = "0.2.1"
 libc = "0.2"
 human-panic = "1.0.1"
 num-traits = "0.2.6"
-serde_json = "1.0"
-async-std = "1.6.0"
-anyhow = "1.0.28"
-thiserror = "1.0.14"
+failure = "0.1.6"
 
 [features]
-default = ["vendored"]
+default = ["vendored", "nightly", "ringbuf"]
 vendored = ["deltachat/vendored"]
 nightly = ["deltachat/nightly"]
-
+ringbuf = ["deltachat/ringbuf"]

deltachat-ffi/src/dc_array.rs

@@ -2,6 +2,7 @@ use crate::location::Location;
 
 /* * the structure behind dc_array_t */
 #[derive(Debug, Clone)]
+#[allow(non_camel_case_types)]
 pub enum dc_array_t {
     Locations(Vec<Location>),
     Uint(Vec<u32>),

deltachat-ffi/src/providers.rs

@@ -0,0 +1,93 @@
+extern crate deltachat_provider_database;
+
+use std::ptr;
+
+use crate::string::{to_string_lossy, StrExt};
+use deltachat_provider_database::StatusState;
+
+#[no_mangle]
+pub type dc_provider_t = deltachat_provider_database::Provider;
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_provider_new_from_domain(
+    domain: *const libc::c_char,
+) -> *const dc_provider_t {
+    match deltachat_provider_database::get_provider_info(&to_string_lossy(domain)) {
+        Some(provider) => provider,
+        None => ptr::null(),
+    }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_provider_new_from_email(
+    email: *const libc::c_char,
+) -> *const dc_provider_t {
+    let email = to_string_lossy(email);
+    let domain = deltachat_provider_database::get_domain_from_email(&email);
+    match deltachat_provider_database::get_provider_info(domain) {
+        Some(provider) => provider,
+        None => ptr::null(),
+    }
+}
+
+macro_rules! null_guard {
+    ($context:tt) => {
+        if $context.is_null() {
+            return ptr::null_mut() as *mut libc::c_char;
+        }
+    };
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_provider_get_overview_page(
+    provider: *const dc_provider_t,
+) -> *mut libc::c_char {
+    null_guard!(provider);
+    format!(
+        "{}/{}",
+        deltachat_provider_database::PROVIDER_OVERVIEW_URL,
+        (*provider).overview_page
+    )
+    .strdup()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_provider_get_name(provider: *const dc_provider_t) -> *mut libc::c_char {
+    null_guard!(provider);
+    (*provider).name.strdup()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_provider_get_markdown(
+    provider: *const dc_provider_t,
+) -> *mut libc::c_char {
+    null_guard!(provider);
+    (*provider).markdown.strdup()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_provider_get_status_date(
+    provider: *const dc_provider_t,
+) -> *mut libc::c_char {
+    null_guard!(provider);
+    (*provider).status.date.strdup()
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_provider_get_status(provider: *const dc_provider_t) -> u32 {
+    if provider.is_null() {
+        return 0;
+    }
+    match (*provider).status.state {
+        StatusState::OK => 1,
+        StatusState::PREPARATION => 2,
+        StatusState::BROKEN => 3,
+    }
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn dc_provider_unref(_provider: *const dc_provider_t) {
+    ()
+}
+
+// TODO expose general provider overview url?

deltachat-ffi/src/string.rs

@@ -1,5 +1,5 @@
+use failure::Fail;
 use std::ffi::{CStr, CString};
-use std::ptr;
 
 /// Duplicates a string
 ///
@@ -8,7 +8,7 @@ use std::ptr;
 /// # Examples
 ///
 /// ```rust,norun
-/// use crate::string::{dc_strdup, to_string_lossy};
+/// use deltachat::dc_tools::{dc_strdup, to_string_lossy};
 /// unsafe {
 ///     let str_a = b"foobar\x00" as *const u8 as *const libc::c_char;
 ///     let str_a_copy = dc_strdup(str_a);
@@ -16,7 +16,7 @@ use std::ptr;
 ///     assert_ne!(str_a, str_a_copy);
 /// }
 /// ```
-unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char {
+pub unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char {
     let ret: *mut libc::c_char;
     if !s.is_null() {
         ret = libc::strdup(s);
@@ -30,13 +30,13 @@ unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char {
 }
 
 /// Error type for the [OsStrExt] trait
-#[derive(Debug, PartialEq, thiserror::Error)]
-pub(crate) enum CStringError {
+#[derive(Debug, Fail, PartialEq)]
+pub enum CStringError {
     /// The string contains an interior null byte
-    #[error("String contains an interior null byte")]
+    #[fail(display = "String contains an interior null byte")]
     InteriorNullByte,
     /// The string is not valid Unicode
-    #[error("String is not valid unicode")]
+    #[fail(display = "String is not valid unicode")]
     NotUnicode,
 }
 
@@ -65,7 +65,7 @@ pub(crate) enum CStringError {
 ///     let mut c_ptr: *mut libc::c_char = dc_strdup(path_c.as_ptr());
 /// }
 /// ```
-pub(crate) trait OsStrExt {
+pub trait OsStrExt {
     /// Convert a  [std::ffi::OsStr] to an [std::ffi::CString]
     ///
     /// This is useful to convert e.g. a [std::path::Path] to
@@ -130,28 +130,27 @@ fn os_str_to_c_string_unicode(
 }
 
 /// Convenience methods/associated functions for working with [CString]
-trait CStringExt {
-    /// Create a new [CString], best effort
+///
+/// This is helps transitioning from unsafe code.
+pub trait CStringExt {
+    /// Create a new [CString], yolo style
     ///
-    /// Like the [to_string_lossy] this doesn't give up in the face of
-    /// bad input (embedded null bytes in this case) instead it does
-    /// the best it can by stripping the embedded null bytes.
-    fn new_lossy<T: Into<Vec<u8>>>(t: T) -> CString {
-        let mut s = t.into();
-        s.retain(|&c| c != 0);
-        CString::new(s).unwrap_or_default()
+    /// This unwrap the result, panicking when there are embedded NULL
+    /// bytes.
+    fn yolo<T: Into<Vec<u8>>>(t: T) -> CString {
+        CString::new(t).expect("String contains null byte, can not be CString")
     }
 }
 
 impl CStringExt for CString {}
 
-/// Convenience methods to turn strings into C strings.
+/// Convenience methods to make transitioning from raw C strings easier.
 ///
 /// To interact with (legacy) C APIs we often need to convert from
 /// Rust strings to raw C strings.  This can be clumsy to do correctly
 /// and the compiler sometimes allows it in an unsafe way.  These
 /// methods make it more succinct and help you get it right.
-pub(crate) trait Strdup {
+pub trait StrExt {
     /// Allocate a new raw C `*char` version of this string.
     ///
     /// This allocates a new raw C string which must be freed using
@@ -168,52 +167,14 @@ pub(crate) trait Strdup {
     unsafe fn strdup(&self) -> *mut libc::c_char;
 }
 
-impl<T: AsRef<str>> Strdup for T {
+impl<T: AsRef<str>> StrExt for T {
     unsafe fn strdup(&self) -> *mut libc::c_char {
-        let tmp = CString::new_lossy(self.as_ref());
+        let tmp = CString::yolo(self.as_ref());
         dc_strdup(tmp.as_ptr())
     }
 }
 
-// We can not implement for AsRef<OsStr> because we already implement
-// AsRev<str> and this conflicts.  So implement for Path directly.
-impl Strdup for std::path::Path {
-    unsafe fn strdup(&self) -> *mut libc::c_char {
-        let tmp = self.to_c_string().unwrap_or_else(|_| CString::default());
-        dc_strdup(tmp.as_ptr())
-    }
-}
-
-/// Convenience methods to turn optional strings into C strings.
-///
-/// This is the same as the [Strdup] trait but a different trait name
-/// to work around the type system not allowing to implement [Strdup]
-/// for `Option<impl Strdup>` When we already have an [Strdup] impl
-/// for `AsRef<&str>`.
-///
-/// When the [Option] is [Option::Some] this behaves just like
-/// [Strdup::strdup], when it is [Option::None] a null pointer is
-/// returned.
-pub(crate) trait OptStrdup {
-    /// Allocate a new raw C `*char` version of this string, or NULL.
-    ///
-    /// See [Strdup::strdup] for details.
-    unsafe fn strdup(&self) -> *mut libc::c_char;
-}
-
-impl<T: AsRef<str>> OptStrdup for Option<T> {
-    unsafe fn strdup(&self) -> *mut libc::c_char {
-        match self {
-            Some(s) => {
-                let tmp = CString::new_lossy(s.as_ref());
-                dc_strdup(tmp.as_ptr())
-            }
-            None => ptr::null_mut(),
-        }
-    }
-}
-
-pub(crate) fn to_string_lossy(s: *const libc::c_char) -> String {
+pub fn to_string_lossy(s: *const libc::c_char) -> String {
     if s.is_null() {
         return "".into();
     }
@@ -223,7 +184,7 @@ pub(crate) fn to_string_lossy(s: *const libc::c_char) -> String {
     cstr.to_string_lossy().to_string()
 }
 
-pub(crate) fn to_opt_string_lossy(s: *const libc::c_char) -> Option<String> {
+pub fn to_opt_string_lossy(s: *const libc::c_char) -> Option<String> {
     if s.is_null() {
         return None;
     }
@@ -244,7 +205,7 @@ pub(crate) fn to_opt_string_lossy(s: *const libc::c_char) -> Option<String> {
 ///
 /// [Path]: std::path::Path
 #[cfg(not(target_os = "windows"))]
-pub(crate) fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path {
+pub fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path {
     assert!(!s.is_null(), "cannot be used on null pointers");
     use std::os::unix::ffi::OsStrExt;
     unsafe {
@@ -256,7 +217,7 @@ pub(crate) fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path {
 
 // as_path() implementation for windows, documented above.
 #[cfg(target_os = "windows")]
-pub(crate) fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path {
+pub fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path {
     as_path_unicode(s)
 }
 
@@ -363,14 +324,8 @@ mod tests {
     }
 
     #[test]
-    fn test_cstring_new_lossy() {
-        assert!(CString::new("hel\x00lo").is_err());
-        assert!(CString::new(String::from("hel\x00o")).is_err());
-        let r = CString::new("hello").unwrap();
-        assert_eq!(CString::new_lossy("hello"), r);
-        assert_eq!(CString::new_lossy("hel\x00lo"), r);
-        assert_eq!(CString::new_lossy(String::from("hello")), r);
-        assert_eq!(CString::new_lossy(String::from("hel\x00lo")), r);
+    fn test_cstring_yolo() {
+        assert_eq!(CString::new("hello").unwrap(), CString::yolo("hello"));
     }
 
     #[test]
@@ -392,19 +347,4 @@ mod tests {
             assert_eq!(cmp, 0);
         }
     }
-
-    #[test]
-    fn test_strdup_opt_string() {
-        unsafe {
-            let s = Some("hello");
-            let c = s.strdup();
-            let cmp = strcmp(c, b"hello\x00" as *const u8 as *const libc::c_char);
-            free(c as *mut libc::c_void);
-            assert_eq!(cmp, 0);
-
-            let s: Option<&str> = None;
-            let c = s.strdup();
-            assert_eq!(c, ptr::null_mut());
-        }
-    }
 }

deltachat_derive/Cargo.toml

@@ -1,13 +1,12 @@
 [package]
 name = "deltachat_derive"
-version = "2.0.0"
-authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
+version = "0.1.0"
+authors = ["Dmitry Bogatov <KAction@debian.org>"]
 edition = "2018"
-license = "MPL-2.0"
 
 [lib]
 proc-macro = true
 
 [dependencies]
-syn = "1.0.13"
-quote = "1.0.2"
+syn = "0.14.4"
+quote = "0.6.3"

deltachat_derive/src/lib.rs

@@ -3,38 +3,42 @@ extern crate proc_macro;
 
 use crate::proc_macro::TokenStream;
 use quote::quote;
+use syn;
 
 // For now, assume (not check) that these macroses are applied to enum without
 // data.  If this assumption is violated, compiler error will point to
 // generated code, which is not very user-friendly.
 
-#[proc_macro_derive(Sqlx)]
-pub fn sqlx_derive(input: TokenStream) -> TokenStream {
+#[proc_macro_derive(ToSql)]
+pub fn to_sql_derive(input: TokenStream) -> TokenStream {
     let ast: syn::DeriveInput = syn::parse(input).unwrap();
     let name = &ast.ident;
 
     let gen = quote! {
-        impl<'q> sqlx::encode::Encode<'q, sqlx::sqlite::Sqlite> for #name {
-            fn encode_by_ref(&self, buf: &mut Vec<sqlx::sqlite::SqliteArgumentValue<'q>>) -> sqlx::encode::IsNull {
-                num_traits::ToPrimitive::to_i32(self).expect("invalid type").encode(buf)
+        impl rusqlite::types::ToSql for #name {
+            fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
+                let num = *self as i64;
+                let value = rusqlite::types::Value::Integer(num);
+                let output = rusqlite::types::ToSqlOutput::Owned(value);
+                std::result::Result::Ok(output)
+            }
+        }
+    };
+    gen.into()
+}
+
+#[proc_macro_derive(FromSql)]
+pub fn from_sql_derive(input: TokenStream) -> TokenStream {
+    let ast: syn::DeriveInput = syn::parse(input).unwrap();
+    let name = &ast.ident;
+
+    let gen = quote! {
+        impl rusqlite::types::FromSql for #name {
+            fn column_result(col: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
+                let inner = rusqlite::types::FromSql::column_result(col)?;
+                Ok(num_traits::FromPrimitive::from_i64(inner).unwrap_or_default())
             }
         }
-
-
-        impl<'de> sqlx::decode::Decode<'de, sqlx::sqlite::Sqlite> for #name {
-            fn decode(value: sqlx::sqlite::SqliteValueRef) -> std::result::Result<Self, Box<dyn std::error::Error + 'static + Send + Sync>> {
-                let raw: i32 = sqlx::decode::Decode::decode(value)?;
-
-                Ok(num_traits::FromPrimitive::from_i32(raw).unwrap_or_default())
-            }
-        }
-
-        impl sqlx::types::Type<sqlx::sqlite::Sqlite> for #name {
-            fn type_info() -> sqlx::sqlite::SqliteTypeInfo {
-                <i32 as sqlx::types::Type<_>>::type_info()
-            }
-        }
-
     };
     gen.into()
 }

draft/group-sync.rst

@@ -1,126 +0,0 @@
-
-Problem: missing eventual group consistency
---------------------------------------------
-
-If group members are concurrently adding new members,
-the new members will miss each other's additions, example:
-
-- Alice and Bob are in a two-member group
-
-- Alice adds Carol, concurrently Bob adds Doris
-
-- Carol will see a three-member group (Alice, Bob, Carol),
-  Doris will see a different three-member group (Alice, Bob, Doris),
-  and only Alice and Bob will have all four members.
-
-Note that for verified groups any mitigation mechanism likely
-needs to make all clients to know who originally added a member.
-
-
-solution: memorize+attach (possible encrypted) chat-meta mime messages
-----------------------------------------------------------------------
-
-For reference, please see https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#add-and-remove-members how MemberAdded/Removed messages are shaped.
-
-
-- All Chat-Group-Member-Added/Removed messages are recorded in their
-  full raw (signed and encrypted) mime-format in the DB
-
-- If an incoming member-add/member-delete messages has a member list
-  which is, apart from the added/removed member, not consistent
-  with our own view, broadcast a "Chat-Group-Member-Correction" message to
-  all members, attaching the original added/removed mime-message for all mismatching
-  contacts.  If we have no relevant add/del information, don't send a
-  correction message out.
-
-- Upong receiving added/removed attachments we don't do the
-  check_consistency+correction message dance.
-  This avoids recursion problems and hard-to-reason-about chatter.
-
-Notes:
-
-- mechanism works for both encrypted and unencrypted add/del messages
-
-- we already have a "mime_headers" column in the DB for each incoming message.
-  We could extend it to also include the payload and store mime unconditionally
-  for member-added/removed messages.
-
-- multiple member-added/removed messages can be attached in a single
-  correction message
-
-- it is minimal on the number of overall messages to reach group consistency
-  (best-case: no extra messages, the ABCD case above: max two extra messages)
-
-- somewhat backward compatible: older clients will probably ignore
-  messages which are signed by someone who is not the outer From-address.
-
-- the correction-protocol also helps with dropped messages.  If a member
-  did not see a member-added/removed message, the next member add/removed
-  message in the group will likely heal group consistency for this member.
-
-- we can quite easily extend the mechanism to also provide the group-avatar or
-  other meta-information.
-
-Discussions of variants
-++++++++++++++++++++++++
-
-- instead of acting on MemberAdded/Removed message we could send
-  corrections for any received message that addresses inconsistent group members but
-  a) this would delay group-membership healing
-  b) could lead to a lot of members sending corrections
-
-- instead of broadcasting correction messages we could only send it to
-  the sender of the inconsistent member-added/removed message.
-  A receiver of such a correction message would then need to forward
-  the message to the members it thinks also have an inconsistent view.
-  This sounds complicated and error-prone.  Concretely, if Alice
-  receives Bob's "Member-added: Doris" message, then Alice
-  broadcasting the correction message with "Member-added: Carol"
-  would reach all four members, healing group consistency in one step.
-  If Bob meanwhile receives Alice's "Member-Added: Carol" message,
-  Bob would broadcast a correction message to all four members as well.
-  (Imagine a situation where Alice/Bob added Carol/Doris
-  while both being in an offline or bad-connection situation).
-
-
-solution2: repeat member-added/removed messages
----------------------------------------------------
-
-Introduce a new Chat-Group-Member-Changed header and deprecate Chat-Group-Member-Added/Removed
-but keep sending out the old headers until the new protocol is sufficiently deployed.
-
-The new Chat-Group-Member-Changed header contains a Time-to-Live number (TTL)
-which controls repetition of the signed "add/del e-mail address" payload.
-
-Example::
-
-    Chat-Group-Member-Changed: TTL add "somedisplayname" someone@example.org
-        owEBYQGe/pANAwACAY47A6J5t3LWAcsxYgBeTQypYWRkICJzb21lZGlzcGxheW5h
-        bWUiIHNvbWVvbmVAZXhhbXBsZS5vcmcgCokBHAQAAQIABgUCXk0MqQAKCRCOOwOi
-        ebdy1hfRB/wJ74tgFQulicthcv9n+ZsqzwOtBKMEVIHqJCzzDB/Hg/2z8ogYoZNR
-        iUKKrv3Y1XuFvdKyOC+wC/unXAWKFHYzY6Tv6qDp6r+amt+ad+8Z02q53h9E55IP
-        FUBdq2rbS8hLGjQB+mVRowYrUACrOqGgNbXMZjQfuV7fSc7y813OsCQgi3tjstup
-        b+uduVzxCp3PChGhcZPs3iOGCnQvSB8VAaLGMWE2d7nTo/yMQ0Jx69x5qwfXogTk
-        mTt5rOJyrosbtf09TMKFzGgtqBcEqHLp3+mQpZQ+WHUKAbsRa8Jc9DOUOSKJ8SNM
-        clKdskprY+4LY0EBwLD3SQ7dPkTITCRD
-        =P6GG
-
-TTL is set to "2" on an initial Chat-Group-Member-Changed add/del message.
-Receivers will apply the add/del change to the group-membership,
-decrease the TTL by 1, and if TTL>0 re-sent the header.
-
-The "add|del e-mail address" payload is pgp-signed and repeated verbatim.
-This allows to propagate, in a cryptographically secured way,
-who added a member. This is particularly important for allowing
-to show in verified groups who added a member (planned).
-
-Disadvantage to solution 1:
-
-- requires to specify encoding and precise rules for what/how is signed.
-
-- causes O(N^2) extra messages
-
-- Not easily extendable for other things (without introducing a new
-  header / encoding)
-
-

examples/repl/main.rs

@@ -6,21 +6,27 @@
 
 #[macro_use]
 extern crate deltachat;
+#[macro_use]
+extern crate failure;
+#[macro_use]
+extern crate lazy_static;
+#[macro_use]
+extern crate rusqlite;
 
 use std::borrow::Cow::{self, Borrowed, Owned};
 use std::io::{self, Write};
+use std::path::Path;
 use std::process::Command;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, Mutex, RwLock};
 
-use ansi_term::Color;
-use anyhow::{bail, Error};
-use async_std::path::Path;
-use deltachat::chat::ChatId;
 use deltachat::config;
+use deltachat::configure::*;
 use deltachat::context::*;
+use deltachat::job::*;
 use deltachat::oauth2::*;
 use deltachat::securejoin::*;
 use deltachat::Event;
-use log::{error, info, warn};
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
 use rustyline::config::OutputStreamType;
 use rustyline::error::ReadlineError;
@@ -33,81 +39,179 @@ use rustyline::{
 mod cmdline;
 use self::cmdline::*;
 
-/// Event Handler
-fn receive_event(event: Event) {
-    let yellow = Color::Yellow.normal();
+// Event Handler
+
+fn receive_event(_context: &Context, event: Event) -> libc::uintptr_t {
     match event {
         Event::Info(msg) => {
             /* do not show the event as this would fill the screen */
-            info!("{}", msg);
+            println!("{}", msg);
         }
         Event::SmtpConnected(msg) => {
-            info!("[SMTP_CONNECTED] {}", msg);
+            println!("[DC_EVENT_SMTP_CONNECTED] {}", msg);
         }
         Event::ImapConnected(msg) => {
-            info!("[IMAP_CONNECTED] {}", msg);
+            println!("[DC_EVENT_IMAP_CONNECTED] {}", msg);
         }
         Event::SmtpMessageSent(msg) => {
-            info!("[SMTP_MESSAGE_SENT] {}", msg);
+            println!("[DC_EVENT_SMTP_MESSAGE_SENT] {}", msg);
         }
         Event::Warning(msg) => {
-            warn!("{}", msg);
+            println!("[Warning] {}", msg);
         }
         Event::Error(msg) => {
-            error!("{}", msg);
+            println!("\x1b[31m[DC_EVENT_ERROR] {}\x1b[0m", msg);
         }
         Event::ErrorNetwork(msg) => {
-            error!("[NETWORK] msg={}", msg);
+            println!("\x1b[31m[DC_EVENT_ERROR_NETWORK] msg={}\x1b[0m", msg);
         }
         Event::ErrorSelfNotInGroup(msg) => {
-            error!("[SELF_NOT_IN_GROUP] {}", msg);
+            println!("\x1b[31m[DC_EVENT_ERROR_SELF_NOT_IN_GROUP] {}\x1b[0m", msg);
         }
         Event::MsgsChanged { chat_id, msg_id } => {
-            info!(
-                "{}",
-                yellow.paint(format!(
-                    "Received MSGS_CHANGED(chat_id={}, msg_id={})",
-                    chat_id, msg_id,
-                ))
+            print!(
+                "\x1b[33m{{Received DC_EVENT_MSGS_CHANGED(chat_id={}, msg_id={})}}\n\x1b[0m",
+                chat_id, msg_id,
             );
         }
         Event::ContactsChanged(_) => {
-            info!("{}", yellow.paint("Received CONTACTS_CHANGED()"));
+            print!("\x1b[33m{{Received DC_EVENT_CONTACTS_CHANGED()}}\n\x1b[0m");
         }
         Event::LocationChanged(contact) => {
-            info!(
-                "{}",
-                yellow.paint(format!("Received LOCATION_CHANGED(contact={:?})", contact))
+            print!(
+                "\x1b[33m{{Received DC_EVENT_LOCATION_CHANGED(contact={:?})}}\n\x1b[0m",
+                contact,
             );
         }
         Event::ConfigureProgress(progress) => {
-            info!(
-                "{}",
-                yellow.paint(format!("Received CONFIGURE_PROGRESS({} ‰)", progress))
+            print!(
+                "\x1b[33m{{Received DC_EVENT_CONFIGURE_PROGRESS({} ‰)}}\n\x1b[0m",
+                progress,
             );
         }
         Event::ImexProgress(progress) => {
-            info!(
-                "{}",
-                yellow.paint(format!("Received IMEX_PROGRESS({} ‰)", progress))
+            print!(
+                "\x1b[33m{{Received DC_EVENT_IMEX_PROGRESS({} ‰)}}\n\x1b[0m",
+                progress,
             );
         }
         Event::ImexFileWritten(file) => {
-            info!(
-                "{}",
-                yellow.paint(format!("Received IMEX_FILE_WRITTEN({})", file.display()))
+            print!(
+                "\x1b[33m{{Received DC_EVENT_IMEX_FILE_WRITTEN({})}}\n\x1b[0m",
+                file.display()
             );
         }
         Event::ChatModified(chat) => {
-            info!(
-                "{}",
-                yellow.paint(format!("Received CHAT_MODIFIED({})", chat))
+            print!(
+                "\x1b[33m{{Received DC_EVENT_CHAT_MODIFIED({})}}\n\x1b[0m",
+                chat
             );
         }
         _ => {
-            info!("Received {:?}", event);
+            print!("\x1b[33m{{Received {:?}}}\n\x1b[0m", event);
         }
     }
+
+    0
+}
+
+// Threads for waiting for messages and for jobs
+
+lazy_static! {
+    static ref HANDLE: Arc<Mutex<Option<Handle>>> = Arc::new(Mutex::new(None));
+    static ref IS_RUNNING: AtomicBool = AtomicBool::new(true);
+}
+
+struct Handle {
+    handle_imap: Option<std::thread::JoinHandle<()>>,
+    handle_mvbox: Option<std::thread::JoinHandle<()>>,
+    handle_sentbox: Option<std::thread::JoinHandle<()>>,
+    handle_smtp: Option<std::thread::JoinHandle<()>>,
+}
+
+macro_rules! while_running {
+    ($code:block) => {
+        if IS_RUNNING.load(Ordering::Relaxed) {
+            $code
+        } else {
+            break;
+        }
+    };
+}
+
+fn start_threads(c: Arc<RwLock<Context>>) {
+    if HANDLE.clone().lock().unwrap().is_some() {
+        return;
+    }
+
+    println!("Starting threads");
+    IS_RUNNING.store(true, Ordering::Relaxed);
+
+    let ctx = c.clone();
+    let handle_imap = std::thread::spawn(move || loop {
+        while_running!({
+            perform_inbox_jobs(&ctx.read().unwrap());
+            perform_inbox_fetch(&ctx.read().unwrap());
+            while_running!({
+                let context = ctx.read().unwrap();
+                perform_inbox_idle(&context);
+            });
+        });
+    });
+
+    let ctx = c.clone();
+    let handle_mvbox = std::thread::spawn(move || loop {
+        while_running!({
+            perform_mvbox_fetch(&ctx.read().unwrap());
+            while_running!({
+                perform_mvbox_idle(&ctx.read().unwrap());
+            });
+        });
+    });
+
+    let ctx = c.clone();
+    let handle_sentbox = std::thread::spawn(move || loop {
+        while_running!({
+            perform_sentbox_fetch(&ctx.read().unwrap());
+            while_running!({
+                perform_sentbox_idle(&ctx.read().unwrap());
+            });
+        });
+    });
+
+    let ctx = c;
+    let handle_smtp = std::thread::spawn(move || loop {
+        while_running!({
+            perform_smtp_jobs(&ctx.read().unwrap());
+            while_running!({
+                perform_smtp_idle(&ctx.read().unwrap());
+            });
+        });
+    });
+
+    *HANDLE.clone().lock().unwrap() = Some(Handle {
+        handle_imap: Some(handle_imap),
+        handle_mvbox: Some(handle_mvbox),
+        handle_sentbox: Some(handle_sentbox),
+        handle_smtp: Some(handle_smtp),
+    });
+}
+
+fn stop_threads(context: &Context) {
+    if let Some(ref mut handle) = *HANDLE.clone().lock().unwrap() {
+        println!("Stopping threads");
+        IS_RUNNING.store(false, Ordering::Relaxed);
+
+        interrupt_inbox_idle(context, true);
+        interrupt_mvbox_idle(context);
+        interrupt_sentbox_idle(context);
+        interrupt_smtp_idle(context);
+
+        handle.handle_imap.take().unwrap().join().unwrap();
+        handle.handle_mvbox.take().unwrap().join().unwrap();
+        handle.handle_sentbox.take().unwrap().join().unwrap();
+        handle.handle_smtp.take().unwrap().join().unwrap();
+    }
 }
 
 // === The main loop
@@ -146,8 +250,10 @@ const IMEX_COMMANDS: [&str; 12] = [
     "stop",
 ];
 
-const DB_COMMANDS: [&str; 9] = [
+const DB_COMMANDS: [&str; 11] = [
     "info",
+    "open",
+    "close",
     "set",
     "get",
     "oauth2",
@@ -158,7 +264,7 @@ const DB_COMMANDS: [&str; 9] = [
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 26] = [
+const CHAT_COMMANDS: [&str; 24] = [
     "listchats",
     "listarchived",
     "chat",
@@ -182,8 +288,6 @@ const CHAT_COMMANDS: [&str; 26] = [
     "listmedia",
     "archive",
     "unarchive",
-    "pin",
-    "unpin",
     "delchat",
 ];
 const MESSAGE_COMMANDS: [&str; 8] = [
@@ -204,17 +308,8 @@ const CONTACT_COMMANDS: [&str; 6] = [
     "delcontact",
     "cleanupcontacts",
 ];
-const MISC_COMMANDS: [&str; 10] = [
-    "getqr",
-    "getbadqr",
-    "checkqr",
-    "event",
-    "fileinfo",
-    "clear",
-    "exit",
-    "quit",
-    "help",
-    "estimatedeletion",
+const MISC_COMMANDS: [&str; 9] = [
+    "getqr", "getbadqr", "checkqr", "event", "fileinfo", "clear", "exit", "quit", "help",
 ];
 
 impl Hinter for DcHelper {
@@ -266,81 +361,69 @@ impl Highlighter for DcHelper {
 
 impl Helper for DcHelper {}
 
-async fn start(args: Vec<String>) -> Result<(), Error> {
+fn main_0(args: Vec<String>) -> Result<(), failure::Error> {
     if args.len() < 2 {
         println!("Error: Bad arguments, expected [db-name].");
-        bail!("No db-name specified");
+        return Err(format_err!("No db-name specified"));
     }
-    let context = Context::new("CLI".into(), Path::new(&args[1]).to_path_buf()).await?;
-
-    let events = context.get_event_emitter();
-    async_std::task::spawn(async move {
-        while let Some(event) = events.recv().await {
-            receive_event(event);
-        }
-    });
+    let context = Context::new(
+        Box::new(receive_event),
+        "CLI".into(),
+        Path::new(&args[1]).to_path_buf(),
+    )?;
 
     println!("Delta Chat Core is awaiting your commands.");
 
+    let ctx = Arc::new(RwLock::new(context));
+
     let config = Config::builder()
         .history_ignore_space(true)
         .completion_type(CompletionType::List)
         .edit_mode(EditMode::Emacs)
         .output_stream(OutputStreamType::Stdout)
         .build();
-    let mut selected_chat = ChatId::default();
-    let (reader_s, reader_r) = async_std::sync::channel(100);
-    let input_loop = async_std::task::spawn_blocking(move || {
-        let h = DcHelper {
-            completer: FilenameCompleter::new(),
-            highlighter: MatchingBracketHighlighter::new(),
-            hinter: HistoryHinter {},
-        };
-        let mut rl = Editor::with_config(config);
-        rl.set_helper(Some(h));
-        rl.bind_sequence(KeyPress::Meta('N'), Cmd::HistorySearchForward);
-        rl.bind_sequence(KeyPress::Meta('P'), Cmd::HistorySearchBackward);
-        if rl.load_history(".dc-history.txt").is_err() {
-            println!("No previous history.");
-        }
+    let h = DcHelper {
+        completer: FilenameCompleter::new(),
+        highlighter: MatchingBracketHighlighter::new(),
+        hinter: HistoryHinter {},
+    };
+    let mut rl = Editor::with_config(config);
+    rl.set_helper(Some(h));
+    rl.bind_sequence(KeyPress::Meta('N'), Cmd::HistorySearchForward);
+    rl.bind_sequence(KeyPress::Meta('P'), Cmd::HistorySearchBackward);
+    if rl.load_history(".dc-history.txt").is_err() {
+        println!("No previous history.");
+    }
 
-        loop {
-            let p = "> ";
-            let readline = rl.readline(&p);
-
-            match readline {
-                Ok(line) => {
-                    // TODO: ignore "set mail_pw"
-                    rl.add_history_entry(line.as_str());
-                    async_std::task::block_on(reader_s.send(line));
-                }
-                Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => {
-                    println!("Exiting...");
-                    drop(reader_s);
-                    break;
-                }
-                Err(err) => {
-                    println!("Error: {}", err);
-                    drop(reader_s);
-                    break;
+    loop {
+        let p = "> ";
+        let readline = rl.readline(&p);
+        match readline {
+            Ok(line) => {
+                // TODO: ignore "set mail_pw"
+                rl.add_history_entry(line.as_str());
+                let ctx = ctx.clone();
+                match handle_cmd(line.trim(), ctx) {
+                    Ok(ExitResult::Continue) => {}
+                    Ok(ExitResult::Exit) => break,
+                    Err(err) => println!("Error: {}", err),
                 }
             }
-        }
-
-        rl.save_history(".dc-history.txt")?;
-        println!("history saved");
-        Ok::<_, Error>(())
-    });
-
-    while let Ok(line) = reader_r.recv().await {
-        match handle_cmd(line.trim(), context.clone(), &mut selected_chat).await {
-            Ok(ExitResult::Continue) => {}
-            Ok(ExitResult::Exit) => break,
-            Err(err) => println!("Error: {}", err),
+            Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => {
+                println!("Exiting...");
+                break;
+            }
+            Err(err) => {
+                println!("Error: {}", err);
+                break;
+            }
         }
     }
-    context.stop_io().await;
-    input_loop.await?;
+    rl.save_history(".dc-history.txt")?;
+    println!("history saved");
+    {
+        stop_threads(&ctx.read().unwrap());
+    }
 
     Ok(())
 }
@@ -351,29 +434,43 @@ enum ExitResult {
     Exit,
 }
 
-async fn handle_cmd(
-    line: &str,
-    ctx: Context,
-    selected_chat: &mut ChatId,
-) -> Result<ExitResult, Error> {
+fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, failure::Error> {
     let mut args = line.splitn(2, ' ');
     let arg0 = args.next().unwrap_or_default();
     let arg1 = args.next().unwrap_or_default();
 
     match arg0 {
         "connect" => {
-            ctx.start_io().await;
+            start_threads(ctx);
         }
         "disconnect" => {
-            ctx.stop_io().await;
+            stop_threads(&ctx.read().unwrap());
+        }
+        "smtp-jobs" => {
+            if HANDLE.clone().lock().unwrap().is_some() {
+                println!("smtp-jobs are already running in a thread.",);
+            } else {
+                perform_smtp_jobs(&ctx.read().unwrap());
+            }
+        }
+        "imap-jobs" => {
+            if HANDLE.clone().lock().unwrap().is_some() {
+                println!("inbox-jobs are already running in a thread.");
+            } else {
+                perform_inbox_jobs(&ctx.read().unwrap());
+            }
         }
         "configure" => {
-            ctx.configure().await?;
+            start_threads(ctx.clone());
+            configure(&ctx.read().unwrap());
         }
         "oauth2" => {
-            if let Some(addr) = ctx.get_config(config::Config::Addr).await {
-                let oauth2_url =
-                    dc_get_oauth2_url(&ctx, &addr, "chat.delta:/com.b44t.messenger").await;
+            if let Some(addr) = ctx.read().unwrap().get_config(config::Config::Addr) {
+                let oauth2_url = dc_get_oauth2_url(
+                    &ctx.read().unwrap(),
+                    &addr,
+                    "chat.delta:/com.b44t.messenger",
+                );
                 if oauth2_url.is_none() {
                     println!("OAuth2 not available for {}.", &addr);
                 } else {
@@ -388,9 +485,9 @@ async fn handle_cmd(
             print!("\x1b[1;1H\x1b[2J");
         }
         "getqr" | "getbadqr" => {
-            ctx.start_io().await;
+            start_threads(ctx.clone());
             if let Some(mut qr) =
-                dc_get_securejoin_qr(&ctx, ChatId::new(arg1.parse().unwrap_or_default())).await
+                dc_get_securejoin_qr(&ctx.read().unwrap(), arg1.parse().unwrap_or_default())
             {
                 if !qr.is_empty() {
                     if arg0 == "getbadqr" && qr.len() > 40 {
@@ -407,23 +504,23 @@ async fn handle_cmd(
             }
         }
         "joinqr" => {
-            ctx.start_io().await;
+            start_threads(ctx.clone());
             if !arg0.is_empty() {
-                dc_join_securejoin(&ctx, arg1).await;
+                dc_join_securejoin(&ctx.read().unwrap(), arg1);
             }
         }
         "exit" | "quit" => return Ok(ExitResult::Exit),
-        _ => cmdline(ctx.clone(), line, selected_chat).await?,
+        _ => dc_cmdline(&ctx.read().unwrap(), line)?,
     }
 
     Ok(ExitResult::Continue)
 }
 
-fn main() -> Result<(), Error> {
+pub fn main() -> Result<(), failure::Error> {
     let _ = pretty_env_logger::try_init();
 
-    let args = std::env::args().collect();
-    async_std::task::block_on(async move { start(args).await })?;
+    let args: Vec<String> = std::env::args().collect();
+    main_0(args)?;
 
     Ok(())
 }

examples/simple.rs

@@ -1,100 +1,115 @@
+extern crate deltachat;
+
+use std::sync::{Arc, RwLock};
+use std::{thread, time};
 use tempfile::tempdir;
 
 use deltachat::chat;
 use deltachat::chatlist::*;
 use deltachat::config;
+use deltachat::configure::*;
 use deltachat::contact::*;
 use deltachat::context::*;
-use deltachat::message::Message;
+use deltachat::job::{
+    perform_inbox_fetch, perform_inbox_idle, perform_inbox_jobs, perform_smtp_idle,
+    perform_smtp_jobs,
+};
 use deltachat::Event;
 
-fn cb(event: Event) {
+fn cb(_ctx: &Context, event: Event) -> usize {
+    print!("[{:?}]", event);
+
     match event {
         Event::ConfigureProgress(progress) => {
-            log::info!("progress: {}", progress);
+            print!("  progress: {}\n", progress);
+            0
         }
-        Event::Info(msg) => {
-            log::info!("{}", msg);
+        Event::Info(msg) | Event::Warning(msg) | Event::Error(msg) | Event::ErrorNetwork(msg) => {
+            print!("  {}\n", msg);
+            0
         }
-        Event::Warning(msg) => {
-            log::warn!("{}", msg);
-        }
-        Event::Error(msg) | Event::ErrorNetwork(msg) => {
-            log::error!("{}", msg);
-        }
-        event => {
-            log::info!("{:?}", event);
+        _ => {
+            print!("\n");
+            0
         }
     }
 }
 
-/// Run with `RUST_LOG=simple=info cargo run --release --example simple --features repl -- email pw`.
-#[async_std::main]
-async fn main() {
-    pretty_env_logger::try_init_timed().ok();
-
+fn main() {
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
-    log::info!("creating database {:?}", dbfile);
-    let ctx = Context::new("FakeOs".into(), dbfile.into())
-        .await
-        .expect("Failed to create context");
-    let info = ctx.get_info().await;
-    log::info!("info: {:#?}", info);
+    println!("creating database {:?}", dbfile);
+    let ctx =
+        Context::new(Box::new(cb), "FakeOs".into(), dbfile).expect("Failed to create context");
+    let running = Arc::new(RwLock::new(true));
+    let info = ctx.get_info();
+    let duration = time::Duration::from_millis(4000);
+    println!("info: {:#?}", info);
 
-    let events = ctx.get_event_emitter();
-    let events_spawn = async_std::task::spawn(async move {
-        while let Some(event) = events.recv().await {
-            cb(event);
+    let ctx = Arc::new(ctx);
+    let ctx1 = ctx.clone();
+    let r1 = running.clone();
+    let t1 = thread::spawn(move || {
+        while *r1.read().unwrap() {
+            perform_inbox_jobs(&ctx1);
+            if *r1.read().unwrap() {
+                perform_inbox_fetch(&ctx1);
+
+                if *r1.read().unwrap() {
+                    perform_inbox_idle(&ctx1);
+                }
+            }
         }
     });
 
-    log::info!("configuring");
+    let ctx1 = ctx.clone();
+    let r1 = running.clone();
+    let t2 = thread::spawn(move || {
+        while *r1.read().unwrap() {
+            perform_smtp_jobs(&ctx1);
+            if *r1.read().unwrap() {
+                perform_smtp_idle(&ctx1);
+            }
+        }
+    });
+
+    println!("configuring");
     let args = std::env::args().collect::<Vec<String>>();
-    assert_eq!(args.len(), 3, "requires email password");
-    let email = args[1].clone();
-    let pw = args[2].clone();
-    ctx.set_config(config::Config::Addr, Some(&email))
-        .await
-        .unwrap();
-    ctx.set_config(config::Config::MailPw, Some(&pw))
-        .await
+    assert_eq!(args.len(), 2, "missing password");
+    let pw = args[1].clone();
+    ctx.set_config(config::Config::Addr, Some("d@testrun.org"))
         .unwrap();
+    ctx.set_config(config::Config::MailPw, Some(&pw)).unwrap();
+    configure(&ctx);
 
-    ctx.configure().await.unwrap();
+    thread::sleep(duration);
 
-    log::info!("------ RUN ------");
-    ctx.start_io().await;
-    log::info!("--- SENDING A MESSAGE ---");
+    println!("sending a message");
+    let contact_id = Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com").unwrap();
+    let chat_id = chat::create_by_contact_id(&ctx, contact_id).unwrap();
+    chat::send_text_msg(&ctx, chat_id, "Hi, here is my first message!".into()).unwrap();
 
-    let contact_id = Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com")
-        .await
-        .unwrap();
-    let chat_id = chat::create_by_contact_id(&ctx, contact_id).await.unwrap();
-
-    for i in 0..1 {
-        log::info!("sending message {}", i);
-        chat::send_text_msg(&ctx, chat_id, format!("Hi, here is my {}nth message!", i))
-            .await
-            .unwrap();
-    }
-
-    // wait for the message to be sent out
-    async_std::task::sleep(std::time::Duration::from_secs(1)).await;
-
-    log::info!("fetching chats..");
-    let chats = Chatlist::try_load(&ctx, 0, None, None).await.unwrap();
+    println!("fetching chats..");
+    let chats = Chatlist::try_load(&ctx, 0, None, None).unwrap();
 
     for i in 0..chats.len() {
-        let msg = Message::load_from_db(&ctx, chats.get_msg_id(i).unwrap())
-            .await
-            .unwrap();
-        log::info!("[{}] msg: {:?}", i, msg);
+        let summary = chats.get_summary(&ctx, 0, None);
+        let text1 = summary.get_text1();
+        let text2 = summary.get_text2();
+        println!("chat: {} - {:?} - {:?}", i, text1, text2,);
     }
 
-    log::info!("stopping");
-    ctx.stop_io().await;
-    log::info!("closing");
-    drop(ctx);
-    events_spawn.await;
+    thread::sleep(duration);
+
+    println!("stopping threads");
+
+    *running.write().unwrap() = false;
+    deltachat::job::interrupt_inbox_idle(&ctx, true);
+    deltachat::job::interrupt_smtp_idle(&ctx);
+
+    println!("joining");
+    t1.join().unwrap();
+    t2.join().unwrap();
+
+    println!("closing");
 }

python/CHANGELOG

@@ -1,29 +1,6 @@
-0.900.0 (DRAFT)
----------------
-
-- refactored internals to use plugin-approach 
-
-- introduced PerAccount and Global hooks that plugins can implement 
-
-- introduced `ac_member_added()` and `ac_member_removed()` plugin events. 
-
-- introduced two documented examples for an echo and a group-membership
-  tracking plugin. 
-
-0.800.0
--------
-
-- use latest core 1.25.0 
-
-- refine tests and some internal changes to core bindings
-
-0.700.0
+0.600.1
 ---------
 
-- lots of new Python APIs
-
-- use rust core-beta23 
-
 - introduce automatic versioning via setuptools_scm, 
   based on py-X.Y.Z tags 
 

python/README.rst

@@ -3,98 +3,41 @@ deltachat python bindings
 =========================
 
 This package provides bindings to the deltachat-core_ Rust -library
-which implements IMAP/SMTP/MIME/PGP e-mail standards and offers
-a low-level Chat/Contact/Message API to user interfaces and bots.
+which provides imap/smtp/crypto handling as well as chat/group/messages
+handling to Android, Desktop and IO user interfaces.
 
+Installing pre-built packages (linux-only)
+==========================================
 
-Installing bindings from source (Updated: 20-Jan-2020)
-=========================================================
-
-Install Rust and Cargo first.  Deltachat needs a specific nightly
-version, the easiest is probably to first install Rust stable from
-rustup and then use this to install the correct nightly version.
-
-Bootstrap Rust and Cargo by using rustup::
-
-   curl https://sh.rustup.rs -sSf | sh
-
-Then GIT clone the deltachat-core-rust repo and get the actual
-rust- and cargo-toolchain needed by deltachat::
-
-   git clone https://github.com/deltachat/deltachat-core-rust
-   cd deltachat-core-rust
-   rustup show
-
-To install the Delta Chat Python bindings make sure you have Python3 installed.
-E.g. on Debian-based systems `apt install python3 python3-pip
-python3-venv` should give you a usable python installation.
-
-Ensure you are in the deltachat-core-rust/python directory, create the
-virtual environment and activate it in your shell::
-
-   cd python
-   python3 -m venv venv  # or: virtualenv venv
-   source venv/bin/activate
-
-You should now be able to build the python bindings using the supplied script::
-
-   ./install_python_bindings.py
-
-The installation might take a while, depending on your machine.
-The bindings will be installed in release mode but with debug symbols.
-The release mode is currently necessary because some tests generate RSA keys
-which is prohibitively slow in non-release mode.
-
-After successful binding installation you can install a few more
-Python packages before running the tests::
-
-    python -m pip install pytest pytest-timeout pytest-rerunfailures requests
-    pytest -v tests
-
-
-running "live" tests with temporary accounts
----------------------------------------------
-
-If you want to run "liveconfig" functional tests you can set
-``DCC_NEW_TMP_EMAIL`` to:
-
-- a particular https-url that you can ask for from the delta
-  chat devs. This is implemented on the server side via
-  the [mailadm](https://github.com/deltachat/mailadm) command line tool.
-
-- or the path of a file that contains two lines, each describing
-  via "addr=... mail_pw=..." a test account login that will
-  be used for the live tests.
-
-With ``DCC_NEW_TMP_EMAIL`` set pytest invocations will use real
-e-mail accounts and run through all functional "liveconfig" tests.
-
-
-Installing pre-built packages (Linux-only)
-========================================================
-
-If you have a Linux system you may try to install the ``deltachat`` binary "wheel" packages
+If you have a linux system you may install the ``deltachat`` binary "wheel" package
 without any "build-from-source" steps.
 
-We suggest to `Install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
-then create a fresh Python virtual environment and activate it in your shell::
+1. `Install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
+   then create a fresh python environment and activate it in your shell::
 
         virtualenv venv  # or: python -m venv
         source venv/bin/activate
 
-Afterwards, invoking ``python`` or ``pip install`` only
-modifies files in your ``venv`` directory and leaves
-your system installation alone.
+   Afterwards, invoking ``python`` or ``pip install`` will only
+   modify files in your ``venv`` directory and leave your system installation
+   alone.
+
+2. Install the wheel for linux::
+
+        pip install deltachat
+
+    Verify it worked by typing::
+
+        python -c "import deltachat"
+
+
+Installing a wheel from a PR/branch
+---------------------------------------
 
 For Linux, we automatically build wheels for all github PR branches
-and push them to a python package index. To install the latest
-github ``master`` branch::
+and push them to a python package index. To install the latest github ``master`` branch::
 
-    pip install --pre -i https://m.devpi.net/dc/master deltachat
-
-To verify it worked::
-
-    python -c "import deltachat"
+    pip install -i https://m.devpi.net/dc/master deltachat
 
 .. note::
 
@@ -103,6 +46,65 @@ To verify it worked::
     `in contact with us <https://delta.chat/en/contribute>`_.
 
 
+Installing bindings from source
+===============================
+
+If you can't use "binary" method above then you need to compile
+to core deltachat library::
+
+    git clone https://github.com/deltachat/deltachat-core-rust
+    cd deltachat-core-rust
+    cd python
+
+If you don't have one active, create and activate a python "virtualenv":
+
+    python virtualenv venv  # or python -m venv
+    source venv/bin/activate
+
+Afterwards ``which python`` tells you that it comes out of the "venv"
+directory that contains all python install artifacts. Let's first
+install test tools::
+
+    pip install pytest pytest-timeout pytest-rerunfailures requests
+
+then cargo-build and install the deltachat bindings::
+
+    python install_python_bindings.py
+
+The bindings will be installed in release mode but with debug symbols.
+The release mode is necessary because some tests generate RSA keys
+which is prohibitively slow in debug mode.
+
+After successful binding installation you can finally run the tests::
+
+    pytest -v tests
+
+.. note::
+
+    Some tests are sometimes failing/hanging because of
+    https://github.com/deltachat/deltachat-core-rust/issues/331
+    and
+    https://github.com/deltachat/deltachat-core-rust/issues/326
+
+
+running "live" tests (experimental)
+-----------------------------------
+
+If you want to run "liveconfig" functional tests you can set
+``DCC_PY_LIVECONFIG`` to:
+
+- a particular https-url that you can ask for from the delta
+  chat devs.
+
+- or the path of a file that contains two lines, each describing
+  via "addr=... mail_pw=..." a test account login that will
+  be used for the live tests.
+
+With ``DCC_PY_LIVECONFIG`` set pytest invocations will use real
+e-mail accounts and run through all functional "liveconfig" tests.
+
+
+
 Code examples
 =============
 
@@ -113,11 +115,15 @@ You may look at `examples <https://py.delta.chat/examples.html>`_.
 .. _`deltachat-core`: https://github.com/deltachat/deltachat-core-rust
 
 
-Building manylinux based wheels
-====================================
+Building manylinux1 wheels
+==========================
 
-Building portable manylinux wheels which come with libdeltachat.so
-can be done with docker-tooling.
+.. note::
+
+   This section may not fully work.
+
+Building portable manylinux1 wheels which come with libdeltachat.so
+and all it's dependencies is easy using the provided docker tooling.
 
 using docker pull / premade images
 ------------------------------------
@@ -130,9 +136,9 @@ organization::
 
 This docker image can be used to run tests and build Python wheels for all interpreters::
 
-    $ docker run -e DCC_NEW_TMP_EMAIL \
-       --rm -it -v \$(pwd):/mnt -w /mnt \
-       deltachat/coredeps ci_scripts/run_all.sh
+    $ bash ci_scripts/ci_run.sh
+
+This command runs tests and build-wheel scripts in a docker container.
 
 
 Optionally build your own docker image

python/doc/_static/custom.css

@@ -15,7 +15,3 @@ div.globaltoc {
 img.logo {
     height: 120px;
 }
-
-div.footer {
-    display: none;
-}

python/doc/api.rst

@@ -2,6 +2,10 @@
 high level API reference
 ========================
 
+.. note::
+
+    This API is work in progress and may change in versions prior to 1.0.
+
 - :class:`deltachat.account.Account` (your main entry point, creates the
   other classes)
 - :class:`deltachat.contact.Contact`

python/doc/capi.rst

@@ -0,0 +1,7 @@
+
+C deltachat interface
+=====================
+
+See :doc:`lapi` for accessing many of the below functions
+through the ``deltachat.capi.lib`` namespace.
+

python/doc/conf.py

@@ -55,7 +55,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'deltachat'
-copyright = u'2020, holger krekel and contributors'
+copyright = u'2018, holger krekel and contributors'
 
 
 # The language for content autogenerated by Sphinx. Refer to documentation

python/doc/examples.rst

@@ -1,60 +1,37 @@
 
+
 examples
 ========
 
+
+Playing around on the commandline
+----------------------------------
+
 Once you have :doc:`installed deltachat bindings <install>`
-you need email/password credentials for an IMAP/SMTP account.
-Delta Chat developers and the CI system use a special URL to create
-temporary e-mail accounts on [testrun.org](https://testrun.org) for testing.
+you can start playing from the python interpreter commandline.
+For example you can type ``python`` and then::
 
-Receiving a Chat message from the command line
-----------------------------------------------
+    # instantiate and configure deltachat account
+    import deltachat
+    ac = deltachat.Account("/tmp/db")
 
-Here is a simple bot that:
+    # start configuration activity and smtp/imap threads
+    ac.start_threads()
+    ac.configure(addr="test2@hq5.merlinux.eu", mail_pw="********")
 
-- receives a message and sends back ("echoes") a message
+    # create a contact and send a message
+    contact = ac.create_contact("someother@email.address")
+    chat = ac.create_chat_by_contact(contact)
+    chat.send_text("hi from the python interpreter command line")
 
-- terminates the bot if the message `/quit` is sent
+Checkout our :doc:`api` for the various high-level things you can do
+to send/receive messages, create contacts and chats.
 
-.. include:: ../examples/echo_and_quit.py
-    :literal:
 
-With this file in your working directory you can run the bot
-by specifying a database path, an e-mail address and password of
-a SMTP-IMAP account::
-
-    $ cd examples
-    $ python echo_and_quit.py /tmp/db --email ADDRESS --password PASSWORD
-
-While this process is running you can start sending chat messages
-to `ADDRESS`.
-
-Track member additions and removals in a group
-----------------------------------------------
-
-Here is a simple bot that:
-
-- echoes messages sent to it
-
-- tracks if configuration completed
-
-- tracks member additions and removals for all chat groups
-
-.. include:: ../examples/group_tracking.py
-    :literal:
-
-With this file in your working directory you can run the bot
-by specifying a database path, an e-mail address and password of
-a SMTP-IMAP account::
-
-    python group_tracking.py --email ADDRESS --password PASSWORD /tmp/db
-
-When this process is running you can start sending chat messages
-to `ADDRESS`.
-
-Writing bots for real
+Looking at a real example
 -------------------------
 
 The `deltabot repository <https://github.com/deltachat/deltabot#deltachat-example-bot>`_
-contains a little framework for writing deltachat bots in Python.
+contains a real-life example of Python bindings usage.
+
 

python/doc/index.rst

@@ -1,15 +1,15 @@
 deltachat python bindings
 =========================
 
-The ``deltachat`` Python package provides two layers of bindings for the
-core Rust-library of the https://delta.chat messaging ecosystem:
+The ``deltachat`` Python package provides two bindings for the core Rust-library
+of the https://delta.chat messaging ecosystem:
 
-- :doc:`api` is a high level interface to deltachat-core.
+- :doc:`api` is a high level interface to deltachat-core which aims
+  to be memory safe and thoroughly tested through continous tox/pytest runs.
 
-- :doc:`plugins` is a brief introduction into implementing plugin hooks.
-
-- :doc:`lapi` is a lowlevel CFFI-binding to the `Rust Core
-  <https://github.com/deltachat/deltachat-core-rust>`_.
+- :doc:`capi` is a lowlevel CFFI-binding to the previous
+  `deltachat-core C-API <https://c.delta.chat>`_ (so far the Rust library
+  replicates exactly the same C-level API).
 
 
 
@@ -28,8 +28,8 @@ getting started
    links
    changelog
    api
+   capi
    lapi
-   plugins
 
 ..
     Indices and tables

python/doc/plugins.rst

@@ -1,38 +0,0 @@
-
-Implementing Plugin Hooks
-==========================
-
-The Delta Chat Python bindings use `pluggy <https://pluggy.readthedocs.io>`_
-for managing global and per-account plugin registration, and performing
-hook calls. There are two kinds of plugins:
-
-- Global plugins that are active for all accounts; they can implement
-  hooks at account-creation and account-shutdown time.
-
-- Account plugins that are only active during the lifetime of a
-  single Account instance.
-
-
-Registering a plugin
---------------------
-
-.. autofunction:: deltachat.register_global_plugin
-    :noindex:
-
-.. automethod:: deltachat.account.Account.add_account_plugin
-    :noindex:
-
-
-Per-Account Hook specifications
--------------------------------
-
-.. autoclass:: deltachat.hookspec.PerAccount
-    :members:
-
-
-Global Hook specifications
---------------------------
-
-.. autoclass:: deltachat.hookspec.Global
-    :members:
-

python/examples/echo_and_quit.py

@@ -1,33 +0,0 @@
-
-# content of echo_and_quit.py
-
-from deltachat import account_hookimpl, run_cmdline
-
-
-class EchoPlugin:
-    @account_hookimpl
-    def ac_incoming_message(self, message):
-        print("process_incoming message", message)
-        if message.text.strip() == "/quit":
-            message.account.shutdown()
-        else:
-            # unconditionally accept the chat
-            message.create_chat()
-            addr = message.get_sender_contact().addr
-            if message.is_system_message():
-                message.chat.send_text("echoing system message from {}:\n{}".format(addr, message))
-            else:
-                text = message.text
-                message.chat.send_text("echoing from {}:\n{}".format(addr, text))
-
-    @account_hookimpl
-    def ac_message_delivered(self, message):
-        print("ac_message_delivered", message)
-
-
-def main(argv=None):
-    run_cmdline(argv=argv, account_plugins=[EchoPlugin()])
-
-
-if __name__ == "__main__":
-    main()

python/examples/group_tracking.py

@@ -1,52 +0,0 @@
-
-# content of group_tracking.py
-
-from deltachat import account_hookimpl, run_cmdline
-
-
-class GroupTrackingPlugin:
-    @account_hookimpl
-    def ac_incoming_message(self, message):
-        print("process_incoming message", message)
-        if message.text.strip() == "/quit":
-            message.account.shutdown()
-        else:
-            # unconditionally accept the chat
-            message.create_chat()
-            addr = message.get_sender_contact().addr
-            text = message.text
-            message.chat.send_text("echoing from {}:\n{}".format(addr, text))
-
-    @account_hookimpl
-    def ac_outgoing_message(self, message):
-        print("ac_outgoing_message:", message)
-
-    @account_hookimpl
-    def ac_configure_completed(self, success):
-        print("ac_configure_completed:", success)
-
-    @account_hookimpl
-    def ac_chat_modified(self, chat):
-        print("ac_chat_modified:", chat.id, chat.get_name())
-        for member in chat.get_contacts():
-            print("chat member: {}".format(member.addr))
-
-    @account_hookimpl
-    def ac_member_added(self, chat, contact, message):
-        print("ac_member_added {} to chat {} from {}".format(
-            contact.addr, chat.id, message.get_sender_contact().addr))
-        for member in chat.get_contacts():
-            print("chat member: {}".format(member.addr))
-
-    @account_hookimpl
-    def ac_member_removed(self, chat, contact, message):
-        print("ac_member_removed {} from chat {} by {}".format(
-            contact.addr, chat.id, message.get_sender_contact().addr))
-
-
-def main(argv=None):
-    run_cmdline(argv=argv, account_plugins=[GroupTrackingPlugin()])
-
-
-if __name__ == "__main__":
-    main()

python/examples/test_examples.py

@@ -1,79 +0,0 @@
-
-import pytest
-import py
-import echo_and_quit
-import group_tracking
-from deltachat.events import FFIEventLogger
-
-
-@pytest.fixture(scope='session')
-def datadir():
-    """The py.path.local object of the test-data/ directory."""
-    for path in reversed(py.path.local(__file__).parts()):
-        datadir = path.join('test-data')
-        if datadir.isdir():
-            return datadir
-    else:
-        pytest.skip('test-data directory not found')
-
-
-def test_echo_quit_plugin(acfactory, lp):
-    lp.sec("creating one echo_and_quit bot")
-    botproc = acfactory.run_bot_process(echo_and_quit)
-
-    lp.sec("creating a temp account to contact the bot")
-    ac1 = acfactory.get_one_online_account()
-
-    lp.sec("sending a message to the bot")
-    bot_contact = ac1.create_contact(botproc.addr)
-    bot_chat = bot_contact.create_chat()
-    bot_chat.send_text("hello")
-
-    lp.sec("waiting for the reply message from the bot to arrive")
-    reply = ac1._evtracker.wait_next_incoming_message()
-    assert reply.chat == bot_chat
-    assert "hello" in reply.text
-    lp.sec("send quit sequence")
-    bot_chat.send_text("/quit")
-    botproc.wait()
-
-
-def test_group_tracking_plugin(acfactory, lp):
-    lp.sec("creating one group-tracking bot and two temp accounts")
-    botproc = acfactory.run_bot_process(group_tracking, ffi=False)
-
-    ac1, ac2 = acfactory.get_two_online_accounts(quiet=True)
-
-    botproc.fnmatch_lines("""
-        *ac_configure_completed*
-    """)
-    ac1.add_account_plugin(FFIEventLogger(ac1))
-    ac2.add_account_plugin(FFIEventLogger(ac2))
-
-    lp.sec("creating bot test group with bot")
-    bot_contact = ac1.create_contact(botproc.addr)
-    ch = ac1.create_group_chat("bot test group")
-    ch.add_contact(bot_contact)
-    ch.send_text("hello")
-
-    botproc.fnmatch_lines("""
-        *ac_chat_modified*bot test group*
-    """)
-
-    lp.sec("adding third member {}".format(ac2.get_config("addr")))
-    contact3 = ac1.create_contact(ac2.get_config("addr"))
-    ch.add_contact(contact3)
-
-    reply = ac1._evtracker.wait_next_incoming_message()
-    assert "hello" in reply.text
-
-    lp.sec("now looking at what the bot received")
-    botproc.fnmatch_lines("""
-        *ac_member_added {}*
-    """.format(contact3.addr))
-
-    lp.sec("contact successfully added, now removing")
-    ch.remove_contact(contact3)
-    botproc.fnmatch_lines("""
-        *ac_member_removed {}*
-    """.format(contact3.addr))

python/fail_test.py

@@ -1,7 +0,0 @@
-from __future__ import print_function
-from deltachat import capi
-from deltachat.capi import ffi, lib
-
-if __name__ == "__main__":
-    ctx = capi.lib.dc_context_new(ffi.NULL, ffi.NULL)
-    lib.dc_stop_io(ctx)

python/install_python_bindings.py

@@ -11,16 +11,18 @@ import sys
 if __name__ == "__main__":
     target = os.environ.get("DCC_RS_TARGET")
     if target is None:
-        os.environ["DCC_RS_TARGET"] = target = "debug"
+        os.environ["DCC_RS_TARGET"] = target = "release"
     if "DCC_RS_DEV" not in os.environ:
         dn = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
         os.environ["DCC_RS_DEV"] = dn
 
-    cmd = ["cargo", "build", "-p", "deltachat_ffi"]
-    if target == 'release':
-        cmd.append("--release")
-    print("running:", " ".join(cmd))
-    subprocess.check_call(cmd)
+    # build the core library  in release + debug mode because
+    # as of Nov 2019 rPGP generates RSA keys which take
+    # prohibitively long for non-release installs
+    os.environ["RUSTFLAGS"] = "-g"
+    subprocess.check_call([
+        "cargo", "build", "-p", "deltachat_ffi", "--" + target
+    ])
     subprocess.check_call("rm -rf build/ src/deltachat/*.so" , shell=True)
 
     if len(sys.argv) <= 1 or sys.argv[1] != "onlybuild":

python/setup.py

@@ -13,20 +13,14 @@ def main():
             "root": "..",
             "relative_to": __file__,
             'tag_regex': r'^(?P<prefix>py-)?(?P<version>[^\+]+)(?P<suffix>.*)?$',
-            'git_describe_command': "git describe --dirty --tags --long --match py-*.*",
         },
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient'],
+        install_requires=['cffi>=1.0.0', 'six'],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],
-        entry_points = {
-            'pytest11': [
-                'deltachat.testplugin = deltachat.testplugin',
-            ],
-        },
         classifiers=[
             'Development Status :: 4 - Beta',
             'Intended Audience :: Developers',

python/src/deltachat/__init__.py

@@ -1,13 +1,6 @@
-import sys
-
-from . import capi, const, hookspec # noqa
-from .capi import ffi  # noqa
-from .account import Account  # noqa
-from .message import Message  # noqa
-from .contact import Contact  # noqa
-from .chat import Chat        # noqa
-from .hookspec import account_hookimpl, global_hookimpl  # noqa
-from . import events
+from deltachat import capi, const
+from deltachat.capi import ffi
+from deltachat.account import Account  # noqa
 
 from pkg_resources import get_distribution, DistributionNotFound
 try:
@@ -17,72 +10,67 @@ except DistributionNotFound:
     __version__ = "0.0.0.dev0-unknown"
 
 
+_DC_CALLBACK_MAP = {}
+
+
+@capi.ffi.def_extern()
+def py_dc_callback(ctx, evt, data1, data2):
+    """The global event handler.
+
+    CFFI only allows us to set one global event handler, so this one
+    looks up the correct event handler for the given context.
+    """
+    try:
+        callback = _DC_CALLBACK_MAP.get(ctx, lambda *a: 0)
+    except AttributeError:
+        # we are in a deep in GC-free/interpreter shutdown land
+        # nothing much better to do here than:
+        return 0
+
+    # the following code relates to the deltachat/_build.py's helper
+    # function which provides us signature info of an event call
+    evt_name = get_dc_event_name(evt)
+    event_sig_types = capi.lib.dc_get_event_signature_types(evt)
+    if data1 and event_sig_types & 1:
+        data1 = ffi.string(ffi.cast('char*', data1)).decode("utf8")
+    if data2 and event_sig_types & 2:
+        data2 = ffi.string(ffi.cast('char*', data2)).decode("utf8")
+        try:
+            if isinstance(data2, bytes):
+                data2 = data2.decode("utf8")
+        except UnicodeDecodeError:
+            # XXX ignoring the decode error is not quite correct but for now
+            # i don't want to hunt down encoding problems in the c lib
+            pass
+    try:
+        ret = callback(ctx, evt_name, data1, data2)
+        if ret is None:
+            ret = 0
+        assert isinstance(ret, int), repr(ret)
+        if event_sig_types & 4:
+            return ffi.cast('uintptr_t', ret)
+        elif event_sig_types & 8:
+            return ffi.cast('int', ret)
+    except:  # noqa
+        raise
+        ret = 0
+    return ret
+
+
+def set_context_callback(dc_context, func):
+    _DC_CALLBACK_MAP[dc_context] = func
+
+
+def clear_context_callback(dc_context):
+    try:
+        _DC_CALLBACK_MAP.pop(dc_context, None)
+    except AttributeError:
+        pass
+
+
 def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}):
     if not _DC_EVENTNAME_MAP:
         for name, val in vars(const).items():
             if name.startswith("DC_EVENT_"):
                 _DC_EVENTNAME_MAP[val] = name
     return _DC_EVENTNAME_MAP[integer]
-
-
-def register_global_plugin(plugin):
-    """ Register a global plugin which implements one or more
-    of the :class:`deltachat.hookspec.Global` hooks.
-    """
-    gm = hookspec.Global._get_plugin_manager()
-    gm.register(plugin)
-    gm.check_pending()
-
-
-def unregister_global_plugin(plugin):
-    gm = hookspec.Global._get_plugin_manager()
-    gm.unregister(plugin)
-
-
-register_global_plugin(events)
-
-
-def run_cmdline(argv=None, account_plugins=None):
-    """ Run a simple default command line app, registering the specified
-    account plugins. """
-    import argparse
-    if argv is None:
-        argv = sys.argv
-
-    parser = argparse.ArgumentParser(prog=argv[0] if argv else None)
-    parser.add_argument("db", action="store", help="database file")
-    parser.add_argument("--show-ffi", action="store_true", help="show low level ffi events")
-    parser.add_argument("--email", action="store", help="email address")
-    parser.add_argument("--password", action="store", help="password")
-
-    args = parser.parse_args(argv[1:])
-
-    ac = Account(args.db)
-
-    if args.show_ffi:
-        ac.set_config("displayname", "bot")
-        log = events.FFIEventLogger(ac)
-        ac.add_account_plugin(log)
-
-    for plugin in account_plugins or []:
-        print("adding plugin", plugin)
-        ac.add_account_plugin(plugin)
-
-    if not ac.is_configured():
-        assert args.email and args.password, (
-            "you must specify --email and --password once to configure this database/account"
-        )
-        ac.set_config("addr", args.email)
-        ac.set_config("mail_pw", args.password)
-        ac.set_config("mvbox_move", "0")
-        ac.set_config("mvbox_watch", "0")
-        ac.set_config("sentbox_watch", "0")
-        configtracker = ac.configure()
-        configtracker.wait_finish()
-
-    # start IO threads and configure if neccessary
-    ac.start_io()
-
-    print("{}: waiting for message".format(ac.get_config("addr")))
-
-    ac.wait_shutdown()

python/src/deltachat/_build.py

@@ -1,64 +1,79 @@
 import distutils.ccompiler
 import distutils.log
 import distutils.sysconfig
-import os
-import platform
-import re
-import shutil
-import subprocess
 import tempfile
-import textwrap
-import types
-from os.path import abspath
-from os.path import dirname as dn
-
+import platform
+import os
 import cffi
+import shutil
+from os.path import dirname as dn
+from os.path import abspath
 
 
-def local_build_flags(projdir, target):
-    """Construct build flags for building against a checkout.
-
-    :param projdir: The root directory of the deltachat-core-rust project.
-    :param target: The rust build target, `debug` or `release`.
-    """
-    flags = types.SimpleNamespace()
-    if platform.system() == 'Darwin':
-        flags.libs = ['resolv', 'dl']
-        flags.extra_link_args = [
-                '-framework', 'CoreFoundation',
-                '-framework', 'CoreServices',
-                '-framework', 'Security',
-        ]
-    elif platform.system() == 'Linux':
-        flags.libs = ['rt', 'dl', 'm']
-        flags.extra_link_args = []
+def ffibuilder():
+    projdir = os.environ.get('DCC_RS_DEV')
+    if not projdir:
+        p = dn(dn(dn(dn(abspath(__file__)))))
+        projdir = os.environ["DCC_RS_DEV"] = p
+    target = os.environ.get('DCC_RS_TARGET', 'release')
+    if projdir:
+        if platform.system() == 'Darwin':
+            libs = ['resolv', 'dl']
+            extra_link_args = [
+                    '-framework', 'CoreFoundation',
+                    '-framework', 'CoreServices',
+                    '-framework', 'Security',
+            ]
+        elif platform.system() == 'Linux':
+            libs = ['rt', 'dl', 'm']
+            extra_link_args = []
+        else:
+            raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
+        target_dir = os.environ.get("CARGO_TARGET_DIR")
+        if target_dir is None:
+            target_dir = os.path.join(projdir, 'target')
+        objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
+        assert os.path.exists(objs[0]), objs
+        incs = [os.path.join(projdir, 'deltachat-ffi')]
     else:
-        raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
-    target_dir = os.environ.get("CARGO_TARGET_DIR")
-    if target_dir is None:
-        target_dir = os.path.join(projdir, 'target')
-    flags.objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
-    assert os.path.exists(flags.objs[0]), flags.objs
-    flags.incs = [os.path.join(projdir, 'deltachat-ffi')]
-    return flags
-
-
-def system_build_flags():
-    """Construct build flags for building against an installed libdeltachat."""
-    flags = types.SimpleNamespace()
-    flags.libs = ['deltachat']
-    flags.objs = []
-    flags.incs = []
-    flags.extra_link_args = []
-
-
-def extract_functions(flags):
-    """Extract the function definitions from deltachat.h.
-
-    This creates a .h file with a single `#include <deltachat.h>` line
-    in it.  It then runs the C preprocessor to create an output file
-    which contains all function definitions found in `deltachat.h`.
-    """
+        libs = ['deltachat']
+        objs = []
+        incs = []
+        extra_link_args = []
+    builder = cffi.FFI()
+    builder.set_source(
+        'deltachat.capi',
+        """
+            #include <deltachat.h>
+            const char * dupstring_helper(const char* string)
+            {
+                return strdup(string);
+            }
+            int dc_get_event_signature_types(int e)
+            {
+                int result = 0;
+                if (DC_EVENT_DATA1_IS_STRING(e))
+                    result |= 1;
+                if (DC_EVENT_DATA2_IS_STRING(e))
+                    result |= 2;
+                if (DC_EVENT_RETURNS_STRING(e))
+                    result |= 4;
+                if (DC_EVENT_RETURNS_INT(e))
+                    result |= 8;
+                return result;
+            }
+        """,
+        include_dirs=incs,
+        libraries=libs,
+        extra_objects=objs,
+        extra_link_args=extra_link_args,
+    )
+    builder.cdef("""
+        typedef int... time_t;
+        void free(void *ptr);
+        extern const char * dupstring_helper(const char* string);
+        extern int dc_get_event_signature_types(int);
+    """)
     distutils.log.set_verbosity(distutils.log.INFO)
     cc = distutils.ccompiler.new_compiler(force=True)
     distutils.sysconfig.customize_compiler(cc)
@@ -70,133 +85,20 @@ def extract_functions(flags):
             src_fp.write('#include <deltachat.h>')
         cc.preprocess(source=src_name,
                       output_file=dst_name,
-                      include_dirs=flags.incs,
+                      include_dirs=incs,
                       macros=[('PY_CFFI', '1')])
         with open(dst_name, "r") as dst_fp:
-            return dst_fp.read()
+            builder.cdef(dst_fp.read())
     finally:
         shutil.rmtree(tmpdir)
 
-
-def find_header(flags):
-    """Use the compiler to find the deltachat.h header location.
-
-    This uses a small utility in deltachat.h to find the location of
-    the header file location.
-    """
-    distutils.log.set_verbosity(distutils.log.INFO)
-    cc = distutils.ccompiler.new_compiler(force=True)
-    distutils.sysconfig.customize_compiler(cc)
-    tmpdir = tempfile.mkdtemp()
-    try:
-        src_name = os.path.join(tmpdir, "where.c")
-        obj_name = os.path.join(tmpdir, "where.o")
-        dst_name = os.path.join(tmpdir, "where")
-        with open(src_name, "w") as src_fp:
-            src_fp.write(textwrap.dedent("""
-                #include <stdio.h>
-                #include <deltachat.h>
-
-                int main(void) {
-                    printf("%s", _dc_header_file_location());
-                    return 0;
-                }
-            """))
-        cwd = os.getcwd()
-        try:
-            os.chdir(tmpdir)
-            cc.compile(sources=["where.c"],
-                       include_dirs=flags.incs,
-                       macros=[("PY_CFFI_INC", "1")])
-        finally:
-            os.chdir(cwd)
-        cc.link_executable(objects=[obj_name],
-                           output_progname="where",
-                           output_dir=tmpdir)
-        return subprocess.check_output(dst_name)
-    finally:
-        shutil.rmtree(tmpdir)
-
-
-def extract_defines(flags):
-    """Extract the required #DEFINEs from deltachat.h.
-
-    Since #DEFINEs are interpreted by the C preprocessor we can not
-    use the compiler to extract these and need to parse the header
-    file ourselves.
-
-    The defines are returned in a string that can be passed to CFFIs
-    cdef() method.
-    """
-    header = find_header(flags)
-    defines_re = re.compile(r"""
-        \#define\s+   # The start of a define.
-        (             # Begin capturing group which captures the define name.
-            (?:       # A nested group which is not captured, this allows us
-                      #    to build the list of prefixes to extract without
-                      #    creation another capture group.
-                DC_EVENT
-                | DC_QR
-                | DC_MSG
-                | DC_LP
-                | DC_EMPTY
-                | DC_CERTCK
-                | DC_STATE
-                | DC_STR
-                | DC_CONTACT_ID
-                | DC_GCL
-                | DC_CHAT
-                | DC_PROVIDER
-                | DC_KEY_GEN
-            )         # End of prefix matching
-            _[\w_]+   # Match the suffix, e.g. _RSA2048 in DC_KEY_GEN_RSA2048
-        )             # Close the capturing group, this contains
-                      # the entire name e.g. DC_MSG_TEXT.
-        \s+\S+        # Ensure there is whitespace followed by a value.
-    """, re.VERBOSE)
-    defines = []
-    with open(header) as fp:
-        for line in fp:
-            match = defines_re.match(line)
-            if match:
-                defines.append(match.group(1))
-    return '\n'.join('#define {} ...'.format(d) for d in defines)
-
-
-def ffibuilder():
-    projdir = os.environ.get('DCC_RS_DEV')
-    if not projdir:
-        p = dn(dn(dn(dn(abspath(__file__)))))
-        projdir = os.environ["DCC_RS_DEV"] = p
-    target = os.environ.get('DCC_RS_TARGET', 'release')
-    if projdir:
-        flags = local_build_flags(projdir, target)
-    else:
-        flags = system_build_flags()
-    builder = cffi.FFI()
-    builder.set_source(
-        'deltachat.capi',
-        """
-            #include <deltachat.h>
-            int dc_event_has_string_data(int e)
-            {
-                return DC_EVENT_DATA2_IS_STRING(e);
-            }
-        """,
-        include_dirs=flags.incs,
-        libraries=flags.libs,
-        extra_objects=flags.objs,
-        extra_link_args=flags.extra_link_args,
-    )
     builder.cdef("""
-        typedef int... time_t;
-        void free(void *ptr);
-        extern int dc_event_has_string_data(int);
+        extern "Python" uintptr_t py_dc_callback(
+            dc_context_t* context,
+            int event,
+            uintptr_t data1,
+            uintptr_t data2);
     """)
-    function_defs = extract_functions(flags)
-    defines = extract_defines(flags)
-    builder.cdef(function_defs)
-    builder.cdef(defines)
     return builder
 
 

python/src/deltachat/account.py

@@ -1,24 +1,24 @@
 """ Account class implementation. """
 
 from __future__ import print_function
-from contextlib import contextmanager
-from email.utils import parseaddr
-from threading import Event
+import atexit
+import threading
 import os
+import re
+import time
 from array import array
+try:
+    from queue import Queue, Empty
+except ImportError:
+    from Queue import Queue, Empty
+
+import deltachat
 from . import const
 from .capi import ffi, lib
 from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array, DCLot
 from .chat import Chat
 from .message import Message
 from .contact import Contact
-from .tracker import ImexTracker, ConfigureTracker
-from . import hookspec
-from .events import EventThread
-
-
-class MissingCredentials(ValueError):
-    """ Account is missing `addr` and `mail_pw` config values. """
 
 
 class Account(object):
@@ -26,53 +26,38 @@ class Account(object):
     by the underlying deltachat core library.  All public Account methods are
     meant to be memory-safe and return memory-safe objects.
     """
-    MissingCredentials = MissingCredentials
-
-    def __init__(self, db_path, os_name=None, logging=True):
+    def __init__(self, db_path, logid=None, eventlogging=True, debug=True):
         """ initialize account object.
 
         :param db_path: a path to the account database. The database
                         will be created if it doesn't exist.
-        :param os_name: this will be put to the X-Mailer header in outgoing messages
+        :param logid: an optional logging prefix that should be used with
+                      the default internal logging.
+        :param eventlogging: if False no eventlogging and no context callback will be configured
+        :param debug: turn on debug logging for events.
         """
-        # initialize per-account plugin system
-        self._pm = hookspec.PerAccount._make_plugin_manager()
-        self._logging = logging
+        self._dc_context = ffi.gc(
+            lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+            _destroy_dc_context,
+        )
+        if eventlogging:
+            self._evlogger = EventLogger(self._dc_context, logid, debug)
+            deltachat.set_context_callback(self._dc_context, self._process_event)
+            self._threads = IOThreads(self._dc_context, self._evlogger._log_event)
+        else:
+            self._threads = IOThreads(self._dc_context)
 
-        self.add_account_plugin(self)
-
-        self.db_path = db_path
         if hasattr(db_path, "encode"):
             db_path = db_path.encode("utf8")
-
-        self._dc_context = ffi.gc(
-            lib.dc_context_new(as_dc_charpointer(os_name), db_path, ffi.NULL),
-            lib.dc_context_unref,
-        )
-        if self._dc_context == ffi.NULL:
-            raise ValueError("Could not dc_context_new: {} {}".format(os_name, db_path))
-
-        self._shutdown_event = Event()
-        self._event_thread = EventThread(self)
+        if not lib.dc_open(self._dc_context, db_path, ffi.NULL):
+            raise ValueError("Could not dc_open: {}".format(db_path))
         self._configkeys = self.get_config("sys.config_keys").split()
-        hook = hookspec.Global._get_plugin_manager().hook
-        hook.dc_account_init(account=self)
-
-    def disable_logging(self):
-        """ disable logging. """
-        self._logging = False
-
-    def enable_logging(self):
-        """ re-enable logging. """
-        self._logging = True
+        self._imex_events = Queue()
+        atexit.register(self.shutdown)
 
     # def __del__(self):
     #    self.shutdown()
 
-    def log(self, msg):
-        if self._logging:
-            self._pm.hook.ac_log_line(message=msg)
-
     def _check_config_key(self, name):
         if name not in self._configkeys:
             raise KeyError("{!r} not a valid config key, existing keys: {!r}".format(
@@ -132,27 +117,16 @@ class Account(object):
         assert res != ffi.NULL, "config value not found for: {!r}".format(name)
         return from_dc_charpointer(res)
 
-    def _preconfigure_keypair(self, addr, public, secret):
-        """See dc_preconfigure_keypair() in deltachat.h.
-
-        In other words, you don't need this.
-        """
-        res = lib.dc_preconfigure_keypair(self._dc_context,
-                                          as_dc_charpointer(addr),
-                                          as_dc_charpointer(public),
-                                          as_dc_charpointer(secret))
-        if res == 0:
-            raise Exception("Failed to set key")
-
-    def update_config(self, kwargs):
-        """ update config values.
+    def configure(self, **kwargs):
+        """ set config values and configure this account.
 
         :param kwargs: name=value config settings for this account.
                        values need to be unicode.
         :returns: None
         """
-        for key, value in kwargs.items():
-            self.set_config(key, str(value))
+        for name, value in kwargs.items():
+            self.set_config(name, value)
+        lib.dc_configure(self._dc_context)
 
     def is_configured(self):
         """ determine if the account is configured already; an initial connection
@@ -160,7 +134,7 @@ class Account(object):
 
         :returns: True if account is configured.
         """
-        return True if lib.dc_is_configured(self._dc_context) else False
+        return lib.dc_is_configured(self._dc_context)
 
     def set_avatar(self, img_path):
         """Set self avatar.
@@ -190,6 +164,11 @@ class Account(object):
             raise ValueError("no flags set")
         lib.dc_empty_server(self._dc_context, flags)
 
+    def get_infostring(self):
+        """ return info of the configured account. """
+        self.check_is_configured()
+        return from_dc_charpointer(lib.dc_get_info(self._dc_context))
+
     def get_latest_backupfile(self, backupdir):
         """ return the latest backup file in a given directory.
         """
@@ -211,43 +190,23 @@ class Account(object):
 
         :returns: :class:`deltachat.contact.Contact`
         """
-        return Contact(self, const.DC_CONTACT_ID_SELF)
+        self.check_is_configured()
+        return Contact(self._dc_context, const.DC_CONTACT_ID_SELF)
 
-    def create_contact(self, obj, name=None):
-        """ create a (new) Contact or return an existing one.
+    def create_contact(self, email, name=None):
+        """ create a (new) Contact. If there already is a Contact
+        with that e-mail address, it is unblocked and its name is
+        updated.
 
-        Calling this method will always resulut in the same
-        underlying contact id.  If there already is a Contact
-        with that e-mail address, it is unblocked and its display
-        `name` is updated if specified.
-
-        :param obj: email-address, Account or Contact instance.
-        :param name: (optional) display name for this contact
+        :param email: email-address (text type)
+        :param name: display name for this contact (optional)
         :returns: :class:`deltachat.contact.Contact` instance.
         """
-        if isinstance(obj, Account):
-            if not obj.is_configured():
-                raise ValueError("can only add addresses from configured accounts")
-            addr, displayname = obj.get_config("addr"), obj.get_config("displayname")
-        elif isinstance(obj, Contact):
-            if obj.account != self:
-                raise ValueError("account mismatch {}".format(obj))
-            addr, displayname = obj.addr, obj.name
-        elif isinstance(obj, str):
-            displayname, addr = parseaddr(obj)
-        else:
-            raise TypeError("don't know how to create chat for %r" % (obj, ))
-
-        if name is None and displayname:
-            name = displayname
-        return self._create_contact(addr, name)
-
-    def _create_contact(self, addr, name):
-        addr = as_dc_charpointer(addr)
         name = as_dc_charpointer(name)
-        contact_id = lib.dc_create_contact(self._dc_context, name, addr)
-        assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL, contact_id
-        return Contact(self, contact_id)
+        email = as_dc_charpointer(email)
+        contact_id = lib.dc_create_contact(self._dc_context, name, email)
+        assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
+        return Contact(self._dc_context, contact_id)
 
     def delete_contact(self, contact):
         """ delete a Contact.
@@ -256,25 +215,10 @@ class Account(object):
         :returns: True if deletion succeeded (contact was deleted)
         """
         contact_id = contact.id
-        assert contact.account == self
+        assert contact._dc_context == self._dc_context
         assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
         return bool(lib.dc_delete_contact(self._dc_context, contact_id))
 
-    def get_contact_by_addr(self, email):
-        """ get a contact for the email address or None if it's blocked or doesn't exist. """
-        _, addr = parseaddr(email)
-        addr = as_dc_charpointer(addr)
-        contact_id = lib.dc_lookup_contact_id_by_addr(self._dc_context, addr)
-        if contact_id:
-            return self.get_contact_by_id(contact_id)
-
-    def get_contact_by_id(self, contact_id):
-        """ return Contact instance or None.
-        :param contact_id: integer id of this contact.
-        :returns: None or :class:`deltachat.contact.Contact` instance.
-        """
-        return Contact(self, contact_id)
-
     def get_contacts(self, query=None, with_self=False, only_verified=False):
         """ get a (filtered) list of contacts.
 
@@ -294,39 +238,52 @@ class Account(object):
             lib.dc_get_contacts(self._dc_context, flags, query),
             lib.dc_array_unref
         )
-        return list(iter_array(dc_array, lambda x: Contact(self, x)))
+        return list(iter_array(dc_array, lambda x: Contact(self._dc_context, x)))
 
-    def get_fresh_messages(self):
-        """ yield all fresh messages from all chats. """
-        dc_array = ffi.gc(
-            lib.dc_get_fresh_msgs(self._dc_context),
-            lib.dc_array_unref
-        )
-        yield from iter_array(dc_array, lambda x: Message.from_db(self, x))
+    def create_chat_by_contact(self, contact):
+        """ create or get an existing 1:1 chat object for the specified contact or contact id.
 
-    def create_chat(self, obj):
-        """ Create a 1:1 chat with Account, Contact or e-mail address. """
-        return self.create_contact(obj).create_chat()
+        :param contact: chat_id (int) or contact object.
+        :returns: a :class:`deltachat.chat.Chat` object.
+        """
+        if hasattr(contact, "id"):
+            if contact._dc_context != self._dc_context:
+                raise ValueError("Contact belongs to a different Account")
+            contact_id = contact.id
+        else:
+            assert isinstance(contact, int)
+            contact_id = contact
+        chat_id = lib.dc_create_chat_by_contact_id(self._dc_context, contact_id)
+        return Chat(self, chat_id)
 
-    def _create_chat_by_message_id(self, msg_id):
-        return Chat(self, lib.dc_create_chat_by_msg_id(self._dc_context, msg_id))
+    def create_chat_by_message(self, message):
+        """ create or get an existing chat object for the
+        the specified message.
 
-    def create_group_chat(self, name, contacts=None, verified=False):
+        :param message: messsage id or message instance.
+        :returns: a :class:`deltachat.chat.Chat` object.
+        """
+        if hasattr(message, "id"):
+            if self._dc_context != message._dc_context:
+                raise ValueError("Message belongs to a different Account")
+            msg_id = message.id
+        else:
+            assert isinstance(message, int)
+            msg_id = message
+        chat_id = lib.dc_create_chat_by_msg_id(self._dc_context, msg_id)
+        return Chat(self, chat_id)
+
+    def create_group_chat(self, name, verified=False):
         """ create a new group chat object.
 
         Chats are unpromoted until the first message is sent.
 
-        :param contacts: list of contacts to add
         :param verified: if true only verified contacts can be added.
         :returns: a :class:`deltachat.chat.Chat` object.
         """
         bytes_name = name.encode("utf8")
         chat_id = lib.dc_create_group_chat(self._dc_context, int(verified), bytes_name)
-        chat = Chat(self, chat_id)
-        if contacts is not None:
-            for contact in contacts:
-                chat.add_contact(contact)
-        return chat
+        return Chat(self, chat_id)
 
     def get_chats(self):
         """ return list of chats.
@@ -399,35 +356,45 @@ class Account(object):
         lib.dc_delete_msgs(self._dc_context, msg_ids, len(msg_ids))
 
     def export_self_keys(self, path):
-        """ export public and private keys to the specified directory.
-
-        Note that the account does not have to be started.
-        """
+        """ export public and private keys to the specified directory. """
         return self._export(path, imex_cmd=1)
 
     def export_all(self, path):
         """return new file containing a backup of all database state
         (chats, contacts, keys, media, ...). The file is created in the
         the `path` directory.
-
-        Note that the account does not have to be started.
         """
         export_files = self._export(path, 11)
         if len(export_files) != 1:
             raise RuntimeError("found more than one new file")
         return export_files[0]
 
+    def _imex_events_clear(self):
+        try:
+            while True:
+                self._imex_events.get_nowait()
+        except Empty:
+            pass
+
     def _export(self, path, imex_cmd):
-        with self.temp_plugin(ImexTracker()) as imex_tracker:
-            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
-            return imex_tracker.wait_finish()
+        self._imex_events_clear()
+        lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
+        if not self._threads.is_started():
+            lib.dc_perform_imap_jobs(self._dc_context)
+        files_written = []
+        while True:
+            ev = self._imex_events.get()
+            if isinstance(ev, str):
+                files_written.append(ev)
+            elif isinstance(ev, bool):
+                if not ev:
+                    raise ValueError("export failed, exp-files: {}".format(files_written))
+                return files_written
 
     def import_self_keys(self, path):
         """ Import private keys found in the `path` directory.
         The last imported key is made the default keys unless its name
         contains the string legacy. Public keys are not imported.
-
-        Note that the account does not have to be started.
         """
         self._import(path, imex_cmd=2)
 
@@ -440,9 +407,12 @@ class Account(object):
         self._import(path, imex_cmd=12)
 
     def _import(self, path, imex_cmd):
-        with self.temp_plugin(ImexTracker()) as imex_tracker:
-            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
-            imex_tracker.wait_finish()
+        self._imex_events_clear()
+        lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
+        if not self._threads.is_started():
+            lib.dc_perform_imap_jobs(self._dc_context)
+        if not self._imex_events.get():
+            raise ValueError("import from path '{}' failed".format(path))
 
     def initiate_key_transfer(self):
         """return setup code after a Autocrypt setup message
@@ -450,8 +420,8 @@ class Account(object):
         If sending out was unsuccessful, a RuntimeError is raised.
         """
         self.check_is_configured()
-        if not self.is_started():
-            raise RuntimeError("IO not running, can not send out")
+        if not self._threads.is_started():
+            raise RuntimeError("threads not running, can not send out")
         res = lib.dc_initiate_key_transfer(self._dc_context)
         if res == ffi.NULL:
             raise RuntimeError("could not send out autocrypt setup message")
@@ -507,6 +477,63 @@ class Account(object):
             raise ValueError("could not join group")
         return Chat(self, chat_id)
 
+    def stop_ongoing(self):
+        lib.dc_stop_ongoing_process(self._dc_context)
+
+    #
+    # meta API for start/stop and event based processing
+    #
+
+    def wait_next_incoming_message(self):
+        """ wait for and return next incoming message. """
+        ev = self._evlogger.get_matching("DC_EVENT_INCOMING_MSG")
+        return self.get_message_by_id(ev[2])
+
+    def start_threads(self, mvbox=False, sentbox=False):
+        """ start IMAP/SMTP threads (and configure account if it hasn't happened).
+
+        :raises: ValueError if 'addr' or 'mail_pw' are not configured.
+        :returns: None
+        """
+        if not self.is_configured():
+            self.configure()
+        self._threads.start(mvbox=mvbox, sentbox=sentbox)
+
+    def stop_threads(self, wait=True):
+        """ stop IMAP/SMTP threads. """
+        if self._threads.is_started():
+            self.stop_ongoing()
+            self._threads.stop(wait=wait)
+
+    def shutdown(self, wait=True):
+        """ stop threads and close and remove underlying dc_context and callbacks. """
+        if hasattr(self, "_dc_context") and hasattr(self, "_threads"):
+            # print("SHUTDOWN", self)
+            self.stop_threads(wait=False)
+            lib.dc_close(self._dc_context)
+            self.stop_threads(wait=wait)  # to wait for threads
+            deltachat.clear_context_callback(self._dc_context)
+            del self._dc_context
+            atexit.unregister(self.shutdown)
+
+    def _process_event(self, ctx, evt_name, data1, data2):
+        assert ctx == self._dc_context
+        if hasattr(self, "_evlogger"):
+            self._evlogger(evt_name, data1, data2)
+            method = getattr(self, "on_" + evt_name.lower(), None)
+            if method is not None:
+                method(data1, data2)
+        return 0
+
+    def on_dc_event_imex_progress(self, data1, data2):
+        if data1 == 1000:
+            self._imex_events.put(True)
+        elif data1 == 0:
+            self._imex_events.put(False)
+
+    def on_dc_event_imex_file_written(self, data1, data2):
+        self._imex_events.put(data1)
+
     def set_location(self, latitude=0.0, longitude=0.0, accuracy=0.0):
         """set a new location. It effects all chats where we currently
         have enabled location streaming.
@@ -521,104 +548,160 @@ class Account(object):
         if dc_res == 0:
             raise ValueError("no chat is streaming locations")
 
-    #
-    # meta API for start/stop and event based processing
-    #
 
-    def add_account_plugin(self, plugin, name=None):
-        """ add an account plugin which implements one or more of
-        the :class:`deltachat.hookspec.PerAccount` hooks.
-        """
-        self._pm.register(plugin, name=name)
-        self._pm.check_pending()
-        return plugin
-
-    def remove_account_plugin(self, plugin, name=None):
-        """ remove an account plugin. """
-        self._pm.unregister(plugin, name=name)
-
-    @contextmanager
-    def temp_plugin(self, plugin):
-        """ run a with-block with the given plugin temporarily registered. """
-        self._pm.register(plugin)
-        yield plugin
-        self._pm.unregister(plugin)
-
-    def stop_ongoing(self):
-        """ Stop ongoing securejoin, configuration or other core jobs. """
-        lib.dc_stop_ongoing_process(self._dc_context)
-
-    def start_io(self):
-        """ start this account's IO scheduling (Rust-core async scheduler)
-
-        If this account is not configured an Exception is raised.
-        You need to call account.configure() and account.wait_configure_finish()
-        before.
-
-        You may call `stop_scheduler`, `wait_shutdown` or `shutdown` after the
-        account is started.
-
-        :raises MissingCredentials: if `addr` and `mail_pw` values are not set.
-        :raises ConfigureFailed: if the account could not be configured.
-
-        :returns: None
-        """
-        if not self.is_configured():
-            raise ValueError("account not configured, cannot start io")
-        lib.dc_start_io(self._dc_context)
-
-    def configure(self):
-        """ Start configuration process and return a Configtracker instance
-        on which you can block with wait_finish() to get a True/False success
-        value for the configuration process.
-        """
-        assert not self.is_configured()
-        if not self.get_config("addr") or not self.get_config("mail_pw"):
-            raise MissingCredentials("addr or mail_pwd not set in config")
-        configtracker = ConfigureTracker(self)
-        self.add_account_plugin(configtracker)
-        lib.dc_configure(self._dc_context)
-        return configtracker
+class IOThreads:
+    def __init__(self, dc_context, log_event=lambda *args: None):
+        self._dc_context = dc_context
+        self._thread_quitflag = False
+        self._name2thread = {}
+        self._log_event = log_event
 
     def is_started(self):
-        return self._event_thread.is_alive() and bool(lib.dc_is_io_running(self._dc_context))
+        return len(self._name2thread) > 0
 
-    def wait_shutdown(self):
-        """ wait until shutdown of this account has completed. """
-        self._shutdown_event.wait()
+    def start(self, imap=True, smtp=True, mvbox=False, sentbox=False):
+        assert not self.is_started()
+        if imap:
+            self._start_one_thread("inbox", self.imap_thread_run)
+        if mvbox:
+            self._start_one_thread("mvbox", self.mvbox_thread_run)
+        if sentbox:
+            self._start_one_thread("sentbox", self.sentbox_thread_run)
+        if smtp:
+            self._start_one_thread("smtp", self.smtp_thread_run)
 
-    def stop_io(self):
-        """ stop core IO scheduler if it is running. """
-        self.log("stop_ongoing")
-        self.stop_ongoing()
+    def _start_one_thread(self, name, func):
+        self._name2thread[name] = t = threading.Thread(target=func, name=name)
+        t.setDaemon(1)
+        t.start()
 
-        if bool(lib.dc_is_io_running(self._dc_context)):
-            self.log("dc_stop_io (stop core IO scheduler)")
-            lib.dc_stop_io(self._dc_context)
-        else:
-            self.log("stop_scheduler called on non-running context")
+    def stop(self, wait=False):
+        self._thread_quitflag = True
+        lib.dc_interrupt_imap_idle(self._dc_context)
+        lib.dc_interrupt_smtp_idle(self._dc_context)
+        lib.dc_interrupt_mvbox_idle(self._dc_context)
+        lib.dc_interrupt_sentbox_idle(self._dc_context)
+        if wait:
+            for name, thread in self._name2thread.items():
+                thread.join()
 
-    def shutdown(self):
-        """ shutdown and destroy account (stop callback thread, close and remove
-        underlying dc_context)."""
-        if self._dc_context is None:
+    def imap_thread_run(self):
+        self._log_event("py-bindings-info", 0, "INBOX THREAD START")
+        while not self._thread_quitflag:
+            lib.dc_perform_imap_jobs(self._dc_context)
+            if not self._thread_quitflag:
+                lib.dc_perform_imap_fetch(self._dc_context)
+            if not self._thread_quitflag:
+                lib.dc_perform_imap_idle(self._dc_context)
+        self._log_event("py-bindings-info", 0, "INBOX THREAD FINISHED")
+
+    def mvbox_thread_run(self):
+        self._log_event("py-bindings-info", 0, "MVBOX THREAD START")
+        while not self._thread_quitflag:
+            lib.dc_perform_mvbox_jobs(self._dc_context)
+            lib.dc_perform_mvbox_fetch(self._dc_context)
+            lib.dc_perform_mvbox_idle(self._dc_context)
+        self._log_event("py-bindings-info", 0, "MVBOX THREAD FINISHED")
+
+    def sentbox_thread_run(self):
+        self._log_event("py-bindings-info", 0, "SENTBOX THREAD START")
+        while not self._thread_quitflag:
+            lib.dc_perform_sentbox_jobs(self._dc_context)
+            lib.dc_perform_sentbox_fetch(self._dc_context)
+            lib.dc_perform_sentbox_idle(self._dc_context)
+        self._log_event("py-bindings-info", 0, "SENTBOX THREAD FINISHED")
+
+    def smtp_thread_run(self):
+        self._log_event("py-bindings-info", 0, "SMTP THREAD START")
+        while not self._thread_quitflag:
+            lib.dc_perform_smtp_jobs(self._dc_context)
+            lib.dc_perform_smtp_idle(self._dc_context)
+        self._log_event("py-bindings-info", 0, "SMTP THREAD FINISHED")
+
+
+class EventLogger:
+    _loglock = threading.RLock()
+
+    def __init__(self, dc_context, logid=None, debug=True):
+        self._dc_context = dc_context
+        self._event_queue = Queue()
+        self._debug = debug
+        if logid is None:
+            logid = str(self._dc_context).strip(">").split()[-1]
+        self.logid = logid
+        self._timeout = None
+        self.init_time = time.time()
+
+    def __call__(self, evt_name, data1, data2):
+        self._log_event(evt_name, data1, data2)
+        self._event_queue.put((evt_name, data1, data2))
+
+    def set_timeout(self, timeout):
+        self._timeout = timeout
+
+    def consume_events(self, check_error=True):
+        while not self._event_queue.empty():
+            self.get()
+
+    def get(self, timeout=None, check_error=True):
+        timeout = timeout or self._timeout
+        ev = self._event_queue.get(timeout=timeout)
+        if check_error and ev[0] == "DC_EVENT_ERROR":
+            raise ValueError("{}({!r},{!r})".format(*ev))
+        return ev
+
+    def ensure_event_not_queued(self, event_name_regex):
+        __tracebackhide__ = True
+        rex = re.compile("(?:{}).*".format(event_name_regex))
+        while 1:
+            try:
+                ev = self._event_queue.get(False)
+            except Empty:
+                break
+            else:
+                assert not rex.match(ev[0]), "event found {}".format(ev)
+
+    def get_matching(self, event_name_regex, check_error=True, timeout=None):
+        self._log("-- waiting for event with regex: {} --".format(event_name_regex))
+        rex = re.compile("(?:{}).*".format(event_name_regex))
+        while 1:
+            ev = self.get(timeout=timeout, check_error=check_error)
+            if rex.match(ev[0]):
+                return ev
+
+    def get_info_matching(self, regex):
+        rex = re.compile("(?:{}).*".format(regex))
+        while 1:
+            ev = self.get_matching("DC_EVENT_INFO")
+            if rex.match(ev[2]):
+                return ev
+
+    def _log_event(self, evt_name, data1, data2):
+        # don't show events that are anyway empty impls now
+        if evt_name == "DC_EVENT_GET_STRING":
             return
+        if self._debug:
+            evpart = "{}({!r},{!r})".format(evt_name, data1, data2)
+            self._log(evpart)
 
-        self.stop_io()
+    def _log(self, msg):
+        t = threading.currentThread()
+        tname = getattr(t, "name", t)
+        if tname == "MainThread":
+            tname = "MAIN"
+        with self._loglock:
+            print("{:2.2f} [{}-{}] {}".format(time.time() - self.init_time, tname, self.logid, msg))
 
-        self.log("remove dc_context references")
-        # the dc_context_unref triggers get_next_event to return ffi.NULL
-        # which in turns makes the event thread finish execution
-        self._dc_context = None
 
-        self.log("wait for event thread to finish")
-        self._event_thread.wait()
-
-        self._shutdown_event.set()
-
-        hook = hookspec.Global._get_plugin_manager().hook
-        hook.dc_account_after_shutdown(account=self)
-        self.log("shutdown finished")
+def _destroy_dc_context(dc_context, dc_context_unref=lib.dc_context_unref):
+    # destructor for dc_context
+    dc_context_unref(dc_context)
+    try:
+        deltachat.clear_context_callback(dc_context)
+    except (TypeError, AttributeError):
+        # we are deep into Python Interpreter shutdown,
+        # so no need to clear the callback context mapping.
+        pass
 
 
 class ScannedQRCode:

python/src/deltachat/chat.py

@@ -18,25 +18,24 @@ class Chat(object):
     """
 
     def __init__(self, account, id):
-        from .account import Account
-        assert isinstance(account, Account), repr(account)
         self.account = account
+        self._dc_context = account._dc_context
         self.id = id
 
     def __eq__(self, other):
         return self.id == getattr(other, "id", None) and \
-               self.account._dc_context == other.account._dc_context
+               self._dc_context == getattr(other, "_dc_context", None)
 
     def __ne__(self, other):
         return not (self == other)
 
     def __repr__(self):
-        return "<Chat id={} name={}>".format(self.id, self.get_name())
+        return "<Chat id={} name={} dc_context={}>".format(self.id, self.get_name(), self._dc_context)
 
     @property
     def _dc_chat(self):
         return ffi.gc(
-            lib.dc_get_chat(self.account._dc_context, self.id),
+            lib.dc_get_chat(self._dc_context, self.id),
             lib.dc_chat_unref
         )
 
@@ -48,20 +47,10 @@ class Chat(object):
         - does not delete messages on server
         - the chat or contact is not blocked, new message will arrive
         """
-        lib.dc_delete_chat(self.account._dc_context, self.id)
+        lib.dc_delete_chat(self._dc_context, self.id)
 
     # ------  chat status/metadata API ------------------------------
 
-    def is_group(self):
-        """ return true if this chat is a group chat.
-
-        :returns: True if chat is a group-chat, false if it's a contact 1:1 chat.
-        """
-        return lib.dc_chat_get_type(self._dc_chat) in (
-            const.DC_CHAT_TYPE_GROUP,
-            const.DC_CHAT_TYPE_VERIFIED_GROUP
-        )
-
     def is_deaddrop(self):
         """ return true if this chat is a deaddrop chat.
 
@@ -69,13 +58,6 @@ class Chat(object):
         """
         return self.id == const.DC_CHAT_ID_DEADDROP
 
-    def is_muted(self):
-        """ return true if this chat is muted.
-
-        :returns: True if chat is muted, False otherwise.
-        """
-        return lib.dc_chat_is_muted(self._dc_chat)
-
     def is_promoted(self):
         """ return True if this chat is promoted, i.e.
         the member contacts are aware of their membership,
@@ -102,45 +84,14 @@ class Chat(object):
     def set_name(self, name):
         """ set name of this chat.
 
-        :param name: as a unicode string.
+        :param: name as a unicode string.
         :returns: None
         """
         name = as_dc_charpointer(name)
-        return lib.dc_set_chat_name(self.account._dc_context, self.id, name)
-
-    def mute(self, duration=None):
-        """ mutes the chat
-
-        :param duration: Number of seconds to mute the chat for. None to mute until unmuted again.
-        :returns: None
-        """
-        if duration is None:
-            mute_duration = -1
-        else:
-            mute_duration = duration
-        ret = lib.dc_set_chat_mute_duration(self.account._dc_context, self.id, mute_duration)
-        if not bool(ret):
-            raise ValueError("Call to dc_set_chat_mute_duration failed")
-
-    def unmute(self):
-        """ unmutes the chat
-
-        :returns: None
-        """
-        ret = lib.dc_set_chat_mute_duration(self.account._dc_context, self.id, 0)
-        if not bool(ret):
-            raise ValueError("Failed to unmute chat")
-
-    def get_mute_duration(self):
-        """ Returns the number of seconds until the mute of this chat is lifted.
-
-        :param duration:
-        :returns: Returns the number of seconds the chat is still muted for. (0 for not muted, -1 forever muted)
-        """
-        return bool(lib.dc_chat_get_remaining_mute_duration(self.id))
+        return lib.dc_set_chat_name(self._dc_context, self.id, name)
 
     def get_type(self):
-        """ (deprecated) return type of this chat.
+        """ return type of this chat.
 
         :returns: one of const.DC_CHAT_TYPE_*
         """
@@ -153,7 +104,7 @@ class Chat(object):
         in a second channel (typically used by mobiles with QRcode-show + scan UX)
         where account.join_with_qrcode(qr) needs to be called.
         """
-        res = lib.dc_get_securejoin_qr(self.account._dc_context, self.id)
+        res = lib.dc_get_securejoin_qr(self._dc_context, self.id)
         return from_dc_charpointer(res)
 
     # ------  chat messaging API ------------------------------
@@ -175,7 +126,7 @@ class Chat(object):
             assert msg.id != 0
             # get a fresh copy of dc_msg, the core needs it
             msg = Message.from_db(self.account, msg.id)
-        sent_id = lib.dc_send_msg(self.account._dc_context, self.id, msg._dc_msg)
+        sent_id = lib.dc_send_msg(self._dc_context, self.id, msg._dc_msg)
         if sent_id == 0:
             raise ValueError("message could not be sent")
         # modify message in place to avoid bad state for the caller
@@ -190,7 +141,7 @@ class Chat(object):
         :returns: the resulting :class:`deltachat.message.Message` instance
         """
         msg = as_dc_charpointer(text)
-        msg_id = lib.dc_send_text_msg(self.account._dc_context, self.id, msg)
+        msg_id = lib.dc_send_text_msg(self._dc_context, self.id, msg)
         if msg_id == 0:
             raise ValueError("message could not be send, does chat exist?")
         return Message.from_db(self.account, msg_id)
@@ -205,7 +156,7 @@ class Chat(object):
         """
         msg = Message.new_empty(self.account, view_type="file")
         msg.set_file(path, mime_type)
-        sent_id = lib.dc_send_msg(self.account._dc_context, self.id, msg._dc_msg)
+        sent_id = lib.dc_send_msg(self._dc_context, self.id, msg._dc_msg)
         if sent_id == 0:
             raise ValueError("message could not be sent")
         return Message.from_db(self.account, sent_id)
@@ -220,7 +171,7 @@ class Chat(object):
         mime_type = mimetypes.guess_type(path)[0]
         msg = Message.new_empty(self.account, view_type="image")
         msg.set_file(path, mime_type)
-        sent_id = lib.dc_send_msg(self.account._dc_context, self.id, msg._dc_msg)
+        sent_id = lib.dc_send_msg(self._dc_context, self.id, msg._dc_msg)
         if sent_id == 0:
             raise ValueError("message could not be sent")
         return Message.from_db(self.account, sent_id)
@@ -231,7 +182,7 @@ class Chat(object):
         :param msg: the message to be prepared.
         :returns: :class:`deltachat.message.Message` instance.
         """
-        msg_id = lib.dc_prepare_msg(self.account._dc_context, self.id, msg._dc_msg)
+        msg_id = lib.dc_prepare_msg(self._dc_context, self.id, msg._dc_msg)
         if msg_id == 0:
             raise ValueError("message could not be prepared")
         # invalidate passed in message which is not safe to use anymore
@@ -267,7 +218,7 @@ class Chat(object):
         msg = Message.from_db(self.account, message.id)
 
         # pass 0 as chat-id because core-docs say it's ok when out-preparing
-        sent_id = lib.dc_send_msg(self.account._dc_context, 0, msg._dc_msg)
+        sent_id = lib.dc_send_msg(self._dc_context, 0, msg._dc_msg)
         if sent_id == 0:
             raise ValueError("message could not be sent")
         assert sent_id == msg.id
@@ -281,9 +232,9 @@ class Chat(object):
         :returns: None
         """
         if message is None:
-            lib.dc_set_draft(self.account._dc_context, self.id, ffi.NULL)
+            lib.dc_set_draft(self._dc_context, self.id, ffi.NULL)
         else:
-            lib.dc_set_draft(self.account._dc_context, self.id, message._dc_msg)
+            lib.dc_set_draft(self._dc_context, self.id, message._dc_msg)
 
     def get_draft(self):
         """ get draft message for this chat.
@@ -291,7 +242,7 @@ class Chat(object):
         :param message: a :class:`Message` instance
         :returns: Message object or None (if no draft available)
         """
-        x = lib.dc_get_draft(self.account._dc_context, self.id)
+        x = lib.dc_get_draft(self._dc_context, self.id)
         if x == ffi.NULL:
             return None
         dc_msg = ffi.gc(x, lib.dc_msg_unref)
@@ -303,7 +254,7 @@ class Chat(object):
         :returns: list of :class:`deltachat.message.Message` objects for this chat.
         """
         dc_array = ffi.gc(
-            lib.dc_get_chat_msgs(self.account._dc_context, self.id, 0, 0),
+            lib.dc_get_chat_msgs(self._dc_context, self.id, 0, 0),
             lib.dc_array_unref
         )
         return list(iter_array(dc_array, lambda x: Message.from_db(self.account, x)))
@@ -313,69 +264,60 @@ class Chat(object):
 
         :returns: number of fresh messages
         """
-        return lib.dc_get_fresh_msg_cnt(self.account._dc_context, self.id)
+        return lib.dc_get_fresh_msg_cnt(self._dc_context, self.id)
 
     def mark_noticed(self):
         """ mark all messages in this chat as noticed.
 
         Noticed messages are no longer fresh.
         """
-        return lib.dc_marknoticed_chat(self.account._dc_context, self.id)
+        return lib.dc_marknoticed_chat(self._dc_context, self.id)
 
     def get_summary(self):
         """ return dictionary with summary information. """
-        dc_res = lib.dc_chat_get_info_json(self.account._dc_context, self.id)
+        dc_res = lib.dc_chat_get_info_json(self._dc_context, self.id)
         s = from_dc_charpointer(dc_res)
         return json.loads(s)
 
     # ------  group management API ------------------------------
 
-    def add_contact(self, obj):
+    def add_contact(self, contact):
         """ add a contact to this chat.
 
-        :params obj: Contact, Account or e-mail address.
+        :params: contact object.
         :raises ValueError: if contact could not be added
         :returns: None
         """
-        contact = self.account.create_contact(obj)
-        ret = lib.dc_add_contact_to_chat(self.account._dc_context, self.id, contact.id)
+        ret = lib.dc_add_contact_to_chat(self._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not add contact {!r} to chat".format(contact))
-        return contact
 
-    def remove_contact(self, obj):
+    def remove_contact(self, contact):
         """ remove a contact from this chat.
 
-        :params obj: Contact, Account or e-mail address.
+        :params: contact object.
         :raises ValueError: if contact could not be removed
         :returns: None
         """
-        contact = self.account.create_contact(obj)
-        ret = lib.dc_remove_contact_from_chat(self.account._dc_context, self.id, contact.id)
+        ret = lib.dc_remove_contact_from_chat(self._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not remove contact {!r} from chat".format(contact))
 
     def get_contacts(self):
         """ get all contacts for this chat.
+        :params: contact object.
         :returns: list of :class:`deltachat.contact.Contact` objects for this chat
+
         """
         from .contact import Contact
         dc_array = ffi.gc(
-            lib.dc_get_chat_contacts(self.account._dc_context, self.id),
+            lib.dc_get_chat_contacts(self._dc_context, self.id),
             lib.dc_array_unref
         )
         return list(iter_array(
-            dc_array, lambda id: Contact(self.account, id))
+            dc_array, lambda id: Contact(self._dc_context, id))
         )
 
-    def num_contacts(self):
-        """ return number of contacts in this chat. """
-        dc_array = ffi.gc(
-            lib.dc_get_chat_contacts(self.account._dc_context, self.id),
-            lib.dc_array_unref
-        )
-        return lib.dc_array_get_cnt(dc_array)
-
     def set_profile_image(self, img_path):
         """Set group profile image.
 
@@ -388,7 +330,7 @@ class Chat(object):
         """
         assert os.path.exists(img_path), img_path
         p = as_dc_charpointer(img_path)
-        res = lib.dc_set_chat_profile_image(self.account._dc_context, self.id, p)
+        res = lib.dc_set_chat_profile_image(self._dc_context, self.id, p)
         if res != 1:
             raise ValueError("Setting Profile Image {!r} failed".format(p))
 
@@ -401,7 +343,7 @@ class Chat(object):
         :raises ValueError: if profile image could not be reset
         :returns: None
         """
-        res = lib.dc_set_chat_profile_image(self.account._dc_context, self.id, ffi.NULL)
+        res = lib.dc_set_chat_profile_image(self._dc_context, self.id, ffi.NULL)
         if res != 1:
             raise ValueError("Removing Profile Image failed")
 
@@ -425,26 +367,32 @@ class Chat(object):
         """
         return lib.dc_chat_get_color(self._dc_chat)
 
+    def get_subtitle(self):
+        """return the subtitle of the chat
+        :returns: the subtitle
+        """
+        return from_dc_charpointer(lib.dc_chat_get_subtitle(self._dc_chat))
+
     # ------  location streaming API ------------------------------
 
     def is_sending_locations(self):
         """return True if this chat has location-sending enabled currently.
         :returns: True if location sending is enabled.
         """
-        return lib.dc_is_sending_locations_to_chat(self.account._dc_context, self.id)
+        return lib.dc_is_sending_locations_to_chat(self._dc_context, self.id)
 
     def is_archived(self):
         """return True if this chat is archived.
         :returns: True if archived.
         """
-        return lib.dc_chat_get_visibility(self._dc_chat) == const.DC_CHAT_VISIBILITY_ARCHIVED
+        return lib.dc_chat_get_archived(self._dc_chat)
 
     def enable_sending_locations(self, seconds):
         """enable sending locations for this chat.
 
         all subsequent messages will carry a location with them.
         """
-        lib.dc_send_locations_to_chat(self.account._dc_context, self.id, seconds)
+        lib.dc_send_locations_to_chat(self._dc_context, self.id, seconds)
 
     def get_locations(self, contact=None, timestamp_from=None, timestamp_to=None):
         """return list of locations for the given contact in the given timespan.
@@ -468,7 +416,7 @@ class Chat(object):
         else:
             contact_id = contact.id
 
-        dc_array = lib.dc_get_locations(self.account._dc_context, self.id, contact_id, time_from, time_to)
+        dc_array = lib.dc_get_locations(self._dc_context, self.id, contact_id, time_from, time_to)
         return [
             Location(
                 latitude=lib.dc_array_get_latitude(dc_array, i),

python/src/deltachat/const.py

@@ -1,7 +1,193 @@
-from .capi import lib
+import sys
+import re
+import os
+from os.path import dirname, abspath
+from os.path import join as joinpath
+
+# the following const are generated from deltachat.h
+# this works well when you in a git-checkout
+# run "python deltachat/const.py" to regenerate events
+# begin const generated
+DC_GCL_ARCHIVED_ONLY = 0x01
+DC_GCL_NO_SPECIALS = 0x02
+DC_GCL_ADD_ALLDONE_HINT = 0x04
+DC_GCL_VERIFIED_ONLY = 0x01
+DC_GCL_ADD_SELF = 0x02
+DC_QR_ASK_VERIFYCONTACT = 200
+DC_QR_ASK_VERIFYGROUP = 202
+DC_QR_FPR_OK = 210
+DC_QR_FPR_MISMATCH = 220
+DC_QR_FPR_WITHOUT_ADDR = 230
+DC_QR_ADDR = 320
+DC_QR_TEXT = 330
+DC_QR_URL = 332
+DC_QR_ERROR = 400
+DC_CHAT_ID_DEADDROP = 1
+DC_CHAT_ID_TRASH = 3
+DC_CHAT_ID_MSGS_IN_CREATION = 4
+DC_CHAT_ID_STARRED = 5
+DC_CHAT_ID_ARCHIVED_LINK = 6
+DC_CHAT_ID_ALLDONE_HINT = 7
+DC_CHAT_ID_LAST_SPECIAL = 9
+DC_CHAT_TYPE_UNDEFINED = 0
+DC_CHAT_TYPE_SINGLE = 100
+DC_CHAT_TYPE_GROUP = 120
+DC_CHAT_TYPE_VERIFIED_GROUP = 130
+DC_MSG_ID_MARKER1 = 1
+DC_MSG_ID_DAYMARKER = 9
+DC_MSG_ID_LAST_SPECIAL = 9
+DC_STATE_UNDEFINED = 0
+DC_STATE_IN_FRESH = 10
+DC_STATE_IN_NOTICED = 13
+DC_STATE_IN_SEEN = 16
+DC_STATE_OUT_PREPARING = 18
+DC_STATE_OUT_DRAFT = 19
+DC_STATE_OUT_PENDING = 20
+DC_STATE_OUT_FAILED = 24
+DC_STATE_OUT_DELIVERED = 26
+DC_STATE_OUT_MDN_RCVD = 28
+DC_CONTACT_ID_SELF = 1
+DC_CONTACT_ID_INFO = 2
+DC_CONTACT_ID_DEVICE = 5
+DC_CONTACT_ID_LAST_SPECIAL = 9
+DC_MSG_TEXT = 10
+DC_MSG_IMAGE = 20
+DC_MSG_GIF = 21
+DC_MSG_STICKER = 23
+DC_MSG_AUDIO = 40
+DC_MSG_VOICE = 41
+DC_MSG_VIDEO = 50
+DC_MSG_FILE = 60
+DC_LP_AUTH_OAUTH2 = 0x2
+DC_LP_AUTH_NORMAL = 0x4
+DC_LP_IMAP_SOCKET_STARTTLS = 0x100
+DC_LP_IMAP_SOCKET_SSL = 0x200
+DC_LP_IMAP_SOCKET_PLAIN = 0x400
+DC_LP_SMTP_SOCKET_STARTTLS = 0x10000
+DC_LP_SMTP_SOCKET_SSL = 0x20000
+DC_LP_SMTP_SOCKET_PLAIN = 0x40000
+DC_CERTCK_AUTO = 0
+DC_CERTCK_STRICT = 1
+DC_CERTCK_ACCEPT_INVALID_CERTIFICATES = 3
+DC_EMPTY_MVBOX = 0x01
+DC_EMPTY_INBOX = 0x02
+DC_EVENT_INFO = 100
+DC_EVENT_SMTP_CONNECTED = 101
+DC_EVENT_IMAP_CONNECTED = 102
+DC_EVENT_SMTP_MESSAGE_SENT = 103
+DC_EVENT_IMAP_MESSAGE_DELETED = 104
+DC_EVENT_IMAP_MESSAGE_MOVED = 105
+DC_EVENT_IMAP_FOLDER_EMPTIED = 106
+DC_EVENT_NEW_BLOB_FILE = 150
+DC_EVENT_DELETED_BLOB_FILE = 151
+DC_EVENT_WARNING = 300
+DC_EVENT_ERROR = 400
+DC_EVENT_ERROR_NETWORK = 401
+DC_EVENT_ERROR_SELF_NOT_IN_GROUP = 410
+DC_EVENT_MSGS_CHANGED = 2000
+DC_EVENT_INCOMING_MSG = 2005
+DC_EVENT_MSG_DELIVERED = 2010
+DC_EVENT_MSG_FAILED = 2012
+DC_EVENT_MSG_READ = 2015
+DC_EVENT_CHAT_MODIFIED = 2020
+DC_EVENT_CONTACTS_CHANGED = 2030
+DC_EVENT_LOCATION_CHANGED = 2035
+DC_EVENT_CONFIGURE_PROGRESS = 2041
+DC_EVENT_IMEX_PROGRESS = 2051
+DC_EVENT_IMEX_FILE_WRITTEN = 2052
+DC_EVENT_SECUREJOIN_INVITER_PROGRESS = 2060
+DC_EVENT_SECUREJOIN_JOINER_PROGRESS = 2061
+DC_EVENT_SECUREJOIN_MEMBER_ADDED = 2062
+DC_EVENT_FILE_COPIED = 2055
+DC_EVENT_IS_OFFLINE = 2081
+DC_EVENT_GET_STRING = 2091
+DC_STR_SELFNOTINGRP = 21
+DC_PROVIDER_STATUS_OK = 1
+DC_PROVIDER_STATUS_PREPARATION = 2
+DC_PROVIDER_STATUS_BROKEN = 3
+DC_STR_NOMESSAGES = 1
+DC_STR_SELF = 2
+DC_STR_DRAFT = 3
+DC_STR_MEMBER = 4
+DC_STR_CONTACT = 6
+DC_STR_VOICEMESSAGE = 7
+DC_STR_DEADDROP = 8
+DC_STR_IMAGE = 9
+DC_STR_VIDEO = 10
+DC_STR_AUDIO = 11
+DC_STR_FILE = 12
+DC_STR_STATUSLINE = 13
+DC_STR_NEWGROUPDRAFT = 14
+DC_STR_MSGGRPNAME = 15
+DC_STR_MSGGRPIMGCHANGED = 16
+DC_STR_MSGADDMEMBER = 17
+DC_STR_MSGDELMEMBER = 18
+DC_STR_MSGGROUPLEFT = 19
+DC_STR_GIF = 23
+DC_STR_ENCRYPTEDMSG = 24
+DC_STR_E2E_AVAILABLE = 25
+DC_STR_ENCR_TRANSP = 27
+DC_STR_ENCR_NONE = 28
+DC_STR_CANTDECRYPT_MSG_BODY = 29
+DC_STR_FINGERPRINTS = 30
+DC_STR_READRCPT = 31
+DC_STR_READRCPT_MAILBODY = 32
+DC_STR_MSGGRPIMGDELETED = 33
+DC_STR_E2E_PREFERRED = 34
+DC_STR_CONTACT_VERIFIED = 35
+DC_STR_CONTACT_NOT_VERIFIED = 36
+DC_STR_CONTACT_SETUP_CHANGED = 37
+DC_STR_ARCHIVEDCHATS = 40
+DC_STR_STARREDMSGS = 41
+DC_STR_AC_SETUP_MSG_SUBJECT = 42
+DC_STR_AC_SETUP_MSG_BODY = 43
+DC_STR_SELFTALK_SUBTITLE = 50
+DC_STR_CANNOT_LOGIN = 60
+DC_STR_SERVER_RESPONSE = 61
+DC_STR_MSGACTIONBYUSER = 62
+DC_STR_MSGACTIONBYME = 63
+DC_STR_MSGLOCATIONENABLED = 64
+DC_STR_MSGLOCATIONDISABLED = 65
+DC_STR_LOCATION = 66
+DC_STR_STICKER = 67
+DC_STR_DEVICE_MESSAGES = 68
+DC_STR_COUNT = 68
+# end const generated
 
 
-for name in dir(lib):
-    if name.startswith("DC_"):
-        globals()[name] = getattr(lib, name)
-del name
+def read_event_defines(f):
+    rex = re.compile(r'#define\s+((?:DC_EVENT|DC_QR|DC_MSG|DC_LP|DC_EMPTY|DC_CERTCK|DC_STATE|DC_STR|'
+                     r'DC_CONTACT_ID|DC_GCL|DC_CHAT|DC_PROVIDER)_\S+)\s+([x\d]+).*')
+    for line in f:
+        m = rex.match(line)
+        if m:
+            yield m.groups()
+
+
+if __name__ == "__main__":
+    here = abspath(__file__).rstrip("oc")
+    here_dir = dirname(here)
+    if len(sys.argv) >= 2:
+        deltah = sys.argv[1]
+    else:
+        deltah = joinpath(dirname(dirname(dirname(here_dir))), "deltachat-ffi", "deltachat.h")
+    assert os.path.exists(deltah)
+
+    lines = []
+    skip_to_end = False
+    for orig_line in open(here):
+        if skip_to_end:
+            if not orig_line.startswith("# end const"):
+                continue
+            skip_to_end = False
+        lines.append(orig_line)
+        if orig_line.startswith("# begin const"):
+            with open(deltah) as f:
+                for name, item in read_event_defines(f):
+                    lines.append("{} = {}\n".format(name, item))
+            skip_to_end = True
+
+    tmpname = here + ".tmp"
+    with open(tmpname, "w") as f:
+        f.write("".join(lines))
+    os.rename(tmpname, here)

python/src/deltachat/contact.py

@@ -3,8 +3,6 @@
 from . import props
 from .cutil import from_dc_charpointer
 from .capi import lib, ffi
-from .chat import Chat
-from . import const
 
 
 class Contact(object):
@@ -12,25 +10,23 @@ class Contact(object):
 
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
-    def __init__(self, account, id):
-        from .account import Account
-        assert isinstance(account, Account), repr(account)
-        self.account = account
+    def __init__(self, dc_context, id):
+        self._dc_context = dc_context
         self.id = id
 
     def __eq__(self, other):
-        return self.account._dc_context == other.account._dc_context and self.id == other.id
+        return self._dc_context == other._dc_context and self.id == other.id
 
     def __ne__(self, other):
         return not (self == other)
 
     def __repr__(self):
-        return "<Contact id={} addr={} dc_context={}>".format(self.id, self.addr, self.account._dc_context)
+        return "<Contact id={} addr={} dc_context={}>".format(self.id, self.addr, self._dc_context)
 
     @property
     def _dc_contact(self):
         return ffi.gc(
-            lib.dc_get_contact(self.account._dc_context, self.id),
+            lib.dc_get_contact(self._dc_context, self.id),
             lib.dc_contact_unref
         )
 
@@ -40,13 +36,10 @@ class Contact(object):
         return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
 
     @props.with_doc
-    def name(self):
+    def display_name(self):
         """ display name for this contact. """
         return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
 
-    # deprecated alias
-    display_name = name
-
     def is_blocked(self):
         """ Return True if the contact is blocked. """
         return lib.dc_contact_is_blocked(self._dc_contact)
@@ -64,17 +57,3 @@ class Contact(object):
         if dc_res == ffi.NULL:
             return None
         return from_dc_charpointer(dc_res)
-
-    def create_chat(self):
-        """ create or get an existing 1:1 chat object for the specified contact or contact id.
-
-        :param contact: chat_id (int) or contact object.
-        :returns: a :class:`deltachat.chat.Chat` object.
-        """
-        dc_context = self.account._dc_context
-        chat_id = lib.dc_create_chat_by_contact_id(dc_context, self.id)
-        assert chat_id > const.DC_CHAT_ID_LAST_SPECIAL, chat_id
-        return Chat(self.account, chat_id)
-
-    # deprecated name
-    get_chat = create_chat

python/src/deltachat/direct_imap.py

@@ -1,213 +0,0 @@
-"""
-Internal Python-level IMAP handling used by the testplugin
-and for cleaning up inbox/mvbox for each test function run.
-"""
-
-import io
-import email
-import ssl
-import pathlib
-from imapclient import IMAPClient
-from imapclient.exceptions import IMAPClientError
-import deltachat
-
-
-SEEN = b'\\Seen'
-DELETED = b'\\Deleted'
-FLAGS = b'FLAGS'
-FETCH = b'FETCH'
-ALL = "1:*"
-
-
-@deltachat.global_hookimpl
-def dc_account_extra_configure(account):
-    """ Reset the account (we reuse accounts across tests)
-    and make 'account.direct_imap' available for direct IMAP ops.
-    """
-    imap = DirectImap(account)
-    if imap.select_config_folder("mvbox"):
-        imap.delete(ALL, expunge=True)
-    assert imap.select_config_folder("inbox")
-    imap.delete(ALL, expunge=True)
-    setattr(account, "direct_imap", imap)
-
-
-@deltachat.global_hookimpl
-def dc_account_after_shutdown(account):
-    """ shutdown the imap connection if there is one. """
-    imap = getattr(account, "direct_imap", None)
-    if imap is not None:
-        imap.shutdown()
-        del account.direct_imap
-
-
-class DirectImap:
-    def __init__(self, account):
-        self.account = account
-        self.logid = account.get_config("displayname") or id(account)
-        self._idling = False
-        self.connect()
-
-    def connect(self):
-        ssl_context = ssl.create_default_context()
-
-        # don't check if certificate hostname doesn't match target hostname
-        ssl_context.check_hostname = False
-
-        # don't check if the certificate is trusted by a certificate authority
-        ssl_context.verify_mode = ssl.CERT_NONE
-
-        host = self.account.get_config("configured_mail_server")
-        user = self.account.get_config("addr")
-        pw = self.account.get_config("mail_pw")
-        self.conn = IMAPClient(host, ssl_context=ssl_context)
-        self.conn.login(user, pw)
-
-        self.select_folder("INBOX")
-
-    def shutdown(self):
-        try:
-            self.conn.idle_done()
-        except (OSError, IMAPClientError):
-            pass
-        try:
-            self.conn.logout()
-        except (OSError, IMAPClientError):
-            print("Could not logout direct_imap conn")
-
-    def select_folder(self, foldername):
-        assert not self._idling
-        return self.conn.select_folder(foldername)
-
-    def select_config_folder(self, config_name):
-        """ Return info about selected folder if it is
-        configured, otherwise None. """
-        if "_" not in config_name:
-            config_name = "configured_{}_folder".format(config_name)
-        foldername = self.account.get_config(config_name)
-        if foldername:
-            return self.select_folder(foldername)
-
-    def list_folders(self):
-        """ return list of all existing folder names"""
-        assert not self._idling
-        folders = []
-        for meta, sep, foldername in self.conn.list_folders():
-            folders.append(foldername)
-        return folders
-
-    def delete(self, range, expunge=True):
-        """ delete a range of messages (imap-syntax).
-        If expunge is true, perform the expunge-operation
-        to make sure the messages are really gone and not
-        just flagged as deleted.
-        """
-        self.conn.set_flags(range, [DELETED])
-        if expunge:
-            self.conn.expunge()
-
-    def get_all_messages(self):
-        assert not self._idling
-        return self.conn.fetch(ALL, [FLAGS])
-
-    def get_unread_messages(self):
-        assert not self._idling
-        res = self.conn.fetch(ALL, [FLAGS])
-        return [uid for uid in res
-                if SEEN not in res[uid][FLAGS]]
-
-    def mark_all_read(self):
-        messages = self.get_unread_messages()
-        if messages:
-            res = self.conn.set_flags(messages, [SEEN])
-            print("marked seen:", messages, res)
-
-    def get_unread_cnt(self):
-        return len(self.get_unread_messages())
-
-    def dump_account_info(self, logfile):
-        def log(*args, **kwargs):
-            kwargs["file"] = logfile
-            print(*args, **kwargs)
-
-        cursor = 0
-        for name, val in self.account.get_info().items():
-            entry = "{}={}".format(name.upper(), val)
-            if cursor + len(entry) > 80:
-                log("")
-                cursor = 0
-            log(entry, end=" ")
-            cursor += len(entry) + 1
-        log("")
-
-    def dump_imap_structures(self, dir, logfile):
-        assert not self._idling
-        stream = io.StringIO()
-
-        def log(*args, **kwargs):
-            kwargs["file"] = stream
-            print(*args, **kwargs)
-
-        empty_folders = []
-        for imapfolder in self.list_folders():
-            self.select_folder(imapfolder)
-            messages = list(self.get_all_messages())
-            if not messages:
-                empty_folders.append(imapfolder)
-                continue
-
-            log("---------", imapfolder, len(messages), "messages ---------")
-            # get message content without auto-marking it as seen
-            # fetching 'RFC822' would mark it as seen.
-            requested = [b'BODY.PEEK[HEADER]', FLAGS]
-            for uid, data in self.conn.fetch(messages, requested).items():
-                body_bytes = data[b'BODY[HEADER]']
-                flags = data[FLAGS]
-                path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
-                path.mkdir(parents=True, exist_ok=True)
-                fn = path.joinpath(str(uid))
-                fn.write_bytes(body_bytes)
-                log("Message", uid, fn)
-                email_message = email.message_from_bytes(body_bytes)
-                log("Message", uid, flags, "Message-Id:", email_message.get("Message-Id"))
-
-        if empty_folders:
-            log("--------- EMPTY FOLDERS:", empty_folders)
-
-        print(stream.getvalue(), file=logfile)
-
-    def idle_start(self):
-        """ switch this connection to idle mode. non-blocking. """
-        assert not self._idling
-        res = self.conn.idle()
-        self._idling = True
-        return res
-
-    def idle_check(self, terminate=False):
-        """ (blocking) wait for next idle message from server. """
-        assert self._idling
-        self.account.log("imap-direct: calling idle_check")
-        res = self.conn.idle_check(timeout=30)
-        if len(res) == 0:
-            raise TimeoutError
-        if terminate:
-            self.idle_done()
-        return res
-
-    def idle_wait_for_seen(self):
-        """ Return first message with SEEN flag
-        from a running idle-stream REtiurn.
-        """
-        while 1:
-            for item in self.idle_check():
-                if item[1] == FETCH:
-                    if item[2][0] == FLAGS:
-                        if SEEN in item[2][1]:
-                            return item[0]
-
-    def idle_done(self):
-        """ send idle-done to server if we are currently in idle mode. """
-        if self._idling:
-            res = self.conn.idle_done()
-            self._idling = False
-            return res

python/src/deltachat/events.py

@@ -1,219 +0,0 @@
-import threading
-import time
-import re
-from queue import Queue, Empty
-
-import deltachat
-from .hookspec import account_hookimpl
-from contextlib import contextmanager
-from .capi import ffi, lib
-from .message import map_system_message
-from .cutil import from_dc_charpointer
-
-
-class FFIEvent:
-    def __init__(self, name, data1, data2):
-        self.name = name
-        self.data1 = data1
-        self.data2 = data2
-
-    def __str__(self):
-        return "{name} data1={data1} data2={data2}".format(**self.__dict__)
-
-
-class FFIEventLogger:
-    """ If you register an instance of this logger with an Account
-    you'll get all ffi-events printed.
-    """
-    # to prevent garbled logging
-    _loglock = threading.RLock()
-
-    def __init__(self, account):
-        self.account = account
-        self.logid = self.account.get_config("displayname")
-        self.init_time = time.time()
-
-    @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        self.account.log(str(ffi_event))
-
-    @account_hookimpl
-    def ac_log_line(self, message):
-        t = threading.currentThread()
-        tname = getattr(t, "name", t)
-        if tname == "MainThread":
-            tname = "MAIN"
-        elapsed = time.time() - self.init_time
-        locname = tname
-        if self.logid:
-            locname += "-" + self.logid
-        s = "{:2.2f} [{}] {}".format(elapsed, locname, message)
-        with self._loglock:
-            print(s, flush=True)
-
-
-class FFIEventTracker:
-    def __init__(self, account, timeout=None):
-        self.account = account
-        self._timeout = timeout
-        self._event_queue = Queue()
-
-    @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        self._event_queue.put(ffi_event)
-
-    def set_timeout(self, timeout):
-        self._timeout = timeout
-
-    def consume_events(self, check_error=True):
-        while not self._event_queue.empty():
-            self.get(check_error=check_error)
-
-    def get(self, timeout=None, check_error=True):
-        timeout = timeout if timeout is not None else self._timeout
-        ev = self._event_queue.get(timeout=timeout)
-        if check_error and ev.name == "DC_EVENT_ERROR":
-            raise ValueError("unexpected event: {}".format(ev))
-        return ev
-
-    def iter_events(self, timeout=None, check_error=True):
-        while 1:
-            yield self.get(timeout=timeout, check_error=check_error)
-
-    def get_matching(self, event_name_regex, check_error=True, timeout=None):
-        rex = re.compile("(?:{}).*".format(event_name_regex))
-        for ev in self.iter_events(timeout=timeout, check_error=check_error):
-            if rex.match(ev.name):
-                return ev
-
-    def get_info_matching(self, regex):
-        rex = re.compile("(?:{}).*".format(regex))
-        while 1:
-            ev = self.get_matching("DC_EVENT_INFO")
-            if rex.match(ev.data2):
-                return ev
-
-    def ensure_event_not_queued(self, event_name_regex):
-        __tracebackhide__ = True
-        rex = re.compile("(?:{}).*".format(event_name_regex))
-        while 1:
-            try:
-                ev = self._event_queue.get(False)
-            except Empty:
-                break
-            else:
-                assert not rex.match(ev.name), "event found {}".format(ev)
-
-    def wait_securejoin_inviter_progress(self, target):
-        while 1:
-            event = self.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS")
-            if event.data2 >= target:
-                print("** SECUREJOINT-INVITER PROGRESS {}".format(target), self.account)
-                break
-
-    def wait_next_incoming_message(self):
-        """ wait for and return next incoming message. """
-        ev = self.get_matching("DC_EVENT_INCOMING_MSG")
-        return self.account.get_message_by_id(ev.data2)
-
-    def wait_next_messages_changed(self):
-        """ wait for and return next message-changed message or None
-        if the event contains no msgid"""
-        ev = self.get_matching("DC_EVENT_MSGS_CHANGED")
-        if ev.data2 > 0:
-            return self.account.get_message_by_id(ev.data2)
-
-    def wait_msg_delivered(self, msg):
-        ev = self.get_matching("DC_EVENT_MSG_DELIVERED")
-        assert ev.data1 == msg.chat.id
-        assert ev.data2 == msg.id
-        assert msg.is_out_delivered()
-
-
-class EventThread(threading.Thread):
-    """ Event Thread for an account.
-
-    With each Account init this callback thread is started.
-    """
-    def __init__(self, account):
-        self.account = account
-        super(EventThread, self).__init__(name="events")
-        self.setDaemon(True)
-        self.start()
-
-    @contextmanager
-    def log_execution(self, message):
-        self.account.log(message + " START")
-        yield
-        self.account.log(message + " FINISHED")
-
-    def wait(self):
-        if self == threading.current_thread():
-            # we are in the callback thread and thus cannot
-            # wait for the thread-loop to finish.
-            return
-        self.join()
-
-    def run(self):
-        """ get and run events until shutdown. """
-        with self.log_execution("EVENT THREAD"):
-            self._inner_run()
-
-    def _inner_run(self):
-        event_emitter = ffi.gc(
-            lib.dc_get_event_emitter(self.account._dc_context),
-            lib.dc_event_emitter_unref,
-        )
-        while 1:
-            event = lib.dc_get_next_event(event_emitter)
-            if event == ffi.NULL:
-                break
-            evt = lib.dc_event_get_id(event)
-            data1 = lib.dc_event_get_data1_int(event)
-            # the following code relates to the deltachat/_build.py's helper
-            # function which provides us signature info of an event call
-            evt_name = deltachat.get_dc_event_name(evt)
-            if lib.dc_event_has_string_data(evt):
-                data2 = from_dc_charpointer(lib.dc_event_get_data2_str(event))
-            else:
-                data2 = lib.dc_event_get_data2_int(event)
-
-            lib.dc_event_unref(event)
-            ffi_event = FFIEvent(name=evt_name, data1=data1, data2=data2)
-            try:
-                self.account._pm.hook.ac_process_ffi_event(account=self, ffi_event=ffi_event)
-                for name, kwargs in self._map_ffi_event(ffi_event):
-                    self.account.log("calling hook name={} kwargs={}".format(name, kwargs))
-                    hook = getattr(self.account._pm.hook, name)
-                    hook(**kwargs)
-            except Exception:
-                if self.account._dc_context is not None:
-                    raise
-
-    def _map_ffi_event(self, ffi_event):
-        name = ffi_event.name
-        account = self.account
-        if name == "DC_EVENT_CONFIGURE_PROGRESS":
-            data1 = ffi_event.data1
-            if data1 == 0 or data1 == 1000:
-                success = data1 == 1000
-                yield "ac_configure_completed", dict(success=success)
-        elif name == "DC_EVENT_INCOMING_MSG":
-            msg = account.get_message_by_id(ffi_event.data2)
-            yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))
-        elif name == "DC_EVENT_MSGS_CHANGED":
-            if ffi_event.data2 != 0:
-                msg = account.get_message_by_id(ffi_event.data2)
-                if msg.is_outgoing():
-                    res = map_system_message(msg)
-                    if res and res[0].startswith("ac_member"):
-                        yield res
-                    yield "ac_outgoing_message", dict(message=msg)
-                elif msg.is_in_fresh():
-                    yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))
-        elif name == "DC_EVENT_MSG_DELIVERED":
-            msg = account.get_message_by_id(ffi_event.data2)
-            yield "ac_message_delivered", dict(message=msg)
-        elif name == "DC_EVENT_CHAT_MODIFIED":
-            chat = account.get_chat_by_id(ffi_event.data1)
-            yield "ac_chat_modified", dict(chat=chat)

python/src/deltachat/hookspec.py

@@ -1,101 +0,0 @@
-""" Hooks for Python bindings to Delta Chat Core Rust CFFI"""
-
-import pluggy
-
-
-account_spec_name = "deltachat-account"
-account_hookspec = pluggy.HookspecMarker(account_spec_name)
-account_hookimpl = pluggy.HookimplMarker(account_spec_name)
-
-global_spec_name = "deltachat-global"
-global_hookspec = pluggy.HookspecMarker(global_spec_name)
-global_hookimpl = pluggy.HookimplMarker(global_spec_name)
-
-
-class PerAccount:
-    """ per-Account-instance hook specifications.
-
-    All hooks are executed in a dedicated Event thread.
-    Hooks are not allowed to block/last long as this
-    blocks overall event processing on the python side.
-    """
-    @classmethod
-    def _make_plugin_manager(cls):
-        pm = pluggy.PluginManager(account_spec_name)
-        pm.add_hookspecs(cls)
-        return pm
-
-    @account_hookspec
-    def ac_process_ffi_event(self, ffi_event):
-        """ process a CFFI low level events for a given account.
-
-        ffi_event has "name", "data1", "data2" values as specified
-        with `DC_EVENT_* <https://c.delta.chat/group__DC__EVENT.html>`_.
-
-        DANGER: this hook is executed from the callback invoked by core.
-        Hook implementations need to be short running and can typically
-        not call back into core because this would easily cause recursion issues.
-        """
-
-    @account_hookspec
-    def ac_log_line(self, message):
-        """ log a message related to the account. """
-
-    @account_hookspec
-    def ac_configure_completed(self, success):
-        """ Called after a configure process completed. """
-
-    @account_hookspec
-    def ac_incoming_message(self, message):
-        """ Called on any incoming message (to deaddrop or chat). """
-
-    @account_hookspec
-    def ac_outgoing_message(self, message):
-        """ Called on each outgoing message (both system and "normal")."""
-
-    @account_hookspec
-    def ac_message_delivered(self, message):
-        """ Called when an outgoing message has been delivered to SMTP. """
-
-    @account_hookspec
-    def ac_chat_modified(self, chat):
-        """ Chat was created or modified regarding membership, avatar, title. """
-
-    @account_hookspec
-    def ac_member_added(self, chat, contact, message):
-        """ Called for each contact added to an accepted chat. """
-
-    @account_hookspec
-    def ac_member_removed(self, chat, contact, message):
-        """ Called for each contact removed from a chat.  """
-
-
-class Global:
-    """ global hook specifications using a per-process singleton
-    plugin manager instance.
-
-    """
-    _plugin_manager = None
-
-    @classmethod
-    def _get_plugin_manager(cls):
-        if cls._plugin_manager is None:
-            cls._plugin_manager = pm = pluggy.PluginManager(global_spec_name)
-            pm.add_hookspecs(cls)
-        return cls._plugin_manager
-
-    @global_hookspec
-    def dc_account_init(self, account):
-        """ called when `Account::__init__()` function starts executing. """
-
-    @global_hookspec
-    def dc_account_extra_configure(self, account):
-        """ Called when account configuration successfully finished.
-
-        This hook can be used to perform extra work before
-        ac_configure_completed is called.
-        """
-
-    @global_hookspec
-    def dc_account_after_shutdown(self, account):
-        """ Called after the account has been shutdown. """

python/src/deltachat/message.py

@@ -16,7 +16,8 @@ class Message(object):
     """
     def __init__(self, account, dc_msg):
         self.account = account
-        assert isinstance(self.account._dc_context, ffi.CData)
+        self._dc_context = account._dc_context
+        assert isinstance(self._dc_context, ffi.CData)
         assert isinstance(dc_msg, ffi.CData)
         assert dc_msg != ffi.NULL
         self._dc_msg = dc_msg
@@ -27,11 +28,7 @@ class Message(object):
         return self.account == other.account and self.id == other.id
 
     def __repr__(self):
-        c = self.get_sender_contact()
-        typ = "outgoing" if self.is_outgoing() else "incoming"
-        return "<Message {} sys={} {} id={} sender={}/{} chat={}/{}>".format(
-            typ, self.is_system_message(), repr(self.text[:10]),
-            self.id, c.id, c.addr, self.chat.id, self.chat.get_name())
+        return "<Message id={} dc_context={}>".format(self.id, self._dc_context)
 
     @classmethod
     def from_db(cls, account, id):
@@ -53,20 +50,6 @@ class Message(object):
             lib.dc_msg_unref
         ))
 
-    def create_chat(self):
-        """ create or get an existing chat (group) object for this message.
-
-        If the message is a deaddrop contact request
-        the sender will become an accepted contact.
-
-        :returns: a :class:`deltachat.chat.Chat` object.
-        """
-        from .chat import Chat
-        chat_id = lib.dc_create_chat_by_msg_id(self.account._dc_context, self.id)
-        ctx = self.account._dc_context
-        self._dc_msg = ffi.gc(lib.dc_get_msg(ctx, self.id), lib.dc_msg_unref)
-        return Chat(self.account, chat_id)
-
     @props.with_doc
     def text(self):
         """unicode text of this messages (might be empty if not a text message). """
@@ -98,10 +81,6 @@ class Message(object):
         """mime type of the file (if it exists)"""
         return from_dc_charpointer(lib.dc_msg_get_filemime(self._dc_msg))
 
-    def is_system_message(self):
-        """ return True if this message is a system/info message. """
-        return bool(lib.dc_msg_is_info(self._dc_msg))
-
     def is_setup_message(self):
         """ return True if this message is a setup message. """
         return lib.dc_msg_is_setupmessage(self._dc_msg)
@@ -123,12 +102,12 @@ class Message(object):
 
         The text is multiline and may contain eg. the raw text of the message.
         """
-        return from_dc_charpointer(lib.dc_get_msg_info(self.account._dc_context, self.id))
+        return from_dc_charpointer(lib.dc_get_msg_info(self._dc_context, self.id))
 
     def continue_key_transfer(self, setup_code):
         """ extract key and use it as primary key for this account. """
         res = lib.dc_continue_key_transfer(
-                self.account._dc_context,
+                self._dc_context,
                 self.id,
                 as_dc_charpointer(setup_code)
         )
@@ -163,7 +142,7 @@ class Message(object):
         :returns: email-mime message object (with headers only, no body).
         """
         import email.parser
-        mime_headers = lib.dc_get_mime_headers(self.account._dc_context, self.id)
+        mime_headers = lib.dc_get_mime_headers(self._dc_context, self.id)
         if mime_headers:
             s = ffi.string(ffi.gc(mime_headers, lib.dc_str_unref))
             if isinstance(s, bytes):
@@ -180,13 +159,6 @@ class Message(object):
         chat_id = lib.dc_msg_get_chat_id(self._dc_msg)
         return Chat(self.account, chat_id)
 
-    def get_sender_chat(self):
-        """return the 1:1 chat with the sender of this message.
-
-        :returns: :class:`deltachat.chat.Chat` instance
-        """
-        return self.get_sender_contact().get_chat()
-
     def get_sender_contact(self):
         """return the contact of who wrote the message.
 
@@ -194,7 +166,7 @@ class Message(object):
         """
         from .contact import Contact
         contact_id = lib.dc_msg_get_from_id(self._dc_msg)
-        return Contact(self.account, contact_id)
+        return Contact(self._dc_context, contact_id)
 
     #
     # Message State query methods
@@ -206,7 +178,7 @@ class Message(object):
         else:
             # load message from db to get a fresh/current state
             dc_msg = ffi.gc(
-                lib.dc_get_msg(self.account._dc_context, self.id),
+                lib.dc_get_msg(self._dc_context, self.id),
                 lib.dc_msg_unref
             )
         return lib.dc_msg_get_state(dc_msg)
@@ -235,13 +207,6 @@ class Message(object):
         """
         return self._msgstate == const.DC_STATE_IN_SEEN
 
-    def is_outgoing(self):
-        """Return True if Message is outgoing. """
-        return self._msgstate in (
-            const.DC_STATE_OUT_PREPARING, const.DC_STATE_OUT_PENDING,
-            const.DC_STATE_OUT_FAILED, const.DC_STATE_OUT_MDN_RCVD,
-            const.DC_STATE_OUT_DELIVERED)
-
     def is_out_preparing(self):
         """Return True if Message is outgoing, but its file is being prepared.
         """
@@ -304,10 +269,6 @@ class Message(object):
         """ return True if it's a file message. """
         return self._view_type == const.DC_MSG_FILE
 
-    def mark_seen(self):
-        """ mark this message as seen. """
-        self.account.mark_seen_messages([self.id])
-
 
 # some code for handling DC_MSG_* view types
 
@@ -327,29 +288,3 @@ def get_viewtype_code_from_name(view_type_name):
             return code
     raise ValueError("message typecode not found for {!r}, "
                      "available {!r}".format(view_type_name, list(_view_type_mapping.values())))
-
-
-#
-# some helper code for turning system messages into hook events
-#
-
-def map_system_message(msg):
-    if msg.is_system_message():
-        res = parse_system_add_remove(msg.text)
-        if res:
-            contact = msg.account.get_contact_by_addr(res[1])
-            if contact:
-                d = dict(chat=msg.chat, contact=contact, message=msg)
-                return "ac_member_" + res[0], d
-
-
-def parse_system_add_remove(text):
-    # Member Me (x@y) removed by a@b.
-    # Member x@y removed by a@b
-    text = text.lower()
-    parts = text.split()
-    if parts[0] == "member":
-        if parts[2] in ("removed", "added"):
-            return parts[2], parts[1]
-        if parts[3] in ("removed", "added"):
-            return parts[3], parts[2].strip("()")

python/src/deltachat/provider.py

@@ -11,18 +11,27 @@ class ProviderNotFoundError(Exception):
 class Provider(object):
     """Provider information.
 
-    :param domain: The email to get the provider info for.
+    :param domain: The domain to get the provider info for, this is
+    normally the part following the `@` of the domain.
     """
 
-    def __init__(self, account, addr):
+    def __init__(self, domain):
         provider = ffi.gc(
-            lib.dc_provider_new_from_email(account._dc_context, as_dc_charpointer(addr)),
+            lib.dc_provider_new_from_domain(as_dc_charpointer(domain)),
             lib.dc_provider_unref,
         )
         if provider == ffi.NULL:
             raise ProviderNotFoundError("Provider not found")
         self._provider = provider
 
+    @classmethod
+    def from_email(cls, email):
+        """Create provider info from an email address.
+
+        :param email: Email address to get provider info for.
+        """
+        return cls(email.split('@')[-1])
+
     @property
     def overview_page(self):
         """URL to the overview page of the provider on providers.delta.chat."""
@@ -30,10 +39,21 @@ class Provider(object):
             lib.dc_provider_get_overview_page(self._provider))
 
     @property
-    def get_before_login_hints(self):
-        """Should be shown to the user on login."""
+    def name(self):
+        """The name of the provider."""
+        return from_dc_charpointer(lib.dc_provider_get_name(self._provider))
+
+    @property
+    def markdown(self):
+        """Content of the information page, formatted as markdown."""
         return from_dc_charpointer(
-            lib.dc_provider_get_before_login_hints(self._provider))
+            lib.dc_provider_get_markdown(self._provider))
+
+    @property
+    def status_date(self):
+        """The date the provider info was last updated, as a string."""
+        return from_dc_charpointer(
+            lib.dc_provider_get_status_date(self._provider))
 
     @property
     def status(self):

python/src/deltachat/testplugin.py

@@ -1,507 +0,0 @@
-from __future__ import print_function
-import os
-import sys
-import io
-import subprocess
-import queue
-import threading
-import fnmatch
-import time
-import weakref
-import tempfile
-
-import pytest
-import requests
-
-from . import Account, const
-from .capi import lib
-from .events import FFIEventLogger, FFIEventTracker
-from _pytest._code import Source
-from deltachat import direct_imap
-
-import deltachat
-
-
-def pytest_addoption(parser):
-    parser.addoption(
-        "--liveconfig", action="store", default=None,
-        help="a file with >=2 lines where each line "
-             "contains NAME=VALUE config settings for one account"
-    )
-    parser.addoption(
-        "--ignored", action="store_true",
-        help="Also run tests marked with the ignored marker",
-    )
-
-
-def pytest_configure(config):
-    cfg = config.getoption('--liveconfig')
-    if not cfg:
-        cfg = os.getenv('DCC_NEW_TMP_EMAIL')
-        if cfg:
-            config.option.liveconfig = cfg
-
-    # Make sure we don't get garbled output because threads keep running
-    # collect all ever created accounts in a weakref-set (so we don't
-    # keep objects unneccessarily alive) and enable/disable logging
-    # for each pytest test phase # (setup/call/teardown).
-    # Additionally make the acfactory use a logging/no-logging default.
-
-    class LoggingAspect:
-        def __init__(self):
-            self._accounts = weakref.WeakSet()
-
-        @deltachat.global_hookimpl
-        def dc_account_init(self, account):
-            self._accounts.add(account)
-
-        def disable_logging(self, item):
-            for acc in self._accounts:
-                acc.disable_logging()
-            acfactory = item.funcargs.get("acfactory")
-            if acfactory:
-                acfactory.set_logging_default(False)
-
-        def enable_logging(self, item):
-            for acc in self._accounts:
-                acc.enable_logging()
-            acfactory = item.funcargs.get("acfactory")
-            if acfactory:
-                acfactory.set_logging_default(True)
-
-        @pytest.hookimpl(hookwrapper=True)
-        def pytest_runtest_setup(self, item):
-            if item.get_closest_marker("ignored"):
-                if not item.config.getvalue("ignored"):
-                    pytest.skip("use --ignored to run this test")
-            self.enable_logging(item)
-            yield
-            self.disable_logging(item)
-
-        @pytest.hookimpl(hookwrapper=True)
-        def pytest_pyfunc_call(self, pyfuncitem):
-            self.enable_logging(pyfuncitem)
-            yield
-            self.disable_logging(pyfuncitem)
-
-        @pytest.hookimpl(hookwrapper=True)
-        def pytest_runtest_teardown(self, item):
-            self.enable_logging(item)
-            yield
-            self.disable_logging(item)
-
-    la = LoggingAspect()
-    config.pluginmanager.register(la)
-    deltachat.register_global_plugin(la)
-
-
-def pytest_report_header(config, startdir):
-    summary = []
-
-    t = tempfile.mktemp()
-    try:
-        ac = Account(t)
-        info = ac.get_info()
-        ac.shutdown()
-    finally:
-        os.remove(t)
-    summary.extend(['Deltachat core={} sqlite={} journal_mode={}'.format(
-         info['deltachat_core_version'],
-         info['sqlite_version'],
-         info['journal_mode'],
-     )])
-
-    cfg = config.option.liveconfig
-    if cfg:
-        if "?" in cfg:
-            url, token = cfg.split("?", 1)
-            summary.append('Liveconfig provider: {}?<token ommitted>'.format(url))
-        else:
-            summary.append('Liveconfig file: {}'.format(cfg))
-    return summary
-
-
-class SessionLiveConfigFromFile:
-    def __init__(self, fn):
-        self.fn = fn
-        self.configlist = []
-        for line in open(fn):
-            if line.strip() and not line.strip().startswith('#'):
-                d = {}
-                for part in line.split():
-                    name, value = part.split("=")
-                    d[name] = value
-                self.configlist.append(d)
-
-    def get(self, index):
-        return self.configlist[index]
-
-    def exists(self):
-        return bool(self.configlist)
-
-
-class SessionLiveConfigFromURL:
-    def __init__(self, url):
-        self.configlist = []
-        self.url = url
-
-    def get(self, index):
-        try:
-            return self.configlist[index]
-        except IndexError:
-            assert index == len(self.configlist), index
-            res = requests.post(self.url)
-            if res.status_code != 200:
-                pytest.skip("creating newtmpuser failed {!r}".format(res))
-            d = res.json()
-            config = dict(addr=d["email"], mail_pw=d["password"])
-            self.configlist.append(config)
-            return config
-
-    def exists(self):
-        return bool(self.configlist)
-
-
-@pytest.fixture(scope="session")
-def session_liveconfig(request):
-    liveconfig_opt = request.config.option.liveconfig
-    if liveconfig_opt:
-        if liveconfig_opt.startswith("http"):
-            return SessionLiveConfigFromURL(liveconfig_opt)
-        else:
-            return SessionLiveConfigFromFile(liveconfig_opt)
-
-
-@pytest.fixture
-def data(request):
-    class Data:
-        def __init__(self):
-            # trying to find test data heuristically
-            # because we are run from a dev-setup with pytest direct,
-            # through tox, and then maybe also from deltachat-binding
-            # users like "deltabot".
-            self.paths = [os.path.normpath(x) for x in [
-                os.path.join(os.path.dirname(request.fspath.strpath), "data"),
-                os.path.join(os.path.dirname(__file__), "..", "..", "..", "test-data")
-            ]]
-
-        def get_path(self, bn):
-            """ return path of file or None if it doesn't exist. """
-            for path in self.paths:
-                fn = os.path.join(path, *bn.split("/"))
-                if os.path.exists(fn):
-                    return fn
-            print("WARNING: path does not exist: {!r}".format(fn))
-
-        def read_path(self, bn, mode="r"):
-            fn = self.get_path(bn)
-            if fn is not None:
-                with open(fn, mode) as f:
-                    return f.read()
-
-    return Data()
-
-
-@pytest.fixture
-def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
-
-    class AccountMaker:
-        def __init__(self):
-            self.live_count = 0
-            self.offline_count = 0
-            self._finalizers = []
-            self._accounts = []
-            self.init_time = time.time()
-            self._generated_keys = ["alice", "bob", "charlie",
-                                    "dom", "elena", "fiona"]
-            self.set_logging_default(False)
-            deltachat.register_global_plugin(direct_imap)
-
-        def finalize(self):
-            while self._finalizers:
-                fin = self._finalizers.pop()
-                fin()
-
-            while self._accounts:
-                acc = self._accounts.pop()
-                acc.shutdown()
-                acc.disable_logging()
-            deltachat.unregister_global_plugin(direct_imap)
-
-        def make_account(self, path, logid, quiet=False):
-            ac = Account(path, logging=self._logging)
-            ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
-            ac.addr = ac.get_self_contact().addr
-            ac.set_config("displayname", logid)
-            if not quiet:
-                ac.add_account_plugin(FFIEventLogger(ac))
-            self._accounts.append(ac)
-            return ac
-
-        def set_logging_default(self, logging):
-            self._logging = bool(logging)
-
-        def get_unconfigured_account(self):
-            self.offline_count += 1
-            tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
-            ac._evtracker.init_time = self.init_time
-            ac._evtracker.set_timeout(2)
-            return ac
-
-        def _preconfigure_key(self, account, addr):
-            # Only set a key if we haven't used it yet for another account.
-            if self._generated_keys:
-                keyname = self._generated_keys.pop(0)
-                fname_pub = data.read_path("key/{name}-public.asc".format(name=keyname))
-                fname_sec = data.read_path("key/{name}-secret.asc".format(name=keyname))
-                if fname_pub and fname_sec:
-                    account._preconfigure_keypair(addr, fname_pub, fname_sec)
-                    return True
-                else:
-                    print("WARN: could not use preconfigured keys for {!r}".format(addr))
-
-        def get_configured_offline_account(self):
-            ac = self.get_unconfigured_account()
-
-            # do a pseudo-configured account
-            addr = "addr{}@offline.org".format(self.offline_count)
-            ac.set_config("addr", addr)
-            self._preconfigure_key(ac, addr)
-            lib.dc_set_config(ac._dc_context, b"configured_addr", addr.encode("ascii"))
-            ac.set_config("mail_pw", "123")
-            lib.dc_set_config(ac._dc_context, b"configured_mail_pw", b"123")
-            lib.dc_set_config(ac._dc_context, b"configured", b"1")
-            return ac
-
-        def get_online_config(self, pre_generated_key=True, quiet=False):
-            if not session_liveconfig:
-                pytest.skip("specify DCC_NEW_TMP_EMAIL or --liveconfig")
-            configdict = session_liveconfig.get(self.live_count)
-            self.live_count += 1
-            if "e2ee_enabled" not in configdict:
-                configdict["e2ee_enabled"] = "1"
-
-            # Enable strict certificate checks for online accounts
-            configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
-            configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
-
-            tmpdb = tmpdir.join("livedb%d" % self.live_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count), quiet=quiet)
-            if pre_generated_key:
-                self._preconfigure_key(ac, configdict['addr'])
-            ac._evtracker.init_time = self.init_time
-            ac._evtracker.set_timeout(30)
-            return ac, dict(configdict)
-
-        def get_online_configuring_account(self, mvbox=False, sentbox=False, move=False,
-                                           pre_generated_key=True, quiet=False, config={}):
-            ac, configdict = self.get_online_config(
-                pre_generated_key=pre_generated_key, quiet=quiet)
-            configdict.update(config)
-            configdict["mvbox_watch"] = str(int(mvbox))
-            configdict["mvbox_move"] = str(int(move))
-            configdict["sentbox_watch"] = str(int(sentbox))
-            ac.update_config(configdict)
-            ac._configtracker = ac.configure()
-            return ac
-
-        def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
-            ac1 = self.get_online_configuring_account(
-                pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
-            self.wait_configure_and_start_io()
-            return ac1
-
-        def get_two_online_accounts(self, move=False, quiet=False):
-            ac1 = self.get_online_configuring_account(move=move, quiet=quiet)
-            ac2 = self.get_online_configuring_account(quiet=quiet)
-            self.wait_configure_and_start_io()
-            return ac1, ac2
-
-        def get_many_online_accounts(self, num, move=True):
-            accounts = [self.get_online_configuring_account(move=move, quiet=True)
-                        for i in range(num)]
-            self.wait_configure_and_start_io()
-            for acc in accounts:
-                acc.add_account_plugin(FFIEventLogger(acc))
-            return accounts
-
-        def clone_online_account(self, account, pre_generated_key=True):
-            self.live_count += 1
-            tmpdb = tmpdir.join("livedb%d" % self.live_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
-            if pre_generated_key:
-                self._preconfigure_key(ac, account.get_config("addr"))
-            ac._evtracker.init_time = self.init_time
-            ac._evtracker.set_timeout(30)
-            ac.update_config(dict(
-                addr=account.get_config("addr"),
-                mail_pw=account.get_config("mail_pw"),
-                mvbox_watch=account.get_config("mvbox_watch"),
-                mvbox_move=account.get_config("mvbox_move"),
-                sentbox_watch=account.get_config("sentbox_watch"),
-            ))
-            ac._configtracker = ac.configure()
-            return ac
-
-        def wait_configure_and_start_io(self):
-            for acc in self._accounts:
-                if hasattr(acc, "_configtracker"):
-                    acc._configtracker.wait_finish()
-                    del acc._configtracker
-                acc.set_config("bcc_self", "0")
-                if acc.is_configured() and not acc.is_started():
-                    acc.start_io()
-                print("{}: {} account was successfully setup".format(
-                    acc.get_config("displayname"), acc.get_config("addr")))
-
-        def run_bot_process(self, module, ffi=True):
-            fn = module.__file__
-
-            bot_ac, bot_cfg = self.get_online_config()
-
-            # Avoid starting ac so we don't interfere with the bot operating on
-            # the same database.
-            self._accounts.remove(bot_ac)
-
-            args = [
-                sys.executable,
-                "-u",
-                fn,
-                "--email", bot_cfg["addr"],
-                "--password", bot_cfg["mail_pw"],
-                bot_ac.db_path,
-            ]
-            if ffi:
-                args.insert(-1, "--show-ffi")
-            print("$", " ".join(args))
-            popen = subprocess.Popen(
-                args=args,
-                stdin=subprocess.DEVNULL,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT,  # combine stdout/stderr in one stream
-                bufsize=0,                 # line buffering
-                close_fds=True,            # close all FDs other than 0/1/2
-                universal_newlines=True    # give back text
-            )
-            bot = BotProcess(popen, bot_cfg)
-            self._finalizers.append(bot.kill)
-            return bot
-
-        def dump_imap_summary(self, logfile):
-            for ac in self._accounts:
-                imap = getattr(ac, "direct_imap", None)
-                if imap is not None:
-                    try:
-                        imap.idle_done()
-                    except Exception:
-                        pass
-                    imap.dump_account_info(logfile=logfile)
-                    imap.dump_imap_structures(tmpdir, logfile=logfile)
-
-        def get_accepted_chat(self, ac1, ac2):
-            ac2.create_chat(ac1)
-            return ac1.create_chat(ac2)
-
-        def introduce_each_other(self, accounts, sending=True):
-            to_wait = []
-            for i, acc in enumerate(accounts):
-                for acc2 in accounts[i + 1:]:
-                    chat = self.get_accepted_chat(acc, acc2)
-                    if sending:
-                        chat.send_text("hi")
-                        to_wait.append(acc2)
-                        acc2.create_chat(acc).send_text("hi back")
-                        to_wait.append(acc)
-            for acc in to_wait:
-                acc._evtracker.wait_next_incoming_message()
-
-    am = AccountMaker()
-    request.addfinalizer(am.finalize)
-    yield am
-    if hasattr(request.node, "rep_call") and request.node.rep_call.failed:
-        logfile = io.StringIO()
-        am.dump_imap_summary(logfile=logfile)
-        print(logfile.getvalue())
-        # request.node.add_report_section("call", "imap-server-state", s)
-
-
-class BotProcess:
-    def __init__(self, popen, bot_cfg):
-        self.popen = popen
-        self.addr = bot_cfg["addr"]
-
-        # we read stdout as quickly as we can in a thread and make
-        # the (unicode) lines available for readers through a queue.
-        self.stdout_queue = queue.Queue()
-        self.stdout_thread = t = threading.Thread(target=self._run_stdout_thread, name="bot-stdout-thread")
-        t.setDaemon(1)
-        t.start()
-
-    def _run_stdout_thread(self):
-        try:
-            while 1:
-                line = self.popen.stdout.readline()
-                if not line:
-                    break
-                line = line.strip()
-                self.stdout_queue.put(line)
-                print("bot-stdout: ", line)
-        finally:
-            self.stdout_queue.put(None)
-
-    def kill(self):
-        self.popen.kill()
-
-    def wait(self, timeout=30):
-        self.popen.wait(timeout=timeout)
-
-    def fnmatch_lines(self, pattern_lines):
-        patterns = [x.strip() for x in Source(pattern_lines.rstrip()).lines if x.strip()]
-        for next_pattern in patterns:
-            print("+++FNMATCH:", next_pattern)
-            ignored = []
-            while 1:
-                line = self.stdout_queue.get(timeout=15)
-                if line is None:
-                    if ignored:
-                        print("BOT stdout terminated after these lines")
-                        for line in ignored:
-                            print(line)
-                    raise IOError("BOT stdout-thread terminated")
-                if fnmatch.fnmatch(line, next_pattern):
-                    print("+++MATCHED:", line)
-                    break
-                else:
-                    print("+++IGN:", line)
-                    ignored.append(line)
-
-
-@pytest.fixture
-def tmp_db_path(tmpdir):
-    return tmpdir.join("test.db").strpath
-
-
-@pytest.fixture
-def lp():
-    class Printer:
-        def sec(self, msg):
-            print()
-            print("=" * 10, msg, "=" * 10)
-
-        def step(self, msg):
-            print("-" * 5, "step " + msg, "-" * 5)
-
-    return Printer()
-
-
-@pytest.hookimpl(tryfirst=True, hookwrapper=True)
-def pytest_runtest_makereport(item, call):
-    # execute all other hooks to obtain the report object
-    outcome = yield
-    rep = outcome.get_result()
-
-    # set a report attribute for each phase of a call, which can
-    # be "setup", "call", "teardown"
-
-    setattr(item, "rep_" + rep.when, rep)

python/src/deltachat/tracker.py

@@ -1,90 +0,0 @@
-
-from queue import Queue
-from threading import Event
-
-from .hookspec import account_hookimpl, Global
-
-
-class ImexFailed(RuntimeError):
-    """ Exception for signalling that import/export operations failed."""
-
-
-class ImexTracker:
-    def __init__(self):
-        self._imex_events = Queue()
-
-    @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        if ffi_event.name == "DC_EVENT_IMEX_PROGRESS":
-            self._imex_events.put(ffi_event.data1)
-        elif ffi_event.name == "DC_EVENT_IMEX_FILE_WRITTEN":
-            self._imex_events.put(ffi_event.data2)
-
-    def wait_finish(self, progress_timeout=60):
-        """ Return list of written files, raise ValueError if ExportFailed. """
-        files_written = []
-        while True:
-            ev = self._imex_events.get(timeout=progress_timeout)
-            if isinstance(ev, str):
-                files_written.append(ev)
-            elif ev == 0:
-                raise ImexFailed("export failed, exp-files: {}".format(files_written))
-            elif ev == 1000:
-                return files_written
-
-
-class ConfigureFailed(RuntimeError):
-    """ Exception for signalling that configuration failed."""
-
-
-class ConfigureTracker:
-    ConfigureFailed = ConfigureFailed
-
-    def __init__(self, account):
-        self.account = account
-        self._configure_events = Queue()
-        self._smtp_finished = Event()
-        self._imap_finished = Event()
-        self._ffi_events = []
-        self._progress = Queue()
-        self._gm = Global._get_plugin_manager()
-
-    @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        self._ffi_events.append(ffi_event)
-        if ffi_event.name == "DC_EVENT_SMTP_CONNECTED":
-            self._smtp_finished.set()
-        elif ffi_event.name == "DC_EVENT_IMAP_CONNECTED":
-            self._imap_finished.set()
-        elif ffi_event.name == "DC_EVENT_CONFIGURE_PROGRESS":
-            self._progress.put(ffi_event.data1)
-
-    @account_hookimpl
-    def ac_configure_completed(self, success):
-        if success:
-            self._gm.hook.dc_account_extra_configure(account=self.account)
-        self._configure_events.put(success)
-        self.account.remove_account_plugin(self)
-
-    def wait_smtp_connected(self):
-        """ wait until smtp is configured. """
-        self._smtp_finished.wait()
-
-    def wait_imap_connected(self):
-        """ wait until smtp is configured. """
-        self._imap_finished.wait()
-
-    def wait_progress(self, data1=None):
-        while 1:
-            evdata = self._progress.get()
-            if data1 is None or evdata == data1:
-                break
-
-    def wait_finish(self):
-        """ wait until configure is completed.
-
-        Raise Exception if Configure failed
-        """
-        if not self._configure_events.get():
-            content = "\n".join(map(str, self._ffi_events))
-            raise ConfigureFailed(content)

python/tests/auditwheels.py

@@ -10,6 +10,4 @@ if __name__ == "__main__":
     for relpath in os.listdir(workspacedir):
         if relpath.startswith("deltachat"):
             p = os.path.join(workspacedir, relpath)
-            subprocess.check_call(
-                ["auditwheel", "repair", p, "-w", workspacedir,
-                 "--plat", "manylinux2014_x86_64"])
+            subprocess.check_call(["auditwheel", "repair", p, "-w", workspacedir])

python/tests/conftest.py

@@ -0,0 +1,273 @@
+from __future__ import print_function
+import os
+import pytest
+import requests
+import time
+from deltachat import Account
+from deltachat import const
+from deltachat.capi import lib
+import tempfile
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--liveconfig", action="store", default=None,
+        help="a file with >=2 lines where each line "
+             "contains NAME=VALUE config settings for one account"
+    )
+
+
+def pytest_configure(config):
+    cfg = config.getoption('--liveconfig')
+    if not cfg:
+        cfg = os.getenv('DCC_PY_LIVECONFIG')
+        if cfg:
+            config.option.liveconfig = cfg
+
+
+def pytest_report_header(config, startdir):
+    summary = []
+
+    t = tempfile.mktemp()
+    try:
+        ac = Account(t, eventlogging=False)
+        info = ac.get_info()
+        ac.shutdown()
+    finally:
+        os.remove(t)
+    summary.extend(['Deltachat core={} sqlite={}'.format(
+         info['deltachat_core_version'],
+         info['sqlite_version'],
+     )])
+
+    cfg = config.option.liveconfig
+    if cfg:
+        if "#" in cfg:
+            url, token = cfg.split("#", 1)
+            summary.append('Liveconfig provider: {}#<token ommitted>'.format(url))
+        else:
+            summary.append('Liveconfig file: {}'.format(cfg))
+    return summary
+
+
+@pytest.fixture(scope="session")
+def data():
+    class Data:
+        def __init__(self):
+            self.path = os.path.join(os.path.dirname(__file__), "data")
+
+        def get_path(self, bn):
+            fn = os.path.join(self.path, bn)
+            assert os.path.exists(fn)
+            return fn
+    return Data()
+
+
+class SessionLiveConfigFromFile:
+    def __init__(self, fn):
+        self.fn = fn
+        self.configlist = []
+        for line in open(fn):
+            if line.strip() and not line.strip().startswith('#'):
+                d = {}
+                for part in line.split():
+                    name, value = part.split("=")
+                    d[name] = value
+                self.configlist.append(d)
+
+    def get(self, index):
+        return self.configlist[index]
+
+    def exists(self):
+        return bool(self.configlist)
+
+
+class SessionLiveConfigFromURL:
+    def __init__(self, url, create_token):
+        self.configlist = []
+        self.url = url
+        self.create_token = create_token
+
+    def get(self, index):
+        try:
+            return self.configlist[index]
+        except IndexError:
+            assert index == len(self.configlist), index
+            res = requests.post(self.url, json={"token_create_user": int(self.create_token)})
+            if res.status_code != 200:
+                pytest.skip("creating newtmpuser failed {!r}".format(res))
+            d = res.json()
+            config = dict(addr=d["email"], mail_pw=d["password"])
+            self.configlist.append(config)
+            return config
+
+    def exists(self):
+        return bool(self.configlist)
+
+
+@pytest.fixture(scope="session")
+def session_liveconfig(request):
+    liveconfig_opt = request.config.option.liveconfig
+    if liveconfig_opt:
+        if liveconfig_opt.startswith("http"):
+            url, create_token = liveconfig_opt.split("#", 1)
+            return SessionLiveConfigFromURL(url, create_token)
+        else:
+            return SessionLiveConfigFromFile(liveconfig_opt)
+
+
+@pytest.fixture
+def acfactory(pytestconfig, tmpdir, request, session_liveconfig):
+
+    class AccountMaker:
+        def __init__(self):
+            self.live_count = 0
+            self.offline_count = 0
+            self._finalizers = []
+            self.init_time = time.time()
+
+        def finalize(self):
+            while self._finalizers:
+                fin = self._finalizers.pop()
+                fin()
+
+        def make_account(self, path, logid):
+            ac = Account(path, logid=logid)
+            self._finalizers.append(ac.shutdown)
+            return ac
+
+        def get_unconfigured_account(self):
+            self.offline_count += 1
+            tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
+            ac._evlogger.init_time = self.init_time
+            ac._evlogger.set_timeout(2)
+            return ac
+
+        def get_configured_offline_account(self):
+            ac = self.get_unconfigured_account()
+
+            # do a pseudo-configured account
+            addr = "addr{}@offline.org".format(self.offline_count)
+            ac.set_config("addr", addr)
+            lib.dc_set_config(ac._dc_context, b"configured_addr", addr.encode("ascii"))
+            ac.set_config("mail_pw", "123")
+            lib.dc_set_config(ac._dc_context, b"configured_mail_pw", b"123")
+            lib.dc_set_config(ac._dc_context, b"configured", b"1")
+            return ac
+
+        def peek_online_config(self):
+            if not session_liveconfig:
+                pytest.skip("specify DCC_PY_LIVECONFIG or --liveconfig")
+            return session_liveconfig.get(self.live_count)
+
+        def get_online_config(self):
+            if not session_liveconfig:
+                pytest.skip("specify DCC_PY_LIVECONFIG or --liveconfig")
+            configdict = session_liveconfig.get(self.live_count)
+            self.live_count += 1
+            if "e2ee_enabled" not in configdict:
+                configdict["e2ee_enabled"] = "1"
+
+            # Enable strict certificate checks for online accounts
+            configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+            configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+
+            tmpdb = tmpdir.join("livedb%d" % self.live_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
+            ac._evlogger.init_time = self.init_time
+            ac._evlogger.set_timeout(30)
+            return ac, dict(configdict)
+
+        def get_online_configuring_account(self, mvbox=False, sentbox=False):
+            ac, configdict = self.get_online_config()
+            ac.configure(**configdict)
+            ac.start_threads(mvbox=mvbox, sentbox=sentbox)
+            return ac
+
+        def get_one_online_account(self):
+            ac1 = self.get_online_configuring_account()
+            wait_successful_IMAP_SMTP_connection(ac1)
+            wait_configuration_progress(ac1, 1000)
+            return ac1
+
+        def get_two_online_accounts(self):
+            ac1 = self.get_online_configuring_account()
+            ac2 = self.get_online_configuring_account()
+            wait_successful_IMAP_SMTP_connection(ac1)
+            wait_configuration_progress(ac1, 1000)
+            wait_successful_IMAP_SMTP_connection(ac2)
+            wait_configuration_progress(ac2, 1000)
+            return ac1, ac2
+
+        def clone_online_account(self, account):
+            self.live_count += 1
+            tmpdb = tmpdir.join("livedb%d" % self.live_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
+            ac._evlogger.init_time = self.init_time
+            ac._evlogger.set_timeout(30)
+            ac.configure(addr=account.get_config("addr"), mail_pw=account.get_config("mail_pw"))
+            ac.start_threads()
+            return ac
+
+    am = AccountMaker()
+    request.addfinalizer(am.finalize)
+    return am
+
+
+@pytest.fixture
+def tmp_db_path(tmpdir):
+    return tmpdir.join("test.db").strpath
+
+
+@pytest.fixture
+def lp():
+    class Printer:
+        def sec(self, msg):
+            print()
+            print("=" * 10, msg, "=" * 10)
+
+        def step(self, msg):
+            print("-" * 5, "step " + msg, "-" * 5)
+    return Printer()
+
+
+def wait_configuration_progress(account, min_target, max_target=1001):
+    min_target = min(min_target, max_target)
+    while 1:
+        evt_name, data1, data2 = \
+            account._evlogger.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
+        if data1 >= min_target and data1 <= max_target:
+            print("** CONFIG PROGRESS {}".format(min_target), account)
+            break
+
+
+def wait_securejoin_inviter_progress(account, target):
+    while 1:
+        evt_name, data1, data2 = \
+            account._evlogger.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS")
+        if data2 >= target:
+            print("** SECUREJOINT-INVITER PROGRESS {}".format(target), account)
+            break
+
+
+def wait_successful_IMAP_SMTP_connection(account):
+    imap_ok = smtp_ok = False
+    while not imap_ok or not smtp_ok:
+        evt_name, data1, data2 = \
+            account._evlogger.get_matching("DC_EVENT_(IMAP|SMTP)_CONNECTED")
+        if evt_name == "DC_EVENT_IMAP_CONNECTED":
+            imap_ok = True
+            print("** IMAP OK", account)
+        if evt_name == "DC_EVENT_SMTP_CONNECTED":
+            smtp_ok = True
+            print("** SMTP OK", account)
+    print("** IMAP and SMTP logins successful", account)
+
+
+def wait_msgs_changed(account, chat_id, msg_id=None):
+    ev = account._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
+    assert ev[1] == chat_id
+    if msg_id is not None:
+        assert ev[2] == msg_id
+    return ev[2]

python/tests/data/key

@@ -1 +0,0 @@
-../../../test-data/key

python/tests/stress_test_db.py

@@ -1,135 +0,0 @@
-
-import time
-import threading
-import pytest
-import os
-from queue import Queue, Empty
-
-import deltachat
-
-
-def test_db_busy_error(acfactory, tmpdir):
-    starttime = time.time()
-    log_lock = threading.RLock()
-
-    def log(string):
-        with log_lock:
-            print("%3.2f %s" % (time.time() - starttime, string))
-
-    # make a number of accounts
-    accounts = acfactory.get_many_online_accounts(3, quiet=True)
-    log("created %s accounts" % len(accounts))
-
-    # put a bigfile into each account
-    for acc in accounts:
-        acc.bigfile = os.path.join(acc.get_blobdir(), "bigfile")
-        with open(acc.bigfile, "wb") as f:
-            f.write(b"01234567890"*1000_000)
-    log("created %s bigfiles" % len(accounts))
-
-    contact_addrs = [acc.get_self_contact().addr for acc in accounts]
-    chat = accounts[0].create_group_chat("stress-group")
-    for addr in contact_addrs[1:]:
-        chat.add_contact(chat.account.create_contact(addr))
-
-    # setup auto-responder bots which report back failures/actions
-    report_queue = Queue()
-
-    def report_func(replier, report_type, *report_args):
-        report_queue.put((replier, report_type, report_args))
-
-    # each replier receives all events and sends report events to receive_queue
-    repliers = []
-    for acc in accounts:
-        replier = AutoReplier(acc, log=log, num_send=500, num_bigfiles=5, report_func=report_func)
-        acc.add_account_plugin(replier)
-        repliers.append(replier)
-
-    # kick off message sending
-    # after which repliers will reply to each other
-    chat.send_text("hello")
-
-    alive_count = len(accounts)
-    while alive_count > 0:
-        try:
-            replier, report_type, report_args = report_queue.get(timeout=10)
-        except Empty:
-            log("timeout waiting for next event")
-            pytest.fail("timeout exceeded")
-        if report_type == ReportType.exit:
-            replier.log("EXIT")
-        elif report_type == ReportType.ffi_error:
-            replier.log("ERROR: {}".format(report_args[0]))
-        elif report_type == ReportType.message_echo:
-            continue
-        else:
-            raise ValueError("{} unknown report type {}, args={}".format(
-                addr, report_type, report_args
-            ))
-        alive_count -= 1
-        replier.log("shutting down")
-        replier.account.shutdown()
-        replier.log("shut down complete, remaining={}".format(alive_count))
-
-
-class ReportType:
-    exit = "exit"
-    ffi_error = "ffi-error"
-    message_echo = "message-echo"
-
-
-class AutoReplier:
-    def __init__(self, account, log, num_send, num_bigfiles, report_func):
-        self.account = account
-        self._log = log
-        self.report_func = report_func
-        self.num_send = num_send
-        self.num_bigfiles = num_bigfiles
-        self.current_sent = 0
-        self.addr = self.account.get_self_contact().addr
-
-        self._thread = threading.Thread(
-            name="Stats{}".format(self.account),
-            target=self.thread_stats
-        )
-        self._thread.setDaemon(True)
-        self._thread.start()
-
-    def log(self, message):
-        self._log("{} {}".format(self.addr, message))
-
-    def thread_stats(self):
-        # XXX later use, for now we just quit
-        return
-        while 1:
-            time.sleep(1.0)
-            break
-
-    @deltachat.account_hookimpl
-    def ac_incoming_message(self, message):
-        if self.current_sent >= self.num_send:
-            self.report_func(self, ReportType.exit)
-            return
-        message.create_chat()
-        message.mark_seen()
-        self.log("incoming message: {}".format(message))
-
-        self.current_sent += 1
-        # we are still alive, let's send a reply
-        if self.num_bigfiles and self.current_sent % (self.num_send / self.num_bigfiles) == 0:
-            message.chat.send_text("send big file as reply to: {}".format(message.text))
-            msg = message.chat.send_file(self.account.bigfile)
-        else:
-            msg = message.chat.send_text("got message id {}, small text reply".format(message.id))
-            assert msg.text
-        self.log("message-sent: {}".format(msg))
-        self.report_func(self, ReportType.message_echo)
-        if self.current_sent >= self.num_send:
-            self.report_func(self, ReportType.exit)
-            return
-
-    @deltachat.account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        self.log(ffi_event)
-        if ffi_event.name == "DC_EVENT_ERROR":
-            self.report_func(self, ReportType.ffi_error, ffi_event)

python/tests/test_increation.py

@@ -6,37 +6,19 @@ import shutil
 import pytest
 from filecmp import cmp
 
+from conftest import wait_configuration_progress, wait_msgs_changed
 from deltachat import const
 
 
-def wait_msg_delivered(account, msg_list):
-    """ wait for one or more MSG_DELIVERED events to match msg_list contents. """
-    msg_list = list(msg_list)
-    while msg_list:
-        ev = account._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
-        msg_list.remove((ev.data1, ev.data2))
-
-
-def wait_msgs_changed(account, msgs_list):
-    """ wait for one or more MSGS_CHANGED events to match msgs_list contents. """
-    account.log("waiting for msgs_list={}".format(msgs_list))
-    msgs_list = list(msgs_list)
-    while msgs_list:
-        ev = account._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
-        for i, (data1, data2) in enumerate(msgs_list):
-            if ev.data1 == data1:
-                if data2 is None or ev.data2 == data2:
-                    del msgs_list[i]
-                    break
-        else:
-            account.log("waiting mismatch data1={} data2={}".format(data1, data2))
-    return ev.data1, ev.data2
-
-
 class TestOnlineInCreation:
     def test_increation_not_blobdir(self, tmpdir, acfactory, lp):
-        ac1, ac2 = acfactory.get_two_online_accounts()
-        chat = ac1.create_chat(ac2)
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.get_online_configuring_account()
+        wait_configuration_progress(ac1, 1000)
+        wait_configuration_progress(ac2, 1000)
+
+        c2 = ac1.create_contact(email=ac2.get_config("addr"))
+        chat = ac1.create_chat_by_contact(c2)
 
         lp.sec("Creating in-creation file outside of blobdir")
         assert tmpdir.strpath != ac1.get_blobdir()
@@ -45,8 +27,13 @@ class TestOnlineInCreation:
             chat.prepare_message_file(src.strpath)
 
     def test_no_increation_copies_to_blobdir(self, tmpdir, acfactory, lp):
-        ac1, ac2 = acfactory.get_two_online_accounts()
-        chat = ac1.create_chat(ac2)
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.get_online_configuring_account()
+        wait_configuration_progress(ac1, 1000)
+        wait_configuration_progress(ac2, 1000)
+
+        c2 = ac1.create_contact(email=ac2.get_config("addr"))
+        chat = ac1.create_chat_by_contact(c2)
 
         lp.sec("Creating file outside of blobdir")
         assert tmpdir.strpath != ac1.get_blobdir()
@@ -58,10 +45,15 @@ class TestOnlineInCreation:
         assert os.path.exists(blob_src), "file.txt not copied to blobdir"
 
     def test_forward_increation(self, acfactory, data, lp):
-        ac1, ac2 = acfactory.get_two_online_accounts()
+        ac1 = acfactory.get_online_configuring_account()
+        ac2 = acfactory.get_online_configuring_account()
+        wait_configuration_progress(ac1, 1000)
+        wait_configuration_progress(ac2, 1000)
 
-        chat = ac1.create_chat(ac2)
-        wait_msgs_changed(ac1, [(0, 0)])  # why no chat id?
+        c2 = ac1.create_contact(email=ac2.get_config("addr"))
+        chat = ac1.create_chat_by_contact(c2)
+        assert chat.id >= const.DC_CHAT_ID_LAST_SPECIAL
+        wait_msgs_changed(ac1, 0, 0)  # why no chat id?
 
         lp.sec("create a message with a file in creation")
         orig = data.get_path("d.png")
@@ -70,16 +62,19 @@ class TestOnlineInCreation:
             fp.write("preparing")
         prepared_original = chat.prepare_message_file(path)
         assert prepared_original.is_out_preparing()
-        wait_msgs_changed(ac1, [(chat.id, prepared_original.id)])
+        wait_msgs_changed(ac1, chat.id, prepared_original.id)
 
         lp.sec("forward the message while still in creation")
         chat2 = ac1.create_group_chat("newgroup")
-        chat2.add_contact(ac2)
-        wait_msgs_changed(ac1, [(0, 0)])  # why not chat id?
+        chat2.add_contact(c2)
+        wait_msgs_changed(ac1, 0, 0)  # why not chat id?
         ac1.forward_messages([prepared_original], chat2)
         # XXX there might be two EVENT_MSGS_CHANGED and only one of them
         # is the one caused by forwarding
-        _, forwarded_id = wait_msgs_changed(ac1, [(chat2.id, None)])
+        forwarded_id = wait_msgs_changed(ac1, chat2.id)
+        if forwarded_id == 0:
+            forwarded_id = wait_msgs_changed(ac1, chat2.id)
+            assert forwarded_id
         forwarded_msg = ac1.get_message_by_id(forwarded_id)
         assert forwarded_msg.is_out_preparing()
 
@@ -88,28 +83,30 @@ class TestOnlineInCreation:
         shutil.copyfile(orig, path)
         chat.send_prepared(prepared_original)
         assert prepared_original.is_out_pending() or prepared_original.is_out_delivered()
+        wait_msgs_changed(ac1, chat.id, prepared_original.id)
 
-        lp.sec("check that both forwarded and original message are proper.")
-        wait_msgs_changed(ac1, [(chat2.id, forwarded_id), (chat.id, prepared_original.id)])
-
+        lp.sec("expect the forwarded message to be sent now too")
+        wait_msgs_changed(ac1, chat2.id, forwarded_id)
         fwd_msg = ac1.get_message_by_id(forwarded_id)
         assert fwd_msg.is_out_pending() or fwd_msg.is_out_delivered()
 
-        lp.sec("wait for both messages to be delivered to SMTP")
-        wait_msg_delivered(ac1, [
-            (chat2.id, forwarded_id),
-            (chat.id, prepared_original.id)
-        ])
+        lp.sec("wait for the messages to be delivered to SMTP")
+        ev = ac1._evlogger.get_matching("DC_EVENT_MSG_DELIVERED")
+        assert ev[1] == chat.id
+        assert ev[2] == prepared_original.id
+        ev = ac1._evlogger.get_matching("DC_EVENT_MSG_DELIVERED")
+        assert ev[1] == chat2.id
+        assert ev[2] == forwarded_id
 
         lp.sec("wait1 for original or forwarded messages to arrive")
-        ev1 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
-        assert ev1.data1 > const.DC_CHAT_ID_LAST_SPECIAL
-        received_original = ac2.get_message_by_id(ev1.data2)
+        ev1 = ac2._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
+        assert ev1[1] > const.DC_CHAT_ID_LAST_SPECIAL
+        received_original = ac2.get_message_by_id(ev1[2])
         assert cmp(received_original.filename, orig, shallow=False)
 
         lp.sec("wait2 for original or forwarded messages to arrive")
-        ev2 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
-        assert ev2.data1 > const.DC_CHAT_ID_LAST_SPECIAL
-        assert ev2.data1 != ev1.data1
-        received_copy = ac2.get_message_by_id(ev2.data2)
+        ev2 = ac2._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
+        assert ev2[1] > const.DC_CHAT_ID_LAST_SPECIAL
+        assert ev2[1] != ev1[1]
+        received_copy = ac2.get_message_by_id(ev2[2])
         assert cmp(received_copy.filename, orig, shallow=False)

python/tests/test_lowlevel.py

@@ -1,52 +1,72 @@
 from __future__ import print_function
-
-from queue import Queue
-from deltachat import capi, cutil, const
-from deltachat import register_global_plugin
-from deltachat.hookspec import global_hookimpl
+from deltachat import capi, cutil, const, set_context_callback, clear_context_callback
 from deltachat.capi import ffi
 from deltachat.capi import lib
-# from deltachat.account import EventLogger
+from deltachat.account import EventLogger
 
 
 def test_empty_context():
     ctx = capi.lib.dc_context_new(capi.ffi.NULL, capi.ffi.NULL, capi.ffi.NULL)
-    capi.lib.dc_context_unref(ctx)
+    capi.lib.dc_close(ctx)
 
 
-def test_dc_close_events(tmpdir, acfactory):
-    ac1 = acfactory.get_unconfigured_account()
+def test_callback_None2int():
+    ctx = capi.lib.dc_context_new(capi.lib.py_dc_callback, ffi.NULL, ffi.NULL)
+    set_context_callback(ctx, lambda *args: None)
+    capi.lib.dc_close(ctx)
+    clear_context_callback(ctx)
 
-    # register after_shutdown function
-    shutdowns = Queue()
 
-    class ShutdownPlugin:
-        @global_hookimpl
-        def dc_account_after_shutdown(self, account):
-            assert account._dc_context is None
-            shutdowns.put(account)
-    register_global_plugin(ShutdownPlugin())
-    assert hasattr(ac1, "_dc_context")
-    ac1.shutdown()
-    shutdowns.get(timeout=2)
+def test_dc_close_events(tmpdir):
+    ctx = ffi.gc(
+        capi.lib.dc_context_new(capi.lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_unref,
+    )
+    evlog = EventLogger(ctx)
+    evlog.set_timeout(5)
+    set_context_callback(
+        ctx,
+        lambda ctx, evt_name, data1, data2: evlog(evt_name, data1, data2)
+    )
+    p = tmpdir.join("hello.db")
+    lib.dc_open(ctx, p.strpath.encode("ascii"), ffi.NULL)
+    capi.lib.dc_close(ctx)
+
+    def find(info_string):
+        while 1:
+            ev = evlog.get_matching("DC_EVENT_INFO", check_error=False)
+            data2 = ev[2]
+            if info_string in data2:
+                return
+            else:
+                print("skipping event", *ev)
+
+    find("disconnecting inbox-thread")
+    find("disconnecting sentbox-thread")
+    find("disconnecting mvbox-thread")
+    find("disconnecting SMTP")
+    find("Database closed")
 
 
 def test_wrong_db(tmpdir):
+    dc_context = ffi.gc(
+        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_unref,
+    )
     p = tmpdir.join("hello.db")
     # write an invalid database file
     p.write("x123" * 10)
-
-    assert ffi.NULL == lib.dc_context_new(ffi.NULL, p.strpath.encode("ascii"), ffi.NULL)
+    assert not lib.dc_open(dc_context, p.strpath.encode("ascii"), ffi.NULL)
 
 
 def test_empty_blobdir(tmpdir):
-    db_fname = tmpdir.join("hello.db")
     # Apparently some client code expects this to be the same as passing NULL.
     ctx = ffi.gc(
-        lib.dc_context_new(ffi.NULL, db_fname.strpath.encode("ascii"), b""),
+        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
         lib.dc_context_unref,
     )
-    assert ctx != ffi.NULL
+    db_fname = tmpdir.join("hello.db")
+    assert lib.dc_open(ctx, db_fname.strpath.encode("ascii"), b"")
 
 
 def test_event_defines():
@@ -55,27 +75,24 @@ def test_event_defines():
 
 
 def test_sig():
-    sig = capi.lib.dc_event_has_string_data
-    assert not sig(const.DC_EVENT_MSGS_CHANGED)
-    assert sig(const.DC_EVENT_INFO)
-    assert sig(const.DC_EVENT_WARNING)
-    assert sig(const.DC_EVENT_ERROR)
-    assert sig(const.DC_EVENT_SMTP_CONNECTED)
-    assert sig(const.DC_EVENT_IMAP_CONNECTED)
-    assert sig(const.DC_EVENT_SMTP_MESSAGE_SENT)
-    assert sig(const.DC_EVENT_IMEX_FILE_WRITTEN)
+    sig = capi.lib.dc_get_event_signature_types
+    assert sig(const.DC_EVENT_INFO) == 2
+    assert sig(const.DC_EVENT_WARNING) == 2
+    assert sig(const.DC_EVENT_ERROR) == 2
+    assert sig(const.DC_EVENT_SMTP_CONNECTED) == 2
+    assert sig(const.DC_EVENT_IMAP_CONNECTED) == 2
+    assert sig(const.DC_EVENT_SMTP_MESSAGE_SENT) == 2
 
 
 def test_markseen_invalid_message_ids(acfactory):
     ac1 = acfactory.get_configured_offline_account()
-
-    contact1 = ac1.create_contact("some1@example.com", name="some1")
-    chat = contact1.create_chat()
+    contact1 = ac1.create_contact(email="some1@example.com", name="some1")
+    chat = ac1.create_chat_by_contact(contact1)
     chat.send_text("one messae")
-    ac1._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+    ac1._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
     msg_ids = [9]
     lib.dc_markseen_msgs(ac1._dc_context, msg_ids, len(msg_ids))
-    ac1._evtracker.ensure_event_not_queued("DC_EVENT_WARNING|DC_EVENT_ERROR")
+    ac1._evlogger.ensure_event_not_queued("DC_EVENT_WARNING|DC_EVENT_ERROR")
 
 
 def test_get_special_message_id_returns_empty_message(acfactory):
@@ -85,20 +102,56 @@ def test_get_special_message_id_returns_empty_message(acfactory):
         assert msg.id == 0
 
 
+def test_provider_info():
+    provider = lib.dc_provider_new_from_email(cutil.as_dc_charpointer("ex@example.com"))
+    assert cutil.from_dc_charpointer(
+        lib.dc_provider_get_overview_page(provider)
+    ) == "https://providers.delta.chat/example.com"
+    assert cutil.from_dc_charpointer(lib.dc_provider_get_name(provider)) == "Example"
+    assert cutil.from_dc_charpointer(lib.dc_provider_get_markdown(provider)) == "\n..."
+    assert cutil.from_dc_charpointer(lib.dc_provider_get_status_date(provider)) == "2018-09"
+    assert lib.dc_provider_get_status(provider) == const.DC_PROVIDER_STATUS_PREPARATION
+
+
 def test_provider_info_none():
-    ctx = ffi.gc(
-        lib.dc_context_new(ffi.NULL, ffi.NULL, ffi.NULL),
-        lib.dc_context_unref,
-    )
-    assert lib.dc_provider_new_from_email(ctx, cutil.as_dc_charpointer("email@unexistent.no")) == ffi.NULL
+    assert lib.dc_provider_new_from_email(cutil.as_dc_charpointer("email@unexistent.no")) == ffi.NULL
 
 
-def test_get_info_open(tmpdir):
-    db_fname = tmpdir.join("test.db")
+def test_get_info_closed():
     ctx = ffi.gc(
-        lib.dc_context_new(ffi.NULL, db_fname.strpath.encode("ascii"), ffi.NULL),
+        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
         lib.dc_context_unref,
     )
     info = cutil.from_dc_charpointer(lib.dc_get_info(ctx))
     assert 'deltachat_core_version' in info
+    assert 'database_dir' not in info
+
+
+def test_get_info_open(tmpdir):
+    ctx = ffi.gc(
+        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_unref,
+    )
+    db_fname = tmpdir.join("test.db")
+    lib.dc_open(ctx, db_fname.strpath.encode("ascii"), ffi.NULL)
+    info = cutil.from_dc_charpointer(lib.dc_get_info(ctx))
+    assert 'deltachat_core_version' in info
     assert 'database_dir' in info
+
+
+def test_is_open_closed():
+    ctx = ffi.gc(
+        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_unref,
+    )
+    assert lib.dc_is_open(ctx) == 0
+
+
+def test_is_open_actually_open(tmpdir):
+    ctx = ffi.gc(
+        lib.dc_context_new(lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_unref,
+    )
+    db_fname = tmpdir.join("test.db")
+    lib.dc_open(ctx, db_fname.strpath.encode("ascii"), ffi.NULL)
+    assert lib.dc_is_open(ctx) == 1

python/tests/test_provider_info.py

@@ -0,0 +1,27 @@
+import pytest
+
+from deltachat import const
+from deltachat import provider
+
+
+def test_provider_info_from_email():
+    example = provider.Provider.from_email("email@example.com")
+    assert example.overview_page == "https://providers.delta.chat/example.com"
+    assert example.name == "Example"
+    assert example.markdown == "\n..."
+    assert example.status_date == "2018-09"
+    assert example.status == const.DC_PROVIDER_STATUS_PREPARATION
+
+
+def test_provider_info_from_domain():
+    example = provider.Provider("example.com")
+    assert example.overview_page == "https://providers.delta.chat/example.com"
+    assert example.name == "Example"
+    assert example.markdown == "\n..."
+    assert example.status_date == "2018-09"
+    assert example.status == const.DC_PROVIDER_STATUS_PREPARATION
+
+
+def test_provider_info_none():
+    with pytest.raises(provider.ProviderNotFoundError):
+        provider.Provider.from_email("email@unexistent.no")

python/tox.ini

@@ -7,14 +7,13 @@ envlist =
 
 [testenv]
 commands = 
-    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored {posargs: tests examples}
-    python tests/package_wheels.py {toxworkdir}/wheelhouse
+    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx {posargs:tests} 
+    # python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 
     DCC_RS_DEV
     DCC_RS_TARGET
     DCC_PY_LIVECONFIG
-    DCC_NEW_TMP_EMAIL 
     CARGO_TARGET_DIR
     RUSTC_WRAPPER
 deps = 
@@ -41,13 +40,13 @@ deps =
     restructuredtext_lint 
 commands =
     flake8 src/deltachat
-    flake8 tests/ examples/
+    flake8 tests/
     rst-lint --encoding 'utf-8' README.rst
 
 [testenv:doc]
 changedir=doc
 deps =
-    sphinx
+    sphinx==2.2.0
     breathe
 commands =
     sphinx-build -Q -w toxdoc-warnings.log -b html . _build/html
@@ -66,13 +65,12 @@ commands =
 
 
 [pytest]
-addopts = -v -ra --strict-markers
+addopts = -v -ra
+python_files = tests/test_*.py
 norecursedirs = .tox 
 xfail_strict=true
-timeout = 90
+timeout = 60 
 timeout_method = thread
-markers = 
-    ignored: ignore this test in default test runs, use --ignored to run.
 
 [flake8]
 max-line-length = 120

rust-toolchain

@@ -1 +1 @@
-1.43.1
+nightly-2019-11-06

scripts/proxy.py

@@ -1,80 +0,0 @@
-#!/usr/bin/env python3
-# Examples:
-#
-# Original server that doesn't use SSL:
-# ./proxy.py 8080 imap.nauta.cu 143
-# ./proxy.py 8081 smtp.nauta.cu 25
-#
-# Original server that uses SSL:
-# ./proxy.py 8080 testrun.org 993 --ssl
-# ./proxy.py 8081 testrun.org 465 --ssl
-
-from datetime import datetime
-import argparse
-import selectors
-import ssl
-import socket
-import socketserver
-
-
-class Proxy(socketserver.ThreadingTCPServer):
-    allow_reuse_address = True
-
-    def __init__(self, proxy_host, proxy_port, real_host, real_port, use_ssl):
-        self.real_host = real_host
-        self.real_port = real_port
-        self.use_ssl = use_ssl
-        super().__init__((proxy_host, proxy_port), RequestHandler)
-
-
-class RequestHandler(socketserver.BaseRequestHandler):
-
-    def handle(self):
-        print('{} - {} CONNECTED.'.format(datetime.now(), self.client_address))
-
-        total = 0
-        real_server = (self.server.real_host, self.server.real_port)
-        with socket.create_connection(real_server) as sock:
-            if self.server.use_ssl:
-                context = ssl.create_default_context()
-                sock = context.wrap_socket(
-                    sock, server_hostname=real_server[0])
-
-            forward = {self.request: sock, sock: self.request}
-
-            sel = selectors.DefaultSelector()
-            sel.register(self.request, selectors.EVENT_READ,
-                         self.client_address)
-            sel.register(sock, selectors.EVENT_READ, real_server)
-
-            active = True
-            while active:
-                events = sel.select()
-                for key, mask in events:
-                    print('\n{} - {} wrote:'.format(datetime.now(), key.data))
-                    data = key.fileobj.recv(1024)
-                    received = len(data)
-                    total += received
-                    print(data)
-                    print('{} Bytes\nTotal: {} Bytes'.format(received, total))
-                    if data:
-                        forward[key.fileobj].sendall(data)
-                    else:
-                        print('\nCLOSING CONNECTION.\n\n')
-                        forward[key.fileobj].close()
-                        key.fileobj.close()
-                        active = False
-
-
-if __name__ == '__main__':
-    p = argparse.ArgumentParser(description='Simple Python Proxy')
-    p.add_argument(
-        "proxy_port", help="the port where the proxy will listen", type=int)
-    p.add_argument('host', help="the real host")
-    p.add_argument('port', help="the port of the real host", type=int)
-    p.add_argument("--ssl", help="use ssl to connect to the real host",
-                   action="store_true")
-    args = p.parse_args()
-
-    with Proxy('', args.proxy_port, args.host, args.port, args.ssl) as proxy:
-        proxy.serve_forever()

set_core_version.py

@@ -35,7 +35,7 @@ if __name__ == "__main__":
     if len(sys.argv) < 2:
         for x in ("Cargo.toml", "deltachat-ffi/Cargo.toml"):
             print("{}: {}".format(x, read_toml_version(x)))
-        raise SystemExit("need argument: new version, example: 1.25.0")
+        raise SystemExit("need argument: new version, example 1.0.0-beta.27")
     newversion = sys.argv[1]
     if newversion.count(".") < 2:
         raise SystemExit("need at least two dots in version")
@@ -45,7 +45,7 @@ if __name__ == "__main__":
     assert core_toml == ffi_toml, (core_toml, ffi_toml)
 
     for line in open("CHANGELOG.md"):
-        ## 1.25.0
+        ## 1.0.0-beta5
         if line.startswith("## "):
             if line[2:].strip().startswith(newversion):
                 break

spec.md

@@ -1,12 +1,10 @@
-# chat-mail specification
+# Chat-over-Email specification
 
-Version: 0.32.0
-Status:  In-progress 
-Format:  [Semantic Line Breaks](https://sembr.org/)
+Version 0.20.0
 
-This document roughly describes how chat-mail 
-apps use the standard e-mail system 
-to implement typical messenger functions.
+This document describes how emails can be used
+to implement typical messenger functions
+while staying compatible to existing MUAs.
 
 - [Encryption](#encryption)
 - [Outgoing messages](#outgoing-messages)
@@ -19,9 +17,6 @@ to implement typical messenger functions.
     - [Change group name](#change-group-name)
     - [Set group image](#set-group-image)
 - [Set profile image](#set-profile-image)
-- [Locations](#locations)
-    - [User locations](#user-locations)
-    - [Points of interest](#points-of-interest)
 - [Miscellaneous](#miscellaneous)
 
 
@@ -32,14 +27,18 @@ Messages SHOULD be encrypted by the
 `prefer-encrypt=mutual` MAY be set by default.
 
 Meta data (at least the subject and all chat-headers) SHOULD be encrypted
-by the [Protected Headers](https://www.ietf.org/id/draft-autocrypt-lamps-protected-headers-02.html) standard.
+by the [Memoryhole](https://github.com/autocrypt/memoryhole) standard.
+If Memoryhole is not used,
+the subject of encrypted messages SHOULD be replaced by the string
+`Chat: Encrypted message` where the part after the colon MAY be localized.
 
 
 # Outgoing messages
 
 Messengers MUST add a `Chat-Version: 1.0` header to outgoing messages.
 For filtering and smart appearance of the messages in normal MUAs,
-the `Subject` header SHOULD be `Message from <sender name>`.
+the `Subject` header SHOULD start with the characters `Chat:`
+and SHOULD be an excerpt of the message.
 Replies to messages MAY follow the typical `Re:`-format.
 
 The body MAY contain text which MUST have the content type `text/plain`
@@ -57,7 +56,7 @@ Full quotes, footers or sth. like that MUST NOT go to the user-text-part.
     To: rcpt@domain
     Chat-Version: 1.0
     Content-Type: text/plain
-    Subject: Message from sender@domain
+    Subject: Chat: Hello ...
 
     Hello world!
 
@@ -114,7 +113,6 @@ but MUAs typically expose the sender in the UI.
 Groups are chats with usually more than one recipient,
 each defined by an email-address.
 The sender plus the recipients are the group members.
-All group members form the member list.
 
 To allow different groups with the same members,
 groups are identified by a group-id.
@@ -137,7 +135,8 @@ The group-name MUST be written to `Chat-Group-Name` header
 to join a group chat on a second device any time).
 
 The `Subject` header of outgoing group messages
-SHOULD be set to the group-name.
+SHOULD start with the characters `Chat:`
+followed by the group-name and a colon followed by an excerpt of the message.
 
 To identify the group-id on replies from normal MUAs,
 the group-id MUST also be added to the message-id of outgoing messages.
@@ -154,8 +153,8 @@ The message-id MUST have the format `Gr.<group-id>.<unique data>`.
     Hello group - this group contains three members
 
 Messengers adding the member list in the form `Name <email-address>`
-MUST take care only to distribute the names authorized by the contacts themselves.
-Otherwise, names as _Daddy_ or _Honey_ may be distributed
+MUST take care only to spread the names authorized by the contacts themselves.
+Otherwise, names as _Daddy_ or _Honey_ may be spread
 (this issue is also true for normal MUAs, however,
 for more contact- and chat-centralized apps
 such situations happen more frequently).
@@ -178,22 +177,12 @@ to a normal single-user chat with the email-address given in `From`.
 
 ## Add and remove members
 
-Messenger clients MUST init the member list
-from the `From`/`To` headers on the first group message.
-
-When a member is added later,
-a `Chat-Group-Member-Added` action header must be sent
-with the value set to the email-address of the added member.
-When receiving a `Chat-Group-Member-Added` header, however,
-_all missing_ members  the `From`/`To` headers has to be added.
-This is to mitigate problems when receiving messages
-in different orders, esp. on creating new groups.
-
-To remove a member, a `Chat-Group-Member-Removed` header must be sent
-with the value set to the email-address of the member to remove.
-When receiving a `Chat-Group-Member-Removed` header,
-only exaxtly the given member has to be removed from the member list.
-
+Messenger clients MUST construct the member list
+from the `From`/`To` headers only on the first group message
+or if they see a `Chat-Group-Member-Added`
+or `Chat-Group-Member-Removed` action header.
+Both headers MUST have the email-address
+of the added or removed member as the value.
 Messenger clients MUST NOT construct the member list
 on other group messages
 (this is to avoid accidentally altered To-lists in normal MUAs;
@@ -299,7 +288,7 @@ to add a `Chat-Group-Avatar` only on image changes.
 
 # Set profile image
 
-A user MAY have a profile-image that MAY be distributed to their contacts.
+A user MAY have a profile-image that MAY be spread to their contacts.
 To change or set the profile-image,
 the messenger MUST attach an image file to a message
 and MUST add the header `Chat-User-Avatar`
@@ -308,7 +297,7 @@ with the value set to the image name.
 To remove the profile-image,
 the messenger MUST add the header `Chat-User-Avatar: 0`.
 
-To distribute the image,
+To spread the image,
 the messenger MAY send the profile image
 together with the next mail to a given contact
 (to do this only once,
@@ -343,64 +332,6 @@ To save data, it is RECOMMENDED to add a `Chat-User-Avatar` header
 only on image changes.
 
 
-# Locations
-
-Locations can be attachted to messages using
-[standard kml-files](https://www.opengeospatial.org/standards/kml/)
-with well-known names.
-
-
-## User locations
-
-To send the location of the sender,
-the app can attach a file with the name `location.kml`.
-The file can contain one or more locations.
-Apps that support location streaming will typically collect some location events
-and send them together in one file.
-As each location has an independent timestamp,
-the apps can show the location as a track.
-
-Note that the `addr` attribute inside the  `location.kml` file
-MUST match the users email-address.
-Otherwise, the file is discarded silently;
-this is to protect against getting wrong locations,
-eg. forwarded from a normal MUA.
-
-    <?xml version="1.0" encoding="UTF-8"?>
-    <kml xmlns="http://www.opengis.net/kml/2.2">
-      <Document addr="ndh@deltachat.de">
-        <Placemark>
-          <Timestamp><when>2020-01-11T20:40:19Z</when></Timestamp>
-          <Point><coordinates accuracy="1.2">1.234,5.678</coordinates></Point>
-        </Placemark>
-        <Placemark>
-          <Timestamp><when>2020-01-11T20:40:25Z</when></Timestamp>
-          <Point><coordinates accuracy="5.4">7.654,3.21</coordinates></Point>
-        </Placemark>
-      </Document>
-    </kml>
-
-
-## Points of interest
-
-To send an "Point of interest", a POI,
-use a normal message and attach a file with the name  `message.kml`.
-In contrast to user locations, this file should contain only one location
-and an address-attribute is not needed -
-as the location belongs to the message content,
-it is fine if the location is detected on forwarding etc.
-
-    <?xml version="1.0" encoding="UTF-8"?>
-    <kml xmlns="http://www.opengis.net/kml/2.2">
-      <Document>
-        <Placemark>
-          <Timestamp><when>2020-01-01T20:40:19Z</when></Timestamp>
-          <Point><coordinates accuracy="1.2">1.234,5.678</coordinates></Point>
-        </Placemark>
-      </Document>
-    </kml>
-
-
 # Miscellaneous
 
 Messengers SHOULD use the header `In-Reply-To` as usual.
@@ -437,4 +368,4 @@ as the sending time of the message as indicated by its Date header,
 or the time of first receipt if that date is in the future or unavailable.
 
 
-Copyright © 2017-2020 Delta Chat contributors.
+Copyright © 2017-2019 Delta Chat contributors.

src/aheader.rs

@@ -6,15 +6,13 @@ use std::collections::BTreeMap;
 use std::str::FromStr;
 use std::{fmt, str};
 
-use deltachat_derive::*;
-
+use crate::constants::*;
 use crate::contact::*;
 use crate::context::Context;
-use crate::headerdef::{HeaderDef, HeaderDefMap};
-use crate::key::{DcKey, SignedPublicKey};
+use crate::key::*;
 
 /// Possible values for encryption preference
-#[derive(PartialEq, Eq, Debug, Clone, Copy, FromPrimitive, ToPrimitive, Sqlx)]
+#[derive(PartialEq, Eq, Debug, Clone, Copy, FromPrimitive, ToPrimitive)]
 #[repr(u8)]
 pub enum EncryptPreference {
     NoPreference = 0,
@@ -54,17 +52,13 @@ impl str::FromStr for EncryptPreference {
 #[derive(Debug)]
 pub struct Aheader {
     pub addr: String,
-    pub public_key: SignedPublicKey,
+    pub public_key: Key,
     pub prefer_encrypt: EncryptPreference,
 }
 
 impl Aheader {
     /// Creates new autocrypt header
-    pub fn new(
-        addr: String,
-        public_key: SignedPublicKey,
-        prefer_encrypt: EncryptPreference,
-    ) -> Self {
+    pub fn new(addr: String, public_key: Key, prefer_encrypt: EncryptPreference) -> Self {
         Aheader {
             addr,
             public_key,
@@ -77,7 +71,9 @@ impl Aheader {
         wanted_from: &str,
         headers: &[mailparse::MailHeader<'_>],
     ) -> Option<Self> {
-        if let Some(value) = headers.get_header_value(HeaderDef::Autocrypt) {
+        use mailparse::MailHeaderMap;
+
+        if let Ok(Some(value)) = headers.get_first_value("Autocrypt") {
             match Self::from_str(&value) {
                 Ok(header) => {
                     if addr_cmp(&header.addr, wanted_from) {
@@ -129,10 +125,14 @@ impl str::FromStr for Aheader {
             .split(';')
             .filter_map(|a| {
                 let attribute: Vec<&str> = a.trim().splitn(2, '=').collect();
-                match &attribute[..] {
-                    [key, value] => Some((key.trim().to_string(), value.trim().to_string())),
-                    _ => None,
+                if attribute.len() < 2 {
+                    return None;
                 }
+
+                Some((
+                    attribute[0].trim().to_string(),
+                    attribute[1].trim().to_string(),
+                ))
             })
             .collect();
 
@@ -142,11 +142,22 @@ impl str::FromStr for Aheader {
                 return Err(());
             }
         };
-        let public_key: SignedPublicKey = attributes
+
+        let public_key = match attributes
             .remove("keydata")
-            .ok_or(())
-            .and_then(|raw| SignedPublicKey::from_base64(&raw).or(Err(())))
-            .and_then(|key| key.verify().and(Ok(key)).or(Err(())))?;
+            .and_then(|raw| Key::from_base64(&raw, KeyType::Public))
+        {
+            Some(key) => {
+                if key.verify() {
+                    key
+                } else {
+                    return Err(());
+                }
+            }
+            None => {
+                return Err(());
+            }
+        };
 
         let prefer_encrypt = attributes
             .remove("prefer-encrypt")
@@ -281,7 +292,7 @@ mod tests {
             "{}",
             Aheader::new(
                 "test@example.com".to_string(),
-                SignedPublicKey::from_base64(RAWKEY).unwrap(),
+                Key::from_base64(RAWKEY, KeyType::Public).unwrap(),
                 EncryptPreference::Mutual
             )
         )
@@ -294,7 +305,7 @@ mod tests {
             "{}",
             Aheader::new(
                 "test@example.com".to_string(),
-                SignedPublicKey::from_base64(RAWKEY).unwrap(),
+                Key::from_base64(RAWKEY, KeyType::Public).unwrap(),
                 EncryptPreference::NoPreference
             )
         )

src/blob.rs

@@ -2,21 +2,16 @@
 
 use std::ffi::OsStr;
 use std::fmt;
+use std::fs;
+use std::io::Write;
+use std::path::{Path, PathBuf};
 
-use async_std::path::{Path, PathBuf};
-use async_std::prelude::*;
-use async_std::{fs, io};
-
-use image::GenericImageView;
-use num_traits::FromPrimitive;
-use thiserror::Error;
-
-use crate::config::Config;
-use crate::constants::*;
+use self::image::GenericImageView;
+use crate::constants::AVATAR_SIZE;
 use crate::context::Context;
-use crate::error::Error;
 use crate::events::Event;
-use crate::message;
+
+extern crate image;
 
 /// Represents a file in the blob directory.
 ///
@@ -48,35 +43,31 @@ impl<'a> BlobObject<'a> {
     /// [BlobError::WriteFailure] is used when the file could not
     /// be written to.  You can expect [BlobError.cause] to contain an
     /// underlying error.
-    pub async fn create(
+    pub fn create(
         context: &'a Context,
         suggested_name: impl AsRef<str>,
         data: &[u8],
     ) -> std::result::Result<BlobObject<'a>, BlobError> {
         let blobdir = context.get_blobdir();
         let (stem, ext) = BlobObject::sanitise_name(suggested_name.as_ref());
-        let (name, mut file) = BlobObject::create_new_file(&blobdir, &stem, &ext).await?;
+        let (name, mut file) = BlobObject::create_new_file(&blobdir, &stem, &ext)?;
         file.write_all(data)
-            .await
             .map_err(|err| BlobError::WriteFailure {
                 blobdir: blobdir.to_path_buf(),
                 blobname: name.clone(),
-                cause: err.into(),
+                cause: err,
+                backtrace: failure::Backtrace::new(),
             })?;
         let blob = BlobObject {
             blobdir,
             name: format!("$BLOBDIR/{}", name),
         };
-        context.emit_event(Event::NewBlobFile(blob.as_name().to_string()));
+        context.call_cb(Event::NewBlobFile(blob.as_name().to_string()));
         Ok(blob)
     }
 
     // Creates a new file, returning a tuple of the name and the handle.
-    async fn create_new_file(
-        dir: &Path,
-        stem: &str,
-        ext: &str,
-    ) -> Result<(String, fs::File), BlobError> {
+    fn create_new_file(dir: &Path, stem: &str, ext: &str) -> Result<(String, fs::File), BlobError> {
         let max_attempt = 15;
         let mut name = format!("{}{}", stem, ext);
         for attempt in 0..max_attempt {
@@ -85,7 +76,6 @@ impl<'a> BlobObject<'a> {
                 .create_new(true)
                 .write(true)
                 .open(&path)
-                .await
             {
                 Ok(file) => return Ok((name, file)),
                 Err(err) => {
@@ -94,6 +84,7 @@ impl<'a> BlobObject<'a> {
                             blobdir: dir.to_path_buf(),
                             blobname: name,
                             cause: err,
+                            backtrace: failure::Backtrace::new(),
                         });
                     } else {
                         name = format!("{}-{}{}", stem, rand::random::<u32>(), ext);
@@ -106,6 +97,7 @@ impl<'a> BlobObject<'a> {
             blobdir: dir.to_path_buf(),
             blobname: name,
             cause: std::io::Error::new(std::io::ErrorKind::Other, "supposedly unreachable"),
+            backtrace: failure::Backtrace::new(),
         })
     }
 
@@ -121,41 +113,39 @@ impl<'a> BlobObject<'a> {
     /// In addition to the errors in [BlobObject::create] the
     /// [BlobError::CopyFailure] is used when the data can not be
     /// copied.
-    pub async fn create_and_copy(
+    pub fn create_and_copy(
         context: &'a Context,
         src: impl AsRef<Path>,
     ) -> std::result::Result<BlobObject<'a>, BlobError> {
-        let mut src_file =
-            fs::File::open(src.as_ref())
-                .await
-                .map_err(|err| BlobError::CopyFailure {
-                    blobdir: context.get_blobdir().to_path_buf(),
-                    blobname: String::from(""),
-                    src: src.as_ref().to_path_buf(),
-                    cause: err,
-                })?;
+        let mut src_file = fs::File::open(src.as_ref()).map_err(|err| BlobError::CopyFailure {
+            blobdir: context.get_blobdir().to_path_buf(),
+            blobname: String::from(""),
+            src: src.as_ref().to_path_buf(),
+            cause: err,
+            backtrace: failure::Backtrace::new(),
+        })?;
         let (stem, ext) = BlobObject::sanitise_name(&src.as_ref().to_string_lossy());
-        let (name, mut dst_file) =
-            BlobObject::create_new_file(context.get_blobdir(), &stem, &ext).await?;
+        let (name, mut dst_file) = BlobObject::create_new_file(context.get_blobdir(), &stem, &ext)?;
         let name_for_err = name.clone();
-        if let Err(err) = io::copy(&mut src_file, &mut dst_file).await {
+        std::io::copy(&mut src_file, &mut dst_file).map_err(|err| {
             {
                 // Attempt to remove the failed file, swallow errors resulting from that.
                 let path = context.get_blobdir().join(&name_for_err);
-                fs::remove_file(path).await.ok();
+                fs::remove_file(path).ok();
             }
-            return Err(BlobError::CopyFailure {
+            BlobError::CopyFailure {
                 blobdir: context.get_blobdir().to_path_buf(),
                 blobname: name_for_err,
                 src: src.as_ref().to_path_buf(),
                 cause: err,
-            });
-        }
+                backtrace: failure::Backtrace::new(),
+            }
+        })?;
         let blob = BlobObject {
             blobdir: context.get_blobdir(),
             name: format!("$BLOBDIR/{}", name),
         };
-        context.emit_event(Event::NewBlobFile(blob.as_name().to_string()));
+        context.call_cb(Event::NewBlobFile(blob.as_name().to_string()));
         Ok(blob)
     }
 
@@ -173,14 +163,14 @@ impl<'a> BlobObject<'a> {
     /// This merely delegates to the [BlobObject::create_and_copy] and
     /// the [BlobObject::from_path] methods.  See those for possible
     /// errors.
-    pub async fn new_from_path(
+    pub fn new_from_path(
         context: &Context,
         src: impl AsRef<Path>,
-    ) -> std::result::Result<BlobObject<'_>, BlobError> {
+    ) -> std::result::Result<BlobObject, BlobError> {
         if src.as_ref().starts_with(context.get_blobdir()) {
             BlobObject::from_path(context, src)
         } else {
-            BlobObject::create_and_copy(context, src).await
+            BlobObject::create_and_copy(context, src)
         }
     }
 
@@ -208,14 +198,17 @@ impl<'a> BlobObject<'a> {
             .map_err(|_| BlobError::WrongBlobdir {
                 blobdir: context.get_blobdir().to_path_buf(),
                 src: path.as_ref().to_path_buf(),
+                backtrace: failure::Backtrace::new(),
             })?;
         if !BlobObject::is_acceptible_blob_name(&rel_path) {
             return Err(BlobError::WrongName {
                 blobname: path.as_ref().to_path_buf(),
+                backtrace: failure::Backtrace::new(),
             });
         }
         let name = rel_path.to_str().ok_or_else(|| BlobError::WrongName {
             blobname: path.as_ref().to_path_buf(),
+            backtrace: failure::Backtrace::new(),
         })?;
         BlobObject::from_name(context, name.to_string())
     }
@@ -243,6 +236,7 @@ impl<'a> BlobObject<'a> {
         if !BlobObject::is_acceptible_blob_name(&name) {
             return Err(BlobError::WrongName {
                 blobname: PathBuf::from(name),
+                backtrace: failure::Backtrace::new(),
             });
         }
         Ok(BlobObject {
@@ -328,12 +322,13 @@ impl<'a> BlobObject<'a> {
 
         let clean = sanitize_filename::sanitize_with_options(name, opts);
         let mut iter = clean.splitn(2, '.');
-        let stem: String = iter.next().unwrap_or_default().chars().take(64).collect();
-        let ext: String = iter.next().unwrap_or_default().chars().take(32).collect();
-        if ext.is_empty() {
-            (stem, "".to_string())
-        } else {
-            (stem, format!(".{}", ext).to_lowercase())
+        let mut stem = iter.next().unwrap_or_default().to_string();
+        let mut ext = iter.next().unwrap_or_default().to_string();
+        stem.truncate(64);
+        ext.truncate(32);
+        match ext.len() {
+            0 => (stem, "".to_string()),
+            _ => (stem, format!(".{}", ext).to_lowercase()),
         }
     }
 
@@ -365,6 +360,7 @@ impl<'a> BlobObject<'a> {
             blobdir: context.get_blobdir().to_path_buf(),
             blobname: blob_abs.to_str().unwrap_or_default().to_string(),
             cause: err,
+            backtrace: failure::Backtrace::new(),
         })?;
 
         if img.width() <= AVATAR_SIZE && img.height() <= AVATAR_SIZE {
@@ -374,75 +370,14 @@ impl<'a> BlobObject<'a> {
         let img = img.thumbnail(AVATAR_SIZE, AVATAR_SIZE);
 
         img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
-            blobdir: context.get_blobdir().to_path_buf(),
-            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err.into(),
-        })?;
-
-        Ok(())
-    }
-
-    pub async fn recode_to_image_size(&self, context: &Context) -> Result<(), BlobError> {
-        let blob_abs = self.to_abs_path();
-        if message::guess_msgtype_from_suffix(Path::new(&blob_abs))
-            != Some((Viewtype::Image, "image/jpeg"))
-        {
-            return Ok(());
-        }
-
-        let img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
             blobdir: context.get_blobdir().to_path_buf(),
             blobname: blob_abs.to_str().unwrap_or_default().to_string(),
             cause: err,
-        })?;
-
-        let img_wh = if MediaQuality::from_i32(context.get_config_int(Config::MediaQuality).await)
-            .unwrap_or_default()
-            == MediaQuality::Balanced
-        {
-            BALANCED_IMAGE_SIZE
-        } else {
-            WORSE_IMAGE_SIZE
-        };
-
-        if img.width() <= img_wh && img.height() <= img_wh {
-            return Ok(());
-        }
-
-        let mut img = img.thumbnail(img_wh, img_wh);
-        match self.get_exif_orientation(context) {
-            Ok(90) => img = img.rotate90(),
-            Ok(180) => img = img.rotate180(),
-            Ok(270) => img = img.rotate270(),
-            _ => {}
-        }
-
-        img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
-            blobdir: context.get_blobdir().to_path_buf(),
-            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err.into(),
+            backtrace: failure::Backtrace::new(),
         })?;
 
         Ok(())
     }
-
-    pub fn get_exif_orientation(&self, context: &Context) -> Result<i32, Error> {
-        let file = std::fs::File::open(self.to_abs_path())?;
-        let mut bufreader = std::io::BufReader::new(&file);
-        let exifreader = exif::Reader::new();
-        let exif = exifreader.read_from_container(&mut bufreader)?;
-        if let Some(orientation) = exif.get_field(exif::Tag::Orientation, exif::In::PRIMARY) {
-            // possible orientation values are described at http://sylvana.net/jpegcrop/exif_orientation.html
-            // we only use rotation, in practise, flipping is not used.
-            match orientation.value.get_uint(0) {
-                Some(3) => return Ok(180),
-                Some(6) => return Ok(90),
-                Some(8) => return Ok(270),
-                other => warn!(context, "exif orientation value ignored: {:?}", other),
-            }
-        }
-        Ok(0)
-    }
 }
 
 impl<'a> fmt::Display for BlobObject<'a> {
@@ -452,41 +387,98 @@ impl<'a> fmt::Display for BlobObject<'a> {
 }
 
 /// Errors for the [BlobObject].
-#[derive(Debug, Error)]
+#[derive(Fail, Debug)]
 pub enum BlobError {
-    #[error("Failed to create blob {blobname} in {}", .blobdir.display())]
     CreateFailure {
         blobdir: PathBuf,
         blobname: String,
-        #[source]
+        #[cause]
         cause: std::io::Error,
+        backtrace: failure::Backtrace,
     },
-    #[error("Failed to write data to blob {blobname} in {}", .blobdir.display())]
     WriteFailure {
         blobdir: PathBuf,
         blobname: String,
-        #[source]
-        cause: anyhow::Error,
+        #[cause]
+        cause: std::io::Error,
+        backtrace: failure::Backtrace,
     },
-    #[error("Failed to copy data from {} to blob {blobname} in {}", .src.display(), .blobdir.display())]
     CopyFailure {
         blobdir: PathBuf,
         blobname: String,
         src: PathBuf,
-        #[source]
+        #[cause]
         cause: std::io::Error,
+        backtrace: failure::Backtrace,
     },
-    #[error("Failed to recode to blob {blobname} in {}", .blobdir.display())]
     RecodeFailure {
         blobdir: PathBuf,
         blobname: String,
-        #[source]
+        #[cause]
         cause: image::ImageError,
+        backtrace: failure::Backtrace,
     },
-    #[error("File path {} is not in {}", .src.display(), .blobdir.display())]
-    WrongBlobdir { blobdir: PathBuf, src: PathBuf },
-    #[error("Blob has a badname {}", .blobname.display())]
-    WrongName { blobname: PathBuf },
+    WrongBlobdir {
+        blobdir: PathBuf,
+        src: PathBuf,
+        backtrace: failure::Backtrace,
+    },
+    WrongName {
+        blobname: PathBuf,
+        backtrace: failure::Backtrace,
+    },
+}
+
+// Implementing Display is done by hand because the failure
+// #[fail(display = "...")] syntax does not allow using
+// `blobdir.display()`.
+impl fmt::Display for BlobError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        // Match on the data rather than kind, they are equivalent for
+        // identifying purposes but contain the actual data we need.
+        match &self {
+            BlobError::CreateFailure {
+                blobdir, blobname, ..
+            } => write!(
+                f,
+                "Failed to create blob {} in {}",
+                blobname,
+                blobdir.display()
+            ),
+            BlobError::WriteFailure {
+                blobdir, blobname, ..
+            } => write!(
+                f,
+                "Failed to write data to blob {} in {}",
+                blobname,
+                blobdir.display()
+            ),
+            BlobError::CopyFailure {
+                blobdir,
+                blobname,
+                src,
+                ..
+            } => write!(
+                f,
+                "Failed to copy data from {} to blob {} in {}",
+                src.display(),
+                blobname,
+                blobdir.display(),
+            ),
+            BlobError::RecodeFailure {
+                blobdir, blobname, ..
+            } => write!(f, "Failed to recode {} in {}", blobname, blobdir.display(),),
+            BlobError::WrongBlobdir { blobdir, src, .. } => write!(
+                f,
+                "File path {} is not in blobdir {}",
+                src.display(),
+                blobdir.display(),
+            ),
+            BlobError::WrongName { blobname, .. } => {
+                write!(f, "Blob has a bad name: {}", blobname.display(),)
+            }
+        }
+    }
 }
 
 #[cfg(test)]
@@ -495,71 +487,58 @@ mod tests {
 
     use crate::test_utils::*;
 
-    #[async_std::test]
-    async fn test_create() {
-        let t = TestContext::new().await;
-        let blob = BlobObject::create(&t.ctx, "foo", b"hello").await.unwrap();
+    #[test]
+    fn test_create() {
+        let t = dummy_context();
+        let blob = BlobObject::create(&t.ctx, "foo", b"hello").unwrap();
         let fname = t.ctx.get_blobdir().join("foo");
-        let data = fs::read(fname).await.unwrap();
+        let data = fs::read(fname).unwrap();
         assert_eq!(data, b"hello");
         assert_eq!(blob.as_name(), "$BLOBDIR/foo");
         assert_eq!(blob.to_abs_path(), t.ctx.get_blobdir().join("foo"));
     }
 
-    #[async_std::test]
-    async fn test_lowercase_ext() {
-        let t = TestContext::new().await;
-        let blob = BlobObject::create(&t.ctx, "foo.TXT", b"hello")
-            .await
-            .unwrap();
+    #[test]
+    fn test_lowercase_ext() {
+        let t = dummy_context();
+        let blob = BlobObject::create(&t.ctx, "foo.TXT", b"hello").unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/foo.txt");
     }
 
-    #[async_std::test]
-    async fn test_as_file_name() {
-        let t = TestContext::new().await;
-        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
-            .await
-            .unwrap();
+    #[test]
+    fn test_as_file_name() {
+        let t = dummy_context();
+        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
         assert_eq!(blob.as_file_name(), "foo.txt");
     }
 
-    #[async_std::test]
-    async fn test_as_rel_path() {
-        let t = TestContext::new().await;
-        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
-            .await
-            .unwrap();
+    #[test]
+    fn test_as_rel_path() {
+        let t = dummy_context();
+        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
         assert_eq!(blob.as_rel_path(), Path::new("foo.txt"));
     }
 
-    #[async_std::test]
-    async fn test_suffix() {
-        let t = TestContext::new().await;
-        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
-            .await
-            .unwrap();
+    #[test]
+    fn test_suffix() {
+        let t = dummy_context();
+        let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
         assert_eq!(blob.suffix(), Some("txt"));
-        let blob = BlobObject::create(&t.ctx, "bar", b"world").await.unwrap();
+        let blob = BlobObject::create(&t.ctx, "bar", b"world").unwrap();
         assert_eq!(blob.suffix(), None);
     }
 
-    #[async_std::test]
-    async fn test_create_dup() {
-        let t = TestContext::new().await;
-        BlobObject::create(&t.ctx, "foo.txt", b"hello")
-            .await
-            .unwrap();
+    #[test]
+    fn test_create_dup() {
+        let t = dummy_context();
+        BlobObject::create(&t.ctx, "foo.txt", b"hello").unwrap();
         let foo_path = t.ctx.get_blobdir().join("foo.txt");
-        assert!(foo_path.exists().await);
-        BlobObject::create(&t.ctx, "foo.txt", b"world")
-            .await
-            .unwrap();
-        let mut dir = fs::read_dir(t.ctx.get_blobdir()).await.unwrap();
-        while let Some(dirent) = dir.next().await {
+        assert!(foo_path.exists());
+        BlobObject::create(&t.ctx, "foo.txt", b"world").unwrap();
+        for dirent in fs::read_dir(t.ctx.get_blobdir()).unwrap() {
             let fname = dirent.unwrap().file_name();
             if fname == foo_path.file_name().unwrap() {
-                assert_eq!(fs::read(&foo_path).await.unwrap(), b"hello");
+                assert_eq!(fs::read(&foo_path).unwrap(), b"hello");
             } else {
                 let name = fname.to_str().unwrap();
                 assert!(name.starts_with("foo"));
@@ -568,22 +547,17 @@ mod tests {
         }
     }
 
-    #[async_std::test]
-    async fn test_double_ext_preserved() {
-        let t = TestContext::new().await;
-        BlobObject::create(&t.ctx, "foo.tar.gz", b"hello")
-            .await
-            .unwrap();
+    #[test]
+    fn test_double_ext_preserved() {
+        let t = dummy_context();
+        BlobObject::create(&t.ctx, "foo.tar.gz", b"hello").unwrap();
         let foo_path = t.ctx.get_blobdir().join("foo.tar.gz");
-        assert!(foo_path.exists().await);
-        BlobObject::create(&t.ctx, "foo.tar.gz", b"world")
-            .await
-            .unwrap();
-        let mut dir = fs::read_dir(t.ctx.get_blobdir()).await.unwrap();
-        while let Some(dirent) = dir.next().await {
+        assert!(foo_path.exists());
+        BlobObject::create(&t.ctx, "foo.tar.gz", b"world").unwrap();
+        for dirent in fs::read_dir(t.ctx.get_blobdir()).unwrap() {
             let fname = dirent.unwrap().file_name();
             if fname == foo_path.file_name().unwrap() {
-                assert_eq!(fs::read(&foo_path).await.unwrap(), b"hello");
+                assert_eq!(fs::read(&foo_path).unwrap(), b"hello");
             } else {
                 let name = fname.to_str().unwrap();
                 println!("{}", name);
@@ -593,55 +567,55 @@ mod tests {
         }
     }
 
-    #[async_std::test]
-    async fn test_create_long_names() {
-        let t = TestContext::new().await;
+    #[test]
+    fn test_create_long_names() {
+        let t = dummy_context();
         let s = "1".repeat(150);
-        let blob = BlobObject::create(&t.ctx, &s, b"data").await.unwrap();
+        let blob = BlobObject::create(&t.ctx, &s, b"data").unwrap();
         let blobname = blob.as_name().split('/').last().unwrap();
         assert!(blobname.len() < 128);
     }
 
-    #[async_std::test]
-    async fn test_create_and_copy() {
-        let t = TestContext::new().await;
+    #[test]
+    fn test_create_and_copy() {
+        let t = dummy_context();
         let src = t.dir.path().join("src");
-        fs::write(&src, b"boo").await.unwrap();
-        let blob = BlobObject::create_and_copy(&t.ctx, &src).await.unwrap();
+        fs::write(&src, b"boo").unwrap();
+        let blob = BlobObject::create_and_copy(&t.ctx, &src).unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/src");
-        let data = fs::read(blob.to_abs_path()).await.unwrap();
+        let data = fs::read(blob.to_abs_path()).unwrap();
         assert_eq!(data, b"boo");
 
         let whoops = t.dir.path().join("whoops");
-        assert!(BlobObject::create_and_copy(&t.ctx, &whoops).await.is_err());
+        assert!(BlobObject::create_and_copy(&t.ctx, &whoops).is_err());
         let whoops = t.ctx.get_blobdir().join("whoops");
-        assert!(!whoops.exists().await);
+        assert!(!whoops.exists());
     }
 
-    #[async_std::test]
-    async fn test_create_from_path() {
-        let t = TestContext::new().await;
+    #[test]
+    fn test_create_from_path() {
+        let t = dummy_context();
 
         let src_ext = t.dir.path().join("external");
-        fs::write(&src_ext, b"boo").await.unwrap();
-        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).await.unwrap();
+        fs::write(&src_ext, b"boo").unwrap();
+        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/external");
-        let data = fs::read(blob.to_abs_path()).await.unwrap();
+        let data = fs::read(blob.to_abs_path()).unwrap();
         assert_eq!(data, b"boo");
 
         let src_int = t.ctx.get_blobdir().join("internal");
-        fs::write(&src_int, b"boo").await.unwrap();
-        let blob = BlobObject::new_from_path(&t.ctx, &src_int).await.unwrap();
+        fs::write(&src_int, b"boo").unwrap();
+        let blob = BlobObject::new_from_path(&t.ctx, &src_int).unwrap();
         assert_eq!(blob.as_name(), "$BLOBDIR/internal");
-        let data = fs::read(blob.to_abs_path()).await.unwrap();
+        let data = fs::read(blob.to_abs_path()).unwrap();
         assert_eq!(data, b"boo");
     }
-    #[async_std::test]
-    async fn test_create_from_name_long() {
-        let t = TestContext::new().await;
+    #[test]
+    fn test_create_from_name_long() {
+        let t = dummy_context();
         let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
-        fs::write(&src_ext, b"boo").await.unwrap();
-        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).await.unwrap();
+        fs::write(&src_ext, b"boo").unwrap();
+        let blob = BlobObject::new_from_path(&t.ctx, &src_ext).unwrap();
         assert_eq!(
             blob.as_name(),
             "$BLOBDIR/autocrypt-setup-message-4137848473.html"
@@ -657,43 +631,4 @@ mod tests {
         assert!(!BlobObject::is_acceptible_blob_name("foo\\bar"));
         assert!(!BlobObject::is_acceptible_blob_name("foo\x00bar"));
     }
-
-    #[test]
-    fn test_sanitise_name() {
-        let (stem, ext) =
-            BlobObject::sanitise_name("Я ЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯ.txt");
-        assert_eq!(ext, ".txt");
-        assert!(!stem.is_empty());
-
-        // the extensions are kept together as between stem and extension a number may be added -
-        // and `foo.tar.gz` should become `foo-1234.tar.gz` and not `foo.tar-1234.gz`
-        let (stem, ext) = BlobObject::sanitise_name("wot.tar.gz");
-        assert_eq!(stem, "wot");
-        assert_eq!(ext, ".tar.gz");
-
-        let (stem, ext) = BlobObject::sanitise_name(".foo.bar");
-        assert_eq!(stem, "");
-        assert_eq!(ext, ".foo.bar");
-
-        let (stem, ext) = BlobObject::sanitise_name("foo?.bar");
-        assert!(stem.contains("foo"));
-        assert!(!stem.contains("?"));
-        assert_eq!(ext, ".bar");
-
-        let (stem, ext) = BlobObject::sanitise_name("no-extension");
-        assert_eq!(stem, "no-extension");
-        assert_eq!(ext, "");
-
-        let (stem, ext) = BlobObject::sanitise_name("path/ignored\\this: is* forbidden?.c");
-        assert_eq!(ext, ".c");
-        assert!(!stem.contains("path"));
-        assert!(!stem.contains("ignored"));
-        assert!(stem.contains("this"));
-        assert!(stem.contains("forbidden"));
-        assert!(!stem.contains("/"));
-        assert!(!stem.contains("\\"));
-        assert!(!stem.contains(":"));
-        assert!(!stem.contains("*"));
-        assert!(!stem.contains("?"));
-    }
 }

src/chatlist.rs

@@ -1,11 +1,10 @@
 //! # Chat list module
 
-use crate::chat;
 use crate::chat::*;
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::*;
-use crate::error::{bail, ensure, Result};
+use crate::error::Result;
 use crate::lot::Lot;
 use crate::message::{Message, MessageState, MsgId};
 use crate::stock::StockMessage;
@@ -37,7 +36,7 @@ use crate::stock::StockMessage;
 #[derive(Debug)]
 pub struct Chatlist {
     /// Stores pairs of `chat_id, message_id`
-    ids: Vec<(ChatId, MsgId)>,
+    ids: Vec<(u32, MsgId)>,
 }
 
 impl Chatlist {
@@ -61,7 +60,7 @@ impl Chatlist {
     ///   or "Not now".
     ///   The UI can also offer a "Close" button that calls dc_marknoticed_contact() then.
     /// - DC_CHAT_ID_ARCHIVED_LINK (6) - this special chat is present if the user has
-    ///   archived *any* chat using dc_set_chat_visibility(). The UI should show a link as
+    ///   archived *any* chat using dc_archive_chat(). The UI should show a link as
     ///   "Show archived chats", if the user clicks this item, the UI should show a
     ///   list of all archived chats that can be created by this function hen using
     ///   the DC_GCL_ARCHIVED_ONLY flag.
@@ -74,9 +73,6 @@ impl Chatlist {
     ///   if DC_GCL_ARCHIVED_ONLY is not set, only unarchived chats are returned and
     ///   the pseudo-chat DC_CHAT_ID_ARCHIVED_LINK is added if there are *any* archived
     ///   chats
-    /// - the flag DC_GCL_FOR_FORWARDING sorts "Saved messages" to the top of the chatlist
-    ///   and hides the device-chat,
-    //    typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS
     /// - if the flag DC_GCL_NO_SPECIALS is set, deaddrop and archive link are not added
     ///   to the list (may be used eg. for selecting chats on forwarding, the flag is
     ///   not needed when DC_GCL_ARCHIVED_ONLY is already set)
@@ -86,32 +82,23 @@ impl Chatlist {
     ///     are returned.
     /// `query_contact_id`: An optional contact ID for filtering the list. Only chats including this contact ID
     ///     are returned.
-    pub async fn try_load(
+    pub fn try_load(
         context: &Context,
         listflags: usize,
         query: Option<&str>,
         query_contact_id: Option<u32>,
     ) -> Result<Self> {
-        let flag_archived_only = 0 != listflags & DC_GCL_ARCHIVED_ONLY;
-        let flag_for_forwarding = 0 != listflags & DC_GCL_FOR_FORWARDING;
-        let flag_no_specials = 0 != listflags & DC_GCL_NO_SPECIALS;
-        let flag_add_alldone_hint = 0 != listflags & DC_GCL_ADD_ALLDONE_HINT;
-
-        // Note that we do not emit DC_EVENT_MSGS_MODIFIED here even if some
-        // messages get deleted to avoid reloading the same chatlist.
-        if let Err(err) = delete_device_expired_messages(context).await {
-            warn!(context, "Failed to hide expired messages: {}", err);
-        }
-
         let mut add_archived_link_item = false;
 
-        let skip_id = if flag_for_forwarding {
-            chat::lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
-                .await
-                .unwrap_or_default()
-                .0
-        } else {
-            ChatId::new(0)
+        let process_row = |row: &rusqlite::Row| {
+            let chat_id: u32 = row.get(0)?;
+            let msg_id: MsgId = row.get(1).unwrap_or_default();
+            Ok((chat_id, msg_id))
+        };
+
+        let process_rows = |rows: rusqlite::MappedRows<_>| {
+            rows.collect::<std::result::Result<Vec<_>, _>>()
+                .map_err(Into::into)
         };
 
         // select with left join and minimum:
@@ -131,131 +118,105 @@ impl Chatlist {
         // shown at all permanent in the chatlist.
         let mut ids = if let Some(query_contact_id) = query_contact_id {
             // show chats shared with a given contact
-            context.sql.query_rows(
+            context.sql.query_map(
                 "SELECT c.id, m.id
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.id=(
-                               SELECT id
+                       AND m.timestamp=(
+                               SELECT MAX(timestamp)
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?)
-                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
+                                  AND (hidden=0 OR state=?))
                  WHERE c.id>9
                    AND c.blocked=0
                    AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)
                  GROUP BY c.id
-                 ORDER BY c.archived=? DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                paramsx![MessageState::OutDraft, query_contact_id as i32, ChatVisibility::Pinned],
-            ).await?
-        } else if flag_archived_only {
+                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                params![MessageState::OutDraft, query_contact_id as i32],
+                process_row,
+                process_rows,
+            )?
+        } else if 0 != listflags & DC_GCL_ARCHIVED_ONLY {
             // show archived chats
-            // (this includes the archived device-chat; we could skip it,
-            // however, then the number of archived chats do not match, which might be even more irritating.
-            // and adapting the number requires larger refactorings and seems not to be worth the effort)
-            context
-                .sql
-                .query_rows(
-                    "SELECT c.id, m.id
+            context.sql.query_map(
+                "SELECT c.id, m.id
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.id=(
-                               SELECT id
+                       AND m.timestamp=(
+                               SELECT MAX(timestamp)
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?)
-                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
+                                  AND (hidden=0 OR state=?))
                  WHERE c.id>9
                    AND c.blocked=0
                    AND c.archived=1
                  GROUP BY c.id
                  ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                    paramsx![MessageState::OutDraft],
-                )
-                .await?
+                params![MessageState::OutDraft],
+                process_row,
+                process_rows,
+            )?
         } else if let Some(query) = query {
             let query = query.trim().to_string();
             ensure!(!query.is_empty(), "missing query");
 
-            // allow searching over special names that may change at any time
-            // when the ui calls set_stock_translation()
-            if let Err(err) = update_special_chat_names(context).await {
-                warn!(context, "cannot update special chat names: {:?}", err)
-            }
-
             let str_like_cmd = format!("%{}%", query);
-            context
-                .sql
-                .query_rows(
-                    "SELECT c.id, m.id
-                 FROM chats c
-                 LEFT JOIN msgs m
-                        ON c.id=m.chat_id
-                       AND m.id=(
-                               SELECT id
-                                 FROM msgs
-                                WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1)
-                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
-                 WHERE c.id>9 AND c.id!=?2
-                   AND c.blocked=0
-                   AND c.name LIKE ?3
-                 GROUP BY c.id
-                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                    paramsx![MessageState::OutDraft, skip_id, str_like_cmd],
-                )
-                .await?
-        } else {
-            //  show normal chatlist
-            let sort_id_up = if flag_for_forwarding {
-                chat::lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
-                    .await
-                    .unwrap_or_default()
-                    .0
-            } else {
-                ChatId::new(0)
-            };
-            let mut ids = context.sql.query_rows(
+            context.sql.query_map(
                 "SELECT c.id, m.id
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.id=(
-                               SELECT id
+                       AND m.timestamp=(
+                               SELECT MAX(timestamp)
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1)
-                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
-                 WHERE c.id>9 AND c.id!=?2
+                                  AND (hidden=0 OR state=?))
+                 WHERE c.id>9
                    AND c.blocked=0
-                   AND NOT c.archived=?3
+                   AND c.name LIKE ?
                  GROUP BY c.id
-                 ORDER BY c.id=?4 DESC, c.archived=?5 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                paramsx![MessageState::OutDraft, skip_id, ChatVisibility::Archived, sort_id_up, ChatVisibility::Pinned],
-            ).await?;
-
-            if !flag_no_specials {
-                if let Some(last_deaddrop_fresh_msg_id) = get_last_deaddrop_fresh_msg(context).await
-                {
-                    if !flag_for_forwarding {
-                        ids.insert(
-                            0,
-                            (ChatId::new(DC_CHAT_ID_DEADDROP), last_deaddrop_fresh_msg_id),
-                        );
-                    }
+                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                params![MessageState::OutDraft, str_like_cmd],
+                process_row,
+                process_rows,
+            )?
+        } else {
+            //  show normal chatlist
+            let mut ids = context.sql.query_map(
+                "SELECT c.id, m.id
+                 FROM chats c
+                 LEFT JOIN msgs m
+                        ON c.id=m.chat_id
+                       AND m.timestamp=(
+                               SELECT MAX(timestamp)
+                                 FROM msgs
+                                WHERE chat_id=c.id
+                                  AND (hidden=0 OR state=?))
+                 WHERE c.id>9
+                   AND c.blocked=0
+                   AND c.archived=0
+                 GROUP BY c.id
+                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                params![MessageState::OutDraft],
+                process_row,
+                process_rows,
+            )?;
+            if 0 == listflags & DC_GCL_NO_SPECIALS {
+                if let Some(last_deaddrop_fresh_msg_id) = get_last_deaddrop_fresh_msg(context) {
+                    ids.insert(0, (DC_CHAT_ID_DEADDROP, last_deaddrop_fresh_msg_id));
                 }
                 add_archived_link_item = true;
             }
             ids
         };
 
-        if add_archived_link_item && dc_get_archived_cnt(context).await > 0 {
-            if ids.is_empty() && flag_add_alldone_hint {
-                ids.push((ChatId::new(DC_CHAT_ID_ALLDONE_HINT), MsgId::new(0)));
+        if add_archived_link_item && dc_get_archived_cnt(context) > 0 {
+            if ids.is_empty() && 0 != listflags & DC_GCL_ADD_ALLDONE_HINT {
+                ids.push((DC_CHAT_ID_ALLDONE_HINT, MsgId::new(0)));
             }
-            ids.push((ChatId::new(DC_CHAT_ID_ARCHIVED_LINK), MsgId::new(0)));
+            ids.push((DC_CHAT_ID_ARCHIVED_LINK, MsgId::new(0)));
         }
 
         Ok(Chatlist { ids })
@@ -274,21 +235,19 @@ impl Chatlist {
     /// Get a single chat ID of a chatlist.
     ///
     /// To get the message object from the message ID, use dc_get_chat().
-    pub fn get_chat_id(&self, index: usize) -> ChatId {
-        match self.ids.get(index) {
-            Some((chat_id, _msg_id)) => *chat_id,
-            None => ChatId::new(0),
+    pub fn get_chat_id(&self, index: usize) -> u32 {
+        if index >= self.ids.len() {
+            return 0;
         }
+        self.ids[index].0
     }
 
     /// Get a single message ID of a chatlist.
     ///
     /// To get the message object from the message ID, use dc_get_msg().
     pub fn get_msg_id(&self, index: usize) -> Result<MsgId> {
-        match self.ids.get(index) {
-            Some((_chat_id, msg_id)) => Ok(*msg_id),
-            None => bail!("Chatlist index out of range"),
-        }
+        ensure!(index < self.ids.len(), "Chatlist index out of range");
+        Ok(self.ids[index].1)
     }
 
     /// Get a summary for a chatlist index.
@@ -305,38 +264,36 @@ impl Chatlist {
     /// - dc_lot_t::timestamp: the timestamp of the message.  0 if not applicable.
     /// - dc_lot_t::state: The state of the message as one of the DC_STATE_* constants (see #dc_msg_get_state()).
     //    0 if not applicable.
-    pub async fn get_summary(&self, context: &Context, index: usize, chat: Option<&Chat>) -> Lot {
+    pub fn get_summary(&self, context: &Context, index: usize, chat: Option<&Chat>) -> Lot {
         // The summary is created by the chat, not by the last message.
         // This is because we may want to display drafts here or stuff as
         // "is typing".
         // Also, sth. as "No messages" would not work if the summary comes from a message.
-        let mut ret = Lot::new();
 
-        let (chat_id, lastmsg_id) = match self.ids.get(index) {
-            Some(ids) => ids,
-            None => {
-                ret.text2 = Some("ErrBadChatlistIndex".to_string());
-                return ret;
-            }
-        };
+        let mut ret = Lot::new();
+        if index >= self.ids.len() {
+            ret.text2 = Some("ErrBadChatlistIndex".to_string());
+            return ret;
+        }
 
         let chat_loaded: Chat;
         let chat = if let Some(chat) = chat {
             chat
-        } else if let Ok(chat) = Chat::load_from_db(context, *chat_id).await {
+        } else if let Ok(chat) = Chat::load_from_db(context, self.ids[index].0) {
             chat_loaded = chat;
             &chat_loaded
         } else {
             return ret;
         };
 
+        let lastmsg_id = self.ids[index].1;
         let mut lastcontact = None;
 
-        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, *lastmsg_id).await {
+        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id) {
             if lastmsg.from_id != DC_CONTACT_ID_SELF
                 && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup)
             {
-                lastcontact = Contact::load_from_db(context, lastmsg.from_id).await.ok();
+                lastcontact = Contact::load_from_db(context, lastmsg.from_id).ok();
             }
 
             Some(lastmsg)
@@ -344,85 +301,66 @@ impl Chatlist {
             None
         };
 
-        if chat.id.is_archived_link() {
+        if chat.id == DC_CHAT_ID_ARCHIVED_LINK {
             ret.text2 = None;
         } else if lastmsg.is_none() || lastmsg.as_ref().unwrap().from_id == DC_CONTACT_ID_UNDEFINED
         {
-            ret.text2 = Some(
-                context
-                    .stock_str(StockMessage::NoMessages)
-                    .await
-                    .to_string(),
-            );
+            ret.text2 = Some(context.stock_str(StockMessage::NoMessages).to_string());
         } else {
-            ret.fill(&mut lastmsg.unwrap(), chat, lastcontact.as_ref(), context)
-                .await;
+            ret.fill(&mut lastmsg.unwrap(), chat, lastcontact.as_ref(), context);
         }
 
         ret
     }
-
-    pub fn get_index_for_id(&self, id: ChatId) -> Option<usize> {
-        self.ids.iter().position(|(chat_id, _)| chat_id == &id)
-    }
 }
 
 /// Returns the number of archived chats
-pub async fn dc_get_archived_cnt(context: &Context) -> u32 {
-    let v: i32 = context
-        .sql
-        .query_value(
-            "SELECT COUNT(*) FROM chats WHERE blocked=0 AND archived=1;",
-            paramsx![],
-        )
-        .await
-        .unwrap_or_default();
-    v as u32
-}
-
-async fn get_last_deaddrop_fresh_msg(context: &Context) -> Option<MsgId> {
-    // We have an index over the state-column, this should be
-    // sufficient as there are typically only few fresh messages.
+pub fn dc_get_archived_cnt(context: &Context) -> u32 {
     context
         .sql
-        .query_value(
-            r#"
-SELECT m.id
-  FROM msgs m
-  LEFT JOIN chats c
-        ON c.id=m.chat_id
- WHERE m.state=10
-   AND m.hidden=0
-   AND c.blocked=2
- ORDER BY m.timestamp DESC, m.id DESC;
-"#,
-            paramsx![],
+        .query_get_value(
+            context,
+            "SELECT COUNT(*) FROM chats WHERE blocked=0 AND archived=1;",
+            params![],
         )
-        .await
-        .ok()
+        .unwrap_or_default()
+}
+
+fn get_last_deaddrop_fresh_msg(context: &Context) -> Option<MsgId> {
+    // We have an index over the state-column, this should be
+    // sufficient as there are typically only few fresh messages.
+    context.sql.query_get_value(
+        context,
+        concat!(
+            "SELECT m.id",
+            " FROM msgs m",
+            " LEFT JOIN chats c",
+            "        ON c.id=m.chat_id",
+            " WHERE m.state=10",
+            "   AND m.hidden=0",
+            "   AND c.blocked=2",
+            " ORDER BY m.timestamp DESC, m.id DESC;"
+        ),
+        params![],
+    )
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
 
+    use crate::chat;
     use crate::test_utils::*;
 
-    #[async_std::test]
-    async fn test_try_load() {
-        let t = TestContext::new().await;
-        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
-            .await
-            .unwrap();
-        let chat_id2 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "b chat")
-            .await
-            .unwrap();
-        let chat_id3 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "c chat")
-            .await
-            .unwrap();
+    #[test]
+    fn test_try_load() {
+        let t = dummy_context();
+        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat").unwrap();
+        let chat_id2 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "b chat").unwrap();
+        let chat_id3 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "c chat").unwrap();
 
         // check that the chatlist starts with the most recent message
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
         assert_eq!(chats.len(), 3);
         assert_eq!(chats.get_chat_id(0), chat_id3);
         assert_eq!(chats.get_chat_id(1), chat_id2);
@@ -431,102 +369,19 @@ mod tests {
         // drafts are sorted to the top
         let mut msg = Message::new(Viewtype::Text);
         msg.set_text(Some("hello".to_string()));
-        chat_id2.set_draft(&t.ctx, Some(&mut msg)).await;
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        set_draft(&t.ctx, chat_id2, Some(&mut msg));
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
         assert_eq!(chats.get_chat_id(0), chat_id2);
 
         // check chatlist query and archive functionality
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("b"), None)
-            .await
-            .unwrap();
+        let chats = Chatlist::try_load(&t.ctx, 0, Some("b"), None).unwrap();
         assert_eq!(chats.len(), 1);
 
-        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None)
-            .await
-            .unwrap();
+        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None).unwrap();
         assert_eq!(chats.len(), 0);
 
-        chat_id1
-            .set_visibility(&t.ctx, ChatVisibility::Archived)
-            .await
-            .ok();
-        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None)
-            .await
-            .unwrap();
+        chat::archive(&t.ctx, chat_id1, true).ok();
+        let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None).unwrap();
         assert_eq!(chats.len(), 1);
     }
-
-    #[async_std::test]
-    async fn test_sort_self_talk_up_on_forward() {
-        let t = TestContext::new().await;
-        t.ctx.update_device_chats().await.unwrap();
-        create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
-            .await
-            .unwrap();
-
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
-        assert!(chats.len() == 3);
-        assert!(!Chat::load_from_db(&t.ctx, chats.get_chat_id(0))
-            .await
-            .unwrap()
-            .is_self_talk());
-
-        let chats = Chatlist::try_load(&t.ctx, DC_GCL_FOR_FORWARDING, None, None)
-            .await
-            .unwrap();
-        assert!(chats.len() == 2); // device chat cannot be written and is skipped on forwarding
-        assert!(Chat::load_from_db(&t.ctx, chats.get_chat_id(0))
-            .await
-            .unwrap()
-            .is_self_talk());
-    }
-
-    #[async_std::test]
-    async fn test_search_special_chat_names() {
-        let t = TestContext::new().await;
-        t.ctx.update_device_chats().await.unwrap();
-
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None)
-            .await
-            .unwrap();
-        assert_eq!(chats.len(), 0);
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None)
-            .await
-            .unwrap();
-        assert_eq!(chats.len(), 0);
-
-        t.ctx
-            .set_stock_translation(StockMessage::SavedMessages, "test-1234-save".to_string())
-            .await
-            .unwrap();
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None)
-            .await
-            .unwrap();
-        assert_eq!(chats.len(), 1);
-
-        t.ctx
-            .set_stock_translation(StockMessage::DeviceMessages, "test-5678-babbel".to_string())
-            .await
-            .unwrap();
-        let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None)
-            .await
-            .unwrap();
-        assert_eq!(chats.len(), 1);
-    }
-
-    #[async_std::test]
-    async fn test_get_summary_unwrap() {
-        let t = TestContext::new().await;
-        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
-            .await
-            .unwrap();
-
-        let mut msg = Message::new(Viewtype::Text);
-        msg.set_text(Some("foo:\nbar \r\n test".to_string()));
-        chat_id1.set_draft(&t.ctx, Some(&mut msg)).await;
-
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
-        let summary = chats.get_summary(&t.ctx, 0, None).await;
-        assert_eq!(summary.get_text2().unwrap(), "foo: bar test"); // the linebreak should be removed from summary
-    }
 }

src/config.rs

@@ -4,14 +4,12 @@ use strum::{EnumProperty, IntoEnumIterator};
 use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString};
 
 use crate::blob::BlobObject;
-use crate::chat::ChatId;
 use crate::constants::DC_VERSION_STR;
 use crate::context::Context;
 use crate::dc_tools::*;
-use crate::events::Event;
-use crate::message::MsgId;
-use crate::mimefactory::RECOMMENDED_FILE_SIZE;
+use crate::job::*;
 use crate::stock::StockMessage;
+use rusqlite::NO_PARAMS;
 
 /// The available configuration keys.
 #[derive(
@@ -63,31 +61,6 @@ pub enum Config {
     #[strum(props(default = "0"))] // also change ShowEmails.default() on changes
     ShowEmails,
 
-    #[strum(props(default = "0"))] // also change MediaQuality.default() on changes
-    MediaQuality,
-
-    #[strum(props(default = "0"))]
-    KeyGenType,
-
-    /// Timer in seconds after which the message is deleted from the
-    /// server.
-    ///
-    /// Equals to 0 by default, which means the message is never
-    /// deleted.
-    ///
-    /// Value 1 is treated as "delete at once": messages are deleted
-    /// immediately, without moving to DeltaChat folder.
-    #[strum(props(default = "0"))]
-    DeleteServerAfter,
-
-    /// Timer in seconds after which the message is deleted from the
-    /// device.
-    ///
-    /// Equals to 0 by default, which means the message is never
-    /// deleted.
-    #[strum(props(default = "0"))]
-    DeleteDeviceAfter,
-
     SaveMimeHeaders,
     ConfiguredAddr,
     ConfiguredMailServer,
@@ -104,9 +77,6 @@ pub enum Config {
     ConfiguredServerFlags,
     ConfiguredSendSecurity,
     ConfiguredE2EEEnabled,
-    ConfiguredInboxFolder,
-    ConfiguredMvboxFolder,
-    ConfiguredSentboxFolder,
     Configured,
 
     #[strum(serialize = "sys.version")]
@@ -120,21 +90,17 @@ pub enum Config {
 }
 
 impl Context {
-    pub async fn config_exists(&self, key: Config) -> bool {
-        self.sql.get_raw_config(key).await.is_some()
-    }
-
     /// Get a configuration key. Returns `None` if no value is set, and no default value found.
-    pub async fn get_config(&self, key: Config) -> Option<String> {
+    pub fn get_config(&self, key: Config) -> Option<String> {
         let value = match key {
             Config::Selfavatar => {
-                let rel_path = self.sql.get_raw_config(key).await;
+                let rel_path = self.sql.get_raw_config(self, key);
                 rel_path.map(|p| dc_get_abs_path(self, &p).to_string_lossy().into_owned())
             }
             Config::SysVersion => Some((&*DC_VERSION_STR).clone()),
-            Config::SysMsgsizeMaxRecommended => Some(format!("{}", RECOMMENDED_FILE_SIZE)),
+            Config::SysMsgsizeMaxRecommended => Some(format!("{}", 24 * 1024 * 1024 / 4 * 3)),
             Config::SysConfigKeys => Some(get_config_keys_string()),
-            _ => self.sql.get_raw_config(key).await,
+            _ => self.sql.get_raw_config(self, key),
         };
 
         if value.is_some() {
@@ -143,88 +109,65 @@ impl Context {
 
         // Default values
         match key {
-            Config::Selfstatus => Some(self.stock_str(StockMessage::StatusLine).await.into_owned()),
-            Config::ConfiguredInboxFolder => Some("INBOX".to_owned()),
+            Config::Selfstatus => Some(self.stock_str(StockMessage::StatusLine).into_owned()),
             _ => key.get_str("default").map(|s| s.to_string()),
         }
     }
 
-    pub async fn get_config_int(&self, key: Config) -> i32 {
+    pub fn get_config_int(&self, key: Config) -> i32 {
         self.get_config(key)
-            .await
             .and_then(|s| s.parse().ok())
             .unwrap_or_default()
     }
 
-    pub async fn get_config_bool(&self, key: Config) -> bool {
-        self.get_config_int(key).await != 0
-    }
-
-    /// Gets configured "delete_server_after" value.
-    ///
-    /// `None` means never delete the message, `Some(0)` means delete
-    /// at once, `Some(x)` means delete after `x` seconds.
-    pub async fn get_config_delete_server_after(&self) -> Option<i64> {
-        match self.get_config_int(Config::DeleteServerAfter).await {
-            0 => None,
-            1 => Some(0),
-            x => Some(x as i64),
-        }
-    }
-
-    /// Gets configured "delete_device_after" value.
-    ///
-    /// `None` means never delete the message, `Some(x)` means delete
-    /// after `x` seconds.
-    pub async fn get_config_delete_device_after(&self) -> Option<i64> {
-        match self.get_config_int(Config::DeleteDeviceAfter).await {
-            0 => None,
-            x => Some(x as i64),
-        }
+    pub fn get_config_bool(&self, key: Config) -> bool {
+        self.get_config_int(key) != 0
     }
 
     /// Set the given config key.
     /// If `None` is passed as a value the value is cleared and set to the default if there is one.
-    pub async fn set_config(&self, key: Config, value: Option<&str>) -> crate::sql::Result<()> {
+    pub fn set_config(&self, key: Config, value: Option<&str>) -> crate::sql::Result<()> {
         match key {
             Config::Selfavatar => {
                 self.sql
-                    .execute("UPDATE contacts SET selfavatar_sent=0;", paramsx![])
-                    .await?;
+                    .execute("UPDATE contacts SET selfavatar_sent=0;", NO_PARAMS)?;
                 self.sql
-                    .set_raw_config_bool(self, "attach_selfavatar", true)
-                    .await?;
+                    .set_raw_config_bool(self, "attach_selfavatar", true)?;
                 match value {
                     Some(value) => {
-                        let blob = BlobObject::new_from_path(&self, value).await?;
+                        let blob = BlobObject::new_from_path(&self, value)?;
                         blob.recode_to_avatar_size(self)?;
-                        self.sql
-                            .set_raw_config(self, key, Some(blob.as_name()))
-                            .await
+                        self.sql.set_raw_config(self, key, Some(blob.as_name()))
                     }
-                    None => self.sql.set_raw_config(self, key, None).await,
+                    None => self.sql.set_raw_config(self, key, None),
                 }
             }
+            Config::InboxWatch => {
+                let ret = self.sql.set_raw_config(self, key, value);
+                interrupt_inbox_idle(self, true);
+                ret
+            }
+            Config::SentboxWatch => {
+                let ret = self.sql.set_raw_config(self, key, value);
+                interrupt_sentbox_idle(self);
+                ret
+            }
+            Config::MvboxWatch => {
+                let ret = self.sql.set_raw_config(self, key, value);
+                interrupt_mvbox_idle(self);
+                ret
+            }
             Config::Selfstatus => {
-                let def = self.stock_str(StockMessage::StatusLine).await;
+                let def = self.stock_str(StockMessage::StatusLine);
                 let val = if value.is_none() || value.unwrap() == def {
                     None
                 } else {
                     value
                 };
 
-                self.sql.set_raw_config(self, key, val).await
+                self.sql.set_raw_config(self, key, val)
             }
-            Config::DeleteDeviceAfter => {
-                let ret = self.sql.set_raw_config(self, key, value).await;
-                // Force chatlist reload to delete old messages immediately.
-                self.emit_event(Event::MsgsChanged {
-                    msg_id: MsgId::new(0),
-                    chat_id: ChatId::new(0),
-                });
-                ret
-            }
-            _ => self.sql.set_raw_config(self, key, value).await,
+            _ => self.sql.set_raw_config(self, key, value),
         }
     }
 }
@@ -247,11 +190,7 @@ mod tests {
     use std::str::FromStr;
     use std::string::ToString;
 
-    use crate::constants;
-    use crate::constants::AVATAR_SIZE;
     use crate::test_utils::*;
-    use image::GenericImageView;
-    use num_traits::FromPrimitive;
     use std::fs::File;
     use std::io::Write;
 
@@ -272,102 +211,33 @@ mod tests {
         assert_eq!(Config::ImapFolder.get_str("default"), Some("INBOX"));
     }
 
-    #[async_std::test]
-    async fn test_selfavatar_outside_blobdir() {
-        let t = TestContext::new().await;
+    #[test]
+    fn test_selfavatar_outside_blobdir() -> failure::Fallible<()> {
+        let t = dummy_context();
         let avatar_src = t.dir.path().join("avatar.jpg");
         let avatar_bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
-        File::create(&avatar_src)
-            .unwrap()
-            .write_all(avatar_bytes)
-            .unwrap();
+        File::create(&avatar_src)?.write_all(avatar_bytes)?;
         let avatar_blob = t.ctx.get_blobdir().join("avatar.jpg");
-        assert!(!avatar_blob.exists().await);
+        assert!(!avatar_blob.exists());
         t.ctx
-            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
-            .await
-            .unwrap();
-        assert!(avatar_blob.exists().await);
+            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))?;
+        assert!(avatar_blob.exists());
         assert!(std::fs::metadata(&avatar_blob).unwrap().len() < avatar_bytes.len() as u64);
-        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
+        let avatar_cfg = t.ctx.get_config(Config::Selfavatar);
         assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
-
-        let img = image::open(avatar_src).unwrap();
-        assert_eq!(img.width(), 1000);
-        assert_eq!(img.height(), 1000);
-
-        let img = image::open(avatar_blob).unwrap();
-        assert_eq!(img.width(), AVATAR_SIZE);
-        assert_eq!(img.height(), AVATAR_SIZE);
+        Ok(())
     }
 
-    #[async_std::test]
-    async fn test_selfavatar_in_blobdir() {
-        let t = TestContext::new().await;
-        let avatar_src = t.ctx.get_blobdir().join("avatar.png");
-        let avatar_bytes = include_bytes!("../test-data/image/avatar900x900.png");
-        File::create(&avatar_src)
-            .unwrap()
-            .write_all(avatar_bytes)
-            .unwrap();
-
-        let img = image::open(&avatar_src).unwrap();
-        assert_eq!(img.width(), 900);
-        assert_eq!(img.height(), 900);
-
+    #[test]
+    fn test_selfavatar_in_blobdir() -> failure::Fallible<()> {
+        let t = dummy_context();
+        let avatar_src = t.ctx.get_blobdir().join("avatar.jpg");
+        let avatar_bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
+        File::create(&avatar_src)?.write_all(avatar_bytes)?;
         t.ctx
-            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
-            .await
-            .unwrap();
-        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
+            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))?;
+        let avatar_cfg = t.ctx.get_config(Config::Selfavatar);
         assert_eq!(avatar_cfg, avatar_src.to_str().map(|s| s.to_string()));
-
-        let img = image::open(avatar_src).unwrap();
-        assert_eq!(img.width(), AVATAR_SIZE);
-        assert_eq!(img.height(), AVATAR_SIZE);
-    }
-
-    #[async_std::test]
-    async fn test_selfavatar_copy_without_recode() {
-        let t = TestContext::new().await;
-        let avatar_src = t.dir.path().join("avatar.png");
-        let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
-        File::create(&avatar_src)
-            .unwrap()
-            .write_all(avatar_bytes)
-            .unwrap();
-        let avatar_blob = t.ctx.get_blobdir().join("avatar.png");
-        assert!(!avatar_blob.exists().await);
-        t.ctx
-            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
-            .await
-            .unwrap();
-        assert!(avatar_blob.exists().await);
-        assert_eq!(
-            std::fs::metadata(&avatar_blob).unwrap().len(),
-            avatar_bytes.len() as u64
-        );
-        let avatar_cfg = t.ctx.get_config(Config::Selfavatar).await;
-        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
-    }
-
-    #[async_std::test]
-    async fn test_media_quality_config_option() {
-        let t = TestContext::new().await;
-        let media_quality = t.ctx.get_config_int(Config::MediaQuality).await;
-        assert_eq!(media_quality, 0);
-        let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
-        assert_eq!(media_quality, constants::MediaQuality::Balanced);
-
-        t.ctx
-            .set_config(Config::MediaQuality, Some("1"))
-            .await
-            .unwrap();
-
-        let media_quality = t.ctx.get_config_int(Config::MediaQuality).await;
-        assert_eq!(media_quality, 1);
-        assert_eq!(constants::MediaQuality::Worse as i32, 1);
-        let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
-        assert_eq!(media_quality, constants::MediaQuality::Worse);
+        Ok(())
     }
 }

src/configure/auto_mozilla.rs

@@ -1,6 +1,7 @@
 //! # Thunderbird's Autoconfiguration implementation
 //!
 //! Documentation: https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration */
+use quick_xml;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
 use crate::constants::*;
@@ -8,7 +9,33 @@ use crate::context::Context;
 use crate::login_param::LoginParam;
 
 use super::read_url::read_url;
-use super::Error;
+
+#[derive(Debug, Fail)]
+pub enum Error {
+    #[fail(display = "Invalid email address: {:?}", _0)]
+    InvalidEmailAddress(String),
+
+    #[fail(display = "XML error at position {}", position)]
+    InvalidXml {
+        position: usize,
+        #[cause]
+        error: quick_xml::Error,
+    },
+
+    #[fail(display = "Bad or incomplete autoconfig")]
+    IncompleteAutoconfig(LoginParam),
+
+    #[fail(display = "Failed to get URL {}", _0)]
+    ReadUrlError(#[cause] super::read_url::Error),
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+impl From<super::read_url::Error> for Error {
+    fn from(err: super::read_url::Error) -> Error {
+        Error::ReadUrlError(err)
+    }
+}
 
 #[derive(Debug)]
 struct MozAutoconfigure<'a> {
@@ -38,16 +65,16 @@ enum MozConfigTag {
     Username,
 }
 
-fn parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result<LoginParam, Error> {
+fn parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result<LoginParam> {
     let mut reader = quick_xml::Reader::from_str(xml_raw);
     reader.trim_text(true);
 
     // Split address into local part and domain part.
-    let parts: Vec<&str> = in_emailaddr.rsplitn(2, '@').collect();
-    let (in_emaillocalpart, in_emaildomain) = match &parts[..] {
-        [domain, local] => (local, domain),
-        _ => return Err(Error::InvalidEmailAddress(in_emailaddr.to_string())),
-    };
+    let p = in_emailaddr
+        .find('@')
+        .ok_or_else(|| Error::InvalidEmailAddress(in_emailaddr.to_string()))?;
+    let (in_emaillocalpart, in_emaildomain) = in_emailaddr.split_at(p);
+    let in_emaildomain = &in_emaildomain[1..];
 
     let mut moz_ac = MozAutoconfigure {
         in_emailaddr,
@@ -94,12 +121,12 @@ fn parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result<LoginParam, Error> {
     }
 }
 
-pub async fn moz_autoconfigure(
+pub fn moz_autoconfigure(
     context: &Context,
     url: &str,
     param_in: &LoginParam,
-) -> Result<LoginParam, Error> {
-    let xml_raw = read_url(context, url).await?;
+) -> Result<LoginParam> {
+    let xml_raw = read_url(context, url)?;
 
     let res = parse_xml(&param_in.addr, &xml_raw);
     if let Err(err) = &res {

src/configure/auto_outlook.rs

@@ -1,5 +1,6 @@
 //! Outlook's Autodiscover
 
+use quick_xml;
 use quick_xml::events::BytesEnd;
 
 use crate::constants::*;
@@ -7,7 +8,33 @@ use crate::context::Context;
 use crate::login_param::LoginParam;
 
 use super::read_url::read_url;
-use super::Error;
+
+#[derive(Debug, Fail)]
+pub enum Error {
+    #[fail(display = "XML error at position {}", position)]
+    InvalidXml {
+        position: usize,
+        #[cause]
+        error: quick_xml::Error,
+    },
+
+    #[fail(display = "Bad or incomplete autoconfig")]
+    IncompleteAutoconfig(LoginParam),
+
+    #[fail(display = "Failed to get URL {}", _0)]
+    ReadUrlError(#[cause] super::read_url::Error),
+
+    #[fail(display = "Number of redirection is exceeded")]
+    RedirectionError,
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+impl From<super::read_url::Error> for Error {
+    fn from(err: super::read_url::Error) -> Error {
+        Error::ReadUrlError(err)
+    }
+}
 
 struct OutlookAutodiscover {
     pub out: LoginParam,
@@ -25,7 +52,7 @@ enum ParsingResult {
     RedirectUrl(String),
 }
 
-fn parse_xml(xml_raw: &str) -> Result<ParsingResult, Error> {
+fn parse_xml(xml_raw: &str) -> Result<ParsingResult> {
     let mut outlk_ad = OutlookAutodiscover {
         out: LoginParam::new(),
         out_imap_set: false,
@@ -112,15 +139,15 @@ fn parse_xml(xml_raw: &str) -> Result<ParsingResult, Error> {
     Ok(res)
 }
 
-pub async fn outlk_autodiscover(
+pub fn outlk_autodiscover(
     context: &Context,
     url: &str,
     _param_in: &LoginParam,
-) -> Result<LoginParam, Error> {
+) -> Result<LoginParam> {
     let mut url = url.to_string();
     /* Follow up to 10 xml-redirects (http-redirects are followed in read_url() */
     for _i in 0..10 {
-        let xml_raw = read_url(context, &url).await?;
+        let xml_raw = read_url(context, &url)?;
         let res = parse_xml(&xml_raw);
         if let Err(err) = &res {
             warn!(context, "{}", err);

src/configure/read_url.rs

@@ -1,15 +1,21 @@
 use crate::context::Context;
 
-#[derive(Debug, thiserror::Error)]
+#[derive(Debug, Fail)]
 pub enum Error {
-    #[error("URL request error")]
-    GetError(surf::Error),
+    #[fail(display = "URL request error")]
+    GetError(#[cause] reqwest::Error),
 }
 
-pub async fn read_url(context: &Context, url: &str) -> Result<String, Error> {
+pub type Result<T> = std::result::Result<T, Error>;
+
+pub fn read_url(context: &Context, url: &str) -> Result<String> {
     info!(context, "Requesting URL {}", url);
 
-    match surf::get(url).recv_string().await {
+    match reqwest::Client::new()
+        .get(url)
+        .send()
+        .and_then(|mut res| res.text())
+    {
         Ok(res) => Ok(res),
         Err(err) => {
             info!(context, "Can\'t read URL {}", url);

src/constants.rs

@@ -1,8 +1,8 @@
 //! # Constants
-#![allow(dead_code)]
+#![allow(non_camel_case_types, dead_code)]
 
+use deltachat_derive::*;
 use lazy_static::lazy_static;
-use serde::{Deserialize, Serialize};
 
 lazy_static! {
     pub static ref DC_VERSION_STR: String = env!("CARGO_PKG_VERSION").to_string();
@@ -15,20 +15,8 @@ const DC_SENTBOX_WATCH_DEFAULT: i32 = 1;
 const DC_MVBOX_WATCH_DEFAULT: i32 = 1;
 const DC_MVBOX_MOVE_DEFAULT: i32 = 1;
 
-#[derive(
-    Debug,
-    Display,
-    Clone,
-    Copy,
-    PartialEq,
-    Eq,
-    FromPrimitive,
-    ToPrimitive,
-    Serialize,
-    Deserialize,
-    sqlx::Type,
-)]
-#[repr(i32)]
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[repr(u8)]
 pub enum Blocked {
     Not = 0,
     Manually = 1,
@@ -41,8 +29,8 @@ impl Default for Blocked {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, sqlx::Type)]
-#[repr(i32)]
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[repr(u8)]
 pub enum ShowEmails {
     Off = 0,
     AcceptedContacts = 1,
@@ -55,43 +43,15 @@ impl Default for ShowEmails {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, sqlx::Type)]
-#[repr(i32)]
-pub enum MediaQuality {
-    Balanced = 0,
-    Worse = 1,
-}
-
-impl Default for MediaQuality {
-    fn default() -> Self {
-        MediaQuality::Balanced // also change Config.MediaQuality props(default) on changes
-    }
-}
-
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, sqlx::Type)]
-#[repr(u8)]
-pub enum KeyGenType {
-    Default = 0,
-    Rsa2048 = 1,
-    Ed25519 = 2,
-}
-
-impl Default for KeyGenType {
-    fn default() -> Self {
-        KeyGenType::Default
-    }
-}
+pub const DC_IMAP_SEEN: u32 = 0x1;
 
 pub const DC_HANDSHAKE_CONTINUE_NORMAL_PROCESSING: i32 = 0x01;
 pub const DC_HANDSHAKE_STOP_NORMAL_PROCESSING: i32 = 0x02;
 pub const DC_HANDSHAKE_ADD_DELETE_JOB: i32 = 0x04;
 
-pub(crate) const DC_FROM_HANDSHAKE: i32 = 0x01;
-
 pub const DC_GCL_ARCHIVED_ONLY: usize = 0x01;
 pub const DC_GCL_NO_SPECIALS: usize = 0x02;
 pub const DC_GCL_ADD_ALLDONE_HINT: usize = 0x04;
-pub const DC_GCL_FOR_FORWARDING: usize = 0x08;
 
 pub const DC_GCM_ADDDAYMARKER: u32 = 0x01;
 
@@ -101,6 +61,10 @@ pub const DC_GCL_ADD_SELF: usize = 0x02;
 // unchanged user avatars are resent to the recipients every some days
 pub const DC_RESEND_USER_AVATAR_DAYS: i64 = 14;
 
+// values for DC_PARAM_FORCE_PLAINTEXT
+pub(crate) const DC_FP_NO_AUTOCRYPT_HEADER: i32 = 2;
+pub(crate) const DC_FP_ADD_AUTOCRYPT_HEADER: i32 = 1;
+
 /// virtual chat showing all messages belonging to chats flagged with chats.blocked=2
 pub(crate) const DC_CHAT_ID_DEADDROP: u32 = 1;
 /// messages that should be deleted get this chat_id; the messages are deleted from the working thread later then. This is also needed as rfc724_mid should be preset as long as the message is not deleted on the server (otherwise it is downloaded again)
@@ -125,12 +89,11 @@ pub const DC_CHAT_ID_LAST_SPECIAL: u32 = 9;
     Eq,
     FromPrimitive,
     ToPrimitive,
+    FromSql,
+    ToSql,
     IntoStaticStr,
-    Serialize,
-    Deserialize,
-    sqlx::Type,
 )]
-#[repr(i32)]
+#[repr(u32)]
 pub enum Chattype {
     Undefined = 0,
     Single = 100,
@@ -151,7 +114,7 @@ pub const DC_MSG_ID_LAST_SPECIAL: u32 = 9;
 /// approx. max. length returned by dc_msg_get_text()
 const DC_MAX_GET_TEXT_LEN: usize = 30000;
 /// approx. max. length returned by dc_get_msg_info()
-const DC_MAX_GET_INFO_LEN: usize = 100_000;
+const DC_MAX_GET_INFO_LEN: usize = 100000;
 
 pub const DC_CONTACT_ID_UNDEFINED: u32 = 0;
 pub const DC_CONTACT_ID_SELF: u32 = 1;
@@ -207,13 +170,13 @@ pub const DC_LP_SMTP_SOCKET_SSL: usize = 0x20000;
 pub const DC_LP_SMTP_SOCKET_PLAIN: usize = 0x40000;
 
 /// if none of these flags are set, the default is chosen
-pub const DC_LP_AUTH_FLAGS: i32 = DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL;
+pub const DC_LP_AUTH_FLAGS: i32 = (DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL);
 /// if none of these flags are set, the default is chosen
 pub const DC_LP_IMAP_SOCKET_FLAGS: i32 =
-    DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_SSL | DC_LP_IMAP_SOCKET_PLAIN;
+    (DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_SSL | DC_LP_IMAP_SOCKET_PLAIN);
 /// if none of these flags are set, the default is chosen
 pub const DC_LP_SMTP_SOCKET_FLAGS: usize =
-    DC_LP_SMTP_SOCKET_STARTTLS | DC_LP_SMTP_SOCKET_SSL | DC_LP_SMTP_SOCKET_PLAIN;
+    (DC_LP_SMTP_SOCKET_STARTTLS | DC_LP_SMTP_SOCKET_SSL | DC_LP_SMTP_SOCKET_PLAIN);
 
 // QR code scanning (view from Bob, the joiner)
 pub const DC_VC_AUTH_REQUIRED: i32 = 2;
@@ -224,30 +187,10 @@ pub const DC_BOB_SUCCESS: i32 = 1;
 // max. width/height of an avatar
 pub const AVATAR_SIZE: u32 = 192;
 
-// max. width/height of images
-pub const BALANCED_IMAGE_SIZE: u32 = 1280;
-pub const WORSE_IMAGE_SIZE: u32 = 640;
-
-// this value can be increased if the folder configuration is changed and must be redone on next program start
-pub const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3;
-
-#[derive(
-    Debug,
-    Display,
-    Clone,
-    Copy,
-    PartialEq,
-    Eq,
-    FromPrimitive,
-    ToPrimitive,
-    Serialize,
-    Deserialize,
-    sqlx::Type,
-)]
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
 #[repr(i32)]
 pub enum Viewtype {
     Unknown = 0,
-
     /// Text message.
     /// The text of the message is set using dc_msg_set_text()
     /// and retrieved with dc_msg_get_text().
@@ -327,6 +270,8 @@ const DC_STR_SELFNOTINGRP: usize = 21; // deprecated;
 const DC_STR_NOMESSAGES: usize = 1;
 const DC_STR_SELF: usize = 2;
 const DC_STR_DRAFT: usize = 3;
+const DC_STR_MEMBER: usize = 4;
+const DC_STR_CONTACT: usize = 6;
 const DC_STR_VOICEMESSAGE: usize = 7;
 const DC_STR_DEADDROP: usize = 8;
 const DC_STR_IMAGE: usize = 9;
@@ -358,6 +303,7 @@ const DC_STR_ARCHIVEDCHATS: usize = 40;
 const DC_STR_STARREDMSGS: usize = 41;
 const DC_STR_AC_SETUP_MSG_SUBJECT: usize = 42;
 const DC_STR_AC_SETUP_MSG_BODY: usize = 43;
+const DC_STR_SELFTALK_SUBTITLE: usize = 50;
 const DC_STR_CANNOT_LOGIN: usize = 60;
 const DC_STR_SERVER_RESPONSE: usize = 61;
 const DC_STR_MSGACTIONBYUSER: usize = 62;

src/context.rs

@@ -1,70 +1,75 @@
 //! Context module
 
-use std::collections::{BTreeMap, HashMap};
+use std::collections::HashMap;
 use std::ffi::OsString;
-use std::ops::Deref;
+use std::path::{Path, PathBuf};
+use std::sync::{Arc, Condvar, Mutex, RwLock};
 
-use async_std::path::{Path, PathBuf};
-use async_std::sync::{channel, Arc, Mutex, Receiver, RwLock, Sender};
+use libc::uintptr_t;
 
 use crate::chat::*;
 use crate::config::Config;
 use crate::constants::*;
 use crate::contact::*;
-use crate::dc_tools::duration_to_str;
 use crate::error::*;
-use crate::events::{Event, EventEmitter, Events};
-use crate::job::{self, Action};
-use crate::key::{DcKey, SignedPublicKey};
+use crate::events::Event;
+use crate::imap::*;
+use crate::job::*;
+use crate::job_thread::JobThread;
+use crate::key::*;
 use crate::login_param::LoginParam;
 use crate::lot::Lot;
-use crate::message::{self, Message, MessengerMessage, MsgId};
+use crate::message::{self, Message, MsgId};
 use crate::param::Params;
-use crate::scheduler::Scheduler;
+use crate::smtp::Smtp;
 use crate::sql::Sql;
-use std::time::SystemTime;
 
-#[derive(Clone, Debug)]
+/// Callback function type for [Context]
+///
+/// # Parameters
+///
+/// * `context` - The context object as returned by [Context::new].
+/// * `event` - One of the [Event] items.
+/// * `data1` - Depends on the event parameter, see [Event].
+/// * `data2` - Depends on the event parameter, see [Event].
+///
+/// # Returns
+///
+/// This callback must return 0 unless stated otherwise in the event
+/// description at [Event].
+pub type ContextCallback = dyn Fn(&Context, Event) -> uintptr_t + Send + Sync;
+
+#[derive(DebugStub)]
 pub struct Context {
-    pub(crate) inner: Arc<InnerContext>,
-}
-
-impl Deref for Context {
-    type Target = InnerContext;
-
-    fn deref(&self) -> &Self::Target {
-        &self.inner
-    }
-}
-
-#[derive(Debug)]
-pub struct InnerContext {
     /// Database file path
-    pub(crate) dbfile: PathBuf,
+    dbfile: PathBuf,
     /// Blob directory path
-    pub(crate) blobdir: PathBuf,
-    pub(crate) sql: Sql,
-    pub(crate) os_name: Option<String>,
-    pub(crate) bob: RwLock<BobStatus>,
-    pub(crate) last_smeared_timestamp: RwLock<i64>,
-    pub(crate) running_state: RwLock<RunningState>,
+    blobdir: PathBuf,
+    pub sql: Sql,
+    pub perform_inbox_jobs_needed: Arc<RwLock<bool>>,
+    pub probe_imap_network: Arc<RwLock<bool>>,
+    pub inbox_thread: Arc<RwLock<JobThread>>,
+    pub sentbox_thread: Arc<RwLock<JobThread>>,
+    pub mvbox_thread: Arc<RwLock<JobThread>>,
+    pub smtp: Arc<Mutex<Smtp>>,
+    pub smtp_state: Arc<(Mutex<SmtpState>, Condvar)>,
+    pub oauth2_critical: Arc<Mutex<()>>,
+    #[debug_stub = "Callback"]
+    cb: Box<ContextCallback>,
+    pub os_name: Option<String>,
+    pub cmdline_sel_chat_id: Arc<RwLock<u32>>,
+    pub bob: Arc<RwLock<BobStatus>>,
+    pub last_smeared_timestamp: RwLock<i64>,
+    pub running_state: Arc<RwLock<RunningState>>,
     /// Mutex to avoid generating the key for the user more than once.
-    pub(crate) generating_key_mutex: Mutex<()>,
-    /// Mutex to enforce only a single running oauth2 is running.
-    pub(crate) oauth2_mutex: Mutex<()>,
-    pub(crate) translated_stockstrings: RwLock<HashMap<usize, String>>,
-    pub(crate) events: Events,
-
-    pub(crate) scheduler: RwLock<Scheduler>,
-
-    creation_time: SystemTime,
+    pub generating_key_mutex: Mutex<()>,
+    pub translated_stockstrings: RwLock<HashMap<usize, String>>,
 }
 
-#[derive(Debug)]
+#[derive(Debug, PartialEq, Eq)]
 pub struct RunningState {
     pub ongoing_running: bool,
     shall_stop_ongoing: bool,
-    cancel_sender: Option<Sender<()>>,
 }
 
 /// Return some info about deltachat-core
@@ -73,106 +78,90 @@ pub struct RunningState {
 /// actual keys and their values which will be present are not
 /// guaranteed.  Calling [Context::get_info] also includes information
 /// about the context on top of the information here.
-pub fn get_info() -> BTreeMap<&'static str, String> {
-    let mut res = BTreeMap::new();
+pub fn get_info() -> HashMap<&'static str, String> {
+    let mut res = HashMap::new();
     res.insert("deltachat_core_version", format!("v{}", &*DC_VERSION_STR));
-
-    let version =
-        String::from_utf8(libsqlite3_sys::SQLITE_VERSION.to_vec()).expect("invalid version");
-    res.insert("sqlite_version", version);
-
-    res.insert("arch", (std::mem::size_of::<usize>() * 8).to_string());
+    res.insert("sqlite_version", rusqlite::version().to_string());
+    res.insert(
+        "sqlite_thread_safe",
+        unsafe { rusqlite::ffi::sqlite3_threadsafe() }.to_string(),
+    );
+    res.insert(
+        "arch",
+        (std::mem::size_of::<*mut libc::c_void>())
+            .wrapping_mul(8)
+            .to_string(),
+    );
     res.insert("level", "awesome".into());
     res
 }
 
 impl Context {
     /// Creates new context.
-    pub async fn new(os_name: String, dbfile: PathBuf) -> Result<Context> {
+    pub fn new(cb: Box<ContextCallback>, os_name: String, dbfile: PathBuf) -> Result<Context> {
         let mut blob_fname = OsString::new();
         blob_fname.push(dbfile.file_name().unwrap_or_default());
         blob_fname.push("-blobs");
         let blobdir = dbfile.with_file_name(blob_fname);
-        if !blobdir.exists().await {
-            async_std::fs::create_dir_all(&blobdir).await?;
+        if !blobdir.exists() {
+            std::fs::create_dir_all(&blobdir)?;
         }
-        Context::with_blobdir(os_name, dbfile, blobdir).await
+        Context::with_blobdir(cb, os_name, dbfile, blobdir)
     }
 
-    pub async fn with_blobdir(
+    pub fn with_blobdir(
+        cb: Box<ContextCallback>,
         os_name: String,
         dbfile: PathBuf,
         blobdir: PathBuf,
     ) -> Result<Context> {
         ensure!(
-            blobdir.is_dir().await,
+            blobdir.is_dir(),
             "Blobdir does not exist: {}",
             blobdir.display()
         );
-
-        let inner = InnerContext {
+        let ctx = Context {
             blobdir,
             dbfile,
+            cb,
             os_name: Some(os_name),
-            running_state: RwLock::new(Default::default()),
+            running_state: Arc::new(RwLock::new(Default::default())),
             sql: Sql::new(),
-            bob: RwLock::new(Default::default()),
+            smtp: Arc::new(Mutex::new(Smtp::new())),
+            smtp_state: Arc::new((Mutex::new(Default::default()), Condvar::new())),
+            oauth2_critical: Arc::new(Mutex::new(())),
+            bob: Arc::new(RwLock::new(Default::default())),
             last_smeared_timestamp: RwLock::new(0),
+            cmdline_sel_chat_id: Arc::new(RwLock::new(0)),
+            inbox_thread: Arc::new(RwLock::new(JobThread::new(
+                "INBOX",
+                "configured_inbox_folder",
+                Imap::new(),
+            ))),
+            sentbox_thread: Arc::new(RwLock::new(JobThread::new(
+                "SENTBOX",
+                "configured_sentbox_folder",
+                Imap::new(),
+            ))),
+            mvbox_thread: Arc::new(RwLock::new(JobThread::new(
+                "MVBOX",
+                "configured_mvbox_folder",
+                Imap::new(),
+            ))),
+            probe_imap_network: Arc::new(RwLock::new(false)),
+            perform_inbox_jobs_needed: Arc::new(RwLock::new(false)),
             generating_key_mutex: Mutex::new(()),
-            oauth2_mutex: Mutex::new(()),
             translated_stockstrings: RwLock::new(HashMap::new()),
-            events: Events::default(),
-            scheduler: RwLock::new(Scheduler::Stopped),
-            creation_time: std::time::SystemTime::now(),
         };
 
-        let ctx = Context {
-            inner: Arc::new(inner),
-        };
-
-        ctx.sql.open(&ctx, &ctx.dbfile, false).await?;
+        ensure!(
+            ctx.sql.open(&ctx, &ctx.dbfile, false),
+            "Failed opening sqlite database"
+        );
 
         Ok(ctx)
     }
 
-    /// Starts the IO scheduler.
-    pub async fn start_io(&self) {
-        info!(self, "starting IO");
-        if self.is_io_running().await {
-            info!(self, "IO is already running");
-            return;
-        }
-
-        {
-            let l = &mut *self.inner.scheduler.write().await;
-            l.start(self.clone()).await;
-        }
-    }
-
-    /// Returns if the IO scheduler is running.
-    pub async fn is_io_running(&self) -> bool {
-        self.inner.is_io_running().await
-    }
-
-    /// Stops the IO scheduler.
-    pub async fn stop_io(&self) {
-        info!(self, "stopping IO");
-        if !self.is_io_running().await {
-            info!(self, "IO is not running");
-            return;
-        }
-
-        self.inner.stop_io().await;
-    }
-
-    /// Returns a reference to the underlying SQL instance.
-    ///
-    /// Warning: this is only here for testing, not part of the public API.
-    #[cfg(feature = "internals")]
-    pub fn sql(&self) -> &Sql {
-        &self.inner.sql
-    }
-
     /// Returns database file path.
     pub fn get_dbfile(&self) -> &Path {
         self.dbfile.as_path()
@@ -183,57 +172,49 @@ impl Context {
         self.blobdir.as_path()
     }
 
-    /// Emits a single event.
-    pub fn emit_event(&self, event: Event) {
-        self.events.emit(event);
+    pub fn call_cb(&self, event: Event) -> uintptr_t {
+        (*self.cb)(self, event)
     }
 
-    /// Get the next queued event.
-    pub fn get_event_emitter(&self) -> EventEmitter {
-        self.events.get_emitter()
-    }
+    /*******************************************************************************
+     * Ongoing process allocation/free/check
+     ******************************************************************************/
 
-    // Ongoing process allocation/free/check
+    pub fn alloc_ongoing(&self) -> bool {
+        if self.has_ongoing() {
+            warn!(self, "There is already another ongoing process running.",);
 
-    pub async fn alloc_ongoing(&self) -> Result<Receiver<()>> {
-        if self.has_ongoing().await {
-            bail!("There is already another ongoing process running.");
+            false
+        } else {
+            let s_a = self.running_state.clone();
+            let mut s = s_a.write().unwrap();
+
+            s.ongoing_running = true;
+            s.shall_stop_ongoing = false;
+
+            true
         }
-
-        let s_a = &self.running_state;
-        let mut s = s_a.write().await;
-
-        s.ongoing_running = true;
-        s.shall_stop_ongoing = false;
-        let (sender, receiver) = channel(1);
-        s.cancel_sender = Some(sender);
-
-        Ok(receiver)
     }
 
-    pub async fn free_ongoing(&self) {
-        let s_a = &self.running_state;
-        let mut s = s_a.write().await;
+    pub fn free_ongoing(&self) {
+        let s_a = self.running_state.clone();
+        let mut s = s_a.write().unwrap();
 
         s.ongoing_running = false;
         s.shall_stop_ongoing = true;
-        s.cancel_sender.take();
     }
 
-    pub async fn has_ongoing(&self) -> bool {
-        let s_a = &self.running_state;
-        let s = s_a.read().await;
+    pub fn has_ongoing(&self) -> bool {
+        let s_a = self.running_state.clone();
+        let s = s_a.read().unwrap();
 
         s.ongoing_running || !s.shall_stop_ongoing
     }
 
     /// Signal an ongoing process to stop.
-    pub async fn stop_ongoing(&self) {
-        let s_a = &self.running_state;
-        let mut s = s_a.write().await;
-        if let Some(cancel) = s.cancel_sender.take() {
-            cancel.send(()).await;
-        }
+    pub fn stop_ongoing(&self) {
+        let s_a = self.running_state.clone();
+        let mut s = s_a.write().unwrap();
 
         if s.ongoing_running && !s.shall_stop_ongoing {
             info!(self, "Signaling the ongoing process to stop ASAP.",);
@@ -243,71 +224,69 @@ impl Context {
         };
     }
 
-    pub async fn shall_stop_ongoing(&self) -> bool {
-        self.running_state.read().await.shall_stop_ongoing
+    pub fn shall_stop_ongoing(&self) -> bool {
+        self.running_state
+            .clone()
+            .read()
+            .unwrap()
+            .shall_stop_ongoing
     }
 
     /*******************************************************************************
      * UI chat/message related API
      ******************************************************************************/
 
-    pub async fn get_info(&self) -> BTreeMap<&'static str, String> {
+    pub fn get_info(&self) -> HashMap<&'static str, String> {
         let unset = "0";
-        let l = LoginParam::from_database(self, "").await;
-        let l2 = LoginParam::from_database(self, "configured_").await;
-        let displayname = self.get_config(Config::Displayname).await;
-        let chats = get_chat_cnt(self).await as usize;
-        let real_msgs = message::get_real_msg_cnt(self).await as usize;
-        let deaddrop_msgs = message::get_deaddrop_msg_cnt(self).await as usize;
-        let contacts = Contact::get_real_cnt(self).await as usize;
-        let is_configured = self.get_config_int(Config::Configured).await;
+        let l = LoginParam::from_database(self, "");
+        let l2 = LoginParam::from_database(self, "configured_");
+        let displayname = self.get_config(Config::Displayname);
+        let chats = get_chat_cnt(self) as usize;
+        let real_msgs = message::get_real_msg_cnt(self) as usize;
+        let deaddrop_msgs = message::get_deaddrop_msg_cnt(self) as usize;
+        let contacts = Contact::get_real_cnt(self) as usize;
+        let is_configured = self.get_config_int(Config::Configured);
         let dbversion = self
             .sql
-            .get_raw_config_int("dbversion")
-            .await
+            .get_raw_config_int(self, "dbversion")
             .unwrap_or_default();
-        let journal_mode = self
-            .sql
-            .query_value("PRAGMA journal_mode;", paramsx![])
-            .await
-            .unwrap_or_else(|_| "unknown".to_string());
-        let e2ee_enabled = self.get_config_int(Config::E2eeEnabled).await;
-        let mdns_enabled = self.get_config_int(Config::MdnsEnabled).await;
-        let bcc_self = self.get_config_int(Config::BccSelf).await;
 
-        let prv_key_cnt: Option<i32> = self
-            .sql
-            .query_value("SELECT COUNT(*) FROM keypairs;", paramsx![])
-            .await
-            .ok();
+        let e2ee_enabled = self.get_config_int(Config::E2eeEnabled);
+        let mdns_enabled = self.get_config_int(Config::MdnsEnabled);
+        let bcc_self = self.get_config_int(Config::BccSelf);
 
-        let pub_key_cnt: Option<i32> = self
-            .sql
-            .query_value("SELECT COUNT(*) FROM acpeerstates;", paramsx![])
-            .await
-            .ok();
-        let fingerprint_str = match SignedPublicKey::load_self(self).await {
-            Ok(key) => key.fingerprint().hex(),
-            Err(err) => format!("<key failure: {}>", err),
+        let prv_key_cnt: Option<isize> =
+            self.sql
+                .query_get_value(self, "SELECT COUNT(*) FROM keypairs;", rusqlite::NO_PARAMS);
+
+        let pub_key_cnt: Option<isize> = self.sql.query_get_value(
+            self,
+            "SELECT COUNT(*) FROM acpeerstates;",
+            rusqlite::NO_PARAMS,
+        );
+
+        let fingerprint_str = if let Some(key) = Key::from_self_public(self, &l2.addr, &self.sql) {
+            key.fingerprint()
+        } else {
+            "<Not yet calculated>".into()
         };
 
-        let inbox_watch = self.get_config_int(Config::InboxWatch).await;
-        let sentbox_watch = self.get_config_int(Config::SentboxWatch).await;
-        let mvbox_watch = self.get_config_int(Config::MvboxWatch).await;
-        let mvbox_move = self.get_config_int(Config::MvboxMove).await;
+        let inbox_watch = self.get_config_int(Config::InboxWatch);
+        let sentbox_watch = self.get_config_int(Config::SentboxWatch);
+        let mvbox_watch = self.get_config_int(Config::MvboxWatch);
+        let mvbox_move = self.get_config_int(Config::MvboxMove);
         let folders_configured = self
             .sql
-            .get_raw_config_int("folders_configured")
-            .await
+            .get_raw_config_int(self, "folders_configured")
             .unwrap_or_default();
 
         let configured_sentbox_folder = self
-            .get_config(Config::ConfiguredSentboxFolder)
-            .await
+            .sql
+            .get_raw_config(self, "configured_sentbox_folder")
             .unwrap_or_else(|| "<unset>".to_string());
         let configured_mvbox_folder = self
-            .get_config(Config::ConfiguredMvboxFolder)
-            .await
+            .sql
+            .get_raw_config(self, "configured_mvbox_folder")
             .unwrap_or_else(|| "<unset>".to_string());
 
         let mut res = get_info();
@@ -317,13 +296,11 @@ impl Context {
         res.insert("number_of_contacts", contacts.to_string());
         res.insert("database_dir", self.get_dbfile().display().to_string());
         res.insert("database_version", dbversion.to_string());
-        res.insert("journal_mode", journal_mode);
         res.insert("blobdir", self.get_blobdir().display().to_string());
         res.insert("display_name", displayname.unwrap_or_else(|| unset.into()));
         res.insert(
             "selfavatar",
             self.get_config(Config::Selfavatar)
-                .await
                 .unwrap_or_else(|| "<unset>".to_string()),
         );
         res.insert("is_configured", is_configured.to_string());
@@ -349,38 +326,42 @@ impl Context {
         );
         res.insert("fingerprint", fingerprint_str);
 
-        let elapsed = self.creation_time.elapsed();
-        res.insert("uptime", duration_to_str(elapsed.unwrap_or_default()));
-
         res
     }
 
-    pub async fn get_fresh_msgs(&self) -> Vec<MsgId> {
-        let show_deaddrop: i32 = 0;
+    pub fn get_fresh_msgs(&self) -> Vec<MsgId> {
+        let show_deaddrop = 0;
         self.sql
-            .query_values(
-                r#"
-SELECT m.id
- FROM msgs m
- LEFT JOIN contacts ct
-        ON m.from_id=ct.id
- LEFT JOIN chats c
-        ON m.chat_id=c.id
- WHERE m.state=?
-   AND m.hidden=0
-   AND m.chat_id>?
-   AND ct.blocked=0
-   AND (c.blocked=0 OR c.blocked=?)
- ORDER BY m.timestamp DESC,m.id DESC;
-"#,
-                paramsx![10, 9, if 0 != show_deaddrop { 2 } else { 0 }],
+            .query_map(
+                concat!(
+                    "SELECT m.id",
+                    " FROM msgs m",
+                    " LEFT JOIN contacts ct",
+                    "        ON m.from_id=ct.id",
+                    " LEFT JOIN chats c",
+                    "        ON m.chat_id=c.id",
+                    " WHERE m.state=?",
+                    "   AND m.hidden=0",
+                    "   AND m.chat_id>?",
+                    "   AND ct.blocked=0",
+                    "   AND (c.blocked=0 OR c.blocked=?)",
+                    " ORDER BY m.timestamp DESC,m.id DESC;"
+                ),
+                &[10, 9, if 0 != show_deaddrop { 2 } else { 0 }],
+                |row| row.get::<_, MsgId>(0),
+                |rows| {
+                    let mut ret = Vec::new();
+                    for row in rows {
+                        ret.push(row?);
+                    }
+                    Ok(ret)
+                },
             )
-            .await
             .unwrap_or_default()
     }
 
     #[allow(non_snake_case)]
-    pub async fn search_msgs(&self, chat_id: ChatId, query: impl AsRef<str>) -> Vec<MsgId> {
+    pub fn search_msgs(&self, chat_id: u32, query: impl AsRef<str>) -> Vec<MsgId> {
         let real_query = query.as_ref().trim();
         if real_query.is_empty() {
             return Vec::new();
@@ -388,100 +369,114 @@ SELECT m.id
         let strLikeInText = format!("%{}%", real_query);
         let strLikeBeg = format!("{}%", real_query);
 
-        let query = if !chat_id.is_unset() {
-            r#"
-SELECT m.id
- FROM msgs 
- LEFT JOIN contacts ct
-        ON m.from_id=ct.id
- WHERE m.chat_id=?
-   AND m.hidden=0
-   AND ct.blocked=0
-   AND (txt LIKE ? OR ct.name LIKE ?)
- ORDER BY m.timestamp,m.id;
-"#
+        let query = if 0 != chat_id {
+            concat!(
+                "SELECT m.id AS id, m.timestamp AS timestamp",
+                " FROM msgs m",
+                " LEFT JOIN contacts ct",
+                "        ON m.from_id=ct.id",
+                " WHERE m.chat_id=?",
+                "   AND m.hidden=0",
+                "   AND ct.blocked=0",
+                "   AND (txt LIKE ? OR ct.name LIKE ?)",
+                " ORDER BY m.timestamp,m.id;"
+            )
         } else {
-            r#"
-SELECT m.id
- FROM msgs m
- LEFT JOIN contacts ct
-        ON m.from_id=ct.id
- LEFT JOIN chats c
-        ON m.chat_id=c.id
- WHERE m.chat_id>9
-   AND m.hidden=0
-   AND (c.blocked=0 OR c.blocked=?)
-   AND ct.blocked=0
-   AND (m.txt LIKE ? OR ct.name LIKE ?)
- ORDER BY m.timestamp DESC,m.id DESC;
-"#
+            concat!(
+                "SELECT m.id AS id, m.timestamp AS timestamp",
+                " FROM msgs m",
+                " LEFT JOIN contacts ct",
+                "        ON m.from_id=ct.id",
+                " LEFT JOIN chats c",
+                "        ON m.chat_id=c.id",
+                " WHERE m.chat_id>9",
+                "   AND m.hidden=0",
+                "   AND (c.blocked=0 OR c.blocked=?)",
+                "   AND ct.blocked=0",
+                "   AND (m.txt LIKE ? OR ct.name LIKE ?)",
+                " ORDER BY m.timestamp DESC,m.id DESC;"
+            )
         };
 
         self.sql
-            .query_values(query, paramsx![chat_id, strLikeInText, strLikeBeg])
-            .await
+            .query_map(
+                query,
+                params![chat_id as i32, &strLikeInText, &strLikeBeg],
+                |row| row.get::<_, MsgId>("id"),
+                |rows| {
+                    let mut ret = Vec::new();
+                    for id in rows {
+                        ret.push(id?);
+                    }
+                    Ok(ret)
+                },
+            )
             .unwrap_or_default()
     }
 
-    pub async fn is_inbox(&self, folder_name: impl AsRef<str>) -> bool {
-        self.get_config(Config::ConfiguredInboxFolder).await
-            == Some(folder_name.as_ref().to_string())
+    pub fn is_inbox(&self, folder_name: impl AsRef<str>) -> bool {
+        folder_name.as_ref() == "INBOX"
     }
 
-    pub async fn is_sentbox(&self, folder_name: impl AsRef<str>) -> bool {
-        self.get_config(Config::ConfiguredSentboxFolder).await
-            == Some(folder_name.as_ref().to_string())
+    pub fn is_sentbox(&self, folder_name: impl AsRef<str>) -> bool {
+        let sentbox_name = self.sql.get_raw_config(self, "configured_sentbox_folder");
+        if let Some(name) = sentbox_name {
+            name == folder_name.as_ref()
+        } else {
+            false
+        }
     }
 
-    pub async fn is_mvbox(&self, folder_name: impl AsRef<str>) -> bool {
-        self.get_config(Config::ConfiguredMvboxFolder).await
-            == Some(folder_name.as_ref().to_string())
+    pub fn is_mvbox(&self, folder_name: impl AsRef<str>) -> bool {
+        let mvbox_name = self.sql.get_raw_config(self, "configured_mvbox_folder");
+
+        if let Some(name) = mvbox_name {
+            name == folder_name.as_ref()
+        } else {
+            false
+        }
     }
 
-    pub async fn do_heuristics_moves(&self, folder: &str, msg_id: MsgId) {
-        if !self.get_config_bool(Config::MvboxMove).await {
+    pub fn do_heuristics_moves(&self, folder: &str, msg_id: MsgId) {
+        if !self.get_config_bool(Config::MvboxMove) {
             return;
         }
 
-        if self.is_mvbox(folder).await {
+        if self.is_mvbox(folder) {
             return;
         }
-        if let Ok(msg) = Message::load_from_db(self, msg_id).await {
+        if let Ok(msg) = Message::load_from_db(self, msg_id) {
             if msg.is_setupmessage() {
                 // do not move setup messages;
                 // there may be a non-delta device that wants to handle it
                 return;
             }
 
-            match msg.is_dc_message {
-                MessengerMessage::No => {}
-                MessengerMessage::Yes | MessengerMessage::Reply => {
-                    job::add(
-                        self,
-                        job::Job::new(Action::MoveMsg, msg.id.to_u32(), Params::new(), 0),
-                    )
-                    .await;
-                }
+            // 1 = dc message, 2 = reply to dc message
+            if 0 != msg.is_dc_message {
+                job_add(
+                    self,
+                    Action::MoveMsg,
+                    msg.id.to_u32() as i32,
+                    Params::new(),
+                    0,
+                );
             }
         }
     }
 }
 
-impl InnerContext {
-    async fn is_io_running(&self) -> bool {
-        self.scheduler.read().await.is_running()
-    }
-
-    async fn stop_io(&self) {
-        assert!(self.is_io_running().await, "context is already stopped");
-        let token = {
-            let lock = &*self.scheduler.read().await;
-            lock.pre_stop().await
-        };
-        {
-            let lock = &mut *self.scheduler.write().await;
-            lock.stop(token).await;
-        }
+impl Drop for Context {
+    fn drop(&mut self) {
+        info!(self, "disconnecting inbox-thread",);
+        self.inbox_thread.read().unwrap().imap.disconnect(self);
+        info!(self, "disconnecting sentbox-thread",);
+        self.sentbox_thread.read().unwrap().imap.disconnect(self);
+        info!(self, "disconnecting mvbox-thread",);
+        self.mvbox_thread.read().unwrap().imap.disconnect(self);
+        info!(self, "disconnecting SMTP");
+        self.smtp.clone().lock().unwrap().disconnect();
+        self.sql.close(self);
     }
 }
 
@@ -490,18 +485,39 @@ impl Default for RunningState {
         RunningState {
             ongoing_running: false,
             shall_stop_ongoing: true,
-            cancel_sender: None,
         }
     }
 }
 
 #[derive(Debug, Default)]
-pub(crate) struct BobStatus {
+pub struct BobStatus {
     pub expects: i32,
     pub status: i32,
     pub qr_scan: Option<Lot>,
 }
 
+#[derive(Debug, PartialEq)]
+pub enum PerformJobsNeeded {
+    Not,
+    AtOnce,
+    AvoidDos,
+}
+
+impl Default for PerformJobsNeeded {
+    fn default() -> Self {
+        Self::Not
+    }
+}
+
+#[derive(Default, Debug)]
+pub struct SmtpState {
+    pub idle: bool,
+    pub suspended: bool,
+    pub doing_jobs: bool,
+    pub perform_jobs_needed: PerformJobsNeeded,
+    pub probe_network: bool,
+}
+
 pub fn get_version_str() -> &'static str {
     &DC_VERSION_STR
 }
@@ -512,81 +528,81 @@ mod tests {
 
     use crate::test_utils::*;
 
-    #[async_std::test]
-    async fn test_wrong_db() {
+    #[test]
+    fn test_wrong_db() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         std::fs::write(&dbfile, b"123").unwrap();
-        let res = Context::new("FakeOs".into(), dbfile.into()).await;
+        let res = Context::new(Box::new(|_, _| 0), "FakeOs".into(), dbfile);
         assert!(res.is_err());
     }
 
-    #[async_std::test]
-    async fn test_get_fresh_msgs() {
-        let t = TestContext::new().await;
-        let fresh = t.ctx.get_fresh_msgs().await;
+    #[test]
+    fn test_get_fresh_msgs() {
+        let t = dummy_context();
+        let fresh = t.ctx.get_fresh_msgs();
         assert!(fresh.is_empty())
     }
 
-    #[async_std::test]
-    async fn test_blobdir_exists() {
+    #[test]
+    fn test_blobdir_exists() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
-        Context::new("FakeOS".into(), dbfile.into()).await.unwrap();
+        Context::new(Box::new(|_, _| 0), "FakeOS".into(), dbfile).unwrap();
         let blobdir = tmp.path().join("db.sqlite-blobs");
         assert!(blobdir.is_dir());
     }
 
-    #[async_std::test]
-    async fn test_wrong_blogdir() {
+    #[test]
+    fn test_wrong_blogdir() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("db.sqlite-blobs");
         std::fs::write(&blobdir, b"123").unwrap();
-        let res = Context::new("FakeOS".into(), dbfile.into()).await;
+        let res = Context::new(Box::new(|_, _| 0), "FakeOS".into(), dbfile);
         assert!(res.is_err());
     }
 
-    #[async_std::test]
-    async fn test_sqlite_parent_not_exists() {
+    #[test]
+    fn test_sqlite_parent_not_exists() {
         let tmp = tempfile::tempdir().unwrap();
         let subdir = tmp.path().join("subdir");
         let dbfile = subdir.join("db.sqlite");
         let dbfile2 = dbfile.clone();
-        Context::new("FakeOS".into(), dbfile.into()).await.unwrap();
+        Context::new(Box::new(|_, _| 0), "FakeOS".into(), dbfile).unwrap();
         assert!(subdir.is_dir());
         assert!(dbfile2.is_file());
     }
 
-    #[async_std::test]
-    async fn test_with_empty_blobdir() {
+    #[test]
+    fn test_with_empty_blobdir() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = PathBuf::new();
-        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir.into()).await;
+        let res = Context::with_blobdir(Box::new(|_, _| 0), "FakeOS".into(), dbfile, blobdir);
         assert!(res.is_err());
     }
 
-    #[async_std::test]
-    async fn test_with_blobdir_not_exists() {
+    #[test]
+    fn test_with_blobdir_not_exists() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("blobs");
-        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir.into()).await;
+        let res = Context::with_blobdir(Box::new(|_, _| 0), "FakeOS".into(), dbfile, blobdir);
         assert!(res.is_err());
     }
 
-    #[async_std::test]
-    async fn no_crashes_on_context_deref() {
-        let t = TestContext::new().await;
+    #[test]
+    fn no_crashes_on_context_deref() {
+        let t = dummy_context();
         std::mem::drop(t.ctx);
     }
 
-    #[async_std::test]
-    async fn test_get_info() {
-        let t = TestContext::new().await;
+    #[test]
+    fn test_get_info() {
+        let t = dummy_context();
 
-        let info = t.ctx.get_info().await;
+        let info = t.ctx.get_info();
         assert!(info.get("database_dir").is_some());
     }
 

src/dc_simplify.rs

@@ -1,41 +1,15 @@
-// protect lines starting with `--` against being treated as a footer.
-// for that, we insert a ZERO WIDTH SPACE (ZWSP, 0x200B);
-// this should be invisible on most systems and there is no need to unescape it again
-// (which won't be done by non-deltas anyway)
-//
-// this escapes a bit more than actually needed by delta (eg. also lines as "-- footer"),
-// but for non-delta-compatibility, that seems to be better.
-// (to be only compatible with delta, only "[\r\n|\n]-- {0,2}[\r\n|\n]" needs to be replaced)
-pub fn escape_message_footer_marks(text: &str) -> String {
-    if text.starts_with("--") {
-        "-\u{200B}-".to_string() + &text[2..].replace("\n--", "\n-\u{200B}-")
-    } else {
-        text.replace("\n--", "\n-\u{200B}-")
-    }
-}
+use crate::dehtml::*;
 
 /// Remove standard (RFC 3676, §4.3) footer if it is found.
 fn remove_message_footer<'a>(lines: &'a [&str]) -> &'a [&'a str] {
-    let mut nearly_standard_footer = None;
     for (ix, &line) in lines.iter().enumerate() {
+        // quoted-printable may encode `-- ` to `-- =20` which is converted
+        // back to `--  `
         match line {
-            // some providers encode `-- ` to `-- =20` which results in `--  `
             "-- " | "--  " => return &lines[..ix],
-            // some providers encode `-- ` to `=2D-` which results in only `--`;
-            // use that only when no other footer is found
-            // and if the line before is empty and the line after is not empty
-            "--" => {
-                if (ix == 0 || lines[ix - 1] == "") && ix != lines.len() - 1 && lines[ix + 1] != ""
-                {
-                    nearly_standard_footer = Some(ix);
-                }
-            }
             _ => (),
         }
     }
-    if let Some(ix) = nearly_standard_footer {
-        return &lines[..ix];
-    }
     lines
 }
 
@@ -64,32 +38,36 @@ fn split_lines(buf: &str) -> Vec<&str> {
 
 /// Simplify message text for chat display.
 /// Remove quotes, signatures, trailing empty lines etc.
-pub fn simplify(mut input: String, is_chat_message: bool) -> (String, bool) {
-    input.retain(|c| c != '\r');
-    let lines = split_lines(&input);
+pub fn simplify(input: &str, is_html: bool, is_chat_message: bool) -> (String, bool) {
+    let mut out = if is_html {
+        dehtml(input)
+    } else {
+        input.to_string()
+    };
+
+    out.retain(|c| c != '\r');
+    let lines = split_lines(&out);
     let (lines, is_forwarded) = skip_forward_header(&lines);
 
-    let original_lines = &lines;
-
     let lines = remove_message_footer(lines);
-
-    let text = if is_chat_message {
-        render_message(lines, false, false)
+    let (lines, has_nonstandard_footer) = remove_nonstandard_footer(lines);
+    let (lines, has_bottom_quote) = if !is_chat_message {
+        remove_bottom_quote(lines)
     } else {
-        let (lines, has_nonstandard_footer) = remove_nonstandard_footer(lines);
-        let (lines, has_bottom_quote) = remove_bottom_quote(lines);
-        let (lines, has_top_quote) = remove_top_quote(lines);
-
-        if lines.iter().all(|it| it.trim().is_empty()) {
-            render_message(original_lines, false, false)
-        } else {
-            render_message(
-                lines,
-                has_top_quote,
-                has_nonstandard_footer || has_bottom_quote,
-            )
-        }
+        (lines, false)
     };
+    let (lines, has_top_quote) = if !is_chat_message {
+        remove_top_quote(lines)
+    } else {
+        (lines, false)
+    };
+
+    // re-create buffer from the remaining lines
+    let text = render_message(
+        lines,
+        has_top_quote,
+        has_nonstandard_footer || has_bottom_quote,
+    );
     (text, is_forwarded)
 }
 
@@ -97,13 +75,14 @@ pub fn simplify(mut input: String, is_chat_message: bool) -> (String, bool) {
 /// Returns message body lines and a boolean indicating whether
 /// a message is forwarded or not.
 fn skip_forward_header<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
-    match lines {
-        ["---------- Forwarded message ----------", first_line, "", rest @ ..]
-            if first_line.starts_with("From: ") =>
-        {
-            (rest, true)
-        }
-        _ => (lines, false),
+    if lines.len() >= 3
+        && lines[0] == "---------- Forwarded message ----------"
+        && lines[1].starts_with("From: ")
+        && lines[2].is_empty()
+    {
+        (&lines[3..], true)
+    } else {
+        (lines, false)
     }
 }
 
@@ -184,8 +163,7 @@ fn render_message(lines: &[&str], is_cut_at_begin: bool, is_cut_at_end: bool) ->
     if is_cut_at_end && (!is_cut_at_begin || !empty_body) {
         ret += " [...]";
     }
-    // redo escaping done by escape_message_footer_marks()
-    ret.replace("\u{200B}", "")
+    ret
 }
 
 /**
@@ -229,48 +207,61 @@ mod tests {
         #[test]
         // proptest does not support [[:graphical:][:space:]] regex.
         fn test_simplify_plain_text_fuzzy(input in "[!-~\t \n]+") {
-            let (output, _is_forwarded) = simplify(input, true);
+            let (output, _is_forwarded) = simplify(&input, false, true);
             assert!(output.split('\n').all(|s| s != "-- "));
         }
     }
 
-    #[test]
-    fn test_dont_remove_whole_message() {
-        let input = "\n------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text".to_string();
-        let (plain, is_forwarded) = simplify(input, false);
-        assert_eq!(
-            plain,
-            "------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text"
-        );
-        assert!(!is_forwarded);
-    }
-
-    #[test]
-    fn test_chat_message() {
-        let input = "Hi! How are you?\n\n---\n\nI am good.\n-- \nSent with my Delta Chat Messenger: https://delta.chat".to_string();
-        let (plain, is_forwarded) = simplify(input, true);
-        assert_eq!(plain, "Hi! How are you?\n\n---\n\nI am good.");
-        assert!(!is_forwarded);
-    }
-
     #[test]
     fn test_simplify_trim() {
-        let input = "line1\n\r\r\rline2".to_string();
-        let (plain, is_forwarded) = simplify(input, false);
+        let html = "\r\r\nline1<br>\r\n\r\n\r\rline2\n\r";
+        let (plain, is_forwarded) = simplify(html, true, false);
 
         assert_eq!(plain, "line1\nline2");
         assert!(!is_forwarded);
     }
 
+    #[test]
+    fn test_simplify_parse_href() {
+        let html = "<a href=url>text</a";
+        let (plain, is_forwarded) = simplify(html, true, false);
+
+        assert_eq!(plain, "[text](url)");
+        assert!(!is_forwarded);
+    }
+
+    #[test]
+    fn test_simplify_bold_text() {
+        let html = "<!DOCTYPE name [<!DOCTYPE ...>]><!-- comment -->text <b><?php echo ... ?>bold</b><![CDATA[<>]]>";
+        let (plain, is_forwarded) = simplify(html, true, false);
+
+        assert_eq!(plain, "text *bold*<>");
+        assert!(!is_forwarded);
+    }
+
     #[test]
     fn test_simplify_forwarded_message() {
-        let input = "---------- Forwarded message ----------\r\nFrom: test@example.com\r\n\r\nForwarded message\r\n-- \r\nSignature goes here".to_string();
-        let (plain, is_forwarded) = simplify(input, false);
+        let text = "---------- Forwarded message ----------\r\nFrom: test@example.com\r\n\r\nForwarded message\r\n-- \r\nSignature goes here";
+        let (plain, is_forwarded) = simplify(text, false, false);
 
         assert_eq!(plain, "Forwarded message");
         assert!(is_forwarded);
     }
 
+    #[test]
+    fn test_simplify_html_encoded() {
+        let html =
+                "&lt;&gt;&quot;&apos;&amp; &auml;&Auml;&ouml;&Ouml;&uuml;&Uuml;&szlig; foo&AElig;&ccedil;&Ccedil; &diams;&lrm;&rlm;&zwnj;&noent;&zwj;";
+
+        let (plain, is_forwarded) = simplify(html, true, false);
+
+        assert_eq!(
+            plain,
+            "<>\"\'& äÄöÖüÜß fooÆçÇ \u{2666}\u{200e}\u{200f}\u{200c}&noent;\u{200d}"
+        );
+        assert!(!is_forwarded);
+    }
+
     #[test]
     fn test_simplify_utilities() {
         assert!(is_empty_line(" \t"));
@@ -297,46 +288,4 @@ mod tests {
         assert_eq!(lines, &["not a quote", "> first", "> second"]);
         assert!(!has_top_quote);
     }
-
-    #[test]
-    fn test_escape_message_footer_marks() {
-        let esc = escape_message_footer_marks("--\n--text --in line");
-        assert_eq!(esc, "-\u{200B}-\n-\u{200B}-text --in line");
-
-        let esc = escape_message_footer_marks("--\r\n--text");
-        assert_eq!(esc, "-\u{200B}-\r\n-\u{200B}-text");
-    }
-
-    #[test]
-    fn test_remove_message_footer() {
-        let input = "text\n--\nno footer".to_string();
-        let (plain, _) = simplify(input, true);
-        assert_eq!(plain, "text\n--\nno footer");
-
-        let input = "text\n\n--\n\nno footer".to_string();
-        let (plain, _) = simplify(input, true);
-        assert_eq!(plain, "text\n\n--\n\nno footer");
-
-        let input = "text\n\n-- no footer\n\n".to_string();
-        let (plain, _) = simplify(input, true);
-        assert_eq!(plain, "text\n\n-- no footer");
-
-        let input = "text\n\n--\nno footer\n-- \nfooter".to_string();
-        let (plain, _) = simplify(input, true);
-        assert_eq!(plain, "text\n\n--\nno footer");
-
-        let input = "text\n\n--\ntreated as footer when unescaped".to_string();
-        let (plain, _) = simplify(input.clone(), true);
-        assert_eq!(plain, "text"); // see remove_message_footer() for some explanations
-        let escaped = escape_message_footer_marks(&input);
-        let (plain, _) = simplify(escaped, true);
-        assert_eq!(plain, "text\n\n--\ntreated as footer when unescaped");
-
-        let input = "--\ntreated as footer when unescaped".to_string();
-        let (plain, _) = simplify(input.clone(), true);
-        assert_eq!(plain, ""); // see remove_message_footer() for some explanations
-        let escaped = escape_message_footer_marks(&input);
-        let (plain, _) = simplify(escaped, true);
-        assert_eq!(plain, "--\ntreated as footer when unescaped");
-    }
 }

src/dc_tools.rs

@@ -3,27 +3,26 @@
 
 use core::cmp::{max, min};
 use std::borrow::Cow;
-use std::fmt;
+use std::path::{Path, PathBuf};
 use std::str::FromStr;
-use std::time::{Duration, SystemTime};
+use std::time::SystemTime;
+use std::{fmt, fs};
 
-use async_std::path::{Path, PathBuf};
-use async_std::{fs, io};
 use chrono::{Local, TimeZone};
 use rand::{thread_rng, Rng};
 
 use crate::context::Context;
-use crate::error::{bail, Error};
+use crate::error::Error;
 use crate::events::Event;
 
 pub(crate) fn dc_exactly_one_bit_set(v: i32) -> bool {
     0 != v && 0 == v & (v - 1)
 }
 
-/// Shortens a string to a specified length and adds "[...]" to the
-/// end of the shortened string.
-pub(crate) fn dc_truncate(buf: &str, approx_chars: usize) -> Cow<str> {
-    let ellipse = "[...]";
+/// Shortens a string to a specified length and adds "..." or "[...]" to the end of
+/// the shortened string.
+pub(crate) fn dc_truncate(buf: &str, approx_chars: usize, do_unwrap: bool) -> Cow<str> {
+    let ellipse = if do_unwrap { "..." } else { "[...]" };
 
     let count = buf.chars().count();
     if approx_chars > 0 && count > approx_chars + ellipse.len() {
@@ -50,8 +49,8 @@ pub(crate) fn dc_truncate(buf: &str, approx_chars: usize) -> Cow<str> {
 /// - harmonize together while being different enough
 /// (therefore, we cannot just use random rgb colors :)
 const COLORS: [u32; 16] = [
-    0xe5_65_55, 0xf2_8c_48, 0x8e_85_ee, 0x76_c8_4d, 0x5b_b6_cc, 0x54_9c_dd, 0xd2_5c_99, 0xb3_78_00,
-    0xf2_30_30, 0x39_b2_49, 0xbb_24_3b, 0x96_40_78, 0x66_87_4f, 0x30_8a_b9, 0x12_7e_d0, 0xbe_45_0c,
+    0xe56555, 0xf28c48, 0x8e85ee, 0x76c84d, 0x5bb6cc, 0x549cdd, 0xd25c99, 0xb37800, 0xf23030,
+    0x39b249, 0xbb243b, 0x964078, 0x66874f, 0x308ab9, 0x127ed0, 0xbe450c,
 ];
 
 pub(crate) fn dc_str_to_color(s: impl AsRef<str>) -> u32 {
@@ -60,7 +59,7 @@ pub(crate) fn dc_str_to_color(s: impl AsRef<str>) -> u32 {
     let bytes = str_lower.as_bytes();
     for (i, byte) in bytes.iter().enumerate() {
         checksum += (i + 1) * *byte as usize;
-        checksum %= 0x00ff_ffff;
+        checksum %= 0xffffff;
     }
     let color_index = checksum % COLORS.len();
 
@@ -76,14 +75,6 @@ pub fn dc_timestamp_to_str(wanted: i64) -> String {
     ts.format("%Y.%m.%d %H:%M:%S").to_string()
 }
 
-pub fn duration_to_str(duration: Duration) -> String {
-    let secs = duration.as_secs();
-    let h = secs / 3600;
-    let m = (secs % 3600) / 60;
-    let s = (secs % 3600) % 60;
-    format!("{}h {}m {}s", h, m, s)
-}
-
 pub(crate) fn dc_gm2local_offset() -> i64 {
     /* returns the offset that must be _added_ to an UTC/GMT-time to create the localtime.
     the function may return negative values. */
@@ -108,9 +99,9 @@ const MAX_SECONDS_TO_LEND_FROM_FUTURE: i64 = 5;
 // returns the currently smeared timestamp,
 // may be used to check if call to dc_create_smeared_timestamp() is needed or not.
 // the returned timestamp MUST NOT be used to be sent out or saved in the database!
-pub(crate) async fn dc_smeared_time(context: &Context) -> i64 {
+pub(crate) fn dc_smeared_time(context: &Context) -> i64 {
     let mut now = time();
-    let ts = *context.last_smeared_timestamp.read().await;
+    let ts = *context.last_smeared_timestamp.read().unwrap();
     if ts >= now {
         now = ts + 1;
     }
@@ -119,11 +110,11 @@ pub(crate) async fn dc_smeared_time(context: &Context) -> i64 {
 }
 
 // returns a timestamp that is guaranteed to be unique.
-pub(crate) async fn dc_create_smeared_timestamp(context: &Context) -> i64 {
+pub(crate) fn dc_create_smeared_timestamp(context: &Context) -> i64 {
     let now = time();
     let mut ret = now;
 
-    let mut last_smeared_timestamp = context.last_smeared_timestamp.write().await;
+    let mut last_smeared_timestamp = context.last_smeared_timestamp.write().unwrap();
     if ret <= *last_smeared_timestamp {
         ret = *last_smeared_timestamp + 1;
         if ret - now > MAX_SECONDS_TO_LEND_FROM_FUTURE {
@@ -138,12 +129,12 @@ pub(crate) async fn dc_create_smeared_timestamp(context: &Context) -> i64 {
 // creates `count` timestamps that are guaranteed to be unique.
 // the frist created timestamps is returned directly,
 // get the other timestamps just by adding 1..count-1
-pub(crate) async fn dc_create_smeared_timestamps(context: &Context, count: usize) -> i64 {
+pub(crate) fn dc_create_smeared_timestamps(context: &Context, count: usize) -> i64 {
     let now = time();
     let count = count as i64;
     let mut start = now + min(count, MAX_SECONDS_TO_LEND_FROM_FUTURE) - count;
 
-    let mut last_smeared_timestamp = context.last_smeared_timestamp.write().await;
+    let mut last_smeared_timestamp = context.last_smeared_timestamp.write().unwrap();
     start = max(*last_smeared_timestamp + 1, start);
 
     *last_smeared_timestamp = start + count - 1;
@@ -231,7 +222,45 @@ pub(crate) fn dc_extract_grpid_from_rfc724_mid(mid: &str) -> Option<&str> {
     None
 }
 
+// Function returns a sanitized basename that does not contain
+// win/linux path separators and also not any non-ascii chars
+fn get_safe_basename(filename: &str) -> String {
+    // return the (potentially mangled) basename of the input filename
+    // this might be a path that comes in from another operating system
+    let mut index: usize = 0;
+
+    if let Some(unix_index) = filename.rfind('/') {
+        index = unix_index + 1;
+    }
+    if let Some(win_index) = filename.rfind('\\') {
+        index = max(index, win_index + 1);
+    }
+    if index >= filename.len() {
+        "nobasename".to_string()
+    } else {
+        // we don't allow any non-ascii to be super-safe
+        filename[index..].replace(|c: char| !c.is_ascii() || c == ':', "-")
+    }
+}
+
+pub fn dc_derive_safe_stem_ext(filename: &str) -> (String, String) {
+    let basename = get_safe_basename(&filename);
+    let (mut stem, mut ext) = if let Some(index) = basename.rfind('.') {
+        (
+            basename[0..index].to_string(),
+            basename[index..].to_string(),
+        )
+    } else {
+        (basename, "".to_string())
+    };
+    // limit length of stem and ext
+    stem.truncate(32);
+    ext.truncate(32);
+    (stem, ext)
+}
+
 // the returned suffix is lower-case
+#[allow(non_snake_case)]
 pub fn dc_get_filesuffix_lc(path_filename: impl AsRef<str>) -> Option<String> {
     Path::new(path_filename.as_ref())
         .extension()
@@ -249,8 +278,11 @@ pub fn dc_get_filemeta(buf: &[u8]) -> Result<(u32, u32), Error> {
 ///
 /// If `path` starts with "$BLOBDIR", replaces it with the blobdir path.
 /// Otherwise, returns path as is.
-pub(crate) fn dc_get_abs_path<P: AsRef<Path>>(context: &Context, path: P) -> PathBuf {
-    let p: &Path = path.as_ref();
+pub(crate) fn dc_get_abs_path<P: AsRef<std::path::Path>>(
+    context: &Context,
+    path: P,
+) -> std::path::PathBuf {
+    let p: &std::path::Path = path.as_ref();
     if let Ok(p) = p.strip_prefix("$BLOBDIR") {
         context.get_blobdir().join(p)
     } else {
@@ -258,20 +290,20 @@ pub(crate) fn dc_get_abs_path<P: AsRef<Path>>(context: &Context, path: P) -> Pat
     }
 }
 
-pub(crate) async fn dc_get_filebytes(context: &Context, path: impl AsRef<Path>) -> u64 {
+pub(crate) fn dc_get_filebytes(context: &Context, path: impl AsRef<std::path::Path>) -> u64 {
     let path_abs = dc_get_abs_path(context, &path);
-    match fs::metadata(&path_abs).await {
+    match fs::metadata(&path_abs) {
         Ok(meta) => meta.len() as u64,
         Err(_err) => 0,
     }
 }
 
-pub(crate) async fn dc_delete_file(context: &Context, path: impl AsRef<Path>) -> bool {
+pub(crate) fn dc_delete_file(context: &Context, path: impl AsRef<std::path::Path>) -> bool {
     let path_abs = dc_get_abs_path(context, &path);
-    if !path_abs.exists().await {
+    if !path_abs.exists() {
         return false;
     }
-    if !path_abs.is_file().await {
+    if !path_abs.is_file() {
         warn!(
             context,
             "refusing to delete non-file \"{}\".",
@@ -281,9 +313,9 @@ pub(crate) async fn dc_delete_file(context: &Context, path: impl AsRef<Path>) ->
     }
 
     let dpath = format!("{}", path.as_ref().to_string_lossy());
-    match fs::remove_file(path_abs).await {
+    match fs::remove_file(path_abs) {
         Ok(_) => {
-            context.emit_event(Event::DeletedBlobFile(dpath));
+            context.call_cb(Event::DeletedBlobFile(dpath));
             true
         }
         Err(err) => {
@@ -293,13 +325,13 @@ pub(crate) async fn dc_delete_file(context: &Context, path: impl AsRef<Path>) ->
     }
 }
 
-pub(crate) async fn dc_copy_file(
+pub(crate) fn dc_copy_file(
     context: &Context,
-    src_path: impl AsRef<Path>,
-    dest_path: impl AsRef<Path>,
+    src_path: impl AsRef<std::path::Path>,
+    dest_path: impl AsRef<std::path::Path>,
 ) -> bool {
     let src_abs = dc_get_abs_path(context, &src_path);
-    let mut src_file = match fs::File::open(&src_abs).await {
+    let mut src_file = match fs::File::open(&src_abs) {
         Ok(file) => file,
         Err(err) => {
             warn!(
@@ -317,7 +349,6 @@ pub(crate) async fn dc_copy_file(
         .create_new(true)
         .write(true)
         .open(&dest_abs)
-        .await
     {
         Ok(file) => file,
         Err(err) => {
@@ -331,7 +362,7 @@ pub(crate) async fn dc_copy_file(
         }
     };
 
-    match io::copy(&mut src_file, &mut dest_file).await {
+    match std::io::copy(&mut src_file, &mut dest_file) {
         Ok(_) => true,
         Err(err) => {
             error!(
@@ -343,20 +374,20 @@ pub(crate) async fn dc_copy_file(
             );
             {
                 // Attempt to remove the failed file, swallow errors resulting from that.
-                fs::remove_file(dest_abs).await.ok();
+                fs::remove_file(dest_abs).ok();
             }
             false
         }
     }
 }
 
-pub(crate) async fn dc_create_folder(
+pub(crate) fn dc_create_folder(
     context: &Context,
-    path: impl AsRef<Path>,
-) -> Result<(), io::Error> {
+    path: impl AsRef<std::path::Path>,
+) -> Result<(), std::io::Error> {
     let path_abs = dc_get_abs_path(context, &path);
-    if !path_abs.exists().await {
-        match fs::create_dir_all(path_abs).await {
+    if !path_abs.exists() {
+        match fs::create_dir_all(path_abs) {
             Ok(_) => Ok(()),
             Err(err) => {
                 warn!(
@@ -374,13 +405,13 @@ pub(crate) async fn dc_create_folder(
 }
 
 /// Write a the given content to provied file path.
-pub(crate) async fn dc_write_file(
+pub(crate) fn dc_write_file(
     context: &Context,
     path: impl AsRef<Path>,
     buf: &[u8],
-) -> Result<(), io::Error> {
+) -> Result<(), std::io::Error> {
     let path_abs = dc_get_abs_path(context, &path);
-    fs::write(&path_abs, buf).await.map_err(|err| {
+    fs::write(&path_abs, buf).map_err(|err| {
         warn!(
             context,
             "Cannot write {} bytes to \"{}\": {}",
@@ -392,10 +423,13 @@ pub(crate) async fn dc_write_file(
     })
 }
 
-pub async fn dc_read_file<P: AsRef<Path>>(context: &Context, path: P) -> Result<Vec<u8>, Error> {
+pub fn dc_read_file<P: AsRef<std::path::Path>>(
+    context: &Context,
+    path: P,
+) -> Result<Vec<u8>, Error> {
     let path_abs = dc_get_abs_path(context, &path);
 
-    match fs::read(&path_abs).await {
+    match fs::read(&path_abs) {
         Ok(bytes) => Ok(bytes),
         Err(err) => {
             warn!(
@@ -409,31 +443,13 @@ pub async fn dc_read_file<P: AsRef<Path>>(context: &Context, path: P) -> Result<
     }
 }
 
-pub async fn dc_open_file<P: AsRef<Path>>(context: &Context, path: P) -> Result<fs::File, Error> {
-    let path_abs = dc_get_abs_path(context, &path);
-
-    match fs::File::open(&path_abs).await {
-        Ok(bytes) => Ok(bytes),
-        Err(err) => {
-            warn!(
-                context,
-                "Cannot read \"{}\" or file is empty: {}",
-                path.as_ref().display(),
-                err
-            );
-            Err(err.into())
-        }
-    }
-}
-
-pub fn dc_open_file_std<P: AsRef<std::path::Path>>(
+pub fn dc_open_file<P: AsRef<std::path::Path>>(
     context: &Context,
     path: P,
 ) -> Result<std::fs::File, Error> {
-    let p: PathBuf = path.as_ref().into();
-    let path_abs = dc_get_abs_path(context, p);
+    let path_abs = dc_get_abs_path(context, &path);
 
-    match std::fs::File::open(&path_abs) {
+    match fs::File::open(&path_abs) {
         Ok(bytes) => Ok(bytes),
         Err(err) => {
             warn!(
@@ -447,7 +463,7 @@ pub fn dc_open_file_std<P: AsRef<std::path::Path>>(
     }
 }
 
-pub(crate) async fn dc_get_next_backup_path(
+pub(crate) fn dc_get_next_backup_path(
     folder: impl AsRef<Path>,
     backup_time: i64,
 ) -> Result<PathBuf, Error> {
@@ -460,7 +476,7 @@ pub(crate) async fn dc_get_next_backup_path(
     for i in 0..64 {
         let mut path = folder.clone();
         path.push(format!("{}-{}.bak", stem, i));
-        if !path.exists().await {
+        if !path.exists() {
             return Ok(path);
         }
     }
@@ -470,18 +486,10 @@ pub(crate) async fn dc_get_next_backup_path(
 pub(crate) fn time() -> i64 {
     SystemTime::now()
         .duration_since(SystemTime::UNIX_EPOCH)
-        .unwrap_or_default()
+        .unwrap()
         .as_secs() as i64
 }
 
-/// An invalid email address was encountered
-#[derive(Debug, thiserror::Error)]
-#[error("Invalid email address: {message} ({addr})")]
-pub struct InvalidEmailError {
-    message: String,
-    addr: String,
-}
-
 /// Very simple email address wrapper.
 ///
 /// Represents an email address, right now just the `name@domain` portion.
@@ -498,14 +506,14 @@ pub struct InvalidEmailError {
 /// assert_eq!(&email.domain, "example.com");
 /// assert_eq!(email.to_string(), "someone@example.com");
 /// ```
-#[derive(Debug, PartialEq, Eq, Clone)]
+#[derive(Debug, PartialEq, Clone)]
 pub struct EmailAddress {
     pub local: String,
     pub domain: String,
 }
 
 impl EmailAddress {
-    pub fn new(input: &str) -> Result<Self, InvalidEmailError> {
+    pub fn new(input: &str) -> Result<Self, Error> {
         input.parse::<EmailAddress>()
     }
 }
@@ -517,54 +525,31 @@ impl fmt::Display for EmailAddress {
 }
 
 impl FromStr for EmailAddress {
-    type Err = InvalidEmailError;
+    type Err = Error;
 
     /// Performs a dead-simple parse of an email address.
-    fn from_str(input: &str) -> Result<EmailAddress, InvalidEmailError> {
-        let err = |msg: &str| {
-            Err(InvalidEmailError {
-                message: msg.to_string(),
-                addr: input.to_string(),
-            })
-        };
-        if input.is_empty() {
-            return err("empty string is not valid");
-        }
+    fn from_str(input: &str) -> Result<EmailAddress, Error> {
+        ensure!(!input.is_empty(), "empty string is not valid");
         let parts: Vec<&str> = input.rsplitn(2, '@').collect();
 
-        if input
-            .chars()
-            .any(|c| c.is_whitespace() || c == '<' || c == '>')
-        {
-            return err("Email must not contain whitespaces, '>' or '<'");
-        }
+        ensure!(parts.len() > 1, "missing '@' character");
+        let local = parts[1];
+        let domain = parts[0];
 
-        match &parts[..] {
-            [domain, local] => {
-                if local.is_empty() {
-                    return err("empty string is not valid for local part");
-                }
-                if domain.len() <= 3 {
-                    return err("domain is too short");
-                }
-                let dot = domain.find('.');
-                match dot {
-                    None => {
-                        return err("invalid domain");
-                    }
-                    Some(dot_idx) => {
-                        if dot_idx >= domain.len() - 2 {
-                            return err("invalid domain");
-                        }
-                    }
-                }
-                Ok(EmailAddress {
-                    local: (*local).to_string(),
-                    domain: (*domain).to_string(),
-                })
-            }
-            _ => err("missing '@' character"),
-        }
+        ensure!(
+            !local.is_empty(),
+            "empty string is not valid for local part"
+        );
+        ensure!(domain.len() > 3, "domain is too short");
+
+        let dot = domain.find('.');
+        ensure!(dot.is_some(), "invalid domain");
+        ensure!(dot.unwrap() < domain.len() - 2, "invalid domain");
+
+        Ok(EmailAddress {
+            local: local.to_string(),
+            domain: domain.to_string(),
+        })
     }
 }
 
@@ -591,42 +576,54 @@ mod tests {
     #[test]
     fn test_dc_truncate_1() {
         let s = "this is a little test string";
-        assert_eq!(dc_truncate(s, 16), "this is a [...]");
+        assert_eq!(dc_truncate(s, 16, false), "this is a [...]");
+        assert_eq!(dc_truncate(s, 16, true), "this is a ...");
     }
 
     #[test]
     fn test_dc_truncate_2() {
-        assert_eq!(dc_truncate("1234", 2), "1234");
+        assert_eq!(dc_truncate("1234", 2, false), "1234");
+        assert_eq!(dc_truncate("1234", 2, true), "1234");
     }
 
     #[test]
     fn test_dc_truncate_3() {
-        assert_eq!(dc_truncate("1234567", 1), "1[...]");
+        assert_eq!(dc_truncate("1234567", 1, false), "1[...]");
+        assert_eq!(dc_truncate("1234567", 1, true), "1...");
     }
 
     #[test]
     fn test_dc_truncate_4() {
-        assert_eq!(dc_truncate("123456", 4), "123456");
+        assert_eq!(dc_truncate("123456", 4, false), "123456");
+        assert_eq!(dc_truncate("123456", 4, true), "123456");
     }
 
     #[test]
     fn test_dc_truncate_edge() {
-        assert_eq!(dc_truncate("", 4), "");
+        assert_eq!(dc_truncate("", 4, false), "");
+        assert_eq!(dc_truncate("", 4, true), "");
 
-        assert_eq!(dc_truncate("\n  hello \n world", 4), "\n  [...]");
+        assert_eq!(dc_truncate("\n  hello \n world", 4, false), "\n  [...]");
+        assert_eq!(dc_truncate("\n  hello \n world", 4, true), "\n  ...");
 
-        assert_eq!(dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 1), "𐠈[...]");
         assert_eq!(
-            dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 0),
+            dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 1, false),
+            "𐠈[...]"
+        );
+        assert_eq!(
+            dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 0, false),
             "𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ"
         );
 
         // 9 characters, so no truncation
-        assert_eq!(dc_truncate("𑒀ὐ￠🜀\u{1e01b}A a🟠", 6), "𑒀ὐ￠🜀\u{1e01b}A a🟠",);
+        assert_eq!(
+            dc_truncate("𑒀ὐ￠🜀\u{1e01b}A a🟠", 6, false),
+            "𑒀ὐ￠🜀\u{1e01b}A a🟠",
+        );
 
         // 12 characters, truncation
         assert_eq!(
-            dc_truncate("𑒀ὐ￠🜀\u{1e01b}A a🟠bcd", 6),
+            dc_truncate("𑒀ὐ￠🜀\u{1e01b}A a🟠bcd", 6, false),
             "𑒀ὐ￠🜀\u{1e01b}A[...]",
         );
     }
@@ -657,6 +654,7 @@ mod tests {
         );
     }
 
+    #[test]
     #[test]
     fn test_dc_extract_grpid_from_rfc724_mid() {
         // Should return None if we pass invalid mid
@@ -711,29 +709,29 @@ mod tests {
 
     #[test]
     fn test_emailaddress_parse() {
-        assert_eq!("".parse::<EmailAddress>().is_ok(), false);
+        assert_eq!(EmailAddress::new("").is_ok(), false);
         assert_eq!(
-            "user@domain.tld".parse::<EmailAddress>().unwrap(),
+            EmailAddress::new("user@domain.tld").unwrap(),
             EmailAddress {
                 local: "user".into(),
                 domain: "domain.tld".into(),
             }
         );
-        assert_eq!("uuu".parse::<EmailAddress>().is_ok(), false);
-        assert_eq!("dd.tt".parse::<EmailAddress>().is_ok(), false);
-        assert_eq!("tt.dd@uu".parse::<EmailAddress>().is_ok(), false);
-        assert_eq!("u@d".parse::<EmailAddress>().is_ok(), false);
-        assert_eq!("u@d.".parse::<EmailAddress>().is_ok(), false);
-        assert_eq!("u@d.t".parse::<EmailAddress>().is_ok(), false);
+        assert_eq!(EmailAddress::new("uuu").is_ok(), false);
+        assert_eq!(EmailAddress::new("dd.tt").is_ok(), false);
+        assert_eq!(EmailAddress::new("tt.dd@uu").is_ok(), false);
+        assert_eq!(EmailAddress::new("u@d").is_ok(), false);
+        assert_eq!(EmailAddress::new("u@d.").is_ok(), false);
+        assert_eq!(EmailAddress::new("u@d.t").is_ok(), false);
         assert_eq!(
-            "u@d.tt".parse::<EmailAddress>().unwrap(),
+            EmailAddress::new("u@d.tt").unwrap(),
             EmailAddress {
                 local: "u".into(),
                 domain: "d.tt".into(),
             }
         );
-        assert_eq!("u@tt".parse::<EmailAddress>().is_ok(), false);
-        assert_eq!("@d.tt".parse::<EmailAddress>().is_ok(), false);
+        assert_eq!(EmailAddress::new("u@.tt").is_ok(), false);
+        assert_eq!(EmailAddress::new("@d.tt").is_ok(), false);
     }
 
     use proptest::prelude::*;
@@ -742,10 +740,11 @@ mod tests {
         #[test]
         fn test_dc_truncate(
             buf: String,
-            approx_chars in 0..10000usize
+            approx_chars in 0..10000usize,
+            do_unwrap: bool,
         ) {
-            let res = dc_truncate(&buf, approx_chars);
-            let el_len = 5;
+            let res = dc_truncate(&buf, approx_chars, do_unwrap);
+            let el_len = if do_unwrap { 3 } else { 5 };
             let l = res.chars().count();
             if approx_chars > 0 {
                 assert!(
@@ -759,40 +758,53 @@ mod tests {
 
             if approx_chars > 0 && buf.chars().count() > approx_chars + el_len {
                 let l = res.len();
-                assert_eq!(&res[l-5..l], "[...]", "missing ellipsis in {}", &res);
+                if do_unwrap {
+                    assert_eq!(&res[l-3..l], "...", "missing ellipsis in {}", &res);
+                } else {
+                    assert_eq!(&res[l-5..l], "[...]", "missing ellipsis in {}", &res);
+                }
             }
         }
     }
 
-    #[async_std::test]
-    async fn test_file_handling() {
-        let t = TestContext::new().await;
-        let context = &t.ctx;
-        macro_rules! dc_file_exist {
-            ($ctx:expr, $fname:expr) => {
-                $ctx.get_blobdir()
-                    .join(Path::new($fname).file_name().unwrap())
-                    .exists()
-            };
-        }
+    #[test]
+    fn test_file_get_safe_basename() {
+        assert_eq!(get_safe_basename("12312/hello"), "hello");
+        assert_eq!(get_safe_basename("12312\\hello"), "hello");
+        assert_eq!(get_safe_basename("//12312\\hello"), "hello");
+        assert_eq!(get_safe_basename("//123:12\\hello"), "hello");
+        assert_eq!(get_safe_basename("//123:12/\\\\hello"), "hello");
+        assert_eq!(get_safe_basename("//123:12//hello"), "hello");
+        assert_eq!(get_safe_basename("//123:12//"), "nobasename");
+        assert_eq!(get_safe_basename("//123:12/"), "nobasename");
+        assert!(get_safe_basename("123\x012.hello").ends_with(".hello"));
+    }
 
-        assert!(!dc_delete_file(context, "$BLOBDIR/lkqwjelqkwlje").await);
-        if dc_file_exist!(context, "$BLOBDIR/foobar").await
-            || dc_file_exist!(context, "$BLOBDIR/dada").await
-            || dc_file_exist!(context, "$BLOBDIR/foobar.dadada").await
-            || dc_file_exist!(context, "$BLOBDIR/foobar-folder").await
+    #[test]
+    fn test_file_handling() {
+        let t = dummy_context();
+        let context = &t.ctx;
+        let dc_file_exist = |ctx: &Context, fname: &str| {
+            ctx.get_blobdir()
+                .join(Path::new(fname).file_name().unwrap())
+                .exists()
+        };
+
+        assert!(!dc_delete_file(context, "$BLOBDIR/lkqwjelqkwlje"));
+        if dc_file_exist(context, "$BLOBDIR/foobar")
+            || dc_file_exist(context, "$BLOBDIR/dada")
+            || dc_file_exist(context, "$BLOBDIR/foobar.dadada")
+            || dc_file_exist(context, "$BLOBDIR/foobar-folder")
         {
-            dc_delete_file(context, "$BLOBDIR/foobar").await;
-            dc_delete_file(context, "$BLOBDIR/dada").await;
-            dc_delete_file(context, "$BLOBDIR/foobar.dadada").await;
-            dc_delete_file(context, "$BLOBDIR/foobar-folder").await;
+            dc_delete_file(context, "$BLOBDIR/foobar");
+            dc_delete_file(context, "$BLOBDIR/dada");
+            dc_delete_file(context, "$BLOBDIR/foobar.dadada");
+            dc_delete_file(context, "$BLOBDIR/foobar-folder");
         }
-        assert!(dc_write_file(context, "$BLOBDIR/foobar", b"content")
-            .await
-            .is_ok());
-        assert!(dc_file_exist!(context, "$BLOBDIR/foobar").await);
-        assert!(!dc_file_exist!(context, "$BLOBDIR/foobarx").await);
-        assert_eq!(dc_get_filebytes(context, "$BLOBDIR/foobar").await, 7);
+        assert!(dc_write_file(context, "$BLOBDIR/foobar", b"content").is_ok());
+        assert!(dc_file_exist(context, "$BLOBDIR/foobar",));
+        assert!(!dc_file_exist(context, "$BLOBDIR/foobarx"));
+        assert_eq!(dc_get_filebytes(context, "$BLOBDIR/foobar"), 7);
 
         let abs_path = context
             .get_blobdir()
@@ -800,33 +812,31 @@ mod tests {
             .to_string_lossy()
             .to_string();
 
-        assert!(dc_file_exist!(context, &abs_path).await);
+        assert!(dc_file_exist(context, &abs_path));
 
-        assert!(dc_copy_file(context, "$BLOBDIR/foobar", "$BLOBDIR/dada").await);
+        assert!(dc_copy_file(context, "$BLOBDIR/foobar", "$BLOBDIR/dada",));
 
         // attempting to copy a second time should fail
-        assert!(!dc_copy_file(context, "$BLOBDIR/foobar", "$BLOBDIR/dada").await);
+        assert!(!dc_copy_file(context, "$BLOBDIR/foobar", "$BLOBDIR/dada",));
 
-        assert_eq!(dc_get_filebytes(context, "$BLOBDIR/dada").await, 7);
+        assert_eq!(dc_get_filebytes(context, "$BLOBDIR/dada",), 7);
 
-        let buf = dc_read_file(context, "$BLOBDIR/dada").await.unwrap();
+        let buf = dc_read_file(context, "$BLOBDIR/dada").unwrap();
 
         assert_eq!(buf.len(), 7);
         assert_eq!(&buf, b"content");
 
-        assert!(dc_delete_file(context, "$BLOBDIR/foobar").await);
-        assert!(dc_delete_file(context, "$BLOBDIR/dada").await);
-        assert!(dc_create_folder(context, "$BLOBDIR/foobar-folder")
-            .await
-            .is_ok());
-        assert!(dc_file_exist!(context, "$BLOBDIR/foobar-folder").await);
-        assert!(!dc_delete_file(context, "$BLOBDIR/foobar-folder").await);
+        assert!(dc_delete_file(context, "$BLOBDIR/foobar"));
+        assert!(dc_delete_file(context, "$BLOBDIR/dada"));
+        assert!(dc_create_folder(context, "$BLOBDIR/foobar-folder").is_ok());
+        assert!(dc_file_exist(context, "$BLOBDIR/foobar-folder",));
+        assert!(!dc_delete_file(context, "$BLOBDIR/foobar-folder"));
 
         let fn0 = "$BLOBDIR/data.data";
-        assert!(dc_write_file(context, &fn0, b"content").await.is_ok());
+        assert!(dc_write_file(context, &fn0, b"content").is_ok());
 
-        assert!(dc_delete_file(context, &fn0).await);
-        assert!(!dc_file_exist!(context, &fn0).await);
+        assert!(dc_delete_file(context, &fn0));
+        assert!(!dc_file_exist(context, &fn0));
     }
 
     #[test]
@@ -843,15 +853,15 @@ mod tests {
         assert!(!listflags_has(listflags, DC_GCL_ADD_SELF));
     }
 
-    #[async_std::test]
-    async fn test_create_smeared_timestamp() {
-        let t = TestContext::new().await;
+    #[test]
+    fn test_create_smeared_timestamp() {
+        let t = dummy_context();
         assert_ne!(
-            dc_create_smeared_timestamp(&t.ctx).await,
-            dc_create_smeared_timestamp(&t.ctx).await
+            dc_create_smeared_timestamp(&t.ctx),
+            dc_create_smeared_timestamp(&t.ctx)
         );
         assert!(
-            dc_create_smeared_timestamp(&t.ctx).await
+            dc_create_smeared_timestamp(&t.ctx)
                 >= SystemTime::now()
                     .duration_since(SystemTime::UNIX_EPOCH)
                     .unwrap()
@@ -859,50 +869,17 @@ mod tests {
         );
     }
 
-    #[async_std::test]
-    async fn test_create_smeared_timestamps() {
-        let t = TestContext::new().await;
+    #[test]
+    fn test_create_smeared_timestamps() {
+        let t = dummy_context();
         let count = MAX_SECONDS_TO_LEND_FROM_FUTURE - 1;
-        let start = dc_create_smeared_timestamps(&t.ctx, count as usize).await;
-        let next = dc_smeared_time(&t.ctx).await;
+        let start = dc_create_smeared_timestamps(&t.ctx, count as usize);
+        let next = dc_smeared_time(&t.ctx);
         assert!((start + count - 1) < next);
 
         let count = MAX_SECONDS_TO_LEND_FROM_FUTURE + 30;
-        let start = dc_create_smeared_timestamps(&t.ctx, count as usize).await;
-        let next = dc_smeared_time(&t.ctx).await;
+        let start = dc_create_smeared_timestamps(&t.ctx, count as usize);
+        let next = dc_smeared_time(&t.ctx);
         assert!((start + count - 1) < next);
     }
-
-    #[test]
-    fn test_duration_to_str() {
-        assert_eq!(duration_to_str(Duration::from_secs(0)), "0h 0m 0s");
-        assert_eq!(duration_to_str(Duration::from_secs(59)), "0h 0m 59s");
-        assert_eq!(duration_to_str(Duration::from_secs(60)), "0h 1m 0s");
-        assert_eq!(duration_to_str(Duration::from_secs(61)), "0h 1m 1s");
-        assert_eq!(duration_to_str(Duration::from_secs(59 * 60)), "0h 59m 0s");
-        assert_eq!(
-            duration_to_str(Duration::from_secs(59 * 60 + 59)),
-            "0h 59m 59s"
-        );
-        assert_eq!(
-            duration_to_str(Duration::from_secs(59 * 60 + 60)),
-            "1h 0m 0s"
-        );
-        assert_eq!(
-            duration_to_str(Duration::from_secs(2 * 60 * 60 + 59 * 60 + 59)),
-            "2h 59m 59s"
-        );
-        assert_eq!(
-            duration_to_str(Duration::from_secs(2 * 60 * 60 + 59 * 60 + 60)),
-            "3h 0m 0s"
-        );
-        assert_eq!(
-            duration_to_str(Duration::from_secs(3 * 60 * 60 + 59)),
-            "3h 0m 59s"
-        );
-        assert_eq!(
-            duration_to_str(Duration::from_secs(3 * 60 * 60 + 60)),
-            "3h 1m 0s"
-        );
-    }
 }

src/dehtml.rs

@@ -3,6 +3,7 @@
 //! A module to remove HTML tags from the email text
 
 use lazy_static::lazy_static;
+use quick_xml;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
 lazy_static! {
@@ -34,7 +35,6 @@ pub fn dehtml(buf: &str) -> String {
     };
 
     let mut reader = quick_xml::Reader::from_str(buf);
-    reader.check_end_names(false);
 
     let mut buf = Vec::new();
 
@@ -139,12 +139,13 @@ fn dehtml_starttag_cb<B: std::io::BufRead>(
             dehtml.add_text = AddText::YesPreserveLineEnds;
         }
         "a" => {
-            if let Some(href) = event
-                .html_attributes()
-                .filter_map(|attr| attr.ok())
-                .find(|attr| String::from_utf8_lossy(attr.key).trim().to_lowercase() == "href")
-            {
+            if let Some(href) = event.html_attributes().find(|attr| {
+                attr.as_ref()
+                    .map(|a| String::from_utf8_lossy(a.key).trim().to_lowercase() == "href")
+                    .unwrap_or_default()
+            }) {
                 let href = href
+                    .unwrap()
                     .unescape_and_decode_value(reader)
                     .unwrap_or_default()
                     .to_lowercase();
@@ -188,60 +189,4 @@ mod tests {
             assert_eq!(dehtml(input), output);
         }
     }
-
-    #[test]
-    fn test_dehtml_parse_br() {
-        let html = "\r\r\nline1<br>\r\n\r\n\r\rline2\n\r";
-        let plain = dehtml(html);
-
-        assert_eq!(plain, "line1\n\r\r\rline2");
-    }
-
-    #[test]
-    fn test_dehtml_parse_href() {
-        let html = "<a href=url>text</a";
-        let plain = dehtml(html);
-
-        assert_eq!(plain, "[text](url)");
-    }
-
-    #[test]
-    fn test_dehtml_bold_text() {
-        let html = "<!DOCTYPE name [<!DOCTYPE ...>]><!-- comment -->text <b><?php echo ... ?>bold</b><![CDATA[<>]]>";
-        let plain = dehtml(html);
-
-        assert_eq!(plain, "text *bold*<>");
-    }
-
-    #[test]
-    fn test_dehtml_html_encoded() {
-        let html =
-                "&lt;&gt;&quot;&apos;&amp; &auml;&Auml;&ouml;&Ouml;&uuml;&Uuml;&szlig; foo&AElig;&ccedil;&Ccedil; &diams;&lrm;&rlm;&zwnj;&noent;&zwj;";
-
-        let plain = dehtml(html);
-
-        assert_eq!(
-            plain,
-            "<>\"\'& äÄöÖüÜß fooÆçÇ \u{2666}\u{200e}\u{200f}\u{200c}&noent;\u{200d}"
-        );
-    }
-
-    #[test]
-    fn test_unclosed_tags() {
-        let input = r##"
-        <!DOCTYPE HTML PUBLIC '-//W3C//DTD HTML 4.01 Transitional//EN'
-        'http://www.w3.org/TR/html4/loose.dtd'>
-        <html>
-        <head>
-        <title>Hi</title>
-        <meta http-equiv='Content-Type' content='text/html; charset=iso-8859-1'>						
-        </head>
-        <body>
-        lots of text
-        </body>
-        </html>
-        "##;
-        let txt = dehtml(input);
-        assert_eq!(txt.trim(), "lots of text");
-    }
 }

src/e2ee.rs

@@ -2,16 +2,14 @@
 
 use std::collections::HashSet;
 
-use mailparse::ParsedMail;
+use mailparse::{MailHeaderMap, ParsedMail};
 use num_traits::FromPrimitive;
 
 use crate::aheader::*;
 use crate::config::Config;
 use crate::context::Context;
 use crate::error::*;
-use crate::headerdef::HeaderDef;
-use crate::headerdef::HeaderDefMap;
-use crate::key::{DcKey, Fingerprint, SignedPublicKey, SignedSecretKey};
+use crate::key::*;
 use crate::keyring::*;
 use crate::peerstate::*;
 use crate::pgp;
@@ -21,22 +19,22 @@ use crate::securejoin::handle_degrade_event;
 pub struct EncryptHelper {
     pub prefer_encrypt: EncryptPreference,
     pub addr: String,
-    pub public_key: SignedPublicKey,
+    pub public_key: Key,
 }
 
 impl EncryptHelper {
-    pub async fn new(context: &Context) -> Result<EncryptHelper> {
+    pub fn new(context: &Context) -> Result<EncryptHelper> {
         let prefer_encrypt =
-            EncryptPreference::from_i32(context.get_config_int(Config::E2eeEnabled).await)
+            EncryptPreference::from_i32(context.get_config_int(Config::E2eeEnabled))
                 .unwrap_or_default();
-        let addr = match context.get_config(Config::ConfiguredAddr).await {
+        let addr = match context.get_config(Config::ConfiguredAddr) {
             None => {
                 bail!("addr not configured!");
             }
             Some(addr) => addr,
         };
 
-        let public_key = SignedPublicKey::load_self(context).await?;
+        let public_key = load_or_generate_self_public_key(context, &addr)?;
 
         Ok(EncryptHelper {
             prefer_encrypt,
@@ -86,44 +84,46 @@ impl EncryptHelper {
     }
 
     /// Tries to encrypt the passed in `mail`.
-    pub async fn encrypt(
-        self,
+    pub fn encrypt(
+        &mut self,
         context: &Context,
         min_verified: PeerstateVerifiedStatus,
         mail_to_encrypt: lettre_email::PartBuilder,
-        peerstates: Vec<(Option<Peerstate>, &str)>,
+        peerstates: &[(Option<Peerstate>, &str)],
     ) -> Result<String> {
-        let mut keyring: Keyring<SignedPublicKey> = Keyring::new();
+        let mut keyring = Keyring::default();
 
         for (peerstate, addr) in peerstates
-            .into_iter()
-            .filter_map(|(state, addr)| state.map(|s| (s, addr)))
+            .iter()
+            .filter_map(|(state, addr)| state.as_ref().map(|s| (s, addr)))
         {
-            let key = peerstate.take_key(min_verified).ok_or_else(|| {
+            let key = peerstate.peek_key(min_verified).ok_or_else(|| {
                 format_err!("proper enc-key for {} missing, cannot encrypt", addr)
             })?;
-            keyring.add(key);
+            keyring.add_ref(key);
         }
-        keyring.add(self.public_key.clone());
-        let sign_key = SignedSecretKey::load_self(context).await?;
+
+        keyring.add_ref(&self.public_key);
+        let sign_key = Key::from_self_private(context, self.addr.clone(), &context.sql)
+            .ok_or_else(|| format_err!("missing own private key"))?;
 
         let raw_message = mail_to_encrypt.build().as_string().into_bytes();
 
-        let ctext = pgp::pk_encrypt(&raw_message, keyring, Some(sign_key)).await?;
+        let ctext = pgp::pk_encrypt(&raw_message, &keyring, Some(&sign_key))?;
 
         Ok(ctext)
     }
 }
 
-pub async fn try_decrypt(
+pub fn try_decrypt(
     context: &Context,
     mail: &ParsedMail<'_>,
     message_time: i64,
-) -> Result<(Option<Vec<u8>>, HashSet<Fingerprint>)> {
+) -> Result<(Option<Vec<u8>>, HashSet<String>)> {
     let from = mail
         .headers
-        .get_header(HeaderDef::From_)
-        .and_then(|from_addr| mailparse::addrparse_header(&from_addr).ok())
+        .get_first_value("From")?
+        .and_then(|from_addr| mailparse::addrparse(&from_addr).ok())
         .and_then(|from| from.extract_single_info())
         .map(|from| from.addr)
         .unwrap_or_default();
@@ -132,54 +132,105 @@ pub async fn try_decrypt(
     let autocryptheader = Aheader::from_headers(context, &from, &mail.headers);
 
     if message_time > 0 {
-        peerstate = Peerstate::from_addr(context, &from).await.ok();
+        peerstate = Peerstate::from_addr(context, &context.sql, &from);
 
         if let Some(ref mut peerstate) = peerstate {
             if let Some(ref header) = autocryptheader {
                 peerstate.apply_header(&header, message_time);
-                peerstate.save_to_db(&context.sql, false).await?;
+                peerstate.save_to_db(&context.sql, false)?;
             } else if message_time > peerstate.last_seen_autocrypt && !contains_report(mail) {
                 peerstate.degrade_encryption(message_time);
-                peerstate.save_to_db(&context.sql, false).await?;
+                peerstate.save_to_db(&context.sql, false)?;
             }
         } else if let Some(ref header) = autocryptheader {
-            let p = Peerstate::from_header(header, message_time);
-            p.save_to_db(&context.sql, true).await?;
+            let p = Peerstate::from_header(context, header, message_time);
+            p.save_to_db(&context.sql, true)?;
             peerstate = Some(p);
         }
     }
 
     /* possibly perform decryption */
-    let private_keyring: Keyring<SignedSecretKey> = Keyring::new_self(context).await?;
-    let mut public_keyring_for_validate: Keyring<SignedPublicKey> = Keyring::new();
+    let mut private_keyring = Keyring::default();
+    let mut public_keyring_for_validate = Keyring::default();
+    let mut out_mail = None;
     let mut signatures = HashSet::default();
+    let self_addr = context.get_config(Config::ConfiguredAddr);
 
-    if peerstate.as_ref().map(|p| p.last_seen).unwrap_or_else(|| 0) == 0 {
-        peerstate = Peerstate::from_addr(&context, &from).await.ok();
-    }
-    if let Some(peerstate) = peerstate {
-        if peerstate.degrade_event.is_some() {
-            handle_degrade_event(context, &peerstate).await?;
-        }
-        if let Some(key) = peerstate.gossip_key {
-            public_keyring_for_validate.add(key);
-        }
-        if let Some(key) = peerstate.public_key {
-            public_keyring_for_validate.add(key);
+    if let Some(self_addr) = self_addr {
+        if private_keyring.load_self_private_for_decrypting(context, self_addr, &context.sql) {
+            if peerstate.as_ref().map(|p| p.last_seen).unwrap_or_else(|| 0) == 0 {
+                peerstate = Peerstate::from_addr(&context, &context.sql, &from);
+            }
+            if let Some(ref peerstate) = peerstate {
+                if peerstate.degrade_event.is_some() {
+                    handle_degrade_event(context, &peerstate)?;
+                }
+                if let Some(ref key) = peerstate.gossip_key {
+                    public_keyring_for_validate.add_ref(key);
+                }
+                if let Some(ref key) = peerstate.public_key {
+                    public_keyring_for_validate.add_ref(key);
+                }
+            }
+
+            out_mail = decrypt_if_autocrypt_message(
+                context,
+                mail,
+                &private_keyring,
+                &public_keyring_for_validate,
+                &mut signatures,
+            )?;
         }
     }
-
-    let out_mail = decrypt_if_autocrypt_message(
-        context,
-        mail,
-        private_keyring,
-        public_keyring_for_validate,
-        &mut signatures,
-    )
-    .await?;
     Ok((out_mail, signatures))
 }
 
+/// Load public key from database or generate a new one.
+///
+/// This will load a public key from the database, generating and
+/// storing a new one when one doesn't exist yet.  Care is taken to
+/// only generate one key per context even when multiple threads call
+/// this function concurrently.
+fn load_or_generate_self_public_key(context: &Context, self_addr: impl AsRef<str>) -> Result<Key> {
+    if let Some(key) = Key::from_self_public(context, &self_addr, &context.sql) {
+        return Ok(key);
+    }
+    let _guard = context.generating_key_mutex.lock().unwrap();
+
+    // Check again in case the key was generated while we were waiting for the lock.
+    if let Some(key) = Key::from_self_public(context, &self_addr, &context.sql) {
+        return Ok(key);
+    }
+
+    let start = std::time::Instant::now();
+    info!(
+        context,
+        "Generating keypair with {} bits, e={} ...", 2048, 65537,
+    );
+    match pgp::create_keypair(&self_addr) {
+        Some((public_key, private_key)) => {
+            if dc_key_save_self_keypair(
+                context,
+                &public_key,
+                &private_key,
+                &self_addr,
+                true,
+                &context.sql,
+            ) {
+                info!(
+                    context,
+                    "Keypair generated in {:.3}s.",
+                    start.elapsed().as_secs()
+                );
+                Ok(public_key)
+            } else {
+                Err(format_err!("Failed to save keypair"))
+            }
+        }
+        None => Err(format_err!("Failed to generate keypair")),
+    }
+}
+
 /// Returns a reference to the encrypted payload and validates the autocrypt structure.
 fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Result<&'a ParsedMail<'b>> {
     ensure!(
@@ -207,12 +258,12 @@ fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Result<&'a ParsedMail
     Ok(&mail.subparts[1])
 }
 
-async fn decrypt_if_autocrypt_message<'a>(
+fn decrypt_if_autocrypt_message<'a>(
     context: &Context,
     mail: &ParsedMail<'a>,
-    private_keyring: Keyring<SignedSecretKey>,
-    public_keyring_for_validate: Keyring<SignedPublicKey>,
-    ret_valid_signatures: &mut HashSet<Fingerprint>,
+    private_keyring: &Keyring,
+    public_keyring_for_validate: &Keyring,
+    ret_valid_signatures: &mut HashSet<String>,
 ) -> Result<Option<Vec<u8>>> {
     //  The returned bool is true if we detected an Autocrypt-encrypted
     // message and successfully decrypted it. Decryption then modifies the
@@ -231,20 +282,21 @@ async fn decrypt_if_autocrypt_message<'a>(
     info!(context, "Detected Autocrypt-mime message");
 
     decrypt_part(
+        context,
         encrypted_data_part,
         private_keyring,
         public_keyring_for_validate,
         ret_valid_signatures,
     )
-    .await
 }
 
 /// Returns Ok(None) if nothing encrypted was found.
-async fn decrypt_part(
+fn decrypt_part(
+    _context: &Context,
     mail: &ParsedMail<'_>,
-    private_keyring: Keyring<SignedSecretKey>,
-    public_keyring_for_validate: Keyring<SignedPublicKey>,
-    ret_valid_signatures: &mut HashSet<Fingerprint>,
+    private_keyring: &Keyring,
+    public_keyring_for_validate: &Keyring,
+    ret_valid_signatures: &mut HashSet<String>,
 ) -> Result<Option<Vec<u8>>> {
     let data = mail.get_body_raw()?;
 
@@ -253,12 +305,11 @@ async fn decrypt_part(
         ensure!(ret_valid_signatures.is_empty(), "corrupt signatures");
 
         let plain = pgp::pk_decrypt(
-            data,
-            private_keyring,
-            public_keyring_for_validate,
+            &data,
+            &private_keyring,
+            &public_keyring_for_validate,
             Some(ret_valid_signatures),
-        )
-        .await?;
+        )?;
 
         ensure!(!ret_valid_signatures.is_empty(), "no valid signatures");
         return Ok(Some(plain));
@@ -301,18 +352,14 @@ fn contains_report(mail: &ParsedMail<'_>) -> bool {
 ///
 /// If this succeeds you are also guaranteed that the
 /// [Config::ConfiguredAddr] is configured, this address is returned.
-// TODO, remove this once deltachat::key::Key no longer exists.
-pub async fn ensure_secret_key_exists(context: &Context) -> Result<String> {
-    let self_addr = context
-        .get_config(Config::ConfiguredAddr)
-        .await
-        .ok_or_else(|| {
-            format_err!(concat!(
-                "Failed to get self address, ",
-                "cannot ensure secret key if not configured."
-            ))
-        })?;
-    SignedPublicKey::load_self(context).await?;
+pub fn ensure_secret_key_exists(context: &Context) -> Result<String> {
+    let self_addr = context.get_config(Config::ConfiguredAddr).ok_or_else(|| {
+        format_err!(concat!(
+            "Failed to get self address, ",
+            "cannot ensure secret key if not configured."
+        ))
+    })?;
+    load_or_generate_self_public_key(context, &self_addr)?;
     Ok(self_addr)
 }
 
@@ -325,17 +372,17 @@ mod tests {
     mod ensure_secret_key_exists {
         use super::*;
 
-        #[async_std::test]
-        async fn test_prexisting() {
-            let t = TestContext::new().await;
-            let test_addr = t.configure_alice().await;
-            assert_eq!(ensure_secret_key_exists(&t.ctx).await.unwrap(), test_addr);
+        #[test]
+        fn test_prexisting() {
+            let t = dummy_context();
+            let test_addr = configure_alice_keypair(&t.ctx);
+            assert_eq!(ensure_secret_key_exists(&t.ctx).unwrap(), test_addr);
         }
 
-        #[async_std::test]
-        async fn test_not_configured() {
-            let t = TestContext::new().await;
-            assert!(ensure_secret_key_exists(&t.ctx).await.is_err());
+        #[test]
+        fn test_not_configured() {
+            let t = dummy_context();
+            assert!(ensure_secret_key_exists(&t.ctx).is_err());
         }
     }
 
@@ -363,6 +410,49 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         );
     }
 
+    mod load_or_generate_self_public_key {
+        use super::*;
+
+        #[test]
+        fn test_existing() {
+            let t = dummy_context();
+            let addr = configure_alice_keypair(&t.ctx);
+            let key = load_or_generate_self_public_key(&t.ctx, addr);
+            assert!(key.is_ok());
+        }
+
+        #[test]
+        #[ignore] // generating keys is expensive
+        fn test_generate() {
+            let t = dummy_context();
+            let addr = "alice@example.org";
+            let key0 = load_or_generate_self_public_key(&t.ctx, addr);
+            assert!(key0.is_ok());
+            let key1 = load_or_generate_self_public_key(&t.ctx, addr);
+            assert!(key1.is_ok());
+            assert_eq!(key0.unwrap(), key1.unwrap());
+        }
+
+        #[test]
+        #[ignore]
+        fn test_generate_concurrent() {
+            use std::sync::Arc;
+            use std::thread;
+
+            let t = dummy_context();
+            let ctx = Arc::new(t.ctx);
+            let ctx0 = Arc::clone(&ctx);
+            let thr0 =
+                thread::spawn(move || load_or_generate_self_public_key(&ctx0, "alice@example.org"));
+            let ctx1 = Arc::clone(&ctx);
+            let thr1 =
+                thread::spawn(move || load_or_generate_self_public_key(&ctx1, "alice@example.org"));
+            let res0 = thr0.join().unwrap();
+            let res1 = thr1.join().unwrap();
+            assert_eq!(res0.unwrap(), res1.unwrap());
+        }
+    }
+
     #[test]
     fn test_has_decrypted_pgp_armor() {
         let data = b" -----BEGIN PGP MESSAGE-----";

src/error.rs

@@ -1,6 +1,156 @@
 //! # Error handling
 
-pub use anyhow::{bail, ensure, format_err, Error, Result};
+use lettre_email::mime;
+
+#[derive(Debug, Fail)]
+pub enum Error {
+    #[fail(display = "{:?}", _0)]
+    Failure(failure::Error),
+    #[fail(display = "SQL error: {:?}", _0)]
+    SqlError(#[cause] crate::sql::Error),
+    #[fail(display = "{:?}", _0)]
+    Io(std::io::Error),
+    #[fail(display = "{:?}", _0)]
+    Message(String),
+    #[fail(display = "{:?}", _0)]
+    Image(image_meta::ImageError),
+    #[fail(display = "{:?}", _0)]
+    Utf8(std::str::Utf8Error),
+    #[fail(display = "PGP: {:?}", _0)]
+    Pgp(pgp::errors::Error),
+    #[fail(display = "Base64Decode: {:?}", _0)]
+    Base64Decode(base64::DecodeError),
+    #[fail(display = "{:?}", _0)]
+    FromUtf8(std::string::FromUtf8Error),
+    #[fail(display = "{}", _0)]
+    BlobError(#[cause] crate::blob::BlobError),
+    #[fail(display = "Invalid Message ID.")]
+    InvalidMsgId,
+    #[fail(display = "Watch folder not found {:?}", _0)]
+    WatchFolderNotFound(String),
+    #[fail(display = "Invalid Email: {:?}", _0)]
+    MailParseError(#[cause] mailparse::MailParseError),
+    #[fail(display = "Building invalid Email: {:?}", _0)]
+    LettreError(#[cause] lettre_email::error::Error),
+    #[fail(display = "FromStr error: {:?}", _0)]
+    FromStr(#[cause] mime::FromStrError),
+    #[fail(display = "Not Configured")]
+    NotConfigured,
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+impl From<crate::sql::Error> for Error {
+    fn from(err: crate::sql::Error) -> Error {
+        Error::SqlError(err)
+    }
+}
+
+impl From<base64::DecodeError> for Error {
+    fn from(err: base64::DecodeError) -> Error {
+        Error::Base64Decode(err)
+    }
+}
+
+impl From<failure::Error> for Error {
+    fn from(err: failure::Error) -> Error {
+        Error::Failure(err)
+    }
+}
+
+impl From<std::io::Error> for Error {
+    fn from(err: std::io::Error) -> Error {
+        Error::Io(err)
+    }
+}
+
+impl From<std::str::Utf8Error> for Error {
+    fn from(err: std::str::Utf8Error) -> Error {
+        Error::Utf8(err)
+    }
+}
+
+impl From<image_meta::ImageError> for Error {
+    fn from(err: image_meta::ImageError) -> Error {
+        Error::Image(err)
+    }
+}
+
+impl From<pgp::errors::Error> for Error {
+    fn from(err: pgp::errors::Error) -> Error {
+        Error::Pgp(err)
+    }
+}
+
+impl From<std::string::FromUtf8Error> for Error {
+    fn from(err: std::string::FromUtf8Error) -> Error {
+        Error::FromUtf8(err)
+    }
+}
+
+impl From<crate::blob::BlobError> for Error {
+    fn from(err: crate::blob::BlobError) -> Error {
+        Error::BlobError(err)
+    }
+}
+
+impl From<crate::message::InvalidMsgId> for Error {
+    fn from(_err: crate::message::InvalidMsgId) -> Error {
+        Error::InvalidMsgId
+    }
+}
+
+impl From<mailparse::MailParseError> for Error {
+    fn from(err: mailparse::MailParseError) -> Error {
+        Error::MailParseError(err)
+    }
+}
+
+impl From<lettre_email::error::Error> for Error {
+    fn from(err: lettre_email::error::Error) -> Error {
+        Error::LettreError(err)
+    }
+}
+
+impl From<mime::FromStrError> for Error {
+    fn from(err: mime::FromStrError) -> Error {
+        Error::FromStr(err)
+    }
+}
+
+#[macro_export]
+macro_rules! bail {
+    ($e:expr) => {
+        return Err($crate::error::Error::Message($e.to_string()));
+    };
+    ($fmt:expr, $($arg:tt)+) => {
+        return Err($crate::error::Error::Message(format!($fmt, $($arg)+)));
+    };
+}
+
+#[macro_export]
+macro_rules! format_err {
+    ($e:expr) => {
+        $crate::error::Error::Message($e.to_string());
+    };
+    ($fmt:expr, $($arg:tt)+) => {
+        $crate::error::Error::Message(format!($fmt, $($arg)+));
+    };
+}
+
+#[macro_export(local_inner_macros)]
+macro_rules! ensure {
+    ($cond:expr, $e:expr) => {
+        if !($cond) {
+            bail!($e);
+        }
+    };
+    ($cond:expr, $fmt:expr, $($arg:tt)+) => {
+        if !($cond) {
+            bail!($fmt, $($arg)+);
+        }
+    };
+}
 
 #[macro_export]
 macro_rules! ensure_eq {

src/events.rs

@@ -1,65 +1,11 @@
 //! # Events specification
 
-use async_std::path::PathBuf;
-use async_std::sync::{channel, Receiver, Sender, TrySendError};
+use std::path::PathBuf;
+
 use strum::EnumProperty;
 
-use crate::chat::ChatId;
 use crate::message::MsgId;
 
-#[derive(Debug)]
-pub struct Events {
-    receiver: Receiver<Event>,
-    sender: Sender<Event>,
-}
-
-impl Default for Events {
-    fn default() -> Self {
-        let (sender, receiver) = channel(1_000);
-
-        Self { receiver, sender }
-    }
-}
-
-impl Events {
-    pub fn emit(&self, event: Event) {
-        match self.sender.try_send(event) {
-            Ok(()) => {}
-            Err(TrySendError::Full(event)) => {
-                // when we are full, we pop remove the oldest event and push on the new one
-                let _ = self.receiver.try_recv();
-
-                // try again
-                self.emit(event);
-            }
-            Err(TrySendError::Disconnected(_)) => {
-                unreachable!("unable to emit event, channel disconnected");
-            }
-        }
-    }
-
-    /// Retrieve the event emitter.
-    pub fn get_emitter(&self) -> EventEmitter {
-        EventEmitter(self.receiver.clone())
-    }
-}
-
-#[derive(Debug, Clone)]
-pub struct EventEmitter(Receiver<Event>);
-
-impl EventEmitter {
-    /// Blocking recv of an event. Return `None` if the `Sender` has been droped.
-    pub fn recv_sync(&self) -> Option<Event> {
-        async_std::task::block_on(self.recv())
-    }
-
-    /// Blocking async recv of an event. Return `None` if the `Sender` has been droped.
-    pub async fn recv(&self) -> Option<Event> {
-        // TODO: change once we can use async channels internally.
-        self.0.recv().await.ok()
-    }
-}
-
 impl Event {
     /// Returns the corresponding Event id.
     pub fn as_id(&self) -> i32 {
@@ -75,38 +21,56 @@ pub enum Event {
     /// The library-user may write an informational string to the log.
     /// Passed to the callback given to dc_context_new().
     /// This event should not be reported to the end-user using a popup or something like that.
+    ///
+    /// @return 0
     #[strum(props(id = "100"))]
     Info(String),
 
     /// Emitted when SMTP connection is established and login was successful.
+    ///
+    /// @return 0
     #[strum(props(id = "101"))]
     SmtpConnected(String),
 
     /// Emitted when IMAP connection is established and login was successful.
+    ///
+    /// @return 0
     #[strum(props(id = "102"))]
     ImapConnected(String),
 
     /// Emitted when a message was successfully sent to the SMTP server.
+    ///
+    /// @return 0
     #[strum(props(id = "103"))]
     SmtpMessageSent(String),
 
     /// Emitted when an IMAP message has been marked as deleted
+    ///
+    /// @return 0
     #[strum(props(id = "104"))]
     ImapMessageDeleted(String),
 
     /// Emitted when an IMAP message has been moved
+    ///
+    /// @return 0
     #[strum(props(id = "105"))]
     ImapMessageMoved(String),
 
     /// Emitted when an IMAP folder was emptied
+    ///
+    /// @return 0
     #[strum(props(id = "106"))]
     ImapFolderEmptied(String),
 
     /// Emitted when an new file in the $BLOBDIR was created
+    ///
+    /// @return 0
     #[strum(props(id = "150"))]
     NewBlobFile(String),
 
     /// Emitted when an new file in the $BLOBDIR was created
+    ///
+    /// @return 0
     #[strum(props(id = "151"))]
     DeletedBlobFile(String),
 
@@ -114,6 +78,8 @@ pub enum Event {
     /// Passed to the callback given to dc_context_new().
     ///
     /// This event should not be reported to the end-user using a popup or something like that.
+    ///
+    /// @return 0
     #[strum(props(id = "300"))]
     Warning(String),
 
@@ -128,6 +94,8 @@ pub enum Event {
     /// it might be better to delay showing these events until the function has really
     /// failed (returned false). It should be sufficient to report only the *last* error
     /// in a messasge box then.
+    ///
+    /// @return
     #[strum(props(id = "400"))]
     Error(String),
 
@@ -144,6 +112,8 @@ pub enum Event {
     /// Moreover, if the UI detects that the device is offline,
     /// it is probably more useful to report this to the user
     /// instead of the string from data2.
+    ///
+    /// @return 0
     #[strum(props(id = "401"))]
     ErrorNetwork(String),
 
@@ -152,6 +122,8 @@ pub enum Event {
     /// dc_set_chat_name(), dc_set_chat_profile_image(),
     /// dc_add_contact_to_chat(), dc_remove_contact_from_chat(),
     /// dc_send_text_msg() or another sending function.
+    ///
+    /// @return 0
     #[strum(props(id = "410"))]
     ErrorSelfNotInGroup(String),
 
@@ -160,41 +132,54 @@ pub enum Event {
     /// - Messages sent, received or removed
     /// - Chats created, deleted or archived
     /// - A draft has been set
+    ///
+    /// @return 0
     #[strum(props(id = "2000"))]
-    MsgsChanged { chat_id: ChatId, msg_id: MsgId },
+    MsgsChanged { chat_id: u32, msg_id: MsgId },
 
     /// There is a fresh message. Typically, the user will show an notification
     /// when receiving this message.
     ///
     /// There is no extra #DC_EVENT_MSGS_CHANGED event send together with this event.
+    ///
+    /// @return 0
     #[strum(props(id = "2005"))]
-    IncomingMsg { chat_id: ChatId, msg_id: MsgId },
+    IncomingMsg { chat_id: u32, msg_id: MsgId },
 
     /// A single message is sent successfully. State changed from  DC_STATE_OUT_PENDING to
     /// DC_STATE_OUT_DELIVERED, see dc_msg_get_state().
+    ///
+    /// @return 0
     #[strum(props(id = "2010"))]
-    MsgDelivered { chat_id: ChatId, msg_id: MsgId },
+    MsgDelivered { chat_id: u32, msg_id: MsgId },
 
     /// A single message could not be sent. State changed from DC_STATE_OUT_PENDING or DC_STATE_OUT_DELIVERED to
     /// DC_STATE_OUT_FAILED, see dc_msg_get_state().
+    ///
+    /// @return 0
     #[strum(props(id = "2012"))]
-    MsgFailed { chat_id: ChatId, msg_id: MsgId },
+    MsgFailed { chat_id: u32, msg_id: MsgId },
 
     /// A single message is read by the receiver. State changed from DC_STATE_OUT_DELIVERED to
     /// DC_STATE_OUT_MDN_RCVD, see dc_msg_get_state().
+    ///
+    /// @return 0
     #[strum(props(id = "2015"))]
-    MsgRead { chat_id: ChatId, msg_id: MsgId },
+    MsgRead { chat_id: u32, msg_id: MsgId },
 
     /// Chat changed.  The name or the image of a chat group was changed or members were added or removed.
     /// Or the verify state of a chat has changed.
     /// See dc_set_chat_name(), dc_set_chat_profile_image(), dc_add_contact_to_chat()
     /// and dc_remove_contact_from_chat().
+    ///
+    /// @return 0
     #[strum(props(id = "2020"))]
-    ChatModified(ChatId),
+    ChatModified(u32),
 
     /// Contact(s) created, renamed, blocked or deleted.
     ///
     /// @param data1 (int) If set, this is the contact_id of an added contact that should be selected.
+    /// @return 0
     #[strum(props(id = "2030"))]
     ContactsChanged(Option<u32>),
 
@@ -203,12 +188,14 @@ pub enum Event {
     /// @param data1 (u32) contact_id of the contact for which the location has changed.
     ///     If the locations of several contacts have been changed,
     ///     eg. after calling dc_delete_all_locations(), this parameter is set to `None`.
+    /// @return 0
     #[strum(props(id = "2035"))]
     LocationChanged(Option<u32>),
 
     /// Inform about the configuration progress started by configure().
     ///
     /// @param data1 (usize) 0=error, 1-999=progress in permille, 1000=success and done
+    /// @return 0
     #[strum(props(id = "2041"))]
     ConfigureProgress(usize),
 
@@ -216,6 +203,7 @@ pub enum Event {
     ///
     /// @param data1 (usize) 0=error, 1-999=progress in permille, 1000=success and done
     /// @param data2 0
+    /// @return 0
     #[strum(props(id = "2051"))]
     ImexProgress(usize),
 
@@ -226,6 +214,7 @@ pub enum Event {
     /// services.
     ///
     /// @param data2 0
+    /// @return 0
     #[strum(props(id = "2052"))]
     ImexFileWritten(PathBuf),
 
@@ -241,6 +230,7 @@ pub enum Event {
     ///     600=vg-/vc-request-with-auth received, vg-member-added/vc-contact-confirm sent, typically shown as "bob@addr verified".
     ///     800=vg-member-added-received received, shown as "bob@addr securely joined GROUP", only sent for the verified-group-protocol.
     ///     1000=Protocol finished for this contact.
+    /// @return 0
     #[strum(props(id = "2060"))]
     SecurejoinInviterProgress { contact_id: u32, progress: usize },
 
@@ -252,6 +242,14 @@ pub enum Event {
     /// @param data2 (int) Progress as:
     ///     400=vg-/vc-request-with-auth sent, typically shown as "alice@addr verified, introducing myself."
     ///     (Bob has verified alice and waits until Alice does the same for him)
+    /// @return 0
     #[strum(props(id = "2061"))]
     SecurejoinJoinerProgress { contact_id: u32, progress: usize },
+
+    /// This event is sent out to the inviter when a joiner successfully joined a group.
+    /// @param data1 (int) chat_id
+    /// @param data2 (int) contact_id
+    /// @return 0
+    #[strum(props(id = "2062"))]
+    SecurejoinMemberAdded { chat_id: u32, contact_id: u32 },
 }

src/headerdef.rs

@@ -1,7 +1,4 @@
-use crate::strum::AsStaticRef;
-use mailparse::{MailHeader, MailHeaderMap};
-
-#[derive(Debug, Display, Clone, PartialEq, Eq, EnumVariantNames, AsStaticStr)]
+#[derive(Debug, Display, Clone, PartialEq, Eq, EnumVariantNames)]
 #[strum(serialize_all = "kebab_case")]
 #[allow(dead_code)]
 pub enum HeaderDef {
@@ -13,20 +10,16 @@ pub enum HeaderDef {
     Cc,
     Disposition,
     OriginalMessageId,
-
-    /// Delta Chat extension for message IDs in combined MDNs
-    AdditionalMessageIds,
-
     ListId,
     References,
     InReplyTo,
     Precedence,
-    ContentType,
     ChatVersion,
     ChatGroupId,
     ChatGroupName,
     ChatGroupNameChanged,
     ChatVerified,
+    ChatGroupImage, // deprecated
     ChatGroupAvatar,
     ChatUserAvatar,
     ChatVoiceMessage,
@@ -35,7 +28,6 @@ pub enum HeaderDef {
     ChatContent,
     ChatDuration,
     ChatDispositionNotificationTo,
-    Autocrypt,
     AutocryptSetupMessage,
     SecureJoin,
     SecureJoinGroup,
@@ -47,22 +39,8 @@ pub enum HeaderDef {
 
 impl HeaderDef {
     /// Returns the corresponding Event id.
-    pub fn get_headername(&self) -> &'static str {
-        self.as_static()
-    }
-}
-
-pub trait HeaderDefMap {
-    fn get_header_value(&self, headerdef: HeaderDef) -> Option<String>;
-    fn get_header(&self, headerdef: HeaderDef) -> Option<&MailHeader>;
-}
-
-impl HeaderDefMap for [MailHeader<'_>] {
-    fn get_header_value(&self, headerdef: HeaderDef) -> Option<String> {
-        self.get_first_value(headerdef.get_headername())
-    }
-    fn get_header(&self, headerdef: HeaderDef) -> Option<&MailHeader> {
-        self.get_first_header(headerdef.get_headername())
+    pub fn get_headername(&self) -> String {
+        self.to_string()
     }
 }
 
@@ -73,24 +51,8 @@ mod tests {
     #[test]
     /// Test that kebab_case serialization works as expected
     fn kebab_test() {
-        assert_eq!(HeaderDef::From_.get_headername(), "from");
+        assert_eq!(HeaderDef::From_.to_string(), "from");
 
-        assert_eq!(HeaderDef::_TestHeader.get_headername(), "test-header");
-    }
-
-    #[test]
-    /// Test that headers are parsed case-insensitively
-    fn test_get_header_value_case() {
-        let (headers, _) =
-            mailparse::parse_headers(b"fRoM: Bob\naUtoCryPt-SeTup-MessAge: v99").unwrap();
-        assert_eq!(
-            headers.get_header_value(HeaderDef::AutocryptSetupMessage),
-            Some("v99".to_string())
-        );
-        assert_eq!(
-            headers.get_header_value(HeaderDef::From_),
-            Some("Bob".to_string())
-        );
-        assert_eq!(headers.get_header_value(HeaderDef::Autocrypt), None);
+        assert_eq!(HeaderDef::_TestHeader.to_string(), "test-header");
     }
 }

src/imap/client.rs

@@ -1,139 +0,0 @@
-use std::ops::{Deref, DerefMut};
-
-use async_imap::{
-    error::{Error as ImapError, Result as ImapResult},
-    Client as ImapClient,
-};
-use async_std::net::{self, TcpStream};
-
-use super::session::Session;
-use crate::login_param::dc_build_tls;
-
-use super::session::SessionStream;
-
-#[derive(Debug)]
-pub(crate) struct Client {
-    is_secure: bool,
-    inner: ImapClient<Box<dyn SessionStream>>,
-}
-
-impl Deref for Client {
-    type Target = ImapClient<Box<dyn SessionStream>>;
-
-    fn deref(&self) -> &Self::Target {
-        &self.inner
-    }
-}
-
-impl DerefMut for Client {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.inner
-    }
-}
-
-impl Client {
-    pub async fn login<U: AsRef<str>, P: AsRef<str>>(
-        self,
-        username: U,
-        password: P,
-    ) -> std::result::Result<Session, (ImapError, Self)> {
-        let Client { inner, is_secure } = self;
-        let session = inner
-            .login(username, password)
-            .await
-            .map_err(|(err, client)| {
-                (
-                    err,
-                    Client {
-                        is_secure,
-                        inner: client,
-                    },
-                )
-            })?;
-        Ok(Session { inner: session })
-    }
-
-    pub async fn authenticate<A: async_imap::Authenticator, S: AsRef<str>>(
-        self,
-        auth_type: S,
-        authenticator: &A,
-    ) -> std::result::Result<Session, (ImapError, Self)> {
-        let Client { inner, is_secure } = self;
-        let session =
-            inner
-                .authenticate(auth_type, authenticator)
-                .await
-                .map_err(|(err, client)| {
-                    (
-                        err,
-                        Client {
-                            is_secure,
-                            inner: client,
-                        },
-                    )
-                })?;
-        Ok(Session { inner: session })
-    }
-
-    pub async fn connect_secure<A: net::ToSocketAddrs, S: AsRef<str>>(
-        addr: A,
-        domain: S,
-        strict_tls: bool,
-    ) -> ImapResult<Self> {
-        let stream = TcpStream::connect(addr).await?;
-        let tls = dc_build_tls(strict_tls);
-        let tls_stream: Box<dyn SessionStream> =
-            Box::new(tls.connect(domain.as_ref(), stream).await?);
-        let mut client = ImapClient::new(tls_stream);
-        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
-            client.debug = true;
-        }
-
-        let _greeting = client
-            .read_response()
-            .await
-            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
-
-        Ok(Client {
-            is_secure: true,
-            inner: client,
-        })
-    }
-
-    pub async fn connect_insecure<A: net::ToSocketAddrs>(addr: A) -> ImapResult<Self> {
-        let stream: Box<dyn SessionStream> = Box::new(TcpStream::connect(addr).await?);
-
-        let mut client = ImapClient::new(stream);
-        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
-            client.debug = true;
-        }
-        let _greeting = client
-            .read_response()
-            .await
-            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
-
-        Ok(Client {
-            is_secure: false,
-            inner: client,
-        })
-    }
-
-    pub async fn secure<S: AsRef<str>>(self, domain: S, strict_tls: bool) -> ImapResult<Client> {
-        if self.is_secure {
-            Ok(self)
-        } else {
-            let Client { mut inner, .. } = self;
-            let tls = dc_build_tls(strict_tls);
-            inner.run_command_and_check_ok("STARTTLS", None).await?;
-
-            let stream = inner.into_inner();
-            let ssl_stream = tls.connect(domain.as_ref(), stream).await?;
-            let boxed: Box<dyn SessionStream> = Box::new(ssl_stream);
-
-            Ok(Client {
-                is_secure: true,
-                inner: ImapClient::new(boxed),
-            })
-        }
-    }
-}

src/imap/idle.rs

@@ -2,230 +2,272 @@ use super::Imap;
 
 use async_imap::extensions::idle::IdleResponse;
 use async_std::prelude::*;
+use async_std::task;
+use std::sync::atomic::Ordering;
 use std::time::{Duration, SystemTime};
 
-use crate::{context::Context, scheduler::InterruptInfo};
+use crate::context::Context;
+use crate::imap_client::*;
 
 use super::select_folder;
-use super::session::Session;
 
 type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug, thiserror::Error)]
+#[derive(Debug, Fail)]
 pub enum Error {
-    #[error("IMAP IDLE protocol failed to init/complete")]
-    IdleProtocolFailed(#[from] async_imap::error::Error),
+    #[fail(display = "IMAP IDLE protocol failed to init/complete")]
+    IdleProtocolFailed(#[cause] async_imap::error::Error),
 
-    #[error("IMAP IDLE protocol timed out")]
-    IdleTimeout(#[from] async_std::future::TimeoutError),
+    #[fail(display = "IMAP IDLE protocol timed out")]
+    IdleTimeout(#[cause] async_std::future::TimeoutError),
 
-    #[error("IMAP server does not have IDLE capability")]
+    #[fail(display = "IMAP server does not have IDLE capability")]
     IdleAbilityMissing,
 
-    #[error("IMAP select folder error")]
-    SelectFolderError(#[from] select_folder::Error),
+    #[fail(display = "IMAP select folder error")]
+    SelectFolderError(#[cause] select_folder::Error),
 
-    #[error("Setup handle error")]
-    SetupHandleError(#[from] super::Error),
+    #[fail(display = "IMAP error")]
+    ImapError(#[cause] async_imap::error::Error),
+
+    #[fail(display = "Setup handle error")]
+    SetupHandleError(#[cause] super::Error),
+}
+
+impl From<select_folder::Error> for Error {
+    fn from(err: select_folder::Error) -> Error {
+        Error::SelectFolderError(err)
+    }
 }
 
 impl Imap {
     pub fn can_idle(&self) -> bool {
-        self.config.can_idle
+        task::block_on(async move { self.config.read().await.can_idle })
     }
 
-    pub async fn idle(
-        &mut self,
-        context: &Context,
-        watch_folder: Option<String>,
-    ) -> Result<InterruptInfo> {
-        use futures::future::FutureExt;
-
-        if !self.can_idle() {
-            return Err(Error::IdleAbilityMissing);
-        }
-        self.setup_handle_if_needed(context).await?;
-
-        self.select_folder(context, watch_folder.clone()).await?;
-
-        let session = self.session.take();
-        let timeout = Duration::from_secs(23 * 60);
-        let mut info = Default::default();
-
-        if let Some(session) = session {
-            let mut handle = session.idle();
-            if let Err(err) = handle.init().await {
-                return Err(Error::IdleProtocolFailed(err));
+    pub fn idle(&self, context: &Context, watch_folder: Option<String>) -> Result<()> {
+        task::block_on(async move {
+            if !self.can_idle() {
+                return Err(Error::IdleAbilityMissing);
             }
 
-            let (idle_wait, interrupt) = handle.wait_with_timeout(timeout);
-
-            enum Event {
-                IdleResponse(IdleResponse),
-                Interrupt(InterruptInfo),
-            }
-
-            if self.skip_next_idle_wait {
-                // interrupt_idle has happened before we
-                // provided self.interrupt
-                self.skip_next_idle_wait = false;
-                drop(idle_wait);
-                drop(interrupt);
-
-                info!(context, "Idle wait was skipped");
-            } else {
-                info!(context, "Idle entering wait-on-remote state");
-                let fut = idle_wait.map(|ev| ev.map(Event::IdleResponse)).race(
-                    self.idle_interrupt.recv().map(|probe_network| {
-                        Ok(Event::Interrupt(probe_network.unwrap_or_default()))
-                    }),
-                );
-
-                match fut.await {
-                    Ok(Event::IdleResponse(IdleResponse::NewData(_))) => {
-                        info!(context, "Idle has NewData");
-                    }
-                    // TODO: idle_wait does not distinguish manual interrupts
-                    // from Timeouts if we would know it's a Timeout we could bail
-                    // directly and reconnect .
-                    Ok(Event::IdleResponse(IdleResponse::Timeout)) => {
-                        info!(context, "Idle-wait timeout or interruption");
-                    }
-                    Ok(Event::IdleResponse(IdleResponse::ManualInterrupt)) => {
-                        info!(context, "Idle wait was interrupted");
-                    }
-                    Ok(Event::Interrupt(i)) => {
-                        info = i;
-                        info!(context, "Idle wait was interrupted");
-                    }
-                    Err(err) => {
-                        warn!(context, "Idle wait errored: {:?}", err);
-                    }
-                }
-            }
-
-            // if we can't properly terminate the idle
-            // protocol let's break the connection.
-            let res = handle
-                .done()
-                .timeout(Duration::from_secs(15))
+            self.setup_handle_if_needed(context)
                 .await
-                .map_err(|err| {
-                    self.trigger_reconnect();
-                    Error::IdleTimeout(err)
-                })?;
+                .map_err(Error::SetupHandleError)?;
 
-            match res {
-                Ok(session) => {
-                    self.session = Some(Session { inner: session });
-                }
-                Err(err) => {
-                    // if we cannot terminate IDLE it probably
-                    // means that we waited long (with idle_wait)
-                    // but the network went away/changed
-                    self.trigger_reconnect();
-                    return Err(Error::IdleProtocolFailed(err));
-                }
-            }
-        }
+            self.select_folder(context, watch_folder.clone()).await?;
 
-        Ok(info)
-    }
+            let session = self.session.lock().await.take();
+            let timeout = Duration::from_secs(23 * 60);
+            if let Some(session) = session {
+                match session.idle() {
+                    // BEWARE: If you change the Secure branch you
+                    // typically also need to change the Insecure branch.
+                    IdleHandle::Secure(mut handle) => {
+                        if let Err(err) = handle.init().await {
+                            return Err(Error::IdleProtocolFailed(err));
+                        }
 
-    pub(crate) async fn fake_idle(
-        &mut self,
-        context: &Context,
-        watch_folder: Option<String>,
-    ) -> InterruptInfo {
-        // Idle using polling. This is also needed if we're not yet configured -
-        // in this case, we're waiting for a configure job (and an interrupt).
+                        let (idle_wait, interrupt) = handle.wait_with_timeout(timeout);
+                        *self.interrupt.lock().await = Some(interrupt);
 
-        let fake_idle_start_time = SystemTime::now();
-        info!(context, "IMAP-fake-IDLEing...");
-
-        // Do not poll, just wait for an interrupt when no folder is passed in.
-        if watch_folder.is_none() {
-            return self.idle_interrupt.recv().await.unwrap_or_default();
-        }
-
-        let mut info: InterruptInfo = Default::default();
-        if self.skip_next_idle_wait {
-            // interrupt_idle has happened before we
-            // provided self.interrupt
-            self.skip_next_idle_wait = false;
-            info!(context, "fake-idle wait was skipped");
-        } else {
-            // check every minute if there are new messages
-            // TODO: grow sleep durations / make them more flexible
-            let mut interval = async_std::stream::interval(Duration::from_secs(60));
-
-            enum Event {
-                Tick,
-                Interrupt(InterruptInfo),
-            }
-            // loop until we are interrupted or if we fetched something
-            info =
-                loop {
-                    use futures::future::FutureExt;
-                    match interval
-                        .next()
-                        .map(|_| Event::Tick)
-                        .race(self.idle_interrupt.recv().map(|probe_network| {
-                            Event::Interrupt(probe_network.unwrap_or_default())
-                        }))
-                        .await
-                    {
-                        Event::Tick => {
-                            // try to connect with proper login params
-                            // (setup_handle_if_needed might not know about them if we
-                            // never successfully connected)
-                            if let Err(err) = self.connect_configured(context).await {
-                                warn!(context, "fake_idle: could not connect: {}", err);
-                                continue;
-                            }
-                            if self.config.can_idle {
-                                // we only fake-idled because network was gone during IDLE, probably
-                                break InterruptInfo::new(false, None);
-                            }
-                            info!(context, "fake_idle is connected");
-                            // we are connected, let's see if fetching messages results
-                            // in anything.  If so, we behave as if IDLE had data but
-                            // will have already fetched the messages so perform_*_fetch
-                            // will not find any new.
-
-                            if let Some(ref watch_folder) = watch_folder {
-                                match self.fetch_new_messages(context, watch_folder).await {
-                                    Ok(res) => {
-                                        info!(context, "fetch_new_messages returned {:?}", res);
-                                        if res {
-                                            break InterruptInfo::new(false, None);
-                                        }
-                                    }
-                                    Err(err) => {
-                                        error!(context, "could not fetch from folder: {}", err);
-                                        self.trigger_reconnect()
-                                    }
+                        if self.skip_next_idle_wait.load(Ordering::SeqCst) {
+                            // interrupt_idle has happened before we
+                            // provided self.interrupt
+                            self.skip_next_idle_wait.store(false, Ordering::SeqCst);
+                            std::mem::drop(idle_wait);
+                            info!(context, "Idle wait was skipped");
+                        } else {
+                            info!(context, "Idle entering wait-on-remote state");
+                            match idle_wait.await {
+                                IdleResponse::NewData(_) => {
+                                    info!(context, "Idle has NewData");
+                                }
+                                // TODO: idle_wait does not distinguish manual interrupts
+                                // from Timeouts if we would know it's a Timeout we could bail
+                                // directly and reconnect .
+                                IdleResponse::Timeout => {
+                                    info!(context, "Idle-wait timeout or interruption");
+                                }
+                                IdleResponse::ManualInterrupt => {
+                                    info!(context, "Idle wait was interrupted");
                                 }
                             }
                         }
-                        Event::Interrupt(info) => {
-                            // Interrupt
-                            break info;
+                        // if we can't properly terminate the idle
+                        // protocol let's break the connection.
+                        let res =
+                            async_std::future::timeout(Duration::from_secs(15), handle.done())
+                                .await
+                                .map_err(|err| {
+                                    self.trigger_reconnect();
+                                    Error::IdleTimeout(err)
+                                })?;
+
+                        match res {
+                            Ok(session) => {
+                                *self.session.lock().await = Some(Session::Secure(session));
+                            }
+                            Err(err) => {
+                                // if we cannot terminate IDLE it probably
+                                // means that we waited long (with idle_wait)
+                                // but the network went away/changed
+                                self.trigger_reconnect();
+                                return Err(Error::IdleProtocolFailed(err));
+                            }
                         }
                     }
-                };
-        }
+                    IdleHandle::Insecure(mut handle) => {
+                        if let Err(err) = handle.init().await {
+                            return Err(Error::IdleProtocolFailed(err));
+                        }
 
-        info!(
-            context,
-            "IMAP-fake-IDLE done after {:.4}s",
-            SystemTime::now()
-                .duration_since(fake_idle_start_time)
-                .unwrap_or_default()
-                .as_millis() as f64
-                / 1000.,
-        );
+                        let (idle_wait, interrupt) = handle.wait_with_timeout(timeout);
+                        *self.interrupt.lock().await = Some(interrupt);
 
-        info
+                        if self.skip_next_idle_wait.load(Ordering::SeqCst) {
+                            // interrupt_idle has happened before we
+                            // provided self.interrupt
+                            self.skip_next_idle_wait.store(false, Ordering::SeqCst);
+                            std::mem::drop(idle_wait);
+                            info!(context, "Idle wait was skipped");
+                        } else {
+                            info!(context, "Idle entering wait-on-remote state");
+                            match idle_wait.await {
+                                IdleResponse::NewData(_) => {
+                                    info!(context, "Idle has NewData");
+                                }
+                                // TODO: idle_wait does not distinguish manual interrupts
+                                // from Timeouts if we would know it's a Timeout we could bail
+                                // directly and reconnect .
+                                IdleResponse::Timeout => {
+                                    info!(context, "Idle-wait timeout or interruption");
+                                }
+                                IdleResponse::ManualInterrupt => {
+                                    info!(context, "Idle wait was interrupted");
+                                }
+                            }
+                        }
+                        // if we can't properly terminate the idle
+                        // protocol let's break the connection.
+                        let res =
+                            async_std::future::timeout(Duration::from_secs(15), handle.done())
+                                .await
+                                .map_err(|err| {
+                                    self.trigger_reconnect();
+                                    Error::IdleTimeout(err)
+                                })?;
+
+                        match res {
+                            Ok(session) => {
+                                *self.session.lock().await = Some(Session::Insecure(session));
+                            }
+                            Err(err) => {
+                                // if we cannot terminate IDLE it probably
+                                // means that we waited long (with idle_wait)
+                                // but the network went away/changed
+                                self.trigger_reconnect();
+                                return Err(Error::IdleProtocolFailed(err));
+                            }
+                        }
+                    }
+                }
+            }
+
+            Ok(())
+        })
+    }
+
+    pub(crate) fn fake_idle(&self, context: &Context, watch_folder: Option<String>) {
+        // Idle using polling. This is also needed if we're not yet configured -
+        // in this case, we're waiting for a configure job (and an interrupt).
+        task::block_on(async move {
+            let fake_idle_start_time = SystemTime::now();
+
+            info!(context, "IMAP-fake-IDLEing...");
+
+            let interrupt = stop_token::StopSource::new();
+
+            // check every minute if there are new messages
+            // TODO: grow sleep durations / make them more flexible
+            let interval = async_std::stream::interval(Duration::from_secs(60));
+            let mut interrupt_interval = interrupt.stop_token().stop_stream(interval);
+            *self.interrupt.lock().await = Some(interrupt);
+            if self.skip_next_idle_wait.load(Ordering::SeqCst) {
+                // interrupt_idle has happened before we
+                // provided self.interrupt
+                self.skip_next_idle_wait.store(false, Ordering::SeqCst);
+                info!(context, "fake-idle wait was skipped");
+            } else {
+                // loop until we are interrupted or if we fetched something
+                while let Some(_) = interrupt_interval.next().await {
+                    // try to connect with proper login params
+                    // (setup_handle_if_needed might not know about them if we
+                    // never successfully connected)
+                    if let Err(err) = self.connect_configured(context) {
+                        warn!(context, "fake_idle: could not connect: {}", err);
+                        continue;
+                    }
+                    if self.config.read().await.can_idle {
+                        // we only fake-idled because network was gone during IDLE, probably
+                        break;
+                    }
+                    info!(context, "fake_idle is connected");
+                    // we are connected, let's see if fetching messages results
+                    // in anything.  If so, we behave as if IDLE had data but
+                    // will have already fetched the messages so perform_*_fetch
+                    // will not find any new.
+
+                    if let Some(ref watch_folder) = watch_folder {
+                        match self.fetch_new_messages(context, watch_folder).await {
+                            Ok(res) => {
+                                info!(context, "fetch_new_messages returned {:?}", res);
+                                if res {
+                                    break;
+                                }
+                            }
+                            Err(err) => {
+                                error!(context, "could not fetch from folder: {}", err);
+                                self.trigger_reconnect()
+                            }
+                        }
+                    }
+                }
+            }
+            self.interrupt.lock().await.take();
+
+            info!(
+                context,
+                "IMAP-fake-IDLE done after {:.4}s",
+                SystemTime::now()
+                    .duration_since(fake_idle_start_time)
+                    .unwrap_or_default()
+                    .as_millis() as f64
+                    / 1000.,
+            );
+        })
+    }
+
+    pub fn interrupt_idle(&self, context: &Context) {
+        task::block_on(async move {
+            let mut interrupt: Option<stop_token::StopSource> = self.interrupt.lock().await.take();
+            if interrupt.is_none() {
+                // idle wait is not running, signal it needs to skip
+                self.skip_next_idle_wait.store(true, Ordering::SeqCst);
+
+                // meanwhile idle-wait may have produced the StopSource
+                interrupt = self.interrupt.lock().await.take();
+            }
+            // let's manually drop the StopSource
+            if interrupt.is_some() {
+                // the imap thread provided us a stop token but might
+                // not have entered idle_wait yet, give it some time
+                // for that to happen. XXX handle this without extra wait
+                // https://github.com/deltachat/deltachat-core-rust/issues/925
+                std::thread::sleep(Duration::from_millis(200));
+                info!(context, "low-level: dropping stop-source to interrupt idle");
+                std::mem::drop(interrupt)
+            }
+        });
     }
 }

src/imap/select_folder.rs

@@ -4,63 +4,36 @@ use crate::context::Context;
 
 type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug, thiserror::Error)]
+#[derive(Debug, Fail)]
 pub enum Error {
-    #[error("IMAP Could not obtain imap-session object.")]
+    #[fail(display = "IMAP Could not obtain imap-session object.")]
     NoSession,
 
-    #[error("IMAP Connection Lost or no connection established")]
+    #[fail(display = "IMAP Connection Lost or no connection established")]
     ConnectionLost,
 
-    #[error("IMAP Folder name invalid: {0}")]
+    #[fail(display = "IMAP Folder name invalid: {:?}", _0)]
     BadFolderName(String),
 
-    #[error("IMAP close/expunge failed")]
-    CloseExpungeFailed(#[from] async_imap::error::Error),
+    #[fail(display = "IMAP close/expunge failed: {}", _0)]
+    CloseExpungeFailed(#[cause] async_imap::error::Error),
 
-    #[error("IMAP other error: {0}")]
+    #[fail(display = "IMAP other error: {:?}", _0)]
     Other(String),
 }
 
 impl Imap {
-    /// Issues a CLOSE command to expunge selected folder.
-    ///
-    /// CLOSE is considerably faster than an EXPUNGE, see
-    /// https://tools.ietf.org/html/rfc3501#section-6.4.2
-    pub(super) async fn close_folder(&mut self, context: &Context) -> Result<()> {
-        if let Some(ref folder) = self.config.selected_folder {
-            info!(context, "Expunge messages in \"{}\".", folder);
-
-            if let Some(ref mut session) = self.session {
-                match session.close().await {
-                    Ok(_) => {
-                        info!(context, "close/expunge succeeded");
-                    }
-                    Err(err) => {
-                        self.trigger_reconnect();
-                        return Err(Error::CloseExpungeFailed(err));
-                    }
-                }
-            } else {
-                return Err(Error::NoSession);
-            }
-        }
-        self.config.selected_folder = None;
-        self.config.selected_folder_needs_expunge = false;
-
-        Ok(())
-    }
-
     /// select a folder, possibly update uid_validity and, if needed,
     /// expunge the folder to remove delete-marked messages.
     pub(super) async fn select_folder<S: AsRef<str>>(
-        &mut self,
+        &self,
         context: &Context,
         folder: Option<S>,
     ) -> Result<()> {
-        if self.session.is_none() {
-            self.config.selected_folder = None;
-            self.config.selected_folder_needs_expunge = false;
+        if self.session.lock().await.is_none() {
+            let mut cfg = self.config.write().await;
+            cfg.selected_folder = None;
+            cfg.selected_folder_needs_expunge = false;
             self.trigger_reconnect();
             return Err(Error::NoSession);
         }
@@ -68,7 +41,7 @@ impl Imap {
         // if there is a new folder and the new folder is equal to the selected one, there's nothing to do.
         // if there is _no_ new folder, we continue as we might want to expunge below.
         if let Some(ref folder) = folder {
-            if let Some(ref selected_folder) = self.config.selected_folder {
+            if let Some(ref selected_folder) = self.config.read().await.selected_folder {
                 if folder.as_ref() == selected_folder {
                     return Ok(());
                 }
@@ -76,14 +49,33 @@ impl Imap {
         }
 
         // deselect existing folder, if needed (it's also done implicitly by SELECT, however, without EXPUNGE then)
-        let needs_expunge = { self.config.selected_folder_needs_expunge };
+        let needs_expunge = { self.config.read().await.selected_folder_needs_expunge };
         if needs_expunge {
-            self.close_folder(context).await?;
+            if let Some(ref folder) = self.config.read().await.selected_folder {
+                info!(context, "Expunge messages in \"{}\".", folder);
+
+                // A CLOSE-SELECT is considerably faster than an EXPUNGE-SELECT, see
+                // https://tools.ietf.org/html/rfc3501#section-6.4.2
+                if let Some(ref mut session) = &mut *self.session.lock().await {
+                    match session.close().await {
+                        Ok(_) => {
+                            info!(context, "close/expunge succeeded");
+                        }
+                        Err(err) => {
+                            self.trigger_reconnect();
+                            return Err(Error::CloseExpungeFailed(err));
+                        }
+                    }
+                } else {
+                    return Err(Error::NoSession);
+                }
+            }
+            self.config.write().await.selected_folder_needs_expunge = false;
         }
 
         // select new folder
         if let Some(ref folder) = folder {
-            if let Some(ref mut session) = &mut self.session {
+            if let Some(ref mut session) = &mut *self.session.lock().await {
                 let res = session.select(folder).await;
 
                 // https://tools.ietf.org/html/rfc3501#section-6.3.1
@@ -92,20 +84,21 @@ impl Imap {
 
                 match res {
                     Ok(mailbox) => {
-                        self.config.selected_folder = Some(folder.as_ref().to_string());
-                        self.config.selected_mailbox = Some(mailbox);
+                        let mut config = self.config.write().await;
+                        config.selected_folder = Some(folder.as_ref().to_string());
+                        config.selected_mailbox = Some(mailbox);
                         Ok(())
                     }
                     Err(async_imap::error::Error::ConnectionLost) => {
                         self.trigger_reconnect();
-                        self.config.selected_folder = None;
+                        self.config.write().await.selected_folder = None;
                         Err(Error::ConnectionLost)
                     }
                     Err(async_imap::error::Error::Validate(_)) => {
                         Err(Error::BadFolderName(folder.as_ref().to_string()))
                     }
                     Err(err) => {
-                        self.config.selected_folder = None;
+                        self.config.write().await.selected_folder = None;
                         self.trigger_reconnect();
                         Err(Error::Other(err.to_string()))
                     }

src/imap/session.rs

@@ -1,40 +0,0 @@
-use std::ops::{Deref, DerefMut};
-
-use async_imap::Session as ImapSession;
-use async_native_tls::TlsStream;
-use async_std::net::TcpStream;
-
-#[derive(Debug)]
-pub(crate) struct Session {
-    pub(super) inner: ImapSession<Box<dyn SessionStream>>,
-}
-
-pub(crate) trait SessionStream:
-    async_std::io::Read + async_std::io::Write + Unpin + Send + Sync + std::fmt::Debug
-{
-}
-
-impl SessionStream for TlsStream<Box<dyn SessionStream>> {}
-impl SessionStream for TlsStream<TcpStream> {}
-impl SessionStream for TcpStream {}
-
-impl Deref for Session {
-    type Target = ImapSession<Box<dyn SessionStream>>;
-
-    fn deref(&self) -> &Self::Target {
-        &self.inner
-    }
-}
-
-impl DerefMut for Session {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.inner
-    }
-}
-
-impl Session {
-    pub fn idle(self) -> async_imap::extensions::idle::Handle<Box<dyn SessionStream>> {
-        let Session { inner } = self;
-        inner.idle()
-    }
-}

src/imap_client.rs

@@ -0,0 +1,292 @@
+use async_imap::{
+    error::{Error as ImapError, Result as ImapResult},
+    extensions::idle::Handle as ImapIdleHandle,
+    types::{Capabilities, Fetch, Mailbox, Name},
+    Client as ImapClient, Session as ImapSession,
+};
+use async_native_tls::TlsStream;
+use async_std::net::{self, TcpStream};
+use async_std::prelude::*;
+
+use crate::login_param::{dc_build_tls, CertificateChecks};
+
+#[derive(Debug)]
+pub(crate) enum Client {
+    Secure(ImapClient<TlsStream<TcpStream>>),
+    Insecure(ImapClient<TcpStream>),
+}
+
+#[derive(Debug)]
+pub(crate) enum Session {
+    Secure(ImapSession<TlsStream<TcpStream>>),
+    Insecure(ImapSession<TcpStream>),
+}
+
+#[derive(Debug)]
+pub(crate) enum IdleHandle {
+    Secure(ImapIdleHandle<TlsStream<TcpStream>>),
+    Insecure(ImapIdleHandle<TcpStream>),
+}
+
+impl Client {
+    pub async fn connect_secure<A: net::ToSocketAddrs, S: AsRef<str>>(
+        addr: A,
+        domain: S,
+        certificate_checks: CertificateChecks,
+    ) -> ImapResult<Self> {
+        let stream = TcpStream::connect(addr).await?;
+        let tls = dc_build_tls(certificate_checks)?;
+        let tls_connector: async_native_tls::TlsConnector = tls.into();
+        let tls_stream = tls_connector.connect(domain.as_ref(), stream).await?;
+        let mut client = ImapClient::new(tls_stream);
+        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
+            client.debug = true;
+        }
+
+        let _greeting = client
+            .read_response()
+            .await
+            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
+
+        Ok(Client::Secure(client))
+    }
+
+    pub async fn connect_insecure<A: net::ToSocketAddrs>(addr: A) -> ImapResult<Self> {
+        let stream = TcpStream::connect(addr).await?;
+
+        let mut client = ImapClient::new(stream);
+        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
+            client.debug = true;
+        }
+        let _greeting = client
+            .read_response()
+            .await
+            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
+
+        Ok(Client::Insecure(client))
+    }
+
+    pub async fn secure<S: AsRef<str>>(
+        self,
+        domain: S,
+        certificate_checks: CertificateChecks,
+    ) -> ImapResult<Client> {
+        match self {
+            Client::Insecure(client) => {
+                let tls = dc_build_tls(certificate_checks)?;
+                let tls_stream = tls.into();
+
+                let client_sec = client.secure(domain, &tls_stream).await?;
+
+                Ok(Client::Secure(client_sec))
+            }
+            // Nothing to do
+            Client::Secure(_) => Ok(self),
+        }
+    }
+
+    pub async fn authenticate<A: async_imap::Authenticator, S: AsRef<str>>(
+        self,
+        auth_type: S,
+        authenticator: &A,
+    ) -> Result<Session, (ImapError, Client)> {
+        match self {
+            Client::Secure(i) => match i.authenticate(auth_type, authenticator).await {
+                Ok(session) => Ok(Session::Secure(session)),
+                Err((err, c)) => Err((err, Client::Secure(c))),
+            },
+            Client::Insecure(i) => match i.authenticate(auth_type, authenticator).await {
+                Ok(session) => Ok(Session::Insecure(session)),
+                Err((err, c)) => Err((err, Client::Insecure(c))),
+            },
+        }
+    }
+
+    pub async fn login<U: AsRef<str>, P: AsRef<str>>(
+        self,
+        username: U,
+        password: P,
+    ) -> Result<Session, (ImapError, Client)> {
+        match self {
+            Client::Secure(i) => match i.login(username, password).await {
+                Ok(session) => Ok(Session::Secure(session)),
+                Err((err, c)) => Err((err, Client::Secure(c))),
+            },
+            Client::Insecure(i) => match i.login(username, password).await {
+                Ok(session) => Ok(Session::Insecure(session)),
+                Err((err, c)) => Err((err, Client::Insecure(c))),
+            },
+        }
+    }
+}
+
+impl Session {
+    pub async fn capabilities(&mut self) -> ImapResult<Capabilities> {
+        let res = match self {
+            Session::Secure(i) => i.capabilities().await?,
+            Session::Insecure(i) => i.capabilities().await?,
+        };
+
+        Ok(res)
+    }
+
+    pub async fn list(
+        &mut self,
+        reference_name: Option<&str>,
+        mailbox_pattern: Option<&str>,
+    ) -> ImapResult<Vec<Name>> {
+        let res = match self {
+            Session::Secure(i) => {
+                i.list(reference_name, mailbox_pattern)
+                    .await?
+                    .collect::<ImapResult<_>>()
+                    .await?
+            }
+            Session::Insecure(i) => {
+                i.list(reference_name, mailbox_pattern)
+                    .await?
+                    .collect::<ImapResult<_>>()
+                    .await?
+            }
+        };
+        Ok(res)
+    }
+
+    pub async fn create<S: AsRef<str>>(&mut self, mailbox_name: S) -> ImapResult<()> {
+        match self {
+            Session::Secure(i) => i.create(mailbox_name).await?,
+            Session::Insecure(i) => i.create(mailbox_name).await?,
+        }
+        Ok(())
+    }
+
+    pub async fn subscribe<S: AsRef<str>>(&mut self, mailbox: S) -> ImapResult<()> {
+        match self {
+            Session::Secure(i) => i.subscribe(mailbox).await?,
+            Session::Insecure(i) => i.subscribe(mailbox).await?,
+        }
+        Ok(())
+    }
+
+    pub async fn close(&mut self) -> ImapResult<()> {
+        match self {
+            Session::Secure(i) => i.close().await?,
+            Session::Insecure(i) => i.close().await?,
+        }
+        Ok(())
+    }
+
+    pub async fn select<S: AsRef<str>>(&mut self, mailbox_name: S) -> ImapResult<Mailbox> {
+        let mbox = match self {
+            Session::Secure(i) => i.select(mailbox_name).await?,
+            Session::Insecure(i) => i.select(mailbox_name).await?,
+        };
+
+        Ok(mbox)
+    }
+
+    pub async fn fetch<S1, S2>(&mut self, sequence_set: S1, query: S2) -> ImapResult<Vec<Fetch>>
+    where
+        S1: AsRef<str>,
+        S2: AsRef<str>,
+    {
+        let res = match self {
+            Session::Secure(i) => {
+                i.fetch(sequence_set, query)
+                    .await?
+                    .collect::<ImapResult<_>>()
+                    .await?
+            }
+            Session::Insecure(i) => {
+                i.fetch(sequence_set, query)
+                    .await?
+                    .collect::<ImapResult<_>>()
+                    .await?
+            }
+        };
+        Ok(res)
+    }
+
+    pub async fn uid_fetch<S1, S2>(&mut self, uid_set: S1, query: S2) -> ImapResult<Vec<Fetch>>
+    where
+        S1: AsRef<str>,
+        S2: AsRef<str>,
+    {
+        let res = match self {
+            Session::Secure(i) => {
+                i.uid_fetch(uid_set, query)
+                    .await?
+                    .collect::<ImapResult<_>>()
+                    .await?
+            }
+            Session::Insecure(i) => {
+                i.uid_fetch(uid_set, query)
+                    .await?
+                    .collect::<ImapResult<_>>()
+                    .await?
+            }
+        };
+
+        Ok(res)
+    }
+
+    pub fn idle(self) -> IdleHandle {
+        match self {
+            Session::Secure(i) => {
+                let h = i.idle();
+                IdleHandle::Secure(h)
+            }
+            Session::Insecure(i) => {
+                let h = i.idle();
+                IdleHandle::Insecure(h)
+            }
+        }
+    }
+
+    pub async fn uid_store<S1, S2>(&mut self, uid_set: S1, query: S2) -> ImapResult<Vec<Fetch>>
+    where
+        S1: AsRef<str>,
+        S2: AsRef<str>,
+    {
+        let res = match self {
+            Session::Secure(i) => {
+                i.uid_store(uid_set, query)
+                    .await?
+                    .collect::<ImapResult<_>>()
+                    .await?
+            }
+            Session::Insecure(i) => {
+                i.uid_store(uid_set, query)
+                    .await?
+                    .collect::<ImapResult<_>>()
+                    .await?
+            }
+        };
+        Ok(res)
+    }
+
+    pub async fn uid_mv<S1: AsRef<str>, S2: AsRef<str>>(
+        &mut self,
+        uid_set: S1,
+        mailbox_name: S2,
+    ) -> ImapResult<()> {
+        match self {
+            Session::Secure(i) => i.uid_mv(uid_set, mailbox_name).await?,
+            Session::Insecure(i) => i.uid_mv(uid_set, mailbox_name).await?,
+        }
+        Ok(())
+    }
+
+    pub async fn uid_copy<S1: AsRef<str>, S2: AsRef<str>>(
+        &mut self,
+        uid_set: S1,
+        mailbox_name: S2,
+    ) -> ImapResult<()> {
+        match self {
+            Session::Secure(i) => i.uid_copy(uid_set, mailbox_name).await?,
+            Session::Insecure(i) => i.uid_copy(uid_set, mailbox_name).await?,
+        }
+
+        Ok(())
+    }
+}