re-enabling smtp_config_task - still fine

Previously empty autoconfig resulted in no servers being tried and no
error displayed.

.circleci/config.yml

@@ -143,7 +143,7 @@ jobs:
     steps:
       - checkout
       - run: ci_scripts/remote_tests_rust.sh
-    
+
   remote_tests_python:
     machine: true
     steps:
@@ -189,10 +189,9 @@ workflows:
               only: /.*/
 
       - remote_python_packaging:
-          requires: 
-             - remote_tests_python
-             - remote_tests_rust 
           filters:
+            branches:
+              only: master
             tags:
               only: /.*/
 
@@ -201,6 +200,8 @@ workflows:
              - remote_python_packaging
              - build_doxygen
           filters:
+            branches:
+              only: master
             tags:
               only: /.*/
       # - rustfmt:
@@ -212,6 +213,8 @@ workflows:
 
       - build_doxygen:
           filters:
+            branches:
+              only: master
             tags:
               only: /.*/
 

.github/workflows/ci.yml

@@ -0,0 +1,103 @@
+name: Rust CI
+
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+      - staging
+      - trying
+
+jobs:
+
+  fmt:
+    name: Rustfmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: 1.45.0
+          override: true
+      - run: rustup component add rustfmt
+      - uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
+
+  run_clippy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+            toolchain: 1.45.0
+            components: clippy
+            override: true
+      - uses: actions-rs/clippy-check@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          args: --workspace --tests --examples
+
+
+  build_and_test:
+    name: Build and test
+    runs-on: ${{ matrix.os }}
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      matrix:
+        # macOS disabled due to random failures related to caching
+        #os: [ubuntu-latest, windows-latest, macOS-latest]
+        os: [ubuntu-latest, windows-latest]
+        rust: [1.45.0]
+        experimental: [false]
+#        include:
+#          - os: ubuntu-latest
+#            rust: nightly
+#            experimental: true
+#          - os: windows-latest
+#            rust: nightly
+#            experimental: true
+#          - os: macOS-latest
+#            rust: nightly
+#            experimental: true
+
+    steps:
+    - uses: actions/checkout@master
+
+    - name: Install ${{ matrix.rust }}
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: ${{ matrix.rust }}
+        override: true
+
+    - name: Cache cargo registry
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/registry
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}
+
+    - name: Cache cargo index
+      uses: actions/cache@v2
+      with:
+        path: ~/.cargo/git
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}
+
+    - name: Cache cargo build
+      uses: actions/cache@v2
+      with:
+        path: target
+        key: ${{ matrix.os }}-${{ matrix.rust }}-cargo-build-target-${{ hashFiles('**/Cargo.toml') }}
+
+    - name: check
+      uses: actions-rs/cargo@v1
+      with:
+        command:  check
+        args: --all --bins --examples --tests --features repl
+
+    - name: tests
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --all

.github/workflows/code-quality.yml

@@ -1,48 +0,0 @@
-on: push
-name: Code Quality
-jobs:
-
-  check:
-    name: Check
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: nightly-2020-03-19
-          override: true
-      - uses: actions-rs/cargo@v1
-        with:
-          command: check
-          args: --workspace --examples --tests --all-features
-
-  fmt:
-    name: Rustfmt
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: nightly-2020-03-19
-          override: true
-      - run: rustup component add rustfmt
-      - uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: --all -- --check
-
-  run_clippy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v1
-      - uses: actions-rs/toolchain@v1
-        with:
-            toolchain: nightly-2020-03-19
-            components: clippy
-            override: true
-      - uses: actions-rs/clippy-check@v1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          args: --all-features

ASYNC-API-TODO.txt

@@ -1,54 +0,0 @@
-
-Delta Chat ASYNC  (friedel, bjoern, floris, friedel)
-
-- smtp fake-idle/load jobs gerade noch alle fuenf sekunden , sollte alle zehn minuten (oder gar nicht)
-
-APIs:
-    dc_context_new                 # opens the database
-    dc_open                             # FFI only
-      -> drop it and move parameters to dc_context_new()
-
-    dc_configure                      # note: dc_start_jobs() is NOT allowed to run concurrently
-    dc_imex NEVER goes through the job system
-    dc_imex import_backup needs to ensure dc_stop_jobs()
-
-    dc_start_io                # start smtp/imap and job handling subsystems
-    dc_stop_io                 # stop smtp/imap and job handling subsystems
-    dc_is_io_running           # return 1 if smtp/imap/jobs susbystem is running
-
-    dc_close                            # FFI only
-      -> can be dropped
-    dc_context_unref
-
-    for ios share-extension:
-    Int dc_direct_send()  -> try send out without going through jobs system, but queue a job in db if it needs to be retried on failure
-        0: message was sent
-        1: message failed to go out, is queued as a job to be retried later
-        2: message permanently failed?
-
-EVENT handling:
-    start a callback thread and call get_next_event() which is BLOCKING
-    it's fine to start this callback thread later, it will see all events.
-    Note that the core infinitely fills the internal queue if you never drain it.
-
-    FFI-get_next_event() returns NULL if the context is unrefed already?
-
-   sidenote: how python's callback thread does it currently:
-         CB-thread runs this while loop:
-         while not QUITFLAG:
-             ev = context.get_next_event( )
-             ...
-         So in order to shutdown properly one has to set QUITFLAG
-         before calling dc_stop_jobs() and dc_context_unref
-
-    event API:
-        get_data1_int
-        get_data2_int
-        get_data3_str
-
-
-- userdata likely only used for the callbacks, likely can be dropped, needs verification
-
-
-- iOS needs for the share app to call "try_send_smtp" wihtout a full dc_context_run and without going
-

CHANGELOG.md

@@ -1,5 +1,322 @@
 # Changelog 
 
+## 1.47.0
+
+- breaking change: `dc_update_device_chats()` removed;
+  this is now done automatically during configure
+  unless the new config-option `bot` is set #1957
+
+- breaking change: split `DC_EVENT_MSGS_NOTICED` off `DC_EVENT_MSGS_CHANGED`
+  and remove `dc_marknoticed_all_chats()` #1942 #1981
+
+- breaking change: remove unused starring options #1965
+
+- breaking change: `DC_CHAT_TYPE_VERIFIED_GROUP` replaced by
+  `dc_chat_is_protected()`; also single-chats may be protected now, this may
+  happen over the wire even if the UI do not offer an option for that #1968
+
+- breaking change: split quotes off message text,
+  UIs should use at least `dc_msg_get_quoted_text()` to show quotes now #1975
+
+- new api for quote handling: `dc_msg_set_quote()`, `dc_msg_get_quoted_text()`,
+  `dc_msg_get_quoted_msg()` #1975 #1984 #1985 #1987 #1989 #2004
+
+- require quorum to enable encryption #1946
+
+- speed up and clean up account creation #1912 #1927 #1960 #1961
+
+- configure now collects recent contacts and fetches last messages
+  unless disabled by `fetch_existing` config-option #1913 #2003
+
+- emit `DC_EVENT_CHAT_MODIFIED` on contact rename
+  and set contact-id on `DC_EVENT_CONTACTS_CHANGED` #1935 #1936 #1937
+
+- add `dc_set_chat_protection()`; the `protect` parameter in
+  `dc_create_group_chat()` will be removed in an upcoming release;
+  up to then, UIs using the "verified group" paradigm
+  should not use `dc_set_chat_protection()` #1968 #2014 #2001 #2012 #2007
+
+- remove unneeded `DC_STR_COUNT` #1991
+
+- mark all failed messages as failed when receiving an NDN #1993
+
+- fix import temporary directory usage #1929
+
+- fix forcing encryption for reset peers #1998
+
+- fix: do not allow to save drafts in non-writeable chats #1997
+
+- fix: do not show HTML if there is no content and there is an attachment #1988
+
+- fix recovering offline/lost connections, fixes background receive bug #1983
+
+- fix ordering of accounts returned by `dc_accounts_get_all()` #1909
+
+- fix whitespace for summaries #1938
+
+- fix: improve sentbox name guessing #1941
+
+- fix: avoid manual poll impl for accounts events #1944
+
+- fix encoding newlines in param as a preparation for storing quotes #1945
+
+- fix: internal and ffi error handling #1967 #1966 #1959 #1911 #1916 #1917 #1915
+
+- fix ci #1928 #1931 #1932 #1933 #1934 #1943
+
+- update provider-database #1940 #2005 #2006
+
+- update dependencies #1919 #1908 #1950 #1963 #1996 #2010 #2013
+
+
+## 1.46.0
+
+- breaking change: `dc_configure()` report errors in
+  `DC_EVENT_CONFIGURE_PROGRESS`: capturing error events is no longer working
+  #1886 #1905
+
+- breaking change: removed `DC_LP_{IMAP|SMTP}_SOCKET*` from `server_flags`;
+  added `mail_security` and `send_security` using `DC_SOCKET` enum #1835
+
+- parse multiple servers in Mozilla autoconfig #1860
+
+- try multiple servers for each protocol #1871
+
+- do IMAP and SMTP configuration in parallel #1891
+
+- configuration cleanup and speedup #1858 #1875 #1889 #1904 #1906
+
+- secure-join cleanup, testing, fixing #1876 #1877 #1887 #1888 #1896 #1899 #1900
+
+- do not reset peerstate on encrypted messages,
+  ignore reordered autocrypt headers #1885 #1890
+
+- always sort message replies after parent message #1852
+
+- add an index to significantly speed up `get_fresh_msg_cnt()` #1881
+
+- improve mimetype guessing for PDF and many other formats #1857 #1861
+
+- improve accepting invalid html #1851
+
+- improve tests, cleanup and ci #1850 #1856 #1859 #1861 #1884 #1894 #1895
+
+- tweak HELO command #1908
+
+- make `dc_accounts_get_all()` return accounts sorted #1909
+
+- fix KML coordinates precision used for location streaming #1872
+
+- fix cancelling import/export #1855
+
+
+## 1.45.0
+
+- add `dc_accounts_t` account manager object and related api functions #1784
+
+- add capability to import backups as .tar files,
+  which will become the default in a subsequent release #1749
+
+- try various server domains on configuration #1780 #1838
+
+- recognize .tgs files as stickers #1826
+
+- remove X-Mailer debug header #1819
+
+- improve guessing message types from extension #1818
+
+- fix showing unprotected subjects in encrypted messages #1822
+
+- fix threading in interaction with non-delta-clients #1843
+
+- fix handling if encryption degrades #1829
+
+- fix webrtc-servers names set by the user #1831
+
+- update provider database #1828
+
+- update async-imap to fix Oauth2 #1837
+
+- optimize jpeg assets with trimage #1840
+
+- add tests and documentations #1809 #1820
+
+
+## 1.44.0
+
+- fix peerstate issues #1800 #1805
+
+- fix a crash related to muted chats #1803
+
+- fix incorrect dimensions sometimes reported for images #1806
+
+- fixed `dc_chat_get_remaining_mute_duration` function #1807
+
+- handle empty tags (e.g. `<br/>`) in HTML mails #1810
+
+- always translate the message about disappearing messages timer change #1813
+
+- improve footer detection in plain text email #1812
+
+- update device chat icon to fix warnings in iOS logs #1802
+
+- fix deletion of multiple messages #1795
+
+
+## 1.43.0
+
+- improve using own jitsi-servers #1785
+
+- fix smtp-timeout tweaks for larger mails #1797
+
+- more bug fixes and updates #1794 #1792 #1789 #1787
+
+
+## 1.42.0
+
+- new qr-code type `DC_QR_WEBRTC` #1779
+
+- new `dc_chatlist_get_summary2()` api #1771
+
+- tweak smtp-timeout for larger mails #1782
+
+- optimize read-receipts #1765
+
+- Allow http scheme for DCACCOUNT URLs #1770
+
+- improve tests #1769
+
+- bug fixes #1766 #1772 #1773 #1775 #1776 #1777
+
+
+## 1.41.0
+
+- new apis to initiate video chats #1718 #1735
+
+- new apis `dc_msg_get_ephemeral_timer()`
+  and `dc_msg_get_ephemeral_timestamp()`
+
+- new api `dc_chatlist_get_summary2()` #1771
+
+- improve IMAP handling #1703 #1704
+
+- improve ephemeral messages #1696 #1705
+
+- mark location-messages as auto-generated #1715
+
+- multi-device avatar-sync #1716 #1717
+
+- improve python bindings #1732 #1733 #1738 #1769
+
+- Allow http scheme for DCACCOUNT urls #1770
+
+- more fixes #1702 #1706 #1707 #1710 #1719 #1721
+  #1723 #1734 #1740 #1744 #1748 #1760 #1766 #1773 #1765
+
+- refactorings #1712 #1714 #1757
+
+- update toolchains and dependencies #1726 #1736 #1737 #1742 #1743 #1746
+
+
+## 1.40.0
+
+- introduce ephemeral messages #1540 #1680 #1683 #1684 #1691 #1692
+
+- `DC_MSG_ID_DAYMARKER` gets timestamp attached #1677 #1685
+
+- improve idle #1690 #1688
+
+- fix message processing issues by sequential processing #1694
+
+- refactorings #1670 #1673
+
+
+## 1.39.0
+
+- fix handling of `mvbox_watch`, `sentbox_watch`, `inbox_watch` #1654 #1658
+
+- fix potential panics, update dependencies #1650 #1655
+
+
+## 1.38.0
+
+- fix sorting, esp. for multi-device
+
+
+## 1.37.0
+
+- improve ndn heuristics #1630
+
+- get oauth2 authorizer from provider-db #1641
+
+- removed linebreaks and spaces from generated qr-code #1631
+
+- more fixes #1633 #1635 #1636 #1637
+
+
+## 1.36.0
+
+- parse ndn (network delivery notification) reports
+  and report failed messages as such #1552 #1622 #1630
+
+- add oauth2 support for gsuite domains #1626
+
+- read image orientation from exif before recoding #1619
+
+- improve logging #1593 #1598
+
+- improve python and bot bindings #1583 #1609
+
+- improve imap logout #1595
+
+- fix sorting #1600 #1604
+
+- fix qr code generation #1631
+
+- update rustcrypto releases #1603
+
+- refactorings #1617
+
+
+## 1.35.0
+
+- enable strict-tls from a new provider-db setting #1587
+
+- new subject 'Message from USER' for one-to-one chats #1395
+
+- recode images #1563
+
+- improve reconnect handling #1549 #1580
+
+- improve importing addresses #1544
+
+- improve configure and folder detection #1539 #1548
+
+- improve test suite #1559 #1564 #1580 #1581 #1582 #1584 #1588:
+
+- fix ad-hoc groups #1566
+
+- preventions against being marked as spam #1575
+
+- refactorings #1542 #1569
+
+
+## 1.34.0
+
+- new api for io, thread and event handling #1356,
+  see the example atop of `deltachat.h` to get an overview
+
+- LOTS of speed improvements due to async processing #1356
+
+- enable WAL mode for sqlite #1492
+
+- process incoming messages in bulk #1527
+
+- improve finding out the sent-folder #1488
+
+- several bug fixes
+
+
 ## 1.33.0
 
 - let `dc_set_muted()` also mute one-to-one chats #1470
@@ -497,4 +814,3 @@
 For a full list of changes, please see our closed Pull Requests: 
 
 https://github.com/deltachat/deltachat-core-rust/pulls?q=is%3Apr+is%3Aclosed
-

Cargo.toml

@@ -1,77 +1,83 @@
 [package]
 name = "deltachat"
-version = "1.33.0"
+version = "1.47.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
 
 [profile.release]
-# lto = true
+lto = true
 
 [dependencies]
 deltachat_derive = { path = "./deltachat_derive" }
 
 libc = "0.2.51"
-pgp = { version = "0.5.1", default-features = false } 
+pgp = { version = "0.7.0", default-features = false } 
 hex = "0.4.0"
-sha2 = "0.8.0"
+sha2 = "0.9.0"
 rand = "0.7.0"
 smallvec = "1.0.0"
-surf = { version = "2.0.0-alpha.2", default-features = false, features = ["h1-client"] }
+surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
 num-derive = "0.3.0"
 num-traits = "0.2.6"
-async-smtp = { version = "0.3" }
+async-smtp = { git = "https://github.com/async-email/async-smtp", rev="2275fd8d13e39b2c58d6605c786ff06ff9e05708" }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
-async-imap = "0.3.0"
+async-imap = "0.4.0"
 async-native-tls = { version = "0.3.3" }
-async-std = { version = "1.6.0", features = ["unstable"] }
-base64 = "0.11"
+async-std = { version = "1.6.4", features = ["unstable"] }
+base64 = "0.12"
 charset = "0.1"
 percent-encoding = "2.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 chrono = "0.4.6"
 indexmap = "1.3.0"
-lazy_static = "1.4.0"
+kamadak-exif = "0.5"
+once_cell = "1.4.1"
 regex = "1.1.6"
-rusqlite = { version = "0.22", features = ["bundled"] }
-r2d2_sqlite = "0.15.0"
+rusqlite = { version = "0.24", features = ["bundled"] }
+r2d2_sqlite = "0.17.0"
 r2d2 = "0.8.5"
-strum = "0.16.0"
-strum_macros = "0.16.0"
+strum = "0.19.0"
+strum_macros = "0.19.0"
 backtrace = "0.3.33"
 byteorder = "1.3.1"
-itertools = "0.8.0"
-image-meta = "0.1.0"
-quick-xml = "0.17.1"
+itertools = "0.9.0"
+quick-xml = "0.18.1"
 escaper = "0.1.0"
 bitflags = "1.1.0"
-debug_stub_derive = "0.3.0"
-sanitize-filename = "0.2.1"
+sanitize-filename = "0.3.0"
 stop-token = { version = "0.1.1", features = ["unstable"] }
-mailparse = "0.12.0"
+mailparse = "0.13.0"
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
 native-tls = "0.2.3"
-image = { version = "0.22.4", default-features=false, features = ["gif_codec", "jpeg", "ico", "png_codec", "pnm", "webp", "bmp"] }
+image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
 futures = "0.3.4"
 thiserror = "1.0.14"
 anyhow = "1.0.28"
 async-trait = "0.1.31"
 url = "2.1.1"
+async-std-resolver = "0.19.5"
+async-tar = "0.3.0"
+uuid = { version = "0.8", features = ["serde", "v4"] }
 
-pretty_env_logger = { version = "0.3.1", optional = true }
+pretty_env_logger = { version = "0.4.0", optional = true }
 log = {version = "0.4.8", optional = true }
 rustyline = { version = "4.1.0", optional = true }
 ansi_term = { version = "0.12.1", optional = true }
+dirs = { version = "3.0.1", optional=true }
+toml = "0.5.6"
 
 
 [dev-dependencies]
 tempfile = "3.0"
 pretty_assertions = "0.6.1"
-pretty_env_logger = "0.3.0"
-proptest = "0.9.4"
-async-std = { version = "1.6.0", features = ["unstable", "attributes"] }
+pretty_env_logger = "0.4.0"
+proptest = "0.10"
+async-std = { version = "1.6.4", features = ["unstable", "attributes"] }
+futures-lite = "1.7.0"
+criterion = "0.3"
 
 [workspace]
 members = [
@@ -82,6 +88,7 @@ members = [
 [[example]]
 name = "simple"
 path = "examples/simple.rs"
+required-features = ["repl"]
 
 [[example]]
 name = "repl"
@@ -89,10 +96,14 @@ path = "examples/repl/main.rs"
 required-features = ["repl"]
 
 
+[[bench]]
+name = "create_account"
+harness = false
+
 [features]
 default = []
 internals = []
-repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term"]
+repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs"]
 vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored"]
 nightly = ["pgp/nightly"]
 

README.md

@@ -95,7 +95,8 @@ $ cargo build -p deltachat_ffi --release
 
 - `DCC_MIME_DEBUG`: if set outgoing and incoming message will be printed 
 
-
+- `RUST_LOG=info,async_imap=trace,async_smtp=trace`: enable IMAP and
+SMTP tracing in addition to info messages.
 
 ### Expensive tests
 
@@ -123,6 +124,7 @@ Language bindings are available for:
 - [Node.js](https://www.npmjs.com/package/deltachat-node)
 - [Python](https://py.delta.chat)
 - [Go](https://github.com/hugot/go-deltachat/)
+- [Free Pascal](https://github.com/deltachat/deltachat-fp/)
 - **Java** and **Swift** (contained in the Android/iOS repos)
 
 The following "frontend" projects make use of the Rust-library
@@ -132,4 +134,5 @@ or its language bindings:
 - [iOS](https://github.com/deltachat/deltachat-ios)
 - [Desktop](https://github.com/deltachat/deltachat-desktop)
 - [Pidgin](https://code.ur.gs/lupine/purple-plugin-delta/)
+- [Telepathy](https://code.ur.gs/lupine/telepathy-padfoot/)
 - several **Bots**

appveyor.yml

@@ -1,19 +0,0 @@
-environment:
-  matrix:
-    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
-
-install:
-  - appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
-  - rustup-init -yv --default-toolchain nightly-2020-03-19
-  - set PATH=%PATH%;%USERPROFILE%\.cargo\bin
-  - rustc -vV
-  - cargo -vV
-
-build: false
-
-test_script:
-  - cargo test --release --all
-
-cache:
-  - target
-  - C:\Users\appveyor\.cargo\registry

benches/create_account.rs

@@ -0,0 +1,26 @@
+use async_std::path::PathBuf;
+use async_std::task::block_on;
+use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use deltachat::accounts::Accounts;
+use tempfile::tempdir;
+
+async fn create_accounts(n: u32) {
+    let dir = tempdir().unwrap();
+    let p: PathBuf = dir.path().join("accounts").into();
+
+    let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+
+    for expected_id in 2..n {
+        let id = accounts.add_account().await.unwrap();
+        assert_eq!(id, expected_id);
+    }
+}
+
+fn criterion_benchmark(c: &mut Criterion) {
+    c.bench_function("create 1 account", |b| {
+        b.iter(|| block_on(async { create_accounts(black_box(1)).await }))
+    });
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

ci_scripts/docker-coredeps/Dockerfile

@@ -16,6 +16,6 @@ ENV PIP_DISABLE_PIP_VERSION_CHECK 1
 ADD deps/build_python.sh /builder/build_python.sh
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_python.sh && cd .. && rm -r tmp1
 
-# Install Rust nightly
+# Install Rust
 ADD deps/build_rust.sh /builder/build_rust.sh
 RUN mkdir tmp1 && cd tmp1 && bash /builder/build_rust.sh && cd .. && rm -r tmp1

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -3,9 +3,9 @@
 set -e -x
 
 # Install Rust
-curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.43.1-x86_64-unknown-linux-gnu -y
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain 1.45.0-x86_64-unknown-linux-gnu -y
 export PATH=/root/.cargo/bin:$PATH
 rustc --version
 
 # remove some 300-400 MB that we don't need for automated builds
-rm -rf /root/.rustup/toolchains/1.43.1-x86_64-unknown-linux-gnu/share
+rm -rf /root/.rustup/toolchains/1.45.0-x86_64-unknown-linux-gnu/share

ci_scripts/remote_python_packaging.sh

@@ -4,8 +4,6 @@ export BRANCH=${CIRCLE_BRANCH:?branch to build}
 export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
 export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
-# we construct the BUILDDIR such that we can easily share the
-# CARGO_TARGET_DIR between runs ("..")
 export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
 
 echo "--- Copying files to $SSHTARGET:$BUILDDIR"
@@ -30,6 +28,7 @@ set +x
 
 ssh $SSHTARGET bash -c "cat >$BUILDDIR/exec_docker_run" <<_HERE
     set +x -e
+    shopt -s huponexit
     cd $BUILDDIR
     export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
     export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL

ci_scripts/remote_tests_python.sh

@@ -4,8 +4,6 @@ export BRANCH=${CIRCLE_BRANCH:?branch to build}
 export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
 export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
-# we construct the BUILDDIR such that we can easily share the
-# CARGO_TARGET_DIR between runs ("..")
 export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
 
 echo "--- Copying files to $SSHTARGET:$BUILDDIR"
@@ -24,10 +22,12 @@ echo "--- Running $CIRCLE_JOB remotely"
 
 ssh $SSHTARGET <<_HERE
     set +x -e
+
+    # make sure all processes exit when ssh dies
+    shopt -s huponexit
+
+    export RUSTC_WRAPPER=\`which sccache\`
     cd $BUILDDIR
-    # let's share the target dir with our last run on this branch/job-type
-    # cargo will make sure to block/unblock us properly 
-    export CARGO_TARGET_DIR=\`pwd\`/../target
     export TARGET=release
     export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
     export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL

ci_scripts/remote_tests_rust.sh

@@ -4,8 +4,6 @@ export BRANCH=${CIRCLE_BRANCH:?branch to build}
 export REPONAME=${CIRCLE_PROJECT_REPONAME:?repository name}
 export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
 
-# we construct the BUILDDIR such that we can easily share the
-# CARGO_TARGET_DIR between runs ("..")
 export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
 
 set -e
@@ -20,10 +18,10 @@ echo "--- Running $CIRCLE_JOB remotely"
 
 ssh $SSHTARGET <<_HERE
     set +x -e
+    # make sure all processes exit when ssh dies
+    shopt -s huponexit
+    export RUSTC_WRAPPER=\`which sccache\`
     cd $BUILDDIR
-    # let's share the target dir with our last run on this branch/job-type
-    # cargo will make sure to block/unblock us properly 
-    export CARGO_TARGET_DIR=\`pwd\`/../target
     export TARGET=x86_64-unknown-linux-gnu
     export RUSTC_WRAPPER=sccache
 

ci_scripts/run-python-test.sh

@@ -4,6 +4,7 @@
 # and tox/pytest. 
 
 set -e -x
+shopt -s huponexit
 
 # for core-building and python install step
 export DCC_RS_TARGET=debug

ci_scripts/run-rust-test.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 set -ex
+shopt -s huponexit
 
 #export RUST_TEST_THREADS=1
 export RUST_BACKTRACE=1

ci_scripts/run_all.sh

@@ -15,6 +15,7 @@ cargo build --release -p deltachat_ffi
 
 # Statically link against libdeltachat.a.
 export DCC_RS_DEV=$(pwd)
+export DCC_RS_TARGET=release
 
 # Configure access to a base python and to several python interpreters
 # needed by tox below.

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.33.0"
+version = "1.47.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
@@ -23,6 +23,7 @@ serde_json = "1.0"
 async-std = "1.6.0"
 anyhow = "1.0.28"
 thiserror = "1.0.14"
+rand = "0.7.3"
 
 [features]
 default = ["vendored"]

deltachat-ffi/Doxyfile

@@ -236,12 +236,6 @@ TAB_SIZE               = 4
 
 ALIASES                =
 
-# This tag can be used to specify a number of word-keyword mappings (TCL only).
-# A mapping has the form "name=value". For example adding "class=itcl::class"
-# will allow you to use the command class in the itcl::class meaning.
-
-TCL_SUBST              =
-
 # Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources
 # only. Doxygen will then generate output that is more tailored for C. For
 # instance, some of the names that are used will be different. The list of all

deltachat-ffi/build.rs

@@ -19,10 +19,10 @@ fn main() {
         include_str!("deltachat.pc.in"),
         name = "deltachat",
         description = env::var("CARGO_PKG_DESCRIPTION").unwrap(),
-        url = env::var("CARGO_PKG_HOMEPAGE").unwrap_or("".to_string()),
+        url = env::var("CARGO_PKG_HOMEPAGE").unwrap_or_else(|_| "".to_string()),
         version = env::var("CARGO_PKG_VERSION").unwrap(),
         libs_priv = libs_priv,
-        prefix = env::var("PREFIX").unwrap_or("/usr/local".to_string()),
+        prefix = env::var("PREFIX").unwrap_or_else(|_| "/usr/local".to_string()),
     );
 
     fs::create_dir_all(target_path.join("pkgconfig")).unwrap();

deltachat-ffi/src/dc_array.rs

@@ -1,46 +1,56 @@
+use crate::chat::ChatItem;
+use crate::constants::{DC_MSG_ID_DAYMARKER, DC_MSG_ID_MARKER1};
 use crate::location::Location;
+use crate::message::MsgId;
 
 /* * the structure behind dc_array_t */
 #[derive(Debug, Clone)]
 pub enum dc_array_t {
+    MsgIds(Vec<MsgId>),
+    Chat(Vec<ChatItem>),
     Locations(Vec<Location>),
     Uint(Vec<u32>),
 }
 
 impl dc_array_t {
-    pub fn new(capacity: usize) -> Self {
-        dc_array_t::Uint(Vec::with_capacity(capacity))
-    }
-
-    /// Constructs a new, empty `dc_array_t` holding locations with specified `capacity`.
-    pub fn new_locations(capacity: usize) -> Self {
-        dc_array_t::Locations(Vec::with_capacity(capacity))
-    }
-
-    pub fn add_id(&mut self, item: u32) {
-        if let Self::Uint(array) = self {
-            array.push(item);
-        } else {
-            panic!("Attempt to add id to array of other type");
-        }
-    }
-
-    pub fn add_location(&mut self, location: Location) {
-        if let Self::Locations(array) = self {
-            array.push(location)
-        } else {
-            panic!("Attempt to add a location to array of other type");
-        }
-    }
-
-    pub fn get_id(&self, index: usize) -> u32 {
+    pub(crate) fn get_id(&self, index: usize) -> u32 {
         match self {
+            Self::MsgIds(array) => array[index].to_u32(),
+            Self::Chat(array) => match array[index] {
+                ChatItem::Message { msg_id } => msg_id.to_u32(),
+                ChatItem::Marker1 => DC_MSG_ID_MARKER1,
+                ChatItem::DayMarker { .. } => DC_MSG_ID_DAYMARKER,
+            },
             Self::Locations(array) => array[index].location_id,
-            Self::Uint(array) => array[index] as u32,
+            Self::Uint(array) => array[index],
         }
     }
 
-    pub fn get_location(&self, index: usize) -> &Location {
+    pub(crate) fn get_timestamp(&self, index: usize) -> Option<i64> {
+        match self {
+            Self::MsgIds(_) => None,
+            Self::Chat(array) => array.get(index).and_then(|item| match item {
+                ChatItem::Message { .. } => None,
+                ChatItem::Marker1 { .. } => None,
+                ChatItem::DayMarker { timestamp } => Some(*timestamp),
+            }),
+            Self::Locations(array) => array.get(index).map(|location| location.timestamp),
+            Self::Uint(_) => None,
+        }
+    }
+
+    pub(crate) fn get_marker(&self, index: usize) -> Option<&str> {
+        match self {
+            Self::MsgIds(_) => None,
+            Self::Chat(_) => None,
+            Self::Locations(array) => array
+                .get(index)
+                .and_then(|location| location.marker.as_deref()),
+            Self::Uint(_) => None,
+        }
+    }
+
+    pub(crate) fn get_location(&self, index: usize) -> &Location {
         if let Self::Locations(array) = self {
             &array[index]
         } else {
@@ -48,55 +58,18 @@ impl dc_array_t {
         }
     }
 
-    pub fn is_empty(&self) -> bool {
-        match self {
-            Self::Locations(array) => array.is_empty(),
-            Self::Uint(array) => array.is_empty(),
-        }
-    }
-
     /// Returns the number of elements in the array.
-    pub fn len(&self) -> usize {
+    pub(crate) fn len(&self) -> usize {
         match self {
+            Self::MsgIds(array) => array.len(),
+            Self::Chat(array) => array.len(),
             Self::Locations(array) => array.len(),
             Self::Uint(array) => array.len(),
         }
     }
 
-    pub fn clear(&mut self) {
-        match self {
-            Self::Locations(array) => array.clear(),
-            Self::Uint(array) => array.clear(),
-        }
-    }
-
-    pub fn search_id(&self, needle: u32) -> Option<usize> {
-        if let Self::Uint(array) = self {
-            for (i, &u) in array.iter().enumerate() {
-                if u == needle {
-                    return Some(i);
-                }
-            }
-            None
-        } else {
-            panic!("Attempt to search for id in array of other type");
-        }
-    }
-
-    pub fn sort_ids(&mut self) {
-        if let dc_array_t::Uint(v) = self {
-            v.sort();
-        } else {
-            panic!("Attempt to sort array of something other than uints");
-        }
-    }
-
-    pub fn as_ptr(&self) -> *const u32 {
-        if let dc_array_t::Uint(v) = self {
-            v.as_ptr()
-        } else {
-            panic!("Attempt to convert array of something other than uints to raw");
-        }
+    pub(crate) fn search_id(&self, needle: u32) -> Option<usize> {
+        (0..self.len()).find(|i| self.get_id(*i) == needle)
     }
 }
 
@@ -106,6 +79,18 @@ impl From<Vec<u32>> for dc_array_t {
     }
 }
 
+impl From<Vec<MsgId>> for dc_array_t {
+    fn from(array: Vec<MsgId>) -> Self {
+        dc_array_t::MsgIds(array)
+    }
+}
+
+impl From<Vec<ChatItem>> for dc_array_t {
+    fn from(array: Vec<ChatItem>) -> Self {
+        dc_array_t::Chat(array)
+    }
+}
+
 impl From<Vec<Location>> for dc_array_t {
     fn from(array: Vec<Location>) -> Self {
         dc_array_t::Locations(array)
@@ -118,12 +103,11 @@ mod tests {
 
     #[test]
     fn test_dc_array() {
-        let mut arr = dc_array_t::new(7);
-        assert!(arr.is_empty());
+        let arr: dc_array_t = Vec::<u32>::new().into();
+        assert!(arr.len() == 0);
 
-        for i in 0..1000 {
-            arr.add_id(i + 2);
-        }
+        let ids: Vec<u32> = (2..1002).collect();
+        let arr: dc_array_t = ids.into();
 
         assert_eq!(arr.len(), 1000);
 
@@ -131,31 +115,15 @@ mod tests {
             assert_eq!(arr.get_id(i), (i + 2) as u32);
         }
 
-        arr.clear();
-
-        assert!(arr.is_empty());
-
-        arr.add_id(13);
-        arr.add_id(7);
-        arr.add_id(666);
-        arr.add_id(0);
-        arr.add_id(5000);
-
-        arr.sort_ids();
-
-        assert_eq!(arr.get_id(0), 0);
-        assert_eq!(arr.get_id(1), 7);
-        assert_eq!(arr.get_id(2), 13);
-        assert_eq!(arr.get_id(3), 666);
+        assert_eq!(arr.search_id(10), Some(8));
+        assert_eq!(arr.search_id(1), None);
     }
 
     #[test]
     #[should_panic]
     fn test_dc_array_out_of_bounds() {
-        let mut arr = dc_array_t::new(7);
-        for i in 0..1000 {
-            arr.add_id(i + 2);
-        }
+        let ids: Vec<u32> = (2..1002).collect();
+        let arr: dc_array_t = ids.into();
         arr.get_id(1000);
     }
 }

deltachat-ffi/src/string.rs

@@ -105,8 +105,9 @@ impl<T: AsRef<std::ffi::OsStr>> OsStrExt for T {
     #[cfg(not(target_os = "windows"))]
     fn to_c_string(&self) -> Result<CString, CStringError> {
         use std::os::unix::ffi::OsStrExt;
-        CString::new(self.as_ref().as_bytes()).map_err(|err| match err {
-            std::ffi::NulError { .. } => CStringError::InteriorNullByte,
+        CString::new(self.as_ref().as_bytes()).map_err(|err| {
+            let std::ffi::NulError { .. } = err;
+            CStringError::InteriorNullByte
         })
     }
 
@@ -122,8 +123,9 @@ fn os_str_to_c_string_unicode(
     os_str: &dyn AsRef<std::ffi::OsStr>,
 ) -> Result<CString, CStringError> {
     match os_str.as_ref().to_str() {
-        Some(val) => CString::new(val.as_bytes()).map_err(|err| match err {
-            std::ffi::NulError { .. } => CStringError::InteriorNullByte,
+        Some(val) => CString::new(val.as_bytes()).map_err(|err| {
+            let std::ffi::NulError { .. } = err;
+            CStringError::InteriorNullByte
         }),
         None => Err(CStringError::NotUnicode),
     }

draft/msgwork_new_imap_jobs.rst

@@ -0,0 +1,66 @@
+
+simplify/streamline mark-seen/delete/move/send-mdn job handling
+---------------------------------------------------------------
+
+Idea: Introduce a new "msgwork" sql table that looks very
+much like the jobs table but has a primary key "msgid"
+and no job id and no foreign-id anymore. This opens up
+bulk-processing by looking at the whole table and combining
+flag-setting to reduce imap-roundtrips and select-folder calls.
+
+Concretely, these IMAP jobs:
+
+    DeleteMsgOnImap
+    MarkseenMsgOnImap
+    MoveMsg
+
+Would be replaced by a few per-message columns in the new msgwork table:
+
+- needs_mark_seen: (bool) message shall be marked as seen on imap
+- needs_to_move: (bool) message should be moved to mvbox_folder
+- deletion_time: (target_time or 0) message shall be deleted at specified time
+- needs_send_mdn: (bool) MDN shall be sent
+
+The various places that currently add the (replaced) jobs
+would now add/modify the respective message record in the message-work table.
+
+Looking at a single message-work entry conceptually looks like this::
+
+    if msg.server_uid==0:
+        return RetryLater  # nothing can be done without server_uid
+
+    if msg.deletion_time > current_time:
+        imap.mark_delete(msg)  # might trigger early exit with a RetryLater/Failed
+        clear(needs_deletion)
+        clear(mark_seen)
+
+    if needs_mark_seen:
+        imap.mark_seen(msg)    # might trigger early exit with a RetryLater/Failed
+        clear(needs_mark_seen)
+
+    if needs_send_mdn:
+        schedule_smtp_send_mdn(msg)
+        clear(needs_send_mdn)
+
+    if any_flag_set():
+        retrylater
+    # remove msgwork entry from table
+
+
+Notes/Questions:
+
+- it's unclear how much we need per-message retry-time tracking/backoff
+
+- drafting bulk processing algo is useful before
+  going for the implementation, i.e. including select_folder calls etc.
+
+- maybe it's better to not have bools for the flags but
+
+  0 (no change)
+  1 (set the imap flag)
+  2 (clear the imap flag)
+
+  and design such that we can cover all imap flags.
+
+- It might not be neccessary to keep needs_send_mdn state in this table
+  if this can be decided rather when we succeed with mark_seen/mark_delete.

examples/repl/cmdline.rs

@@ -1,8 +1,10 @@
+extern crate dirs;
+
 use std::str::FromStr;
 
 use anyhow::{bail, ensure};
 use async_std::path::Path;
-use deltachat::chat::{self, Chat, ChatId, ChatVisibility};
+use deltachat::chat::{self, Chat, ChatId, ChatItem, ChatVisibility, ProtectionStatus};
 use deltachat::chatlist::*;
 use deltachat::constants::*;
 use deltachat::contact::*;
@@ -17,7 +19,7 @@ use deltachat::message::{self, Message, MessageState, MsgId};
 use deltachat::peerstate::*;
 use deltachat::qr::*;
 use deltachat::sql;
-use deltachat::Event;
+use deltachat::EventType;
 use deltachat::{config, provider};
 
 /// Reset database tables.
@@ -86,7 +88,7 @@ async fn reset_tables(context: &Context, bits: i32) {
         println!("(8) Rest but server config reset.");
     }
 
-    context.emit_event(Event::MsgsChanged {
+    context.emit_event(EventType::MsgsChanged {
         chat_id: ChatId::new(0),
         msg_id: MsgId::new(0),
     });
@@ -157,7 +159,7 @@ async fn poke_spec(context: &Context, spec: Option<&str>) -> bool {
     }
     println!("Import: {} items read from \"{}\".", read_cnt, &real_spec);
     if read_cnt > 0 {
-        context.emit_event(Event::MsgsChanged {
+        context.emit_event(EventType::MsgsChanged {
             chat_id: ChatId::new(0),
             msg_id: MsgId::new(0),
         });
@@ -191,7 +193,6 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         &contact_name,
         contact_id,
         msgtext.unwrap_or_default(),
-        if msg.is_starred() { "★" } else { "" },
         if msg.get_from_id() == 1 as libc::c_uint {
             ""
         } else if msg.get_state() == MessageState::InSeen {
@@ -202,6 +203,15 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
             "[FRESH]"
         },
         if msg.is_info() { "[INFO]" } else { "" },
+        if msg.get_viewtype() == Viewtype::VideochatInvitation {
+            format!(
+                "[VIDEOCHAT-INVITATION: {}, type={}]",
+                msg.get_videochat_url().unwrap_or_default(),
+                msg.get_videochat_type().unwrap_or_default()
+            )
+        } else {
+            "".to_string()
+        },
         if msg.is_forwarded() {
             "[FORWARDED]"
         } else {
@@ -215,7 +225,7 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
 async fn log_msglist(context: &Context, msglist: &[MsgId]) -> Result<(), Error> {
     let mut lines_out = 0;
     for &msg_id in msglist {
-        if msg_id.is_daymarker() {
+        if msg_id == MsgId::new(DC_MSG_ID_DAYMARKER) {
             println!(
                 "--------------------------------------------------------------------------------"
             );
@@ -275,7 +285,9 @@ async fn log_contactlist(context: &Context, contacts: &[u32]) {
                     "addr unset"
                 }
             );
-            let peerstate = Peerstate::from_addr(context, &addr).await;
+            let peerstate = Peerstate::from_addr(context, &addr)
+                .await
+                .expect("peerstate error");
             if peerstate.is_some() && contact_id != 1 as libc::c_uint {
                 line2 = format!(
                     ", prefer-encrypt={}",
@@ -345,7 +357,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  createchat <contact-id>\n\
                  createchatbymsg <msg-id>\n\
                  creategroup <name>\n\
-                 createverified <name>\n\
+                 createprotected <name>\n\
                  addmember <contact-id>\n\
                  removemember <contact-id>\n\
                  groupname <name>\n\
@@ -359,6 +371,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  send-garbage\n\
                  sendimage <file> [<text>]\n\
                  sendfile <file> [<text>]\n\
+                 videochat\n\
                  draft [<text>]\n\
                  devicemsg <text>\n\
                  listmedia\n\
@@ -366,6 +379,8 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  unarchive <chat-id>\n\
                  pin <chat-id>\n\
                  unpin <chat-id>\n\
+                 protect <chat-id>\n\
+                 unprotect <chat-id>\n\
                  delchat <chat-id>\n\
                  ===========================Message commands==\n\
                  listmsgs <query>\n\
@@ -373,8 +388,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  listfresh\n\
                  forward <msg-id> <chat-id>\n\
                  markseen <msg-id>\n\
-                 star <msg-id>\n\
-                 unstar <msg-id>\n\
                  delmsg <msg-id>\n\
                  ===========================Contact commands==\n\
                  listcontacts [<query>]\n\
@@ -391,7 +404,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  event <event-id to test>\n\
                  fileinfo <file>\n\
                  estimatedeletion <seconds>\n\
-                 emptyserver <flags> (1=MVBOX 2=INBOX)\n\
                  clear -- clear screen\n\
                  exit or quit\n\
                  ============================================="
@@ -430,17 +442,21 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             has_backup(&context, blobdir).await?;
         }
         "export-backup" => {
-            imex(&context, ImexMode::ExportBackup, Some(blobdir)).await?;
+            let dir = dirs::home_dir().unwrap_or_default();
+            imex(&context, ImexMode::ExportBackup, Some(&dir)).await?;
+            println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-backup" => {
             ensure!(!arg1.is_empty(), "Argument <backup-file> missing.");
             imex(&context, ImexMode::ImportBackup, Some(arg1)).await?;
         }
         "export-keys" => {
-            imex(&context, ImexMode::ExportSelfKeys, Some(blobdir)).await?;
+            let dir = dirs::home_dir().unwrap_or_default();
+            imex(&context, ImexMode::ExportSelfKeys, Some(&dir)).await?;
+            println!("Exported to {}.", dir.to_string_lossy());
         }
         "import-keys" => {
-            imex(&context, ImexMode::ImportSelfKeys, Some(blobdir)).await?;
+            imex(&context, ImexMode::ImportSelfKeys, Some(arg1)).await?;
         }
         "export-setup" => {
             let setup_code = create_setup_code(&context);
@@ -488,6 +504,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "listchats" | "listarchived" | "chats" => {
             let listflags = if arg0 == "listarchived" { 0x01 } else { 0 };
+            let time_start = std::time::SystemTime::now();
             let chatlist = Chatlist::try_load(
                 &context,
                 listflags,
@@ -495,6 +512,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 None,
             )
             .await?;
+            let time_needed = std::time::SystemTime::now()
+                .duration_since(time_start)
+                .unwrap_or_default();
 
             let cnt = chatlist.len();
             if cnt > 0 {
@@ -505,7 +525,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 for i in (0..cnt).rev() {
                     let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)).await?;
                     println!(
-                        "{}#{}: {} [{} fresh] {}",
+                        "{}#{}: {} [{} fresh] {}{}",
                         chat_prefix(&chat),
                         chat.get_id(),
                         chat.get_name(),
@@ -515,6 +535,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                             ChatVisibility::Archived => "📦",
                             ChatVisibility::Pinned => "📌",
                         },
+                        if chat.is_protected() { "🛡️" } else { "" },
                     );
                     let lot = chatlist.get_summary(&context, i, Some(&chat)).await;
                     let statestr = if chat.visibility == ChatVisibility::Archived {
@@ -553,6 +574,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 println!("Location streaming enabled.");
             }
             println!("{} chats", cnt);
+            println!("{:?} to create this list", time_needed);
         }
         "chat" => {
             if sel_chat.is_none() && arg1.is_empty() {
@@ -569,6 +591,15 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let sel_chat = sel_chat.as_ref().unwrap();
 
             let msglist = chat::get_chat_msgs(&context, sel_chat.get_id(), 0x1, None).await;
+            let msglist: Vec<MsgId> = msglist
+                .into_iter()
+                .map(|x| match x {
+                    ChatItem::Message { msg_id } => msg_id,
+                    ChatItem::Marker1 => MsgId::new(DC_MSG_ID_MARKER1),
+                    ChatItem::DayMarker { .. } => MsgId::new(DC_MSG_ID_DAYMARKER),
+                })
+                .collect();
+
             let members = chat::get_chat_contacts(&context, sel_chat.id).await;
             let subtitle = if sel_chat.is_device_talk() {
                 "device-talk".to_string()
@@ -579,7 +610,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 format!("{} member(s)", members.len())
             };
             println!(
-                "{}#{}: {} [{}]{}{}",
+                "{}#{}: {} [{}]{}{} {}",
                 chat_prefix(sel_chat),
                 sel_chat.get_id(),
                 sel_chat.get_name(),
@@ -596,6 +627,11 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     },
                     _ => "".to_string(),
                 },
+                if sel_chat.is_protected() {
+                    "🛡️"
+                } else {
+                    ""
+                },
             );
             log_msglist(&context, &msglist).await?;
             if let Some(draft) = sel_chat.get_id().get_draft(&context).await? {
@@ -626,15 +662,16 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "creategroup" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
             let chat_id =
-                chat::create_group_chat(&context, VerifiedStatus::Unverified, arg1).await?;
+                chat::create_group_chat(&context, ProtectionStatus::Unprotected, arg1).await?;
 
             println!("Group#{} created successfully.", chat_id);
         }
-        "createverified" => {
+        "createprotected" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
-            let chat_id = chat::create_group_chat(&context, VerifiedStatus::Verified, arg1).await?;
+            let chat_id =
+                chat::create_group_chat(&context, ProtectionStatus::Protected, arg1).await?;
 
-            println!("VerifiedGroup#{} created successfully.", chat_id);
+            println!("Group#{} created and protected successfully.", chat_id);
         }
         "addmember" => {
             ensure!(sel_chat.is_some(), "No chat selected");
@@ -794,6 +831,10 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             }
             chat::send_msg(&context, sel_chat.as_ref().unwrap().get_id(), &mut msg).await?;
         }
+        "videochat" => {
+            ensure!(sel_chat.is_some(), "No chat selected.");
+            chat::send_videochat_invitation(&context, sel_chat.as_ref().unwrap().get_id()).await?;
+        }
         "listmsgs" => {
             ensure!(!arg1.is_empty(), "Argument <query> missing.");
 
@@ -840,9 +881,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             msg.set_text(Some(arg1.to_string()));
             chat::add_device_msg(&context, None, Some(&mut msg)).await?;
         }
-        "updatedevicechats" => {
-            context.update_device_chats().await?;
-        }
         "listmedia" => {
             ensure!(sel_chat.is_some(), "No chat selected.");
 
@@ -874,7 +912,21 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                         "archive" => ChatVisibility::Archived,
                         "unarchive" | "unpin" => ChatVisibility::Normal,
                         "pin" => ChatVisibility::Pinned,
-                        _ => panic!("Unexpected command (This should never happen)"),
+                        _ => unreachable!("arg0={:?}", arg0),
+                    },
+                )
+                .await?;
+        }
+        "protect" | "unprotect" => {
+            ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
+            let chat_id = ChatId::new(arg1.parse()?);
+            chat_id
+                .set_protection(
+                    &context,
+                    match arg0 {
+                        "protect" => ProtectionStatus::Protected,
+                        "unprotect" => ProtectionStatus::Unprotected,
+                        _ => unreachable!("arg0={:?}", arg0),
                     },
                 )
                 .await?;
@@ -913,12 +965,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             msg_ids[0] = MsgId::new(arg1.parse()?);
             message::markseen_msgs(&context, msg_ids).await;
         }
-        "star" | "unstar" => {
-            ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let mut msg_ids = vec![MsgId::new(0); 1];
-            msg_ids[0] = MsgId::new(arg1.parse()?);
-            message::star_msgs(&context, msg_ids, arg0 == "star").await;
-        }
         "delmsg" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let mut ids = [MsgId::new(0); 1];
@@ -1029,7 +1075,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         // "event" => {
         //     ensure!(!arg1.is_empty(), "Argument <id> missing.");
         //     let event = arg1.parse()?;
-        //     let event = Event::from_u32(event).ok_or(format_err!("Event::from_u32({})", event))?;
+        //     let event = EventType::from_u32(event).ok_or(format_err!("EventType::from_u32({})", event))?;
         //     let r = context.emit_event(event, 0 as libc::uintptr_t, 0 as libc::uintptr_t);
         //     println!(
         //         "Sending event {:?}({}), received value {}.",
@@ -1056,11 +1102,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 seconds, device_cnt, server_cnt
             );
         }
-        "emptyserver" => {
-            ensure!(!arg1.is_empty(), "Argument <flags> missing");
-
-            message::dc_empty_server(&context, arg1.parse()?).await;
-        }
         "" => (),
         _ => bail!("Unknown command: \"{}\" type ? for help.", arg0),
     }

examples/repl/main.rs

@@ -19,7 +19,7 @@ use deltachat::config;
 use deltachat::context::*;
 use deltachat::oauth2::*;
 use deltachat::securejoin::*;
-use deltachat::Event;
+use deltachat::EventType;
 use log::{error, info, warn};
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
 use rustyline::config::OutputStreamType;
@@ -34,35 +34,35 @@ mod cmdline;
 use self::cmdline::*;
 
 /// Event Handler
-fn receive_event(event: Event) {
+fn receive_event(event: EventType) {
     let yellow = Color::Yellow.normal();
     match event {
-        Event::Info(msg) => {
+        EventType::Info(msg) => {
             /* do not show the event as this would fill the screen */
             info!("{}", msg);
         }
-        Event::SmtpConnected(msg) => {
+        EventType::SmtpConnected(msg) => {
             info!("[SMTP_CONNECTED] {}", msg);
         }
-        Event::ImapConnected(msg) => {
+        EventType::ImapConnected(msg) => {
             info!("[IMAP_CONNECTED] {}", msg);
         }
-        Event::SmtpMessageSent(msg) => {
+        EventType::SmtpMessageSent(msg) => {
             info!("[SMTP_MESSAGE_SENT] {}", msg);
         }
-        Event::Warning(msg) => {
+        EventType::Warning(msg) => {
             warn!("{}", msg);
         }
-        Event::Error(msg) => {
+        EventType::Error(msg) => {
             error!("{}", msg);
         }
-        Event::ErrorNetwork(msg) => {
+        EventType::ErrorNetwork(msg) => {
             error!("[NETWORK] msg={}", msg);
         }
-        Event::ErrorSelfNotInGroup(msg) => {
+        EventType::ErrorSelfNotInGroup(msg) => {
             error!("[SELF_NOT_IN_GROUP] {}", msg);
         }
-        Event::MsgsChanged { chat_id, msg_id } => {
+        EventType::MsgsChanged { chat_id, msg_id } => {
             info!(
                 "{}",
                 yellow.paint(format!(
@@ -71,34 +71,44 @@ fn receive_event(event: Event) {
                 ))
             );
         }
-        Event::ContactsChanged(_) => {
+        EventType::ContactsChanged(_) => {
             info!("{}", yellow.paint("Received CONTACTS_CHANGED()"));
         }
-        Event::LocationChanged(contact) => {
+        EventType::LocationChanged(contact) => {
             info!(
                 "{}",
                 yellow.paint(format!("Received LOCATION_CHANGED(contact={:?})", contact))
             );
         }
-        Event::ConfigureProgress(progress) => {
-            info!(
-                "{}",
-                yellow.paint(format!("Received CONFIGURE_PROGRESS({} ‰)", progress))
-            );
+        EventType::ConfigureProgress { progress, comment } => {
+            if let Some(comment) = comment {
+                info!(
+                    "{}",
+                    yellow.paint(format!(
+                        "Received CONFIGURE_PROGRESS({} ‰, {})",
+                        progress, comment
+                    ))
+                );
+            } else {
+                info!(
+                    "{}",
+                    yellow.paint(format!("Received CONFIGURE_PROGRESS({} ‰)", progress))
+                );
+            }
         }
-        Event::ImexProgress(progress) => {
+        EventType::ImexProgress(progress) => {
             info!(
                 "{}",
                 yellow.paint(format!("Received IMEX_PROGRESS({} ‰)", progress))
             );
         }
-        Event::ImexFileWritten(file) => {
+        EventType::ImexFileWritten(file) => {
             info!(
                 "{}",
                 yellow.paint(format!("Received IMEX_FILE_WRITTEN({})", file.display()))
             );
         }
-        Event::ChatModified(chat) => {
+        EventType::ChatModified(chat) => {
             info!(
                 "{}",
                 yellow.paint(format!("Received CHAT_MODIFIED({})", chat))
@@ -146,10 +156,8 @@ const IMEX_COMMANDS: [&str; 12] = [
     "stop",
 ];
 
-const DB_COMMANDS: [&str; 11] = [
+const DB_COMMANDS: [&str; 9] = [
     "info",
-    "open",
-    "close",
     "set",
     "get",
     "oauth2",
@@ -160,7 +168,7 @@ const DB_COMMANDS: [&str; 11] = [
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 26] = [
+const CHAT_COMMANDS: [&str; 27] = [
     "listchats",
     "listarchived",
     "chat",
@@ -180,6 +188,7 @@ const CHAT_COMMANDS: [&str; 26] = [
     "send",
     "sendimage",
     "sendfile",
+    "videochat",
     "draft",
     "listmedia",
     "archive",
@@ -273,12 +282,12 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         println!("Error: Bad arguments, expected [db-name].");
         bail!("No db-name specified");
     }
-    let context = Context::new("CLI".into(), Path::new(&args[1]).to_path_buf()).await?;
+    let context = Context::new("CLI".into(), Path::new(&args[1]).to_path_buf(), 0).await?;
 
     let events = context.get_event_emitter();
     async_std::task::spawn(async move {
         while let Some(event) = events.recv().await {
-            receive_event(event);
+            receive_event(event.typ);
         }
     });
 
@@ -290,48 +299,59 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         .edit_mode(EditMode::Emacs)
         .output_stream(OutputStreamType::Stdout)
         .build();
-    let h = DcHelper {
-        completer: FilenameCompleter::new(),
-        highlighter: MatchingBracketHighlighter::new(),
-        hinter: HistoryHinter {},
-    };
-    let mut rl = Editor::with_config(config);
-    rl.set_helper(Some(h));
-    rl.bind_sequence(KeyPress::Meta('N'), Cmd::HistorySearchForward);
-    rl.bind_sequence(KeyPress::Meta('P'), Cmd::HistorySearchBackward);
-    if rl.load_history(".dc-history.txt").is_err() {
-        println!("No previous history.");
-    }
-
     let mut selected_chat = ChatId::default();
+    let (reader_s, reader_r) = async_std::sync::channel(100);
+    let input_loop = async_std::task::spawn_blocking(move || {
+        let h = DcHelper {
+            completer: FilenameCompleter::new(),
+            highlighter: MatchingBracketHighlighter::new(),
+            hinter: HistoryHinter {},
+        };
+        let mut rl = Editor::with_config(config);
+        rl.set_helper(Some(h));
+        rl.bind_sequence(KeyPress::Meta('N'), Cmd::HistorySearchForward);
+        rl.bind_sequence(KeyPress::Meta('P'), Cmd::HistorySearchBackward);
+        if rl.load_history(".dc-history.txt").is_err() {
+            println!("No previous history.");
+        }
 
-    loop {
-        let p = "> ";
-        let readline = rl.readline(&p);
-        match readline {
-            Ok(line) => {
-                // TODO: ignore "set mail_pw"
-                rl.add_history_entry(line.as_str());
-                match handle_cmd(line.trim(), context.clone(), &mut selected_chat).await {
-                    Ok(ExitResult::Continue) => {}
-                    Ok(ExitResult::Exit) => break,
-                    Err(err) => println!("Error: {}", err),
+        loop {
+            let p = "> ";
+            let readline = rl.readline(&p);
+
+            match readline {
+                Ok(line) => {
+                    // TODO: ignore "set mail_pw"
+                    rl.add_history_entry(line.as_str());
+                    async_std::task::block_on(reader_s.send(line));
+                }
+                Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => {
+                    println!("Exiting...");
+                    drop(reader_s);
+                    break;
+                }
+                Err(err) => {
+                    println!("Error: {}", err);
+                    drop(reader_s);
+                    break;
                 }
             }
-            Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => {
-                println!("Exiting...");
-                context.stop_io().await;
-                break;
-            }
-            Err(err) => {
-                println!("Error: {}", err);
-                break;
-            }
+        }
+
+        rl.save_history(".dc-history.txt")?;
+        println!("history saved");
+        Ok::<_, Error>(())
+    });
+
+    while let Ok(line) = reader_r.recv().await {
+        match handle_cmd(line.trim(), context.clone(), &mut selected_chat).await {
+            Ok(ExitResult::Continue) => {}
+            Ok(ExitResult::Exit) => break,
+            Err(err) => println!("Error: {}", err),
         }
     }
-
-    rl.save_history(".dc-history.txt")?;
-    println!("history saved");
+    context.stop_io().await;
+    input_loop.await?;
 
     Ok(())
 }
@@ -400,7 +420,7 @@ async fn handle_cmd(
         "joinqr" => {
             ctx.start_io().await;
             if !arg0.is_empty() {
-                dc_join_securejoin(&ctx, arg1).await;
+                dc_join_securejoin(&ctx, arg1).await?;
             }
         }
         "exit" | "quit" => return Ok(ExitResult::Exit),

examples/simple.rs

@@ -1,6 +1,3 @@
-extern crate deltachat;
-
-use std::time;
 use tempfile::tempdir;
 
 use deltachat::chat;
@@ -8,44 +5,51 @@ use deltachat::chatlist::*;
 use deltachat::config;
 use deltachat::contact::*;
 use deltachat::context::*;
-use deltachat::Event;
-
-fn cb(event: Event) {
-    print!("[{:?}]", event);
+use deltachat::message::Message;
+use deltachat::EventType;
 
+fn cb(event: EventType) {
     match event {
-        Event::ConfigureProgress(progress) => {
-            println!("  progress: {}", progress);
+        EventType::ConfigureProgress { progress, .. } => {
+            log::info!("progress: {}", progress);
         }
-        Event::Info(msg) | Event::Warning(msg) | Event::Error(msg) | Event::ErrorNetwork(msg) => {
-            println!("  {}", msg);
+        EventType::Info(msg) => {
+            log::info!("{}", msg);
         }
-        _ => {
-            println!();
+        EventType::Warning(msg) => {
+            log::warn!("{}", msg);
+        }
+        EventType::Error(msg) | EventType::ErrorNetwork(msg) => {
+            log::error!("{}", msg);
+        }
+        event => {
+            log::info!("{:?}", event);
         }
     }
 }
 
+/// Run with `RUST_LOG=simple=info cargo run --release --example simple --features repl -- email pw`.
 #[async_std::main]
 async fn main() {
+    pretty_env_logger::try_init_timed().ok();
+
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
-    println!("creating database {:?}", dbfile);
-    let ctx = Context::new("FakeOs".into(), dbfile.into())
+    log::info!("creating database {:?}", dbfile);
+    let ctx = Context::new("FakeOs".into(), dbfile.into(), 0)
         .await
         .expect("Failed to create context");
     let info = ctx.get_info().await;
-    let duration = time::Duration::from_millis(4000);
-    println!("info: {:#?}", info);
+    log::info!("info: {:#?}", info);
 
     let events = ctx.get_event_emitter();
     let events_spawn = async_std::task::spawn(async move {
         while let Some(event) = events.recv().await {
-            cb(event);
+            cb(event.typ);
         }
     });
 
-    println!("configuring");
+    log::info!("configuring");
     let args = std::env::args().collect::<Vec<String>>();
     assert_eq!(args.len(), 3, "requires email password");
     let email = args[1].clone();
@@ -59,35 +63,38 @@ async fn main() {
 
     ctx.configure().await.unwrap();
 
-    println!("------ RUN ------");
-    ctx.clone().start_io().await;
-    println!("--- SENDING A MESSAGE ---");
+    log::info!("------ RUN ------");
+    ctx.start_io().await;
+    log::info!("--- SENDING A MESSAGE ---");
 
     let contact_id = Contact::create(&ctx, "dignifiedquire", "dignifiedquire@gmail.com")
         .await
         .unwrap();
     let chat_id = chat::create_by_contact_id(&ctx, contact_id).await.unwrap();
 
-    for i in 0..2 {
+    for i in 0..1 {
+        log::info!("sending message {}", i);
         chat::send_text_msg(&ctx, chat_id, format!("Hi, here is my {}nth message!", i))
             .await
             .unwrap();
     }
 
-    println!("fetching chats..");
+    // wait for the message to be sent out
+    async_std::task::sleep(std::time::Duration::from_secs(1)).await;
+
+    log::info!("fetching chats..");
     let chats = Chatlist::try_load(&ctx, 0, None, None).await.unwrap();
 
     for i in 0..chats.len() {
-        let summary = chats.get_summary(&ctx, 0, None).await;
-        let text1 = summary.get_text1();
-        let text2 = summary.get_text2();
-        println!("chat: {} - {:?} - {:?}", i, text1, text2,);
+        let msg = Message::load_from_db(&ctx, chats.get_msg_id(i).unwrap())
+            .await
+            .unwrap();
+        log::info!("[{}] msg: {:?}", i, msg);
     }
 
-    async_std::task::sleep(duration).await;
-
-    println!("stopping");
+    log::info!("stopping");
     ctx.stop_io().await;
-    println!("closing");
+    log::info!("closing");
+    drop(ctx);
     events_spawn.await;
 }

python/CHANGELOG

@@ -1,6 +1,22 @@
-0.900.0 (DRAFT)
+1.44.0
+------
+
+- fix Chat.get_mute_duration()
+
+1.40.1
 ---------------
 
+- emit "ac_member_removed" event (with 'actor' being the removed contact) 
+  for when a user leaves a group.
+
+- fix create_contact(addr) when addr is the self-contact. 
+
+
+1.40.0
+---------------
+
+- uses latest 1.40+ Delta Chat core
+
 - refactored internals to use plugin-approach 
 
 - introduced PerAccount and Global hooks that plugins can implement 
@@ -10,6 +26,7 @@
 - introduced two documented examples for an echo and a group-membership
   tracking plugin. 
 
+
 0.800.0
 -------
 

python/README.rst

@@ -7,76 +7,14 @@ which implements IMAP/SMTP/MIME/PGP e-mail standards and offers
 a low-level Chat/Contact/Message API to user interfaces and bots.
 
 
-Installing bindings from source (Updated: 20-Jan-2020)
-=========================================================
-
-Install Rust and Cargo first.  Deltachat needs a specific nightly
-version, the easiest is probably to first install Rust stable from
-rustup and then use this to install the correct nightly version.
-
-Bootstrap Rust and Cargo by using rustup::
-
-   curl https://sh.rustup.rs -sSf | sh
-
-Then GIT clone the deltachat-core-rust repo and get the actual
-rust- and cargo-toolchain needed by deltachat::
-
-   git clone https://github.com/deltachat/deltachat-core-rust
-   cd deltachat-core-rust
-   rustup show
-
-To install the Delta Chat Python bindings make sure you have Python3 installed.
-E.g. on Debian-based systems `apt install python3 python3-pip
-python3-venv` should give you a usable python installation.
-
-Ensure you are in the deltachat-core-rust/python directory, create the
-virtual environment and activate it in your shell::
-
-   cd python
-   python3 -m venv venv  # or: virtualenv venv
-   source venv/bin/activate
-
-You should now be able to build the python bindings using the supplied script::
-
-   ./install_python_bindings.py
-
-The installation might take a while, depending on your machine.
-The bindings will be installed in release mode but with debug symbols.
-The release mode is currently necessary because some tests generate RSA keys
-which is prohibitively slow in non-release mode.
-
-After successful binding installation you can install a few more
-Python packages before running the tests::
-
-    python -m pip install pytest pytest-timeout pytest-rerunfailures requests
-    pytest -v tests
-
-
-running "live" tests with temporary accounts
----------------------------------------------
-
-If you want to run "liveconfig" functional tests you can set
-``DCC_NEW_TMP_EMAIL`` to:
-
-- a particular https-url that you can ask for from the delta
-  chat devs. This is implemented on the server side via
-  the [mailadm](https://github.com/deltachat/mailadm) command line tool.
-
-- or the path of a file that contains two lines, each describing
-  via "addr=... mail_pw=..." a test account login that will
-  be used for the live tests.
-
-With ``DCC_NEW_TMP_EMAIL`` set pytest invocations will use real
-e-mail accounts and run through all functional "liveconfig" tests.
-
-
 Installing pre-built packages (Linux-only)
 ========================================================
 
 If you have a Linux system you may try to install the ``deltachat`` binary "wheel" packages
-without any "build-from-source" steps.
+without any "build-from-source" steps. Otherwise you need to `compile the Delta Chat bindings
+yourself <sourceinstall>`_.
 
-We suggest to `Install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
+We recommend to first `install virtualenv <https://virtualenv.pypa.io/en/stable/installation/>`_,
 then create a fresh Python virtual environment and activate it in your shell::
 
         virtualenv venv  # or: python -m venv
@@ -103,6 +41,78 @@ To verify it worked::
     `in contact with us <https://delta.chat/en/contribute>`_.
 
 
+Running tests
+=============
+
+After successful binding installation you can install a few more
+Python packages before running the tests::
+
+    python -m pip install pytest pytest-xdist pytest-timeout pytest-rerunfailures requests
+    pytest -v tests
+
+This will run all "offline" tests and skip all functional
+end-to-end tests that require accounts on real e-mail servers.
+
+.. _livetests:
+
+running "live" tests with temporary accounts
+---------------------------------------------
+
+If you want to run live functional tests you can set ``DCC_NEW_TMP_EMAIL``::
+
+    export DCC_NEW_TMP_EMAIL=https://testrun.org/new_email?t=1h_4w4r8h7y9nmcdsy
+
+With this, pytest runs create ephemeral e-mail accounts on the http://testrun.org server.
+These accounts exists for one 1hour and then are removed completely.
+One hour is enough to invoke pytest and run all offline and online tests:
+
+    pytest
+
+    # or if you have installed pytest-xdist for parallel test execution
+    pytest -n6
+
+Each test run creates new accounts.
+
+
+.. _sourceinstall:
+
+Installing bindings from source (Updated: July 2020)
+=========================================================
+
+Install Rust and Cargo first.
+The easiest is probably to use `rustup <https://rustup.rs/>`_.
+
+Bootstrap Rust and Cargo by using rustup::
+
+   curl https://sh.rustup.rs -sSf | sh
+
+Then clone the deltachat-core-rust repo::
+
+   git clone https://github.com/deltachat/deltachat-core-rust
+   cd deltachat-core-rust
+
+To install the Delta Chat Python bindings make sure you have Python3 installed.
+E.g. on Debian-based systems `apt install python3 python3-pip
+python3-venv` should give you a usable python installation.
+
+Ensure you are in the deltachat-core-rust/python directory, create the
+virtual environment and activate it in your shell::
+
+   cd python
+   python3 -m venv venv  # or: virtualenv venv
+   source venv/bin/activate
+
+You should now be able to build the python bindings using the supplied script::
+
+   python install_python_bindings.py
+
+The core compilation and bindings building might take a while,
+depending on the speed of your machine.
+The bindings will be installed in release mode but with debug symbols.
+The release mode is currently necessary because some tests generate RSA keys
+which is prohibitively slow in non-release mode.
+
+
 Code examples
 =============
 

python/doc/_templates/globaltoc.html

@@ -9,8 +9,7 @@
 </ul>
 <b>external links:</b>
 <ul>
-  <li><a href="https://github.com/deltachat/deltachat-core">github repository</a></li>
-  <!-- <li><a href="https://lists.codespeak.net/postorius/lists/muacrypt.lists.codespeak.net">Mailing list</></li> <-->
+  <li><a href="https://github.com/deltachat/deltachat-core-rust">github repository</a></li>
   <li><a href="https://pypi.python.org/pypi/deltachat">pypi: deltachat</a></li>
 </ul>
 

python/examples/echo_and_quit.py

@@ -12,7 +12,7 @@ class EchoPlugin:
             message.account.shutdown()
         else:
             # unconditionally accept the chat
-            message.accept_sender_contact()
+            message.create_chat()
             addr = message.get_sender_contact().addr
             if message.is_system_message():
                 message.chat.send_text("echoing system message from {}:\n{}".format(addr, message))

python/examples/group_tracking.py

@@ -12,7 +12,7 @@ class GroupTrackingPlugin:
             message.account.shutdown()
         else:
             # unconditionally accept the chat
-            message.accept_sender_contact()
+            message.create_chat()
             addr = message.get_sender_contact().addr
             text = message.text
             message.chat.send_text("echoing from {}:\n{}".format(addr, text))
@@ -32,16 +32,16 @@ class GroupTrackingPlugin:
             print("chat member: {}".format(member.addr))
 
     @account_hookimpl
-    def ac_member_added(self, chat, contact, message):
+    def ac_member_added(self, chat, contact, actor, message):
         print("ac_member_added {} to chat {} from {}".format(
-            contact.addr, chat.id, message.get_sender_contact().addr))
+            contact.addr, chat.id, actor or message.get_sender_contact().addr))
         for member in chat.get_contacts():
             print("chat member: {}".format(member.addr))
 
     @account_hookimpl
-    def ac_member_removed(self, chat, contact, message):
+    def ac_member_removed(self, chat, contact, actor, message):
         print("ac_member_removed {} from chat {} by {}".format(
-            contact.addr, chat.id, message.get_sender_contact().addr))
+            contact.addr, chat.id, actor or message.get_sender_contact().addr))
 
 
 def main(argv=None):

python/examples/test_examples.py

@@ -26,15 +26,15 @@ def test_echo_quit_plugin(acfactory, lp):
 
     lp.sec("sending a message to the bot")
     bot_contact = ac1.create_contact(botproc.addr)
-    ch1 = ac1.create_chat_by_contact(bot_contact)
-    ch1.send_text("hello")
+    bot_chat = bot_contact.create_chat()
+    bot_chat.send_text("hello")
 
-    lp.sec("waiting for the bot-reply to arrive")
+    lp.sec("waiting for the reply message from the bot to arrive")
     reply = ac1._evtracker.wait_next_incoming_message()
+    assert reply.chat == bot_chat
     assert "hello" in reply.text
-    assert reply.chat == ch1
     lp.sec("send quit sequence")
-    ch1.send_text("/quit")
+    bot_chat.send_text("/quit")
     botproc.wait()
 
 
@@ -47,8 +47,8 @@ def test_group_tracking_plugin(acfactory, lp):
     botproc.fnmatch_lines("""
         *ac_configure_completed*
     """)
-    ac1.add_account_plugin(FFIEventLogger(ac1, "ac1"))
-    ac2.add_account_plugin(FFIEventLogger(ac2, "ac2"))
+    ac1.add_account_plugin(FFIEventLogger(ac1))
+    ac2.add_account_plugin(FFIEventLogger(ac2))
 
     lp.sec("creating bot test group with bot")
     bot_contact = ac1.create_contact(botproc.addr)
@@ -69,11 +69,11 @@ def test_group_tracking_plugin(acfactory, lp):
 
     lp.sec("now looking at what the bot received")
     botproc.fnmatch_lines("""
-        *ac_member_added {}*
-    """.format(contact3.addr))
+        *ac_member_added {}*from*{}*
+    """.format(contact3.addr, ac1.get_config("addr")))
 
     lp.sec("contact successfully added, now removing")
     ch.remove_contact(contact3)
     botproc.fnmatch_lines("""
-        *ac_member_removed {}*
-    """.format(contact3.addr))
+        *ac_member_removed {}*from*{}*
+    """.format(contact3.addr, ac1.get_config("addr")))

python/install_python_bindings.py

@@ -17,8 +17,12 @@ if __name__ == "__main__":
         os.environ["DCC_RS_DEV"] = dn
 
     cmd = ["cargo", "build", "-p", "deltachat_ffi"]
+
     if target == 'release':
+        extra = " -C lto=on -C embed-bitcode=yes"
+        os.environ["RUSTFLAGS"] = os.environ.get("RUSTFLAGS", "") + extra
         cmd.append("--release")
+
     print("running:", " ".join(cmd))
     subprocess.check_call(cmd)
     subprocess.check_call("rm -rf build/ src/deltachat/*.so" , shell=True)

python/setup.py

@@ -18,7 +18,7 @@ def main():
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy'],
+        install_requires=['cffi>=1.0.0', 'pluggy', 'imapclient'],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],

python/src/deltachat/__init__.py

@@ -60,7 +60,8 @@ def run_cmdline(argv=None, account_plugins=None):
     ac = Account(args.db)
 
     if args.show_ffi:
-        log = events.FFIEventLogger(ac, "bot")
+        ac.set_config("displayname", "bot")
+        log = events.FFIEventLogger(ac)
         ac.add_account_plugin(log)
 
     for plugin in account_plugins or []:
@@ -76,8 +77,9 @@ def run_cmdline(argv=None, account_plugins=None):
         ac.set_config("mvbox_move", "0")
         ac.set_config("mvbox_watch", "0")
         ac.set_config("sentbox_watch", "0")
-        ac.configure()
-        ac.wait_configure_finish()
+        ac.set_config("bot", "1")
+        configtracker = ac.configure()
+        configtracker.wait_finish()
 
     # start IO threads and configure if neccessary
     ac.start_io()

python/src/deltachat/_build.py

@@ -1,65 +1,64 @@
 import distutils.ccompiler
 import distutils.log
 import distutils.sysconfig
-import tempfile
-import platform
 import os
-import cffi
+import platform
+import re
 import shutil
-from os.path import dirname as dn
+import subprocess
+import tempfile
+import textwrap
+import types
 from os.path import abspath
+from os.path import dirname as dn
+
+import cffi
 
 
-def ffibuilder():
-    projdir = os.environ.get('DCC_RS_DEV')
-    if not projdir:
-        p = dn(dn(dn(dn(abspath(__file__)))))
-        projdir = os.environ["DCC_RS_DEV"] = p
-    target = os.environ.get('DCC_RS_TARGET', 'release')
-    if projdir:
-        if platform.system() == 'Darwin':
-            libs = ['resolv', 'dl']
-            extra_link_args = [
-                    '-framework', 'CoreFoundation',
-                    '-framework', 'CoreServices',
-                    '-framework', 'Security',
-            ]
-        elif platform.system() == 'Linux':
-            libs = ['rt', 'dl', 'm']
-            extra_link_args = []
-        else:
-            raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
-        target_dir = os.environ.get("CARGO_TARGET_DIR")
-        if target_dir is None:
-            target_dir = os.path.join(projdir, 'target')
-        objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
-        assert os.path.exists(objs[0]), objs
-        incs = [os.path.join(projdir, 'deltachat-ffi')]
+def local_build_flags(projdir, target):
+    """Construct build flags for building against a checkout.
+
+    :param projdir: The root directory of the deltachat-core-rust project.
+    :param target: The rust build target, `debug` or `release`.
+    """
+    flags = types.SimpleNamespace()
+    if platform.system() == 'Darwin':
+        flags.libs = ['resolv', 'dl']
+        flags.extra_link_args = [
+                '-framework', 'CoreFoundation',
+                '-framework', 'CoreServices',
+                '-framework', 'Security',
+        ]
+    elif platform.system() == 'Linux':
+        flags.libs = ['rt', 'dl', 'm']
+        flags.extra_link_args = []
     else:
-        libs = ['deltachat']
-        objs = []
-        incs = []
-        extra_link_args = []
-    builder = cffi.FFI()
-    builder.set_source(
-        'deltachat.capi',
-        """
-            #include <deltachat.h>
-            int dc_event_has_string_data(int e)
-            {
-                return DC_EVENT_DATA2_IS_STRING(e);
-            }
-        """,
-        include_dirs=incs,
-        libraries=libs,
-        extra_objects=objs,
-        extra_link_args=extra_link_args,
-    )
-    builder.cdef("""
-        typedef int... time_t;
-        void free(void *ptr);
-        extern int dc_event_has_string_data(int);
-    """)
+        raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
+    target_dir = os.environ.get("CARGO_TARGET_DIR")
+    if target_dir is None:
+        target_dir = os.path.join(projdir, 'target')
+    flags.objs = [os.path.join(target_dir, target, 'libdeltachat.a')]
+    assert os.path.exists(flags.objs[0]), flags.objs
+    flags.incs = [os.path.join(projdir, 'deltachat-ffi')]
+    return flags
+
+
+def system_build_flags():
+    """Construct build flags for building against an installed libdeltachat."""
+    flags = types.SimpleNamespace()
+    flags.libs = ['deltachat']
+    flags.objs = []
+    flags.incs = []
+    flags.extra_link_args = []
+
+
+def extract_functions(flags):
+    """Extract the function definitions from deltachat.h.
+
+    This creates a .h file with a single `#include <deltachat.h>` line
+    in it.  It then runs the C preprocessor to create an output file
+    which contains all function definitions found in `deltachat.h`.
+    """
     distutils.log.set_verbosity(distutils.log.INFO)
     cc = distutils.ccompiler.new_compiler(force=True)
     distutils.sysconfig.customize_compiler(cc)
@@ -71,13 +70,133 @@ def ffibuilder():
             src_fp.write('#include <deltachat.h>')
         cc.preprocess(source=src_name,
                       output_file=dst_name,
-                      include_dirs=incs,
+                      include_dirs=flags.incs,
                       macros=[('PY_CFFI', '1')])
         with open(dst_name, "r") as dst_fp:
-            builder.cdef(dst_fp.read())
+            return dst_fp.read()
     finally:
         shutil.rmtree(tmpdir)
 
+
+def find_header(flags):
+    """Use the compiler to find the deltachat.h header location.
+
+    This uses a small utility in deltachat.h to find the location of
+    the header file location.
+    """
+    distutils.log.set_verbosity(distutils.log.INFO)
+    cc = distutils.ccompiler.new_compiler(force=True)
+    distutils.sysconfig.customize_compiler(cc)
+    tmpdir = tempfile.mkdtemp()
+    try:
+        src_name = os.path.join(tmpdir, "where.c")
+        obj_name = os.path.join(tmpdir, "where.o")
+        dst_name = os.path.join(tmpdir, "where")
+        with open(src_name, "w") as src_fp:
+            src_fp.write(textwrap.dedent("""
+                #include <stdio.h>
+                #include <deltachat.h>
+
+                int main(void) {
+                    printf("%s", _dc_header_file_location());
+                    return 0;
+                }
+            """))
+        cwd = os.getcwd()
+        try:
+            os.chdir(tmpdir)
+            cc.compile(sources=["where.c"],
+                       include_dirs=flags.incs,
+                       macros=[("PY_CFFI_INC", "1")])
+        finally:
+            os.chdir(cwd)
+        cc.link_executable(objects=[obj_name],
+                           output_progname="where",
+                           output_dir=tmpdir)
+        return subprocess.check_output(dst_name)
+    finally:
+        shutil.rmtree(tmpdir)
+
+
+def extract_defines(flags):
+    """Extract the required #DEFINEs from deltachat.h.
+
+    Since #DEFINEs are interpreted by the C preprocessor we can not
+    use the compiler to extract these and need to parse the header
+    file ourselves.
+
+    The defines are returned in a string that can be passed to CFFIs
+    cdef() method.
+    """
+    header = find_header(flags)
+    defines_re = re.compile(r"""
+        \#define\s+   # The start of a define.
+        (             # Begin capturing group which captures the define name.
+            (?:       # A nested group which is not captured, this allows us
+                      #    to build the list of prefixes to extract without
+                      #    creation another capture group.
+                DC_EVENT
+                | DC_QR
+                | DC_MSG
+                | DC_LP
+                | DC_EMPTY
+                | DC_CERTCK
+                | DC_STATE
+                | DC_STR
+                | DC_CONTACT_ID
+                | DC_GCL
+                | DC_CHAT
+                | DC_PROVIDER
+                | DC_KEY_GEN
+            )         # End of prefix matching
+            _[\w_]+   # Match the suffix, e.g. _RSA2048 in DC_KEY_GEN_RSA2048
+        )             # Close the capturing group, this contains
+                      # the entire name e.g. DC_MSG_TEXT.
+        \s+\S+        # Ensure there is whitespace followed by a value.
+    """, re.VERBOSE)
+    defines = []
+    with open(header) as fp:
+        for line in fp:
+            match = defines_re.match(line)
+            if match:
+                defines.append(match.group(1))
+    return '\n'.join('#define {} ...'.format(d) for d in defines)
+
+
+def ffibuilder():
+    projdir = os.environ.get('DCC_RS_DEV')
+    if not projdir:
+        p = dn(dn(dn(dn(abspath(__file__)))))
+        projdir = os.environ["DCC_RS_DEV"] = p
+    target = os.environ.get('DCC_RS_TARGET', 'release')
+    if projdir:
+        flags = local_build_flags(projdir, target)
+    else:
+        flags = system_build_flags()
+    builder = cffi.FFI()
+    builder.set_source(
+        'deltachat.capi',
+        """
+            #include <deltachat.h>
+            int dc_event_has_string_data(int e)
+            {
+                return DC_EVENT_DATA2_IS_STRING(e);
+            }
+        """,
+        include_dirs=flags.incs,
+        libraries=flags.libs,
+        extra_objects=flags.objs,
+        extra_link_args=flags.extra_link_args,
+    )
+    builder.cdef("""
+        typedef int... time_t;
+        void free(void *ptr);
+        extern int dc_event_has_string_data(int);
+    """)
+    function_defs = extract_functions(flags)
+    defines = extract_defines(flags)
+    builder.cdef(function_defs)
+    builder.cdef(defines)
     return builder
 
 

python/src/deltachat/account.py

@@ -50,7 +50,7 @@ class Account(object):
             lib.dc_context_unref,
         )
         if self._dc_context == ffi.NULL:
-            raise ValueError("FAILED dc_context_new: {} {}".format(os_name, db_path))
+            raise ValueError("Could not dc_context_new: {} {}".format(os_name, db_path))
 
         self._shutdown_event = Event()
         self._event_thread = EventThread(self)
@@ -179,17 +179,6 @@ class Account(object):
         if not self.is_configured():
             raise ValueError("need to configure first")
 
-    def empty_server_folders(self, inbox=False, mvbox=False):
-        """ empty server folders. """
-        flags = 0
-        if inbox:
-            flags |= const.DC_EMPTY_INBOX
-        if mvbox:
-            flags |= const.DC_EMPTY_MVBOX
-        if not flags:
-            raise ValueError("no flags set")
-        lib.dc_empty_server(self._dc_context, flags)
-
     def get_latest_backupfile(self, backupdir):
         """ return the latest backup file in a given directory.
         """
@@ -213,22 +202,39 @@ class Account(object):
         """
         return Contact(self, const.DC_CONTACT_ID_SELF)
 
-    def create_contact(self, email, name=None):
-        """ create a (new) Contact. If there already is a Contact
-        with that e-mail address, it is unblocked and its name is
-        updated.
+    def create_contact(self, obj, name=None):
+        """ create a (new) Contact or return an existing one.
 
-        :param email: email-address (text type)
-        :param name: display name for this contact (optional)
+        Calling this method will always result in the same
+        underlying contact id.  If there already is a Contact
+        with that e-mail address, it is unblocked and its display
+        `name` is updated if specified.
+
+        :param obj: email-address, Account or Contact instance.
+        :param name: (optional) display name for this contact
         :returns: :class:`deltachat.contact.Contact` instance.
         """
-        realname, addr = parseaddr(email)
-        if name:
-            realname = name
-        realname = as_dc_charpointer(realname)
+        if isinstance(obj, Account):
+            if not obj.is_configured():
+                raise ValueError("can only add addresses from configured accounts")
+            addr, displayname = obj.get_config("addr"), obj.get_config("displayname")
+        elif isinstance(obj, Contact):
+            if obj.account != self:
+                raise ValueError("account mismatch {}".format(obj))
+            addr, displayname = obj.addr, obj.name
+        elif isinstance(obj, str):
+            displayname, addr = parseaddr(obj)
+        else:
+            raise TypeError("don't know how to create chat for %r" % (obj, ))
+
+        if name is None and displayname:
+            name = displayname
+        return self._create_contact(addr, name)
+
+    def _create_contact(self, addr, name):
         addr = as_dc_charpointer(addr)
-        contact_id = lib.dc_create_contact(self._dc_context, realname, addr)
-        assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
+        name = as_dc_charpointer(name)
+        contact_id = lib.dc_create_contact(self._dc_context, name, addr)
         return Contact(self, contact_id)
 
     def delete_contact(self, contact):
@@ -250,6 +256,24 @@ class Account(object):
         if contact_id:
             return self.get_contact_by_id(contact_id)
 
+    def get_contact_by_id(self, contact_id):
+        """ return Contact instance or None.
+        :param contact_id: integer id of this contact.
+        :returns: None or :class:`deltachat.contact.Contact` instance.
+        """
+        return Contact(self, contact_id)
+
+    def get_blocked_contacts(self):
+        """ return a list of all blocked contacts.
+
+        :returns: list of :class:`deltachat.contact.Contact` objects.
+        """
+        dc_array = ffi.gc(
+            lib.dc_get_blocked_contacts(self._dc_context),
+            lib.dc_array_unref
+        )
+        return list(iter_array(dc_array, lambda x: Contact(self, x)))
+
     def get_contacts(self, query=None, with_self=False, only_verified=False):
         """ get a (filtered) list of contacts.
 
@@ -279,53 +303,29 @@ class Account(object):
         )
         yield from iter_array(dc_array, lambda x: Message.from_db(self, x))
 
-    def create_chat_by_contact(self, contact):
-        """ create or get an existing 1:1 chat object for the specified contact or contact id.
+    def create_chat(self, obj):
+        """ Create a 1:1 chat with Account, Contact or e-mail address. """
+        return self.create_contact(obj).create_chat()
 
-        :param contact: chat_id (int) or contact object.
-        :returns: a :class:`deltachat.chat.Chat` object.
-        """
-        if hasattr(contact, "id"):
-            if contact.account != self:
-                raise ValueError("Contact belongs to a different Account")
-            contact_id = contact.id
-        else:
-            assert isinstance(contact, int)
-            contact_id = contact
-        chat_id = lib.dc_create_chat_by_contact_id(self._dc_context, contact_id)
-        return Chat(self, chat_id)
+    def _create_chat_by_message_id(self, msg_id):
+        return Chat(self, lib.dc_create_chat_by_msg_id(self._dc_context, msg_id))
 
-    def create_chat_by_message(self, message):
-        """ create or get an existing chat object for the
-        the specified message.
-
-        If this message is in the deaddrop chat then
-        the sender will become an accepted contact.
-
-        :param message: messsage id or message instance.
-        :returns: a :class:`deltachat.chat.Chat` object.
-        """
-        if hasattr(message, "id"):
-            if message.account != self:
-                raise ValueError("Message belongs to a different Account")
-            msg_id = message.id
-        else:
-            assert isinstance(message, int)
-            msg_id = message
-        chat_id = lib.dc_create_chat_by_msg_id(self._dc_context, msg_id)
-        return Chat(self, chat_id)
-
-    def create_group_chat(self, name, verified=False):
+    def create_group_chat(self, name, contacts=None, verified=False):
         """ create a new group chat object.
 
         Chats are unpromoted until the first message is sent.
 
+        :param contacts: list of contacts to add
         :param verified: if true only verified contacts can be added.
         :returns: a :class:`deltachat.chat.Chat` object.
         """
         bytes_name = name.encode("utf8")
         chat_id = lib.dc_create_group_chat(self._dc_context, int(verified), bytes_name)
-        return Chat(self, chat_id)
+        chat = Chat(self, chat_id)
+        if contacts is not None:
+            for contact in contacts:
+                chat.add_contact(contact)
+        return chat
 
     def get_chats(self):
         """ return list of chats.
@@ -347,6 +347,9 @@ class Account(object):
     def get_deaddrop_chat(self):
         return Chat(self, const.DC_CHAT_ID_DEADDROP)
 
+    def get_device_chat(self):
+        return Contact(self, const.DC_CONTACT_ID_DEVICE).create_chat()
+
     def get_message_by_id(self, msg_id):
         """ return Message instance.
         :param msg_id: integer id of this message.
@@ -354,13 +357,6 @@ class Account(object):
         """
         return Message.from_db(self, msg_id)
 
-    def get_contact_by_id(self, contact_id):
-        """ return Contact instance or None.
-        :param contact_id: integer id of this contact.
-        :returns: None or :class:`deltachat.contact.Contact` instance.
-        """
-        return Contact(self, contact_id)
-
     def get_chat_by_id(self, chat_id):
         """ return Chat instance.
         :param chat_id: integer id of this chat.
@@ -564,32 +560,30 @@ class Account(object):
         You may call `stop_scheduler`, `wait_shutdown` or `shutdown` after the
         account is started.
 
+        If you are using this from a test, you may want to call
+        wait_all_initial_fetches() afterwards.
+
         :raises MissingCredentials: if `addr` and `mail_pw` values are not set.
         :raises ConfigureFailed: if the account could not be configured.
 
-        :returns: None (account is configured and with io-scheduling running)
+        :returns: None
         """
         if not self.is_configured():
             raise ValueError("account not configured, cannot start io")
         lib.dc_start_io(self._dc_context)
 
     def configure(self):
+        """ Start configuration process and return a Configtracker instance
+        on which you can block with wait_finish() to get a True/False success
+        value for the configuration process.
+        """
         assert not self.is_configured()
-        assert not hasattr(self, "_configtracker")
         if not self.get_config("addr") or not self.get_config("mail_pw"):
             raise MissingCredentials("addr or mail_pwd not set in config")
-        if hasattr(self, "_configtracker"):
-            self.remove_account_plugin(self._configtracker)
-        self._configtracker = ConfigureTracker()
-        self.add_account_plugin(self._configtracker)
+        configtracker = ConfigureTracker(self)
+        self.add_account_plugin(configtracker)
         lib.dc_configure(self._dc_context)
-
-    def wait_configure_finish(self):
-        try:
-            self._configtracker.wait_finish()
-        finally:
-            self.remove_account_plugin(self._configtracker)
-            del self._configtracker
+        return configtracker
 
     def is_started(self):
         return self._event_thread.is_alive() and bool(lib.dc_is_io_running(self._dc_context))
@@ -618,12 +612,24 @@ class Account(object):
         self.stop_io()
 
         self.log("remove dc_context references")
-        # the dc_context_unref triggers get_next_event to return ffi.NULL
-        # which in turns makes the event thread finish execution
+
+        # if _dc_context is unref'ed the event thread should quickly
+        # receive the termination signal. However, some python code might
+        # still hold a reference and so we use a secondary signal
+        # to make sure the even thread terminates if it receives any new
+        # event, indepedently from waiting for the core to send NULL to
+        # get_next_event().
+        self._event_thread.mark_shutdown()
         self._dc_context = None
 
         self.log("wait for event thread to finish")
-        self._event_thread.wait()
+        try:
+            self._event_thread.wait(timeout=2)
+        except RuntimeError as e:
+            self.log("Waiting for event thread failed: {}".format(e))
+
+        if self._event_thread.is_alive():
+            self.log("WARN: event thread did not terminate yet, ignoring.")
 
         self._shutdown_event.set()
 

python/src/deltachat/chat.py

@@ -18,6 +18,8 @@ class Chat(object):
     """
 
     def __init__(self, account, id):
+        from .account import Account
+        assert isinstance(account, Account), repr(account)
         self.account = account
         self.id = id
 
@@ -55,10 +57,7 @@ class Chat(object):
 
         :returns: True if chat is a group-chat, false if it's a contact 1:1 chat.
         """
-        return lib.dc_chat_get_type(self._dc_chat) in (
-            const.DC_CHAT_TYPE_GROUP,
-            const.DC_CHAT_TYPE_VERIFIED_GROUP
-        )
+        return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_GROUP
 
     def is_deaddrop(self):
         """ return true if this chat is a deaddrop chat.
@@ -83,12 +82,20 @@ class Chat(object):
         """
         return not lib.dc_chat_is_unpromoted(self._dc_chat)
 
-    def is_verified(self):
-        """ return True if this chat is a verified group.
+    def can_send(self):
+        """Check if messages can be sent to a give chat.
+        This is not true eg. for the deaddrop or for the device-talk
 
-        :returns: True if chat is verified, False otherwise.
+        :returns: True if the chat is writable, False otherwise
         """
-        return lib.dc_chat_is_verified(self._dc_chat)
+        return lib.dc_chat_can_send(self._dc_chat)
+
+    def is_protected(self):
+        """ return True if this chat is a protected chat.
+
+        :returns: True if chat is protected, False otherwise.
+        """
+        return lib.dc_chat_is_protected(self._dc_chat)
 
     def get_name(self):
         """ return name of this chat.
@@ -135,7 +142,23 @@ class Chat(object):
         :param duration:
         :returns: Returns the number of seconds the chat is still muted for. (0 for not muted, -1 forever muted)
         """
-        return bool(lib.dc_chat_get_remaining_mute_duration(self.id))
+        return lib.dc_chat_get_remaining_mute_duration(self._dc_chat)
+
+    def get_ephemeral_timer(self):
+        """ get ephemeral timer.
+
+        :returns: ephemeral timer value in seconds
+        """
+        return lib.dc_get_chat_ephemeral_timer(self.account._dc_context, self.id)
+
+    def set_ephemeral_timer(self, timer):
+        """ set ephemeral timer.
+
+        :param: timer value in seconds
+
+        :returns: None
+        """
+        return lib.dc_set_chat_ephemeral_timer(self.account._dc_context, self.id, timer)
 
     def get_type(self):
         """ (deprecated) return type of this chat.
@@ -328,33 +351,34 @@ class Chat(object):
 
     # ------  group management API ------------------------------
 
-    def add_contact(self, contact):
+    def add_contact(self, obj):
         """ add a contact to this chat.
 
-        :params: contact object.
+        :params obj: Contact, Account or e-mail address.
         :raises ValueError: if contact could not be added
         :returns: None
         """
+        contact = self.account.create_contact(obj)
         ret = lib.dc_add_contact_to_chat(self.account._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not add contact {!r} to chat".format(contact))
+        return contact
 
-    def remove_contact(self, contact):
+    def remove_contact(self, obj):
         """ remove a contact from this chat.
 
-        :params: contact object.
+        :params obj: Contact, Account or e-mail address.
         :raises ValueError: if contact could not be removed
         :returns: None
         """
+        contact = self.account.create_contact(obj)
         ret = lib.dc_remove_contact_from_chat(self.account._dc_context, self.id, contact.id)
         if ret != 1:
             raise ValueError("could not remove contact {!r} from chat".format(contact))
 
     def get_contacts(self):
         """ get all contacts for this chat.
-        :params: contact object.
         :returns: list of :class:`deltachat.contact.Contact` objects for this chat
-
         """
         from .contact import Contact
         dc_array = ffi.gc(
@@ -365,6 +389,14 @@ class Chat(object):
             dc_array, lambda id: Contact(self.account, id))
         )
 
+    def num_contacts(self):
+        """ return number of contacts in this chat. """
+        dc_array = ffi.gc(
+            lib.dc_get_chat_contacts(self.account._dc_context, self.id),
+            lib.dc_array_unref
+        )
+        return lib.dc_array_get_cnt(dc_array)
+
     def set_profile_image(self, img_path):
         """Set group profile image.
 

python/src/deltachat/const.py

@@ -1,197 +1,7 @@
-import sys
-import re
-import os
-from os.path import dirname, abspath
-from os.path import join as joinpath
-
-# the following const are generated from deltachat.h
-# this works well when you in a git-checkout
-# run "python deltachat/const.py" to regenerate events
-# begin const generated
-DC_GCL_ARCHIVED_ONLY = 0x01
-DC_GCL_NO_SPECIALS = 0x02
-DC_GCL_ADD_ALLDONE_HINT = 0x04
-DC_GCL_FOR_FORWARDING = 0x08
-DC_GCL_VERIFIED_ONLY = 0x01
-DC_GCL_ADD_SELF = 0x02
-DC_QR_ASK_VERIFYCONTACT = 200
-DC_QR_ASK_VERIFYGROUP = 202
-DC_QR_FPR_OK = 210
-DC_QR_FPR_MISMATCH = 220
-DC_QR_FPR_WITHOUT_ADDR = 230
-DC_QR_ACCOUNT = 250
-DC_QR_ADDR = 320
-DC_QR_TEXT = 330
-DC_QR_URL = 332
-DC_QR_ERROR = 400
-DC_CHAT_ID_DEADDROP = 1
-DC_CHAT_ID_TRASH = 3
-DC_CHAT_ID_MSGS_IN_CREATION = 4
-DC_CHAT_ID_STARRED = 5
-DC_CHAT_ID_ARCHIVED_LINK = 6
-DC_CHAT_ID_ALLDONE_HINT = 7
-DC_CHAT_ID_LAST_SPECIAL = 9
-DC_CHAT_TYPE_UNDEFINED = 0
-DC_CHAT_TYPE_SINGLE = 100
-DC_CHAT_TYPE_GROUP = 120
-DC_CHAT_TYPE_VERIFIED_GROUP = 130
-DC_MSG_ID_MARKER1 = 1
-DC_MSG_ID_DAYMARKER = 9
-DC_MSG_ID_LAST_SPECIAL = 9
-DC_STATE_UNDEFINED = 0
-DC_STATE_IN_FRESH = 10
-DC_STATE_IN_NOTICED = 13
-DC_STATE_IN_SEEN = 16
-DC_STATE_OUT_PREPARING = 18
-DC_STATE_OUT_DRAFT = 19
-DC_STATE_OUT_PENDING = 20
-DC_STATE_OUT_FAILED = 24
-DC_STATE_OUT_DELIVERED = 26
-DC_STATE_OUT_MDN_RCVD = 28
-DC_CONTACT_ID_SELF = 1
-DC_CONTACT_ID_INFO = 2
-DC_CONTACT_ID_DEVICE = 5
-DC_CONTACT_ID_LAST_SPECIAL = 9
-DC_MSG_TEXT = 10
-DC_MSG_IMAGE = 20
-DC_MSG_GIF = 21
-DC_MSG_STICKER = 23
-DC_MSG_AUDIO = 40
-DC_MSG_VOICE = 41
-DC_MSG_VIDEO = 50
-DC_MSG_FILE = 60
-DC_LP_AUTH_OAUTH2 = 0x2
-DC_LP_AUTH_NORMAL = 0x4
-DC_LP_IMAP_SOCKET_STARTTLS = 0x100
-DC_LP_IMAP_SOCKET_SSL = 0x200
-DC_LP_IMAP_SOCKET_PLAIN = 0x400
-DC_LP_SMTP_SOCKET_STARTTLS = 0x10000
-DC_LP_SMTP_SOCKET_SSL = 0x20000
-DC_LP_SMTP_SOCKET_PLAIN = 0x40000
-DC_CERTCK_AUTO = 0
-DC_CERTCK_STRICT = 1
-DC_CERTCK_ACCEPT_INVALID_CERTIFICATES = 3
-DC_EMPTY_MVBOX = 0x01
-DC_EMPTY_INBOX = 0x02
-DC_EVENT_INFO = 100
-DC_EVENT_SMTP_CONNECTED = 101
-DC_EVENT_IMAP_CONNECTED = 102
-DC_EVENT_SMTP_MESSAGE_SENT = 103
-DC_EVENT_IMAP_MESSAGE_DELETED = 104
-DC_EVENT_IMAP_MESSAGE_MOVED = 105
-DC_EVENT_IMAP_FOLDER_EMPTIED = 106
-DC_EVENT_NEW_BLOB_FILE = 150
-DC_EVENT_DELETED_BLOB_FILE = 151
-DC_EVENT_WARNING = 300
-DC_EVENT_ERROR = 400
-DC_EVENT_ERROR_NETWORK = 401
-DC_EVENT_ERROR_SELF_NOT_IN_GROUP = 410
-DC_EVENT_MSGS_CHANGED = 2000
-DC_EVENT_INCOMING_MSG = 2005
-DC_EVENT_MSG_DELIVERED = 2010
-DC_EVENT_MSG_FAILED = 2012
-DC_EVENT_MSG_READ = 2015
-DC_EVENT_CHAT_MODIFIED = 2020
-DC_EVENT_CONTACTS_CHANGED = 2030
-DC_EVENT_LOCATION_CHANGED = 2035
-DC_EVENT_CONFIGURE_PROGRESS = 2041
-DC_EVENT_IMEX_PROGRESS = 2051
-DC_EVENT_IMEX_FILE_WRITTEN = 2052
-DC_EVENT_SECUREJOIN_INVITER_PROGRESS = 2060
-DC_EVENT_SECUREJOIN_JOINER_PROGRESS = 2061
-DC_EVENT_FILE_COPIED = 2055
-DC_EVENT_IS_OFFLINE = 2081
-DC_EVENT_GET_STRING = 2091
-DC_STR_SELFNOTINGRP = 21
-DC_KEY_GEN_DEFAULT = 0
-DC_KEY_GEN_RSA2048 = 1
-DC_KEY_GEN_ED25519 = 2
-DC_PROVIDER_STATUS_OK = 1
-DC_PROVIDER_STATUS_PREPARATION = 2
-DC_PROVIDER_STATUS_BROKEN = 3
-DC_CHAT_VISIBILITY_NORMAL = 0
-DC_CHAT_VISIBILITY_ARCHIVED = 1
-DC_CHAT_VISIBILITY_PINNED = 2
-DC_STR_NOMESSAGES = 1
-DC_STR_SELF = 2
-DC_STR_DRAFT = 3
-DC_STR_VOICEMESSAGE = 7
-DC_STR_DEADDROP = 8
-DC_STR_IMAGE = 9
-DC_STR_VIDEO = 10
-DC_STR_AUDIO = 11
-DC_STR_FILE = 12
-DC_STR_STATUSLINE = 13
-DC_STR_NEWGROUPDRAFT = 14
-DC_STR_MSGGRPNAME = 15
-DC_STR_MSGGRPIMGCHANGED = 16
-DC_STR_MSGADDMEMBER = 17
-DC_STR_MSGDELMEMBER = 18
-DC_STR_MSGGROUPLEFT = 19
-DC_STR_GIF = 23
-DC_STR_ENCRYPTEDMSG = 24
-DC_STR_E2E_AVAILABLE = 25
-DC_STR_ENCR_TRANSP = 27
-DC_STR_ENCR_NONE = 28
-DC_STR_CANTDECRYPT_MSG_BODY = 29
-DC_STR_FINGERPRINTS = 30
-DC_STR_READRCPT = 31
-DC_STR_READRCPT_MAILBODY = 32
-DC_STR_MSGGRPIMGDELETED = 33
-DC_STR_E2E_PREFERRED = 34
-DC_STR_CONTACT_VERIFIED = 35
-DC_STR_CONTACT_NOT_VERIFIED = 36
-DC_STR_CONTACT_SETUP_CHANGED = 37
-DC_STR_ARCHIVEDCHATS = 40
-DC_STR_STARREDMSGS = 41
-DC_STR_AC_SETUP_MSG_SUBJECT = 42
-DC_STR_AC_SETUP_MSG_BODY = 43
-DC_STR_CANNOT_LOGIN = 60
-DC_STR_SERVER_RESPONSE = 61
-DC_STR_MSGACTIONBYUSER = 62
-DC_STR_MSGACTIONBYME = 63
-DC_STR_MSGLOCATIONENABLED = 64
-DC_STR_MSGLOCATIONDISABLED = 65
-DC_STR_LOCATION = 66
-DC_STR_STICKER = 67
-DC_STR_DEVICE_MESSAGES = 68
-DC_STR_COUNT = 68
-# end const generated
+from .capi import lib
 
 
-def read_event_defines(f):
-    rex = re.compile(r'#define\s+((?:DC_EVENT|DC_QR|DC_MSG|DC_LP|DC_EMPTY|DC_CERTCK|DC_STATE|DC_STR|'
-                     r'DC_CONTACT_ID|DC_GCL|DC_CHAT|DC_PROVIDER|DC_KEY_GEN)_\S+)\s+([x\d]+).*')
-    for line in f:
-        m = rex.match(line)
-        if m:
-            yield m.groups()
-
-
-if __name__ == "__main__":
-    here = abspath(__file__).rstrip("oc")
-    here_dir = dirname(here)
-    if len(sys.argv) >= 2:
-        deltah = sys.argv[1]
-    else:
-        deltah = joinpath(dirname(dirname(dirname(here_dir))), "deltachat-ffi", "deltachat.h")
-    assert os.path.exists(deltah)
-
-    lines = []
-    skip_to_end = False
-    for orig_line in open(here):
-        if skip_to_end:
-            if not orig_line.startswith("# end const"):
-                continue
-            skip_to_end = False
-        lines.append(orig_line)
-        if orig_line.startswith("# begin const"):
-            with open(deltah) as f:
-                for name, item in read_event_defines(f):
-                    lines.append("{} = {}\n".format(name, item))
-            skip_to_end = True
-
-    tmpname = here + ".tmp"
-    with open(tmpname, "w") as f:
-        f.write("".join(lines))
-    os.rename(tmpname, here)
+for name in dir(lib):
+    if name.startswith("DC_"):
+        globals()[name] = getattr(lib, name)
+del name

python/src/deltachat/contact.py

@@ -3,6 +3,8 @@
 from . import props
 from .cutil import from_dc_charpointer
 from .capi import lib, ffi
+from .chat import Chat
+from . import const
 
 
 class Contact(object):
@@ -11,6 +13,8 @@ class Contact(object):
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
     def __init__(self, account, id):
+        from .account import Account
+        assert isinstance(account, Account), repr(account)
         self.account = account
         self.id = id
 
@@ -36,14 +40,29 @@ class Contact(object):
         return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
 
     @props.with_doc
-    def display_name(self):
+    def name(self):
         """ display name for this contact. """
         return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
 
+    # deprecated alias
+    display_name = name
+
     def is_blocked(self):
         """ Return True if the contact is blocked. """
         return lib.dc_contact_is_blocked(self._dc_contact)
 
+    def set_blocked(self, block=True):
+        """ [Deprecated, use block/unblock methods] Block or unblock a contact. """
+        return lib.dc_block_contact(self.account._dc_context, self.id, block)
+
+    def block(self):
+        """ Block this contact. Message will not be seen/retrieved from this contact. """
+        return lib.dc_block_contact(self.account._dc_context, self.id, True)
+
+    def unblock(self):
+        """ Unblock this contact. Messages from this contact will be retrieved (again)."""
+        return lib.dc_block_contact(self.account._dc_context, self.id, False)
+
     def is_verified(self):
         """ Return True if the contact is verified. """
         return lib.dc_contact_is_verified(self._dc_contact)
@@ -58,6 +77,16 @@ class Contact(object):
             return None
         return from_dc_charpointer(dc_res)
 
-    def get_chat(self):
-        """return 1:1 chat for this contact. """
-        return self.account.create_chat_by_contact(self)
+    def create_chat(self):
+        """ create or get an existing 1:1 chat object for the specified contact or contact id.
+
+        :param contact: chat_id (int) or contact object.
+        :returns: a :class:`deltachat.chat.Chat` object.
+        """
+        dc_context = self.account._dc_context
+        chat_id = lib.dc_create_chat_by_contact_id(dc_context, self.id)
+        assert chat_id > const.DC_CHAT_ID_LAST_SPECIAL, chat_id
+        return Chat(self.account, chat_id)
+
+    # deprecated name
+    get_chat = create_chat

python/src/deltachat/cutil.py

@@ -17,7 +17,8 @@ def iter_array(dc_array_t, constructor):
 
 
 def from_dc_charpointer(obj):
-    return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
+    if obj != ffi.NULL:
+        return ffi.string(ffi.gc(obj, lib.dc_str_unref)).decode("utf8")
 
 
 class DCLot:

python/src/deltachat/direct_imap.py

@@ -0,0 +1,228 @@
+"""
+Internal Python-level IMAP handling used by the testplugin
+and for cleaning up inbox/mvbox for each test function run.
+"""
+
+import io
+import email
+import ssl
+import pathlib
+from imapclient import IMAPClient
+from imapclient.exceptions import IMAPClientError
+import deltachat
+
+
+SEEN = b'\\Seen'
+DELETED = b'\\Deleted'
+FLAGS = b'FLAGS'
+FETCH = b'FETCH'
+ALL = "1:*"
+
+
+@deltachat.global_hookimpl
+def dc_account_extra_configure(account):
+    """ Reset the account (we reuse accounts across tests)
+    and make 'account.direct_imap' available for direct IMAP ops.
+    """
+    if not hasattr(account, "direct_imap"):
+        imap = DirectImap(account)
+        if imap.select_config_folder("mvbox"):
+            imap.delete(ALL, expunge=True)
+        assert imap.select_config_folder("inbox")
+        imap.delete(ALL, expunge=True)
+        setattr(account, "direct_imap", imap)
+
+
+@deltachat.global_hookimpl
+def dc_account_after_shutdown(account):
+    """ shutdown the imap connection if there is one. """
+    imap = getattr(account, "direct_imap", None)
+    if imap is not None:
+        imap.shutdown()
+        del account.direct_imap
+
+
+class DirectImap:
+    def __init__(self, account):
+        self.account = account
+        self.logid = account.get_config("displayname") or id(account)
+        self._idling = False
+        self.connect()
+
+    def connect(self):
+        ssl_context = ssl.create_default_context()
+
+        # don't check if certificate hostname doesn't match target hostname
+        ssl_context.check_hostname = False
+
+        # don't check if the certificate is trusted by a certificate authority
+        ssl_context.verify_mode = ssl.CERT_NONE
+
+        host = self.account.get_config("configured_mail_server")
+        user = self.account.get_config("addr")
+        pw = self.account.get_config("mail_pw")
+        self.conn = IMAPClient(host, ssl_context=ssl_context)
+        self.conn.login(user, pw)
+
+        self.select_folder("INBOX")
+
+    def shutdown(self):
+        try:
+            self.conn.idle_done()
+        except (OSError, IMAPClientError):
+            pass
+        try:
+            self.conn.logout()
+        except (OSError, IMAPClientError):
+            print("Could not logout direct_imap conn")
+
+    def select_folder(self, foldername):
+        assert not self._idling
+        return self.conn.select_folder(foldername)
+
+    def select_config_folder(self, config_name):
+        """ Return info about selected folder if it is
+        configured, otherwise None. """
+        if "_" not in config_name:
+            config_name = "configured_{}_folder".format(config_name)
+        foldername = self.account.get_config(config_name)
+        if foldername:
+            return self.select_folder(foldername)
+
+    def list_folders(self):
+        """ return list of all existing folder names"""
+        assert not self._idling
+        folders = []
+        for meta, sep, foldername in self.conn.list_folders():
+            folders.append(foldername)
+        return folders
+
+    def delete(self, range, expunge=True):
+        """ delete a range of messages (imap-syntax).
+        If expunge is true, perform the expunge-operation
+        to make sure the messages are really gone and not
+        just flagged as deleted.
+        """
+        self.conn.set_flags(range, [DELETED])
+        if expunge:
+            self.conn.expunge()
+
+    def get_all_messages(self):
+        assert not self._idling
+
+        # Flush unsolicited responses. IMAPClient has problems
+        # dealing with them: https://github.com/mjs/imapclient/issues/334
+        # When this NOOP was introduced, next FETCH returned empty
+        # result instead of a single message, even though IMAP server
+        # can only return more untagged responses than required, not
+        # less.
+        self.conn.noop()
+
+        return self.conn.fetch(ALL, [FLAGS])
+
+    def get_unread_messages(self):
+        assert not self._idling
+        res = self.conn.fetch(ALL, [FLAGS])
+        return [uid for uid in res
+                if SEEN not in res[uid][FLAGS]]
+
+    def mark_all_read(self):
+        messages = self.get_unread_messages()
+        if messages:
+            res = self.conn.set_flags(messages, [SEEN])
+            print("marked seen:", messages, res)
+
+    def get_unread_cnt(self):
+        return len(self.get_unread_messages())
+
+    def dump_account_info(self, logfile):
+        def log(*args, **kwargs):
+            kwargs["file"] = logfile
+            print(*args, **kwargs)
+
+        cursor = 0
+        for name, val in self.account.get_info().items():
+            entry = "{}={}".format(name.upper(), val)
+            if cursor + len(entry) > 80:
+                log("")
+                cursor = 0
+            log(entry, end=" ")
+            cursor += len(entry) + 1
+        log("")
+
+    def dump_imap_structures(self, dir, logfile):
+        assert not self._idling
+        stream = io.StringIO()
+
+        def log(*args, **kwargs):
+            kwargs["file"] = stream
+            print(*args, **kwargs)
+
+        empty_folders = []
+        for imapfolder in self.list_folders():
+            self.select_folder(imapfolder)
+            messages = list(self.get_all_messages())
+            if not messages:
+                empty_folders.append(imapfolder)
+                continue
+
+            log("---------", imapfolder, len(messages), "messages ---------")
+            # get message content without auto-marking it as seen
+            # fetching 'RFC822' would mark it as seen.
+            requested = [b'BODY.PEEK[]', FLAGS]
+            for uid, data in self.conn.fetch(messages, requested).items():
+                body_bytes = data[b'BODY[]']
+                if not body_bytes:
+                    log("Message", uid, "has empty body")
+                    continue
+
+                flags = data[FLAGS]
+                path = pathlib.Path(str(dir)).joinpath("IMAP", self.logid, imapfolder)
+                path.mkdir(parents=True, exist_ok=True)
+                fn = path.joinpath(str(uid))
+                fn.write_bytes(body_bytes)
+                log("Message", uid, fn)
+                email_message = email.message_from_bytes(body_bytes)
+                log("Message", uid, flags, "Message-Id:", email_message.get("Message-Id"))
+
+        if empty_folders:
+            log("--------- EMPTY FOLDERS:", empty_folders)
+
+        print(stream.getvalue(), file=logfile)
+
+    def idle_start(self):
+        """ switch this connection to idle mode. non-blocking. """
+        assert not self._idling
+        res = self.conn.idle()
+        self._idling = True
+        return res
+
+    def idle_check(self, terminate=False):
+        """ (blocking) wait for next idle message from server. """
+        assert self._idling
+        self.account.log("imap-direct: calling idle_check")
+        res = self.conn.idle_check(timeout=30)
+        if len(res) == 0:
+            raise TimeoutError
+        if terminate:
+            self.idle_done()
+        self.account.log("imap-direct: idle_check returned {!r}".format(res))
+        return res
+
+    def idle_wait_for_seen(self):
+        """ Return first message with SEEN flag
+        from a running idle-stream REtiurn.
+        """
+        while 1:
+            for item in self.idle_check():
+                if item[1] == FETCH:
+                    if item[2][0] == FLAGS:
+                        if SEEN in item[2][1]:
+                            return item[0]
+
+    def idle_done(self):
+        """ send idle-done to server if we are currently in idle mode. """
+        if self._idling:
+            res = self.conn.idle_done()
+            self._idling = False
+            return res

python/src/deltachat/events.py

@@ -1,6 +1,7 @@
 import threading
 import time
 import re
+import os
 from queue import Queue, Empty
 
 import deltachat
@@ -28,13 +29,9 @@ class FFIEventLogger:
     # to prevent garbled logging
     _loglock = threading.RLock()
 
-    def __init__(self, account, logid):
-        """
-        :param logid: an optional logging prefix that should be used with
-                      the default internal logging.
-        """
+    def __init__(self, account):
         self.account = account
-        self.logid = logid
+        self.logid = self.account.get_config("displayname")
         self.init_time = time.time()
 
     @account_hookimpl
@@ -52,6 +49,15 @@ class FFIEventLogger:
         if self.logid:
             locname += "-" + self.logid
         s = "{:2.2f} [{}] {}".format(elapsed, locname, message)
+
+        if os.name == "posix":
+            WARN = '\033[93m'
+            ERROR = '\033[91m'
+            ENDC = '\033[0m'
+            if message.startswith("DC_EVENT_WARNING"):
+                s = WARN + s + ENDC
+            if message.startswith("DC_EVENT_ERROR"):
+                s = ERROR + s + ENDC
         with self._loglock:
             print(s, flush=True)
 
@@ -90,11 +96,11 @@ class FFIEventTracker:
             if rex.match(ev.name):
                 return ev
 
-    def get_info_matching(self, regex):
-        rex = re.compile("(?:{}).*".format(regex))
+    def get_info_contains(self, regex):
+        rex = re.compile(regex)
         while 1:
             ev = self.get_matching("DC_EVENT_INFO")
-            if rex.match(ev.data2):
+            if rex.search(ev.data2):
                 return ev
 
     def ensure_event_not_queued(self, event_name_regex):
@@ -115,6 +121,15 @@ class FFIEventTracker:
                 print("** SECUREJOINT-INVITER PROGRESS {}".format(target), self.account)
                 break
 
+    def wait_all_initial_fetches(self):
+        """Has to be called after start_io() to wait for fetch_existing_msgs to run
+        so that new messages are not mistaken for old ones:
+        - ac1 and ac2 are created
+        - ac1 sends a message to ac2
+        - ac2 is still running FetchExsistingMsgs job and thinks it's an existing, old message
+        - therefore no DC_EVENT_INCOMING_MSG is sent"""
+        self.get_info_contains("Done fetching existing messages")
+
     def wait_next_incoming_message(self):
         """ wait for and return next incoming message. """
         ev = self.get_matching("DC_EVENT_INCOMING_MSG")
@@ -127,6 +142,12 @@ class FFIEventTracker:
         if ev.data2 > 0:
             return self.account.get_message_by_id(ev.data2)
 
+    def wait_msg_delivered(self, msg):
+        ev = self.get_matching("DC_EVENT_MSG_DELIVERED")
+        assert ev.data1 == msg.chat.id
+        assert ev.data2 == msg.id
+        assert msg.is_out_delivered()
+
 
 class EventThread(threading.Thread):
     """ Event Thread for an account.
@@ -137,6 +158,7 @@ class EventThread(threading.Thread):
         self.account = account
         super(EventThread, self).__init__(name="events")
         self.setDaemon(True)
+        self._marked_for_shutdown = False
         self.start()
 
     @contextmanager
@@ -145,12 +167,15 @@ class EventThread(threading.Thread):
         yield
         self.account.log(message + " FINISHED")
 
-    def wait(self):
+    def mark_shutdown(self):
+        self._marked_for_shutdown = True
+
+    def wait(self, timeout=None):
         if self == threading.current_thread():
             # we are in the callback thread and thus cannot
             # wait for the thread-loop to finish.
             return
-        self.join()
+        self.join(timeout=timeout)
 
     def run(self):
         """ get and run events until shutdown. """
@@ -162,17 +187,19 @@ class EventThread(threading.Thread):
             lib.dc_get_event_emitter(self.account._dc_context),
             lib.dc_event_emitter_unref,
         )
-        while 1:
+        while not self._marked_for_shutdown:
             event = lib.dc_get_next_event(event_emitter)
             if event == ffi.NULL:
                 break
+            if self._marked_for_shutdown:
+                break
             evt = lib.dc_event_get_id(event)
             data1 = lib.dc_event_get_data1_int(event)
             # the following code relates to the deltachat/_build.py's helper
             # function which provides us signature info of an event call
             evt_name = deltachat.get_dc_event_name(evt)
             if lib.dc_event_has_string_data(evt):
-                data2 = from_dc_charpointer(lib.dc_event_get_data3_str(event))
+                data2 = from_dc_charpointer(lib.dc_event_get_data2_str(event))
             else:
                 data2 = lib.dc_event_get_data2_int(event)
 

python/src/deltachat/hookspec.py

@@ -16,7 +16,7 @@ class PerAccount:
     """ per-Account-instance hook specifications.
 
     All hooks are executed in a dedicated Event thread.
-    Hooks are not allowed to block/last long as this
+    Hooks are generally not allowed to block/last long as this
     blocks overall event processing on the python side.
     """
     @classmethod
@@ -31,10 +31,6 @@ class PerAccount:
 
         ffi_event has "name", "data1", "data2" values as specified
         with `DC_EVENT_* <https://c.delta.chat/group__DC__EVENT.html>`_.
-
-        DANGER: this hook is executed from the callback invoked by core.
-        Hook implementations need to be short running and can typically
-        not call back into core because this would easily cause recursion issues.
         """
 
     @account_hookspec
@@ -43,7 +39,7 @@ class PerAccount:
 
     @account_hookspec
     def ac_configure_completed(self, success):
-        """ Called when a configure process completed. """
+        """ Called after a configure process completed. """
 
     @account_hookspec
     def ac_incoming_message(self, message):
@@ -55,19 +51,37 @@ class PerAccount:
 
     @account_hookspec
     def ac_message_delivered(self, message):
-        """ Called when an outgoing message has been delivered to SMTP. """
+        """ Called when an outgoing message has been delivered to SMTP.
+
+        :param message: Message that was just delivered.
+        """
 
     @account_hookspec
     def ac_chat_modified(self, chat):
-        """ Chat was created or modified regarding membership, avatar, title. """
+        """ Chat was created or modified regarding membership, avatar, title.
+
+        :param chat: Chat which was modified.
+        """
 
     @account_hookspec
-    def ac_member_added(self, chat, contact, message):
-        """ Called for each contact added to an accepted chat. """
+    def ac_member_added(self, chat, contact, actor, message):
+        """ Called for each contact added to an accepted chat.
+
+        :param chat: Chat where contact was added.
+        :param contact: Contact that was added.
+        :param actor: Who added the contact (None if it was our self-addr)
+        :param message: The original system message that reports the addition.
+        """
 
     @account_hookspec
-    def ac_member_removed(self, chat, contact, message):
-        """ Called for each contact removed from a chat.  """
+    def ac_member_removed(self, chat, contact, actor, message):
+        """ Called for each contact removed from a chat.
+
+        :param chat: Chat where contact was removed.
+        :param contact: Contact that was removed.
+        :param actor: Who removed the contact (None if it was our self-addr)
+        :param message: The original system message that reports the removal.
+        """
 
 
 class Global:
@@ -88,6 +102,14 @@ class Global:
     def dc_account_init(self, account):
         """ called when `Account::__init__()` function starts executing. """
 
+    @global_hookspec
+    def dc_account_extra_configure(self, account):
+        """ Called when account configuration successfully finished.
+
+        This hook can be used to perform extra work before
+        ac_configure_completed is called.
+        """
+
     @global_hookspec
     def dc_account_after_shutdown(self, account):
         """ Called after the account has been shutdown. """

python/src/deltachat/message.py

@@ -1,6 +1,7 @@
 """ The Message object. """
 
 import os
+import re
 from . import props
 from .cutil import from_dc_charpointer, as_dc_charpointer
 from .capi import lib, ffi
@@ -53,15 +54,19 @@ class Message(object):
             lib.dc_msg_unref
         ))
 
-    def accept_sender_contact(self):
-        """ ensure that the sender is an accepted contact
-        and that the message has a non-deaddrop chat object.
+    def create_chat(self):
+        """ create or get an existing chat (group) object for this message.
+
+        If the message is a deaddrop contact request
+        the sender will become an accepted contact.
+
+        :returns: a :class:`deltachat.chat.Chat` object.
         """
-        self.account.create_chat_by_message(self)
-        self._dc_msg = ffi.gc(
-                lib.dc_get_msg(self.account._dc_context, self.id),
-                lib.dc_msg_unref
-        )
+        from .chat import Chat
+        chat_id = lib.dc_create_chat_by_msg_id(self.account._dc_context, self.id)
+        ctx = self.account._dc_context
+        self._dc_msg = ffi.gc(lib.dc_get_msg(ctx, self.id), lib.dc_msg_unref)
+        return Chat(self.account, chat_id)
 
     @props.with_doc
     def text(self):
@@ -150,6 +155,47 @@ class Message(object):
         if ts:
             return datetime.utcfromtimestamp(ts)
 
+    @props.with_doc
+    def ephemeral_timer(self):
+        """Ephemeral timer in seconds
+
+        :returns: timer in seconds or None if there is no timer
+        """
+        timer = lib.dc_msg_get_ephemeral_timer(self._dc_msg)
+        if timer:
+            return timer
+
+    @props.with_doc
+    def ephemeral_timestamp(self):
+        """UTC time when the message will be deleted.
+
+        :returns: naive datetime.datetime() object or None if the timer is not started.
+        """
+        ts = lib.dc_msg_get_ephemeral_timestamp(self._dc_msg)
+        if ts:
+            return datetime.utcfromtimestamp(ts)
+
+    @property
+    def quoted_text(self):
+        """Text inside the quote
+
+        :returns: Quoted text"""
+        return from_dc_charpointer(lib.dc_msg_get_quoted_text(self._dc_msg))
+
+    @property
+    def quote(self):
+        """Quote getter
+
+        :returns: Quoted message, if found in the database"""
+        msg = lib.dc_msg_get_quoted_msg(self._dc_msg)
+        if msg:
+            return Message(self.account, ffi.gc(msg, lib.dc_msg_unref))
+
+    @quote.setter
+    def quote(self, quoted_message):
+        """Quote setter"""
+        lib.dc_msg_set_quote(self._dc_msg, quoted_message._dc_msg)
+
     def get_mime_headers(self):
         """ return mime-header object for an incoming message.
 
@@ -332,20 +378,43 @@ def get_viewtype_code_from_name(view_type_name):
 def map_system_message(msg):
     if msg.is_system_message():
         res = parse_system_add_remove(msg.text)
-        if res:
-            contact = msg.account.get_contact_by_addr(res[1])
-            if contact:
-                d = dict(chat=msg.chat, contact=contact, message=msg)
-                return "ac_member_" + res[0], d
+        if not res:
+            return
+        action, affected, actor = res
+        affected = msg.account.get_contact_by_addr(affected)
+        if actor == "me":
+            actor = None
+        else:
+            actor = msg.account.get_contact_by_addr(actor)
+        d = dict(chat=msg.chat, contact=affected, actor=actor, message=msg)
+        return "ac_member_" + res[0], d
+
+
+def extract_addr(text):
+    m = re.match(r'.*\((.+@.+)\)', text)
+    if m:
+        text = m.group(1)
+    text = text.rstrip(".")
+    return text.strip()
 
 
 def parse_system_add_remove(text):
+    """ return add/remove info from parsing the given system message text.
+
+    returns a (action, affected, actor) triple """
+
     # Member Me (x@y) removed by a@b.
-    # Member x@y removed by a@b
+    # Member x@y added by a@b
+    # Member With space (tmp1@x.org) removed by tmp2@x.org.
+    # Member With space (tmp1@x.org) removed by Another member (tmp2@x.org).",
+    # Group left by some one (tmp1@x.org).
+    # Group left by tmp1@x.org.
     text = text.lower()
-    parts = text.split()
-    if parts[0] == "member":
-        if parts[2] in ("removed", "added"):
-            return parts[2], parts[1]
-        if parts[3] in ("removed", "added"):
-            return parts[3], parts[2].strip("()")
+    m = re.match(r'member (.+) (removed|added) by (.+)', text)
+    if m:
+        affected, action, actor = m.groups()
+        return action, extract_addr(affected), extract_addr(actor)
+    if text.startswith("group left by "):
+        addr = extract_addr(text[13:])
+        if addr:
+            return "removed", addr, addr

python/src/deltachat/testplugin.py

@@ -1,6 +1,7 @@
 from __future__ import print_function
 import os
 import sys
+import io
 import subprocess
 import queue
 import threading
@@ -16,6 +17,7 @@ from . import Account, const
 from .capi import lib
 from .events import FFIEventLogger, FFIEventTracker
 from _pytest._code import Source
+from deltachat import direct_imap
 
 import deltachat
 
@@ -30,12 +32,13 @@ def pytest_addoption(parser):
         "--ignored", action="store_true",
         help="Also run tests marked with the ignored marker",
     )
+    parser.addoption(
+        "--strict-tls", action="store_true",
+        help="Never accept invalid TLS certificates for test accounts",
+    )
 
 
 def pytest_configure(config):
-    config.addinivalue_line(
-        "markers", "ignored: Mark test as bing slow, skipped unless --ignored is used."
-    )
     cfg = config.getoption('--liveconfig')
     if not cfg:
         cfg = os.getenv('DCC_NEW_TMP_EMAIL')
@@ -153,7 +156,7 @@ class SessionLiveConfigFromURL:
             assert index == len(self.configlist), index
             res = requests.post(self.url)
             if res.status_code != 200:
-                pytest.skip("creating newtmpuser failed {!r}".format(res))
+                pytest.skip("creating newtmpuser failed with code {}: '{}'".format(res.status_code, res.text))
             d = res.json()
             config = dict(addr=d["email"], mail_pw=d["password"])
             self.configlist.append(config)
@@ -216,6 +219,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             self._generated_keys = ["alice", "bob", "charlie",
                                     "dom", "elena", "fiona"]
             self.set_logging_default(False)
+            deltachat.register_global_plugin(direct_imap)
 
         def finalize(self):
             while self._finalizers:
@@ -226,13 +230,18 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 acc = self._accounts.pop()
                 acc.shutdown()
                 acc.disable_logging()
+            deltachat.unregister_global_plugin(direct_imap)
 
         def make_account(self, path, logid, quiet=False):
             ac = Account(path, logging=self._logging)
             ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
+            ac._evtracker.set_timeout(30)
             ac.addr = ac.get_self_contact().addr
+            ac.set_config("displayname", logid)
             if not quiet:
-                ac.add_account_plugin(FFIEventLogger(ac, logid=logid))
+                logger = FFIEventLogger(ac)
+                logger.init_time = self.init_time
+                ac.add_account_plugin(logger)
             self._accounts.append(ac)
             return ac
 
@@ -242,10 +251,7 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
         def get_unconfigured_account(self):
             self.offline_count += 1
             tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
-            ac._evtracker.init_time = self.init_time
-            ac._evtracker.set_timeout(2)
-            return ac
+            return self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
 
         def _preconfigure_key(self, account, addr):
             # Only set a key if we haven't used it yet for another account.
@@ -280,16 +286,15 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             if "e2ee_enabled" not in configdict:
                 configdict["e2ee_enabled"] = "1"
 
-            # Enable strict certificate checks for online accounts
-            configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
-            configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+            if pytestconfig.getoption("--strict-tls"):
+                # Enable strict certificate checks for online accounts
+                configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+                configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
 
             tmpdb = tmpdir.join("livedb%d" % self.live_count)
             ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count), quiet=quiet)
             if pre_generated_key:
                 self._preconfigure_key(ac, configdict['addr'])
-            ac._evtracker.init_time = self.init_time
-            ac._evtracker.set_timeout(30)
             return ac, dict(configdict)
 
         def get_online_configuring_account(self, mvbox=False, sentbox=False, move=False,
@@ -301,31 +306,27 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             configdict["mvbox_move"] = str(int(move))
             configdict["sentbox_watch"] = str(int(sentbox))
             ac.update_config(configdict)
-            ac.configure()
+            ac._configtracker = ac.configure()
             return ac
 
         def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
             ac1 = self.get_online_configuring_account(
                 pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
-            ac1.wait_configure_finish()
-            ac1.start_io()
+            self.wait_configure_and_start_io()
             return ac1
 
         def get_two_online_accounts(self, move=False, quiet=False):
-            ac1 = self.get_online_configuring_account(move=True, quiet=quiet)
+            ac1 = self.get_online_configuring_account(move=move, quiet=quiet)
             ac2 = self.get_online_configuring_account(quiet=quiet)
-            ac1.wait_configure_finish()
-            ac1.start_io()
-            ac2.wait_configure_finish()
-            ac2.start_io()
+            self.wait_configure_and_start_io()
             return ac1, ac2
 
-        def get_many_online_accounts(self, num, move=True, quiet=True):
-            accounts = [self.get_online_configuring_account(move=move, quiet=quiet)
+        def get_many_online_accounts(self, num, move=True):
+            accounts = [self.get_online_configuring_account(move=move, quiet=True)
                         for i in range(num)]
+            self.wait_configure_and_start_io()
             for acc in accounts:
-                acc._configtracker.wait_finish()
-                acc.start_io()
+                acc.add_account_plugin(FFIEventLogger(acc))
             return accounts
 
         def clone_online_account(self, account, pre_generated_key=True):
@@ -334,8 +335,6 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
             if pre_generated_key:
                 self._preconfigure_key(ac, account.get_config("addr"))
-            ac._evtracker.init_time = self.init_time
-            ac._evtracker.set_timeout(30)
             ac.update_config(dict(
                 addr=account.get_config("addr"),
                 mail_pw=account.get_config("mail_pw"),
@@ -343,14 +342,39 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
                 mvbox_move=account.get_config("mvbox_move"),
                 sentbox_watch=account.get_config("sentbox_watch"),
             ))
-            ac.configure()
+            if hasattr(account, "direct_imap"):
+                # Attach the existing direct_imap. If we did not do this, a new one would be created and
+                # delete existing messages (see dc_account_extra_configure(configure))
+                ac.direct_imap = account.direct_imap
+            ac._configtracker = ac.configure()
             return ac
 
+        def wait_configure_and_start_io(self):
+            started_accounts = []
+            for acc in self._accounts:
+                if hasattr(acc, "_configtracker"):
+                    acc._configtracker.wait_finish()
+                    acc._evtracker.consume_events()
+                    acc.get_device_chat().mark_noticed()
+                    del acc._configtracker
+                acc.set_config("bcc_self", "0")
+                if acc.is_configured() and not acc.is_started():
+                    acc.start_io()
+                    started_accounts.append(acc)
+                print("{}: {} account was successfully setup".format(
+                    acc.get_config("displayname"), acc.get_config("addr")))
+            for acc in started_accounts:
+                acc._evtracker.wait_all_initial_fetches()
+
         def run_bot_process(self, module, ffi=True):
             fn = module.__file__
 
             bot_ac, bot_cfg = self.get_online_config()
 
+            # Avoid starting ac so we don't interfere with the bot operating on
+            # the same database.
+            self._accounts.remove(bot_ac)
+
             args = [
                 sys.executable,
                 "-u",
@@ -375,9 +399,42 @@ def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
             self._finalizers.append(bot.kill)
             return bot
 
+        def dump_imap_summary(self, logfile):
+            for ac in self._accounts:
+                imap = getattr(ac, "direct_imap", None)
+                if imap is not None:
+                    try:
+                        imap.idle_done()
+                    except Exception:
+                        pass
+                    imap.dump_account_info(logfile=logfile)
+                    imap.dump_imap_structures(tmpdir, logfile=logfile)
+
+        def get_accepted_chat(self, ac1, ac2):
+            ac2.create_chat(ac1)
+            return ac1.create_chat(ac2)
+
+        def introduce_each_other(self, accounts, sending=True):
+            to_wait = []
+            for i, acc in enumerate(accounts):
+                for acc2 in accounts[i + 1:]:
+                    chat = self.get_accepted_chat(acc, acc2)
+                    if sending:
+                        chat.send_text("hi")
+                        to_wait.append(acc2)
+                        acc2.create_chat(acc).send_text("hi back")
+                        to_wait.append(acc)
+            for acc in to_wait:
+                acc._evtracker.wait_next_incoming_message()
+
     am = AccountMaker()
     request.addfinalizer(am.finalize)
-    return am
+    yield am
+    if hasattr(request.node, "rep_call") and request.node.rep_call.failed:
+        logfile = io.StringIO()
+        am.dump_imap_summary(logfile=logfile)
+        print(logfile.getvalue())
+        # request.node.add_report_section("call", "imap-server-state", s)
 
 
 class BotProcess:
@@ -445,4 +502,17 @@ def lp():
 
         def step(self, msg):
             print("-" * 5, "step " + msg, "-" * 5)
+
     return Printer()
+
+
+@pytest.hookimpl(tryfirst=True, hookwrapper=True)
+def pytest_runtest_makereport(item, call):
+    # execute all other hooks to obtain the report object
+    outcome = yield
+    rep = outcome.get_result()
+
+    # set a report attribute for each phase of a call, which can
+    # be "setup", "call", "teardown"
+
+    setattr(item, "rep_" + rep.when, rep)

python/src/deltachat/tracker.py

@@ -2,7 +2,7 @@
 from queue import Queue
 from threading import Event
 
-from .hookspec import account_hookimpl
+from .hookspec import account_hookimpl, Global
 
 
 class ImexFailed(RuntimeError):
@@ -40,12 +40,14 @@ class ConfigureFailed(RuntimeError):
 class ConfigureTracker:
     ConfigureFailed = ConfigureFailed
 
-    def __init__(self):
+    def __init__(self, account):
+        self.account = account
         self._configure_events = Queue()
         self._smtp_finished = Event()
         self._imap_finished = Event()
         self._ffi_events = []
         self._progress = Queue()
+        self._gm = Global._get_plugin_manager()
 
     @account_hookimpl
     def ac_process_ffi_event(self, ffi_event):
@@ -59,7 +61,10 @@ class ConfigureTracker:
 
     @account_hookimpl
     def ac_configure_completed(self, success):
+        if success:
+            self._gm.hook.dc_account_extra_configure(account=self.account)
         self._configure_events.put(success)
+        self.account.remove_account_plugin(self)
 
     def wait_smtp_connected(self):
         """ wait until smtp is configured. """

python/tests/stress_test_db.py

@@ -17,7 +17,7 @@ def test_db_busy_error(acfactory, tmpdir):
             print("%3.2f %s" % (time.time() - starttime, string))
 
     # make a number of accounts
-    accounts = acfactory.get_many_online_accounts(5, quiet=True)
+    accounts = acfactory.get_many_online_accounts(3, quiet=True)
     log("created %s accounts" % len(accounts))
 
     # put a bigfile into each account
@@ -41,7 +41,7 @@ def test_db_busy_error(acfactory, tmpdir):
     # each replier receives all events and sends report events to receive_queue
     repliers = []
     for acc in accounts:
-        replier = AutoReplier(acc, log=log, num_send=1000, num_bigfiles=0, report_func=report_func)
+        replier = AutoReplier(acc, log=log, num_send=500, num_bigfiles=5, report_func=report_func)
         acc.add_account_plugin(replier)
         repliers.append(replier)
 
@@ -57,9 +57,9 @@ def test_db_busy_error(acfactory, tmpdir):
             log("timeout waiting for next event")
             pytest.fail("timeout exceeded")
         if report_type == ReportType.exit:
-            replier.log("EXIT".format(alive_count))
+            replier.log("EXIT")
         elif report_type == ReportType.ffi_error:
-            replier.log("ERROR: {}".format(addr, report_args[0]))
+            replier.log("ERROR: {}".format(report_args[0]))
         elif report_type == ReportType.message_echo:
             continue
         else:
@@ -108,14 +108,15 @@ class AutoReplier:
     @deltachat.account_hookimpl
     def ac_incoming_message(self, message):
         if self.current_sent >= self.num_send:
+            self.report_func(self, ReportType.exit)
             return
-        message.accept_sender_contact()
+        message.create_chat()
         message.mark_seen()
         self.log("incoming message: {}".format(message))
 
         self.current_sent += 1
         # we are still alive, let's send a reply
-        if self.num_bigfiles and self.current_sent % self.num_bigfiles == 0:
+        if self.num_bigfiles and self.current_sent % (self.num_send / self.num_bigfiles) == 0:
             message.chat.send_text("send big file as reply to: {}".format(message.text))
             msg = message.chat.send_file(self.account.bigfile)
         else:

python/tests/test_increation.py

@@ -36,9 +36,7 @@ def wait_msgs_changed(account, msgs_list):
 class TestOnlineInCreation:
     def test_increation_not_blobdir(self, tmpdir, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
-
-        c2 = ac1.create_contact(email=ac2.get_config("addr"))
-        chat = ac1.create_chat_by_contact(c2)
+        chat = ac1.create_chat(ac2)
 
         lp.sec("Creating in-creation file outside of blobdir")
         assert tmpdir.strpath != ac1.get_blobdir()
@@ -48,9 +46,7 @@ class TestOnlineInCreation:
 
     def test_no_increation_copies_to_blobdir(self, tmpdir, acfactory, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
-
-        c2 = ac1.create_contact(email=ac2.get_config("addr"))
-        chat = ac1.create_chat_by_contact(c2)
+        chat = ac1.create_chat(ac2)
 
         lp.sec("Creating file outside of blobdir")
         assert tmpdir.strpath != ac1.get_blobdir()
@@ -64,9 +60,7 @@ class TestOnlineInCreation:
     def test_forward_increation(self, acfactory, data, lp):
         ac1, ac2 = acfactory.get_two_online_accounts()
 
-        c2 = ac1.create_contact(email=ac2.get_config("addr"))
-        chat = ac1.create_chat_by_contact(c2)
-        assert chat.id >= const.DC_CHAT_ID_LAST_SPECIAL
+        chat = ac1.create_chat(ac2)
         wait_msgs_changed(ac1, [(0, 0)])  # why no chat id?
 
         lp.sec("create a message with a file in creation")
@@ -80,7 +74,7 @@ class TestOnlineInCreation:
 
         lp.sec("forward the message while still in creation")
         chat2 = ac1.create_group_chat("newgroup")
-        chat2.add_contact(c2)
+        chat2.add_contact(ac2)
         wait_msgs_changed(ac1, [(0, 0)])  # why not chat id?
         ac1.forward_messages([prepared_original], chat2)
         # XXX there might be two EVENT_MSGS_CHANGED and only one of them

python/tests/test_lowlevel.py

@@ -69,8 +69,8 @@ def test_sig():
 def test_markseen_invalid_message_ids(acfactory):
     ac1 = acfactory.get_configured_offline_account()
 
-    contact1 = ac1.create_contact(email="some1@example.com", name="some1")
-    chat = ac1.create_chat_by_contact(contact1)
+    contact1 = ac1.create_contact("some1@example.com", name="some1")
+    chat = contact1.create_chat()
     chat.send_text("one messae")
     ac1._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
     msg_ids = [9]

python/tox.ini

@@ -7,7 +7,7 @@ envlist =
 
 [testenv]
 commands = 
-    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored {posargs: tests examples}
+    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
     python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 
@@ -71,6 +71,8 @@ norecursedirs = .tox
 xfail_strict=true
 timeout = 90
 timeout_method = thread
+markers = 
+    ignored: ignore this test in default test runs, use --ignored to run.
 
 [flake8]
 max-line-length = 120

rust-toolchain

@@ -1 +1 @@
-1.43.1
+1.45.0

set_core_version.py

@@ -5,18 +5,27 @@ import sys
 import re
 import pathlib
 import subprocess
+from argparse import ArgumentParser
 
 rex = re.compile(r'version = "(\S+)"')
 
-def read_toml_version(relpath):
+
+def regex_matches(relpath, regex=rex):
     p = pathlib.Path(relpath)
     assert p.exists()
     for line in open(str(p)):
-        m = rex.match(line)
+        m = regex.match(line)
         if m is not None:
-            return m.group(1)
+            return m
+
+
+def read_toml_version(relpath):
+    res = regex_matches(relpath, rex)
+    if res is not None:
+        return res.group(1)
     raise ValueError("no version found in {}".format(relpath))
 
+
 def replace_toml_version(relpath, newversion):
     p = pathlib.Path(relpath)
     assert p.exists()
@@ -25,18 +34,28 @@ def replace_toml_version(relpath, newversion):
         for line in open(str(p)):
             m = rex.match(line)
             if m is not None:
+                print("{}: set version={}".format(relpath, newversion))
                 f.write('version = "{}"\n'.format(newversion))
             else:
                 f.write(line)
     os.rename(tmp_path, str(p))
 
-if __name__ == "__main__":
 
-    if len(sys.argv) < 2:
-        for x in ("Cargo.toml", "deltachat-ffi/Cargo.toml"):
+def main():
+    parser = ArgumentParser(prog="set_core_version")
+    parser.add_argument("newversion")
+
+    toml_list = ["Cargo.toml", "deltachat-ffi/Cargo.toml"]
+    try:
+        opts = parser.parse_args()
+    except SystemExit:
+        print()
+        for x in toml_list:
             print("{}: {}".format(x, read_toml_version(x)))
+        print()
         raise SystemExit("need argument: new version, example: 1.25.0")
-    newversion = sys.argv[1]
+
+    newversion = opts.newversion
     if newversion.count(".") < 2:
         raise SystemExit("need at least two dots in version")
 
@@ -55,7 +74,10 @@ if __name__ == "__main__":
     replace_toml_version("Cargo.toml", newversion)
     replace_toml_version("deltachat-ffi/Cargo.toml", newversion)
 
+    print("running cargo check")
     subprocess.call(["cargo", "check"])
+
+    print("adding changes to git index")
     subprocess.call(["git", "add", "-u"])
     # subprocess.call(["cargo", "update", "-p", "deltachat"])
 
@@ -63,3 +85,8 @@ if __name__ == "__main__":
     print("")
     print("   git tag {}".format(newversion))
     print("")
+
+
+if __name__ == "__main__":
+    main()
+

spec.md

@@ -1,10 +1,12 @@
-# Chat-over-Email specification
+# chat-mail specification
 
-Version 0.30.0
+Version: 0.32.0
+Status:  In-progress 
+Format:  [Semantic Line Breaks](https://sembr.org/)
 
-This document describes how emails can be used
-to implement typical messenger functions
-while staying compatible to existing MUAs.
+This document roughly describes how chat-mail 
+apps use the standard e-mail system 
+to implement typical messenger functions.
 
 - [Encryption](#encryption)
 - [Outgoing messages](#outgoing-messages)
@@ -30,17 +32,14 @@ Messages SHOULD be encrypted by the
 `prefer-encrypt=mutual` MAY be set by default.
 
 Meta data (at least the subject and all chat-headers) SHOULD be encrypted
-by the [Memoryhole](https://github.com/autocrypt/memoryhole) standard.
-If Memoryhole is not used,
-the subject of encrypted messages SHOULD be replaced by the string `...`.
+by the [Protected Headers](https://tools.ietf.org/id/draft-autocrypt-lamps-protected-headers-02.html) standard.
 
 
 # Outgoing messages
 
 Messengers MUST add a `Chat-Version: 1.0` header to outgoing messages.
 For filtering and smart appearance of the messages in normal MUAs,
-the `Subject` header SHOULD start with the characters `Chat:`
-and SHOULD be an excerpt of the message.
+the `Subject` header SHOULD be `Message from <sender name>`.
 Replies to messages MAY follow the typical `Re:`-format.
 
 The body MAY contain text which MUST have the content type `text/plain`
@@ -58,7 +57,7 @@ Full quotes, footers or sth. like that MUST NOT go to the user-text-part.
     To: rcpt@domain
     Chat-Version: 1.0
     Content-Type: text/plain
-    Subject: Chat: Hello ...
+    Subject: Message from sender@domain
 
     Hello world!
 

src/accounts.rs

@@ -0,0 +1,531 @@
+use std::collections::BTreeMap;
+
+use async_std::fs;
+use async_std::path::PathBuf;
+use async_std::prelude::*;
+use async_std::sync::{Arc, RwLock};
+use uuid::Uuid;
+
+use anyhow::{ensure, Context as _};
+use serde::{Deserialize, Serialize};
+
+use crate::context::Context;
+use crate::error::Result;
+use crate::events::Event;
+
+/// Account manager, that can handle multiple accounts in a single place.
+#[derive(Debug, Clone)]
+pub struct Accounts {
+    dir: PathBuf,
+    config: Config,
+    accounts: Arc<RwLock<BTreeMap<u32, Context>>>,
+}
+
+impl Accounts {
+    /// Loads or creates an accounts folder at the given `dir`.
+    pub async fn new(os_name: String, dir: PathBuf) -> Result<Self> {
+        if !dir.exists().await {
+            Accounts::create(os_name, &dir).await?;
+        }
+
+        Accounts::open(dir).await
+    }
+
+    /// Creates a new default structure, including a default account.
+    pub async fn create(os_name: String, dir: &PathBuf) -> Result<()> {
+        fs::create_dir_all(dir)
+            .await
+            .context("failed to create folder")?;
+
+        // create default account
+        let config = Config::new(os_name.clone(), dir).await?;
+        let account_config = config.new_account(dir).await?;
+
+        Context::new(os_name, account_config.dbfile().into(), account_config.id)
+            .await
+            .context("failed to create default account")?;
+
+        Ok(())
+    }
+
+    /// Opens an existing accounts structure. Will error if the folder doesn't exist,
+    /// no account exists and no config exists.
+    pub async fn open(dir: PathBuf) -> Result<Self> {
+        ensure!(dir.exists().await, "directory does not exist");
+
+        let config_file = dir.join(CONFIG_NAME);
+        ensure!(config_file.exists().await, "accounts.toml does not exist");
+
+        let config = Config::from_file(config_file).await?;
+        let accounts = config.load_accounts().await?;
+
+        Ok(Self {
+            dir,
+            config,
+            accounts: Arc::new(RwLock::new(accounts)),
+        })
+    }
+
+    /// Get an account by its `id`:
+    pub async fn get_account(&self, id: u32) -> Option<Context> {
+        self.accounts.read().await.get(&id).cloned()
+    }
+
+    /// Get the currently selected account.
+    pub async fn get_selected_account(&self) -> Context {
+        let id = self.config.get_selected_account().await;
+        self.accounts
+            .read()
+            .await
+            .get(&id)
+            .cloned()
+            .expect("inconsistent state")
+    }
+
+    /// Select the given account.
+    pub async fn select_account(&self, id: u32) -> Result<()> {
+        self.config.select_account(id).await?;
+
+        Ok(())
+    }
+
+    /// Add a new account.
+    pub async fn add_account(&self) -> Result<u32> {
+        let os_name = self.config.os_name().await;
+        let account_config = self.config.new_account(&self.dir).await?;
+
+        let ctx = Context::new(os_name, account_config.dbfile().into(), account_config.id).await?;
+        self.accounts.write().await.insert(account_config.id, ctx);
+
+        Ok(account_config.id)
+    }
+
+    /// Remove an account.
+    pub async fn remove_account(&self, id: u32) -> Result<()> {
+        let ctx = self.accounts.write().await.remove(&id);
+        ensure!(ctx.is_some(), "no account with this id: {}", id);
+        let ctx = ctx.unwrap();
+        ctx.stop_io().await;
+        drop(ctx);
+
+        if let Some(cfg) = self.config.get_account(id).await {
+            fs::remove_dir_all(async_std::path::PathBuf::from(&cfg.dir))
+                .await
+                .context("failed to remove account data")?;
+        }
+        self.config.remove_account(id).await?;
+
+        Ok(())
+    }
+
+    /// Migrate an existing account into this structure.
+    pub async fn migrate_account(&self, dbfile: PathBuf) -> Result<u32> {
+        let blobdir = Context::derive_blobdir(&dbfile);
+
+        ensure!(
+            dbfile.exists().await,
+            "no database found: {}",
+            dbfile.display()
+        );
+        ensure!(
+            blobdir.exists().await,
+            "no blobdir found: {}",
+            blobdir.display()
+        );
+
+        let old_id = self.config.get_selected_account().await;
+
+        // create new account
+        let account_config = self.config.new_account(&self.dir).await?;
+
+        let new_dbfile = account_config.dbfile().into();
+        let new_blobdir = Context::derive_blobdir(&new_dbfile);
+
+        let res = {
+            fs::create_dir_all(&account_config.dir).await?;
+            fs::rename(&dbfile, &new_dbfile).await?;
+            fs::rename(&blobdir, &new_blobdir).await?;
+            Ok(())
+        };
+
+        match res {
+            Ok(_) => {
+                let ctx = Context::with_blobdir(
+                    self.config.os_name().await,
+                    new_dbfile,
+                    new_blobdir,
+                    account_config.id,
+                )
+                .await?;
+                self.accounts.write().await.insert(account_config.id, ctx);
+                Ok(account_config.id)
+            }
+            Err(err) => {
+                // remove temp account
+                fs::remove_dir_all(async_std::path::PathBuf::from(&account_config.dir))
+                    .await
+                    .context("failed to remove account data")?;
+
+                self.config.remove_account(account_config.id).await?;
+
+                // set selection back
+                self.select_account(old_id).await?;
+
+                Err(err)
+            }
+        }
+    }
+
+    /// Get a list of all account ids.
+    pub async fn get_all(&self) -> Vec<u32> {
+        self.accounts.read().await.keys().copied().collect()
+    }
+
+    /// Import a backup using a new account and selects it.
+    pub async fn import_account(&self, file: PathBuf) -> Result<u32> {
+        let old_id = self.config.get_selected_account().await;
+
+        let id = self.add_account().await?;
+        let ctx = self.get_account(id).await.expect("just added");
+
+        match crate::imex::imex(&ctx, crate::imex::ImexMode::ImportBackup, Some(file)).await {
+            Ok(_) => Ok(id),
+            Err(err) => {
+                // remove temp account
+                self.remove_account(id).await?;
+                // set selection back
+                self.select_account(old_id).await?;
+                Err(err)
+            }
+        }
+    }
+
+    pub async fn start_io(&self) {
+        let accounts = &*self.accounts.read().await;
+        for account in accounts.values() {
+            account.start_io().await;
+        }
+    }
+
+    pub async fn stop_io(&self) {
+        let accounts = &*self.accounts.read().await;
+        for account in accounts.values() {
+            account.stop_io().await;
+        }
+    }
+
+    pub async fn maybe_network(&self) {
+        let accounts = &*self.accounts.read().await;
+        for account in accounts.values() {
+            account.maybe_network().await;
+        }
+    }
+
+    /// Unified event emitter.
+    pub async fn get_event_emitter(&self) -> EventEmitter {
+        let emitters: Vec<_> = self
+            .accounts
+            .read()
+            .await
+            .iter()
+            .map(|(_id, a)| a.get_event_emitter())
+            .collect();
+
+        EventEmitter(futures::stream::select_all(emitters))
+    }
+}
+
+#[derive(Debug)]
+pub struct EventEmitter(futures::stream::SelectAll<crate::events::EventEmitter>);
+
+impl EventEmitter {
+    /// Blocking recv of an event. Return `None` if all `Sender`s have been droped.
+    pub fn recv_sync(&mut self) -> Option<Event> {
+        async_std::task::block_on(self.recv())
+    }
+
+    /// Async recv of an event. Return `None` if all `Sender`s have been droped.
+    pub async fn recv(&mut self) -> Option<Event> {
+        self.0.next().await
+    }
+}
+
+impl async_std::stream::Stream for EventEmitter {
+    type Item = Event;
+
+    fn poll_next(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Self::Item>> {
+        std::pin::Pin::new(&mut self.0).poll_next(cx)
+    }
+}
+
+pub const CONFIG_NAME: &str = "accounts.toml";
+pub const DB_NAME: &str = "dc.db";
+
+#[derive(Debug, Clone)]
+pub struct Config {
+    file: PathBuf,
+    inner: Arc<RwLock<InnerConfig>>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
+struct InnerConfig {
+    pub os_name: String,
+    /// The currently selected account.
+    pub selected_account: u32,
+    pub next_id: u32,
+    pub accounts: Vec<AccountConfig>,
+}
+
+impl Config {
+    pub async fn new(os_name: String, dir: &PathBuf) -> Result<Self> {
+        let cfg = Config {
+            file: dir.join(CONFIG_NAME),
+            inner: Arc::new(RwLock::new(InnerConfig {
+                os_name,
+                accounts: Vec::new(),
+                selected_account: 0,
+                next_id: 1,
+            })),
+        };
+
+        cfg.sync().await?;
+
+        Ok(cfg)
+    }
+
+    pub async fn os_name(&self) -> String {
+        self.inner.read().await.os_name.clone()
+    }
+
+    /// Sync the inmemory representation to disk.
+    async fn sync(&self) -> Result<()> {
+        fs::write(
+            &self.file,
+            toml::to_string_pretty(&*self.inner.read().await)?,
+        )
+        .await
+        .context("failed to write config")
+    }
+
+    /// Read a configuration from the given file into memory.
+    pub async fn from_file(file: PathBuf) -> Result<Self> {
+        let bytes = fs::read(&file).await.context("failed to read file")?;
+        let inner: InnerConfig = toml::from_slice(&bytes).context("failed to parse config")?;
+
+        Ok(Config {
+            file,
+            inner: Arc::new(RwLock::new(inner)),
+        })
+    }
+
+    pub async fn load_accounts(&self) -> Result<BTreeMap<u32, Context>> {
+        let cfg = &*self.inner.read().await;
+        let mut accounts = BTreeMap::new();
+        for account_config in &cfg.accounts {
+            let ctx = Context::new(
+                cfg.os_name.clone(),
+                account_config.dbfile().into(),
+                account_config.id,
+            )
+            .await?;
+            accounts.insert(account_config.id, ctx);
+        }
+
+        Ok(accounts)
+    }
+
+    /// Create a new account in the given root directory.
+    pub async fn new_account(&self, dir: &PathBuf) -> Result<AccountConfig> {
+        let id = {
+            let inner = &mut self.inner.write().await;
+            let id = inner.next_id;
+            let uuid = Uuid::new_v4();
+            let target_dir = dir.join(uuid.to_simple_ref().to_string());
+
+            inner.accounts.push(AccountConfig {
+                id,
+                name: String::new(),
+                dir: target_dir.into(),
+                uuid,
+            });
+            inner.next_id += 1;
+            id
+        };
+
+        self.sync().await?;
+
+        self.select_account(id).await.expect("just added");
+        let cfg = self.get_account(id).await.expect("just added");
+        Ok(cfg)
+    }
+
+    /// Removes an existing acccount entirely.
+    pub async fn remove_account(&self, id: u32) -> Result<()> {
+        {
+            let inner = &mut *self.inner.write().await;
+            if let Some(idx) = inner.accounts.iter().position(|e| e.id == id) {
+                // remove account from the configs
+                inner.accounts.remove(idx);
+            }
+            if inner.selected_account == id {
+                // reset selected account
+                inner.selected_account = inner.accounts.get(0).map(|e| e.id).unwrap_or_default();
+            }
+        }
+
+        self.sync().await
+    }
+
+    pub async fn get_account(&self, id: u32) -> Option<AccountConfig> {
+        self.inner
+            .read()
+            .await
+            .accounts
+            .iter()
+            .find(|e| e.id == id)
+            .cloned()
+    }
+
+    pub async fn get_selected_account(&self) -> u32 {
+        self.inner.read().await.selected_account
+    }
+
+    pub async fn select_account(&self, id: u32) -> Result<()> {
+        {
+            let inner = &mut *self.inner.write().await;
+            ensure!(
+                inner.accounts.iter().any(|e| e.id == id),
+                "invalid account id: {}",
+                id
+            );
+
+            inner.selected_account = id;
+        }
+
+        self.sync().await?;
+        Ok(())
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
+pub struct AccountConfig {
+    /// Unique id.
+    pub id: u32,
+    /// Display name
+    pub name: String,
+    /// Root directory for all data for this account.
+    pub dir: std::path::PathBuf,
+    pub uuid: Uuid,
+}
+
+impl AccountConfig {
+    /// Get the canoncial dbfile name for this configuration.
+    pub fn dbfile(&self) -> std::path::PathBuf {
+        self.dir.join(DB_NAME)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[async_std::test]
+    async fn test_account_new_open() {
+        let dir = tempfile::tempdir().unwrap();
+        let p: PathBuf = dir.path().join("accounts1").into();
+
+        let accounts1 = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+        let accounts2 = Accounts::open(p).await.unwrap();
+
+        assert_eq!(accounts1.accounts.read().await.len(), 1);
+        assert_eq!(accounts1.config.get_selected_account().await, 1);
+
+        assert_eq!(accounts1.dir, accounts2.dir);
+        assert_eq!(
+            &*accounts1.config.inner.read().await,
+            &*accounts2.config.inner.read().await,
+        );
+        assert_eq!(
+            accounts1.accounts.read().await.len(),
+            accounts2.accounts.read().await.len()
+        );
+    }
+
+    #[async_std::test]
+    async fn test_account_new_add_remove() {
+        let dir = tempfile::tempdir().unwrap();
+        let p: PathBuf = dir.path().join("accounts").into();
+
+        let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+
+        assert_eq!(accounts.accounts.read().await.len(), 1);
+        assert_eq!(accounts.config.get_selected_account().await, 1);
+
+        let id = accounts.add_account().await.unwrap();
+        assert_eq!(id, 2);
+        assert_eq!(accounts.config.get_selected_account().await, id);
+        assert_eq!(accounts.accounts.read().await.len(), 2);
+
+        accounts.select_account(1).await.unwrap();
+        assert_eq!(accounts.config.get_selected_account().await, 1);
+
+        accounts.remove_account(1).await.unwrap();
+        assert_eq!(accounts.config.get_selected_account().await, 2);
+        assert_eq!(accounts.accounts.read().await.len(), 1);
+    }
+
+    #[async_std::test]
+    async fn test_migrate_account() {
+        let dir = tempfile::tempdir().unwrap();
+        let p: PathBuf = dir.path().join("accounts").into();
+
+        let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+        assert_eq!(accounts.accounts.read().await.len(), 1);
+        assert_eq!(accounts.config.get_selected_account().await, 1);
+
+        let extern_dbfile: PathBuf = dir.path().join("other").into();
+        let ctx = Context::new("my_os".into(), extern_dbfile.clone(), 0)
+            .await
+            .unwrap();
+        ctx.set_config(crate::config::Config::Addr, Some("me@mail.com"))
+            .await
+            .unwrap();
+
+        drop(ctx);
+
+        accounts
+            .migrate_account(extern_dbfile.clone())
+            .await
+            .unwrap();
+        assert_eq!(accounts.accounts.read().await.len(), 2);
+        assert_eq!(accounts.config.get_selected_account().await, 2);
+
+        let ctx = accounts.get_selected_account().await;
+        assert_eq!(
+            "me@mail.com",
+            ctx.get_config(crate::config::Config::Addr).await.unwrap()
+        );
+    }
+
+    /// Tests that accounts are sorted by ID.
+    #[async_std::test]
+    async fn test_accounts_sorted() {
+        let dir = tempfile::tempdir().unwrap();
+        let p: PathBuf = dir.path().join("accounts").into();
+
+        let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+
+        for expected_id in 2..10 {
+            let id = accounts.add_account().await.unwrap();
+            assert_eq!(id, expected_id);
+        }
+
+        let ids = accounts.get_all().await;
+        for (i, expected_id) in (1..10).enumerate() {
+            assert_eq!(ids.get(i), Some(&expected_id));
+        }
+    }
+}

src/blob.rs

@@ -8,11 +8,15 @@ use async_std::prelude::*;
 use async_std::{fs, io};
 
 use image::GenericImageView;
+use num_traits::FromPrimitive;
 use thiserror::Error;
 
-use crate::constants::AVATAR_SIZE;
+use crate::config::Config;
+use crate::constants::*;
 use crate::context::Context;
-use crate::events::Event;
+use crate::error::Error;
+use crate::events::EventType;
+use crate::message;
 
 /// Represents a file in the blob directory.
 ///
@@ -57,13 +61,13 @@ impl<'a> BlobObject<'a> {
             .map_err(|err| BlobError::WriteFailure {
                 blobdir: blobdir.to_path_buf(),
                 blobname: name.clone(),
-                cause: err,
+                cause: err.into(),
             })?;
         let blob = BlobObject {
             blobdir,
             name: format!("$BLOBDIR/{}", name),
         };
-        context.emit_event(Event::NewBlobFile(blob.as_name().to_string()));
+        context.emit_event(EventType::NewBlobFile(blob.as_name().to_string()));
         Ok(blob)
     }
 
@@ -151,7 +155,7 @@ impl<'a> BlobObject<'a> {
             blobdir: context.get_blobdir(),
             name: format!("$BLOBDIR/{}", name),
         };
-        context.emit_event(Event::NewBlobFile(blob.as_name().to_string()));
+        context.emit_event(EventType::NewBlobFile(blob.as_name().to_string()));
         Ok(blob)
     }
 
@@ -164,6 +168,9 @@ impl<'a> BlobObject<'a> {
     /// subdirectory is used and [BlobObject::sanitise_name] does not
     /// modify the filename.
     ///
+    /// Paths into the blob directory may be either defined by an absolute path
+    /// or by the relative prefix `$BLOBDIR`.
+    ///
     /// # Errors
     ///
     /// This merely delegates to the [BlobObject::create_and_copy] and
@@ -175,6 +182,11 @@ impl<'a> BlobObject<'a> {
     ) -> std::result::Result<BlobObject<'_>, BlobError> {
         if src.as_ref().starts_with(context.get_blobdir()) {
             BlobObject::from_path(context, src)
+        } else if src.as_ref().starts_with("$BLOBDIR/") {
+            BlobObject::from_name(
+                context,
+                src.as_ref().to_str().unwrap_or_default().to_string(),
+            )
         } else {
             BlobObject::create_and_copy(context, src).await
         }
@@ -372,11 +384,73 @@ impl<'a> BlobObject<'a> {
         img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
             blobdir: context.get_blobdir().to_path_buf(),
             blobname: blob_abs.to_str().unwrap_or_default().to_string(),
-            cause: err,
+            cause: err.into(),
         })?;
 
         Ok(())
     }
+
+    pub async fn recode_to_image_size(&self, context: &Context) -> Result<(), BlobError> {
+        let blob_abs = self.to_abs_path();
+        if message::guess_msgtype_from_suffix(Path::new(&blob_abs))
+            != Some((Viewtype::Image, "image/jpeg"))
+        {
+            return Ok(());
+        }
+
+        let img = image::open(&blob_abs).map_err(|err| BlobError::RecodeFailure {
+            blobdir: context.get_blobdir().to_path_buf(),
+            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+            cause: err,
+        })?;
+
+        let img_wh = if MediaQuality::from_i32(context.get_config_int(Config::MediaQuality).await)
+            .unwrap_or_default()
+            == MediaQuality::Balanced
+        {
+            BALANCED_IMAGE_SIZE
+        } else {
+            WORSE_IMAGE_SIZE
+        };
+
+        if img.width() <= img_wh && img.height() <= img_wh {
+            return Ok(());
+        }
+
+        let mut img = img.thumbnail(img_wh, img_wh);
+        match self.get_exif_orientation(context) {
+            Ok(90) => img = img.rotate90(),
+            Ok(180) => img = img.rotate180(),
+            Ok(270) => img = img.rotate270(),
+            _ => {}
+        }
+
+        img.save(&blob_abs).map_err(|err| BlobError::WriteFailure {
+            blobdir: context.get_blobdir().to_path_buf(),
+            blobname: blob_abs.to_str().unwrap_or_default().to_string(),
+            cause: err.into(),
+        })?;
+
+        Ok(())
+    }
+
+    pub fn get_exif_orientation(&self, context: &Context) -> Result<i32, Error> {
+        let file = std::fs::File::open(self.to_abs_path())?;
+        let mut bufreader = std::io::BufReader::new(&file);
+        let exifreader = exif::Reader::new();
+        let exif = exifreader.read_from_container(&mut bufreader)?;
+        if let Some(orientation) = exif.get_field(exif::Tag::Orientation, exif::In::PRIMARY) {
+            // possible orientation values are described at http://sylvana.net/jpegcrop/exif_orientation.html
+            // we only use rotation, in practise, flipping is not used.
+            match orientation.value.get_uint(0) {
+                Some(3) => return Ok(180),
+                Some(6) => return Ok(90),
+                Some(8) => return Ok(270),
+                other => warn!(context, "exif orientation value ignored: {:?}", other),
+            }
+        }
+        Ok(0)
+    }
 }
 
 impl<'a> fmt::Display for BlobObject<'a> {
@@ -400,7 +474,7 @@ pub enum BlobError {
         blobdir: PathBuf,
         blobname: String,
         #[source]
-        cause: std::io::Error,
+        cause: anyhow::Error,
     },
     #[error("Failed to copy data from {} to blob {blobname} in {}", .src.display(), .blobdir.display())]
     CopyFailure {
@@ -431,7 +505,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let blob = BlobObject::create(&t.ctx, "foo", b"hello").await.unwrap();
         let fname = t.ctx.get_blobdir().join("foo");
         let data = fs::read(fname).await.unwrap();
@@ -442,7 +516,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_lowercase_ext() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let blob = BlobObject::create(&t.ctx, "foo.TXT", b"hello")
             .await
             .unwrap();
@@ -451,7 +525,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_as_file_name() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
             .await
             .unwrap();
@@ -460,7 +534,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_as_rel_path() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
             .await
             .unwrap();
@@ -469,7 +543,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_suffix() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let blob = BlobObject::create(&t.ctx, "foo.txt", b"hello")
             .await
             .unwrap();
@@ -480,7 +554,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_dup() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         BlobObject::create(&t.ctx, "foo.txt", b"hello")
             .await
             .unwrap();
@@ -504,7 +578,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_double_ext_preserved() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         BlobObject::create(&t.ctx, "foo.tar.gz", b"hello")
             .await
             .unwrap();
@@ -529,7 +603,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_long_names() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let s = "1".repeat(150);
         let blob = BlobObject::create(&t.ctx, &s, b"data").await.unwrap();
         let blobname = blob.as_name().split('/').last().unwrap();
@@ -538,7 +612,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_and_copy() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let src = t.dir.path().join("src");
         fs::write(&src, b"boo").await.unwrap();
         let blob = BlobObject::create_and_copy(&t.ctx, &src).await.unwrap();
@@ -554,7 +628,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_from_path() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         let src_ext = t.dir.path().join("external");
         fs::write(&src_ext, b"boo").await.unwrap();
@@ -572,7 +646,7 @@ mod tests {
     }
     #[async_std::test]
     async fn test_create_from_name_long() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let src_ext = t.dir.path().join("autocrypt-setup-message-4137848473.html");
         fs::write(&src_ext, b"boo").await.unwrap();
         let blob = BlobObject::new_from_path(&t.ctx, &src_ext).await.unwrap();
@@ -594,8 +668,40 @@ mod tests {
 
     #[test]
     fn test_sanitise_name() {
-        let (_, ext) =
+        let (stem, ext) =
             BlobObject::sanitise_name("Я ЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯЯ.txt");
         assert_eq!(ext, ".txt");
+        assert!(!stem.is_empty());
+
+        // the extensions are kept together as between stem and extension a number may be added -
+        // and `foo.tar.gz` should become `foo-1234.tar.gz` and not `foo.tar-1234.gz`
+        let (stem, ext) = BlobObject::sanitise_name("wot.tar.gz");
+        assert_eq!(stem, "wot");
+        assert_eq!(ext, ".tar.gz");
+
+        let (stem, ext) = BlobObject::sanitise_name(".foo.bar");
+        assert_eq!(stem, "");
+        assert_eq!(ext, ".foo.bar");
+
+        let (stem, ext) = BlobObject::sanitise_name("foo?.bar");
+        assert!(stem.contains("foo"));
+        assert!(!stem.contains('?'));
+        assert_eq!(ext, ".bar");
+
+        let (stem, ext) = BlobObject::sanitise_name("no-extension");
+        assert_eq!(stem, "no-extension");
+        assert_eq!(ext, "");
+
+        let (stem, ext) = BlobObject::sanitise_name("path/ignored\\this: is* forbidden?.c");
+        assert_eq!(ext, ".c");
+        assert!(!stem.contains("path"));
+        assert!(!stem.contains("ignored"));
+        assert!(stem.contains("this"));
+        assert!(stem.contains("forbidden"));
+        assert!(!stem.contains('/'));
+        assert!(!stem.contains('\\'));
+        assert!(!stem.contains(':'));
+        assert!(!stem.contains('*'));
+        assert!(!stem.contains('?'));
     }
 }

src/chatlist.rs

@@ -5,6 +5,7 @@ use crate::chat::*;
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::*;
+use crate::ephemeral::delete_expired_messages;
 use crate::error::{bail, ensure, Result};
 use crate::lot::Lot;
 use crate::message::{Message, MessageState, MsgId};
@@ -76,7 +77,7 @@ impl Chatlist {
     ///   chats
     /// - the flag DC_GCL_FOR_FORWARDING sorts "Saved messages" to the top of the chatlist
     ///   and hides the device-chat,
-    //    typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS
+    ///   typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS
     /// - if the flag DC_GCL_NO_SPECIALS is set, deaddrop and archive link are not added
     ///   to the list (may be used eg. for selecting chats on forwarding, the flag is
     ///   not needed when DC_GCL_ARCHIVED_ONLY is already set)
@@ -99,7 +100,7 @@ impl Chatlist {
 
         // Note that we do not emit DC_EVENT_MSGS_MODIFIED here even if some
         // messages get deleted to avoid reloading the same chatlist.
-        if let Err(err) = delete_device_expired_messages(context).await {
+        if let Err(err) = delete_expired_messages(context).await {
             warn!(context, "Failed to hide expired messages: {}", err);
         }
 
@@ -147,11 +148,12 @@ impl Chatlist {
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
+                       AND m.id=(
+                               SELECT id
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1))
+                                  AND (hidden=0 OR state=?1)
+                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9
                    AND c.blocked=0
                    AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?2)
@@ -173,11 +175,12 @@ impl Chatlist {
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
+                       AND m.id=(
+                               SELECT id
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?))
+                                  AND (hidden=0 OR state=?)
+                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9
                    AND c.blocked=0
                    AND c.archived=1
@@ -206,11 +209,12 @@ impl Chatlist {
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
+                       AND m.id=(
+                               SELECT id
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1))
+                                  AND (hidden=0 OR state=?1)
+                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9 AND c.id!=?2
                    AND c.blocked=0
                    AND c.name LIKE ?3
@@ -236,11 +240,12 @@ impl Chatlist {
                  FROM chats c
                  LEFT JOIN msgs m
                         ON c.id=m.chat_id
-                       AND m.timestamp=(
-                               SELECT MAX(timestamp)
+                       AND m.id=(
+                               SELECT id
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1))
+                                  AND (hidden=0 OR state=?1)
+                                  ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9 AND c.id!=?2
                    AND c.blocked=0
                    AND NOT c.archived=?3
@@ -324,20 +329,30 @@ impl Chatlist {
         // This is because we may want to display drafts here or stuff as
         // "is typing".
         // Also, sth. as "No messages" would not work if the summary comes from a message.
-        let mut ret = Lot::new();
-
         let (chat_id, lastmsg_id) = match self.ids.get(index) {
             Some(ids) => ids,
             None => {
+                let mut ret = Lot::new();
                 ret.text2 = Some("ErrBadChatlistIndex".to_string());
-                return ret;
+                return Lot::new();
             }
         };
 
+        Chatlist::get_summary2(context, *chat_id, *lastmsg_id, chat).await
+    }
+
+    pub async fn get_summary2(
+        context: &Context,
+        chat_id: ChatId,
+        lastmsg_id: MsgId,
+        chat: Option<&Chat>,
+    ) -> Lot {
+        let mut ret = Lot::new();
+
         let chat_loaded: Chat;
         let chat = if let Some(chat) = chat {
             chat
-        } else if let Ok(chat) = Chat::load_from_db(context, *chat_id).await {
+        } else if let Ok(chat) = Chat::load_from_db(context, chat_id).await {
             chat_loaded = chat;
             &chat_loaded
         } else {
@@ -346,10 +361,8 @@ impl Chatlist {
 
         let mut lastcontact = None;
 
-        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, *lastmsg_id).await {
-            if lastmsg.from_id != DC_CONTACT_ID_SELF
-                && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup)
-            {
+        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id).await {
+            if lastmsg.from_id != DC_CONTACT_ID_SELF && chat.typ == Chattype::Group {
                 lastcontact = Contact::load_from_db(context, lastmsg.from_id).await.ok();
             }
 
@@ -424,14 +437,14 @@ mod tests {
 
     #[async_std::test]
     async fn test_try_load() {
-        let t = dummy_context().await;
-        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
+        let t = TestContext::new().await;
+        let chat_id1 = create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "a chat")
             .await
             .unwrap();
-        let chat_id2 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "b chat")
+        let chat_id2 = create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "b chat")
             .await
             .unwrap();
-        let chat_id3 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "c chat")
+        let chat_id3 = create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "c chat")
             .await
             .unwrap();
 
@@ -472,9 +485,9 @@ mod tests {
 
     #[async_std::test]
     async fn test_sort_self_talk_up_on_forward() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx.update_device_chats().await.unwrap();
-        create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
+        create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "a chat")
             .await
             .unwrap();
 
@@ -497,7 +510,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_search_special_chat_names() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx.update_device_chats().await.unwrap();
 
         let chats = Chatlist::try_load(&t.ctx, 0, Some("t-1234-s"), None)
@@ -530,8 +543,8 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_summary_unwrap() {
-        let t = dummy_context().await;
-        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat")
+        let t = TestContext::new().await;
+        let chat_id1 = create_group_chat(&t.ctx, ProtectionStatus::Unprotected, "a chat")
             .await
             .unwrap();
 

src/config.rs

@@ -8,7 +8,8 @@ use crate::chat::ChatId;
 use crate::constants::DC_VERSION_STR;
 use crate::context::Context;
 use crate::dc_tools::*;
-use crate::events::Event;
+use crate::events::EventType;
+use crate::job;
 use crate::message::MsgId;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::stock::StockMessage;
@@ -24,11 +25,13 @@ pub enum Config {
     MailUser,
     MailPw,
     MailPort,
+    MailSecurity,
     ImapCertificateChecks,
     SendServer,
     SendUser,
     SendPw,
     SendPort,
+    SendSecurity,
     SmtpCertificateChecks,
     ServerFlags,
 
@@ -66,6 +69,9 @@ pub enum Config {
     #[strum(props(default = "0"))] // also change MediaQuality.default() on changes
     MediaQuality,
 
+    #[strum(props(default = "1"))]
+    FetchExisting,
+
     #[strum(props(default = "0"))]
     KeyGenType,
 
@@ -104,6 +110,9 @@ pub enum Config {
     ConfiguredServerFlags,
     ConfiguredSendSecurity,
     ConfiguredE2EEEnabled,
+    ConfiguredInboxFolder,
+    ConfiguredMvboxFolder,
+    ConfiguredSentboxFolder,
     Configured,
 
     #[strum(serialize = "sys.version")]
@@ -114,9 +123,23 @@ pub enum Config {
 
     #[strum(serialize = "sys.config_keys")]
     SysConfigKeys,
+
+    Bot,
+
+    /// Whether we send a warning if the password is wrong (set to false when we send a warning
+    /// because we do not want to send a second warning)
+    #[strum(props(default = "0"))]
+    NotifyAboutWrongPw,
+
+    /// address to webrtc instance to use for videochats
+    WebrtcInstance,
 }
 
 impl Context {
+    pub async fn config_exists(&self, key: Config) -> bool {
+        self.sql.get_raw_config(self, key).await.is_some()
+    }
+
     /// Get a configuration key. Returns `None` if no value is set, and no default value found.
     pub async fn get_config(&self, key: Config) -> Option<String> {
         let value = match key {
@@ -137,6 +160,7 @@ impl Context {
         // Default values
         match key {
             Config::Selfstatus => Some(self.stock_str(StockMessage::StatusLine).await.into_owned()),
+            Config::ConfiguredInboxFolder => Some("INBOX".to_owned()),
             _ => key.get_str("default").map(|s| s.to_string()),
         }
     }
@@ -197,21 +221,6 @@ impl Context {
                     None => self.sql.set_raw_config(self, key, None).await,
                 }
             }
-            Config::InboxWatch => {
-                let ret = self.sql.set_raw_config(self, key, value).await;
-                self.interrupt_inbox().await;
-                ret
-            }
-            Config::SentboxWatch => {
-                let ret = self.sql.set_raw_config(self, key, value).await;
-                self.interrupt_sentbox().await;
-                ret
-            }
-            Config::MvboxWatch => {
-                let ret = self.sql.set_raw_config(self, key, value).await;
-                self.interrupt_mvbox().await;
-                ret
-            }
             Config::Selfstatus => {
                 let def = self.stock_str(StockMessage::StatusLine).await;
                 let val = if value.is_none() || value.unwrap() == def {
@@ -225,12 +234,21 @@ impl Context {
             Config::DeleteDeviceAfter => {
                 let ret = self.sql.set_raw_config(self, key, value).await;
                 // Force chatlist reload to delete old messages immediately.
-                self.emit_event(Event::MsgsChanged {
+                self.emit_event(EventType::MsgsChanged {
                     msg_id: MsgId::new(0),
                     chat_id: ChatId::new(0),
                 });
                 ret
             }
+            Config::Displayname => {
+                let value = value.map(improve_single_line_input);
+                self.sql.set_raw_config(self, key, value.as_deref()).await
+            }
+            Config::DeleteServerAfter => {
+                let ret = self.sql.set_raw_config(self, key, value).await;
+                job::schedule_resync(self).await;
+                ret
+            }
             _ => self.sql.set_raw_config(self, key, value).await,
         }
     }
@@ -281,7 +299,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_selfavatar_outside_blobdir() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let avatar_src = t.dir.path().join("avatar.jpg");
         let avatar_bytes = include_bytes!("../test-data/image/avatar1000x1000.jpg");
         File::create(&avatar_src)
@@ -310,7 +328,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_selfavatar_in_blobdir() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let avatar_src = t.ctx.get_blobdir().join("avatar.png");
         let avatar_bytes = include_bytes!("../test-data/image/avatar900x900.png");
         File::create(&avatar_src)
@@ -336,7 +354,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_selfavatar_copy_without_recode() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let avatar_src = t.dir.path().join("avatar.png");
         let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
         File::create(&avatar_src)
@@ -360,7 +378,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_media_quality_config_option() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let media_quality = t.ctx.get_config_int(Config::MediaQuality).await;
         assert_eq!(media_quality, 0);
         let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();

src/configure/auto_mozilla.rs

@@ -1,32 +1,31 @@
 //! # Thunderbird's Autoconfiguration implementation
 //!
-//! Documentation: https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration */
-use quick_xml::events::{BytesEnd, BytesStart, BytesText};
+//! Documentation: https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration
+use quick_xml::events::{BytesStart, Event};
+
+use std::io::BufRead;
+use std::str::FromStr;
 
-use crate::constants::*;
 use crate::context::Context;
 use crate::login_param::LoginParam;
+use crate::provider::{Protocol, Socket};
 
 use super::read_url::read_url;
-use super::Error;
+use super::{Error, ServerParams};
 
 #[derive(Debug)]
-struct MozAutoconfigure<'a> {
-    pub in_emailaddr: &'a str,
-    pub in_emaildomain: &'a str,
-    pub in_emaillocalpart: &'a str,
-    pub out: LoginParam,
-    pub out_imap_set: bool,
-    pub out_smtp_set: bool,
-    pub tag_server: MozServer,
-    pub tag_config: MozConfigTag,
+struct Server {
+    pub typ: String,
+    pub hostname: String,
+    pub port: u16,
+    pub sockettype: Socket,
+    pub username: String,
 }
 
-#[derive(Debug, PartialEq)]
-enum MozServer {
-    Undefined,
-    Imap,
-    Smtp,
+#[derive(Debug)]
+struct MozAutoconfigure {
+    pub incoming_servers: Vec<Server>,
+    pub outgoing_servers: Vec<Server>,
 }
 
 #[derive(Debug)]
@@ -38,10 +37,147 @@ enum MozConfigTag {
     Username,
 }
 
-fn parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result<LoginParam, Error> {
-    let mut reader = quick_xml::Reader::from_str(xml_raw);
-    reader.trim_text(true);
+impl Default for MozConfigTag {
+    fn default() -> Self {
+        Self::Undefined
+    }
+}
 
+impl FromStr for MozConfigTag {
+    type Err = ();
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s.trim().to_lowercase().as_ref() {
+            "hostname" => Ok(MozConfigTag::Hostname),
+            "port" => Ok(MozConfigTag::Port),
+            "sockettype" => Ok(MozConfigTag::Sockettype),
+            "username" => Ok(MozConfigTag::Username),
+            _ => Err(()),
+        }
+    }
+}
+
+/// Parses a single IncomingServer or OutgoingServer section.
+fn parse_server<B: BufRead>(
+    reader: &mut quick_xml::Reader<B>,
+    server_event: &BytesStart,
+) -> Result<Option<Server>, quick_xml::Error> {
+    let end_tag = String::from_utf8_lossy(server_event.name())
+        .trim()
+        .to_lowercase();
+
+    let typ = server_event
+        .attributes()
+        .find(|attr| {
+            attr.as_ref()
+                .map(|a| String::from_utf8_lossy(a.key).trim().to_lowercase() == "type")
+                .unwrap_or_default()
+        })
+        .map(|typ| {
+            typ.unwrap()
+                .unescape_and_decode_value(reader)
+                .unwrap_or_default()
+                .to_lowercase()
+        })
+        .unwrap_or_default();
+
+    let mut hostname = None;
+    let mut port = None;
+    let mut sockettype = Socket::Automatic;
+    let mut username = None;
+
+    let mut tag_config = MozConfigTag::Undefined;
+    let mut buf = Vec::new();
+    loop {
+        match reader.read_event(&mut buf)? {
+            Event::Start(ref event) => {
+                tag_config = String::from_utf8_lossy(event.name())
+                    .parse()
+                    .unwrap_or_default();
+            }
+            Event::End(ref event) => {
+                let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
+
+                if tag == end_tag {
+                    break;
+                }
+            }
+            Event::Text(ref event) => {
+                let val = event
+                    .unescape_and_decode(reader)
+                    .unwrap_or_default()
+                    .trim()
+                    .to_owned();
+
+                match tag_config {
+                    MozConfigTag::Hostname => hostname = Some(val),
+                    MozConfigTag::Port => port = Some(val.parse().unwrap_or_default()),
+                    MozConfigTag::Username => username = Some(val),
+                    MozConfigTag::Sockettype => {
+                        sockettype = match val.to_lowercase().as_ref() {
+                            "ssl" => Socket::SSL,
+                            "starttls" => Socket::STARTTLS,
+                            "plain" => Socket::Plain,
+                            _ => Socket::Automatic,
+                        }
+                    }
+                    _ => {}
+                }
+            }
+            Event::Eof => break,
+            _ => (),
+        }
+    }
+
+    if let (Some(hostname), Some(port), Some(username)) = (hostname, port, username) {
+        Ok(Some(Server {
+            typ,
+            hostname,
+            port,
+            sockettype,
+            username,
+        }))
+    } else {
+        Ok(None)
+    }
+}
+
+fn parse_xml_reader<B: BufRead>(
+    reader: &mut quick_xml::Reader<B>,
+) -> Result<MozAutoconfigure, quick_xml::Error> {
+    let mut incoming_servers = Vec::new();
+    let mut outgoing_servers = Vec::new();
+
+    let mut buf = Vec::new();
+    loop {
+        match reader.read_event(&mut buf)? {
+            Event::Start(ref event) => {
+                let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
+
+                if tag == "incomingserver" {
+                    if let Some(incoming_server) = parse_server(reader, event)? {
+                        incoming_servers.push(incoming_server);
+                    }
+                } else if tag == "outgoingserver" {
+                    if let Some(outgoing_server) = parse_server(reader, event)? {
+                        outgoing_servers.push(outgoing_server);
+                    }
+                }
+            }
+            Event::Eof => break,
+            _ => (),
+        }
+        buf.clear();
+    }
+
+    Ok(MozAutoconfigure {
+        incoming_servers,
+        outgoing_servers,
+    })
+}
+
+/// Parses XML and fills in address and domain placeholders.
+fn parse_xml_with_address(in_emailaddr: &str, xml_raw: &str) -> Result<MozAutoconfigure, Error> {
     // Split address into local part and domain part.
     let parts: Vec<&str> = in_emailaddr.rsplitn(2, '@').collect();
     let (in_emaillocalpart, in_emaildomain) = match &parts[..] {
@@ -49,59 +185,78 @@ fn parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result<LoginParam, Error> {
         _ => return Err(Error::InvalidEmailAddress(in_emailaddr.to_string())),
     };
 
-    let mut moz_ac = MozAutoconfigure {
-        in_emailaddr,
-        in_emaildomain,
-        in_emaillocalpart,
-        out: LoginParam::new(),
-        out_imap_set: false,
-        out_smtp_set: false,
-        tag_server: MozServer::Undefined,
-        tag_config: MozConfigTag::Undefined,
+    let mut reader = quick_xml::Reader::from_str(xml_raw);
+    reader.trim_text(true);
+
+    let moz_ac = parse_xml_reader(&mut reader).map_err(|error| Error::InvalidXml {
+        position: reader.buffer_position(),
+        error,
+    })?;
+
+    let fill_placeholders = |val: &str| -> String {
+        val.replace("%EMAILADDRESS%", in_emailaddr)
+            .replace("%EMAILLOCALPART%", in_emaillocalpart)
+            .replace("%EMAILDOMAIN%", in_emaildomain)
     };
 
-    let mut buf = Vec::new();
-    loop {
-        let event = reader
-            .read_event(&mut buf)
-            .map_err(|error| Error::InvalidXml {
-                position: reader.buffer_position(),
-                error,
-            })?;
-
-        match event {
-            quick_xml::events::Event::Start(ref e) => {
-                moz_autoconfigure_starttag_cb(e, &mut moz_ac, &reader)
-            }
-            quick_xml::events::Event::End(ref e) => moz_autoconfigure_endtag_cb(e, &mut moz_ac),
-            quick_xml::events::Event::Text(ref e) => {
-                moz_autoconfigure_text_cb(e, &mut moz_ac, &reader)
-            }
-            quick_xml::events::Event::Eof => break,
-            _ => (),
+    let fill_server_placeholders = |server: Server| -> Server {
+        Server {
+            typ: server.typ,
+            hostname: fill_placeholders(&server.hostname),
+            port: server.port,
+            sockettype: server.sockettype,
+            username: fill_placeholders(&server.username),
         }
-        buf.clear();
-    }
+    };
 
-    if moz_ac.out.mail_server.is_empty()
-        || moz_ac.out.mail_port == 0
-        || moz_ac.out.send_server.is_empty()
-        || moz_ac.out.send_port == 0
-    {
-        Err(Error::IncompleteAutoconfig(moz_ac.out))
-    } else {
-        Ok(moz_ac.out)
-    }
+    Ok(MozAutoconfigure {
+        incoming_servers: moz_ac
+            .incoming_servers
+            .into_iter()
+            .map(fill_server_placeholders)
+            .collect(),
+        outgoing_servers: moz_ac
+            .outgoing_servers
+            .into_iter()
+            .map(fill_server_placeholders)
+            .collect(),
+    })
 }
 
-pub async fn moz_autoconfigure(
+/// Parses XML into `ServerParams` vector.
+fn parse_serverparams(in_emailaddr: &str, xml_raw: &str) -> Result<Vec<ServerParams>, Error> {
+    let moz_ac = parse_xml_with_address(in_emailaddr, xml_raw)?;
+
+    let res = moz_ac
+        .incoming_servers
+        .into_iter()
+        .chain(moz_ac.outgoing_servers.into_iter())
+        .filter_map(|server| {
+            let protocol = match server.typ.as_ref() {
+                "imap" => Some(Protocol::IMAP),
+                "smtp" => Some(Protocol::SMTP),
+                _ => None,
+            };
+            Some(ServerParams {
+                protocol: protocol?,
+                socket: server.sockettype,
+                hostname: server.hostname,
+                port: server.port,
+                username: server.username,
+            })
+        })
+        .collect();
+    Ok(res)
+}
+
+pub(crate) async fn moz_autoconfigure(
     context: &Context,
     url: &str,
     param_in: &LoginParam,
-) -> Result<LoginParam, Error> {
+) -> Result<Vec<ServerParams>, Error> {
     let xml_raw = read_url(context, url).await?;
 
-    let res = parse_xml(&param_in.addr, &xml_raw);
+    let res = parse_serverparams(&param_in.addr, &xml_raw);
     if let Err(err) = &res {
         warn!(
             context,
@@ -111,212 +266,62 @@ pub async fn moz_autoconfigure(
     res
 }
 
-fn moz_autoconfigure_text_cb<B: std::io::BufRead>(
-    event: &BytesText,
-    moz_ac: &mut MozAutoconfigure,
-    reader: &quick_xml::Reader<B>,
-) {
-    let val = event.unescape_and_decode(reader).unwrap_or_default();
-
-    let addr = moz_ac.in_emailaddr;
-    let email_local = moz_ac.in_emaillocalpart;
-    let email_domain = moz_ac.in_emaildomain;
-
-    let val = val
-        .trim()
-        .replace("%EMAILADDRESS%", addr)
-        .replace("%EMAILLOCALPART%", email_local)
-        .replace("%EMAILDOMAIN%", email_domain);
-
-    match moz_ac.tag_server {
-        MozServer::Imap => match moz_ac.tag_config {
-            MozConfigTag::Hostname => moz_ac.out.mail_server = val,
-            MozConfigTag::Port => moz_ac.out.mail_port = val.parse().unwrap_or_default(),
-            MozConfigTag::Username => moz_ac.out.mail_user = val,
-            MozConfigTag::Sockettype => {
-                let val_lower = val.to_lowercase();
-                if val_lower == "ssl" {
-                    moz_ac.out.server_flags |= DC_LP_IMAP_SOCKET_SSL as i32
-                }
-                if val_lower == "starttls" {
-                    moz_ac.out.server_flags |= DC_LP_IMAP_SOCKET_STARTTLS as i32
-                }
-                if val_lower == "plain" {
-                    moz_ac.out.server_flags |= DC_LP_IMAP_SOCKET_PLAIN as i32
-                }
-            }
-            _ => {}
-        },
-        MozServer::Smtp => match moz_ac.tag_config {
-            MozConfigTag::Hostname => moz_ac.out.send_server = val,
-            MozConfigTag::Port => moz_ac.out.send_port = val.parse().unwrap_or_default(),
-            MozConfigTag::Username => moz_ac.out.send_user = val,
-            MozConfigTag::Sockettype => {
-                let val_lower = val.to_lowercase();
-                if val_lower == "ssl" {
-                    moz_ac.out.server_flags |= DC_LP_SMTP_SOCKET_SSL as i32
-                }
-                if val_lower == "starttls" {
-                    moz_ac.out.server_flags |= DC_LP_SMTP_SOCKET_STARTTLS as i32
-                }
-                if val_lower == "plain" {
-                    moz_ac.out.server_flags |= DC_LP_SMTP_SOCKET_PLAIN as i32
-                }
-            }
-            _ => {}
-        },
-        MozServer::Undefined => {}
-    }
-}
-
-fn moz_autoconfigure_endtag_cb(event: &BytesEnd, moz_ac: &mut MozAutoconfigure) {
-    let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
-
-    if tag == "incomingserver" {
-        if moz_ac.tag_server == MozServer::Imap {
-            moz_ac.out_imap_set = true;
-        }
-        moz_ac.tag_server = MozServer::Undefined;
-        moz_ac.tag_config = MozConfigTag::Undefined;
-    } else if tag == "outgoingserver" {
-        if moz_ac.tag_server == MozServer::Smtp {
-            moz_ac.out_smtp_set = true;
-        }
-        moz_ac.tag_server = MozServer::Undefined;
-        moz_ac.tag_config = MozConfigTag::Undefined;
-    } else {
-        moz_ac.tag_config = MozConfigTag::Undefined;
-    }
-}
-
-fn moz_autoconfigure_starttag_cb<B: std::io::BufRead>(
-    event: &BytesStart,
-    moz_ac: &mut MozAutoconfigure,
-    reader: &quick_xml::Reader<B>,
-) {
-    let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
-
-    if tag == "incomingserver" {
-        moz_ac.tag_server = if let Some(typ) = event.attributes().find(|attr| {
-            attr.as_ref()
-                .map(|a| String::from_utf8_lossy(a.key).trim().to_lowercase() == "type")
-                .unwrap_or_default()
-        }) {
-            let typ = typ
-                .unwrap()
-                .unescape_and_decode_value(reader)
-                .unwrap_or_default()
-                .to_lowercase();
-
-            if typ == "imap" && !moz_ac.out_imap_set {
-                MozServer::Imap
-            } else {
-                MozServer::Undefined
-            }
-        } else {
-            MozServer::Undefined
-        };
-        moz_ac.tag_config = MozConfigTag::Undefined;
-    } else if tag == "outgoingserver" {
-        moz_ac.tag_server = if !moz_ac.out_smtp_set {
-            MozServer::Smtp
-        } else {
-            MozServer::Undefined
-        };
-        moz_ac.tag_config = MozConfigTag::Undefined;
-    } else if tag == "hostname" {
-        moz_ac.tag_config = MozConfigTag::Hostname;
-    } else if tag == "port" {
-        moz_ac.tag_config = MozConfigTag::Port;
-    } else if tag == "sockettype" {
-        moz_ac.tag_config = MozConfigTag::Sockettype;
-    } else if tag == "username" {
-        moz_ac.tag_config = MozConfigTag::Username;
-    }
-}
-
 #[cfg(test)]
 mod tests {
+    #![allow(clippy::indexing_slicing)]
+
     use super::*;
 
     #[test]
     fn test_parse_outlook_autoconfig() {
-        // Copied from https://autoconfig.thunderbird.net/v1.1/outlook.com on 2019-10-11
-        let xml_raw =
-"<clientConfig version=\"1.1\">
-  <emailProvider id=\"outlook.com\">
-    <domain>hotmail.com</domain>
-    <domain>hotmail.co.uk</domain>
-    <domain>hotmail.co.jp</domain>
-    <domain>hotmail.com.br</domain>
-    <domain>hotmail.de</domain>
-    <domain>hotmail.fr</domain>
-    <domain>hotmail.it</domain>
-    <domain>hotmail.es</domain>
-    <domain>live.com</domain>
-    <domain>live.co.uk</domain>
-    <domain>live.co.jp</domain>
-    <domain>live.de</domain>
-    <domain>live.fr</domain>
-    <domain>live.it</domain>
-    <domain>live.jp</domain>
-    <domain>msn.com</domain>
-    <domain>outlook.com</domain>
-    <displayName>Outlook.com (Microsoft)</displayName>
-    <displayShortName>Outlook</displayShortName>
-    <incomingServer type=\"exchange\">
-      <hostname>outlook.office365.com</hostname>
-      <port>443</port>
-      <username>%EMAILADDRESS%</username>
-      <socketType>SSL</socketType>
-      <authentication>OAuth2</authentication>
-      <owaURL>https://outlook.office365.com/owa/</owaURL>
-      <ewsURL>https://outlook.office365.com/ews/exchange.asmx</ewsURL>
-      <useGlobalPreferredServer>true</useGlobalPreferredServer>
-    </incomingServer>
-    <incomingServer type=\"imap\">
-      <hostname>outlook.office365.com</hostname>
-      <port>993</port>
-      <socketType>SSL</socketType>
-      <authentication>password-cleartext</authentication>
-      <username>%EMAILADDRESS%</username>
-    </incomingServer>
-    <incomingServer type=\"pop3\">
-      <hostname>outlook.office365.com</hostname>
-      <port>995</port>
-      <socketType>SSL</socketType>
-      <authentication>password-cleartext</authentication>
-      <username>%EMAILADDRESS%</username>
-      <pop3>
-        <leaveMessagesOnServer>true</leaveMessagesOnServer>
-        <!-- Outlook.com docs specifically mention that POP3 deletes have effect on the main inbox on webmail and IMAP -->
-      </pop3>
-    </incomingServer>
-    <outgoingServer type=\"smtp\">
-      <hostname>smtp.office365.com</hostname>
-      <port>587</port>
-      <socketType>STARTTLS</socketType>
-      <authentication>password-cleartext</authentication>
-      <username>%EMAILADDRESS%</username>
-    </outgoingServer>
-    <documentation url=\"http://windows.microsoft.com/en-US/windows/outlook/send-receive-from-app\">
-      <descr lang=\"en\">Set up an email app with Outlook.com</descr>
-    </documentation>
-  </emailProvider>
-  <webMail>
-    <loginPage url=\"https://www.outlook.com/\"/>
-    <loginPageInfo url=\"https://www.outlook.com/\">
-      <username>%EMAILADDRESS%</username>
-      <usernameField id=\"i0116\" name=\"login\"/>
-      <passwordField id=\"i0118\" name=\"passwd\"/>
-      <loginButton id=\"idSIButton9\" name=\"SI\"/>
-    </loginPageInfo>
-  </webMail>
-</clientConfig>";
-        let res = parse_xml("example@outlook.com", xml_raw).expect("XML parsing failed");
-        assert_eq!(res.mail_server, "outlook.office365.com");
-        assert_eq!(res.mail_port, 993);
-        assert_eq!(res.send_server, "smtp.office365.com");
-        assert_eq!(res.send_port, 587);
+        let xml_raw = include_str!("../../test-data/autoconfig/outlook.com.xml");
+        let res = parse_serverparams("example@outlook.com", xml_raw).expect("XML parsing failed");
+        assert_eq!(res[0].protocol, Protocol::IMAP);
+        assert_eq!(res[0].hostname, "outlook.office365.com");
+        assert_eq!(res[0].port, 993);
+        assert_eq!(res[1].protocol, Protocol::SMTP);
+        assert_eq!(res[1].hostname, "smtp.office365.com");
+        assert_eq!(res[1].port, 587);
+    }
+
+    #[test]
+    fn test_parse_lakenet_autoconfig() {
+        let xml_raw = include_str!("../../test-data/autoconfig/lakenet.ch.xml");
+        let res =
+            parse_xml_with_address("example@lakenet.ch", xml_raw).expect("XML parsing failed");
+
+        assert_eq!(res.incoming_servers.len(), 4);
+
+        assert_eq!(res.incoming_servers[0].typ, "imap");
+        assert_eq!(res.incoming_servers[0].hostname, "mail.lakenet.ch");
+        assert_eq!(res.incoming_servers[0].port, 993);
+        assert_eq!(res.incoming_servers[0].sockettype, Socket::SSL);
+        assert_eq!(res.incoming_servers[0].username, "example@lakenet.ch");
+
+        assert_eq!(res.incoming_servers[1].typ, "imap");
+        assert_eq!(res.incoming_servers[1].hostname, "mail.lakenet.ch");
+        assert_eq!(res.incoming_servers[1].port, 143);
+        assert_eq!(res.incoming_servers[1].sockettype, Socket::STARTTLS);
+        assert_eq!(res.incoming_servers[1].username, "example@lakenet.ch");
+
+        assert_eq!(res.incoming_servers[2].typ, "pop3");
+        assert_eq!(res.incoming_servers[2].hostname, "mail.lakenet.ch");
+        assert_eq!(res.incoming_servers[2].port, 995);
+        assert_eq!(res.incoming_servers[2].sockettype, Socket::SSL);
+        assert_eq!(res.incoming_servers[2].username, "example@lakenet.ch");
+
+        assert_eq!(res.incoming_servers[3].typ, "pop3");
+        assert_eq!(res.incoming_servers[3].hostname, "mail.lakenet.ch");
+        assert_eq!(res.incoming_servers[3].port, 110);
+        assert_eq!(res.incoming_servers[3].sockettype, Socket::STARTTLS);
+        assert_eq!(res.incoming_servers[3].username, "example@lakenet.ch");
+
+        assert_eq!(res.outgoing_servers.len(), 1);
+
+        assert_eq!(res.outgoing_servers[0].typ, "smtp");
+        assert_eq!(res.outgoing_servers[0].hostname, "mail.lakenet.ch");
+        assert_eq!(res.outgoing_servers[0].port, 587);
+        assert_eq!(res.outgoing_servers[0].sockettype, Socket::STARTTLS);
+        assert_eq!(res.outgoing_servers[0].username, "example@lakenet.ch");
     }
 }

src/configure/auto_outlook.rs

@@ -1,122 +1,194 @@
-//! Outlook's Autodiscover
+//! # Outlook's Autodiscover
+//!
+//! This module implements autoconfiguration via POX (Plain Old XML) interface to Autodiscover
+//! Service. Newer SOAP interface, introduced in Exchange 2010, is not used.
 
-use quick_xml::events::BytesEnd;
+use quick_xml::events::Event;
+
+use std::io::BufRead;
 
-use crate::constants::*;
 use crate::context::Context;
-use crate::login_param::LoginParam;
+use crate::provider::{Protocol, Socket};
 
 use super::read_url::read_url;
-use super::Error;
+use super::{Error, ServerParams};
 
-struct OutlookAutodiscover {
-    pub out: LoginParam,
-    pub out_imap_set: bool,
-    pub out_smtp_set: bool,
-    pub config_type: Option<String>,
-    pub config_server: String,
-    pub config_port: i32,
-    pub config_ssl: String,
-    pub config_redirecturl: Option<String>,
+/// Result of parsing a single `Protocol` tag.
+///
+/// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/protocol-pox
+#[derive(Debug)]
+struct ProtocolTag {
+    /// Server type, such as "IMAP", "SMTP" or "POP3".
+    ///
+    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/type-pox
+    pub typ: String,
+
+    /// Server identifier, hostname or IP address for IMAP and SMTP.
+    ///
+    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/server-pox
+    pub server: String,
+
+    /// Network port.
+    ///
+    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/port-pox
+    pub port: u16,
+
+    /// Whether connection should be secure, "on" or "off", default is "on".
+    ///
+    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/ssl-pox
+    pub ssl: bool,
 }
 
 enum ParsingResult {
-    LoginParam(LoginParam),
+    Protocols(Vec<ProtocolTag>),
+
+    /// XML redirect via `RedirectUrl` tag.
     RedirectUrl(String),
 }
 
-fn parse_xml(xml_raw: &str) -> Result<ParsingResult, Error> {
-    let mut outlk_ad = OutlookAutodiscover {
-        out: LoginParam::new(),
-        out_imap_set: false,
-        out_smtp_set: false,
-        config_type: None,
-        config_server: String::new(),
-        config_port: 0,
-        config_ssl: String::new(),
-        config_redirecturl: None,
-    };
-
-    let mut reader = quick_xml::Reader::from_str(&xml_raw);
-    reader.trim_text(true);
+/// Parses a single Protocol section.
+fn parse_protocol<B: BufRead>(
+    reader: &mut quick_xml::Reader<B>,
+) -> Result<Option<ProtocolTag>, quick_xml::Error> {
+    let mut protocol_type = None;
+    let mut protocol_server = None;
+    let mut protocol_port = None;
+    let mut protocol_ssl = true;
 
     let mut buf = Vec::new();
 
     let mut current_tag: Option<String> = None;
-
     loop {
-        let event = reader
-            .read_event(&mut buf)
-            .map_err(|error| Error::InvalidXml {
-                position: reader.buffer_position(),
-                error,
-            })?;
-
-        match event {
-            quick_xml::events::Event::Start(ref e) => {
-                let tag = String::from_utf8_lossy(e.name()).trim().to_lowercase();
-
+        match reader.read_event(&mut buf)? {
+            Event::Start(ref event) => {
+                current_tag = Some(String::from_utf8_lossy(event.name()).trim().to_lowercase());
+            }
+            Event::End(ref event) => {
+                let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
                 if tag == "protocol" {
-                    outlk_ad.config_type = None;
-                    outlk_ad.config_server = String::new();
-                    outlk_ad.config_port = 0;
-                    outlk_ad.config_ssl = String::new();
-                    outlk_ad.config_redirecturl = None;
-
+                    break;
+                }
+                if Some(tag) == current_tag {
                     current_tag = None;
-                } else {
-                    current_tag = Some(tag);
                 }
             }
-            quick_xml::events::Event::End(ref e) => {
-                outlk_autodiscover_endtag_cb(e, &mut outlk_ad);
-                current_tag = None;
-            }
-            quick_xml::events::Event::Text(ref e) => {
+            Event::Text(ref e) => {
                 let val = e.unescape_and_decode(&reader).unwrap_or_default();
 
                 if let Some(ref tag) = current_tag {
                     match tag.as_str() {
-                        "type" => {
-                            outlk_ad.config_type = Some(val.trim().to_lowercase().to_string())
+                        "type" => protocol_type = Some(val.trim().to_string()),
+                        "server" => protocol_server = Some(val.trim().to_string()),
+                        "port" => protocol_port = Some(val.trim().parse().unwrap_or_default()),
+                        "ssl" => {
+                            protocol_ssl = match val.trim() {
+                                "on" => true,
+                                "off" => false,
+                                _ => true,
+                            }
                         }
-                        "server" => outlk_ad.config_server = val.trim().to_string(),
-                        "port" => outlk_ad.config_port = val.trim().parse().unwrap_or_default(),
-                        "ssl" => outlk_ad.config_ssl = val.trim().to_string(),
-                        "redirecturl" => outlk_ad.config_redirecturl = Some(val.trim().to_string()),
                         _ => {}
                     };
                 }
             }
-            quick_xml::events::Event::Eof => break,
+            Event::Eof => break,
+            _ => {}
+        }
+    }
+
+    if let (Some(protocol_type), Some(protocol_server), Some(protocol_port)) =
+        (protocol_type, protocol_server, protocol_port)
+    {
+        Ok(Some(ProtocolTag {
+            typ: protocol_type,
+            server: protocol_server,
+            port: protocol_port,
+            ssl: protocol_ssl,
+        }))
+    } else {
+        Ok(None)
+    }
+}
+
+/// Parses `RedirectUrl` tag.
+fn parse_redirecturl<B: BufRead>(
+    reader: &mut quick_xml::Reader<B>,
+) -> Result<String, quick_xml::Error> {
+    let mut buf = Vec::new();
+    match reader.read_event(&mut buf)? {
+        Event::Text(ref e) => {
+            let val = e.unescape_and_decode(&reader).unwrap_or_default();
+            Ok(val.trim().to_string())
+        }
+        _ => Ok("".to_string()),
+    }
+}
+
+fn parse_xml_reader<B: BufRead>(
+    reader: &mut quick_xml::Reader<B>,
+) -> Result<ParsingResult, quick_xml::Error> {
+    let mut protocols = Vec::new();
+
+    let mut buf = Vec::new();
+    loop {
+        match reader.read_event(&mut buf)? {
+            Event::Start(ref e) => {
+                let tag = String::from_utf8_lossy(e.name()).trim().to_lowercase();
+
+                if tag == "protocol" {
+                    if let Some(protocol) = parse_protocol(reader)? {
+                        protocols.push(protocol);
+                    }
+                } else if tag == "redirecturl" {
+                    let redirecturl = parse_redirecturl(reader)?;
+                    return Ok(ParsingResult::RedirectUrl(redirecturl));
+                }
+            }
+            Event::Eof => break,
             _ => (),
         }
         buf.clear();
     }
 
-    // XML redirect via redirecturl
-    let res = if outlk_ad.config_redirecturl.is_none()
-        || outlk_ad.config_redirecturl.as_ref().unwrap().is_empty()
-    {
-        if outlk_ad.out.mail_server.is_empty()
-            || outlk_ad.out.mail_port == 0
-            || outlk_ad.out.send_server.is_empty()
-            || outlk_ad.out.send_port == 0
-        {
-            return Err(Error::IncompleteAutoconfig(outlk_ad.out));
-        }
-        ParsingResult::LoginParam(outlk_ad.out)
-    } else {
-        ParsingResult::RedirectUrl(outlk_ad.config_redirecturl.unwrap())
-    };
-    Ok(res)
+    Ok(ParsingResult::Protocols(protocols))
 }
 
-pub async fn outlk_autodiscover(
+fn parse_xml(xml_raw: &str) -> Result<ParsingResult, Error> {
+    let mut reader = quick_xml::Reader::from_str(&xml_raw);
+    reader.trim_text(true);
+
+    parse_xml_reader(&mut reader).map_err(|error| Error::InvalidXml {
+        position: reader.buffer_position(),
+        error,
+    })
+}
+
+fn protocols_to_serverparams(protocols: Vec<ProtocolTag>) -> Vec<ServerParams> {
+    protocols
+        .into_iter()
+        .filter_map(|protocol| {
+            Some(ServerParams {
+                protocol: match protocol.typ.to_lowercase().as_ref() {
+                    "imap" => Some(Protocol::IMAP),
+                    "smtp" => Some(Protocol::SMTP),
+                    _ => None,
+                }?,
+                socket: match protocol.ssl {
+                    true => Socket::Automatic,
+                    false => Socket::Plain,
+                },
+                hostname: protocol.server,
+                port: protocol.port,
+                username: String::new(),
+            })
+        })
+        .collect()
+}
+
+pub(crate) async fn outlk_autodiscover(
     context: &Context,
     url: &str,
-    _param_in: &LoginParam,
-) -> Result<LoginParam, Error> {
+) -> Result<Vec<ServerParams>, Error> {
     let mut url = url.to_string();
     /* Follow up to 10 xml-redirects (http-redirects are followed in read_url() */
     for _i in 0..10 {
@@ -127,47 +199,18 @@ pub async fn outlk_autodiscover(
         }
         match res? {
             ParsingResult::RedirectUrl(redirect_url) => url = redirect_url,
-            ParsingResult::LoginParam(login_param) => return Ok(login_param),
+            ParsingResult::Protocols(protocols) => {
+                return Ok(protocols_to_serverparams(protocols));
+            }
         }
     }
     Err(Error::RedirectionError)
 }
 
-fn outlk_autodiscover_endtag_cb(event: &BytesEnd, outlk_ad: &mut OutlookAutodiscover) {
-    let tag = String::from_utf8_lossy(event.name()).trim().to_lowercase();
-
-    if tag == "protocol" {
-        if let Some(type_) = &outlk_ad.config_type {
-            let port = outlk_ad.config_port;
-            let ssl_on = outlk_ad.config_ssl == "on";
-            let ssl_off = outlk_ad.config_ssl == "off";
-            if type_ == "imap" && !outlk_ad.out_imap_set {
-                outlk_ad.out.mail_server =
-                    std::mem::replace(&mut outlk_ad.config_server, String::new());
-                outlk_ad.out.mail_port = port;
-                if ssl_on {
-                    outlk_ad.out.server_flags |= DC_LP_IMAP_SOCKET_SSL as i32
-                } else if ssl_off {
-                    outlk_ad.out.server_flags |= DC_LP_IMAP_SOCKET_PLAIN as i32
-                }
-                outlk_ad.out_imap_set = true
-            } else if type_ == "smtp" && !outlk_ad.out_smtp_set {
-                outlk_ad.out.send_server =
-                    std::mem::replace(&mut outlk_ad.config_server, String::new());
-                outlk_ad.out.send_port = outlk_ad.config_port;
-                if ssl_on {
-                    outlk_ad.out.server_flags |= DC_LP_SMTP_SOCKET_SSL as i32
-                } else if ssl_off {
-                    outlk_ad.out.server_flags |= DC_LP_SMTP_SOCKET_PLAIN as i32
-                }
-                outlk_ad.out_smtp_set = true
-            }
-        }
-    }
-}
-
 #[cfg(test)]
 mod tests {
+    #![allow(clippy::indexing_slicing)]
+
     use super::*;
 
     #[test]
@@ -184,16 +227,13 @@ mod tests {
     </Response>
   </Autodiscover>
  ").expect("XML is not parsed successfully");
-        match res {
-            ParsingResult::LoginParam(_lp) => {
-                panic!("redirecturl is not found");
-            }
-            ParsingResult::RedirectUrl(url) => {
-                assert_eq!(
-                    url,
-                    "https://mail.example.com/autodiscover/autodiscover.xml"
-                );
-            }
+        if let ParsingResult::RedirectUrl(url) = res {
+            assert_eq!(
+                url,
+                "https://mail.example.com/autodiscover/autodiscover.xml"
+            );
+        } else {
+            panic!("redirecturl is not found");
         }
     }
 
@@ -228,11 +268,16 @@ mod tests {
         .expect("XML is not parsed successfully");
 
         match res {
-            ParsingResult::LoginParam(lp) => {
-                assert_eq!(lp.mail_server, "example.com");
-                assert_eq!(lp.mail_port, 993);
-                assert_eq!(lp.send_server, "smtp.example.com");
-                assert_eq!(lp.send_port, 25);
+            ParsingResult::Protocols(protocols) => {
+                assert_eq!(protocols[0].typ, "IMAP");
+                assert_eq!(protocols[0].server, "example.com");
+                assert_eq!(protocols[0].port, 993);
+                assert_eq!(protocols[0].ssl, true);
+
+                assert_eq!(protocols[1].typ, "SMTP");
+                assert_eq!(protocols[1].server, "smtp.example.com");
+                assert_eq!(protocols[1].port, 25);
+                assert_eq!(protocols[1].ssl, false);
             }
             ParsingResult::RedirectUrl(_) => {
                 panic!("RedirectUrl is not expected");

src/configure/read_url.rs

@@ -12,7 +12,7 @@ pub async fn read_url(context: &Context, url: &str) -> Result<String, Error> {
     match surf::get(url).recv_string().await {
         Ok(res) => Ok(res),
         Err(err) => {
-            info!(context, "Can\'t read URL {}", url);
+            info!(context, "Can\'t read URL {}: {}", url, err);
 
             Err(Error::GetError(err))
         }

src/configure/server_params.rs

@@ -0,0 +1,164 @@
+//! Variable server parameters lists
+
+use crate::provider::{Protocol, Socket};
+
+/// Set of variable parameters to try during configuration.
+///
+/// Can be loaded from offline provider database, online configuraiton
+/// or derived from user entered parameters.
+#[derive(Debug, Clone, PartialEq)]
+pub(crate) struct ServerParams {
+    /// Protocol, such as IMAP or SMTP.
+    pub protocol: Protocol,
+
+    /// Server hostname, empty if unknown.
+    pub hostname: String,
+
+    /// Server port, zero if unknown.
+    pub port: u16,
+
+    /// Socket security, such as TLS or STARTTLS, Socket::Automatic if unknown.
+    pub socket: Socket,
+
+    /// Username, empty if unknown.
+    pub username: String,
+}
+
+impl ServerParams {
+    pub(crate) fn expand_usernames(mut self, addr: &str) -> Vec<ServerParams> {
+        let mut res = Vec::new();
+
+        if self.username.is_empty() {
+            self.username = addr.to_string();
+            res.push(self.clone());
+
+            if let Some(at) = addr.find('@') {
+                self.username = addr.split_at(at).0.to_string();
+                res.push(self);
+            }
+        } else {
+            res.push(self)
+        }
+        res
+    }
+
+    pub(crate) fn expand_hostnames(mut self, param_domain: &str) -> Vec<ServerParams> {
+        let mut res = Vec::new();
+        if self.hostname.is_empty() {
+            self.hostname = param_domain.to_string();
+            res.push(self.clone());
+
+            self.hostname = match self.protocol {
+                Protocol::IMAP => "imap.".to_string() + param_domain,
+                Protocol::SMTP => "smtp.".to_string() + param_domain,
+            };
+            res.push(self.clone());
+
+            self.hostname = "mail.".to_string() + param_domain;
+            res.push(self);
+        } else {
+            res.push(self);
+        }
+        res
+    }
+
+    pub(crate) fn expand_ports(mut self) -> Vec<ServerParams> {
+        // Try to infer port from socket security.
+        if self.port == 0 {
+            self.port = match self.socket {
+                Socket::SSL => match self.protocol {
+                    Protocol::IMAP => 993,
+                    Protocol::SMTP => 465,
+                },
+                Socket::STARTTLS | Socket::Plain => match self.protocol {
+                    Protocol::IMAP => 143,
+                    Protocol::SMTP => 587,
+                },
+                Socket::Automatic => 0,
+            }
+        }
+
+        let mut res = Vec::new();
+        if self.port == 0 {
+            // Neither port nor security is set.
+            //
+            // Try common secure combinations.
+
+            // Try STARTTLS
+            self.socket = Socket::STARTTLS;
+            self.port = match self.protocol {
+                Protocol::IMAP => 143,
+                Protocol::SMTP => 587,
+            };
+            res.push(self.clone());
+
+            // Try TLS
+            self.socket = Socket::SSL;
+            self.port = match self.protocol {
+                Protocol::IMAP => 993,
+                Protocol::SMTP => 465,
+            };
+            res.push(self);
+        } else if self.socket == Socket::Automatic {
+            // Try TLS over user-provided port.
+            self.socket = Socket::SSL;
+            res.push(self.clone());
+
+            // Try STARTTLS over user-provided port.
+            self.socket = Socket::STARTTLS;
+            res.push(self);
+        } else {
+            res.push(self);
+        }
+        res
+    }
+}
+
+/// Expands vector of `ServerParams`, replacing placeholders with
+/// variants to try.
+pub(crate) fn expand_param_vector(
+    v: Vec<ServerParams>,
+    addr: &str,
+    domain: &str,
+) -> Vec<ServerParams> {
+    v.into_iter()
+        // The order of expansion is important: ports are expanded the
+        // last, so they are changed the first. Username is only
+        // changed if default value (address with domain) didn't work
+        // for all available hosts and ports.
+        .flat_map(|params| params.expand_usernames(addr).into_iter())
+        .flat_map(|params| params.expand_hostnames(domain).into_iter())
+        .flat_map(|params| params.expand_ports().into_iter())
+        .collect()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_expand_param_vector() {
+        let v = expand_param_vector(
+            vec![ServerParams {
+                protocol: Protocol::IMAP,
+                hostname: "example.net".to_string(),
+                port: 0,
+                socket: Socket::SSL,
+                username: "foobar".to_string(),
+            }],
+            "foobar@example.net",
+            "example.net",
+        );
+
+        assert_eq!(
+            v,
+            vec![ServerParams {
+                protocol: Protocol::IMAP,
+                hostname: "example.net".to_string(),
+                port: 993,
+                socket: Socket::SSL,
+                username: "foobar".to_string(),
+            }],
+        );
+    }
+}

src/constants.rs

@@ -1,20 +1,9 @@
 //! # Constants
-#![allow(dead_code)]
-
 use deltachat_derive::*;
-use lazy_static::lazy_static;
+use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
 
-lazy_static! {
-    pub static ref DC_VERSION_STR: String = env!("CARGO_PKG_VERSION").to_string();
-}
-
-// some defaults
-const DC_E2EE_DEFAULT_ENABLED: i32 = 1;
-const DC_INBOX_WATCH_DEFAULT: i32 = 1;
-const DC_SENTBOX_WATCH_DEFAULT: i32 = 1;
-const DC_MVBOX_WATCH_DEFAULT: i32 = 1;
-const DC_MVBOX_MOVE_DEFAULT: i32 = 1;
+pub static DC_VERSION_STR: Lazy<String> = Lazy::new(|| env!("CARGO_PKG_VERSION").to_string());
 
 #[derive(
     Debug,
@@ -84,6 +73,20 @@ impl Default for KeyGenType {
     }
 }
 
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
+#[repr(i8)]
+pub enum VideochatType {
+    Unknown = 0,
+    BasicWebrtc = 1,
+    Jitsi = 2,
+}
+
+impl Default for VideochatType {
+    fn default() -> Self {
+        VideochatType::Unknown
+    }
+}
+
 pub const DC_HANDSHAKE_CONTINUE_NORMAL_PROCESSING: i32 = 0x01;
 pub const DC_HANDSHAKE_STOP_NORMAL_PROCESSING: i32 = 0x02;
 pub const DC_HANDSHAKE_ADD_DELETE_JOB: i32 = 0x04;
@@ -103,14 +106,16 @@ pub const DC_GCL_ADD_SELF: usize = 0x02;
 // unchanged user avatars are resent to the recipients every some days
 pub const DC_RESEND_USER_AVATAR_DAYS: i64 = 14;
 
+// warn about an outdated app after a given number of days.
+// as we use the "provider-db generation date" as reference (that might not be updated very often)
+// and as not all system get speedy updates,
+// do not use too small value that will annoy users checking for nonexistant updates.
+pub const DC_OUTDATED_WARNING_DAYS: i64 = 365;
+
 /// virtual chat showing all messages belonging to chats flagged with chats.blocked=2
-pub(crate) const DC_CHAT_ID_DEADDROP: u32 = 1;
+pub const DC_CHAT_ID_DEADDROP: u32 = 1;
 /// messages that should be deleted get this chat_id; the messages are deleted from the working thread later then. This is also needed as rfc724_mid should be preset as long as the message is not deleted on the server (otherwise it is downloaded again)
 pub const DC_CHAT_ID_TRASH: u32 = 3;
-/// a message is just in creation but not yet assigned to a chat (eg. we may need the message ID to set up blobs; this avoids unready message to be sent and shown)
-const DC_CHAT_ID_MSGS_IN_CREATION: u32 = 4;
-/// virtual chat showing all messages flagged with msgs.starred=2
-pub const DC_CHAT_ID_STARRED: u32 = 5;
 /// only an indicator in a chatlist
 pub const DC_CHAT_ID_ARCHIVED_LINK: u32 = 6;
 /// only an indicator in a chatlist
@@ -138,7 +143,6 @@ pub enum Chattype {
     Undefined = 0,
     Single = 100,
     Group = 120,
-    VerifiedGroup = 130,
 }
 
 impl Default for Chattype {
@@ -152,9 +156,9 @@ pub const DC_MSG_ID_DAYMARKER: u32 = 9;
 pub const DC_MSG_ID_LAST_SPECIAL: u32 = 9;
 
 /// approx. max. length returned by dc_msg_get_text()
-const DC_MAX_GET_TEXT_LEN: usize = 30000;
+pub const DC_MAX_GET_TEXT_LEN: usize = 30000;
 /// approx. max. length returned by dc_get_msg_info()
-const DC_MAX_GET_INFO_LEN: usize = 100_000;
+pub const DC_MAX_GET_INFO_LEN: usize = 100_000;
 
 pub const DC_CONTACT_ID_UNDEFINED: u32 = 0;
 pub const DC_CONTACT_ID_SELF: u32 = 1;
@@ -185,48 +189,19 @@ pub const DC_LP_AUTH_OAUTH2: i32 = 0x2;
 /// If this flag is set, automatic configuration is skipped.
 pub const DC_LP_AUTH_NORMAL: i32 = 0x4;
 
-/// Connect to IMAP via STARTTLS.
-/// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_IMAP_SOCKET_STARTTLS: i32 = 0x100;
-
-/// Connect to IMAP via SSL.
-/// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_IMAP_SOCKET_SSL: i32 = 0x200;
-
-/// Connect to IMAP unencrypted, this should not be used.
-/// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_IMAP_SOCKET_PLAIN: i32 = 0x400;
-
-/// Connect to SMTP via STARTTLS.
-/// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_SMTP_SOCKET_STARTTLS: usize = 0x10000;
-
-/// Connect to SMTP via SSL.
-/// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_SMTP_SOCKET_SSL: usize = 0x20000;
-
-/// Connect to SMTP unencrypted, this should not be used.
-/// If this flag is set, automatic configuration is skipped.
-pub const DC_LP_SMTP_SOCKET_PLAIN: usize = 0x40000;
-
 /// if none of these flags are set, the default is chosen
 pub const DC_LP_AUTH_FLAGS: i32 = DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL;
-/// if none of these flags are set, the default is chosen
-pub const DC_LP_IMAP_SOCKET_FLAGS: i32 =
-    DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_SSL | DC_LP_IMAP_SOCKET_PLAIN;
-/// if none of these flags are set, the default is chosen
-pub const DC_LP_SMTP_SOCKET_FLAGS: usize =
-    DC_LP_SMTP_SOCKET_STARTTLS | DC_LP_SMTP_SOCKET_SSL | DC_LP_SMTP_SOCKET_PLAIN;
 
-// QR code scanning (view from Bob, the joiner)
-pub const DC_VC_AUTH_REQUIRED: i32 = 2;
-pub const DC_VC_CONTACT_CONFIRM: i32 = 6;
-pub const DC_BOB_ERROR: i32 = 0;
-pub const DC_BOB_SUCCESS: i32 = 1;
+/// How many existing messages shall be fetched after configuration.
+pub const DC_FETCH_EXISTING_MSGS_COUNT: i64 = 100;
 
 // max. width/height of an avatar
 pub const AVATAR_SIZE: u32 = 192;
 
+// max. width/height of images
+pub const BALANCED_IMAGE_SIZE: u32 = 1280;
+pub const WORSE_IMAGE_SIZE: u32 = 640;
+
 // this value can be increased if the folder configuration is changed and must be redone on next program start
 pub const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3;
 
@@ -292,6 +267,9 @@ pub enum Viewtype {
     /// The file is set via dc_msg_set_file()
     /// and retrieved via dc_msg_get_file().
     File = 60,
+
+    /// Message is an invitation to a videochat.
+    VideochatInvitation = 70,
 }
 
 impl Default for Viewtype {
@@ -310,64 +288,6 @@ mod tests {
     }
 }
 
-// These constants are used as events
-// reported to the callback given to dc_context_new().
-// If you do not want to handle an event, it is always safe to return 0,
-// so there is no need to add a "case" for every event.
-
-const DC_EVENT_FILE_COPIED: usize = 2055; // deprecated;
-const DC_EVENT_IS_OFFLINE: usize = 2081; // deprecated;
-const DC_ERROR_SEE_STRING: usize = 0; // deprecated;
-const DC_ERROR_SELF_NOT_IN_GROUP: usize = 1; // deprecated;
-const DC_STR_SELFNOTINGRP: usize = 21; // deprecated;
-
-// TODO: Strings need some doumentation about used placeholders.
-// These constants are used to set stock translation strings
-
-const DC_STR_NOMESSAGES: usize = 1;
-const DC_STR_SELF: usize = 2;
-const DC_STR_DRAFT: usize = 3;
-const DC_STR_VOICEMESSAGE: usize = 7;
-const DC_STR_DEADDROP: usize = 8;
-const DC_STR_IMAGE: usize = 9;
-const DC_STR_VIDEO: usize = 10;
-const DC_STR_AUDIO: usize = 11;
-const DC_STR_FILE: usize = 12;
-const DC_STR_STATUSLINE: usize = 13;
-const DC_STR_NEWGROUPDRAFT: usize = 14;
-const DC_STR_MSGGRPNAME: usize = 15;
-const DC_STR_MSGGRPIMGCHANGED: usize = 16;
-const DC_STR_MSGADDMEMBER: usize = 17;
-const DC_STR_MSGDELMEMBER: usize = 18;
-const DC_STR_MSGGROUPLEFT: usize = 19;
-const DC_STR_GIF: usize = 23;
-const DC_STR_ENCRYPTEDMSG: usize = 24;
-const DC_STR_E2E_AVAILABLE: usize = 25;
-const DC_STR_ENCR_TRANSP: usize = 27;
-const DC_STR_ENCR_NONE: usize = 28;
-const DC_STR_CANTDECRYPT_MSG_BODY: usize = 29;
-const DC_STR_FINGERPRINTS: usize = 30;
-const DC_STR_READRCPT: usize = 31;
-const DC_STR_READRCPT_MAILBODY: usize = 32;
-const DC_STR_MSGGRPIMGDELETED: usize = 33;
-const DC_STR_E2E_PREFERRED: usize = 34;
-const DC_STR_CONTACT_VERIFIED: usize = 35;
-const DC_STR_CONTACT_NOT_VERIFIED: usize = 36;
-const DC_STR_CONTACT_SETUP_CHANGED: usize = 37;
-const DC_STR_ARCHIVEDCHATS: usize = 40;
-const DC_STR_STARREDMSGS: usize = 41;
-const DC_STR_AC_SETUP_MSG_SUBJECT: usize = 42;
-const DC_STR_AC_SETUP_MSG_BODY: usize = 43;
-const DC_STR_CANNOT_LOGIN: usize = 60;
-const DC_STR_SERVER_RESPONSE: usize = 61;
-const DC_STR_MSGACTIONBYUSER: usize = 62;
-const DC_STR_MSGACTIONBYME: usize = 63;
-const DC_STR_MSGLOCATIONENABLED: usize = 64;
-const DC_STR_MSGLOCATIONDISABLED: usize = 65;
-const DC_STR_LOCATION: usize = 66;
-const DC_STR_STICKER: usize = 67;
-const DC_STR_COUNT: usize = 67;
-
 pub const DC_JOB_DELETE_MSG_ON_IMAP: i32 = 110;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive)]

src/contact.rs

@@ -3,6 +3,8 @@
 use async_std::path::PathBuf;
 use deltachat_derive::*;
 use itertools::Itertools;
+use once_cell::sync::Lazy;
+use regex::Regex;
 
 use crate::aheader::EncryptPreference;
 use crate::chat::ChatId;
@@ -11,13 +13,14 @@ use crate::constants::*;
 use crate::context::Context;
 use crate::dc_tools::*;
 use crate::error::{bail, ensure, format_err, Result};
-use crate::events::Event;
-use crate::key::{DcKey, Key, SignedPublicKey};
+use crate::events::EventType;
+use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
-use crate::message::{MessageState, MsgId};
+use crate::message::MessageState;
 use crate::mimeparser::AvatarAction;
 use crate::param::*;
 use crate::peerstate::*;
+use crate::provider::Socket;
 use crate::stock::StockMessage;
 
 /// An object representing a single contact in memory.
@@ -233,21 +236,23 @@ impl Contact {
         name: impl AsRef<str>,
         addr: impl AsRef<str>,
     ) -> Result<u32> {
+        let name = improve_single_line_input(name);
         ensure!(
             !addr.as_ref().is_empty(),
             "Cannot create contact with empty address"
         );
 
+        let (name, addr) = sanitize_name_and_addr(name, addr);
+
         let (contact_id, sth_modified) =
             Contact::add_or_lookup(context, name, addr, Origin::ManuallyCreated).await?;
         let blocked = Contact::is_blocked_load(context, contact_id).await;
-        context.emit_event(Event::ContactsChanged(
-            if sth_modified == Modifier::Created {
-                Some(contact_id)
-            } else {
-                None
-            },
-        ));
+        match sth_modified {
+            Modifier::None => {}
+            Modifier::Modified | Modifier::Created => {
+                context.emit_event(EventType::ContactsChanged(Some(contact_id)))
+            }
+        }
         if blocked {
             Contact::unblock(context, contact_id).await;
         }
@@ -255,10 +260,9 @@ impl Contact {
         Ok(contact_id)
     }
 
-    /// Mark all messages sent by the given contact
-    /// as *noticed*.  See also dc_marknoticed_chat() and dc_markseen_msgs()
-    ///
-    /// Calling this function usually results in the event `#DC_EVENT_MSGS_CHANGED`.
+    /// Mark messages from a contact as noticed.
+    /// The contact is expected to belong to the deaddrop,
+    /// therefore, DC_EVENT_MSGS_NOTICED(DC_CHAT_ID_DEADDROP) is emitted.
     pub async fn mark_noticed(context: &Context, id: u32) {
         if context
             .sql
@@ -269,10 +273,7 @@ impl Contact {
             .await
             .is_ok()
         {
-            context.emit_event(Event::MsgsChanged {
-                chat_id: ChatId::new(0),
-                msg_id: MsgId::new(0),
-            });
+            context.emit_event(EventType::MsgsNoticed(ChatId::new(DC_CHAT_ID_DEADDROP)));
         }
     }
 
@@ -451,10 +452,20 @@ impl Contact {
                 if update_name {
                     // Update the contact name also if it is used as a group name.
                     // This is one of the few duplicated data, however, getting the chat list is easier this way.
-                    context.sql.execute(
-                    "UPDATE chats SET name=? WHERE type=? AND id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?);",
-                    paramsv![new_name, Chattype::Single, row_id]
-                ).await.ok();
+                    let chat_id = context.sql.query_get_value::<i32>(
+                        context,
+                        "SELECT id FROM chats WHERE type=? AND id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)",
+                        paramsv![Chattype::Single, row_id]
+                    ).await;
+                    if let Some(chat_id) = chat_id {
+                        match context.sql.execute("UPDATE chats SET name=? WHERE id=? AND name!=?1", paramsv![new_name, chat_id]).await {
+                            Err(err) => warn!(context, "Can't update chat name: {}", err),
+                            Ok(count) => if count > 0 {
+                                // Chat name updated
+                                context.emit_event(EventType::ChatModified(ChatId::new(chat_id as u32)));
+                            }
+                        }
+                    }
                 }
                 sth_modified = Modifier::Modified;
             }
@@ -512,8 +523,9 @@ impl Contact {
         let mut modify_cnt = 0;
 
         for (name, addr) in split_address_book(addr_book.as_ref()).into_iter() {
+            let (name, addr) = sanitize_name_and_addr(name, addr);
             let name = normalize_name(name);
-            match Contact::add_or_lookup(context, name, addr, Origin::AddressBook).await {
+            match Contact::add_or_lookup(context, name, &addr, Origin::AddressBook).await {
                 Err(err) => {
                     warn!(
                         context,
@@ -528,7 +540,7 @@ impl Contact {
             }
         }
         if modify_cnt > 0 {
-            context.emit_event(Event::ContactsChanged(None));
+            context.emit_event(EventType::ContactsChanged(None));
         }
 
         Ok(modify_cnt)
@@ -673,7 +685,7 @@ impl Contact {
         let mut ret = String::new();
 
         if let Ok(contact) = Contact::load_from_db(context, contact_id).await {
-            let peerstate = Peerstate::from_addr(context, &contact.addr).await;
+            let peerstate = Peerstate::from_addr(context, &contact.addr).await?;
             let loginparam = LoginParam::from_database(context, "configured_").await;
 
             if peerstate.is_some()
@@ -691,18 +703,20 @@ impl Contact {
                     })
                     .await;
                 ret += &p;
-                let self_key = Key::from(SignedPublicKey::load_self(context).await?);
                 let p = context.stock_str(StockMessage::FingerPrints).await;
                 ret += &format!(" {}:", p);
 
-                let fingerprint_self = self_key.formatted_fingerprint();
+                let fingerprint_self = SignedPublicKey::load_self(context)
+                    .await?
+                    .fingerprint()
+                    .to_string();
                 let fingerprint_other_verified = peerstate
                     .peek_key(PeerstateVerifiedStatus::BidirectVerified)
-                    .map(|k| k.formatted_fingerprint())
+                    .map(|k| k.fingerprint().to_string())
                     .unwrap_or_default();
                 let fingerprint_other_unverified = peerstate
                     .peek_key(PeerstateVerifiedStatus::Unverified)
-                    .map(|k| k.formatted_fingerprint())
+                    .map(|k| k.fingerprint().to_string())
                     .unwrap_or_default();
                 if loginparam.addr < peerstate.addr {
                     cat_fingerprint(&mut ret, &loginparam.addr, &fingerprint_self, "");
@@ -721,8 +735,8 @@ impl Contact {
                     );
                     cat_fingerprint(&mut ret, &loginparam.addr, &fingerprint_self, "");
                 }
-            } else if 0 == loginparam.server_flags & DC_LP_IMAP_SOCKET_PLAIN as i32
-                && 0 == loginparam.server_flags & DC_LP_SMTP_SOCKET_PLAIN as i32
+            } else if loginparam.imap.security != Socket::Plain
+                && loginparam.smtp.security != Socket::Plain
             {
                 ret += &context.stock_str(StockMessage::EncrTransp).await;
             } else {
@@ -777,7 +791,7 @@ impl Contact {
                 .await
             {
                 Ok(_) => {
-                    context.emit_event(Event::ContactsChanged(None));
+                    context.emit_event(EventType::ContactsChanged(None));
                     return Ok(());
                 }
                 Err(err) => {
@@ -932,7 +946,17 @@ impl Contact {
             }
         }
 
-        let peerstate = Peerstate::from_addr(context, &self.addr).await;
+        let peerstate = match Peerstate::from_addr(context, &self.addr).await {
+            Ok(peerstate) => peerstate,
+            Err(err) => {
+                warn!(
+                    context,
+                    "Failed to load peerstate for address {}: {}", self.addr, err
+                );
+                return VerifiedStatus::Unverified;
+            }
+        };
+
         if let Some(ps) = peerstate {
             if ps.verified_key.is_some() {
                 return VerifiedStatus::BidirectVerified;
@@ -1022,10 +1046,30 @@ pub fn addr_normalize(addr: &str) -> &str {
     let norm = addr.trim();
 
     if norm.starts_with("mailto:") {
-        return &norm[7..];
+        norm.get(7..).unwrap_or(norm)
+    } else {
+        norm
     }
+}
 
-    norm
+fn sanitize_name_and_addr(name: impl AsRef<str>, addr: impl AsRef<str>) -> (String, String) {
+    static ADDR_WITH_NAME_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new("(.*)<(.*)>").unwrap());
+    if let Some(captures) = ADDR_WITH_NAME_REGEX.captures(addr.as_ref()) {
+        (
+            if name.as_ref().is_empty() {
+                captures
+                    .get(1)
+                    .map_or("".to_string(), |m| normalize_name(m.as_str()))
+            } else {
+                name.as_ref().to_string()
+            },
+            captures
+                .get(2)
+                .map_or("".to_string(), |m| m.as_str().to_string()),
+        )
+    } else {
+        (name.as_ref().to_string(), addr.as_ref().to_string())
+    }
 }
 
 async fn set_block_contact(context: &Context, contact_id: u32, new_blocking: bool) {
@@ -1050,37 +1094,61 @@ async fn set_block_contact(context: &Context, contact_id: u32, new_blocking: boo
             // However, I'm not sure about this point; it may be confusing if the user wants to add other people;
             // this would result in recreating the same group...)
             if context.sql.execute(
-                    "UPDATE chats SET blocked=? WHERE type=? AND id IN (SELECT chat_id FROM chats_contacts WHERE contact_id=?);",
-                    paramsv![new_blocking, 100, contact_id as i32],
-                ).await.is_ok() {
-                    Contact::mark_noticed(context, contact_id).await;
-                    context.emit_event(Event::ContactsChanged(None));
-                }
+                "UPDATE chats SET blocked=? WHERE type=? AND id IN (SELECT chat_id FROM chats_contacts WHERE contact_id=?);",
+                paramsv![new_blocking, 100, contact_id as i32]).await.is_ok()
+            {
+                Contact::mark_noticed(context, contact_id).await;
+                context.emit_event(EventType::ContactsChanged(Some(contact_id)));
+            }
         }
     }
 }
 
+/// Set profile image for a contact.
+///
+/// The given profile image is expected to be already in the blob directory
+/// as profile images can be set only by receiving messages, this should be always the case, however.
+///
+/// For contact SELF, the image is not saved in the contact-database but as Config::Selfavatar;
+/// this typically happens if we see message with our own profile image, sent from another device.
 pub(crate) async fn set_profile_image(
     context: &Context,
     contact_id: u32,
     profile_image: &AvatarAction,
+    was_encrypted: bool,
 ) -> Result<()> {
-    // the given profile image is expected to be already in the blob directory
-    // as profile images can be set only by receiving messages, this should be always the case, however.
     let mut contact = Contact::load_from_db(context, contact_id).await?;
     let changed = match profile_image {
         AvatarAction::Change(profile_image) => {
-            contact.param.set(Param::ProfileImage, profile_image);
+            if contact_id == DC_CONTACT_ID_SELF {
+                if was_encrypted {
+                    context
+                        .set_config(Config::Selfavatar, Some(profile_image))
+                        .await?;
+                } else {
+                    info!(context, "Do not use unencrypted selfavatar.");
+                }
+            } else {
+                contact.param.set(Param::ProfileImage, profile_image);
+            }
             true
         }
         AvatarAction::Delete => {
-            contact.param.remove(Param::ProfileImage);
+            if contact_id == DC_CONTACT_ID_SELF {
+                if was_encrypted {
+                    context.set_config(Config::Selfavatar, None).await?;
+                } else {
+                    info!(context, "Do not use unencrypted selfavatar deletion.");
+                }
+            } else {
+                contact.param.remove(Param::ProfileImage);
+            }
             true
         }
     };
     if changed {
         contact.update_param(context).await?;
-        context.emit_event(Event::ContactsChanged(Some(contact_id)));
+        context.emit_event(EventType::ContactsChanged(Some(contact_id)));
     }
     Ok(())
 }
@@ -1088,38 +1156,21 @@ pub(crate) async fn set_profile_image(
 /// Normalize a name.
 ///
 /// - Remove quotes (come from some bad MUA implementations)
-/// - Convert names as "Petersen, Björn" to "Björn Petersen"
 /// - Trims the resulting string
 ///
 /// Typically, this function is not needed as it is called implicitly by `Contact::add_address_book`.
 pub fn normalize_name(full_name: impl AsRef<str>) -> String {
-    let mut full_name = full_name.as_ref().trim();
+    let full_name = full_name.as_ref().trim();
     if full_name.is_empty() {
         return full_name.into();
     }
 
-    let len = full_name.len();
-    if len > 1 {
-        let firstchar = full_name.as_bytes()[0];
-        let lastchar = full_name.as_bytes()[len - 1];
-        if firstchar == b'\'' && lastchar == b'\''
-            || firstchar == b'\"' && lastchar == b'\"'
-            || firstchar == b'<' && lastchar == b'>'
-        {
-            full_name = &full_name[1..len - 1];
-        }
+    match full_name.as_bytes() {
+        [b'\'', .., b'\''] | [b'\"', .., b'\"'] | [b'<', .., b'>'] => full_name
+            .get(1..full_name.len() - 1)
+            .map_or("".to_string(), |s| s.trim().into()),
+        _ => full_name.to_string(),
     }
-
-    if let Some(p1) = full_name.find(',') {
-        let (last_name, first_name) = full_name.split_at(p1);
-
-        let last_name = last_name.trim();
-        let first_name = (&first_name[1..]).trim();
-
-        return format!("{} {}", first_name, last_name);
-    }
-
-    full_name.trim().into()
 }
 
 fn cat_fingerprint(
@@ -1202,11 +1253,14 @@ mod tests {
         assert_eq!(may_be_valid_addr("u@d.tt"), true);
         assert_eq!(may_be_valid_addr("u@.tt"), false);
         assert_eq!(may_be_valid_addr("@d.tt"), false);
+        assert_eq!(may_be_valid_addr("<da@d.tt"), false);
+        assert_eq!(may_be_valid_addr("sk <@d.tt>"), false);
+        assert_eq!(may_be_valid_addr("as@sd.de>"), false);
+        assert_eq!(may_be_valid_addr("ask dkl@dd.tt"), false);
     }
 
     #[test]
     fn test_normalize_name() {
-        assert_eq!(&normalize_name("Doe, John"), "John Doe");
         assert_eq!(&normalize_name(" hello world   "), "hello world");
         assert_eq!(&normalize_name("<"), "<");
         assert_eq!(&normalize_name(">"), ">");
@@ -1241,7 +1295,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_contacts() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
         let contacts = Contact::get_all(&context.ctx, 0, Some("some2"))
             .await
             .unwrap();
@@ -1265,10 +1319,10 @@ mod tests {
 
     #[async_std::test]
     async fn test_is_self_addr() -> Result<()> {
-        let t = test_context().await;
+        let t = TestContext::new().await;
         assert!(t.ctx.is_self_addr("me@me.org").await.is_err());
 
-        let addr = configure_alice_keypair(&t.ctx).await;
+        let addr = t.configure_alice().await;
         assert_eq!(t.ctx.is_self_addr("me@me.org").await?, false);
         assert_eq!(t.ctx.is_self_addr(&addr).await?, true);
 
@@ -1278,15 +1332,16 @@ mod tests {
     #[async_std::test]
     async fn test_add_or_lookup() {
         // add some contacts, this also tests add_address_book()
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let book = concat!(
             "  Name one  \n one@eins.org \n",
             "Name two\ntwo@deux.net\n",
             "Invalid\n+1234567890\n", // invalid, should be ignored
             "\nthree@drei.sam\n",
-            "Name two\ntwo@deux.net\n" // should not be added again
+            "Name two\ntwo@deux.net\n", // should not be added again
+            "\nWonderland, Alice <alice@w.de>\n",
         );
-        assert_eq!(Contact::add_address_book(&t.ctx, book).await.unwrap(), 3);
+        assert_eq!(Contact::add_address_book(&t.ctx, book).await.unwrap(), 4);
 
         // check first added contact, this does not modify because of lower origin
         let (contact_id, sth_modified) =
@@ -1362,6 +1417,19 @@ mod tests {
         assert_eq!(contact.get_name_n_addr(), "schnucki (three@drei.sam)");
         assert!(!contact.is_blocked());
 
+        // Fourth contact:
+        let (contact_id, sth_modified) =
+            Contact::add_or_lookup(&t.ctx, "", "alice@w.de", Origin::IncomingUnknownTo)
+                .await
+                .unwrap();
+        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert_eq!(sth_modified, Modifier::None);
+        let contact = Contact::load_from_db(&t.ctx, contact_id).await.unwrap();
+        assert_eq!(contact.get_name(), "Wonderland, Alice");
+        assert_eq!(contact.get_display_name(), "Wonderland, Alice");
+        assert_eq!(contact.get_addr(), "alice@w.de");
+        assert_eq!(contact.get_name_n_addr(), "Wonderland, Alice (alice@w.de)");
+
         // check SELF
         let contact = Contact::load_from_db(&t.ctx, DC_CONTACT_ID_SELF)
             .await
@@ -1377,7 +1445,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_remote_authnames() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         // incoming mail `From: bob1 <bob@example.org>` - this should init authname and name
         let (contact_id, sth_modified) = Contact::add_or_lookup(
@@ -1440,7 +1508,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_remote_authnames_create_empty() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         // manually create "claire@example.org" without a given name
         let contact_id = Contact::create(&t.ctx, "", "claire@example.org")
@@ -1487,7 +1555,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_remote_authnames_edit_empty() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         // manually create "dave@example.org"
         let contact_id = Contact::create(&t.ctx, "dave1", "dave@example.org")
@@ -1528,4 +1596,50 @@ mod tests {
         assert!(addr_cmp(" aa@aa.ORG ", "AA@AA.ORG"));
         assert!(addr_cmp(" mailto:AA@AA.ORG", "Aa@Aa.orG"));
     }
+
+    #[async_std::test]
+    async fn test_name_in_address() {
+        let t = TestContext::new().await;
+
+        let contact_id = Contact::create(&t.ctx, "", "<dave@example.org>")
+            .await
+            .unwrap();
+        let contact = Contact::load_from_db(&t.ctx, contact_id).await.unwrap();
+        assert_eq!(contact.get_name(), "");
+        assert_eq!(contact.get_addr(), "dave@example.org");
+
+        let contact_id = Contact::create(&t.ctx, "", "Mueller, Dave <dave@example.org>")
+            .await
+            .unwrap();
+        let contact = Contact::load_from_db(&t.ctx, contact_id).await.unwrap();
+        assert_eq!(contact.get_name(), "Mueller, Dave");
+        assert_eq!(contact.get_addr(), "dave@example.org");
+
+        let contact_id = Contact::create(&t.ctx, "name1", "name2 <dave@example.org>")
+            .await
+            .unwrap();
+        let contact = Contact::load_from_db(&t.ctx, contact_id).await.unwrap();
+        assert_eq!(contact.get_name(), "name1");
+        assert_eq!(contact.get_addr(), "dave@example.org");
+
+        assert!(Contact::create(&t.ctx, "", "<dskjfdslk@sadklj.dk")
+            .await
+            .is_err());
+        assert!(Contact::create(&t.ctx, "", "<dskjf>dslk@sadklj.dk>")
+            .await
+            .is_err());
+        assert!(Contact::create(&t.ctx, "", "dskjfdslksadklj.dk")
+            .await
+            .is_err());
+        assert!(Contact::create(&t.ctx, "", "dskjfdslk@sadklj.dk>")
+            .await
+            .is_err());
+        assert!(Contact::create(&t.ctx, "", "dskjf@dslk@sadkljdk")
+            .await
+            .is_err());
+        assert!(Contact::create(&t.ctx, "", "dskjf dslk@d.e").await.is_err());
+        assert!(Contact::create(&t.ctx, "", "<dskjf dslk@sadklj.dk")
+            .await
+            .is_err());
+    }
 }

src/context.rs

@@ -6,6 +6,7 @@ use std::ops::Deref;
 
 use async_std::path::{Path, PathBuf};
 use async_std::sync::{channel, Arc, Mutex, Receiver, RwLock, Sender};
+use async_std::task;
 
 use crate::chat::*;
 use crate::config::Config;
@@ -13,14 +14,12 @@ use crate::constants::*;
 use crate::contact::*;
 use crate::dc_tools::duration_to_str;
 use crate::error::*;
-use crate::events::{Event, EventEmitter, Events};
-use crate::job::{self, Action};
-use crate::key::{DcKey, Key, SignedPublicKey};
+use crate::events::{Event, EventEmitter, EventType, Events};
+use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
-use crate::lot::Lot;
-use crate::message::{self, Message, MessengerMessage, MsgId};
-use crate::param::Params;
+use crate::message::{self, MsgId};
 use crate::scheduler::Scheduler;
+use crate::securejoin::Bob;
 use crate::sql::Sql;
 use std::time::SystemTime;
 
@@ -45,17 +44,23 @@ pub struct InnerContext {
     pub(crate) blobdir: PathBuf,
     pub(crate) sql: Sql,
     pub(crate) os_name: Option<String>,
-    pub(crate) bob: RwLock<BobStatus>,
+    pub(crate) bob: RwLock<Bob>,
     pub(crate) last_smeared_timestamp: RwLock<i64>,
     pub(crate) running_state: RwLock<RunningState>,
     /// Mutex to avoid generating the key for the user more than once.
     pub(crate) generating_key_mutex: Mutex<()>,
     /// Mutex to enforce only a single running oauth2 is running.
     pub(crate) oauth2_mutex: Mutex<()>,
+    /// Mutex to prevent a race condition when a "your pw is wrong" warning is sent, resulting in multiple messeges being sent.
+    pub(crate) wrong_pw_warning_mutex: Mutex<()>,
     pub(crate) translated_stockstrings: RwLock<HashMap<usize, String>>,
     pub(crate) events: Events,
 
     pub(crate) scheduler: RwLock<Scheduler>,
+    pub(crate) ephemeral_task: RwLock<Option<task::JoinHandle<()>>>,
+
+    /// Id for this context on the current device.
+    pub(crate) id: u32,
 
     creation_time: SystemTime,
 }
@@ -84,7 +89,7 @@ pub fn get_info() -> BTreeMap<&'static str, String> {
 
 impl Context {
     /// Creates new context.
-    pub async fn new(os_name: String, dbfile: PathBuf) -> Result<Context> {
+    pub async fn new(os_name: String, dbfile: PathBuf, id: u32) -> Result<Context> {
         // pretty_env_logger::try_init_timed().ok();
 
         let mut blob_fname = OsString::new();
@@ -94,13 +99,14 @@ impl Context {
         if !blobdir.exists().await {
             async_std::fs::create_dir_all(&blobdir).await?;
         }
-        Context::with_blobdir(os_name, dbfile, blobdir).await
+        Context::with_blobdir(os_name, dbfile, blobdir, id).await
     }
 
-    pub async fn with_blobdir(
+    pub(crate) async fn with_blobdir(
         os_name: String,
         dbfile: PathBuf,
         blobdir: PathBuf,
+        id: u32,
     ) -> Result<Context> {
         ensure!(
             blobdir.is_dir().await,
@@ -109,6 +115,7 @@ impl Context {
         );
 
         let inner = InnerContext {
+            id,
             blobdir,
             dbfile,
             os_name: Some(os_name),
@@ -118,19 +125,18 @@ impl Context {
             last_smeared_timestamp: RwLock::new(0),
             generating_key_mutex: Mutex::new(()),
             oauth2_mutex: Mutex::new(()),
+            wrong_pw_warning_mutex: Mutex::new(()),
             translated_stockstrings: RwLock::new(HashMap::new()),
             events: Events::default(),
             scheduler: RwLock::new(Scheduler::Stopped),
+            ephemeral_task: RwLock::new(None),
             creation_time: std::time::SystemTime::now(),
         };
 
         let ctx = Context {
             inner: Arc::new(inner),
         };
-        ensure!(
-            ctx.sql.open(&ctx, &ctx.dbfile, false).await,
-            "Failed opening sqlite database"
-        );
+        ctx.sql.open(&ctx, &ctx.dbfile, false).await?;
 
         Ok(ctx)
     }
@@ -138,10 +144,15 @@ impl Context {
     /// Starts the IO scheduler.
     pub async fn start_io(&self) {
         info!(self, "starting IO");
-        assert!(!self.is_io_running().await, "context is already running");
+        if self.is_io_running().await {
+            info!(self, "IO is already running");
+            return;
+        }
 
-        let l = &mut *self.inner.scheduler.write().await;
-        l.start(self.clone()).await;
+        {
+            let l = &mut *self.inner.scheduler.write().await;
+            l.start(self.clone()).await;
+        }
     }
 
     /// Returns if the IO scheduler is running.
@@ -152,6 +163,11 @@ impl Context {
     /// Stops the IO scheduler.
     pub async fn stop_io(&self) {
         info!(self, "stopping IO");
+        if !self.is_io_running().await {
+            info!(self, "IO is not running");
+            return;
+        }
+
         self.inner.stop_io().await;
     }
 
@@ -174,8 +190,11 @@ impl Context {
     }
 
     /// Emits a single event.
-    pub fn emit_event(&self, event: Event) {
-        self.events.emit(event);
+    pub fn emit_event(&self, event: EventType) {
+        self.events.emit(Event {
+            id: self.id,
+            typ: event,
+        });
     }
 
     /// Get the next queued event.
@@ -183,6 +202,11 @@ impl Context {
         self.events.get_emitter()
     }
 
+    /// Get the ID of this context.
+    pub fn get_id(&self) -> u32 {
+        self.id
+    }
+
     // Ongoing process allocation/free/check
 
     pub async fn alloc_ongoing(&self) -> Result<Receiver<()>> {
@@ -275,7 +299,7 @@ impl Context {
             .query_get_value(self, "SELECT COUNT(*) FROM acpeerstates;", paramsv![])
             .await;
         let fingerprint_str = match SignedPublicKey::load_self(self).await {
-            Ok(key) => Key::from(key).fingerprint(),
+            Ok(key) => key.fingerprint().hex(),
             Err(err) => format!("<key failure: {}>", err),
         };
 
@@ -290,13 +314,11 @@ impl Context {
             .unwrap_or_default();
 
         let configured_sentbox_folder = self
-            .sql
-            .get_raw_config(self, "configured_sentbox_folder")
+            .get_config(Config::ConfiguredSentboxFolder)
             .await
             .unwrap_or_else(|| "<unset>".to_string());
         let configured_mvbox_folder = self
-            .sql
-            .get_raw_config(self, "configured_mvbox_folder")
+            .get_config(Config::ConfiguredMvboxFolder)
             .await
             .unwrap_or_else(|| "<unset>".to_string());
 
@@ -432,61 +454,26 @@ impl Context {
             .unwrap_or_default()
     }
 
-    pub fn is_inbox(&self, folder_name: impl AsRef<str>) -> bool {
-        folder_name.as_ref() == "INBOX"
+    pub async fn is_inbox(&self, folder_name: impl AsRef<str>) -> bool {
+        self.get_config(Config::ConfiguredInboxFolder).await
+            == Some(folder_name.as_ref().to_string())
     }
 
     pub async fn is_sentbox(&self, folder_name: impl AsRef<str>) -> bool {
-        let sentbox_name = self
-            .sql
-            .get_raw_config(self, "configured_sentbox_folder")
-            .await;
-        if let Some(name) = sentbox_name {
-            name == folder_name.as_ref()
-        } else {
-            false
-        }
+        self.get_config(Config::ConfiguredSentboxFolder).await
+            == Some(folder_name.as_ref().to_string())
     }
 
     pub async fn is_mvbox(&self, folder_name: impl AsRef<str>) -> bool {
-        let mvbox_name = self
-            .sql
-            .get_raw_config(self, "configured_mvbox_folder")
-            .await;
-
-        if let Some(name) = mvbox_name {
-            name == folder_name.as_ref()
-        } else {
-            false
-        }
+        self.get_config(Config::ConfiguredMvboxFolder).await
+            == Some(folder_name.as_ref().to_string())
     }
 
-    pub async fn do_heuristics_moves(&self, folder: &str, msg_id: MsgId) {
-        if !self.get_config_bool(Config::MvboxMove).await {
-            return;
-        }
-
-        if self.is_mvbox(folder).await {
-            return;
-        }
-        if let Ok(msg) = Message::load_from_db(self, msg_id).await {
-            if msg.is_setupmessage() {
-                // do not move setup messages;
-                // there may be a non-delta device that wants to handle it
-                return;
-            }
-
-            match msg.is_dc_message {
-                MessengerMessage::No => {}
-                MessengerMessage::Yes | MessengerMessage::Reply => {
-                    job::add(
-                        self,
-                        job::Job::new(Action::MoveMsg, msg.id.to_u32(), Params::new(), 0),
-                    )
-                    .await;
-                }
-            }
-        }
+    pub fn derive_blobdir(dbfile: &PathBuf) -> PathBuf {
+        let mut blob_fname = OsString::new();
+        blob_fname.push(dbfile.file_name().unwrap_or_default());
+        blob_fname.push("-blobs");
+        dbfile.with_file_name(blob_fname)
     }
 }
 
@@ -518,13 +505,6 @@ impl Default for RunningState {
     }
 }
 
-#[derive(Debug, Default)]
-pub(crate) struct BobStatus {
-    pub expects: i32,
-    pub status: i32,
-    pub qr_scan: Option<Lot>,
-}
-
 pub fn get_version_str() -> &'static str {
     &DC_VERSION_STR
 }
@@ -540,13 +520,13 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         std::fs::write(&dbfile, b"123").unwrap();
-        let res = Context::new("FakeOs".into(), dbfile.into()).await;
+        let res = Context::new("FakeOs".into(), dbfile.into(), 1).await;
         assert!(res.is_err());
     }
 
     #[async_std::test]
     async fn test_get_fresh_msgs() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let fresh = t.ctx.get_fresh_msgs().await;
         assert!(fresh.is_empty())
     }
@@ -555,7 +535,9 @@ mod tests {
     async fn test_blobdir_exists() {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
-        Context::new("FakeOS".into(), dbfile.into()).await.unwrap();
+        Context::new("FakeOS".into(), dbfile.into(), 1)
+            .await
+            .unwrap();
         let blobdir = tmp.path().join("db.sqlite-blobs");
         assert!(blobdir.is_dir());
     }
@@ -566,7 +548,7 @@ mod tests {
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("db.sqlite-blobs");
         std::fs::write(&blobdir, b"123").unwrap();
-        let res = Context::new("FakeOS".into(), dbfile.into()).await;
+        let res = Context::new("FakeOS".into(), dbfile.into(), 1).await;
         assert!(res.is_err());
     }
 
@@ -576,7 +558,9 @@ mod tests {
         let subdir = tmp.path().join("subdir");
         let dbfile = subdir.join("db.sqlite");
         let dbfile2 = dbfile.clone();
-        Context::new("FakeOS".into(), dbfile.into()).await.unwrap();
+        Context::new("FakeOS".into(), dbfile.into(), 1)
+            .await
+            .unwrap();
         assert!(subdir.is_dir());
         assert!(dbfile2.is_file());
     }
@@ -586,7 +570,7 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = PathBuf::new();
-        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir.into()).await;
+        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir, 1).await;
         assert!(res.is_err());
     }
 
@@ -595,19 +579,19 @@ mod tests {
         let tmp = tempfile::tempdir().unwrap();
         let dbfile = tmp.path().join("db.sqlite");
         let blobdir = tmp.path().join("blobs");
-        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir.into()).await;
+        let res = Context::with_blobdir("FakeOS".into(), dbfile.into(), blobdir.into(), 1).await;
         assert!(res.is_err());
     }
 
     #[async_std::test]
     async fn no_crashes_on_context_deref() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         std::mem::drop(t.ctx);
     }
 
     #[async_std::test]
     async fn test_get_info() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
 
         let info = t.ctx.get_info().await;
         assert!(info.get("database_dir").is_some());

src/dc_tools.rs

@@ -4,24 +4,29 @@
 use core::cmp::{max, min};
 use std::borrow::Cow;
 use std::fmt;
+use std::io::Cursor;
 use std::str::FromStr;
 use std::time::{Duration, SystemTime};
 
 use async_std::path::{Path, PathBuf};
+use async_std::prelude::*;
 use async_std::{fs, io};
+
 use chrono::{Local, TimeZone};
 use rand::{thread_rng, Rng};
 
+use crate::chat::{add_device_msg, add_device_msg_with_importance};
+use crate::constants::{Viewtype, DC_OUTDATED_WARNING_DAYS};
 use crate::context::Context;
 use crate::error::{bail, Error};
-use crate::events::Event;
-
-pub(crate) fn dc_exactly_one_bit_set(v: i32) -> bool {
-    0 != v && 0 == v & (v - 1)
-}
+use crate::events::EventType;
+use crate::message::Message;
+use crate::provider::get_provider_update_timestamp;
+use crate::stock::StockMessage;
 
 /// Shortens a string to a specified length and adds "[...]" to the
 /// end of the shortened string.
+#[allow(clippy::indexing_slicing)]
 pub(crate) fn dc_truncate(buf: &str, approx_chars: usize) -> Cow<str> {
     let ellipse = "[...]";
 
@@ -54,6 +59,7 @@ const COLORS: [u32; 16] = [
     0xf2_30_30, 0x39_b2_49, 0xbb_24_3b, 0x96_40_78, 0x66_87_4f, 0x30_8a_b9, 0x12_7e_d0, 0xbe_45_0c,
 ];
 
+#[allow(clippy::indexing_slicing)]
 pub(crate) fn dc_str_to_color(s: impl AsRef<str>) -> u32 {
     let str_lower = s.as_ref().to_lowercase();
     let mut checksum = 0;
@@ -150,6 +156,73 @@ pub(crate) async fn dc_create_smeared_timestamps(context: &Context, count: usize
     start
 }
 
+// if the system time is not plausible, once a day, add a device message.
+// for testing we're using time() as that is also used for message timestamps.
+// moreover, add a warning if the app is outdated.
+pub(crate) async fn maybe_add_time_based_warnings(context: &Context) {
+    if !maybe_warn_on_bad_time(context, time(), get_provider_update_timestamp()).await {
+        maybe_warn_on_outdated(context, time(), get_provider_update_timestamp()).await;
+    }
+}
+
+async fn maybe_warn_on_bad_time(context: &Context, now: i64, known_past_timestamp: i64) -> bool {
+    if now < known_past_timestamp {
+        let mut msg = Message::new(Viewtype::Text);
+        msg.text = Some(
+            context
+                .stock_string_repl_str(
+                    StockMessage::BadTimeMsgBody,
+                    Local
+                        .timestamp(now, 0)
+                        .format("%Y-%m-%d %H:%M:%S")
+                        .to_string(),
+                )
+                .await,
+        );
+        add_device_msg_with_importance(
+            context,
+            Some(
+                format!(
+                    "bad-time-warning-{}",
+                    chrono::NaiveDateTime::from_timestamp(now, 0).format("%Y-%m-%d") // repeat every day
+                )
+                .as_str(),
+            ),
+            Some(&mut msg),
+            true,
+        )
+        .await
+        .ok();
+        return true;
+    }
+    false
+}
+
+async fn maybe_warn_on_outdated(context: &Context, now: i64, approx_compile_time: i64) {
+    if now > approx_compile_time + DC_OUTDATED_WARNING_DAYS * 24 * 60 * 60 {
+        let mut msg = Message::new(Viewtype::Text);
+        msg.text = Some(
+            context
+                .stock_str(StockMessage::UpdateReminderMsgBody)
+                .await
+                .into(),
+        );
+        add_device_msg(
+            context,
+            Some(
+                format!(
+                    "outdated-warning-{}",
+                    chrono::NaiveDateTime::from_timestamp(now, 0).format("%Y-%m") // repeat every month
+                )
+                .as_str(),
+            ),
+            Some(&mut msg),
+        )
+        .await
+        .ok();
+    }
+}
+
 /* Message-ID tools */
 pub(crate) fn dc_create_id() -> String {
     /* generate an id. the generated ID should be as short and as unique as possible:
@@ -198,7 +271,7 @@ fn encode_66bits_as_base64(v1: u32, v2: u32, fill: u32) -> String {
 pub(crate) fn dc_create_outgoing_rfc724_mid(grpid: Option<&str>, from_addr: &str) -> String {
     let hostname = from_addr
         .find('@')
-        .map(|k| &from_addr[k..])
+        .and_then(|k| from_addr.get(k..))
         .unwrap_or("@nohost");
     match grpid {
         Some(grpid) => format!("Gr.{}.{}{}", grpid, dc_create_id(), hostname),
@@ -240,9 +313,9 @@ pub fn dc_get_filesuffix_lc(path_filename: impl AsRef<str>) -> Option<String> {
 
 /// Returns the `(width, height)` of the given image buffer.
 pub fn dc_get_filemeta(buf: &[u8]) -> Result<(u32, u32), Error> {
-    let meta = image_meta::load_from_buf(buf)?;
-
-    Ok((meta.dimensions.width, meta.dimensions.height))
+    let image = image::io::Reader::new(Cursor::new(buf)).with_guessed_format()?;
+    let dimensions = image.into_dimensions()?;
+    Ok(dimensions)
 }
 
 /// Expand paths relative to $BLOBDIR into absolute paths.
@@ -283,7 +356,7 @@ pub(crate) async fn dc_delete_file(context: &Context, path: impl AsRef<Path>) ->
     let dpath = format!("{}", path.as_ref().to_string_lossy());
     match fs::remove_file(path_abs).await {
         Ok(_) => {
-            context.emit_event(Event::DeletedBlobFile(dpath));
+            context.emit_event(EventType::DeletedBlobFile(dpath));
             true
         }
         Err(err) => {
@@ -293,6 +366,23 @@ pub(crate) async fn dc_delete_file(context: &Context, path: impl AsRef<Path>) ->
     }
 }
 
+pub async fn dc_delete_files_in_dir(context: &Context, path: impl AsRef<Path>) {
+    match async_std::fs::read_dir(path).await {
+        Ok(mut read_dir) => {
+            while let Some(entry) = read_dir.next().await {
+                match entry {
+                    Ok(file) => {
+                        dc_delete_file(context, file.file_name()).await;
+                    }
+                    Err(e) => warn!(context, "Could not read file to delete: {}", e),
+                }
+            }
+        }
+
+        Err(e) => warn!(context, "Could not read dir to delete: {}", e),
+    }
+}
+
 pub(crate) async fn dc_copy_file(
     context: &Context,
     src_path: impl AsRef<Path>,
@@ -447,7 +537,7 @@ pub fn dc_open_file_std<P: AsRef<std::path::Path>>(
     }
 }
 
-pub(crate) async fn dc_get_next_backup_path(
+pub(crate) async fn get_next_backup_path_old(
     folder: impl AsRef<Path>,
     backup_time: i64,
 ) -> Result<PathBuf, Error> {
@@ -467,6 +557,32 @@ pub(crate) async fn dc_get_next_backup_path(
     bail!("could not create backup file, disk full?");
 }
 
+/// Returns Ok((temp_path, dest_path)) on success. The backup can then be written to temp_path. If the backup succeeded,
+/// it can be renamed to dest_path. This guarantees that the backup is complete.
+pub(crate) async fn get_next_backup_path_new(
+    folder: impl AsRef<Path>,
+    backup_time: i64,
+) -> Result<(PathBuf, PathBuf), Error> {
+    let folder = PathBuf::from(folder.as_ref());
+    let stem = chrono::NaiveDateTime::from_timestamp(backup_time, 0)
+        .format("delta-chat-backup-%Y-%m-%d")
+        .to_string();
+
+    // 64 backup files per day should be enough for everyone
+    for i in 0..64 {
+        let mut tempfile = folder.clone();
+        tempfile.push(format!("{}-{:02}.tar.part", stem, i));
+
+        let mut destfile = folder.clone();
+        destfile.push(format!("{}-{:02}.tar", stem, i));
+
+        if !tempfile.exists().await && !destfile.exists().await {
+            return Ok((tempfile, destfile));
+        }
+    }
+    bail!("could not create backup file, disk full?");
+}
+
 pub(crate) fn time() -> i64 {
     SystemTime::now()
         .duration_since(SystemTime::UNIX_EPOCH)
@@ -482,15 +598,6 @@ pub struct InvalidEmailError {
     addr: String,
 }
 
-impl InvalidEmailError {
-    fn new(msg: impl Into<String>, addr: impl Into<String>) -> InvalidEmailError {
-        InvalidEmailError {
-            message: msg.into(),
-            addr: addr.into(),
-        }
-    }
-}
-
 /// Very simple email address wrapper.
 ///
 /// Represents an email address, right now just the `name@domain` portion.
@@ -530,17 +637,24 @@ impl FromStr for EmailAddress {
 
     /// Performs a dead-simple parse of an email address.
     fn from_str(input: &str) -> Result<EmailAddress, InvalidEmailError> {
-        if input.is_empty() {
-            return Err(InvalidEmailError::new("empty string is not valid", input));
-        }
-        let parts: Vec<&str> = input.rsplitn(2, '@').collect();
-
         let err = |msg: &str| {
             Err(InvalidEmailError {
                 message: msg.to_string(),
                 addr: input.to_string(),
             })
         };
+        if input.is_empty() {
+            return err("empty string is not valid");
+        }
+        let parts: Vec<&str> = input.rsplitn(2, '@').collect();
+
+        if input
+            .chars()
+            .any(|c| c.is_whitespace() || c == '<' || c == '>')
+        {
+            return err("Email must not contain whitespaces, '>' or '<'");
+        }
+
         match &parts[..] {
             [domain, local] => {
                 if local.is_empty() {
@@ -585,8 +699,32 @@ pub(crate) fn listflags_has(listflags: u32, bitindex: usize) -> bool {
     (listflags & bitindex) == bitindex
 }
 
+/// Makes sure that a user input that is not supposed to contain newlines does not contain newlines.
+pub(crate) fn improve_single_line_input(input: impl AsRef<str>) -> String {
+    input
+        .as_ref()
+        .replace("\n", " ")
+        .replace("\r", " ")
+        .trim()
+        .to_string()
+}
+
+pub(crate) trait IsNoneOrEmpty<T> {
+    fn is_none_or_empty(&self) -> bool;
+}
+impl<T> IsNoneOrEmpty<T> for Option<T>
+where
+    T: AsRef<str>,
+{
+    fn is_none_or_empty(&self) -> bool {
+        !matches!(self, Some(s) if !s.as_ref().is_empty())
+    }
+}
+
 #[cfg(test)]
 mod tests {
+    #![allow(clippy::indexing_slicing)]
+
     use super::*;
     use std::convert::TryInto;
 
@@ -746,6 +884,9 @@ mod tests {
         assert_eq!("@d.tt".parse::<EmailAddress>().is_ok(), false);
     }
 
+    use crate::chat;
+    use crate::chatlist::Chatlist;
+    use chrono::{NaiveDate, NaiveDateTime, NaiveTime};
     use proptest::prelude::*;
 
     proptest! {
@@ -776,7 +917,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_file_handling() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let context = &t.ctx;
         macro_rules! dc_file_exist {
             ($ctx:expr, $fname:expr) => {
@@ -855,7 +996,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_smeared_timestamp() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         assert_ne!(
             dc_create_smeared_timestamp(&t.ctx).await,
             dc_create_smeared_timestamp(&t.ctx).await
@@ -871,7 +1012,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_smeared_timestamps() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let count = MAX_SECONDS_TO_LEND_FROM_FUTURE - 1;
         let start = dc_create_smeared_timestamps(&t.ctx, count as usize).await;
         let next = dc_smeared_time(&t.ctx).await;
@@ -915,4 +1056,147 @@ mod tests {
             "3h 1m 0s"
         );
     }
+
+    #[test]
+    fn test_get_filemeta() {
+        let data = include_bytes!("../test-data/image/avatar900x900.png");
+        let (w, h) = dc_get_filemeta(data).unwrap();
+        assert_eq!(w, 900);
+        assert_eq!(h, 900);
+
+        let data = include_bytes!("../test-data/image/avatar1000x1000.jpg");
+        let (w, h) = dc_get_filemeta(data).unwrap();
+        assert_eq!(w, 1000);
+        assert_eq!(h, 1000);
+
+        let data = include_bytes!("../test-data/image/image100x50.gif");
+        let (w, h) = dc_get_filemeta(data).unwrap();
+        assert_eq!(w, 100);
+        assert_eq!(h, 50);
+    }
+
+    #[test]
+    fn test_improve_single_line_input() {
+        assert_eq!(improve_single_line_input("Hi\naiae "), "Hi aiae");
+        assert_eq!(improve_single_line_input("\r\nahte\n\r"), "ahte");
+    }
+
+    #[async_std::test]
+    async fn test_maybe_warn_on_bad_time() {
+        let t = TestContext::new().await;
+        let timestamp_now = time();
+        let timestamp_future = timestamp_now + 60 * 60 * 24 * 7;
+        let timestamp_past = NaiveDateTime::new(
+            NaiveDate::from_ymd(2020, 9, 1),
+            NaiveTime::from_hms(0, 0, 0),
+        )
+        .timestamp_millis()
+            / 1_000;
+
+        // a correct time must not add a device message
+        maybe_warn_on_bad_time(&t.ctx, timestamp_now, get_provider_update_timestamp()).await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 0);
+
+        // we cannot find out if a date in the future is wrong - a device message is not added
+        maybe_warn_on_bad_time(&t.ctx, timestamp_future, get_provider_update_timestamp()).await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 0);
+
+        // a date in the past must add a device message
+        maybe_warn_on_bad_time(&t.ctx, timestamp_past, get_provider_update_timestamp()).await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 1);
+        let device_chat_id = chats.get_chat_id(0);
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 1);
+
+        // the message should be added only once a day - test that an hour later and nearly a day later
+        maybe_warn_on_bad_time(
+            &t.ctx,
+            timestamp_past + 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 1);
+
+        maybe_warn_on_bad_time(
+            &t.ctx,
+            timestamp_past + 60 * 60 * 24 - 1,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 1);
+
+        // next day, there should be another device message
+        maybe_warn_on_bad_time(
+            &t.ctx,
+            timestamp_past + 60 * 60 * 24,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 1);
+        assert_eq!(device_chat_id, chats.get_chat_id(0));
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 2);
+    }
+
+    #[async_std::test]
+    async fn test_maybe_warn_on_outdated() {
+        let t = TestContext::new().await;
+        let timestamp_now: i64 = time();
+
+        // in about 6 months, the app should not be outdated
+        // (if this fails, provider-db is not updated since 6 months)
+        maybe_warn_on_outdated(
+            &t.ctx,
+            timestamp_now + 180 * 24 * 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 0);
+
+        // in 1 year, the app should be considered as outdated
+        maybe_warn_on_outdated(
+            &t.ctx,
+            timestamp_now + 365 * 24 * 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 1);
+        let device_chat_id = chats.get_chat_id(0);
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 1);
+
+        // do not repeat the warning every day ...
+        maybe_warn_on_outdated(
+            &t.ctx,
+            timestamp_now + (365 + 1) * 24 * 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 1);
+        let device_chat_id = chats.get_chat_id(0);
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 1);
+
+        // ... but every month
+        maybe_warn_on_outdated(
+            &t.ctx,
+            timestamp_now + (365 + 31) * 24 * 60 * 60,
+            get_provider_update_timestamp(),
+        )
+        .await;
+        let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
+        assert_eq!(chats.len(), 1);
+        let device_chat_id = chats.get_chat_id(0);
+        let msgs = chat::get_chat_msgs(&t.ctx, device_chat_id, 0, None).await;
+        assert_eq!(msgs.len(), 2);
+    }
 }

src/dehtml.rs

@@ -2,12 +2,10 @@
 //!
 //! A module to remove HTML tags from the email text
 
-use lazy_static::lazy_static;
+use once_cell::sync::Lazy;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
-lazy_static! {
-    static ref LINE_RE: regex::Regex = regex::Regex::new(r"(\r?\n)+").unwrap();
-}
+static LINE_RE: Lazy<regex::Regex> = Lazy::new(|| regex::Regex::new(r"(\r?\n)+").unwrap());
 
 struct Dehtml {
     strbuilder: String,
@@ -24,8 +22,20 @@ enum AddText {
 
 // dehtml() returns way too many newlines; however, an optimisation on this issue is not needed as
 // the newlines are typically removed in further processing by the caller
-pub fn dehtml(buf: &str) -> String {
-    let buf = buf.trim();
+pub fn dehtml(buf: &str) -> Option<String> {
+    let s = dehtml_quick_xml(buf);
+    if !s.trim().is_empty() {
+        return Some(s);
+    }
+    let s = dehtml_manually(buf);
+    if !s.trim().is_empty() {
+        return Some(s);
+    }
+    None
+}
+
+pub fn dehtml_quick_xml(buf: &str) -> String {
+    let buf = buf.trim().trim_start_matches("<!doctype html>");
 
     let mut dehtml = Dehtml {
         strbuilder: String::with_capacity(buf.len()),
@@ -46,6 +56,12 @@ pub fn dehtml(buf: &str) -> String {
             Ok(quick_xml::events::Event::End(ref e)) => dehtml_endtag_cb(e, &mut dehtml),
             Ok(quick_xml::events::Event::Text(ref e)) => dehtml_text_cb(e, &mut dehtml),
             Ok(quick_xml::events::Event::CData(ref e)) => dehtml_cdata_cb(e, &mut dehtml),
+            Ok(quick_xml::events::Event::Empty(ref e)) => {
+                // Handle empty tags as a start tag immediately followed by end tag.
+                // For example, `<p/>` is treated as `<p></p>`.
+                dehtml_starttag_cb(e, &mut dehtml, &reader);
+                dehtml_endtag_cb(&BytesEnd::borrowed(e.name()), &mut dehtml);
+            }
             Err(e) => {
                 eprintln!(
                     "Parse html error: Error at position {}: {:?}",
@@ -165,9 +181,28 @@ fn dehtml_starttag_cb<B: std::io::BufRead>(
     }
 }
 
+pub fn dehtml_manually(buf: &str) -> String {
+    // Just strip out everything between "<" and ">"
+    let mut strbuilder = String::new();
+    let mut show_next_chars = true;
+    for c in buf.chars() {
+        match c {
+            '<' => show_next_chars = false,
+            '>' => show_next_chars = true,
+            _ => {
+                if show_next_chars {
+                    strbuilder.push(c)
+                }
+            }
+        }
+    }
+    strbuilder
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::simplify::simplify;
 
     #[test]
     fn test_dehtml() {
@@ -176,31 +211,40 @@ mod tests {
                 "<a href='https://example.com'> Foo </a>",
                 "[ Foo ](https://example.com)",
             ),
-            ("<img href='/foo.png'>", ""),
             ("<b> bar </b>", "* bar *"),
             ("<b> bar <i> foo", "* bar _ foo"),
             ("&amp; bar", "& bar"),
-            // Note missing '
-            ("<a href='/foo.png>Hi</a> ", ""),
-            ("", ""),
+            // Despite missing ', this should be shown:
+            ("<a href='/foo.png>Hi</a> ", "Hi "),
+            (
+                "<a href='https://get.delta.chat/'/>",
+                "[](https://get.delta.chat/)",
+            ),
+            ("<!doctype html>\n<b>fat text</b>", "*fat text*"),
+            // Invalid html (at least DC should show the text if the html is invalid):
+            ("<!some invalid html code>\n<b>some text</b>", "some text"),
         ];
         for (input, output) in cases {
-            assert_eq!(dehtml(input), output);
+            assert_eq!(simplify(dehtml(input).unwrap(), true).0, output);
+        }
+        let none_cases = vec!["<html> </html>", ""];
+        for input in none_cases {
+            assert_eq!(dehtml(input), None);
         }
     }
 
     #[test]
     fn test_dehtml_parse_br() {
-        let html = "\r\r\nline1<br>\r\n\r\n\r\rline2\n\r";
-        let plain = dehtml(html);
+        let html = "\r\r\nline1<br>\r\n\r\n\r\rline2<br/>line3\n\r";
+        let plain = dehtml(html).unwrap();
 
-        assert_eq!(plain, "line1\n\r\r\rline2");
+        assert_eq!(plain, "line1\n\r\r\rline2\nline3");
     }
 
     #[test]
     fn test_dehtml_parse_href() {
         let html = "<a href=url>text</a";
-        let plain = dehtml(html);
+        let plain = dehtml(html).unwrap();
 
         assert_eq!(plain, "[text](url)");
     }
@@ -208,7 +252,7 @@ mod tests {
     #[test]
     fn test_dehtml_bold_text() {
         let html = "<!DOCTYPE name [<!DOCTYPE ...>]><!-- comment -->text <b><?php echo ... ?>bold</b><![CDATA[<>]]>";
-        let plain = dehtml(html);
+        let plain = dehtml(html).unwrap();
 
         assert_eq!(plain, "text *bold*<>");
     }
@@ -218,7 +262,7 @@ mod tests {
         let html =
                 "&lt;&gt;&quot;&apos;&amp; &auml;&Auml;&ouml;&Ouml;&uuml;&Uuml;&szlig; foo&AElig;&ccedil;&Ccedil; &diams;&lrm;&rlm;&zwnj;&noent;&zwj;";
 
-        let plain = dehtml(html);
+        let plain = dehtml(html).unwrap();
 
         assert_eq!(
             plain,
@@ -241,7 +285,7 @@ mod tests {
         </body>
         </html>
         "##;
-        let txt = dehtml(input);
+        let txt = dehtml(input).unwrap();
         assert_eq!(txt.trim(), "lots of text");
     }
 }

src/e2ee.rs

@@ -11,11 +11,10 @@ use crate::context::Context;
 use crate::error::*;
 use crate::headerdef::HeaderDef;
 use crate::headerdef::HeaderDefMap;
-use crate::key::{DcKey, Key, SignedPublicKey, SignedSecretKey};
+use crate::key::{DcKey, Fingerprint, SignedPublicKey, SignedSecretKey};
 use crate::keyring::*;
 use crate::peerstate::*;
 use crate::pgp;
-use crate::securejoin::handle_degrade_event;
 
 #[derive(Debug)]
 pub struct EncryptHelper {
@@ -52,23 +51,42 @@ impl EncryptHelper {
     }
 
     /// Determines if we can and should encrypt.
+    ///
+    /// For encryption to be enabled, `e2ee_guaranteed` should be true, or strictly more than a half
+    /// of peerstates should prefer encryption. Own preference is counted equally to peer
+    /// preferences, even if message copy is not sent to self.
+    ///
+    /// `e2ee_guaranteed` should be set to true for replies to encrypted messages (as required by
+    /// Autocrypt Level 1, version 1.1) and for messages sent in protected groups.
+    ///
+    /// Returns an error if `e2ee_guaranteed` is true, but one or more keys are missing.
     pub fn should_encrypt(
         &self,
         context: &Context,
         e2ee_guaranteed: bool,
         peerstates: &[(Option<Peerstate>, &str)],
     ) -> Result<bool> {
-        if !(self.prefer_encrypt == EncryptPreference::Mutual || e2ee_guaranteed) {
-            return Ok(false);
-        }
-
+        let mut prefer_encrypt_count = if self.prefer_encrypt == EncryptPreference::Mutual {
+            1
+        } else {
+            0
+        };
         for (peerstate, addr) in peerstates {
             match peerstate {
                 Some(peerstate) => {
-                    if peerstate.prefer_encrypt != EncryptPreference::Mutual && !e2ee_guaranteed {
-                        info!(context, "peerstate for {:?} is no-encrypt", addr);
-                        return Ok(false);
-                    }
+                    info!(
+                        context,
+                        "peerstate for {:?} is {}", addr, peerstate.prefer_encrypt
+                    );
+                    match peerstate.prefer_encrypt {
+                        EncryptPreference::NoPreference => {}
+                        EncryptPreference::Mutual => prefer_encrypt_count += 1,
+                        EncryptPreference::Reset => {
+                            if !e2ee_guaranteed {
+                                return Ok(false);
+                            }
+                        }
+                    };
                 }
                 None => {
                     let msg = format!("peerstate for {:?} missing, cannot encrypt", addr);
@@ -82,45 +100,56 @@ impl EncryptHelper {
             }
         }
 
-        Ok(true)
+        // Count number of recipients, including self.
+        // This does not depend on whether we send a copy to self or not.
+        let recipients_count = peerstates.len() + 1;
+
+        Ok(e2ee_guaranteed || 2 * prefer_encrypt_count > recipients_count)
     }
 
     /// Tries to encrypt the passed in `mail`.
     pub async fn encrypt(
-        &mut self,
+        self,
         context: &Context,
         min_verified: PeerstateVerifiedStatus,
         mail_to_encrypt: lettre_email::PartBuilder,
-        peerstates: &[(Option<Peerstate<'_>>, &str)],
+        peerstates: Vec<(Option<Peerstate<'_>>, &str)>,
     ) -> Result<String> {
-        let mut keyring = Keyring::default();
+        let mut keyring: Keyring<SignedPublicKey> = Keyring::new();
 
         for (peerstate, addr) in peerstates
-            .iter()
-            .filter_map(|(state, addr)| state.as_ref().map(|s| (s, addr)))
+            .into_iter()
+            .filter_map(|(state, addr)| state.map(|s| (s, addr)))
         {
-            let key = peerstate.peek_key(min_verified).ok_or_else(|| {
+            let key = peerstate.take_key(min_verified).ok_or_else(|| {
                 format_err!("proper enc-key for {} missing, cannot encrypt", addr)
             })?;
-            keyring.add_ref(key);
+            keyring.add(key);
         }
-        let public_key = Key::from(self.public_key.clone());
-        keyring.add_ref(&public_key);
-        let sign_key = Key::from(SignedSecretKey::load_self(context).await?);
+        keyring.add(self.public_key.clone());
+        let sign_key = SignedSecretKey::load_self(context).await?;
 
         let raw_message = mail_to_encrypt.build().as_string().into_bytes();
 
-        let ctext = pgp::pk_encrypt(&raw_message, &keyring, Some(&sign_key))?;
+        let ctext = pgp::pk_encrypt(&raw_message, keyring, Some(sign_key)).await?;
 
         Ok(ctext)
     }
 }
 
+/// Tries to decrypt a message, but only if it is structured as an
+/// Autocrypt message.
+///
+/// Returns decrypted body and a set of valid signature fingerprints
+/// if successful.
+///
+/// If the message is wrongly signed, this will still return the decrypted
+/// message but the HashSet will be empty.
 pub async fn try_decrypt(
     context: &Context,
     mail: &ParsedMail<'_>,
     message_time: i64,
-) -> Result<(Option<Vec<u8>>, HashSet<String>)> {
+) -> Result<(Option<Vec<u8>>, HashSet<Fingerprint>)> {
     let from = mail
         .headers
         .get_header(HeaderDef::From_)
@@ -129,63 +158,54 @@ pub async fn try_decrypt(
         .map(|from| from.addr)
         .unwrap_or_default();
 
-    let mut peerstate = None;
-    let autocryptheader = Aheader::from_headers(context, &from, &mail.headers);
-
-    if message_time > 0 {
-        peerstate = Peerstate::from_addr(context, &from).await;
+    let mut peerstate = Peerstate::from_addr(context, &from).await?;
 
+    // Apply Autocrypt header
+    if let Some(ref header) = Aheader::from_headers(context, &from, &mail.headers) {
         if let Some(ref mut peerstate) = peerstate {
-            if let Some(ref header) = autocryptheader {
-                peerstate.apply_header(&header, message_time);
-                peerstate.save_to_db(&context.sql, false).await?;
-            } else if message_time > peerstate.last_seen_autocrypt && !contains_report(mail) {
-                peerstate.degrade_encryption(message_time);
-                peerstate.save_to_db(&context.sql, false).await?;
-            }
-        } else if let Some(ref header) = autocryptheader {
+            peerstate.apply_header(&header, message_time);
+            peerstate.save_to_db(&context.sql, false).await?;
+        } else {
             let p = Peerstate::from_header(context, header, message_time);
             p.save_to_db(&context.sql, true).await?;
             peerstate = Some(p);
         }
     }
 
-    /* possibly perform decryption */
-    let mut private_keyring = Keyring::default();
-    let mut public_keyring_for_validate = Keyring::default();
-    let mut out_mail = None;
+    // Possibly perform decryption
+    let private_keyring: Keyring<SignedSecretKey> = Keyring::new_self(context).await?;
+    let mut public_keyring_for_validate: Keyring<SignedPublicKey> = Keyring::new();
     let mut signatures = HashSet::default();
-    let self_addr = context.get_config(Config::ConfiguredAddr).await;
 
-    if let Some(self_addr) = self_addr {
-        if private_keyring
-            .load_self_private_for_decrypting(context, self_addr, &context.sql)
-            .await
-        {
-            if peerstate.as_ref().map(|p| p.last_seen).unwrap_or_else(|| 0) == 0 {
-                peerstate = Peerstate::from_addr(&context, &from).await;
-            }
-            if let Some(ref peerstate) = peerstate {
-                if peerstate.degrade_event.is_some() {
-                    handle_degrade_event(context, &peerstate).await?;
-                }
-                if let Some(ref key) = peerstate.gossip_key {
-                    public_keyring_for_validate.add_ref(key);
-                }
-                if let Some(ref key) = peerstate.public_key {
-                    public_keyring_for_validate.add_ref(key);
-                }
-            }
-
-            out_mail = decrypt_if_autocrypt_message(
-                context,
-                mail,
-                &private_keyring,
-                &public_keyring_for_validate,
-                &mut signatures,
-            )?;
+    if let Some(ref mut peerstate) = peerstate {
+        peerstate.handle_fingerprint_change(context).await?;
+        if let Some(key) = &peerstate.public_key {
+            public_keyring_for_validate.add(key.clone());
+        } else if let Some(key) = &peerstate.gossip_key {
+            public_keyring_for_validate.add(key.clone());
         }
     }
+
+    let out_mail = decrypt_if_autocrypt_message(
+        context,
+        mail,
+        private_keyring,
+        public_keyring_for_validate,
+        &mut signatures,
+    )
+    .await?;
+
+    if let Some(mut peerstate) = peerstate {
+        // If message is not encrypted and it is not a read receipt, degrade encryption.
+        if out_mail.is_none()
+            && message_time > peerstate.last_seen_autocrypt
+            && !contains_report(mail)
+        {
+            peerstate.degrade_encryption(message_time);
+            peerstate.save_to_db(&context.sql, false).await?;
+        }
+    }
+
     Ok((out_mail, signatures))
 }
 
@@ -196,40 +216,32 @@ fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Result<&'a ParsedMail
         "Not a multipart/encrypted message: {}",
         mail.ctype.mimetype
     );
-    ensure!(
-        mail.subparts.len() == 2,
-        "Invalid Autocrypt Level 1 Mime Parts"
-    );
+    if let [first_part, second_part] = &mail.subparts[..] {
+        ensure!(
+            first_part.ctype.mimetype == "application/pgp-encrypted",
+            "Invalid Autocrypt Level 1 version part: {:?}",
+            first_part.ctype,
+        );
 
-    ensure!(
-        mail.subparts[0].ctype.mimetype == "application/pgp-encrypted",
-        "Invalid Autocrypt Level 1 version part: {:?}",
-        mail.subparts[0].ctype,
-    );
+        ensure!(
+            second_part.ctype.mimetype == "application/octet-stream",
+            "Invalid Autocrypt Level 1 encrypted part: {:?}",
+            second_part.ctype
+        );
 
-    ensure!(
-        mail.subparts[1].ctype.mimetype == "application/octet-stream",
-        "Invalid Autocrypt Level 1 encrypted part: {:?}",
-        mail.subparts[1].ctype
-    );
-
-    Ok(&mail.subparts[1])
+        Ok(second_part)
+    } else {
+        bail!("Invalid Autocrypt Level 1 Mime Parts")
+    }
 }
 
-fn decrypt_if_autocrypt_message<'a>(
+async fn decrypt_if_autocrypt_message(
     context: &Context,
-    mail: &ParsedMail<'a>,
-    private_keyring: &Keyring,
-    public_keyring_for_validate: &Keyring,
-    ret_valid_signatures: &mut HashSet<String>,
+    mail: &ParsedMail<'_>,
+    private_keyring: Keyring<SignedSecretKey>,
+    public_keyring_for_validate: Keyring<SignedPublicKey>,
+    ret_valid_signatures: &mut HashSet<Fingerprint>,
 ) -> Result<Option<Vec<u8>>> {
-    //  The returned bool is true if we detected an Autocrypt-encrypted
-    // message and successfully decrypted it. Decryption then modifies the
-    // passed in mime structure in place. The returned bool is false
-    // if it was not an Autocrypt message.
-    //
-    // Errors are returned for failures related to decryption of AC-messages.
-
     let encrypted_data_part = match get_autocrypt_mime(mail) {
         Err(_) => {
             // not an autocrypt mime message, abort and ignore
@@ -240,21 +252,20 @@ fn decrypt_if_autocrypt_message<'a>(
     info!(context, "Detected Autocrypt-mime message");
 
     decrypt_part(
-        context,
         encrypted_data_part,
         private_keyring,
         public_keyring_for_validate,
         ret_valid_signatures,
     )
+    .await
 }
 
 /// Returns Ok(None) if nothing encrypted was found.
-fn decrypt_part(
-    _context: &Context,
+async fn decrypt_part(
     mail: &ParsedMail<'_>,
-    private_keyring: &Keyring,
-    public_keyring_for_validate: &Keyring,
-    ret_valid_signatures: &mut HashSet<String>,
+    private_keyring: Keyring<SignedSecretKey>,
+    public_keyring_for_validate: Keyring<SignedPublicKey>,
+    ret_valid_signatures: &mut HashSet<Fingerprint>,
 ) -> Result<Option<Vec<u8>>> {
     let data = mail.get_body_raw()?;
 
@@ -263,19 +274,23 @@ fn decrypt_part(
         ensure!(ret_valid_signatures.is_empty(), "corrupt signatures");
 
         let plain = pgp::pk_decrypt(
-            &data,
-            &private_keyring,
-            &public_keyring_for_validate,
+            data,
+            private_keyring,
+            public_keyring_for_validate,
             Some(ret_valid_signatures),
-        )?;
+        )
+        .await?;
+
+        // If the message was wrongly or not signed, still return the plain text.
+        // The caller has to check the signatures then.
 
-        ensure!(!ret_valid_signatures.is_empty(), "no valid signatures");
         return Ok(Some(plain));
     }
 
     Ok(None)
 }
 
+#[allow(clippy::indexing_slicing)]
 fn has_decrypted_pgp_armor(input: &[u8]) -> bool {
     if let Some(index) = input.iter().position(|b| *b > b' ') {
         if input.len() - index > 26 {
@@ -329,6 +344,11 @@ pub async fn ensure_secret_key_exists(context: &Context) -> Result<String> {
 mod tests {
     use super::*;
 
+    use crate::chat;
+    use crate::constants::Viewtype;
+    use crate::contact::{Contact, Origin};
+    use crate::message::Message;
+    use crate::param::Param;
     use crate::test_utils::*;
 
     mod ensure_secret_key_exists {
@@ -336,14 +356,14 @@ mod tests {
 
         #[async_std::test]
         async fn test_prexisting() {
-            let t = dummy_context().await;
-            let test_addr = configure_alice_keypair(&t.ctx).await;
+            let t = TestContext::new().await;
+            let test_addr = t.configure_alice().await;
             assert_eq!(ensure_secret_key_exists(&t.ctx).await.unwrap(), test_addr);
         }
 
         #[async_std::test]
         async fn test_not_configured() {
-            let t = dummy_context().await;
+            let t = TestContext::new().await;
             assert!(ensure_secret_key_exists(&t.ctx).await.is_err());
         }
     }
@@ -389,4 +409,161 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         let data = b"blas";
         assert_eq!(has_decrypted_pgp_armor(data), false);
     }
+
+    #[async_std::test]
+    async fn test_encrypted_no_autocrypt() -> crate::error::Result<()> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+
+        let (contact_alice_id, _modified) = Contact::add_or_lookup(
+            &bob.ctx,
+            "Alice",
+            "alice@example.com",
+            Origin::ManuallyCreated,
+        )
+        .await?;
+        let (contact_bob_id, _modified) = Contact::add_or_lookup(
+            &alice.ctx,
+            "Bob",
+            "bob@example.net",
+            Origin::ManuallyCreated,
+        )
+        .await?;
+
+        let chat_alice = chat::create_by_contact_id(&alice.ctx, contact_bob_id).await?;
+        let chat_bob = chat::create_by_contact_id(&bob.ctx, contact_alice_id).await?;
+
+        // Alice sends unencrypted message to Bob
+        let mut msg = Message::new(Viewtype::Text);
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;
+
+        // Bob receives unencrypted message from Alice
+        let msg = bob.parse_msg(&sent).await;
+        assert!(!msg.was_encrypted());
+
+        // Parsing a message is enough to update peerstate
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
+            .await?
+            .expect("no peerstate found in the database");
+        assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Mutual);
+
+        // Bob sends encrypted message to Alice
+        let mut msg = Message::new(Viewtype::Text);
+        chat::prepare_msg(&bob.ctx, chat_bob, &mut msg).await?;
+        chat::send_msg(&bob.ctx, chat_bob, &mut msg).await?;
+        let sent = bob.pop_sent_msg().await;
+
+        // Alice receives encrypted message from Bob
+        let msg = alice.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+
+        let peerstate_bob = Peerstate::from_addr(&alice.ctx, "bob@example.net")
+            .await?
+            .expect("no peerstate found in the database");
+        assert_eq!(peerstate_bob.prefer_encrypt, EncryptPreference::Mutual);
+
+        // Now Alice and Bob have established keys.
+
+        // Alice sends encrypted message without Autocrypt header.
+        let mut msg = Message::new(Viewtype::Text);
+        msg.param.set_int(Param::SkipAutocrypt, 1);
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;
+
+        let msg = bob.parse_msg(&sent).await;
+        assert!(msg.was_encrypted());
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
+            .await?
+            .expect("no peerstate found in the database");
+        assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Mutual);
+
+        // Alice sends plaintext message with Autocrypt header.
+        let mut msg = Message::new(Viewtype::Text);
+        msg.param.set_int(Param::ForcePlaintext, 1);
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;
+
+        let msg = bob.parse_msg(&sent).await;
+        assert!(!msg.was_encrypted());
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
+            .await?
+            .expect("no peerstate found in the database");
+        assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Mutual);
+
+        // Alice sends plaintext message without Autocrypt header.
+        let mut msg = Message::new(Viewtype::Text);
+        msg.param.set_int(Param::ForcePlaintext, 1);
+        msg.param.set_int(Param::SkipAutocrypt, 1);
+        chat::prepare_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        chat::send_msg(&alice.ctx, chat_alice, &mut msg).await?;
+        let sent = alice.pop_sent_msg().await;
+
+        let msg = bob.parse_msg(&sent).await;
+        assert!(!msg.was_encrypted());
+        let peerstate_alice = Peerstate::from_addr(&bob.ctx, "alice@example.com")
+            .await?
+            .expect("no peerstate found in the database");
+        assert_eq!(peerstate_alice.prefer_encrypt, EncryptPreference::Reset);
+
+        Ok(())
+    }
+
+    fn new_peerstates(
+        ctx: &Context,
+        prefer_encrypt: EncryptPreference,
+    ) -> Vec<(Option<Peerstate<'_>>, &str)> {
+        let addr = "bob@foo.bar";
+        let pub_key = bob_keypair().public;
+        let peerstate = Peerstate {
+            context: &ctx,
+            addr: addr.into(),
+            last_seen: 13,
+            last_seen_autocrypt: 14,
+            prefer_encrypt,
+            public_key: Some(pub_key.clone()),
+            public_key_fingerprint: Some(pub_key.fingerprint()),
+            gossip_key: Some(pub_key.clone()),
+            gossip_timestamp: 15,
+            gossip_key_fingerprint: Some(pub_key.fingerprint()),
+            verified_key: Some(pub_key.clone()),
+            verified_key_fingerprint: Some(pub_key.fingerprint()),
+            to_save: Some(ToSave::All),
+            fingerprint_changed: false,
+        };
+        let mut peerstates = Vec::new();
+        peerstates.push((Some(peerstate), addr));
+        peerstates
+    }
+
+    #[async_std::test]
+    async fn test_should_encrypt() {
+        let t = TestContext::new_alice().await;
+        let encrypt_helper = EncryptHelper::new(&t.ctx).await.unwrap();
+
+        // test with EncryptPreference::NoPreference:
+        // if e2ee_eguaranteed is unset, there is no encryption as not more than half of peers want encryption
+        let ps = new_peerstates(&t.ctx, EncryptPreference::NoPreference);
+        assert!(encrypt_helper.should_encrypt(&t.ctx, true, &ps).unwrap());
+        assert!(!encrypt_helper.should_encrypt(&t.ctx, false, &ps).unwrap());
+
+        // test with EncryptPreference::Reset
+        let ps = new_peerstates(&t.ctx, EncryptPreference::Reset);
+        assert!(encrypt_helper.should_encrypt(&t.ctx, true, &ps).unwrap());
+        assert!(!encrypt_helper.should_encrypt(&t.ctx, false, &ps).unwrap());
+
+        // test with EncryptPreference::Mutual (self is also Mutual)
+        let ps = new_peerstates(&t.ctx, EncryptPreference::Mutual);
+        assert!(encrypt_helper.should_encrypt(&t.ctx, true, &ps).unwrap());
+        assert!(encrypt_helper.should_encrypt(&t.ctx, false, &ps).unwrap());
+
+        // test with missing peerstate
+        let mut ps = Vec::new();
+        ps.push((None, "bob@foo.bar"));
+        assert!(encrypt_helper.should_encrypt(&t.ctx, true, &ps).is_err());
+        assert!(!encrypt_helper.should_encrypt(&t.ctx, false, &ps).unwrap());
+    }
 }

src/ephemeral.rs

@@ -0,0 +1,528 @@
+//! # Ephemeral messages
+//!
+//! Ephemeral messages are messages that have an Ephemeral-Timer
+//! header attached to them, which specifies time in seconds after
+//! which the message should be deleted both from the device and from
+//! the server. The timer is started when the message is marked as
+//! seen, which usually happens when its contents is displayed on
+//! device screen.
+//!
+//! Each chat, including 1:1, group chats and "saved messages" chat,
+//! has its own ephemeral timer setting, which is applied to all
+//! messages sent to the chat. The setting is synchronized to all the
+//! devices participating in the chat by applying the timer value from
+//! all received messages, including BCC-self ones, to the chat. This
+//! way the setting is eventually synchronized among all participants.
+//!
+//! When user changes ephemeral timer setting for the chat, a system
+//! message is automatically sent to update the setting for all
+//! participants. This allows changing the setting for a chat like any
+//! group chat setting, e.g. name and avatar, without the need to
+//! write an actual message.
+//!
+//! ## Device settings
+//!
+//! In addition to per-chat ephemeral message setting, each device has
+//! two global user-configured settings that complement per-chat
+//! settings: `delete_device_after` and `delete_server_after`. These
+//! settings are not synchronized among devices and apply to all
+//! messages known to the device, including messages sent or received
+//! before configuring the setting.
+//!
+//! `delete_device_after` configures the maximum time device is
+//! storing the messages locally. `delete_server_after` configures the
+//! time after which device will delete the messages it knows about
+//! from the server.
+//!
+//! ## How messages are deleted
+//!
+//! When the message is deleted locally, its contents is removed and
+//! it is moved to the trash chat. This database entry is then used to
+//! track the Message-ID and corresponding IMAP folder and UID until
+//! the message is deleted from the server. Vice versa, when device
+//! deletes the message from the server, it removes IMAP folder and
+//! UID information, but keeps the message contents. When database
+//! entry is both moved to trash chat and does not contain UID
+//! information, it is deleted from the database, leaving no trace of
+//! the message.
+//!
+//! ## When messages are deleted
+//!
+//! Local deletion happens when the chatlist or chat is loaded. A
+//! `MsgsChanged` event is emitted when a message deletion is due, to
+//! make UI reload displayed messages and cause actual deletion.
+//!
+//! Server deletion happens by generating IMAP deletion jobs based on
+//! the database entries which are expired either according to their
+//! ephemeral message timers or global `delete_server_after` setting.
+
+use crate::chat::{lookup_by_contact_id, send_msg, ChatId};
+use crate::constants::{
+    Viewtype, DC_CHAT_ID_LAST_SPECIAL, DC_CHAT_ID_TRASH, DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_SELF,
+};
+use crate::context::Context;
+use crate::dc_tools::time;
+use crate::error::{ensure, Error};
+use crate::events::EventType;
+use crate::message::{Message, MessageState, MsgId};
+use crate::mimeparser::SystemMessage;
+use crate::sql;
+use crate::stock::StockMessage;
+use async_std::task;
+use serde::{Deserialize, Serialize};
+use std::convert::{TryFrom, TryInto};
+use std::num::ParseIntError;
+use std::str::FromStr;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+#[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize)]
+pub enum Timer {
+    Disabled,
+    Enabled { duration: u32 },
+}
+
+impl Timer {
+    pub fn to_u32(self) -> u32 {
+        match self {
+            Self::Disabled => 0,
+            Self::Enabled { duration } => duration,
+        }
+    }
+
+    pub fn from_u32(duration: u32) -> Self {
+        if duration == 0 {
+            Self::Disabled
+        } else {
+            Self::Enabled { duration }
+        }
+    }
+}
+
+impl Default for Timer {
+    fn default() -> Self {
+        Self::Disabled
+    }
+}
+
+impl ToString for Timer {
+    fn to_string(&self) -> String {
+        self.to_u32().to_string()
+    }
+}
+
+impl FromStr for Timer {
+    type Err = ParseIntError;
+
+    fn from_str(input: &str) -> Result<Timer, ParseIntError> {
+        input.parse::<u32>().map(Self::from_u32)
+    }
+}
+
+impl rusqlite::types::ToSql for Timer {
+    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
+        let val = rusqlite::types::Value::Integer(match self {
+            Self::Disabled => 0,
+            Self::Enabled { duration } => i64::from(*duration),
+        });
+        let out = rusqlite::types::ToSqlOutput::Owned(val);
+        Ok(out)
+    }
+}
+
+impl rusqlite::types::FromSql for Timer {
+    fn column_result(value: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
+        i64::column_result(value).and_then(|value| {
+            if value == 0 {
+                Ok(Self::Disabled)
+            } else if let Ok(duration) = u32::try_from(value) {
+                Ok(Self::Enabled { duration })
+            } else {
+                Err(rusqlite::types::FromSqlError::OutOfRange(value))
+            }
+        })
+    }
+}
+
+impl ChatId {
+    /// Get ephemeral message timer value in seconds.
+    pub async fn get_ephemeral_timer(self, context: &Context) -> Result<Timer, Error> {
+        let timer = context
+            .sql
+            .query_get_value_result(
+                "SELECT ephemeral_timer FROM chats WHERE id=?;",
+                paramsv![self],
+            )
+            .await?;
+        Ok(timer.unwrap_or_default())
+    }
+
+    /// Set ephemeral timer value without sending a message.
+    ///
+    /// Used when a message arrives indicating that someone else has
+    /// changed the timer value for a chat.
+    pub(crate) async fn inner_set_ephemeral_timer(
+        self,
+        context: &Context,
+        timer: Timer,
+    ) -> Result<(), Error> {
+        ensure!(!self.is_special(), "Invalid chat ID");
+
+        context
+            .sql
+            .execute(
+                "UPDATE chats
+             SET ephemeral_timer=?
+             WHERE id=?;",
+                paramsv![timer, self],
+            )
+            .await?;
+
+        context.emit_event(EventType::ChatEphemeralTimerModified {
+            chat_id: self,
+            timer,
+        });
+        Ok(())
+    }
+
+    /// Set ephemeral message timer value in seconds.
+    ///
+    /// If timer value is 0, disable ephemeral message timer.
+    pub async fn set_ephemeral_timer(self, context: &Context, timer: Timer) -> Result<(), Error> {
+        if timer == self.get_ephemeral_timer(context).await? {
+            return Ok(());
+        }
+        self.inner_set_ephemeral_timer(context, timer).await?;
+        let mut msg = Message::new(Viewtype::Text);
+        msg.text = Some(stock_ephemeral_timer_changed(context, timer, DC_CONTACT_ID_SELF).await);
+        msg.param.set_cmd(SystemMessage::EphemeralTimerChanged);
+        if let Err(err) = send_msg(context, self, &mut msg).await {
+            error!(
+                context,
+                "Failed to send a message about ephemeral message timer change: {:?}", err
+            );
+        }
+        Ok(())
+    }
+}
+
+/// Returns a stock message saying that ephemeral timer is changed to `timer` by `from_id`.
+pub(crate) async fn stock_ephemeral_timer_changed(
+    context: &Context,
+    timer: Timer,
+    from_id: u32,
+) -> String {
+    let stock_message = match timer {
+        Timer::Disabled => StockMessage::MsgEphemeralTimerDisabled,
+        Timer::Enabled { duration } => match duration {
+            60 => StockMessage::MsgEphemeralTimerMinute,
+            3600 => StockMessage::MsgEphemeralTimerHour,
+            86400 => StockMessage::MsgEphemeralTimerDay,
+            604_800 => StockMessage::MsgEphemeralTimerWeek,
+            2_419_200 => StockMessage::MsgEphemeralTimerFourWeeks,
+            _ => StockMessage::MsgEphemeralTimerEnabled,
+        },
+    };
+
+    context
+        .stock_system_msg(stock_message, timer.to_string(), "", from_id)
+        .await
+}
+
+impl MsgId {
+    /// Returns ephemeral message timer value for the message.
+    pub(crate) async fn ephemeral_timer(self, context: &Context) -> crate::sql::Result<Timer> {
+        let res = match context
+            .sql
+            .query_get_value_result(
+                "SELECT ephemeral_timer FROM msgs WHERE id=?",
+                paramsv![self],
+            )
+            .await?
+        {
+            None | Some(0) => Timer::Disabled,
+            Some(duration) => Timer::Enabled { duration },
+        };
+        Ok(res)
+    }
+
+    /// Starts ephemeral message timer for the message if it is not started yet.
+    pub(crate) async fn start_ephemeral_timer(self, context: &Context) -> crate::sql::Result<()> {
+        if let Timer::Enabled { duration } = self.ephemeral_timer(context).await? {
+            let ephemeral_timestamp = time() + i64::from(duration);
+
+            context
+                .sql
+                .execute(
+                    "UPDATE msgs SET ephemeral_timestamp = ? \
+                WHERE (ephemeral_timestamp == 0 OR ephemeral_timestamp > ?) \
+                AND id = ?",
+                    paramsv![ephemeral_timestamp, ephemeral_timestamp, self],
+                )
+                .await?;
+            schedule_ephemeral_task(context).await;
+        }
+        Ok(())
+    }
+}
+
+/// Deletes messages which are expired according to
+/// `delete_device_after` setting or `ephemeral_timestamp` column.
+///
+/// Returns true if any message is deleted, so caller can emit
+/// MsgsChanged event. If nothing has been deleted, returns
+/// false. This function does not emit the MsgsChanged event itself,
+/// because it is also called when chatlist is reloaded, and emitting
+/// MsgsChanged there will cause infinite reload loop.
+pub(crate) async fn delete_expired_messages(context: &Context) -> Result<bool, Error> {
+    let mut updated = context
+        .sql
+        .execute(
+            "UPDATE msgs \
+             SET txt = 'DELETED', chat_id = ? \
+             WHERE \
+             ephemeral_timestamp != 0 \
+             AND ephemeral_timestamp < ? \
+             AND chat_id != ?",
+            paramsv![DC_CHAT_ID_TRASH, time(), DC_CHAT_ID_TRASH],
+        )
+        .await?
+        > 0;
+
+    if let Some(delete_device_after) = context.get_config_delete_device_after().await {
+        let self_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
+            .await
+            .unwrap_or_default()
+            .0;
+        let device_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
+            .await
+            .unwrap_or_default()
+            .0;
+
+        let threshold_timestamp = time() - delete_device_after;
+
+        // Delete expired messages
+        //
+        // Only update the rows that have to be updated, to avoid emitting
+        // unnecessary "chat modified" events.
+        let rows_modified = context
+            .sql
+            .execute(
+                "UPDATE msgs \
+             SET txt = 'DELETED', chat_id = ? \
+             WHERE timestamp < ? \
+             AND chat_id > ? \
+             AND chat_id != ? \
+             AND chat_id != ?",
+                paramsv![
+                    DC_CHAT_ID_TRASH,
+                    threshold_timestamp,
+                    DC_CHAT_ID_LAST_SPECIAL,
+                    self_chat_id,
+                    device_chat_id
+                ],
+            )
+            .await?;
+
+        updated |= rows_modified > 0;
+    }
+
+    schedule_ephemeral_task(context).await;
+    Ok(updated)
+}
+
+/// Schedule a task to emit MsgsChanged event when the next local
+/// deletion happens. Existing task is cancelled to make sure at most
+/// one such task is scheduled at a time.
+///
+/// UI is expected to reload the chatlist or the chat in response to
+/// MsgsChanged event, this will trigger actual deletion.
+///
+/// This takes into account only per-chat timeouts, because global device
+/// timeouts are at least one hour long and deletion is triggered often enough
+/// by user actions.
+pub async fn schedule_ephemeral_task(context: &Context) {
+    let ephemeral_timestamp: Option<i64> = match context
+        .sql
+        .query_get_value_result(
+            "SELECT ephemeral_timestamp \
+         FROM msgs \
+         WHERE ephemeral_timestamp != 0 \
+           AND chat_id != ? \
+         ORDER BY ephemeral_timestamp ASC \
+         LIMIT 1",
+            paramsv![DC_CHAT_ID_TRASH], // Trash contains already deleted messages, skip them
+        )
+        .await
+    {
+        Err(err) => {
+            warn!(context, "Can't calculate next ephemeral timeout: {}", err);
+            return;
+        }
+        Ok(ephemeral_timestamp) => ephemeral_timestamp,
+    };
+
+    // Cancel existing task, if any
+    if let Some(ephemeral_task) = context.ephemeral_task.write().await.take() {
+        ephemeral_task.cancel().await;
+    }
+
+    if let Some(ephemeral_timestamp) = ephemeral_timestamp {
+        let now = SystemTime::now();
+        let until = UNIX_EPOCH
+            + Duration::from_secs(ephemeral_timestamp.try_into().unwrap_or(u64::MAX))
+            + Duration::from_secs(1);
+
+        if let Ok(duration) = until.duration_since(now) {
+            // Schedule a task, ephemeral_timestamp is in the future
+            let context1 = context.clone();
+            let ephemeral_task = task::spawn(async move {
+                async_std::task::sleep(duration).await;
+                emit_event!(
+                    context1,
+                    EventType::MsgsChanged {
+                        chat_id: ChatId::new(0),
+                        msg_id: MsgId::new(0)
+                    }
+                );
+            });
+            *context.ephemeral_task.write().await = Some(ephemeral_task);
+        } else {
+            // Emit event immediately
+            emit_event!(
+                context,
+                EventType::MsgsChanged {
+                    chat_id: ChatId::new(0),
+                    msg_id: MsgId::new(0)
+                }
+            );
+        }
+    }
+}
+
+/// Returns ID of any expired message that should be deleted from the server.
+///
+/// It looks up the trash chat too, to find messages that are already
+/// deleted locally, but not deleted on the server.
+pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<Option<MsgId>> {
+    let now = time();
+
+    let threshold_timestamp = match context.get_config_delete_server_after().await {
+        None => 0,
+        Some(delete_server_after) => now - delete_server_after,
+    };
+
+    context
+        .sql
+        .query_row_optional(
+            "SELECT id FROM msgs \
+         WHERE ( \
+         timestamp < ? \
+         OR (ephemeral_timestamp != 0 AND ephemeral_timestamp < ?) \
+         ) \
+         AND server_uid != 0 \
+         LIMIT 1",
+            paramsv![threshold_timestamp, now],
+            |row| row.get::<_, MsgId>(0),
+        )
+        .await
+}
+
+/// Start ephemeral timers for seen messages if they are not started
+/// yet.
+///
+/// It is possible that timers are not started due to a missing or
+/// failed `MsgId.start_ephemeral_timer()` call, either in the current
+/// or previous version of Delta Chat.
+///
+/// This function is supposed to be called in the background,
+/// e.g. from housekeeping task.
+pub(crate) async fn start_ephemeral_timers(context: &Context) -> sql::Result<()> {
+    context
+        .sql
+        .execute(
+            "UPDATE msgs \
+    SET ephemeral_timestamp = ? + ephemeral_timer \
+    WHERE ephemeral_timer > 0 \
+    AND ephemeral_timestamp = 0 \
+    AND state NOT IN (?, ?, ?)",
+            paramsv![
+                time(),
+                MessageState::InFresh,
+                MessageState::InNoticed,
+                MessageState::OutDraft
+            ],
+        )
+        .await?;
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::test_utils::*;
+
+    #[async_std::test]
+    async fn test_stock_ephemeral_messages() {
+        let context = TestContext::new().await.ctx;
+
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Disabled, DC_CONTACT_ID_SELF).await,
+            "Message deletion timer is disabled by me."
+        );
+
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Disabled, 0).await,
+            "Message deletion timer is disabled."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 1 }, 0).await,
+            "Message deletion timer is set to 1 s."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 30 }, 0).await,
+            "Message deletion timer is set to 30 s."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 60 }, 0).await,
+            "Message deletion timer is set to 1 minute."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(&context, Timer::Enabled { duration: 60 * 60 }, 0).await,
+            "Message deletion timer is set to 1 hour."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled {
+                    duration: 24 * 60 * 60
+                },
+                0
+            )
+            .await,
+            "Message deletion timer is set to 1 day."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled {
+                    duration: 7 * 24 * 60 * 60
+                },
+                0
+            )
+            .await,
+            "Message deletion timer is set to 1 week."
+        );
+        assert_eq!(
+            stock_ephemeral_timer_changed(
+                &context,
+                Timer::Enabled {
+                    duration: 4 * 7 * 24 * 60 * 60
+                },
+                0
+            )
+            .await,
+            "Message deletion timer is set to 4 weeks."
+        );
+    }
+}

src/events.rs

@@ -1,10 +1,13 @@
 //! # Events specification
 
+use std::ops::Deref;
+
 use async_std::path::PathBuf;
 use async_std::sync::{channel, Receiver, Sender, TrySendError};
 use strum::EnumProperty;
 
 use crate::chat::ChatId;
+use crate::ephemeral::Timer as EphemeralTimer;
 use crate::message::MsgId;
 
 #[derive(Debug)]
@@ -53,14 +56,38 @@ impl EventEmitter {
         async_std::task::block_on(self.recv())
     }
 
-    /// Blocking async recv of an event. Return `None` if the `Sender` has been droped.
+    /// Async recv of an event. Return `None` if the `Sender` has been droped.
     pub async fn recv(&self) -> Option<Event> {
-        // TODO: change once we can use async channels internally.
         self.0.recv().await.ok()
     }
 }
 
-impl Event {
+impl async_std::stream::Stream for EventEmitter {
+    type Item = Event;
+
+    fn poll_next(
+        mut self: std::pin::Pin<&mut Self>,
+        cx: &mut std::task::Context<'_>,
+    ) -> std::task::Poll<Option<Self::Item>> {
+        std::pin::Pin::new(&mut self.0).poll_next(cx)
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Event {
+    pub id: u32,
+    pub typ: EventType,
+}
+
+impl Deref for Event {
+    type Target = EventType;
+
+    fn deref(&self) -> &EventType {
+        &self.typ
+    }
+}
+
+impl EventType {
     /// Returns the corresponding Event id.
     pub fn as_id(&self) -> i32 {
         self.get_str("id")
@@ -71,7 +98,7 @@ impl Event {
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, EnumProperty)]
-pub enum Event {
+pub enum EventType {
     /// The library-user may write an informational string to the log.
     /// Passed to the callback given to dc_context_new().
     /// This event should not be reported to the end-user using a popup or something like that.
@@ -98,15 +125,11 @@ pub enum Event {
     #[strum(props(id = "105"))]
     ImapMessageMoved(String),
 
-    /// Emitted when an IMAP folder was emptied
-    #[strum(props(id = "106"))]
-    ImapFolderEmptied(String),
-
     /// Emitted when an new file in the $BLOBDIR was created
     #[strum(props(id = "150"))]
     NewBlobFile(String),
 
-    /// Emitted when an new file in the $BLOBDIR was created
+    /// Emitted when an file in the $BLOBDIR was deleted
     #[strum(props(id = "151"))]
     DeletedBlobFile(String),
 
@@ -139,7 +162,6 @@ pub enum Event {
     /// Network errors should be reported to users in a non-disturbing way,
     /// however, as network errors may come in a sequence,
     /// it is not useful to raise each an every error to the user.
-    /// For this purpose, data1 is set to 1 if the error is probably worth reporting.
     ///
     /// Moreover, if the UI detects that the device is offline,
     /// it is probably more useful to report this to the user
@@ -170,6 +192,11 @@ pub enum Event {
     #[strum(props(id = "2005"))]
     IncomingMsg { chat_id: ChatId, msg_id: MsgId },
 
+    /// Messages were seen or noticed.
+    /// chat id is always set.
+    #[strum(props(id = "2008"))]
+    MsgsNoticed(ChatId),
+
     /// A single message is sent successfully. State changed from  DC_STATE_OUT_PENDING to
     /// DC_STATE_OUT_DELIVERED, see dc_msg_get_state().
     #[strum(props(id = "2010"))]
@@ -189,9 +216,19 @@ pub enum Event {
     /// Or the verify state of a chat has changed.
     /// See dc_set_chat_name(), dc_set_chat_profile_image(), dc_add_contact_to_chat()
     /// and dc_remove_contact_from_chat().
+    ///
+    /// This event does not include ephemeral timer modification, which
+    /// is a separate event.
     #[strum(props(id = "2020"))]
     ChatModified(ChatId),
 
+    /// Chat ephemeral timer changed.
+    #[strum(props(id = "2021"))]
+    ChatEphemeralTimerModified {
+        chat_id: ChatId,
+        timer: EphemeralTimer,
+    },
+
     /// Contact(s) created, renamed, blocked or deleted.
     ///
     /// @param data1 (int) If set, this is the contact_id of an added contact that should be selected.
@@ -207,10 +244,16 @@ pub enum Event {
     LocationChanged(Option<u32>),
 
     /// Inform about the configuration progress started by configure().
-    ///
-    /// @param data1 (usize) 0=error, 1-999=progress in permille, 1000=success and done
     #[strum(props(id = "2041"))]
-    ConfigureProgress(usize),
+    ConfigureProgress {
+        /// Progress.
+        ///
+        /// 0=error, 1-999=progress in permille, 1000=success and done
+        progress: usize,
+
+        /// Progress comment or error, something to display to the user.
+        comment: Option<String>,
+    },
 
     /// Inform about the import/export progress started by imex().
     ///

src/format_flowed.rs

@@ -0,0 +1,177 @@
+///! # format=flowed support
+///!
+///! Format=flowed is defined in
+///! [RFC 3676](https://tools.ietf.org/html/rfc3676).
+///!
+///! Older [RFC 2646](https://tools.ietf.org/html/rfc2646) is used
+///! during formatting, i.e., DelSp parameter introduced in RFC 3676
+///! is assumed to be set to "no".
+///!
+///! For received messages, DelSp parameter is honoured.
+
+/// Wraps line to 72 characters using format=flowed soft breaks.
+///
+/// 72 characters is the limit recommended by RFC 3676.
+///
+/// The function breaks line only after SP and before non-whitespace
+/// characters. It also does not insert breaks before ">" to avoid the
+/// need to do space stuffing (see RFC 3676) for quotes.
+///
+/// If there are long words, line may still exceed the limits on line
+/// length. However, this should be rare and should not result in
+/// immediate mail rejection: SMTP (RFC 2821) limit is 998 characters,
+/// and Spam Assassin limit is 78 characters.
+fn format_line_flowed(line: &str, prefix: &str) -> String {
+    let mut result = String::new();
+    let mut buffer = prefix.to_string();
+    let mut after_space = false;
+
+    for c in line.chars() {
+        if c == ' ' {
+            buffer.push(c);
+            after_space = true;
+        } else if c == '>' {
+            if buffer.is_empty() {
+                // Space stuffing, see RFC 3676
+                buffer.push(' ');
+            }
+            buffer.push(c);
+            after_space = false;
+        } else {
+            if after_space && buffer.len() >= 72 && !c.is_whitespace() {
+                // Flush the buffer and insert soft break (SP CRLF).
+                result += &buffer;
+                result += "\r\n";
+                buffer = prefix.to_string();
+            }
+            buffer.push(c);
+            after_space = false;
+        }
+    }
+    result + &buffer
+}
+
+fn format_flowed_prefix(text: &str, prefix: &str) -> String {
+    let mut result = String::new();
+
+    for line in text.split('\n') {
+        if !result.is_empty() {
+            result += "\r\n";
+        }
+        let line = line.trim_end();
+        if prefix.len() + line.len() > 78 {
+            result += &format_line_flowed(line, prefix);
+        } else {
+            result += prefix;
+            if prefix.is_empty() && line.starts_with('>') {
+                // Space stuffing, see RFC 3676
+                result.push(' ');
+            }
+            result += line;
+        }
+    }
+    result
+}
+
+/// Returns text formatted according to RFC 3767 (format=flowed).
+///
+/// This function accepts text separated by LF, but returns text
+/// separated by CRLF.
+///
+/// RFC 2646 technique is used to insert soft line breaks, so DelSp
+/// SHOULD be set to "no" when sending.
+pub fn format_flowed(text: &str) -> String {
+    format_flowed_prefix(text, "")
+}
+
+/// Same as format_flowed(), but adds "> " prefix to each line.
+pub fn format_flowed_quote(text: &str) -> String {
+    format_flowed_prefix(text, "> ")
+}
+
+/// Joins lines in format=flowed text.
+///
+/// Lines must be separated by single LF.
+///
+/// Quote processing is not supported, it is assumed that they are
+/// deleted during simplification.
+///
+/// Signature separator line is not processed here, it is assumed to
+/// be stripped beforehand.
+pub fn unformat_flowed(text: &str, delsp: bool) -> String {
+    let mut result = String::new();
+    let mut skip_newline = true;
+
+    for line in text.split('\n') {
+        // Revert space-stuffing
+        let line = line.strip_prefix(" ").unwrap_or(line);
+
+        if !skip_newline {
+            result.push('\n');
+        }
+
+        if let Some(line) = line.strip_suffix(" ") {
+            // Flowed line
+            result += line;
+            if !delsp {
+                result.push(' ');
+            }
+            skip_newline = true;
+        } else {
+            // Fixed line
+            result += line;
+            skip_newline = false;
+        }
+    }
+    result
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_format_flowed() {
+        let text = "Foo bar baz";
+        assert_eq!(format_flowed(text), "Foo bar baz");
+
+        let text = "This is the Autocrypt Setup Message used to transfer your key between clients.\n\
+                    \n\
+                    To decrypt and use your key, open the message in an Autocrypt-compliant client and enter the setup code presented on the generating device.";
+        let expected = "This is the Autocrypt Setup Message used to transfer your key between clients.\r\n\
+                        \r\n\
+                        To decrypt and use your key, open the message in an Autocrypt-compliant \r\n\
+                        client and enter the setup code presented on the generating device.";
+        assert_eq!(format_flowed(text), expected);
+
+        let text = "> Not a quote";
+        assert_eq!(format_flowed(text), " > Not a quote");
+    }
+
+    #[test]
+    fn test_unformat_flowed() {
+        let text = "this is a very long message that should be wrapped using format=flowed and \n\
+            unwrapped on the receiver";
+        let expected =
+            "this is a very long message that should be wrapped using format=flowed and \
+                        unwrapped on the receiver";
+        assert_eq!(unformat_flowed(text, false), expected);
+    }
+
+    #[test]
+    fn test_format_flowed_quote() {
+        let quote = "this is a quoted line";
+        let expected = "> this is a quoted line";
+        assert_eq!(format_flowed_quote(quote), expected);
+
+        let quote = "> foo bar baz";
+        let expected = "> > foo bar baz";
+        assert_eq!(format_flowed_quote(quote), expected);
+
+        let quote = "this is a very long quote that should be wrapped using format=flowed and unwrapped on the receiver";
+        let expected =
+            "> this is a very long quote that should be wrapped using format=flowed and \r\n\
+            > unwrapped on the receiver";
+        assert_eq!(format_flowed_quote(quote), expected);
+    }
+}

src/headerdef.rs

@@ -3,7 +3,6 @@ use mailparse::{MailHeader, MailHeaderMap};
 
 #[derive(Debug, Display, Clone, PartialEq, Eq, EnumVariantNames, AsStaticStr)]
 #[strum(serialize_all = "kebab_case")]
-#[allow(dead_code)]
 pub enum HeaderDef {
     MessageId,
     Subject,
@@ -21,6 +20,7 @@ pub enum HeaderDef {
     References,
     InReplyTo,
     Precedence,
+    ContentType,
     ChatVersion,
     ChatGroupId,
     ChatGroupName,
@@ -34,6 +34,7 @@ pub enum HeaderDef {
     ChatContent,
     ChatDuration,
     ChatDispositionNotificationTo,
+    ChatWebrtcRoom,
     Autocrypt,
     AutocryptSetupMessage,
     SecureJoin,
@@ -41,6 +42,7 @@ pub enum HeaderDef {
     SecureJoinFingerprint,
     SecureJoinInvitenumber,
     SecureJoinAuth,
+    EphemeralTimer,
     _TestHeader,
 }
 

src/imap/client.rs

@@ -7,7 +7,7 @@ use async_imap::{
 use async_std::net::{self, TcpStream};
 
 use super::session::Session;
-use crate::login_param::{dc_build_tls, CertificateChecks};
+use crate::login_param::dc_build_tls;
 
 use super::session::SessionStream;
 
@@ -56,7 +56,7 @@ impl Client {
     pub async fn authenticate<A: async_imap::Authenticator, S: AsRef<str>>(
         self,
         auth_type: S,
-        authenticator: &A,
+        authenticator: A,
     ) -> std::result::Result<Session, (ImapError, Self)> {
         let Client { inner, is_secure } = self;
         let session =
@@ -78,16 +78,13 @@ impl Client {
     pub async fn connect_secure<A: net::ToSocketAddrs, S: AsRef<str>>(
         addr: A,
         domain: S,
-        certificate_checks: CertificateChecks,
+        strict_tls: bool,
     ) -> ImapResult<Self> {
         let stream = TcpStream::connect(addr).await?;
-        let tls = dc_build_tls(certificate_checks);
+        let tls = dc_build_tls(strict_tls);
         let tls_stream: Box<dyn SessionStream> =
             Box::new(tls.connect(domain.as_ref(), stream).await?);
         let mut client = ImapClient::new(tls_stream);
-        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
-            client.debug = true;
-        }
 
         let _greeting = client
             .read_response()
@@ -104,9 +101,6 @@ impl Client {
         let stream: Box<dyn SessionStream> = Box::new(TcpStream::connect(addr).await?);
 
         let mut client = ImapClient::new(stream);
-        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
-            client.debug = true;
-        }
         let _greeting = client
             .read_response()
             .await
@@ -118,16 +112,12 @@ impl Client {
         })
     }
 
-    pub async fn secure<S: AsRef<str>>(
-        self,
-        domain: S,
-        certificate_checks: CertificateChecks,
-    ) -> ImapResult<Client> {
+    pub async fn secure<S: AsRef<str>>(self, domain: S, strict_tls: bool) -> ImapResult<Client> {
         if self.is_secure {
             Ok(self)
         } else {
             let Client { mut inner, .. } = self;
-            let tls = dc_build_tls(certificate_checks);
+            let tls = dc_build_tls(strict_tls);
             inner.run_command_and_check_ok("STARTTLS", None).await?;
 
             let stream = inner.into_inner();

src/imap/idle.rs

@@ -1,188 +1,186 @@
 use super::Imap;
 
 use async_imap::extensions::idle::IdleResponse;
+use async_imap::types::UnsolicitedResponse;
 use async_std::prelude::*;
 use std::time::{Duration, SystemTime};
 
-use crate::context::Context;
+use crate::error::{bail, format_err, Result};
+use crate::{context::Context, scheduler::InterruptInfo};
 
-use super::select_folder;
 use super::session::Session;
 
-type Result<T> = std::result::Result<T, Error>;
-
-#[derive(Debug, thiserror::Error)]
-pub enum Error {
-    #[error("IMAP IDLE protocol failed to init/complete")]
-    IdleProtocolFailed(#[from] async_imap::error::Error),
-
-    #[error("IMAP IDLE protocol timed out")]
-    IdleTimeout(#[from] async_std::future::TimeoutError),
-
-    #[error("IMAP server does not have IDLE capability")]
-    IdleAbilityMissing,
-
-    #[error("IMAP select folder error")]
-    SelectFolderError(#[from] select_folder::Error),
-
-    #[error("Setup handle error")]
-    SetupHandleError(#[from] super::Error),
-}
-
 impl Imap {
     pub fn can_idle(&self) -> bool {
         self.config.can_idle
     }
 
-    pub async fn idle(&mut self, context: &Context, watch_folder: Option<String>) -> Result<()> {
+    pub async fn idle(
+        &mut self,
+        context: &Context,
+        watch_folder: Option<String>,
+    ) -> Result<InterruptInfo> {
         use futures::future::FutureExt;
 
         if !self.can_idle() {
-            return Err(Error::IdleAbilityMissing);
+            bail!("IMAP server does not have IDLE capability");
         }
-        self.setup_handle_if_needed(context).await?;
+        self.setup_handle(context).await?;
 
         self.select_folder(context, watch_folder.clone()).await?;
 
-        let session = self.session.take();
         let timeout = Duration::from_secs(23 * 60);
+        let mut info = Default::default();
+
+        if let Some(session) = self.session.take() {
+            // if we have unsolicited responses we directly return
+            let mut unsolicited_exists = false;
+            while let Ok(response) = session.unsolicited_responses.try_recv() {
+                match response {
+                    UnsolicitedResponse::Exists(_) => {
+                        warn!(context, "skip idle, got unsolicited EXISTS {:?}", response);
+                        unsolicited_exists = true;
+                    }
+                    _ => info!(context, "ignoring unsolicited response {:?}", response),
+                }
+            }
+
+            if unsolicited_exists {
+                self.session = Some(session);
+                return Ok(info);
+            }
 
-        if let Some(session) = session {
             let mut handle = session.idle();
             if let Err(err) = handle.init().await {
-                return Err(Error::IdleProtocolFailed(err));
+                bail!("IMAP IDLE protocol failed to init/complete: {}", err);
             }
 
             let (idle_wait, interrupt) = handle.wait_with_timeout(timeout);
 
-            if self.skip_next_idle_wait {
-                // interrupt_idle has happened before we
-                // provided self.interrupt
-                self.skip_next_idle_wait = false;
-                drop(idle_wait);
+            enum Event {
+                IdleResponse(IdleResponse),
+                Interrupt(InterruptInfo),
+            }
+
+            info!(context, "Idle entering wait-on-remote state");
+            let fut = idle_wait.map(|ev| ev.map(Event::IdleResponse)).race(async {
+                let probe_network = self.idle_interrupt.recv().await;
+
+                // cancel imap idle connection properly
                 drop(interrupt);
 
-                info!(context, "Idle wait was skipped");
-            } else {
-                info!(context, "Idle entering wait-on-remote state");
-                let fut = idle_wait.race(
-                    self.idle_interrupt
-                        .recv()
-                        .map(|_| Ok(IdleResponse::ManualInterrupt)),
-                );
+                Ok(Event::Interrupt(probe_network.unwrap_or_default()))
+            });
 
-                match fut.await {
-                    Ok(IdleResponse::NewData(_)) => {
-                        info!(context, "Idle has NewData");
-                    }
-                    // TODO: idle_wait does not distinguish manual interrupts
-                    // from Timeouts if we would know it's a Timeout we could bail
-                    // directly and reconnect .
-                    Ok(IdleResponse::Timeout) => {
-                        info!(context, "Idle-wait timeout or interruption");
-                    }
-                    Ok(IdleResponse::ManualInterrupt) => {
-                        info!(context, "Idle wait was interrupted");
-                    }
-                    Err(err) => {
-                        warn!(context, "Idle wait errored: {:?}", err);
-                    }
+            match fut.await {
+                Ok(Event::IdleResponse(IdleResponse::NewData(x))) => {
+                    info!(context, "Idle has NewData {:?}", x);
+                }
+                Ok(Event::IdleResponse(IdleResponse::Timeout)) => {
+                    info!(context, "Idle-wait timeout or interruption");
+                }
+                Ok(Event::IdleResponse(IdleResponse::ManualInterrupt)) => {
+                    info!(context, "Idle wait was interrupted");
+                }
+                Ok(Event::Interrupt(i)) => {
+                    info = i;
+                    info!(context, "Idle wait was interrupted");
+                }
+                Err(err) => {
+                    warn!(context, "Idle wait errored: {:?}", err);
                 }
             }
 
-            // if we can't properly terminate the idle
-            // protocol let's break the connection.
-            let res = handle
+            let session = handle
                 .done()
                 .timeout(Duration::from_secs(15))
                 .await
-                .map_err(|err| {
-                    self.trigger_reconnect();
-                    Error::IdleTimeout(err)
-                })?;
-
-            match res {
-                Ok(session) => {
-                    self.session = Some(Session { inner: session });
-                }
-                Err(err) => {
-                    // if we cannot terminate IDLE it probably
-                    // means that we waited long (with idle_wait)
-                    // but the network went away/changed
-                    self.trigger_reconnect();
-                    return Err(Error::IdleProtocolFailed(err));
-                }
-            }
+                .map_err(|err| format_err!("IMAP IDLE protocol timed out: {}", err))??;
+            self.session = Some(Session { inner: session });
+        } else {
+            warn!(context, "Attempted to idle without a session");
         }
 
-        Ok(())
+        Ok(info)
     }
 
-    pub(crate) async fn fake_idle(&mut self, context: &Context, watch_folder: Option<String>) {
+    pub(crate) async fn fake_idle(
+        &mut self,
+        context: &Context,
+        watch_folder: Option<String>,
+    ) -> InterruptInfo {
         // Idle using polling. This is also needed if we're not yet configured -
         // in this case, we're waiting for a configure job (and an interrupt).
 
         let fake_idle_start_time = SystemTime::now();
-        info!(context, "IMAP-fake-IDLEing...");
 
-        if self.skip_next_idle_wait {
-            // interrupt_idle has happened before we
-            // provided self.interrupt
-            self.skip_next_idle_wait = false;
-            info!(context, "fake-idle wait was skipped");
-        } else {
-            // check every minute if there are new messages
-            // TODO: grow sleep durations / make them more flexible
-            let mut interval = async_std::stream::interval(Duration::from_secs(60));
+        // Do not poll, just wait for an interrupt when no folder is passed in.
+        if watch_folder.is_none() {
+            info!(context, "IMAP-fake-IDLE: no folder, waiting for interrupt");
+            return self.idle_interrupt.recv().await.unwrap_or_default();
+        }
+        info!(context, "IMAP-fake-IDLEing folder={:?}", watch_folder);
 
-            // loop until we are interrupted or if we fetched something
-            loop {
-                use futures::future::FutureExt;
-                match interval
-                    .next()
-                    .race(self.idle_interrupt.recv().map(|_| None))
-                    .await
-                {
-                    Some(_) => {
-                        // try to connect with proper login params
-                        // (setup_handle_if_needed might not know about them if we
-                        // never successfully connected)
-                        if let Err(err) = self.connect_configured(context).await {
-                            warn!(context, "fake_idle: could not connect: {}", err);
-                            continue;
-                        }
-                        if self.config.can_idle {
-                            // we only fake-idled because network was gone during IDLE, probably
-                            break;
-                        }
-                        info!(context, "fake_idle is connected");
-                        // we are connected, let's see if fetching messages results
-                        // in anything.  If so, we behave as if IDLE had data but
-                        // will have already fetched the messages so perform_*_fetch
-                        // will not find any new.
+        // check every minute if there are new messages
+        // TODO: grow sleep durations / make them more flexible
+        let mut interval = async_std::stream::interval(Duration::from_secs(60));
 
-                        if let Some(ref watch_folder) = watch_folder {
-                            match self.fetch_new_messages(context, watch_folder).await {
-                                Ok(res) => {
-                                    info!(context, "fetch_new_messages returned {:?}", res);
-                                    if res {
-                                        break;
-                                    }
-                                }
-                                Err(err) => {
-                                    error!(context, "could not fetch from folder: {}", err);
-                                    self.trigger_reconnect()
+        enum Event {
+            Tick,
+            Interrupt(InterruptInfo),
+        }
+        // loop until we are interrupted or if we fetched something
+        let info = loop {
+            use futures::future::FutureExt;
+            match interval
+                .next()
+                .map(|_| Event::Tick)
+                .race(
+                    self.idle_interrupt
+                        .recv()
+                        .map(|probe_network| Event::Interrupt(probe_network.unwrap_or_default())),
+                )
+                .await
+            {
+                Event::Tick => {
+                    // try to connect with proper login params
+                    // (setup_handle_if_needed might not know about them if we
+                    // never successfully connected)
+                    if let Err(err) = self.connect_configured(context).await {
+                        warn!(context, "fake_idle: could not connect: {}", err);
+                        continue;
+                    }
+                    if self.config.can_idle {
+                        // we only fake-idled because network was gone during IDLE, probably
+                        break InterruptInfo::new(false, None);
+                    }
+                    info!(context, "fake_idle is connected");
+                    // we are connected, let's see if fetching messages results
+                    // in anything.  If so, we behave as if IDLE had data but
+                    // will have already fetched the messages so perform_*_fetch
+                    // will not find any new.
+
+                    if let Some(ref watch_folder) = watch_folder {
+                        match self.fetch_new_messages(context, watch_folder, false).await {
+                            Ok(res) => {
+                                info!(context, "fetch_new_messages returned {:?}", res);
+                                if res {
+                                    break InterruptInfo::new(false, None);
                                 }
                             }
+                            Err(err) => {
+                                error!(context, "could not fetch from folder: {}", err);
+                                self.trigger_reconnect()
+                            }
                         }
                     }
-                    None => {
-                        // Interrupt
-                        break;
-                    }
+                }
+                Event::Interrupt(info) => {
+                    // Interrupt
+                    break info;
                 }
             }
-        }
+        };
 
         info!(
             context,
@@ -193,5 +191,7 @@ impl Imap {
                 .as_millis() as f64
                 / 1000.,
         );
+
+        info
     }
 }

src/imap/select_folder.rs

@@ -27,7 +27,7 @@ impl Imap {
     ///
     /// CLOSE is considerably faster than an EXPUNGE, see
     /// https://tools.ietf.org/html/rfc3501#section-6.4.2
-    async fn close_folder(&mut self, context: &Context) -> Result<()> {
+    pub(super) async fn close_folder(&mut self, context: &Context) -> Result<()> {
         if let Some(ref folder) = self.config.selected_folder {
             info!(context, "Expunge messages in \"{}\".", folder);
 
@@ -51,6 +51,14 @@ impl Imap {
         Ok(())
     }
 
+    /// Issues a CLOSE command if selected folder needs expunge.
+    pub(crate) async fn maybe_close_folder(&mut self, context: &Context) -> Result<()> {
+        if self.config.selected_folder_needs_expunge {
+            self.close_folder(context).await?;
+        }
+        Ok(())
+    }
+
     /// select a folder, possibly update uid_validity and, if needed,
     /// expunge the folder to remove delete-marked messages.
     pub(super) async fn select_folder<S: AsRef<str>>(
@@ -76,10 +84,7 @@ impl Imap {
         }
 
         // deselect existing folder, if needed (it's also done implicitly by SELECT, however, without EXPUNGE then)
-        let needs_expunge = { self.config.selected_folder_needs_expunge };
-        if needs_expunge {
-            self.close_folder(context).await?;
-        }
+        self.maybe_close_folder(context).await?;
 
         // select new folder
         if let Some(ref folder) = folder {

src/imex.rs

@@ -1,9 +1,17 @@
 //! # Import/export module
 
-use std::cmp::{max, min};
+use std::any::Any;
+use std::{
+    cmp::{max, min},
+    ffi::OsStr,
+};
 
+use anyhow::Context as _;
 use async_std::path::{Path, PathBuf};
-use async_std::prelude::*;
+use async_std::{
+    fs::{self, File},
+    prelude::*,
+};
 use rand::{thread_rng, Rng};
 
 use crate::blob::BlobObject;
@@ -15,14 +23,19 @@ use crate::context::Context;
 use crate::dc_tools::*;
 use crate::e2ee;
 use crate::error::*;
-use crate::events::Event;
-use crate::key::{self, DcKey, Key, SignedSecretKey};
+use crate::events::EventType;
+use crate::key::{self, DcKey, DcSecretKey, SignedPublicKey, SignedSecretKey};
 use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::param::*;
 use crate::pgp;
 use crate::sql::{self, Sql};
 use crate::stock::StockMessage;
+use async_tar::Archive;
+
+// Name of the database file in the backup.
+const DBFILE_BACKUP_NAME: &str = "dc_database_backup.sqlite";
+const BLOBS_BACKUP_NAME: &str = "blobs_backup";
 
 #[derive(Debug, Display, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive)]
 #[repr(i32)]
@@ -41,8 +54,8 @@ pub enum ImexMode {
     /// Export a backup to the directory given as `param1`.
     /// The backup contains all contacts, chats, images and other data and device independent settings.
     /// The backup does not contain device dependent settings as ringtones or LED notification settings.
-    /// The name of the backup is typically `delta-chat.<day>.bak`, if more than one backup is create on a day,
-    /// the format is `delta-chat.<day>-<number>.bak`
+    /// The name of the backup is typically `delta-chat-<day>.tar`, if more than one backup is create on a day,
+    /// the format is `delta-chat-<day>-<number>.tar`
     ExportBackup = 11,
 
     /// `param1` is the file (not: directory) to import. The file is normally
@@ -70,20 +83,82 @@ pub async fn imex(
     what: ImexMode,
     param1: Option<impl AsRef<Path>>,
 ) -> Result<()> {
-    use futures::future::FutureExt;
-
     let cancel = context.alloc_ongoing().await?;
-    let res = imex_inner(context, what, param1)
-        .race(cancel.recv().map(|_| Err(format_err!("canceled"))))
-        .await;
+
+    let res = async {
+        let success = imex_inner(context, what, param1).await;
+        match success {
+            Ok(()) => {
+                info!(context, "IMEX successfully completed");
+                context.emit_event(EventType::ImexProgress(1000));
+                Ok(())
+            }
+            Err(err) => {
+                cleanup_aborted_imex(context, what).await;
+                // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+                error!(context, "{:#}", err);
+                context.emit_event(EventType::ImexProgress(0));
+                bail!("IMEX FAILED to complete: {}", err);
+            }
+        }
+    }
+    .race(async {
+        cancel.recv().await.ok();
+        cleanup_aborted_imex(context, what).await;
+        Err(format_err!("canceled"))
+    })
+    .await;
 
     context.free_ongoing().await;
 
     res
 }
 
+async fn cleanup_aborted_imex(context: &Context, what: ImexMode) {
+    if what == ImexMode::ImportBackup {
+        dc_delete_file(context, context.get_dbfile()).await;
+        dc_delete_files_in_dir(context, context.get_blobdir()).await;
+    }
+    if what == ImexMode::ExportBackup || what == ImexMode::ImportBackup {
+        if let Err(e) = context.sql.open(context, context.get_dbfile(), false).await {
+            warn!(context, "Re-opening db after imex failed: {}", e);
+        }
+    }
+}
+
 /// Returns the filename of the backup found (otherwise an error)
 pub async fn has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Result<String> {
+    let dir_name = dir_name.as_ref();
+    let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
+    let mut newest_backup_name = "".to_string();
+    let mut newest_backup_path: Option<PathBuf> = None;
+
+    while let Some(dirent) = dir_iter.next().await {
+        if let Ok(dirent) = dirent {
+            let path = dirent.path();
+            let name = dirent.file_name();
+            let name: String = name.to_string_lossy().into();
+            if name.starts_with("delta-chat")
+                && name.ends_with(".tar")
+                && (newest_backup_name.is_empty() || name > newest_backup_name)
+            {
+                // We just use string comparison to determine which backup is newer.
+                // This works fine because the filenames have the form ...delta-chat-backup-2020-07-24-00.tar
+                newest_backup_path = Some(path);
+                newest_backup_name = name;
+            }
+        }
+    }
+
+    match newest_backup_path {
+        Some(path) => Ok(path.to_string_lossy().into_owned()),
+        None => has_backup_old(context, dir_name).await,
+        // When we decide to remove support for .bak backups, we can replace this with `None => bail!("no backup found in {}", dir_name.display()),`.
+    }
+}
+
+/// Returns the filename of the backup found (otherwise an error)
+pub async fn has_backup_old(context: &Context, dir_name: impl AsRef<Path>) -> Result<String> {
     let dir_name = dir_name.as_ref();
     let mut dir_iter = async_std::fs::read_dir(dir_name).await?;
     let mut newest_backup_time = 0;
@@ -95,17 +170,23 @@ pub async fn has_backup(context: &Context, dir_name: impl AsRef<Path>) -> Result
             let name = name.to_string_lossy();
             if name.starts_with("delta-chat") && name.ends_with(".bak") {
                 let sql = Sql::new();
-                if sql.open(context, &path, true).await {
-                    let curr_backup_time = sql
-                        .get_raw_config_int(context, "backup_time")
-                        .await
-                        .unwrap_or_default();
-                    if curr_backup_time > newest_backup_time {
-                        newest_backup_path = Some(path);
-                        newest_backup_time = curr_backup_time;
+                match sql.open(context, &path, true).await {
+                    Ok(_) => {
+                        let curr_backup_time = sql
+                            .get_raw_config_int(context, "backup_time")
+                            .await
+                            .unwrap_or_default();
+                        if curr_backup_time > newest_backup_time {
+                            newest_backup_path = Some(path);
+                            newest_backup_time = curr_backup_time;
+                        }
+                        info!(context, "backup_time of {} is {}", name, curr_backup_time);
+                        sql.close().await;
                     }
-                    info!(context, "backup_time of {} is {}", name, curr_backup_time);
-                    sql.close().await;
+                    Err(e) => warn!(
+                        context,
+                        "Found backup file {} which could not be opened: {}", name, e
+                    ),
                 }
             }
         }
@@ -149,10 +230,8 @@ async fn do_initiate_key_transfer(context: &Context) -> Result<String> {
     msg.param
         .set(Param::MimeType, "application/autocrypt-setup");
     msg.param.set_cmd(SystemMessage::AutocryptSetupMessage);
-    msg.param.set_int(
-        Param::ForcePlaintext,
-        ForcePlaintext::NoAutocryptHeader as i32,
-    );
+    msg.param.set_int(Param::ForcePlaintext, 1);
+    msg.param.set_int(Param::SkipAutocrypt, 1);
 
     let msg_id = chat::send_msg(context, chat_id, &mut msg).await?;
     info!(context, "Wait for setup message being sent ...",);
@@ -177,17 +256,18 @@ async fn do_initiate_key_transfer(context: &Context) -> Result<String> {
 ///
 /// The `passphrase` must be at least 2 characters long.
 pub async fn render_setup_file(context: &Context, passphrase: &str) -> Result<String> {
-    ensure!(
-        passphrase.len() >= 2,
-        "Passphrase must be at least 2 chars long."
-    );
-    let private_key = Key::from(SignedSecretKey::load_self(context).await?);
+    let passphrase_begin = if let Some(passphrase_begin) = passphrase.get(..2) {
+        passphrase_begin
+    } else {
+        bail!("Passphrase must be at least 2 chars long.");
+    };
+    let private_key = SignedSecretKey::load_self(context).await?;
     let ac_headers = match context.get_config_bool(Config::E2eeEnabled).await {
         false => None,
         true => Some(("Autocrypt-Prefer-Encrypt", "mutual")),
     };
     let private_key_asc = private_key.to_asc(ac_headers);
-    let encr = pgp::symm_encrypt(&passphrase, private_key_asc.as_bytes())?;
+    let encr = pgp::symm_encrypt(&passphrase, private_key_asc.as_bytes()).await?;
 
     let replacement = format!(
         concat!(
@@ -195,7 +275,7 @@ pub async fn render_setup_file(context: &Context, passphrase: &str) -> Result<St
             "Passphrase-Format: numeric9x4\r\n",
             "Passphrase-Begin: {}"
         ),
-        &passphrase[..2]
+        passphrase_begin
     );
     let pgp_msg = encr.replace("-----BEGIN PGP MESSAGE-----", &replacement);
 
@@ -274,7 +354,7 @@ pub async fn continue_key_transfer(
     if let Some(filename) = msg.get_file(context) {
         let file = dc_open_file_std(context, filename)?;
         let sc = normalize_setup_code(setup_code);
-        let armored_key = decrypt_setup_file(context, &sc, file)?;
+        let armored_key = decrypt_setup_file(&sc, file).await?;
         set_self_key(context, &armored_key, true, true).await?;
         maybe_add_bcc_self_device_msg(context).await?;
 
@@ -291,12 +371,8 @@ async fn set_self_key(
     prefer_encrypt_required: bool,
 ) -> Result<()> {
     // try hard to only modify key-state
-    let keys = Key::from_armored_string(armored, KeyType::Private)
-        .and_then(|(k, h)| if k.verify() { Some((k, h)) } else { None })
-        .and_then(|(k, h)| k.split_key().map(|pub_key| (k, pub_key, h)));
-
-    ensure!(keys.is_some(), "Not a valid private key");
-    let (private_key, public_key, header) = keys.unwrap();
+    let (private_key, header) = SignedSecretKey::from_asc(armored)?;
+    let public_key = private_key.split_public_key()?;
     let preferencrypt = header.get("Autocrypt-Prefer-Encrypt");
     match preferencrypt.map(|s| s.as_str()) {
         Some(headerval) => {
@@ -322,15 +398,10 @@ async fn set_self_key(
     let self_addr = context.get_config(Config::ConfiguredAddr).await;
     ensure!(self_addr.is_some(), "Missing self addr");
     let addr = EmailAddress::new(&self_addr.unwrap_or_default())?;
-
-    let (public, secret) = match (public_key, private_key) {
-        (Key::Public(p), Key::Secret(s)) => (p, s),
-        _ => bail!("wrong keys unpacked"),
-    };
     let keypair = pgp::KeyPair {
         addr,
-        public,
-        secret,
+        public: public_key,
+        secret: private_key,
     };
     key::store_self_keypair(
         context,
@@ -345,12 +416,11 @@ async fn set_self_key(
     Ok(())
 }
 
-fn decrypt_setup_file<T: std::io::Read + std::io::Seek>(
-    _context: &Context,
+async fn decrypt_setup_file<T: std::io::Read + std::io::Seek>(
     passphrase: &str,
     file: T,
 ) -> Result<String> {
-    let plain_bytes = pgp::symm_decrypt(passphrase, file)?;
+    let plain_bytes = pgp::symm_decrypt(passphrase, file).await?;
     let plain_text = std::string::String::from_utf8(plain_bytes)?;
 
     Ok(plain_text)
@@ -377,11 +447,11 @@ async fn imex_inner(
     ensure!(param.is_some(), "No Import/export dir/file given.");
 
     info!(context, "Import/export process started.");
-    context.emit_event(Event::ImexProgress(10));
+    context.emit_event(EventType::ImexProgress(10));
 
     ensure!(context.sql.is_open().await, "Database not opened.");
 
-    let path = param.unwrap();
+    let path = param.ok_or_else(|| format_err!("Imex: Param was None"))?;
     if what == ImexMode::ExportBackup || what == ImexMode::ExportSelfKeys {
         // before we export anything, make sure the private key exists
         if e2ee::ensure_secret_key_exists(context).await.is_err() {
@@ -391,28 +461,87 @@ async fn imex_inner(
         }
     }
 
-    let success = match what {
+    match what {
         ImexMode::ExportSelfKeys => export_self_keys(context, path).await,
         ImexMode::ImportSelfKeys => import_self_keys(context, path).await,
-        ImexMode::ExportBackup => export_backup(context, path).await,
-        ImexMode::ImportBackup => import_backup(context, path).await,
-    };
 
-    match success {
-        Ok(()) => {
-            info!(context, "IMEX successfully completed");
-            context.emit_event(Event::ImexProgress(1000));
-            Ok(())
-        }
-        Err(err) => {
-            context.emit_event(Event::ImexProgress(0));
-            bail!("IMEX FAILED to complete: {}", err);
-        }
+        // TODO In some months we can change the export_backup_old() call to export_backup() and delete export_backup_old().
+        // (now is 07/2020)
+        ImexMode::ExportBackup => export_backup_old(context, path).await,
+        // import_backup() will call import_backup_old() if this is an old backup.
+        ImexMode::ImportBackup => import_backup(context, path).await,
     }
 }
 
 /// Import Backup
 async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Result<()> {
+    if backup_to_import
+        .as_ref()
+        .to_string_lossy()
+        .ends_with(".bak")
+    {
+        // Backwards compability
+        return import_backup_old(context, backup_to_import).await;
+    }
+
+    info!(
+        context,
+        "Import \"{}\" to \"{}\".",
+        backup_to_import.as_ref().display(),
+        context.get_dbfile().display()
+    );
+
+    ensure!(
+        !context.is_configured().await,
+        "Cannot import backups to accounts in use."
+    );
+    context.sql.close().await;
+    dc_delete_file(context, context.get_dbfile()).await;
+    ensure!(
+        !context.get_dbfile().exists().await,
+        "Cannot delete old database."
+    );
+
+    let backup_file = File::open(backup_to_import).await?;
+    let archive = Archive::new(backup_file);
+    let mut entries = archive.entries()?;
+    while let Some(file) = entries.next().await {
+        let f = &mut file?;
+        if f.path()?.file_name() == Some(OsStr::new(DBFILE_BACKUP_NAME)) {
+            // async_tar can't unpack to a specified file name, so we just unpack to the blobdir and then move the unpacked file.
+            f.unpack_in(context.get_blobdir()).await?;
+            fs::rename(
+                context.get_blobdir().join(DBFILE_BACKUP_NAME),
+                context.get_dbfile(),
+            )
+            .await?;
+            context.emit_event(EventType::ImexProgress(400)); // Just guess the progress, we at least have the dbfile by now
+        } else {
+            // async_tar will unpack to blobdir/BLOBS_BACKUP_NAME, so we move the file afterwards.
+            f.unpack_in(context.get_blobdir()).await?;
+            let from_path = context.get_blobdir().join(f.path()?);
+            if from_path.is_file().await {
+                if let Some(name) = from_path.file_name() {
+                    fs::rename(&from_path, context.get_blobdir().join(name)).await?;
+                } else {
+                    warn!(context, "No file name");
+                }
+            }
+        }
+    }
+
+    context
+        .sql
+        .open(&context, &context.get_dbfile(), false)
+        .await
+        .context("Could not re-open db")?;
+
+    delete_and_reset_all_device_msgs(&context).await?;
+
+    Ok(())
+}
+
+async fn import_backup_old(context: &Context, backup_to_import: impl AsRef<Path>) -> Result<()> {
     info!(
         context,
         "Import \"{}\" to \"{}\".",
@@ -437,13 +566,11 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
     );
     /* error already logged */
     /* re-open copied database file */
-    ensure!(
-        context
-            .sql
-            .open(&context, &context.get_dbfile(), false)
-            .await,
-        "could not re-open db"
-    );
+    context
+        .sql
+        .open(&context, &context.get_dbfile(), false)
+        .await
+        .context("Could not re-open db")?;
 
     delete_and_reset_all_device_msgs(&context).await?;
 
@@ -457,27 +584,33 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
         "***IMPORT-in-progress: total_files_cnt={:?}", total_files_cnt,
     );
 
-    let files = context
+    // Load IDs only for now, without the file contents, to avoid
+    // consuming too much memory.
+    let file_ids = context
         .sql
         .query_map(
-            "SELECT file_name, file_content FROM backup_blobs ORDER BY id;",
+            "SELECT id FROM backup_blobs ORDER BY id",
             paramsv![],
-            |row| {
-                let name: String = row.get(0)?;
-                let blob: Vec<u8> = row.get(1)?;
-
-                Ok((name, blob))
-            },
-            |files| {
-                files
-                    .collect::<std::result::Result<Vec<_>, _>>()
+            |row| row.get(0),
+            |ids| {
+                ids.collect::<std::result::Result<Vec<i64>, _>>()
                     .map_err(Into::into)
             },
         )
         .await?;
 
     let mut all_files_extracted = true;
-    for (processed_files_cnt, (file_name, file_blob)) in files.into_iter().enumerate() {
+    for (processed_files_cnt, file_id) in file_ids.into_iter().enumerate() {
+        // Load a single blob into memory
+        let (file_name, file_blob) = context
+            .sql
+            .query_row(
+                "SELECT file_name, file_content FROM backup_blobs WHERE id = ?",
+                paramsv![file_id],
+                |row| Ok((row.get::<_, String>(0)?, row.get::<_, Vec<u8>>(1)?)),
+            )
+            .await?;
+
         if context.shall_stop_ongoing().await {
             all_files_extracted = false;
             break;
@@ -489,7 +622,7 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
         if permille > 990 {
             permille = 990
         }
-        context.emit_event(Event::ImexProgress(permille));
+        context.emit_event(EventType::ImexProgress(permille));
         if file_blob.is_empty() {
             continue;
         }
@@ -514,14 +647,90 @@ async fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) ->
 /*******************************************************************************
  * Export backup
  ******************************************************************************/
-/* the FILE_PROGRESS macro calls the callback with the permille of files processed.
-The macro avoids weird values of 0% or 100% while still working. */
+#[allow(unused)]
 async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
+    // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
+    let now = time();
+    let (temp_path, dest_path) = get_next_backup_path_new(dir, now).await?;
+    let _d = DeleteOnDrop(temp_path.clone());
+
+    context
+        .sql
+        .set_raw_config_int(context, "backup_time", now as i32)
+        .await?;
+    sql::housekeeping(context).await;
+
+    context
+        .sql
+        .execute("VACUUM;", paramsv![])
+        .await
+        .map_err(|e| warn!(context, "Vacuum failed, exporting anyway {}", e));
+
+    // we close the database during the export
+    context.sql.close().await;
+
+    info!(
+        context,
+        "Backup '{}' to '{}'.",
+        context.get_dbfile().display(),
+        dest_path.display(),
+    );
+
+    let res = export_backup_inner(context, &temp_path).await;
+
+    // we re-open the database after export is finished
+    context
+        .sql
+        .open(&context, &context.get_dbfile(), false)
+        .await;
+
+    match &res {
+        Ok(_) => {
+            fs::rename(temp_path, &dest_path).await?;
+            context.emit_event(EventType::ImexFileWritten(dest_path));
+        }
+        Err(e) => {
+            error!(context, "backup failed: {}", e);
+        }
+    }
+
+    res
+}
+struct DeleteOnDrop(PathBuf);
+impl Drop for DeleteOnDrop {
+    fn drop(&mut self) {
+        let file = self.0.clone();
+        // Not using dc_delete_file() here because it would send a DeletedBlobFile event
+        async_std::task::block_on(async move { fs::remove_file(file).await.ok() });
+    }
+}
+
+async fn export_backup_inner(context: &Context, temp_path: &PathBuf) -> Result<()> {
+    let file = File::create(temp_path).await?;
+
+    let mut builder = async_tar::Builder::new(file);
+
+    // append_path_with_name() wants the source path as the first argument, append_dir_all() wants it as the second argument.
+    builder
+        .append_path_with_name(context.get_dbfile(), DBFILE_BACKUP_NAME)
+        .await?;
+
+    context.emit_event(EventType::ImexProgress(500));
+
+    builder
+        .append_dir_all(BLOBS_BACKUP_NAME, context.get_blobdir())
+        .await?;
+
+    builder.finish().await?;
+    Ok(())
+}
+
+async fn export_backup_old(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     // get a fine backup file name (the name includes the date so that multiple backup instances are possible)
     // FIXME: we should write to a temporary file first and rename it on success. this would guarantee the backup is complete.
     // let dest_path_filename = dc_get_next_backup_file(context, dir, res);
     let now = time();
-    let dest_path_filename = dc_get_next_backup_path(dir, now).await?;
+    let dest_path_filename = get_next_backup_path_old(dir, now).await?;
     let dest_path_string = dest_path_filename.to_string_lossy().to_string();
 
     sql::housekeeping(context).await;
@@ -540,7 +749,7 @@ async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
     context
         .sql
         .open(&context, &context.get_dbfile(), false)
-        .await;
+        .await?;
 
     if !copied {
         bail!(
@@ -550,11 +759,11 @@ async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
         );
     }
     let dest_sql = Sql::new();
-    ensure!(
-        dest_sql.open(context, &dest_path_filename, false).await,
-        "could not open exported database {}",
-        dest_path_string
-    );
+    dest_sql
+        .open(context, &dest_path_filename, false)
+        .await
+        .with_context(|| format!("could not open exported database {}", dest_path_string))?;
+
     let res = match add_files_to_export(context, &dest_sql).await {
         Err(err) => {
             dc_delete_file(context, &dest_path_filename).await;
@@ -565,7 +774,7 @@ async fn export_backup(context: &Context, dir: impl AsRef<Path>) -> Result<()> {
             dest_sql
                 .set_raw_config_int(context, "backup_time", now as i32)
                 .await?;
-            context.emit_event(Event::ImexFileWritten(dest_path_filename));
+            context.emit_event(EventType::ImexFileWritten(dest_path_filename));
             Ok(())
         }
     };
@@ -604,7 +813,7 @@ async fn add_files_to_export(context: &Context, sql: &Sql) -> Result<()> {
             }
             processed_files_cnt += 1;
             let permille = max(min(processed_files_cnt * 1000 / total_files_cnt, 990), 10);
-            context.emit_event(Event::ImexProgress(permille));
+            context.emit_event(EventType::ImexProgress(permille));
 
             let name_f = entry.file_name();
             let name = name_f.to_string_lossy();
@@ -697,9 +906,9 @@ async fn export_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()
             |row| {
                 let id = row.get(0)?;
                 let public_key_blob: Vec<u8> = row.get(1)?;
-                let public_key = Key::from_slice(&public_key_blob, KeyType::Public);
+                let public_key = SignedPublicKey::from_slice(&public_key_blob);
                 let private_key_blob: Vec<u8> = row.get(2)?;
-                let private_key = Key::from_slice(&private_key_blob, KeyType::Private);
+                let private_key = SignedSecretKey::from_slice(&private_key_blob);
                 let is_default: i32 = row.get(3)?;
 
                 Ok((id, public_key, private_key, is_default))
@@ -713,7 +922,7 @@ async fn export_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()
 
     for (id, public_key, private_key, is_default) in keys {
         let id = Some(id).filter(|_| is_default != 0);
-        if let Some(key) = public_key {
+        if let Ok(key) = public_key {
             if export_key_to_asc_file(context, &dir, id, &key)
                 .await
                 .is_err()
@@ -723,7 +932,7 @@ async fn export_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()
         } else {
             export_errors += 1;
         }
-        if let Some(key) = private_key {
+        if let Ok(key) = private_key {
             if export_key_to_asc_file(context, &dir, id, &key)
                 .await
                 .is_err()
@@ -742,26 +951,36 @@ async fn export_self_keys(context: &Context, dir: impl AsRef<Path>) -> Result<()
 /*******************************************************************************
  * Classic key export
  ******************************************************************************/
-async fn export_key_to_asc_file(
+async fn export_key_to_asc_file<T>(
     context: &Context,
     dir: impl AsRef<Path>,
     id: Option<i64>,
-    key: &Key,
-) -> std::io::Result<()> {
+    key: &T,
+) -> std::io::Result<()>
+where
+    T: DcKey + Any,
+{
     let file_name = {
-        let kind = if key.is_public() { "public" } else { "private" };
+        let any_key = key as &dyn Any;
+        let kind = if any_key.downcast_ref::<SignedPublicKey>().is_some() {
+            "public"
+        } else if any_key.downcast_ref::<SignedPublicKey>().is_some() {
+            "private"
+        } else {
+            "unknown"
+        };
         let id = id.map_or("default".into(), |i| i.to_string());
-
         dir.as_ref().join(format!("{}-key-{}.asc", kind, &id))
     };
     info!(context, "Exporting key {}", file_name.display());
     dc_delete_file(context, &file_name).await;
 
-    let res = key.write_asc_to_file(&file_name, context).await;
+    let content = key.to_asc(None).into_bytes();
+    let res = dc_write_file(context, &file_name, &content).await;
     if res.is_err() {
         error!(context, "Cannot write key to {}", file_name.display());
     } else {
-        context.emit_event(Event::ImexFileWritten(file_name));
+        context.emit_event(EventType::ImexFileWritten(file_name));
     }
     res
 }
@@ -775,9 +994,9 @@ mod tests {
 
     #[async_std::test]
     async fn test_render_setup_file() {
-        let t = test_context().await;
+        let t = TestContext::new().await;
 
-        configure_alice_keypair(&t.ctx).await;
+        t.configure_alice().await;
         let msg = render_setup_file(&t.ctx, "hello").await.unwrap();
         println!("{}", &msg);
         // Check some substrings, indicating things got substituted.
@@ -794,12 +1013,12 @@ mod tests {
 
     #[async_std::test]
     async fn test_render_setup_file_newline_replace() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx
             .set_stock_translation(StockMessage::AcSetupMsgBody, "hello\r\nthere".to_string())
             .await
             .unwrap();
-        configure_alice_keypair(&t.ctx).await;
+        t.configure_alice().await;
         let msg = render_setup_file(&t.ctx, "pw").await.unwrap();
         println!("{}", &msg);
         assert!(msg.contains("<p>hello<br>there</p>"));
@@ -807,7 +1026,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_create_setup_code() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let setupcode = create_setup_code(&t.ctx);
         assert_eq!(setupcode.len(), 44);
         assert_eq!(setupcode.chars().nth(4).unwrap(), '-');
@@ -822,8 +1041,8 @@ mod tests {
 
     #[async_std::test]
     async fn test_export_key_to_asc_file() {
-        let context = dummy_context().await;
-        let key = Key::from(alice_keypair().public);
+        let context = TestContext::new().await;
+        let key = alice_keypair().public;
         let blobdir = "$BLOBDIR";
         assert!(export_key_to_asc_file(&context.ctx, blobdir, None, &key)
             .await
@@ -835,6 +1054,21 @@ mod tests {
         assert_eq!(bytes, key.to_asc(None).into_bytes());
     }
 
+    #[async_std::test]
+    async fn test_export_and_import_key() {
+        let context = TestContext::new().await;
+        context.configure_alice().await;
+        let blobdir = "$BLOBDIR";
+        assert!(imex(&context.ctx, ImexMode::ExportSelfKeys, Some(blobdir))
+            .await
+            .is_ok());
+
+        let blobdir = context.ctx.get_blobdir().to_str().unwrap();
+        assert!(imex(&context.ctx, ImexMode::ImportSelfKeys, Some(blobdir))
+            .await
+            .is_ok());
+    }
+
     #[test]
     fn test_normalize_setup_code() {
         let norm = normalize_setup_code("123422343234423452346234723482349234");
@@ -852,9 +1086,6 @@ mod tests {
 
     #[async_std::test]
     async fn test_split_and_decrypt() {
-        let ctx = dummy_context().await;
-        let context = &ctx.ctx;
-
         let buf_1 = S_EM_SETUPFILE.as_bytes().to_vec();
         let (typ, headers, base64) = split_armored_data(&buf_1).unwrap();
         assert_eq!(typ, BlockType::Message);
@@ -864,12 +1095,10 @@ mod tests {
         assert!(!base64.is_empty());
 
         let setup_file = S_EM_SETUPFILE.to_string();
-        let decrypted = decrypt_setup_file(
-            context,
-            S_EM_SETUPCODE,
-            std::io::Cursor::new(setup_file.as_bytes()),
-        )
-        .unwrap();
+        let decrypted =
+            decrypt_setup_file(S_EM_SETUPCODE, std::io::Cursor::new(setup_file.as_bytes()))
+                .await
+                .unwrap();
 
         let (typ, headers, _base64) = split_armored_data(decrypted.as_bytes()).unwrap();
 

src/job.rs

@@ -14,24 +14,23 @@ use async_smtp::smtp::response::Category;
 use async_smtp::smtp::response::Code;
 use async_smtp::smtp::response::Detail;
 
-use crate::blob::BlobObject;
 use crate::chat::{self, ChatId};
 use crate::config::Config;
-use crate::constants::*;
 use crate::contact::Contact;
 use crate::context::Context;
 use crate::dc_tools::*;
+use crate::ephemeral::load_imap_deletion_msgid;
 use crate::error::{bail, ensure, format_err, Error, Result};
-use crate::events::Event;
+use crate::events::EventType;
 use crate::imap::*;
 use crate::location;
-use crate::login_param::LoginParam;
 use crate::message::MsgId;
 use crate::message::{self, Message, MessageState};
 use crate::mimefactory::MimeFactory;
 use crate::param::*;
 use crate::smtp::Smtp;
-use crate::sql;
+use crate::{blob::BlobObject, contact::normalize_name, contact::Modifier, contact::Origin};
+use crate::{scheduler::InterruptInfo, sql};
 
 // results in ~3 weeks for the last backoff timespan
 const JOB_RETRIES: u32 = 17;
@@ -93,8 +92,7 @@ pub enum Action {
 
     // Jobs in the INBOX-thread, range from DC_IMAP_THREAD..DC_IMAP_THREAD+999
     Housekeeping = 105, // low priority ...
-    EmptyServer = 107,
-    OldDeleteMsgOnImap = 110,
+    FetchExistingMsgs = 110,
     MarkseenMsgOnImap = 130,
 
     // Moving message is prioritized lower than deletion so we don't
@@ -102,6 +100,10 @@ pub enum Action {
     MoveMsg = 200,
     DeleteMsgOnImap = 210,
 
+    // UID synchronization is high-priority to make sure correct UIDs
+    // are used by message moving/deletion.
+    ResyncFolders = 300,
+
     // Jobs in the SMTP-thread, range from DC_SMTP_THREAD..DC_SMTP_THREAD+999
     MaybeSendLocations = 5005, // low priority ...
     MaybeSendLocationsEnded = 5007,
@@ -123,9 +125,9 @@ impl From<Action> for Thread {
             Unknown => Thread::Unknown,
 
             Housekeeping => Thread::Imap,
-            OldDeleteMsgOnImap => Thread::Imap,
+            FetchExistingMsgs => Thread::Imap,
             DeleteMsgOnImap => Thread::Imap,
-            EmptyServer => Thread::Imap,
+            ResyncFolders => Thread::Imap,
             MarkseenMsgOnImap => Thread::Imap,
             MoveMsg => Thread::Imap,
 
@@ -190,7 +192,7 @@ impl Job {
     /// Saves the job to the database, creating a new entry if necessary.
     ///
     /// The Job is consumed by this method.
-    pub async fn save(self, context: &Context) -> Result<()> {
+    pub(crate) async fn save(self, context: &Context) -> Result<()> {
         let thread: Thread = self.action.into();
 
         info!(context, "saving job for {}-thread: {:?}", thread, self);
@@ -254,40 +256,48 @@ impl Job {
 
                 let res = match err {
                     async_smtp::smtp::error::Error::Permanent(ref response) => {
-                        match response.code {
+                        // Workaround for incorrectly configured servers returning permanent errors
+                        // instead of temporary ones.
+                        let maybe_transient = match response.code {
                             // Sometimes servers send a permanent error when actually it is a temporary error
                             // For documentation see https://tools.ietf.org/html/rfc3463
-
-                            // Code 5.5.0, see https://support.delta.chat/t/every-other-message-gets-stuck/877/2
                             Code {
                                 category: Category::MailSystem,
                                 detail: Detail::Zero,
                                 ..
-                            } => Status::RetryLater,
-
-                            _ => {
-                                // If we do not retry, add an info message to the chat
-                                // Error 5.7.1 should definitely go here: Yandex sends 5.7.1 with a link when it thinks that the email is SPAM.
-                                match Message::load_from_db(context, MsgId::new(self.foreign_id))
-                                    .await
-                                {
-                                    Ok(message) => {
-                                        chat::add_info_msg(
-                                            context,
-                                            message.chat_id,
-                                            err.to_string(),
-                                        )
-                                        .await
-                                    }
-                                    Err(e) => warn!(
-                                        context,
-                                        "couldn't load chat_id to inform user about SMTP error: {}",
-                                        e
-                                    ),
-                                };
-
-                                Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
+                            } => {
+                                // Ignore status code 5.5.0, see https://support.delta.chat/t/every-other-message-gets-stuck/877/2
+                                // Maybe incorrectly configured Postfix milter with "reject" instead of "tempfail", which returns
+                                // "550 5.5.0 Service unavailable" instead of "451 4.7.1 Service unavailable - try again later".
+                                //
+                                // Other enhanced status codes, such as Postfix
+                                // "550 5.1.1 <foobar@example.org>: Recipient address rejected: User unknown in local recipient table"
+                                // are not ignored.
+                                response.message.get(0) == Some(&"5.5.0".to_string())
                             }
+                            _ => false,
+                        };
+
+                        if maybe_transient {
+                            Status::RetryLater
+                        } else {
+                            // If we do not retry, add an info message to the chat.
+                            // Yandex error "554 5.7.1 [2] Message rejected under suspicion of SPAM; https://ya.cc/..."
+                            // should definitely go here, because user has to open the link to
+                            // resume message sending.
+                            let msg_id = MsgId::new(self.foreign_id);
+                            message::set_msg_failed(context, msg_id, Some(err.to_string())).await;
+                            match Message::load_from_db(context, msg_id).await {
+                                Ok(message) => {
+                                    chat::add_info_msg(context, message.chat_id, err.to_string())
+                                        .await
+                                }
+                                Err(e) => error!(
+                                    context,
+                                    "couldn't load chat_id to inform user about SMTP error: {}", e
+                                ),
+                            };
+                            Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
                         }
                     }
                     async_smtp::smtp::error::Error::Transient(_) => {
@@ -329,14 +339,11 @@ impl Job {
         }
     }
 
-    pub async fn send_msg_to_smtp(&mut self, context: &Context, smtp: &mut Smtp) -> Status {
+    pub(crate) async fn send_msg_to_smtp(&mut self, context: &Context, smtp: &mut Smtp) -> Status {
         //  SMTP server, if not yet done
-        if !smtp.is_connected().await {
-            let loginparam = LoginParam::from_database(context, "configured_").await;
-            if let Err(err) = smtp.connect(context, &loginparam).await {
-                warn!(context, "SMTP connection failure: {:?}", err);
-                return Status::RetryLater;
-            }
+        if let Err(err) = smtp.connect_configured(context).await {
+            warn!(context, "SMTP connection failure: {:?}", err);
+            return Status::RetryLater;
         }
 
         let filename = job_try!(job_try!(self
@@ -479,12 +486,9 @@ impl Job {
         let recipients = vec![recipient];
 
         // connect to SMTP server, if not yet done
-        if !smtp.is_connected().await {
-            let loginparam = LoginParam::from_database(context, "configured_").await;
-            if let Err(err) = smtp.connect(context, &loginparam).await {
-                warn!(context, "SMTP connection failure: {:?}", err);
-                return Status::RetryLater;
-            }
+        if let Err(err) = smtp.connect_configured(context).await {
+            warn!(context, "SMTP connection failure: {:?}", err);
+            return Status::RetryLater;
         }
 
         self.smtp_send(context, recipients, body, self.job_id, smtp, || {
@@ -498,16 +502,13 @@ impl Job {
     }
 
     async fn move_msg(&mut self, context: &Context, imap: &mut Imap) -> Status {
-        let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
-
-        if let Err(err) = imap.ensure_configured_folders(context, true).await {
-            warn!(context, "could not configure folders: {:?}", err);
+        if let Err(err) = imap.connect_configured(context).await {
+            warn!(context, "could not connect: {:?}", err);
             return Status::RetryLater;
         }
-        let dest_folder = context
-            .sql
-            .get_raw_config(context, "configured_mvbox_folder")
-            .await;
+
+        let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
+        let dest_folder = context.get_config(Config::ConfiguredMvboxFolder).await;
 
         if let Some(dest_folder) = dest_folder {
             let server_folder = msg.server_folder.as_ref().unwrap();
@@ -518,7 +519,7 @@ impl Job {
             {
                 ImapActionResult::RetryLater => Status::RetryLater,
                 ImapActionResult::Success => {
-                    // XXX Rust-Imap provides no target uid on mv, so just set it to 0
+                    // Rust-Imap provides no target uid on mv, so just set it to 0, update again when precheck_imf() is called for the moved message
                     message::update_server_uid(context, &msg.rfc724_mid, &dest_folder, 0).await;
                     Status::Finished(Ok(()))
                 }
@@ -541,6 +542,11 @@ impl Job {
     /// records pointing to the same message on the server, the job
     /// also removes the message on the server.
     async fn delete_msg_on_imap(&mut self, context: &Context, imap: &mut Imap) -> Status {
+        if let Err(err) = imap.connect_configured(context).await {
+            warn!(context, "could not connect: {:?}", err);
+            return Status::RetryLater;
+        }
+
         let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
 
         if !msg.rfc724_mid.is_empty() {
@@ -556,6 +562,11 @@ impl Job {
                     context,
                     "The message is deleted from the server when all parts are deleted.",
                 );
+            } else if cnt == 0 {
+                warn!(
+                    context,
+                    "The message {} has no UID on the server to delete", &msg.rfc724_mid
+                );
             } else {
                 /* if this is the last existing part of the message,
                 we delete the message from the server */
@@ -610,27 +621,105 @@ impl Job {
         }
     }
 
-    async fn empty_server(&mut self, context: &Context, imap: &mut Imap) -> Status {
-        if self.foreign_id & DC_EMPTY_MVBOX > 0 {
-            if let Some(mvbox_folder) = context
-                .sql
-                .get_raw_config(context, "configured_mvbox_folder")
-                .await
-            {
-                imap.empty_folder(context, &mvbox_folder).await;
+    /// Read the recipients from old emails sent by the user and add them as contacts.
+    /// This way, we can already offer them some email addresses they can write to.
+    ///
+    /// Then, Fetch the last messages DC_FETCH_EXISTING_MSGS_COUNT emails from the server
+    /// and show them in the chat list.
+    async fn fetch_existing_msgs(&mut self, context: &Context, imap: &mut Imap) -> Status {
+        if context.get_config_bool(Config::Bot).await {
+            return Status::Finished(Ok(())); // Bots don't want those messages
+        }
+        if let Err(err) = imap.connect_configured(context).await {
+            warn!(context, "could not connect: {:?}", err);
+            return Status::RetryLater;
+        }
+
+        add_all_recipients_as_contacts(context, imap, Config::ConfiguredSentboxFolder).await;
+        add_all_recipients_as_contacts(context, imap, Config::ConfiguredMvboxFolder).await;
+        add_all_recipients_as_contacts(context, imap, Config::ConfiguredInboxFolder).await;
+
+        if context.get_config_bool(Config::FetchExisting).await {
+            for config in &[
+                Config::ConfiguredMvboxFolder,
+                Config::ConfiguredInboxFolder,
+                Config::ConfiguredSentboxFolder,
+            ] {
+                if let Some(folder) = context.get_config(*config).await {
+                    if let Err(e) = imap.fetch_new_messages(context, folder, true).await {
+                        // We are using Anyhow's .context() and to show the inner error, too, we need the {:#}:
+                        warn!(context, "Could not fetch messages, retrying: {:#}", e);
+                        return Status::RetryLater;
+                    };
+                }
             }
         }
-        if self.foreign_id & DC_EMPTY_INBOX > 0 {
-            imap.empty_folder(context, "INBOX").await;
+        info!(context, "Done fetching existing messages.");
+        Status::Finished(Ok(()))
+    }
+
+    /// Synchronizes UIDs for sentbox, inbox and mvbox, in this order.
+    ///
+    /// If a copy of the message is present in multiple folders, mvbox
+    /// is preferred to inbox, which is in turn preferred to
+    /// sentbox. This is because in the database it is impossible to
+    /// store multiple UIDs for one message, so we prefer to
+    /// automatically delete messages in the folders managed by Delta
+    /// Chat in contrast to the Sent folder, which is normally managed
+    /// by the user via webmail or another email client.
+    async fn resync_folders(&mut self, context: &Context, imap: &mut Imap) -> Status {
+        if let Err(err) = imap.connect_configured(context).await {
+            warn!(context, "could not connect: {:?}", err);
+            return Status::RetryLater;
+        }
+
+        if let Some(sentbox_folder) = &context.get_config(Config::ConfiguredSentboxFolder).await {
+            job_try!(
+                imap.resync_folder_uids(context, sentbox_folder.to_string())
+                    .await
+            );
+        }
+
+        if let Some(inbox_folder) = &context.get_config(Config::ConfiguredInboxFolder).await {
+            job_try!(
+                imap.resync_folder_uids(context, inbox_folder.to_string())
+                    .await
+            );
+        }
+
+        if let Some(mvbox_folder) = &context.get_config(Config::ConfiguredMvboxFolder).await {
+            job_try!(
+                imap.resync_folder_uids(context, mvbox_folder.to_string())
+                    .await
+            );
         }
         Status::Finished(Ok(()))
     }
 
     async fn markseen_msg_on_imap(&mut self, context: &Context, imap: &mut Imap) -> Status {
+        if let Err(err) = imap.connect_configured(context).await {
+            warn!(context, "could not connect: {:?}", err);
+            return Status::RetryLater;
+        }
+
         let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
 
         let folder = msg.server_folder.as_ref().unwrap();
-        match imap.set_seen(context, folder, msg.server_uid).await {
+
+        let result = if msg.server_uid == 0 {
+            // The message is moved or deleted by us.
+            //
+            // Do not call set_seen with zero UID, as it will return
+            // ImapActionResult::RetryLater, but we do not want to
+            // retry. If the message was moved, we will create another
+            // job to mark the message as seen later. If it was
+            // deleted, there is nothing to do.
+            ImapActionResult::Failed
+        } else {
+            imap.set_seen(context, folder, msg.server_uid).await
+        };
+
+        match result {
             ImapActionResult::RetryLater => Status::RetryLater,
             ImapActionResult::AlreadyDone => Status::Finished(Ok(())),
             ImapActionResult::Success | ImapActionResult::Failed => {
@@ -638,7 +727,18 @@ impl Job {
                 // we want to send out an MDN anyway
                 // The job will not be retried so locally
                 // there is no risk of double-sending MDNs.
+                //
+                // Read receipts for system messages are never
+                // sent. These messages have no place to display
+                // received read receipt anyway.  And since their text
+                // is locally generated, quoting them is dangerous as
+                // it may contain contact names. E.g., for original
+                // message "Group left by me", a read receipt will
+                // quote "Group left by <name>", and the name can be a
+                // display name stored in address book rather than
+                // the name sent in the From field by the user.
                 if msg.param.get_bool(Param::WantsMdn).unwrap_or_default()
+                    && !msg.is_system_message()
                     && context.get_config_bool(Config::MdnsEnabled).await
                 {
                     if let Err(err) = send_mdn(context, &msg).await {
@@ -695,7 +795,51 @@ async fn set_delivered(context: &Context, msg_id: MsgId) {
         )
         .await
         .unwrap_or_default();
-    context.emit_event(Event::MsgDelivered { chat_id, msg_id });
+    context.emit_event(EventType::MsgDelivered { chat_id, msg_id });
+}
+
+async fn add_all_recipients_as_contacts(context: &Context, imap: &mut Imap, folder: Config) {
+    let mailbox = if let Some(m) = context.get_config(folder).await {
+        m
+    } else {
+        return;
+    };
+    if let Err(e) = imap.select_with_uidvalidity(context, &mailbox).await {
+        warn!(context, "Could not select {}: {}", mailbox, e);
+        return;
+    }
+    match imap.get_all_recipients(context).await {
+        Ok(contacts) => {
+            let mut any_modified = false;
+            for contact in contacts {
+                let display_name_normalized = contact
+                    .display_name
+                    .as_ref()
+                    .map(normalize_name)
+                    .unwrap_or_default();
+
+                match Contact::add_or_lookup(
+                    context,
+                    display_name_normalized,
+                    contact.addr,
+                    Origin::OutgoingTo,
+                )
+                .await
+                {
+                    Ok((_, modified)) => {
+                        if modified != Modifier::None {
+                            any_modified = true;
+                        }
+                    }
+                    Err(e) => warn!(context, "Could not add recipient: {}", e),
+                }
+            }
+            if any_modified {
+                context.emit_event(EventType::ContactsChanged(None));
+            }
+        }
+        Err(e) => warn!(context, "Could not add recipients: {}", e),
+    };
 }
 
 /// Constructs a job for sending a message.
@@ -798,7 +942,7 @@ pub async fn send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<Job
 
     if rendered_msg.is_encrypted && !needs_encryption {
         msg.param.set_int(Param::GuaranteeE2ee, 1);
-        msg.save_param_to_disk(context).await;
+        msg.update_param(context).await;
     }
 
     ensure!(!recipients.is_empty(), "no recipients for smtp job set");
@@ -815,31 +959,11 @@ pub async fn send_msg_job(context: &Context, msg_id: MsgId) -> Result<Option<Job
     Ok(Some(job))
 }
 
-#[derive(Debug)]
-pub enum Connection<'a> {
+pub(crate) enum Connection<'a> {
     Inbox(&'a mut Imap),
     Smtp(&'a mut Smtp),
 }
 
-async fn load_imap_deletion_msgid(context: &Context) -> sql::Result<Option<MsgId>> {
-    if let Some(delete_server_after) = context.get_config_delete_server_after().await {
-        let threshold_timestamp = time() - delete_server_after;
-
-        context
-            .sql
-            .query_row_optional(
-                "SELECT id FROM msgs \
-             WHERE timestamp < ? \
-             AND server_uid != 0",
-                paramsv![threshold_timestamp],
-                |row| row.get::<_, MsgId>(0),
-            )
-            .await
-    } else {
-        Ok(None)
-    }
-}
-
 async fn load_imap_deletion_job(context: &Context) -> sql::Result<Option<Job>> {
     let res = if let Some(msg_id) = load_imap_deletion_msgid(context).await? {
         Some(Job::new(
@@ -962,21 +1086,18 @@ async fn perform_job_action(
         Action::MaybeSendLocationsEnded => {
             location::job_maybe_send_locations_ended(context, job).await
         }
-        Action::EmptyServer => job.empty_server(context, connection.inbox()).await,
-        Action::OldDeleteMsgOnImap => job.delete_msg_on_imap(context, connection.inbox()).await,
         Action::DeleteMsgOnImap => job.delete_msg_on_imap(context, connection.inbox()).await,
+        Action::ResyncFolders => job.resync_folders(context, connection.inbox()).await,
         Action::MarkseenMsgOnImap => job.markseen_msg_on_imap(context, connection.inbox()).await,
         Action::MoveMsg => job.move_msg(context, connection.inbox()).await,
+        Action::FetchExistingMsgs => job.fetch_existing_msgs(context, connection.inbox()).await,
         Action::Housekeeping => {
             sql::housekeeping(context).await;
             Status::Finished(Ok(()))
         }
     };
 
-    info!(
-        context,
-        "Inbox finished immediate try {} of job {}", tries, job
-    );
+    info!(context, "Finished immediate try {} of job {}", tries, job);
 
     try_res
 }
@@ -1001,6 +1122,15 @@ async fn send_mdn(context: &Context, msg: &Message) -> Result<()> {
     Ok(())
 }
 
+pub(crate) async fn schedule_resync(context: &Context) {
+    kill_action(context, Action::ResyncFolders).await;
+    add(
+        context,
+        Job::new(Action::ResyncFolders, 0, Params::new(), 0),
+    )
+    .await;
+}
+
 /// Creates a job.
 pub fn create(action: Action, foreign_id: i32, param: Params, delay_seconds: i64) -> Result<Job> {
     ensure!(
@@ -1023,18 +1153,24 @@ pub async fn add(context: &Context, job: Job) {
         match action {
             Action::Unknown => unreachable!(),
             Action::Housekeeping
-            | Action::EmptyServer
-            | Action::OldDeleteMsgOnImap
             | Action::DeleteMsgOnImap
+            | Action::ResyncFolders
             | Action::MarkseenMsgOnImap
+            | Action::FetchExistingMsgs
             | Action::MoveMsg => {
-                context.interrupt_inbox().await;
+                info!(context, "interrupt: imap");
+                context
+                    .interrupt_inbox(InterruptInfo::new(false, None))
+                    .await;
             }
             Action::MaybeSendLocations
             | Action::MaybeSendLocationsEnded
             | Action::SendMdn
             | Action::SendMsgToSmtp => {
-                context.interrupt_smtp().await;
+                info!(context, "interrupt: smtp");
+                context
+                    .interrupt_smtp(InterruptInfo::new(false, None))
+                    .await;
             }
         }
     }
@@ -1049,38 +1185,49 @@ pub async fn add(context: &Context, job: Job) {
 pub(crate) async fn load_next(
     context: &Context,
     thread: Thread,
-    probe_network: bool,
+    info: &InterruptInfo,
 ) -> Option<Job> {
     info!(context, "loading job for {}-thread", thread);
-    let query = if !probe_network {
+
+    let query;
+    let params;
+    let t = time();
+    let m;
+    let thread_i = thread as i64;
+
+    if let Some(msg_id) = info.msg_id {
+        query = r#"
+SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries
+FROM jobs
+WHERE thread=? AND foreign_id=?
+ORDER BY action DESC, added_timestamp
+LIMIT 1;
+"#;
+        m = msg_id;
+        params = paramsv![thread_i, m];
+    } else if !info.probe_network {
         // processing for first-try and after backoff-timeouts:
         // process jobs in the order they were added.
-        r#"
+        query = r#"
 SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries
 FROM jobs
 WHERE thread=? AND desired_timestamp<=?
 ORDER BY action DESC, added_timestamp
 LIMIT 1;
-"#
+"#;
+        params = paramsv![thread_i, t];
     } else {
         // processing after call to dc_maybe_network():
         // process _all_ pending jobs that failed before
         // in the order of their backoff-times.
-        r#"
+        query = r#"
 SELECT id, action, foreign_id, param, added_timestamp, desired_timestamp, tries
 FROM jobs
 WHERE thread=? AND tries>0
 ORDER BY desired_timestamp, action DESC
 LIMIT 1;
-"#
-    };
-
-    let thread_i = thread as i64;
-    let t = time();
-    let params = if !probe_network {
-        paramsv![thread_i, t]
-    } else {
-        paramsv![thread_i]
+"#;
+        params = paramsv![thread_i];
     };
 
     let job = loop {
@@ -1088,13 +1235,13 @@ LIMIT 1;
             .sql
             .query_row_optional(query, params.clone(), |row| {
                 let job = Job {
-                    job_id: row.get(0)?,
-                    action: row.get(1)?,
-                    foreign_id: row.get(2)?,
-                    desired_timestamp: row.get(5)?,
-                    added_timestamp: row.get(4)?,
-                    tries: row.get(6)?,
-                    param: row.get::<_, String>(3)?.parse().unwrap_or_default(),
+                    job_id: row.get("id")?,
+                    action: row.get("action")?,
+                    foreign_id: row.get("foreign_id")?,
+                    desired_timestamp: row.get("desired_timestamp")?,
+                    added_timestamp: row.get("added_timestamp")?,
+                    tries: row.get("tries")?,
+                    param: row.get::<_, String>("param")?.parse().unwrap_or_default(),
                     pending_error: None,
                 };
 
@@ -1104,8 +1251,9 @@ LIMIT 1;
 
         match job_res {
             Ok(job) => break job,
-            Err(_) => {
+            Err(err) => {
                 // Remove invalid job from the DB
+                info!(context, "cleaning up job, because of {}", err);
 
                 // TODO: improve by only doing a single query
                 match context
@@ -1116,7 +1264,7 @@ LIMIT 1;
                     Ok(id) => {
                         context
                             .sql
-                            .execute("DELETE FROM jobs WHERE id=?", paramsv![id])
+                            .execute("DELETE FROM jobs WHERE id=?;", paramsv![id])
                             .await
                             .ok();
                     }
@@ -1129,21 +1277,26 @@ LIMIT 1;
         }
     };
 
-    if thread == Thread::Imap {
-        if let Some(job) = job {
-            if job.action < Action::DeleteMsgOnImap {
-                load_imap_deletion_job(context)
-                    .await
-                    .unwrap_or_default()
-                    .or(Some(job))
-            } else {
-                Some(job)
-            }
-        } else {
-            load_imap_deletion_job(context).await.unwrap_or_default()
+    match thread {
+        Thread::Unknown => {
+            error!(context, "unknown thread for job");
+            None
         }
-    } else {
-        job
+        Thread::Imap => {
+            if let Some(job) = job {
+                if job.action < Action::DeleteMsgOnImap {
+                    load_imap_deletion_job(context)
+                        .await
+                        .unwrap_or_default()
+                        .or(Some(job))
+                } else {
+                    Some(job)
+                }
+            } else {
+                load_imap_deletion_job(context).await.unwrap_or_default()
+            }
+        }
+        Thread::Smtp => job,
     }
 }
 
@@ -1175,17 +1328,42 @@ mod tests {
     }
 
     #[async_std::test]
-    async fn test_load_next_job() {
+    async fn test_load_next_job_two() {
         // We want to ensure that loading jobs skips over jobs which
         // fails to load from the database instead of failing to load
         // all jobs.
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         insert_job(&t.ctx, -1).await; // This can not be loaded into Job struct.
-        let jobs = load_next(&t.ctx, Thread::from(Action::MoveMsg), false).await;
+        let jobs = load_next(
+            &t.ctx,
+            Thread::from(Action::MoveMsg),
+            &InterruptInfo::new(false, None),
+        )
+        .await;
         assert!(jobs.is_none());
 
         insert_job(&t.ctx, 1).await;
-        let jobs = load_next(&t.ctx, Thread::from(Action::MoveMsg), false).await;
+        let jobs = load_next(
+            &t.ctx,
+            Thread::from(Action::MoveMsg),
+            &InterruptInfo::new(false, None),
+        )
+        .await;
+        assert!(jobs.is_some());
+    }
+
+    #[async_std::test]
+    async fn test_load_next_job_one() {
+        let t = TestContext::new().await;
+
+        insert_job(&t.ctx, 1).await;
+
+        let jobs = load_next(
+            &t.ctx,
+            Thread::from(Action::MoveMsg),
+            &InterruptInfo::new(false, None),
+        )
+        .await;
         assert!(jobs.is_some());
     }
 }

src/key.rs

@@ -1,19 +1,20 @@
 //! Cryptographic key module
 
 use std::collections::BTreeMap;
+use std::fmt;
 use std::io::Cursor;
 
-use async_std::path::Path;
 use async_trait::async_trait;
 use num_traits::FromPrimitive;
 use pgp::composed::Deserializable;
 use pgp::ser::Serialize;
 use pgp::types::{KeyTrait, SecretKeyTrait};
+use thiserror::Error;
 
 use crate::config::Config;
 use crate::constants::*;
 use crate::context::Context;
-use crate::dc_tools::{dc_write_file, time, EmailAddress, InvalidEmailError};
+use crate::dc_tools::{time, EmailAddress, InvalidEmailError};
 use crate::sql;
 
 // Re-export key types
@@ -38,6 +39,8 @@ pub enum Error {
     NoConfiguredAddr,
     #[error("Configured address is invalid: {}", _0)]
     InvalidConfiguredAddr(#[from] InvalidEmailError),
+    #[error("no data provided")]
+    Empty,
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -48,8 +51,8 @@ pub type Result<T> = std::result::Result<T, Error>;
 /// [SignedSecretKey] types and makes working with them a little
 /// easier in the deltachat world.
 #[async_trait]
-pub trait DcKey: Serialize + Deserializable {
-    type KeyType: Serialize + Deserializable;
+pub trait DcKey: Serialize + Deserializable + KeyTrait + Clone {
+    type KeyType: Serialize + Deserializable + KeyTrait + Clone;
 
     /// Create a key from some bytes.
     fn from_slice(bytes: &[u8]) -> Result<Self::KeyType> {
@@ -66,18 +69,45 @@ pub trait DcKey: Serialize + Deserializable {
         Self::from_slice(&bytes)
     }
 
+    /// Create a key from an ASCII-armored string.
+    ///
+    /// Returns the key and a map of any headers which might have been set in
+    /// the ASCII-armored representation.
+    fn from_asc(data: &str) -> Result<(Self::KeyType, BTreeMap<String, String>)> {
+        let bytes = data.as_bytes();
+        Self::KeyType::from_armor_single(Cursor::new(bytes)).map_err(Error::Pgp)
+    }
+
     /// Load the users' default key from the database.
     async fn load_self(context: &Context) -> Result<Self::KeyType>;
 
-    /// Serialise the key to a base64 string.
-    fn to_base64(&self) -> String {
+    /// Serialise the key as bytes.
+    fn to_bytes(&self) -> Vec<u8> {
         // Not using Serialize::to_bytes() to make clear *why* it is
         // safe to ignore this error.
         // Because we write to a Vec<u8> the io::Write impls never
         // fail and we can hide this error.
         let mut buf = Vec::new();
         self.to_writer(&mut buf).unwrap();
-        base64::encode(&buf)
+        buf
+    }
+
+    /// Serialise the key to a base64 string.
+    fn to_base64(&self) -> String {
+        base64::encode(&DcKey::to_bytes(self))
+    }
+
+    /// Serialise the key to ASCII-armored representation.
+    ///
+    /// Each header line must be terminated by `\r\n`.  Only allows setting one
+    /// header as a simplification since that's the only way it's used so far.
+    // Since .to_armored_string() are actual methods on SignedPublicKey and
+    // SignedSecretKey we can not generically implement this.
+    fn to_asc(&self, header: Option<(&str, &str)>) -> String;
+
+    /// The fingerprint for the key.
+    fn fingerprint(&self) -> Fingerprint {
+        Fingerprint::new(KeyTrait::fingerprint(self)).expect("Invalid fingerprint from rpgp")
     }
 }
 
@@ -108,6 +138,22 @@ impl DcKey for SignedPublicKey {
             Err(err) => Err(err.into()),
         }
     }
+
+    fn to_asc(&self, header: Option<(&str, &str)>) -> String {
+        // Not using .to_armored_string() to make clear *why* it is
+        // safe to ignore this error.
+        // Because we write to a Vec<u8> the io::Write impls never
+        // fail and we can hide this error.
+        let headers = header.map(|(key, value)| {
+            let mut m = BTreeMap::new();
+            m.insert(key.to_string(), value.to_string());
+            m
+        });
+        let mut buf = Vec::new();
+        self.to_armored_writer(&mut buf, headers.as_ref())
+            .unwrap_or_default();
+        std::string::String::from_utf8(buf).unwrap_or_default()
+    }
 }
 
 #[async_trait]
@@ -137,13 +183,46 @@ impl DcKey for SignedSecretKey {
             Err(err) => Err(err.into()),
         }
     }
+
+    fn to_asc(&self, header: Option<(&str, &str)>) -> String {
+        // Not using .to_armored_string() to make clear *why* it is
+        // safe to do these unwraps.
+        // Because we write to a Vec<u8> the io::Write impls never
+        // fail and we can hide this error.  The string is always ASCII.
+        let headers = header.map(|(key, value)| {
+            let mut m = BTreeMap::new();
+            m.insert(key.to_string(), value.to_string());
+            m
+        });
+        let mut buf = Vec::new();
+        self.to_armored_writer(&mut buf, headers.as_ref())
+            .unwrap_or_default();
+        std::string::String::from_utf8(buf).unwrap_or_default()
+    }
+}
+
+/// Deltachat extension trait for secret keys.
+///
+/// Provides some convenience wrappers only applicable to [SignedSecretKey].
+pub trait DcSecretKey {
+    /// Create a public key from a private one.
+    fn split_public_key(&self) -> Result<SignedPublicKey>;
+}
+
+impl DcSecretKey for SignedSecretKey {
+    fn split_public_key(&self) -> Result<SignedPublicKey> {
+        self.verify()?;
+        let unsigned_pubkey = SecretKeyTrait::public_key(self);
+        let signed_pubkey = unsigned_pubkey.sign(self, || "".into())?;
+        Ok(signed_pubkey)
+    }
 }
 
 async fn generate_keypair(context: &Context) -> Result<KeyPair> {
     let addr = context
         .get_config(Config::ConfiguredAddr)
         .await
-        .ok_or_else(|| Error::NoConfiguredAddr)?;
+        .ok_or(Error::NoConfiguredAddr)?;
     let addr = EmailAddress::new(&addr)?;
     let _guard = context.generating_key_mutex.lock().await;
 
@@ -168,16 +247,18 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
             secret: SignedSecretKey::from_slice(&sec_bytes)?,
         }),
         Err(sql::Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => {
-            let start = std::time::Instant::now();
+            let start = std::time::SystemTime::now();
             let keytype = KeyGenType::from_i32(context.get_config_int(Config::KeyGenType).await)
                 .unwrap_or_default();
             info!(context, "Generating keypair with type {}", keytype);
-            let keypair = crate::pgp::create_keypair(addr, keytype)?;
+            let keypair =
+                async_std::task::spawn_blocking(move || crate::pgp::create_keypair(addr, keytype))
+                    .await?;
             store_self_keypair(context, &keypair, KeyPairUse::Default).await?;
             info!(
                 context,
                 "Keypair generated in {:.3}s.",
-                start.elapsed().as_secs()
+                start.elapsed().unwrap_or_default().as_secs()
             );
             Ok(keypair)
         }
@@ -185,198 +266,6 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
     }
 }
 
-/// Cryptographic key
-#[derive(Debug, PartialEq, Eq, Clone)]
-pub enum Key {
-    Public(SignedPublicKey),
-    Secret(SignedSecretKey),
-}
-
-impl From<SignedPublicKey> for Key {
-    fn from(key: SignedPublicKey) -> Self {
-        Key::Public(key)
-    }
-}
-
-impl From<SignedSecretKey> for Key {
-    fn from(key: SignedSecretKey) -> Self {
-        Key::Secret(key)
-    }
-}
-
-impl std::convert::TryFrom<Key> for SignedSecretKey {
-    type Error = ();
-
-    fn try_from(value: Key) -> std::result::Result<Self, Self::Error> {
-        match value {
-            Key::Public(_) => Err(()),
-            Key::Secret(key) => Ok(key),
-        }
-    }
-}
-
-impl<'a> std::convert::TryFrom<&'a Key> for &'a SignedSecretKey {
-    type Error = ();
-
-    fn try_from(value: &'a Key) -> std::result::Result<Self, Self::Error> {
-        match value {
-            Key::Public(_) => Err(()),
-            Key::Secret(key) => Ok(key),
-        }
-    }
-}
-
-impl std::convert::TryFrom<Key> for SignedPublicKey {
-    type Error = ();
-
-    fn try_from(value: Key) -> std::result::Result<Self, Self::Error> {
-        match value {
-            Key::Public(key) => Ok(key),
-            Key::Secret(_) => Err(()),
-        }
-    }
-}
-
-impl<'a> std::convert::TryFrom<&'a Key> for &'a SignedPublicKey {
-    type Error = ();
-
-    fn try_from(value: &'a Key) -> std::result::Result<Self, Self::Error> {
-        match value {
-            Key::Public(key) => Ok(key),
-            Key::Secret(_) => Err(()),
-        }
-    }
-}
-
-impl Key {
-    pub fn is_public(&self) -> bool {
-        match self {
-            Key::Public(_) => true,
-            Key::Secret(_) => false,
-        }
-    }
-
-    pub fn is_secret(&self) -> bool {
-        !self.is_public()
-    }
-
-    pub fn from_slice(bytes: &[u8], key_type: KeyType) -> Option<Self> {
-        if bytes.is_empty() {
-            return None;
-        }
-        let res: std::result::Result<Key, _> = match key_type {
-            KeyType::Public => SignedPublicKey::from_bytes(Cursor::new(bytes)).map(Into::into),
-            KeyType::Private => SignedSecretKey::from_bytes(Cursor::new(bytes)).map(Into::into),
-        };
-
-        match res {
-            Ok(key) => Some(key),
-            Err(err) => {
-                eprintln!("Invalid key bytes: {:?}", err);
-                None
-            }
-        }
-    }
-
-    pub fn from_armored_string(
-        data: &str,
-        key_type: KeyType,
-    ) -> Option<(Self, BTreeMap<String, String>)> {
-        let bytes = data.as_bytes();
-        let res: std::result::Result<(Key, _), _> = match key_type {
-            KeyType::Public => SignedPublicKey::from_armor_single(Cursor::new(bytes))
-                .map(|(k, h)| (Into::into(k), h)),
-            KeyType::Private => SignedSecretKey::from_armor_single(Cursor::new(bytes))
-                .map(|(k, h)| (Into::into(k), h)),
-        };
-
-        match res {
-            Ok(res) => Some(res),
-            Err(err) => {
-                eprintln!("Invalid key bytes: {:?}", err);
-                None
-            }
-        }
-    }
-
-    pub fn to_bytes(&self) -> Vec<u8> {
-        match self {
-            Key::Public(k) => k.to_bytes().unwrap_or_default(),
-            Key::Secret(k) => k.to_bytes().unwrap_or_default(),
-        }
-    }
-
-    pub fn verify(&self) -> bool {
-        match self {
-            Key::Public(k) => k.verify().is_ok(),
-            Key::Secret(k) => k.verify().is_ok(),
-        }
-    }
-
-    pub fn to_base64(&self) -> String {
-        let buf = self.to_bytes();
-        base64::encode(&buf)
-    }
-
-    pub fn to_armored_string(
-        &self,
-        headers: Option<&BTreeMap<String, String>>,
-    ) -> pgp::errors::Result<String> {
-        match self {
-            Key::Public(k) => k.to_armored_string(headers),
-            Key::Secret(k) => k.to_armored_string(headers),
-        }
-    }
-
-    /// Each header line must be terminated by `\r\n`
-    pub fn to_asc(&self, header: Option<(&str, &str)>) -> String {
-        let headers = header.map(|(key, value)| {
-            let mut m = BTreeMap::new();
-            m.insert(key.to_string(), value.to_string());
-            m
-        });
-
-        self.to_armored_string(headers.as_ref())
-            .expect("failed to serialize key")
-    }
-
-    pub async fn write_asc_to_file(
-        &self,
-        file: impl AsRef<Path>,
-        context: &Context,
-    ) -> std::io::Result<()> {
-        let file_content = self.to_asc(None).into_bytes();
-
-        let res = dc_write_file(context, &file, &file_content).await;
-        if res.is_err() {
-            error!(context, "Cannot write key to {}", file.as_ref().display());
-        }
-        res
-    }
-
-    pub fn fingerprint(&self) -> String {
-        match self {
-            Key::Public(k) => hex::encode_upper(k.fingerprint()),
-            Key::Secret(k) => hex::encode_upper(k.fingerprint()),
-        }
-    }
-
-    pub fn formatted_fingerprint(&self) -> String {
-        let rawhex = self.fingerprint();
-        dc_format_fingerprint(&rawhex)
-    }
-
-    pub fn split_key(&self) -> Option<Key> {
-        match self {
-            Key::Public(_) => None,
-            Key::Secret(k) => {
-                let pub_key = k.public_key();
-                pub_key.sign(k, || "".into()).map(Key::Public).ok()
-            }
-        }
-    }
-}
-
 /// Use of a [KeyPair] for encryption or decryption.
 ///
 /// This is used by [store_self_keypair] to know what kind of key is
@@ -426,14 +315,8 @@ pub async fn store_self_keypair(
 ) -> std::result::Result<(), SaveKeyError> {
     // Everything should really be one transaction, more refactoring
     // is needed for that.
-    let public_key = keypair
-        .public
-        .to_bytes()
-        .map_err(|err| SaveKeyError::new("failed to serialise public key", err))?;
-    let secret_key = keypair
-        .secret
-        .to_bytes()
-        .map_err(|err| SaveKeyError::new("failed to serialise secret key", err))?;
+    let public_key = DcKey::to_bytes(&keypair.public);
+    let secret_key = DcKey::to_bytes(&keypair.secret);
     context
         .sql
         .execute(
@@ -471,55 +354,90 @@ pub async fn store_self_keypair(
     Ok(())
 }
 
-/// Make a fingerprint human-readable, in hex format.
-pub fn dc_format_fingerprint(fingerprint: &str) -> String {
-    // split key into chunks of 4 with space, and 20 newline
-    let mut res = String::new();
+/// A key fingerprint
+#[derive(Clone, Eq, PartialEq, Hash)]
+pub struct Fingerprint(Vec<u8>);
 
-    for (i, c) in fingerprint.chars().enumerate() {
-        if i > 0 && i % 20 == 0 {
-            res += "\n";
-        } else if i > 0 && i % 4 == 0 {
-            res += " ";
+impl Fingerprint {
+    pub fn new(v: Vec<u8>) -> std::result::Result<Fingerprint, FingerprintError> {
+        match v.len() {
+            20 => Ok(Fingerprint(v)),
+            _ => Err(FingerprintError::WrongLength),
         }
-
-        res += &c.to_string();
     }
 
-    res
+    /// Make a hex string from the fingerprint.
+    ///
+    /// Use [std::fmt::Display] or [ToString::to_string] to get a
+    /// human-readable formatted string.
+    pub fn hex(&self) -> String {
+        hex::encode_upper(&self.0)
+    }
 }
 
-/// Bring a human-readable or otherwise formatted fingerprint back to the 40-characters-uppercase-hex format.
-pub fn dc_normalize_fingerprint(fp: &str) -> String {
-    fp.to_uppercase()
-        .chars()
-        .filter(|&c| c >= '0' && c <= '9' || c >= 'A' && c <= 'F')
-        .collect()
+impl fmt::Debug for Fingerprint {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("Fingerprint")
+            .field("hex", &self.hex())
+            .finish()
+    }
+}
+
+/// Make a human-readable fingerprint.
+impl fmt::Display for Fingerprint {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        // Split key into chunks of 4 with space and newline at 20 chars
+        for (i, c) in self.hex().chars().enumerate() {
+            if i > 0 && i % 20 == 0 {
+                writeln!(f)?;
+            } else if i > 0 && i % 4 == 0 {
+                write!(f, " ")?;
+            }
+            write!(f, "{}", c)?;
+        }
+        Ok(())
+    }
+}
+
+/// Parse a human-readable or otherwise formatted fingerprint.
+impl std::str::FromStr for Fingerprint {
+    type Err = FingerprintError;
+
+    fn from_str(input: &str) -> std::result::Result<Self, Self::Err> {
+        let hex_repr: String = input
+            .to_uppercase()
+            .chars()
+            .filter(|&c| c >= '0' && c <= '9' || c >= 'A' && c <= 'F')
+            .collect();
+        let v: Vec<u8> = hex::decode(hex_repr)?;
+        let fp = Fingerprint::new(v)?;
+        Ok(fp)
+    }
+}
+
+#[derive(Debug, Error)]
+pub enum FingerprintError {
+    #[error("Invalid hex characters")]
+    NotHex(#[from] hex::FromHexError),
+    #[error("Incorrect fingerprint lengths")]
+    WrongLength,
 }
 
 #[cfg(test)]
 mod tests {
     use super::*;
     use crate::test_utils::*;
-    use std::convert::TryFrom;
+
+    use std::error::Error;
 
     use async_std::sync::Arc;
-    use lazy_static::lazy_static;
+    use once_cell::sync::Lazy;
 
-    lazy_static! {
-        static ref KEYPAIR: KeyPair = alice_keypair();
-    }
-
-    #[test]
-    fn test_normalize_fingerprint() {
-        let fingerprint = dc_normalize_fingerprint(" 1234  567890 \n AbcD abcdef ABCDEF ");
-
-        assert_eq!(fingerprint, "1234567890ABCDABCDEFABCDEF");
-    }
+    static KEYPAIR: Lazy<KeyPair> = Lazy::new(alice_keypair);
 
     #[test]
     fn test_from_armored_string() {
-        let (private_key, _) = Key::from_armored_string(
+        let (private_key, _) = SignedSecretKey::from_asc(
             "-----BEGIN PGP PRIVATE KEY BLOCK-----
 
 xcLYBF0fgz4BCADnRUV52V4xhSsU56ZaAn3+3oG86MZhXy4X8w14WZZDf0VJGeTh
@@ -577,63 +495,69 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
 7yPJeQ==
 =KZk/
 -----END PGP PRIVATE KEY BLOCK-----",
-            KeyType::Private,
         )
-        .expect("failed to decode"); // NOTE: if you take out the ===GU1/ part, everything passes!
-        let binary = private_key.to_bytes();
-        Key::from_slice(&binary, KeyType::Private).expect("invalid private key");
+        .expect("failed to decode");
+        let binary = DcKey::to_bytes(&private_key);
+        SignedSecretKey::from_slice(&binary).expect("invalid private key");
     }
 
     #[test]
-    fn test_format_fingerprint() {
-        let fingerprint = dc_format_fingerprint("1234567890ABCDABCDEFABCDEF1234567890ABCD");
+    fn test_asc_roundtrip() {
+        let key = KEYPAIR.public.clone();
+        let asc = key.to_asc(Some(("spam", "ham")));
+        let (key2, hdrs) = SignedPublicKey::from_asc(&asc).unwrap();
+        assert_eq!(key, key2);
+        assert_eq!(hdrs.len(), 1);
+        assert_eq!(hdrs.get("spam"), Some(&String::from("ham")));
 
-        assert_eq!(
-            fingerprint,
-            "1234 5678 90AB CDAB CDEF\nABCD EF12 3456 7890 ABCD"
-        );
+        let key = KEYPAIR.secret.clone();
+        let asc = key.to_asc(Some(("spam", "ham")));
+        let (key2, hdrs) = SignedSecretKey::from_asc(&asc).unwrap();
+        assert_eq!(key, key2);
+        assert_eq!(hdrs.len(), 1);
+        assert_eq!(hdrs.get("spam"), Some(&String::from("ham")));
     }
 
     #[test]
     fn test_from_slice_roundtrip() {
-        let public_key = Key::from(KEYPAIR.public.clone());
-        let private_key = Key::from(KEYPAIR.secret.clone());
+        let public_key = KEYPAIR.public.clone();
+        let private_key = KEYPAIR.secret.clone();
 
-        let binary = public_key.to_bytes();
-        let public_key2 = Key::from_slice(&binary, KeyType::Public).expect("invalid public key");
+        let binary = DcKey::to_bytes(&public_key);
+        let public_key2 = SignedPublicKey::from_slice(&binary).expect("invalid public key");
         assert_eq!(public_key, public_key2);
 
-        let binary = private_key.to_bytes();
-        let private_key2 = Key::from_slice(&binary, KeyType::Private).expect("invalid private key");
+        let binary = DcKey::to_bytes(&private_key);
+        let private_key2 = SignedSecretKey::from_slice(&binary).expect("invalid private key");
         assert_eq!(private_key, private_key2);
     }
 
     #[test]
     fn test_from_slice_bad_data() {
         let mut bad_data: [u8; 4096] = [0; 4096];
-
-        for i in 0..4096 {
-            bad_data[i] = (i & 0xff) as u8;
+        for (i, v) in bad_data.iter_mut().enumerate() {
+            *v = (i & 0xff) as u8;
         }
-
         for j in 0..(4096 / 40) {
-            let bad_key = Key::from_slice(
-                &bad_data[j..j + 4096 / 2 + j],
-                if 0 != j & 1 {
-                    KeyType::Public
-                } else {
-                    KeyType::Private
-                },
-            );
-            assert!(bad_key.is_none());
+            let slice = &bad_data.get(j..j + 4096 / 2 + j).unwrap();
+            assert!(SignedPublicKey::from_slice(slice).is_err());
+            assert!(SignedSecretKey::from_slice(slice).is_err());
         }
     }
 
+    #[test]
+    fn test_base64_roundtrip() {
+        let key = KEYPAIR.public.clone();
+        let base64 = key.to_base64();
+        let key2 = SignedPublicKey::from_base64(&base64).unwrap();
+        assert_eq!(key, key2);
+    }
+
     #[async_std::test]
     async fn test_load_self_existing() {
         let alice = alice_keypair();
-        let t = dummy_context().await;
-        configure_alice_keypair(&t.ctx).await;
+        let t = TestContext::new().await;
+        t.configure_alice().await;
         let pubkey = SignedPublicKey::load_self(&t.ctx).await.unwrap();
         assert_eq!(alice.public, pubkey);
         let seckey = SignedSecretKey::load_self(&t.ctx).await.unwrap();
@@ -641,9 +565,8 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     }
 
     #[async_std::test]
-    #[ignore] // generating keys is expensive
     async fn test_load_self_generate_public() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx
             .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
             .await
@@ -653,9 +576,8 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     }
 
     #[async_std::test]
-    #[ignore] // generating keys is expensive
     async fn test_load_self_generate_secret() {
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx
             .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
             .await
@@ -665,11 +587,10 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     }
 
     #[async_std::test]
-    #[ignore] // generating keys is expensive
     async fn test_load_self_generate_concurrent() {
         use std::thread;
 
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         t.ctx
             .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
             .await
@@ -678,7 +599,7 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         let ctx0 = ctx.clone();
         let thr0 =
             thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx0)));
-        let ctx1 = ctx.clone();
+        let ctx1 = ctx;
         let thr1 =
             thread::spawn(move || async_std::task::block_on(SignedPublicKey::load_self(&ctx1)));
         let res0 = thr0.join().unwrap();
@@ -686,36 +607,17 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         assert_eq!(res0.unwrap(), res1.unwrap());
     }
 
-    #[test]
-    fn test_ascii_roundtrip() {
-        let public_key = Key::from(KEYPAIR.public.clone());
-        let private_key = Key::from(KEYPAIR.secret.clone());
-
-        let s = public_key.to_armored_string(None).unwrap();
-        let (public_key2, _) =
-            Key::from_armored_string(&s, KeyType::Public).expect("invalid public key");
-        assert_eq!(public_key, public_key2);
-
-        let s = private_key.to_armored_string(None).unwrap();
-        println!("{}", &s);
-        let (private_key2, _) =
-            Key::from_armored_string(&s, KeyType::Private).expect("invalid private key");
-        assert_eq!(private_key, private_key2);
-    }
-
     #[test]
     fn test_split_key() {
-        let private_key = Key::from(KEYPAIR.secret.clone());
-        let public_wrapped = private_key.split_key().unwrap();
-        let public = SignedPublicKey::try_from(public_wrapped).unwrap();
-        assert_eq!(public.primary_key, KEYPAIR.public.primary_key);
+        let pubkey = KEYPAIR.secret.split_public_key().unwrap();
+        assert_eq!(pubkey.primary_key, KEYPAIR.public.primary_key);
     }
 
     #[async_std::test]
     async fn test_save_self_key_twice() {
         // Saving the same key twice should result in only one row in
         // the keypairs table.
-        let t = dummy_context().await;
+        let t = TestContext::new().await;
         let ctx = Arc::new(t.ctx);
 
         let ctx1 = ctx.clone();
@@ -756,4 +658,49 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
     //     )
     //     .unwrap();
     // }
+
+    #[test]
+    fn test_fingerprint_from_str() {
+        let res = Fingerprint::new(vec![
+            1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+        ])
+        .unwrap();
+
+        let fp: Fingerprint = "0102030405060708090A0B0c0d0e0F1011121314".parse().unwrap();
+        assert_eq!(fp, res);
+
+        let fp: Fingerprint = "zzzz 0102 0304 0506\n0708090a0b0c0D0E0F1011121314 yyy"
+            .parse()
+            .unwrap();
+        assert_eq!(fp, res);
+
+        let err = "1".parse::<Fingerprint>().err().unwrap();
+        match err {
+            FingerprintError::NotHex(_) => (),
+            _ => panic!("Wrong error"),
+        }
+        let src_err = err.source().unwrap().downcast_ref::<hex::FromHexError>();
+        assert_eq!(src_err, Some(&hex::FromHexError::OddLength));
+    }
+
+    #[test]
+    fn test_fingerprint_hex() {
+        let fp = Fingerprint::new(vec![
+            1, 2, 4, 8, 16, 32, 64, 128, 255, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+        ])
+        .unwrap();
+        assert_eq!(fp.hex(), "0102040810204080FF0A0B0C0D0E0F1011121314");
+    }
+
+    #[test]
+    fn test_fingerprint_to_string() {
+        let fp = Fingerprint::new(vec![
+            1, 2, 4, 8, 16, 32, 64, 128, 255, 1, 2, 4, 8, 16, 32, 64, 128, 255, 19, 20,
+        ])
+        .unwrap();
+        assert_eq!(
+            fp.to_string(),
+            "0102 0408 1020 4080 FF01\n0204 0810 2040 80FF 1314"
+        );
+    }
 }

src/keyring.rs

@@ -1,46 +1,92 @@
-use std::borrow::Cow;
+//! Keyring to perform rpgp operations with.
+
+use anyhow::Result;
 
-use crate::constants::KeyType;
 use crate::context::Context;
-use crate::key::Key;
-use crate::sql::Sql;
+use crate::key::{self, DcKey};
 
-#[derive(Default, Clone, Debug)]
-pub struct Keyring<'a> {
-    keys: Vec<Cow<'a, Key>>,
+/// An in-memory keyring.
+///
+/// Instances are usually constructed just for the rpgp operation and
+/// short-lived.
+#[derive(Clone, Debug, Default)]
+pub struct Keyring<T>
+where
+    T: DcKey,
+{
+    keys: Vec<T>,
 }
 
-impl<'a> Keyring<'a> {
-    pub fn add_owned(&mut self, key: Key) {
-        self.add(Cow::Owned(key))
+impl<T> Keyring<T>
+where
+    T: DcKey<KeyType = T>,
+{
+    /// New empty keyring.
+    pub fn new() -> Keyring<T> {
+        Keyring { keys: Vec::new() }
     }
 
-    pub fn add_ref(&mut self, key: &'a Key) {
-        self.add(Cow::Borrowed(key))
+    /// Create a new keyring with the the user's secret key loaded.
+    pub async fn new_self(context: &Context) -> Result<Keyring<T>, key::Error> {
+        let mut keyring: Keyring<T> = Keyring::new();
+        keyring.load_self(context).await?;
+        Ok(keyring)
     }
 
-    fn add(&mut self, key: Cow<'a, Key>) {
+    /// Load the user's key into the keyring.
+    pub async fn load_self(&mut self, context: &Context) -> Result<(), key::Error> {
+        self.add(T::load_self(context).await?);
+        Ok(())
+    }
+
+    /// Add a key to the keyring.
+    pub fn add(&mut self, key: T) {
         self.keys.push(key);
     }
 
-    pub fn keys(&self) -> &[Cow<'a, Key>] {
-        &self.keys
+    pub fn len(&self) -> usize {
+        self.keys.len()
     }
 
-    pub async fn load_self_private_for_decrypting(
-        &mut self,
-        context: &Context,
-        self_addr: impl AsRef<str>,
-        sql: &Sql,
-    ) -> bool {
-        sql.query_get_value(
-            context,
-            "SELECT private_key FROM keypairs ORDER BY addr=? DESC, is_default DESC;",
-            paramsv![self_addr.as_ref().to_string()],
-        )
-        .await
-        .and_then(|blob: Vec<u8>| Key::from_slice(&blob, KeyType::Private))
-        .map(|key| self.add_owned(key))
-        .is_some()
+    pub fn is_empty(&self) -> bool {
+        self.keys.is_empty()
+    }
+
+    /// A vector with reference to all the keys in the keyring.
+    pub fn keys(&self) -> &[T] {
+        &self.keys
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::key::{SignedPublicKey, SignedSecretKey};
+    use crate::test_utils::*;
+
+    #[test]
+    fn test_keyring_add_keys() {
+        let alice = alice_keypair();
+        let mut pub_ring: Keyring<SignedPublicKey> = Keyring::new();
+        pub_ring.add(alice.public.clone());
+        assert_eq!(pub_ring.keys(), [alice.public]);
+
+        let mut sec_ring: Keyring<SignedSecretKey> = Keyring::new();
+        sec_ring.add(alice.secret.clone());
+        assert_eq!(sec_ring.keys(), [alice.secret]);
+    }
+
+    #[async_std::test]
+    async fn test_keyring_load_self() {
+        // new_self() implies load_self()
+        let t = TestContext::new().await;
+        t.configure_alice().await;
+        let alice = alice_keypair();
+
+        let pub_ring: Keyring<SignedPublicKey> = Keyring::new_self(&t.ctx).await.unwrap();
+        assert_eq!(pub_ring.keys(), [alice.public]);
+
+        let sec_ring: Keyring<SignedSecretKey> = Keyring::new_self(&t.ctx).await.unwrap();
+        assert_eq!(sec_ring.keys(), [alice.secret]);
     }
 }

src/lib.rs

@@ -1,6 +1,11 @@
 #![forbid(unsafe_code)]
-#![deny(clippy::correctness, missing_debug_implementations, clippy::all)]
-#![allow(clippy::match_bool)]
+#![deny(
+    clippy::correctness,
+    missing_debug_implementations,
+    clippy::all,
+    clippy::indexing_slicing
+)]
+#![allow(clippy::match_bool, clippy::eval_order_dependence)]
 
 #[macro_use]
 extern crate num_derive;
@@ -11,8 +16,6 @@ extern crate rusqlite;
 extern crate strum;
 #[macro_use]
 extern crate strum_macros;
-#[macro_use]
-extern crate debug_stub_derive;
 
 pub trait ToSql: rusqlite::ToSql + Send + Sync {}
 
@@ -45,11 +48,13 @@ pub mod constants;
 pub mod contact;
 pub mod context;
 mod e2ee;
+pub mod ephemeral;
 mod imap;
 pub mod imex;
 mod scheduler;
 #[macro_use]
 pub mod job;
+mod format_flowed;
 pub mod key;
 mod keyring;
 pub mod location;
@@ -75,6 +80,8 @@ mod dehtml;
 pub mod dc_receive_imf;
 pub mod dc_tools;
 
+pub mod accounts;
+
 /// if set imap/incoming and smtp/outgoing MIME messages will be printed
 pub const DCC_MIME_DEBUG: &str = "DCC_MIME_DEBUG";
 

src/location.rs

@@ -9,7 +9,7 @@ use crate::constants::*;
 use crate::context::*;
 use crate::dc_tools::*;
 use crate::error::{ensure, Error};
-use crate::events::Event;
+use crate::events::EventType;
 use crate::job::{self, Job};
 use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
@@ -227,7 +227,7 @@ pub async fn send_locations_to_chat(context: &Context, chat_id: ChatId, seconds:
                     .await;
                 chat::add_info_msg(context, chat_id, stock_str).await;
             }
-            context.emit_event(Event::ChatModified(chat_id));
+            context.emit_event(EventType::ChatModified(chat_id));
             if 0 != seconds {
                 schedule_maybe_send_locations(context, false).await;
                 job::add(
@@ -301,7 +301,7 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
             }
         }
         if continue_streaming {
-            context.emit_event(Event::LocationChanged(Some(DC_CONTACT_ID_SELF)));
+            context.emit_event(EventType::LocationChanged(Some(DC_CONTACT_ID_SELF)));
         };
         schedule_maybe_send_locations(context, false).await;
     }
@@ -381,7 +381,7 @@ pub async fn delete_all(context: &Context) -> Result<(), Error> {
         .sql
         .execute("DELETE FROM locations;", paramsv![])
         .await?;
-    context.emit_event(Event::LocationChanged(None));
+    context.emit_event(EventType::LocationChanged(None));
     Ok(())
 }
 
@@ -469,7 +469,7 @@ pub fn get_message_kml(timestamp: i64, latitude: f64, longitude: f64) -> String
          <Document>\n\
          <Placemark>\
          <Timestamp><when>{}</when></Timestamp>\
-         <Point><coordinates>{:.2},{:.2}</coordinates></Point>\
+         <Point><coordinates>{},{}</coordinates></Point>\
          </Placemark>\n\
          </Document>\n\
          </kml>",
@@ -530,7 +530,7 @@ pub async fn save(
             accuracy,
             ..
         } = location;
-        context
+        let (loc_id, ts) = context
             .sql
             .with_conn(move |mut conn| {
                 let mut stmt_test = conn
@@ -569,9 +569,11 @@ pub async fn save(
                         )?;
                     }
                 }
-                Ok(())
+                Ok((newest_location_id, newest_timestamp))
             })
             .await?;
+        newest_timestamp = ts;
+        newest_location_id = loc_id;
     }
 
     Ok(newest_location_id)
@@ -713,7 +715,7 @@ pub(crate) async fn job_maybe_send_locations_ended(
                 .stock_system_msg(StockMessage::MsgLocationDisabled, "", "", 0)
                 .await;
             chat::add_info_msg(context, chat_id, stock_str).await;
-            context.emit_event(Event::ChatModified(chat_id));
+            context.emit_event(EventType::ChatModified(chat_id));
         }
     }
     job::Status::Finished(Ok(()))
@@ -721,12 +723,14 @@ pub(crate) async fn job_maybe_send_locations_ended(
 
 #[cfg(test)]
 mod tests {
+    #![allow(clippy::indexing_slicing)]
+
     use super::*;
-    use crate::test_utils::dummy_context;
+    use crate::test_utils::TestContext;
 
     #[async_std::test]
     async fn test_kml_parse() {
-        let context = dummy_context().await;
+        let context = TestContext::new().await;
 
         let xml =
             b"<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<kml xmlns=\"http://www.opengis.net/kml/2.2\">\n<Document addr=\"user@example.org\">\n<Placemark><Timestamp><when>2019-03-06T21:09:57Z</when></Timestamp><Point><coordinates accuracy=\"32.000000\">9.423110,53.790302</coordinates></Point></Placemark>\n<PlaceMARK>\n<Timestamp><WHEN > \n\t2018-12-13T22:11:12Z\t</WHEN></Timestamp><Point><coordinates aCCuracy=\"2.500000\"> 19.423110 \t , \n 63.790302\n </coordinates></Point></PlaceMARK>\n</Document>\n</kml>";
@@ -755,4 +759,22 @@ mod tests {
         assert!(locations_ref[1].accuracy < 2.6f64);
         assert_eq!(locations_ref[1].timestamp, 1544739072);
     }
+
+    #[async_std::test]
+    async fn test_get_message_kml() {
+        let context = TestContext::new().await;
+        let timestamp = 1598490000;
+
+        let xml = get_message_kml(timestamp, 51.423723f64, 8.552556f64);
+        let kml = Kml::parse(&context.ctx, xml.as_bytes()).expect("parsing failed");
+        let locations_ref = &kml.locations;
+        assert_eq!(locations_ref.len(), 1);
+
+        assert!(locations_ref[0].latitude >= 51.423723f64);
+        assert!(locations_ref[0].latitude < 51.423724f64);
+        assert!(locations_ref[0].longitude >= 8.552556f64);
+        assert!(locations_ref[0].longitude < 8.552557f64);
+        assert!(locations_ref[0].accuracy.abs() < f64::EPSILON);
+        assert_eq!(locations_ref[0].timestamp, timestamp);
+    }
 }

src/log.rs

@@ -11,7 +11,7 @@ macro_rules! info {
                            file = file!(),
                            line = line!(),
                            msg = &formatted);
-        emit_event!($ctx, $crate::Event::Info(full));
+        emit_event!($ctx, $crate::EventType::Info(full));
     }};
 }
 
@@ -26,7 +26,7 @@ macro_rules! warn {
                            file = file!(),
                            line = line!(),
                            msg = &formatted);
-        emit_event!($ctx, $crate::Event::Warning(full));
+        emit_event!($ctx, $crate::EventType::Warning(full));
     }};
 }
 
@@ -37,7 +37,18 @@ macro_rules! error {
     };
     ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
         let formatted = format!($msg, $($args),*);
-        emit_event!($ctx, $crate::Event::Error(formatted));
+        emit_event!($ctx, $crate::EventType::Error(formatted));
+    }};
+}
+
+#[macro_export]
+macro_rules! error_network {
+    ($ctx:expr, $msg:expr) => {
+        error_network!($ctx, $msg,)
+    };
+    ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
+        let formatted = format!($msg, $($args),*);
+        emit_event!($ctx, $crate::EventType::ErrorNetwork(formatted));
     }};
 }
 

src/login_param.rs

@@ -3,13 +3,16 @@
 use std::borrow::Cow;
 use std::fmt;
 
-use crate::context::Context;
+use crate::{context::Context, provider::Socket};
 
-#[derive(Copy, Clone, Debug, Display, FromPrimitive)]
+#[derive(Copy, Clone, Debug, Display, FromPrimitive, PartialEq, Eq)]
 #[repr(i32)]
 #[strum(serialize_all = "snake_case")]
 pub enum CertificateChecks {
+    /// Same as AcceptInvalidCertificates unless overridden by
+    /// `strict_tls` setting in provider database.
     Automatic = 0,
+
     Strict = 1,
 
     /// Same as AcceptInvalidCertificates
@@ -25,30 +28,29 @@ impl Default for CertificateChecks {
     }
 }
 
-#[derive(Default, Debug)]
+/// Login parameters for a single server, either IMAP or SMTP
+#[derive(Default, Debug, Clone)]
+pub struct ServerLoginParam {
+    pub server: String,
+    pub user: String,
+    pub password: String,
+    pub port: u16,
+    pub security: Socket,
+
+    /// TLS options: whether to allow invalid certificates and/or
+    /// invalid hostnames
+    pub certificate_checks: CertificateChecks,
+}
+
+#[derive(Default, Debug, Clone)]
 pub struct LoginParam {
     pub addr: String,
-    pub mail_server: String,
-    pub mail_user: String,
-    pub mail_pw: String,
-    pub mail_port: i32,
-    /// IMAP TLS options: whether to allow invalid certificates and/or invalid hostnames
-    pub imap_certificate_checks: CertificateChecks,
-    pub send_server: String,
-    pub send_user: String,
-    pub send_pw: String,
-    pub send_port: i32,
-    /// SMTP TLS options: whether to allow invalid certificates and/or invalid hostnames
-    pub smtp_certificate_checks: CertificateChecks,
+    pub imap: ServerLoginParam,
+    pub smtp: ServerLoginParam,
     pub server_flags: i32,
 }
 
 impl LoginParam {
-    /// Create a new `LoginParam` with default values.
-    pub fn new() -> Self {
-        Default::default()
-    }
-
     /// Read the login parameters from the database.
     pub async fn from_database(context: &Context, prefix: impl AsRef<str>) -> Self {
         let prefix = prefix.as_ref();
@@ -77,6 +79,13 @@ impl LoginParam {
         let key = format!("{}mail_pw", prefix);
         let mail_pw = sql.get_raw_config(context, key).await.unwrap_or_default();
 
+        let key = format!("{}mail_security", prefix);
+        let mail_security = sql
+            .get_raw_config_int(context, key)
+            .await
+            .and_then(num_traits::FromPrimitive::from_i32)
+            .unwrap_or_default();
+
         let key = format!("{}imap_certificate_checks", prefix);
         let imap_certificate_checks =
             if let Some(certificate_checks) = sql.get_raw_config_int(context, key).await {
@@ -100,6 +109,13 @@ impl LoginParam {
         let key = format!("{}send_pw", prefix);
         let send_pw = sql.get_raw_config(context, key).await.unwrap_or_default();
 
+        let key = format!("{}send_security", prefix);
+        let send_security = sql
+            .get_raw_config_int(context, key)
+            .await
+            .and_then(num_traits::FromPrimitive::from_i32)
+            .unwrap_or_default();
+
         let key = format!("{}smtp_certificate_checks", prefix);
         let smtp_certificate_checks =
             if let Some(certificate_checks) = sql.get_raw_config_int(context, key).await {
@@ -116,24 +132,26 @@ impl LoginParam {
 
         LoginParam {
             addr,
-            mail_server,
-            mail_user,
-            mail_pw,
-            mail_port,
-            imap_certificate_checks,
-            send_server,
-            send_user,
-            send_pw,
-            send_port,
-            smtp_certificate_checks,
+            imap: ServerLoginParam {
+                server: mail_server,
+                user: mail_user,
+                password: mail_pw,
+                port: mail_port as u16,
+                security: mail_security,
+                certificate_checks: imap_certificate_checks,
+            },
+            smtp: ServerLoginParam {
+                server: send_server,
+                user: send_user,
+                password: send_pw,
+                port: send_port as u16,
+                security: send_security,
+                certificate_checks: smtp_certificate_checks,
+            },
             server_flags,
         }
     }
 
-    pub fn addr_str(&self) -> &str {
-        self.addr.as_str()
-    }
-
     /// Save this loginparam to the database.
     pub async fn save_to_database(
         &self,
@@ -147,41 +165,51 @@ impl LoginParam {
         sql.set_raw_config(context, key, Some(&self.addr)).await?;
 
         let key = format!("{}mail_server", prefix);
-        sql.set_raw_config(context, key, Some(&self.mail_server))
+        sql.set_raw_config(context, key, Some(&self.imap.server))
             .await?;
 
         let key = format!("{}mail_port", prefix);
-        sql.set_raw_config_int(context, key, self.mail_port).await?;
+        sql.set_raw_config_int(context, key, self.imap.port as i32)
+            .await?;
 
         let key = format!("{}mail_user", prefix);
-        sql.set_raw_config(context, key, Some(&self.mail_user))
+        sql.set_raw_config(context, key, Some(&self.imap.user))
             .await?;
 
         let key = format!("{}mail_pw", prefix);
-        sql.set_raw_config(context, key, Some(&self.mail_pw))
+        sql.set_raw_config(context, key, Some(&self.imap.password))
+            .await?;
+
+        let key = format!("{}mail_security", prefix);
+        sql.set_raw_config_int(context, key, self.imap.security as i32)
             .await?;
 
         let key = format!("{}imap_certificate_checks", prefix);
-        sql.set_raw_config_int(context, key, self.imap_certificate_checks as i32)
+        sql.set_raw_config_int(context, key, self.imap.certificate_checks as i32)
             .await?;
 
         let key = format!("{}send_server", prefix);
-        sql.set_raw_config(context, key, Some(&self.send_server))
+        sql.set_raw_config(context, key, Some(&self.smtp.server))
             .await?;
 
         let key = format!("{}send_port", prefix);
-        sql.set_raw_config_int(context, key, self.send_port).await?;
+        sql.set_raw_config_int(context, key, self.smtp.port as i32)
+            .await?;
 
         let key = format!("{}send_user", prefix);
-        sql.set_raw_config(context, key, Some(&self.send_user))
+        sql.set_raw_config(context, key, Some(&self.smtp.user))
             .await?;
 
         let key = format!("{}send_pw", prefix);
-        sql.set_raw_config(context, key, Some(&self.send_pw))
+        sql.set_raw_config(context, key, Some(&self.smtp.password))
+            .await?;
+
+        let key = format!("{}send_security", prefix);
+        sql.set_raw_config_int(context, key, self.smtp.security as i32)
             .await?;
 
         let key = format!("{}smtp_certificate_checks", prefix);
-        sql.set_raw_config_int(context, key, self.smtp_certificate_checks as i32)
+        sql.set_raw_config_int(context, key, self.smtp.certificate_checks as i32)
             .await?;
 
         let key = format!("{}server_flags", prefix);
@@ -203,16 +231,24 @@ impl fmt::Display for LoginParam {
             f,
             "{} imap:{}:{}:{}:{}:cert_{} smtp:{}:{}:{}:{}:cert_{} {}",
             unset_empty(&self.addr),
-            unset_empty(&self.mail_user),
-            if !self.mail_pw.is_empty() { pw } else { unset },
-            unset_empty(&self.mail_server),
-            self.mail_port,
-            self.imap_certificate_checks,
-            unset_empty(&self.send_user),
-            if !self.send_pw.is_empty() { pw } else { unset },
-            unset_empty(&self.send_server),
-            self.send_port,
-            self.smtp_certificate_checks,
+            unset_empty(&self.imap.user),
+            if !self.imap.password.is_empty() {
+                pw
+            } else {
+                unset
+            },
+            unset_empty(&self.imap.server),
+            self.imap.port,
+            self.imap.certificate_checks,
+            unset_empty(&self.smtp.user),
+            if !self.smtp.password.is_empty() {
+                pw
+            } else {
+                unset
+            },
+            unset_empty(&self.smtp.server),
+            self.smtp.port,
+            self.smtp.certificate_checks,
             flags_readable,
         )
     }
@@ -241,30 +277,6 @@ fn get_readable_flags(flags: i32) -> String {
                 res += "AUTH_NORMAL ";
                 flag_added = true;
             }
-            if 1 << bit == 0x100 {
-                res += "IMAP_STARTTLS ";
-                flag_added = true;
-            }
-            if 1 << bit == 0x200 {
-                res += "IMAP_SSL ";
-                flag_added = true;
-            }
-            if 1 << bit == 0x400 {
-                res += "IMAP_PLAIN ";
-                flag_added = true;
-            }
-            if 1 << bit == 0x10000 {
-                res += "SMTP_STARTTLS ";
-                flag_added = true;
-            }
-            if 1 << bit == 0x20000 {
-                res += "SMTP_SSL ";
-                flag_added = true;
-            }
-            if 1 << bit == 0x40000 {
-                res += "SMTP_PLAIN ";
-                flag_added = true;
-            }
             if flag_added {
                 res += &format!("{:#0x}", 1 << bit);
             }
@@ -277,21 +289,15 @@ fn get_readable_flags(flags: i32) -> String {
     res
 }
 
-pub fn dc_build_tls(certificate_checks: CertificateChecks) -> async_native_tls::TlsConnector {
+pub fn dc_build_tls(strict_tls: bool) -> async_native_tls::TlsConnector {
     let tls_builder = async_native_tls::TlsConnector::new();
-    match certificate_checks {
-        CertificateChecks::Automatic => {
-            // Same as AcceptInvalidCertificates for now.
-            // TODO: use provider database when it becomes available
-            tls_builder
-                .danger_accept_invalid_hostnames(true)
-                .danger_accept_invalid_certs(true)
-        }
-        CertificateChecks::Strict => tls_builder,
-        CertificateChecks::AcceptInvalidCertificates
-        | CertificateChecks::AcceptInvalidCertificates2 => tls_builder
+
+    if strict_tls {
+        tls_builder
+    } else {
+        tls_builder
             .danger_accept_invalid_hostnames(true)
-            .danger_accept_invalid_certs(true),
+            .danger_accept_invalid_certs(true)
     }
 }
 

src/lot.rs

@@ -1,5 +1,7 @@
 use deltachat_derive::{FromSql, ToSql};
 
+use crate::key::Fingerprint;
+
 /// An object containing a set of values.
 /// The meaning of the values is defined by the function returning the object.
 /// Lot objects are created
@@ -14,7 +16,7 @@ pub struct Lot {
     pub(crate) timestamp: i64,
     pub(crate) state: LotState,
     pub(crate) id: u32,
-    pub(crate) fingerprint: Option<String>,
+    pub(crate) fingerprint: Option<Fingerprint>,
     pub(crate) invitenumber: Option<String>,
     pub(crate) auth: Option<String>,
 }
@@ -89,6 +91,9 @@ pub enum LotState {
     /// text1=domain
     QrAccount = 250,
 
+    /// text1=domain, text2=instance pattern
+    QrWebrtcInstance = 260,
+
     /// id=contact
     QrAddr = 320,
 

src/mimefactory.rs

@@ -9,7 +9,9 @@ use crate::contact::*;
 use crate::context::{get_version_str, Context};
 use crate::dc_tools::*;
 use crate::e2ee::*;
+use crate::ephemeral::Timer as EphemeralTimer;
 use crate::error::{bail, ensure, format_err, Error};
+use crate::format_flowed::{format_flowed, format_flowed_quote};
 use crate::location;
 use crate::message::{self, Message};
 use crate::mimeparser::SystemMessage;
@@ -112,12 +114,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                 )
                 .await?;
 
-            let command = msg.param.get_cmd();
-
-            if command != SystemMessage::AutocryptSetupMessage
-                && command != SystemMessage::SecurejoinMessage
-                && context.get_config_bool(Config::MdnsEnabled).await
-            {
+            if !msg.is_system_message() && context.get_config_bool(Config::MdnsEnabled).await {
                 req_mdn = true;
             }
         }
@@ -148,7 +145,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             selfstatus: context
                 .get_config(Config::Selfstatus)
                 .await
-                .unwrap_or_else(|| default_str),
+                .unwrap_or(default_str),
             recipients,
             timestamp: msg.timestamp_sort,
             loaded: Loaded::Message { chat },
@@ -186,7 +183,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let selfstatus = context
             .get_config(Config::Selfstatus)
             .await
-            .unwrap_or_else(|| default_str);
+            .unwrap_or(default_str);
         let timestamp = dc_create_smeared_timestamp(context).await;
 
         let res = MimeFactory::<'a, 'b> {
@@ -225,7 +222,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             .filter(|(_, addr)| addr != &self_addr)
         {
             res.push((
-                Peerstate::from_addr(self.context, addr).await,
+                Peerstate::from_addr(self.context, addr).await?,
                 addr.as_str(),
             ));
         }
@@ -236,26 +233,20 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
     fn is_e2ee_guaranteed(&self) -> bool {
         match &self.loaded {
             Loaded::Message { chat } => {
-                if chat.typ == Chattype::VerifiedGroup {
+                if chat.is_protected() {
                     return true;
                 }
 
-                let force_plaintext = self
+                !self
                     .msg
                     .param
-                    .get_int(Param::ForcePlaintext)
-                    .unwrap_or_default();
-
-                if force_plaintext == 0 {
-                    return self
+                    .get_bool(Param::ForcePlaintext)
+                    .unwrap_or_default()
+                    && self
                         .msg
                         .param
-                        .get_int(Param::GuaranteeE2ee)
+                        .get_bool(Param::GuaranteeE2ee)
                         .unwrap_or_default()
-                        != 0;
-                }
-
-                false
             }
             Loaded::MDN { .. } => false,
         }
@@ -264,7 +255,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
     fn min_verified(&self) -> PeerstateVerifiedStatus {
         match &self.loaded {
             Loaded::Message { chat } => {
-                if chat.typ == Chattype::VerifiedGroup {
+                if chat.is_protected() {
                     PeerstateVerifiedStatus::BidirectVerified
                 } else {
                     PeerstateVerifiedStatus::Unverified
@@ -274,19 +265,30 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         }
     }
 
-    fn should_force_plaintext(&self) -> i32 {
+    fn should_force_plaintext(&self) -> bool {
         match &self.loaded {
             Loaded::Message { chat } => {
-                if chat.typ == Chattype::VerifiedGroup {
-                    0
+                if chat.is_protected() {
+                    false
                 } else {
                     self.msg
                         .param
-                        .get_int(Param::ForcePlaintext)
+                        .get_bool(Param::ForcePlaintext)
                         .unwrap_or_default()
                 }
             }
-            Loaded::MDN { .. } => ForcePlaintext::NoAutocryptHeader as i32,
+            Loaded::MDN { .. } => true,
+        }
+    }
+
+    fn should_skip_autocrypt(&self) -> bool {
+        match &self.loaded {
+            Loaded::Message { .. } => self
+                .msg
+                .param
+                .get_bool(Param::SkipAutocrypt)
+                .unwrap_or_default(),
+            Loaded::MDN { .. } => true,
         }
     }
 
@@ -343,7 +345,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                         .stock_str(StockMessage::AcSetupMsgSubject)
                         .await
                         .into_owned()
-                } else if chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup {
+                } else if chat.typ == Chattype::Group {
                     let re = if self.in_reply_to.is_empty() {
                         ""
                     } else {
@@ -351,16 +353,47 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                     };
                     format!("{}{}", re, chat.name)
                 } else {
-                    let raw = message::get_summarytext_by_raw(
-                        self.msg.viewtype,
-                        self.msg.text.as_ref(),
-                        &self.msg.param,
-                        32,
-                        self.context,
-                    )
-                    .await;
-                    let raw_subject = raw.lines().next().unwrap_or_default();
-                    format!("Chat: {}", raw_subject)
+                    match chat.param.get(Param::LastSubject) {
+                        Some(last_subject) => {
+                            let subject_start = if last_subject.starts_with("Chat:") {
+                                0
+                            } else {
+                                // "Antw:" is the longest abbreviation in
+                                // https://en.wikipedia.org/wiki/List_of_email_subject_abbreviations#Abbreviations_in_other_languages,
+                                // so look at the first _5_ characters:
+                                match last_subject.chars().take(5).position(|c| c == ':') {
+                                    Some(prefix_end) => prefix_end + 1,
+                                    None => 0,
+                                }
+                            };
+                            format!(
+                                "Re: {}",
+                                last_subject
+                                    .chars()
+                                    .skip(subject_start)
+                                    .collect::<String>()
+                                    .trim()
+                            )
+                        }
+                        None => {
+                            let self_name = match self.context.get_config(Config::Displayname).await
+                            {
+                                Some(name) => name,
+                                None => self
+                                    .context
+                                    .get_config(Config::Addr)
+                                    .await
+                                    .unwrap_or_default(),
+                            };
+
+                            self.context
+                                .stock_string_repl_str(
+                                    StockMessage::SubjectForNewContact,
+                                    self_name,
+                                )
+                                .await
+                        }
+                    }
                 }
             }
             Loaded::MDN { .. } => self
@@ -409,6 +442,8 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             to.push(from.clone());
         }
 
+        unprotected_headers.push(Header::new("MIME-Version".into(), "1.0".into()));
+
         if !self.references.is_empty() {
             unprotected_headers.push(Header::new("References".into(), self.references.clone()));
         }
@@ -424,21 +459,6 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
         unprotected_headers.push(Header::new("Date".into(), date));
 
-        let os_name = &self.context.os_name;
-        let os_part = os_name
-            .as_ref()
-            .map(|s| format!("/{}", s))
-            .unwrap_or_default();
-        let version = get_version_str();
-
-        // Add a X-Mailer header.
-        // This is only informational for debugging and may be removed in the release.
-        // We do not rely on this header as it may be removed by MTAs.
-
-        unprotected_headers.push(Header::new(
-            "X-Mailer".into(),
-            format!("Delta Chat Core {}{}", version, os_part),
-        ));
         unprotected_headers.push(Header::new("Chat-Version".to_string(), "1.0".to_string()));
 
         if let Loaded::MDN { .. } = self.loaded {
@@ -461,11 +481,21 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let min_verified = self.min_verified();
         let grpimage = self.grpimage();
         let force_plaintext = self.should_force_plaintext();
+        let skip_autocrypt = self.should_skip_autocrypt();
         let subject_str = self.subject_str().await;
         let e2ee_guaranteed = self.is_e2ee_guaranteed();
-        let mut encrypt_helper = EncryptHelper::new(self.context).await?;
+        let encrypt_helper = EncryptHelper::new(self.context).await?;
 
-        let subject = encode_words(&subject_str);
+        let subject = if subject_str
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || c == ' ')
+        // We do not use needs_encoding() here because needs_encoding() returns true if the string contains a space
+        // but we do not want to encode all subjects just because they contain a space.
+        {
+            subject_str
+        } else {
+            encode_words(&subject_str)
+        };
 
         let mut message = match self.loaded {
             Loaded::Message { .. } => {
@@ -475,7 +505,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             Loaded::MDN { .. } => self.render_mdn().await?,
         };
 
-        if force_plaintext != ForcePlaintext::NoAutocryptHeader as i32 {
+        if !skip_autocrypt {
             // unless determined otherwise we add the Autocrypt header
             let aheader = encrypt_helper.get_aheader().to_string();
             unprotected_headers.push(Header::new("Autocrypt".into(), aheader));
@@ -486,13 +516,21 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let peerstates = self.peerstates_for_recipients().await?;
         let should_encrypt =
             encrypt_helper.should_encrypt(self.context, e2ee_guaranteed, &peerstates)?;
-        let is_encrypted = should_encrypt && force_plaintext == 0;
+        let is_encrypted = should_encrypt && !force_plaintext;
 
         let rfc724_mid = match self.loaded {
             Loaded::Message { .. } => self.msg.rfc724_mid.clone(),
             Loaded::MDN { .. } => dc_create_outgoing_rfc724_mid(None, &self.from_addr),
         };
 
+        let ephemeral_timer = self.msg.chat_id.get_ephemeral_timer(self.context).await?;
+        if let EphemeralTimer::Enabled { duration } = ephemeral_timer {
+            protected_headers.push(Header::new(
+                "Ephemeral-Timer".to_string(),
+                duration.to_string(),
+            ));
+        }
+
         // we could also store the message-id in the protected headers
         // which would probably help to survive providers like
         // Outlook.com or hotmail which mangle the Message-ID.
@@ -560,7 +598,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             }
 
             let encrypted = encrypt_helper
-                .encrypt(self.context, min_verified, message, &peerstates)
+                .encrypt(self.context, min_verified, message, peerstates)
                 .await?;
 
             outer_message = outer_message
@@ -670,11 +708,11 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let mut placeholdertext = None;
         let mut meta_part = None;
 
-        if chat.typ == Chattype::VerifiedGroup {
+        if chat.is_protected() {
             protected_headers.push(Header::new("Chat-Verified".to_string(), "1".to_string()));
         }
 
-        if chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup {
+        if chat.typ == Chattype::Group {
             protected_headers.push(Header::new("Chat-Group-ID".into(), chat.grpid.clone()));
 
             let encoded = encode_words(&chat.name);
@@ -743,6 +781,26 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                     "location-streaming-enabled".into(),
                 ));
             }
+            SystemMessage::EphemeralTimerChanged => {
+                protected_headers.push(Header::new(
+                    "Chat-Content".to_string(),
+                    "ephemeral-timer-changed".to_string(),
+                ));
+            }
+            SystemMessage::LocationOnly => {
+                // This should prevent automatic replies,
+                // such as non-delivery reports.
+                //
+                // See https://tools.ietf.org/html/rfc3834
+                //
+                // Adding this header without encryption leaks some
+                // information about the message contents, but it can
+                // already be easily guessed from message timing and size.
+                unprotected_headers.push(Header::new(
+                    "Auto-Submitted".to_string(),
+                    "auto-generated".to_string(),
+                ));
+            }
             SystemMessage::AutocryptSetupMessage => {
                 unprotected_headers
                     .push(Header::new("Autocrypt-Setup-Message".into(), "v1".into()));
@@ -788,6 +846,18 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                     };
                 }
             }
+            SystemMessage::ChatProtectionEnabled => {
+                protected_headers.push(Header::new(
+                    "Chat-Content".to_string(),
+                    "protection-enabled".to_string(),
+                ));
+            }
+            SystemMessage::ChatProtectionDisabled => {
+                protected_headers.push(Header::new(
+                    "Chat-Content".to_string(),
+                    "protection-disabled".to_string(),
+                ));
+            }
             _ => {}
         }
 
@@ -804,6 +874,19 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
         if self.msg.viewtype == Viewtype::Sticker {
             protected_headers.push(Header::new("Chat-Content".into(), "sticker".into()));
+        } else if self.msg.viewtype == Viewtype::VideochatInvitation {
+            protected_headers.push(Header::new(
+                "Chat-Content".into(),
+                "videochat-invitation".into(),
+            ));
+            protected_headers.push(Header::new(
+                "Chat-Webrtc-Room".into(),
+                self.msg
+                    .param
+                    .get(Param::WebrtcRoom)
+                    .unwrap_or_default()
+                    .into(),
+            ));
         }
 
         if self.msg.viewtype == Viewtype::Voice
@@ -846,11 +929,18 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             }
         };
 
+        let quoted_text = self
+            .msg
+            .quoted_text()
+            .map(|quote| format_flowed_quote(&quote) + "\r\n\r\n");
+        let flowed_text = format_flowed(final_text);
+
         let footer = &self.selfstatus;
         let message_text = format!(
-            "{}{}{}{}{}",
+            "{}{}{}{}{}{}",
             fwdhint.unwrap_or_default(),
-            escape_message_footer_marks(final_text),
+            quoted_text.unwrap_or_default(),
+            escape_message_footer_marks(&flowed_text),
             if !final_text.is_empty() && !footer.is_empty() {
                 "\r\n\r\n"
             } else {
@@ -862,7 +952,10 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
         // Message is sent as text/plain, with charset = utf-8
         let main_part = PartBuilder::new()
-            .content_type(&mime::TEXT_PLAIN_UTF_8)
+            .header((
+                "Content-Type".to_string(),
+                "text/plain; charset=utf-8; format=flowed; delsp=no".to_string(),
+            ))
             .body(message_text);
         let mut parts = Vec::new();
 
@@ -1174,6 +1267,10 @@ pub fn needs_encoding(to_check: impl AsRef<str>) -> bool {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::chatlist::Chatlist;
+    use crate::dc_receive_imf::dc_receive_imf;
+    use crate::mimeparser::*;
+    use crate::test_utils::TestContext;
 
     #[test]
     fn test_render_email_address() {
@@ -1181,6 +1278,9 @@ mod tests {
         let addr = "x@y.org";
 
         assert!(!display_name.is_ascii());
+        assert!(!display_name
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || c == ' '));
 
         let s = format!(
             "{}",
@@ -1192,6 +1292,25 @@ mod tests {
         assert_eq!(s, "=?utf-8?q?=C3=A4_space?= <x@y.org>");
     }
 
+    #[test]
+    fn test_render_email_address_noescape() {
+        let display_name = "a space";
+        let addr = "x@y.org";
+
+        assert!(display_name.is_ascii());
+        assert!(display_name
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || c == ' '));
+
+        let s = format!(
+            "{}",
+            Address::new_mailbox_with_name(display_name.to_string(), addr.to_string())
+        );
+
+        // Addresses should not be unnecessarily be encoded, see https://github.com/deltachat/deltachat-core-rust/issues/1575:
+        assert_eq!(s, "a space <x@y.org>");
+    }
+
     #[test]
     fn test_render_rfc724_mid() {
         assert_eq!(
@@ -1234,4 +1353,241 @@ mod tests {
         assert!(needs_encoding(" "));
         assert!(needs_encoding("foo bar"));
     }
+
+    #[async_std::test]
+    async fn test_subject() {
+        // 1.: Receive a mail from an MUA or Delta Chat
+        assert_eq!(
+            msg_to_subject_str(
+                b"From: Bob <bob@example.com>\n\
+                To: alice@example.com\n\
+                Subject: Antw: Chat: hello\n\
+                Message-ID: <2222@example.com>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n"
+            )
+            .await,
+            "Re: Chat: hello"
+        );
+
+        assert_eq!(
+            msg_to_subject_str(
+                b"From: Bob <bob@example.com>\n\
+                To: alice@example.com\n\
+                Subject: Infos: 42\n\
+                Message-ID: <2222@example.com>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n"
+            )
+            .await,
+            "Re: Infos: 42"
+        );
+
+        // 2. Receive a message from Delta Chat when we did not send any messages before
+        assert_eq!(
+            msg_to_subject_str(
+                b"From: Charlie <charlie@example.com>\n\
+                To: alice@example.com\n\
+                Subject: Chat: hello\n\
+                Chat-Version: 1.0\n\
+                Message-ID: <2223@example.com>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n"
+            )
+            .await,
+            "Re: Chat: hello"
+        );
+
+        // 3. Send the first message to a new contact
+        let t = TestContext::new_alice().await;
+
+        assert_eq!(first_subject_str(t).await, "Message from alice@example.com");
+
+        let t = TestContext::new_alice().await;
+        t.ctx
+            .set_config(Config::Displayname, Some("Alice"))
+            .await
+            .unwrap();
+        assert_eq!(first_subject_str(t).await, "Message from Alice");
+
+        // 4. Receive messages with unicode characters and make sure that we do not panic (we do not care about the result)
+        msg_to_subject_str(
+            "From: Charlie <charlie@example.com>\n\
+            To: alice@example.com\n\
+            Subject: äääää\n\
+            Chat-Version: 1.0\n\
+            Message-ID: <2893@example.com>\n\
+            Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+            \n\
+            hello\n"
+                .as_bytes(),
+        )
+        .await;
+
+        msg_to_subject_str(
+            "From: Charlie <charlie@example.com>\n\
+            To: alice@example.com\n\
+            Subject: aäääää\n\
+            Chat-Version: 1.0\n\
+            Message-ID: <2893@example.com>\n\
+            Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+            \n\
+            hello\n"
+                .as_bytes(),
+        )
+        .await;
+
+        // 5. Receive an mdn (read receipt) and make sure the mdn's subject is not used
+        let t = TestContext::new_alice().await;
+        dc_receive_imf(
+            &t.ctx,
+            b"From: alice@example.com\n\
+            To: Charlie <charlie@example.com>\n\
+            Subject: Hello, Charlie\n\
+            Chat-Version: 1.0\n\
+            Message-ID: <2893@example.com>\n\
+            Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+            \n\
+            hello\n",
+            "INBOX",
+            1,
+            false,
+        )
+        .await
+        .unwrap();
+        let new_msg = incoming_msg_to_reply_msg(b"From: charlie@example.com\n\
+                 To: alice@example.com\n\
+                 Subject: message opened\n\
+                 Date: Sun, 22 Mar 2020 23:37:57 +0000\n\
+                 Chat-Version: 1.0\n\
+                 Message-ID: <Mr.12345678902@example.com>\n\
+                 Content-Type: multipart/report; report-type=disposition-notification; boundary=\"SNIPP\"\n\
+                 \n\
+                 \n\
+                 --SNIPP\n\
+                 Content-Type: text/plain; charset=utf-8\n\
+                 \n\
+                 Read receipts do not guarantee sth. was read.\n\
+                 \n\
+                 \n\
+                 --SNIPP\n\
+                 Content-Type: message/disposition-notification\n\
+                 \n\
+                 Reporting-UA: Delta Chat 1.28.0\n\
+                 Original-Recipient: rfc822;charlie@example.com\n\
+                 Final-Recipient: rfc822;charlie@example.com\n\
+                 Original-Message-ID: <2893@example.com>\n\
+                 Disposition: manual-action/MDN-sent-automatically; displayed\n\
+                 \n", &t.ctx).await;
+        let mf = MimeFactory::from_msg(&t.ctx, &new_msg, false)
+            .await
+            .unwrap();
+        // The subject string should not be "Re: message opened"
+        assert_eq!("Re: Hello, Charlie", mf.subject_str().await);
+    }
+
+    async fn first_subject_str(t: TestContext) -> String {
+        let contact_id =
+            Contact::add_or_lookup(&t.ctx, "Dave", "dave@example.com", Origin::ManuallyCreated)
+                .await
+                .unwrap()
+                .0;
+
+        let chat_id = chat::create_by_contact_id(&t.ctx, contact_id)
+            .await
+            .unwrap();
+
+        let mut new_msg = Message::new(Viewtype::Text);
+        new_msg.set_text(Some("Hi".to_string()));
+        new_msg.chat_id = chat_id;
+        chat::prepare_msg(&t.ctx, chat_id, &mut new_msg)
+            .await
+            .unwrap();
+
+        let mf = MimeFactory::from_msg(&t.ctx, &new_msg, false)
+            .await
+            .unwrap();
+
+        mf.subject_str().await
+    }
+
+    async fn msg_to_subject_str(imf_raw: &[u8]) -> String {
+        let t = TestContext::new_alice().await;
+        let new_msg = incoming_msg_to_reply_msg(imf_raw, &t.ctx).await;
+        let mf = MimeFactory::from_msg(&t.ctx, &new_msg, false)
+            .await
+            .unwrap();
+        mf.subject_str().await
+    }
+
+    // Creates a `Message` that replies "Hi" to the incoming email in `imf_raw`.
+    async fn incoming_msg_to_reply_msg(imf_raw: &[u8], context: &Context) -> Message {
+        context
+            .set_config(Config::ShowEmails, Some("2"))
+            .await
+            .unwrap();
+
+        dc_receive_imf(context, imf_raw, "INBOX", 1, false)
+            .await
+            .unwrap();
+
+        let chats = Chatlist::try_load(context, 0, None, None).await.unwrap();
+
+        let chat_id = chat::create_by_msg_id(context, chats.get_msg_id(0).unwrap())
+            .await
+            .unwrap();
+
+        let mut new_msg = Message::new(Viewtype::Text);
+        new_msg.set_text(Some("Hi".to_string()));
+        new_msg.chat_id = chat_id;
+        chat::prepare_msg(context, chat_id, &mut new_msg)
+            .await
+            .unwrap();
+
+        new_msg
+    }
+
+    #[async_std::test]
+    // This test could still be extended
+    async fn test_render_reply() {
+        let t = TestContext::new_alice().await;
+        let context = &t.ctx;
+
+        let msg = incoming_msg_to_reply_msg(
+            b"From: Charlie <charlie@example.com>\n\
+                To: alice@example.com\n\
+                Subject: Chat: hello\n\
+                Chat-Version: 1.0\n\
+                Message-ID: <2223@example.com>\n\
+                Date: Sun, 22 Mar 2020 22:37:56 +0000\n\
+                \n\
+                hello\n",
+            context,
+        )
+        .await;
+
+        let mimefactory = MimeFactory::from_msg(&t.ctx, &msg, false).await.unwrap();
+
+        let recipients = mimefactory.recipients();
+        assert_eq!(recipients, vec!["charlie@example.com"]);
+
+        let rendered_msg = mimefactory.render().await.unwrap();
+
+        let mail = mailparse::parse_mail(&rendered_msg.message).unwrap();
+        assert_eq!(
+            mail.headers
+                .iter()
+                .find(|h| h.get_key() == "MIME-Version")
+                .unwrap()
+                .get_value(),
+            "1.0"
+        );
+
+        let _mime_msg = MimeMessage::from_bytes(context, &rendered_msg.message)
+            .await
+            .unwrap();
+    }
 }

src/oauth2.rs

@@ -1,12 +1,16 @@
 //! OAuth 2 module
 
+use regex::Regex;
 use std::collections::HashMap;
 
+use async_std_resolver::{config, resolver};
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 use serde::Deserialize;
 
 use crate::context::Context;
 use crate::dc_tools::*;
+use crate::provider;
+use crate::provider::Oauth2Authorizer;
 
 const OAUTH2_GMAIL: Oauth2 = Oauth2 {
     // see https://developers.google.com/identity/protocols/OAuth2InstalledApp
@@ -15,6 +19,7 @@ const OAUTH2_GMAIL: Oauth2 = Oauth2 {
     init_token: "https://accounts.google.com/o/oauth2/token?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&code=$CODE&grant_type=authorization_code",
     refresh_token: "https://accounts.google.com/o/oauth2/token?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&refresh_token=$REFRESH_TOKEN&grant_type=refresh_token",
     get_userinfo: Some("https://www.googleapis.com/oauth2/v1/userinfo?alt=json&access_token=$ACCESS_TOKEN"),
+    mx_pattern: Some(r"^aspmx\.l\.google\.com\.$"),
 };
 
 const OAUTH2_YANDEX: Oauth2 = Oauth2 {
@@ -24,8 +29,11 @@ const OAUTH2_YANDEX: Oauth2 = Oauth2 {
     init_token: "https://oauth.yandex.com/token?grant_type=authorization_code&code=$CODE&client_id=$CLIENT_ID&client_secret=58b8c6e94cf44fbe952da8511955dacf",
     refresh_token: "https://oauth.yandex.com/token?grant_type=refresh_token&refresh_token=$REFRESH_TOKEN&client_id=$CLIENT_ID&client_secret=58b8c6e94cf44fbe952da8511955dacf",
     get_userinfo: None,
+    mx_pattern: None,
 };
 
+const OAUTH2_PROVIDERS: [Oauth2; 1] = [OAUTH2_GMAIL];
+
 #[derive(Debug, Clone, PartialEq, Eq)]
 struct Oauth2 {
     client_id: &'static str,
@@ -33,6 +41,7 @@ struct Oauth2 {
     init_token: &'static str,
     refresh_token: &'static str,
     get_userinfo: Option<&'static str>,
+    mx_pattern: Option<&'static str>,
 }
 
 /// OAuth 2 Access Token Response
@@ -53,7 +62,7 @@ pub async fn dc_get_oauth2_url(
     addr: impl AsRef<str>,
     redirect_uri: impl AsRef<str>,
 ) -> Option<String> {
-    if let Some(oauth2) = Oauth2::from_address(addr) {
+    if let Some(oauth2) = Oauth2::from_address(addr).await {
         if context
             .sql
             .set_raw_config(
@@ -81,7 +90,7 @@ pub async fn dc_get_oauth2_access_token(
     code: impl AsRef<str>,
     regenerate: bool,
 ) -> Option<String> {
-    if let Some(oauth2) = Oauth2::from_address(addr) {
+    if let Some(oauth2) = Oauth2::from_address(addr).await {
         let lock = context.oauth2_mutex.lock().await;
 
         // read generated token
@@ -161,16 +170,14 @@ pub async fn dc_get_oauth2_access_token(
         }
 
         // ... and POST
-        let response = surf::post(post_url).body_form(&post_param);
-        if response.is_err() {
-            warn!(
-                context,
-                "Error calling OAuth2 at {}: {:?}", token_url, response
-            );
+        let mut req = surf::post(post_url).build();
+        if let Err(err) = req.body_form(&post_param) {
+            warn!(context, "Error calling OAuth2 at {}: {:?}", token_url, err);
             return None;
         }
 
-        let parsed: Result<Response, _> = response.unwrap().recv_json().await;
+        let client = surf::Client::new();
+        let parsed: Result<Response, _> = client.recv_json(req).await;
         if parsed.is_err() {
             warn!(
                 context,
@@ -239,7 +246,7 @@ pub async fn dc_get_oauth2_addr(
     addr: impl AsRef<str>,
     code: impl AsRef<str>,
 ) -> Option<String> {
-    let oauth2 = Oauth2::from_address(addr.as_ref())?;
+    let oauth2 = Oauth2::from_address(addr.as_ref()).await?;
     oauth2.get_userinfo?;
 
     if let Some(access_token) =
@@ -263,25 +270,58 @@ pub async fn dc_get_oauth2_addr(
 }
 
 impl Oauth2 {
-    fn from_address(addr: impl AsRef<str>) -> Option<Self> {
+    async fn from_address(addr: impl AsRef<str>) -> Option<Self> {
         let addr_normalized = normalize_addr(addr.as_ref());
         if let Some(domain) = addr_normalized
             .find('@')
             .map(|index| addr_normalized.split_at(index + 1).1)
         {
-            match domain {
-                "gmail.com" | "googlemail.com" => Some(OAUTH2_GMAIL),
-                "yandex.com" | "yandex.by" | "yandex.kz" | "yandex.ru" | "yandex.ua" | "ya.ru"
-                | "narod.ru" => Some(OAUTH2_YANDEX),
-                _ => None,
+            if let Some(provider) = provider::get_provider_info(&addr_normalized) {
+                match &provider.oauth2_authorizer {
+                    Some(Oauth2Authorizer::Gmail) => Some(OAUTH2_GMAIL),
+                    Some(Oauth2Authorizer::Yandex) => Some(OAUTH2_YANDEX),
+                    None => None, // provider known to not support oauth2, no mx-lookup required
+                }
+            } else {
+                Oauth2::lookup_mx(domain).await
             }
         } else {
             None
         }
     }
 
+    async fn lookup_mx(domain: impl AsRef<str>) -> Option<Self> {
+        if let Ok(resolver) = resolver(
+            config::ResolverConfig::default(),
+            config::ResolverOpts::default(),
+        )
+        .await
+        {
+            for provider in OAUTH2_PROVIDERS.iter() {
+                if let Some(pattern) = provider.mx_pattern {
+                    let re = Regex::new(pattern).unwrap();
+
+                    let mut fqdn: String = String::from(domain.as_ref());
+                    if !fqdn.ends_with('.') {
+                        fqdn.push('.');
+                    }
+
+                    if let Ok(res) = resolver.mx_lookup(fqdn).await {
+                        for rr in res.iter() {
+                            if re.is_match(&rr.exchange().to_lowercase().to_utf8()) {
+                                return Some(provider.clone());
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        None
+    }
+
     async fn get_addr(&self, context: &Context, access_token: impl AsRef<str>) -> Option<String> {
-        let userinfo_url = self.get_userinfo.unwrap_or_else(|| "");
+        let userinfo_url = self.get_userinfo.unwrap_or("");
         let userinfo_url = replace_in_uri(&userinfo_url, "$ACCESS_TOKEN", access_token);
 
         // should returns sth. as
@@ -362,25 +402,39 @@ mod tests {
         );
     }
 
-    #[test]
-    fn test_oauth_from_address() {
-        assert_eq!(Oauth2::from_address("hello@gmail.com"), Some(OAUTH2_GMAIL));
+    #[async_std::test]
+    async fn test_oauth_from_address() {
         assert_eq!(
-            Oauth2::from_address("hello@googlemail.com"),
+            Oauth2::from_address("hello@gmail.com").await,
             Some(OAUTH2_GMAIL)
         );
         assert_eq!(
-            Oauth2::from_address("hello@yandex.com"),
+            Oauth2::from_address("hello@googlemail.com").await,
+            Some(OAUTH2_GMAIL)
+        );
+        assert_eq!(
+            Oauth2::from_address("hello@yandex.com").await,
+            Some(OAUTH2_YANDEX)
+        );
+        assert_eq!(
+            Oauth2::from_address("hello@yandex.ru").await,
             Some(OAUTH2_YANDEX)
         );
-        assert_eq!(Oauth2::from_address("hello@yandex.ru"), Some(OAUTH2_YANDEX));
 
-        assert_eq!(Oauth2::from_address("hello@web.de"), None);
+        assert_eq!(Oauth2::from_address("hello@web.de").await, None);
+    }
+
+    #[async_std::test]
+    async fn test_oauth_from_mx() {
+        assert_eq!(
+            Oauth2::from_address("hello@google.com").await,
+            Some(OAUTH2_GMAIL)
+        );
     }
 
     #[async_std::test]
     async fn test_dc_get_oauth2_addr() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
         let addr = "dignifiedquire@gmail.com";
         let code = "fail";
         let res = dc_get_oauth2_addr(&ctx.ctx, addr, code).await;
@@ -390,7 +444,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_dc_get_oauth2_url() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
         let addr = "dignifiedquire@gmail.com";
         let redirect_uri = "chat.delta:/com.b44t.messenger";
         let res = dc_get_oauth2_url(&ctx.ctx, addr, redirect_uri).await;
@@ -400,7 +454,7 @@ mod tests {
 
     #[async_std::test]
     async fn test_dc_get_oauth2_token() {
-        let ctx = dummy_context().await;
+        let ctx = TestContext::new().await;
         let addr = "dignifiedquire@gmail.com";
         let code = "fail";
         let res = dc_get_oauth2_access_token(&ctx.ctx, addr, code, false).await;