Robust blob filename backup import/export

This ignores invalid UTF-8 filenames during import and export. Rust core
does not produce them, but it was possible to have such filenames with
C core.

.circleci/config.yml

@@ -0,0 +1,77 @@
+version: 2.1
+executors:
+  default:
+    docker:
+      - image: filecoin/rust:latest
+    working_directory: /mnt/crate
+  doxygen:
+    docker:
+      - image: hrektts/doxygen
+            
+jobs:
+  build_doxygen:
+    executor: doxygen
+    steps:
+      - checkout
+      - run: bash scripts/run-doxygen.sh
+      - run: mkdir -p workspace/c-docs
+      - run: cp -av deltachat-ffi/{html,xml} workspace/c-docs/
+      - persist_to_workspace:
+          root: workspace
+          paths:
+            - c-docs
+
+  remote_python_packaging:
+    machine: true
+    steps:
+      - checkout
+      # the following commands on success produces
+      # workspace/{wheelhouse,py-docs} as artefact directories
+      - run:
+          # building aarch64 packages under qemu is very slow
+          no_output_timeout: 60m
+          command: bash scripts/remote_python_packaging.sh
+      - persist_to_workspace:
+          root: workspace
+          paths:
+            # - c-docs
+            - py-docs
+            - wheelhouse
+
+  upload_docs_wheels:
+    machine: true
+    steps:
+      - checkout
+      - attach_workspace:
+          at: workspace
+      - run: ls -laR workspace
+      - run: scripts/ci_upload.sh workspace/py-docs workspace/wheelhouse workspace/c-docs
+
+workflows:
+  version: 2.1
+
+  test:
+    jobs:
+      - remote_python_packaging:
+          filters:
+            tags:
+              only: /.+/
+            branches:
+              ignore: /.*/
+
+      - build_doxygen:
+          filters:
+            tags:
+              only: /.+/
+            branches:
+              ignore: /.*/
+
+      - upload_docs_wheels:
+          requires:
+             - remote_python_packaging
+             - build_doxygen
+          filters:
+            tags:
+              only: /.+/
+            branches:
+              ignore: /.*/

.github/workflows/ci.yml

@@ -18,7 +18,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: stable
+          toolchain: 1.50.0
           override: true
       - run: rustup component add rustfmt
       - uses: actions-rs/cargo@v1
@@ -32,7 +32,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions-rs/toolchain@v1
         with:
-            toolchain: stable
+            toolchain: 1.50.0
             components: clippy
             override: true
       - uses: actions-rs/clippy-check@v1
@@ -68,23 +68,12 @@ jobs:
     strategy:
       matrix:
         include:
-          # Currently used Rust version, same as in `rust-toolchain` file.
           - os: ubuntu-latest
-            rust: 1.54.0
-            python: 3.9
+            rust: 1.50.0
+            python: 3.6
           - os: windows-latest
-            rust: 1.54.0
+            rust: 1.50.0
             python: false # Python bindings compilation on Windows is not supported.
-
-          # Minimum Supported Rust Version = 1.48.0
-          # This is the Debian "bullseye" release version of Rust.
-          #
-          # Minimum Supported Python Version = 3.7
-          # This is the minimum version for which manylinux Python wheels are
-          # built.
-          - os: ubuntu-latest
-            rust: 1.48.0
-            python: 3.7
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@master

.gitignore

@@ -28,4 +28,3 @@ deltachat-ffi/xml
 .rsynclist
 
 coverage/
-.DS_Store

CHANGELOG.md

@@ -1,157 +1,4 @@
-# Changelog
-
-## Unreleased
-
-### Changes
-- `Accounts` is not cloneable anymore #2654 #2658
-- always check certificates strictly when connecting over SOCKS5 in Automatic mode #2657
-- update chat/contact data only when there was no newer update #2642
-- improve Doxygen documentation style #2647
-
-### Fixes
-- ignore MDNs sent to self #2674
-- fix pkg-config file #2660
-
-## 1.60.0
-
-### Added
-- add device message to warn about QUOTA #2621
-- add SOCKS5 support #2474 #2620
-
-### Changes
-- don't emit multiple events with the same import/export progress number #2639
-- reduce message length limit to 5000 chars #2615
-
-### Fixes
-- keep event emitter from closing when there are no accounts #2636
-
-## 1.59.0
-
-### Added
-- add quota information to `dc_get_connectivity_html()`
-
-### Changes
-- refactorings #2592 #2570 #2581
-- add 'device chat about' to now existing status #2613
-- update provider database #2608
-
-### Fixes
-- provider database supports socket=PLAIN and dotless domains now #2604 #2608
-- add migrated accounts to events emitter #2607
-- fix forwarding quote-only mails #2600
-- do not set WantsMdn param for outgoing messages #2603
-- set timestamps for system messages #2593
-- do not treat gmail labels as folders #2587
-- avoid timing problems in `dc_maybe_network_lost()` #2551
-- only set smtp to "connected" if the last message was actually sent #2541
-
-
-## 1.58.0
-
-### Fixes
-
-- move WAL file together with database
-  and avoid using data if the database was not closed correctly before #2583
-
-
-## 1.57.0
-
-### API Changes
-
-- breaking change: removed deaddrop chat #2514 #2563
-
-  Contact request chats are not merged into a single virtual
-  "deaddrop" chat anymore. Instead, they are shown in the chatlist the
-  same way as other chats, but sending of messages to them is not
-  allowed and MDNs are not sent automatically until the chat is
-  "accepted" by the user.
-
-  New API:
-  - `dc_chat_is_contact_request()`: returns true if chat is a contact
-    request.  In this case an option to accept the chat via
-    `dc_accept_chat()` should be shown in the UI.
-  - `dc_accept_chat()`: unblock the chat or accept contact request
-  - `dc_block_chat()`: block the chat, currently works only for mailing
-    lists.
-
-  Removed API:
-  - `dc_create_chat_by_msg_id()`: deprecated 2021-02-07 in favor of
-    `dc_decide_on_contact_request()`
-  - `dc_marknoticed_contact()`: deprecated 2021-02-07 in favor of
-    `dc_decide_on_contact_request()`
-  - `dc_decide_on_contact_request()`: this call requires a message ID
-    from deaddrop chat as input. As deaddrop chat is removed, this
-    call can't be used anymore.
-  - `dc_msg_get_real_chat_id()`: use `dc_msg_get_chat_id()` instead, the
-    only difference between these calls was in handling of deaddrop
-    chat
-  - removed `DC_CHAT_ID_DEADDROP` and `DC_STR_DEADDROP` constants
-
-- breaking change: removed `DC_EVENT_ERROR_NETWORK` and `DC_STR_SERVER_RESPONSE`
-  Instead, there is a new api `dc_get_connectivity()`
-  and `dc_get_connectivity_html()`;
-  `DC_EVENT_CONNECTIVITY_CHANGED` is emitted on changes
-
-- breaking change: removed `dc_accounts_import_account()`
-  Instead you need to add an account and call `dc_imex(DC_IMEX_IMPORT_BACKUP)`
-  on its context
-
-- update account api, 2 new methods:
-  `int dc_all_work_done (dc_context_t* context);`
-  `int dc_accounts_all_work_done (dc_accounts_t* accounts);`
-
-- add api to check if a message was `Auto-Submitted`
-  cffi: `int dc_msg_is_bot (const dc_msg_t* msg);`
-  python: `Message.is_bot()`
-
-- `dc_context_t* dc_accounts_get_selected_account (dc_accounts_t* accounts);`
-  now returns `NULL` if there is no selected account
-
-- added `dc_accounts_maybe_network_lost()` for systems core cannot find out
-  connectivity loss on its own (eg. iOS) #2550
-
-### Added
-- use Auto-Submitted: auto-generated header to identify bots #2502
-- allow sending stickers via repl tool
-- chat: make `get_msg_cnt()` and `get_fresh_msg_cnt()` work for deaddrop chat #2493
-- withdraw/revive own qr-codes #2512
-- add Connectivity view (a better api for getting the connection status) #2319 #2549 #2542
-
-### Changes
-- updated spec: new `Chat-User-Avatar` usage, `Chat-Content: sticker`, structure, copyright year #2480
-- update documentation #2548 #2561 #2569
-- breaking: `Accounts::create` does not also create an default account anymore #2500
-- remove "forwarded" from stickers, as the primary way of getting stickers
-  is by asking a bot and then forwarding them currently #2526
-- mimeparser: use mailparse to parse RFC 2231 filenames #2543
-- allow email addresses without dot in the domain part #2112
-- allow installing lib and include under different prefixes #2558
-- remove counter from name provided by `DC_CHAT_ID_ARCHIVED_LINK` #2566
-- improve tests #2487 #2491 #2497
-- refactorings #2492 #2503 #2504 #2506 #2515 #2520 #2567 #2575 #2577 #2579
-- improve ci #2494
-- update provider-database #2565
-
-### Removed
-- remove `dc_accounts_import_account()` api #2521
-- remove `DC_EVENT_ERROR_NETWORK` and `DC_STR_SERVER_RESPONSE` #2319
-
-### Fixes
-- allow stickers with gif-images #2481
-- fix database migration #2486
-- do not count hidden messages in get_msg_cnt(). #2493
-- improve drafts detection #2489
-- fix panic when removing last, selected account from account manager #2500
-- set_draft's message-changed-event returns now draft's msg id instead of 0 #2304
-- avoid hiding outgoing classic emails #2505
-- fixes for message timestamps #2517
-- do not process names, avatars, location XMLs, message signature etc.
-  for duplicate messages #2513
-- fix `can_send` for users not in group #2479
-- fix receiving events for accounts added by `dc_accounts_add_account()` #2559
-- fix which chats messages are assigned to #2465
-- fix: don't create chats when MDNs are received #2578
-
+# Changelog 
 
 ## 1.56.0
 

CMakeLists.txt

@@ -1,6 +1,5 @@
 cmake_minimum_required(VERSION 3.16)
 project(deltachat LANGUAGES C)
-include(GNUInstallDirs)
 
 find_program(CARGO cargo)
 
@@ -9,11 +8,7 @@ add_custom_command(
 	"target/release/libdeltachat.a"
 	"target/release/libdeltachat.so"
 	"target/release/pkgconfig/deltachat.pc"
-        COMMAND
-        PREFIX=${CMAKE_INSTALL_PREFIX}
-        LIBDIR=${CMAKE_INSTALL_FULL_LIBDIR}
-        INCLUDEDIR=${CMAKE_INSTALL_FULL_INCLUDEDIR}
-        ${CARGO} build --release --no-default-features
+	COMMAND PREFIX=${CMAKE_INSTALL_PREFIX} ${CARGO} build --release --no-default-features
 
 	# Build in `deltachat-ffi` directory instead of using
 	# `--package deltachat_ffi` to avoid feature resolver version
@@ -36,6 +31,7 @@ add_custom_target(
 	"target/release/pkgconfig/deltachat.pc"
 )
 
+include(GNUInstallDirs)
 install(FILES "deltachat-ffi/deltachat.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 install(FILES "target/release/libdeltachat.a" DESTINATION ${CMAKE_INSTALL_LIBDIR})
 install(FILES "target/release/libdeltachat.so" DESTINATION ${CMAKE_INSTALL_LIBDIR})

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.60.0"
+version = "1.56.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
@@ -15,76 +15,75 @@ lto = true
 deltachat_derive = { path = "./deltachat_derive" }
 
 ansi_term = { version = "0.12.1", optional = true }
-anyhow = "1"
-async-imap = { git = "https://github.com/async-email/async-imap" }
-async-native-tls = { version = "0.3" }
-async-smtp = { git = "https://github.com/async-email/async-smtp", branch="master", features = ["socks5"] }
-async-std-resolver = "0.20"
-async-std = { version = "1", features = ["unstable"] }
-async-tar = "0.3"
-async-trait = "0.1"
-backtrace = "0.3"
+anyhow = "1.0.41"
+async-imap = "0.5.0"
+async-native-tls = { version = "0.3.3" }
+async-smtp = { git = "https://github.com/async-email/async-smtp", rev="2275fd8d13e39b2c58d6605c786ff06ff9e05708" }
+async-std-resolver = "0.20.3"
+async-std = { version = "~1.9.0", features = ["unstable"] }
+async-tar = "0.3.0"
+async-trait = "0.1.50"
+backtrace = "0.3.59"
 base64 = "0.13"
-bitflags = "1.3"
-byteorder = "1.3"
-chrono = "0.4"
-dirs = { version = "4", optional=true }
+bitflags = "1.1.0"
+byteorder = "1.3.1"
+charset = "0.1"
+chrono = "0.4.6"
+dirs = { version = "3.0.2", optional=true }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
-escaper = "0.1"
-futures = "0.3"
+escaper = "0.1.1"
+futures = "0.3.15"
 hex = "0.4.0"
 image = { version = "0.23.5", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
-indexmap = "1.7"
-itertools = "0.10"
+indexmap = "1.3.0"
+itertools = "0.10.1"
 kamadak-exif = "0.5"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
-libc = "0.2"
+libc = "0.2.97"
 log = {version = "0.4.8", optional = true }
-mailparse = "0.13"
-native-tls = "0.2"
-num_cpus = "1.13"
-num-derive = "0.3"
-num-traits = "0.2"
+mailparse = "0.13.4"
+native-tls = "0.2.3"
+num_cpus = "1.13.0"
+num-derive = "0.3.0"
+num-traits = "0.2.6"
 once_cell = "1.8.0"
 percent-encoding = "2.0"
-pgp = { version = "0.7", default-features = false }
-pretty_env_logger = { version = "0.4", optional = true }
-quick-xml = "0.22"
-r2d2 = "0.8"
-r2d2_sqlite = "0.18"
-rand = "0.7"
-regex = "1.5"
+pgp = { version = "0.7.0", default-features = false }
+pretty_env_logger = { version = "0.4.0", optional = true }
+quick-xml = "0.22.0"
+r2d2 = "0.8.9"
+r2d2_sqlite = "0.18.0"
+rand = "0.7.0"
+regex = "1.4.6"
 rusqlite = "0.25"
-rust-hsluv = "0.1"
-rustyline = { version = "8.2", optional = true }
-sanitize-filename = "0.3"
+rust-hsluv = "0.1.4"
+rustyline = { version = "8.2.0", optional = true }
+sanitize-filename = "0.3.0"
 serde_json = "1.0"
 serde = { version = "1.0", features = ["derive"] }
-sha-1 = "0.9"
-sha2 = "0.9"
-smallvec = "1"
-stop-token = "0.2"
-strum = "0.21"
-strum_macros = "0.21"
-surf = { version = "2.3", default-features = false, features = ["h1-client"] }
-thiserror = "1"
-toml = "0.5"
-url = "2"
+sha-1 = "0.9.6"
+sha2 = "0.9.5"
+smallvec = "1.0.0"
+stop-token = "0.2.0"
+strum = "0.21.0"
+strum_macros = "0.21.1"
+surf = { version = "2.0.0-alpha.4", default-features = false, features = ["h1-client"] }
+thiserror = "1.0.25"
+toml = "0.5.6"
+url = "2.2.2"
 uuid = { version = "0.8", features = ["serde", "v4"] }
-fast-socks5 = "0.4"
-humansize = "1"
 
 [dev-dependencies]
 ansi_term = "0.12.0"
-async-std = { version = "1.10.0", features = ["unstable", "attributes"] }
+async-std = { version = "1.9.0", features = ["unstable", "attributes"] }
 criterion = "0.3"
-futures-lite = "1.12"
-log = "0.4"
-pretty_assertions = "0.7"
-pretty_env_logger = "0.4"
-proptest = "1"
-tempfile = "3"
+futures-lite = "1.12.0"
+log = "0.4.11"
+pretty_assertions = "0.7.2"
+pretty_env_logger = "0.4.0"
+proptest = "1.0"
+tempfile = "3.0"
 
 [workspace]
 members = [
@@ -116,7 +115,7 @@ name = "search_msgs"
 harness = false
 
 [features]
-default = ["vendored"]
+default = []
 internals = []
 repl = ["internals", "rustyline", "log", "pretty_env_logger", "ansi_term", "dirs"]
 vendored = ["async-native-tls/vendored", "async-smtp/native-tls-vendored", "rusqlite/bundled"]

README.md

@@ -3,6 +3,7 @@
 > Deltachat-core written in Rust 
 
 [![Rust CI](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml/badge.svg)](https://github.com/deltachat/deltachat-core-rust/actions/workflows/ci.yml)
+[![CircleCI](https://circleci.com/gh/deltachat/deltachat-core-rust.svg?style=shield)](https://circleci.com/gh/deltachat/deltachat-core-rust/)
 
 ## Installing Rust and Cargo
 
@@ -79,16 +80,6 @@ For more commands type:
 > help
 ```
 
-## Installing libdeltachat system wide
-
-```
-$ git clone https://github.com/deltachat/deltachat-core-rust.git
-$ cd deltachat-core-rust
-$ cmake -B build . -DCMAKE_INSTALL_PREFIX=/usr
-$ cmake --build build
-$ sudo cmake --install build
-```
-
 ## Development
 
 ```sh

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.60.0"
+version = "1.56.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
@@ -20,9 +20,9 @@ libc = "0.2"
 human-panic = "1.0.1"
 num-traits = "0.2.6"
 serde_json = "1.0"
-async-std = "1.10.0"
-anyhow = "1.0.44"
-thiserror = "1.0.29"
+async-std = "1.9.0"
+anyhow = "1.0.41"
+thiserror = "1.0.25"
 rand = "0.7.3"
 
 [features]

deltachat-ffi/Doxyfile

@@ -583,7 +583,7 @@ SORT_MEMBERS_CTORS_1ST = NO
 # appear in their defined order.
 # The default value is: NO.
 
-SORT_GROUP_NAMES       = YES
+SORT_GROUP_NAMES       = NO
 
 # If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by
 # fully-qualified names, including namespaces. If set to NO, the class list will

deltachat-ffi/Doxyfile.css

@@ -4,16 +4,4 @@ div.fragment {
 	background-color: #e0e0e0;
 	border: 0;
 	padding: 1em;
-	border-radius: 6px;
 }
-
-code {
-	background-color: #e0e0e0;
-	padding-left: .5em;
-	padding-right: .5em;
-	border-radius: 6px;
-}
-
-li {
-	margin-bottom: .5em;
-}

deltachat-ffi/build.rs

@@ -23,13 +23,11 @@ fn main() {
         version = env::var("CARGO_PKG_VERSION").unwrap(),
         libs_priv = libs_priv,
         prefix = env::var("PREFIX").unwrap_or_else(|_| "/usr/local".to_string()),
-        libdir = env::var("LIBDIR").unwrap_or_else(|_| "/usr/local/lib".to_string()),
-        includedir = env::var("INCLUDEDIR").unwrap_or_else(|_| "/usr/local/include".to_string()),
     );
 
     fs::create_dir_all(target_path.join("pkgconfig")).unwrap();
     fs::File::create(target_path.join("pkgconfig").join("deltachat.pc"))
         .unwrap()
-        .write_all(pkg_config.as_bytes())
+        .write_all(&pkg_config.as_bytes())
         .unwrap();
 }

deltachat-ffi/deltachat.pc.in

@@ -1,6 +1,6 @@
 prefix={prefix}
-libdir={libdir}
-includedir={includedir}
+libdir=${{prefix}}/lib
+includedir=${{prefix}}/include
 
 Name: {name}
 Description: {description}

deltachat-ffi/src/string.rs

@@ -17,12 +17,15 @@ use std::ptr;
 /// }
 /// ```
 unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char {
-    let ret: *mut libc::c_char = if !s.is_null() {
-        libc::strdup(s)
+    let ret: *mut libc::c_char;
+    if !s.is_null() {
+        ret = libc::strdup(s);
+        assert!(!ret.is_null());
     } else {
-        libc::calloc(1, 1) as *mut libc::c_char
-    };
-    assert!(!ret.is_null());
+        ret = libc::calloc(1, 1) as *mut libc::c_char;
+        assert!(!ret.is_null());
+    }
+
     ret
 }
 
@@ -167,20 +170,15 @@ pub(crate) trait Strdup {
     unsafe fn strdup(&self) -> *mut libc::c_char;
 }
 
-impl Strdup for str {
+impl<T: AsRef<str>> Strdup for T {
     unsafe fn strdup(&self) -> *mut libc::c_char {
-        let tmp = CString::new_lossy(self);
+        let tmp = CString::new_lossy(self.as_ref());
         dc_strdup(tmp.as_ptr())
     }
 }
 
-impl Strdup for String {
-    unsafe fn strdup(&self) -> *mut libc::c_char {
-        let s: &str = self;
-        s.strdup()
-    }
-}
-
+// We can not implement for AsRef<OsStr> because we already implement
+// AsRev<str> and this conflicts.  So implement for Path directly.
 impl Strdup for std::path::Path {
     unsafe fn strdup(&self) -> *mut libc::c_char {
         let tmp = self.to_c_string().unwrap_or_else(|_| CString::default());
@@ -188,13 +186,6 @@ impl Strdup for std::path::Path {
     }
 }
 
-impl Strdup for [u8] {
-    unsafe fn strdup(&self) -> *mut libc::c_char {
-        let tmp = CString::new_lossy(self);
-        dc_strdup(tmp.as_ptr())
-    }
-}
-
 /// Convenience methods to turn optional strings into C strings.
 ///
 /// This is the same as the [Strdup] trait but a different trait name

deltachat_derive/Cargo.toml

@@ -9,5 +9,5 @@ license = "MPL-2.0"
 proc-macro = true
 
 [dependencies]
-syn = "1.0.76"
+syn = "1.0.73"
 quote = "1.0.2"

examples/repl/cmdline.rs

@@ -13,11 +13,11 @@ use deltachat::contact::*;
 use deltachat::context::*;
 use deltachat::dc_receive_imf::*;
 use deltachat::dc_tools::*;
-use deltachat::download::DownloadState;
 use deltachat::imex::*;
 use deltachat::location;
 use deltachat::log::LogExt;
-use deltachat::message::{self, Message, MessageState, MsgId};
+use deltachat::lot::LotState;
+use deltachat::message::{self, ContactRequestDecision, Message, MessageState, MsgId};
 use deltachat::peerstate::*;
 use deltachat::qr::*;
 use deltachat::sql;
@@ -189,18 +189,10 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
         MessageState::OutFailed => " !!",
         _ => "",
     };
-
-    let downloadstate = match msg.download_state() {
-        DownloadState::Done => "",
-        DownloadState::Available => " [⬇ Download available]",
-        DownloadState::InProgress => " [⬇ Download in progress...]️",
-        DownloadState::Failure => " [⬇ Download failed]",
-    };
-
     let temp2 = dc_timestamp_to_str(msg.get_timestamp());
     let msgtext = msg.get_text();
     println!(
-        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{}{}{} [{}]",
+        "{}{}{}{}: {} (Contact#{}): {} {}{}{}{}{}{} [{}]",
         prefix.as_ref(),
         msg.get_id(),
         if msg.get_showpadlock() { "🔒" } else { "" },
@@ -234,7 +226,6 @@ async fn log_msg(context: &Context, prefix: impl AsRef<str>, msg: &Message) {
             ""
         },
         statestr,
-        downloadstate,
         &temp2,
     );
 }
@@ -298,7 +289,7 @@ async fn log_contactlist(context: &Context, contacts: &[u32]) {
                     "addr unset"
                 }
             );
-            let peerstate = Peerstate::from_addr(context, addr)
+            let peerstate = Peerstate::from_addr(context, &addr)
                 .await
                 .expect("peerstate error");
             if peerstate.is_some() && *contact_id != 1 {
@@ -360,7 +351,6 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  configure\n\
                  connect\n\
                  disconnect\n\
-                 connectivity\n\
                  maybenetwork\n\
                  housekeeping\n\
                  help imex (Import/Export)\n\
@@ -398,12 +388,13 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                  protect <chat-id>\n\
                  unprotect <chat-id>\n\
                  delchat <chat-id>\n\
-                 accept <chat-id>\n\
-                 decline <chat-id>\n\
+                 ===========================Contact requests==\n\
+                 decidestartchat <msg-id>\n\
+                 decideblock <msg-id>\n\
+                 decidenotnow <msg-id>\n\
                  ===========================Message commands==\n\
                  listmsgs <query>\n\
                  msginfo <msg-id>\n\
-                 download <msg-id>\n\
                  html <msg-id>\n\
                  listfresh\n\
                  forward <msg-id> <chat-id>\n\
@@ -460,7 +451,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 !arg1.is_empty() && !arg2.is_empty(),
                 "Arguments <msg-id> <setup-code> expected"
             );
-            continue_key_transfer(&context, MsgId::new(arg1.parse()?), arg2).await?;
+            continue_key_transfer(&context, MsgId::new(arg1.parse()?), &arg2).await?;
         }
         "has-backup" => {
             has_backup(&context, blobdir).await?;
@@ -507,33 +498,19 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "set" => {
             ensure!(!arg1.is_empty(), "Argument <key> missing.");
-            let key = config::Config::from_str(arg1)?;
+            let key = config::Config::from_str(&arg1)?;
             let value = if arg2.is_empty() { None } else { Some(arg2) };
             context.set_config(key, value).await?;
         }
         "get" => {
             ensure!(!arg1.is_empty(), "Argument <key> missing.");
-            let key = config::Config::from_str(arg1)?;
+            let key = config::Config::from_str(&arg1)?;
             let val = context.get_config(key).await;
             println!("{}={:?}", key, val);
         }
         "info" => {
             println!("{:#?}", context.get_info().await);
         }
-        "connectivity" => {
-            let file = dirs::home_dir()
-                .unwrap_or_default()
-                .join("connectivity.html");
-            match context.get_connectivity_html().await {
-                Ok(html) => {
-                    fs::write(&file, html)?;
-                    println!("Report written to: {:#?}", file);
-                }
-                Err(err) => {
-                    bail!("Failed to get connectivity html: {}", err);
-                }
-            }
-        }
         "maybenetwork" => {
             context.maybe_network().await;
         }
@@ -561,7 +538,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 for i in (0..cnt).rev() {
                     let chat = Chat::load_from_db(&context, chatlist.get_chat_id(i)).await?;
                     println!(
-                        "{}#{}: {} [{} fresh] {}{}{}{}",
+                        "{}#{}: {} [{} fresh] {}{}{}",
                         chat_prefix(&chat),
                         chat.get_id(),
                         chat.get_name(),
@@ -573,31 +550,27 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                             ChatVisibility::Pinned => "📌",
                         },
                         if chat.is_protected() { "🛡️" } else { "" },
-                        if chat.is_contact_request() {
-                            "🆕"
-                        } else {
-                            ""
-                        },
                     );
-                    let summary = chatlist.get_summary(&context, i, Some(&chat)).await?;
+                    let lot = chatlist.get_summary(&context, i, Some(&chat)).await;
                     let statestr = if chat.visibility == ChatVisibility::Archived {
                         " [Archived]"
                     } else {
-                        match summary.state {
-                            MessageState::OutPending => " o",
-                            MessageState::OutDelivered => " √",
-                            MessageState::OutMdnRcvd => " √√",
-                            MessageState::OutFailed => " !!",
+                        match lot.get_state() {
+                            LotState::MsgOutPending => " o",
+                            LotState::MsgOutDelivered => " √",
+                            LotState::MsgOutMdnRcvd => " √√",
+                            LotState::MsgOutFailed => " !!",
                             _ => "",
                         }
                     };
-                    let timestr = dc_timestamp_to_str(summary.timestamp);
+                    let timestr = dc_timestamp_to_str(lot.get_timestamp());
+                    let text1 = lot.get_text1();
+                    let text2 = lot.get_text2();
                     println!(
-                        "{}{}{} [{}]{}",
-                        summary
-                            .prefix
-                            .map_or_else(String::new, |prefix| format!("{}: ", prefix)),
-                        summary.text,
+                        "{}{}{}{} [{}]{}",
+                        text1.unwrap_or(""),
+                        if text1.is_some() { ": " } else { "" },
+                        text2.unwrap_or(""),
                         statestr,
                         &timestr,
                         if chat.is_sending_locations() {
@@ -611,7 +584,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     );
                 }
             }
-            if location::is_sending_locations_to_chat(&context, None).await? {
+            if location::is_sending_locations_to_chat(&context, None).await {
                 println!("Location streaming enabled.");
             }
             println!("{} chats", cnt);
@@ -707,6 +680,35 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
 
             println!("Single#{} created successfully.", chat_id,);
         }
+        "decidestartchat" | "createchatbymsg" => {
+            ensure!(!arg1.is_empty(), "Argument <msg-id> missing");
+            let msg_id = MsgId::new(arg1.parse()?);
+            match message::decide_on_contact_request(
+                &context,
+                msg_id,
+                ContactRequestDecision::StartChat,
+            )
+            .await
+            {
+                Some(chat_id) => {
+                    let chat = Chat::load_from_db(&context, chat_id).await?;
+                    println!("{}#{} created successfully.", chat_prefix(&chat), chat_id);
+                }
+                None => println!("Cannot crate chat."),
+            }
+        }
+        "decidenotnow" => {
+            ensure!(!arg1.is_empty(), "Argument <msg-id> missing");
+            let msg_id = MsgId::new(arg1.parse()?);
+            message::decide_on_contact_request(&context, msg_id, ContactRequestDecision::NotNow)
+                .await;
+        }
+        "decideblock" => {
+            ensure!(!arg1.is_empty(), "Argument <msg-id> missing");
+            let msg_id = MsgId::new(arg1.parse()?);
+            message::decide_on_contact_request(&context, msg_id, ContactRequestDecision::Block)
+                .await;
+        }
         "creategroup" => {
             ensure!(!arg1.is_empty(), "Argument <name> missing.");
             let chat_id =
@@ -782,7 +784,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                     &context,
                     Some(sel_chat.as_ref().unwrap().get_id())
                 )
-                .await?,
+                .await,
             );
         }
         "getlocations" => {
@@ -827,7 +829,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
                 sel_chat.as_ref().unwrap().get_id(),
                 seconds,
             )
-            .await?;
+            .await;
             println!(
                 "Locations will be sent to Chat#{} for {} seconds. Use 'setlocation <lat> <lng>' to play around.",
                 sel_chat.as_ref().unwrap().get_id(),
@@ -904,7 +906,12 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         "listmsgs" => {
             ensure!(!arg1.is_empty(), "Argument <query> missing.");
 
-            let chat = sel_chat.as_ref().map(|sel_chat| sel_chat.get_id());
+            let chat = if let Some(ref sel_chat) = sel_chat {
+                Some(sel_chat.get_id())
+            } else {
+                None
+            };
+
             let time_start = std::time::SystemTime::now();
             let msglist = context.search_msgs(chat, arg1).await?;
             let time_needed = time_start.elapsed().unwrap_or_default();
@@ -1018,28 +1025,12 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             let chat_id = ChatId::new(arg1.parse()?);
             chat_id.delete(&context).await?;
         }
-        "accept" => {
-            ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
-            let chat_id = ChatId::new(arg1.parse()?);
-            chat_id.accept(&context).await?;
-        }
-        "blockchat" => {
-            ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
-            let chat_id = ChatId::new(arg1.parse()?);
-            chat_id.block(&context).await?;
-        }
         "msginfo" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let id = MsgId::new(arg1.parse()?);
             let res = message::get_msg_info(&context, id).await?;
             println!("{}", res);
         }
-        "download" => {
-            ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
-            let id = MsgId::new(arg1.parse()?);
-            println!("Scheduling download for {:?}", id);
-            id.download_full(&context).await?;
-        }
         "html" => {
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let id = MsgId::new(arg1.parse()?);
@@ -1077,7 +1068,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             ensure!(!arg1.is_empty(), "Argument <msg-id> missing.");
             let mut ids = [MsgId::new(0); 1];
             ids[0] = MsgId::new(arg1.parse()?);
-            message::delete_msgs(&context, &ids).await?;
+            message::delete_msgs(&context, &ids).await;
         }
         "listcontacts" | "contacts" | "listverified" => {
             let contacts = Contact::get_all(
@@ -1178,10 +1169,7 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
         }
         "providerinfo" => {
             ensure!(!arg1.is_empty(), "Argument <addr> missing.");
-            let socks5_enabled = context
-                .get_config_bool(config::Config::Socks5Enabled)
-                .await?;
-            match provider::get_provider_info(arg1, socks5_enabled).await {
+            match provider::get_provider_info(arg1).await {
                 Some(info) => {
                     println!("Information for provider belonging to {}:", arg1);
                     println!("status: {}", info.status as u32);

examples/repl/main.rs

@@ -57,6 +57,9 @@ fn receive_event(event: EventType) {
         EventType::Error(msg) => {
             error!("{}", msg);
         }
+        EventType::ErrorNetwork(msg) => {
+            error!("[NETWORK] msg={}", msg);
+        }
         EventType::ErrorSelfNotInGroup(msg) => {
             error!("[SELF_NOT_IN_GROUP] {}", msg);
         }
@@ -154,7 +157,7 @@ const IMEX_COMMANDS: [&str; 12] = [
     "stop",
 ];
 
-const DB_COMMANDS: [&str; 10] = [
+const DB_COMMANDS: [&str; 9] = [
     "info",
     "set",
     "get",
@@ -162,16 +165,18 @@ const DB_COMMANDS: [&str; 10] = [
     "configure",
     "connect",
     "disconnect",
-    "connectivity",
     "maybenetwork",
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 33] = [
+const CHAT_COMMANDS: [&str; 34] = [
     "listchats",
     "listarchived",
     "chat",
     "createchat",
+    "decidestartchat",
+    "decideblock",
+    "decidenotnow",
     "creategroup",
     "createverified",
     "addmember",
@@ -199,17 +204,14 @@ const CHAT_COMMANDS: [&str; 33] = [
     "protect",
     "unprotect",
     "delchat",
-    "accept",
-    "blockchat",
 ];
-const MESSAGE_COMMANDS: [&str; 7] = [
+const MESSAGE_COMMANDS: [&str; 6] = [
     "listmsgs",
     "msginfo",
     "listfresh",
     "forward",
     "markseen",
     "delmsg",
-    "download",
 ];
 const CONTACT_COMMANDS: [&str; 9] = [
     "listcontacts",
@@ -326,7 +328,7 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
 
         loop {
             let p = "> ";
-            let readline = rl.readline(p);
+            let readline = rl.readline(&p);
 
             match readline {
                 Ok(line) => {
@@ -393,7 +395,7 @@ async fn handle_cmd(
         "oauth2" => {
             if let Some(addr) = ctx.get_config(config::Config::Addr).await? {
                 let oauth2_url =
-                    dc_get_oauth2_url(&ctx, &addr, "chat.delta:/com.b44t.messenger").await?;
+                    dc_get_oauth2_url(&ctx, &addr, "chat.delta:/com.b44t.messenger").await;
                 if oauth2_url.is_none() {
                     println!("OAuth2 not available for {}.", &addr);
                 } else {
@@ -409,7 +411,7 @@ async fn handle_cmd(
         }
         "getqr" | "getbadqr" => {
             ctx.start_io().await;
-            let group = arg1.parse::<u32>().ok().map(ChatId::new);
+            let group = arg1.parse::<u32>().ok().map(|id| ChatId::new(id));
             if let Some(mut qr) = dc_get_securejoin_qr(&ctx, group).await {
                 if !qr.is_empty() {
                     if arg0 == "getbadqr" && qr.len() > 40 {

examples/simple.rs

@@ -19,7 +19,7 @@ fn cb(event: EventType) {
         EventType::Warning(msg) => {
             log::warn!("{}", msg);
         }
-        EventType::Error(msg) => {
+        EventType::Error(msg) | EventType::ErrorNetwork(msg) => {
             log::error!("{}", msg);
         }
         event => {
@@ -86,7 +86,7 @@ async fn main() {
     let chats = Chatlist::try_load(&ctx, 0, None, None).await.unwrap();
 
     for i in 0..chats.len() {
-        let msg = Message::load_from_db(&ctx, chats.get_msg_id(i).unwrap().unwrap())
+        let msg = Message::load_from_db(&ctx, chats.get_msg_id(i).unwrap())
             .await
             .unwrap();
         log::info!("[{}] msg: {:?}", i, msg);

python/README.rst

@@ -58,13 +58,12 @@ end-to-end tests that require accounts on real e-mail servers.
 running "live" tests with temporary accounts
 ---------------------------------------------
 
-If you want to run live functional tests you can set ``DCC_NEW_TMP_EMAIL`` to a URL that creates e-mail accounts.  Most developers use https://testrun.org URLS created and managed by [mailadm](https://mailadm.readthedocs.io/en/latest/).
+If you want to run live functional tests you can set ``DCC_NEW_TMP_EMAIL``::
 
-Please feel free to contact us through a github issue or by e-mail and we'll send you a URL that you can then use for functional tests like this:
+    export DCC_NEW_TMP_EMAIL=https://testrun.org/new_email?t=1h_4w4r8h7y9nmcdsy
 
-    export DCC_NEW_TMP_EMAIL=<URL you got from us>
-
-With this account-creation setting, pytest runs create ephemeral e-mail accounts on the http://testrun.org server.  These accounts exists only for one hour and then are removed completely.
+With this, pytest runs create ephemeral e-mail accounts on the http://testrun.org server.
+These accounts exists for one 1hour and then are removed completely.
 One hour is enough to invoke pytest and run all offline and online tests:
 
     pytest

python/src/deltachat/_build.py

@@ -150,7 +150,6 @@ def extract_defines(flags):
                 | DC_PROVIDER
                 | DC_KEY_GEN
                 | DC_IMEX
-                | DC_CONNECTIVITY
             )         # End of prefix matching
             _[\w_]+   # Match the suffix, e.g. _RSA2048 in DC_KEY_GEN_RSA2048
         )             # Close the capturing group, this contains

python/src/deltachat/account.py

@@ -330,6 +330,9 @@ class Account(object):
         """ Create a 1:1 chat with Account, Contact or e-mail address. """
         return self.create_contact(obj).create_chat()
 
+    def _create_chat_by_message_id(self, msg_id):
+        return Chat(self, lib.dc_create_chat_by_msg_id(self._dc_context, msg_id))
+
     def create_group_chat(self, name, contacts=None, verified=False):
         """ create a new group chat object.
 
@@ -364,6 +367,9 @@ class Account(object):
             chatlist.append(Chat(self, chat_id))
         return chatlist
 
+    def get_deaddrop_chat(self):
+        return Chat(self, const.DC_CHAT_ID_DEADDROP)
+
     def get_device_chat(self):
         return Contact(self, const.DC_CONTACT_ID_DEVICE).create_chat()
 
@@ -568,15 +574,6 @@ class Account(object):
         """ Stop ongoing securejoin, configuration or other core jobs. """
         lib.dc_stop_ongoing_process(self._dc_context)
 
-    def get_connectivity(self):
-        return lib.dc_get_connectivity(self._dc_context)
-
-    def get_connectivity_html(self):
-        return from_dc_charpointer(lib.dc_get_connectivity_html(self._dc_context))
-
-    def all_work_done(self):
-        return lib.dc_all_work_done(self._dc_context)
-
     def start_io(self):
         """ start this account's IO scheduling (Rust-core async scheduler)
 

python/src/deltachat/chat.py

@@ -50,14 +50,6 @@ class Chat(object):
         """
         lib.dc_delete_chat(self.account._dc_context, self.id)
 
-    def block(self):
-        """Block this chat."""
-        lib.dc_block_chat(self.account._dc_context, self.id)
-
-    def accept(self):
-        """Accept this contact request chat."""
-        lib.dc_accept_chat(self.account._dc_context, self.id)
-
     # ------  chat status/metadata API ------------------------------
 
     def is_group(self):
@@ -67,6 +59,13 @@ class Chat(object):
         """
         return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_GROUP
 
+    def is_deaddrop(self):
+        """ return true if this chat is a deaddrop chat.
+
+        :returns: True if chat is the deaddrop chat, False otherwise.
+        """
+        return self.id == const.DC_CHAT_ID_DEADDROP
+
     def is_muted(self):
         """ return true if this chat is muted.
 
@@ -74,13 +73,6 @@ class Chat(object):
         """
         return lib.dc_chat_is_muted(self._dc_chat)
 
-    def is_contact_request(self):
-        """ return True if this chat is a contact request chat.
-
-        :returns: True if chat is a contact request chat, False otherwise.
-        """
-        return lib.dc_chat_is_contact_request(self._dc_chat)
-
     def is_promoted(self):
         """ return True if this chat is promoted, i.e.
         the member contacts are aware of their membership,
@@ -92,7 +84,7 @@ class Chat(object):
 
     def can_send(self):
         """Check if messages can be sent to a give chat.
-        This is not true eg. for the contact requests or for the device-talk
+        This is not true eg. for the deaddrop or for the device-talk
 
         :returns: True if the chat is writable, False otherwise
         """

python/src/deltachat/events.py

@@ -111,33 +111,6 @@ class FFIEventTracker:
             if m is not None:
                 return m.groups()
 
-    def wait_for_connectivity(self, connectivity):
-        """Wait for the specified connectivity.
-        This only works reliably if the connectivity doesn't change
-        again too quickly, otherwise we might miss it."""
-        while 1:
-            if self.account.get_connectivity() == connectivity:
-                return
-            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
-    def wait_for_connectivity_change(self, previous, expected_next):
-        """Wait until the connectivity changes to `expected_next`.
-        Fails the test if it changes to something else."""
-        while 1:
-            current = self.account.get_connectivity()
-            if current == expected_next:
-                return
-            elif current != previous:
-                raise Exception("Expected connectivity " + str(expected_next) + " but got " + str(current))
-
-            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
-    def wait_for_all_work_done(self):
-        while 1:
-            if self.account.all_work_done():
-                return
-            self.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
     def ensure_event_not_queued(self, event_name_regex):
         __tracebackhide__ = True
         rex = re.compile("(?:{}).*".format(event_name_regex))

python/src/deltachat/hookspec.py

@@ -43,7 +43,7 @@ class PerAccount:
 
     @account_hookspec
     def ac_incoming_message(self, message):
-        """ Called on any incoming message (both existing chats and contact requests). """
+        """ Called on any incoming message (to deaddrop or chat). """
 
     @account_hookspec
     def ac_outgoing_message(self, message):

python/src/deltachat/message.py

@@ -61,13 +61,16 @@ class Message(object):
     def create_chat(self):
         """ create or get an existing chat (group) object for this message.
 
-        If the message is a contact request
+        If the message is a deaddrop contact request
         the sender will become an accepted contact.
 
         :returns: a :class:`deltachat.chat.Chat` object.
         """
-        self.chat.accept()
-        return self.chat
+        from .chat import Chat
+        chat_id = lib.dc_create_chat_by_msg_id(self.account._dc_context, self.id)
+        ctx = self.account._dc_context
+        self._dc_msg = ffi.gc(lib.dc_get_msg(ctx, self.id), lib.dc_msg_unref)
+        return Chat(self.account, chat_id)
 
     @props.with_doc
     def id(self):
@@ -138,10 +141,6 @@ class Message(object):
         """ return True if this message was encrypted. """
         return bool(lib.dc_msg_get_showpadlock(self._dc_msg))
 
-    def is_bot(self):
-        """ return True if this message is submitted automatically. """
-        return bool(lib.dc_msg_is_bot(self._dc_msg))
-
     def is_forwarded(self):
         """ return True if this message was forwarded. """
         return bool(lib.dc_msg_is_forwarded(self._dc_msg))

python/tests/test_account.py

@@ -909,11 +909,12 @@ class TestOnlineAccount:
         msg_in = ac2.get_message_by_id(msg_out.id)
         assert msg_in.text == "message2"
 
-        lp.sec("ac2: check that the message arrived in a chat")
+        lp.sec("ac2: check that the message arrive in deaddrop")
         chat2 = msg_in.chat
         assert msg_in in chat2.get_messages()
         assert not msg_in.is_forwarded()
-        assert chat2.is_contact_request()
+        assert chat2.is_deaddrop()
+        assert chat2 == ac2.get_deaddrop_chat()
 
         lp.sec("ac2: create new chat and forward message to it")
         chat3 = ac2.create_group_chat("newgroup")
@@ -978,16 +979,16 @@ class TestOnlineAccount:
         assert not msg2.is_forwarded()
         assert msg2.get_sender_contact().display_name == ac1.get_config("displayname")
 
-        lp.sec("check the message arrived in contact request chat")
+        lp.sec("check the message arrived in contact-requests/deaddrop")
         chat2 = msg2.chat
         assert msg2 in chat2.get_messages()
-        assert chat2.is_contact_request()
+        assert chat2.is_deaddrop()
         assert chat2.count_fresh_messages() == 1
         assert msg2.time_received >= msg1.time_sent
 
         lp.sec("create new chat with contact and verify it's proper")
         chat2b = msg2.create_chat()
-        assert not chat2b.is_contact_request()
+        assert not chat2b.is_deaddrop()
         assert chat2b.count_fresh_messages() == 1
 
         lp.sec("mark chat as noticed")
@@ -1429,33 +1430,6 @@ class TestOnlineAccount:
         # Majority prefers encryption now
         assert msg5.is_encrypted()
 
-    def test_bot(self, acfactory, lp):
-        """Test that bot messages can be identified as such"""
-        ac1, ac2 = acfactory.get_two_online_accounts()
-        ac1.set_config("bot", "0")
-        ac2.set_config("bot", "1")
-
-        lp.sec("ac1: create chat with ac2")
-        chat = acfactory.get_accepted_chat(ac1, ac2)
-
-        lp.sec("sending a message from ac1 to ac2")
-        text1 = "hello"
-        chat.send_text(text1)
-
-        lp.sec("wait for ac2 to receive a message")
-        msg_in = ac2._evtracker.wait_next_incoming_message()
-        assert msg_in.text == text1
-        assert not msg_in.is_bot()
-
-        lp.sec("sending a message from ac2 to ac1")
-        text2 = "reply"
-        msg_in.chat.send_text(text2)
-
-        lp.sec("wait for ac1 to receive a message")
-        msg_in = ac1._evtracker.wait_next_incoming_message()
-        assert msg_in.text == text2
-        assert msg_in.is_bot()
-
     def test_quote_encrypted(self, acfactory, lp):
         """Test that replies to encrypted messages with quotes are encrypted."""
         ac1, ac2 = acfactory.get_two_online_accounts()
@@ -1852,7 +1826,7 @@ class TestOnlineAccount:
 
         lp.sec("ac2: wait for receiving message and avatar from ac1")
         msg2 = ac2._evtracker.wait_next_messages_changed()
-        assert msg2.chat.is_contact_request()
+        assert msg2.chat.is_deaddrop()
         received_path = msg2.get_sender_contact().get_profile_image()
         assert open(received_path, "rb").read() == open(p, "rb").read()
 
@@ -1927,8 +1901,6 @@ class TestOnlineAccount:
         ev = in_list.get(timeout=10)
         assert ev.action == "chat-modified"
         ev = in_list.get(timeout=10)
-        assert ev.action == "chat-modified"
-        ev = in_list.get(timeout=10)
         assert ev.action == "added"
         assert ev.message.get_sender_contact().addr == ac1_addr
         assert ev.contact.addr == "devnull@testrun.org"
@@ -2043,84 +2015,6 @@ class TestOnlineAccount:
         assert msg_back.chat == chat
         assert chat.get_profile_image() is None
 
-    @pytest.mark.parametrize("inbox_watch", ["0", "1"])
-    def test_connectivity(self, acfactory, lp, inbox_watch):
-        ac1, ac2 = acfactory.get_two_online_accounts()
-        ac1.set_config("inbox_watch", inbox_watch)
-        ac1.set_config("scan_all_folders_debounce_secs", "0")
-
-        ac1._evtracker.wait_for_connectivity(const.DC_CONNECTIVITY_CONNECTED)
-
-        lp.sec("Test stop_io() and start_io()")
-        ac1.stop_io()
-        ac1._evtracker.wait_for_connectivity(const.DC_CONNECTIVITY_NOT_CONNECTED)
-
-        ac1.start_io()
-        ac1._evtracker.wait_for_connectivity(const.DC_CONNECTIVITY_CONNECTING)
-        ac1._evtracker.wait_for_connectivity_change(const.DC_CONNECTIVITY_CONNECTING, const.DC_CONNECTIVITY_CONNECTED)
-
-        lp.sec("Test that after calling start_io(), maybe_network() and waiting for `all_work_done()`, " +
-               "all messages are fetched")
-
-        ac1.direct_imap.select_config_folder("inbox")
-        ac1.direct_imap.idle_start()
-        ac2.create_chat(ac1).send_text("Hi")
-
-        ac1.direct_imap.idle_check(terminate=False)
-        ac1.maybe_network()
-
-        ac1._evtracker.wait_for_all_work_done()
-        msgs = ac1.create_chat(ac2).get_messages()
-        assert len(msgs) == 1
-        assert msgs[0].text == "Hi"
-
-        lp.sec("Test that the connectivity changes to WORKING while new messages are fetched")
-
-        ac2.create_chat(ac1).send_text("Hi 2")
-
-        ac1.direct_imap.idle_check(terminate=True)
-        ac1.maybe_network()
-        ac1._evtracker.wait_for_connectivity_change(const.DC_CONNECTIVITY_CONNECTED, const.DC_CONNECTIVITY_WORKING)
-        ac1._evtracker.wait_for_connectivity_change(const.DC_CONNECTIVITY_WORKING, const.DC_CONNECTIVITY_CONNECTED)
-
-        msgs = ac1.create_chat(ac2).get_messages()
-        assert len(msgs) == 2
-        assert msgs[1].text == "Hi 2"
-
-        lp.sec("Test that the connectivity doesn't flicker to WORKING if there are no new messages")
-
-        ac1.maybe_network()
-        while 1:
-            assert ac1.get_connectivity() == const.DC_CONNECTIVITY_CONNECTED
-            if ac1.all_work_done():
-                break
-            ac1._evtracker.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
-        lp.sec("Test that the connectivity doesn't flicker to WORKING if the sender of the message is blocked")
-        ac1.create_contact(ac2).block()
-
-        ac1.direct_imap.select_config_folder("inbox")
-        ac1.direct_imap.idle_start()
-        ac2.create_chat(ac1).send_text("Hi")
-
-        ac1.direct_imap.idle_check(terminate=True)
-        ac1.maybe_network()
-
-        while 1:
-            assert ac1.get_connectivity() == const.DC_CONNECTIVITY_CONNECTED
-            if ac1.all_work_done():
-                break
-            ac1._evtracker.get_matching("DC_EVENT_CONNECTIVITY_CHANGED")
-
-        lp.sec("Test that the connectivity is NOT_CONNECTED if the password is wrong")
-
-        ac1.set_config("configured_mail_pw", "abc")
-        ac1.stop_io()
-        ac1._evtracker.wait_for_connectivity(const.DC_CONNECTIVITY_NOT_CONNECTED)
-        ac1.start_io()
-        ac1._evtracker.wait_for_connectivity(const.DC_CONNECTIVITY_CONNECTING)
-        ac1._evtracker.wait_for_connectivity(const.DC_CONNECTIVITY_NOT_CONNECTED)
-
     def test_fetch_deleted_msg(self, acfactory, lp):
         """This is a regression test: Messages with \\Deleted flag were downloaded again and again,
         hundreds of times, because uid_next was not updated.
@@ -2837,6 +2731,7 @@ class TestOnlineConfigureFails:
         configtracker = ac1.configure()
         configtracker.wait_progress(500)
         configtracker.wait_progress(0)
+        ac1._evtracker.ensure_event_not_queued("DC_EVENT_ERROR_NETWORK")
 
     def test_invalid_user(self, acfactory):
         ac1, configdict = acfactory.get_online_config()
@@ -2844,6 +2739,7 @@ class TestOnlineConfigureFails:
         configtracker = ac1.configure()
         configtracker.wait_progress(500)
         configtracker.wait_progress(0)
+        ac1._evtracker.ensure_event_not_queued("DC_EVENT_ERROR_NETWORK")
 
     def test_invalid_domain(self, acfactory):
         ac1, configdict = acfactory.get_online_config()
@@ -2851,3 +2747,4 @@ class TestOnlineConfigureFails:
         configtracker = ac1.configure()
         configtracker.wait_progress(500)
         configtracker.wait_progress(0)
+        ac1._evtracker.ensure_event_not_queued("DC_EVENT_ERROR_NETWORK")

rust-toolchain

@@ -1 +1 @@
-1.54.0
+1.50.0

scripts/README.md

@@ -1,13 +1,16 @@
 # Continuous Integration Scripts for Delta Chat
 
 Continuous Integration, run through [GitHub
-Actions](https://docs.github.com/actions)
-and an own build machine.
+Actions](https://docs.github.com/actions),
+[CircleCI](https://app.circleci.com/) and an own build machine.
 
 ## Description of scripts 
 
 - `../.github/workflows` contains jobs run by GitHub Actions.
 
+- `../.circleci/config.yml` describing the build jobs that are run
+  by CircleCI.
+
 - `remote_tests_python.sh` rsyncs to a build machine and runs
   `run-python-test.sh` remotely on the build machine. 
 

scripts/ci_upload.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+if [ -z "$DEVPI_LOGIN" ] ; then 
+    echo "required: password for 'dc' user on https://m.devpi/net/dc index"
+    exit 0
+fi
+
+set -xe
+
+PYDOCDIR=${1:?directory with python docs}
+WHEELHOUSEDIR=${2:?directory with pre-built wheels}
+DOXYDOCDIR=${3:?directory where doxygen docs to be found}
+SSHTARGET=ci@b1.delta.chat
+
+
+# if CIRCLE_BRANCH is not set we are called for a tag with empty CIRCLE_BRANCH variable. 
+export BRANCH=${CIRCLE_BRANCH:master}
+
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}/wheelhouse
+
+
+# python docs to py.delta.chat
+ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@py.delta.chat mkdir -p build/${BRANCH}
+rsync -avz \
+  --delete \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  "$PYDOCDIR/html/" \
+  delta@py.delta.chat:build/${BRANCH}
+
+# C docs to c.delta.chat
+ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null delta@c.delta.chat mkdir -p build-c/${BRANCH}
+rsync -avz \
+  --delete \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  "$DOXYDOCDIR/html/" \
+  delta@c.delta.chat:build-c/${BRANCH}
+
+echo -----------------------
+echo upload wheels 
+echo -----------------------
+
+# Bundle external shared libraries into the wheels
+
+ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null $SSHTARGET mkdir -p $BUILDDIR 
+scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null scripts/cleanup_devpi_indices.py $SSHTARGET:$BUILDDIR 
+rsync -avz \
+  -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" \
+  $WHEELHOUSEDIR \
+  $SSHTARGET:$BUILDDIR 
+
+ssh $SSHTARGET <<_HERE
+    set +x -e
+    # make sure all processes exit when ssh dies
+    shopt -s huponexit
+
+    # we rely on the "venv" virtualenv on the remote account to exist 
+    source venv/bin/activate
+    cd $BUILDDIR
+
+    devpi use https://m.devpi.net
+    devpi login dc --password $DEVPI_LOGIN
+
+    N_BRANCH=${BRANCH//[\/]}
+
+    devpi use dc/\$N_BRANCH || {
+        devpi index -c \$N_BRANCH 
+        devpi use dc/\$N_BRANCH
+    }
+    devpi index \$N_BRANCH bases=/root/pypi
+    devpi upload wheelhouse/deltachat*
+
+    # remove devpi non-master dc indices if thy are too old
+    # this script was copied above 
+    python cleanup_devpi_indices.py
+_HERE

scripts/concourse/README.md

@@ -1,21 +0,0 @@
-# Concourse CI pipeline
-
-`docs_wheels.yml` is a pipeline for [Concourse CI](https://concourse-ci.org/)
-that builds C documentation, Python documentation, Python wheels for `x86_64`
-and `aarch64` and Python source packages, and uploads them.
-
-To setup the pipeline run
-```
-fly -t <your-target> set-pipeline -c docs_wheels.yml -p docs_wheels -l secret.yml
-```
-where `secret.yml` contains the following secrets:
-```
-c.delta.chat:
-  private_key: |
-    -----BEGIN RSA PRIVATE KEY-----
-    ...
-    -----END RSA PRIVATE KEY-----
-devpi:
-  login: dc
-  password: ...
-```

scripts/concourse/docs_wheels.yml

@@ -1,232 +0,0 @@
-resources:
-- name: deltachat-core-rust
-  type: git
-  icon: github
-  source:
-    branch: master
-    uri: https://github.com/deltachat/deltachat-core-rust.git
-
-- name: deltachat-core-rust-release
-  type: git
-  icon: github
-  source:
-    branch: master
-    uri: https://github.com/deltachat/deltachat-core-rust.git
-    tag_filter: "py-*"
-
-jobs:
-- name: doxygen
-  plan:
-  - get: deltachat-core-rust
-    trigger: true
-
-  # Build Doxygen documentation
-  - task: build-doxygen
-    config:
-      inputs:
-        - name: deltachat-core-rust
-      outputs:
-        - name: c-docs
-      image_resource:
-        source:
-          repository: hrektts/doxygen
-        type: registry-image
-      platform: linux
-      run:
-        path: bash
-        args:
-        - -exc
-        - |
-          cd deltachat-core-rust
-          bash scripts/run-doxygen.sh
-          cd ..
-          cp -av deltachat-core-rust/deltachat-ffi/{html,xml} c-docs/
-
-  - task: upload-c-docs
-    config:
-      inputs:
-        - name: c-docs
-      image_resource:
-        type: registry-image
-        source:
-          repository: alpine
-      platform: linux
-      run:
-        path: sh
-        args:
-        - -ec
-        - |
-          apk add --no-cache rsync openssh-client
-          mkdir -p ~/.ssh
-          chmod 700 ~/.ssh
-          echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519
-          chmod 600 ~/.ssh/id_ed25519
-          rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete c-docs/html/ delta@c.delta.chat:build-c/master
-
-- name: python-x86_64
-  plan:
-  - get: deltachat-core-rust
-  - get: deltachat-core-rust-release
-    trigger: true
-
-  # Build manylinux image with additional dependencies
-  - task: build-coredeps
-    privileged: true
-    config:
-      inputs:
-        # Building the latest, not tagged coredeps
-        - name: deltachat-core-rust
-      image_resource:
-        source:
-          repository: vito/oci-build-task
-        type: registry-image
-      outputs:
-        - name: coredeps-image
-          path: image
-      params:
-        CONTEXT: deltachat-core-rust/scripts/docker-coredeps
-        UNPACK_ROOTFS: "true"
-      platform: linux
-      caches:
-      - path: cache
-      run:
-        path: build
-
-  # Use built image to build python wheels
-  - task: build-wheels
-    image: coredeps-image
-    config:
-      inputs:
-        - name: deltachat-core-rust-release
-          path: .
-      outputs:
-        - name: py-docs
-          path: ./python/doc/_build/
-        # Source packages
-        - name: py-dist
-          path: ./python/.docker-tox/dist/
-        # Binary wheels
-        - name: py-wheels
-          path: ./python/.docker-tox/wheelhouse/
-      platform: linux
-      run:
-        path: bash
-        args:
-        - -exc
-        - |
-          scripts/run_all.sh
-
-  # Upload python docs to py.delta.chat
-  - task: upload-py-docs
-    config:
-      inputs:
-        - name: py-docs
-      image_resource:
-        type: registry-image
-        source:
-          repository: alpine
-      platform: linux
-      run:
-        path: sh
-        args:
-        - -ec
-        - |
-          apk add --no-cache rsync openssh-client
-          mkdir -p ~/.ssh
-          chmod 700 ~/.ssh
-          echo "(("c.delta.chat".private_key))" > ~/.ssh/id_ed25519
-          chmod 600 ~/.ssh/id_ed25519
-          rsync -e "ssh -o StrictHostKeyChecking=no" -avz --delete py-docs/html/ delta@py.delta.chat:build/master
-
-  # Upload x86_64 wheels and source packages
-  - task: upload-wheels
-    config:
-      inputs:
-        - name: py-wheels
-        - name: py-dist
-      image_resource:
-        type: registry-image
-        source:
-          repository: debian
-      platform: linux
-      run:
-        path: sh
-        args:
-        - -ec
-        - |
-          apt-get update -y
-          apt-get install -y --no-install-recommends python3-pip python3-setuptools
-          pip3 install devpi
-          devpi use https://m.devpi.net/dc/master
-          devpi login ((devpi.login)) --password ((devpi.password))
-          devpi upload py-wheels/*manylinux201*
-          devpi upload py-dist/*
-
-- name: python-aarch64
-  plan:
-  - get: deltachat-core-rust
-  - get: deltachat-core-rust-release
-    trigger: true
-
-  # Build manylinux image with additional dependencies
-  - task: build-coredeps
-    privileged: true
-    config:
-      inputs:
-        # Building the latest, not tagged coredeps
-        - name: deltachat-core-rust
-      image_resource:
-        source:
-          repository: vito/oci-build-task
-        type: registry-image
-      outputs:
-        - name: coredeps-image
-          path: image
-      params:
-        CONTEXT: deltachat-core-rust/scripts/docker-coredeps-arm64
-        UNPACK_ROOTFS: "true"
-      platform: linux
-      caches:
-      - path: cache
-      run:
-        path: build
-
-  # Use built image to build python wheels
-  - task: build-wheels
-    image: coredeps-image
-    config:
-      inputs:
-        - name: deltachat-core-rust-release
-          path: .
-      outputs:
-        - name: py-wheels
-          path: ./python/.docker-tox/wheelhouse/
-      platform: linux
-      run:
-        path: bash
-        args:
-        - -exc
-        - |
-          scripts/run_all.sh
-
-  # Upload aarch64 wheels
-  - task: upload-wheels
-    config:
-      inputs:
-        - name: py-wheels
-      image_resource:
-        type: registry-image
-        source:
-          repository: debian
-      platform: linux
-      run:
-        path: sh
-        args:
-        - -ec
-        - |
-          apt-get update -y
-          apt-get install -y --no-install-recommends python3-pip python3-setuptools
-          pip3 install devpi
-          devpi use https://m.devpi.net/dc/master
-          devpi login ((devpi.login)) --password ((devpi.password))
-          devpi upload py-wheels/*manylinux201*

scripts/coverage.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 set -eu
 
-if ! command -v grcov >/dev/null; then
+if ! which grcov 2>/dev/null 1>&2; then
     echo >&2 '`grcov` not found. Check README at https://github.com/mozilla/grcov for setup instructions.'
     echo >&2 'Run `cargo install grcov` to build `grcov` from source.'
     exit 1

scripts/docker-coredeps-arm64/deps/build_perl.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-PERL_VERSION=5.34.0
-# PERL_SHA256=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a
+PERL_VERSION=5.30.0
+# PERL_SHA256=7e929f64d4cb0e9d1159d4a59fc89394e27fa1f7004d0836ca0d514685406ea8
 curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz
 # echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
 tar -xzf perl-${PERL_VERSION}.tar.gz

scripts/docker-coredeps-arm64/deps/build_rust.sh

@@ -8,11 +8,9 @@ set -e -x
 #
 # Avoid using rustup here as it depends on reading /proc/self/exe and
 # has problems running under QEMU.
-RUST_VERSION=1.54.0
-
-curl "https://static.rust-lang.org/dist/rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xz
-cd "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"
+curl "https://static.rust-lang.org/dist/rust-1.52.1-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xz
+cd "rust-1.52.1-$(uname -m)-unknown-linux-gnu"
 ./install.sh --prefix=/usr --components=rustc,cargo,"rust-std-$(uname -m)-unknown-linux-gnu"
 rustc --version
 cd ..
-rm -fr "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"
+rm -fr "rust-1.52.1-$(uname -m)-unknown-linux-gnu"

scripts/docker-coredeps/deps/build_perl.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-PERL_VERSION=5.34.0
-# PERL_SHA256=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a
+PERL_VERSION=5.30.0
+# PERL_SHA256=7e929f64d4cb0e9d1159d4a59fc89394e27fa1f7004d0836ca0d514685406ea8
 curl -O https://www.cpan.org/src/5.0/perl-${PERL_VERSION}.tar.gz
 # echo "${PERL_SHA256}  perl-${PERL_VERSION}.tar.gz" | sha256sum -c -
 tar -xzf perl-${PERL_VERSION}.tar.gz

scripts/docker-coredeps/deps/build_rust.sh

@@ -3,16 +3,9 @@
 set -e -x
 
 # Install Rust
-#
-# Path from https://forge.rust-lang.org/infra/other-installation-methods.html
-#
-# Avoid using rustup here as it depends on reading /proc/self/exe and
-# has problems running under QEMU.
-RUST_VERSION=1.54.0
-
-curl "https://static.rust-lang.org/dist/rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu.tar.gz" | tar xz
-cd "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"
-./install.sh --prefix=/usr --components=rustc,cargo,"rust-std-$(uname -m)-unknown-linux-gnu"
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain "1.50.0-$(uname -m)-unknown-linux-gnu" -y
+export PATH=/root/.cargo/bin:$PATH
 rustc --version
-cd ..
-rm -fr "rust-${RUST_VERSION}-$(uname -m)-unknown-linux-gnu"
+
+# remove some 300-400 MB that we don't need for automated builds
+rm -rf "/root/.rustup/toolchains/1.50.0-$(uname -m)-unknown-linux-gnu/share"

scripts/remote_python_packaging.sh

@@ -0,0 +1,67 @@
+#!/bin/bash 
+
+export BRANCH=${CIRCLE_BRANCH:-master}
+export REPONAME=${CIRCLE_PROJECT_REPONAME:-deltachat-core-rust}
+export SSHTARGET=${SSHTARGET-ci@b1.delta.chat}
+
+export BUILDDIR=ci_builds/$REPONAME/$BRANCH/${CIRCLE_JOB:?jobname}/${CIRCLE_BUILD_NUM:?circle-build-number}
+
+echo "--- Copying files to $SSHTARGET:$BUILDDIR"
+
+set -xe
+
+ssh -oBatchMode=yes -oStrictHostKeyChecking=no  $SSHTARGET mkdir -p "$BUILDDIR"
+git ls-files >.rsynclist 
+# we seem to need .git for setuptools_scm versioning 
+find .git >>.rsynclist
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR"
+rsync --delete --files-from=.rsynclist -az ./ "$SSHTARGET:$BUILDDIR-arm64"
+
+set +x
+
+# we have to create a remote file for the remote-docker run 
+# so we can do a simple ssh command with a TTY
+# so that when our job dies, all container-runs are aborted.
+# sidenote: the circle-ci machinery will kill ongoing jobs
+# if there are new commits and we want to ensure that 
+# everything is terminated/cleaned up and we have no orphaned
+# useless still-running docker-containers consuming resources. 
+
+for arch in "" "-arm64"; do
+
+ssh $SSHTARGET bash -c "cat >${BUILDDIR}${arch}/exec_docker_run" <<_HERE
+    set +x -e
+    shopt -s huponexit
+    cd ${BUILDDIR}${arch}
+    export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
+    set -x
+
+    # run everything else inside docker 
+    docker run -e DCC_NEW_TMP_EMAIL \
+       --rm -it -v \$(pwd):/mnt -w /mnt \
+       deltachat/coredeps${arch} scripts/run_all.sh
+
+_HERE
+
+done
+
+echo "--- Running $CIRCLE_JOB remotely"
+
+echo "--- Building aarch64 wheels"
+ssh -o ServerAliveInterval=30 -t $SSHTARGET bash "$BUILDDIR-arm64/exec_docker_run"
+
+echo "--- Building x86_64 wheels"
+ssh -o ServerAliveInterval=30 -t $SSHTARGET bash "$BUILDDIR/exec_docker_run"
+
+mkdir -p workspace 
+
+# Wheels
+for arch in "" "-arm64"; do
+rsync -avz "$SSHTARGET:$BUILDDIR${arch}/python/.docker-tox/wheelhouse/*manylinux201*" workspace/wheelhouse/
+done
+
+# Source packages
+rsync -avz "$SSHTARGET:$BUILDDIR${arch}/python/.docker-tox/dist/*" workspace/wheelhouse/
+
+# Documentation
+rsync -avz "$SSHTARGET:$BUILDDIR/python/doc/_build/" workspace/py-docs

src/accounts.rs

@@ -1,8 +1,5 @@
-//! # Account manager module.
-
 use std::collections::BTreeMap;
 
-use async_std::channel::{self, Receiver, Sender};
 use async_std::fs;
 use async_std::path::PathBuf;
 use async_std::prelude::*;
@@ -16,19 +13,11 @@ use crate::context::Context;
 use crate::events::Event;
 
 /// Account manager, that can handle multiple accounts in a single place.
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct Accounts {
     dir: PathBuf,
     config: Config,
-    accounts: BTreeMap<u32, Context>,
-    emitter: EventEmitter,
-
-    /// Sender side of the fake event channel.
-    ///
-    /// We never send any events over this channel, but hold it during the account manager lifetime
-    /// to prevent `EventEmitter` from returning `None` as long as account manager is alive, even if
-    /// it holds no accounts which could emit events.
-    fake_sender: Sender<crate::events::Event>,
+    accounts: Arc<RwLock<BTreeMap<u32, Context>>>,
 }
 
 impl Accounts {
@@ -41,13 +30,19 @@ impl Accounts {
         Accounts::open(dir).await
     }
 
-    /// Creates a new default structure.
+    /// Creates a new default structure, including a default account.
     pub async fn create(os_name: String, dir: &PathBuf) -> Result<()> {
         fs::create_dir_all(dir)
             .await
             .context("failed to create folder")?;
 
-        Config::new(os_name.clone(), dir).await?;
+        // create default account
+        let config = Config::new(os_name.clone(), dir).await?;
+        let account_config = config.new_account(dir).await?;
+
+        Context::new(os_name, account_config.dbfile().into(), account_config.id)
+            .await
+            .context("failed to create default account")?;
 
         Ok(())
     }
@@ -63,66 +58,50 @@ impl Accounts {
         let config = Config::from_file(config_file).await?;
         let accounts = config.load_accounts().await?;
 
-        let emitter = EventEmitter::new();
-
-        // Fake event stream to prevent event emitter from closing.
-        let (fake_sender, fake_receiver) = channel::bounded(1);
-        emitter.sender.send(fake_receiver).await?;
-
-        for account in accounts.values() {
-            emitter.add_account(account).await?;
-        }
-
         Ok(Self {
             dir,
             config,
-            accounts,
-            emitter,
-            fake_sender,
+            accounts: Arc::new(RwLock::new(accounts)),
         })
     }
 
     /// Get an account by its `id`:
     pub async fn get_account(&self, id: u32) -> Option<Context> {
-        self.accounts.get(&id).cloned()
+        self.accounts.read().await.get(&id).cloned()
     }
 
     /// Get the currently selected account.
-    pub async fn get_selected_account(&self) -> Option<Context> {
+    pub async fn get_selected_account(&self) -> Context {
         let id = self.config.get_selected_account().await;
-        self.accounts.get(&id).cloned()
-    }
-
-    /// Returns the currently selected account's id or None if no account is selected.
-    pub async fn get_selected_account_id(&self) -> Option<u32> {
-        match self.config.get_selected_account().await {
-            0 => None,
-            id => Some(id),
-        }
+        self.accounts
+            .read()
+            .await
+            .get(&id)
+            .cloned()
+            .expect("inconsistent state")
     }
 
     /// Select the given account.
-    pub async fn select_account(&mut self, id: u32) -> Result<()> {
+    pub async fn select_account(&self, id: u32) -> Result<()> {
         self.config.select_account(id).await?;
 
         Ok(())
     }
 
     /// Add a new account.
-    pub async fn add_account(&mut self) -> Result<u32> {
+    pub async fn add_account(&self) -> Result<u32> {
         let os_name = self.config.os_name().await;
         let account_config = self.config.new_account(&self.dir).await?;
 
         let ctx = Context::new(os_name, account_config.dbfile().into(), account_config.id).await?;
-        self.emitter.add_account(&ctx).await?;
-        self.accounts.insert(account_config.id, ctx);
+        self.accounts.write().await.insert(account_config.id, ctx);
 
         Ok(account_config.id)
     }
 
     /// Remove an account.
-    pub async fn remove_account(&mut self, id: u32) -> Result<()> {
-        let ctx = self.accounts.remove(&id);
+    pub async fn remove_account(&self, id: u32) -> Result<()> {
+        let ctx = self.accounts.write().await.remove(&id);
         ensure!(ctx.is_some(), "no account with this id: {}", id);
         let ctx = ctx.unwrap();
         ctx.stop_io().await;
@@ -139,9 +118,8 @@ impl Accounts {
     }
 
     /// Migrate an existing account into this structure.
-    pub async fn migrate_account(&mut self, dbfile: PathBuf) -> Result<u32> {
+    pub async fn migrate_account(&self, dbfile: PathBuf) -> Result<u32> {
         let blobdir = Context::derive_blobdir(&dbfile);
-        let walfile = Context::derive_walfile(&dbfile);
 
         ensure!(
             dbfile.exists().await,
@@ -165,7 +143,6 @@ impl Accounts {
 
         let new_dbfile = account_config.dbfile().into();
         let new_blobdir = Context::derive_blobdir(&new_dbfile);
-        let new_walfile = Context::derive_walfile(&new_dbfile);
 
         let res = {
             fs::create_dir_all(&account_config.dir)
@@ -177,11 +154,6 @@ impl Accounts {
             fs::rename(&blobdir, &new_blobdir)
                 .await
                 .context("failed to rename blobdir")?;
-            if walfile.exists().await {
-                fs::rename(&walfile, &new_walfile)
-                    .await
-                    .context("failed to rename walfile")?;
-            }
             Ok(())
         };
 
@@ -194,8 +166,7 @@ impl Accounts {
                     account_config.id,
                 )
                 .await?;
-                self.emitter.add_account(&ctx).await?;
-                self.accounts.insert(account_config.id, ctx);
+                self.accounts.write().await.insert(account_config.id, ctx);
                 Ok(account_config.id)
             }
             Err(err) => {
@@ -216,111 +187,75 @@ impl Accounts {
 
     /// Get a list of all account ids.
     pub async fn get_all(&self) -> Vec<u32> {
-        self.accounts.keys().copied().collect()
+        self.accounts.read().await.keys().copied().collect()
     }
 
-    /// This is meant especially for iOS, because iOS needs to tell the system when its background work is done.
-    ///
-    /// Returns whether all accounts finished their background work.
-    /// DC_EVENT_CONNECTIVITY_CHANGED will be sent when this turns to true.
-    ///
-    /// iOS can:
-    /// - call dc_start_io() (in case IO was not running)
-    /// - call dc_maybe_network()
-    /// - while dc_accounts_all_work_done() returns false:
-    ///   -  Wait for DC_EVENT_CONNECTIVITY_CHANGED
-    pub async fn all_work_done(&self) -> bool {
-        for account in self.accounts.values() {
-            if !account.all_work_done().await {
-                return false;
+    /// Import a backup using a new account and selects it.
+    pub async fn import_account(&self, file: PathBuf) -> Result<u32> {
+        let old_id = self.config.get_selected_account().await;
+
+        let id = self.add_account().await?;
+        let ctx = self.get_account(id).await.expect("just added");
+
+        match crate::imex::imex(&ctx, crate::imex::ImexMode::ImportBackup, &file).await {
+            Ok(_) => Ok(id),
+            Err(err) => {
+                // remove temp account
+                self.remove_account(id).await?;
+                // set selection back
+                self.select_account(old_id).await?;
+                Err(err)
             }
         }
-        true
     }
 
     pub async fn start_io(&self) {
-        for account in self.accounts.values() {
+        let accounts = &*self.accounts.read().await;
+        for account in accounts.values() {
             account.start_io().await;
         }
     }
 
     pub async fn stop_io(&self) {
-        for account in self.accounts.values() {
+        let accounts = &*self.accounts.read().await;
+        for account in accounts.values() {
             account.stop_io().await;
         }
     }
 
     pub async fn maybe_network(&self) {
-        for account in self.accounts.values() {
+        let accounts = &*self.accounts.read().await;
+        for account in accounts.values() {
             account.maybe_network().await;
         }
     }
 
-    pub async fn maybe_network_lost(&self) {
-        for account in self.accounts.values() {
-            account.maybe_network_lost().await;
-        }
-    }
-
-    /// Returns unified event emitter.
+    /// Unified event emitter.
     pub async fn get_event_emitter(&self) -> EventEmitter {
-        self.emitter.clone()
+        let emitters: Vec<_> = self
+            .accounts
+            .read()
+            .await
+            .iter()
+            .map(|(_id, a)| a.get_event_emitter())
+            .collect();
+
+        EventEmitter(futures::stream::select_all(emitters))
     }
 }
 
-/// Unified event emitter for multiple accounts.
-#[derive(Debug, Clone)]
-pub struct EventEmitter {
-    /// Aggregate stream of events from all accounts.
-    stream: Arc<RwLock<futures::stream::SelectAll<Receiver<crate::events::Event>>>>,
-
-    /// Sender for the channel where new account emitters will be pushed.
-    sender: Sender<Receiver<crate::events::Event>>,
-
-    /// Receiver for the channel where new account emitters will be pushed.
-    receiver: Receiver<Receiver<crate::events::Event>>,
-}
+#[derive(Debug)]
+pub struct EventEmitter(futures::stream::SelectAll<crate::events::EventEmitter>);
 
 impl EventEmitter {
-    pub fn new() -> Self {
-        let (sender, receiver) = channel::unbounded();
-        Self {
-            stream: Arc::new(RwLock::new(futures::stream::SelectAll::new())),
-            sender,
-            receiver,
-        }
-    }
-
     /// Blocking recv of an event. Return `None` if all `Sender`s have been droped.
     pub fn recv_sync(&mut self) -> Option<Event> {
-        async_std::task::block_on(self.recv()).unwrap_or_default()
+        async_std::task::block_on(self.recv())
     }
 
-    /// Async recv of an event. Return `None` if all `Sender`s have been dropped.
-    pub async fn recv(&mut self) -> Result<Option<Event>> {
-        let mut stream = self.stream.write().await;
-        loop {
-            match futures::future::select(self.receiver.recv(), stream.next()).await {
-                futures::future::Either::Left((emitter, _)) => {
-                    stream.push(emitter?);
-                }
-                futures::future::Either::Right((ev, _)) => return Ok(ev),
-            }
-        }
-    }
-
-    /// Add event emitter of a new account to the aggregate event emitter.
-    pub async fn add_account(&self, context: &Context) -> Result<()> {
-        self.sender
-            .send(context.get_event_emitter().into_inner())
-            .await?;
-        Ok(())
-    }
-}
-
-impl Default for EventEmitter {
-    fn default() -> Self {
-        Self::new()
+    /// Async recv of an event. Return `None` if all `Sender`s have been droped.
+    pub async fn recv(&mut self) -> Option<Event> {
+        self.0.next().await
     }
 }
 
@@ -331,23 +266,19 @@ impl async_std::stream::Stream for EventEmitter {
         mut self: std::pin::Pin<&mut Self>,
         cx: &mut std::task::Context<'_>,
     ) -> std::task::Poll<Option<Self::Item>> {
-        std::pin::Pin::new(&mut self).poll_next(cx)
+        std::pin::Pin::new(&mut self.0).poll_next(cx)
     }
 }
 
 pub const CONFIG_NAME: &str = "accounts.toml";
 pub const DB_NAME: &str = "dc.db";
 
-/// Account manager configuration file.
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone)]
 pub struct Config {
     file: PathBuf,
-    inner: InnerConfig,
+    inner: Arc<RwLock<InnerConfig>>,
 }
 
-/// Account manager configuration file contents.
-///
-/// This is serialized into TOML.
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
 struct InnerConfig {
     pub os_name: String,
@@ -359,15 +290,14 @@ struct InnerConfig {
 
 impl Config {
     pub async fn new(os_name: String, dir: &PathBuf) -> Result<Self> {
-        let inner = InnerConfig {
-            os_name,
-            accounts: Vec::new(),
-            selected_account: 0,
-            next_id: 1,
-        };
         let cfg = Config {
             file: dir.join(CONFIG_NAME),
-            inner,
+            inner: Arc::new(RwLock::new(InnerConfig {
+                os_name,
+                accounts: Vec::new(),
+                selected_account: 0,
+                next_id: 1,
+            })),
         };
 
         cfg.sync().await?;
@@ -376,14 +306,17 @@ impl Config {
     }
 
     pub async fn os_name(&self) -> String {
-        self.inner.os_name.clone()
+        self.inner.read().await.os_name.clone()
     }
 
     /// Sync the inmemory representation to disk.
     async fn sync(&self) -> Result<()> {
-        fs::write(&self.file, toml::to_string_pretty(&self.inner)?)
-            .await
-            .context("failed to write config")
+        fs::write(
+            &self.file,
+            toml::to_string_pretty(&*self.inner.read().await)?,
+        )
+        .await
+        .context("failed to write config")
     }
 
     /// Read a configuration from the given file into memory.
@@ -391,14 +324,18 @@ impl Config {
         let bytes = fs::read(&file).await.context("failed to read file")?;
         let inner: InnerConfig = toml::from_slice(&bytes).context("failed to parse config")?;
 
-        Ok(Config { file, inner })
+        Ok(Config {
+            file,
+            inner: Arc::new(RwLock::new(inner)),
+        })
     }
 
     pub async fn load_accounts(&self) -> Result<BTreeMap<u32, Context>> {
+        let cfg = &*self.inner.read().await;
         let mut accounts = BTreeMap::new();
-        for account_config in &self.inner.accounts {
+        for account_config in &cfg.accounts {
             let ctx = Context::new(
-                self.inner.os_name.clone(),
+                cfg.os_name.clone(),
                 account_config.dbfile().into(),
                 account_config.id,
             )
@@ -410,18 +347,19 @@ impl Config {
     }
 
     /// Create a new account in the given root directory.
-    async fn new_account(&mut self, dir: &PathBuf) -> Result<AccountConfig> {
+    pub async fn new_account(&self, dir: &PathBuf) -> Result<AccountConfig> {
         let id = {
-            let id = self.inner.next_id;
+            let inner = &mut self.inner.write().await;
+            let id = inner.next_id;
             let uuid = Uuid::new_v4();
             let target_dir = dir.join(uuid.to_simple_ref().to_string());
 
-            self.inner.accounts.push(AccountConfig {
+            inner.accounts.push(AccountConfig {
                 id,
                 dir: target_dir.into(),
                 uuid,
             });
-            self.inner.next_id += 1;
+            inner.next_id += 1;
             id
         };
 
@@ -433,39 +371,46 @@ impl Config {
     }
 
     /// Removes an existing acccount entirely.
-    pub async fn remove_account(&mut self, id: u32) -> Result<()> {
+    pub async fn remove_account(&self, id: u32) -> Result<()> {
         {
-            if let Some(idx) = self.inner.accounts.iter().position(|e| e.id == id) {
+            let inner = &mut *self.inner.write().await;
+            if let Some(idx) = inner.accounts.iter().position(|e| e.id == id) {
                 // remove account from the configs
-                self.inner.accounts.remove(idx);
+                inner.accounts.remove(idx);
             }
-            if self.inner.selected_account == id {
+            if inner.selected_account == id {
                 // reset selected account
-                self.inner.selected_account =
-                    self.inner.accounts.get(0).map(|e| e.id).unwrap_or_default();
+                inner.selected_account = inner.accounts.get(0).map(|e| e.id).unwrap_or_default();
             }
         }
 
         self.sync().await
     }
 
-    async fn get_account(&self, id: u32) -> Option<AccountConfig> {
-        self.inner.accounts.iter().find(|e| e.id == id).cloned()
+    pub async fn get_account(&self, id: u32) -> Option<AccountConfig> {
+        self.inner
+            .read()
+            .await
+            .accounts
+            .iter()
+            .find(|e| e.id == id)
+            .cloned()
     }
 
     pub async fn get_selected_account(&self) -> u32 {
-        self.inner.selected_account
+        self.inner.read().await.selected_account
     }
 
-    pub async fn select_account(&mut self, id: u32) -> Result<()> {
+    pub async fn select_account(&self, id: u32) -> Result<()> {
         {
+            let inner = &mut *self.inner.write().await;
             ensure!(
-                self.inner.accounts.iter().any(|e| e.id == id),
+                inner.accounts.iter().any(|e| e.id == id),
                 "invalid account id: {}",
                 id
             );
 
-            self.inner.selected_account = id;
+            inner.selected_account = id;
         }
 
         self.sync().await?;
@@ -473,9 +418,8 @@ impl Config {
     }
 }
 
-/// Configuration of a single account.
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
-struct AccountConfig {
+pub struct AccountConfig {
     /// Unique id.
     pub id: u32,
     /// Root directory for all data for this account.
@@ -499,17 +443,21 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts1").into();
 
-        let mut accounts1 = Accounts::new("my_os".into(), p.clone()).await.unwrap();
-        accounts1.add_account().await.unwrap();
-
+        let accounts1 = Accounts::new("my_os".into(), p.clone()).await.unwrap();
         let accounts2 = Accounts::open(p).await.unwrap();
 
-        assert_eq!(accounts1.accounts.len(), 1);
+        assert_eq!(accounts1.accounts.read().await.len(), 1);
         assert_eq!(accounts1.config.get_selected_account().await, 1);
 
         assert_eq!(accounts1.dir, accounts2.dir);
-        assert_eq!(accounts1.config, accounts2.config,);
-        assert_eq!(accounts1.accounts.len(), accounts2.accounts.len());
+        assert_eq!(
+            &*accounts1.config.inner.read().await,
+            &*accounts2.config.inner.read().await,
+        );
+        assert_eq!(
+            accounts1.accounts.read().await.len(),
+            accounts2.accounts.read().await.len()
+        );
     }
 
     #[async_std::test]
@@ -517,47 +465,22 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts").into();
 
-        let mut accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
-        assert_eq!(accounts.accounts.len(), 0);
-        assert_eq!(accounts.config.get_selected_account().await, 0);
+        let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
 
-        let id = accounts.add_account().await.unwrap();
-        assert_eq!(id, 1);
-        assert_eq!(accounts.accounts.len(), 1);
+        assert_eq!(accounts.accounts.read().await.len(), 1);
         assert_eq!(accounts.config.get_selected_account().await, 1);
 
         let id = accounts.add_account().await.unwrap();
         assert_eq!(id, 2);
         assert_eq!(accounts.config.get_selected_account().await, id);
-        assert_eq!(accounts.accounts.len(), 2);
+        assert_eq!(accounts.accounts.read().await.len(), 2);
 
         accounts.select_account(1).await.unwrap();
         assert_eq!(accounts.config.get_selected_account().await, 1);
 
         accounts.remove_account(1).await.unwrap();
         assert_eq!(accounts.config.get_selected_account().await, 2);
-        assert_eq!(accounts.accounts.len(), 1);
-    }
-
-    #[async_std::test]
-    async fn test_accounts_remove_last() -> Result<()> {
-        let dir = tempfile::tempdir()?;
-        let p: PathBuf = dir.path().join("accounts").into();
-
-        let mut accounts = Accounts::new("my_os".into(), p.clone()).await?;
-        assert!(accounts.get_selected_account().await.is_none());
-        assert_eq!(accounts.config.get_selected_account().await, 0);
-
-        let id = accounts.add_account().await?;
-        assert!(accounts.get_selected_account().await.is_some());
-        assert_eq!(id, 1);
-        assert_eq!(accounts.accounts.len(), 1);
-        assert_eq!(accounts.config.get_selected_account().await, id);
-
-        accounts.remove_account(id).await?;
-        assert!(accounts.get_selected_account().await.is_none());
-
-        Ok(())
+        assert_eq!(accounts.accounts.read().await.len(), 1);
     }
 
     #[async_std::test]
@@ -565,9 +488,9 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts").into();
 
-        let mut accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
-        assert_eq!(accounts.accounts.len(), 0);
-        assert_eq!(accounts.config.get_selected_account().await, 0);
+        let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+        assert_eq!(accounts.accounts.read().await.len(), 1);
+        assert_eq!(accounts.config.get_selected_account().await, 1);
 
         let extern_dbfile: PathBuf = dir.path().join("other").into();
         let ctx = Context::new("my_os".into(), extern_dbfile.clone(), 0)
@@ -583,10 +506,10 @@ mod tests {
             .migrate_account(extern_dbfile.clone())
             .await
             .unwrap();
-        assert_eq!(accounts.accounts.len(), 1);
-        assert_eq!(accounts.config.get_selected_account().await, 1);
+        assert_eq!(accounts.accounts.read().await.len(), 2);
+        assert_eq!(accounts.config.get_selected_account().await, 2);
 
-        let ctx = accounts.get_selected_account().await.unwrap();
+        let ctx = accounts.get_selected_account().await;
         assert_eq!(
             "me@mail.com",
             ctx.get_config(crate::config::Config::Addr)
@@ -602,9 +525,9 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let p: PathBuf = dir.path().join("accounts").into();
 
-        let mut accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
+        let accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
 
-        for expected_id in 1..10 {
+        for expected_id in 2..10 {
             let id = accounts.add_account().await.unwrap();
             assert_eq!(id, expected_id);
         }
@@ -614,136 +537,4 @@ mod tests {
             assert_eq!(ids.get(i), Some(&expected_id));
         }
     }
-
-    #[async_std::test]
-    async fn test_accounts_ids_unique_increasing_and_persisted() -> Result<()> {
-        let dir = tempfile::tempdir()?;
-        let p: PathBuf = dir.path().join("accounts").into();
-        let dummy_accounts = 10;
-
-        let (id0, id1, id2) = {
-            let mut accounts = Accounts::new("my_os".into(), p.clone()).await?;
-            accounts.add_account().await?;
-            let ids = accounts.get_all().await;
-            assert_eq!(ids.len(), 1);
-
-            let id0 = *ids.get(0).unwrap();
-            let ctx = accounts.get_account(id0).await.unwrap();
-            ctx.set_config(crate::config::Config::Addr, Some("one@example.org"))
-                .await?;
-
-            let id1 = accounts.add_account().await?;
-            let ctx = accounts.get_account(id1).await.unwrap();
-            ctx.set_config(crate::config::Config::Addr, Some("two@example.org"))
-                .await?;
-
-            // add and remove some accounts and force a gap (ids must not be reused)
-            for _ in 0..dummy_accounts {
-                let to_delete = accounts.add_account().await?;
-                accounts.remove_account(to_delete).await?;
-            }
-
-            let id2 = accounts.add_account().await?;
-            let ctx = accounts.get_account(id2).await.unwrap();
-            ctx.set_config(crate::config::Config::Addr, Some("three@example.org"))
-                .await?;
-
-            accounts.select_account(id1).await?;
-
-            (id0, id1, id2)
-        };
-        assert!(id0 > 0);
-        assert!(id1 > id0);
-        assert!(id2 > id1 + dummy_accounts);
-
-        let (id0_reopened, id1_reopened, id2_reopened) = {
-            let accounts = Accounts::new("my_os".into(), p.clone()).await?;
-            let ctx = accounts.get_selected_account().await.unwrap();
-            assert_eq!(
-                ctx.get_config(crate::config::Config::Addr).await?,
-                Some("two@example.org".to_string())
-            );
-
-            let ids = accounts.get_all().await;
-            assert_eq!(ids.len(), 3);
-
-            let id0 = *ids.get(0).unwrap();
-            let ctx = accounts.get_account(id0).await.unwrap();
-            assert_eq!(
-                ctx.get_config(crate::config::Config::Addr).await?,
-                Some("one@example.org".to_string())
-            );
-
-            let id1 = *ids.get(1).unwrap();
-            let t = accounts.get_account(id1).await.unwrap();
-            assert_eq!(
-                t.get_config(crate::config::Config::Addr).await?,
-                Some("two@example.org".to_string())
-            );
-
-            let id2 = *ids.get(2).unwrap();
-            let ctx = accounts.get_account(id2).await.unwrap();
-            assert_eq!(
-                ctx.get_config(crate::config::Config::Addr).await?,
-                Some("three@example.org".to_string())
-            );
-
-            (id0, id1, id2)
-        };
-        assert_eq!(id0, id0_reopened);
-        assert_eq!(id1, id1_reopened);
-        assert_eq!(id2, id2_reopened);
-
-        Ok(())
-    }
-
-    #[async_std::test]
-    async fn test_no_accounts_event_emitter() -> Result<()> {
-        let dir = tempfile::tempdir().unwrap();
-        let p: PathBuf = dir.path().join("accounts").into();
-
-        let accounts = Accounts::new("my_os".into(), p.clone()).await?;
-
-        // Make sure there are no accounts.
-        assert_eq!(accounts.accounts.len(), 0);
-
-        // Create event emitter.
-        let mut event_emitter = accounts.get_event_emitter().await;
-
-        // Test that event emitter does not return `None` immediately.
-        let duration = std::time::Duration::from_millis(1);
-        assert!(async_std::future::timeout(duration, event_emitter.recv())
-            .await
-            .is_err());
-
-        // When account manager is dropped, event emitter is exhausted.
-        drop(accounts);
-        assert_eq!(event_emitter.recv().await?, None);
-
-        Ok(())
-    }
-
-    #[async_std::test]
-    async fn test_account_new_add_remove_with_active_event_emmitter() {
-        let dir = tempfile::tempdir().unwrap();
-        let p: PathBuf = dir.path().join("accounts").into();
-
-        let mut accounts = Accounts::new("my_os".into(), p.clone()).await.unwrap();
-        assert_eq!(accounts.accounts.len(), 0);
-        assert_eq!(accounts.config.get_selected_account().await, 0);
-
-        accounts.add_account().await.unwrap();
-        accounts.add_account().await.unwrap();
-
-        // Create event emitter.
-        let mut event_emitter = accounts.get_event_emitter().await;
-
-        // make sure that account is still removed on windows
-        accounts.remove_account(1).await.unwrap();
-        assert_eq!(accounts.config.get_selected_account().await, 2);
-        assert_eq!(accounts.accounts.len(), 1);
-
-        // make sure event emitter is not dropped before the test
-        println!("{:?}", event_emitter.recv().await);
-    }
 }

src/aheader.rs

@@ -1,4 +1,4 @@
-//! # Autocrypt header module.
+//! # Autocrypt header module
 //!
 //! Parse and create [Autocrypt-headers](https://autocrypt.org/en/latest/level1.html#the-autocrypt-header).
 

src/blob.rs

@@ -1,4 +1,4 @@
-//! # Blob directory management.
+//! # Blob directory management
 
 use core::cmp::max;
 use std::ffi::OsStr;
@@ -74,7 +74,7 @@ impl<'a> BlobObject<'a> {
 
         // workaround a bug in async-std
         // (the executor does not handle blocking operation in Drop correctly,
-        // see <https://github.com/async-rs/async-std/issues/900>)
+        // see https://github.com/async-rs/async-std/issues/900 )
         let _ = file.flush().await;
 
         let blob = BlobObject {

src/chatlist.rs

@@ -1,19 +1,19 @@
-//! # Chat list module.
+//! # Chat list module
 
 use anyhow::{bail, ensure, Result};
 
 use crate::chat::{update_special_chat_names, Chat, ChatId, ChatVisibility};
 use crate::constants::{
-    Blocked, Chattype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK, DC_CONTACT_ID_DEVICE,
-    DC_CONTACT_ID_SELF, DC_CONTACT_ID_UNDEFINED, DC_GCL_ADD_ALLDONE_HINT, DC_GCL_ARCHIVED_ONLY,
-    DC_GCL_FOR_FORWARDING, DC_GCL_NO_SPECIALS,
+    Chattype, DC_CHAT_ID_ALLDONE_HINT, DC_CHAT_ID_ARCHIVED_LINK, DC_CHAT_ID_DEADDROP,
+    DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_SELF, DC_CONTACT_ID_UNDEFINED, DC_GCL_ADD_ALLDONE_HINT,
+    DC_GCL_ARCHIVED_ONLY, DC_GCL_FOR_FORWARDING, DC_GCL_NO_SPECIALS,
 };
 use crate::contact::Contact;
 use crate::context::Context;
 use crate::ephemeral::delete_expired_messages;
+use crate::lot::Lot;
 use crate::message::{Message, MessageState, MsgId};
 use crate::stock_str;
-use crate::summary::Summary;
 
 /// An object representing a single chatlist in memory.
 ///
@@ -34,12 +34,15 @@ use crate::summary::Summary;
 /// and for each messages that is scrolled into view, dc_get_msg() is called then.
 ///
 /// Why no listflags?
-/// Without listflags, dc_get_chatlist() adds the archive "link" automatically as needed.
-/// The UI can just render these items differently then.
+/// Without listflags, dc_get_chatlist() adds the deaddrop and the archive "link" automatically as needed.
+/// The UI can just render these items differently then. Although the deaddrop link is currently always the
+/// first entry and only present on new messages, there is the rough idea that it can be optionally always
+/// present and sorted into the list by date. Rendering the deaddrop in the described way
+/// would not add extra work in the UI then.
 #[derive(Debug)]
 pub struct Chatlist {
     /// Stores pairs of `chat_id, message_id`
-    ids: Vec<(ChatId, Option<MsgId>)>,
+    ids: Vec<(ChatId, MsgId)>,
 }
 
 impl Chatlist {
@@ -55,6 +58,12 @@ impl Chatlist {
     ///
     /// By default, the function adds some special entries to the list.
     /// These special entries can be identified by the ID returned by chatlist.get_chat_id():
+    /// - DC_CHAT_ID_DEADDROP (1) - this special chat is present if there are
+    ///   messages from addresses that have no relationship to the configured account.
+    ///   The last of these messages is represented by DC_CHAT_ID_DEADDROP and you can retrieve details
+    ///   about it with chatlist.get_msg_id(). Typically, the UI asks the user "Do you want to chat with NAME?"
+    ///   and offers the options "Start chat", "Block" and "Not now";
+    ///   The decision should be passed to dc_decide_on_contact_request().
     /// - DC_CHAT_ID_ARCHIVED_LINK (6) - this special chat is present if the user has
     ///   archived *any* chat using dc_set_chat_visibility(). The UI should show a link as
     ///   "Show archived chats", if the user clicks this item, the UI should show a
@@ -70,9 +79,9 @@ impl Chatlist {
     ///   the pseudo-chat DC_CHAT_ID_ARCHIVED_LINK is added if there are *any* archived
     ///   chats
     /// - the flag DC_GCL_FOR_FORWARDING sorts "Saved messages" to the top of the chatlist
-    ///   and hides the device-chat and contact requests
+    ///   and hides the device-chat,
     ///   typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS
-    /// - if the flag DC_GCL_NO_SPECIALS is set, archive link is not added
+    /// - if the flag DC_GCL_NO_SPECIALS is set, deaddrop and archive link are not added
     ///   to the list (may be used eg. for selecting chats on forwarding, the flag is
     ///   not needed when DC_GCL_ARCHIVED_ONLY is already set)
     /// - if the flag DC_GCL_ADD_ALLDONE_HINT is set, DC_CHAT_ID_ALLDONE_HINT
@@ -102,7 +111,7 @@ impl Chatlist {
 
         let process_row = |row: &rusqlite::Row| {
             let chat_id: ChatId = row.get(0)?;
-            let msg_id: Option<MsgId> = row.get(1)?;
+            let msg_id: MsgId = row.get(1).unwrap_or_default();
             Ok((chat_id, msg_id))
         };
 
@@ -127,8 +136,13 @@ impl Chatlist {
         //   timestamp
         // - the list starts with the newest chats
         //
-        // The query shows messages from blocked contacts in
-        // groups. Otherwise it would be hard to follow conversations.
+        // nb: the query currently shows messages from blocked
+        // contacts in groups.  however, for normal-groups, this is
+        // okay as the message is also returned by dc_get_chat_msgs()
+        // (otherwise it would be hard to follow conversations, wa and
+        // tg do the same) for the deaddrop, however, they should
+        // really be hidden, however, _currently_ the deaddrop is not
+        // shown at all permanent in the chatlist.
         let mut ids = if let Some(query_contact_id) = query_contact_id {
             // show chats shared with a given contact
             context.sql.query_map(
@@ -143,7 +157,7 @@ impl Chatlist {
                                   AND (hidden=0 OR state=?1)
                                   ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9
-                   AND c.blocked!=1
+                   AND c.blocked=0
                    AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?2)
                  GROUP BY c.id
                  ORDER BY c.archived=?3 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
@@ -170,7 +184,7 @@ impl Chatlist {
                                   AND (hidden=0 OR state=?)
                                   ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9
-                   AND c.blocked!=1
+                   AND c.blocked=0
                    AND c.archived=1
                  GROUP BY c.id
                  ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
@@ -204,7 +218,7 @@ impl Chatlist {
                                   AND (hidden=0 OR state=?1)
                                   ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9 AND c.id!=?2
-                   AND c.blocked!=1
+                   AND c.blocked=0
                    AND c.name LIKE ?3
                  GROUP BY c.id
                  ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
@@ -222,7 +236,7 @@ impl Chatlist {
             } else {
                 ChatId::new(0)
             };
-            let ids = context.sql.query_map(
+            let mut ids = context.sql.query_map(
                 "SELECT c.id, m.id
                  FROM chats c
                  LEFT JOIN msgs m
@@ -234,15 +248,22 @@ impl Chatlist {
                                   AND (hidden=0 OR state=?1)
                                   ORDER BY timestamp DESC, id DESC LIMIT 1)
                  WHERE c.id>9 AND c.id!=?2
-                   AND (c.blocked=0 OR (c.blocked=2 AND NOT ?3))
-                   AND NOT c.archived=?4
+                   AND c.blocked=0
+                   AND NOT c.archived=?3
                  GROUP BY c.id
-                 ORDER BY c.id=?5 DESC, c.archived=?6 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                paramsv![MessageState::OutDraft, skip_id, flag_for_forwarding, ChatVisibility::Archived, sort_id_up, ChatVisibility::Pinned],
+                 ORDER BY c.id=?4 DESC, c.archived=?5 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                paramsv![MessageState::OutDraft, skip_id, ChatVisibility::Archived, sort_id_up, ChatVisibility::Pinned],
                 process_row,
                 process_rows,
             ).await?;
             if !flag_no_specials {
+                if let Some(last_deaddrop_fresh_msg_id) =
+                    get_last_deaddrop_fresh_msg(context).await?
+                {
+                    if !flag_for_forwarding {
+                        ids.insert(0, (DC_CHAT_ID_DEADDROP, last_deaddrop_fresh_msg_id));
+                    }
+                }
                 add_archived_link_item = true;
             }
             ids
@@ -250,9 +271,9 @@ impl Chatlist {
 
         if add_archived_link_item && dc_get_archived_cnt(context).await? > 0 {
             if ids.is_empty() && flag_add_alldone_hint {
-                ids.push((DC_CHAT_ID_ALLDONE_HINT, None));
+                ids.push((DC_CHAT_ID_ALLDONE_HINT, MsgId::new(0)));
             }
-            ids.push((DC_CHAT_ID_ARCHIVED_LINK, None));
+            ids.push((DC_CHAT_ID_ARCHIVED_LINK, MsgId::new(0)));
         }
 
         Ok(Chatlist { ids })
@@ -281,75 +302,91 @@ impl Chatlist {
     /// Get a single message ID of a chatlist.
     ///
     /// To get the message object from the message ID, use dc_get_msg().
-    pub fn get_msg_id(&self, index: usize) -> Result<Option<MsgId>> {
+    pub fn get_msg_id(&self, index: usize) -> Result<MsgId> {
         match self.ids.get(index) {
             Some((_chat_id, msg_id)) => Ok(*msg_id),
             None => bail!("Chatlist index out of range"),
         }
     }
 
-    /// Returns a summary for a given chatlist index.
-    pub async fn get_summary(
-        &self,
-        context: &Context,
-        index: usize,
-        chat: Option<&Chat>,
-    ) -> Result<Summary> {
+    /// Get a summary for a chatlist index.
+    ///
+    /// The summary is returned by a dc_lot_t object with the following fields:
+    ///
+    /// - dc_lot_t::text1: contains the username or the strings "Me", "Draft" and so on.
+    ///   The string may be colored by having a look at text1_meaning.
+    ///   If there is no such name or it should not be displayed, the element is NULL.
+    /// - dc_lot_t::text1_meaning: one of DC_TEXT1_USERNAME, DC_TEXT1_SELF or DC_TEXT1_DRAFT.
+    ///   Typically used to show dc_lot_t::text1 with different colors. 0 if not applicable.
+    /// - dc_lot_t::text2: contains an excerpt of the message text or strings as
+    ///   "No messages".  May be NULL of there is no such text (eg. for the archive link)
+    /// - dc_lot_t::timestamp: the timestamp of the message.  0 if not applicable.
+    /// - dc_lot_t::state: The state of the message as one of the DC_STATE_* constants (see #dc_msg_get_state()).
+    //    0 if not applicable.
+    pub async fn get_summary(&self, context: &Context, index: usize, chat: Option<&Chat>) -> Lot {
         // The summary is created by the chat, not by the last message.
         // This is because we may want to display drafts here or stuff as
         // "is typing".
         // Also, sth. as "No messages" would not work if the summary comes from a message.
         let (chat_id, lastmsg_id) = match self.ids.get(index) {
             Some(ids) => ids,
-            None => bail!("Chatlist index out of range"),
+            None => {
+                let mut ret = Lot::new();
+                ret.text2 = Some("ErrBadChatlistIndex".to_string());
+                return Lot::new();
+            }
         };
 
         Chatlist::get_summary2(context, *chat_id, *lastmsg_id, chat).await
     }
 
-    /// Returns a summary for a given chatlist item.
     pub async fn get_summary2(
         context: &Context,
         chat_id: ChatId,
-        lastmsg_id: Option<MsgId>,
+        lastmsg_id: MsgId,
         chat: Option<&Chat>,
-    ) -> Result<Summary> {
+    ) -> Lot {
+        let mut ret = Lot::new();
+
         let chat_loaded: Chat;
         let chat = if let Some(chat) = chat {
             chat
-        } else {
-            let chat = Chat::load_from_db(context, chat_id).await?;
+        } else if let Ok(chat) = Chat::load_from_db(context, chat_id).await {
             chat_loaded = chat;
             &chat_loaded
+        } else {
+            return ret;
         };
 
-        let (lastmsg, lastcontact) = if let Some(lastmsg_id) = lastmsg_id {
-            let lastmsg = Message::load_from_db(context, lastmsg_id).await?;
-            if lastmsg.from_id == DC_CONTACT_ID_SELF {
-                (Some(lastmsg), None)
-            } else {
-                match chat.typ {
-                    Chattype::Group | Chattype::Mailinglist => {
-                        let lastcontact = Contact::load_from_db(context, lastmsg.from_id).await?;
-                        (Some(lastmsg), Some(lastcontact))
+        let (lastmsg, lastcontact) =
+            if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id).await {
+                if lastmsg.from_id == DC_CONTACT_ID_SELF {
+                    (Some(lastmsg), None)
+                } else {
+                    match chat.typ {
+                        Chattype::Group | Chattype::Mailinglist => {
+                            let lastcontact =
+                                Contact::load_from_db(context, lastmsg.from_id).await.ok();
+                            (Some(lastmsg), lastcontact)
+                        }
+                        Chattype::Single | Chattype::Undefined => (Some(lastmsg), None),
                     }
-                    Chattype::Single | Chattype::Undefined => (Some(lastmsg), None),
                 }
-            }
-        } else {
-            (None, None)
-        };
+            } else {
+                (None, None)
+            };
 
         if chat.id.is_archived_link() {
-            Ok(Default::default())
-        } else if let Some(lastmsg) = lastmsg.filter(|msg| msg.from_id != DC_CONTACT_ID_UNDEFINED) {
-            Ok(Summary::new(context, &lastmsg, chat, lastcontact.as_ref()).await)
+            ret.text2 = None;
+        } else if lastmsg.is_none() || lastmsg.as_ref().unwrap().from_id == DC_CONTACT_ID_UNDEFINED
+        {
+            ret.text2 = Some(stock_str::no_messages(context).await);
         } else {
-            Ok(Summary {
-                text: stock_str::no_messages(context).await,
-                ..Default::default()
-            })
+            ret.fill(&mut lastmsg.unwrap(), chat, lastcontact.as_ref(), context)
+                .await;
         }
+
+        ret
     }
 
     pub fn get_index_for_id(&self, id: ChatId) -> Option<usize> {
@@ -362,13 +399,35 @@ pub async fn dc_get_archived_cnt(context: &Context) -> Result<usize> {
     let count = context
         .sql
         .count(
-            "SELECT COUNT(*) FROM chats WHERE blocked!=? AND archived=?;",
-            paramsv![Blocked::Manually, ChatVisibility::Archived],
+            "SELECT COUNT(*) FROM chats WHERE blocked=0 AND archived=1;",
+            paramsv![],
         )
         .await?;
     Ok(count)
 }
 
+async fn get_last_deaddrop_fresh_msg(context: &Context) -> Result<Option<MsgId>> {
+    // We have an index over the state-column, this should be
+    // sufficient as there are typically only few fresh messages.
+    let id = context
+        .sql
+        .query_get_value(
+            concat!(
+                "SELECT m.id",
+                " FROM msgs m",
+                " LEFT JOIN chats c",
+                "        ON c.id=m.chat_id",
+                " WHERE m.state=10",
+                "   AND m.hidden=0",
+                "   AND c.blocked=2",
+                " ORDER BY m.timestamp DESC, m.id DESC;"
+            ),
+            paramsv![],
+        )
+        .await?;
+    Ok(id)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -376,6 +435,8 @@ mod tests {
     use crate::chat::{create_group_chat, get_chat_contacts, ProtectionStatus};
     use crate::constants::Viewtype;
     use crate::dc_receive_imf::dc_receive_imf;
+    use crate::message;
+    use crate::message::ContactRequestDecision;
     use crate::stock_str::StockMessage;
     use crate::test_utils::TestContext;
 
@@ -485,7 +546,7 @@ mod tests {
     async fn test_search_single_chat() -> anyhow::Result<()> {
         let t = TestContext::new_alice().await;
 
-        // receive a one-to-one-message
+        // receive a one-to-one-message, accept contact request
         dc_receive_imf(
             &t,
             b"From: Bob Authname <bob@example.org>\n\
@@ -503,13 +564,15 @@ mod tests {
         .await?;
 
         let chats = Chatlist::try_load(&t, 0, Some("Bob Authname"), None).await?;
-        // Contact request should be searchable
-        assert_eq!(chats.len(), 1);
+        assert_eq!(chats.len(), 0);
 
         let msg = t.get_last_msg().await;
-        let chat_id = msg.get_chat_id();
-        chat_id.accept(&t).await.unwrap();
+        assert_eq!(msg.get_chat_id(), DC_CHAT_ID_DEADDROP);
 
+        let chat_id =
+            message::decide_on_contact_request(&t, msg.get_id(), ContactRequestDecision::StartChat)
+                .await
+                .unwrap();
         let contacts = get_chat_contacts(&t, chat_id).await?;
         let contact_id = *contacts.first().unwrap();
         let chat = Chat::load_from_db(&t, chat_id).await?;
@@ -547,7 +610,7 @@ mod tests {
     async fn test_search_single_chat_without_authname() -> anyhow::Result<()> {
         let t = TestContext::new_alice().await;
 
-        // receive a one-to-one-message without authname set
+        // receive a one-to-one-message without authname set, accept contact request
         dc_receive_imf(
             &t,
             b"From: bob@example.org\n\
@@ -565,8 +628,10 @@ mod tests {
         .await?;
 
         let msg = t.get_last_msg().await;
-        let chat_id = msg.get_chat_id();
-        chat_id.accept(&t).await.unwrap();
+        let chat_id =
+            message::decide_on_contact_request(&t, msg.get_id(), ContactRequestDecision::StartChat)
+                .await
+                .unwrap();
         let contacts = get_chat_contacts(&t, chat_id).await?;
         let contact_id = *contacts.first().unwrap();
         let chat = Chat::load_from_db(&t, chat_id).await?;
@@ -619,7 +684,7 @@ mod tests {
         chat_id1.set_draft(&t, Some(&mut msg)).await.unwrap();
 
         let chats = Chatlist::try_load(&t, 0, None, None).await.unwrap();
-        let summary = chats.get_summary(&t, 0, None).await.unwrap();
-        assert_eq!(summary.text, "foo: bar test"); // the linebreak should be removed from summary
+        let summary = chats.get_summary(&t, 0, None).await;
+        assert_eq!(summary.get_text2().unwrap(), "foo: bar test"); // the linebreak should be removed from summary
     }
 }

src/color.rs

@@ -1,4 +1,4 @@
-//! Implementation of Consistent Color Generation.
+//! Implementation of Consistent Color Generation
 //!
 //! Consistent Color Generation is defined in XEP-0392.
 //!
@@ -42,7 +42,7 @@ mod tests {
     #[test]
     fn test_str_to_angle() {
         // Test against test vectors from
-        // <https://xmpp.org/extensions/xep-0392.html#testvectors-fullrange-no-cvd>
+        // https://xmpp.org/extensions/xep-0392.html#testvectors-fullrange-no-cvd
         assert!((str_to_angle("Romeo") - 327.255249).abs() < 1e-6);
         assert!((str_to_angle("juliet@capulet.lit") - 209.410400).abs() < 1e-6);
         assert!((str_to_angle("😺") - 331.199341).abs() < 1e-6);

src/config.rs

@@ -1,4 +1,4 @@
-//! # Key-value configuration management.
+//! # Key-value configuration management
 
 use anyhow::Result;
 use strum::{EnumProperty, IntoEnumIterator};
@@ -48,12 +48,6 @@ pub enum Config {
     SmtpCertificateChecks,
     ServerFlags,
 
-    Socks5Enabled,
-    Socks5Host,
-    Socks5Port,
-    Socks5User,
-    Socks5Password,
-
     Displayname,
     Selfstatus,
     Selfavatar,
@@ -156,11 +150,6 @@ pub enum Config {
     #[strum(props(default = "0"))]
     NotifyAboutWrongPw,
 
-    /// If a warning about exceeding quota was shown recently,
-    /// this is the percentage of quota at the time the warning was given.
-    /// Unset, when quota falls below minimal warning threshold again.
-    QuotaExceeding,
-
     /// address to webrtc instance to use for videochats
     WebrtcInstance,
 
@@ -170,11 +159,6 @@ pub enum Config {
     /// To how many seconds to debounce scan_all_folders. Used mainly in tests, to disable debouncing completely.
     #[strum(props(default = "60"))]
     ScanAllFoldersDebounceSecs,
-
-    /// Defines the max. size (in bytes) of messages downloaded automatically.
-    /// 0 = no limit.
-    #[strum(props(default = "0"))]
-    DownloadLimit,
 }
 
 impl Context {
@@ -322,7 +306,7 @@ impl Context {
                     .set_raw_config(key, value)
                     .await
                     .map_err(Into::into);
-                job::schedule_resync(self).await?;
+                job::schedule_resync(self).await;
                 ret
             }
             _ => {

src/configure.rs

@@ -1,4 +1,4 @@
-//! Email accounts autoconfiguration process module.
+//! Email accounts autoconfiguration process module
 
 mod auto_mozilla;
 mod auto_outlook;
@@ -14,7 +14,6 @@ use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 
 use crate::dc_tools::EmailAddress;
 use crate::imap::Imap;
-use crate::login_param::Socks5Config;
 use crate::login_param::{LoginParam, ServerLoginParam};
 use crate::message::Message;
 use crate::oauth2::dc_get_oauth2_addr;
@@ -171,17 +170,12 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         DC_LP_AUTH_NORMAL as i32
     };
 
-    let socks5_config = param.socks5_config.clone();
-    let socks5_enabled = socks5_config.is_some();
-
     let ctx2 = ctx.clone();
     let update_device_chats_handle = task::spawn(async move { ctx2.update_device_chats().await });
 
     // Step 1: Load the parameters and check email-address and password
 
-    // Do oauth2 only if socks5 is disabled. As soon as we have a http library that can do
-    // socks5 requests, this can work with socks5 too
-    if oauth2 && !socks5_enabled {
+    if oauth2 {
         // the used oauth2 addr may differ, check this.
         // if dc_get_oauth2_addr() is not available in the oauth2 implementation, just use the given one.
         progress!(ctx, 10);
@@ -223,7 +217,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             "checking internal provider-info for offline autoconfig"
         );
 
-        if let Some(provider) = provider::get_provider_info(&param_domain, socks5_enabled).await {
+        if let Some(provider) = provider::get_provider_info(&param_domain).await {
             param.provider = Some(provider);
             match provider.status {
                 provider::Status::Ok | provider::Status::Preparation => {
@@ -262,16 +256,9 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
                 }
             }
         } else {
-            // Try receiving autoconfig
             info!(ctx, "no offline autoconfig found");
-            param_autoconfig = if socks5_enabled {
-                // Currently we can't do http requests through socks5, to not leak
-                // the ip, just don't do online autoconfig
-                info!(ctx, "socks5 enabled, skipping autoconfig");
-                None
-            } else {
-                get_autoconfig(ctx, param, &param_domain, &param_addr_urlencoded).await
-            }
+            param_autoconfig =
+                get_autoconfig(ctx, param, &param_domain, &param_addr_urlencoded).await;
         }
     } else {
         param_autoconfig = None;
@@ -319,9 +306,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         .filter(|params| params.protocol == Protocol::Smtp)
         .cloned()
         .collect();
-    let provider_strict_tls = param
-        .provider
-        .map_or(socks5_config.is_some(), |provider| provider.strict_tls);
+    let provider_strict_tls = param.provider.map_or(false, |provider| provider.strict_tls);
 
     let smtp_config_task = task::spawn(async move {
         let mut smtp_configured = false;
@@ -335,7 +320,6 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
             match try_smtp_one_param(
                 &context_smtp,
                 &smtp_param,
-                &socks5_config,
                 &smtp_addr,
                 oauth2,
                 provider_strict_tls,
@@ -375,16 +359,7 @@ async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
         param.imap.port = imap_server.port;
         param.imap.security = imap_server.socket;
 
-        match try_imap_one_param(
-            ctx,
-            &param.imap,
-            &param.socks5_config,
-            &param.addr,
-            oauth2,
-            provider_strict_tls,
-        )
-        .await
-        {
+        match try_imap_one_param(ctx, &param.imap, &param.addr, oauth2, provider_strict_tls).await {
             Ok(configured_imap) => {
                 imap = Some(configured_imap);
                 break;
@@ -478,7 +453,7 @@ async fn get_autoconfig(
 
     if let Ok(res) = moz_autoconfigure(
         ctx,
-        // the doc does not mention `emailaddress=`, however, Thunderbird adds it, see <https://releases.mozilla.org/pub/thunderbird/>,  which makes some sense
+        // the doc does not mention `emailaddress=`, however, Thunderbird adds it, see https://releases.mozilla.org/pub/thunderbird/ ,  which makes some sense
         &format!(
             "https://{}/.well-known/autoconfig/mail/config-v1.1.xml?emailaddress={}",
             &param_domain, &param_addr_urlencoded
@@ -532,7 +507,6 @@ async fn get_autoconfig(
 async fn try_imap_one_param(
     context: &Context,
     param: &ServerLoginParam,
-    socks5_config: &Option<Socks5Config>,
     addr: &str,
     oauth2: bool,
     provider_strict_tls: bool,
@@ -545,16 +519,7 @@ async fn try_imap_one_param(
 
     let (_s, r) = async_std::channel::bounded(1);
 
-    let mut imap = match Imap::new(
-        param,
-        socks5_config.clone(),
-        addr,
-        oauth2,
-        provider_strict_tls,
-        r,
-    )
-    .await
-    {
+    let mut imap = match Imap::new(param, addr, oauth2, provider_strict_tls, r).await {
         Err(err) => {
             info!(context, "failure: {}", err);
             return Err(ConfigurationError {
@@ -568,14 +533,14 @@ async fn try_imap_one_param(
     match imap.connect(context).await {
         Err(err) => {
             info!(context, "failure: {}", err);
-            Err(ConfigurationError {
+            return Err(ConfigurationError {
                 config: inf,
                 msg: err.to_string(),
-            })
+            });
         }
         Ok(()) => {
             info!(context, "success: {}", inf);
-            Ok(imap)
+            return Ok(imap);
         }
     }
 }
@@ -583,37 +548,19 @@ async fn try_imap_one_param(
 async fn try_smtp_one_param(
     context: &Context,
     param: &ServerLoginParam,
-    socks5_config: &Option<Socks5Config>,
     addr: &str,
     oauth2: bool,
     provider_strict_tls: bool,
     smtp: &mut Smtp,
 ) -> Result<(), ConfigurationError> {
     let inf = format!(
-        "smtp: {}@{}:{} security={} certificate_checks={} oauth2={} socks5_config={}",
-        param.user,
-        param.server,
-        param.port,
-        param.security,
-        param.certificate_checks,
-        oauth2,
-        if let Some(socks5_config) = socks5_config {
-            socks5_config.to_string()
-        } else {
-            "None".to_string()
-        }
+        "smtp: {}@{}:{} security={} certificate_checks={} oauth2={}",
+        param.user, param.server, param.port, param.security, param.certificate_checks, oauth2
     );
     info!(context, "Trying: {}", inf);
 
     if let Err(err) = smtp
-        .connect(
-            context,
-            param,
-            socks5_config,
-            addr,
-            oauth2,
-            provider_strict_tls,
-        )
+        .connect(context, param, addr, oauth2, provider_strict_tls)
         .await
     {
         info!(context, "failure: {}", err);
@@ -671,10 +618,10 @@ pub enum Error {
     },
 
     #[error("Failed to get URL: {0}")]
-    ReadUrl(#[from] self::read_url::Error),
+    ReadUrlError(#[from] self::read_url::Error),
 
     #[error("Number of redirection is exceeded")]
-    Redirection,
+    RedirectionError,
 }
 
 #[cfg(test)]

src/configure/auto_mozilla.rs

@@ -1,6 +1,6 @@
 //! # Thunderbird's Autoconfiguration implementation
 //!
-//! Documentation: <https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration>
+//! Documentation: https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration
 use quick_xml::events::{BytesStart, Event};
 
 use std::io::BufRead;

src/configure/auto_outlook.rs

@@ -15,27 +15,27 @@ use super::{Error, ServerParams};
 
 /// Result of parsing a single `Protocol` tag.
 ///
-/// <https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/protocol-pox>
+/// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/protocol-pox
 #[derive(Debug)]
 struct ProtocolTag {
     /// Server type, such as "IMAP", "SMTP" or "POP3".
     ///
-    /// <https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/type-pox>
+    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/type-pox
     pub typ: String,
 
     /// Server identifier, hostname or IP address for IMAP and SMTP.
     ///
-    /// <https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/server-pox>
+    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/server-pox
     pub server: String,
 
     /// Network port.
     ///
-    /// <https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/port-pox>
+    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/port-pox
     pub port: u16,
 
     /// Whether connection should be secure, "on" or "off", default is "on".
     ///
-    /// <https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/ssl-pox>
+    /// https://docs.microsoft.com/en-us/exchange/client-developer/web-service-reference/ssl-pox
     pub ssl: bool,
 }
 
@@ -203,7 +203,7 @@ pub(crate) async fn outlk_autodiscover(
             }
         }
     }
-    Err(Error::Redirection)
+    Err(Error::RedirectionError)
 }
 
 #[cfg(test)]

src/configure/server_params.rs

@@ -25,20 +25,16 @@ pub(crate) struct ServerParams {
 }
 
 impl ServerParams {
-    fn expand_usernames(self, addr: &str) -> Vec<ServerParams> {
+    fn expand_usernames(mut self, addr: &str) -> Vec<ServerParams> {
         let mut res = Vec::new();
 
         if self.username.is_empty() {
-            res.push(Self {
-                username: addr.to_string(),
-                ..self.clone()
-            });
+            self.username = addr.to_string();
+            res.push(self.clone());
 
             if let Some(at) = addr.find('@') {
-                res.push(Self {
-                    username: addr.split_at(at).0.to_string(),
-                    ..self
-                });
+                self.username = addr.split_at(at).0.to_string();
+                res.push(self);
             }
         } else {
             res.push(self)
@@ -46,28 +42,24 @@ impl ServerParams {
         res
     }
 
-    fn expand_hostnames(self, param_domain: &str) -> Vec<ServerParams> {
+    fn expand_hostnames(mut self, param_domain: &str) -> Vec<ServerParams> {
+        let mut res = Vec::new();
         if self.hostname.is_empty() {
-            vec![
-                Self {
-                    hostname: param_domain.to_string(),
-                    ..self.clone()
-                },
-                Self {
-                    hostname: match self.protocol {
-                        Protocol::Imap => "imap.".to_string() + param_domain,
-                        Protocol::Smtp => "smtp.".to_string() + param_domain,
-                    },
-                    ..self.clone()
-                },
-                Self {
-                    hostname: "mail.".to_string() + param_domain,
-                    ..self
-                },
-            ]
+            self.hostname = param_domain.to_string();
+            res.push(self.clone());
+
+            self.hostname = match self.protocol {
+                Protocol::Imap => "imap.".to_string() + param_domain,
+                Protocol::Smtp => "smtp.".to_string() + param_domain,
+            };
+            res.push(self.clone());
+
+            self.hostname = "mail.".to_string() + param_domain;
+            res.push(self);
         } else {
-            vec![self]
+            res.push(self);
         }
+        res
     }
 
     fn expand_ports(mut self) -> Vec<ServerParams> {
@@ -86,47 +78,39 @@ impl ServerParams {
             }
         }
 
+        let mut res = Vec::new();
         if self.port == 0 {
             // Neither port nor security is set.
             //
             // Try common secure combinations.
 
-            vec![
-                // Try STARTTLS
-                Self {
-                    socket: Socket::Starttls,
-                    port: match self.protocol {
-                        Protocol::Imap => 143,
-                        Protocol::Smtp => 587,
-                    },
-                    ..self.clone()
-                },
-                // Try TLS
-                Self {
-                    socket: Socket::Ssl,
-                    port: match self.protocol {
-                        Protocol::Imap => 993,
-                        Protocol::Smtp => 465,
-                    },
-                    ..self
-                },
-            ]
+            // Try STARTTLS
+            self.socket = Socket::Starttls;
+            self.port = match self.protocol {
+                Protocol::Imap => 143,
+                Protocol::Smtp => 587,
+            };
+            res.push(self.clone());
+
+            // Try TLS
+            self.socket = Socket::Ssl;
+            self.port = match self.protocol {
+                Protocol::Imap => 993,
+                Protocol::Smtp => 465,
+            };
+            res.push(self);
         } else if self.socket == Socket::Automatic {
-            vec![
-                // Try TLS over user-provided port.
-                Self {
-                    socket: Socket::Ssl,
-                    ..self.clone()
-                },
-                // Try STARTTLS over user-provided port.
-                Self {
-                    socket: Socket::Starttls,
-                    ..self
-                },
-            ]
+            // Try TLS over user-provided port.
+            self.socket = Socket::Ssl;
+            res.push(self.clone());
+
+            // Try STARTTLS over user-provided port.
+            self.socket = Socket::Starttls;
+            res.push(self);
         } else {
-            vec![self]
+            res.push(self);
         }
+        res
     }
 }
 

src/constants.rs

@@ -1,4 +1,4 @@
-//! # Constants.
+//! # Constants
 use deltachat_derive::{FromSql, ToSql};
 use once_cell::sync::Lazy;
 use serde::{Deserialize, Serialize};
@@ -25,7 +25,7 @@ pub static DC_VERSION_STR: Lazy<String> = Lazy::new(|| env!("CARGO_PKG_VERSION")
 pub enum Blocked {
     Not = 0,
     Manually = 1,
-    Request = 2,
+    Deaddrop = 2,
 }
 
 impl Default for Blocked {
@@ -123,6 +123,8 @@ pub const DC_RESEND_USER_AVATAR_DAYS: i64 = 14;
 // do not use too small value that will annoy users checking for nonexistant updates.
 pub const DC_OUTDATED_WARNING_DAYS: i64 = 365;
 
+/// virtual chat showing all messages belonging to chats flagged with chats.blocked=2
+pub const DC_CHAT_ID_DEADDROP: ChatId = ChatId::new(1);
 /// messages that should be deleted get this chat_id; the messages are deleted from the working thread later then. This is also needed as rfc724_mid should be preset as long as the message is not deleted on the server (otherwise it is downloaded again)
 pub const DC_CHAT_ID_TRASH: ChatId = ChatId::new(3);
 /// only an indicator in a chatlist
@@ -165,18 +167,36 @@ pub const DC_MSG_ID_MARKER1: u32 = 1;
 pub const DC_MSG_ID_DAYMARKER: u32 = 9;
 pub const DC_MSG_ID_LAST_SPECIAL: u32 = 9;
 
-/// String that indicates that something is left out or truncated.
-pub const DC_ELLIPSIS: &str = "[...]";
+/// string that indicates sth. is left out or truncated
+pub const DC_ELLIPSE: &str = "[...]";
 
-/// Message length limit.
+/// to keep bubbles and chat flow usable,
+/// and to avoid problems with controls using very long texts,
+/// we limit the text length to DC_DESIRED_TEXT_LEN.
+/// if the text is longer, the full text can be retrieved usind has_html()/get_html().
 ///
-/// To keep bubbles and chat flow usable and to avoid problems with controls using very long texts,
-/// we limit the text length to `DC_DESIRED_TEXT_LEN`.  If the text is longer, the full text can be
-/// retrieved using has_html()/get_html().
+/// we are using a bit less than DC_MAX_GET_TEXT_LEN to avoid cutting twice
+/// (a bit less as truncation may not be exact and ellipses may be added).
 ///
-/// Note that for simplicity maximum length is defined as the number of Unicode Scalar Values (Rust
-/// `char`s), not Unicode Grapheme Clusters.
-pub const DC_DESIRED_TEXT_LEN: usize = 5000;
+/// note, that DC_DESIRED_TEXT_LEN and DC_MAX_GET_TEXT_LEN
+/// define max. number of bytes, _not_ unicode graphemes.
+/// in general, that seems to be okay for such an upper limit,
+/// esp. as calculating the number of graphemes is not simple
+/// (one graphemes may be a sequence of code points which is a sequence of bytes).
+/// also even if we have the exact number of graphemes,
+/// that would not always help on getting an idea about the screen space used
+/// (to keep bubbles and chat flow usable).
+///
+/// therefore, the number of bytes is only a very rough estimation,
+/// however, the ~30K seems to work okayish for a while,
+/// if it turns out, it is too few for some alphabet, we can still increase.
+pub const DC_DESIRED_TEXT_LEN: usize = 29_000;
+
+/// approx. max. length (number of bytes) returned by dc_msg_get_text()
+pub const DC_MAX_GET_TEXT_LEN: usize = 30_000;
+
+/// approx. max. length returned by dc_get_msg_info()
+pub const DC_MAX_GET_INFO_LEN: usize = 100_000;
 
 pub const DC_CONTACT_ID_UNDEFINED: u32 = 0;
 pub const DC_CONTACT_ID_SELF: u32 = 1;
@@ -384,7 +404,7 @@ mod tests {
         assert_eq!(Blocked::Not, Blocked::default());
         assert_eq!(Blocked::Not, Blocked::from_i32(0).unwrap());
         assert_eq!(Blocked::Manually, Blocked::from_i32(1).unwrap());
-        assert_eq!(Blocked::Request, Blocked::from_i32(2).unwrap());
+        assert_eq!(Blocked::Deaddrop, Blocked::from_i32(2).unwrap());
     }
 
     #[test]

src/contact.rs

@@ -14,8 +14,8 @@ use crate::chat::ChatId;
 use crate::color::str_to_color;
 use crate::config::Config;
 use crate::constants::{
-    Blocked, Chattype, DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_DEVICE_ADDR, DC_CONTACT_ID_LAST_SPECIAL,
-    DC_CONTACT_ID_SELF, DC_GCL_ADD_SELF, DC_GCL_VERIFIED_ONLY,
+    Blocked, Chattype, DC_CHAT_ID_DEADDROP, DC_CONTACT_ID_DEVICE, DC_CONTACT_ID_DEVICE_ADDR,
+    DC_CONTACT_ID_LAST_SPECIAL, DC_CONTACT_ID_SELF, DC_GCL_ADD_SELF, DC_GCL_VERIFIED_ONLY,
 };
 use crate::context::Context;
 use crate::dc_tools::{dc_get_abs_path, improve_single_line_input, EmailAddress};
@@ -218,7 +218,6 @@ impl Contact {
         } else if contact_id == DC_CONTACT_ID_DEVICE {
             contact.name = stock_str::device_messages(context).await;
             contact.addr = DC_CONTACT_ID_DEVICE_ADDR.to_string();
-            contact.status = stock_str::device_messages_hint(context).await;
         }
         Ok(contact)
     }
@@ -278,15 +277,20 @@ impl Contact {
     }
 
     /// Mark messages from a contact as noticed.
-    pub async fn mark_noticed(context: &Context, id: u32) -> Result<()> {
-        context
+    /// The contact is expected to belong to the deaddrop,
+    /// therefore, DC_EVENT_MSGS_NOTICED(DC_CHAT_ID_DEADDROP) is emitted.
+    pub async fn mark_noticed(context: &Context, id: u32) {
+        if context
             .sql
             .execute(
                 "UPDATE msgs SET state=? WHERE from_id=? AND state=?;",
                 paramsv![MessageState::InNoticed, id as i32, MessageState::InFresh],
             )
-            .await?;
-        Ok(())
+            .await
+            .is_ok()
+        {
+            context.emit_event(EventType::MsgsNoticed(DC_CHAT_ID_DEADDROP));
+        }
     }
 
     /// Check if an e-mail address belongs to a known and unblocked contact.
@@ -1109,19 +1113,15 @@ impl Contact {
             .unwrap_or_default()
     }
 
-    pub async fn scaleup_origin_by_id(
-        context: &Context,
-        contact_id: u32,
-        origin: Origin,
-    ) -> Result<()> {
+    pub async fn scaleup_origin_by_id(context: &Context, contact_id: u32, origin: Origin) -> bool {
         context
             .sql
             .execute(
                 "UPDATE contacts SET origin=? WHERE id=? AND origin<?;",
                 paramsv![origin, contact_id as i32, origin],
             )
-            .await?;
-        Ok(())
+            .await
+            .is_ok()
     }
 }
 
@@ -1200,16 +1200,15 @@ WHERE type=? AND id IN (
             .await
             .is_ok()
         {
-            Contact::mark_noticed(context, contact_id).await?;
+            Contact::mark_noticed(context, contact_id).await;
             context.emit_event(EventType::ContactsChanged(Some(contact_id)));
         }
 
         // also unblock mailinglist
         // if the contact is a mailinglist address explicitly created to allow unblocking
         if !new_blocking && contact.origin == Origin::MailinglistAddress {
-            if let Some((chat_id, _, _)) = chat::get_chat_id_by_grpid(context, contact.addr).await?
-            {
-                chat_id.unblock(context).await?;
+            if let Ok((chat_id, _, _)) = chat::get_chat_id_by_grpid(context, contact.addr).await {
+                chat_id.set_blocked(context, Blocked::Not).await;
             }
         }
     }
@@ -1391,12 +1390,12 @@ mod tests {
         assert_eq!(may_be_valid_addr("user@domain.tld"), true);
         assert_eq!(may_be_valid_addr("uuu"), false);
         assert_eq!(may_be_valid_addr("dd.tt"), false);
-        assert_eq!(may_be_valid_addr("tt.dd@uu"), true);
-        assert_eq!(may_be_valid_addr("u@d"), true);
-        assert_eq!(may_be_valid_addr("u@d."), true);
-        assert_eq!(may_be_valid_addr("u@d.t"), true);
+        assert_eq!(may_be_valid_addr("tt.dd@uu"), false);
+        assert_eq!(may_be_valid_addr("u@d"), false);
+        assert_eq!(may_be_valid_addr("u@d."), false);
+        assert_eq!(may_be_valid_addr("u@d.t"), false);
         assert_eq!(may_be_valid_addr("u@d.tt"), true);
-        assert_eq!(may_be_valid_addr("u@.tt"), true);
+        assert_eq!(may_be_valid_addr("u@.tt"), false);
         assert_eq!(may_be_valid_addr("@d.tt"), false);
         assert_eq!(may_be_valid_addr("<da@d.tt"), false);
         assert_eq!(may_be_valid_addr("sk <@d.tt>"), false);
@@ -1840,6 +1839,9 @@ mod tests {
         assert!(Contact::create(&t, "", "dskjfdslk@sadklj.dk>")
             .await
             .is_err());
+        assert!(Contact::create(&t, "", "dskjf@dslk@sadkljdk")
+            .await
+            .is_err());
         assert!(Contact::create(&t, "", "dskjf dslk@d.e").await.is_err());
         assert!(Contact::create(&t, "", "<dskjf dslk@sadklj.dk")
             .await

src/context.rs

@@ -1,4 +1,4 @@
-//! Context module.
+//! Context module
 
 use std::collections::{BTreeMap, HashMap};
 use std::ffi::OsString;
@@ -22,7 +22,6 @@ use crate::events::{Event, EventEmitter, EventType, Events};
 use crate::key::{DcKey, SignedPublicKey};
 use crate::login_param::LoginParam;
 use crate::message::{self, MessageState, MsgId};
-use crate::quota::QuotaInfo;
 use crate::scheduler::Scheduler;
 use crate::securejoin::Bob;
 use crate::sql::Sql;
@@ -63,10 +62,6 @@ pub struct InnerContext {
     pub(crate) scheduler: RwLock<Scheduler>,
     pub(crate) ephemeral_task: RwLock<Option<task::JoinHandle<()>>>,
 
-    /// Recently loaded quota information, if any.
-    /// Set to `None` if quota was never tried to load.
-    pub(crate) quota: RwLock<Option<QuotaInfo>>,
-
     pub(crate) last_full_folder_scan: Mutex<Option<Instant>>,
 
     /// ID for this `Context` in the current process.
@@ -144,7 +139,6 @@ impl Context {
             events: Events::default(),
             scheduler: RwLock::new(Scheduler::Stopped),
             ephemeral_task: RwLock::new(None),
-            quota: RwLock::new(None),
             creation_time: std::time::SystemTime::now(),
             last_full_folder_scan: Mutex::new(None),
         };
@@ -285,11 +279,10 @@ impl Context {
         let l2 = LoginParam::from_database(self, "configured_").await?;
         let displayname = self.get_config(Config::Displayname).await?;
         let chats = get_chat_cnt(self).await? as usize;
-        let unblocked_msgs = message::get_unblocked_msg_cnt(self).await as usize;
-        let request_msgs = message::get_request_msg_cnt(self).await as usize;
+        let real_msgs = message::get_real_msg_cnt(self).await as usize;
+        let deaddrop_msgs = message::get_deaddrop_msg_cnt(self).await as usize;
         let contacts = Contact::get_real_cnt(self).await? as usize;
         let is_configured = self.get_config_int(Config::Configured).await?;
-        let socks5_enabled = self.get_config_int(Config::Socks5Enabled).await?;
         let dbversion = self
             .sql
             .get_raw_config_int("dbversion")
@@ -343,8 +336,8 @@ impl Context {
         // insert values
         res.insert("bot", self.get_config_int(Config::Bot).await?.to_string());
         res.insert("number_of_chats", chats.to_string());
-        res.insert("number_of_chat_messages", unblocked_msgs.to_string());
-        res.insert("messages_in_contact_requests", request_msgs.to_string());
+        res.insert("number_of_chat_messages", real_msgs.to_string());
+        res.insert("messages_in_contact_requests", deaddrop_msgs.to_string());
         res.insert("number_of_contacts", contacts.to_string());
         res.insert("database_dir", self.get_dbfile().display().to_string());
         res.insert("database_version", dbversion.to_string());
@@ -358,7 +351,6 @@ impl Context {
                 .unwrap_or_else(|| "<unset>".to_string()),
         );
         res.insert("is_configured", is_configured.to_string());
-        res.insert("socks5_enabled", socks5_enabled.to_string());
         res.insert("entered_account_settings", l.to_string());
         res.insert("used_account_settings", l2.to_string());
         res.insert(
@@ -371,12 +363,6 @@ impl Context {
             "show_emails",
             self.get_config_int(Config::ShowEmails).await?.to_string(),
         );
-        res.insert(
-            "download_limit",
-            self.get_config_int(Config::DownloadLimit)
-                .await?
-                .to_string(),
-        );
         res.insert("inbox_watch", inbox_watch.to_string());
         res.insert("sentbox_watch", sentbox_watch.to_string());
         res.insert("mvbox_watch", mvbox_watch.to_string());
@@ -429,12 +415,6 @@ impl Context {
                 .await?
                 .to_string(),
         );
-        res.insert(
-            "quota_exceeding",
-            self.get_config_int(Config::QuotaExceeding)
-                .await?
-                .to_string(),
-        );
 
         let elapsed = self.creation_time.elapsed();
         res.insert("uptime", duration_to_str(elapsed.unwrap_or_default()));
@@ -442,7 +422,7 @@ impl Context {
         Ok(res)
     }
 
-    /// Get a list of fresh, unmuted messages in unblocked chats.
+    /// Get a list of fresh, unmuted messages in any chat but deaddrop.
     ///
     /// The list starts with the most recent message
     /// and is typically used to show notifications.
@@ -579,13 +559,6 @@ impl Context {
         blob_fname.push("-blobs");
         dbfile.with_file_name(blob_fname)
     }
-
-    pub fn derive_walfile(dbfile: &PathBuf) -> PathBuf {
-        let mut wal_fname = OsString::new();
-        wal_fname.push(dbfile.file_name().unwrap_or_default());
-        wal_fname.push("-wal");
-        dbfile.with_file_name(wal_fname)
-    }
 }
 
 impl InnerContext {
@@ -892,10 +865,6 @@ mod tests {
             "send_security",
             "server_flags",
             "smtp_certificate_checks",
-            "socks5_host",
-            "socks5_port",
-            "socks5_user",
-            "socks5_password",
         ];
         let t = TestContext::new().await;
         let info = t.get_info().await.unwrap();

src/dc_tools.rs

@@ -17,7 +17,7 @@ use chrono::{Local, TimeZone};
 use rand::{thread_rng, Rng};
 
 use crate::chat::{add_device_msg, add_device_msg_with_importance};
-use crate::constants::{Viewtype, DC_ELLIPSIS, DC_OUTDATED_WARNING_DAYS};
+use crate::constants::{Viewtype, DC_ELLIPSE, DC_OUTDATED_WARNING_DAYS};
 use crate::context::Context;
 use crate::events::EventType;
 use crate::message::Message;
@@ -29,7 +29,7 @@ use crate::stock_str;
 #[allow(clippy::indexing_slicing)]
 pub(crate) fn dc_truncate(buf: &str, approx_chars: usize) -> Cow<str> {
     let count = buf.chars().count();
-    if count > approx_chars + DC_ELLIPSIS.len() {
+    if approx_chars > 0 && count > approx_chars + DC_ELLIPSE.len() {
         let end_pos = buf
             .char_indices()
             .nth(approx_chars)
@@ -37,9 +37,9 @@ pub(crate) fn dc_truncate(buf: &str, approx_chars: usize) -> Cow<str> {
             .unwrap_or_default();
 
         if let Some(index) = buf[..end_pos].rfind(|c| c == ' ' || c == '\n') {
-            Cow::Owned(format!("{}{}", &buf[..=index], DC_ELLIPSIS))
+            Cow::Owned(format!("{}{}", &buf[..=index], DC_ELLIPSE))
         } else {
-            Cow::Owned(format!("{}{}", &buf[..end_pos], DC_ELLIPSIS))
+            Cow::Owned(format!("{}{}", &buf[..end_pos], DC_ELLIPSE))
         }
     } else {
         Cow::Borrowed(buf)
@@ -608,8 +608,19 @@ impl FromStr for EmailAddress {
                 if local.is_empty() {
                     return err("empty string is not valid for local part");
                 }
-                if domain.is_empty() {
-                    return err("missing domain after '@'");
+                if domain.len() <= 3 {
+                    return err("domain is too short");
+                }
+                let dot = domain.find('.');
+                match dot {
+                    None => {
+                        return err("invalid domain");
+                    }
+                    Some(dot_idx) => {
+                        if dot_idx >= domain.len() - 2 {
+                            return err("invalid domain");
+                        }
+                    }
                 }
                 Ok(EmailAddress {
                     local: (*local).to_string(),
@@ -655,7 +666,7 @@ pub fn remove_subject_prefix(last_subject: &str) -> String {
         0
     } else {
         // "Antw:" is the longest abbreviation in
-        // <https://en.wikipedia.org/wiki/List_of_email_subject_abbreviations#Abbreviations_in_other_languages>,
+        // https://en.wikipedia.org/wiki/List_of_email_subject_abbreviations#Abbreviations_in_other_languages,
         // so look at the first _5_ characters:
         match last_subject.chars().take(5).position(|c| c == ':') {
             Some(prefix_end) => prefix_end + 1,
@@ -711,7 +722,10 @@ mod tests {
         assert_eq!(dc_truncate("\n  hello \n world", 4), "\n  [...]");
 
         assert_eq!(dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 1), "𐠈[...]");
-        assert_eq!(dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 0), "[...]");
+        assert_eq!(
+            dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 0),
+            "𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ"
+        );
 
         // 9 characters, so no truncation
         assert_eq!(dc_truncate("𑒀ὐ￠🜀\u{1e01b}A a🟠", 6), "𑒀ὐ￠🜀\u{1e01b}A a🟠",);
@@ -811,19 +825,12 @@ mod tests {
                 domain: "domain.tld".into(),
             }
         );
-        assert_eq!(
-            "user@localhost".parse::<EmailAddress>().unwrap(),
-            EmailAddress {
-                local: "user".into(),
-                domain: "localhost".into()
-            }
-        );
         assert_eq!("uuu".parse::<EmailAddress>().is_ok(), false);
         assert_eq!("dd.tt".parse::<EmailAddress>().is_ok(), false);
-        assert!("tt.dd@uu".parse::<EmailAddress>().is_ok());
-        assert!("u@d".parse::<EmailAddress>().is_ok());
-        assert!("u@d.".parse::<EmailAddress>().is_ok());
-        assert!("u@d.t".parse::<EmailAddress>().is_ok());
+        assert_eq!("tt.dd@uu".parse::<EmailAddress>().is_ok(), false);
+        assert_eq!("u@d".parse::<EmailAddress>().is_ok(), false);
+        assert_eq!("u@d.".parse::<EmailAddress>().is_ok(), false);
+        assert_eq!("u@d.t".parse::<EmailAddress>().is_ok(), false);
         assert_eq!(
             "u@d.tt".parse::<EmailAddress>().unwrap(),
             EmailAddress {
@@ -831,7 +838,7 @@ mod tests {
                 domain: "d.tt".into(),
             }
         );
-        assert!("u@tt".parse::<EmailAddress>().is_ok());
+        assert_eq!("u@tt".parse::<EmailAddress>().is_ok(), false);
         assert_eq!("@d.tt".parse::<EmailAddress>().is_ok(), false);
     }
 

src/dehtml.rs

@@ -1,4 +1,4 @@
-//! De-HTML.
+//! De-HTML
 //!
 //! A module to remove HTML tags from the email text
 

src/download.rs

@@ -1,346 +0,0 @@
-//! # Download large messages manually.
-
-use anyhow::{anyhow, Result};
-use deltachat_derive::{FromSql, ToSql};
-use serde::{Deserialize, Serialize};
-
-use crate::config::Config;
-use crate::constants::Viewtype;
-use crate::context::Context;
-use crate::dc_tools::time;
-use crate::imap::{Imap, ImapActionResult};
-use crate::job::{self, Action, Job, Status};
-use crate::message::{Message, MsgId};
-use crate::mimeparser::{MimeMessage, Part};
-use crate::param::Params;
-use crate::{job_try, stock_str, EventType};
-use std::cmp::max;
-
-/// Download limits should not be used below `MIN_DOWNLOAD_LIMIT`.
-///
-/// Some messages as non-delivery-reports (NDN) or read-receipts (MDN)
-/// need to be downloaded completely to handle them correctly,
-/// eg. to assign them to the correct chat.
-/// As these messages are typically small,
-/// they're catched by `MIN_DOWNLOAD_LIMIT`.
-const MIN_DOWNLOAD_LIMIT: u32 = 32768;
-
-/// If a message is downloaded only partially
-/// and `delete_server_after` is set to small timeouts (eg. "at once"),
-/// the user might have no chance to actually download that message.
-/// `MIN_DELETE_SERVER_AFTER` increases the timeout in this case.
-pub(crate) const MIN_DELETE_SERVER_AFTER: i64 = 48 * 60 * 60;
-
-#[derive(
-    Debug,
-    Display,
-    Clone,
-    Copy,
-    PartialEq,
-    Eq,
-    FromPrimitive,
-    ToPrimitive,
-    FromSql,
-    ToSql,
-    Serialize,
-    Deserialize,
-)]
-#[repr(u32)]
-pub enum DownloadState {
-    Done = 0,
-    Available = 10,
-    Failure = 20,
-    InProgress = 1000,
-}
-
-impl Default for DownloadState {
-    fn default() -> Self {
-        DownloadState::Done
-    }
-}
-
-impl Context {
-    // Returns validated download limit or `None` for "no limit".
-    pub(crate) async fn download_limit(&self) -> Result<Option<u32>> {
-        let download_limit = self.get_config_int(Config::DownloadLimit).await?;
-        if download_limit <= 0 {
-            Ok(None)
-        } else {
-            Ok(Some(max(MIN_DOWNLOAD_LIMIT, download_limit as u32)))
-        }
-    }
-}
-
-impl MsgId {
-    /// Schedules full message download for partially downloaded message.
-    pub async fn download_full(self, context: &Context) -> Result<()> {
-        let msg = Message::load_from_db(context, self).await?;
-        match msg.download_state() {
-            DownloadState::Done => return Err(anyhow!("Nothing to download.")),
-            DownloadState::InProgress => return Err(anyhow!("Download already in progress.")),
-            DownloadState::Available | DownloadState::Failure => {
-                self.update_download_state(context, DownloadState::InProgress)
-                    .await?;
-                job::add(
-                    context,
-                    Job::new(Action::DownloadMsg, self.to_u32(), Params::new(), 0),
-                )
-                .await;
-            }
-        }
-        Ok(())
-    }
-
-    pub(crate) async fn update_download_state(
-        self,
-        context: &Context,
-        download_state: DownloadState,
-    ) -> Result<()> {
-        let msg = Message::load_from_db(context, self).await?;
-        context
-            .sql
-            .execute(
-                "UPDATE msgs SET download_state=? WHERE id=?;",
-                paramsv![download_state, self],
-            )
-            .await?;
-        context.emit_event(EventType::MsgsChanged {
-            chat_id: msg.chat_id,
-            msg_id: self,
-        });
-        Ok(())
-    }
-}
-
-impl Message {
-    /// Returns the download state of the message.
-    pub fn download_state(&self) -> DownloadState {
-        self.download_state
-    }
-}
-
-impl Job {
-    /// Actually download a message.
-    /// Called in response to `Action::DownloadMsg`.
-    pub(crate) async fn download_msg(&self, context: &Context, imap: &mut Imap) -> Status {
-        if let Err(err) = imap.prepare(context).await {
-            warn!(context, "download: could not connect: {:?}", err);
-            return Status::RetryNow;
-        }
-
-        let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)).await);
-        let server_folder = msg.server_folder.unwrap_or_default();
-        match imap
-            .fetch_single_msg(context, &server_folder, msg.server_uid)
-            .await
-        {
-            ImapActionResult::RetryLater | ImapActionResult::Failed => {
-                job_try!(
-                    msg.id
-                        .update_download_state(context, DownloadState::Failure)
-                        .await
-                );
-                Status::Finished(Err(anyhow!("Call download_full() again to try over.")))
-            }
-            ImapActionResult::Success | ImapActionResult::AlreadyDone => {
-                // update_download_state() not needed as receive_imf() already
-                // set the state and emitted the event.
-                Status::Finished(Ok(()))
-            }
-        }
-    }
-}
-
-impl Imap {
-    /// Download a single message and pipe it to receive_imf().
-    ///
-    /// receive_imf() is not directly aware that this is a result of a call to download_msg(),
-    /// however, implicitly knows that as the existing message is flagged as being partly.
-    async fn fetch_single_msg(
-        &mut self,
-        context: &Context,
-        folder: &str,
-        uid: u32,
-    ) -> ImapActionResult {
-        if let Some(imapresult) = self
-            .prepare_imap_operation_on_msg(context, folder, uid)
-            .await
-        {
-            return imapresult;
-        }
-
-        // we are connected, and the folder is selected
-        info!(context, "Downloading message {}/{} fully...", folder, uid);
-
-        let (_, error_cnt) = self
-            .fetch_many_msgs(context, folder, vec![uid], false, false)
-            .await;
-        if error_cnt > 0 {
-            return ImapActionResult::Failed;
-        }
-
-        ImapActionResult::Success
-    }
-}
-
-impl MimeMessage {
-    /// Creates a placeholder part and add that to `parts`.
-    ///
-    /// To create the placeholder, only the outermost header can be used,
-    /// the mime-structure itself is not available.
-    ///
-    /// The placeholder part currently contains a text with size and availability of the message;
-    /// in the future, we may do more advanced things as previews here.
-    pub(crate) async fn create_stub_from_partial_download(
-        &mut self,
-        context: &Context,
-        org_bytes: u32,
-    ) -> Result<()> {
-        let mut text = format!(
-            "[{}]",
-            stock_str::partial_download_msg_body(context, org_bytes).await
-        );
-        if let Some(delete_server_after) = context.get_config_delete_server_after().await? {
-            let until = stock_str::download_availability(
-                context,
-                time() + max(delete_server_after, MIN_DELETE_SERVER_AFTER),
-            )
-            .await;
-            text += format!(" [{}]", until).as_str();
-        };
-
-        info!(context, "Partial download: {}", text);
-
-        self.parts.push(Part {
-            typ: Viewtype::Text,
-            msg: text,
-            ..Default::default()
-        });
-
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::chat::send_msg;
-    use crate::constants::Viewtype;
-    use crate::dc_receive_imf::dc_receive_imf_inner;
-    use crate::test_utils::TestContext;
-    use num_traits::FromPrimitive;
-
-    #[test]
-    fn test_downloadstate_values() {
-        // values may be written to disk and must not change
-        assert_eq!(DownloadState::Done, DownloadState::default());
-        assert_eq!(DownloadState::Done, DownloadState::from_i32(0).unwrap());
-        assert_eq!(
-            DownloadState::Available,
-            DownloadState::from_i32(10).unwrap()
-        );
-        assert_eq!(DownloadState::Failure, DownloadState::from_i32(20).unwrap());
-        assert_eq!(
-            DownloadState::InProgress,
-            DownloadState::from_i32(1000).unwrap()
-        );
-    }
-
-    #[async_std::test]
-    async fn test_download_limit() -> Result<()> {
-        let t = TestContext::new_alice().await;
-
-        assert_eq!(t.download_limit().await?, None);
-
-        t.set_config(Config::DownloadLimit, Some("200000")).await?;
-        assert_eq!(t.download_limit().await?, Some(200000));
-
-        t.set_config(Config::DownloadLimit, Some("20000")).await?;
-        assert_eq!(t.download_limit().await?, Some(MIN_DOWNLOAD_LIMIT));
-
-        t.set_config(Config::DownloadLimit, None).await?;
-        assert_eq!(t.download_limit().await?, None);
-
-        for val in &["0", "-1", "-100", "", "foo"] {
-            t.set_config(Config::DownloadLimit, Some(val)).await?;
-            assert_eq!(t.download_limit().await?, None);
-        }
-
-        Ok(())
-    }
-
-    #[async_std::test]
-    async fn test_update_download_state() -> Result<()> {
-        let t = TestContext::new_alice().await;
-        let chat = t.create_chat_with_contact("Bob", "bob@example.org").await;
-
-        let mut msg = Message::new(Viewtype::Text);
-        msg.set_text(Some("Hi Bob".to_owned()));
-        let msg_id = send_msg(&t, chat.id, &mut msg).await?;
-        let msg = Message::load_from_db(&t, msg_id).await?;
-        assert_eq!(msg.download_state(), DownloadState::Done);
-
-        for s in &[
-            DownloadState::Available,
-            DownloadState::InProgress,
-            DownloadState::Failure,
-            DownloadState::Done,
-        ] {
-            msg_id.update_download_state(&t, *s).await?;
-            let msg = Message::load_from_db(&t, msg_id).await?;
-            assert_eq!(msg.download_state(), *s);
-        }
-
-        Ok(())
-    }
-
-    #[async_std::test]
-    async fn test_partial_receive_imf() -> Result<()> {
-        let t = TestContext::new_alice().await;
-
-        let header =
-            "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
-             From: bob@example.com\n\
-             To: alice@example.org\n\
-             Subject: foo\n\
-             Message-ID: <Mr.12345678901@example.com>\n\
-             Chat-Version: 1.0\n\
-             Date: Sun, 22 Mar 2020 22:37:57 +0000\
-             Content-Type: text/plain";
-
-        dc_receive_imf_inner(
-            &t,
-            header.as_bytes(),
-            "INBOX",
-            1,
-            false,
-            Some(100000),
-            false,
-        )
-        .await?;
-        let msg = t.get_last_msg().await;
-        assert_eq!(msg.download_state(), DownloadState::Available);
-        assert_eq!(msg.get_subject(), "foo");
-        assert!(msg
-            .get_text()
-            .unwrap()
-            .contains(&stock_str::partial_download_msg_body(&t, 100000).await));
-
-        dc_receive_imf_inner(
-            &t,
-            format!("{}\n\n100k text...", header).as_bytes(),
-            "INBOX",
-            1,
-            false,
-            None,
-            false,
-        )
-        .await?;
-        let msg = t.get_last_msg().await;
-        assert_eq!(msg.download_state(), DownloadState::Done);
-        assert_eq!(msg.get_subject(), "foo");
-        assert_eq!(msg.get_text(), Some("100k text...".to_string()));
-
-        Ok(())
-    }
-}

src/e2ee.rs

@@ -178,9 +178,7 @@ pub async fn try_decrypt(
     let mut signatures = HashSet::default();
 
     if let Some(ref mut peerstate) = peerstate {
-        peerstate
-            .handle_fingerprint_change(context, message_time)
-            .await?;
+        peerstate.handle_fingerprint_change(context).await?;
         if let Some(key) = &peerstate.public_key {
             public_keyring_for_validate.add(key.clone());
         } else if let Some(key) = &peerstate.gossip_key {
@@ -335,7 +333,7 @@ fn has_decrypted_pgp_armor(input: &[u8]) -> bool {
     false
 }
 
-/// Checks if a MIME structure contains a multipart/report part.
+/// Check if a MIME structure contains a multipart/report part.
 ///
 /// As reports are often unencrypted, we do not reset the Autocrypt header in
 /// this case.

src/ephemeral.rs

@@ -1,4 +1,4 @@
-//! # Ephemeral messages.
+//! # Ephemeral messages
 //!
 //! Ephemeral messages are messages that have an Ephemeral-Timer
 //! header attached to them, which specifies time in seconds after
@@ -71,13 +71,11 @@ use crate::constants::{
 };
 use crate::context::Context;
 use crate::dc_tools::time;
-use crate::download::MIN_DELETE_SERVER_AFTER;
 use crate::events::EventType;
 use crate::job;
 use crate::message::{Message, MessageState, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::stock_str;
-use std::cmp::max;
 
 #[derive(Debug, PartialEq, Eq, Copy, Clone, Serialize, Deserialize)]
 pub enum Timer {
@@ -281,7 +279,7 @@ impl MsgId {
     /// Starts ephemeral message timer for the message if it is not started yet.
     pub(crate) async fn start_ephemeral_timer(self, context: &Context) -> anyhow::Result<()> {
         if let Timer::Enabled { duration } = self.ephemeral_timer(context).await? {
-            let ephemeral_timestamp = time().saturating_add(duration.into());
+            let ephemeral_timestamp = time() + i64::from(duration);
 
             context
                 .sql
@@ -418,18 +416,24 @@ pub async fn schedule_ephemeral_task(context: &Context) {
             let context1 = context.clone();
             let ephemeral_task = task::spawn(async move {
                 async_std::task::sleep(duration).await;
-                context1.emit_event(EventType::MsgsChanged {
-                    chat_id: ChatId::new(0),
-                    msg_id: MsgId::new(0),
-                });
+                emit_event!(
+                    context1,
+                    EventType::MsgsChanged {
+                        chat_id: ChatId::new(0),
+                        msg_id: MsgId::new(0)
+                    }
+                );
             });
             *context.ephemeral_task.write().await = Some(ephemeral_task);
         } else {
             // Emit event immediately
-            context.emit_event(EventType::MsgsChanged {
-                chat_id: ChatId::new(0),
-                msg_id: MsgId::new(0),
-            });
+            emit_event!(
+                context,
+                EventType::MsgsChanged {
+                    chat_id: ChatId::new(0),
+                    msg_id: MsgId::new(0)
+                }
+            );
         }
     }
 }
@@ -441,32 +445,23 @@ pub async fn schedule_ephemeral_task(context: &Context) {
 pub(crate) async fn load_imap_deletion_msgid(context: &Context) -> anyhow::Result<Option<MsgId>> {
     let now = time();
 
-    let (threshold_timestamp, threshold_timestamp_extended) =
-        match context.get_config_delete_server_after().await? {
-            None => (0, 0),
-            Some(delete_server_after) => (
-                now - delete_server_after,
-                now - max(delete_server_after, MIN_DELETE_SERVER_AFTER),
-            ),
-        };
+    let threshold_timestamp = match context.get_config_delete_server_after().await? {
+        None => 0,
+        Some(delete_server_after) => now - delete_server_after,
+    };
 
     context
         .sql
         .query_row_optional(
             "SELECT id FROM msgs \
          WHERE ( \
-         ((download_state = 0 AND timestamp < ?) OR (download_state != 0 AND timestamp < ?)) \
+         timestamp < ? \
          OR (ephemeral_timestamp != 0 AND ephemeral_timestamp <= ?) \
          ) \
          AND server_uid != 0 \
          AND NOT id IN (SELECT foreign_id FROM jobs WHERE action = ?)
          LIMIT 1",
-            paramsv![
-                threshold_timestamp,
-                threshold_timestamp_extended,
-                now,
-                job::Action::DeleteMsgOnImap
-            ],
+            paramsv![threshold_timestamp, now, job::Action::DeleteMsgOnImap],
             |row| {
                 let msg_id: MsgId = row.get(0)?;
                 Ok(msg_id)
@@ -511,8 +506,6 @@ mod tests {
     use async_std::task::sleep;
 
     use super::*;
-    use crate::config::Config;
-    use crate::download::DownloadState;
     use crate::test_utils::TestContext;
     use crate::{
         chat::{self, Chat, ChatItem},
@@ -784,58 +777,4 @@ mod tests {
             assert!(rawtxt.is_none_or_empty(), "{:?}", rawtxt);
         }
     }
-
-    #[async_std::test]
-    async fn test_load_imap_deletion_msgid() -> Result<()> {
-        let t = TestContext::new_alice().await;
-        const HOUR: i64 = 60 * 60;
-        let now = time();
-        for (id, timestamp, ephemeral_timestamp) in &[
-            (900, now - 2 * HOUR, 0),
-            (1000, now - 23 * HOUR - MIN_DELETE_SERVER_AFTER, 0),
-            (1010, now - 23 * HOUR, 0),
-            (1020, now - 21 * HOUR, 0),
-            (1030, now - 19 * HOUR, 0),
-            (2000, now - 18 * HOUR, now - HOUR),
-            (2020, now - 17 * HOUR, now + HOUR),
-        ] {
-            t.sql
-                .execute(
-                    "INSERT INTO msgs (id, server_uid, timestamp, ephemeral_timestamp) VALUES (?,?,?,?);",
-                    paramsv![id, id, timestamp, ephemeral_timestamp],
-                )
-                .await?;
-        }
-
-        assert_eq!(load_imap_deletion_msgid(&t).await?, Some(MsgId::new(2000)));
-
-        MsgId::new(2000).delete_from_db(&t).await?;
-        assert_eq!(load_imap_deletion_msgid(&t).await?, None);
-
-        t.set_config(Config::DeleteServerAfter, Some(&*(25 * HOUR).to_string()))
-            .await?;
-        assert_eq!(load_imap_deletion_msgid(&t).await?, Some(MsgId::new(1000)));
-
-        MsgId::new(1000)
-            .update_download_state(&t, DownloadState::Available)
-            .await?;
-        assert_eq!(load_imap_deletion_msgid(&t).await?, Some(MsgId::new(1000))); // delete downloadable anyway
-
-        MsgId::new(1000).delete_from_db(&t).await?;
-        assert_eq!(load_imap_deletion_msgid(&t).await?, None);
-
-        t.set_config(Config::DeleteServerAfter, Some(&*(22 * HOUR).to_string()))
-            .await?;
-        assert_eq!(load_imap_deletion_msgid(&t).await?, Some(MsgId::new(1010)));
-
-        MsgId::new(1010)
-            .update_download_state(&t, DownloadState::Available)
-            .await?;
-        assert_eq!(load_imap_deletion_msgid(&t).await?, None); // keep downloadable for now
-
-        MsgId::new(1010).delete_from_db(&t).await?;
-        assert_eq!(load_imap_deletion_msgid(&t).await?, None);
-
-        Ok(())
-    }
 }

src/events.rs

@@ -1,4 +1,4 @@
-//! # Events specification.
+//! # Events specification
 
 use std::ops::Deref;
 
@@ -62,10 +62,6 @@ impl Events {
 pub struct EventEmitter(Receiver<Event>);
 
 impl EventEmitter {
-    pub(crate) fn into_inner(self) -> Receiver<Event> {
-        self.0
-    }
-
     /// Blocking recv of an event. Return `None` if the `Sender` has been droped.
     pub fn recv_sync(&self) -> Option<Event> {
         async_std::task::block_on(self.recv())
@@ -189,6 +185,21 @@ pub enum EventType {
     #[strum(props(id = "400"))]
     Error(String),
 
+    /// An action cannot be performed because there is no network available.
+    ///
+    /// The library will typically try over after a some time
+    /// and when dc_maybe_network() is called.
+    ///
+    /// Network errors should be reported to users in a non-disturbing way,
+    /// however, as network errors may come in a sequence,
+    /// it is not useful to raise each an every error to the user.
+    ///
+    /// Moreover, if the UI detects that the device is offline,
+    /// it is probably more useful to report this to the user
+    /// instead of the string from data2.
+    #[strum(props(id = "401"))]
+    ErrorNetwork(String),
+
     /// An action cannot be performed because the user is not in the group.
     /// Reported eg. after a call to
     /// dc_set_chat_name(), dc_set_chat_profile_image(),
@@ -319,11 +330,4 @@ pub enum EventType {
     ///     (Bob has verified alice and waits until Alice does the same for him)
     #[strum(props(id = "2061"))]
     SecurejoinJoinerProgress { contact_id: u32, progress: usize },
-
-    /// The connectivity to the server changed.
-    /// This means that you should refresh the connectivity view
-    /// and possibly the connectivtiy HTML; see dc_get_connectivity() and
-    /// dc_get_connectivity_html() for details.
-    #[strum(props(id = "2100"))]
-    ConnectivityChanged,
 }

src/format_flowed.rs

@@ -1,13 +1,13 @@
-//! # format=flowed support.
-//!
-//! Format=flowed is defined in
-//! [RFC 3676](https://tools.ietf.org/html/rfc3676).
-//!
-//! Older [RFC 2646](https://tools.ietf.org/html/rfc2646) is used
-//! during formatting, i.e., DelSp parameter introduced in RFC 3676
-//! is assumed to be set to "no".
-//!
-//! For received messages, DelSp parameter is honoured.
+///! # format=flowed support
+///!
+///! Format=flowed is defined in
+///! [RFC 3676](https://tools.ietf.org/html/rfc3676).
+///!
+///! Older [RFC 2646](https://tools.ietf.org/html/rfc2646) is used
+///! during formatting, i.e., DelSp parameter introduced in RFC 3676
+///! is assumed to be set to "no".
+///!
+///! For received messages, DelSp parameter is honoured.
 
 /// Wraps line to 72 characters using format=flowed soft breaks.
 ///
@@ -104,13 +104,13 @@ pub fn unformat_flowed(text: &str, delsp: bool) -> String {
 
     for line in text.split('\n') {
         // Revert space-stuffing
-        let line = line.strip_prefix(' ').unwrap_or(line);
+        let line = line.strip_prefix(" ").unwrap_or(line);
 
         if !skip_newline {
             result.push('\n');
         }
 
-        if let Some(line) = line.strip_suffix(' ') {
+        if let Some(line) = line.strip_suffix(" ") {
             // Flowed line
             result += line;
             if !delsp {

src/headerdef.rs

@@ -1,5 +1,3 @@
-//! # List of email headers.
-
 use crate::strum::AsStaticRef;
 use mailparse::{MailHeader, MailHeaderMap};
 

src/html.rs

@@ -1,12 +1,11 @@
-//! # Get message as HTML.
-//!
-//! Use `Message.has_html()` to check if the UI shall render a
-//! corresponding button and `MsgId.get_html()` to get the full message.
-//!
-//! Even when the original mime-message is not HTML,
-//! `MsgId.get_html()` will return HTML -
-//! this allows nice quoting, handling linebreaks properly etc.
-
+///! # Get message as HTML.
+///!
+///! Use `Message.has_html()` to check if the UI shall render a
+///! corresponding button and `MsgId.get_html()` to get the full message.
+///!
+///! Even when the original mime-message is not HTML,
+///! `MsgId.get_html()` will return HTML -
+///! this allows nice quoting, handling linebreaks properly etc.
 use futures::future::FutureExt;
 use std::future::Future;
 use std::pin::Pin;
@@ -249,7 +248,7 @@ impl MsgId {
         let rawmime = message::get_mime_headers(context, self).await?;
 
         if !rawmime.is_empty() {
-            match HtmlMsgParser::from_bytes(context, &rawmime).await {
+            match HtmlMsgParser::from_bytes(context, rawmime.as_bytes()).await {
                 Err(err) => {
                     warn!(context, "get_html: parser error: {}", err);
                     Ok(None)
@@ -425,10 +424,10 @@ test some special html-characters as < > and & but also " and &#x
     }
 
     #[async_std::test]
-    async fn test_get_html_invalid_msgid() {
+    async fn test_get_html_empty() {
         let t = TestContext::new().await;
         let msg_id = MsgId::new(100);
-        assert!(msg_id.get_html(&t).await.is_err())
+        assert!(msg_id.get_html(&t).await.unwrap().is_none())
     }
 
     #[async_std::test]
@@ -551,26 +550,4 @@ test some special html-characters as < > and & but also " and &#x
         let html = msg.get_id().get_html(&bob).await.unwrap().unwrap();
         assert!(html.contains("<b>html</b> text"));
     }
-
-    #[async_std::test]
-    async fn test_cp1252_html() -> Result<()> {
-        let t = TestContext::new_alice().await;
-        t.set_config(Config::ShowEmails, Some("2")).await?;
-        dc_receive_imf(
-            &t,
-            include_bytes!("../test-data/message/cp1252-html.eml"),
-            "INBOX",
-            0,
-            false,
-        )
-        .await?;
-        let msg = t.get_last_msg().await;
-        assert_eq!(msg.viewtype, Viewtype::Text);
-        assert!(msg.text.as_ref().unwrap().contains("foo bar ä ö ü ß"));
-        assert!(msg.has_html());
-        let html = msg.get_id().get_html(&t).await?.unwrap();
-        println!("{}", html);
-        assert!(html.contains("foo bar ä ö ü ß"));
-        Ok(())
-    }
 }

src/imap.rs

@@ -1,18 +1,20 @@
-//! # IMAP handling module.
+//! # Imap handling module
 //!
 //! uses [async-email/async-imap](https://github.com/async-email/async-imap)
 //! to implement connect, fetch, delete functionality with standard IMAP servers.
 
 use std::{cmp, cmp::max, collections::BTreeMap};
 
-use anyhow::{anyhow, bail, format_err, Context as _, Result};
-use async_imap::types::{
-    Fetch, Flag, Mailbox, Name, NameAttribute, Quota, QuotaRoot, UnsolicitedResponse,
+use anyhow::{bail, format_err, Context as _, Result};
+use async_imap::{
+    error::Result as ImapResult,
+    types::{Fetch, Flag, Mailbox, Name, NameAttribute},
 };
 use async_std::channel::Receiver;
 use async_std::prelude::*;
 use num_traits::FromPrimitive;
 
+use crate::config::Config;
 use crate::constants::{
     Chattype, ShowEmails, Viewtype, DC_FETCH_EXISTING_MSGS_COUNT, DC_FOLDERS_CONFIGURED_VERSION,
     DC_LP_AUTH_OAUTH2,
@@ -26,7 +28,6 @@ use crate::events::EventType;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
 use crate::job::{self, Action};
 use crate::login_param::{CertificateChecks, LoginParam, ServerLoginParam};
-use crate::login_param::{ServerAddress, Socks5Config};
 use crate::message::{self, update_server_uid, MessageState};
 use crate::mimeparser;
 use crate::oauth2::dc_get_oauth2_access_token;
@@ -35,7 +36,6 @@ use crate::provider::Socket;
 use crate::scheduler::InterruptInfo;
 use crate::stock_str;
 use crate::{chat, constants::DC_CONTACT_ID_SELF};
-use crate::{config::Config, scheduler::connectivity::ConnectivityStore};
 
 mod client;
 mod idle;
@@ -65,7 +65,7 @@ pub enum ImapActionResult {
 /// - Chat-Version to check if a message is a chat message
 /// - Autocrypt-Setup-Message to check if a message is an autocrypt setup message,
 ///   not necessarily sent by Delta Chat.
-const PREFETCH_FLAGS: &str = "(UID RFC822.SIZE BODY.PEEK[HEADER.FIELDS (\
+const PREFETCH_FLAGS: &str = "(UID BODY.PEEK[HEADER.FIELDS (\
                               MESSAGE-ID \
                               FROM \
                               IN-REPLY-TO REFERENCES \
@@ -81,14 +81,14 @@ const RFC724MID_UID: &str = "(UID BODY.PEEK[HEADER.FIELDS (\
                              X-MICROSOFT-ORIGINAL-MESSAGE-ID\
                              )])";
 const JUST_UID: &str = "(UID)";
-const BODY_FULL: &str = "(FLAGS BODY.PEEK[])";
-const BODY_PARTIAL: &str = "(FLAGS RFC822.SIZE BODY.PEEK[HEADER])";
+const BODY_FLAGS: &str = "(FLAGS BODY.PEEK[])";
 
 #[derive(Debug)]
 pub struct Imap {
     idle_interrupt: Receiver<InterruptInfo>,
     config: ImapConfig,
     session: Option<Session>,
+    connected: bool,
     interrupt: Option<stop_token::StopSource>,
     should_reconnect: bool,
     login_failed_once: bool,
@@ -96,8 +96,6 @@ pub struct Imap {
     /// True if CAPABILITY command was run successfully once and config.can_* contain correct
     /// values.
     capabilities_determined: bool,
-
-    pub(crate) connectivity: ConnectivityStore,
 }
 
 #[derive(Debug)]
@@ -142,7 +140,6 @@ impl FolderMeaning {
 struct ImapConfig {
     pub addr: String,
     pub lp: ServerLoginParam,
-    pub socks5_config: Option<Socks5Config>,
     pub strict_tls: bool,
     pub oauth2: bool,
     pub selected_folder: Option<String>,
@@ -152,12 +149,8 @@ struct ImapConfig {
     pub can_idle: bool,
 
     /// True if the server has MOVE capability as defined in
-    /// <https://tools.ietf.org/html/rfc6851>
+    /// https://tools.ietf.org/html/rfc6851
     pub can_move: bool,
-
-    /// True if the server has QUOTA capability as defined in
-    /// <https://tools.ietf.org/html/rfc2087>
-    pub can_check_quota: bool,
 }
 
 impl Imap {
@@ -166,7 +159,6 @@ impl Imap {
     /// `addr` is used to renew token if OAuth2 authentication is used.
     pub async fn new(
         lp: &ServerLoginParam,
-        socks5_config: Option<Socks5Config>,
         addr: &str,
         oauth2: bool,
         provider_strict_tls: bool,
@@ -185,7 +177,6 @@ impl Imap {
         let config = ImapConfig {
             addr: addr.to_string(),
             lp: lp.clone(),
-            socks5_config,
             strict_tls,
             oauth2,
             selected_folder: None,
@@ -193,17 +184,16 @@ impl Imap {
             selected_folder_needs_expunge: false,
             can_idle: false,
             can_move: false,
-            can_check_quota: false,
         };
 
         let imap = Imap {
             idle_interrupt,
             config,
             session: None,
+            connected: false,
             interrupt: None,
             should_reconnect: false,
             login_failed_once: false,
-            connectivity: Default::default(),
             capabilities_determined: false,
         };
 
@@ -224,14 +214,9 @@ impl Imap {
 
         let imap = Self::new(
             &param.imap,
-            param.socks5_config.clone(),
             &param.addr,
             param.server_flags & DC_LP_AUTH_OAUTH2 != 0,
-            param
-                .provider
-                .map_or(param.socks5_config.is_some(), |provider| {
-                    provider.strict_tls
-                }),
+            param.provider.map_or(false, |provider| provider.strict_tls),
             idle_interrupt,
         )
         .await?;
@@ -242,6 +227,9 @@ impl Imap {
     ///
     /// It is safe to call this function if already connected, actions are performed only as needed.
     ///
+    /// Does not emit network errors, can be used to try various parameters during
+    /// autoconfiguration.
+    ///
     /// Calling this function is not enough to perform IMAP operations. Use [`Imap::prepare`]
     /// instead if you are going to actually use connection rather than trying connection
     /// parameters.
@@ -253,35 +241,20 @@ impl Imap {
         if self.should_reconnect() {
             self.disconnect(context).await;
             self.should_reconnect = false;
-        } else if self.session.is_some() {
+        } else if self.is_connected() {
             return Ok(());
         }
 
-        self.connectivity.set_connecting(context).await;
-
         let oauth2 = self.config.oauth2;
 
-        let connection_res: Result<Client> = if self.config.lp.security == Socket::Starttls
+        let connection_res: ImapResult<Client> = if self.config.lp.security == Socket::Starttls
             || self.config.lp.security == Socket::Plain
         {
             let config = &mut self.config;
             let imap_server: &str = config.lp.server.as_ref();
             let imap_port = config.lp.port;
 
-            let connection = if let Some(socks5_config) = &config.socks5_config {
-                Client::connect_insecure_socks5(
-                    &ServerAddress {
-                        host: imap_server.to_string(),
-                        port: imap_port,
-                    },
-                    socks5_config.clone(),
-                )
-                .await
-            } else {
-                Client::connect_insecure((imap_server, imap_port)).await
-            };
-
-            match connection {
+            match Client::connect_insecure((imap_server, imap_port)).await {
                 Ok(client) => {
                     if config.lp.security == Socket::Starttls {
                         client.secure(imap_server, config.strict_tls).await
@@ -296,20 +269,7 @@ impl Imap {
             let imap_server: &str = config.lp.server.as_ref();
             let imap_port = config.lp.port;
 
-            if let Some(socks5_config) = &config.socks5_config {
-                Client::connect_secure_socks5(
-                    &ServerAddress {
-                        host: imap_server.to_string(),
-                        port: imap_port,
-                    },
-                    config.strict_tls,
-                    socks5_config.clone(),
-                )
-                .await
-            } else {
-                Client::connect_secure((imap_server, imap_port), imap_server, config.strict_tls)
-                    .await
-            }
+            Client::connect_secure((imap_server, imap_port), imap_server, config.strict_tls).await
         };
 
         let login_res = match connection_res {
@@ -346,16 +306,17 @@ impl Imap {
         match login_res {
             Ok(session) => {
                 // needs to be set here to ensure it is set on reconnects.
+                self.connected = true;
                 self.session = Some(session);
                 self.login_failed_once = false;
-                context.emit_event(EventType::ImapConnected(format!(
-                    "IMAP-LOGIN as {}",
-                    self.config.lp.user
-                )));
+                emit_event!(
+                    context,
+                    EventType::ImapConnected(format!("IMAP-LOGIN as {}", self.config.lp.user))
+                );
                 Ok(())
             }
 
-            Err(err) => {
+            Err((err, _)) => {
                 let imap_user = self.config.lp.user.to_owned();
                 let message = stock_str::cannot_login(context, &imap_user).await;
 
@@ -382,7 +343,7 @@ impl Imap {
                     self.login_failed_once = true;
                 }
 
-                self.trigger_reconnect(context).await;
+                self.trigger_reconnect();
                 Err(format_err!("{}\n\n{}", message, err))
             }
         }
@@ -399,7 +360,6 @@ impl Imap {
                 Ok(caps) => {
                     self.config.can_idle = caps.has_str("IDLE");
                     self.config.can_move = caps.has_str("MOVE");
-                    self.config.can_check_quota = caps.has_str("QUOTA");
                     self.capabilities_determined = true;
                     Ok(())
                 }
@@ -417,15 +377,18 @@ impl Imap {
     ///
     /// Ensure that IMAP client is connected, folders are created and IMAP capabilities are
     /// determined.
+    ///
+    /// This function emits network error if it fails.  It should not be used during configuration
+    /// to avoid showing failed attempt errors to the user.
     pub async fn prepare(&mut self, context: &Context) -> Result<()> {
-        if let Err(err) = self.connect(context).await {
-            self.connectivity.set_err(context, &err).await;
-            return Err(err);
+        let res = self.connect(context).await;
+        if let Err(ref err) = res {
+            emit_event!(context, EventType::ErrorNetwork(err.to_string()));
         }
 
         self.ensure_configured_folders(context, true).await?;
         self.determine_capabilities().await?;
-        Ok(())
+        res
     }
 
     async fn disconnect(&mut self, context: &Context) {
@@ -440,17 +403,21 @@ impl Imap {
                 warn!(context, "failed to logout: {:?}", err);
             }
         }
+        self.connected = false;
         self.capabilities_determined = false;
         self.config.selected_folder = None;
         self.config.selected_mailbox = None;
     }
 
+    pub fn is_connected(&self) -> bool {
+        self.connected
+    }
+
     pub fn should_reconnect(&self) -> bool {
         self.should_reconnect
     }
 
-    pub async fn trigger_reconnect(&mut self, context: &Context) {
-        self.connectivity.set_connecting(context).await;
+    pub fn trigger_reconnect(&mut self) {
         self.should_reconnect = true;
     }
 
@@ -580,7 +547,7 @@ impl Imap {
                         folder, old_uid_next, uid_next, new_uid_validity,
                     );
                     set_uid_next(context, folder, uid_next).await?;
-                    job::schedule_resync(context).await?;
+                    job::schedule_resync(context).await;
                 }
                 uid_next != old_uid_next // If uid_next changed, there are new emails
             } else {
@@ -633,7 +600,7 @@ impl Imap {
         set_uid_next(context, folder, new_uid_next).await?;
         set_uidvalidity(context, folder, new_uid_validity).await?;
         if old_uid_validity != 0 || old_uid_next != 0 {
-            job::schedule_resync(context).await?;
+            job::schedule_resync(context).await;
         }
         info!(
             context,
@@ -655,7 +622,6 @@ impl Imap {
     ) -> Result<bool> {
         let show_emails = ShowEmails::from_i32(context.get_config_int(Config::ShowEmails).await?)
             .unwrap_or_default();
-        let download_limit = context.download_limit().await?;
 
         let new_emails = self
             .select_with_uidvalidity(context, folder.as_ref())
@@ -677,8 +643,7 @@ impl Imap {
         let folder: &str = folder.as_ref();
 
         let mut read_errors = 0;
-        let mut uids_fetch_fully = Vec::with_capacity(msgs.len());
-        let mut uids_fetch_partially = Vec::with_capacity(msgs.len());
+        let mut uids = Vec::with_capacity(msgs.len());
         let mut largest_uid_skipped = None;
 
         for (current_uid, msg) in msgs.into_iter() {
@@ -705,16 +670,7 @@ impl Imap {
             )
             .await
             {
-                match download_limit {
-                    Some(download_limit) => {
-                        if msg.size.unwrap_or_default() > download_limit {
-                            uids_fetch_partially.push(current_uid);
-                        } else {
-                            uids_fetch_fully.push(current_uid)
-                        }
-                    }
-                    None => uids_fetch_fully.push(current_uid),
-                }
+                uids.push(current_uid);
             } else if read_errors == 0 {
                 // If there were errors (`read_errors != 0`), stop updating largest_uid_skipped so that uid_next will
                 // not be updated and we will retry prefetching next time
@@ -722,29 +678,8 @@ impl Imap {
             }
         }
 
-        if !uids_fetch_fully.is_empty() || !uids_fetch_partially.is_empty() {
-            self.connectivity.set_working(context).await;
-        }
-
-        let (largest_uid_fully_fetched, error_cnt) = self
-            .fetch_many_msgs(
-                context,
-                folder,
-                uids_fetch_fully,
-                false,
-                fetch_existing_msgs,
-            )
-            .await;
-        read_errors += error_cnt;
-
-        let (largest_uid_partially_fetched, error_cnt) = self
-            .fetch_many_msgs(
-                context,
-                folder,
-                uids_fetch_partially,
-                true,
-                fetch_existing_msgs,
-            )
+        let (largest_uid_processed, error_cnt) = self
+            .fetch_many_msgs(context, folder, uids, fetch_existing_msgs)
             .await;
         read_errors += error_cnt;
 
@@ -755,10 +690,7 @@ impl Imap {
         // So: Update the uid_next to the largest uid that did NOT recoverably fail. Not perfect because if there was
         // another message afterwards that succeeded, we will not retry. The upside is that we will not retry an infinite amount of times.
         let largest_uid_without_errors = max(
-            max(
-                largest_uid_fully_fetched.unwrap_or(0),
-                largest_uid_partially_fetched.unwrap_or(0),
-            ),
+            largest_uid_processed.unwrap_or(0),
             largest_uid_skipped.unwrap_or(0),
         );
         let new_uid_next = largest_uid_without_errors + 1;
@@ -852,7 +784,7 @@ impl Imap {
         // at least one UID, even if last_seen_uid+1 is past
         // the last UID in the mailbox.  It happens because
         // uid:* is interpreted the same way as *:uid.
-        // See <https://tools.ietf.org/html/rfc3501#page-61> for
+        // See https://tools.ietf.org/html/rfc3501#page-61 for
         // standard reference. Therefore, sometimes we receive
         // already seen messages and have to filter them out.
         let new_msgs = msgs.split_off(&uid_next);
@@ -895,25 +827,30 @@ impl Imap {
     /// Fetches a list of messages by server UID.
     ///
     /// Returns the last uid fetch successfully and an error count.
-    pub(crate) async fn fetch_many_msgs(
+    async fn fetch_many_msgs(
         &mut self,
         context: &Context,
         folder: &str,
         server_uids: Vec<u32>,
-        fetch_partially: bool,
         fetching_existing_messages: bool,
     ) -> (Option<u32>, usize) {
         if server_uids.is_empty() {
             return (None, 0);
         }
 
-        let session = match self.session.as_mut() {
-            Some(session) => session,
-            None => {
-                warn!(context, "Not connected");
-                return (None, server_uids.len());
-            }
-        };
+        if !self.is_connected() {
+            warn!(context, "Not connected");
+            return (None, server_uids.len());
+        }
+
+        if self.session.is_none() {
+            // we could not get a valid imap session, this should be retried
+            self.trigger_reconnect();
+            warn!(context, "Could not get IMAP session");
+            return (None, server_uids.len());
+        }
+
+        let session = self.session.as_mut().unwrap();
 
         let sets = build_sequence_sets(server_uids.clone());
         let mut read_errors = 0;
@@ -921,17 +858,7 @@ impl Imap {
         let mut last_uid = None;
 
         for set in sets.iter() {
-            let mut msgs = match session
-                .uid_fetch(
-                    &set,
-                    if fetch_partially {
-                        BODY_PARTIAL
-                    } else {
-                        BODY_FULL
-                    },
-                )
-                .await
-            {
+            let mut msgs = match session.uid_fetch(&set, BODY_FLAGS).await {
                 Ok(msgs) => msgs,
                 Err(err) => {
                     // TODO: maybe differentiate between IO and input/parsing problems
@@ -966,13 +893,7 @@ impl Imap {
                 count += 1;
 
                 let is_deleted = msg.flags().any(|flag| flag == Flag::Deleted);
-                let (body, partial) = if fetch_partially {
-                    (msg.header(), msg.size) // `BODY.PEEK[HEADER]` goes to header() ...
-                } else {
-                    (msg.body(), None) // ... while `BODY.PEEK[]` goes to body() - and includes header()
-                };
-
-                if is_deleted || body.is_none() {
+                if is_deleted || msg.body().is_none() {
                     info!(
                         context,
                         "Not processing deleted or empty msg {}", server_uid
@@ -986,7 +907,7 @@ impl Imap {
                 let folder = folder.clone();
 
                 // safe, as we checked above that there is a body.
-                let body = body.unwrap();
+                let body = msg.body().unwrap();
                 let is_seen = msg.flags().any(|flag| flag == Flag::Seen);
 
                 match dc_receive_imf_inner(
@@ -995,7 +916,6 @@ impl Imap {
                     &folder,
                     server_uid,
                     is_seen,
-                    partial,
                     fetching_existing_messages,
                 )
                 .await
@@ -1051,10 +971,13 @@ impl Imap {
             if let Some(ref mut session) = &mut self.session {
                 match session.uid_mv(&set, &dest_folder).await {
                     Ok(_) => {
-                        context.emit_event(EventType::ImapMessageMoved(format!(
-                            "IMAP Message {} moved to {}",
-                            display_folder_id, dest_folder
-                        )));
+                        emit_event!(
+                            context,
+                            EventType::ImapMessageMoved(format!(
+                                "IMAP Message {} moved to {}",
+                                display_folder_id, dest_folder
+                            ))
+                        );
                         return ImapActionResult::Success;
                     }
                     Err(err) => {
@@ -1092,17 +1015,23 @@ impl Imap {
 
         if !self.add_flag_finalized(context, uid, "\\Deleted").await {
             warn!(context, "Cannot mark {} as \"Deleted\" after copy.", uid);
-            context.emit_event(EventType::ImapMessageMoved(format!(
-                "IMAP Message {} copied to {} (delete FAILED)",
-                display_folder_id, dest_folder
-            )));
+            emit_event!(
+                context,
+                EventType::ImapMessageMoved(format!(
+                    "IMAP Message {} copied to {} (delete FAILED)",
+                    display_folder_id, dest_folder
+                ))
+            );
             ImapActionResult::Failed
         } else {
             self.config.selected_folder_needs_expunge = true;
-            context.emit_event(EventType::ImapMessageMoved(format!(
-                "IMAP Message {} copied to {} (delete successfull)",
-                display_folder_id, dest_folder
-            )));
+            emit_event!(
+                context,
+                EventType::ImapMessageMoved(format!(
+                    "IMAP Message {} copied to {} (delete successfull)",
+                    display_folder_id, dest_folder
+                ))
+            );
             ImapActionResult::Success
         }
     }
@@ -1159,7 +1088,7 @@ impl Imap {
         if uid == 0 {
             return Some(ImapActionResult::RetryLater);
         }
-        if self.session.is_none() {
+        if !self.is_connected() {
             // currently jobs are only performed on the INBOX thread
             // TODO: make INBOX/SENT/MVBOX perform the jobs on their
             // respective folders to avoid select_folder network traffic
@@ -1296,10 +1225,13 @@ impl Imap {
             );
             ImapActionResult::RetryLater
         } else {
-            context.emit_event(EventType::ImapMessageDeleted(format!(
-                "IMAP Message {} marked as deleted [{}]",
-                display_imap_id, message_id
-            )));
+            emit_event!(
+                context,
+                EventType::ImapMessageDeleted(format!(
+                    "IMAP Message {} marked as deleted [{}]",
+                    display_imap_id, message_id
+                ))
+            );
             self.config.selected_folder_needs_expunge = true;
             ImapActionResult::Success
         }
@@ -1319,158 +1251,118 @@ impl Imap {
     }
 
     pub async fn configure_folders(&mut self, context: &Context, create_mvbox: bool) -> Result<()> {
-        let session = match self.session {
-            Some(ref mut session) => session,
-            None => bail!("no IMAP connection established"),
-        };
-
-        let mut folders = match session.list(Some(""), Some("*")).await {
-            Ok(f) => f,
-            Err(err) => {
-                bail!("list_folders failed: {}", err);
-            }
-        };
-
-        let mut delimiter = ".".to_string();
-        let mut delimiter_is_default = true;
-        let mut mvbox_folder = None;
-        let mut folder_configs = BTreeMap::new();
-        let mut fallback_folder = get_fallback_folder(&delimiter);
-
-        while let Some(folder) = folders.next().await {
-            let folder = folder?;
-            info!(context, "Scanning folder: {:?}", folder);
-
-            // Update the delimiter iff there is a different one, but only once.
-            if let Some(d) = folder.delimiter() {
-                if delimiter_is_default && !d.is_empty() && delimiter != d {
-                    delimiter = d.to_string();
-                    fallback_folder = get_fallback_folder(&delimiter);
-                    delimiter_is_default = false;
-                }
-            }
-
-            let folder_meaning = get_folder_meaning(&folder);
-            let folder_name_meaning = get_folder_meaning_by_name(folder.name());
-            if folder.name() == "DeltaChat" {
-                // Always takes precedence
-                mvbox_folder = Some(folder.name().to_string());
-            } else if folder.name() == fallback_folder {
-                // only set if none has been already set
-                if mvbox_folder.is_none() {
-                    mvbox_folder = Some(folder.name().to_string());
-                }
-            } else if let Some(config) = folder_meaning.to_config() {
-                // Always takes precedence
-                folder_configs.insert(config, folder.name().to_string());
-            } else if let Some(config) = folder_name_meaning.to_config() {
-                // only set if none has been already set
-                folder_configs
-                    .entry(config)
-                    .or_insert_with(|| folder.name().to_string());
-            }
+        if !self.is_connected() {
+            bail!("IMAP No Connection established");
         }
-        drop(folders);
 
-        info!(context, "Using \"{}\" as folder-delimiter.", delimiter);
-
-        if mvbox_folder.is_none() && create_mvbox {
-            info!(context, "Creating MVBOX-folder \"DeltaChat\"...",);
-
-            match session.create("DeltaChat").await {
-                Ok(_) => {
-                    mvbox_folder = Some("DeltaChat".into());
-                    info!(context, "MVBOX-folder created.",);
-                }
+        if let Some(ref mut session) = &mut self.session {
+            let mut folders = match session.list(Some(""), Some("*")).await {
+                Ok(f) => f,
                 Err(err) => {
-                    warn!(
-                        context,
-                        "Cannot create MVBOX-folder, trying to create INBOX subfolder. ({})", err
-                    );
+                    bail!("list_folders failed: {}", err);
+                }
+            };
 
-                    match session.create(&fallback_folder).await {
-                        Ok(_) => {
-                            mvbox_folder = Some(fallback_folder);
-                            info!(
-                                context,
-                                "MVBOX-folder created as INBOX subfolder. ({})", err
-                            );
-                        }
-                        Err(err) => {
-                            warn!(context, "Cannot create MVBOX-folder. ({})", err);
+            let mut delimiter = ".".to_string();
+            let mut delimiter_is_default = true;
+            let mut mvbox_folder = None;
+            let mut folder_configs = BTreeMap::new();
+            let mut fallback_folder = get_fallback_folder(&delimiter);
+
+            while let Some(folder) = folders.next().await {
+                let folder = folder?;
+                info!(context, "Scanning folder: {:?}", folder);
+
+                // Update the delimiter iff there is a different one, but only once.
+                if let Some(d) = folder.delimiter() {
+                    if delimiter_is_default && !d.is_empty() && delimiter != d {
+                        delimiter = d.to_string();
+                        fallback_folder = get_fallback_folder(&delimiter);
+                        delimiter_is_default = false;
+                    }
+                }
+
+                let folder_meaning = get_folder_meaning(&folder);
+                let folder_name_meaning = get_folder_meaning_by_name(folder.name());
+                if folder.name() == "DeltaChat" {
+                    // Always takes precedence
+                    mvbox_folder = Some(folder.name().to_string());
+                } else if folder.name() == fallback_folder {
+                    // only set if none has been already set
+                    if mvbox_folder.is_none() {
+                        mvbox_folder = Some(folder.name().to_string());
+                    }
+                } else if let Some(config) = folder_meaning.to_config() {
+                    // Always takes precedence
+                    folder_configs.insert(config, folder.name().to_string());
+                } else if let Some(config) = folder_name_meaning.to_config() {
+                    // only set if none has been already set
+                    folder_configs
+                        .entry(config)
+                        .or_insert_with(|| folder.name().to_string());
+                }
+            }
+            drop(folders);
+
+            info!(context, "Using \"{}\" as folder-delimiter.", delimiter);
+
+            if mvbox_folder.is_none() && create_mvbox {
+                info!(context, "Creating MVBOX-folder \"DeltaChat\"...",);
+
+                match session.create("DeltaChat").await {
+                    Ok(_) => {
+                        mvbox_folder = Some("DeltaChat".into());
+                        info!(context, "MVBOX-folder created.",);
+                    }
+                    Err(err) => {
+                        warn!(
+                            context,
+                            "Cannot create MVBOX-folder, trying to create INBOX subfolder. ({})",
+                            err
+                        );
+
+                        match session.create(&fallback_folder).await {
+                            Ok(_) => {
+                                mvbox_folder = Some(fallback_folder);
+                                info!(
+                                    context,
+                                    "MVBOX-folder created as INBOX subfolder. ({})", err
+                                );
+                            }
+                            Err(err) => {
+                                warn!(context, "Cannot create MVBOX-folder. ({})", err);
+                            }
                         }
                     }
                 }
-            }
-            // SUBSCRIBE is needed to make the folder visible to the LSUB command
-            // that may be used by other MUAs to list folders.
-            // for the LIST command, the folder is always visible.
-            if let Some(ref mvbox) = mvbox_folder {
-                if let Err(err) = session.subscribe(mvbox).await {
-                    warn!(context, "could not subscribe to {:?}: {:?}", mvbox, err);
+                // SUBSCRIBE is needed to make the folder visible to the LSUB command
+                // that may be used by other MUAs to list folders.
+                // for the LIST command, the folder is always visible.
+                if let Some(ref mvbox) = mvbox_folder {
+                    if let Err(err) = session.subscribe(mvbox).await {
+                        warn!(context, "could not subscribe to {:?}: {:?}", mvbox, err);
+                    }
                 }
             }
-        }
-        context
-            .set_config(Config::ConfiguredInboxFolder, Some("INBOX"))
-            .await?;
-        if let Some(ref mvbox_folder) = mvbox_folder {
             context
-                .set_config(Config::ConfiguredMvboxFolder, Some(mvbox_folder))
+                .set_config(Config::ConfiguredInboxFolder, Some("INBOX"))
+                .await?;
+            if let Some(ref mvbox_folder) = mvbox_folder {
+                context
+                    .set_config(Config::ConfiguredMvboxFolder, Some(mvbox_folder))
+                    .await?;
+            }
+            for (config, name) in folder_configs {
+                context.set_config(config, Some(&name)).await?;
+            }
+            context
+                .sql
+                .set_raw_config_int("folders_configured", DC_FOLDERS_CONFIGURED_VERSION)
                 .await?;
         }
-        for (config, name) in folder_configs {
-            context.set_config(config, Some(&name)).await?;
-        }
-        context
-            .sql
-            .set_raw_config_int("folders_configured", DC_FOLDERS_CONFIGURED_VERSION)
-            .await?;
-
         info!(context, "FINISHED configuring IMAP-folders.");
         Ok(())
     }
-
-    /// Return whether the server sent an unsolicited EXISTS response.
-    /// Drains all responses from `session.unsolicited_responses` in the process.
-    /// If this returns `true`, this means that new emails arrived and you should
-    /// fetch again, even if you just fetched.
-    fn server_sent_unsolicited_exists(&self, context: &Context) -> bool {
-        let session = match &self.session {
-            Some(s) => s,
-            None => return false,
-        };
-        let mut unsolicited_exists = false;
-        while let Ok(response) = session.unsolicited_responses.try_recv() {
-            match response {
-                UnsolicitedResponse::Exists(_) => {
-                    info!(
-                        context,
-                        "Need to fetch again, got unsolicited EXISTS {:?}", response
-                    );
-                    unsolicited_exists = true;
-                }
-                _ => info!(context, "ignoring unsolicited response {:?}", response),
-            }
-        }
-        unsolicited_exists
-    }
-
-    pub fn can_check_quota(&self) -> bool {
-        self.config.can_check_quota
-    }
-
-    pub async fn get_quota_roots(
-        &mut self,
-        mailbox_name: &str,
-    ) -> Result<(Vec<QuotaRoot>, Vec<Quota>)> {
-        if let Some(session) = self.session.as_mut() {
-            let quota_roots = session.get_quota_root(mailbox_name).await?;
-            Ok(quota_roots)
-        } else {
-            Err(anyhow!("Not connected to IMAP, no session"))
-        }
-    }
 }
 
 /// Try to get the folder meaning by the name of the folder only used if the server does not support XLIST.
@@ -1480,7 +1372,7 @@ impl Imap {
 // CAVE: if possible, take care not to add a name here that is "sent" in one language
 // but sth. different in others - a hard job.
 fn get_folder_meaning_by_name(folder_name: &str) -> FolderMeaning {
-    // source: <https://stackoverflow.com/questions/2185391/localized-gmail-imap-folders>
+    // source: https://stackoverflow.com/questions/2185391/localized-gmail-imap-folders
     const SENT_NAMES: &[&str] = &[
         "sent",
         "sentmail",
@@ -1703,7 +1595,7 @@ pub(crate) async fn prefetch_should_download(
     let is_reply_to_chat_message = parent.is_some();
     if let Some(parent) = &parent {
         let chat = chat::Chat::load_from_db(context, parent.get_chat_id()).await?;
-        if chat.typ == Chattype::Group && !chat.id.is_special() {
+        if chat.typ == Chattype::Group {
             // This might be a group command, like removing a group member.
             // We really need to fetch this to avoid inconsistent group state.
             return Ok(true);
@@ -1715,7 +1607,7 @@ pub(crate) async fn prefetch_should_download(
     // deleted from the database or has not arrived yet.
     if let Some(rfc724_mid) = headers.get_header_value(HeaderDef::MessageId) {
         if let Some(group_id) = dc_extract_grpid_from_rfc724_mid(&rfc724_mid) {
-            if get_chat_id_by_grpid(context, group_id).await?.is_some() {
+            if let Ok((_chat_id, _, _)) = get_chat_id_by_grpid(context, group_id).await {
                 return Ok(true);
             }
         }
@@ -1810,7 +1702,7 @@ fn get_fallback_folder(delimiter: &str) -> String {
 }
 
 /// uid_next is the next unique identifier value from the last time we fetched a folder
-/// See <https://tools.ietf.org/html/rfc3501#section-2.3.1.1>
+/// See https://tools.ietf.org/html/rfc3501#section-2.3.1.1
 /// This function is used to update our uid_next after fetching messages.
 pub(crate) async fn set_uid_next(context: &Context, folder: &str, uid_next: u32) -> Result<()> {
     context
@@ -1825,7 +1717,7 @@ pub(crate) async fn set_uid_next(context: &Context, folder: &str, uid_next: u32)
 }
 
 /// uid_next is the next unique identifier value from the last time we fetched a folder
-/// See <https://tools.ietf.org/html/rfc3501#section-2.3.1.1>
+/// See https://tools.ietf.org/html/rfc3501#section-2.3.1.1
 /// This method returns the uid_next from the last time we fetched messages.
 /// We can compare this to the current uid_next to find out whether there are new messages
 /// and fetch from this value on to get all new messages.
@@ -1886,7 +1778,7 @@ pub async fn get_config_last_seen_uid<S: AsRef<str>>(
 }
 
 /// Builds a list of sequence/uid sets. The returned sets have each no more than around 1000
-/// characters because according to <https://tools.ietf.org/html/rfc2683#section-3.2.1.5>
+/// characters because according to https://tools.ietf.org/html/rfc2683#section-3.2.1.5
 /// command lines should not be much more than 1000 chars (servers should allow at least 8000 chars)
 fn build_sequence_sets(mut uids: Vec<u32>) -> Vec<String> {
     uids.sort_unstable();

src/imap/client.rs

@@ -1,23 +1,16 @@
-use std::{
-    ops::{Deref, DerefMut},
-    time::Duration,
+use std::ops::{Deref, DerefMut};
+
+use async_imap::{
+    error::{Error as ImapError, Result as ImapResult},
+    Client as ImapClient,
 };
-
-use anyhow::{Context as _, Result};
-
-use async_imap::Client as ImapClient;
-
-use async_smtp::ServerAddress;
 use async_std::net::{self, TcpStream};
 
 use super::session::Session;
-use crate::login_param::{dc_build_tls, Socks5Config};
+use crate::login_param::dc_build_tls;
 
 use super::session::SessionStream;
 
-/// IMAP write and read timeout in seconds.
-const IMAP_TIMEOUT: u64 = 30;
-
 #[derive(Debug)]
 pub(crate) struct Client {
     is_secure: bool,
@@ -39,12 +32,24 @@ impl DerefMut for Client {
 }
 
 impl Client {
-    pub async fn login(self, username: &str, password: &str) -> Result<Session> {
-        let Client { inner, .. } = self;
+    pub async fn login(
+        self,
+        username: &str,
+        password: &str,
+    ) -> std::result::Result<Session, (ImapError, Self)> {
+        let Client { inner, is_secure } = self;
         let session = inner
             .login(username, password)
             .await
-            .map_err(|(err, _client)| err)?;
+            .map_err(|(err, client)| {
+                (
+                    err,
+                    Client {
+                        is_secure,
+                        inner: client,
+                    },
+                )
+            })?;
         Ok(Session { inner: session })
     }
 
@@ -52,12 +57,21 @@ impl Client {
         self,
         auth_type: &str,
         authenticator: impl async_imap::Authenticator,
-    ) -> Result<Session> {
-        let Client { inner, .. } = self;
-        let session = inner
-            .authenticate(auth_type, authenticator)
-            .await
-            .map_err(|(err, _client)| err)?;
+    ) -> std::result::Result<Session, (ImapError, Self)> {
+        let Client { inner, is_secure } = self;
+        let session =
+            inner
+                .authenticate(auth_type, authenticator)
+                .await
+                .map_err(|(err, client)| {
+                    (
+                        err,
+                        Client {
+                            is_secure,
+                            inner: client,
+                        },
+                    )
+                })?;
         Ok(Session { inner: session })
     }
 
@@ -65,7 +79,7 @@ impl Client {
         addr: impl net::ToSocketAddrs,
         domain: &str,
         strict_tls: bool,
-    ) -> Result<Self> {
+    ) -> ImapResult<Self> {
         let stream = TcpStream::connect(addr).await?;
         let tls = dc_build_tls(strict_tls);
         let tls_stream: Box<dyn SessionStream> = Box::new(tls.connect(domain, stream).await?);
@@ -74,7 +88,7 @@ impl Client {
         let _greeting = client
             .read_response()
             .await
-            .context("failed to read greeting")?;
+            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
 
         Ok(Client {
             is_secure: true,
@@ -82,14 +96,14 @@ impl Client {
         })
     }
 
-    pub async fn connect_insecure(addr: impl net::ToSocketAddrs) -> Result<Self> {
+    pub async fn connect_insecure(addr: impl net::ToSocketAddrs) -> ImapResult<Self> {
         let stream: Box<dyn SessionStream> = Box::new(TcpStream::connect(addr).await?);
 
         let mut client = ImapClient::new(stream);
         let _greeting = client
             .read_response()
             .await
-            .context("failed to read greeting")?;
+            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
 
         Ok(Client {
             is_secure: false,
@@ -97,56 +111,7 @@ impl Client {
         })
     }
 
-    pub async fn connect_secure_socks5(
-        target_addr: &ServerAddress,
-        strict_tls: bool,
-        socks5_config: Socks5Config,
-    ) -> Result<Self> {
-        let socks5_stream: Box<dyn SessionStream> = Box::new(
-            socks5_config
-                .connect(target_addr, Some(Duration::from_secs(IMAP_TIMEOUT)))
-                .await?,
-        );
-
-        let tls = dc_build_tls(strict_tls);
-        let tls_stream: Box<dyn SessionStream> =
-            Box::new(tls.connect(target_addr.host.clone(), socks5_stream).await?);
-        let mut client = ImapClient::new(tls_stream);
-
-        let _greeting = client
-            .read_response()
-            .await
-            .context("failed to read greeting")?;
-
-        Ok(Client {
-            is_secure: true,
-            inner: client,
-        })
-    }
-
-    pub async fn connect_insecure_socks5(
-        target_addr: &ServerAddress,
-        socks5_config: Socks5Config,
-    ) -> Result<Self> {
-        let socks5_stream: Box<dyn SessionStream> = Box::new(
-            socks5_config
-                .connect(target_addr, Some(Duration::from_secs(IMAP_TIMEOUT)))
-                .await?,
-        );
-
-        let mut client = ImapClient::new(socks5_stream);
-        let _greeting = client
-            .read_response()
-            .await
-            .context("failed to read greeting")?;
-
-        Ok(Client {
-            is_secure: false,
-            inner: client,
-        })
-    }
-
-    pub async fn secure(self, domain: &str, strict_tls: bool) -> Result<Client> {
+    pub async fn secure(self, domain: &str, strict_tls: bool) -> ImapResult<Client> {
         if self.is_secure {
             Ok(self)
         } else {

src/imap/idle.rs

@@ -2,6 +2,7 @@ use super::Imap;
 
 use anyhow::{bail, format_err, Result};
 use async_imap::extensions::idle::IdleResponse;
+use async_imap::types::UnsolicitedResponse;
 use async_std::prelude::*;
 use std::time::{Duration, SystemTime};
 
@@ -31,11 +32,24 @@ impl Imap {
         let timeout = Duration::from_secs(23 * 60);
         let mut info = Default::default();
 
-        if self.server_sent_unsolicited_exists(context) {
-            return Ok(info);
-        }
-
         if let Some(session) = self.session.take() {
+            // if we have unsolicited responses we directly return
+            let mut unsolicited_exists = false;
+            while let Ok(response) = session.unsolicited_responses.try_recv() {
+                match response {
+                    UnsolicitedResponse::Exists(_) => {
+                        warn!(context, "skip idle, got unsolicited EXISTS {:?}", response);
+                        unsolicited_exists = true;
+                    }
+                    _ => info!(context, "ignoring unsolicited response {:?}", response),
+                }
+            }
+
+            if unsolicited_exists {
+                self.session = Some(session);
+                return Ok(info);
+            }
+
             if let Ok(info) = self.idle_interrupt.try_recv() {
                 info!(context, "skip idle, got interrupt {:?}", info);
                 self.session = Some(session);
@@ -167,7 +181,7 @@ impl Imap {
                         }
                         Err(err) => {
                             error!(context, "could not fetch from folder: {:#}", err);
-                            self.trigger_reconnect(context).await;
+                            self.trigger_reconnect()
                         }
                     }
                 }

src/imap/scan_folders.rs

@@ -42,15 +42,6 @@ impl Imap {
                 }
             };
 
-            // Gmail labels are not folders and should be skipped. For example,
-            // emails appear in the inbox and under "All Mail" as soon as it is
-            // received. The code used to wrongly conclude that the email had
-            // already been moved and left it in the inbox.
-            let folder_name = folder.name();
-            if folder_name.starts_with("[Gmail]") {
-                continue;
-            }
-
             let folder_meaning = get_folder_meaning(&folder);
             let folder_name_meaning = get_folder_meaning_by_name(folder.name());
 
@@ -70,19 +61,9 @@ impl Imap {
 
             // Don't scan folders that are watched anyway
             if !watched_folders.contains(&folder.name().to_string()) && !is_drafts {
-                // Drain leftover unsolicited EXISTS messages
-                self.server_sent_unsolicited_exists(context);
-
-                loop {
-                    self.fetch_new_messages(context, folder.name(), false)
-                        .await
-                        .ok_or_log_msg(context, "Can't fetch new msgs in scanned folder");
-
-                    // If the server sent an unsocicited EXISTS during the fetch, we need to fetch again
-                    if !self.server_sent_unsolicited_exists(context) {
-                        break;
-                    }
-                }
+                self.fetch_new_messages(context, folder.name(), false)
+                    .await
+                    .ok_or_log_msg(context, "Can't fetch new msgs in scanned folder");
             }
         }
 
@@ -102,7 +83,7 @@ impl Imap {
     }
 }
 
-pub(crate) async fn get_watched_folders(context: &Context) -> Vec<String> {
+async fn get_watched_folders(context: &Context) -> Vec<String> {
     let mut res = Vec::new();
     let folder_watched_configured = &[
         (Config::SentboxWatch, Config::ConfiguredSentboxFolder),

src/imap/select_folder.rs

@@ -26,7 +26,7 @@ impl Imap {
     /// Issues a CLOSE command to expunge selected folder.
     ///
     /// CLOSE is considerably faster than an EXPUNGE, see
-    /// <https://tools.ietf.org/html/rfc3501#section-6.4.2>
+    /// https://tools.ietf.org/html/rfc3501#section-6.4.2
     pub(super) async fn close_folder(&mut self, context: &Context) -> Result<()> {
         if let Some(ref folder) = self.config.selected_folder {
             info!(context, "Expunge messages in \"{}\".", folder);
@@ -37,7 +37,7 @@ impl Imap {
                         info!(context, "close/expunge succeeded");
                     }
                     Err(err) => {
-                        self.trigger_reconnect(context).await;
+                        self.trigger_reconnect();
                         return Err(Error::CloseExpungeFailed(err));
                     }
                 }
@@ -70,7 +70,7 @@ impl Imap {
         if self.session.is_none() {
             self.config.selected_folder = None;
             self.config.selected_folder_needs_expunge = false;
-            self.trigger_reconnect(context).await;
+            self.trigger_reconnect();
             return Err(Error::NoSession);
         }
 
@@ -92,7 +92,7 @@ impl Imap {
             if let Some(ref mut session) = &mut self.session {
                 let res = session.select(folder).await;
 
-                // <https://tools.ietf.org/html/rfc3501#section-6.3.1>
+                // https://tools.ietf.org/html/rfc3501#section-6.3.1
                 // says that if the server reports select failure we are in
                 // authenticated (not-select) state.
 
@@ -103,7 +103,7 @@ impl Imap {
                         Ok(NewlySelected::Yes)
                     }
                     Err(async_imap::error::Error::ConnectionLost) => {
-                        self.trigger_reconnect(context).await;
+                        self.trigger_reconnect();
                         self.config.selected_folder = None;
                         Err(Error::ConnectionLost)
                     }
@@ -112,7 +112,7 @@ impl Imap {
                     }
                     Err(err) => {
                         self.config.selected_folder = None;
-                        self.trigger_reconnect(context).await;
+                        self.trigger_reconnect();
                         Err(Error::Other(err.to_string()))
                     }
                 }

src/imap/session.rs

@@ -3,7 +3,6 @@ use std::ops::{Deref, DerefMut};
 use async_imap::Session as ImapSession;
 use async_native_tls::TlsStream;
 use async_std::net::TcpStream;
-use fast_socks5::client::Socks5Stream;
 
 #[derive(Debug)]
 pub(crate) struct Session {
@@ -18,7 +17,6 @@ pub(crate) trait SessionStream:
 impl SessionStream for TlsStream<Box<dyn SessionStream>> {}
 impl SessionStream for TlsStream<TcpStream> {}
 impl SessionStream for TcpStream {}
-impl SessionStream for Socks5Stream<TcpStream> {}
 
 impl Deref for Session {
     type Target = ImapSession<Box<dyn SessionStream>>;

src/imex.rs

@@ -1,4 +1,4 @@
-//! # Import/export module.
+//! # Import/export module
 
 use std::any::Any;
 use std::ffi::OsStr;
@@ -66,15 +66,15 @@ pub enum ImexMode {
 
 /// Import/export things.
 ///
-/// What to do is defined by the `what` parameter.
+/// What to do is defined by the *what* parameter.
 ///
 /// During execution of the job,
 /// some events are sent out:
 ///
-/// - A number of `DC_EVENT_IMEX_PROGRESS` events are sent and may be used to create
+/// - A number of #DC_EVENT_IMEX_PROGRESS events are sent and may be used to create
 ///   a progress bar or stuff like that. Moreover, you'll be informed when the imex-job is done.
 ///
-/// - For each file written on export, the function sends `DC_EVENT_IMEX_FILE_WRITTEN`
+/// - For each file written on export, the function sends #DC_EVENT_IMEX_FILE_WRITTEN
 ///
 /// Only one import-/export-progress can run at the same time.
 /// To cancel an import-/export-progress, drop the future returned by this function.
@@ -184,7 +184,7 @@ pub async fn has_backup_old(context: &Context, dir_name: &Path) -> Result<String
                             "Found backup file {} which could not be opened: {}", name, e
                         );
                         // On some Android devices we can't open sql files that are not in our private directory
-                        // (see <https://github.com/deltachat/deltachat-android/issues/1768>). So, compare names
+                        // (see https://github.com/deltachat/deltachat-android/issues/1768). So, compare names
                         // to still find the newest backup.
                         let name: String = name.into();
                         if newest_backup_time == 0
@@ -204,7 +204,6 @@ pub async fn has_backup_old(context: &Context, dir_name: &Path) -> Result<String
     }
 }
 
-/// Initiates key transfer via Autocrypt Setup Message.
 pub async fn initiate_key_transfer(context: &Context) -> Result<String> {
     use futures::future::FutureExt;
 
@@ -234,7 +233,7 @@ async fn do_initiate_key_transfer(context: &Context) -> Result<String> {
     msg = Message::default();
     msg.viewtype = Viewtype::File;
     msg.param.set(Param::File, setup_file_blob.as_name());
-    msg.subject = stock_str::ac_setup_msg_subject(context).await;
+
     msg.param
         .set(Param::MimeType, "application/autocrypt-setup");
     msg.param.set_cmd(SystemMessage::AutocryptSetupMessage);
@@ -436,7 +435,7 @@ async fn decrypt_setup_file<T: std::io::Read + std::io::Seek>(
     Ok(plain_text)
 }
 
-fn normalize_setup_code(s: &str) -> String {
+pub fn normalize_setup_code(s: &str) -> String {
     let mut out = String::new();
     for c in s.chars() {
         if ('0'..='9').contains(&c) {
@@ -507,16 +506,14 @@ async fn import_backup(context: &Context, backup_to_import: &Path) -> Result<()>
     let archive = Archive::new(backup_file);
 
     let mut entries = archive.entries()?;
-    let mut last_progress = 0;
     while let Some(file) = entries.next().await {
         let f = &mut file?;
 
         let current_pos = f.raw_file_position();
         let progress = 1000 * current_pos / file_size;
-        if progress != last_progress && progress > 10 && progress < 1000 {
+        if progress > 10 && progress < 1000 {
             // We already emitted ImexProgress(10) above
             context.emit_event(EventType::ImexProgress(progress as usize));
-            last_progress = progress;
         }
 
         if f.path()?.file_name() == Some(OsStr::new(DBFILE_BACKUP_NAME)) {
@@ -615,8 +612,13 @@ async fn import_backup_old(context: &Context, backup_to_import: &Path) -> Result
 
     let mut all_files_extracted = true;
     for (processed_files_cnt, file_id) in file_ids.into_iter().enumerate() {
+        if context.shall_stop_ongoing().await {
+            all_files_extracted = false;
+            break;
+        }
+
         // Load a single blob into memory
-        let (file_name, file_blob) = context
+        match context
             .sql
             .query_row(
                 "SELECT file_name, file_content FROM backup_blobs WHERE id = ?",
@@ -627,12 +629,17 @@ async fn import_backup_old(context: &Context, backup_to_import: &Path) -> Result
                     Ok((file_name, file_blob))
                 },
             )
-            .await?;
-
-        if context.shall_stop_ongoing().await {
-            all_files_extracted = false;
-            break;
+            .await
+        {
+            Ok((file_name, file_blob)) => {
+                let path_filename = context.get_blobdir().join(file_name);
+                dc_write_file(context, &path_filename, &file_blob).await?;
+            }
+            Err(err) => {
+                error!(context, "Can't import file {}: {}", file_id, err);
+            }
         }
+
         let mut permille = processed_files_cnt * 1000 / total_files_cnt;
         if permille < 10 {
             permille = 10
@@ -641,12 +648,6 @@ async fn import_backup_old(context: &Context, backup_to_import: &Path) -> Result
             permille = 990
         }
         context.emit_event(EventType::ImexProgress(permille));
-        if file_blob.is_empty() {
-            continue;
-        }
-
-        let path_filename = context.get_blobdir().join(file_name);
-        dc_write_file(context, &path_filename, &file_blob).await?;
     }
 
     if all_files_extracted {
@@ -739,7 +740,6 @@ async fn export_backup_inner(context: &Context, temp_path: &PathBuf) -> Result<(
     let count = read_dir.len();
     let mut written_files = 0;
 
-    let mut last_progress = 0;
     for entry in read_dir.into_iter() {
         let entry = entry?;
         let name = entry.file_name();
@@ -757,10 +757,9 @@ async fn export_backup_inner(context: &Context, temp_path: &PathBuf) -> Result<(
 
         written_files += 1;
         let progress = 1000 * written_files / count;
-        if progress != last_progress && progress > 10 && progress < 1000 {
+        if progress > 10 && progress < 1000 {
             // We already emitted ImexProgress(10) above
-            context.emit_event(EventType::ImexProgress(progress));
-            last_progress = progress;
+            emit_event!(context, EventType::ImexProgress(progress));
         }
     }
 

src/job.rs

@@ -1,19 +1,21 @@
-//! # Job module.
+//! # Job module
 //!
 //! This module implements a job queue maintained in the SQLite database
 //! and job types.
-use std::fmt;
 use std::future::Future;
+use std::{fmt, time::Duration};
 
 use anyhow::{bail, ensure, format_err, Context as _, Error, Result};
 use async_smtp::smtp::response::{Category, Code, Detail};
+use async_std::task::sleep;
 use deltachat_derive::{FromSql, ToSql};
 use itertools::Itertools;
 use rand::{thread_rng, Rng};
 
 use crate::blob::BlobObject;
-use crate::chat::{self, ChatId};
+use crate::chat::{self, Chat, ChatId, ChatIdBlocked, ChatItem};
 use crate::config::Config;
+use crate::constants::{Blocked, Chattype, DC_CHAT_ID_DEADDROP};
 use crate::contact::{normalize_name, Contact, Modifier, Origin};
 use crate::context::Context;
 use crate::dc_tools::{dc_delete_file, dc_read_file, time};
@@ -94,20 +96,11 @@ pub enum Action {
     FetchExistingMsgs = 110,
     MarkseenMsgOnImap = 130,
 
-    // this is user initiated so it should have a fairly high priority
-    UpdateRecentQuota = 140,
-
     // Moving message is prioritized lower than deletion so we don't
     // bother moving message if it is already scheduled for deletion.
     MoveMsg = 200,
     DeleteMsgOnImap = 210,
 
-    // This job will download partially downloaded messages completely
-    // and is added when download_full() is called.
-    // Most messages are downloaded automatically on fetch
-    // and do not go through this job.
-    DownloadMsg = 250,
-
     // UID synchronization is high-priority to make sure correct UIDs
     // are used by message moving/deletion.
     ResyncFolders = 300,
@@ -138,8 +131,6 @@ impl From<Action> for Thread {
             ResyncFolders => Thread::Imap,
             MarkseenMsgOnImap => Thread::Imap,
             MoveMsg => Thread::Imap,
-            UpdateRecentQuota => Thread::Imap,
-            DownloadMsg => Thread::Imap,
 
             MaybeSendLocations => Thread::Smtp,
             MaybeSendLocationsEnded => Thread::Smtp,
@@ -158,6 +149,7 @@ pub struct Job {
     pub added_timestamp: i64,
     pub tries: u32,
     pub param: Params,
+    pub pending_error: Option<String>,
 }
 
 impl fmt::Display for Job {
@@ -178,6 +170,7 @@ impl Job {
             added_timestamp: timestamp,
             tries: 0,
             param,
+            pending_error: None,
         }
     }
 
@@ -256,16 +249,11 @@ impl Job {
             info!(context, "smtp-sending out mime message:");
             println!("{}", String::from_utf8_lossy(&message));
         }
-
-        smtp.connectivity.set_working(context).await;
-
-        let send_result = smtp.send(context, recipients, message, job_id).await;
-        smtp.last_send_error = send_result.as_ref().err().map(|e| e.to_string());
-
-        let status = match send_result {
-            Err(crate::smtp::send::Error::SmtpSend(err)) => {
+        let status = match smtp.send(context, recipients, message, job_id).await {
+            Err(crate::smtp::send::Error::SendError(err)) => {
                 // Remote error, retry later.
-                warn!(context, "SMTP failed to send: {:?}", &err);
+                warn!(context, "SMTP failed to send: {:?}", err);
+                self.pending_error = Some(err.to_string());
 
                 let res = match err {
                     async_smtp::smtp::error::Error::Permanent(ref response) => {
@@ -273,13 +261,13 @@ impl Job {
                         // instead of temporary ones.
                         let maybe_transient = match response.code {
                             // Sometimes servers send a permanent error when actually it is a temporary error
-                            // For documentation see <https://tools.ietf.org/html/rfc3463>
+                            // For documentation see https://tools.ietf.org/html/rfc3463
                             Code {
                                 category: Category::MailSystem,
                                 detail: Detail::Zero,
                                 ..
                             } => {
-                                // Ignore status code 5.5.0, see <https://support.delta.chat/t/every-other-message-gets-stuck/877/2>
+                                // Ignore status code 5.5.0, see https://support.delta.chat/t/every-other-message-gets-stuck/877/2
                                 // Maybe incorrectly configured Postfix milter with "reject" instead of "tempfail", which returns
                                 // "550 5.5.0 Service unavailable" instead of "451 4.7.1 Service unavailable - try again later".
                                 //
@@ -313,7 +301,7 @@ impl Job {
                                 // Sometimes we receive transient errors that should be permanent.
                                 // Any extended smtp status codes like x.1.1, x.1.2 or x.1.3 that we
                                 // receive as a transient error are misconfigurations of the smtp server.
-                                // See <https://tools.ietf.org/html/rfc3463#section-3.2>
+                                // See https://tools.ietf.org/html/rfc3463#section-3.2
                                 info!(context, "Smtp-job #{} Received extended status code {} for a transient error. This looks like a misconfigured smtp server, let's fail immediatly", self.job_id, first_word);
                                 Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
                             } else {
@@ -338,7 +326,7 @@ impl Job {
 
                 res
             }
-            Err(crate::smtp::send::Error::Envelope(err)) => {
+            Err(crate::smtp::send::Error::EnvelopeError(err)) => {
                 // Local error, job is invalid, do not retry.
                 smtp.disconnect().await;
                 warn!(context, "SMTP job is invalid: {}", err);
@@ -374,7 +362,6 @@ impl Job {
         //  SMTP server, if not yet done
         if let Err(err) = smtp.connect_configured(context).await {
             warn!(context, "SMTP connection failure: {:?}", err);
-            smtp.last_send_error = Some(format!("SMTP connection failure: {:#}", err));
             return Status::RetryLater;
         }
 
@@ -417,8 +404,6 @@ impl Job {
                 }
                 Err(err) => {
                     warn!(context, "failed to check message existence: {:?}", err);
-                    smtp.last_send_error =
-                        Some(format!("failed to check message existence: {:#}", err));
                     return Status::RetryLater;
                 }
             }
@@ -533,7 +518,6 @@ impl Job {
         // connect to SMTP server, if not yet done
         if let Err(err) = smtp.connect_configured(context).await {
             warn!(context, "SMTP connection failure: {:?}", err);
-            smtp.last_send_error = Some(err.to_string());
             return Status::RetryLater;
         }
 
@@ -603,13 +587,12 @@ impl Job {
 
     /// Deletes a message on the server.
     ///
-    /// `foreign_id` is a MsgId.
+    /// foreign_id is a MsgId pointing to a message in the trash chat
+    /// or a hidden message.
     ///
-    /// If the message is in the trash chat or hidden, this job
-    /// removes database record, otherwise it only clears the
-    /// `server_uid` column.  If there are no more records pointing to
-    /// the same message on the server, the job actually removes the
-    /// message on the server.
+    /// This job removes the database record. If there are no more
+    /// records pointing to the same message on the server, the job
+    /// also removes the message on the server.
     async fn delete_msg_on_imap(&mut self, context: &Context, imap: &mut Imap) -> Status {
         if let Err(err) = imap.prepare(context).await {
             warn!(context, "could not connect: {:?}", err);
@@ -724,6 +707,40 @@ impl Job {
             }
         }
 
+        // Make sure that if there now is a chat with a contact (created by an outgoing
+        // message), then group contact requests from this contact should also be unblocked.
+        // See https://github.com/deltachat/deltachat-core-rust/issues/2097.
+        for item in job_try!(chat::get_chat_msgs(context, DC_CHAT_ID_DEADDROP, 0, None).await) {
+            if let ChatItem::Message { msg_id } = item {
+                let msg = match Message::load_from_db(context, msg_id).await {
+                    Err(e) => {
+                        warn!(context, "can't get msg: {:#}", e);
+                        return Status::RetryLater;
+                    }
+                    Ok(m) => m,
+                };
+                let chat = match Chat::load_from_db(context, msg.chat_id).await {
+                    Err(e) => {
+                        warn!(context, "can't get chat: {:#}", e);
+                        return Status::RetryLater;
+                    }
+                    Ok(c) => c,
+                };
+                match chat.typ {
+                    Chattype::Group | Chattype::Mailinglist => {
+                        if let Ok(Some(one_to_one_chat)) =
+                            ChatIdBlocked::lookup_by_contact(context, msg.from_id).await
+                        {
+                            if one_to_one_chat.blocked == Blocked::Not {
+                                chat.id.unblock(context).await;
+                            }
+                        }
+                    }
+                    Chattype::Single | Chattype::Undefined => {}
+                }
+            }
+        }
+
         info!(context, "Done fetching existing messages.");
         Status::Finished(Ok(()))
     }
@@ -821,12 +838,12 @@ impl Job {
 }
 
 /// Delete all pending jobs with the given action.
-pub async fn kill_action(context: &Context, action: Action) -> Result<()> {
+pub async fn kill_action(context: &Context, action: Action) -> bool {
     context
         .sql
         .execute("DELETE FROM jobs WHERE action=?;", paramsv![action])
-        .await?;
-    Ok(())
+        .await
+        .is_ok()
 }
 
 /// Remove jobs with specified IDs.
@@ -1035,9 +1052,16 @@ pub(crate) enum Connection<'a> {
 }
 
 pub(crate) async fn load_imap_deletion_job(context: &Context) -> Result<Option<Job>> {
-    let res = load_imap_deletion_msgid(context)
-        .await?
-        .map(|msg_id| Job::new(Action::DeleteMsgOnImap, msg_id.to_u32(), Params::new(), 0));
+    let res = if let Some(msg_id) = load_imap_deletion_msgid(context).await? {
+        Some(Job::new(
+            Action::DeleteMsgOnImap,
+            msg_id.to_u32(),
+            Params::new(),
+            0,
+        ))
+    } else {
+        None
+    };
     Ok(res)
 }
 
@@ -1158,11 +1182,6 @@ async fn perform_job_action(
             sql::housekeeping(context).await.ok_or_log(context);
             Status::Finished(Ok(()))
         }
-        Action::UpdateRecentQuota => match context.update_recent_quota(connection.inbox()).await {
-            Ok(status) => status,
-            Err(err) => Status::Finished(Err(err)),
-        },
-        Action::DownloadMsg => job.download_msg(context, connection.inbox()).await,
     };
 
     info!(context, "Finished immediate try {} of job {}", tries, job);
@@ -1190,14 +1209,13 @@ async fn send_mdn(context: &Context, msg: &Message) -> Result<()> {
     Ok(())
 }
 
-pub(crate) async fn schedule_resync(context: &Context) -> Result<()> {
-    kill_action(context, Action::ResyncFolders).await?;
+pub(crate) async fn schedule_resync(context: &Context) {
+    kill_action(context, Action::ResyncFolders).await;
     add(
         context,
         Job::new(Action::ResyncFolders, 0, Params::new(), 0),
     )
     .await;
-    Ok(())
 }
 
 /// Creates a job.
@@ -1226,9 +1244,7 @@ pub async fn add(context: &Context, job: Job) {
             | Action::ResyncFolders
             | Action::MarkseenMsgOnImap
             | Action::FetchExistingMsgs
-            | Action::MoveMsg
-            | Action::UpdateRecentQuota
-            | Action::DownloadMsg => {
+            | Action::MoveMsg => {
                 info!(context, "interrupt: imap");
                 context
                     .interrupt_inbox(InterruptInfo::new(false, None))
@@ -1247,15 +1263,21 @@ pub async fn add(context: &Context, job: Job) {
     }
 }
 
-async fn load_housekeeping_job(context: &Context) -> Result<Option<Job>> {
-    let last_time = context.get_config_i64(Config::LastHousekeeping).await?;
+async fn load_housekeeping_job(context: &Context) -> Option<Job> {
+    let last_time = match context.get_config_i64(Config::LastHousekeeping).await {
+        Ok(last_time) => last_time,
+        Err(err) => {
+            warn!(context, "failed to load housekeeping config: {:?}", err);
+            return None;
+        }
+    };
 
     let next_time = last_time + (60 * 60 * 24);
     if next_time <= time() {
-        kill_action(context, Action::Housekeeping).await?;
-        Ok(Some(Job::new(Action::Housekeeping, 0, Params::new(), 0)))
+        kill_action(context, Action::Housekeeping).await;
+        Some(Job::new(Action::Housekeeping, 0, Params::new(), 0))
     } else {
-        Ok(None)
+        None
     }
 }
 
@@ -1269,9 +1291,20 @@ pub(crate) async fn load_next(
     context: &Context,
     thread: Thread,
     info: &InterruptInfo,
-) -> Result<Option<Job>> {
+) -> Option<Job> {
     info!(context, "loading job for {}-thread", thread);
 
+    while !context.sql.is_open().await {
+        // The db is closed, which means that this thread should not be running.
+        // Wait until the db is re-opened (if we returned None, this thread might do further damage)
+        warn!(
+            context,
+            "{}: load_next() was called but the db was not opened, THIS SHOULD NOT HAPPEN. Waiting...",
+            thread
+        );
+        sleep(Duration::from_millis(500)).await;
+    }
+
     let query;
     let params;
     let t = time();
@@ -1325,6 +1358,7 @@ LIMIT 1;
                     added_timestamp: row.get("added_timestamp")?,
                     tries: row.get("tries")?,
                     param: row.get::<_, String>("param")?.parse().unwrap_or_default(),
+                    pending_error: None,
                 };
 
                 Ok(job)
@@ -1338,38 +1372,51 @@ LIMIT 1;
                 info!(context, "cleaning up job, because of {}", err);
 
                 // TODO: improve by only doing a single query
-                let id = context
+                match context
                     .sql
                     .query_row(query, params.clone(), |row| row.get::<_, i32>(0))
                     .await
-                    .context("Failed to retrieve invalid job ID from the database")?;
-                context
-                    .sql
-                    .execute("DELETE FROM jobs WHERE id=?;", paramsv![id])
-                    .await
-                    .with_context(|| format!("Failed to delete invalid job {}", id))?;
+                {
+                    Ok(id) => {
+                        if let Err(err) = context
+                            .sql
+                            .execute("DELETE FROM jobs WHERE id=?;", paramsv![id])
+                            .await
+                        {
+                            warn!(context, "failed to delete job {}: {:?}", id, err);
+                        }
+                    }
+                    Err(err) => {
+                        error!(context, "failed to retrieve invalid job from DB: {}", err);
+                        break None;
+                    }
+                }
             }
         }
     };
 
     match thread {
         Thread::Unknown => {
-            bail!("unknown thread for job")
+            error!(context, "unknown thread for job");
+            None
         }
         Thread::Imap => {
             if let Some(job) = job {
                 if job.action < Action::DeleteMsgOnImap {
-                    Ok(load_imap_deletion_job(context).await?.or(Some(job)))
+                    load_imap_deletion_job(context)
+                        .await
+                        .unwrap_or_default()
+                        .or(Some(job))
                 } else {
-                    Ok(Some(job))
+                    Some(job)
                 }
-            } else if let Some(job) = load_imap_deletion_job(context).await? {
-                Ok(Some(job))
+            } else if let Some(job) = load_imap_deletion_job(context).await.unwrap_or_default() {
+                Some(job)
             } else {
-                Ok(load_housekeeping_job(context).await?)
+                load_housekeeping_job(context).await
             }
         }
-        Thread::Smtp => Ok(job),
+        Thread::Smtp => job,
     }
 }
 
@@ -1401,7 +1448,7 @@ mod tests {
     }
 
     #[async_std::test]
-    async fn test_load_next_job_two() -> Result<()> {
+    async fn test_load_next_job_two() {
         // We want to ensure that loading jobs skips over jobs which
         // fails to load from the database instead of failing to load
         // all jobs.
@@ -1412,7 +1459,7 @@ mod tests {
             Thread::from(Action::MoveMsg),
             &InterruptInfo::new(false, None),
         )
-        .await?;
+        .await;
         // The housekeeping job should be loaded as we didn't run housekeeping in the last day:
         assert_eq!(jobs.unwrap().action, Action::Housekeeping);
 
@@ -1422,13 +1469,12 @@ mod tests {
             Thread::from(Action::MoveMsg),
             &InterruptInfo::new(false, None),
         )
-        .await?;
+        .await;
         assert!(jobs.is_some());
-        Ok(())
     }
 
     #[async_std::test]
-    async fn test_load_next_job_one() -> Result<()> {
+    async fn test_load_next_job_one() {
         let t = TestContext::new().await;
 
         insert_job(&t, 1, true).await;
@@ -1438,8 +1484,7 @@ mod tests {
             Thread::from(Action::MoveMsg),
             &InterruptInfo::new(false, None),
         )
-        .await?;
+        .await;
         assert!(jobs.is_some());
-        Ok(())
     }
 }

src/key.rs

@@ -1,25 +1,49 @@
-//! Cryptographic key module.
+//! Cryptographic key module
 
 use std::collections::BTreeMap;
 use std::fmt;
 use std::io::Cursor;
 
-use anyhow::{format_err, Result};
 use async_trait::async_trait;
 use num_traits::FromPrimitive;
 use pgp::composed::Deserializable;
 use pgp::ser::Serialize;
 use pgp::types::{KeyTrait, SecretKeyTrait};
+use thiserror::Error;
 
 use crate::config::Config;
 use crate::constants::KeyGenType;
 use crate::context::Context;
-use crate::dc_tools::{time, EmailAddress};
+use crate::dc_tools::{time, EmailAddress, InvalidEmailError};
 
 // Re-export key types
 pub use crate::pgp::KeyPair;
 pub use pgp::composed::{SignedPublicKey, SignedSecretKey};
 
+/// Error type for deltachat key handling.
+#[derive(Debug, thiserror::Error)]
+#[non_exhaustive]
+pub enum Error {
+    #[error("Could not decode base64")]
+    Base64Decode(#[from] base64::DecodeError),
+    #[error("rPGP error: {}", _0)]
+    Pgp(#[from] pgp::errors::Error),
+    #[error("Failed to generate PGP key: {}", _0)]
+    Keygen(#[from] crate::pgp::PgpKeygenError),
+    #[error("Failed to save generated key: {}", _0)]
+    StoreKey(#[from] SaveKeyError),
+    #[error("No address configured")]
+    NoConfiguredAddr,
+    #[error("Configured address is invalid: {}", _0)]
+    InvalidConfiguredAddr(#[from] InvalidEmailError),
+    #[error("no data provided")]
+    Empty,
+    #[error("{0}")]
+    Other(#[from] anyhow::Error),
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
 /// Convenience trait for working with keys.
 ///
 /// This trait is implemented for rPGP's [SignedPublicKey] and
@@ -50,8 +74,7 @@ pub trait DcKey: Serialize + Deserializable + KeyTrait + Clone {
     /// the ASCII-armored representation.
     fn from_asc(data: &str) -> Result<(Self::KeyType, BTreeMap<String, String>)> {
         let bytes = data.as_bytes();
-        Self::KeyType::from_armor_single(Cursor::new(bytes))
-            .map_err(|err| format_err!("rPGP error: {}", err))
+        Self::KeyType::from_armor_single(Cursor::new(bytes)).map_err(Error::Pgp)
     }
 
     /// Load the users' default key from the database.
@@ -202,7 +225,7 @@ async fn generate_keypair(context: &Context) -> Result<KeyPair> {
     let addr = context
         .get_config(Config::ConfiguredAddr)
         .await?
-        .ok_or_else(|| format_err!("No address configured"))?;
+        .ok_or(Error::NoConfiguredAddr)?;
     let addr = EmailAddress::new(&addr)?;
     let _guard = context.generating_key_mutex.lock().await;
 
@@ -261,6 +284,24 @@ pub enum KeyPairUse {
     ReadOnly,
 }
 
+/// Error saving a keypair to the database.
+#[derive(Debug, thiserror::Error)]
+#[error("SaveKeyError: {message}")]
+pub struct SaveKeyError {
+    message: String,
+    #[source]
+    cause: anyhow::Error,
+}
+
+impl SaveKeyError {
+    fn new(message: impl Into<String>, cause: impl Into<anyhow::Error>) -> Self {
+        Self {
+            message: message.into(),
+            cause: cause.into(),
+        }
+    }
+}
+
 /// Store the keypair as an owned keypair for addr in the database.
 ///
 /// This will save the keypair as keys for the given address.  The
@@ -277,7 +318,7 @@ pub async fn store_self_keypair(
     context: &Context,
     keypair: &KeyPair,
     default: KeyPairUse,
-) -> Result<()> {
+) -> std::result::Result<(), SaveKeyError> {
     // Everything should really be one transaction, more refactoring
     // is needed for that.
     let public_key = DcKey::to_bytes(&keypair.public);
@@ -289,13 +330,13 @@ pub async fn store_self_keypair(
             paramsv![public_key, secret_key],
         )
         .await
-        .map_err(|err| err.context("failed to remove old use of key"))?;
+        .map_err(|err| SaveKeyError::new("failed to remove old use of key", err))?;
     if default == KeyPairUse::Default {
         context
             .sql
             .execute("UPDATE keypairs SET is_default=0;", paramsv![])
             .await
-            .map_err(|err| err.context("failed to clear default"))?;
+            .map_err(|err| SaveKeyError::new("failed to clear default", err))?;
     }
     let is_default = match default {
         KeyPairUse::Default => true as i32,
@@ -313,7 +354,7 @@ pub async fn store_self_keypair(
             paramsv![addr, is_default, public_key, secret_key, t],
         )
         .await
-        .map_err(|err| err.context("failed to insert keypair"))?;
+        .map_err(|err| SaveKeyError::new("failed to insert keypair", err))?;
 
     Ok(())
 }
@@ -323,10 +364,10 @@ pub async fn store_self_keypair(
 pub struct Fingerprint(Vec<u8>);
 
 impl Fingerprint {
-    pub fn new(v: Vec<u8>) -> Result<Fingerprint> {
+    pub fn new(v: Vec<u8>) -> std::result::Result<Fingerprint, FingerprintError> {
         match v.len() {
             20 => Ok(Fingerprint(v)),
-            _ => Err(format_err!("Wrong fingerprint length")),
+            _ => Err(FingerprintError::WrongLength),
         }
     }
 
@@ -365,7 +406,7 @@ impl fmt::Display for Fingerprint {
 
 /// Parse a human-readable or otherwise formatted fingerprint.
 impl std::str::FromStr for Fingerprint {
-    type Err = anyhow::Error;
+    type Err = FingerprintError;
 
     fn from_str(input: &str) -> std::result::Result<Self, Self::Err> {
         let hex_repr: String = input
@@ -379,11 +420,21 @@ impl std::str::FromStr for Fingerprint {
     }
 }
 
+#[derive(Debug, Error)]
+pub enum FingerprintError {
+    #[error("Invalid hex characters")]
+    NotHex(#[from] hex::FromHexError),
+    #[error("Incorrect fingerprint lengths")]
+    WrongLength,
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
     use crate::test_utils::{alice_keypair, TestContext};
 
+    use std::error::Error;
+
     use async_std::sync::Arc;
     use once_cell::sync::Lazy;
 
@@ -625,7 +676,13 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
             .unwrap();
         assert_eq!(fp, res);
 
-        assert!("1".parse::<Fingerprint>().is_err());
+        let err = "1".parse::<Fingerprint>().err().unwrap();
+        match err {
+            FingerprintError::NotHex(_) => (),
+            _ => panic!("Wrong error"),
+        }
+        let src_err = err.source().unwrap().downcast_ref::<hex::FromHexError>();
+        assert_eq!(src_err, Some(&hex::FromHexError::OddLength));
     }
 
     #[test]

src/keyring.rs

@@ -3,7 +3,7 @@
 use anyhow::Result;
 
 use crate::context::Context;
-use crate::key::DcKey;
+use crate::key::{self, DcKey};
 
 /// An in-memory keyring.
 ///
@@ -27,14 +27,14 @@ where
     }
 
     /// Create a new keyring with the the user's secret key loaded.
-    pub async fn new_self(context: &Context) -> Result<Keyring<T>> {
+    pub async fn new_self(context: &Context) -> Result<Keyring<T>, key::Error> {
         let mut keyring: Keyring<T> = Keyring::new();
         keyring.load_self(context).await?;
         Ok(keyring)
     }
 
     /// Load the user's key into the keyring.
-    pub async fn load_self(&mut self, context: &Context) -> Result<()> {
+    pub async fn load_self(&mut self, context: &Context) -> Result<(), key::Error> {
         self.add(T::load_self(context).await?);
         Ok(())
     }

src/lib.rs

@@ -1,5 +1,3 @@
-//! # Delta Chat Core Library.
-
 #![forbid(unsafe_code)]
 #![deny(
     clippy::correctness,
@@ -9,11 +7,7 @@
     clippy::wildcard_imports,
     clippy::needless_borrow
 )]
-#![allow(
-    clippy::match_bool,
-    clippy::eval_order_dependence,
-    clippy::bool_assert_comparison
-)]
+#![allow(clippy::match_bool, clippy::eval_order_dependence)]
 
 #[macro_use]
 extern crate num_derive;
@@ -55,14 +49,13 @@ mod configure;
 pub mod constants;
 pub mod contact;
 pub mod context;
-pub mod download;
 mod e2ee;
 pub mod ephemeral;
 mod imap;
 pub mod imex;
 mod scheduler;
 #[macro_use]
-mod job;
+pub mod job;
 mod format_flowed;
 pub mod key;
 mod keyring;
@@ -78,19 +71,16 @@ pub mod peerstate;
 pub mod pgp;
 pub mod provider;
 pub mod qr;
-pub mod quota;
 pub mod securejoin;
 mod simplify;
 mod smtp;
 pub mod stock_str;
 mod token;
-mod update_helper;
 #[macro_use]
 mod dehtml;
 mod color;
 pub mod html;
 pub mod plaintext;
-pub mod summary;
 
 pub mod dc_receive_imf;
 pub mod dc_tools;

src/location.rs

@@ -1,7 +1,7 @@
-//! Location handling.
+//! Location handling
 use std::convert::TryFrom;
 
-use anyhow::{ensure, Result};
+use anyhow::{ensure, Error};
 use bitflags::bitflags;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
@@ -63,7 +63,7 @@ impl Kml {
         Default::default()
     }
 
-    pub fn parse(context: &Context, to_parse: &[u8]) -> Result<Self> {
+    pub fn parse(context: &Context, to_parse: &[u8]) -> Result<Self, Error> {
         ensure!(to_parse.len() <= 1024 * 1024, "kml-file is too large");
 
         let mut reader = quick_xml::Reader::from_reader(to_parse);
@@ -190,56 +190,55 @@ impl Kml {
     }
 }
 
-/// Enables location streaming in chat identified by `chat_id` for `seconds` seconds.
-pub async fn send_locations_to_chat(
-    context: &Context,
-    chat_id: ChatId,
-    seconds: i64,
-) -> Result<()> {
-    ensure!(seconds >= 0);
-    ensure!(!chat_id.is_special());
+// location streaming
+pub async fn send_locations_to_chat(context: &Context, chat_id: ChatId, seconds: i64) {
     let now = time();
-    let is_sending_locations_before = is_sending_locations_to_chat(context, Some(chat_id)).await?;
-    context
-        .sql
-        .execute(
-            "UPDATE chats    \
-         SET locations_send_begin=?,        \
-         locations_send_until=?  \
-         WHERE id=?",
-            paramsv![
-                if 0 != seconds { now } else { 0 },
-                if 0 != seconds { now + seconds } else { 0 },
-                chat_id,
-            ],
-        )
-        .await?;
-    if 0 != seconds && !is_sending_locations_before {
-        let mut msg = Message::new(Viewtype::Text);
-        msg.text = Some(stock_str::msg_location_enabled(context).await);
-        msg.param.set_cmd(SystemMessage::LocationStreamingEnabled);
-        chat::send_msg(context, chat_id, &mut msg)
+    if !(seconds < 0 || chat_id.is_special()) {
+        let is_sending_locations_before =
+            is_sending_locations_to_chat(context, Some(chat_id)).await;
+        if context
+            .sql
+            .execute(
+                "UPDATE chats    \
+             SET locations_send_begin=?,        \
+             locations_send_until=?  \
+             WHERE id=?",
+                paramsv![
+                    if 0 != seconds { now } else { 0 },
+                    if 0 != seconds { now + seconds } else { 0 },
+                    chat_id,
+                ],
+            )
             .await
-            .unwrap_or_default();
-    } else if 0 == seconds && is_sending_locations_before {
-        let stock_str = stock_str::msg_location_disabled(context).await;
-        chat::add_info_msg(context, chat_id, stock_str, now).await;
+            .is_ok()
+        {
+            if 0 != seconds && !is_sending_locations_before {
+                let mut msg = Message::new(Viewtype::Text);
+                msg.text = Some(stock_str::msg_location_enabled(context).await);
+                msg.param.set_cmd(SystemMessage::LocationStreamingEnabled);
+                chat::send_msg(context, chat_id, &mut msg)
+                    .await
+                    .unwrap_or_default();
+            } else if 0 == seconds && is_sending_locations_before {
+                let stock_str = stock_str::msg_location_disabled(context).await;
+                chat::add_info_msg(context, chat_id, stock_str).await;
+            }
+            context.emit_event(EventType::ChatModified(chat_id));
+            if 0 != seconds {
+                schedule_maybe_send_locations(context, false).await;
+                job::add(
+                    context,
+                    job::Job::new(
+                        job::Action::MaybeSendLocationsEnded,
+                        chat_id.to_u32(),
+                        Params::new(),
+                        seconds + 1,
+                    ),
+                )
+                .await;
+            }
+        }
     }
-    context.emit_event(EventType::ChatModified(chat_id));
-    if 0 != seconds {
-        schedule_maybe_send_locations(context, false).await;
-        job::add(
-            context,
-            job::Job::new(
-                job::Action::MaybeSendLocationsEnded,
-                chat_id.to_u32(),
-                Params::new(),
-                seconds + 1,
-            ),
-        )
-        .await;
-    }
-    Ok(())
 }
 
 async fn schedule_maybe_send_locations(context: &Context, force_schedule: bool) {
@@ -256,31 +255,25 @@ async fn schedule_maybe_send_locations(context: &Context, force_schedule: bool)
 ///
 /// If `chat_id` is `Some` only that chat is checked, otherwise returns `true` if any chat
 /// is sending locations.
-pub async fn is_sending_locations_to_chat(
-    context: &Context,
-    chat_id: Option<ChatId>,
-) -> Result<bool> {
-    let exists = match chat_id {
-        Some(chat_id) => {
-            context
-                .sql
-                .exists(
-                    "SELECT COUNT(id) FROM chats  WHERE id=?  AND locations_send_until>?;",
-                    paramsv![chat_id, time()],
-                )
-                .await?
-        }
-        None => {
-            context
-                .sql
-                .exists(
-                    "SELECT COUNT(id) FROM chats  WHERE locations_send_until>?;",
-                    paramsv![time()],
-                )
-                .await?
-        }
-    };
-    Ok(exists)
+pub async fn is_sending_locations_to_chat(context: &Context, chat_id: Option<ChatId>) -> bool {
+    match chat_id {
+        Some(chat_id) => context
+            .sql
+            .exists(
+                "SELECT COUNT(id) FROM chats  WHERE id=?  AND locations_send_until>?;",
+                paramsv![chat_id, time()],
+            )
+            .await
+            .unwrap_or_default(),
+        None => context
+            .sql
+            .exists(
+                "SELECT COUNT(id) FROM chats  WHERE locations_send_until>?;",
+                paramsv![time()],
+            )
+            .await
+            .unwrap_or_default(),
+    }
 }
 
 pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64) -> bool {
@@ -295,11 +288,7 @@ pub async fn set(context: &Context, latitude: f64, longitude: f64, accuracy: f64
             "SELECT id FROM chats WHERE locations_send_until>?;",
             paramsv![time()],
             |row| row.get::<_, i32>(0),
-            |chats| {
-                chats
-                    .collect::<std::result::Result<Vec<_>, _>>()
-                    .map_err(Into::into)
-            },
+            |chats| chats.collect::<Result<Vec<_>, _>>().map_err(Into::into),
         )
         .await
     {
@@ -336,7 +325,7 @@ pub async fn get_range(
     contact_id: Option<u32>,
     timestamp_from: i64,
     mut timestamp_to: i64,
-) -> Result<Vec<Location>> {
+) -> Result<Vec<Location>, Error> {
     if timestamp_to == 0 {
         timestamp_to = time() + 10;
     }
@@ -411,7 +400,7 @@ fn is_marker(txt: &str) -> bool {
 }
 
 /// Deletes all locations from the database.
-pub async fn delete_all(context: &Context) -> Result<()> {
+pub async fn delete_all(context: &Context) -> Result<(), Error> {
     context
         .sql
         .execute("DELETE FROM locations;", paramsv![])
@@ -420,7 +409,7 @@ pub async fn delete_all(context: &Context) -> Result<()> {
     Ok(())
 }
 
-pub async fn get_kml(context: &Context, chat_id: ChatId) -> Result<(String, u32)> {
+pub async fn get_kml(context: &Context, chat_id: ChatId) -> Result<(String, u32), Error> {
     let mut last_added_location_id = 0;
 
     let self_addr = context
@@ -528,7 +517,7 @@ pub async fn set_kml_sent_timestamp(
     context: &Context,
     chat_id: ChatId,
     timestamp: i64,
-) -> Result<()> {
+) -> Result<(), Error> {
     context
         .sql
         .execute(
@@ -539,7 +528,11 @@ pub async fn set_kml_sent_timestamp(
     Ok(())
 }
 
-pub async fn set_msg_location_id(context: &Context, msg_id: MsgId, location_id: u32) -> Result<()> {
+pub async fn set_msg_location_id(
+    context: &Context,
+    msg_id: MsgId,
+    location_id: u32,
+) -> Result<(), Error> {
     context
         .sql
         .execute(
@@ -557,7 +550,7 @@ pub async fn save(
     contact_id: u32,
     locations: &[Location],
     independent: bool,
-) -> Result<u32> {
+) -> Result<u32, Error> {
     ensure!(!chat_id.is_special(), "Invalid chat id");
 
     let mut newest_timestamp = 0;
@@ -637,7 +630,7 @@ pub(crate) async fn job_maybe_send_locations(context: &Context, _job: &Job) -> j
             },
             |rows| {
                 rows.filter_map(|v| v.transpose())
-                    .collect::<std::result::Result<Vec<_>, _>>()
+                    .collect::<Result<Vec<_>, _>>()
                     .map_err(Into::into)
             },
         )
@@ -723,8 +716,7 @@ pub(crate) async fn job_maybe_send_locations_ended(
             .await
     );
 
-    let now = time();
-    if !(send_begin != 0 && now <= send_until) {
+    if !(send_begin != 0 && time() <= send_until) {
         // still streaming -
         // may happen as several calls to dc_send_locations_to_chat()
         // do not un-schedule pending DC_MAYBE_SEND_LOC_ENDED jobs
@@ -743,7 +735,7 @@ pub(crate) async fn job_maybe_send_locations_ended(
             );
 
             let stock_str = stock_str::msg_location_disabled(context).await;
-            chat::add_info_msg(context, chat_id, stock_str, now).await;
+            chat::add_info_msg(context, chat_id, stock_str).await;
             context.emit_event(EventType::ChatModified(chat_id));
         }
     }

src/log.rs

@@ -1,5 +1,4 @@
-//! # Logging.
-
+//! # Logging
 use crate::context::Context;
 
 #[macro_export]
@@ -13,7 +12,7 @@ macro_rules! info {
                            file = file!(),
                            line = line!(),
                            msg = &formatted);
-        $ctx.emit_event($crate::EventType::Info(full));
+        emit_event!($ctx, $crate::EventType::Info(full));
     }};
 }
 
@@ -28,7 +27,7 @@ macro_rules! warn {
                            file = file!(),
                            line = line!(),
                            msg = &formatted);
-        $ctx.emit_event($crate::EventType::Warning(full));
+        emit_event!($ctx, $crate::EventType::Warning(full));
     }};
 }
 
@@ -39,10 +38,28 @@ macro_rules! error {
     };
     ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
         let formatted = format!($msg, $($args),*);
-        $ctx.emit_event($crate::EventType::Error(formatted));
+        emit_event!($ctx, $crate::EventType::Error(formatted));
     }};
 }
 
+#[macro_export]
+macro_rules! error_network {
+    ($ctx:expr, $msg:expr) => {
+        error_network!($ctx, $msg,)
+    };
+    ($ctx:expr, $msg:expr, $($args:expr),* $(,)?) => {{
+        let formatted = format!($msg, $($args),*);
+        emit_event!($ctx, $crate::EventType::ErrorNetwork(formatted));
+    }};
+}
+
+#[macro_export]
+macro_rules! emit_event {
+    ($ctx:expr, $event:expr) => {
+        $ctx.emit_event($event);
+    };
+}
+
 pub trait LogExt<T, E>
 where
     Self: std::marker::Sized,
@@ -59,7 +76,7 @@ where
     /// Once it is, you can add `#[track_caller]` to helper functions that use one of the log helpers here
     /// so that the location of the caller can be seen in the log. (this won't work with the macros,
     /// like warn!(), since the file!() and line!() macros don't work with track_caller)  
-    /// See <https://github.com/rust-lang/rust/issues/78840> for progress on this.
+    /// See https://github.com/rust-lang/rust/issues/78840 for progress on this.
     #[track_caller]
     fn log_err(self, context: &Context, msg: &str) -> Result<T, E> {
         self.log_err_inner(context, Some(msg))
@@ -129,7 +146,7 @@ impl<T, E: std::fmt::Display> LogExt<T, E> for Result<T, E> {
             );
             // We can't use the warn!() macro here as the file!() and line!() macros
             // don't work with #[track_caller]
-            context.emit_event(crate::EventType::Warning(full));
+            emit_event!(context, crate::EventType::Warning(full));
         };
         self
     }

src/login_param.rs

@@ -1,19 +1,12 @@
-//! # Login parameters.
+//! # Login parameters
 
 use std::borrow::Cow;
 use std::fmt;
-use std::time::Duration;
 
 use crate::provider::{get_provider_by_id, Provider};
 use crate::{context::Context, provider::Socket};
 use anyhow::Result;
 
-use async_std::io;
-use async_std::net::TcpStream;
-
-pub use async_smtp::ServerAddress;
-use fast_socks5::client::Socks5Stream;
-
 #[derive(Copy, Clone, Debug, Display, FromPrimitive, PartialEq, Eq)]
 #[repr(u32)]
 #[strum(serialize_all = "snake_case")]
@@ -51,81 +44,6 @@ pub struct ServerLoginParam {
     pub certificate_checks: CertificateChecks,
 }
 
-#[derive(Default, Debug, Clone, PartialEq)]
-pub struct Socks5Config {
-    pub host: String,
-    pub port: u16,
-    pub user_password: Option<(String, String)>,
-}
-
-impl Socks5Config {
-    /// Reads SOCKS5 configuration from the database.
-    pub async fn from_database(context: &Context) -> Result<Option<Self>> {
-        let sql = &context.sql;
-
-        let enabled = sql.get_raw_config_bool("socks5_enabled").await?;
-        if enabled {
-            let host = sql.get_raw_config("socks5_host").await?.unwrap_or_default();
-            let port: u16 = sql
-                .get_raw_config_int("socks5_port")
-                .await?
-                .unwrap_or_default() as u16;
-            let user = sql.get_raw_config("socks5_user").await?.unwrap_or_default();
-            let password = sql
-                .get_raw_config("socks5_password")
-                .await?
-                .unwrap_or_default();
-
-            let socks5_config = Self {
-                host,
-                port,
-                user_password: if !user.is_empty() {
-                    Some((user, password))
-                } else {
-                    None
-                },
-            };
-            Ok(Some(socks5_config))
-        } else {
-            Ok(None)
-        }
-    }
-
-    pub async fn connect(
-        &self,
-        target_addr: &ServerAddress,
-        timeout: Option<Duration>,
-    ) -> io::Result<Socks5Stream<TcpStream>> {
-        self.to_async_smtp_socks5_config()
-            .connect(target_addr, timeout)
-            .await
-    }
-
-    pub fn to_async_smtp_socks5_config(&self) -> async_smtp::smtp::Socks5Config {
-        async_smtp::smtp::Socks5Config {
-            host: self.host.clone(),
-            port: self.port,
-            user_password: self.user_password.clone(),
-        }
-    }
-}
-
-impl fmt::Display for Socks5Config {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(
-            f,
-            "host:{},port:{},user_password:{}",
-            self.host,
-            self.port,
-            if let Some(user_password) = self.user_password.clone() {
-                format!("user: {}, password: ***", user_password.0)
-            } else {
-                "user: None".to_string()
-            }
-        )
-    }
-}
-
 #[derive(Default, Debug, Clone, PartialEq)]
 pub struct LoginParam {
     pub addr: String,
@@ -133,7 +51,6 @@ pub struct LoginParam {
     pub smtp: ServerLoginParam,
     pub server_flags: i32,
     pub provider: Option<&'static Provider>,
-    pub socks5_config: Option<Socks5Config>,
 }
 
 impl LoginParam {
@@ -213,8 +130,6 @@ impl LoginParam {
             .await?
             .and_then(|provider_id| get_provider_by_id(&provider_id));
 
-        let socks5_config = Socks5Config::from_database(context).await?;
-
         Ok(LoginParam {
             addr,
             imap: ServerLoginParam {
@@ -235,7 +150,6 @@ impl LoginParam {
             },
             provider,
             server_flags,
-            socks5_config,
         })
     }
 
@@ -420,8 +334,6 @@ mod tests {
             },
             server_flags: 0,
             provider: get_provider_by_id("example.com"),
-            // socks5_config is not saved by `save_to_database`, using default value
-            socks5_config: None,
         };
 
         param.save_to_database(&t, "foobar_").await?;

src/lot.rs

@@ -1,10 +1,6 @@
-//! # Legacy generic return values for C API.
-
 use deltachat_derive::{FromSql, ToSql};
 
 use crate::key::Fingerprint;
-use crate::message::MessageState;
-use crate::summary::{Summary, SummaryPrefix};
 
 /// An object containing a set of values.
 /// The meaning of the values is defined by the function returning the object.
@@ -114,16 +110,6 @@ pub enum LotState {
     /// text1=error string
     QrError = 400,
 
-    QrWithdrawVerifyContact = 500,
-
-    /// text1=groupname
-    QrWithdrawVerifyGroup = 502,
-
-    QrReviveVerifyContact = 510,
-
-    /// text1=groupname
-    QrReviveVerifyGroup = 512,
-
     // Message States
     MsgInFresh = 10,
     MsgInNoticed = 13,
@@ -141,40 +127,3 @@ impl Default for LotState {
         LotState::Undefined
     }
 }
-
-impl From<MessageState> for LotState {
-    fn from(s: MessageState) -> Self {
-        use MessageState::*;
-        match s {
-            Undefined => LotState::Undefined,
-            InFresh => LotState::MsgInFresh,
-            InNoticed => LotState::MsgInNoticed,
-            InSeen => LotState::MsgInSeen,
-            OutPreparing => LotState::MsgOutPreparing,
-            OutDraft => LotState::MsgOutDraft,
-            OutPending => LotState::MsgOutPending,
-            OutFailed => LotState::MsgOutFailed,
-            OutDelivered => LotState::MsgOutDelivered,
-            OutMdnRcvd => LotState::MsgOutMdnRcvd,
-        }
-    }
-}
-
-impl From<Summary> for Lot {
-    fn from(summary: Summary) -> Self {
-        let (text1, text1_meaning) = match summary.prefix {
-            None => (None, Meaning::None),
-            Some(SummaryPrefix::Draft(text)) => (Some(text), Meaning::Text1Draft),
-            Some(SummaryPrefix::Username(username)) => (Some(username), Meaning::Text1Username),
-            Some(SummaryPrefix::Me(text)) => (Some(text), Meaning::Text1Self),
-        };
-        Self {
-            text1_meaning,
-            text1,
-            text2: Some(summary.text),
-            timestamp: summary.timestamp,
-            state: summary.state.into(),
-            ..Default::default()
-        }
-    }
-}

src/mimefactory.rs

@@ -1,5 +1,3 @@
-//! # MIME message production.
-
 use std::convert::TryInto;
 
 use anyhow::{bail, ensure, format_err, Result};
@@ -388,6 +386,10 @@ impl<'a> MimeFactory<'a> {
 
         let subject = match self.loaded {
             Loaded::Message { ref chat } => {
+                if self.msg.param.get_cmd() == SystemMessage::AutocryptSetupMessage {
+                    return Ok(stock_str::ac_setup_msg_subject(context).await);
+                }
+
                 if !self.msg.subject.is_empty() {
                     return Ok(self.msg.subject.clone());
                 }
@@ -489,11 +491,6 @@ impl<'a> MimeFactory<'a> {
                 "Auto-Submitted".to_string(),
                 "auto-replied".to_string(),
             ));
-        } else if context.get_config_bool(Config::Bot).await? {
-            headers.unprotected.push(Header::new(
-                "Auto-Submitted".to_string(),
-                "auto-generated".to_string(),
-            ));
         }
 
         if self.req_mdn {
@@ -863,7 +860,7 @@ impl<'a> MimeFactory<'a> {
                 // This should prevent automatic replies,
                 // such as non-delivery reports.
                 //
-                // See <https://tools.ietf.org/html/rfc3834>
+                // See https://tools.ietf.org/html/rfc3834
                 //
                 // Adding this header without encryption leaks some
                 // information about the message contents, but it can
@@ -1080,7 +1077,7 @@ impl<'a> MimeFactory<'a> {
             parts.push(msg_kml_part);
         }
 
-        if location::is_sending_locations_to_chat(context, Some(self.msg.chat_id)).await? {
+        if location::is_sending_locations_to_chat(context, Some(self.msg.chat_id)).await {
             match self.get_location_kml_part(context).await {
                 Ok(part) => parts.push(part),
                 Err(err) => {
@@ -1398,7 +1395,7 @@ mod tests {
             Address::new_mailbox_with_name(display_name.to_string(), addr.to_string())
         );
 
-        // Addresses should not be unnecessarily be encoded, see <https://github.com/deltachat/deltachat-core-rust/issues/1575>:
+        // Addresses should not be unnecessarily be encoded, see https://github.com/deltachat/deltachat-core-rust/issues/1575:
         assert_eq!(s, "a space <x@y.org>");
     }
 
@@ -1804,8 +1801,9 @@ mod tests {
 
         let chats = Chatlist::try_load(context, 0, None, None).await.unwrap();
 
-        let chat_id = chats.get_chat_id(0);
-        chat_id.accept(context).await.unwrap();
+        let chat_id = chat::create_by_msg_id(context, chats.get_msg_id(0).unwrap())
+            .await
+            .unwrap();
 
         let mut new_msg = Message::new(Viewtype::Text);
         new_msg.set_text(Some("Hi".to_string()));
@@ -1861,7 +1859,7 @@ mod tests {
 
     #[test]
     fn test_no_empty_lines_in_header() {
-        // See <https://github.com/deltachat/deltachat-core-rust/issues/2118>
+        // See https://github.com/deltachat/deltachat-core-rust/issues/2118
         let to_tuples = [
             ("Nnnn", "nnn@ttttttttt.de"),
             ("😀 ttttttt", "ttttttt@rrrrrr.net"),

src/mimeparser.rs

@@ -1,18 +1,18 @@
-//! # MIME message parsing module.
-
 use std::collections::{HashMap, HashSet};
 use std::future::Future;
 use std::pin::Pin;
 
 use anyhow::{bail, Result};
+use charset::Charset;
 use deltachat_derive::{FromSql, ToSql};
 use lettre_email::mime::{self, Mime};
 use mailparse::{addrparse_header, DispositionType, MailHeader, MailHeaderMap, SingleInfo};
 use once_cell::sync::Lazy;
+use percent_encoding::percent_decode_str;
 
 use crate::aheader::Aheader;
 use crate::blob::BlobObject;
-use crate::constants::{Viewtype, DC_DESIRED_TEXT_LEN, DC_ELLIPSIS};
+use crate::constants::{Viewtype, DC_DESIRED_TEXT_LEN, DC_ELLIPSE};
 use crate::contact::addr_normalize;
 use crate::context::Context;
 use crate::dc_tools::{dc_get_filemeta, dc_truncate};
@@ -136,18 +136,6 @@ const MIME_AC_SETUP_FILE: &str = "application/autocrypt-setup";
 
 impl MimeMessage {
     pub async fn from_bytes(context: &Context, body: &[u8]) -> Result<Self> {
-        MimeMessage::from_bytes_with_partial(context, body, None).await
-    }
-
-    /// Parse a mime message.
-    ///
-    /// If `partial` is set, it contains the full message size in bytes
-    /// and `body` contains the header only.
-    pub async fn from_bytes_with_partial(
-        context: &Context,
-        body: &[u8],
-        partial: Option<u32>,
-    ) -> Result<Self> {
         let mail = mailparse::parse_mail(body)?;
 
         let message_time = mail
@@ -210,7 +198,7 @@ impl MimeMessage {
                         }
 
                         // Handle any gossip headers if the mail was encrypted.  See section
-                        // "3.6 Key Gossip" of <https://autocrypt.org/autocrypt-spec-1.1.0.pdf>
+                        // "3.6 Key Gossip" of https://autocrypt.org/autocrypt-spec-1.1.0.pdf
                         // but only if the mail was correctly signed:
                         if !signatures.is_empty() {
                             let gossip_headers =
@@ -232,7 +220,7 @@ impl MimeMessage {
                         let mut throwaway_from = from.clone();
 
                         // We do not want to allow unencrypted subject in encrypted emails because the user might falsely think that the subject is safe.
-                        // See <https://github.com/deltachat/deltachat-core-rust/issues/1790>.
+                        // See https://github.com/deltachat/deltachat-core-rust/issues/1790.
                         headers.remove("subject");
 
                         MimeMessage::merge_headers(
@@ -286,22 +274,11 @@ impl MimeMessage {
             is_mime_modified: false,
             decoded_data: Vec::new(),
         };
-
-        match partial {
-            Some(org_bytes) => {
-                parser
-                    .create_stub_from_partial_download(context, org_bytes)
-                    .await?;
-            }
-            None => {
-                parser.parse_mime_recursive(context, &mail, false).await?;
-            }
-        };
-
+        parser.parse_mime_recursive(context, &mail, false).await?;
         parser.maybe_remove_bad_parts();
         parser.maybe_remove_inline_mailinglist_footer();
         parser.heuristically_parse_ndn(context).await;
-        parser.parse_headers(context).await?;
+        parser.parse_headers(context).await;
 
         if warn_empty_signature && parser.signatures.is_empty() {
             for part in parser.parts.iter_mut() {
@@ -318,7 +295,7 @@ impl MimeMessage {
 
     /// Parses system messages.
     fn parse_system_message_headers(&mut self, context: &Context) {
-        if self.get_header(HeaderDef::AutocryptSetupMessage).is_some() {
+        if self.get(HeaderDef::AutocryptSetupMessage).is_some() {
             self.parts = self
                 .parts
                 .iter()
@@ -334,7 +311,7 @@ impl MimeMessage {
             } else {
                 warn!(context, "could not determine ASM mime-part");
             }
-        } else if let Some(value) = self.get_header(HeaderDef::ChatContent) {
+        } else if let Some(value) = self.get(HeaderDef::ChatContent) {
             if value == "location-streaming-enabled" {
                 self.is_system_message = SystemMessage::LocationStreamingEnabled;
             } else if value == "ephemeral-timer-changed" {
@@ -349,19 +326,19 @@ impl MimeMessage {
 
     /// Parses avatar action headers.
     async fn parse_avatar_headers(&mut self, context: &Context) {
-        if let Some(header_value) = self.get_header(HeaderDef::ChatGroupAvatar).cloned() {
+        if let Some(header_value) = self.get(HeaderDef::ChatGroupAvatar).cloned() {
             self.group_avatar = self.avatar_action_from_header(context, header_value).await;
         }
 
-        if let Some(header_value) = self.get_header(HeaderDef::ChatUserAvatar).cloned() {
+        if let Some(header_value) = self.get(HeaderDef::ChatUserAvatar).cloned() {
             self.user_avatar = self.avatar_action_from_header(context, header_value).await;
         }
     }
 
     fn parse_videochat_headers(&mut self) {
-        if let Some(value) = self.get_header(HeaderDef::ChatContent).cloned() {
+        if let Some(value) = self.get(HeaderDef::ChatContent).cloned() {
             if value == "videochat-invitation" {
-                let instance = self.get_header(HeaderDef::ChatWebrtcRoom).cloned();
+                let instance = self.get(HeaderDef::ChatWebrtcRoom).cloned();
                 if let Some(part) = self.parts.first_mut() {
                     part.typ = Viewtype::VideochatInvitation;
                     part.param
@@ -418,12 +395,11 @@ impl MimeMessage {
         }
 
         if let Some(mut part) = self.parts.pop() {
-            if part.typ == Viewtype::Audio && self.get_header(HeaderDef::ChatVoiceMessage).is_some()
-            {
+            if part.typ == Viewtype::Audio && self.get(HeaderDef::ChatVoiceMessage).is_some() {
                 part.typ = Viewtype::Voice;
             }
             if part.typ == Viewtype::Image || part.typ == Viewtype::Gif {
-                if let Some(value) = self.get_header(HeaderDef::ChatContent) {
+                if let Some(value) = self.get(HeaderDef::ChatContent) {
                     if value == "sticker" {
                         part.typ = Viewtype::Sticker;
                     }
@@ -433,7 +409,7 @@ impl MimeMessage {
                 || part.typ == Viewtype::Voice
                 || part.typ == Viewtype::Video
             {
-                if let Some(field_0) = self.get_header(HeaderDef::ChatDuration) {
+                if let Some(field_0) = self.get(HeaderDef::ChatDuration) {
                     let duration_ms = field_0.parse().unwrap_or_default();
                     if duration_ms > 0 && duration_ms < 24 * 60 * 60 * 1000 {
                         part.param.set_int(Param::Duration, duration_ms);
@@ -445,7 +421,7 @@ impl MimeMessage {
         }
     }
 
-    async fn parse_headers(&mut self, context: &Context) -> Result<()> {
+    async fn parse_headers(&mut self, context: &Context) {
         self.parse_system_message_headers(context);
         self.parse_avatar_headers(context).await;
         self.parse_videochat_headers();
@@ -490,20 +466,15 @@ impl MimeMessage {
         if !self.decrypting_failed && !self.parts.is_empty() {
             if let Some(ref dn_to) = self.chat_disposition_notification_to {
                 if let Some(from) = self.from.get(0) {
-                    // Check that the message is not outgoing.
-                    if !context.is_self_addr(&from.addr).await? {
-                        if from.addr.to_lowercase() == dn_to.addr.to_lowercase() {
-                            if let Some(part) = self.parts.last_mut() {
-                                part.param.set_int(Param::WantsMdn, 1);
-                            }
-                        } else {
-                            warn!(
-                                context,
-                                "{} requested a read receipt to {}, ignoring",
-                                from.addr,
-                                dn_to.addr
-                            );
+                    if from.addr.to_lowercase() == dn_to.addr.to_lowercase() {
+                        if let Some(part) = self.parts.last_mut() {
+                            part.param.set_int(Param::WantsMdn, 1);
                         }
+                    } else {
+                        warn!(
+                            context,
+                            "{} requested a read receipt to {}, ignoring", from.addr, dn_to.addr
+                        );
                     }
                 }
             }
@@ -527,14 +498,6 @@ impl MimeMessage {
 
             self.parts.push(part);
         }
-
-        if self.header.contains_key("auto-submitted") {
-            for part in &mut self.parts {
-                part.param.set(Param::Bot, "1");
-            }
-        }
-
-        Ok(())
     }
 
     async fn avatar_action_from_header(
@@ -615,12 +578,12 @@ impl MimeMessage {
     }
 
     pub(crate) fn get_subject(&self) -> Option<String> {
-        self.get_header(HeaderDef::Subject)
+        self.get(HeaderDef::Subject)
             .filter(|s| !s.is_empty())
             .map(|s| s.to_string())
     }
 
-    pub fn get_header(&self, headerdef: HeaderDef) -> Option<&String> {
+    pub fn get(&self, headerdef: HeaderDef) -> Option<&String> {
         self.header.get(headerdef.get_headername())
     }
 
@@ -768,7 +731,7 @@ impl MimeMessage {
                 The second body part contains the control information necessary to
                 verify the digital signature." We simply take the first body part and
                 skip the rest.  (see
-                <https://k9mail.github.io/2016/11/24/OpenPGP-Considerations-Part-I.html>
+                https://k9mail.github.io/2016/11/24/OpenPGP-Considerations-Part-I.html
                 for background information why we use encrypted+signed) */
                 if let Some(first) = mail.subparts.get(0) {
                     any_part_added = self
@@ -929,14 +892,13 @@ impl MimeMessage {
                             (simplified_txt, top_quote)
                         };
 
-                        let simplified_txt = if simplified_txt.chars().count()
-                            > DC_DESIRED_TEXT_LEN + DC_ELLIPSIS.len()
-                        {
-                            self.is_mime_modified = true;
-                            dc_truncate(&*simplified_txt, DC_DESIRED_TEXT_LEN).to_string()
-                        } else {
-                            simplified_txt
-                        };
+                        let simplified_txt =
+                            if simplified_txt.len() > DC_DESIRED_TEXT_LEN + DC_ELLIPSE.len() {
+                                self.is_mime_modified = true;
+                                dc_truncate(&*simplified_txt, DC_DESIRED_TEXT_LEN).to_string()
+                            } else {
+                                simplified_txt
+                            };
 
                         if !simplified_txt.is_empty() || simplified_quote.is_some() {
                             let mut part = Part {
@@ -1043,12 +1005,12 @@ impl MimeMessage {
     }
 
     pub(crate) fn get_mailinglist_type(&self) -> MailinglistType {
-        if self.get_header(HeaderDef::ListId).is_some() {
+        if self.get(HeaderDef::ListId).is_some() {
             return MailinglistType::ListIdBased;
-        } else if self.get_header(HeaderDef::Sender).is_some() {
+        } else if self.get(HeaderDef::Sender).is_some() {
             // the `Sender:`-header alone is no indicator for mailing list
             // as also used for bot-impersonation via `set_override_sender_name()`
-            if let Some(precedence) = self.get_header(HeaderDef::Precedence) {
+            if let Some(precedence) = self.get(HeaderDef::Precedence) {
                 if precedence == "list" || precedence == "bulk" {
                     return MailinglistType::SenderBased;
                 }
@@ -1074,8 +1036,8 @@ impl MimeMessage {
     }
 
     pub fn get_rfc724_mid(&self) -> Option<String> {
-        self.get_header(HeaderDef::XMicrosoftOriginalMessageId)
-            .or_else(|| self.get_header(HeaderDef::MessageId))
+        self.get(HeaderDef::XMicrosoftOriginalMessageId)
+            .or_else(|| self.get(HeaderDef::MessageId))
             .and_then(|msgid| parse_message_id(msgid).ok())
     }
 
@@ -1164,11 +1126,11 @@ impl MimeMessage {
         report: &mailparse::ParsedMail<'_>,
     ) -> Result<Option<FailureReport>> {
         // parse as mailheaders
-        if let Some(original_msg) = report.subparts.iter().find(|p| {
-            p.ctype.mimetype.contains("rfc822")
-                || p.ctype.mimetype == "message/global"
-                || p.ctype.mimetype == "message/global-headers"
-        }) {
+        if let Some(original_msg) = report
+            .subparts
+            .iter()
+            .find(|p| p.ctype.mimetype.contains("rfc822") || p.ctype.mimetype == "message/global")
+        {
             let report_body = original_msg.get_body_raw()?;
             let (report_fields, _) = mailparse::parse_headers(&report_body)?;
 
@@ -1262,7 +1224,7 @@ impl MimeMessage {
     /// Also you should add a test in dc_receive_imf.rs (there already are lots of test_parse_ndn_* tests).
     #[allow(clippy::indexing_slicing)]
     async fn heuristically_parse_ndn(&mut self, context: &Context) {
-        let maybe_ndn = if let Some(from) = self.get_header(HeaderDef::From_) {
+        let maybe_ndn = if let Some(from) = self.get(HeaderDef::From_) {
             let from = from.to_ascii_lowercase();
             from.contains("mailer-daemon") || from.contains("mail-daemon")
         } else {
@@ -1337,7 +1299,7 @@ impl MimeMessage {
     /// database, returns None.
     pub async fn get_parent_timestamp(&self, context: &Context) -> Result<Option<i64>> {
         let parent_timestamp = if let Some(field) = self
-            .get_header(HeaderDef::InReplyTo)
+            .get(HeaderDef::InReplyTo)
             .and_then(|msgid| parse_message_id(msgid).ok())
         {
             context
@@ -1381,9 +1343,7 @@ async fn update_gossip_peerstates(
                     peerstate = Some(p);
                 }
                 if let Some(peerstate) = peerstate {
-                    peerstate
-                        .handle_fingerprint_change(context, message_time)
-                        .await?;
+                    peerstate.handle_fingerprint_change(context).await?;
                 }
 
                 gossipped_addr.insert(header.addr.clone());
@@ -1466,9 +1426,9 @@ pub struct Part {
     pub msg_raw: Option<String>,
     pub bytes: usize,
     pub param: Params,
-    pub(crate) org_filename: Option<String>,
+    org_filename: Option<String>,
     pub error: Option<String>,
-    pub(crate) dehtml_failed: bool,
+    dehtml_failed: bool,
 
     /// the part is a child or a descendant of multipart/related.
     /// typically, these are images that are referenced from text/html part
@@ -1476,7 +1436,7 @@ pub struct Part {
     ///
     /// note that multipart/related may contain further multipart nestings
     /// and all of them needs to be marked with `is_related`.
-    pub(crate) is_related: bool,
+    is_related: bool,
 }
 
 /// return mimetype and viewtype for a parsed mail
@@ -1503,7 +1463,7 @@ fn get_mime_type(mail: &mailparse::ParsedMail<'_>) -> Result<(Mime, Viewtype)> {
         mime::VIDEO => Viewtype::Video,
         mime::MULTIPART => Viewtype::Unknown,
         mime::MESSAGE => {
-            // Enacapsulated messages, see <https://www.w3.org/Protocols/rfc1341/7_3_Message.html>
+            // Enacapsulated messages, see https://www.w3.org/Protocols/rfc1341/7_3_Message.html
             // Also used as part "message/disposition-notification" of "multipart/report", which, however, will
             // be handled separatedly.
             // I've not seen any messages using this, so we do not attach these parts (maybe they're used to attach replies,
@@ -1544,13 +1504,55 @@ fn get_attachment_filename(
     // `Content-Disposition: ... filename=...`
     let mut desired_filename = ct.params.get("filename").map(|s| s.to_string());
 
+    // try to get file name from
+    // `Content-Disposition: ... filename*0*=... filename*1*=... filename*2*=...`
+    // encoded as CHARSET'LANG'test%2E%70%64%66 (key ends with `*`)
+    // or as "encoded-words" (key does not end with `*`)
     if desired_filename.is_none() {
-        if let Some(name) = ct.params.get("filename*").map(|s| s.to_string()) {
-            // be graceful and just use the original name.
-            // some MUA, including Delta Chat up to core1.50,
-            // use `filename*` mistakenly for simple encoded-words without following rfc2231
-            warn!(context, "apostrophed encoding invalid: {}", name);
-            desired_filename = Some(name);
+        let mut apostrophe_encoded = false;
+        desired_filename = ct
+            .params
+            .iter()
+            .filter(|(key, _value)| key.starts_with("filename*"))
+            .fold(None, |acc, (key, value)| {
+                if key.ends_with('*') {
+                    apostrophe_encoded = true;
+                }
+                if let Some(acc) = acc {
+                    Some(acc + value)
+                } else {
+                    Some(value.to_string())
+                }
+            });
+        if apostrophe_encoded {
+            if let Some(name) = desired_filename {
+                let mut parts = name.splitn(3, '\'');
+                desired_filename =
+                    if let (Some(charset), Some(value)) = (parts.next(), parts.last()) {
+                        let decoded_bytes = percent_decode_str(value);
+                        if charset.to_lowercase() == "utf-8" {
+                            Some(decoded_bytes.decode_utf8_lossy().to_string())
+                        } else {
+                            // encoded_words crate say, latin-1 is not reported; moreover, latin1 is a good default
+                            if let Some(charset) = Charset::for_label(charset.as_bytes())
+                                .or_else(|| Charset::for_label(b"latin1"))
+                            {
+                                let decoded_bytes = decoded_bytes.collect::<Vec<u8>>();
+                                let (utf8_str, _, _) = charset.decode(&*decoded_bytes);
+                                Some(utf8_str.into())
+                            } else {
+                                warn!(context, "latin1 encoding does not exist");
+                                None
+                            }
+                        }
+                    } else {
+                        warn!(context, "apostrophed encoding invalid: {}", name);
+                        // be graceful and just use the original name.
+                        // some MUA, including Delta Chat up to core1.50,
+                        // use `filename*` mistakenly for simple encoded-words without following rfc2231
+                        Some(name)
+                    }
+            }
         }
     }
 
@@ -1631,6 +1633,7 @@ mod tests {
     #![allow(clippy::indexing_slicing)]
 
     use super::*;
+    use crate::constants::DC_MAX_GET_TEXT_LEN;
     use crate::{
         chatlist::Chatlist,
         config::Config,
@@ -1911,6 +1914,13 @@ mod tests {
         assert_eq!(filename, Some("Maßnahmen Okt. 2020.html".to_string()))
     }
 
+    #[test]
+    fn test_charset_latin1() {
+        // make sure, latin1 exists under this name
+        // as we're using it as default in get_attachment_filename() for non-utf-8
+        assert!(Charset::for_label(b"latin1").is_some());
+    }
+
     #[test]
     fn test_mailparse_content_type() {
         let ctype =
@@ -2006,11 +2016,11 @@ mod tests {
             .unwrap();
 
         // non-overwritten headers do not bubble up
-        let of = mimeparser.get_header(HeaderDef::SecureJoinGroup).unwrap();
+        let of = mimeparser.get(HeaderDef::SecureJoinGroup).unwrap();
         assert_eq!(of, "no");
 
         // unknown headers do not bubble upwards
-        let of = mimeparser.get_header(HeaderDef::_TestHeader).unwrap();
+        let of = mimeparser.get(HeaderDef::_TestHeader).unwrap();
         assert_eq!(of, "Bar");
 
         // the following fields would bubble up
@@ -2019,15 +2029,13 @@ mod tests {
         // for Chat-Version, also the case-insensivity is tested.
         assert_eq!(mimeparser.get_subject(), Some("outer-subject".into()));
 
-        let of = mimeparser.get_header(HeaderDef::ChatVersion).unwrap();
+        let of = mimeparser.get(HeaderDef::ChatVersion).unwrap();
         assert_eq!(of, "0.0");
         assert_eq!(mimeparser.parts.len(), 1);
 
         // make sure, headers that are only allowed in the encrypted part
         // cannot be set from the outer part
-        assert!(mimeparser
-            .get_header(HeaderDef::SecureJoinFingerprint)
-            .is_none());
+        assert!(mimeparser.get(HeaderDef::SecureJoinFingerprint).is_none());
     }
 
     #[async_std::test]
@@ -2820,7 +2828,7 @@ On 2020-10-25, Bob wrote:
         .unwrap();
 
         let chats = Chatlist::try_load(&t.ctx, 0, None, None).await.unwrap();
-        let msg_id = chats.get_msg_id(0).unwrap().unwrap();
+        let msg_id = chats.get_msg_id(0).unwrap();
         let msg = Message::load_from_db(&t.ctx, msg_id).await.unwrap();
 
         assert_eq!(
@@ -2830,7 +2838,7 @@ On 2020-10-25, Bob wrote:
         assert_eq!(msg.viewtype, Viewtype::Image);
         assert_eq!(msg.error(), None);
         assert_eq!(msg.is_dc_message, MessengerMessage::No);
-        assert_eq!(msg.chat_blocked, Blocked::Request);
+        assert_eq!(msg.chat_blocked, Blocked::Deaddrop);
         assert_eq!(msg.state, MessageState::InFresh);
         assert_eq!(msg.get_filebytes(&t).await, 2115);
         assert!(msg.get_file(&t).is_some());
@@ -2908,20 +2916,21 @@ On 2020-10-25, Bob wrote:
         let t = TestContext::new().await;
 
         static REPEAT_TXT: &str = "this text with 42 chars is just repeated.\n";
-        static REPEAT_CNT: usize = 2000; // results in a text of 84k, should be more than DC_DESIRED_TEXT_LEN
+        static REPEAT_CNT: usize = 2000; // results in a text of 84k, should be more than DC_MAX_GET_TEXT_LEN
         let long_txt = format!("From: alice@c.de\n\n{}", REPEAT_TXT.repeat(REPEAT_CNT));
+        assert!(DC_DESIRED_TEXT_LEN + DC_ELLIPSE.len() < DC_MAX_GET_TEXT_LEN);
 
         let mimemsg = MimeMessage::from_bytes(&t, long_txt.as_ref())
             .await
             .unwrap();
         assert_eq!(long_txt.matches("just repeated").count(), REPEAT_CNT);
-        assert!(long_txt.len() > DC_DESIRED_TEXT_LEN);
+        assert!(long_txt.len() > DC_MAX_GET_TEXT_LEN);
         assert!(mimemsg.is_mime_modified);
         assert!(
             mimemsg.parts[0].msg.matches("just repeated").count()
-                <= DC_DESIRED_TEXT_LEN / REPEAT_TXT.len()
+                < DC_MAX_GET_TEXT_LEN / REPEAT_TXT.len()
         );
-        assert!(mimemsg.parts[0].msg.len() <= DC_DESIRED_TEXT_LEN + DC_ELLIPSIS.len());
+        assert!(mimemsg.parts[0].msg.len() <= DC_MAX_GET_TEXT_LEN);
     }
 
     #[async_std::test]
@@ -2988,106 +2997,4 @@ Some reply
 
         Ok(())
     }
-
-    // Test that WantsMdn parameter is not set on outgoing messages.
-    #[async_std::test]
-    async fn test_outgoing_wants_mdn() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-        let bob = TestContext::new_bob().await;
-
-        let raw = br###"Date: Thu, 28 Jan 2021 00:26:57 +0000
-Chat-Version: 1.0\n\
-Message-ID: <foobarbaz@example.org>
-To: Bob <bob@example.org>
-From: Alice <alice@example.com>
-Subject: subject
-Chat-Disposition-Notification-To: alice@example.com
-
-Message.
-"###;
-
-        // Bob receives message.
-        dc_receive_imf(&bob, raw, "INBOX", 1, false).await?;
-        let msg = bob.get_last_msg().await;
-        // Message is incoming.
-        assert!(msg.param.get_bool(Param::WantsMdn).unwrap());
-
-        // Alice receives copy-to-self.
-        dc_receive_imf(&alice, raw, "INBOX", 1, false).await?;
-        let msg = alice.get_last_msg().await;
-        // Message is outgoing, don't send read receipt to self.
-        assert!(msg.param.get_bool(Param::WantsMdn).is_none());
-
-        Ok(())
-    }
-
-    #[async_std::test]
-    async fn test_ignore_read_receipt_to_self() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-
-        // Alice receives BCC-self copy of a message sent to Bob.
-        dc_receive_imf(
-            &alice,
-            "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
-                 From: alice@example.com\n\
-                 To: bob@example.net\n\
-                 Subject: foo\n\
-                 Message-ID: first@example.com\n\
-                 Chat-Version: 1.0\n\
-                 Chat-Disposition-Notification-To: alice@example.com\n\
-                 Date: Sun, 22 Mar 2020 22:37:57 +0000\n\
-                 \n\
-                 hello\n"
-                .as_bytes(),
-            "INBOX",
-            1,
-            false,
-        )
-        .await?;
-        let msg = alice.get_last_msg().await;
-        assert_eq!(msg.state, MessageState::OutDelivered);
-
-        // Due to a bug in the old version running on the other device, Alice receives a read
-        // receipt from self.
-        dc_receive_imf(
-            &alice,
-                "Received: (Postfix, from userid 1000); Mon, 4 Dec 2006 14:51:39 +0100 (CET)\n\
-                 From: alice@example.com\n\
-                 To: alice@example.com\n\
-                 Subject: message opened\n\
-                 Date: Sun, 22 Mar 2020 23:37:57 +0000\n\
-                 Chat-Version: 1.0\n\
-                 Message-ID: second@example.com\n\
-                 Content-Type: multipart/report; report-type=disposition-notification; boundary=\"SNIPP\"\n\
-                 \n\
-                 \n\
-                 --SNIPP\n\
-                 Content-Type: text/plain; charset=utf-8\n\
-                 \n\
-                 Read receipts do not guarantee sth. was read.\n\
-                 \n\
-                 \n\
-                 --SNIPP\n\
-                 Content-Type: message/disposition-notification\n\
-                 \n\
-                 Original-Recipient: rfc822;bob@example.com\n\
-                 Final-Recipient: rfc822;bob@example.com\n\
-                 Original-Message-ID: <first@example.com>\n\
-                 Disposition: manual-action/MDN-sent-automatically; displayed\n\
-                 \n\
-                 \n\
-                 --SNIPP--"
-            .as_bytes(),
-            "INBOX",
-            2,
-            false,
-        )
-        .await?;
-
-        // Check that the state has not changed to `MessageState::OutMdnRcvd`.
-        let msg = Message::load_from_db(&alice, msg.id).await?;
-        assert_eq!(msg.state, MessageState::OutDelivered);
-
-        Ok(())
-    }
 }

src/oauth2.rs

@@ -1,4 +1,4 @@
-//! OAuth 2 module.
+//! OAuth 2 module
 
 use std::collections::HashMap;
 
@@ -6,14 +6,13 @@ use anyhow::Result;
 use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 use serde::Deserialize;
 
-use crate::config::Config;
 use crate::context::Context;
 use crate::dc_tools::time;
 use crate::provider;
 use crate::provider::Oauth2Authorizer;
 
 const OAUTH2_GMAIL: Oauth2 = Oauth2 {
-    // see <https://developers.google.com/identity/protocols/OAuth2InstalledApp>
+    // see https://developers.google.com/identity/protocols/OAuth2InstalledApp
     client_id: "959970109878-4mvtgf6feshskf7695nfln6002mom908.apps.googleusercontent.com",
     get_code: "https://accounts.google.com/o/oauth2/auth?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&response_type=code&scope=https%3A%2F%2Fmail.google.com%2F%20email&access_type=offline",
     init_token: "https://accounts.google.com/o/oauth2/token?client_id=$CLIENT_ID&redirect_uri=$REDIRECT_URI&code=$CODE&grant_type=authorization_code",
@@ -22,7 +21,7 @@ const OAUTH2_GMAIL: Oauth2 = Oauth2 {
 };
 
 const OAUTH2_YANDEX: Oauth2 = Oauth2 {
-    // see <https://tech.yandex.com/oauth/doc/dg/reference/auto-code-client-docpage/>
+    // see https://tech.yandex.com/oauth/doc/dg/reference/auto-code-client-docpage/
     client_id: "c4d0b6735fc8420a816d7e1303469341",
     get_code: "https://oauth.yandex.com/authorize?client_id=$CLIENT_ID&response_type=code&scope=mail%3Aimap_full%20mail%3Asmtp&force_confirm=true",
     init_token: "https://oauth.yandex.com/token?grant_type=authorization_code&code=$CODE&client_id=$CLIENT_ID&client_secret=58b8c6e94cf44fbe952da8511955dacf",
@@ -42,7 +41,7 @@ struct Oauth2 {
 /// OAuth 2 Access Token Response
 #[derive(Debug, Deserialize)]
 struct Response {
-    // Should always be there according to: <https://www.oauth.com/oauth2-servers/access-tokens/access-token-response/>
+    // Should always be there according to: https://www.oauth.com/oauth2-servers/access-tokens/access-token-response/
     // but previous code handled its abscense.
     access_token: Option<String>,
     token_type: String,
@@ -56,19 +55,22 @@ pub async fn dc_get_oauth2_url(
     context: &Context,
     addr: &str,
     redirect_uri: &str,
-) -> Result<Option<String>> {
-    let socks5_enabled = context.get_config_bool(Config::Socks5Enabled).await?;
-    if let Some(oauth2) = Oauth2::from_address(addr, socks5_enabled).await {
-        context
+) -> Option<String> {
+    if let Some(oauth2) = Oauth2::from_address(addr).await {
+        if context
             .sql
             .set_raw_config("oauth2_pending_redirect_uri", Some(redirect_uri))
-            .await?;
+            .await
+            .is_err()
+        {
+            return None;
+        }
         let oauth2_url = replace_in_uri(oauth2.get_code, "$CLIENT_ID", oauth2.client_id);
         let oauth2_url = replace_in_uri(&oauth2_url, "$REDIRECT_URI", redirect_uri);
 
-        Ok(Some(oauth2_url))
+        Some(oauth2_url)
     } else {
-        Ok(None)
+        None
     }
 }
 
@@ -78,8 +80,7 @@ pub async fn dc_get_oauth2_access_token(
     code: &str,
     regenerate: bool,
 ) -> Result<Option<String>> {
-    let socks5_enabled = context.get_config_bool(Config::Socks5Enabled).await?;
-    if let Some(oauth2) = Oauth2::from_address(addr, socks5_enabled).await {
+    if let Some(oauth2) = Oauth2::from_address(addr).await {
         let lock = context.oauth2_mutex.lock().await;
 
         // read generated token
@@ -128,7 +129,7 @@ pub async fn dc_get_oauth2_access_token(
             };
 
         // to allow easier specification of different configurations,
-        // token_url is in GET-method-format, sth. as <https://domain?param1=val1&param2=val2> -
+        // token_url is in GET-method-format, sth. as https://domain?param1=val1&param2=val2 -
         // convert this to POST-format ...
         let mut parts = token_url.splitn(2, '?');
         let post_url = parts.next().unwrap_or_default();
@@ -224,8 +225,7 @@ pub async fn dc_get_oauth2_addr(
     addr: &str,
     code: &str,
 ) -> Result<Option<String>> {
-    let socks5_enabled = context.get_config_bool(Config::Socks5Enabled).await?;
-    let oauth2 = match Oauth2::from_address(addr, socks5_enabled).await {
+    let oauth2 = match Oauth2::from_address(addr).await {
         Some(o) => o,
         None => return Ok(None),
     };
@@ -253,13 +253,13 @@ pub async fn dc_get_oauth2_addr(
 }
 
 impl Oauth2 {
-    async fn from_address(addr: &str, skip_mx: bool) -> Option<Self> {
+    async fn from_address(addr: &str) -> Option<Self> {
         let addr_normalized = normalize_addr(addr);
         if let Some(domain) = addr_normalized
             .find('@')
             .map(|index| addr_normalized.split_at(index + 1).1)
         {
-            if let Some(oauth2_authorizer) = provider::get_provider_info(domain, skip_mx)
+            if let Some(oauth2_authorizer) = provider::get_provider_info(domain)
                 .await
                 .and_then(|provider| provider.oauth2_authorizer.as_ref())
             {
@@ -357,29 +357,29 @@ mod tests {
     #[async_std::test]
     async fn test_oauth_from_address() {
         assert_eq!(
-            Oauth2::from_address("hello@gmail.com", false).await,
+            Oauth2::from_address("hello@gmail.com").await,
             Some(OAUTH2_GMAIL)
         );
         assert_eq!(
-            Oauth2::from_address("hello@googlemail.com", false).await,
+            Oauth2::from_address("hello@googlemail.com").await,
             Some(OAUTH2_GMAIL)
         );
         assert_eq!(
-            Oauth2::from_address("hello@yandex.com", false).await,
+            Oauth2::from_address("hello@yandex.com").await,
             Some(OAUTH2_YANDEX)
         );
         assert_eq!(
-            Oauth2::from_address("hello@yandex.ru", false).await,
+            Oauth2::from_address("hello@yandex.ru").await,
             Some(OAUTH2_YANDEX)
         );
 
-        assert_eq!(Oauth2::from_address("hello@web.de", false).await, None);
+        assert_eq!(Oauth2::from_address("hello@web.de").await, None);
     }
 
     #[async_std::test]
     async fn test_oauth_from_mx() {
         assert_eq!(
-            Oauth2::from_address("hello@google.com", false).await,
+            Oauth2::from_address("hello@google.com").await,
             Some(OAUTH2_GMAIL)
         );
     }
@@ -399,9 +399,7 @@ mod tests {
         let ctx = TestContext::new().await;
         let addr = "dignifiedquire@gmail.com";
         let redirect_uri = "chat.delta:/com.b44t.messenger";
-        let res = dc_get_oauth2_url(&ctx.ctx, addr, redirect_uri)
-            .await
-            .unwrap();
+        let res = dc_get_oauth2_url(&ctx.ctx, addr, redirect_uri).await;
 
         assert_eq!(res, Some("https://accounts.google.com/o/oauth2/auth?client_id=959970109878%2D4mvtgf6feshskf7695nfln6002mom908%2Eapps%2Egoogleusercontent%2Ecom&redirect_uri=chat%2Edelta%3A%2Fcom%2Eb44t%2Emessenger&response_type=code&scope=https%3A%2F%2Fmail.google.com%2F%20email&access_type=offline".into()));
     }

src/param.rs

@@ -60,9 +60,6 @@ pub enum Param {
     /// For Messages
     WantsMdn = b'r',
 
-    /// For Messages: a message with Auto-Submitted header ("bot").
-    Bot = b'b',
-
     /// For Messages: unset or 0=not forwarded,
     /// 1=forwarded from unknown msg_id, >9 forwarded from msg_id
     Forwarded = b'a',
@@ -139,27 +136,6 @@ pub enum Param {
 
     /// For MDN-sending job
     MsgId = b'I',
-
-    /// For Contacts: timestamp of status (aka signature or footer) update.
-    StatusTimestamp = b'j',
-
-    /// For Contacts and Chats: timestamp of avatar update.
-    AvatarTimestamp = b'J',
-
-    /// For Chats: timestamp of status/signature/footer update.
-    EphemeralSettingsTimestamp = b'B',
-
-    /// For Chats: timestamp of subject update.
-    SubjectTimestamp = b'C',
-
-    /// For Chats: timestamp of group name update.
-    GroupNameTimestamp = b'g',
-
-    /// For Chats: timestamp of group name update.
-    MemberListTimestamp = b'k',
-
-    /// For Chats: timestamp of protection settings update.
-    ProtectionSettingsTimestamp = b'L',
 }
 
 /// An object for handling key=value parameter lists.
@@ -266,11 +242,6 @@ impl Params {
         self.get(key).and_then(|s| s.parse().ok())
     }
 
-    /// Get the given parameter and parse as `i64`.
-    pub fn get_i64(&self, key: Param) -> Option<i64> {
-        self.get(key).and_then(|s| s.parse().ok())
-    }
-
     /// Get the given parameter and parse as `bool`.
     pub fn get_bool(&self, key: Param) -> Option<bool> {
         self.get_int(key).map(|v| v != 0)
@@ -372,12 +343,6 @@ impl Params {
         self
     }
 
-    /// Set the given paramter to the passed in `i64`.
-    pub fn set_i64(&mut self, key: Param, value: i64) -> &mut Self {
-        self.set(key, value.to_string());
-        self
-    }
-
     /// Set the given parameter to the passed in `f64` .
     pub fn set_float(&mut self, key: Param, value: f64) -> &mut Self {
         self.set(key, format!("{}", value));

src/peerstate.rs

@@ -1,4 +1,4 @@
-//! # [Autocrypt Peer State](https://autocrypt.org/level1.html#peer-state-management) module.
+//! # [Autocrypt Peer State](https://autocrypt.org/level1.html#peer-state-management) module
 
 use std::collections::HashSet;
 use std::fmt;
@@ -260,25 +260,22 @@ impl Peerstate {
     }
 
     /// Adds a warning to the chat corresponding to peerstate if fingerprint has changed.
-    pub(crate) async fn handle_fingerprint_change(
-        &self,
-        context: &Context,
-        timestamp: i64,
-    ) -> Result<()> {
+    pub(crate) async fn handle_fingerprint_change(&self, context: &Context) -> Result<()> {
         if self.fingerprint_changed {
             if let Some(contact_id) = context
                 .sql
                 .query_get_value("SELECT id FROM contacts WHERE addr=?;", paramsv![self.addr])
                 .await?
             {
-                let chat_id = ChatIdBlocked::get_for_contact(context, contact_id, Blocked::Request)
-                    .await?
-                    .id;
+                let chat_id =
+                    ChatIdBlocked::get_for_contact(context, contact_id, Blocked::Deaddrop)
+                        .await?
+                        .id;
 
                 let msg = stock_str::contact_setup_changed(context, self.addr.clone()).await;
 
-                chat::add_info_msg(context, chat_id, msg, timestamp).await;
-                context.emit_event(EventType::ChatModified(chat_id));
+                chat::add_info_msg(context, chat_id, msg).await;
+                emit_event!(context, EventType::ChatModified(chat_id));
             } else {
                 bail!("contact with peerstate.addr {:?} not found", &self.addr);
             }
@@ -496,6 +493,12 @@ impl Peerstate {
     }
 }
 
+impl From<crate::key::FingerprintError> for rusqlite::Error {
+    fn from(_source: crate::key::FingerprintError) -> Self {
+        Self::InvalidColumnType(0, "Invalid fingerprint".into(), rusqlite::types::Type::Text)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/pgp.rs

@@ -1,4 +1,4 @@
-//! OpenPGP helper module using [rPGP facilities](https://github.com/rpgp/rpgp).
+//! OpenPGP helper module using [rPGP facilities](https://github.com/rpgp/rpgp)
 
 use std::collections::{BTreeMap, HashSet};
 use std::io;
@@ -86,7 +86,7 @@ impl<'a> PublicKeyTrait for SignedPublicKeyOrSubkey<'a> {
     }
 }
 
-/// Split data from PGP Armored Data as defined in <https://tools.ietf.org/html/rfc4880#section-6.2>.
+/// Split data from PGP Armored Data as defined in https://tools.ietf.org/html/rfc4880#section-6.2.
 ///
 /// Returns (type, headers, base64 encoded body).
 pub fn split_armored_data(buf: &[u8]) -> Result<(BlockType, BTreeMap<String, String>, Vec<u8>)> {

src/plaintext.rs

@@ -1,5 +1,4 @@
-//! Handle plain text together with some attributes.
-
+///! Handle plain text together with some attributes.
 use crate::simplify::split_lines;
 use once_cell::sync::Lazy;
 
@@ -65,7 +64,7 @@ impl PlainText {
                 // flowed text as of RFC 3676 -
                 // a leading space shall be removed
                 // and is only there to allow > at the beginning of a line that is no quote.
-                line = line.strip_prefix(' ').unwrap_or(&line).to_string();
+                line = line.strip_prefix(" ").unwrap_or(&line).to_string();
                 if is_quote {
                     line = "<em>".to_owned() + &line + "</em>";
                 }

src/provider.rs

@@ -1,4 +1,4 @@
-//! [Provider database](https://providers.delta.chat/) module.
+//! [Provider database](https://providers.delta.chat/) module
 
 mod data;
 
@@ -89,17 +89,15 @@ pub struct Provider {
 ///
 /// For compatibility, email address can be passed to this function
 /// instead of the domain.
-pub async fn get_provider_info(domain: &str, skip_mx: bool) -> Option<&'static Provider> {
+pub async fn get_provider_info(domain: &str) -> Option<&'static Provider> {
     let domain = domain.rsplitn(2, '@').next()?;
 
     if let Some(provider) = get_provider_by_domain(domain) {
         return Some(provider);
     }
 
-    if !skip_mx {
-        if let Some(provider) = get_provider_by_mx(domain).await {
-            return Some(provider);
-        }
+    if let Some(provider) = get_provider_by_mx(domain).await {
+        return Some(provider);
     }
 
     None
@@ -223,17 +221,11 @@ mod tests {
 
     #[async_std::test]
     async fn test_get_provider_info() {
-        assert!(get_provider_info("", false).await.is_none());
-        assert!(get_provider_info("google.com", false).await.unwrap().id == "gmail");
+        assert!(get_provider_info("").await.is_none());
+        assert!(get_provider_info("google.com").await.unwrap().id == "gmail");
 
         // get_provider_info() accepts email addresses for backwards compatibility
-        assert!(
-            get_provider_info("example@google.com", false)
-                .await
-                .unwrap()
-                .id
-                == "gmail"
-        );
+        assert!(get_provider_info("example@google.com").await.unwrap().id == "gmail");
     }
 
     #[test]

src/provider/data.rs

@@ -8,23 +8,6 @@ use std::collections::HashMap;
 
 use once_cell::sync::Lazy;
 
-// 163.md: 163.com
-static P_163: Lazy<Provider> = Lazy::new(|| {
-    Provider {
-    id: "163",
-    status: Status::Broken,
-    before_login_hint: "163 Mail does not work since it forces the email clients to connect with an IMAP ID, which is currently not the case of Delta Chat.",
-    after_login_hint: "",
-    overview_page: "https://providers.delta.chat/163",
-    server: vec![
-    ],
-    config_defaults: None,
-    strict_tls: true,
-    max_smtp_rcpt_to: None,
-    oauth2_authorizer: None,
-}
-});
-
 // aktivix.org.md: aktivix.org
 static P_AKTIVIX_ORG: Lazy<Provider> = Lazy::new(|| Provider {
     id: "aktivix.org",
@@ -585,8 +568,8 @@ static P_HEY_COM: Lazy<Provider> = Lazy::new(|| {
 // i.ua.md: i.ua
 static P_I_UA: Lazy<Provider> = Lazy::new(|| Provider {
     id: "i.ua",
-    status: Status::Broken,
-    before_login_hint: "Протокол IMAP не предоставляется и не планируется.",
+    status: Status::Ok,
+    before_login_hint: "",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/i-ua",
     server: vec![],
@@ -614,7 +597,8 @@ static P_I3_NET: Lazy<Provider> = Lazy::new(|| Provider {
 static P_ICLOUD: Lazy<Provider> = Lazy::new(|| Provider {
     id: "icloud",
     status: Status::Preparation,
-    before_login_hint: "You must create an app-specific password for Delta Chat before login.",
+    before_login_hint:
+        "You must create an app-specific password for Delta Chat before you can login.",
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/icloud",
     server: vec![
@@ -667,7 +651,7 @@ static P_KONTENT_COM: Lazy<Provider> = Lazy::new(|| Provider {
     oauth2_authorizer: None,
 });
 
-// mail.ru.md: mail.ru, inbox.ru, internet.ru, bk.ru, list.ru
+// mail.ru.md: mail.ru, inbox.ru, bk.ru, list.ru
 static P_MAIL_RU: Lazy<Provider> = Lazy::new(|| {
     Provider {
     id: "mail.ru",
@@ -676,8 +660,6 @@ static P_MAIL_RU: Lazy<Provider> = Lazy::new(|| {
     after_login_hint: "",
     overview_page: "https://providers.delta.chat/mail-ru",
     server: vec![
-        Server { protocol: Imap, socket: Ssl, hostname: "imap.mail.ru", port: 993, username_pattern: Email },
-        Server { protocol: Smtp, socket: Ssl, hostname: "smtp.mail.ru", port: 465, username_pattern: Email },
     ],
     config_defaults: None,
     strict_tls: true,
@@ -686,35 +668,6 @@ static P_MAIL_RU: Lazy<Provider> = Lazy::new(|| {
 }
 });
 
-// mail2tor.md: mail2tor.com
-static P_MAIL2TOR: Lazy<Provider> = Lazy::new(|| Provider {
-    id: "mail2tor",
-    status: Status::Preparation,
-    before_login_hint: "Tor is needed to connect to the email servers.",
-    after_login_hint: "",
-    overview_page: "https://providers.delta.chat/mail2tor",
-    server: vec![
-        Server {
-            protocol: Imap,
-            socket: Plain,
-            hostname: "g77kjrad6bafzzyldqvffq6kxlsgphcygptxhnn4xlnktfgaqshilmyd.onion",
-            port: 143,
-            username_pattern: Email,
-        },
-        Server {
-            protocol: Smtp,
-            socket: Plain,
-            hostname: "xc7tgk2c5onxni2wsy76jslfsitxjbbptejnqhw6gy2ft7khpevhc7ad.onion",
-            port: 25,
-            username_pattern: Email,
-        },
-    ],
-    config_defaults: None,
-    strict_tls: true,
-    max_smtp_rcpt_to: None,
-    oauth2_authorizer: None,
-});
-
 // mailbox.org.md: mailbox.org, secure.mailbox.org
 static P_MAILBOX_ORG: Lazy<Provider> = Lazy::new(|| Provider {
     id: "mailbox.org",
@@ -820,35 +773,6 @@ static P_NAUTA_CU: Lazy<Provider> = Lazy::new(|| Provider {
     oauth2_authorizer: None,
 });
 
-// naver.md: naver.com
-static P_NAVER: Lazy<Provider> = Lazy::new(|| Provider {
-    id: "naver",
-    status: Status::Preparation,
-    before_login_hint: "Manually enabling IMAP/SMTP is required.",
-    after_login_hint: "",
-    overview_page: "https://providers.delta.chat/naver",
-    server: vec![
-        Server {
-            protocol: Imap,
-            socket: Ssl,
-            hostname: "imap.naver.com",
-            port: 993,
-            username_pattern: Emaillocalpart,
-        },
-        Server {
-            protocol: Smtp,
-            socket: Starttls,
-            hostname: "smtp.naver.com",
-            port: 587,
-            username_pattern: Email,
-        },
-    ],
-    config_defaults: None,
-    strict_tls: true,
-    max_smtp_rcpt_to: None,
-    oauth2_authorizer: None,
-});
-
 // outlook.com.md: hotmail.com, outlook.com, office365.com, outlook.com.tr, live.com
 static P_OUTLOOK_COM: Lazy<Provider> = Lazy::new(|| Provider {
     id: "outlook.com",
@@ -878,7 +802,7 @@ static P_OUTLOOK_COM: Lazy<Provider> = Lazy::new(|| Provider {
     oauth2_authorizer: None,
 });
 
-// posteo.md: posteo.de, posteo.af, posteo.at, posteo.be, posteo.ch, posteo.cl, posteo.co, posteo.co.uk, posteo.com.br, posteo.cr, posteo.cz, posteo.dk, posteo.ee, posteo.es, posteo.eu, posteo.fi, posteo.gl, posteo.gr, posteo.hn, posteo.hr, posteo.hu, posteo.ie, posteo.in, posteo.is, posteo.it, posteo.jp, posteo.la, posteo.li, posteo.lt, posteo.lu, posteo.me, posteo.mx, posteo.my, posteo.net, posteo.nl, posteo.no, posteo.nz, posteo.org, posteo.pe, posteo.pl, posteo.pm, posteo.pt, posteo.ro, posteo.ru, posteo.se, posteo.sg, posteo.si, posteo.tn, posteo.uk, posteo.us
+// posteo.md: posteo.de, posteo.af, posteo.at, posteo.be, posteo.ch, posteo.cl, posteo.co, posteo.co.uk, posteo.com.br, posteo.cr, posteo.cz, posteo.dk, posteo.ee, posteo.es, posteo.eu, posteo.fi, posteo.gl, posteo.gr, posteo.hn, posteo.hr, posteo.hu, posteo.ie, posteo.in, posteo.is, posteo.jp, posteo.la, posteo.li, posteo.lt, posteo.lu, posteo.me, posteo.mx, posteo.my, posteo.net, posteo.nl, posteo.no, posteo.nz, posteo.org, posteo.pe, posteo.pl, posteo.pm, posteo.pt, posteo.ro, posteo.ru, posteo.se, posteo.sg, posteo.si, posteo.tn, posteo.uk, posteo.us
 static P_POSTEO: Lazy<Provider> = Lazy::new(|| Provider {
     id: "posteo",
     status: Status::Ok,
@@ -924,25 +848,6 @@ static P_PROTONMAIL: Lazy<Provider> = Lazy::new(|| {
 }
 });
 
-// qq.md: qq.com, foxmail.com
-static P_QQ: Lazy<Provider> = Lazy::new(|| {
-    Provider {
-    id: "qq",
-    status: Status::Preparation,
-    before_login_hint: "Manually enabling IMAP/SMTP and creating an app-specific password for Delta Chat are required.",
-    after_login_hint: "",
-    overview_page: "https://providers.delta.chat/qq",
-    server: vec![
-        Server { protocol: Imap, socket: Ssl, hostname: "imap.qq.com", port: 993, username_pattern: Emaillocalpart },
-        Server { protocol: Smtp, socket: Starttls, hostname: "smtp.qq.com", port: 465, username_pattern: Email },
-    ],
-    config_defaults: None,
-    strict_tls: true,
-    max_smtp_rcpt_to: None,
-    oauth2_authorizer: None,
-}
-});
-
 // riseup.net.md: riseup.net
 static P_RISEUP_NET: Lazy<Provider> = Lazy::new(|| Provider {
     id: "riseup.net",
@@ -1189,35 +1094,6 @@ static P_VFEMAIL: Lazy<Provider> = Lazy::new(|| Provider {
     oauth2_authorizer: None,
 });
 
-// vivaldi.md: vivaldi.net
-static P_VIVALDI: Lazy<Provider> = Lazy::new(|| Provider {
-    id: "vivaldi",
-    status: Status::Ok,
-    before_login_hint: "",
-    after_login_hint: "",
-    overview_page: "https://providers.delta.chat/vivaldi",
-    server: vec![
-        Server {
-            protocol: Imap,
-            socket: Starttls,
-            hostname: "imap.vivaldi.net",
-            port: 143,
-            username_pattern: Email,
-        },
-        Server {
-            protocol: Smtp,
-            socket: Starttls,
-            hostname: "smtp.vivaldi.net",
-            port: 587,
-            username_pattern: Email,
-        },
-    ],
-    config_defaults: None,
-    strict_tls: true,
-    max_smtp_rcpt_to: None,
-    oauth2_authorizer: None,
-});
-
 // vodafone.de.md: vodafone.de, vodafonemail.de
 static P_VODAFONE_DE: Lazy<Provider> = Lazy::new(|| Provider {
     id: "vodafone.de",
@@ -1315,25 +1191,6 @@ static P_YANDEX_RU: Lazy<Provider> = Lazy::new(|| Provider {
     oauth2_authorizer: Some(Oauth2Authorizer::Yandex),
 });
 
-// yggmail.md: yggmail
-static P_YGGMAIL: Lazy<Provider> = Lazy::new(|| {
-    Provider {
-    id: "yggmail",
-    status: Status::Preparation,
-    before_login_hint: "An Yggmail companion app needs to be installed on your device to access the Yggmail network.",
-    after_login_hint: "Make sure, the Yggmail companion app runs whenever you want to use this account. Note, that you usually cannot write from @yggmail addresses to normal e-mail-addresses (as @gmx.net). However, you can create another account in the normal e-mail-network for this purpose.",
-    overview_page: "https://providers.delta.chat/yggmail",
-    server: vec![
-        Server { protocol: Imap, socket: Plain, hostname: "localhost", port: 1143, username_pattern: Email },
-        Server { protocol: Smtp, socket: Plain, hostname: "localhost", port: 1025, username_pattern: Email },
-    ],
-    config_defaults: None,
-    strict_tls: true,
-    max_smtp_rcpt_to: None,
-    oauth2_authorizer: None,
-}
-});
-
 // ziggo.nl.md: ziggo.nl
 static P_ZIGGO_NL: Lazy<Provider> = Lazy::new(|| Provider {
     id: "ziggo.nl",
@@ -1363,38 +1220,8 @@ static P_ZIGGO_NL: Lazy<Provider> = Lazy::new(|| Provider {
     oauth2_authorizer: None,
 });
 
-// zoho.md: zohomail.eu, zoho.com
-static P_ZOHO: Lazy<Provider> = Lazy::new(|| Provider {
-    id: "zoho",
-    status: Status::Preparation,
-    before_login_hint: "To use Zoho Mail, you have to turn on IMAP in the Zoho Mail backend.",
-    after_login_hint: "",
-    overview_page: "https://providers.delta.chat/zoho",
-    server: vec![
-        Server {
-            protocol: Imap,
-            socket: Ssl,
-            hostname: "imap.zoho.eu",
-            port: 993,
-            username_pattern: Email,
-        },
-        Server {
-            protocol: Smtp,
-            socket: Ssl,
-            hostname: "smtp.zoho.eu",
-            port: 465,
-            username_pattern: Email,
-        },
-    ],
-    config_defaults: None,
-    strict_tls: true,
-    max_smtp_rcpt_to: None,
-    oauth2_authorizer: None,
-});
-
 pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| {
     [
-        ("163.com", &*P_163),
         ("aktivix.org", &*P_AKTIVIX_ORG),
         ("aol.com", &*P_AOL),
         ("arcor.de", &*P_ARCOR_DE),
@@ -1440,15 +1267,12 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
         ("kontent.com", &*P_KONTENT_COM),
         ("mail.ru", &*P_MAIL_RU),
         ("inbox.ru", &*P_MAIL_RU),
-        ("internet.ru", &*P_MAIL_RU),
         ("bk.ru", &*P_MAIL_RU),
         ("list.ru", &*P_MAIL_RU),
-        ("mail2tor.com", &*P_MAIL2TOR),
         ("mailbox.org", &*P_MAILBOX_ORG),
         ("secure.mailbox.org", &*P_MAILBOX_ORG),
         ("mailo.com", &*P_MAILO_COM),
         ("nauta.cu", &*P_NAUTA_CU),
-        ("naver.com", &*P_NAVER),
         ("hotmail.com", &*P_OUTLOOK_COM),
         ("outlook.com", &*P_OUTLOOK_COM),
         ("office365.com", &*P_OUTLOOK_COM),
@@ -1478,7 +1302,6 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
         ("posteo.ie", &*P_POSTEO),
         ("posteo.in", &*P_POSTEO),
         ("posteo.is", &*P_POSTEO),
-        ("posteo.it", &*P_POSTEO),
         ("posteo.jp", &*P_POSTEO),
         ("posteo.la", &*P_POSTEO),
         ("posteo.li", &*P_POSTEO),
@@ -1506,8 +1329,6 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
         ("posteo.us", &*P_POSTEO),
         ("protonmail.com", &*P_PROTONMAIL),
         ("protonmail.ch", &*P_PROTONMAIL),
-        ("qq.com", &*P_QQ),
-        ("foxmail.com", &*P_QQ),
         ("riseup.net", &*P_RISEUP_NET),
         ("rogers.com", &*P_ROGERS_COM),
         ("systemausfall.org", &*P_SYSTEMAUSFALL_ORG),
@@ -1525,7 +1346,6 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
         ("ukr.net", &*P_UKR_NET),
         ("undernet.uy", &*P_UNDERNET_UY),
         ("vfemail.net", &*P_VFEMAIL),
-        ("vivaldi.net", &*P_VIVALDI),
         ("vodafone.de", &*P_VODAFONE_DE),
         ("vodafonemail.de", &*P_VODAFONE_DE),
         ("web.de", &*P_WEB_DE),
@@ -1577,10 +1397,7 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
         ("yandex.ua", &*P_YANDEX_RU),
         ("ya.ru", &*P_YANDEX_RU),
         ("narod.ru", &*P_YANDEX_RU),
-        ("yggmail", &*P_YGGMAIL),
         ("ziggo.nl", &*P_ZIGGO_NL),
-        ("zohomail.eu", &*P_ZOHO),
-        ("zoho.com", &*P_ZOHO),
     ]
     .iter()
     .copied()
@@ -1589,7 +1406,6 @@ pub(crate) static PROVIDER_DATA: Lazy<HashMap<&'static str, &'static Provider>>
 
 pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> = Lazy::new(|| {
     [
-        ("163", &*P_163),
         ("aktivix.org", &*P_AKTIVIX_ORG),
         ("aol", &*P_AOL),
         ("arcor.de", &*P_ARCOR_DE),
@@ -1618,15 +1434,12 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
         ("kolst.com", &*P_KOLST_COM),
         ("kontent.com", &*P_KONTENT_COM),
         ("mail.ru", &*P_MAIL_RU),
-        ("mail2tor", &*P_MAIL2TOR),
         ("mailbox.org", &*P_MAILBOX_ORG),
         ("mailo.com", &*P_MAILO_COM),
         ("nauta.cu", &*P_NAUTA_CU),
-        ("naver", &*P_NAVER),
         ("outlook.com", &*P_OUTLOOK_COM),
         ("posteo", &*P_POSTEO),
         ("protonmail", &*P_PROTONMAIL),
-        ("qq", &*P_QQ),
         ("riseup.net", &*P_RISEUP_NET),
         ("rogers.com", &*P_ROGERS_COM),
         ("systemausfall.org", &*P_SYSTEMAUSFALL_ORG),
@@ -1638,14 +1451,11 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
         ("ukr.net", &*P_UKR_NET),
         ("undernet.uy", &*P_UNDERNET_UY),
         ("vfemail", &*P_VFEMAIL),
-        ("vivaldi", &*P_VIVALDI),
         ("vodafone.de", &*P_VODAFONE_DE),
         ("web.de", &*P_WEB_DE),
         ("yahoo", &*P_YAHOO),
         ("yandex.ru", &*P_YANDEX_RU),
-        ("yggmail", &*P_YGGMAIL),
         ("ziggo.nl", &*P_ZIGGO_NL),
-        ("zoho", &*P_ZOHO),
     ]
     .iter()
     .copied()
@@ -1653,4 +1463,4 @@ pub(crate) static PROVIDER_IDS: Lazy<HashMap<&'static str, &'static Provider>> =
 });
 
 pub static PROVIDER_UPDATED: Lazy<chrono::NaiveDate> =
-    Lazy::new(|| chrono::NaiveDate::from_ymd(2021, 8, 17));
+    Lazy::new(|| chrono::NaiveDate::from_ymd(2021, 6, 7));

src/provider/update.py

@@ -63,7 +63,7 @@ def process_data(data, file):
         raise TypeError("no domains found")
     for domain in data["domains"]:
         domain = cleanstr(domain)
-        if domain == "" or domain.lower() != domain:
+        if domain == "" or domain.count(".") < 1 or domain.lower() != domain:
             raise TypeError("bad domain: " + domain)
 
         global domains_set
@@ -84,7 +84,7 @@ def process_data(data, file):
         for s in data["server"]:
             hostname = cleanstr(s.get("hostname", ""))
             port = int(s.get("port", ""))
-            if hostname == "" or hostname.lower() != hostname or port <= 0:
+            if hostname == "" or hostname.count(".") < 1 or port <= 0:
                 raise TypeError("bad hostname or port")
 
             protocol = s.get("type", "").upper()
@@ -96,7 +96,7 @@ def process_data(data, file):
                 raise TypeError("bad protocol")
 
             socket = s.get("socket", "").upper()
-            if socket != "STARTTLS" and socket != "SSL" and socket != "PLAIN":
+            if socket != "STARTTLS" and socket != "SSL":
                 raise TypeError("bad socket")
 
             username_pattern = s.get("username_pattern", "EMAIL").upper()

src/qr.rs

@@ -1,4 +1,4 @@
-//! # QR code module.
+//! # QR code module
 
 use anyhow::{bail, ensure, format_err, Error};
 use once_cell::sync::Lazy;
@@ -6,18 +6,16 @@ use percent_encoding::percent_decode_str;
 use serde::Deserialize;
 use std::collections::BTreeMap;
 
-use crate::chat::{self, get_chat_id_by_grpid, ChatIdBlocked};
+use crate::chat::{self, ChatIdBlocked};
 use crate::config::Config;
 use crate::constants::Blocked;
 use crate::contact::{addr_normalize, may_be_valid_addr, Contact, Origin};
 use crate::context::Context;
-use crate::dc_tools::time;
 use crate::key::Fingerprint;
 use crate::log::LogExt;
 use crate::lot::{Lot, LotState};
 use crate::message::Message;
 use crate::peerstate::Peerstate;
-use crate::token;
 
 const OPENPGP4FPR_SCHEME: &str = "OPENPGP4FPR:"; // yes: uppercase
 const DCACCOUNT_SCHEME: &str = "DCACCOUNT:";
@@ -88,7 +86,11 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
     };
     let fingerprint: Fingerprint = match fingerprint.parse() {
         Ok(fp) => fp,
-        Err(err) => return err.context("Failed to parse fingerprint in QR code").into(),
+        Err(err) => {
+            return Error::new(err)
+                .context("Failed to parse fingerprint in QR code")
+                .into()
+        }
     };
 
     let param: BTreeMap<&str, &str> = fragment
@@ -157,17 +159,11 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
                     .map(|(id, _)| id)
                     .unwrap_or_default();
 
-            if let Ok(chat) = ChatIdBlocked::get_for_contact(context, lot.id, Blocked::Request)
+            if let Ok(chat) = ChatIdBlocked::get_for_contact(context, lot.id, Blocked::Deaddrop)
                 .await
                 .log_err(context, "Failed to create (new) chat for contact")
             {
-                chat::add_info_msg(
-                    context,
-                    chat.id,
-                    format!("{} verified.", peerstate.addr),
-                    time(),
-                )
-                .await;
+                chat::add_info_msg(context, chat.id, format!("{} verified.", peerstate.addr)).await;
             }
         } else if let Some(addr) = addr {
             lot.state = LotState::QrFprMismatch;
@@ -197,25 +193,6 @@ async fn decode_openpgp(context: &Context, qr: &str) -> Lot {
         lot.fingerprint = Some(fingerprint);
         lot.invitenumber = invitenumber;
         lot.auth = auth;
-
-        // scanning own qr-code offers withdraw/revive instead of secure-join
-        if context.is_self_addr(&addr).await.unwrap_or_default() {
-            if let Some(ref invitenumber) = lot.invitenumber {
-                lot.state =
-                    if token::exists(context, token::Namespace::InviteNumber, &*invitenumber).await
-                    {
-                        if lot.state == LotState::QrAskVerifyContact {
-                            LotState::QrWithdrawVerifyContact
-                        } else {
-                            LotState::QrWithdrawVerifyGroup
-                        }
-                    } else if lot.state == LotState::QrAskVerifyContact {
-                        LotState::QrReviveVerifyContact
-                    } else {
-                        LotState::QrReviveVerifyGroup
-                    }
-            }
-        }
     } else {
         return format_err!("Missing address").into();
     }
@@ -302,8 +279,7 @@ async fn set_account_from_qr(context: &Context, qr: &str) -> Result<(), Error> {
 }
 
 pub async fn set_config_from_qr(context: &Context, qr: &str) -> Result<(), Error> {
-    let lot = check_qr(context, qr).await;
-    match lot.state {
+    match check_qr(context, qr).await.state {
         LotState::QrAccount => set_account_from_qr(context, qr).await,
         LotState::QrWebrtcInstance => {
             let val = decode_webrtc_instance(context, qr).text2;
@@ -312,45 +288,6 @@ pub async fn set_config_from_qr(context: &Context, qr: &str) -> Result<(), Error
                 .await?;
             Ok(())
         }
-        LotState::QrWithdrawVerifyContact | LotState::QrWithdrawVerifyGroup => {
-            token::delete(
-                context,
-                token::Namespace::InviteNumber,
-                lot.invitenumber.unwrap_or_default().as_str(),
-            )
-            .await?;
-            token::delete(
-                context,
-                token::Namespace::Auth,
-                lot.auth.unwrap_or_default().as_str(),
-            )
-            .await?;
-            Ok(())
-        }
-        LotState::QrReviveVerifyContact | LotState::QrReviveVerifyGroup => {
-            let chat_id = if lot.state == LotState::QrReviveVerifyContact {
-                None
-            } else {
-                get_chat_id_by_grpid(context, &lot.text2.unwrap_or_default())
-                    .await?
-                    .map(|(chat_id, _protected, _blocked)| chat_id)
-            };
-            token::save(
-                context,
-                token::Namespace::InviteNumber,
-                chat_id,
-                &lot.invitenumber.unwrap_or_default(),
-            )
-            .await?;
-            token::save(
-                context,
-                token::Namespace::Auth,
-                chat_id,
-                &lot.auth.unwrap_or_default(),
-            )
-            .await?;
-            Ok(())
-        }
         _ => bail!("qr code does not contain config: {}", qr),
     }
 }
@@ -504,12 +441,9 @@ mod tests {
     use super::*;
 
     use crate::aheader::EncryptPreference;
-    use crate::chat::{create_group_chat, ProtectionStatus};
     use crate::key::DcKey;
     use crate::peerstate::ToSave;
-    use crate::securejoin::dc_get_securejoin_qr;
     use crate::test_utils::{alice_keypair, TestContext};
-    use anyhow::Result;
 
     #[async_std::test]
     async fn test_decode_http() {
@@ -786,62 +720,6 @@ mod tests {
         assert_eq!(res.get_id(), 0);
     }
 
-    #[async_std::test]
-    async fn test_withdraw_verfifycontact() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-        let qr = dc_get_securejoin_qr(&alice, None).await.unwrap();
-
-        // scanning own verfify-contact code offers withdrawing
-        let check = check_qr(&alice, &qr).await;
-        assert_eq!(check.state, LotState::QrWithdrawVerifyContact);
-        assert!(check.text1.is_none());
-        set_config_from_qr(&alice, &qr).await?;
-
-        // scanning withdrawn verfify-contact code offers reviving
-        let check = check_qr(&alice, &qr).await;
-        assert_eq!(check.state, LotState::QrReviveVerifyContact);
-        assert!(check.text1.is_none());
-        set_config_from_qr(&alice, &qr).await?;
-        let check = check_qr(&alice, &qr).await;
-        assert_eq!(check.state, LotState::QrWithdrawVerifyContact);
-
-        // someone else always scans as ask-verify-contact
-        let bob = TestContext::new_bob().await;
-        let check = check_qr(&bob, &qr).await;
-        assert_eq!(check.state, LotState::QrAskVerifyContact);
-        assert!(check.text1.is_none());
-        assert!(set_config_from_qr(&bob, &qr).await.is_err());
-
-        Ok(())
-    }
-
-    #[async_std::test]
-    async fn test_withdraw_verfifygroup() -> Result<()> {
-        let alice = TestContext::new_alice().await;
-        let chat_id = create_group_chat(&alice, ProtectionStatus::Unprotected, "foo").await?;
-        let qr = dc_get_securejoin_qr(&alice, Some(chat_id)).await.unwrap();
-
-        // scanning own verfify-group code offers withdrawing
-        let check = check_qr(&alice, &qr).await;
-        assert_eq!(check.state, LotState::QrWithdrawVerifyGroup);
-        assert_eq!(check.text1, Some("foo".to_string()));
-        set_config_from_qr(&alice, &qr).await?;
-
-        // scanning withdrawn verfify-group code offers reviving
-        let check = check_qr(&alice, &qr).await;
-        assert_eq!(check.state, LotState::QrReviveVerifyGroup);
-        assert_eq!(check.text1, Some("foo".to_string()));
-
-        // someone else always scans as ask-verify-group
-        let bob = TestContext::new_bob().await;
-        let check = check_qr(&bob, &qr).await;
-        assert_eq!(check.state, LotState::QrAskVerifyGroup);
-        assert_eq!(check.text1, Some("foo".to_string()));
-        assert!(set_config_from_qr(&bob, &qr).await.is_err());
-
-        Ok(())
-    }
-
     #[async_std::test]
     async fn test_decode_account() {
         let ctx = TestContext::new().await;

src/quota.rs

@@ -1,181 +0,0 @@
-//! # Support for IMAP QUOTA extension.
-
-use anyhow::{anyhow, Result};
-use async_imap::types::{Quota, QuotaResource};
-use indexmap::IndexMap;
-
-use crate::chat::add_device_msg_with_importance;
-use crate::config::Config;
-use crate::constants::Viewtype;
-use crate::context::Context;
-use crate::dc_tools::time;
-use crate::imap::scan_folders::get_watched_folders;
-use crate::imap::Imap;
-use crate::job::{Action, Status};
-use crate::message::Message;
-use crate::param::Params;
-use crate::{job, stock_str, EventType};
-
-/// warn about a nearly full mailbox after this usage percentage is reached.
-/// quota icon is "yellow".
-pub const QUOTA_WARN_THRESHOLD_PERCENTAGE: u64 = 80;
-
-// warning is already issued at QUOTA_WARN_THRESHOLD_PERCENTAGE,
-// this threshold only makes the quota icon "red".
-pub const QUOTA_ERROR_THRESHOLD_PERCENTAGE: u64 = 99;
-
-/// if quota is below this value (again),
-/// QuotaExceeding is cleared.
-/// This value should be a bit below QUOTA_WARN_THRESHOLD_PERCENTAGE to
-/// avoid jittering and lots of warnings when quota is exactly at the warning threshold.
-pub const QUOTA_ALLCLEAR_PERCENTAGE: u64 = 75;
-
-// if recent quota is older,
-// it is re-fetched on dc_get_connectivity_html()
-pub const QUOTA_MAX_AGE_SECONDS: i64 = 60;
-
-#[derive(Debug)]
-pub struct QuotaInfo {
-    /// Recently loaded quota information.
-    /// set to `Err()` if the provider does not support quota or on other errors,
-    /// set to `Ok()` for valid quota information.
-    /// Updated by `Action::UpdateRecentQuota`
-    pub(crate) recent: Result<IndexMap<String, Vec<QuotaResource>>>,
-
-    /// Timestamp when structure was modified.
-    pub(crate) modified: i64,
-}
-
-async fn get_unique_quota_roots_and_usage(
-    folders: Vec<String>,
-    imap: &mut Imap,
-) -> Result<IndexMap<String, Vec<QuotaResource>>> {
-    let mut unique_quota_roots: IndexMap<String, Vec<QuotaResource>> = IndexMap::new();
-    for folder in folders {
-        let (quota_roots, quotas) = &imap.get_quota_roots(&folder).await?;
-        // if there are new quota roots found in this imap folder, add them to the list
-        for qr_entries in quota_roots {
-            for quota_root_name in &qr_entries.quota_root_names {
-                // the quota for that quota root
-                let quota: Quota = quotas
-                    .iter()
-                    .find(|q| &q.root_name == quota_root_name)
-                    .cloned()
-                    .ok_or_else(|| anyhow!("quota_root should have a quota"))?;
-                // replace old quotas, because between fetching quotaroots for folders,
-                // messages could be recieved and so the usage could have been changed
-                *unique_quota_roots
-                    .entry(quota_root_name.clone())
-                    .or_insert(vec![]) = quota.resources;
-            }
-        }
-    }
-    Ok(unique_quota_roots)
-}
-
-fn get_highest_usage<'t>(
-    unique_quota_roots: &'t IndexMap<String, Vec<QuotaResource>>,
-) -> Result<(u64, &'t String, &QuotaResource)> {
-    let mut highest: Option<(u64, &'t String, &QuotaResource)> = None;
-    for (name, resources) in unique_quota_roots {
-        for r in resources {
-            let usage_percent = r.get_usage_percentage();
-            match highest {
-                None => {
-                    highest = Some((usage_percent, name, r));
-                }
-                Some((up, ..)) => {
-                    if up <= usage_percent {
-                        highest = Some((usage_percent, name, r));
-                    }
-                }
-            };
-        }
-    }
-
-    highest.ok_or_else(|| anyhow!("no quota_resource found, this is unexpected"))
-}
-
-impl Context {
-    // Adds a job to update `quota.recent`
-    pub(crate) async fn schedule_quota_update(&self) -> Result<()> {
-        job::kill_action(self, Action::UpdateRecentQuota).await?;
-        job::add(
-            self,
-            job::Job::new(Action::UpdateRecentQuota, 0, Params::new(), 0),
-        )
-        .await;
-        Ok(())
-    }
-
-    /// Updates `quota.recent`, sets `quota.modified` to the current time
-    /// and emits an event to let the UIs update connectivity view.
-    ///
-    /// Moreover, once each time quota gets larger than `QUOTA_WARN_THRESHOLD_PERCENTAGE`,
-    /// a device message is added.
-    /// As the message is added only once, the user is not spammed
-    /// in case for some providers the quota is always at ~100%
-    /// and new space is allocated as needed.
-    ///
-    /// Called in response to `Action::UpdateRecentQuota`.
-    pub(crate) async fn update_recent_quota(&self, imap: &mut Imap) -> Result<Status> {
-        if let Err(err) = imap.prepare(self).await {
-            warn!(self, "could not connect: {:?}", err);
-            return Ok(Status::RetryNow);
-        }
-
-        let quota = if imap.can_check_quota() {
-            let folders = get_watched_folders(self).await;
-            get_unique_quota_roots_and_usage(folders, imap).await
-        } else {
-            Err(anyhow!("Quota not supported by your provider."))
-        };
-
-        if let Ok(quota) = &quota {
-            match get_highest_usage(quota) {
-                Ok((highest, _, _)) => {
-                    if highest >= QUOTA_WARN_THRESHOLD_PERCENTAGE {
-                        if self.get_config_int(Config::QuotaExceeding).await? == 0 {
-                            self.set_config(Config::QuotaExceeding, Some(&highest.to_string()))
-                                .await?;
-
-                            let mut msg = Message::new(Viewtype::Text);
-                            msg.text = Some(stock_str::quota_exceeding(self, highest).await);
-                            add_device_msg_with_importance(self, None, Some(&mut msg), true)
-                                .await?;
-                        }
-                    } else if highest <= QUOTA_ALLCLEAR_PERCENTAGE {
-                        self.set_config(Config::QuotaExceeding, None).await?;
-                    }
-                }
-                Err(err) => warn!(self, "cannot get highest quota usage: {:?}", err),
-            }
-        }
-
-        *self.quota.write().await = Some(QuotaInfo {
-            recent: quota,
-            modified: time(),
-        });
-
-        self.emit_event(EventType::ConnectivityChanged);
-        Ok(Status::Finished(Ok(())))
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use crate::quota::{
-        QUOTA_ALLCLEAR_PERCENTAGE, QUOTA_ERROR_THRESHOLD_PERCENTAGE,
-        QUOTA_WARN_THRESHOLD_PERCENTAGE,
-    };
-
-    #[allow(clippy::assertions_on_constants)]
-    #[async_std::test]
-    async fn test_quota_thresholds() -> anyhow::Result<()> {
-        assert!(QUOTA_ALLCLEAR_PERCENTAGE > 50);
-        assert!(QUOTA_ALLCLEAR_PERCENTAGE < QUOTA_WARN_THRESHOLD_PERCENTAGE);
-        assert!(QUOTA_WARN_THRESHOLD_PERCENTAGE < QUOTA_ERROR_THRESHOLD_PERCENTAGE);
-        assert!(QUOTA_ERROR_THRESHOLD_PERCENTAGE < 100);
-        Ok(())
-    }
-}

src/scheduler.rs

@@ -13,10 +13,6 @@ use crate::job::{self, Thread};
 use crate::message::MsgId;
 use crate::smtp::Smtp;
 
-use self::connectivity::ConnectivityStore;
-
-pub(crate) mod connectivity;
-
 pub(crate) struct StopToken;
 
 /// Job and connection scheduler.
@@ -39,16 +35,7 @@ pub(crate) enum Scheduler {
 impl Context {
     /// Indicate that the network likely has come back.
     pub async fn maybe_network(&self) {
-        let lock = self.scheduler.read().await;
-        lock.maybe_network().await;
-        connectivity::idle_interrupted(lock).await;
-    }
-
-    /// Indicate that the network likely is lost.
-    pub async fn maybe_network_lost(&self) {
-        let lock = self.scheduler.read().await;
-        lock.maybe_network_lost().await;
-        connectivity::maybe_network_lost(self, lock).await;
+        self.scheduler.read().await.maybe_network().await;
     }
 
     pub(crate) async fn interrupt_inbox(&self, info: InterruptInfo) {
@@ -82,11 +69,7 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
         let mut jobs_loaded = 0;
         let mut info = InterruptInfo::default();
         loop {
-            match job::load_next(&ctx, Thread::Imap, &info)
-                .await
-                .ok()
-                .flatten()
-            {
+            match job::load_next(&ctx, Thread::Imap, &info).await {
                 Some(job) if jobs_loaded <= 20 => {
                     jobs_loaded += 1;
                     job::perform_job(&ctx, job::Connection::Inbox(&mut connection), job).await;
@@ -124,9 +107,6 @@ async fn inbox_loop(ctx: Context, started: Sender<()>, inbox_handlers: ImapConne
                     } else {
                         if let Err(err) = connection.scan_folders(&ctx).await {
                             warn!(ctx, "{}", err);
-                            connection.connectivity.set_err(&ctx, err).await;
-                        } else {
-                            connection.connectivity.set_not_configured(&ctx).await;
                         }
                         connection.fake_idle(&ctx, None).await
                     };
@@ -152,24 +132,26 @@ async fn fetch(ctx: &Context, connection: &mut Imap) {
     match ctx.get_config(Config::ConfiguredInboxFolder).await {
         Ok(Some(watch_folder)) => {
             if let Err(err) = connection.prepare(ctx).await {
-                warn!(ctx, "Could not connect: {}", err);
+                error_network!(ctx, "{}", err);
                 return;
             }
 
             // fetch
             if let Err(err) = connection.fetch(ctx, &watch_folder).await {
-                connection.trigger_reconnect(ctx).await;
+                connection.trigger_reconnect();
                 warn!(ctx, "{:#}", err);
             }
         }
         Ok(None) => {
-            info!(ctx, "Can not fetch inbox folder, not set");
+            warn!(ctx, "Can not fetch inbox folder, not set");
+            connection.fake_idle(ctx, None).await;
         }
         Err(err) => {
             warn!(
                 ctx,
                 "Can not fetch inbox folder, failed to get config: {:?}", err
             );
+            connection.fake_idle(ctx, None).await;
         }
     }
 }
@@ -185,9 +167,8 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
 
             // fetch
             if let Err(err) = connection.fetch(ctx, &watch_folder).await {
-                connection.trigger_reconnect(ctx).await;
+                connection.trigger_reconnect();
                 warn!(ctx, "{:#}", err);
-                return InterruptInfo::new(false, None);
             }
 
             if folder == Config::ConfiguredInboxFolder {
@@ -199,25 +180,22 @@ async fn fetch_idle(ctx: &Context, connection: &mut Imap, folder: Config) -> Int
                 }
             }
 
-            connection.connectivity.set_connected(ctx).await;
-
             // idle
             if connection.can_idle() {
-                match connection.idle(ctx, Some(watch_folder)).await {
-                    Ok(v) => v,
-                    Err(err) => {
-                        connection.trigger_reconnect(ctx).await;
+                connection
+                    .idle(ctx, Some(watch_folder))
+                    .await
+                    .unwrap_or_else(|err| {
+                        connection.trigger_reconnect();
                         warn!(ctx, "{}", err);
                         InterruptInfo::new(false, None)
-                    }
-                }
+                    })
             } else {
                 connection.fake_idle(ctx, Some(watch_folder)).await
             }
         }
         Ok(None) => {
-            connection.connectivity.set_not_configured(ctx).await;
-            info!(ctx, "Can not watch {} folder, not set", folder);
+            warn!(ctx, "Can not watch {} folder, not set", folder);
             connection.fake_idle(ctx, None).await
         }
         Err(err) => {
@@ -293,11 +271,7 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
 
         let mut interrupt_info = Default::default();
         loop {
-            match job::load_next(&ctx, Thread::Smtp, &interrupt_info)
-                .await
-                .ok()
-                .flatten()
-            {
+            match job::load_next(&ctx, Thread::Smtp, &interrupt_info).await {
                 Some(job) => {
                     info!(ctx, "executing smtp job");
                     job::perform_job(&ctx, job::Connection::Smtp(&mut connection), job).await;
@@ -306,11 +280,6 @@ async fn smtp_loop(ctx: Context, started: Sender<()>, smtp_handlers: SmtpConnect
                 None => {
                     // Fake Idle
                     info!(ctx, "smtp fake idle - started");
-                    match &connection.last_send_error {
-                        None => connection.connectivity.set_connected(&ctx).await,
-                        Some(err) => connection.connectivity.set_err(&ctx, err).await,
-                    }
-
                     interrupt_info = idle_interrupt_receiver.recv().await.unwrap_or_default();
                     info!(ctx, "smtp fake idle - interrupted")
                 }
@@ -369,11 +338,6 @@ impl Scheduler {
                 .send(())
                 .await
                 .expect("mvbox start send, missing receiver");
-            mvbox_handlers
-                .connection
-                .connectivity
-                .set_not_configured(&ctx)
-                .await
         }
 
         if ctx.get_config_bool(Config::SentboxWatch).await? {
@@ -392,11 +356,6 @@ impl Scheduler {
                 .send(())
                 .await
                 .expect("sentbox start send, missing receiver");
-            sentbox_handlers
-                .connection
-                .connectivity
-                .set_not_configured(&ctx)
-                .await
         }
 
         let smtp_handle = {
@@ -444,18 +403,6 @@ impl Scheduler {
             .await;
     }
 
-    async fn maybe_network_lost(&self) {
-        if !self.is_running() {
-            return;
-        }
-
-        self.interrupt_inbox(InterruptInfo::new(false, None))
-            .join(self.interrupt_mvbox(InterruptInfo::new(false, None)))
-            .join(self.interrupt_sentbox(InterruptInfo::new(false, None)))
-            .join(self.interrupt_smtp(InterruptInfo::new(false, None)))
-            .await;
-    }
-
     async fn interrupt_inbox(&self, info: InterruptInfo) {
         if let Scheduler::Running { ref inbox, .. } = self {
             inbox.interrupt(info).await;
@@ -561,8 +508,6 @@ struct ConnectionState {
     stop_sender: Sender<()>,
     /// Channel to interrupt idle.
     idle_interrupt_sender: Sender<InterruptInfo>,
-    /// Mutex to pass connectivity info between IMAP/SMTP threads and the API
-    connectivity: ConnectivityStore,
 }
 
 impl ConnectionState {
@@ -605,7 +550,6 @@ impl SmtpConnectionState {
             shutdown_receiver,
             stop_sender,
             idle_interrupt_sender,
-            connectivity: handlers.connection.connectivity.clone(),
         };
 
         let conn = SmtpConnectionState { state };
@@ -653,7 +597,6 @@ impl ImapConnectionState {
             shutdown_receiver,
             stop_sender,
             idle_interrupt_sender,
-            connectivity: handlers.connection.connectivity.clone(),
         };
 
         let conn = ImapConnectionState { state };

src/scheduler/connectivity.rs

@@ -1,543 +0,0 @@
-use core::fmt;
-use std::{ops::Deref, sync::Arc};
-
-use async_std::sync::{Mutex, RwLockReadGuard};
-
-use crate::dc_tools::time;
-use crate::events::EventType;
-use crate::quota::{
-    QUOTA_ERROR_THRESHOLD_PERCENTAGE, QUOTA_MAX_AGE_SECONDS, QUOTA_WARN_THRESHOLD_PERCENTAGE,
-};
-use crate::{config::Config, dc_tools, scheduler::Scheduler};
-use crate::{context::Context, log::LogExt};
-use anyhow::{anyhow, Result};
-use humansize::{file_size_opts, FileSize};
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, EnumProperty, PartialOrd, Ord)]
-pub enum Connectivity {
-    NotConnected = 1000,
-    Connecting = 2000,
-    /// Fetching or sending messages
-    Working = 3000,
-    Connected = 4000,
-}
-
-// The order of the connectivities is important: worse connectivities (i.e. those at
-// the top) take priority. This means that e.g. if any folder has an error - usually
-// because there is no internet connection - the connectivity for the whole
-// account will be `Notconnected`.
-#[derive(Debug, Clone, PartialEq, Eq, EnumProperty, PartialOrd)]
-enum DetailedConnectivity {
-    Error(String),
-    Uninitialized,
-    Connecting,
-    Working,
-    InterruptingIdle,
-    Connected,
-
-    /// The folder was configured not to be watched or configured_*_folder is not set
-    NotConfigured,
-}
-
-impl Default for DetailedConnectivity {
-    fn default() -> Self {
-        DetailedConnectivity::Uninitialized
-    }
-}
-
-impl DetailedConnectivity {
-    fn to_basic(&self) -> Option<Connectivity> {
-        match self {
-            DetailedConnectivity::Error(_) => Some(Connectivity::NotConnected),
-            DetailedConnectivity::Uninitialized => Some(Connectivity::NotConnected),
-            DetailedConnectivity::Connecting => Some(Connectivity::Connecting),
-            DetailedConnectivity::Working => Some(Connectivity::Working),
-            DetailedConnectivity::InterruptingIdle => Some(Connectivity::Connected),
-            DetailedConnectivity::Connected => Some(Connectivity::Connected),
-
-            // Just don't return a connectivity, probably the folder is configured not to be
-            // watched or there is e.g. no "Sent" folder, so we are not interested in it
-            DetailedConnectivity::NotConfigured => None,
-        }
-    }
-
-    fn to_icon(&self) -> String {
-        match self {
-            DetailedConnectivity::Error(_)
-            | DetailedConnectivity::Uninitialized
-            | DetailedConnectivity::NotConfigured => "<span class=\"red dot\"></span>".to_string(),
-            DetailedConnectivity::Connecting => "<span class=\"yellow dot\"></span>".to_string(),
-            DetailedConnectivity::Working
-            | DetailedConnectivity::InterruptingIdle
-            | DetailedConnectivity::Connected => "<span class=\"green dot\"></span>".to_string(),
-        }
-    }
-
-    fn to_string_imap(&self, _context: &Context) -> String {
-        match self {
-            DetailedConnectivity::Error(e) => format!("Error: {}", e),
-            DetailedConnectivity::Uninitialized => "Not started".to_string(),
-            DetailedConnectivity::Connecting => "Connecting…".to_string(),
-            DetailedConnectivity::Working => "Getting new messages…".to_string(),
-            DetailedConnectivity::InterruptingIdle | DetailedConnectivity::Connected => {
-                "Connected".to_string()
-            }
-            DetailedConnectivity::NotConfigured => "Not configured".to_string(),
-        }
-    }
-
-    fn to_string_smtp(&self, _context: &Context) -> String {
-        match self {
-            DetailedConnectivity::Error(e) => format!("Error: {}", e),
-            DetailedConnectivity::Uninitialized => {
-                "(You did not try to send a message recently)".to_string()
-            }
-            DetailedConnectivity::Connecting => "Connecting…".to_string(),
-            DetailedConnectivity::Working => "Sending…".to_string(),
-
-            // We don't know any more than that the last message was sent successfully;
-            // since sending the last message, connectivity could have changed, which we don't notice
-            // until another message is sent
-            DetailedConnectivity::InterruptingIdle | DetailedConnectivity::Connected => {
-                "Your last message was sent successfully".to_string()
-            }
-            DetailedConnectivity::NotConfigured => "Not configured".to_string(),
-        }
-    }
-
-    fn all_work_done(&self) -> bool {
-        match self {
-            DetailedConnectivity::Error(_) => true,
-            DetailedConnectivity::Uninitialized => false,
-            DetailedConnectivity::Connecting => false,
-            DetailedConnectivity::Working => false,
-            DetailedConnectivity::InterruptingIdle => false,
-            DetailedConnectivity::Connected => true,
-            DetailedConnectivity::NotConfigured => true,
-        }
-    }
-}
-
-#[derive(Clone, Default)]
-pub(crate) struct ConnectivityStore(Arc<Mutex<DetailedConnectivity>>);
-
-impl ConnectivityStore {
-    async fn set(&self, context: &Context, v: DetailedConnectivity) {
-        {
-            *self.0.lock().await = v;
-        }
-        context.emit_event(EventType::ConnectivityChanged);
-    }
-
-    pub(crate) async fn set_err(&self, context: &Context, e: impl ToString) {
-        self.set(context, DetailedConnectivity::Error(e.to_string()))
-            .await;
-    }
-    pub(crate) async fn set_connecting(&self, context: &Context) {
-        self.set(context, DetailedConnectivity::Connecting).await;
-    }
-    pub(crate) async fn set_working(&self, context: &Context) {
-        self.set(context, DetailedConnectivity::Working).await;
-    }
-    pub(crate) async fn set_connected(&self, context: &Context) {
-        self.set(context, DetailedConnectivity::Connected).await;
-    }
-    pub(crate) async fn set_not_configured(&self, context: &Context) {
-        self.set(context, DetailedConnectivity::NotConfigured).await;
-    }
-
-    async fn get_detailed(&self) -> DetailedConnectivity {
-        self.0.lock().await.deref().clone()
-    }
-    async fn get_basic(&self) -> Option<Connectivity> {
-        self.0.lock().await.to_basic()
-    }
-    async fn get_all_work_done(&self) -> bool {
-        self.0.lock().await.all_work_done()
-    }
-}
-
-/// Set all folder states to InterruptingIdle in case they were `Connected` before.
-/// Called during `dc_maybe_network()` to make sure that `dc_accounts_all_work_done()`
-/// returns false immediately after `dc_maybe_network()`.
-pub(crate) async fn idle_interrupted(scheduler: RwLockReadGuard<'_, Scheduler>) {
-    let [inbox, mvbox, sentbox] = match &*scheduler {
-        Scheduler::Running {
-            inbox,
-            mvbox,
-            sentbox,
-            ..
-        } => [
-            inbox.state.connectivity.clone(),
-            mvbox.state.connectivity.clone(),
-            sentbox.state.connectivity.clone(),
-        ],
-        Scheduler::Stopped => return,
-    };
-    drop(scheduler);
-
-    let mut connectivity_lock = inbox.0.lock().await;
-    // For the inbox, we also have to set the connectivity to InterruptingIdle if it was
-    // NotConfigured before: If all folders are NotConfigured, dc_get_connectivity()
-    // returns Connected. But after dc_maybe_network(), dc_get_connectivity() must not
-    // return Connected until DC is completely done with fetching folders; this also
-    // includes scan_folders() which happens on the inbox thread.
-    if *connectivity_lock == DetailedConnectivity::Connected
-        || *connectivity_lock == DetailedConnectivity::NotConfigured
-    {
-        *connectivity_lock = DetailedConnectivity::InterruptingIdle;
-    }
-    drop(connectivity_lock);
-
-    for state in &[&mvbox, &sentbox] {
-        let mut connectivity_lock = state.0.lock().await;
-        if *connectivity_lock == DetailedConnectivity::Connected {
-            *connectivity_lock = DetailedConnectivity::InterruptingIdle;
-        }
-    }
-    // No need to send ConnectivityChanged, the user-facing connectivity doesn't change because
-    // of what we do here.
-}
-
-/// Set the connectivity to "Not connected" after a call to dc_maybe_network_lost().
-/// If we did not do this, the connectivity would stay "Connected" for quite a long time
-/// after `maybe_network_lost()` was called.
-pub(crate) async fn maybe_network_lost(
-    context: &Context,
-    scheduler: RwLockReadGuard<'_, Scheduler>,
-) {
-    let stores = match &*scheduler {
-        Scheduler::Running {
-            inbox,
-            mvbox,
-            sentbox,
-            ..
-        } => [
-            inbox.state.connectivity.clone(),
-            mvbox.state.connectivity.clone(),
-            sentbox.state.connectivity.clone(),
-        ],
-        Scheduler::Stopped => return,
-    };
-    drop(scheduler);
-
-    for store in &stores {
-        let mut connectivity_lock = store.0.lock().await;
-        if !matches!(
-            *connectivity_lock,
-            DetailedConnectivity::Uninitialized
-                | DetailedConnectivity::Error(_)
-                | DetailedConnectivity::NotConfigured,
-        ) {
-            *connectivity_lock = DetailedConnectivity::Error("Connection lost".to_string());
-        }
-        drop(connectivity_lock);
-    }
-    context.emit_event(EventType::ConnectivityChanged);
-}
-
-impl fmt::Debug for ConnectivityStore {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        if let Some(guard) = self.0.try_lock() {
-            write!(f, "ConnectivityStore {:?}", &*guard)
-        } else {
-            write!(f, "ConnectivityStore [LOCKED]")
-        }
-    }
-}
-
-impl Context {
-    /// Get the current connectivity, i.e. whether the device is connected to the IMAP server.
-    /// One of:
-    /// - DC_CONNECTIVITY_NOT_CONNECTED (1000-1999): Show e.g. the string "Not connected" or a red dot
-    /// - DC_CONNECTIVITY_CONNECTING (2000-2999): Show e.g. the string "Connecting…" or a yellow dot
-    /// - DC_CONNECTIVITY_WORKING (3000-3999): Show e.g. the string "Getting new messages" or a spinning wheel
-    /// - DC_CONNECTIVITY_CONNECTED (>=4000): Show e.g. the string "Connected" or a green dot
-    ///
-    /// We don't use exact values but ranges here so that we can split up
-    /// states into multiple states in the future.
-    ///
-    /// Meant as a rough overview that can be shown
-    /// e.g. in the title of the main screen.
-    ///
-    /// If the connectivity changes, a DC_EVENT_CONNECTIVITY_CHANGED will be emitted.
-    pub async fn get_connectivity(&self) -> Connectivity {
-        let lock = self.scheduler.read().await;
-        let stores: Vec<_> = match &*lock {
-            Scheduler::Running {
-                inbox,
-                mvbox,
-                sentbox,
-                ..
-            } => [&inbox.state, &mvbox.state, &sentbox.state]
-                .iter()
-                .map(|state| state.connectivity.clone())
-                .collect(),
-            Scheduler::Stopped => return Connectivity::NotConnected,
-        };
-        drop(lock);
-
-        let mut connectivities = Vec::new();
-        for s in stores {
-            if let Some(connectivity) = s.get_basic().await {
-                connectivities.push(connectivity);
-            }
-        }
-        connectivities
-            .into_iter()
-            .min()
-            .unwrap_or(Connectivity::Connected)
-    }
-
-    /// Get an overview of the current connectivity, and possibly more statistics.
-    /// Meant to give the user more insight about the current status than
-    /// the basic connectivity info returned by dc_get_connectivity(); show this
-    /// e.g., if the user taps on said basic connectivity info.
-    ///
-    /// If this page changes, a DC_EVENT_CONNECTIVITY_CHANGED will be emitted.
-    ///
-    /// This comes as an HTML from the core so that we can easily improve it
-    /// and the improvement instantly reaches all UIs.
-    pub async fn get_connectivity_html(&self) -> Result<String> {
-        let mut ret = r#"<!DOCTYPE html>
-            <html>
-            <head>
-                <meta charset="UTF-8" />
-                <meta name="viewport" content="initial-scale=1.0" />
-                <style>
-                    ul {
-                        list-style-type: none;
-                        padding-left: 1em;
-                    }
-                    .dot {
-                        height: 0.9em; width: 0.9em;
-                        border: 1px solid #888;
-                        border-radius: 50%;
-                        display: inline-block;
-                        position: relative; left: -0.1em; top: 0.1em;
-                    }
-                    .bar {
-                        width: 90%;
-                        border: 1px solid #888;
-                        border-radius: .5em;
-                        margin-top: .2em;
-                        margin-bottom: 1em;
-                        position: relative; left: -0.2em;
-                    }
-                    .progress {
-                        min-width:1.8em;
-                        height: 1em;
-                        border-radius: .45em;
-                        color: white;
-                        text-align: center;
-                        padding-bottom: 2px;
-                    }
-                    .red {
-                        background-color: #f33b2d;
-                    }
-                    .green {
-                        background-color: #34c759;
-                    }
-                    .yellow {
-                        background-color: #fdc625;
-                    }
-                </style>
-            </head>
-            <body>"#
-            .to_string();
-
-        let lock = self.scheduler.read().await;
-        let (folders_states, smtp) = match &*lock {
-            Scheduler::Running {
-                inbox,
-                mvbox,
-                sentbox,
-                smtp,
-                ..
-            } => (
-                [
-                    (
-                        Config::ConfiguredInboxFolder,
-                        Config::InboxWatch,
-                        inbox.state.connectivity.clone(),
-                    ),
-                    (
-                        Config::ConfiguredMvboxFolder,
-                        Config::MvboxWatch,
-                        mvbox.state.connectivity.clone(),
-                    ),
-                    (
-                        Config::ConfiguredSentboxFolder,
-                        Config::SentboxWatch,
-                        sentbox.state.connectivity.clone(),
-                    ),
-                ],
-                smtp.state.connectivity.clone(),
-            ),
-            Scheduler::Stopped => {
-                return Err(anyhow!("Not started"));
-            }
-        };
-        drop(lock);
-
-        ret += "<h3>Incoming messages</h3><ul>";
-        for (folder, watch, state) in &folders_states {
-            let w = self.get_config(*watch).await.ok_or_log(self);
-
-            let mut folder_added = false;
-            if w.flatten() == Some("1".to_string()) {
-                let f = self.get_config(*folder).await.ok_or_log(self).flatten();
-
-                if let Some(foldername) = f {
-                    let detailed = &state.get_detailed().await;
-                    ret += "<li>";
-                    ret += &*detailed.to_icon();
-                    ret += " <b>";
-                    ret += &*escaper::encode_minimal(&foldername);
-                    ret += ":</b> ";
-                    ret += &*escaper::encode_minimal(&*detailed.to_string_imap(self));
-                    ret += "</li>";
-
-                    folder_added = true;
-                }
-            }
-
-            if !folder_added && folder == &Config::ConfiguredInboxFolder {
-                let detailed = &state.get_detailed().await;
-                if let DetailedConnectivity::Error(_) = detailed {
-                    // On the inbox thread, we also do some other things like scan_folders and run jobs
-                    // so, maybe, the inbox is not watched, but something else went wrong
-                    ret += "<li>";
-                    ret += &*detailed.to_icon();
-                    ret += " ";
-                    ret += &*escaper::encode_minimal(&detailed.to_string_imap(self));
-                    ret += "</li>";
-                }
-            }
-        }
-        ret += "</ul>";
-
-        ret += "<h3>Outgoing messages</h3><ul><li>";
-        let detailed = smtp.get_detailed().await;
-        ret += &*detailed.to_icon();
-        ret += " ";
-        ret += &*escaper::encode_minimal(&detailed.to_string_smtp(self));
-        ret += "</li></ul>";
-
-        let domain = dc_tools::EmailAddress::new(
-            &self
-                .get_config(Config::ConfiguredAddr)
-                .await?
-                .unwrap_or_default(),
-        )?
-        .domain;
-        ret += &format!("<h3>Storage on {}</h3><ul>", domain);
-        let quota = self.quota.read().await;
-        if let Some(quota) = &*quota {
-            match &quota.recent {
-                Ok(quota) => {
-                    let roots_cnt = quota.len();
-                    for (root_name, resources) in quota {
-                        use async_imap::types::QuotaResourceName::*;
-                        for resource in resources {
-                            ret += "<li>";
-
-                            // root name is empty eg. for gmail and redundant eg. for riseup.
-                            // therefore, use it only if there are really several roots.
-                            if roots_cnt > 1 && !root_name.is_empty() {
-                                ret +=
-                                    &format!("<b>{}:</b> ", &*escaper::encode_minimal(root_name));
-                            } else {
-                                info!(self, "connectivity: root name hidden: \"{}\"", root_name);
-                            }
-
-                            ret += &match &resource.name {
-                                Atom(resource_name) => {
-                                    format!(
-                                        "<b>{}:</b> {} of {} used",
-                                        &*escaper::encode_minimal(resource_name),
-                                        resource.usage.to_string(),
-                                        resource.limit.to_string(),
-                                    )
-                                }
-                                Message => {
-                                    format!(
-                                        "<b>Messages:</b> {} of {} used",
-                                        resource.usage.to_string(),
-                                        resource.limit.to_string(),
-                                    )
-                                }
-                                Storage => {
-                                    // do not use a special title needed for "Storage":
-                                    // - it is usually shown directly under the "Storage" headline
-                                    // - by the units "1 MB of 10 MB used" there is some difference to eg. "Messages: 1 of 10 used"
-                                    // - the string is not longer than the other strings that way (minus title, plus units) -
-                                    //   additional linebreaks on small displays are unlikely therefore
-                                    // - most times, this is the only item anyway
-                                    let usage = (resource.usage * 1024)
-                                        .file_size(file_size_opts::BINARY)
-                                        .unwrap_or_default();
-                                    let limit = (resource.limit * 1024)
-                                        .file_size(file_size_opts::BINARY)
-                                        .unwrap_or_default();
-                                    format!("{} of {} used", usage, limit)
-                                }
-                            };
-
-                            let percent = resource.get_usage_percentage();
-                            let color = if percent >= QUOTA_ERROR_THRESHOLD_PERCENTAGE {
-                                "red"
-                            } else if percent >= QUOTA_WARN_THRESHOLD_PERCENTAGE {
-                                "yellow"
-                            } else {
-                                "green"
-                            };
-                            ret += &format!("<div class=\"bar\"><div class=\"progress {}\" style=\"width: {}%\">{}%</div></div>", color, percent, percent);
-
-                            ret += "</li>";
-                        }
-                    }
-                }
-                Err(e) => {
-                    ret += format!("<li>{}</li>", e).as_str();
-                }
-            }
-
-            if quota.modified + QUOTA_MAX_AGE_SECONDS < time() {
-                self.schedule_quota_update().await?;
-            }
-        } else {
-            ret += "<li>One moment...</li>";
-            self.schedule_quota_update().await?;
-        }
-        ret += "</ul>";
-
-        ret += "</body></html>\n";
-        Ok(ret)
-    }
-
-    pub async fn all_work_done(&self) -> bool {
-        let lock = self.scheduler.read().await;
-        let stores: Vec<_> = match &*lock {
-            Scheduler::Running {
-                inbox,
-                mvbox,
-                sentbox,
-                smtp,
-                ..
-            } => [&inbox.state, &mvbox.state, &sentbox.state, &smtp.state]
-                .iter()
-                .map(|state| state.connectivity.clone())
-                .collect(),
-            Scheduler::Stopped => return false,
-        };
-        drop(lock);
-
-        for s in &stores {
-            if !s.get_all_work_done().await {
-                return false;
-            }
-        }
-        true
-    }
-}

src/securejoin.rs

@@ -1,9 +1,9 @@
-//! Verified contact protocol implementation as [specified by countermitm project](https://countermitm.readthedocs.io/en/stable/new.html#setup-contact-protocol).
+//! Verified contact protocol implementation as [specified by countermitm project](https://countermitm.readthedocs.io/en/stable/new.html#setup-contact-protocol)
 
 use std::convert::TryFrom;
 use std::time::{Duration, Instant};
 
-use anyhow::{anyhow, bail, Context as _, Error, Result};
+use anyhow::{bail, Context as _, Error, Result};
 use async_std::channel::Receiver;
 use async_std::sync::Mutex;
 use percent_encoding::{utf8_percent_encode, AsciiSet, NON_ALPHANUMERIC};
@@ -14,11 +14,10 @@ use crate::config::Config;
 use crate::constants::{Blocked, Viewtype, DC_CONTACT_ID_LAST_SPECIAL};
 use crate::contact::{Contact, Origin, VerifiedStatus};
 use crate::context::Context;
-use crate::dc_tools::time;
 use crate::e2ee::ensure_secret_key_exists;
 use crate::events::EventType;
 use crate::headerdef::HeaderDef;
-use crate::key::{DcKey, Fingerprint, SignedPublicKey};
+use crate::key::{self, DcKey, Fingerprint, SignedPublicKey};
 use crate::message::Message;
 use crate::mimeparser::{MimeMessage, SystemMessage};
 use crate::param::Param;
@@ -328,14 +327,14 @@ async fn securejoin(context: &Context, qr: &str) -> Result<ChatId, JoinError> {
             let start = Instant::now();
             let chatid = loop {
                 {
-                    match chat::get_chat_id_by_grpid(context, &group_id).await? {
-                        Some((chatid, _is_protected, _blocked)) => break chatid,
-                        None => {
+                    match chat::get_chat_id_by_grpid(context, &group_id).await {
+                        Ok((chatid, _is_protected, _blocked)) => break chatid,
+                        Err(err) => {
                             if start.elapsed() > Duration::from_secs(7) {
                                 context.free_ongoing().await;
-                                return Err(JoinError::Other(anyhow!(
-                                    "Ongoing sender dropped (this is a bug)"
-                                )));
+                                return Err(err
+                                    .context("Ongoing sender dropped (this is a bug)")
+                                    .into());
                             }
                         }
                     }
@@ -356,6 +355,12 @@ async fn securejoin(context: &Context, qr: &str) -> Result<ChatId, JoinError> {
 #[error("Failed sending handshake message")]
 pub struct SendMsgError(#[from] anyhow::Error);
 
+impl From<key::Error> for SendMsgError {
+    fn from(source: key::Error) -> Self {
+        Self(anyhow::Error::new(source))
+    }
+}
+
 async fn send_handshake_msg(
     context: &Context,
     contact_chat_id: ChatId,
@@ -484,7 +489,7 @@ pub(crate) async fn handle_securejoin_handshake(
         return Err(Error::msg("Can not be called with special contact ID"));
     }
     let step = mime_message
-        .get_header(HeaderDef::SecureJoin)
+        .get(HeaderDef::SecureJoin)
         .context("Not a Secure-Join message")?;
 
     info!(
@@ -502,7 +507,7 @@ pub(crate) async fn handle_securejoin_handshake(
                 )
             })?;
         if chat.blocked != Blocked::Not {
-            chat.id.unblock(context).await?;
+            chat.id.unblock(context).await;
         }
         chat.id
     };
@@ -520,7 +525,7 @@ pub(crate) async fn handle_securejoin_handshake(
             // it just ensures, we have Bobs key now. If we do _not_ have the key because eg. MitM has removed it,
             // send_message() will fail with the error "End-to-end-encryption unavailable unexpectedly.", so, there is no additional check needed here.
             // verify that the `Secure-Join-Invitenumber:`-header matches invitenumber written to the QR code
-            let invitenumber = match mime_message.get_header(HeaderDef::SecureJoinInvitenumber) {
+            let invitenumber = match mime_message.get(HeaderDef::SecureJoinInvitenumber) {
                 Some(n) => n,
                 None => {
                     warn!(context, "Secure-join denied (invitenumber missing)");
@@ -576,19 +581,19 @@ pub(crate) async fn handle_securejoin_handshake(
             ==========================================================*/
 
             // verify that Secure-Join-Fingerprint:-header matches the fingerprint of Bob
-            let fingerprint: Fingerprint =
-                match mime_message.get_header(HeaderDef::SecureJoinFingerprint) {
-                    Some(fp) => fp.parse()?,
-                    None => {
-                        could_not_establish_secure_connection(
-                            context,
-                            contact_chat_id,
-                            "Fingerprint not provided.",
-                        )
-                        .await?;
-                        return Ok(HandshakeMessage::Ignore);
-                    }
-                };
+            let fingerprint: Fingerprint = match mime_message.get(HeaderDef::SecureJoinFingerprint)
+            {
+                Some(fp) => fp.parse()?,
+                None => {
+                    could_not_establish_secure_connection(
+                        context,
+                        contact_chat_id,
+                        "Fingerprint not provided.",
+                    )
+                    .await?;
+                    return Ok(HandshakeMessage::Ignore);
+                }
+            };
             if !encrypted_and_signed(context, mime_message, Some(&fingerprint)) {
                 could_not_establish_secure_connection(
                     context,
@@ -609,7 +614,7 @@ pub(crate) async fn handle_securejoin_handshake(
             }
             info!(context, "Fingerprint verified.",);
             // verify that the `Secure-Join-Auth:`-header matches the secret written to the QR code
-            let auth_0 = match mime_message.get_header(HeaderDef::SecureJoinAuth) {
+            let auth_0 = match mime_message.get(HeaderDef::SecureJoinAuth) {
                 Some(auth) => auth,
                 None => {
                     could_not_establish_secure_connection(
@@ -635,24 +640,24 @@ pub(crate) async fn handle_securejoin_handshake(
                 .await?;
                 return Ok(HandshakeMessage::Ignore);
             }
-            Contact::scaleup_origin_by_id(context, contact_id, Origin::SecurejoinInvited).await?;
+            Contact::scaleup_origin_by_id(context, contact_id, Origin::SecurejoinInvited).await;
             info!(context, "Auth verified.",);
             secure_connection_established(context, contact_chat_id).await?;
-            context.emit_event(EventType::ContactsChanged(Some(contact_id)));
+            emit_event!(context, EventType::ContactsChanged(Some(contact_id)));
             inviter_progress!(context, contact_id, 600);
             if join_vg {
                 // the vg-member-added message is special:
                 // this is a normal Chat-Group-Member-Added message
                 // with an additional Secure-Join header
-                let field_grpid = match mime_message.get_header(HeaderDef::SecureJoinGroup) {
+                let field_grpid = match mime_message.get(HeaderDef::SecureJoinGroup) {
                     Some(s) => s.as_str(),
                     None => {
                         warn!(context, "Missing Secure-Join-Group header");
                         return Ok(HandshakeMessage::Ignore);
                     }
                 };
-                match chat::get_chat_id_by_grpid(context, field_grpid).await? {
-                    Some((group_chat_id, _, _)) => {
+                match chat::get_chat_id_by_grpid(context, field_grpid).await {
+                    Ok((group_chat_id, _, _)) => {
                         if let Err(err) =
                             chat::add_contact_to_chat_ex(context, group_chat_id, contact_id, true)
                                 .await
@@ -660,7 +665,12 @@ pub(crate) async fn handle_securejoin_handshake(
                             error!(context, "failed to add contact: {}", err);
                         }
                     }
-                    None => bail!("Chat {} not found", &field_grpid),
+                    Err(err) => {
+                        error!(context, "Chat {} not found: {}", &field_grpid, err);
+                        return Err(
+                            err.context(format!("Chat for group {} not found", &field_grpid))
+                        );
+                    }
                 }
             } else {
                 // Alice -> Bob
@@ -729,7 +739,7 @@ pub(crate) async fn handle_securejoin_handshake(
                     inviter_progress!(context, contact_id, 800);
                     inviter_progress!(context, contact_id, 1000);
                     let field_grpid = mime_message
-                        .get_header(HeaderDef::SecureJoinGroup)
+                        .get(HeaderDef::SecureJoinGroup)
                         .map(|s| s.as_str())
                         .unwrap_or_else(|| "");
                     if let Err(err) = chat::get_chat_id_by_grpid(context, &field_grpid).await {
@@ -778,7 +788,7 @@ pub(crate) async fn observe_securejoin_on_other_device(
         return Err(Error::msg("Can not be called with special contact ID"));
     }
     let step = mime_message
-        .get_header(HeaderDef::SecureJoin)
+        .get(HeaderDef::SecureJoin)
         .context("Not a Secure-Join message")?;
     info!(context, "observing secure-join message \'{}\'", step);
 
@@ -792,7 +802,7 @@ pub(crate) async fn observe_securejoin_on_other_device(
                 )
             })?;
         if chat.blocked != Blocked::Not {
-            chat.id.unblock(context).await?;
+            chat.id.unblock(context).await;
         }
         chat.id
     };
@@ -815,19 +825,19 @@ pub(crate) async fn observe_securejoin_on_other_device(
                 .await?;
                 return Ok(HandshakeMessage::Ignore);
             }
-            let fingerprint: Fingerprint =
-                match mime_message.get_header(HeaderDef::SecureJoinFingerprint) {
-                    Some(fp) => fp.parse()?,
-                    None => {
-                        could_not_establish_secure_connection(
+            let fingerprint: Fingerprint = match mime_message.get(HeaderDef::SecureJoinFingerprint)
+            {
+                Some(fp) => fp.parse()?,
+                None => {
+                    could_not_establish_secure_connection(
                         context,
                         contact_chat_id,
                         "Fingerprint not provided, please update Delta Chat on all your devices.",
                     )
                     .await?;
-                        return Ok(HandshakeMessage::Ignore);
-                    }
-                };
+                    return Ok(HandshakeMessage::Ignore);
+                }
+            };
             if mark_peer_as_verified(context, &fingerprint).await.is_err() {
                 could_not_establish_secure_connection(
                     context,
@@ -860,8 +870,8 @@ async fn secure_connection_established(
         "?"
     };
     let msg = stock_str::contact_verified(context, addr).await;
-    chat::add_info_msg(context, contact_chat_id, msg, time()).await;
-    context.emit_event(EventType::ChatModified(contact_chat_id));
+    chat::add_info_msg(context, contact_chat_id, msg).await;
+    emit_event!(context, EventType::ChatModified(contact_chat_id));
     info!(context, "StockMessage::ContactVerified posted to 1:1 chat");
 
     Ok(())
@@ -884,7 +894,7 @@ async fn could_not_establish_secure_connection(
     )
     .await;
 
-    chat::add_info_msg(context, contact_chat_id, &msg, time()).await;
+    chat::add_info_msg(context, contact_chat_id, &msg).await;
     error!(
         context,
         "StockMessage::ContactNotVerified posted to 1:1 chat ({})", details
@@ -985,8 +995,8 @@ mod tests {
         assert_eq!(sent.recipient(), "alice@example.com".parse().unwrap());
         let msg = alice.parse_msg(&sent).await;
         assert!(!msg.was_encrypted());
-        assert_eq!(msg.get_header(HeaderDef::SecureJoin).unwrap(), "vc-request");
-        assert!(msg.get_header(HeaderDef::SecureJoinInvitenumber).is_some());
+        assert_eq!(msg.get(HeaderDef::SecureJoin).unwrap(), "vc-request");
+        assert!(msg.get(HeaderDef::SecureJoinInvitenumber).is_some());
 
         // Step 3: Alice receives vc-request, sends vc-auth-required
         alice.recv_msg(&sent).await;
@@ -994,10 +1004,7 @@ mod tests {
         let sent = alice.pop_sent_msg().await;
         let msg = bob.parse_msg(&sent).await;
         assert!(msg.was_encrypted());
-        assert_eq!(
-            msg.get_header(HeaderDef::SecureJoin).unwrap(),
-            "vc-auth-required"
-        );
+        assert_eq!(msg.get(HeaderDef::SecureJoin).unwrap(), "vc-auth-required");
 
         // Step 4: Bob receives vc-auth-required, sends vc-request-with-auth
         bob.recv_msg(&sent).await;
@@ -1032,16 +1039,16 @@ mod tests {
         let msg = alice.parse_msg(&sent).await;
         assert!(msg.was_encrypted());
         assert_eq!(
-            msg.get_header(HeaderDef::SecureJoin).unwrap(),
+            msg.get(HeaderDef::SecureJoin).unwrap(),
             "vc-request-with-auth"
         );
-        assert!(msg.get_header(HeaderDef::SecureJoinAuth).is_some());
+        assert!(msg.get(HeaderDef::SecureJoinAuth).is_some());
         let bob_fp = SignedPublicKey::load_self(&bob.ctx)
             .await
             .unwrap()
             .fingerprint();
         assert_eq!(
-            *msg.get_header(HeaderDef::SecureJoinFingerprint).unwrap(),
+            *msg.get(HeaderDef::SecureJoinFingerprint).unwrap(),
             bob_fp.hex()
         );
 
@@ -1090,7 +1097,7 @@ mod tests {
         let msg = bob.parse_msg(&sent).await;
         assert!(msg.was_encrypted());
         assert_eq!(
-            msg.get_header(HeaderDef::SecureJoin).unwrap(),
+            msg.get(HeaderDef::SecureJoin).unwrap(),
             "vc-contact-confirm"
         );
 
@@ -1139,7 +1146,7 @@ mod tests {
         let msg = alice.parse_msg(&sent).await;
         assert!(msg.was_encrypted());
         assert_eq!(
-            msg.get_header(HeaderDef::SecureJoin).unwrap(),
+            msg.get(HeaderDef::SecureJoin).unwrap(),
             "vc-contact-confirm-received"
         );
     }
@@ -1224,16 +1231,16 @@ mod tests {
         let msg = alice.parse_msg(&sent).await;
         assert!(msg.was_encrypted());
         assert_eq!(
-            msg.get_header(HeaderDef::SecureJoin).unwrap(),
+            msg.get(HeaderDef::SecureJoin).unwrap(),
             "vc-request-with-auth"
         );
-        assert!(msg.get_header(HeaderDef::SecureJoinAuth).is_some());
+        assert!(msg.get(HeaderDef::SecureJoinAuth).is_some());
         let bob_fp = SignedPublicKey::load_self(&bob.ctx)
             .await
             .unwrap()
             .fingerprint();
         assert_eq!(
-            *msg.get_header(HeaderDef::SecureJoinFingerprint).unwrap(),
+            *msg.get(HeaderDef::SecureJoinFingerprint).unwrap(),
             bob_fp.hex()
         );
 
@@ -1265,7 +1272,7 @@ mod tests {
         let msg = bob.parse_msg(&sent).await;
         assert!(msg.was_encrypted());
         assert_eq!(
-            msg.get_header(HeaderDef::SecureJoin).unwrap(),
+            msg.get(HeaderDef::SecureJoin).unwrap(),
             "vc-contact-confirm"
         );
 
@@ -1294,7 +1301,7 @@ mod tests {
         let msg = alice.parse_msg(&sent).await;
         assert!(msg.was_encrypted());
         assert_eq!(
-            msg.get_header(HeaderDef::SecureJoin).unwrap(),
+            msg.get(HeaderDef::SecureJoin).unwrap(),
             "vc-contact-confirm-received"
         );
     }
@@ -1337,8 +1344,8 @@ mod tests {
         assert_eq!(sent.recipient(), "alice@example.com".parse().unwrap());
         let msg = alice.parse_msg(&sent).await;
         assert!(!msg.was_encrypted());
-        assert_eq!(msg.get_header(HeaderDef::SecureJoin).unwrap(), "vg-request");
-        assert!(msg.get_header(HeaderDef::SecureJoinInvitenumber).is_some());
+        assert_eq!(msg.get(HeaderDef::SecureJoin).unwrap(), "vg-request");
+        assert!(msg.get(HeaderDef::SecureJoinInvitenumber).is_some());
 
         // Step 3: Alice receives vg-request, sends vg-auth-required
         alice.recv_msg(&sent).await;
@@ -1346,10 +1353,7 @@ mod tests {
         let sent = alice.pop_sent_msg().await;
         let msg = bob.parse_msg(&sent).await;
         assert!(msg.was_encrypted());
-        assert_eq!(
-            msg.get_header(HeaderDef::SecureJoin).unwrap(),
-            "vg-auth-required"
-        );
+        assert_eq!(msg.get(HeaderDef::SecureJoin).unwrap(), "vg-auth-required");
 
         // Step 4: Bob receives vg-auth-required, sends vg-request-with-auth
         bob.recv_msg(&sent).await;
@@ -1384,16 +1388,16 @@ mod tests {
         let msg = alice.parse_msg(&sent).await;
         assert!(msg.was_encrypted());
         assert_eq!(
-            msg.get_header(HeaderDef::SecureJoin).unwrap(),
+            msg.get(HeaderDef::SecureJoin).unwrap(),
             "vg-request-with-auth"
         );
-        assert!(msg.get_header(HeaderDef::SecureJoinAuth).is_some());
+        assert!(msg.get(HeaderDef::SecureJoinAuth).is_some());
         let bob_fp = SignedPublicKey::load_self(&bob.ctx)
             .await
             .unwrap()
             .fingerprint();
         assert_eq!(
-            *msg.get_header(HeaderDef::SecureJoinFingerprint).unwrap(),
+            *msg.get(HeaderDef::SecureJoinFingerprint).unwrap(),
             bob_fp.hex()
         );
 
@@ -1421,10 +1425,7 @@ mod tests {
         let sent = alice.pop_sent_msg().await;
         let msg = bob.parse_msg(&sent).await;
         assert!(msg.was_encrypted());
-        assert_eq!(
-            msg.get_header(HeaderDef::SecureJoin).unwrap(),
-            "vg-member-added"
-        );
+        assert_eq!(msg.get(HeaderDef::SecureJoin).unwrap(), "vg-member-added");
 
         // Bob should not yet have Alice verified
         let contact_alice_id =
@@ -1451,7 +1452,7 @@ mod tests {
         let msg = alice.parse_msg(&sent).await;
         assert!(msg.was_encrypted());
         assert_eq!(
-            msg.get_header(HeaderDef::SecureJoin).unwrap(),
+            msg.get(HeaderDef::SecureJoin).unwrap(),
             "vg-member-added-received"
         );
 

src/securejoin/bobstate.rs

@@ -12,7 +12,7 @@ use anyhow::{Error, Result};
 use async_std::sync::MutexGuard;
 
 use crate::chat::{self, ChatId};
-use crate::constants::Viewtype;
+use crate::constants::{Blocked, Viewtype};
 use crate::contact::{Contact, Origin};
 use crate::context::Context;
 use crate::events::EventType;
@@ -236,7 +236,7 @@ impl BobState {
         context: &Context,
         mime_message: &MimeMessage,
     ) -> Result<Option<BobHandshakeStage>> {
-        let step = match mime_message.get_header(HeaderDef::SecureJoin) {
+        let step = match mime_message.get(HeaderDef::SecureJoin) {
             Some(step) => step,
             None => {
                 warn!(
@@ -336,10 +336,9 @@ impl BobState {
                 // the very handshake message we're handling now.  But
                 // only after we have returned.  It does not impact
                 // the security invariants of secure-join however.
-
-                let is_verified_group = chat::get_chat_id_by_grpid(context, grpid)
-                    .await?
-                    .map_or(false, |(_chat_id, is_protected, _blocked)| is_protected);
+                let (_, is_verified_group, _) = chat::get_chat_id_by_grpid(context, grpid)
+                    .await
+                    .unwrap_or((ChatId::new(0), false, Blocked::Not));
                 // when joining a non-verified group
                 // the vg-member-added message may be unencrypted
                 // when not all group members have keys or prefer encryption.
@@ -357,12 +356,12 @@ impl BobState {
         }
         mark_peer_as_verified(context, self.invite.fingerprint()).await?;
         Contact::scaleup_origin_by_id(context, self.invite.contact_id(), Origin::SecurejoinJoined)
-            .await?;
-        context.emit_event(EventType::ContactsChanged(None));
+            .await;
+        emit_event!(context, EventType::ContactsChanged(None));
 
         if let QrInvite::Group { .. } = self.invite {
             let member_added = mime_message
-                .get_header(HeaderDef::ChatGroupMemberAdded)
+                .get(HeaderDef::ChatGroupMemberAdded)
                 .map(|s| s.as_str())
                 .ok_or_else(|| Error::msg("Missing Chat-Group-Member-Added header"))?;
             if !context.is_self_addr(member_added).await? {

src/securejoin/qrinvite.rs

@@ -9,7 +9,7 @@ use std::convert::TryFrom;
 
 use anyhow::Result;
 
-use crate::key::Fingerprint;
+use crate::key::{Fingerprint, FingerprintError};
 use crate::lot::{Lot, LotState};
 
 /// Represents the data from a QR-code scan.
@@ -103,6 +103,8 @@ impl TryFrom<Lot> for QrInvite {
 pub enum QrError {
     #[error("Unsupported protocol in QR-code")]
     UnsupportedProtocol,
+    #[error("Failed to read fingerprint")]
+    InvalidFingerprint(#[from] FingerprintError),
     #[error("Missing fingerprint")]
     MissingFingerprint,
     #[error("Missing invitenumber")]

src/simplify.rs

@@ -1,5 +1,3 @@
-//! # Simplify incoming plaintext.
-
 use itertools::Itertools;
 
 // protect lines starting with `--` against being treated as a footer.
@@ -155,8 +153,8 @@ fn remove_bottom_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], Option<String>)
         let quoted_text = lines[l_last..first_quoted_line]
             .iter()
             .map(|s| {
-                s.strip_prefix('>')
-                    .map_or(*s, |u| u.strip_prefix(' ').unwrap_or(u))
+                s.strip_prefix(">")
+                    .map_or(*s, |u| u.strip_prefix(" ").unwrap_or(u))
             })
             .join("\n");
         if l_last > 1 && is_empty_line(lines[l_last - 1]) {
@@ -201,8 +199,8 @@ fn remove_top_quote<'a>(lines: &'a [&str]) -> (&'a [&'a str], Option<String>) {
                 lines[first_quoted_line..last_quoted_line + 1]
                     .iter()
                     .map(|s| {
-                        s.strip_prefix('>')
-                            .map_or(*s, |u| u.strip_prefix(' ').unwrap_or(u))
+                        s.strip_prefix(">")
+                            .map_or(*s, |u| u.strip_prefix(" ").unwrap_or(u))
                     })
                     .join("\n"),
             ),
@@ -243,12 +241,14 @@ fn render_message(lines: &[&str], is_cut_at_end: bool) -> String {
     ret.replace("\u{200B}", "")
 }
 
-/// Returns true if the line contains only whitespace.
+/**
+ * Tools
+ */
 fn is_empty_line(buf: &str) -> bool {
     buf.chars().all(char::is_whitespace)
     // for some time, this checked for `char <= ' '`,
-    // see discussion at: <https://github.com/deltachat/deltachat-core-rust/pull/402#discussion_r317062392>
-    // and <https://github.com/deltachat/deltachat-core-rust/pull/2104/files#r538973613>
+    // see discussion at: https://github.com/deltachat/deltachat-core-rust/pull/402#discussion_r317062392
+    // and https://github.com/deltachat/deltachat-core-rust/pull/2104/files#r538973613
 }
 
 fn is_quoted_headline(buf: &str) -> bool {
@@ -396,7 +396,7 @@ mod tests {
         assert!(!is_cut);
         assert_eq!(footer, None);
 
-        // Nonstandard footer sent by <https://siju.es/>
+        // Nonstandard footer sent by https://siju.es/
         let input = "Message text here\n---Desde mi teléfono con SIJÚ\n\nQuote here".to_string();
         let (plain, _, is_cut, _, footer) = simplify(input.clone(), false);
         assert_eq!(plain, "Message text here [...]");

src/smtp.rs

@@ -1,20 +1,19 @@
-//! # SMTP transport module.
+//! # SMTP transport module
 
 pub mod send;
 
 use std::time::{Duration, SystemTime};
 
 use async_smtp::smtp::client::net::ClientTlsParameters;
-use async_smtp::{error, smtp, EmailAddress, ServerAddress};
+use async_smtp::{error, smtp, EmailAddress};
 
 use crate::constants::DC_LP_AUTH_OAUTH2;
+use crate::context::Context;
 use crate::events::EventType;
-use crate::login_param::{
-    dc_build_tls, CertificateChecks, LoginParam, ServerLoginParam, Socks5Config,
-};
+use crate::login_param::{dc_build_tls, CertificateChecks, LoginParam, ServerLoginParam};
 use crate::oauth2::dc_get_oauth2_access_token;
 use crate::provider::Socket;
-use crate::{context::Context, scheduler::connectivity::ConnectivityStore};
+use crate::stock_str;
 
 /// SMTP write and read timeout in seconds.
 const SMTP_TIMEOUT: u64 = 30;
@@ -29,10 +28,12 @@ pub enum Error {
         #[source]
         error: error::Error,
     },
-    #[error("SMTP failed to connect: {0}")]
+    #[error("SMTP: failed to connect: {0}")]
     ConnectionFailure(#[source] smtp::error::Error),
-    #[error("SMTP oauth2 error {address}")]
-    Oauth2 { address: String },
+    #[error("SMTP: failed to setup connection {0:?}")]
+    ConnectionSetupFailure(#[source] smtp::error::Error),
+    #[error("SMTP: oauth2 error {address}")]
+    Oauth2Error { address: String },
     #[error("TLS error {0}")]
     Tls(#[from] async_native_tls::Error),
     #[error("{0}")]
@@ -52,11 +53,6 @@ pub(crate) struct Smtp {
     /// (eg connect or send succeeded). On initialization and disconnect
     /// it is set to None.
     last_success: Option<SystemTime>,
-
-    pub(crate) connectivity: ConnectivityStore,
-
-    /// If sending the last message failed, contains the error message.
-    pub(crate) last_send_error: Option<String>,
 }
 
 impl Smtp {
@@ -101,18 +97,27 @@ impl Smtp {
             return Ok(());
         }
 
-        self.connectivity.set_connecting(context).await;
         let lp = LoginParam::from_database(context, "configured_").await?;
-        self.connect(
-            context,
-            &lp.smtp,
-            &lp.socks5_config,
-            &lp.addr,
-            lp.server_flags & DC_LP_AUTH_OAUTH2 != 0,
-            lp.provider
-                .map_or(lp.socks5_config.is_some(), |provider| provider.strict_tls),
-        )
-        .await
+        let res = self
+            .connect(
+                context,
+                &lp.smtp,
+                &lp.addr,
+                lp.server_flags & DC_LP_AUTH_OAUTH2 != 0,
+                lp.provider.map_or(false, |provider| provider.strict_tls),
+            )
+            .await;
+        if let Err(ref err) = res {
+            let message = stock_str::server_response(
+                context,
+                format!("SMTP {}:{}", lp.smtp.server, lp.smtp.port),
+                err.to_string(),
+            )
+            .await;
+
+            context.emit_event(EventType::ErrorNetwork(message));
+        };
+        res
     }
 
     /// Connect using the provided login params.
@@ -120,7 +125,6 @@ impl Smtp {
         &mut self,
         context: &Context,
         lp: &ServerLoginParam,
-        socks5_config: &Option<Socks5Config>,
         addr: &str,
         oauth2: bool,
         provider_strict_tls: bool,
@@ -159,7 +163,7 @@ impl Smtp {
             let send_pw = &lp.password;
             let access_token = dc_get_oauth2_access_token(context, addr, send_pw, false).await?;
             if access_token.is_none() {
-                return Err(Error::Oauth2 {
+                return Err(Error::Oauth2Error {
                     address: addr.to_string(),
                 });
             }
@@ -190,20 +194,17 @@ impl Smtp {
             _ => smtp::ClientSecurity::Wrapper(tls_parameters),
         };
 
-        let client =
-            smtp::SmtpClient::with_security(ServerAddress::new(domain.to_string(), port), security);
+        let client = smtp::SmtpClient::with_security((domain.as_str(), port), security)
+            .await
+            .map_err(Error::ConnectionSetupFailure)?;
 
-        let mut client = client
+        let client = client
             .smtp_utf8(true)
             .credentials(creds)
             .authentication_mechanism(mechanism)
             .connection_reuse(smtp::ConnectionReuseParameters::ReuseUnlimited)
             .timeout(Some(Duration::from_secs(SMTP_TIMEOUT)));
 
-        if let Some(socks5_config) = socks5_config {
-            client = client.use_socks5(socks5_config.to_async_smtp_socks5_config());
-        }
-
         let mut trans = client.into_transport();
         if let Err(err) = trans.connect().await {
             return Err(Error::ConnectionFailure(err));

src/smtp/send.rs

@@ -14,9 +14,9 @@ pub type Result<T> = std::result::Result<T, Error>;
 #[derive(Debug, thiserror::Error)]
 pub enum Error {
     #[error("Envelope error: {}", _0)]
-    Envelope(#[from] async_smtp::error::Error),
+    EnvelopeError(#[from] async_smtp::error::Error),
     #[error("Send error: {}", _0)]
-    SmtpSend(#[from] async_smtp::smtp::error::Error),
+    SendError(#[from] async_smtp::smtp::error::Error),
     #[error("SMTP has no transport")]
     NoTransport,
     #[error("{}", _0)]
@@ -46,7 +46,8 @@ impl Smtp {
             let recipients = recipients_chunk.to_vec();
             let recipients_display = recipients.iter().map(|x| x.to_string()).join(",");
 
-            let envelope = Envelope::new(self.from.clone(), recipients).map_err(Error::Envelope)?;
+            let envelope =
+                Envelope::new(self.from.clone(), recipients).map_err(Error::EnvelopeError)?;
             let mail = SendableEmail::new(
                 envelope,
                 format!("{}", job_id), // only used for internal logging
@@ -59,7 +60,7 @@ impl Smtp {
                 transport
                     .send_with_timeout(mail, Some(&Duration::from_secs(timeout)))
                     .await
-                    .map_err(Error::SmtpSend)?;
+                    .map_err(Error::SendError)?;
 
                 context.emit_event(EventType::SmtpMessageSent(format!(
                     "Message len={} was smtp-sent to {}",

src/sql.rs

@@ -1,4 +1,4 @@
-//! # SQLite wrapper.
+//! # SQLite wrapper
 
 use async_std::path::Path;
 use async_std::sync::RwLock;
@@ -598,8 +598,6 @@ pub async fn housekeeping(context: &Context) -> Result<()> {
         );
     }
 
-    context.schedule_quota_update().await?;
-
     if let Err(e) = context
         .set_config(Config::LastHousekeeping, Some(&time().to_string()))
         .await
@@ -643,8 +641,12 @@ async fn maybe_add_from_param(
         paramsv![],
         |row| row.get::<_, String>(0),
         |rows| {
-            for row in rows {
-                let param: Params = row?.parse().unwrap_or_default();
+            // Rows that can't be parsed, for example if they are
+            // not UTF-8 strings, are ignored. It is possible
+            // when upgrading from C core to Rust core, which
+            // guarantees UTF-8 strings everywhere.
+            for row in rows.filter_map(|row| row.ok()) {
+                let param: Params = row.parse().unwrap_or_default();
                 if let Some(file) = param.get(param_id) {
                     maybe_add_file(files_in_use, file);
                 }
@@ -772,7 +774,7 @@ mod test {
     /// existed and `PRAGMA` returned non-empty result.
     ///
     /// Statements were not finalized due to a bug in sqlx:
-    /// <https://github.com/launchbadge/sqlx/issues/1147>
+    /// https://github.com/launchbadge/sqlx/issues/1147
     #[async_std::test]
     async fn test_db_reopen() -> Result<()> {
         use tempfile::tempdir;