Update mailparse to parse names with comma

.circleci/config.yml

@@ -7,10 +7,6 @@ executors:
   doxygen:
     docker:
       - image: hrektts/doxygen
-  python:
-    docker:
-      - image: 3.7.7-stretch
-            
 
 restore-workspace: &restore-workspace
   attach_workspace:

.github/workflows/code-quality.yml

@@ -10,7 +10,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly-2020-03-12
+          toolchain: nightly-2019-11-06
           override: true
       - uses: actions-rs/cargo@v1
         with:
@@ -25,7 +25,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly-2020-03-12
+          toolchain: nightly-2019-11-06
           override: true
       - run: rustup component add rustfmt
       - uses: actions-rs/cargo@v1
@@ -39,10 +39,10 @@ jobs:
       - uses: actions/checkout@v1
       - uses: actions-rs/toolchain@v1
         with:
-            toolchain: nightly-2020-03-12
+            toolchain: nightly-2019-11-06
             components: clippy
             override: true
       - uses: actions-rs/clippy-check@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          args: --all-features
+          args: --all-features

CHANGELOG.md

@@ -1,156 +1,5 @@
 # Changelog 
 
-## 1.33.0
-
-- let `dc_set_muted()` also mute one-to-one chats #1470
-
-- fix a bug that led to load and traffic if the server does not use sent-folder
-  #1472
-
-
-## 1.32.0
-
-- fix endless loop when trying to download messages with bad RFC Message-ID,
-  also be more reliable on similar errors #1463 #1466 #1462
-
-- fix bug with comma in contact request #1438
-
-- do not refer to hidden messages on replies #1459
-
-- improve error handling #1468 #1465 #1464
-
-
-## 1.31.0
-
-- always describe the context of the displayed error #1451
-
-- do not emit `DC_EVENT_ERROR` when message sending fails;
-  `dc_msg_get_state()` and `dc_get_msg_info()` are sufficient #1451
-
-- new config-option `media_quality` #1449
-
-- try over if writing message to database fails #1447
-
-
-## 1.30.0
-
-- expunge deleted messages #1440
-
-- do not send `DC_EVENT_MSGS_CHANGED|INCOMING_MSG` on hidden messages #1439
-
-
-## 1.29.0
-
-- new config options `delete_device_after` and `delete_server_after`,
-  each taking an amount of seconds after which messages
-  are deleted from the device and/or the server #1310 #1335 #1411 #1417 #1423
-
-- new api `dc_estimate_deletion_cnt()` to estimate the effect
-  of `delete_device_after` and `delete_server_after`
-
-- use Ed25519 keys by default, these keys are much shorter
-  than RSA keys, which results in saving traffic and speed improvements #1362
-
-- improve message ellipsizing #1397 #1430
-
-- emit `DC_EVENT_ERROR_NETWORK` also on smtp-errors #1378
-
-- do not show badly formatted non-delta-messages as empty #1384
-
-- try over SMTP on potentially recoverable error 5.5.0 #1379
-
-- remove device-chat from forward-to-chat-list #1367
-
-- improve group-handling #1368
-
-- `dc_get_info()` returns uptime (how long the context is in use)
-
-- python improvements and adaptions #1408 #1415
-
-- log to the stdout and stderr in tests #1416
-
-- refactoring, code improvements #1363 #1365 #1366 #1370 #1375 #1389 #1390 #1418 #1419
-
-- removed api: `dc_chat_get_subtitle()`, `dc_get_version_str()`, `dc_array_add_id()`
-
-- removed events: `DC_EVENT_MEMBER_ADDED`, `DC_EVENT_MEMBER_REMOVED`
-
-
-## 1.28.0
-
-- new flag DC_GCL_FOR_FORWARDING for dc_get_chatlist()
-  that will sort the "saved messages" chat to the top of the chatlist #1336
-- mark mails as being deleted from server in dc_empty_server() #1333
-- fix interaction with servers that do not allow folder creation on root-level;
-  use path separator as defined by the email server #1359
-- fix group creation if group was created by non-delta clients #1357
-- fix showing replies from non-delta clients #1353
-- fix member list on rejoining left groups #1343
-- fix crash when using empty groups #1354
-- fix potential crash on special names #1350
-
-
-## 1.27.0
-
-- handle keys reliably on armv7 #1327
-
-
-## 1.26.0
-
-- change generated key type back to RSA as shipped versions
-  have problems to encrypt to Ed25519 keys
-
-- update rPGP to encrypt reliably to Ed25519 keys;
-  one of the next versions can finally use Ed25519 keys then
-
-
-## 1.25.0
-
-- save traffic by downloading only messages that are really displayed #1236
-
-- change generated key type to Ed25519, these keys are much shorter
-  than RSA keys, which results in saving traffic and speed improvements #1287
-
-- improve key handling #1237 #1240 #1242 #1247
-
-- mute handling, apis are dc_set_chat_mute_duration()
-  dc_chat_is_muted() and dc_chat_get_remaining_mute_duration() #1143
-
-- pinning chats, new apis are dc_set_chat_visibility() and
-  dc_chat_get_visibility() #1248
-
-- add dc_provider_new_from_email() api that queries the new, integrated
-  provider-database #1207
-
-- account creation by scanning a qr code
-  in the DCACCOUNT scheme (https://mailadm.readthedocs.io),
-  new api is dc_set_config_from_qr() #1249
-
-- if possible, dc_join_securejoin(), returns the new chat-id immediately
-  and does the handshake in background #1225
-
-- update imap and smtp dependencies #1115
-
-- check for MOVE capability before using MOVE command #1263
-
-- allow inline attachments from RFC 2183 #1280
-
-- fix updating names from incoming mails #1298
-
-- fix error messages shown on import #1234
-
-- directly attempt to re-connect if the smtp connection is maybe stale #1296
-
-- improve adding group members #1291
-
-- improve rust-api #1261
-
-- cleanup #1302 #1283 #1282 #1276 #1270-#1274 #1267 #1258-#1260
-  #1257 #1239 #1231 #1224
-
-- update spec #1286 #1291
-
-
 ## 1.0.0-beta.24
 
 - fix oauth2/gmail bug introduced in beta23 (not used in releases) #1219

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.33.0"
+version = "1.0.0-beta.24"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
 license = "MPL-2.0"
@@ -12,7 +12,7 @@ lto = true
 deltachat_derive = { path = "./deltachat_derive" }
 
 libc = "0.2.51"
-pgp = { version = "0.5.1", default-features = false } 
+pgp = { version = "0.4.0", default-features = false }
 hex = "0.4.0"
 sha2 = "0.8.0"
 rand = "0.7.0"
@@ -32,6 +32,8 @@ percent-encoding = "2.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 chrono = "0.4.6"
+failure = "0.1.5"
+failure_derive = "0.1.5"
 indexmap = "1.3.0"
 lazy_static = "1.4.0"
 regex = "1.1.6"
@@ -51,15 +53,13 @@ bitflags = "1.1.0"
 debug_stub_derive = "0.3.0"
 sanitize-filename = "0.2.1"
 stop-token = { version = "0.1.1", features = ["unstable"] }
-mailparse = "0.12.0"
+mailparse = { git = "https://github.com/link2xt/mailparse", branch="address-comma" }
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
 native-tls = "0.2.3"
 image = { version = "0.22.4", default-features=false, features = ["gif_codec", "jpeg", "ico", "png_codec", "pnm", "webp", "bmp"] }
 pretty_env_logger = "0.3.1"
 
 rustyline = { version = "4.1.0", optional = true }
-thiserror = "1.0.14"
-anyhow = "1.0.28"
 
 [dev-dependencies]
 tempfile = "3.0"
@@ -84,7 +84,7 @@ required-features = ["rustyline"]
 
 
 [features]
-default = ["nightly"]
+default = ["nightly", "ringbuf"]
 vendored = ["async-native-tls/vendored", "reqwest/native-tls-vendored", "async-smtp/native-tls-vendored"]
 nightly = ["pgp/nightly"]
-
+ringbuf = ["pgp/ringbuf"]

README.md

@@ -17,9 +17,8 @@ curl https://sh.rustup.rs -sSf | sh
 Compile and run Delta Chat Core command line utility, using `cargo`:
 
 ```
-cargo run --example repl -- ~/deltachat-db
+cargo run --example repl -- /path/to/db
 ```
-where ~/deltachat-db is the database file. Delta Chat will create it if it does not exist.
 
 Configure your account (if not already configured):
 
@@ -109,6 +108,7 @@ $ cargo test -- --ignored
 
 - `vendored`: When using Openssl for TLS, this bundles a vendored version.
 - `nightly`: Enable nightly only performance and security related features.
+- `ringbuf`: Enable the use of [`slice_deque`](https://github.com/gnzlbg/slice_deque) in pgp.
 
 [circle-shield]: https://img.shields.io/circleci/project/github/deltachat/deltachat-core-rust/master.svg?style=flat-square
 [circle]: https://circleci.com/gh/deltachat/deltachat-core-rust/

appveyor.yml

@@ -4,7 +4,7 @@ environment:
 
 install:
   - appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
-  - rustup-init -yv --default-toolchain nightly-2020-03-12
+  - rustup-init -yv --default-toolchain nightly-2019-07-10
   - set PATH=%PATH%;%USERPROFILE%\.cargo\bin
   - rustc -vV
   - cargo -vV

ci_scripts/ci_upload.sh

@@ -37,7 +37,7 @@ echo -----------------------
 # Bundle external shared libraries into the wheels
 pushd $WHEELHOUSEDIR
 
-pip3 install -U pip setuptools
+pip3 install -U pip
 pip3 install devpi-client
 devpi use https://m.devpi.net
 devpi login dc --password $DEVPI_LOGIN

ci_scripts/cleanup_devpi_indices.py

@@ -48,7 +48,7 @@ def run():
         projectnames = get_projectnames(baseurl, username, indexname)
         if indexname == "master" or not indexname:
             continue
-        clear_index = not projectnames
+        assert projectnames == ["deltachat"]
         for projectname in projectnames:
             dates = get_release_dates(baseurl, username, indexname, projectname)
             if not dates:
@@ -60,11 +60,8 @@ def run():
                 date = datetime.datetime(*max(dates))
             if (datetime.datetime.now() - date) > datetime.timedelta(days=MAXDAYS):
                 assert username and indexname
-                clear_index = True
-                break
-        if clear_index:
-            url = baseurl + username + "/" + indexname
-            subprocess.check_call(["devpi", "index", "-y", "--delete", url])
+                url = baseurl + username + "/" + indexname
+                subprocess.check_call(["devpi", "index", "-y", "--delete", url])
 
 
 

ci_scripts/docker-coredeps/deps/build_rust.sh

@@ -3,9 +3,9 @@
 set -e -x
 
 # Install Rust
-curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2020-03-12 -y
+curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-11-06 -y
 export PATH=/root/.cargo/bin:$PATH
 rustc --version
 
 # remove some 300-400 MB that we don't need for automated builds
-rm -rf /root/.rustup/toolchains/nightly-2020-03-12-x86_64-unknown-linux-gnu/share/
+rm -rf /root/.rustup/toolchains/nightly-2019-11-06-x86_64-unknown-linux-gnu/share/

ci_scripts/old/run-python.sh

@@ -46,7 +46,6 @@ if [ -n "$TESTS" ]; then
     tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "not qr"
     tox --workdir "$TOXWORKDIR" -e py37 -- --reruns 3 -k "qr"
     unset DCC_PY_LIVECONFIG
-    unset DCC_NEW_TMP_EMAIL
     tox --workdir "$TOXWORKDIR" -p4 -e lint,py35,py36,doc
     tox --workdir "$TOXWORKDIR" -e auditwheels
     popd

ci_scripts/remote_python_packaging.sh

@@ -32,11 +32,11 @@ ssh $SSHTARGET bash -c "cat >$BUILDDIR/exec_docker_run" <<_HERE
     set +x -e
     cd $BUILDDIR
     export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
-    export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
+
     set -x
 
     # run everything else inside docker 
-    docker run -e DCC_NEW_TMP_EMAIL -e DCC_PY_LIVECONFIG \
+    docker run -e DCC_PY_LIVECONFIG \
        --rm -it -v \$(pwd):/mnt -w /mnt \
        deltachat/coredeps ci_scripts/run_all.sh
 

ci_scripts/remote_tests_python.sh

@@ -30,7 +30,6 @@ ssh $SSHTARGET <<_HERE
     export CARGO_TARGET_DIR=\`pwd\`/../target
     export TARGET=release
     export DCC_PY_LIVECONFIG=$DCC_PY_LIVECONFIG
-    export DCC_NEW_TMP_EMAIL=$DCC_NEW_TMP_EMAIL
 
     #we rely on tox/virtualenv being available in the host
     #rm -rf virtualenv venv

ci_scripts/run_all.sh

@@ -37,8 +37,7 @@ mkdir -p $TOXWORKDIR
 # XXX we may switch on some live-tests on for better ensurances 
 # Note that the independent remote_tests_python step does all kinds of
 # live-testing already. 
-unset DCC_PY_LIVECONFIG
-unset DCC_NEW_TMP_EMAIL
+unset DCC_PY_LIVECONFIG 
 tox --workdir "$TOXWORKDIR" -e py35,py36,py37,py38,auditwheels
 popd
 

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.33.0"
+version = "1.0.0-beta.24"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
@@ -19,11 +19,11 @@ deltachat = { path = "../", default-features = false }
 libc = "0.2"
 human-panic = "1.0.1"
 num-traits = "0.2.6"
+failure = "0.1.6"
 serde_json = "1.0"
-anyhow = "1.0.28"
-thiserror = "1.0.14"
 
 [features]
-default = ["vendored", "nightly"]
+default = ["vendored", "nightly", "ringbuf"]
 vendored = ["deltachat/vendored"]
 nightly = ["deltachat/nightly"]
+ringbuf = ["deltachat/ringbuf"]

deltachat-ffi/deltachat.h

@@ -41,7 +41,7 @@ typedef struct _dc_provider dc_provider_t;
  * uintptr_t event_handler_func(dc_context_t* context, int event,
  *                              uintptr_t data1, uintptr_t data2)
  * {
- *     return 0;
+ *     return 0; // for unhandled events, it is always safe to return 0
  * }
  *
  * dc_context_t* context = dc_context_new(event_handler_func, NULL, NULL);
@@ -208,7 +208,7 @@ typedef struct _dc_provider dc_provider_t;
  * @param event one of the @ref DC_EVENT constants
  * @param data1 depends on the event parameter
  * @param data2 depends on the event parameter
- * @return events do not expect a return value, just always return 0
+ * @return return 0 unless stated otherwise in the event parameter documentation
  */
 typedef uintptr_t (*dc_callback_t) (dc_context_t* context, int event, uintptr_t data1, uintptr_t data2);
 
@@ -229,7 +229,7 @@ typedef uintptr_t (*dc_callback_t) (dc_context_t* context, int event, uintptr_t
  *       otherwise!
  *     - The callback SHOULD return _fast_, for GUI updates etc. you should
  *       post yourself an asynchronous message to your GUI thread, if needed.
- *     - events do not expect a return value, just always return 0.
+ *     - If not mentioned otherweise, the callback should return 0.
  * @param userdata can be used by the client for any purpuse.  He finds it
  *     later in dc_get_userdata().
  * @param os_name is only for decorative use
@@ -342,8 +342,7 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  * - `smtp_certificate_checks` = how to check SMTP certificates, one of the @ref DC_CERTCK flags, defaults to #DC_CERTCK_AUTO (0)
  * - `displayname`  = Own name to use when sending messages.  MUAs are allowed to spread this way eg. using CC, defaults to empty
  * - `selfstatus`   = Own status to display eg. in email footers, defaults to a standard text
- * - `selfavatar`   = File containing avatar. Will immediately be copied to the 
- *                    `blobdir`; the original image will not be needed anymore.
+ * - `selfavatar`   = File containing avatar. Will be copied to blob directory.
  *                    NULL to remove the avatar.
  *                    It is planned for future versions
  *                    to send this image together with the next messages.
@@ -365,34 +364,9 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    also show all mails of confirmed contacts,
  *                    DC_SHOW_EMAILS_ALL (2)=
  *                    also show mails of unconfirmed contacts in the deaddrop.
- * - `key_gen_type` = DC_KEY_GEN_DEFAULT (0)=
- *                    generate recommended key type (default),
- *                    DC_KEY_GEN_RSA2048 (1)=
- *                    generate RSA 2048 keypair
- *                    DC_KEY_GEN_ED25519 (2)=
- *                    generate Ed25519 keypair
  * - `save_mime_headers` = 1=save mime headers
  *                    and make dc_get_mime_headers() work for subsequent calls,
  *                    0=do not save mime headers (default)
- * - `delete_device_after` = 0=do not delete messages from device automatically (default),
- *                    >=1=seconds, after which messages are deleted automatically from the device.
- *                    Messages in the "saved messages" chat (see dc_chat_is_self_talk()) are skipped.
- *                    Messages are deleted whether they were seen or not, the UI should clearly point that out.
- *                    See also dc_estimate_deletion_cnt().
- * - `delete_server_after` = 0=do not delete messages from server automatically (default),
- *                    1=delete messages directly after receiving from server, mvbox is skipped.
- *                    >1=seconds, after which messages are deleted automatically from the server, mvbox is used as defined.
- *                    "Saved messages" are deleted from the server as well as
- *                    emails matching the `show_emails` settings above, the UI should clearly point that out.
- *                    See also dc_estimate_deletion_cnt().
- * - `media_quality` = DC_MEDIA_QUALITY_BALANCED (0) =
- *                    good outgoing images/videos/voice quality at reasonable sizes (default)
- *                    DC_MEDIA_QUALITY_WORSE (1)
- *                    allow worse images/videos/voice quality to gain smaller sizes,
- *                    suitable for providers or areas known to have a bad connection.
- *                    In contrast to other options, the implementation of this option is currently up to the UIs;
- *                    this may change in future, however,
- *                    having the option in the core allows provider-specific-defaults already today.
  *
  * If you want to retrieve a value, use dc_get_config().
  *
@@ -921,7 +895,6 @@ int             dc_preconfigure_keypair        (dc_context_t* context, const cha
 #define         DC_GCL_ARCHIVED_ONLY         0x01
 #define         DC_GCL_NO_SPECIALS           0x02
 #define         DC_GCL_ADD_ALLDONE_HINT      0x04
-#define         DC_GCL_FOR_FORWARDING        0x08
 
 
 /**
@@ -945,7 +918,7 @@ int             dc_preconfigure_keypair        (dc_context_t* context, const cha
  *   or "Not now".
  *   The UI can also offer a "Close" button that calls dc_marknoticed_contact() then.
  * - DC_CHAT_ID_ARCHIVED_LINK (6) - this special chat is present if the user has
- *   archived _any_ chat using dc_set_chat_visibility(). The UI should show a link as
+ *   archived _any_ chat using dc_archive_chat(). The UI should show a link as
  *   "Show archived chats", if the user clicks this item, the UI should show a
  *   list of all archived chats that can be created by this function hen using
  *   the DC_GCL_ARCHIVED_ONLY flag.
@@ -960,10 +933,6 @@ int             dc_preconfigure_keypair        (dc_context_t* context, const cha
  *       if DC_GCL_ARCHIVED_ONLY is not set, only unarchived chats are returned and
  *       the pseudo-chat DC_CHAT_ID_ARCHIVED_LINK is added if there are _any_ archived
  *       chats
- *     - the flag DC_GCL_FOR_FORWARDING sorts "Saved messages" to the top of the chatlist
- *       and hides the "Device chat" and the deaddrop.
- *       typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS
- *       to also hide the archive link.
  *     - if the flag DC_GCL_NO_SPECIALS is set, deaddrop and archive link are not added
  *       to the list (may be used eg. for selecting chats on forwarding, the flag is
  *       not needed when DC_GCL_ARCHIVED_ONLY is already set)
@@ -1317,22 +1286,6 @@ int             dc_get_msg_cnt               (dc_context_t* context, uint32_t ch
 int             dc_get_fresh_msg_cnt         (dc_context_t* context, uint32_t chat_id);
 
 
-
-/**
- * Estimate the number of messages that will be deleted
- * by the dc_set_config()-options `delete_device_after` or `delete_server_after`.
- * This is typically used to show the estimated impact to the user before actually enabling ephemeral messages.
- *
- * @memberof dc_context_t
- * @param context The context object as returned from dc_context_new().
- * @param from_server 1=Estimate deletion count for server, 0=Estimate deletion count for device
- * @param seconds Count messages older than the given number of seconds.
- * @return Number of messages that are older than the given number of seconds.
- *     This includes emails downloaded due to the `show_emails` option.
- *     Messages in the "saved messages" folder are not counted as they will not be deleted automatically.
- */
-int             dc_estimate_deletion_cnt    (dc_context_t* context, int from_server, int64_t seconds);
-
 /**
  * Returns the message IDs of all _fresh_ messages of any chat.
  * Typically used for implementing notification summaries.
@@ -1419,18 +1372,25 @@ uint32_t        dc_get_next_media            (dc_context_t* context, uint32_t ms
 
 
 /**
- * Set chat visibility to pinned, archived or normal.
+ * Archive or unarchive a chat.
  *
- * Calling this function usually results in the event #DC_EVENT_MSGS_CHANGED
- * See @ref DC_CHAT_VISIBILITY for detailed information about the visibilities.
+ * Archived chats are not included in the default chatlist returned
+ * by dc_get_chatlist().  Instead, if there are _any_ archived chats,
+ * the pseudo-chat with the chat_id DC_CHAT_ID_ARCHIVED_LINK will be added the the
+ * end of the chatlist.
+ *
+ * - To get a list of archived chats, use dc_get_chatlist() with the flag DC_GCL_ARCHIVED_ONLY.
+ * - To find out the archived state of a given chat, use dc_chat_get_archived()
+ * - Messages in archived chats are marked as being noticed, so they do not count as "fresh"
+ * - Calling this function usually results in the event #DC_EVENT_MSGS_CHANGED
  *
  * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
- * @param chat_id The ID of the chat to change the visibility for.
- * @param visibility one of @ref DC_CHAT_VISIBILITY
+ * @param chat_id The ID of the chat to archive or unarchive.
+ * @param archive 1=archive chat, 0=unarchive chat, all other values are reserved for future use
  * @return None.
  */
-void            dc_set_chat_visibility       (dc_context_t* context, uint32_t chat_id, int visibility);
+void            dc_archive_chat              (dc_context_t* context, uint32_t chat_id, int archive);
 
 
 /**
@@ -1627,10 +1587,8 @@ int             dc_set_chat_name             (dc_context_t* context, uint32_t ch
  * @memberof dc_context_t
  * @param context The context as created by dc_context_new().
  * @param chat_id The chat ID to set the image for.
- * @param image Full path of the image to use as the group image. The image will immediately be copied to the 
- *     `blobdir`; the original image will not be needed anymore.
- *      If you pass NULL here, the group image is deleted (for promoted groups, all members are informed about 
- *      this change anyway).
+ * @param image Full path of the image to use as the group image.  If you pass NULL here,
+ *     the group image is deleted (for promoted groups, all members are informed about this change anyway).
  * @return 1=success, 0=error
  */
 int             dc_set_chat_profile_image    (dc_context_t* context, uint32_t chat_id, const char* image);
@@ -1640,7 +1598,7 @@ int             dc_set_chat_profile_image    (dc_context_t* context, uint32_t ch
 /**
  * Set mute duration of a chat.
  *
- * The ui can then call dc_chat_is_muted() when receiving a new message to decide whether it should trigger an notification.
+ * This value can be checked by the ui upon receiving a new message to decide whether it should trigger an notification.
  *
  * Sends out #DC_EVENT_CHAT_MODIFIED.
  *
@@ -1698,9 +1656,8 @@ char*           dc_get_mime_headers          (dc_context_t* context, uint32_t ms
  */
 void            dc_delete_msgs               (dc_context_t* context, const uint32_t* msg_ids, int msg_cnt);
 
-/*
+/**
  * Empty IMAP server folder: delete all messages.
- * Deprecated, use dc_set_config() with the key "delete_server_after" instead.
  *
  * @memberof dc_context_t
  * @param context The context object as created by dc_context_new()
@@ -1803,7 +1760,7 @@ int             dc_may_be_valid_addr         (const char* addr);
 
 /**
  * Check if an e-mail address belongs to a known and unblocked contact.
- * To get a list of all known and unblocked contacts, use dc_get_contacts().
+ * Known and unblocked contacts will be returned by dc_get_contacts().
  *
  * To validate an e-mail address independently of the contact database
  * use dc_may_be_valid_addr().
@@ -1811,8 +1768,7 @@ int             dc_may_be_valid_addr         (const char* addr);
  * @memberof dc_context_t
  * @param context The context object as created by dc_context_new().
  * @param addr The e-mail-address to check.
- * @return Contact ID of the contact belonging to the e-mail-address
- *     or 0 if there is no contact that is or was introduced by an accepted contact.  
+ * @return 1=address is a contact in use, 0=address is not a contact in use.
  */
 uint32_t        dc_lookup_contact_id_by_addr (dc_context_t* context, const char* addr);
 
@@ -2845,6 +2801,19 @@ int             dc_chat_get_type             (const dc_chat_t* chat);
 char*           dc_chat_get_name             (const dc_chat_t* chat);
 
 
+/*
+ * Get a subtitle for a chat.  The subtitle is eg. the email-address or the
+ * number of group members.
+ *
+ * Deprecated function. Subtitles should be created in the ui
+ * where plural forms and other specials can be handled more gracefully.
+ *
+ * @param chat The chat object to calulate the subtitle for.
+ * @return Subtitle as a string. Must be released using dc_str_unref() after usage. Never NULL.
+ */
+char*           dc_chat_get_subtitle         (const dc_chat_t* chat);
+
+
 /**
  * Get the chat's profile image.
  * For groups, this is the image set by any group member
@@ -2877,14 +2846,21 @@ uint32_t        dc_chat_get_color            (const dc_chat_t* chat);
 
 
 /**
- * Get visibility of chat.
- * See @ref DC_CHAT_VISIBILITY for detailed information about the visibilities.
+ * Get archived state.
+ *
+ * - 0 = normal chat, not archived, not sticky.
+ * - 1 = chat archived
+ * - 2 = chat sticky (reserved for future use, if you do not support this value, just treat the chat as a normal one)
+ *
+ * To archive or unarchive chats, use dc_archive_chat().
+ * If chats are archived, this should be shown in the UI by a little icon or text,
+ * eg. the search will also return archived chats.
  *
  * @memberof dc_chat_t
  * @param chat The chat object.
- * @return One of @ref DC_CHAT_VISIBILITY
+ * @return Archived state.
  */
-int             dc_chat_get_visibility       (const dc_chat_t* chat);
+int             dc_chat_get_archived         (const dc_chat_t* chat);
 
 
 /**
@@ -2978,7 +2954,7 @@ int             dc_chat_is_sending_locations (const dc_chat_t* chat);
 
 
 /**
- * Check whether the chat is currently muted (can be changed by dc_set_chat_mute_duration()).
+ * Check whether the chat is currently muted
  *
  * @memberof dc_chat_t
  * @param chat The chat object.
@@ -3797,7 +3773,7 @@ int             dc_contact_is_verified       (dc_contact_t* contact);
  *     accessor functions.  If no provider info is found, NULL will be
  *     returned.
  */
-dc_provider_t*  dc_provider_new_from_email            (const dc_context_t* context, const char* email);
+dc_provider_t*  dc_provider_new_from_email            (const dc_context_t*, const char* email);
 
 
 /**
@@ -4156,8 +4132,28 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  */
 
 
-#define DC_EMPTY_MVBOX 0x01 // Deprecated, flag for dc_empty_server(): Clear all mvbox messages
-#define DC_EMPTY_INBOX 0x02 // Deprecated, flag for dc_empty_server(): Clear all INBOX messages
+/**
+ * @defgroup DC_EMPTY DC_EMPTY
+ *
+ * These constants configure emptying imap folders with dc_empty_server()
+ *
+ * @addtogroup DC_EMPTY
+ * @{
+ */
+
+/**
+ * Clear all mvbox messages.
+ */
+#define DC_EMPTY_MVBOX 0x01
+
+/**
+ * Clear all INBOX messages.
+ */
+#define DC_EMPTY_INBOX 0x02
+
+/**
+ * @}
+ */
 
 
 /**
@@ -4165,6 +4161,8 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * These constants are used as events
  * reported to the callback given to dc_context_new().
+ * If you do not want to handle an event, it is always safe to return 0,
+ * so there is no need to add a "case" for every event.
  *
  * @addtogroup DC_EVENT
  * @{
@@ -4179,6 +4177,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
  *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
  */
 #define DC_EVENT_INFO                     100
 
@@ -4189,6 +4188,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
  *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
  */
 #define DC_EVENT_SMTP_CONNECTED           101
 
@@ -4199,6 +4199,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
  *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
  */
 #define DC_EVENT_IMAP_CONNECTED           102
 
@@ -4208,6 +4209,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
  *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
  */
 #define DC_EVENT_SMTP_MESSAGE_SENT        103
 
@@ -4217,6 +4219,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
  *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
  */
 #define DC_EVENT_IMAP_MESSAGE_DELETED   104
 
@@ -4226,6 +4229,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) Info string in english language.
  *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
  */
 #define DC_EVENT_IMAP_MESSAGE_MOVED   105
 
@@ -4235,6 +4239,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) folder name.
  *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
  */
 #define DC_EVENT_IMAP_FOLDER_EMPTIED  106
 
@@ -4244,6 +4249,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) path name
  *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
  */
 #define DC_EVENT_NEW_BLOB_FILE 150
 
@@ -4253,6 +4259,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) path name
  *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
  */
 #define DC_EVENT_DELETED_BLOB_FILE 151
 
@@ -4265,6 +4272,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 0
  * @param data2 (const char*) Warning string in english language.
  *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
  */
 #define DC_EVENT_WARNING                  300
 
@@ -4287,6 +4295,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *     Some error strings are taken from dc_set_stock_translation(),
  *     however, most error strings will be in english language.
  *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
  */
 #define DC_EVENT_ERROR                    400
 
@@ -4310,6 +4319,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *     0=subsequent network error, should be logged only
  * @param data2 (const char*) Error string, always set, never NULL.
  *     Must not be unref'd or modified and is valid only until the callback returns.
+ * @return 0
  */
 #define DC_EVENT_ERROR_NETWORK            401
 
@@ -4325,6 +4335,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data2 (const char*) Info string in english language.
  *     Must not be unref'd or modified
  *     and is valid only until the callback returns.
+ * @return 0
  */
 #define DC_EVENT_ERROR_SELF_NOT_IN_GROUP  410
 
@@ -4338,6 +4349,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) chat_id for single added messages
  * @param data2 (int) msg_id for single added messages
+ * @return 0
  */
 #define DC_EVENT_MSGS_CHANGED             2000
 
@@ -4350,6 +4362,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) chat_id
  * @param data2 (int) msg_id
+ * @return 0
  */
 #define DC_EVENT_INCOMING_MSG             2005
 
@@ -4360,6 +4373,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) chat_id
  * @param data2 (int) msg_id
+ * @return 0
  */
 #define DC_EVENT_MSG_DELIVERED            2010
 
@@ -4370,6 +4384,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) chat_id
  * @param data2 (int) msg_id
+ * @return 0
  */
 #define DC_EVENT_MSG_FAILED               2012
 
@@ -4380,6 +4395,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) chat_id
  * @param data2 (int) msg_id
+ * @return 0
  */
 #define DC_EVENT_MSG_READ                 2015
 
@@ -4392,6 +4408,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) chat_id
  * @param data2 0
+ * @return 0
  */
 #define DC_EVENT_CHAT_MODIFIED            2020
 
@@ -4401,6 +4418,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) If not 0, this is the contact_id of an added contact that should be selected.
  * @param data2 0
+ * @return 0
  */
 #define DC_EVENT_CONTACTS_CHANGED         2030
 
@@ -4413,6 +4431,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *     If the locations of several contacts have been changed,
  *     eg. after calling dc_delete_all_locations(), this parameter is set to 0.
  * @param data2 0
+ * @return 0
  */
 #define DC_EVENT_LOCATION_CHANGED         2035
 
@@ -4422,6 +4441,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) 0=error, 1-999=progress in permille, 1000=success and done
  * @param data2 0
+ * @return 0
  */
 #define DC_EVENT_CONFIGURE_PROGRESS       2041
 
@@ -4431,6 +4451,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *
  * @param data1 (int) 0=error, 1-999=progress in permille, 1000=success and done
  * @param data2 0
+ * @return 0
  */
 #define DC_EVENT_IMEX_PROGRESS            2051
 
@@ -4445,6 +4466,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data1 (const char*) Path and file name.
  *     Must not be unref'd or modified and is valid only until the callback returns.
  * @param data2 0
+ * @return 0
  */
 #define DC_EVENT_IMEX_FILE_WRITTEN        2052
 
@@ -4462,6 +4484,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  *     600=vg-/vc-request-with-auth received, vg-member-added/vc-contact-confirm sent, typically shown as "bob@addr verified".
  *     800=vg-member-added-received received, shown as "bob@addr securely joined GROUP", only sent for the verified-group-protocol.
  *     1000=Protocol finished for this contact.
+ * @return 0
  */
 #define DC_EVENT_SECUREJOIN_INVITER_PROGRESS      2060
 
@@ -4477,9 +4500,21 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  * @param data2 (int) Progress as:
  *     400=vg-/vc-request-with-auth sent, typically shown as "alice@addr verified, introducing myself."
  *     (Bob has verified alice and waits until Alice does the same for him)
+ * @return 0
  */
 #define DC_EVENT_SECUREJOIN_JOINER_PROGRESS       2061
 
+
+/**
+ * This event is sent out to the inviter when a joiner successfully joined a group.
+ *
+ * @param data1 (int) chat_id
+ * @param data2 (int) contact_id
+ * @return 0
+ */
+#define DC_EVENT_SECUREJOIN_MEMBER_ADDED 2062
+
+
 /**
  * @}
  */
@@ -4495,8 +4530,8 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_EVENT_DATA2_IS_STRING(e)  ((e)>=100 && (e)<=499)
 #define DC_EVENT_RETURNS_INT(e)      ((e)==DC_EVENT_IS_OFFLINE) // not used anymore
 #define DC_EVENT_RETURNS_STRING(e)   ((e)==DC_EVENT_GET_STRING) // not used anymore
-#define dc_archive_chat(a,b,c)  dc_set_chat_visibility((a), (b), (c)? 1 : 0) // not used anymore
-#define dc_chat_get_archived(a) (dc_chat_get_visibility((a))==1? 1 : 0)      // not used anymore
+char*           dc_get_version_str           (void); // deprecated
+void            dc_array_add_id              (dc_array_t*, uint32_t); // deprecated
 
 
 /*
@@ -4507,21 +4542,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_SHOW_EMAILS_ALL               2
 
 
-/*
- * Values for dc_get|set_config("media_quality")
- */
-#define DC_MEDIA_QUALITY_BALANCED 0
-#define DC_MEDIA_QUALITY_WORSE    1
-
-
-/*
- * Values for dc_get|set_config("key_gen_type")
- */
-#define DC_KEY_GEN_DEFAULT 0
-#define DC_KEY_GEN_RSA2048 1
-#define DC_KEY_GEN_ED25519 2
-
-
 /**
  * @defgroup DC_PROVIDER_STATUS DC_PROVIDER_STATUS
  *
@@ -4577,48 +4597,6 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  */
 
 
-/**
- * @defgroup DC_CHAT_VISIBILITY DC_CHAT_VISIBILITY
- *
- * These constants describe the visibility of a chat.
- * The chat visibiliry can be get using dc_chat_get_visibility()
- * and set using dc_set_chat_visibility().
- *
- * @addtogroup DC_CHAT_VISIBILITY
- * @{
- */
-
-/**
- * Chats with normal visibility are not archived and are shown below all pinned chats.
- * Archived chats, that receive new messages automatically become normal chats.
- */
-#define         DC_CHAT_VISIBILITY_NORMAL      0
-
-/**
- * Archived chats are not included in the default chatlist returned by dc_get_chatlist().
- * Instead, if there are _any_ archived chats, the pseudo-chat
- * with the chat_id DC_CHAT_ID_ARCHIVED_LINK will be added the the end of the chatlist.
- *
- * The UI typically shows a little icon or chats beside archived chats in the chatlist,
- * this is needed as eg. the search will also return archived chats.
- *
- * If archived chats receive new messages, they become normal chats again.
- *
- * To get a list of archived chats, use dc_get_chatlist() with the flag DC_GCL_ARCHIVED_ONLY.
- */
-#define         DC_CHAT_VISIBILITY_ARCHIVED    1
-
-/**
- * Pinned chats are included in the default chatlist. moreover,
- * they are always the first items, whether they have fresh messages or not.
- */
-#define         DC_CHAT_VISIBILITY_PINNED      2
-
-/**
- * @}
- */
-
-
 /*
  * TODO: Strings need some doumentation about used placeholders.
  *
@@ -4632,6 +4610,8 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_STR_NOMESSAGES                 1
 #define DC_STR_SELF                       2
 #define DC_STR_DRAFT                      3
+#define DC_STR_MEMBER                     4
+#define DC_STR_CONTACT                    6
 #define DC_STR_VOICEMESSAGE               7
 #define DC_STR_DEADDROP                   8
 #define DC_STR_IMAGE                      9
@@ -4663,6 +4643,7 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
 #define DC_STR_STARREDMSGS                41
 #define DC_STR_AC_SETUP_MSG_SUBJECT       42
 #define DC_STR_AC_SETUP_MSG_BODY          43
+#define DC_STR_SELFTALK_SUBTITLE          50
 #define DC_STR_CANNOT_LOGIN               60
 #define DC_STR_SERVER_RESPONSE            61
 #define DC_STR_MSGACTIONBYUSER            62

deltachat-ffi/src/lib.rs

@@ -13,7 +13,7 @@ extern crate human_panic;
 extern crate num_traits;
 extern crate serde_json;
 
-use std::collections::BTreeMap;
+use std::collections::HashMap;
 use std::convert::TryInto;
 use std::ffi::CString;
 use std::fmt::Write;
@@ -25,9 +25,10 @@ use std::time::{Duration, SystemTime};
 use libc::uintptr_t;
 use num_traits::{FromPrimitive, ToPrimitive};
 
-use deltachat::chat::{ChatId, ChatVisibility, MuteDuration};
+use deltachat::chat::ChatId;
+use deltachat::chat::MuteDuration;
 use deltachat::constants::DC_MSG_ID_LAST_SPECIAL;
-use deltachat::contact::{Contact, Origin};
+use deltachat::contact::Contact;
 use deltachat::context::Context;
 use deltachat::key::DcKey;
 use deltachat::message::MsgId;
@@ -85,6 +86,17 @@ pub type dc_callback_t =
 pub type dc_context_t = ContextWrapper;
 
 impl ContextWrapper {
+    /// Log an error on the FFI context.
+    ///
+    /// As soon as a [ContextWrapper] exist it can be used to log an
+    /// error using the callback, even before [dc_context_open] is
+    /// called and an actual [Context] exists.
+    ///
+    /// This function makes it easy to log an error.
+    unsafe fn error(&self, msg: &str) {
+        self.translate_cb(Event::Error(msg.to_string()));
+    }
+
     /// Log a warning on the FFI context.
     ///
     /// Like [error] but logs as a warning which only goes to the
@@ -108,6 +120,10 @@ impl ContextWrapper {
     /// the appropriate return value for an error return since this
     /// differs for various functions on the FFI API: sometimes 0,
     /// NULL, an empty string etc.
+    ///
+    /// Prefer to use [ContextWrapper::try_inner], we might want to
+    /// remove this function at some point to reduce the cognitive
+    /// overload of having two functions which are too similar.
     unsafe fn with_inner<T, F>(&self, ctxfn: F) -> Result<T, ()>
     where
         F: FnOnce(&Context) -> T,
@@ -120,17 +136,17 @@ impl ContextWrapper {
     /// Unlock the context and execute a closure with it.
     ///
     /// This is like [ContextWrapper::with_inner] but uses
-    /// [anyhow::Error] as error type.  This allows you to write a
+    /// [failure::Error] as error type.  This allows you to write a
     /// closure which could produce many errors, use the `?` operator
     /// to return them and handle them all as the return of this call.
-    fn try_inner<T, F>(&self, ctxfn: F) -> Result<T, anyhow::Error>
+    fn try_inner<T, F>(&self, ctxfn: F) -> Result<T, failure::Error>
     where
-        F: FnOnce(&Context) -> Result<T, anyhow::Error>,
+        F: FnOnce(&Context) -> Result<T, failure::Error>,
     {
         let guard = self.inner.read().unwrap();
         match guard.as_ref() {
             Some(ref ctx) => ctxfn(ctx),
-            None => Err(anyhow::format_err!("context not open")),
+            None => Err(failure::err_msg("context not open")),
         }
     }
 
@@ -196,6 +212,17 @@ impl ContextWrapper {
                         progress as uintptr_t,
                     );
                 }
+                Event::SecurejoinMemberAdded {
+                    chat_id,
+                    contact_id,
+                } => {
+                    ffi_cb(
+                        self,
+                        event_id,
+                        chat_id.to_u32() as uintptr_t,
+                        contact_id as uintptr_t,
+                    );
+                }
             }
         }
     }
@@ -340,7 +367,7 @@ pub unsafe extern "C" fn dc_set_config(
             })
             .unwrap_or(0),
         Err(_) => {
-            ffi_context.warning("dc_set_config(): invalid key");
+            ffi_context.error("dc_set_config(): invalid key");
             0
         }
     }
@@ -361,7 +388,7 @@ pub unsafe extern "C" fn dc_get_config(
             .with_inner(|ctx| ctx.get_config(key).unwrap_or_default().strdup())
             .unwrap_or_else(|_| "".strdup()),
         Err(_) => {
-            ffi_context.warning("dc_get_config(): invalid key");
+            ffi_context.error("dc_get_config(): invalid key");
             "".strdup()
         }
     }
@@ -422,7 +449,7 @@ pub unsafe extern "C" fn dc_set_config_from_qr(
 pub unsafe extern "C" fn dc_get_info(context: *mut dc_context_t) -> *mut libc::c_char {
     if context.is_null() {
         eprintln!("ignoring careless call to dc_get_info()");
-        return "".strdup();
+        return dc_strdup(ptr::null());
     }
     let ffi_context = &*context;
     let guard = ffi_context.inner.read().unwrap();
@@ -434,7 +461,7 @@ pub unsafe extern "C" fn dc_get_info(context: *mut dc_context_t) -> *mut libc::c
 }
 
 fn render_info(
-    info: BTreeMap<&'static str, String>,
+    info: HashMap<&'static str, String>,
 ) -> std::result::Result<String, std::fmt::Error> {
     let mut res = String::new();
     for (key, value) in &info {
@@ -465,6 +492,11 @@ pub unsafe extern "C" fn dc_get_oauth2_url(
         .unwrap_or_else(|_| ptr::null_mut())
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_get_version_str() -> *mut libc::c_char {
+    context::get_version_str().strdup()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_configure(context: *mut dc_context_t) {
     if context.is_null() {
@@ -472,7 +504,9 @@ pub unsafe extern "C" fn dc_configure(context: *mut dc_context_t) {
         return;
     }
     let ffi_context = &*context;
-    ffi_context.with_inner(|ctx| ctx.configure()).unwrap_or(())
+    ffi_context
+        .with_inner(|ctx| configure::configure(ctx))
+        .unwrap_or(())
 }
 
 #[no_mangle]
@@ -483,7 +517,7 @@ pub unsafe extern "C" fn dc_is_configured(context: *mut dc_context_t) -> libc::c
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| ctx.is_configured() as libc::c_int)
+        .with_inner(|ctx| configure::dc_is_configured(ctx) as libc::c_int)
         .unwrap_or(0)
 }
 
@@ -705,7 +739,7 @@ pub unsafe extern "C" fn dc_preconfigure_keypair(
             key::store_self_keypair(ctx, &keypair, key::KeyPairUse::Default)?;
             Ok(1)
         })
-        .log_err(ffi_context, "Failed to save keypair")
+        .log_warn(ffi_context, "Failed to save keypair")
         .unwrap_or(0)
 }
 
@@ -752,7 +786,7 @@ pub unsafe extern "C" fn dc_create_chat_by_msg_id(context: *mut dc_context_t, ms
     ffi_context
         .with_inner(|ctx| {
             chat::create_by_msg_id(ctx, MsgId::new(msg_id))
-                .log_err(ffi_context, "Failed to create chat from msg_id")
+                .log_err(ctx, "Failed to create chat from msg_id")
                 .map(|id| id.to_u32())
                 .unwrap_or(0)
         })
@@ -772,7 +806,7 @@ pub unsafe extern "C" fn dc_create_chat_by_contact_id(
     ffi_context
         .with_inner(|ctx| {
             chat::create_by_contact_id(ctx, contact_id)
-                .log_err(ffi_context, "Failed to create chat from contact_id")
+                .log_err(ctx, "Failed to create chat from contact_id")
                 .map(|id| id.to_u32())
                 .unwrap_or(0)
         })
@@ -792,7 +826,7 @@ pub unsafe extern "C" fn dc_get_chat_id_by_contact_id(
     ffi_context
         .with_inner(|ctx| {
             chat::get_by_contact_id(ctx, contact_id)
-                .log_err(ffi_context, "Failed to get chat for contact_id")
+                .log_err(ctx, "Failed to get chat for contact_id")
                 .map(|id| id.to_u32())
                 .unwrap_or(0)
         })
@@ -1029,25 +1063,6 @@ pub unsafe extern "C" fn dc_get_fresh_msg_cnt(
         .unwrap_or(0)
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_estimate_deletion_cnt(
-    context: *mut dc_context_t,
-    from_server: libc::c_int,
-    seconds: i64,
-) -> libc::c_int {
-    if context.is_null() || seconds < 0 {
-        eprintln!("ignoring careless call to dc_estimate_deletion_cnt()");
-        return 0;
-    }
-    let ffi_context = &*context;
-    ffi_context
-        .with_inner(|ctx| {
-            message::estimate_deletion_cnt(ctx, from_server != 0, seconds).unwrap_or(0)
-                as libc::c_int
-        })
-        .unwrap_or(0)
-}
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_get_fresh_msgs(
     context: *mut dc_context_t,
@@ -1080,7 +1095,7 @@ pub unsafe extern "C" fn dc_marknoticed_chat(context: *mut dc_context_t, chat_id
     ffi_context
         .with_inner(|ctx| {
             chat::marknoticed_chat(ctx, ChatId::new(chat_id))
-                .log_err(ffi_context, "Failed marknoticed chat")
+                .log_err(ctx, "Failed marknoticed chat")
                 .unwrap_or(())
         })
         .unwrap_or(())
@@ -1096,7 +1111,7 @@ pub unsafe extern "C" fn dc_marknoticed_all_chats(context: *mut dc_context_t) {
     ffi_context
         .with_inner(|ctx| {
             chat::marknoticed_all_chats(ctx)
-                .log_err(ffi_context, "Failed marknoticed all chats")
+                .log_err(ctx, "Failed marknoticed all chats")
                 .unwrap_or(())
         })
         .unwrap_or(())
@@ -1189,32 +1204,28 @@ pub unsafe extern "C" fn dc_get_next_media(
 }
 
 #[no_mangle]
-pub unsafe extern "C" fn dc_set_chat_visibility(
+pub unsafe extern "C" fn dc_archive_chat(
     context: *mut dc_context_t,
     chat_id: u32,
     archive: libc::c_int,
 ) {
     if context.is_null() {
-        eprintln!("ignoring careless call to dc_set_chat_visibility()");
+        eprintln!("ignoring careless call to dc_archive_chat()");
         return;
     }
     let ffi_context = &*context;
-    let visibility = match archive {
-        0 => ChatVisibility::Normal,
-        1 => ChatVisibility::Archived,
-        2 => ChatVisibility::Pinned,
-        _ => {
-            ffi_context.warning(
-                "ignoring careless call to dc_set_chat_visibility(): unknown archived state",
-            );
-            return;
-        }
+    let archive = if archive == 0 {
+        false
+    } else if archive == 1 {
+        true
+    } else {
+        return;
     };
     ffi_context
         .with_inner(|ctx| {
             ChatId::new(chat_id)
-                .set_visibility(ctx, visibility)
-                .log_err(ffi_context, "Failed setting chat visibility")
+                .set_archived(ctx, archive)
+                .log_err(ctx, "Failed archive chat")
                 .unwrap_or(())
         })
         .unwrap_or(())
@@ -1231,7 +1242,7 @@ pub unsafe extern "C" fn dc_delete_chat(context: *mut dc_context_t, chat_id: u32
         .with_inner(|ctx| {
             ChatId::new(chat_id)
                 .delete(ctx)
-                .log_err(ffi_context, "Failed chat delete")
+                .log_err(ctx, "Failed chat delete")
                 .unwrap_or(())
         })
         .unwrap_or(())
@@ -1318,7 +1329,7 @@ pub unsafe extern "C" fn dc_create_group_chat(
     ffi_context
         .with_inner(|ctx| {
             chat::create_group_chat(ctx, verified, to_string_lossy(name))
-                .log_err(ffi_context, "Failed to create group chat")
+                .log_err(ctx, "Failed to create group chat")
                 .map(|id| id.to_u32())
                 .unwrap_or(0)
         })
@@ -1458,7 +1469,7 @@ pub unsafe extern "C" fn dc_get_msg_info(
 ) -> *mut libc::c_char {
     if context.is_null() {
         eprintln!("ignoring careless call to dc_get_msg_info()");
-        return "".strdup();
+        return dc_strdup(ptr::null());
     }
     let ffi_context = &*context;
     ffi_context
@@ -1641,9 +1652,7 @@ pub unsafe extern "C" fn dc_lookup_contact_id_by_addr(
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| {
-            Contact::lookup_id_by_addr(ctx, to_string_lossy(addr), Origin::IncomingReplyTo)
-        })
+        .with_inner(|ctx| Contact::lookup_id_by_addr(ctx, to_string_lossy(addr)))
         .unwrap_or(0)
 }
 
@@ -2059,9 +2068,7 @@ pub unsafe extern "C" fn dc_delete_all_locations(context: *mut dc_context_t) {
     }
     let ffi_context = &*context;
     ffi_context
-        .with_inner(|ctx| {
-            location::delete_all(ctx).log_err(ffi_context, "Failed to delete locations")
-        })
+        .with_inner(|ctx| location::delete_all(ctx).log_err(ctx, "Failed to delete locations"))
         .ok();
 }
 
@@ -2080,6 +2087,16 @@ pub unsafe extern "C" fn dc_array_unref(a: *mut dc_array::dc_array_t) {
     Box::from_raw(a);
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_array_add_id(array: *mut dc_array_t, item: libc::c_uint) {
+    if array.is_null() {
+        eprintln!("ignoring careless call to dc_array_add_id()");
+        return;
+    }
+
+    (*array).add_id(item);
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_array_get_cnt(array: *const dc_array_t) -> libc::size_t {
     if array.is_null() {
@@ -2401,12 +2418,25 @@ pub unsafe extern "C" fn dc_chat_get_type(chat: *mut dc_chat_t) -> libc::c_int {
 pub unsafe extern "C" fn dc_chat_get_name(chat: *mut dc_chat_t) -> *mut libc::c_char {
     if chat.is_null() {
         eprintln!("ignoring careless call to dc_chat_get_name()");
-        return "".strdup();
+        return dc_strdup(ptr::null());
     }
     let ffi_chat = &*chat;
     ffi_chat.chat.get_name().strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_chat_get_subtitle(chat: *mut dc_chat_t) -> *mut libc::c_char {
+    if chat.is_null() {
+        eprintln!("ignoring careless call to dc_chat_get_subtitle()");
+        return "".strdup();
+    }
+    let ffi_chat = &*chat;
+    let ffi_context: &ContextWrapper = &*ffi_chat.context;
+    ffi_context
+        .with_inner(|ctx| ffi_chat.chat.get_subtitle(ctx).strdup())
+        .unwrap_or_else(|_| "".strdup())
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_chat_get_profile_image(chat: *mut dc_chat_t) -> *mut libc::c_char {
     if chat.is_null() {
@@ -2437,17 +2467,13 @@ pub unsafe extern "C" fn dc_chat_get_color(chat: *mut dc_chat_t) -> u32 {
 }
 
 #[no_mangle]
-pub unsafe extern "C" fn dc_chat_get_visibility(chat: *mut dc_chat_t) -> libc::c_int {
+pub unsafe extern "C" fn dc_chat_get_archived(chat: *mut dc_chat_t) -> libc::c_int {
     if chat.is_null() {
-        eprintln!("ignoring careless call to dc_chat_get_visibility()");
+        eprintln!("ignoring careless call to dc_chat_get_archived()");
         return 0;
     }
     let ffi_chat = &*chat;
-    match ffi_chat.chat.visibility {
-        ChatVisibility::Normal => 0,
-        ChatVisibility::Archived => 1,
-        ChatVisibility::Pinned => 2,
-    }
+    ffi_chat.chat.is_archived() as libc::c_int
 }
 
 #[no_mangle]
@@ -2710,7 +2736,7 @@ pub unsafe extern "C" fn dc_msg_get_sort_timestamp(msg: *mut dc_msg_t) -> i64 {
 pub unsafe extern "C" fn dc_msg_get_text(msg: *mut dc_msg_t) -> *mut libc::c_char {
     if msg.is_null() {
         eprintln!("ignoring careless call to dc_msg_get_text()");
-        return "".strdup();
+        return dc_strdup(ptr::null());
     }
     let ffi_msg = &*msg;
     ffi_msg.message.get_text().unwrap_or_default().strdup()
@@ -2729,7 +2755,8 @@ pub unsafe extern "C" fn dc_msg_get_file(msg: *mut dc_msg_t) -> *mut libc::c_cha
             ffi_msg
                 .message
                 .get_file(ctx)
-                .map(|p| p.strdup())
+                .and_then(|p| p.to_c_string().ok())
+                .map(|cs| dc_strdup(cs.as_ptr()))
                 .unwrap_or_else(|| "".strdup())
         })
         .unwrap_or_else(|_| "".strdup())
@@ -2739,7 +2766,7 @@ pub unsafe extern "C" fn dc_msg_get_file(msg: *mut dc_msg_t) -> *mut libc::c_cha
 pub unsafe extern "C" fn dc_msg_get_filename(msg: *mut dc_msg_t) -> *mut libc::c_char {
     if msg.is_null() {
         eprintln!("ignoring careless call to dc_msg_get_filename()");
-        return "".strdup();
+        return dc_strdup(ptr::null());
     }
     let ffi_msg = &*msg;
     ffi_msg.message.get_filename().unwrap_or_default().strdup()
@@ -2749,13 +2776,13 @@ pub unsafe extern "C" fn dc_msg_get_filename(msg: *mut dc_msg_t) -> *mut libc::c
 pub unsafe extern "C" fn dc_msg_get_filemime(msg: *mut dc_msg_t) -> *mut libc::c_char {
     if msg.is_null() {
         eprintln!("ignoring careless call to dc_msg_get_filemime()");
-        return "".strdup();
+        return dc_strdup(ptr::null());
     }
     let ffi_msg = &*msg;
     if let Some(x) = ffi_msg.message.get_filemime() {
         x.strdup()
     } else {
-        "".strdup()
+        dc_strdup(ptr::null())
     }
 }
 
@@ -3079,7 +3106,7 @@ pub unsafe extern "C" fn dc_contact_get_id(contact: *mut dc_contact_t) -> u32 {
 pub unsafe extern "C" fn dc_contact_get_addr(contact: *mut dc_contact_t) -> *mut libc::c_char {
     if contact.is_null() {
         eprintln!("ignoring careless call to dc_contact_get_addr()");
-        return "".strdup();
+        return dc_strdup(ptr::null());
     }
     let ffi_contact = &*contact;
     ffi_contact.contact.get_addr().strdup()
@@ -3089,7 +3116,7 @@ pub unsafe extern "C" fn dc_contact_get_addr(contact: *mut dc_contact_t) -> *mut
 pub unsafe extern "C" fn dc_contact_get_name(contact: *mut dc_contact_t) -> *mut libc::c_char {
     if contact.is_null() {
         eprintln!("ignoring careless call to dc_contact_get_name()");
-        return "".strdup();
+        return dc_strdup(ptr::null());
     }
     let ffi_contact = &*contact;
     ffi_contact.contact.get_name().strdup()
@@ -3101,7 +3128,7 @@ pub unsafe extern "C" fn dc_contact_get_display_name(
 ) -> *mut libc::c_char {
     if contact.is_null() {
         eprintln!("ignoring careless call to dc_contact_get_display_name()");
-        return "".strdup();
+        return dc_strdup(ptr::null());
     }
     let ffi_contact = &*contact;
     ffi_contact.contact.get_display_name().strdup()
@@ -3113,7 +3140,7 @@ pub unsafe extern "C" fn dc_contact_get_name_n_addr(
 ) -> *mut libc::c_char {
     if contact.is_null() {
         eprintln!("ignoring careless call to dc_contact_get_name_n_addr()");
-        return "".strdup();
+        return dc_strdup(ptr::null());
     }
     let ffi_contact = &*contact;
     ffi_contact.contact.get_name_n_addr().strdup()
@@ -3125,7 +3152,7 @@ pub unsafe extern "C" fn dc_contact_get_first_name(
 ) -> *mut libc::c_char {
     if contact.is_null() {
         eprintln!("ignoring careless call to dc_contact_get_first_name()");
-        return "".strdup();
+        return dc_strdup(ptr::null());
     }
     let ffi_contact = &*contact;
     ffi_contact.contact.get_first_name().strdup()
@@ -3208,7 +3235,7 @@ pub unsafe extern "C" fn dc_lot_get_text1(lot: *mut dc_lot_t) -> *mut libc::c_ch
     }
 
     let lot = &*lot;
-    lot.get_text1().strdup()
+    strdup_opt(lot.get_text1())
 }
 
 #[no_mangle]
@@ -3219,7 +3246,7 @@ pub unsafe extern "C" fn dc_lot_get_text2(lot: *mut dc_lot_t) -> *mut libc::c_ch
     }
 
     let lot = &*lot;
-    lot.get_text2().strdup()
+    strdup_opt(lot.get_text2())
 }
 
 #[no_mangle]
@@ -3271,16 +3298,21 @@ pub unsafe extern "C" fn dc_str_unref(s: *mut libc::c_char) {
     libc::free(s as *mut _)
 }
 
-trait ResultExt<T, E> {
+pub trait ResultExt<T, E> {
     fn unwrap_or_log_default(self, context: &context::Context, message: &str) -> T;
+    fn log_err(self, context: &context::Context, message: &str) -> Result<T, E>;
 
     /// Log a warning to a [ContextWrapper] for an [Err] result.
     ///
-    /// Does nothing for an [Ok].
+    /// Does nothing for an [Ok].  This is usually preferable over
+    /// [ResultExt::log_err] because warnings go to the logfile and
+    /// errors are displayed directly to the user.  Usually problems
+    /// on the FFI layer are coding errors and not errors which need
+    /// to be displayed to the user.
     ///
     /// You can do this as soon as the wrapper exists, it does not
     /// have to be open (which is required for the `warn!()` macro).
-    fn log_err(self, wrapper: &ContextWrapper, message: &str) -> Result<T, E>;
+    fn log_warn(self, wrapper: &ContextWrapper, message: &str) -> Result<T, E>;
 }
 
 impl<T: Default, E: std::fmt::Display> ResultExt<T, E> for Result<T, E> {
@@ -3294,7 +3326,14 @@ impl<T: Default, E: std::fmt::Display> ResultExt<T, E> for Result<T, E> {
         }
     }
 
-    fn log_err(self, wrapper: &ContextWrapper, message: &str) -> Result<T, E> {
+    fn log_err(self, context: &context::Context, message: &str) -> Result<T, E> {
+        self.map_err(|err| {
+            warn!(context, "{}: {}", message, err);
+            err
+        })
+    }
+
+    fn log_warn(self, wrapper: &ContextWrapper, message: &str) -> Result<T, E> {
         self.map_err(|err| {
             unsafe {
                 wrapper.warning(&format!("{}: {}", message, err));
@@ -3304,7 +3343,14 @@ impl<T: Default, E: std::fmt::Display> ResultExt<T, E> for Result<T, E> {
     }
 }
 
-trait ResultNullableExt<T> {
+unsafe fn strdup_opt(s: Option<impl AsRef<str>>) -> *mut libc::c_char {
+    match s {
+        Some(s) => s.as_ref().strdup(),
+        None => ptr::null_mut(),
+    }
+}
+
+pub trait ResultNullableExt<T> {
     fn into_raw(self) -> *mut T;
 }
 

deltachat-ffi/src/string.rs

@@ -1,5 +1,5 @@
+use failure::Fail;
 use std::ffi::{CStr, CString};
-use std::ptr;
 
 /// Duplicates a string
 ///
@@ -8,7 +8,7 @@ use std::ptr;
 /// # Examples
 ///
 /// ```rust,norun
-/// use crate::string::{dc_strdup, to_string_lossy};
+/// use deltachat::dc_tools::{dc_strdup, to_string_lossy};
 /// unsafe {
 ///     let str_a = b"foobar\x00" as *const u8 as *const libc::c_char;
 ///     let str_a_copy = dc_strdup(str_a);
@@ -16,7 +16,7 @@ use std::ptr;
 ///     assert_ne!(str_a, str_a_copy);
 /// }
 /// ```
-unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char {
+pub unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char {
     let ret: *mut libc::c_char;
     if !s.is_null() {
         ret = libc::strdup(s);
@@ -30,13 +30,13 @@ unsafe fn dc_strdup(s: *const libc::c_char) -> *mut libc::c_char {
 }
 
 /// Error type for the [OsStrExt] trait
-#[derive(Debug, PartialEq, thiserror::Error)]
-pub(crate) enum CStringError {
+#[derive(Debug, Fail, PartialEq)]
+pub enum CStringError {
     /// The string contains an interior null byte
-    #[error("String contains an interior null byte")]
+    #[fail(display = "String contains an interior null byte")]
     InteriorNullByte,
     /// The string is not valid Unicode
-    #[error("String is not valid unicode")]
+    #[fail(display = "String is not valid unicode")]
     NotUnicode,
 }
 
@@ -65,7 +65,7 @@ pub(crate) enum CStringError {
 ///     let mut c_ptr: *mut libc::c_char = dc_strdup(path_c.as_ptr());
 /// }
 /// ```
-pub(crate) trait OsStrExt {
+pub trait OsStrExt {
     /// Convert a  [std::ffi::OsStr] to an [std::ffi::CString]
     ///
     /// This is useful to convert e.g. a [std::path::Path] to
@@ -130,28 +130,27 @@ fn os_str_to_c_string_unicode(
 }
 
 /// Convenience methods/associated functions for working with [CString]
-trait CStringExt {
-    /// Create a new [CString], best effort
+///
+/// This is helps transitioning from unsafe code.
+pub trait CStringExt {
+    /// Create a new [CString], yolo style
     ///
-    /// Like the [to_string_lossy] this doesn't give up in the face of
-    /// bad input (embedded null bytes in this case) instead it does
-    /// the best it can by stripping the embedded null bytes.
-    fn new_lossy<T: Into<Vec<u8>>>(t: T) -> CString {
-        let mut s = t.into();
-        s.retain(|&c| c != 0);
-        CString::new(s).unwrap_or_default()
+    /// This unwrap the result, panicking when there are embedded NULL
+    /// bytes.
+    fn yolo<T: Into<Vec<u8>>>(t: T) -> CString {
+        CString::new(t).expect("String contains null byte, can not be CString")
     }
 }
 
 impl CStringExt for CString {}
 
-/// Convenience methods to turn strings into C strings.
+/// Convenience methods to make transitioning from raw C strings easier.
 ///
 /// To interact with (legacy) C APIs we often need to convert from
 /// Rust strings to raw C strings.  This can be clumsy to do correctly
 /// and the compiler sometimes allows it in an unsafe way.  These
 /// methods make it more succinct and help you get it right.
-pub(crate) trait Strdup {
+pub trait StrExt {
     /// Allocate a new raw C `*char` version of this string.
     ///
     /// This allocates a new raw C string which must be freed using
@@ -168,52 +167,14 @@ pub(crate) trait Strdup {
     unsafe fn strdup(&self) -> *mut libc::c_char;
 }
 
-impl<T: AsRef<str>> Strdup for T {
+impl<T: AsRef<str>> StrExt for T {
     unsafe fn strdup(&self) -> *mut libc::c_char {
-        let tmp = CString::new_lossy(self.as_ref());
+        let tmp = CString::yolo(self.as_ref());
         dc_strdup(tmp.as_ptr())
     }
 }
 
-// We can not implement for AsRef<OsStr> because we already implement
-// AsRev<str> and this conflicts.  So implement for Path directly.
-impl Strdup for std::path::Path {
-    unsafe fn strdup(&self) -> *mut libc::c_char {
-        let tmp = self.to_c_string().unwrap_or_else(|_| CString::default());
-        dc_strdup(tmp.as_ptr())
-    }
-}
-
-/// Convenience methods to turn optional strings into C strings.
-///
-/// This is the same as the [Strdup] trait but a different trait name
-/// to work around the type system not allowing to implement [Strdup]
-/// for `Option<impl Strdup>` When we already have an [Strdup] impl
-/// for `AsRef<&str>`.
-///
-/// When the [Option] is [Option::Some] this behaves just like
-/// [Strdup::strdup], when it is [Option::None] a null pointer is
-/// returned.
-pub(crate) trait OptStrdup {
-    /// Allocate a new raw C `*char` version of this string, or NULL.
-    ///
-    /// See [Strdup::strdup] for details.
-    unsafe fn strdup(&self) -> *mut libc::c_char;
-}
-
-impl<T: AsRef<str>> OptStrdup for Option<T> {
-    unsafe fn strdup(&self) -> *mut libc::c_char {
-        match self {
-            Some(s) => {
-                let tmp = CString::new_lossy(s.as_ref());
-                dc_strdup(tmp.as_ptr())
-            }
-            None => ptr::null_mut(),
-        }
-    }
-}
-
-pub(crate) fn to_string_lossy(s: *const libc::c_char) -> String {
+pub fn to_string_lossy(s: *const libc::c_char) -> String {
     if s.is_null() {
         return "".into();
     }
@@ -223,7 +184,7 @@ pub(crate) fn to_string_lossy(s: *const libc::c_char) -> String {
     cstr.to_string_lossy().to_string()
 }
 
-pub(crate) fn to_opt_string_lossy(s: *const libc::c_char) -> Option<String> {
+pub fn to_opt_string_lossy(s: *const libc::c_char) -> Option<String> {
     if s.is_null() {
         return None;
     }
@@ -244,7 +205,7 @@ pub(crate) fn to_opt_string_lossy(s: *const libc::c_char) -> Option<String> {
 ///
 /// [Path]: std::path::Path
 #[cfg(not(target_os = "windows"))]
-pub(crate) fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path {
+pub fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path {
     assert!(!s.is_null(), "cannot be used on null pointers");
     use std::os::unix::ffi::OsStrExt;
     unsafe {
@@ -256,7 +217,7 @@ pub(crate) fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path {
 
 // as_path() implementation for windows, documented above.
 #[cfg(target_os = "windows")]
-pub(crate) fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path {
+pub fn as_path<'a>(s: *const libc::c_char) -> &'a std::path::Path {
     as_path_unicode(s)
 }
 
@@ -363,14 +324,8 @@ mod tests {
     }
 
     #[test]
-    fn test_cstring_new_lossy() {
-        assert!(CString::new("hel\x00lo").is_err());
-        assert!(CString::new(String::from("hel\x00o")).is_err());
-        let r = CString::new("hello").unwrap();
-        assert_eq!(CString::new_lossy("hello"), r);
-        assert_eq!(CString::new_lossy("hel\x00lo"), r);
-        assert_eq!(CString::new_lossy(String::from("hello")), r);
-        assert_eq!(CString::new_lossy(String::from("hel\x00lo")), r);
+    fn test_cstring_yolo() {
+        assert_eq!(CString::new("hello").unwrap(), CString::yolo("hello"));
     }
 
     #[test]
@@ -392,19 +347,4 @@ mod tests {
             assert_eq!(cmp, 0);
         }
     }
-
-    #[test]
-    fn test_strdup_opt_string() {
-        unsafe {
-            let s = Some("hello");
-            let c = s.strdup();
-            let cmp = strcmp(c, b"hello\x00" as *const u8 as *const libc::c_char);
-            free(c as *mut libc::c_void);
-            assert_eq!(cmp, 0);
-
-            let s: Option<&str> = None;
-            let c = s.strdup();
-            assert_eq!(c, ptr::null_mut());
-        }
-    }
 }

deltachat_derive/src/lib.rs

@@ -3,6 +3,7 @@ extern crate proc_macro;
 
 use crate::proc_macro::TokenStream;
 use quote::quote;
+use syn;
 
 // For now, assume (not check) that these macroses are applied to enum without
 // data.  If this assumption is violated, compiler error will point to

draft/group-sync.rst

@@ -1,126 +0,0 @@
-
-Problem: missing eventual group consistency
---------------------------------------------
-
-If group members are concurrently adding new members,
-the new members will miss each other's additions, example:
-
-- Alice and Bob are in a two-member group
-
-- Alice adds Carol, concurrently Bob adds Doris
-
-- Carol will see a three-member group (Alice, Bob, Carol),
-  Doris will see a different three-member group (Alice, Bob, Doris),
-  and only Alice and Bob will have all four members.
-
-Note that for verified groups any mitigation mechanism likely
-needs to make all clients to know who originally added a member.
-
-
-solution: memorize+attach (possible encrypted) chat-meta mime messages
-----------------------------------------------------------------------
-
-For reference, please see https://github.com/deltachat/deltachat-core-rust/blob/master/spec.md#add-and-remove-members how MemberAdded/Removed messages are shaped.
-
-
-- All Chat-Group-Member-Added/Removed messages are recorded in their
-  full raw (signed and encrypted) mime-format in the DB
-
-- If an incoming member-add/member-delete messages has a member list
-  which is, apart from the added/removed member, not consistent
-  with our own view, broadcast a "Chat-Group-Member-Correction" message to
-  all members, attaching the original added/removed mime-message for all mismatching
-  contacts.  If we have no relevant add/del information, don't send a
-  correction message out.
-
-- Upong receiving added/removed attachments we don't do the
-  check_consistency+correction message dance.
-  This avoids recursion problems and hard-to-reason-about chatter.
-
-Notes:
-
-- mechanism works for both encrypted and unencrypted add/del messages
-
-- we already have a "mime_headers" column in the DB for each incoming message.
-  We could extend it to also include the payload and store mime unconditionally
-  for member-added/removed messages.
-
-- multiple member-added/removed messages can be attached in a single
-  correction message
-
-- it is minimal on the number of overall messages to reach group consistency
-  (best-case: no extra messages, the ABCD case above: max two extra messages)
-
-- somewhat backward compatible: older clients will probably ignore
-  messages which are signed by someone who is not the outer From-address.
-
-- the correction-protocol also helps with dropped messages.  If a member
-  did not see a member-added/removed message, the next member add/removed
-  message in the group will likely heal group consistency for this member.
-
-- we can quite easily extend the mechanism to also provide the group-avatar or
-  other meta-information.
-
-Discussions of variants
-++++++++++++++++++++++++
-
-- instead of acting on MemberAdded/Removed message we could send
-  corrections for any received message that addresses inconsistent group members but
-  a) this would delay group-membership healing
-  b) could lead to a lot of members sending corrections
-
-- instead of broadcasting correction messages we could only send it to
-  the sender of the inconsistent member-added/removed message.
-  A receiver of such a correction message would then need to forward
-  the message to the members it thinks also have an inconsistent view.
-  This sounds complicated and error-prone.  Concretely, if Alice
-  receives Bob's "Member-added: Doris" message, then Alice
-  broadcasting the correction message with "Member-added: Carol"
-  would reach all four members, healing group consistency in one step.
-  If Bob meanwhile receives Alice's "Member-Added: Carol" message,
-  Bob would broadcast a correction message to all four members as well.
-  (Imagine a situation where Alice/Bob added Carol/Doris
-  while both being in an offline or bad-connection situation).
-
-
-solution2: repeat member-added/removed messages
----------------------------------------------------
-
-Introduce a new Chat-Group-Member-Changed header and deprecate Chat-Group-Member-Added/Removed
-but keep sending out the old headers until the new protocol is sufficiently deployed.
-
-The new Chat-Group-Member-Changed header contains a Time-to-Live number (TTL)
-which controls repetition of the signed "add/del e-mail address" payload.
-
-Example::
-
-    Chat-Group-Member-Changed: TTL add "somedisplayname" someone@example.org
-        owEBYQGe/pANAwACAY47A6J5t3LWAcsxYgBeTQypYWRkICJzb21lZGlzcGxheW5h
-        bWUiIHNvbWVvbmVAZXhhbXBsZS5vcmcgCokBHAQAAQIABgUCXk0MqQAKCRCOOwOi
-        ebdy1hfRB/wJ74tgFQulicthcv9n+ZsqzwOtBKMEVIHqJCzzDB/Hg/2z8ogYoZNR
-        iUKKrv3Y1XuFvdKyOC+wC/unXAWKFHYzY6Tv6qDp6r+amt+ad+8Z02q53h9E55IP
-        FUBdq2rbS8hLGjQB+mVRowYrUACrOqGgNbXMZjQfuV7fSc7y813OsCQgi3tjstup
-        b+uduVzxCp3PChGhcZPs3iOGCnQvSB8VAaLGMWE2d7nTo/yMQ0Jx69x5qwfXogTk
-        mTt5rOJyrosbtf09TMKFzGgtqBcEqHLp3+mQpZQ+WHUKAbsRa8Jc9DOUOSKJ8SNM
-        clKdskprY+4LY0EBwLD3SQ7dPkTITCRD
-        =P6GG
-
-TTL is set to "2" on an initial Chat-Group-Member-Changed add/del message.
-Receivers will apply the add/del change to the group-membership,
-decrease the TTL by 1, and if TTL>0 re-sent the header.
-
-The "add|del e-mail address" payload is pgp-signed and repeated verbatim.
-This allows to propagate, in a cryptographically secured way,
-who added a member. This is particularly important for allowing
-to show in verified groups who added a member (planned).
-
-Disadvantage to solution 1:
-
-- requires to specify encoding and precise rules for what/how is signed.
-
-- causes O(N^2) extra messages
-
-- Not easily extendable for other things (without introducing a new
-  header / encoding)
-
-

examples/repl/cmdline.rs

@@ -1,8 +1,7 @@
 use std::path::Path;
 use std::str::FromStr;
 
-use anyhow::{bail, ensure};
-use deltachat::chat::{self, Chat, ChatId, ChatVisibility};
+use deltachat::chat::{self, Chat, ChatId};
 use deltachat::chatlist::*;
 use deltachat::constants::*;
 use deltachat::contact::*;
@@ -92,10 +91,10 @@ fn dc_reset_tables(context: &Context, bits: i32) -> i32 {
     1
 }
 
-fn dc_poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<(), anyhow::Error> {
+fn dc_poke_eml_file(context: &Context, filename: impl AsRef<Path>) -> Result<(), Error> {
     let data = dc_read_file(context, filename)?;
 
-    if let Err(err) = dc_receive_imf(context, &data, "import", 0, false) {
+    if let Err(err) = dc_receive_imf(context, &data, "import", 0, 0) {
         println!("dc_receive_imf errored: {:?}", err);
     }
     Ok(())
@@ -298,7 +297,7 @@ fn chat_prefix(chat: &Chat) -> &'static str {
     chat.typ.into()
 }
 
-pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
+pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), failure::Error> {
     let chat_id = *context.cmdline_sel_chat_id.read().unwrap();
     let mut sel_chat = if !chat_id.is_unset() {
         Chat::load_from_db(context, chat_id).ok()
@@ -372,8 +371,6 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                  listmedia\n\
                  archive <chat-id>\n\
                  unarchive <chat-id>\n\
-                 pin <chat-id>\n\
-                 unpin <chat-id>\n\
                  delchat <chat-id>\n\
                  ===========================Message commands==\n\
                  listmsgs <query>\n\
@@ -398,7 +395,6 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                  providerinfo <addr>\n\
                  event <event-id to test>\n\
                  fileinfo <file>\n\
-                 estimatedeletion <seconds>\n\
                  emptyserver <flags> (1=MVBOX 2=INBOX)\n\
                  clear -- clear screen\n\
                  exit or quit\n\
@@ -515,19 +511,14 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                 for i in (0..cnt).rev() {
                     let chat = Chat::load_from_db(context, chatlist.get_chat_id(i))?;
                     println!(
-                        "{}#{}: {} [{} fresh] {}",
+                        "{}#{}: {} [{} fresh]",
                         chat_prefix(&chat),
                         chat.get_id(),
                         chat.get_name(),
                         chat.get_id().get_fresh_msg_cnt(context),
-                        match chat.visibility {
-                            ChatVisibility::Normal => "",
-                            ChatVisibility::Archived => "📦",
-                            ChatVisibility::Pinned => "📌",
-                        },
                     );
                     let lot = chatlist.get_summary(context, i, Some(&chat));
-                    let statestr = if chat.visibility == ChatVisibility::Archived {
+                    let statestr = if chat.is_archived() {
                         " [Archived]"
                     } else {
                         match lot.get_state() {
@@ -851,18 +842,10 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
             }
             print!("\n");
         }
-        "archive" | "unarchive" | "pin" | "unpin" => {
+        "archive" | "unarchive" => {
             ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
             let chat_id = ChatId::new(arg1.parse()?);
-            chat_id.set_visibility(
-                context,
-                match arg0 {
-                    "archive" => ChatVisibility::Archived,
-                    "unarchive" | "unpin" => ChatVisibility::Normal,
-                    "pin" => ChatVisibility::Pinned,
-                    _ => panic!("Unexpected command (This should never happen)"),
-                },
-            )?;
+            chat_id.set_archived(context, arg0 == "archive")?;
         }
         "delchat" => {
             ensure!(!arg1.is_empty(), "Argument <chat-id> missing.");
@@ -986,10 +969,7 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
         }
         "setqr" => {
             ensure!(!arg1.is_empty(), "Argument <qr-content> missing.");
-            match set_config_from_qr(context, arg1) {
-                Ok(()) => println!("Config set from QR code, you can now call 'configure'"),
-                Err(err) => println!("Cannot set config from QR code: {:?}", err),
-            }
+            set_config_from_qr(context, arg1);
         }
         "providerinfo" => {
             ensure!(!arg1.is_empty(), "Argument <addr> missing.");
@@ -1030,16 +1010,6 @@ pub fn dc_cmdline(context: &Context, line: &str) -> Result<(), Error> {
                 bail!("Command failed.");
             }
         }
-        "estimatedeletion" => {
-            ensure!(!arg1.is_empty(), "Argument <seconds> missing");
-            let seconds = arg1.parse()?;
-            let device_cnt = message::estimate_deletion_cnt(context, false, seconds)?;
-            let server_cnt = message::estimate_deletion_cnt(context, true, seconds)?;
-            println!(
-                "estimated count of messages older than {} seconds:\non device: {}\non server: {}",
-                seconds, device_cnt, server_cnt
-            );
-        }
         "emptyserver" => {
             ensure!(!arg1.is_empty(), "Argument <flags> missing");
 

examples/repl/main.rs

@@ -7,6 +7,8 @@
 #[macro_use]
 extern crate deltachat;
 #[macro_use]
+extern crate failure;
+#[macro_use]
 extern crate lazy_static;
 #[macro_use]
 extern crate rusqlite;
@@ -18,9 +20,9 @@ use std::process::Command;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Mutex, RwLock};
 
-use anyhow::{bail, Error};
 use deltachat::chat::ChatId;
 use deltachat::config;
+use deltachat::configure::*;
 use deltachat::context::*;
 use deltachat::job::*;
 use deltachat::oauth2::*;
@@ -261,7 +263,7 @@ const DB_COMMANDS: [&str; 11] = [
     "housekeeping",
 ];
 
-const CHAT_COMMANDS: [&str; 26] = [
+const CHAT_COMMANDS: [&str; 24] = [
     "listchats",
     "listarchived",
     "chat",
@@ -285,8 +287,6 @@ const CHAT_COMMANDS: [&str; 26] = [
     "listmedia",
     "archive",
     "unarchive",
-    "pin",
-    "unpin",
     "delchat",
 ];
 const MESSAGE_COMMANDS: [&str; 8] = [
@@ -307,17 +307,8 @@ const CONTACT_COMMANDS: [&str; 6] = [
     "delcontact",
     "cleanupcontacts",
 ];
-const MISC_COMMANDS: [&str; 10] = [
-    "getqr",
-    "getbadqr",
-    "checkqr",
-    "event",
-    "fileinfo",
-    "clear",
-    "exit",
-    "quit",
-    "help",
-    "estimatedeletion",
+const MISC_COMMANDS: [&str; 9] = [
+    "getqr", "getbadqr", "checkqr", "event", "fileinfo", "clear", "exit", "quit", "help",
 ];
 
 impl Hinter for DcHelper {
@@ -369,10 +360,10 @@ impl Highlighter for DcHelper {
 
 impl Helper for DcHelper {}
 
-fn main_0(args: Vec<String>) -> Result<(), Error> {
+fn main_0(args: Vec<String>) -> Result<(), failure::Error> {
     if args.len() < 2 {
         println!("Error: Bad arguments, expected [db-name].");
-        bail!("No db-name specified");
+        return Err(format_err!("No db-name specified"));
     }
     let context = Context::new(
         Box::new(receive_event),
@@ -442,7 +433,7 @@ enum ExitResult {
     Exit,
 }
 
-fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, Error> {
+fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, failure::Error> {
     let mut args = line.splitn(2, ' ');
     let arg0 = args.next().unwrap_or_default();
     let arg1 = args.next().unwrap_or_default();
@@ -470,7 +461,7 @@ fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, Error
         }
         "configure" => {
             start_threads(ctx.clone());
-            ctx.read().unwrap().configure();
+            configure(&ctx.read().unwrap());
         }
         "oauth2" => {
             if let Some(addr) = ctx.read().unwrap().get_config(config::Config::Addr) {
@@ -525,7 +516,7 @@ fn handle_cmd(line: &str, ctx: Arc<RwLock<Context>>) -> Result<ExitResult, Error
     Ok(ExitResult::Continue)
 }
 
-pub fn main() -> Result<(), Error> {
+pub fn main() -> Result<(), failure::Error> {
     let _ = pretty_env_logger::try_init();
 
     let args: Vec<String> = std::env::args().collect();

examples/simple.rs

@@ -7,6 +7,7 @@ use tempfile::tempdir;
 use deltachat::chat;
 use deltachat::chatlist::*;
 use deltachat::config;
+use deltachat::configure::*;
 use deltachat::contact::*;
 use deltachat::context::*;
 use deltachat::job::{
@@ -76,7 +77,7 @@ fn main() {
     ctx.set_config(config::Config::Addr, Some("d@testrun.org"))
         .unwrap();
     ctx.set_config(config::Config::MailPw, Some(&pw)).unwrap();
-    ctx.configure();
+    configure(&ctx);
 
     thread::sleep(duration);
 

python/CHANGELOG

@@ -1,22 +1,3 @@
-0.900.0 (DRAFT)
----------------
-
-- refactored internals to use plugin-approach 
-
-- introduced PerAccount and Global hooks that plugins can implement 
-
-- introduced `ac_member_added()` and `ac_member_removed()` plugin events. 
-
-- introduced two documented examples for an echo and a group-membership
-  tracking plugin. 
-
-0.800.0
--------
-
-- use latest core 1.25.0 
-
-- refine tests and some internal changes to core bindings
-
 0.700.0
 ---------
 

python/doc/_static/custom.css

@@ -15,7 +15,3 @@ div.globaltoc {
 img.logo {
     height: 120px;
 }
-
-div.footer {
-    display: none;
-}

python/doc/api.rst

@@ -2,6 +2,10 @@
 high level API reference
 ========================
 
+.. note::
+
+    This API is work in progress and may change in versions prior to 1.0.
+
 - :class:`deltachat.account.Account` (your main entry point, creates the
   other classes)
 - :class:`deltachat.contact.Contact`

python/doc/capi.rst

@@ -0,0 +1,7 @@
+
+C deltachat interface
+=====================
+
+See :doc:`lapi` for accessing many of the below functions
+through the ``deltachat.capi.lib`` namespace.
+

python/doc/conf.py

@@ -55,7 +55,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'deltachat'
-copyright = u'2020, holger krekel and contributors'
+copyright = u'2018, holger krekel and contributors'
 
 
 # The language for content autogenerated by Sphinx. Refer to documentation

python/doc/examples.rst

@@ -1,60 +1,37 @@
 
+
 examples
 ========
 
+
+Playing around on the commandline
+----------------------------------
+
 Once you have :doc:`installed deltachat bindings <install>`
-you need email/password credentials for an IMAP/SMTP account.
-Delta Chat developers and the CI system use a special URL to create
-temporary e-mail accounts on [testrun.org](https://testrun.org) for testing.
+you can start playing from the python interpreter commandline.
+For example you can type ``python`` and then::
 
-Receiving a Chat message from the command line
-----------------------------------------------
+    # instantiate and configure deltachat account
+    import deltachat
+    ac = deltachat.Account("/tmp/db")
 
-Here is a simple bot that:
+    # start configuration activity and smtp/imap threads
+    ac.start_threads()
+    ac.configure(addr="test2@hq5.merlinux.eu", mail_pw="********")
 
-- receives a message and sends back ("echoes") a message
+    # create a contact and send a message
+    contact = ac.create_contact("someother@email.address")
+    chat = ac.create_chat_by_contact(contact)
+    chat.send_text("hi from the python interpreter command line")
 
-- terminates the bot if the message `/quit` is sent
+Checkout our :doc:`api` for the various high-level things you can do
+to send/receive messages, create contacts and chats.
 
-.. include:: ../examples/echo_and_quit.py
-    :literal:
 
-With this file in your working directory you can run the bot
-by specifying a database path, an e-mail address and password of
-a SMTP-IMAP account::
-
-    $ cd examples
-    $ python echo_and_quit.py /tmp/db --email ADDRESS --password PASSWORD
-
-While this process is running you can start sending chat messages
-to `ADDRESS`.
-
-Track member additions and removals in a group
-----------------------------------------------
-
-Here is a simple bot that:
-
-- echoes messages sent to it
-
-- tracks if configuration completed
-
-- tracks member additions and removals for all chat groups
-
-.. include:: ../examples/group_tracking.py
-    :literal:
-
-With this file in your working directory you can run the bot
-by specifying a database path, an e-mail address and password of
-a SMTP-IMAP account::
-
-    python group_tracking.py --email ADDRESS --password PASSWORD /tmp/db
-
-When this process is running you can start sending chat messages
-to `ADDRESS`.
-
-Writing bots for real
+Looking at a real example
 -------------------------
 
 The `deltabot repository <https://github.com/deltachat/deltabot#deltachat-example-bot>`_
-contains a little framework for writing deltachat bots in Python.
+contains a real-life example of Python bindings usage.
+
 

python/doc/index.rst

@@ -1,15 +1,15 @@
 deltachat python bindings
 =========================
 
-The ``deltachat`` Python package provides two layers of bindings for the
-core Rust-library of the https://delta.chat messaging ecosystem:
+The ``deltachat`` Python package provides two bindings for the core Rust-library
+of the https://delta.chat messaging ecosystem:
 
-- :doc:`api` is a high level interface to deltachat-core.
+- :doc:`api` is a high level interface to deltachat-core which aims
+  to be memory safe and thoroughly tested through continous tox/pytest runs.
 
-- :doc:`plugins` is a brief introduction into implementing plugin hooks.
-
-- :doc:`lapi` is a lowlevel CFFI-binding to the `Rust Core
-  <https://github.com/deltachat/deltachat-core-rust>`_.
+- :doc:`capi` is a lowlevel CFFI-binding to the previous
+  `deltachat-core C-API <https://c.delta.chat>`_ (so far the Rust library
+  replicates exactly the same C-level API).
 
 
 
@@ -28,8 +28,8 @@ getting started
    links
    changelog
    api
+   capi
    lapi
-   plugins
 
 ..
     Indices and tables

python/doc/plugins.rst

@@ -1,38 +0,0 @@
-
-Implementing Plugin Hooks
-==========================
-
-The Delta Chat Python bindings use `pluggy <https://pluggy.readthedocs.io>`_
-for managing global and per-account plugin registration, and performing
-hook calls. There are two kinds of plugins:
-
-- Global plugins that are active for all accounts; they can implement
-  hooks at account-creation and account-shutdown time.
-
-- Account plugins that are only active during the lifetime of a
-  single Account instance.
-
-
-Registering a plugin
---------------------
-
-.. autofunction:: deltachat.register_global_plugin
-    :noindex:
-
-.. automethod:: deltachat.account.Account.add_account_plugin
-    :noindex:
-
-
-Per-Account Hook specifications
--------------------------------
-
-.. autoclass:: deltachat.hookspec.PerAccount
-    :members:
-
-
-Global Hook specifications
---------------------------
-
-.. autoclass:: deltachat.hookspec.Global
-    :members:
-

python/examples/echo_and_quit.py

@@ -1,30 +0,0 @@
-
-# content of echo_and_quit.py
-
-from deltachat import account_hookimpl, run_cmdline
-
-
-class EchoPlugin:
-    @account_hookimpl
-    def ac_incoming_message(self, message):
-        print("process_incoming message", message)
-        if message.text.strip() == "/quit":
-            message.account.shutdown()
-        else:
-            # unconditionally accept the chat
-            message.accept_sender_contact()
-            addr = message.get_sender_contact().addr
-            text = message.text
-            message.chat.send_text("echoing from {}:\n{}".format(addr, text))
-
-    @account_hookimpl
-    def ac_message_delivered(self, message):
-        print("ac_message_delivered", message)
-
-
-def main(argv=None):
-    run_cmdline(argv=argv, account_plugins=[EchoPlugin()])
-
-
-if __name__ == "__main__":
-    main()

python/examples/group_tracking.py

@@ -1,52 +0,0 @@
-
-# content of group_tracking.py
-
-from deltachat import account_hookimpl, run_cmdline
-
-
-class GroupTrackingPlugin:
-    @account_hookimpl
-    def ac_incoming_message(self, message):
-        print("process_incoming message", message)
-        if message.text.strip() == "/quit":
-            message.account.shutdown()
-        else:
-            # unconditionally accept the chat
-            message.accept_sender_contact()
-            addr = message.get_sender_contact().addr
-            text = message.text
-            message.chat.send_text("echoing from {}:\n{}".format(addr, text))
-
-    @account_hookimpl
-    def ac_outgoing_message(self, message):
-        print("ac_outgoing_message:", message)
-
-    @account_hookimpl
-    def ac_configure_completed(self, success):
-        print("ac_configure_completed:", success)
-
-    @account_hookimpl
-    def ac_chat_modified(self, chat):
-        print("ac_chat_modified:", chat.id, chat.get_name())
-        for member in chat.get_contacts():
-            print("chat member: {}".format(member.addr))
-
-    @account_hookimpl
-    def ac_member_added(self, chat, contact, message):
-        print("ac_member_added {} to chat {} from {}".format(
-            contact.addr, chat.id, message.get_sender_contact().addr))
-        for member in chat.get_contacts():
-            print("chat member: {}".format(member.addr))
-
-    @account_hookimpl
-    def ac_member_removed(self, chat, contact, message):
-        print("ac_member_removed {} from chat {} by {}".format(
-            contact.addr, chat.id, message.get_sender_contact().addr))
-
-
-def main(argv=None):
-    run_cmdline(argv=argv, account_plugins=[GroupTrackingPlugin()])
-
-
-if __name__ == "__main__":
-    main()

python/examples/test_examples.py

@@ -1,72 +0,0 @@
-
-import pytest
-import py
-import echo_and_quit
-import group_tracking
-from deltachat.eventlogger import FFIEventLogger
-
-
-@pytest.fixture(scope='session')
-def datadir():
-    """The py.path.local object of the test-data/ directory."""
-    for path in reversed(py.path.local(__file__).parts()):
-        datadir = path.join('test-data')
-        if datadir.isdir():
-            return datadir
-    else:
-        pytest.skip('test-data directory not found')
-
-
-def test_echo_quit_plugin(acfactory):
-    botproc = acfactory.run_bot_process(echo_and_quit)
-
-    ac1 = acfactory.get_one_online_account()
-    bot_contact = ac1.create_contact(botproc.addr)
-    ch1 = ac1.create_chat_by_contact(bot_contact)
-    ch1.send_text("hello")
-    reply = ac1._evtracker.wait_next_incoming_message()
-    assert "hello" in reply.text
-    assert reply.chat == ch1
-    ch1.send_text("/quit")
-    botproc.wait()
-
-
-def test_group_tracking_plugin(acfactory, lp):
-    lp.sec("creating one group-tracking bot and two temp accounts")
-    botproc = acfactory.run_bot_process(group_tracking, ffi=False)
-
-    ac1, ac2 = acfactory.get_two_online_accounts(quiet=True)
-
-    botproc.fnmatch_lines("""
-        *ac_configure_completed*
-    """)
-    ac1.add_account_plugin(FFIEventLogger(ac1, "ac1"))
-    ac2.add_account_plugin(FFIEventLogger(ac2, "ac2"))
-
-    lp.sec("creating bot test group with bot")
-    bot_contact = ac1.create_contact(botproc.addr)
-    ch = ac1.create_group_chat("bot test group")
-    ch.add_contact(bot_contact)
-    ch.send_text("hello")
-
-    botproc.fnmatch_lines("""
-        *ac_chat_modified*bot test group*
-    """)
-
-    lp.sec("adding third member {}".format(ac2.get_config("addr")))
-    contact3 = ac1.create_contact(ac2.get_config("addr"))
-    ch.add_contact(contact3)
-
-    reply = ac1._evtracker.wait_next_incoming_message()
-    assert "hello" in reply.text
-
-    lp.sec("now looking at what the bot received")
-    botproc.fnmatch_lines("""
-        *ac_member_added {}*
-    """.format(contact3.addr))
-
-    lp.sec("contact successfully added, now removing")
-    ch.remove_contact(contact3)
-    botproc.fnmatch_lines("""
-        *ac_member_removed {}*
-    """.format(contact3.addr))

python/setup.py

@@ -18,15 +18,10 @@ def main():
         description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
         long_description=long_description,
         author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy'],
+        install_requires=['cffi>=1.0.0', 'six'],
         packages=setuptools.find_packages('src'),
         package_dir={'': 'src'},
         cffi_modules=['src/deltachat/_build.py:ffibuilder'],
-        entry_points = {
-            'pytest11': [
-                'deltachat.testplugin = deltachat.testplugin',
-            ],
-        },
         classifiers=[
             'Development Status :: 4 - Beta',
             'Intended Audience :: Developers',

python/src/deltachat/__init__.py

@@ -1,13 +1,6 @@
-import sys
-
-from . import capi, const, hookspec
-from .capi import ffi
-from .account import Account  # noqa
-from .message import Message  # noqa
-from .contact import Contact  # noqa
-from .chat import Chat        # noqa
-from .hookspec import account_hookimpl, global_hookimpl  # noqa
-from . import eventlogger
+from deltachat import capi, const
+from deltachat.capi import ffi
+from deltachat.account import Account  # noqa
 
 from pkg_resources import get_distribution, DistributionNotFound
 try:
@@ -81,62 +74,3 @@ def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}):
             if name.startswith("DC_EVENT_"):
                 _DC_EVENTNAME_MAP[val] = name
     return _DC_EVENTNAME_MAP[integer]
-
-
-def register_global_plugin(plugin):
-    """ Register a global plugin which implements one or more
-    of the :class:`deltachat.hookspec.Global` hooks.
-    """
-    gm = hookspec.Global._get_plugin_manager()
-    gm.register(plugin)
-    gm.check_pending()
-
-
-def unregister_global_plugin(plugin):
-    gm = hookspec.Global._get_plugin_manager()
-    gm.unregister(plugin)
-
-
-register_global_plugin(eventlogger)
-
-
-def run_cmdline(argv=None, account_plugins=None):
-    """ Run a simple default command line app, registering the specified
-    account plugins. """
-    import argparse
-    if argv is None:
-        argv = sys.argv
-
-    parser = argparse.ArgumentParser(prog=argv[0] if argv else None)
-    parser.add_argument("db", action="store", help="database file")
-    parser.add_argument("--show-ffi", action="store_true", help="show low level ffi events")
-    parser.add_argument("--email", action="store", help="email address")
-    parser.add_argument("--password", action="store", help="password")
-
-    args = parser.parse_args(argv[1:])
-
-    ac = Account(args.db)
-
-    if args.show_ffi:
-        log = eventlogger.FFIEventLogger(ac, "bot")
-        ac.add_account_plugin(log)
-
-    if not ac.is_configured():
-        assert args.email and args.password, (
-            "you must specify --email and --password once to configure this database/account"
-        )
-        ac.set_config("addr", args.email)
-        ac.set_config("mail_pw", args.password)
-        ac.set_config("mvbox_move", "0")
-        ac.set_config("mvbox_watch", "0")
-        ac.set_config("sentbox_watch", "0")
-
-    for plugin in account_plugins or []:
-        ac.add_account_plugin(plugin)
-
-    # start IO threads and configure if neccessary
-    ac.start()
-
-    print("{}: waiting for message".format(ac.get_config("addr")))
-
-    ac.wait_shutdown()

python/src/deltachat/account.py

@@ -2,25 +2,23 @@
 
 from __future__ import print_function
 import atexit
-from contextlib import contextmanager
-from email.utils import parseaddr
-import queue
-from threading import Event
+import threading
 import os
+import re
+import time
 from array import array
+try:
+    from queue import Queue, Empty
+except ImportError:
+    from Queue import Queue, Empty
+
 import deltachat
 from . import const
 from .capi import ffi, lib
 from .cutil import as_dc_charpointer, from_dc_charpointer, iter_array, DCLot
 from .chat import Chat
-from .message import Message, map_system_message
+from .message import Message
 from .contact import Contact
-from .tracker import ImexTracker
-from . import hookspec, iothreads
-
-
-class MissingCredentials(ValueError):
-    """ Account is missing `addr` and `mail_pw` config values. """
 
 
 class Account(object):
@@ -28,63 +26,39 @@ class Account(object):
     by the underlying deltachat core library.  All public Account methods are
     meant to be memory-safe and return memory-safe objects.
     """
-    MissingCredentials = MissingCredentials
-
-    def __init__(self, db_path, os_name=None, logging=True):
+    def __init__(self, db_path, logid=None, eventlogging=True, os_name=None, debug=True):
         """ initialize account object.
 
         :param db_path: a path to the account database. The database
                         will be created if it doesn't exist.
+        :param logid: an optional logging prefix that should be used with
+                      the default internal logging.
+        :param eventlogging: if False no eventlogging and no context callback will be configured
         :param os_name: this will be put to the X-Mailer header in outgoing messages
+        :param debug: turn on debug logging for events.
         """
-        # initialize per-account plugin system
-        self._pm = hookspec.PerAccount._make_plugin_manager()
-        self._logging = logging
-        self.add_account_plugin(self)
-
         self._dc_context = ffi.gc(
             lib.dc_context_new(lib.py_dc_callback, ffi.NULL, as_dc_charpointer(os_name)),
             _destroy_dc_context,
         )
+        if eventlogging:
+            self._evlogger = EventLogger(self._dc_context, logid, debug)
+            deltachat.set_context_callback(self._dc_context, self._process_event)
+            self._threads = IOThreads(self._dc_context, self._evlogger._log_event)
+        else:
+            self._threads = IOThreads(self._dc_context)
 
-        hook = hookspec.Global._get_plugin_manager().hook
-
-        self._threads = iothreads.IOThreads(self)
-        self._hook_event_queue = queue.Queue()
-        self._in_use_iter_events = False
-        self._shutdown_event = Event()
-
-        # open database
-        self.db_path = db_path
         if hasattr(db_path, "encode"):
             db_path = db_path.encode("utf8")
         if not lib.dc_open(self._dc_context, db_path, ffi.NULL):
             raise ValueError("Could not dc_open: {}".format(db_path))
         self._configkeys = self.get_config("sys.config_keys").split()
+        self._imex_events = Queue()
         atexit.register(self.shutdown)
-        hook.dc_account_init(account=self)
-
-    def disable_logging(self):
-        """ disable logging. """
-        self._logging = False
-
-    def enable_logging(self):
-        """ re-enable logging. """
-        self._logging = True
-
-    @hookspec.account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        for name, kwargs in self._map_ffi_event(ffi_event):
-            ev = HookEvent(self, name=name, kwargs=kwargs)
-            self._hook_event_queue.put(ev)
 
     # def __del__(self):
     #    self.shutdown()
 
-    def ac_log_line(self, msg):
-        if self._logging:
-            self._pm.hook.ac_log_line(message=msg)
-
     def _check_config_key(self, name):
         if name not in self._configkeys:
             raise KeyError("{!r} not a valid config key, existing keys: {!r}".format(
@@ -156,15 +130,16 @@ class Account(object):
         if res == 0:
             raise Exception("Failed to set key")
 
-    def update_config(self, kwargs):
-        """ update config values.
+    def configure(self, **kwargs):
+        """ set config values and configure this account.
 
         :param kwargs: name=value config settings for this account.
                        values need to be unicode.
         :returns: None
         """
-        for key, value in kwargs.items():
-            self.set_config(key, str(value))
+        for name, value in kwargs.items():
+            self.set_config(name, value)
+        lib.dc_configure(self._dc_context)
 
     def is_configured(self):
         """ determine if the account is configured already; an initial connection
@@ -172,7 +147,7 @@ class Account(object):
 
         :returns: True if account is configured.
         """
-        return bool(lib.dc_is_configured(self._dc_context))
+        return lib.dc_is_configured(self._dc_context)
 
     def set_avatar(self, img_path):
         """Set self avatar.
@@ -202,6 +177,11 @@ class Account(object):
             raise ValueError("no flags set")
         lib.dc_empty_server(self._dc_context, flags)
 
+    def get_infostring(self):
+        """ return info of the configured account. """
+        self.check_is_configured()
+        return from_dc_charpointer(lib.dc_get_info(self._dc_context))
+
     def get_latest_backupfile(self, backupdir):
         """ return the latest backup file in a given directory.
         """
@@ -223,7 +203,8 @@ class Account(object):
 
         :returns: :class:`deltachat.contact.Contact`
         """
-        return Contact(self, const.DC_CONTACT_ID_SELF)
+        self.check_is_configured()
+        return Contact(self._dc_context, const.DC_CONTACT_ID_SELF)
 
     def create_contact(self, email, name=None):
         """ create a (new) Contact. If there already is a Contact
@@ -234,14 +215,11 @@ class Account(object):
         :param name: display name for this contact (optional)
         :returns: :class:`deltachat.contact.Contact` instance.
         """
-        realname, addr = parseaddr(email)
-        if name:
-            realname = name
-        realname = as_dc_charpointer(realname)
-        addr = as_dc_charpointer(addr)
-        contact_id = lib.dc_create_contact(self._dc_context, realname, addr)
+        name = as_dc_charpointer(name)
+        email = as_dc_charpointer(email)
+        contact_id = lib.dc_create_contact(self._dc_context, name, email)
         assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
-        return Contact(self, contact_id)
+        return Contact(self._dc_context, contact_id)
 
     def delete_contact(self, contact):
         """ delete a Contact.
@@ -254,14 +232,6 @@ class Account(object):
         assert contact_id > const.DC_CHAT_ID_LAST_SPECIAL
         return bool(lib.dc_delete_contact(self._dc_context, contact_id))
 
-    def get_contact_by_addr(self, email):
-        """ get a contact for the email address or None if it's blocked or doesn't exist. """
-        _, addr = parseaddr(email)
-        addr = as_dc_charpointer(addr)
-        contact_id = lib.dc_lookup_contact_id_by_addr(self._dc_context, addr)
-        if contact_id:
-            return self.get_contact_by_id(contact_id)
-
     def get_contacts(self, query=None, with_self=False, only_verified=False):
         """ get a (filtered) list of contacts.
 
@@ -281,15 +251,7 @@ class Account(object):
             lib.dc_get_contacts(self._dc_context, flags, query),
             lib.dc_array_unref
         )
-        return list(iter_array(dc_array, lambda x: Contact(self, x)))
-
-    def get_fresh_messages(self):
-        """ yield all fresh messages from all chats. """
-        dc_array = ffi.gc(
-            lib.dc_get_fresh_msgs(self._dc_context),
-            lib.dc_array_unref
-        )
-        yield from iter_array(dc_array, lambda x: Message.from_db(self, x))
+        return list(iter_array(dc_array, lambda x: Contact(self._dc_context, x)))
 
     def create_chat_by_contact(self, contact):
         """ create or get an existing 1:1 chat object for the specified contact or contact id.
@@ -311,9 +273,6 @@ class Account(object):
         """ create or get an existing chat object for the
         the specified message.
 
-        If this message is in the deaddrop chat then
-        the sender will become an accepted contact.
-
         :param message: messsage id or message instance.
         :returns: a :class:`deltachat.chat.Chat` object.
         """
@@ -366,13 +325,6 @@ class Account(object):
         """
         return Message.from_db(self, msg_id)
 
-    def get_contact_by_id(self, contact_id):
-        """ return Contact instance or None.
-        :param contact_id: integer id of this contact.
-        :returns: None or :class:`deltachat.contact.Contact` instance.
-        """
-        return Contact(self, contact_id)
-
     def get_chat_by_id(self, chat_id):
         """ return Chat instance.
         :param chat_id: integer id of this chat.
@@ -417,37 +369,45 @@ class Account(object):
         lib.dc_delete_msgs(self._dc_context, msg_ids, len(msg_ids))
 
     def export_self_keys(self, path):
-        """ export public and private keys to the specified directory.
-
-        Note that the account does not have to be started.
-        """
+        """ export public and private keys to the specified directory. """
         return self._export(path, imex_cmd=1)
 
     def export_all(self, path):
         """return new file containing a backup of all database state
         (chats, contacts, keys, media, ...). The file is created in the
         the `path` directory.
-
-        Note that the account does not have to be started.
         """
         export_files = self._export(path, 11)
         if len(export_files) != 1:
             raise RuntimeError("found more than one new file")
         return export_files[0]
 
+    def _imex_events_clear(self):
+        try:
+            while True:
+                self._imex_events.get_nowait()
+        except Empty:
+            pass
+
     def _export(self, path, imex_cmd):
-        with self.temp_plugin(ImexTracker()) as imex_tracker:
-            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
-            if not self._threads.is_started():
-                lib.dc_perform_imap_jobs(self._dc_context)
-            return imex_tracker.wait_finish()
+        self._imex_events_clear()
+        lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
+        if not self._threads.is_started():
+            lib.dc_perform_imap_jobs(self._dc_context)
+        files_written = []
+        while True:
+            ev = self._imex_events.get()
+            if isinstance(ev, str):
+                files_written.append(ev)
+            elif isinstance(ev, bool):
+                if not ev:
+                    raise ValueError("export failed, exp-files: {}".format(files_written))
+                return files_written
 
     def import_self_keys(self, path):
         """ Import private keys found in the `path` directory.
         The last imported key is made the default keys unless its name
         contains the string legacy. Public keys are not imported.
-
-        Note that the account does not have to be started.
         """
         self._import(path, imex_cmd=2)
 
@@ -460,11 +420,12 @@ class Account(object):
         self._import(path, imex_cmd=12)
 
     def _import(self, path, imex_cmd):
-        with self.temp_plugin(ImexTracker()) as imex_tracker:
-            lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
-            if not self._threads.is_started():
-                lib.dc_perform_imap_jobs(self._dc_context)
-            imex_tracker.wait_finish()
+        self._imex_events_clear()
+        lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
+        if not self._threads.is_started():
+            lib.dc_perform_imap_jobs(self._dc_context)
+        if not self._imex_events.get():
+            raise ValueError("import from path '{}' failed".format(path))
 
     def initiate_key_transfer(self):
         """return setup code after a Autocrypt setup message
@@ -529,6 +490,63 @@ class Account(object):
             raise ValueError("could not join group")
         return Chat(self, chat_id)
 
+    def stop_ongoing(self):
+        lib.dc_stop_ongoing_process(self._dc_context)
+
+    #
+    # meta API for start/stop and event based processing
+    #
+
+    def wait_next_incoming_message(self):
+        """ wait for and return next incoming message. """
+        ev = self._evlogger.get_matching("DC_EVENT_INCOMING_MSG")
+        return self.get_message_by_id(ev[2])
+
+    def start_threads(self, mvbox=False, sentbox=False):
+        """ start IMAP/SMTP threads (and configure account if it hasn't happened).
+
+        :raises: ValueError if 'addr' or 'mail_pw' are not configured.
+        :returns: None
+        """
+        if not self.is_configured():
+            self.configure()
+        self._threads.start(mvbox=mvbox, sentbox=sentbox)
+
+    def stop_threads(self, wait=True):
+        """ stop IMAP/SMTP threads. """
+        if self._threads.is_started():
+            self.stop_ongoing()
+            self._threads.stop(wait=wait)
+
+    def shutdown(self, wait=True):
+        """ stop threads and close and remove underlying dc_context and callbacks. """
+        if hasattr(self, "_dc_context") and hasattr(self, "_threads"):
+            # print("SHUTDOWN", self)
+            self.stop_threads(wait=False)
+            lib.dc_close(self._dc_context)
+            self.stop_threads(wait=wait)  # to wait for threads
+            deltachat.clear_context_callback(self._dc_context)
+            del self._dc_context
+            atexit.unregister(self.shutdown)
+
+    def _process_event(self, ctx, evt_name, data1, data2):
+        assert ctx == self._dc_context
+        if hasattr(self, "_evlogger"):
+            self._evlogger(evt_name, data1, data2)
+            method = getattr(self, "on_" + evt_name.lower(), None)
+            if method is not None:
+                method(data1, data2)
+        return 0
+
+    def on_dc_event_imex_progress(self, data1, data2):
+        if data1 == 1000:
+            self._imex_events.put(True)
+        elif data1 == 0:
+            self._imex_events.put(False)
+
+    def on_dc_event_imex_file_written(self, data1, data2):
+        self._imex_events.put(data1)
+
     def set_location(self, latitude=0.0, longitude=0.0, accuracy=0.0):
         """set a new location. It effects all chats where we currently
         have enabled location streaming.
@@ -543,122 +561,159 @@ class Account(object):
         if dc_res == 0:
             raise ValueError("no chat is streaming locations")
 
-    #
-    # meta API for start/stop and event based processing
-    #
 
-    def add_account_plugin(self, plugin, name=None):
-        """ add an account plugin which implements one or more of
-        the :class:`deltachat.hookspec.PerAccount` hooks.
-        """
-        self._pm.register(plugin, name=name)
-        self._pm.check_pending()
-        return plugin
+class IOThreads:
+    def __init__(self, dc_context, log_event=lambda *args: None):
+        self._dc_context = dc_context
+        self._thread_quitflag = False
+        self._name2thread = {}
+        self._log_event = log_event
 
-    @contextmanager
-    def temp_plugin(self, plugin):
-        """ run a with-block with the given plugin temporarily registered. """
-        self._pm.register(plugin)
-        yield plugin
-        self._pm.unregister(plugin)
+    def is_started(self):
+        return len(self._name2thread) > 0
 
-    def stop_ongoing(self):
-        """ Stop ongoing securejoin, configuration or other core jobs. """
-        lib.dc_stop_ongoing_process(self._dc_context)
+    def start(self, imap=True, smtp=True, mvbox=False, sentbox=False):
+        assert not self.is_started()
+        if imap:
+            self._start_one_thread("inbox", self.imap_thread_run)
+        if mvbox:
+            self._start_one_thread("mvbox", self.mvbox_thread_run)
+        if sentbox:
+            self._start_one_thread("sentbox", self.sentbox_thread_run)
+        if smtp:
+            self._start_one_thread("smtp", self.smtp_thread_run)
 
-    def start(self, callback_thread=True):
-        """ start this account (activate imap/smtp threads etc.)
-        and return immediately.
+    def _start_one_thread(self, name, func):
+        self._name2thread[name] = t = threading.Thread(target=func, name=name)
+        t.setDaemon(1)
+        t.start()
 
-        If this account is not configured, an internal configuration
-        job will be scheduled if config values are sufficiently specified.
+    def stop(self, wait=False):
+        self._thread_quitflag = True
 
-        You may call `wait_shutdown` or `shutdown` after the
-        account is in started mode.
+        # Workaround for a race condition. Make sure that thread is
+        # not in between checking for quitflag and entering idle.
+        time.sleep(0.5)
 
-        :raises MissingCredentials: if `addr` and `mail_pw` values are not set.
+        lib.dc_interrupt_imap_idle(self._dc_context)
+        lib.dc_interrupt_smtp_idle(self._dc_context)
+        lib.dc_interrupt_mvbox_idle(self._dc_context)
+        lib.dc_interrupt_sentbox_idle(self._dc_context)
+        if wait:
+            for name, thread in self._name2thread.items():
+                thread.join()
 
-        :returns: None
-        """
-        if not self.is_configured():
-            if not self.get_config("addr") or not self.get_config("mail_pw"):
-                raise MissingCredentials("addr or mail_pwd not set in config")
-            lib.dc_configure(self._dc_context)
-        self._threads.start(callback_thread=callback_thread)
+    def imap_thread_run(self):
+        self._log_event("py-bindings-info", 0, "INBOX THREAD START")
+        while not self._thread_quitflag:
+            lib.dc_perform_imap_jobs(self._dc_context)
+            if not self._thread_quitflag:
+                lib.dc_perform_imap_fetch(self._dc_context)
+            if not self._thread_quitflag:
+                lib.dc_perform_imap_idle(self._dc_context)
+        self._log_event("py-bindings-info", 0, "INBOX THREAD FINISHED")
 
-    def wait_shutdown(self):
-        """ wait until shutdown of this account has completed. """
-        self._shutdown_event.wait()
+    def mvbox_thread_run(self):
+        self._log_event("py-bindings-info", 0, "MVBOX THREAD START")
+        while not self._thread_quitflag:
+            lib.dc_perform_mvbox_jobs(self._dc_context)
+            if not self._thread_quitflag:
+                lib.dc_perform_mvbox_fetch(self._dc_context)
+            if not self._thread_quitflag:
+                lib.dc_perform_mvbox_idle(self._dc_context)
+        self._log_event("py-bindings-info", 0, "MVBOX THREAD FINISHED")
 
-    def shutdown(self, wait=True):
-        """ shutdown account, stop threads and close and remove
-        underlying dc_context and callbacks. """
-        dc_context = self._dc_context
-        if dc_context is None:
-            return
+    def sentbox_thread_run(self):
+        self._log_event("py-bindings-info", 0, "SENTBOX THREAD START")
+        while not self._thread_quitflag:
+            lib.dc_perform_sentbox_jobs(self._dc_context)
+            if not self._thread_quitflag:
+                lib.dc_perform_sentbox_fetch(self._dc_context)
+            if not self._thread_quitflag:
+                lib.dc_perform_sentbox_idle(self._dc_context)
+        self._log_event("py-bindings-info", 0, "SENTBOX THREAD FINISHED")
 
-        if self._threads.is_started():
-            self.stop_ongoing()
-            self._threads.stop(wait=False)
-        lib.dc_close(dc_context)
-        self._hook_event_queue.put(None)
-        self._threads.stop(wait=wait)  # to wait for threads
-        self._dc_context = None
-        atexit.unregister(self.shutdown)
-        self._shutdown_event.set()
-        hook = hookspec.Global._get_plugin_manager().hook
-        hook.dc_account_after_shutdown(account=self, dc_context=dc_context)
+    def smtp_thread_run(self):
+        self._log_event("py-bindings-info", 0, "SMTP THREAD START")
+        while not self._thread_quitflag:
+            lib.dc_perform_smtp_jobs(self._dc_context)
+            if not self._thread_quitflag:
+                lib.dc_perform_smtp_idle(self._dc_context)
+        self._log_event("py-bindings-info", 0, "SMTP THREAD FINISHED")
 
-    def _handle_current_events(self):
-        """ handle all currently queued events and then return. """
+
+class EventLogger:
+    _loglock = threading.RLock()
+
+    def __init__(self, dc_context, logid=None, debug=True):
+        self._dc_context = dc_context
+        self._event_queue = Queue()
+        self._debug = debug
+        if logid is None:
+            logid = str(self._dc_context).strip(">").split()[-1]
+        self.logid = logid
+        self._timeout = None
+        self.init_time = time.time()
+
+    def __call__(self, evt_name, data1, data2):
+        self._log_event(evt_name, data1, data2)
+        self._event_queue.put((evt_name, data1, data2))
+
+    def set_timeout(self, timeout):
+        self._timeout = timeout
+
+    def consume_events(self, check_error=True):
+        while not self._event_queue.empty():
+            self.get()
+
+    def get(self, timeout=None, check_error=True):
+        timeout = timeout or self._timeout
+        ev = self._event_queue.get(timeout=timeout)
+        if check_error and ev[0] == "DC_EVENT_ERROR":
+            raise ValueError("{}({!r},{!r})".format(*ev))
+        return ev
+
+    def ensure_event_not_queued(self, event_name_regex):
+        __tracebackhide__ = True
+        rex = re.compile("(?:{}).*".format(event_name_regex))
         while 1:
             try:
-                event = self._hook_event_queue.get(block=False)
-            except queue.Empty:
+                ev = self._event_queue.get(False)
+            except Empty:
                 break
             else:
-                event.call_hook()
+                assert not rex.match(ev[0]), "event found {}".format(ev)
 
-    def iter_events(self, timeout=None):
-        """ yield hook events until shutdown.
-
-        It is not allowed to call iter_events() from multiple threads.
-        """
-        if self._in_use_iter_events:
-            raise RuntimeError("can only call iter_events() from one thread")
-        self._in_use_iter_events = True
+    def get_matching(self, event_name_regex, check_error=True, timeout=None):
+        self._log("-- waiting for event with regex: {} --".format(event_name_regex))
+        rex = re.compile("(?:{}).*".format(event_name_regex))
         while 1:
-            event = self._hook_event_queue.get(timeout=timeout)
-            if event is None:
-                break
-            yield event
+            ev = self.get(timeout=timeout, check_error=check_error)
+            if rex.match(ev[0]):
+                return ev
 
-    def _map_ffi_event(self, ffi_event):
-        name = ffi_event.name
-        if name == "DC_EVENT_CONFIGURE_PROGRESS":
-            data1 = ffi_event.data1
-            if data1 == 0 or data1 == 1000:
-                success = data1 == 1000
-                yield "ac_configure_completed", dict(success=success)
-        elif name == "DC_EVENT_INCOMING_MSG":
-            msg = self.get_message_by_id(ffi_event.data2)
-            yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))
-        elif name == "DC_EVENT_MSGS_CHANGED":
-            if ffi_event.data2 != 0:
-                msg = self.get_message_by_id(ffi_event.data2)
-                if msg.is_outgoing():
-                    res = map_system_message(msg)
-                    if res and res[0].startswith("ac_member"):
-                        yield res
-                    yield "ac_outgoing_message", dict(message=msg)
-                elif msg.is_in_fresh():
-                    yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))
-        elif name == "DC_EVENT_MSG_DELIVERED":
-            msg = self.get_message_by_id(ffi_event.data2)
-            yield "ac_message_delivered", dict(message=msg)
-        elif name == "DC_EVENT_CHAT_MODIFIED":
-            chat = self.get_chat_by_id(ffi_event.data1)
-            yield "ac_chat_modified", dict(chat=chat)
+    def get_info_matching(self, regex):
+        rex = re.compile("(?:{}).*".format(regex))
+        while 1:
+            ev = self.get_matching("DC_EVENT_INFO")
+            if rex.match(ev[2]):
+                return ev
+
+    def _log_event(self, evt_name, data1, data2):
+        # don't show events that are anyway empty impls now
+        if evt_name == "DC_EVENT_GET_STRING":
+            return
+        if self._debug:
+            evpart = "{}({!r},{!r})".format(evt_name, data1, data2)
+            self._log(evpart)
+
+    def _log(self, msg):
+        t = threading.currentThread()
+        tname = getattr(t, "name", t)
+        if tname == "MainThread":
+            tname = "MAIN"
+        with self._loglock:
+            print("{:2.2f} [{}-{}] {}".format(time.time() - self.init_time, tname, self.logid, msg))
 
 
 def _destroy_dc_context(dc_context, dc_context_unref=lib.dc_context_unref):
@@ -685,17 +740,3 @@ class ScannedQRCode:
     @property
     def contact_id(self):
         return self._dc_lot.id()
-
-
-class HookEvent:
-    def __init__(self, account, name, kwargs):
-        assert hasattr(account._pm.hook, name), name
-        self.account = account
-        self.name = name
-        self.kwargs = kwargs
-
-    def call_hook(self):
-        hook = getattr(self.account._pm.hook, self.name, None)
-        if hook is None:
-            raise ValueError("event_name {} unknown".format(self.name))
-        return hook(**self.kwargs)

python/src/deltachat/chat.py

@@ -30,7 +30,7 @@ class Chat(object):
         return not (self == other)
 
     def __repr__(self):
-        return "<Chat id={} name={}>".format(self.id, self.get_name())
+        return "<Chat id={} name={} dc_context={}>".format(self.id, self.get_name(), self._dc_context)
 
     @property
     def _dc_chat(self):
@@ -51,16 +51,6 @@ class Chat(object):
 
     # ------  chat status/metadata API ------------------------------
 
-    def is_group(self):
-        """ return true if this chat is a group chat.
-
-        :returns: True if chat is a group-chat, false if it's a contact 1:1 chat.
-        """
-        return lib.dc_chat_get_type(self._dc_chat) in (
-            const.DC_CHAT_TYPE_GROUP,
-            const.DC_CHAT_TYPE_VERIFIED_GROUP
-        )
-
     def is_deaddrop(self):
         """ return true if this chat is a deaddrop chat.
 
@@ -139,7 +129,7 @@ class Chat(object):
         return bool(lib.dc_chat_get_remaining_mute_duration(self.id))
 
     def get_type(self):
-        """ (deprecated) return type of this chat.
+        """ return type of this chat.
 
         :returns: one of const.DC_CHAT_TYPE_*
         """
@@ -363,7 +353,7 @@ class Chat(object):
             lib.dc_array_unref
         )
         return list(iter_array(
-            dc_array, lambda id: Contact(self.account, id))
+            dc_array, lambda id: Contact(self._dc_context, id))
         )
 
     def set_profile_image(self, img_path):
@@ -415,6 +405,12 @@ class Chat(object):
         """
         return lib.dc_chat_get_color(self._dc_chat)
 
+    def get_subtitle(self):
+        """return the subtitle of the chat
+        :returns: the subtitle
+        """
+        return from_dc_charpointer(lib.dc_chat_get_subtitle(self._dc_chat))
+
     # ------  location streaming API ------------------------------
 
     def is_sending_locations(self):
@@ -427,7 +423,7 @@ class Chat(object):
         """return True if this chat is archived.
         :returns: True if archived.
         """
-        return lib.dc_chat_get_visibility(self._dc_chat) == const.DC_CHAT_VISIBILITY_ARCHIVED
+        return lib.dc_chat_get_archived(self._dc_chat)
 
     def enable_sending_locations(self, seconds):
         """enable sending locations for this chat.

python/src/deltachat/const.py

@@ -11,7 +11,6 @@ from os.path import join as joinpath
 DC_GCL_ARCHIVED_ONLY = 0x01
 DC_GCL_NO_SPECIALS = 0x02
 DC_GCL_ADD_ALLDONE_HINT = 0x04
-DC_GCL_FOR_FORWARDING = 0x08
 DC_GCL_VERIFIED_ONLY = 0x01
 DC_GCL_ADD_SELF = 0x02
 DC_QR_ASK_VERIFYCONTACT = 200
@@ -19,7 +18,6 @@ DC_QR_ASK_VERIFYGROUP = 202
 DC_QR_FPR_OK = 210
 DC_QR_FPR_MISMATCH = 220
 DC_QR_FPR_WITHOUT_ADDR = 230
-DC_QR_ACCOUNT = 250
 DC_QR_ADDR = 320
 DC_QR_TEXT = 330
 DC_QR_URL = 332
@@ -99,22 +97,19 @@ DC_EVENT_IMEX_PROGRESS = 2051
 DC_EVENT_IMEX_FILE_WRITTEN = 2052
 DC_EVENT_SECUREJOIN_INVITER_PROGRESS = 2060
 DC_EVENT_SECUREJOIN_JOINER_PROGRESS = 2061
+DC_EVENT_SECUREJOIN_MEMBER_ADDED = 2062
 DC_EVENT_FILE_COPIED = 2055
 DC_EVENT_IS_OFFLINE = 2081
 DC_EVENT_GET_STRING = 2091
 DC_STR_SELFNOTINGRP = 21
-DC_KEY_GEN_DEFAULT = 0
-DC_KEY_GEN_RSA2048 = 1
-DC_KEY_GEN_ED25519 = 2
 DC_PROVIDER_STATUS_OK = 1
 DC_PROVIDER_STATUS_PREPARATION = 2
 DC_PROVIDER_STATUS_BROKEN = 3
-DC_CHAT_VISIBILITY_NORMAL = 0
-DC_CHAT_VISIBILITY_ARCHIVED = 1
-DC_CHAT_VISIBILITY_PINNED = 2
 DC_STR_NOMESSAGES = 1
 DC_STR_SELF = 2
 DC_STR_DRAFT = 3
+DC_STR_MEMBER = 4
+DC_STR_CONTACT = 6
 DC_STR_VOICEMESSAGE = 7
 DC_STR_DEADDROP = 8
 DC_STR_IMAGE = 9
@@ -146,6 +141,7 @@ DC_STR_ARCHIVEDCHATS = 40
 DC_STR_STARREDMSGS = 41
 DC_STR_AC_SETUP_MSG_SUBJECT = 42
 DC_STR_AC_SETUP_MSG_BODY = 43
+DC_STR_SELFTALK_SUBTITLE = 50
 DC_STR_CANNOT_LOGIN = 60
 DC_STR_SERVER_RESPONSE = 61
 DC_STR_MSGACTIONBYUSER = 62
@@ -161,7 +157,7 @@ DC_STR_COUNT = 68
 
 def read_event_defines(f):
     rex = re.compile(r'#define\s+((?:DC_EVENT|DC_QR|DC_MSG|DC_LP|DC_EMPTY|DC_CERTCK|DC_STATE|DC_STR|'
-                     r'DC_CONTACT_ID|DC_GCL|DC_CHAT|DC_PROVIDER|DC_KEY_GEN)_\S+)\s+([x\d]+).*')
+                     r'DC_CONTACT_ID|DC_GCL|DC_CHAT|DC_PROVIDER)_\S+)\s+([x\d]+).*')
     for line in f:
         m = rex.match(line)
         if m:

python/src/deltachat/contact.py

@@ -10,9 +10,8 @@ class Contact(object):
 
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
-    def __init__(self, account, id):
-        self.account = account
-        self._dc_context = account._dc_context
+    def __init__(self, dc_context, id):
+        self._dc_context = dc_context
         self.id = id
 
     def __eq__(self, other):
@@ -58,7 +57,3 @@ class Contact(object):
         if dc_res == ffi.NULL:
             return None
         return from_dc_charpointer(dc_res)
-
-    def get_chat(self):
-        """return 1:1 chat for this contact. """
-        return self.account.create_chat_by_contact(self)

python/src/deltachat/eventlogger.py

@@ -1,137 +0,0 @@
-import deltachat
-import threading
-import time
-import re
-from queue import Queue, Empty
-from .hookspec import account_hookimpl, global_hookimpl
-
-
-@global_hookimpl
-def dc_account_init(account):
-    # send all FFI events for this account to a plugin hook
-    def _ll_event(ctx, evt_name, data1, data2):
-        assert ctx == account._dc_context
-        ffi_event = FFIEvent(name=evt_name, data1=data1, data2=data2)
-        account._pm.hook.ac_process_ffi_event(
-            account=account, ffi_event=ffi_event
-        )
-    deltachat.set_context_callback(account._dc_context, _ll_event)
-
-
-@global_hookimpl
-def dc_account_after_shutdown(dc_context):
-    deltachat.clear_context_callback(dc_context)
-
-
-class FFIEvent:
-    def __init__(self, name, data1, data2):
-        self.name = name
-        self.data1 = data1
-        self.data2 = data2
-
-    def __str__(self):
-        return "{name} data1={data1} data2={data2}".format(**self.__dict__)
-
-
-class FFIEventLogger:
-    """ If you register an instance of this logger with an Account
-    you'll get all ffi-events printed.
-    """
-    # to prevent garbled logging
-    _loglock = threading.RLock()
-
-    def __init__(self, account, logid):
-        """
-        :param logid: an optional logging prefix that should be used with
-                      the default internal logging.
-        """
-        self.account = account
-        self.logid = logid
-        self.init_time = time.time()
-
-    @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        self._log_event(ffi_event)
-
-    def _log_event(self, ffi_event):
-        # don't show events that are anyway empty impls now
-        if ffi_event.name == "DC_EVENT_GET_STRING":
-            return
-        self.account.ac_log_line(str(ffi_event))
-
-    @account_hookimpl
-    def ac_log_line(self, message):
-        t = threading.currentThread()
-        tname = getattr(t, "name", t)
-        if tname == "MainThread":
-            tname = "MAIN"
-        elapsed = time.time() - self.init_time
-        locname = tname
-        if self.logid:
-            locname += "-" + self.logid
-        s = "{:2.2f} [{}] {}".format(elapsed, locname, message)
-        with self._loglock:
-            print(s, flush=True)
-
-
-class FFIEventTracker:
-    def __init__(self, account, timeout=None):
-        self.account = account
-        self._timeout = timeout
-        self._event_queue = Queue()
-
-    @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        self._event_queue.put(ffi_event)
-
-    def set_timeout(self, timeout):
-        self._timeout = timeout
-
-    def consume_events(self, check_error=True):
-        while not self._event_queue.empty():
-            self.get(check_error=check_error)
-
-    def get(self, timeout=None, check_error=True):
-        timeout = timeout if timeout is not None else self._timeout
-        ev = self._event_queue.get(timeout=timeout)
-        if check_error and ev.name == "DC_EVENT_ERROR":
-            raise ValueError(str(ev))
-        return ev
-
-    def ensure_event_not_queued(self, event_name_regex):
-        __tracebackhide__ = True
-        rex = re.compile("(?:{}).*".format(event_name_regex))
-        while 1:
-            try:
-                ev = self._event_queue.get(False)
-            except Empty:
-                break
-            else:
-                assert not rex.match(ev.name), "event found {}".format(ev)
-
-    def get_matching(self, event_name_regex, check_error=True, timeout=None):
-        self.account.ac_log_line("-- waiting for event with regex: {} --".format(event_name_regex))
-        rex = re.compile("(?:{}).*".format(event_name_regex))
-        while 1:
-            ev = self.get(timeout=timeout, check_error=check_error)
-            if rex.match(ev.name):
-                return ev
-
-    def get_info_matching(self, regex):
-        rex = re.compile("(?:{}).*".format(regex))
-        while 1:
-            ev = self.get_matching("DC_EVENT_INFO")
-            if rex.match(ev.data2):
-                return ev
-
-    def wait_next_incoming_message(self):
-        """ wait for and return next incoming message. """
-        ev = self.get_matching("DC_EVENT_INCOMING_MSG")
-        return self.account.get_message_by_id(ev.data2)
-
-    def wait_next_messages_changed(self):
-        """ wait for and return next message-changed message or None
-        if the event contains no msgid"""
-        ev = self.get_matching("DC_EVENT_MSGS_CHANGED")
-        if ev.data2 > 0:
-            return self.account.get_message_by_id(ev.data2)

python/src/deltachat/hookspec.py

@@ -1,92 +0,0 @@
-""" Hooks for Python bindings to Delta Chat Core Rust CFFI"""
-
-import pluggy
-
-
-account_spec_name = "deltachat-account"
-account_hookspec = pluggy.HookspecMarker(account_spec_name)
-account_hookimpl = pluggy.HookimplMarker(account_spec_name)
-
-global_spec_name = "deltachat-global"
-global_hookspec = pluggy.HookspecMarker(global_spec_name)
-global_hookimpl = pluggy.HookimplMarker(global_spec_name)
-
-
-class PerAccount:
-    """ per-Account-instance hook specifications.
-
-    Except for ac_process_ffi_event all hooks are executed
-    in the thread which calls Account.wait_shutdown().
-    """
-    @classmethod
-    def _make_plugin_manager(cls):
-        pm = pluggy.PluginManager(account_spec_name)
-        pm.add_hookspecs(cls)
-        return pm
-
-    @account_hookspec
-    def ac_process_ffi_event(self, ffi_event):
-        """ process a CFFI low level events for a given account.
-
-        ffi_event has "name", "data1", "data2" values as specified
-        with `DC_EVENT_* <https://c.delta.chat/group__DC__EVENT.html>`_.
-
-        DANGER: this hook is executed from the callback invoked by core.
-        Hook implementations need to be short running and can typically
-        not call back into core because this would easily cause recursion issues.
-        """
-
-    @account_hookspec
-    def ac_log_line(self, message):
-        """ log a message related to the account. """
-
-    @account_hookspec
-    def ac_configure_completed(self, success):
-        """ Called when a configure process completed. """
-
-    @account_hookspec
-    def ac_incoming_message(self, message):
-        """ Called on any incoming message (to deaddrop or chat). """
-
-    @account_hookspec
-    def ac_outgoing_message(self, message):
-        """ Called on each outgoing message (both system and "normal")."""
-
-    @account_hookspec
-    def ac_message_delivered(self, message):
-        """ Called when an outgoing message has been delivered to SMTP. """
-
-    @account_hookspec
-    def ac_chat_modified(self, chat):
-        """ Chat was created or modified regarding membership, avatar, title. """
-
-    @account_hookspec
-    def ac_member_added(self, chat, contact, message):
-        """ Called for each contact added to an accepted chat. """
-
-    @account_hookspec
-    def ac_member_removed(self, chat, contact, message):
-        """ Called for each contact removed from a chat.  """
-
-
-class Global:
-    """ global hook specifications using a per-process singleton
-    plugin manager instance.
-
-    """
-    _plugin_manager = None
-
-    @classmethod
-    def _get_plugin_manager(cls):
-        if cls._plugin_manager is None:
-            cls._plugin_manager = pm = pluggy.PluginManager(global_spec_name)
-            pm.add_hookspecs(cls)
-        return cls._plugin_manager
-
-    @global_hookspec
-    def dc_account_init(self, account):
-        """ called when `Account::__init__()` function starts executing. """
-
-    @global_hookspec
-    def dc_account_after_shutdown(self, account, dc_context):
-        """ Called after the account has been shutdown. """

python/src/deltachat/iothreads.py

@@ -1,106 +0,0 @@
-
-import threading
-import time
-
-from contextlib import contextmanager
-
-from .capi import lib
-
-
-class IOThreads:
-    def __init__(self, account):
-        self.account = account
-        self._dc_context = account._dc_context
-        self._thread_quitflag = False
-        self._name2thread = {}
-
-    def is_started(self):
-        return len(self._name2thread) > 0
-
-    def start(self, callback_thread):
-        assert not self.is_started()
-        self._start_one_thread("inbox", self.imap_thread_run)
-        self._start_one_thread("smtp", self.smtp_thread_run)
-
-        if callback_thread:
-            self._start_one_thread("cb", self.cb_thread_run)
-
-        if int(self.account.get_config("mvbox_watch")):
-            self._start_one_thread("mvbox", self.mvbox_thread_run)
-
-        if int(self.account.get_config("sentbox_watch")):
-            self._start_one_thread("sentbox", self.sentbox_thread_run)
-
-    def _start_one_thread(self, name, func):
-        self._name2thread[name] = t = threading.Thread(target=func, name=name)
-        t.setDaemon(1)
-        t.start()
-
-    @contextmanager
-    def log_execution(self, message):
-        self.account.ac_log_line(message + " START")
-        yield
-        self.account.ac_log_line(message + " FINISHED")
-
-    def stop(self, wait=False):
-        self._thread_quitflag = True
-
-        # Workaround for a race condition. Make sure that thread is
-        # not in between checking for quitflag and entering idle.
-        time.sleep(0.5)
-
-        lib.dc_interrupt_imap_idle(self._dc_context)
-        lib.dc_interrupt_smtp_idle(self._dc_context)
-        if "mvbox" in self._name2thread:
-            lib.dc_interrupt_mvbox_idle(self._dc_context)
-        if "sentbox" in self._name2thread:
-            lib.dc_interrupt_sentbox_idle(self._dc_context)
-        if wait:
-            for name, thread in self._name2thread.items():
-                if thread != threading.currentThread():
-                    thread.join()
-
-    def cb_thread_run(self):
-        with self.log_execution("CALLBACK THREAD START"):
-            it = self.account.iter_events()
-            while not self._thread_quitflag:
-                try:
-                    ev = next(it)
-                except StopIteration:
-                    break
-                self.account.ac_log_line("calling hook name={} kwargs={}".format(ev.name, ev.kwargs))
-                ev.call_hook()
-
-    def imap_thread_run(self):
-        with self.log_execution("INBOX THREAD START"):
-            while not self._thread_quitflag:
-                lib.dc_perform_imap_jobs(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_imap_fetch(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_imap_idle(self._dc_context)
-
-    def mvbox_thread_run(self):
-        with self.log_execution("MVBOX THREAD"):
-            while not self._thread_quitflag:
-                lib.dc_perform_mvbox_jobs(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_mvbox_fetch(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_mvbox_idle(self._dc_context)
-
-    def sentbox_thread_run(self):
-        with self.log_execution("SENTBOX THREAD"):
-            while not self._thread_quitflag:
-                lib.dc_perform_sentbox_jobs(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_sentbox_fetch(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_sentbox_idle(self._dc_context)
-
-    def smtp_thread_run(self):
-        with self.log_execution("SMTP THREAD"):
-            while not self._thread_quitflag:
-                lib.dc_perform_smtp_jobs(self._dc_context)
-                if not self._thread_quitflag:
-                    lib.dc_perform_smtp_idle(self._dc_context)

python/src/deltachat/message.py

@@ -28,9 +28,7 @@ class Message(object):
         return self.account == other.account and self.id == other.id
 
     def __repr__(self):
-        c = self.get_sender_contact()
-        return "<Message id={} sender={}/{} outgoing={} chat={}/{}>".format(
-            self.id, c.id, c.addr, self.is_outgoing(), self.chat.id, self.chat.get_name())
+        return "<Message id={} dc_context={}>".format(self.id, self._dc_context)
 
     @classmethod
     def from_db(cls, account, id):
@@ -52,16 +50,6 @@ class Message(object):
             lib.dc_msg_unref
         ))
 
-    def accept_sender_contact(self):
-        """ ensure that the sender is an accepted contact
-        and that the message has a non-deaddrop chat object.
-        """
-        self.account.create_chat_by_message(self)
-        self._dc_msg = ffi.gc(
-                lib.dc_get_msg(self._dc_context, self.id),
-                lib.dc_msg_unref
-        )
-
     @props.with_doc
     def text(self):
         """unicode text of this messages (might be empty if not a text message). """
@@ -93,10 +81,6 @@ class Message(object):
         """mime type of the file (if it exists)"""
         return from_dc_charpointer(lib.dc_msg_get_filemime(self._dc_msg))
 
-    def is_system_message(self):
-        """ return True if this message is a system/info message. """
-        return lib.dc_msg_is_info(self._dc_msg)
-
     def is_setup_message(self):
         """ return True if this message is a setup message. """
         return lib.dc_msg_is_setupmessage(self._dc_msg)
@@ -175,13 +159,6 @@ class Message(object):
         chat_id = lib.dc_msg_get_chat_id(self._dc_msg)
         return Chat(self.account, chat_id)
 
-    def get_sender_chat(self):
-        """return the 1:1 chat with the sender of this message.
-
-        :returns: :class:`deltachat.chat.Chat` instance
-        """
-        return self.get_sender_contact().get_chat()
-
     def get_sender_contact(self):
         """return the contact of who wrote the message.
 
@@ -189,7 +166,7 @@ class Message(object):
         """
         from .contact import Contact
         contact_id = lib.dc_msg_get_from_id(self._dc_msg)
-        return Contact(self.account, contact_id)
+        return Contact(self._dc_context, contact_id)
 
     #
     # Message State query methods
@@ -230,13 +207,6 @@ class Message(object):
         """
         return self._msgstate == const.DC_STATE_IN_SEEN
 
-    def is_outgoing(self):
-        """Return True if Message is outgoing. """
-        return self._msgstate in (
-            const.DC_STATE_OUT_PREPARING, const.DC_STATE_OUT_PENDING,
-            const.DC_STATE_OUT_FAILED, const.DC_STATE_OUT_MDN_RCVD,
-            const.DC_STATE_OUT_DELIVERED)
-
     def is_out_preparing(self):
         """Return True if Message is outgoing, but its file is being prepared.
         """
@@ -299,10 +269,6 @@ class Message(object):
         """ return True if it's a file message. """
         return self._view_type == const.DC_MSG_FILE
 
-    def mark_seen(self):
-        """ mark this message as seen. """
-        self.account.mark_seen_messages([self.id])
-
 
 # some code for handling DC_MSG_* view types
 
@@ -322,29 +288,3 @@ def get_viewtype_code_from_name(view_type_name):
             return code
     raise ValueError("message typecode not found for {!r}, "
                      "available {!r}".format(view_type_name, list(_view_type_mapping.values())))
-
-
-#
-# some helper code for turning system messages into hook events
-#
-
-def map_system_message(msg):
-    if msg.is_system_message():
-        res = parse_system_add_remove(msg.text)
-        if res:
-            contact = msg.account.get_contact_by_addr(res[1])
-            if contact:
-                d = dict(chat=msg.chat, contact=contact, message=msg)
-                return "ac_member_" + res[0], d
-
-
-def parse_system_add_remove(text):
-    # Member Me (x@y) removed by a@b.
-    # Member x@y removed by a@b
-    text = text.lower()
-    parts = text.split()
-    if parts[0] == "member":
-        if parts[2] in ("removed", "added"):
-            return parts[2], parts[1]
-        if parts[3] in ("removed", "added"):
-            return parts[3], parts[2].strip("()")

python/src/deltachat/testplugin.py

@@ -1,447 +0,0 @@
-from __future__ import print_function
-import os
-import sys
-import subprocess
-import queue
-import threading
-import fnmatch
-import time
-import weakref
-import tempfile
-
-import pytest
-import requests
-
-from . import Account, const
-from .tracker import ConfigureTracker
-from .capi import lib
-from .eventlogger import FFIEventLogger, FFIEventTracker
-from _pytest.monkeypatch import MonkeyPatch
-from _pytest._code import Source
-
-import deltachat
-
-
-def pytest_addoption(parser):
-    parser.addoption(
-        "--liveconfig", action="store", default=None,
-        help="a file with >=2 lines where each line "
-             "contains NAME=VALUE config settings for one account"
-    )
-    parser.addoption(
-        "--ignored", action="store_true",
-        help="Also run tests marked with the ignored marker",
-    )
-
-
-def pytest_configure(config):
-    config.addinivalue_line(
-        "markers", "ignored: Mark test as bing slow, skipped unless --ignored is used."
-    )
-    cfg = config.getoption('--liveconfig')
-    if not cfg:
-        cfg = os.getenv('DCC_NEW_TMP_EMAIL')
-        if cfg:
-            config.option.liveconfig = cfg
-
-    # Make sure we don't get garbled output because threads keep running
-    # collect all ever created accounts in a weakref-set (so we don't
-    # keep objects unneccessarily alive) and enable/disable logging
-    # for each pytest test phase # (setup/call/teardown).
-    # Additionally make the acfactory use a logging/no-logging default.
-
-    class LoggingAspect:
-        def __init__(self):
-            self._accounts = weakref.WeakSet()
-
-        @deltachat.global_hookimpl
-        def dc_account_init(self, account):
-            self._accounts.add(account)
-
-        def disable_logging(self, item):
-            for acc in self._accounts:
-                acc.disable_logging()
-            acfactory = item.funcargs.get("acfactory")
-            if acfactory:
-                acfactory.set_logging_default(False)
-
-        def enable_logging(self, item):
-            for acc in self._accounts:
-                acc.enable_logging()
-            acfactory = item.funcargs.get("acfactory")
-            if acfactory:
-                acfactory.set_logging_default(True)
-
-        @pytest.hookimpl(hookwrapper=True)
-        def pytest_runtest_setup(self, item):
-            self.enable_logging(item)
-            yield
-            self.disable_logging(item)
-
-        @pytest.hookimpl(hookwrapper=True)
-        def pytest_pyfunc_call(self, pyfuncitem):
-            self.enable_logging(pyfuncitem)
-            yield
-            self.disable_logging(pyfuncitem)
-
-        @pytest.hookimpl(hookwrapper=True)
-        def pytest_runtest_teardown(self, item):
-            self.enable_logging(item)
-            yield
-            self.disable_logging(item)
-
-    la = LoggingAspect()
-    config.pluginmanager.register(la)
-    deltachat.register_global_plugin(la)
-
-
-def pytest_report_header(config, startdir):
-    summary = []
-
-    t = tempfile.mktemp()
-    m = MonkeyPatch()
-    try:
-        m.setattr(sys.stdout, "write", lambda x: len(x))
-        ac = Account(t)
-        info = ac.get_info()
-        ac.shutdown()
-    finally:
-        m.undo()
-        os.remove(t)
-    summary.extend(['Deltachat core={} sqlite={}'.format(
-         info['deltachat_core_version'],
-         info['sqlite_version'],
-     )])
-
-    cfg = config.option.liveconfig
-    if cfg:
-        if "?" in cfg:
-            url, token = cfg.split("?", 1)
-            summary.append('Liveconfig provider: {}?<token ommitted>'.format(url))
-        else:
-            summary.append('Liveconfig file: {}'.format(cfg))
-    return summary
-
-
-class SessionLiveConfigFromFile:
-    def __init__(self, fn):
-        self.fn = fn
-        self.configlist = []
-        for line in open(fn):
-            if line.strip() and not line.strip().startswith('#'):
-                d = {}
-                for part in line.split():
-                    name, value = part.split("=")
-                    d[name] = value
-                self.configlist.append(d)
-
-    def get(self, index):
-        return self.configlist[index]
-
-    def exists(self):
-        return bool(self.configlist)
-
-
-class SessionLiveConfigFromURL:
-    def __init__(self, url):
-        self.configlist = []
-        self.url = url
-
-    def get(self, index):
-        try:
-            return self.configlist[index]
-        except IndexError:
-            assert index == len(self.configlist), index
-            res = requests.post(self.url)
-            if res.status_code != 200:
-                pytest.skip("creating newtmpuser failed {!r}".format(res))
-            d = res.json()
-            config = dict(addr=d["email"], mail_pw=d["password"])
-            self.configlist.append(config)
-            return config
-
-    def exists(self):
-        return bool(self.configlist)
-
-
-@pytest.fixture(scope="session")
-def session_liveconfig(request):
-    liveconfig_opt = request.config.option.liveconfig
-    if liveconfig_opt:
-        if liveconfig_opt.startswith("http"):
-            return SessionLiveConfigFromURL(liveconfig_opt)
-        else:
-            return SessionLiveConfigFromFile(liveconfig_opt)
-
-
-@pytest.fixture
-def data(request):
-    class Data:
-        def __init__(self):
-            # trying to find test data heuristically
-            # because we are run from a dev-setup with pytest direct,
-            # through tox, and then maybe also from deltachat-binding
-            # users like "deltabot".
-            self.paths = [os.path.normpath(x) for x in [
-                os.path.join(os.path.dirname(request.fspath.strpath), "data"),
-                os.path.join(os.path.dirname(__file__), "..", "..", "..", "test-data")
-            ]]
-
-        def get_path(self, bn):
-            """ return path of file or None if it doesn't exist. """
-            for path in self.paths:
-                fn = os.path.join(path, *bn.split("/"))
-                if os.path.exists(fn):
-                    return fn
-            print("WARNING: path does not exist: {!r}".format(fn))
-
-        def read_path(self, bn, mode="r"):
-            fn = self.get_path(bn)
-            if fn is not None:
-                with open(fn, mode) as f:
-                    return f.read()
-
-    return Data()
-
-
-@pytest.fixture
-def acfactory(pytestconfig, tmpdir, request, session_liveconfig, data):
-
-    class AccountMaker:
-        def __init__(self):
-            self.live_count = 0
-            self.offline_count = 0
-            self._finalizers = []
-            self._accounts = []
-            self.init_time = time.time()
-            self._generated_keys = ["alice", "bob", "charlie",
-                                    "dom", "elena", "fiona"]
-            self.set_logging_default(False)
-
-        def finalize(self):
-            while self._finalizers:
-                fin = self._finalizers.pop()
-                fin()
-
-            while self._accounts:
-                acc = self._accounts.pop()
-                acc.shutdown()
-                acc.disable_logging()
-
-        def make_account(self, path, logid, quiet=False):
-            ac = Account(path, logging=self._logging)
-            ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
-            ac._configtracker = ac.add_account_plugin(ConfigureTracker())
-            ac.addr = ac.get_self_contact().addr
-            if not quiet:
-                ac.add_account_plugin(FFIEventLogger(ac, logid=logid))
-            self._accounts.append(ac)
-            return ac
-
-        def set_logging_default(self, logging):
-            self._logging = bool(logging)
-
-        def get_unconfigured_account(self):
-            self.offline_count += 1
-            tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
-            ac._evtracker.init_time = self.init_time
-            ac._evtracker.set_timeout(2)
-            return ac
-
-        def _preconfigure_key(self, account, addr):
-            # Only set a key if we haven't used it yet for another account.
-            if self._generated_keys:
-                keyname = self._generated_keys.pop(0)
-                fname_pub = data.read_path("key/{name}-public.asc".format(name=keyname))
-                fname_sec = data.read_path("key/{name}-secret.asc".format(name=keyname))
-                if fname_pub and fname_sec:
-                    account._preconfigure_keypair(addr, fname_pub, fname_sec)
-                    return True
-                else:
-                    print("WARN: could not use preconfigured keys for {!r}".format(addr))
-
-        def get_configured_offline_account(self):
-            ac = self.get_unconfigured_account()
-
-            # do a pseudo-configured account
-            addr = "addr{}@offline.org".format(self.offline_count)
-            ac.set_config("addr", addr)
-            self._preconfigure_key(ac, addr)
-            lib.dc_set_config(ac._dc_context, b"configured_addr", addr.encode("ascii"))
-            ac.set_config("mail_pw", "123")
-            lib.dc_set_config(ac._dc_context, b"configured_mail_pw", b"123")
-            lib.dc_set_config(ac._dc_context, b"configured", b"1")
-            return ac
-
-        def get_online_config(self, pre_generated_key=True, quiet=False):
-            if not session_liveconfig:
-                pytest.skip("specify DCC_NEW_TMP_EMAIL or --liveconfig")
-            configdict = session_liveconfig.get(self.live_count)
-            self.live_count += 1
-            if "e2ee_enabled" not in configdict:
-                configdict["e2ee_enabled"] = "1"
-
-            # Enable strict certificate checks for online accounts
-            configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
-            configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
-
-            tmpdb = tmpdir.join("livedb%d" % self.live_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count), quiet=quiet)
-            if pre_generated_key:
-                self._preconfigure_key(ac, configdict['addr'])
-            ac._evtracker.init_time = self.init_time
-            ac._evtracker.set_timeout(30)
-            return ac, dict(configdict)
-
-        def get_online_configuring_account(self, mvbox=False, sentbox=False, move=False,
-                                           pre_generated_key=True, quiet=False, config={}):
-            ac, configdict = self.get_online_config(
-                pre_generated_key=pre_generated_key, quiet=quiet)
-            configdict.update(config)
-            configdict["mvbox_watch"] = str(int(mvbox))
-            configdict["mvbox_move"] = str(int(move))
-            configdict["sentbox_watch"] = str(int(sentbox))
-            ac.update_config(configdict)
-            ac.start()
-            return ac
-
-        def get_one_online_account(self, pre_generated_key=True, mvbox=False, move=False):
-            ac1 = self.get_online_configuring_account(
-                pre_generated_key=pre_generated_key, mvbox=mvbox, move=move)
-            ac1._configtracker.wait_imap_connected()
-            ac1._configtracker.wait_smtp_connected()
-            ac1._configtracker.wait_finish()
-            return ac1
-
-        def get_two_online_accounts(self, move=False, quiet=False):
-            ac1 = self.get_online_configuring_account(move=True, quiet=quiet)
-            ac2 = self.get_online_configuring_account(quiet=quiet)
-            ac1._configtracker.wait_finish()
-            ac2._configtracker.wait_finish()
-            return ac1, ac2
-
-        def get_many_online_accounts(self, num, move=True, quiet=True):
-            accounts = [self.get_online_configuring_account(move=move, quiet=quiet)
-                        for i in range(num)]
-            for acc in accounts:
-                acc._configtracker.wait_finish()
-            return accounts
-
-        def clone_online_account(self, account, pre_generated_key=True):
-            self.live_count += 1
-            tmpdb = tmpdir.join("livedb%d" % self.live_count)
-            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
-            if pre_generated_key:
-                self._preconfigure_key(ac, account.get_config("addr"))
-            ac._evtracker.init_time = self.init_time
-            ac._evtracker.set_timeout(30)
-            ac.update_config(dict(
-                addr=account.get_config("addr"),
-                mail_pw=account.get_config("mail_pw"),
-                mvbox_watch=account.get_config("mvbox_watch"),
-                mvbox_move=account.get_config("mvbox_move"),
-                sentbox_watch=account.get_config("sentbox_watch"),
-            ))
-            ac.start()
-            return ac
-
-        def run_bot_process(self, module, ffi=True):
-            fn = module.__file__
-
-            bot_ac, bot_cfg = self.get_online_config()
-
-            args = [
-                sys.executable,
-                "-u",
-                fn,
-                "--email", bot_cfg["addr"],
-                "--password", bot_cfg["mail_pw"],
-                bot_ac.db_path,
-            ]
-            if ffi:
-                args.insert(-1, "--show-ffi")
-            print("$", " ".join(args))
-            popen = subprocess.Popen(
-                args=args,
-                stdin=subprocess.DEVNULL,
-                stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT,  # combine stdout/stderr in one stream
-                bufsize=0,                 # line buffering
-                close_fds=True,            # close all FDs other than 0/1/2
-                universal_newlines=True    # give back text
-            )
-            bot = BotProcess(popen, bot_cfg)
-            self._finalizers.append(bot.kill)
-            return bot
-
-    am = AccountMaker()
-    request.addfinalizer(am.finalize)
-    return am
-
-
-class BotProcess:
-    def __init__(self, popen, bot_cfg):
-        self.popen = popen
-        self.addr = bot_cfg["addr"]
-
-        # we read stdout as quickly as we can in a thread and make
-        # the (unicode) lines available for readers through a queue.
-        self.stdout_queue = queue.Queue()
-        self.stdout_thread = t = threading.Thread(target=self._run_stdout_thread, name="bot-stdout-thread")
-        t.setDaemon(1)
-        t.start()
-
-    def _run_stdout_thread(self):
-        try:
-            while 1:
-                line = self.popen.stdout.readline()
-                if not line:
-                    break
-                line = line.strip()
-                self.stdout_queue.put(line)
-        finally:
-            self.stdout_queue.put(None)
-
-    def kill(self):
-        self.popen.kill()
-
-    def wait(self, timeout=30):
-        self.popen.wait(timeout=timeout)
-
-    def fnmatch_lines(self, pattern_lines):
-        patterns = [x.strip() for x in Source(pattern_lines.rstrip()).lines if x.strip()]
-        for next_pattern in patterns:
-            print("+++FNMATCH:", next_pattern)
-            ignored = []
-            while 1:
-                line = self.stdout_queue.get(timeout=15)
-                if line is None:
-                    if ignored:
-                        print("BOT stdout terminated after these lines")
-                        for line in ignored:
-                            print(line)
-                    raise IOError("BOT stdout-thread terminated")
-                if fnmatch.fnmatch(line, next_pattern):
-                    print("+++MATCHED:", line)
-                    break
-                else:
-                    print("+++IGN:", line)
-                    ignored.append(line)
-
-
-@pytest.fixture
-def tmp_db_path(tmpdir):
-    return tmpdir.join("test.db").strpath
-
-
-@pytest.fixture
-def lp():
-    class Printer:
-        def sec(self, msg):
-            print()
-            print("=" * 10, msg, "=" * 10)
-
-        def step(self, msg):
-            print("-" * 5, "step " + msg, "-" * 5)
-    return Printer()

python/src/deltachat/tracker.py

@@ -1,76 +0,0 @@
-
-from queue import Queue
-from threading import Event
-
-from .hookspec import account_hookimpl
-
-
-class ImexFailed(RuntimeError):
-    """ Exception for signalling that import/export operations failed."""
-
-
-class ImexTracker:
-    def __init__(self):
-        self._imex_events = Queue()
-
-    @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        if ffi_event.name == "DC_EVENT_IMEX_PROGRESS":
-            self._imex_events.put(ffi_event.data1)
-        elif ffi_event.name == "DC_EVENT_IMEX_FILE_WRITTEN":
-            self._imex_events.put(ffi_event.data1)
-
-    def wait_finish(self, progress_timeout=60):
-        """ Return list of written files, raise ValueError if ExportFailed. """
-        files_written = []
-        while True:
-            ev = self._imex_events.get(timeout=progress_timeout)
-            if isinstance(ev, str):
-                files_written.append(ev)
-            elif ev == 0:
-                raise ImexFailed("export failed, exp-files: {}".format(files_written))
-            elif ev == 1000:
-                return files_written
-
-
-class ConfigureFailed(RuntimeError):
-    """ Exception for signalling that configuration failed."""
-
-
-class ConfigureTracker:
-    ConfigureFailed = ConfigureFailed
-
-    def __init__(self):
-        self._configure_events = Queue()
-        self._smtp_finished = Event()
-        self._imap_finished = Event()
-        self._ffi_events = []
-
-    @account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        self._ffi_events.append(ffi_event)
-        if ffi_event.name == "DC_EVENT_SMTP_CONNECTED":
-            self._smtp_finished.set()
-        elif ffi_event.name == "DC_EVENT_IMAP_CONNECTED":
-            self._imap_finished.set()
-
-    @account_hookimpl
-    def ac_configure_completed(self, success):
-        self._configure_events.put(success)
-
-    def wait_smtp_connected(self):
-        """ wait until smtp is configured. """
-        self._smtp_finished.wait()
-
-    def wait_imap_connected(self):
-        """ wait until smtp is configured. """
-        self._imap_finished.wait()
-
-    def wait_finish(self):
-        """ wait until configure is completed.
-
-        Raise Exception if Configure failed
-        """
-        if not self._configure_events.get():
-            content = "\n".join(map(str, self._ffi_events))
-            raise ConfigureFailed(content)

python/tests/conftest.py

@@ -1,18 +1,311 @@
 from __future__ import print_function
+import os
+import py
+import pytest
+import requests
+import time
+from deltachat import Account
+from deltachat import const
+from deltachat.capi import lib
+import tempfile
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--liveconfig", action="store", default=None,
+        help="a file with >=2 lines where each line "
+             "contains NAME=VALUE config settings for one account"
+    )
+    parser.addoption(
+        "--ignored", action="store_true",
+        help="Also run tests marked with the ignored marker",
+    )
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers", "ignored: Mark test as bing slow, skipped unless --ignored is used."
+    )
+    cfg = config.getoption('--liveconfig')
+    if not cfg:
+        cfg = os.getenv('DCC_NEW_TMP_EMAIL')
+        if cfg:
+            config.option.liveconfig = cfg
+
+
+def pytest_runtest_setup(item):
+    if (list(item.iter_markers(name="ignored"))
+            and not item.config.getoption("ignored")):
+        pytest.skip("Ignored tests not requested, use --ignored")
+
+
+def pytest_report_header(config, startdir):
+    summary = []
+
+    t = tempfile.mktemp()
+    try:
+        ac = Account(t, eventlogging=False)
+        info = ac.get_info()
+        ac.shutdown()
+    finally:
+        os.remove(t)
+    summary.extend(['Deltachat core={} sqlite={}'.format(
+         info['deltachat_core_version'],
+         info['sqlite_version'],
+     )])
+
+    cfg = config.option.liveconfig
+    if cfg:
+        if "#" in cfg:
+            url, token = cfg.split("#", 1)
+            summary.append('Liveconfig provider: {}#<token ommitted>'.format(url))
+        else:
+            summary.append('Liveconfig file: {}'.format(cfg))
+    return summary
+
+
+@pytest.fixture(scope="session")
+def data():
+    class Data:
+        def __init__(self):
+            self.path = os.path.join(os.path.dirname(__file__), "data")
+
+        def get_path(self, bn):
+            fn = os.path.join(self.path, bn)
+            assert os.path.exists(fn)
+            return fn
+    return Data()
+
+
+class SessionLiveConfigFromFile:
+    def __init__(self, fn):
+        self.fn = fn
+        self.configlist = []
+        for line in open(fn):
+            if line.strip() and not line.strip().startswith('#'):
+                d = {}
+                for part in line.split():
+                    name, value = part.split("=")
+                    d[name] = value
+                self.configlist.append(d)
+
+    def get(self, index):
+        return self.configlist[index]
+
+    def exists(self):
+        return bool(self.configlist)
+
+
+class SessionLiveConfigFromURL:
+    def __init__(self, url):
+        self.configlist = []
+        self.url = url
+
+    def get(self, index):
+        try:
+            return self.configlist[index]
+        except IndexError:
+            assert index == len(self.configlist), index
+            res = requests.post(self.url)
+            if res.status_code != 200:
+                pytest.skip("creating newtmpuser failed {!r}".format(res))
+            d = res.json()
+            config = dict(addr=d["email"], mail_pw=d["password"])
+            self.configlist.append(config)
+            return config
+
+    def exists(self):
+        return bool(self.configlist)
+
+
+@pytest.fixture(scope="session")
+def session_liveconfig(request):
+    liveconfig_opt = request.config.option.liveconfig
+    if liveconfig_opt:
+        if liveconfig_opt.startswith("http"):
+            return SessionLiveConfigFromURL(liveconfig_opt)
+        else:
+            return SessionLiveConfigFromFile(liveconfig_opt)
+
+
+@pytest.fixture(scope='session')
+def datadir():
+    """The py.path.local object of the test-data/ directory."""
+    for path in reversed(py.path.local(__file__).parts()):
+        datadir = path.join('test-data')
+        if datadir.isdir():
+            return datadir
+    else:
+        pytest.skip('test-data directory not found')
+
+
+@pytest.fixture
+def acfactory(pytestconfig, tmpdir, request, session_liveconfig, datadir):
+
+    class AccountMaker:
+        def __init__(self):
+            self.live_count = 0
+            self.offline_count = 0
+            self._finalizers = []
+            self.init_time = time.time()
+            self._generated_keys = ["alice", "bob", "charlie",
+                                    "dom", "elena", "fiona"]
+
+        def finalize(self):
+            while self._finalizers:
+                fin = self._finalizers.pop()
+                fin()
+
+        def make_account(self, path, logid):
+            ac = Account(path, logid=logid)
+            self._finalizers.append(ac.shutdown)
+            return ac
+
+        def get_unconfigured_account(self):
+            self.offline_count += 1
+            tmpdb = tmpdir.join("offlinedb%d" % self.offline_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.offline_count))
+            ac._evlogger.init_time = self.init_time
+            ac._evlogger.set_timeout(2)
+            return ac
+
+        def _preconfigure_key(self, account, addr):
+            # Only set a key if we haven't used it yet for another account.
+            if self._generated_keys:
+                keyname = self._generated_keys.pop(0)
+                fname_pub = "key/{name}-public.asc".format(name=keyname)
+                fname_sec = "key/{name}-secret.asc".format(name=keyname)
+                account._preconfigure_keypair(addr,
+                                              datadir.join(fname_pub).read(),
+                                              datadir.join(fname_sec).read())
+
+        def get_configured_offline_account(self):
+            ac = self.get_unconfigured_account()
+
+            # do a pseudo-configured account
+            addr = "addr{}@offline.org".format(self.offline_count)
+            ac.set_config("addr", addr)
+            self._preconfigure_key(ac, addr)
+            lib.dc_set_config(ac._dc_context, b"configured_addr", addr.encode("ascii"))
+            ac.set_config("mail_pw", "123")
+            lib.dc_set_config(ac._dc_context, b"configured_mail_pw", b"123")
+            lib.dc_set_config(ac._dc_context, b"configured", b"1")
+            return ac
+
+        def get_online_config(self, pre_generated_key=True):
+            if not session_liveconfig:
+                pytest.skip("specify DCC_NEW_TMP_EMAIL or --liveconfig")
+            configdict = session_liveconfig.get(self.live_count)
+            self.live_count += 1
+            if "e2ee_enabled" not in configdict:
+                configdict["e2ee_enabled"] = "1"
+
+            # Enable strict certificate checks for online accounts
+            configdict["imap_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+            configdict["smtp_certificate_checks"] = str(const.DC_CERTCK_STRICT)
+
+            tmpdb = tmpdir.join("livedb%d" % self.live_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
+            if pre_generated_key:
+                self._preconfigure_key(ac, configdict['addr'])
+            ac._evlogger.init_time = self.init_time
+            ac._evlogger.set_timeout(30)
+            return ac, dict(configdict)
+
+        def get_online_configuring_account(self, mvbox=False, sentbox=False,
+                                           pre_generated_key=True):
+            ac, configdict = self.get_online_config(
+                pre_generated_key=pre_generated_key)
+            ac.configure(**configdict)
+            ac.start_threads(mvbox=mvbox, sentbox=sentbox)
+            return ac
+
+        def get_one_online_account(self, pre_generated_key=True):
+            ac1 = self.get_online_configuring_account(
+                pre_generated_key=pre_generated_key)
+            wait_successful_IMAP_SMTP_connection(ac1)
+            wait_configuration_progress(ac1, 1000)
+            return ac1
+
+        def get_two_online_accounts(self):
+            ac1 = self.get_online_configuring_account()
+            ac2 = self.get_online_configuring_account()
+            wait_successful_IMAP_SMTP_connection(ac1)
+            wait_configuration_progress(ac1, 1000)
+            wait_successful_IMAP_SMTP_connection(ac2)
+            wait_configuration_progress(ac2, 1000)
+            return ac1, ac2
+
+        def clone_online_account(self, account, pre_generated_key=True):
+            self.live_count += 1
+            tmpdb = tmpdir.join("livedb%d" % self.live_count)
+            ac = self.make_account(tmpdb.strpath, logid="ac{}".format(self.live_count))
+            if pre_generated_key:
+                self._preconfigure_key(ac, account.get_config("addr"))
+            ac._evlogger.init_time = self.init_time
+            ac._evlogger.set_timeout(30)
+            ac.configure(addr=account.get_config("addr"), mail_pw=account.get_config("mail_pw"))
+            ac.start_threads()
+            return ac
+
+    am = AccountMaker()
+    request.addfinalizer(am.finalize)
+    return am
+
+
+@pytest.fixture
+def tmp_db_path(tmpdir):
+    return tmpdir.join("test.db").strpath
+
+
+@pytest.fixture
+def lp():
+    class Printer:
+        def sec(self, msg):
+            print()
+            print("=" * 10, msg, "=" * 10)
+
+        def step(self, msg):
+            print("-" * 5, "step " + msg, "-" * 5)
+    return Printer()
 
 
 def wait_configuration_progress(account, min_target, max_target=1001):
     min_target = min(min_target, max_target)
     while 1:
-        event = account._evtracker.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
-        if event.data1 >= min_target and event.data1 <= max_target:
+        evt_name, data1, data2 = \
+            account._evlogger.get_matching("DC_EVENT_CONFIGURE_PROGRESS")
+        if data1 >= min_target and data1 <= max_target:
             print("** CONFIG PROGRESS {}".format(min_target), account)
             break
 
 
 def wait_securejoin_inviter_progress(account, target):
     while 1:
-        event = account._evtracker.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS")
-        if event.data2 >= target:
+        evt_name, data1, data2 = \
+            account._evlogger.get_matching("DC_EVENT_SECUREJOIN_INVITER_PROGRESS")
+        if data2 >= target:
             print("** SECUREJOINT-INVITER PROGRESS {}".format(target), account)
             break
+
+
+def wait_successful_IMAP_SMTP_connection(account):
+    imap_ok = smtp_ok = False
+    while not imap_ok or not smtp_ok:
+        evt_name, data1, data2 = \
+            account._evlogger.get_matching("DC_EVENT_(IMAP|SMTP)_CONNECTED")
+        if evt_name == "DC_EVENT_IMAP_CONNECTED":
+            imap_ok = True
+            print("** IMAP OK", account)
+        if evt_name == "DC_EVENT_SMTP_CONNECTED":
+            smtp_ok = True
+            print("** SMTP OK", account)
+    print("** IMAP and SMTP logins successful", account)
+
+
+def wait_msgs_changed(account, chat_id, msg_id=None):
+    ev = account._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
+    assert ev[1] == chat_id
+    if msg_id is not None:
+        assert ev[2] == msg_id
+    return ev[2]

python/tests/data/key

@@ -1 +0,0 @@
-../../../test-data/key

python/tests/stress_test_db.py

@@ -1,105 +0,0 @@
-import time
-import os
-from queue import Queue
-
-import deltachat
-
-
-def test_db_busy_error(acfactory, tmpdir):
-    starttime = time.time()
-
-    def log(string):
-        print("%3.2f %s" % (time.time() - starttime, string))
-
-    # make a number of accounts
-    accounts = acfactory.get_many_online_accounts(5, quiet=False)
-    log("created %s accounts" % len(accounts))
-
-    # put a bigfile into each account
-    for acc in accounts:
-        acc.bigfile = os.path.join(acc.get_blobdir(), "bigfile")
-        with open(acc.bigfile, "wb") as f:
-            f.write(b"01234567890"*1000_000)
-    log("created %s bigfiles" % len(accounts))
-
-    contact_addrs = [acc.get_self_contact().addr for acc in accounts]
-    chat = accounts[0].create_group_chat("stress-group")
-    for addr in contact_addrs[1:]:
-        chat.add_contact(chat.account.create_contact(addr))
-
-    # setup auto-responder bots which report back failures/actions
-    report_queue = Queue()
-
-    def report_func(replier, report_type, *report_args):
-        report_queue.put((replier, report_type, report_args))
-
-    # each replier receives all events and sends report events to receive_queue
-    repliers = []
-    for acc in accounts:
-        replier = AutoReplier(acc, num_send=1000, num_bigfiles=0, report_func=report_func)
-        acc.add_account_plugin(replier)
-        repliers.append(replier)
-
-    # kick off message sending
-    # after which repliers will reply to each other
-    chat.send_text("hello")
-
-    alive_count = len(accounts)
-    while alive_count > 0:
-        replier, report_type, report_args = report_queue.get(10)
-        addr = replier.account.get_self_contact().addr
-        assert addr
-        if report_type == ReportType.exit:
-            alive_count -= 1
-            log("{} EXIT -- remaining: {}".format(addr, alive_count))
-            replier.account.shutdown(wait=True)
-        elif report_type == ReportType.message_sent:
-            log("{} sent message: {}".format(addr, report_args[0].text))
-        elif report_type == ReportType.message_incoming:
-            log("{} incoming message: {}".format(addr, report_args[0].text))
-        elif report_type == ReportType.ffi_error:
-            log("{} ERROR: {}".format(addr, report_args[0]))
-            replier.account.shutdown(wait=True)
-            alive_count -= 1
-
-
-class ReportType:
-    exit = "exit"
-    message_sent = "message-sent"
-    ffi_error = "ffi-error"
-    message_incoming = "message-incoming"
-
-
-class AutoReplier:
-    def __init__(self, account, report_func, num_send, num_bigfiles):
-        self.account = account
-        self.report_func = report_func
-        self.num_send = num_send
-        self.num_bigfiles = num_bigfiles
-        self.current_sent = 0
-
-    @deltachat.account_hookimpl
-    def ac_incoming_message(self, message):
-        if self.current_sent >= self.num_send:
-            return
-        message.accept_sender_contact()
-        message.mark_seen()
-        self.report_func(self, ReportType.message_incoming, message)
-
-        self.current_sent += 1
-        # we are still alive, let's send a reply
-        if self.num_bigfiles and self.current_sent % self.num_bigfiles == 0:
-            message.chat.send_text("send big file as reply to: {}".format(message.text))
-            msg = message.chat.send_file(self.account.bigfile)
-        else:
-            msg = message.chat.send_text("got message id {}, small text reply".format(message.id))
-            assert msg.text
-        self.report_func(self, ReportType.message_sent, msg)
-        if self.current_sent >= self.num_send:
-            self.report_func(self, ReportType.exit)
-            return
-
-    @deltachat.account_hookimpl
-    def ac_process_ffi_event(self, ffi_event):
-        if ffi_event.name == "DC_EVENT_ERROR":
-            self.report_func(self, ReportType.ffi_error, ffi_event)

python/tests/test_increation.py

@@ -6,18 +6,10 @@ import shutil
 import pytest
 from filecmp import cmp
 
-from conftest import wait_configuration_progress
+from conftest import wait_configuration_progress, wait_msgs_changed
 from deltachat import const
 
 
-def wait_msgs_changed(account, chat_id, msg_id=None):
-    ev = account._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
-    assert ev.data1 == chat_id
-    if msg_id is not None:
-        assert ev.data2 == msg_id
-    return ev.data2
-
-
 class TestOnlineInCreation:
     def test_increation_not_blobdir(self, tmpdir, acfactory, lp):
         ac1 = acfactory.get_online_configuring_account()
@@ -99,22 +91,22 @@ class TestOnlineInCreation:
         assert fwd_msg.is_out_pending() or fwd_msg.is_out_delivered()
 
         lp.sec("wait for the messages to be delivered to SMTP")
-        ev = ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
-        assert ev.data1 == chat.id
-        assert ev.data2 == prepared_original.id
-        ev = ac1._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
-        assert ev.data1 == chat2.id
-        assert ev.data2 == forwarded_id
+        ev = ac1._evlogger.get_matching("DC_EVENT_MSG_DELIVERED")
+        assert ev[1] == chat.id
+        assert ev[2] == prepared_original.id
+        ev = ac1._evlogger.get_matching("DC_EVENT_MSG_DELIVERED")
+        assert ev[1] == chat2.id
+        assert ev[2] == forwarded_id
 
         lp.sec("wait1 for original or forwarded messages to arrive")
-        ev1 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
-        assert ev1.data1 > const.DC_CHAT_ID_LAST_SPECIAL
-        received_original = ac2.get_message_by_id(ev1.data2)
+        ev1 = ac2._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
+        assert ev1[1] > const.DC_CHAT_ID_LAST_SPECIAL
+        received_original = ac2.get_message_by_id(ev1[2])
         assert cmp(received_original.filename, orig, shallow=False)
 
         lp.sec("wait2 for original or forwarded messages to arrive")
-        ev2 = ac2._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
-        assert ev2.data1 > const.DC_CHAT_ID_LAST_SPECIAL
-        assert ev2.data1 != ev1.data1
-        received_copy = ac2.get_message_by_id(ev2.data2)
+        ev2 = ac2._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
+        assert ev2[1] > const.DC_CHAT_ID_LAST_SPECIAL
+        assert ev2[1] != ev1[1]
+        received_copy = ac2.get_message_by_id(ev2[2])
         assert cmp(received_copy.filename, orig, shallow=False)

python/tests/test_lowlevel.py

@@ -1,9 +1,8 @@
 from __future__ import print_function
 from deltachat import capi, cutil, const, set_context_callback, clear_context_callback
-from deltachat import register_global_plugin
-from deltachat.hookspec import global_hookimpl
 from deltachat.capi import ffi
 from deltachat.capi import lib
+from deltachat.account import EventLogger
 
 
 def test_empty_context():
@@ -18,31 +17,29 @@ def test_callback_None2int():
     clear_context_callback(ctx)
 
 
-def test_dc_close_events(tmpdir, acfactory):
-    ac1 = acfactory.get_unconfigured_account()
-
-    # register after_shutdown function
-    shutdowns = []
-
-    class ShutdownPlugin:
-        @global_hookimpl
-        def dc_account_after_shutdown(self, account):
-            assert account._dc_context is None
-            shutdowns.append(account)
-    register_global_plugin(ShutdownPlugin())
-    assert hasattr(ac1, "_dc_context")
-    ac1.shutdown()
-    assert shutdowns == [ac1]
+def test_dc_close_events(tmpdir):
+    ctx = ffi.gc(
+        capi.lib.dc_context_new(capi.lib.py_dc_callback, ffi.NULL, ffi.NULL),
+        lib.dc_context_unref,
+    )
+    evlog = EventLogger(ctx)
+    evlog.set_timeout(5)
+    set_context_callback(
+        ctx,
+        lambda ctx, evt_name, data1, data2: evlog(evt_name, data1, data2)
+    )
+    p = tmpdir.join("hello.db")
+    lib.dc_open(ctx, p.strpath.encode("ascii"), ffi.NULL)
+    capi.lib.dc_close(ctx)
 
     def find(info_string):
-        evlog = ac1._evtracker
         while 1:
             ev = evlog.get_matching("DC_EVENT_INFO", check_error=False)
-            data2 = ev.data2
+            data2 = ev[2]
             if info_string in data2:
                 return
             else:
-                print("skipping event", ev)
+                print("skipping event", *ev)
 
     find("disconnecting inbox-thread")
     find("disconnecting sentbox-thread")
@@ -92,10 +89,10 @@ def test_markseen_invalid_message_ids(acfactory):
     contact1 = ac1.create_contact(email="some1@example.com", name="some1")
     chat = ac1.create_chat_by_contact(contact1)
     chat.send_text("one messae")
-    ac1._evtracker.get_matching("DC_EVENT_MSGS_CHANGED")
+    ac1._evlogger.get_matching("DC_EVENT_MSGS_CHANGED")
     msg_ids = [9]
     lib.dc_markseen_msgs(ac1._dc_context, msg_ids, len(msg_ids))
-    ac1._evtracker.ensure_event_not_queued("DC_EVENT_WARNING|DC_EVENT_ERROR")
+    ac1._evlogger.ensure_event_not_queued("DC_EVENT_WARNING|DC_EVENT_ERROR")
 
 
 def test_get_special_message_id_returns_empty_message(acfactory):

python/tox.ini

@@ -7,14 +7,13 @@ envlist =
 
 [testenv]
 commands = 
-    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored {posargs: tests examples}
+    pytest -n6 --reruns 2 --reruns-delay 5 -v -rsXx --ignored {posargs:tests}
     python tests/package_wheels.py {toxworkdir}/wheelhouse
 passenv = 
     TRAVIS 
     DCC_RS_DEV
     DCC_RS_TARGET
     DCC_PY_LIVECONFIG
-    DCC_NEW_TMP_EMAIL 
     CARGO_TARGET_DIR
     RUSTC_WRAPPER
 deps = 
@@ -41,13 +40,13 @@ deps =
     restructuredtext_lint 
 commands =
     flake8 src/deltachat
-    flake8 tests/ examples/
+    flake8 tests/
     rst-lint --encoding 'utf-8' README.rst
 
 [testenv:doc]
 changedir=doc
 deps =
-    sphinx
+    sphinx==2.2.0
     breathe
 commands =
     sphinx-build -Q -w toxdoc-warnings.log -b html . _build/html
@@ -67,6 +66,7 @@ commands =
 
 [pytest]
 addopts = -v -ra --strict-markers
+python_files = tests/test_*.py
 norecursedirs = .tox 
 xfail_strict=true
 timeout = 90

rust-toolchain

@@ -1 +1 @@
-nightly-2020-03-12
+nightly-2019-11-06

scripts/proxy.py

@@ -1,80 +0,0 @@
-#!/usr/bin/env python3
-# Examples:
-#
-# Original server that doesn't use SSL:
-# ./proxy.py 8080 imap.nauta.cu 143
-# ./proxy.py 8081 smtp.nauta.cu 25
-#
-# Original server that uses SSL:
-# ./proxy.py 8080 testrun.org 993 --ssl
-# ./proxy.py 8081 testrun.org 465 --ssl
-
-from datetime import datetime
-import argparse
-import selectors
-import ssl
-import socket
-import socketserver
-
-
-class Proxy(socketserver.ThreadingTCPServer):
-    allow_reuse_address = True
-
-    def __init__(self, proxy_host, proxy_port, real_host, real_port, use_ssl):
-        self.real_host = real_host
-        self.real_port = real_port
-        self.use_ssl = use_ssl
-        super().__init__((proxy_host, proxy_port), RequestHandler)
-
-
-class RequestHandler(socketserver.BaseRequestHandler):
-
-    def handle(self):
-        print('{} - {} CONNECTED.'.format(datetime.now(), self.client_address))
-
-        total = 0
-        real_server = (self.server.real_host, self.server.real_port)
-        with socket.create_connection(real_server) as sock:
-            if self.server.use_ssl:
-                context = ssl.create_default_context()
-                sock = context.wrap_socket(
-                    sock, server_hostname=real_server[0])
-
-            forward = {self.request: sock, sock: self.request}
-
-            sel = selectors.DefaultSelector()
-            sel.register(self.request, selectors.EVENT_READ,
-                         self.client_address)
-            sel.register(sock, selectors.EVENT_READ, real_server)
-
-            active = True
-            while active:
-                events = sel.select()
-                for key, mask in events:
-                    print('\n{} - {} wrote:'.format(datetime.now(), key.data))
-                    data = key.fileobj.recv(1024)
-                    received = len(data)
-                    total += received
-                    print(data)
-                    print('{} Bytes\nTotal: {} Bytes'.format(received, total))
-                    if data:
-                        forward[key.fileobj].sendall(data)
-                    else:
-                        print('\nCLOSING CONNECTION.\n\n')
-                        forward[key.fileobj].close()
-                        key.fileobj.close()
-                        active = False
-
-
-if __name__ == '__main__':
-    p = argparse.ArgumentParser(description='Simple Python Proxy')
-    p.add_argument(
-        "proxy_port", help="the port where the proxy will listen", type=int)
-    p.add_argument('host', help="the real host")
-    p.add_argument('port', help="the port of the real host", type=int)
-    p.add_argument("--ssl", help="use ssl to connect to the real host",
-                   action="store_true")
-    args = p.parse_args()
-
-    with Proxy('', args.proxy_port, args.host, args.port, args.ssl) as proxy:
-        proxy.serve_forever()

set_core_version.py

@@ -35,7 +35,7 @@ if __name__ == "__main__":
     if len(sys.argv) < 2:
         for x in ("Cargo.toml", "deltachat-ffi/Cargo.toml"):
             print("{}: {}".format(x, read_toml_version(x)))
-        raise SystemExit("need argument: new version, example: 1.25.0")
+        raise SystemExit("need argument: new version, example 1.0.0-beta.27")
     newversion = sys.argv[1]
     if newversion.count(".") < 2:
         raise SystemExit("need at least two dots in version")
@@ -45,7 +45,7 @@ if __name__ == "__main__":
     assert core_toml == ffi_toml, (core_toml, ffi_toml)
 
     for line in open("CHANGELOG.md"):
-        ## 1.25.0
+        ## 1.0.0-beta5
         if line.startswith("## "):
             if line[2:].strip().startswith(newversion):
                 break

spec.md

@@ -1,6 +1,6 @@
 # Chat-over-Email specification
 
-Version 0.30.0
+Version 0.20.0
 
 This document describes how emails can be used
 to implement typical messenger functions
@@ -17,9 +17,6 @@ while staying compatible to existing MUAs.
     - [Change group name](#change-group-name)
     - [Set group image](#set-group-image)
 - [Set profile image](#set-profile-image)
-- [Locations](#locations)
-    - [User locations](#user-locations)
-    - [Points of interest](#points-of-interest)
 - [Miscellaneous](#miscellaneous)
 
 
@@ -32,7 +29,8 @@ Messages SHOULD be encrypted by the
 Meta data (at least the subject and all chat-headers) SHOULD be encrypted
 by the [Memoryhole](https://github.com/autocrypt/memoryhole) standard.
 If Memoryhole is not used,
-the subject of encrypted messages SHOULD be replaced by the string `...`.
+the subject of encrypted messages SHOULD be replaced by the string
+`Chat: Encrypted message` where the part after the colon MAY be localized.
 
 
 # Outgoing messages
@@ -115,7 +113,6 @@ but MUAs typically expose the sender in the UI.
 Groups are chats with usually more than one recipient,
 each defined by an email-address.
 The sender plus the recipients are the group members.
-All group members form the member list.
 
 To allow different groups with the same members,
 groups are identified by a group-id.
@@ -138,7 +135,8 @@ The group-name MUST be written to `Chat-Group-Name` header
 to join a group chat on a second device any time).
 
 The `Subject` header of outgoing group messages
-SHOULD be set to the group-name.
+SHOULD start with the characters `Chat:`
+followed by the group-name and a colon followed by an excerpt of the message.
 
 To identify the group-id on replies from normal MUAs,
 the group-id MUST also be added to the message-id of outgoing messages.
@@ -179,22 +177,12 @@ to a normal single-user chat with the email-address given in `From`.
 
 ## Add and remove members
 
-Messenger clients MUST init the member list
-from the `From`/`To` headers on the first group message.
-
-When a member is added later,
-a `Chat-Group-Member-Added` action header must be sent
-with the value set to the email-address of the added member.
-When receiving a `Chat-Group-Member-Added` header, however,
-_all missing_ members  the `From`/`To` headers has to be added.
-This is to mitigate problems when receiving messages
-in different orders, esp. on creating new groups.
-
-To remove a member, a `Chat-Group-Member-Removed` header must be sent
-with the value set to the email-address of the member to remove.
-When receiving a `Chat-Group-Member-Removed` header,
-only exaxtly the given member has to be removed from the member list.
-
+Messenger clients MUST construct the member list
+from the `From`/`To` headers only on the first group message
+or if they see a `Chat-Group-Member-Added`
+or `Chat-Group-Member-Removed` action header.
+Both headers MUST have the email-address
+of the added or removed member as the value.
 Messenger clients MUST NOT construct the member list
 on other group messages
 (this is to avoid accidentally altered To-lists in normal MUAs;
@@ -344,64 +332,6 @@ To save data, it is RECOMMENDED to add a `Chat-User-Avatar` header
 only on image changes.
 
 
-# Locations
-
-Locations can be attachted to messages using
-[standard kml-files](https://www.opengeospatial.org/standards/kml/)
-with well-known names.
-
-
-## User locations
-
-To send the location of the sender,
-the app can attach a file with the name `location.kml`.
-The file can contain one or more locations.
-Apps that support location streaming will typically collect some location events
-and send them together in one file.
-As each location has an independent timestamp,
-the apps can show the location as a track.
-
-Note that the `addr` attribute inside the  `location.kml` file
-MUST match the users email-address.
-Otherwise, the file is discarded silently;
-this is to protect against getting wrong locations,
-eg. forwarded from a normal MUA.
-
-    <?xml version="1.0" encoding="UTF-8"?>
-    <kml xmlns="http://www.opengis.net/kml/2.2">
-      <Document addr="ndh@deltachat.de">
-        <Placemark>
-          <Timestamp><when>2020-01-11T20:40:19Z</when></Timestamp>
-          <Point><coordinates accuracy="1.2">1.234,5.678</coordinates></Point>
-        </Placemark>
-        <Placemark>
-          <Timestamp><when>2020-01-11T20:40:25Z</when></Timestamp>
-          <Point><coordinates accuracy="5.4">7.654,3.21</coordinates></Point>
-        </Placemark>
-      </Document>
-    </kml>
-
-
-## Points of interest
-
-To send an "Point of interest", a POI,
-use a normal message and attach a file with the name  `message.kml`.
-In contrast to user locations, this file should contain only one location
-and an address-attribute is not needed -
-as the location belongs to the message content,
-it is fine if the location is detected on forwarding etc.
-
-    <?xml version="1.0" encoding="UTF-8"?>
-    <kml xmlns="http://www.opengis.net/kml/2.2">
-      <Document>
-        <Placemark>
-          <Timestamp><when>2020-01-01T20:40:19Z</when></Timestamp>
-          <Point><coordinates accuracy="1.2">1.234,5.678</coordinates></Point>
-        </Placemark>
-      </Document>
-    </kml>
-
-
 # Miscellaneous
 
 Messengers SHOULD use the header `In-Reply-To` as usual.
@@ -438,4 +368,4 @@ as the sending time of the message as indicated by its Date header,
 or the time of first receipt if that date is in the future or unavailable.
 
 
-Copyright © 2017-2020 Delta Chat contributors.
+Copyright © 2017-2019 Delta Chat contributors.

src/aheader.rs

@@ -75,7 +75,7 @@ impl Aheader {
         wanted_from: &str,
         headers: &[mailparse::MailHeader<'_>],
     ) -> Option<Self> {
-        if let Some(value) = headers.get_header_value(HeaderDef::Autocrypt) {
+        if let Ok(Some(value)) = headers.get_header_value(HeaderDef::Autocrypt) {
             match Self::from_str(&value) {
                 Ok(header) => {
                     if addr_cmp(&header.addr, wanted_from) {
@@ -127,10 +127,14 @@ impl str::FromStr for Aheader {
             .split(';')
             .filter_map(|a| {
                 let attribute: Vec<&str> = a.trim().splitn(2, '=').collect();
-                match &attribute[..] {
-                    [key, value] => Some((key.trim().to_string(), value.trim().to_string())),
-                    _ => None,
+                if attribute.len() < 2 {
+                    return None;
                 }
+
+                Some((
+                    attribute[0].trim().to_string(),
+                    attribute[1].trim().to_string(),
+                ))
             })
             .collect();
 

src/blob.rs

@@ -6,13 +6,13 @@ use std::fs;
 use std::io::Write;
 use std::path::{Path, PathBuf};
 
-use image::GenericImageView;
-use thiserror::Error;
-
+use self::image::GenericImageView;
 use crate::constants::AVATAR_SIZE;
 use crate::context::Context;
 use crate::events::Event;
 
+extern crate image;
+
 /// Represents a file in the blob directory.
 ///
 /// The object has a name, which will always be valid UTF-8.  Having a
@@ -56,6 +56,7 @@ impl<'a> BlobObject<'a> {
                 blobdir: blobdir.to_path_buf(),
                 blobname: name.clone(),
                 cause: err,
+                backtrace: failure::Backtrace::new(),
             })?;
         let blob = BlobObject {
             blobdir,
@@ -83,6 +84,7 @@ impl<'a> BlobObject<'a> {
                             blobdir: dir.to_path_buf(),
                             blobname: name,
                             cause: err,
+                            backtrace: failure::Backtrace::new(),
                         });
                     } else {
                         name = format!("{}-{}{}", stem, rand::random::<u32>(), ext);
@@ -95,6 +97,7 @@ impl<'a> BlobObject<'a> {
             blobdir: dir.to_path_buf(),
             blobname: name,
             cause: std::io::Error::new(std::io::ErrorKind::Other, "supposedly unreachable"),
+            backtrace: failure::Backtrace::new(),
         })
     }
 
@@ -119,6 +122,7 @@ impl<'a> BlobObject<'a> {
             blobname: String::from(""),
             src: src.as_ref().to_path_buf(),
             cause: err,
+            backtrace: failure::Backtrace::new(),
         })?;
         let (stem, ext) = BlobObject::sanitise_name(&src.as_ref().to_string_lossy());
         let (name, mut dst_file) = BlobObject::create_new_file(context.get_blobdir(), &stem, &ext)?;
@@ -134,6 +138,7 @@ impl<'a> BlobObject<'a> {
                 blobname: name_for_err,
                 src: src.as_ref().to_path_buf(),
                 cause: err,
+                backtrace: failure::Backtrace::new(),
             }
         })?;
         let blob = BlobObject {
@@ -193,14 +198,17 @@ impl<'a> BlobObject<'a> {
             .map_err(|_| BlobError::WrongBlobdir {
                 blobdir: context.get_blobdir().to_path_buf(),
                 src: path.as_ref().to_path_buf(),
+                backtrace: failure::Backtrace::new(),
             })?;
         if !BlobObject::is_acceptible_blob_name(&rel_path) {
             return Err(BlobError::WrongName {
                 blobname: path.as_ref().to_path_buf(),
+                backtrace: failure::Backtrace::new(),
             });
         }
         let name = rel_path.to_str().ok_or_else(|| BlobError::WrongName {
             blobname: path.as_ref().to_path_buf(),
+            backtrace: failure::Backtrace::new(),
         })?;
         BlobObject::from_name(context, name.to_string())
     }
@@ -228,6 +236,7 @@ impl<'a> BlobObject<'a> {
         if !BlobObject::is_acceptible_blob_name(&name) {
             return Err(BlobError::WrongName {
                 blobname: PathBuf::from(name),
+                backtrace: failure::Backtrace::new(),
             });
         }
         Ok(BlobObject {
@@ -350,6 +359,7 @@ impl<'a> BlobObject<'a> {
             blobdir: context.get_blobdir().to_path_buf(),
             blobname: blob_abs.to_str().unwrap_or_default().to_string(),
             cause: err,
+            backtrace: failure::Backtrace::new(),
         })?;
 
         if img.width() <= AVATAR_SIZE && img.height() <= AVATAR_SIZE {
@@ -362,6 +372,7 @@ impl<'a> BlobObject<'a> {
             blobdir: context.get_blobdir().to_path_buf(),
             blobname: blob_abs.to_str().unwrap_or_default().to_string(),
             cause: err,
+            backtrace: failure::Backtrace::new(),
         })?;
 
         Ok(())
@@ -375,41 +386,98 @@ impl<'a> fmt::Display for BlobObject<'a> {
 }
 
 /// Errors for the [BlobObject].
-#[derive(Debug, Error)]
+#[derive(Fail, Debug)]
 pub enum BlobError {
-    #[error("Failed to create blob {blobname} in {}", .blobdir.display())]
     CreateFailure {
         blobdir: PathBuf,
         blobname: String,
-        #[source]
+        #[cause]
         cause: std::io::Error,
+        backtrace: failure::Backtrace,
     },
-    #[error("Failed to write data to blob {blobname} in {}", .blobdir.display())]
     WriteFailure {
         blobdir: PathBuf,
         blobname: String,
-        #[source]
+        #[cause]
         cause: std::io::Error,
+        backtrace: failure::Backtrace,
     },
-    #[error("Failed to copy data from {} to blob {blobname} in {}", .src.display(), .blobdir.display())]
     CopyFailure {
         blobdir: PathBuf,
         blobname: String,
         src: PathBuf,
-        #[source]
+        #[cause]
         cause: std::io::Error,
+        backtrace: failure::Backtrace,
     },
-    #[error("Failed to recode to blob {blobname} in {}", .blobdir.display())]
     RecodeFailure {
         blobdir: PathBuf,
         blobname: String,
-        #[source]
+        #[cause]
         cause: image::ImageError,
+        backtrace: failure::Backtrace,
     },
-    #[error("File path {} is not in {}", .src.display(), .blobdir.display())]
-    WrongBlobdir { blobdir: PathBuf, src: PathBuf },
-    #[error("Blob has a badname {}", .blobname.display())]
-    WrongName { blobname: PathBuf },
+    WrongBlobdir {
+        blobdir: PathBuf,
+        src: PathBuf,
+        backtrace: failure::Backtrace,
+    },
+    WrongName {
+        blobname: PathBuf,
+        backtrace: failure::Backtrace,
+    },
+}
+
+// Implementing Display is done by hand because the failure
+// #[fail(display = "...")] syntax does not allow using
+// `blobdir.display()`.
+impl fmt::Display for BlobError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        // Match on the data rather than kind, they are equivalent for
+        // identifying purposes but contain the actual data we need.
+        match &self {
+            BlobError::CreateFailure {
+                blobdir, blobname, ..
+            } => write!(
+                f,
+                "Failed to create blob {} in {}",
+                blobname,
+                blobdir.display()
+            ),
+            BlobError::WriteFailure {
+                blobdir, blobname, ..
+            } => write!(
+                f,
+                "Failed to write data to blob {} in {}",
+                blobname,
+                blobdir.display()
+            ),
+            BlobError::CopyFailure {
+                blobdir,
+                blobname,
+                src,
+                ..
+            } => write!(
+                f,
+                "Failed to copy data from {} to blob {} in {}",
+                src.display(),
+                blobname,
+                blobdir.display(),
+            ),
+            BlobError::RecodeFailure {
+                blobdir, blobname, ..
+            } => write!(f, "Failed to recode {} in {}", blobname, blobdir.display(),),
+            BlobError::WrongBlobdir { blobdir, src, .. } => write!(
+                f,
+                "File path {} is not in blobdir {}",
+                src.display(),
+                blobdir.display(),
+            ),
+            BlobError::WrongName { blobname, .. } => {
+                write!(f, "Blob has a bad name: {}", blobname.display(),)
+            }
+        }
+    }
 }
 
 #[cfg(test)]

src/chat.rs

@@ -15,7 +15,7 @@ use crate::constants::*;
 use crate::contact::*;
 use crate::context::Context;
 use crate::dc_tools::*;
-use crate::error::{bail, ensure, format_err, Error};
+use crate::error::Error;
 use crate::events::Event;
 use crate::job::*;
 use crate::message::{self, InvalidMsgId, Message, MessageState, MsgId};
@@ -28,9 +28,7 @@ use crate::stock::StockMessage;
 ///
 /// Some chat IDs are reserved to identify special chat types.  This
 /// type can represent both the special as well as normal chats.
-#[derive(
-    Debug, Copy, Clone, Default, PartialEq, Eq, Serialize, Deserialize, Hash, PartialOrd, Ord,
-)]
+#[derive(Debug, Copy, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
 pub struct ChatId(u32);
 
 impl ChatId {
@@ -137,18 +135,14 @@ impl ChatId {
     }
 
     /// Archives or unarchives a chat.
-    pub fn set_visibility(
-        self,
-        context: &Context,
-        visibility: ChatVisibility,
-    ) -> Result<(), Error> {
+    pub fn set_archived(self, context: &Context, new_archived: bool) -> Result<(), Error> {
         ensure!(
             !self.is_special(),
             "bad chat_id, can not be special chat: {}",
             self
         );
 
-        if visibility == ChatVisibility::Archived {
+        if new_archived {
             sql::execute(
                 context,
                 &context.sql,
@@ -161,7 +155,7 @@ impl ChatId {
             context,
             &context.sql,
             "UPDATE chats SET archived=? WHERE id=?;",
-            params![visibility, self],
+            params![new_archived, self],
         )?;
 
         context.call_cb(Event::MsgsChanged {
@@ -172,13 +166,13 @@ impl ChatId {
         Ok(())
     }
 
-    // note that unarchive() is not the same as set_visibility(Normal) -
-    // eg. unarchive() does not modify pinned chats and does not send events.
+    // note that unarchive() is not the same as set_archived(false) -
+    // eg. unarchive() does not send events as done for set_archived(false).
     pub fn unarchive(self, context: &Context) -> Result<(), Error> {
         sql::execute(
             context,
             &context.sql,
-            "UPDATE chats SET archived=0 WHERE id=? and archived=1",
+            "UPDATE chats SET archived=0 WHERE id=?",
             params![self],
         )?;
         Ok(())
@@ -286,7 +280,10 @@ impl ChatId {
     /// Returns `true`, if message was deleted, `false` otherwise.
     fn maybe_delete_draft(self, context: &Context) -> bool {
         match self.get_draft_msg_id(context) {
-            Some(msg_id) => msg_id.delete_from_db(context).is_ok(),
+            Some(msg_id) => {
+                Message::delete_from_db(context, msg_id);
+                true
+            }
             None => false,
         }
     }
@@ -324,7 +321,7 @@ impl ChatId {
                 time(),
                 msg.viewtype,
                 MessageState::OutDraft,
-                msg.text.as_deref().unwrap_or(""),
+                msg.text.as_ref().map(String::as_str).unwrap_or(""),
                 msg.param.to_string(),
                 1,
             ],
@@ -359,74 +356,6 @@ impl ChatId {
             .unwrap_or_default() as usize
     }
 
-    pub(crate) fn get_param(self, context: &Context) -> Result<Params, Error> {
-        let res: Option<String> = context
-            .sql
-            .query_get_value_result("SELECT param FROM chats WHERE id=?", params![self])?;
-        Ok(res
-            .map(|s| s.parse().unwrap_or_default())
-            .unwrap_or_default())
-    }
-
-    // Returns true if chat is a saved messages chat.
-    pub fn is_self_talk(self, context: &Context) -> Result<bool, Error> {
-        Ok(self.get_param(context)?.exists(Param::Selftalk))
-    }
-
-    /// Returns true if chat is a device chat.
-    pub fn is_device_talk(self, context: &Context) -> Result<bool, Error> {
-        Ok(self.get_param(context)?.exists(Param::Devicetalk))
-    }
-
-    fn parent_query<T, F>(self, context: &Context, fields: &str, f: F) -> sql::Result<Option<T>>
-    where
-        F: FnOnce(&rusqlite::Row) -> rusqlite::Result<T>,
-    {
-        let sql = &context.sql;
-        let query = format!(
-            "SELECT {} \
-             FROM msgs WHERE chat_id=? AND state NOT IN (?, ?, ?, ?) AND NOT hidden \
-             ORDER BY timestamp DESC, id DESC \
-             LIMIT 1;",
-            fields
-        );
-        sql.query_row_optional(
-            query,
-            params![
-                self,
-                MessageState::OutPreparing,
-                MessageState::OutDraft,
-                MessageState::OutPending,
-                MessageState::OutFailed
-            ],
-            f,
-        )
-    }
-
-    fn get_parent_mime_headers(self, context: &Context) -> Option<(String, String, String)> {
-        let collect = |row: &rusqlite::Row| Ok((row.get(0)?, row.get(1)?, row.get(2)?));
-        self.parent_query(
-            context,
-            "rfc724_mid, mime_in_reply_to, mime_references",
-            collect,
-        )
-        .ok()
-        .flatten()
-    }
-
-    fn parent_is_encrypted(self, context: &Context) -> Result<bool, Error> {
-        let collect = |row: &rusqlite::Row| Ok(row.get(0)?);
-        let packed: Option<String> = self.parent_query(context, "param", collect)?;
-
-        if let Some(ref packed) = packed {
-            let param = packed.parse::<Params>()?;
-            Ok(param.exists(Param::GuaranteeE2ee))
-        } else {
-            // No messages
-            Ok(false)
-        }
-    }
-
     /// Bad evil escape hatch.
     ///
     /// Avoid using this, eventually types should be cleaned up enough
@@ -485,12 +414,12 @@ impl rusqlite::types::FromSql for ChatId {
 /// Chat objects are created using eg. `Chat::load_from_db`
 /// and are not updated on database changes;
 /// if you want an update, you have to recreate the object.
-#[derive(Debug, Clone, Deserialize, Serialize)]
+#[derive(Debug, Clone)]
 pub struct Chat {
     pub id: ChatId,
     pub typ: Chattype,
     pub name: String,
-    pub visibility: ChatVisibility,
+    archived: bool,
     pub grpid: String,
     blocked: Blocked,
     pub param: Params,
@@ -514,7 +443,7 @@ impl Chat {
                     name: row.get::<_, String>(1)?,
                     grpid: row.get::<_, String>(2)?,
                     param: row.get::<_, String>(3)?.parse().unwrap_or_default(),
-                    visibility: row.get(4)?,
+                    archived: row.get(4)?,
                     blocked: row.get::<_, Option<_>>(5)?.unwrap_or_default(),
                     is_sending_locations: row.get(6)?,
                     mute_duration: row.get(7)?,
@@ -604,6 +533,76 @@ impl Chat {
         &self.name
     }
 
+    pub fn get_subtitle(&self, context: &Context) -> String {
+        // returns either the address or the number of chat members
+
+        if self.typ == Chattype::Single && self.param.exists(Param::Selftalk) {
+            return context.stock_str(StockMessage::SelfTalkSubTitle).into();
+        }
+
+        if self.typ == Chattype::Single {
+            return context
+                .sql
+                .query_get_value(
+                    context,
+                    "SELECT c.addr
+                       FROM chats_contacts cc
+                       LEFT JOIN contacts c ON c.id=cc.contact_id
+                      WHERE cc.chat_id=?;",
+                    params![self.id],
+                )
+                .unwrap_or_else(|| "Err".into());
+        }
+
+        if self.typ == Chattype::Group || self.typ == Chattype::VerifiedGroup {
+            if self.id.is_deaddrop() {
+                return context.stock_str(StockMessage::DeadDrop).into();
+            }
+            let cnt = get_chat_contact_cnt(context, self.id);
+            return context.stock_string_repl_int(StockMessage::Member, cnt as i32);
+        }
+
+        "Err".to_string()
+    }
+
+    fn parent_query(fields: &str) -> String {
+        // Check for server_uid guarantees that we don't
+        // select a draft or undelivered message.
+        format!(
+            "SELECT {} \
+             FROM msgs WHERE chat_id=?1 AND server_uid!=0 \
+             ORDER BY timestamp DESC, id DESC \
+             LIMIT 1;",
+            fields
+        )
+    }
+
+    fn get_parent_mime_headers(&self, context: &Context) -> Option<(String, String, String)> {
+        let collect = |row: &rusqlite::Row| Ok((row.get(0)?, row.get(1)?, row.get(2)?));
+        let params = params![self.id];
+        let sql = &context.sql;
+
+        let query = Self::parent_query("rfc724_mid, mime_in_reply_to, mime_references");
+
+        sql.query_row(&query, params, collect).ok()
+    }
+
+    fn parent_is_encrypted(&self, context: &Context) -> Result<bool, Error> {
+        let sql = &context.sql;
+        let params = params![self.id];
+        let query = Self::parent_query("param");
+
+        let packed: Option<String> = sql.query_get_value_result(&query, params)?;
+
+        if let Some(ref packed) = packed {
+            let param = packed.parse::<Params>()?;
+            Ok(param.exists(Param::GuaranteeE2ee))
+        } else {
+            // No messages
+            Ok(false)
+        }
+    }
+
     pub fn get_profile_image(&self, context: &Context) -> Option<PathBuf> {
         if let Some(image_rel) = self.param.get(Param::ProfileImage) {
             if !image_rel.is_empty() {
@@ -655,19 +654,21 @@ impl Chat {
             id: self.id,
             type_: self.typ as u32,
             name: self.name.clone(),
-            archived: self.visibility == ChatVisibility::Archived,
+            archived: self.archived,
             param: self.param.to_string(),
             gossiped_timestamp: self.get_gossiped_timestamp(context),
             is_sending_locations: self.is_sending_locations,
             color: self.get_color(context),
             profile_image: self.get_profile_image(context).unwrap_or_else(PathBuf::new),
+            subtitle: self.get_subtitle(context),
             draft,
             is_muted: self.is_muted(),
         })
     }
 
-    pub fn get_visibility(&self) -> ChatVisibility {
-        self.visibility
+    /// Returns true if the chat is archived.
+    pub fn is_archived(&self) -> bool {
+        self.archived
     }
 
     pub fn is_unpromoted(&self) -> bool {
@@ -809,7 +810,7 @@ impl Chat {
                     }
                 }
 
-                if can_encrypt && (all_mutual || self.id.parent_is_encrypted(context)?) {
+                if can_encrypt && (all_mutual || self.parent_is_encrypted(context)?) {
                     msg.param.set_int(Param::GuaranteeE2ee, 1);
                 }
             }
@@ -824,7 +825,7 @@ impl Chat {
             // we do not set In-Reply-To/References in this case.
             if !self.is_self_talk() {
                 if let Some((parent_rfc724_mid, parent_in_reply_to, parent_references)) =
-                    self.id.get_parent_mime_headers(context)
+                    self.get_parent_mime_headers(context)
                 {
                     if !parent_rfc724_mid.is_empty() {
                         new_in_reply_to = parent_rfc724_mid.clone();
@@ -925,40 +926,6 @@ impl Chat {
     }
 }
 
-#[derive(Debug, Copy, Eq, PartialEq, Clone, Serialize, Deserialize)]
-pub enum ChatVisibility {
-    Normal,
-    Archived,
-    Pinned,
-}
-
-impl rusqlite::types::ToSql for ChatVisibility {
-    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
-        let visibility = match &self {
-            ChatVisibility::Normal => 0,
-            ChatVisibility::Archived => 1,
-            ChatVisibility::Pinned => 2,
-        };
-        let val = rusqlite::types::Value::Integer(visibility);
-        let out = rusqlite::types::ToSqlOutput::Owned(val);
-        Ok(out)
-    }
-}
-
-impl rusqlite::types::FromSql for ChatVisibility {
-    fn column_result(value: rusqlite::types::ValueRef) -> rusqlite::types::FromSqlResult<Self> {
-        i64::column_result(value).and_then(|val| {
-            match val {
-                2 => Ok(ChatVisibility::Pinned),
-                1 => Ok(ChatVisibility::Archived),
-                0 => Ok(ChatVisibility::Normal),
-                // fallback to to Normal for unknown values, may happen eg. on imports created by a newer version.
-                _ => Ok(ChatVisibility::Normal),
-            }
-        })
-    }
-}
-
 /// The current state of a chat.
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 #[non_exhaustive]
@@ -1003,6 +970,9 @@ pub struct ChatInfo {
     /// currently.
     pub profile_image: PathBuf,
 
+    /// Subtitle for the chat.
+    pub subtitle: String,
+
     /// The draft message text.
     ///
     /// If the chat has not draft this is an empty string.
@@ -1105,7 +1075,7 @@ pub fn create_by_contact_id(context: &Context, contact_id: u32) -> Result<ChatId
     Ok(chat_id)
 }
 
-pub(crate) fn update_saved_messages_icon(context: &Context) -> Result<(), Error> {
+pub fn update_saved_messages_icon(context: &Context) -> Result<(), Error> {
     // if there is no saved-messages chat, there is nothing to update. this is no error.
     if let Ok((chat_id, _)) = lookup_by_contact_id(context, DC_CONTACT_ID_SELF) {
         let icon = include_bytes!("../assets/icon-saved-messages.png");
@@ -1119,7 +1089,7 @@ pub(crate) fn update_saved_messages_icon(context: &Context) -> Result<(), Error>
     Ok(())
 }
 
-pub(crate) fn update_device_icon(context: &Context) -> Result<(), Error> {
+pub fn update_device_icon(context: &Context) -> Result<(), Error> {
     // if there is no device-chat, there is nothing to update. this is no error.
     if let Ok((chat_id, _)) = lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE) {
         let icon = include_bytes!("../assets/icon-device.png");
@@ -1153,13 +1123,13 @@ fn update_special_chat_name(
     Ok(())
 }
 
-pub(crate) fn update_special_chat_names(context: &Context) -> Result<(), Error> {
+pub fn update_special_chat_names(context: &Context) -> Result<(), Error> {
     update_special_chat_name(context, DC_CONTACT_ID_DEVICE, StockMessage::DeviceMessages)?;
     update_special_chat_name(context, DC_CONTACT_ID_SELF, StockMessage::SavedMessages)?;
     Ok(())
 }
 
-pub(crate) fn create_or_lookup_by_contact_id(
+pub fn create_or_lookup_by_contact_id(
     context: &Context,
     contact_id: u32,
     create_blocked: Blocked,
@@ -1175,31 +1145,33 @@ pub(crate) fn create_or_lookup_by_contact_id(
     let contact = Contact::load_from_db(context, contact_id)?;
     let chat_name = contact.get_display_name();
 
-    context
-        .sql
-        .start_stmt("create_or_lookup_by_contact_id transaction");
-    context.sql.with_conn(|conn| {
-        let tx = conn.transaction()?;
-        tx.execute(
-        "INSERT INTO chats (type, name, param, blocked, created_timestamp) VALUES(?, ?, ?, ?, ?)",
+    sql::execute(
+        context,
+        &context.sql,
+        "INSERT INTO chats (type, name, param, blocked, grpid, created_timestamp) VALUES(?, ?, ?, ?, ?, ?)",
         params![
-            Chattype::Single,
+            100,
             chat_name,
             match contact_id {
                 DC_CONTACT_ID_SELF => "K=1".to_string(), // K = Param::Selftalk
                 DC_CONTACT_ID_DEVICE => "D=1".to_string(), // D = Param::Devicetalk
-                _ => "".to_string(),
+                _ => "".to_string()
             },
             create_blocked as u8,
+            contact.get_addr(),
             time(),
         ]
-        )?;
-        tx.execute(
-            "INSERT INTO chats_contacts (chat_id, contact_id) VALUES((SELECT last_insert_rowid()), ?)",
-            params![contact_id])?;
-        tx.commit()?;
-        Ok(())
-    })?;
+    )?;
+
+    let row_id = sql::get_rowid(context, &context.sql, "chats", "grpid", contact.get_addr());
+    let chat_id = ChatId::new(row_id);
+
+    sql::execute(
+        context,
+        &context.sql,
+        "INSERT INTO chats_contacts (chat_id, contact_id) VALUES(?, ?)",
+        params![chat_id, contact_id],
+    )?;
 
     if contact_id == DC_CONTACT_ID_SELF {
         update_saved_messages_icon(context)?;
@@ -1207,10 +1179,10 @@ pub(crate) fn create_or_lookup_by_contact_id(
         update_device_icon(context)?;
     }
 
-    lookup_by_contact_id(context, contact_id)
+    Ok((chat_id, create_blocked))
 }
 
-pub(crate) fn lookup_by_contact_id(
+pub fn lookup_by_contact_id(
     context: &Context,
     contact_id: u32,
 ) -> Result<(ChatId, Blocked), Error> {
@@ -1264,7 +1236,7 @@ pub fn prepare_msg<'a>(
     Ok(msg_id)
 }
 
-pub(crate) fn msgtype_has_file(msgtype: Viewtype) -> bool {
+pub fn msgtype_has_file(msgtype: Viewtype) -> bool {
     match msgtype {
         Viewtype::Unknown => false,
         Viewtype::Text => false,
@@ -1441,18 +1413,6 @@ pub fn get_chat_msgs(
     flags: u32,
     marker1before: Option<MsgId>,
 ) -> Vec<MsgId> {
-    match delete_device_expired_messages(context) {
-        Err(err) => warn!(context, "Failed to delete expired messages: {}", err),
-        Ok(messages_deleted) => {
-            if messages_deleted {
-                context.call_cb(Event::MsgsChanged {
-                    msg_id: MsgId::new(0),
-                    chat_id: ChatId::new(0),
-                })
-            }
-        }
-    }
-
     let process_row =
         |row: &rusqlite::Row| Ok((row.get::<_, MsgId>("id")?, row.get::<_, i64>("timestamp")?));
     let process_rows = |rows: rusqlite::MappedRows<_>| {
@@ -1587,47 +1547,6 @@ pub fn marknoticed_all_chats(context: &Context) -> Result<(), Error> {
     Ok(())
 }
 
-/// Deletes messages which are expired according to "delete_device_after" setting.
-///
-/// Returns true if any message is deleted, so event can be emitted. If nothing
-/// has been deleted, returns false.
-pub fn delete_device_expired_messages(context: &Context) -> Result<bool, Error> {
-    if let Some(delete_device_after) = context.get_config_delete_device_after() {
-        let threshold_timestamp = time() - delete_device_after;
-
-        let self_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
-            .unwrap_or_default()
-            .0;
-        let device_chat_id = lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
-            .unwrap_or_default()
-            .0;
-
-        // Delete expired messages
-        //
-        // Only update the rows that have to be updated, to avoid emitting
-        // unnecessary "chat modified" events.
-        let rows_modified = context.sql.execute(
-            "UPDATE msgs \
-             SET txt = 'DELETED', chat_id = ? \
-             WHERE timestamp < ? \
-             AND chat_id > ? \
-             AND chat_id != ? \
-             AND chat_id != ?",
-            params![
-                DC_CHAT_ID_TRASH,
-                threshold_timestamp,
-                DC_CHAT_ID_LAST_SPECIAL,
-                self_chat_id,
-                device_chat_id
-            ],
-        )?;
-
-        Ok(rows_modified > 0)
-    } else {
-        Ok(false)
-    }
-}
-
 pub fn get_chat_media(
     context: &Context,
     chat_id: ChatId,
@@ -1702,17 +1621,17 @@ pub fn get_next_media(
             msg_type2,
             msg_type3,
         );
-        for (i, msg_id) in list.iter().enumerate() {
-            if curr_msg_id == *msg_id {
+        for i in 0..list.len() {
+            if curr_msg_id == list[i] {
                 match direction {
                     Direction::Forward => {
                         if i + 1 < list.len() {
-                            ret = list.get(i + 1).copied();
+                            ret = Some(list[i + 1]);
                         }
                     }
                     Direction::Backward => {
                         if i >= 1 {
-                            ret = list.get(i - 1).copied();
+                            ret = Some(list[i - 1]);
                         }
                     }
                 }
@@ -1794,52 +1713,18 @@ pub fn create_group_chat(
     Ok(chat_id)
 }
 
-/// add a contact to the chats_contact table
-pub(crate) fn add_to_chat_contacts_table(
-    context: &Context,
-    chat_id: ChatId,
-    contact_id: u32,
-) -> bool {
-    match sql::execute(
+/* you MUST NOT modify this or the following strings */
+// Context functions to work with chats
+pub fn add_to_chat_contacts_table(context: &Context, chat_id: ChatId, contact_id: u32) -> bool {
+    // add a contact to a chat; the function does not check the type or if any of the record exist or are already
+    // added to the chat!
+    sql::execute(
         context,
         &context.sql,
         "INSERT INTO chats_contacts (chat_id, contact_id) VALUES(?, ?)",
         params![chat_id, contact_id as i32],
-    ) {
-        Ok(()) => true,
-        Err(err) => {
-            error!(
-                context,
-                "could not add {} to chat {} table: {}", contact_id, chat_id, err
-            );
-
-            false
-        }
-    }
-}
-
-/// remove a contact from the chats_contact table
-pub(crate) fn remove_from_chat_contacts_table(
-    context: &Context,
-    chat_id: ChatId,
-    contact_id: u32,
-) -> bool {
-    match sql::execute(
-        context,
-        &context.sql,
-        "DELETE FROM chats_contacts WHERE chat_id=? AND contact_id=?",
-        params![chat_id, contact_id as i32],
-    ) {
-        Ok(()) => true,
-        Err(_) => {
-            warn!(
-                context,
-                "could not remove contact {:?} from chat {:?}", contact_id, chat_id
-            );
-
-            false
-        }
-    }
+    )
+    .is_ok()
 }
 
 /// Adds a contact to the chat.
@@ -1933,9 +1818,16 @@ pub(crate) fn add_contact_to_chat_ex(
         msg.param.set_cmd(SystemMessage::MemberAddedToGroup);
         msg.param.set(Param::Arg, contact.get_addr());
         msg.param.set_int(Param::Arg2, from_handshake.into());
-
         msg.id = send_msg(context, chat_id, &mut msg)?;
+        context.call_cb(Event::MsgsChanged {
+            chat_id,
+            msg_id: msg.id,
+        });
     }
+    context.call_cb(Event::MsgsChanged {
+        chat_id,
+        msg_id: MsgId::new(0),
+    });
     context.call_cb(Event::ChatModified(chat_id));
     Ok(true)
 }
@@ -1955,7 +1847,7 @@ fn real_group_exists(context: &Context, chat_id: ChatId) -> bool {
         .unwrap_or_default()
 }
 
-pub(crate) fn reset_gossiped_timestamp(context: &Context, chat_id: ChatId) -> Result<(), Error> {
+pub fn reset_gossiped_timestamp(context: &Context, chat_id: ChatId) -> Result<(), Error> {
     set_gossiped_timestamp(context, chat_id, 0)
 }
 
@@ -1972,7 +1864,7 @@ pub fn get_gossiped_timestamp(context: &Context, chat_id: ChatId) -> i64 {
         .unwrap_or_default()
 }
 
-pub(crate) fn set_gossiped_timestamp(
+pub fn set_gossiped_timestamp(
     context: &Context,
     chat_id: ChatId,
     timestamp: i64,
@@ -1992,7 +1884,7 @@ pub(crate) fn set_gossiped_timestamp(
     Ok(())
 }
 
-pub(crate) fn shall_attach_selfavatar(context: &Context, chat_id: ChatId) -> Result<bool, Error> {
+pub fn shall_attach_selfavatar(context: &Context, chat_id: ChatId) -> Result<bool, Error> {
     // versions before 12/2019 already allowed to set selfavatar, however, it was never sent to others.
     // to avoid sending out previously set selfavatars unexpectedly we added this additional check.
     // it can be removed after some time.
@@ -2027,7 +1919,7 @@ pub(crate) fn shall_attach_selfavatar(context: &Context, chat_id: ChatId) -> Res
     Ok(needs_attach)
 }
 
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq)]
 pub enum MuteDuration {
     NotMuted,
     Forever,
@@ -2071,17 +1963,18 @@ impl rusqlite::types::FromSql for MuteDuration {
 
 pub fn set_muted(context: &Context, chat_id: ChatId, duration: MuteDuration) -> Result<(), Error> {
     ensure!(!chat_id.is_special(), "Invalid chat ID");
-    if sql::execute(
-        context,
-        &context.sql,
-        "UPDATE chats SET muted_until=? WHERE id=?;",
-        params![duration, chat_id],
-    )
-    .is_ok()
+    if real_group_exists(context, chat_id)
+        && sql::execute(
+            context,
+            &context.sql,
+            "UPDATE chats SET muted_until=? WHERE id=?;",
+            params![duration, chat_id],
+        )
+        .is_ok()
     {
         context.call_cb(Event::ChatModified(chat_id));
     } else {
-        bail!("Failed to set mute duration, chat might not exist -");
+        bail!("Failed to set name");
     }
     Ok(())
 }
@@ -2116,6 +2009,7 @@ pub fn remove_contact_from_chat(
                     )
                 );
             } else {
+                /* we should respect this - whatever we send to the group, it gets discarded anyway! */
                 if let Ok(contact) = Contact::get_by_id(context, contact_id) {
                     if chat.is_promoted() {
                         msg.viewtype = Viewtype::Text;
@@ -2138,20 +2032,23 @@ pub fn remove_contact_from_chat(
                         msg.param.set_cmd(SystemMessage::MemberRemovedFromGroup);
                         msg.param.set(Param::Arg, contact.get_addr());
                         msg.id = send_msg(context, chat_id, &mut msg)?;
+                        context.call_cb(Event::MsgsChanged {
+                            chat_id,
+                            msg_id: msg.id,
+                        });
                     }
                 }
-                // we remove the member from the chat after constructing the
-                // to-be-send message. If between send_msg() and here the
-                // process dies the user will have to re-do the action.  It's
-                // better than the other way round: you removed
-                // someone from DB but no peer or device gets to know about it and
-                // group membership is thus different on different devices.
-                // Note also that sending a message needs all recipients
-                // in order to correctly determine encryption so if we
-                // removed it first, it would complicate the
-                // check/encryption logic.
-                success = remove_from_chat_contacts_table(context, chat_id, contact_id);
-                context.call_cb(Event::ChatModified(chat_id));
+                if sql::execute(
+                    context,
+                    &context.sql,
+                    "DELETE FROM chats_contacts WHERE chat_id=? AND contact_id=?;",
+                    params![chat_id, contact_id as i32],
+                )
+                .is_ok()
+                {
+                    context.call_cb(Event::ChatModified(chat_id));
+                    success = true;
+                }
             }
         }
     }
@@ -2176,10 +2073,7 @@ fn set_group_explicitly_left(context: &Context, grpid: impl AsRef<str>) -> Resul
     Ok(())
 }
 
-pub(crate) fn is_group_explicitly_left(
-    context: &Context,
-    grpid: impl AsRef<str>,
-) -> Result<bool, Error> {
+pub fn is_group_explicitly_left(context: &Context, grpid: impl AsRef<str>) -> Result<bool, Error> {
     context
         .sql
         .exists(
@@ -2401,7 +2295,7 @@ pub fn forward_msgs(context: &Context, msg_ids: &[MsgId], chat_id: ChatId) -> Re
     Ok(())
 }
 
-pub(crate) fn get_chat_contact_cnt(context: &Context, chat_id: ChatId) -> usize {
+pub fn get_chat_contact_cnt(context: &Context, chat_id: ChatId) -> usize {
     context
         .sql
         .query_get_value::<_, isize>(
@@ -2412,7 +2306,7 @@ pub(crate) fn get_chat_contact_cnt(context: &Context, chat_id: ChatId) -> usize
         .unwrap_or_default() as usize
 }
 
-pub(crate) fn get_chat_cnt(context: &Context) -> usize {
+pub fn get_chat_cnt(context: &Context) -> usize {
     if context.sql.is_open() {
         /* no database, no chats - this is no error (needed eg. for information) */
         context
@@ -2527,7 +2421,7 @@ pub fn was_device_msg_ever_added(context: &Context, label: &str) -> Result<bool,
 //   no wrong information are shown in the device chat
 // - deletion in `devmsglabels` makes sure,
 //   deleted messages are resetted and useful messages can be added again
-pub(crate) fn delete_and_reset_all_device_msgs(context: &Context) -> Result<(), Error> {
+pub fn delete_and_reset_all_device_msgs(context: &Context) -> Result<(), Error> {
     context.sql.execute(
         "DELETE FROM msgs WHERE from_id=?;",
         params![DC_CONTACT_ID_DEVICE],
@@ -2541,7 +2435,7 @@ pub(crate) fn delete_and_reset_all_device_msgs(context: &Context) -> Result<(),
 /// Adds an informational message to chat.
 ///
 /// For example, it can be a message showing that a member was added to a group.
-pub(crate) fn add_info_msg(context: &Context, chat_id: ChatId, text: impl AsRef<str>) {
+pub fn add_info_msg(context: &Context, chat_id: ChatId, text: impl AsRef<str>) {
     let rfc724_mid = dc_create_outgoing_rfc724_mid(None, "@device");
 
     if context.sql.execute(
@@ -2596,6 +2490,7 @@ mod tests {
                 "is_sending_locations": false,
                 "color": 15895624,
                 "profile_image": "",
+                "subtitle": "bob@example.com",
                 "draft": "",
                 "is_muted": false
             }
@@ -2663,7 +2558,7 @@ mod tests {
         let chat = Chat::load_from_db(&t.ctx, chat_id).unwrap();
         assert_eq!(chat.id, chat_id);
         assert!(chat.is_self_talk());
-        assert!(chat.visibility == ChatVisibility::Normal);
+        assert!(!chat.archived);
         assert!(!chat.is_device_talk());
         assert!(chat.can_send());
         assert_eq!(chat.name, t.ctx.stock_str(StockMessage::SavedMessages));
@@ -2677,7 +2572,7 @@ mod tests {
         assert_eq!(DC_CHAT_ID_DEADDROP, 1);
         assert!(chat.id.is_deaddrop());
         assert!(!chat.is_self_talk());
-        assert!(chat.visibility == ChatVisibility::Normal);
+        assert!(!chat.archived);
         assert!(!chat.is_device_talk());
         assert!(!chat.can_send());
         assert_eq!(chat.name, t.ctx.stock_str(StockMessage::DeadDrop));
@@ -2883,134 +2778,35 @@ mod tests {
         assert_eq!(DC_GCL_NO_SPECIALS, 0x02);
 
         // archive first chat
-        assert!(chat_id1
-            .set_visibility(&t.ctx, ChatVisibility::Archived)
-            .is_ok());
-        assert!(
-            Chat::load_from_db(&t.ctx, chat_id1)
-                .unwrap()
-                .get_visibility()
-                == ChatVisibility::Archived
-        );
-        assert!(
-            Chat::load_from_db(&t.ctx, chat_id2)
-                .unwrap()
-                .get_visibility()
-                == ChatVisibility::Normal
-        );
+        assert!(chat_id1.set_archived(&t.ctx, true).is_ok());
+        assert!(Chat::load_from_db(&t.ctx, chat_id1).unwrap().is_archived());
+        assert!(!Chat::load_from_db(&t.ctx, chat_id2).unwrap().is_archived());
         assert_eq!(get_chat_cnt(&t.ctx), 2);
         assert_eq!(chatlist_len(&t.ctx, 0), 2); // including DC_CHAT_ID_ARCHIVED_LINK now
         assert_eq!(chatlist_len(&t.ctx, DC_GCL_NO_SPECIALS), 1);
         assert_eq!(chatlist_len(&t.ctx, DC_GCL_ARCHIVED_ONLY), 1);
 
         // archive second chat
-        assert!(chat_id2
-            .set_visibility(&t.ctx, ChatVisibility::Archived)
-            .is_ok());
-        assert!(
-            Chat::load_from_db(&t.ctx, chat_id1)
-                .unwrap()
-                .get_visibility()
-                == ChatVisibility::Archived
-        );
-        assert!(
-            Chat::load_from_db(&t.ctx, chat_id2)
-                .unwrap()
-                .get_visibility()
-                == ChatVisibility::Archived
-        );
+        assert!(chat_id2.set_archived(&t.ctx, true).is_ok());
+        assert!(Chat::load_from_db(&t.ctx, chat_id1).unwrap().is_archived());
+        assert!(Chat::load_from_db(&t.ctx, chat_id2).unwrap().is_archived());
         assert_eq!(get_chat_cnt(&t.ctx), 2);
         assert_eq!(chatlist_len(&t.ctx, 0), 1); // only DC_CHAT_ID_ARCHIVED_LINK now
         assert_eq!(chatlist_len(&t.ctx, DC_GCL_NO_SPECIALS), 0);
         assert_eq!(chatlist_len(&t.ctx, DC_GCL_ARCHIVED_ONLY), 2);
 
         // archive already archived first chat, unarchive second chat two times
-        assert!(chat_id1
-            .set_visibility(&t.ctx, ChatVisibility::Archived)
-            .is_ok());
-        assert!(chat_id2
-            .set_visibility(&t.ctx, ChatVisibility::Normal)
-            .is_ok());
-        assert!(chat_id2
-            .set_visibility(&t.ctx, ChatVisibility::Normal)
-            .is_ok());
-        assert!(
-            Chat::load_from_db(&t.ctx, chat_id1)
-                .unwrap()
-                .get_visibility()
-                == ChatVisibility::Archived
-        );
-        assert!(
-            Chat::load_from_db(&t.ctx, chat_id2)
-                .unwrap()
-                .get_visibility()
-                == ChatVisibility::Normal
-        );
+        assert!(chat_id1.set_archived(&t.ctx, true).is_ok());
+        assert!(chat_id2.set_archived(&t.ctx, false).is_ok());
+        assert!(chat_id2.set_archived(&t.ctx, false).is_ok());
+        assert!(Chat::load_from_db(&t.ctx, chat_id1).unwrap().is_archived());
+        assert!(!Chat::load_from_db(&t.ctx, chat_id2).unwrap().is_archived());
         assert_eq!(get_chat_cnt(&t.ctx), 2);
         assert_eq!(chatlist_len(&t.ctx, 0), 2);
         assert_eq!(chatlist_len(&t.ctx, DC_GCL_NO_SPECIALS), 1);
         assert_eq!(chatlist_len(&t.ctx, DC_GCL_ARCHIVED_ONLY), 1);
     }
 
-    fn get_chats_from_chat_list(ctx: &Context, listflags: usize) -> Vec<ChatId> {
-        let chatlist = Chatlist::try_load(ctx, listflags, None, None).unwrap();
-        let mut result = Vec::new();
-        for chatlist_index in 0..chatlist.len() {
-            result.push(chatlist.get_chat_id(chatlist_index))
-        }
-        result
-    }
-
-    #[test]
-    fn test_pinned() {
-        let t = dummy_context();
-
-        // create 3 chats, wait 1 second in between to get a reliable order (we order by time)
-        let mut msg = Message::new(Viewtype::Text);
-        msg.text = Some("foo".to_string());
-        let msg_id = add_device_msg(&t.ctx, None, Some(&mut msg)).unwrap();
-        let chat_id1 = message::Message::load_from_db(&t.ctx, msg_id)
-            .unwrap()
-            .chat_id;
-        std::thread::sleep(std::time::Duration::from_millis(1000));
-        let chat_id2 = create_by_contact_id(&t.ctx, DC_CONTACT_ID_SELF).unwrap();
-        std::thread::sleep(std::time::Duration::from_millis(1000));
-        let chat_id3 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo").unwrap();
-
-        let chatlist = get_chats_from_chat_list(&t.ctx, DC_GCL_NO_SPECIALS);
-        assert_eq!(chatlist, vec![chat_id3, chat_id2, chat_id1]);
-
-        // pin
-        assert!(chat_id1
-            .set_visibility(&t.ctx, ChatVisibility::Pinned)
-            .is_ok());
-        assert_eq!(
-            Chat::load_from_db(&t.ctx, chat_id1)
-                .unwrap()
-                .get_visibility(),
-            ChatVisibility::Pinned
-        );
-
-        // check if chat order changed
-        let chatlist = get_chats_from_chat_list(&t.ctx, DC_GCL_NO_SPECIALS);
-        assert_eq!(chatlist, vec![chat_id1, chat_id3, chat_id2]);
-
-        // unpin
-        assert!(chat_id1
-            .set_visibility(&t.ctx, ChatVisibility::Normal)
-            .is_ok());
-        assert_eq!(
-            Chat::load_from_db(&t.ctx, chat_id1)
-                .unwrap()
-                .get_visibility(),
-            ChatVisibility::Normal
-        );
-
-        // check if chat order changed back
-        let chatlist = get_chats_from_chat_list(&t.ctx, DC_GCL_NO_SPECIALS);
-        assert_eq!(chatlist, vec![chat_id3, chat_id2, chat_id1]);
-    }
-
     #[test]
     fn test_set_chat_name() {
         let t = dummy_context();
@@ -3105,16 +2901,4 @@ mod tests {
             false
         );
     }
-
-    #[test]
-    fn test_parent_is_encrypted() {
-        let t = dummy_context();
-        let chat_id = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "foo").unwrap();
-        assert!(!chat_id.parent_is_encrypted(&t.ctx).unwrap());
-
-        let mut msg = Message::new(Viewtype::Text);
-        msg.set_text(Some("hello".to_string()));
-        chat_id.set_draft(&t.ctx, Some(&mut msg));
-        assert!(!chat_id.parent_is_encrypted(&t.ctx).unwrap());
-    }
 }

src/chatlist.rs

@@ -1,11 +1,10 @@
 //! # Chat list module
 
-use crate::chat;
 use crate::chat::*;
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::*;
-use crate::error::{bail, ensure, Result};
+use crate::error::Result;
 use crate::lot::Lot;
 use crate::message::{Message, MessageState, MsgId};
 use crate::stock::StockMessage;
@@ -61,7 +60,7 @@ impl Chatlist {
     ///   or "Not now".
     ///   The UI can also offer a "Close" button that calls dc_marknoticed_contact() then.
     /// - DC_CHAT_ID_ARCHIVED_LINK (6) - this special chat is present if the user has
-    ///   archived *any* chat using dc_set_chat_visibility(). The UI should show a link as
+    ///   archived *any* chat using dc_archive_chat(). The UI should show a link as
     ///   "Show archived chats", if the user clicks this item, the UI should show a
     ///   list of all archived chats that can be created by this function hen using
     ///   the DC_GCL_ARCHIVED_ONLY flag.
@@ -74,9 +73,6 @@ impl Chatlist {
     ///   if DC_GCL_ARCHIVED_ONLY is not set, only unarchived chats are returned and
     ///   the pseudo-chat DC_CHAT_ID_ARCHIVED_LINK is added if there are *any* archived
     ///   chats
-    /// - the flag DC_GCL_FOR_FORWARDING sorts "Saved messages" to the top of the chatlist
-    ///   and hides the device-chat,
-    //    typically used on forwarding, may be combined with DC_GCL_NO_SPECIALS
     /// - if the flag DC_GCL_NO_SPECIALS is set, deaddrop and archive link are not added
     ///   to the list (may be used eg. for selecting chats on forwarding, the flag is
     ///   not needed when DC_GCL_ARCHIVED_ONLY is already set)
@@ -92,17 +88,6 @@ impl Chatlist {
         query: Option<&str>,
         query_contact_id: Option<u32>,
     ) -> Result<Self> {
-        let flag_archived_only = 0 != listflags & DC_GCL_ARCHIVED_ONLY;
-        let flag_for_forwarding = 0 != listflags & DC_GCL_FOR_FORWARDING;
-        let flag_no_specials = 0 != listflags & DC_GCL_NO_SPECIALS;
-        let flag_add_alldone_hint = 0 != listflags & DC_GCL_ADD_ALLDONE_HINT;
-
-        // Note that we do not emit DC_EVENT_MSGS_MODIFIED here even if some
-        // messages get deleted to avoid reloading the same chatlist.
-        if let Err(err) = delete_device_expired_messages(context) {
-            warn!(context, "Failed to hide expired messages: {}", err);
-        }
-
         let mut add_archived_link_item = false;
 
         let process_row = |row: &rusqlite::Row| {
@@ -116,14 +101,6 @@ impl Chatlist {
                 .map_err(Into::into)
         };
 
-        let skip_id = if flag_for_forwarding {
-            chat::lookup_by_contact_id(context, DC_CONTACT_ID_DEVICE)
-                .unwrap_or_default()
-                .0
-        } else {
-            ChatId::new(0)
-        };
-
         // select with left join and minimum:
         //
         // - the inner select must use `hidden` and _not_ `m.hidden`
@@ -150,21 +127,18 @@ impl Chatlist {
                                SELECT MAX(timestamp)
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1))
+                                  AND (hidden=0 OR state=?))
                  WHERE c.id>9
                    AND c.blocked=0
-                   AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?2)
+                   AND c.id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?)
                  GROUP BY c.id
-                 ORDER BY c.archived=?3 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                params![MessageState::OutDraft, query_contact_id as i32, ChatVisibility::Pinned],
+                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                params![MessageState::OutDraft, query_contact_id as i32],
                 process_row,
                 process_rows,
             )?
-        } else if flag_archived_only {
+        } else if 0 != listflags & DC_GCL_ARCHIVED_ONLY {
             // show archived chats
-            // (this includes the archived device-chat; we could skip it,
-            // however, then the number of archived chats do not match, which might be even more irritating.
-            // and adapting the number requires larger refactorings and seems not to be worth the effort)
             context.sql.query_map(
                 "SELECT c.id, m.id
                  FROM chats c
@@ -204,25 +178,18 @@ impl Chatlist {
                                SELECT MAX(timestamp)
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1))
-                 WHERE c.id>9 AND c.id!=?2
+                                  AND (hidden=0 OR state=?))
+                 WHERE c.id>9
                    AND c.blocked=0
-                   AND c.name LIKE ?3
+                   AND c.name LIKE ?
                  GROUP BY c.id
                  ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                params![MessageState::OutDraft, skip_id, str_like_cmd],
+                params![MessageState::OutDraft, str_like_cmd],
                 process_row,
                 process_rows,
             )?
         } else {
             //  show normal chatlist
-            let sort_id_up = if flag_for_forwarding {
-                chat::lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
-                    .unwrap_or_default()
-                    .0
-            } else {
-                ChatId::new(0)
-            };
             let mut ids = context.sql.query_map(
                 "SELECT c.id, m.id
                  FROM chats c
@@ -232,24 +199,22 @@ impl Chatlist {
                                SELECT MAX(timestamp)
                                  FROM msgs
                                 WHERE chat_id=c.id
-                                  AND (hidden=0 OR state=?1))
-                 WHERE c.id>9 AND c.id!=?2
+                                  AND (hidden=0 OR state=?))
+                 WHERE c.id>9
                    AND c.blocked=0
-                   AND NOT c.archived=?3
+                   AND c.archived=0
                  GROUP BY c.id
-                 ORDER BY c.id=?4 DESC, c.archived=?5 DESC, IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
-                params![MessageState::OutDraft, skip_id, ChatVisibility::Archived, sort_id_up, ChatVisibility::Pinned],
+                 ORDER BY IFNULL(m.timestamp,c.created_timestamp) DESC, m.id DESC;",
+                params![MessageState::OutDraft],
                 process_row,
                 process_rows,
             )?;
-            if !flag_no_specials {
+            if 0 == listflags & DC_GCL_NO_SPECIALS {
                 if let Some(last_deaddrop_fresh_msg_id) = get_last_deaddrop_fresh_msg(context) {
-                    if !flag_for_forwarding {
-                        ids.insert(
-                            0,
-                            (ChatId::new(DC_CHAT_ID_DEADDROP), last_deaddrop_fresh_msg_id),
-                        );
-                    }
+                    ids.insert(
+                        0,
+                        (ChatId::new(DC_CHAT_ID_DEADDROP), last_deaddrop_fresh_msg_id),
+                    );
                 }
                 add_archived_link_item = true;
             }
@@ -257,7 +222,7 @@ impl Chatlist {
         };
 
         if add_archived_link_item && dc_get_archived_cnt(context) > 0 {
-            if ids.is_empty() && flag_add_alldone_hint {
+            if ids.is_empty() && 0 != listflags & DC_GCL_ADD_ALLDONE_HINT {
                 ids.push((ChatId::new(DC_CHAT_ID_ALLDONE_HINT), MsgId::new(0)));
             }
             ids.push((ChatId::new(DC_CHAT_ID_ARCHIVED_LINK), MsgId::new(0)));
@@ -280,20 +245,18 @@ impl Chatlist {
     ///
     /// To get the message object from the message ID, use dc_get_chat().
     pub fn get_chat_id(&self, index: usize) -> ChatId {
-        match self.ids.get(index) {
-            Some((chat_id, _msg_id)) => *chat_id,
-            None => ChatId::new(0),
+        if index >= self.ids.len() {
+            return ChatId::new(0);
         }
+        self.ids[index].0
     }
 
     /// Get a single message ID of a chatlist.
     ///
     /// To get the message object from the message ID, use dc_get_msg().
     pub fn get_msg_id(&self, index: usize) -> Result<MsgId> {
-        match self.ids.get(index) {
-            Some((_chat_id, msg_id)) => Ok(*msg_id),
-            None => bail!("Chatlist index out of range"),
-        }
+        ensure!(index < self.ids.len(), "Chatlist index out of range");
+        Ok(self.ids[index].1)
     }
 
     /// Get a summary for a chatlist index.
@@ -315,29 +278,27 @@ impl Chatlist {
         // This is because we may want to display drafts here or stuff as
         // "is typing".
         // Also, sth. as "No messages" would not work if the summary comes from a message.
-        let mut ret = Lot::new();
 
-        let (chat_id, lastmsg_id) = match self.ids.get(index) {
-            Some(ids) => ids,
-            None => {
-                ret.text2 = Some("ErrBadChatlistIndex".to_string());
-                return ret;
-            }
-        };
+        let mut ret = Lot::new();
+        if index >= self.ids.len() {
+            ret.text2 = Some("ErrBadChatlistIndex".to_string());
+            return ret;
+        }
 
         let chat_loaded: Chat;
         let chat = if let Some(chat) = chat {
             chat
-        } else if let Ok(chat) = Chat::load_from_db(context, *chat_id) {
+        } else if let Ok(chat) = Chat::load_from_db(context, self.ids[index].0) {
             chat_loaded = chat;
             &chat_loaded
         } else {
             return ret;
         };
 
+        let lastmsg_id = self.ids[index].1;
         let mut lastcontact = None;
 
-        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, *lastmsg_id) {
+        let lastmsg = if let Ok(lastmsg) = Message::load_from_db(context, lastmsg_id) {
             if lastmsg.from_id != DC_CONTACT_ID_SELF
                 && (chat.typ == Chattype::Group || chat.typ == Chattype::VerifiedGroup)
             {
@@ -360,10 +321,6 @@ impl Chatlist {
 
         ret
     }
-
-    pub fn get_index_for_id(&self, id: ChatId) -> Option<usize> {
-        self.ids.iter().position(|(chat_id, _)| chat_id == &id)
-    }
 }
 
 /// Returns the number of archived chats
@@ -431,32 +388,11 @@ mod tests {
         let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None).unwrap();
         assert_eq!(chats.len(), 0);
 
-        chat_id1
-            .set_visibility(&t.ctx, ChatVisibility::Archived)
-            .ok();
+        chat_id1.set_archived(&t.ctx, true).ok();
         let chats = Chatlist::try_load(&t.ctx, DC_GCL_ARCHIVED_ONLY, None, None).unwrap();
         assert_eq!(chats.len(), 1);
     }
 
-    #[test]
-    fn test_sort_self_talk_up_on_forward() {
-        let t = dummy_context();
-        t.ctx.update_device_chats().unwrap();
-        create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat").unwrap();
-
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
-        assert!(chats.len() == 3);
-        assert!(!Chat::load_from_db(&t.ctx, chats.get_chat_id(0))
-            .unwrap()
-            .is_self_talk());
-
-        let chats = Chatlist::try_load(&t.ctx, DC_GCL_FOR_FORWARDING, None, None).unwrap();
-        assert!(chats.len() == 2); // device chat cannot be written and is skipped on forwarding
-        assert!(Chat::load_from_db(&t.ctx, chats.get_chat_id(0))
-            .unwrap()
-            .is_self_talk());
-    }
-
     #[test]
     fn test_search_special_chat_names() {
         let t = dummy_context();
@@ -479,18 +415,4 @@ mod tests {
         let chats = Chatlist::try_load(&t.ctx, 0, Some("t-5678-b"), None).unwrap();
         assert_eq!(chats.len(), 1);
     }
-
-    #[test]
-    fn test_get_summary_unwrap() {
-        let t = dummy_context();
-        let chat_id1 = create_group_chat(&t.ctx, VerifiedStatus::Unverified, "a chat").unwrap();
-
-        let mut msg = Message::new(Viewtype::Text);
-        msg.set_text(Some("foo:\nbar \r\n test".to_string()));
-        chat_id1.set_draft(&t.ctx, Some(&mut msg));
-
-        let chats = Chatlist::try_load(&t.ctx, 0, None, None).unwrap();
-        let summary = chats.get_summary(&t.ctx, 0, None);
-        assert_eq!(summary.get_text2().unwrap(), "foo: bar test"); // the linebreak should be removed from summary
-    }
 }

src/config.rs

@@ -4,13 +4,10 @@ use strum::{EnumProperty, IntoEnumIterator};
 use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString};
 
 use crate::blob::BlobObject;
-use crate::chat::ChatId;
 use crate::constants::DC_VERSION_STR;
 use crate::context::Context;
 use crate::dc_tools::*;
-use crate::events::Event;
 use crate::job::*;
-use crate::message::MsgId;
 use crate::mimefactory::RECOMMENDED_FILE_SIZE;
 use crate::stock::StockMessage;
 use rusqlite::NO_PARAMS;
@@ -65,31 +62,6 @@ pub enum Config {
     #[strum(props(default = "0"))] // also change ShowEmails.default() on changes
     ShowEmails,
 
-    #[strum(props(default = "0"))] // also change MediaQuality.default() on changes
-    MediaQuality,
-
-    #[strum(props(default = "0"))]
-    KeyGenType,
-
-    /// Timer in seconds after which the message is deleted from the
-    /// server.
-    ///
-    /// Equals to 0 by default, which means the message is never
-    /// deleted.
-    ///
-    /// Value 1 is treated as "delete at once": messages are deleted
-    /// immediately, without moving to DeltaChat folder.
-    #[strum(props(default = "0"))]
-    DeleteServerAfter,
-
-    /// Timer in seconds after which the message is deleted from the
-    /// device.
-    ///
-    /// Equals to 0 by default, which means the message is never
-    /// deleted.
-    #[strum(props(default = "0"))]
-    DeleteDeviceAfter,
-
     SaveMimeHeaders,
     ConfiguredAddr,
     ConfiguredMailServer,
@@ -153,29 +125,6 @@ impl Context {
         self.get_config_int(key) != 0
     }
 
-    /// Gets configured "delete_server_after" value.
-    ///
-    /// `None` means never delete the message, `Some(0)` means delete
-    /// at once, `Some(x)` means delete after `x` seconds.
-    pub fn get_config_delete_server_after(&self) -> Option<i64> {
-        match self.get_config_int(Config::DeleteServerAfter) {
-            0 => None,
-            1 => Some(0),
-            x => Some(x as i64),
-        }
-    }
-
-    /// Gets configured "delete_device_after" value.
-    ///
-    /// `None` means never delete the message, `Some(x)` means delete
-    /// after `x` seconds.
-    pub fn get_config_delete_device_after(&self) -> Option<i64> {
-        match self.get_config_int(Config::DeleteDeviceAfter) {
-            0 => None,
-            x => Some(x as i64),
-        }
-    }
-
     /// Set the given config key.
     /// If `None` is passed as a value the value is cleared and set to the default if there is one.
     pub fn set_config(&self, key: Config, value: Option<&str>) -> crate::sql::Result<()> {
@@ -219,15 +168,6 @@ impl Context {
 
                 self.sql.set_raw_config(self, key, val)
             }
-            Config::DeleteDeviceAfter => {
-                let ret = self.sql.set_raw_config(self, key, value);
-                // Force chatlist reload to delete old messages immediately.
-                self.call_cb(Event::MsgsChanged {
-                    msg_id: MsgId::new(0),
-                    chat_id: ChatId::new(0),
-                });
-                ret
-            }
             _ => self.sql.set_raw_config(self, key, value),
         }
     }
@@ -251,11 +191,9 @@ mod tests {
     use std::str::FromStr;
     use std::string::ToString;
 
-    use crate::constants;
     use crate::constants::AVATAR_SIZE;
     use crate::test_utils::*;
     use image::GenericImageView;
-    use num_traits::FromPrimitive;
     use std::fs::File;
     use std::io::Write;
 
@@ -328,44 +266,4 @@ mod tests {
         assert_eq!(img.width(), AVATAR_SIZE);
         assert_eq!(img.height(), AVATAR_SIZE);
     }
-
-    #[test]
-    fn test_selfavatar_copy_without_recode() {
-        let t = dummy_context();
-        let avatar_src = t.dir.path().join("avatar.png");
-        let avatar_bytes = include_bytes!("../test-data/image/avatar64x64.png");
-        File::create(&avatar_src)
-            .unwrap()
-            .write_all(avatar_bytes)
-            .unwrap();
-        let avatar_blob = t.ctx.get_blobdir().join("avatar.png");
-        assert!(!avatar_blob.exists());
-        t.ctx
-            .set_config(Config::Selfavatar, Some(&avatar_src.to_str().unwrap()))
-            .unwrap();
-        assert!(avatar_blob.exists());
-        assert_eq!(
-            std::fs::metadata(&avatar_blob).unwrap().len(),
-            avatar_bytes.len() as u64
-        );
-        let avatar_cfg = t.ctx.get_config(Config::Selfavatar);
-        assert_eq!(avatar_cfg, avatar_blob.to_str().map(|s| s.to_string()));
-    }
-
-    #[test]
-    fn test_media_quality_config_option() {
-        let t = dummy_context();
-        let media_quality = t.ctx.get_config_int(Config::MediaQuality);
-        assert_eq!(media_quality, 0);
-        let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
-        assert_eq!(media_quality, constants::MediaQuality::Balanced);
-
-        t.ctx.set_config(Config::MediaQuality, Some("1")).unwrap();
-
-        let media_quality = t.ctx.get_config_int(Config::MediaQuality);
-        assert_eq!(media_quality, 1);
-        assert_eq!(constants::MediaQuality::Worse as i32, 1);
-        let media_quality = constants::MediaQuality::from_i32(media_quality).unwrap_or_default();
-        assert_eq!(media_quality, constants::MediaQuality::Worse);
-    }
 }

src/configure/auto_mozilla.rs

@@ -1,6 +1,7 @@
 //! # Thunderbird's Autoconfiguration implementation
 //!
 //! Documentation: https://developer.mozilla.org/en-US/docs/Mozilla/Thunderbird/Autoconfiguration */
+use quick_xml;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
 use crate::constants::*;
@@ -8,7 +9,33 @@ use crate::context::Context;
 use crate::login_param::LoginParam;
 
 use super::read_url::read_url;
-use super::Error;
+
+#[derive(Debug, Fail)]
+pub enum Error {
+    #[fail(display = "Invalid email address: {:?}", _0)]
+    InvalidEmailAddress(String),
+
+    #[fail(display = "XML error at position {}", position)]
+    InvalidXml {
+        position: usize,
+        #[cause]
+        error: quick_xml::Error,
+    },
+
+    #[fail(display = "Bad or incomplete autoconfig")]
+    IncompleteAutoconfig(LoginParam),
+
+    #[fail(display = "Failed to get URL {}", _0)]
+    ReadUrlError(#[cause] super::read_url::Error),
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+impl From<super::read_url::Error> for Error {
+    fn from(err: super::read_url::Error) -> Error {
+        Error::ReadUrlError(err)
+    }
+}
 
 #[derive(Debug)]
 struct MozAutoconfigure<'a> {
@@ -38,16 +65,16 @@ enum MozConfigTag {
     Username,
 }
 
-fn parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result<LoginParam, Error> {
+fn parse_xml(in_emailaddr: &str, xml_raw: &str) -> Result<LoginParam> {
     let mut reader = quick_xml::Reader::from_str(xml_raw);
     reader.trim_text(true);
 
     // Split address into local part and domain part.
-    let parts: Vec<&str> = in_emailaddr.rsplitn(2, '@').collect();
-    let (in_emaillocalpart, in_emaildomain) = match &parts[..] {
-        [domain, local] => (local, domain),
-        _ => return Err(Error::InvalidEmailAddress(in_emailaddr.to_string())),
-    };
+    let p = in_emailaddr
+        .find('@')
+        .ok_or_else(|| Error::InvalidEmailAddress(in_emailaddr.to_string()))?;
+    let (in_emaillocalpart, in_emaildomain) = in_emailaddr.split_at(p);
+    let in_emaildomain = &in_emaildomain[1..];
 
     let mut moz_ac = MozAutoconfigure {
         in_emailaddr,
@@ -98,7 +125,7 @@ pub fn moz_autoconfigure(
     context: &Context,
     url: &str,
     param_in: &LoginParam,
-) -> Result<LoginParam, Error> {
+) -> Result<LoginParam> {
     let xml_raw = read_url(context, url)?;
 
     let res = parse_xml(&param_in.addr, &xml_raw);

src/configure/auto_outlook.rs

@@ -1,5 +1,6 @@
 //! Outlook's Autodiscover
 
+use quick_xml;
 use quick_xml::events::BytesEnd;
 
 use crate::constants::*;
@@ -7,7 +8,33 @@ use crate::context::Context;
 use crate::login_param::LoginParam;
 
 use super::read_url::read_url;
-use super::Error;
+
+#[derive(Debug, Fail)]
+pub enum Error {
+    #[fail(display = "XML error at position {}", position)]
+    InvalidXml {
+        position: usize,
+        #[cause]
+        error: quick_xml::Error,
+    },
+
+    #[fail(display = "Bad or incomplete autoconfig")]
+    IncompleteAutoconfig(LoginParam),
+
+    #[fail(display = "Failed to get URL {}", _0)]
+    ReadUrlError(#[cause] super::read_url::Error),
+
+    #[fail(display = "Number of redirection is exceeded")]
+    RedirectionError,
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+impl From<super::read_url::Error> for Error {
+    fn from(err: super::read_url::Error) -> Error {
+        Error::ReadUrlError(err)
+    }
+}
 
 struct OutlookAutodiscover {
     pub out: LoginParam,
@@ -25,7 +52,7 @@ enum ParsingResult {
     RedirectUrl(String),
 }
 
-fn parse_xml(xml_raw: &str) -> Result<ParsingResult, Error> {
+fn parse_xml(xml_raw: &str) -> Result<ParsingResult> {
     let mut outlk_ad = OutlookAutodiscover {
         out: LoginParam::new(),
         out_imap_set: false,
@@ -116,7 +143,7 @@ pub fn outlk_autodiscover(
     context: &Context,
     url: &str,
     _param_in: &LoginParam,
-) -> Result<LoginParam, Error> {
+) -> Result<LoginParam> {
     let mut url = url.to_string();
     /* Follow up to 10 xml-redirects (http-redirects are followed in read_url() */
     for _i in 0..10 {

src/configure/mod.rs

@@ -12,7 +12,6 @@ use crate::config::Config;
 use crate::constants::*;
 use crate::context::Context;
 use crate::dc_tools::*;
-use crate::error::format_err;
 use crate::job::{self, job_add, job_kill_action};
 use crate::login_param::{CertificateChecks, LoginParam};
 use crate::oauth2::*;
@@ -33,28 +32,26 @@ macro_rules! progress {
     };
 }
 
-impl Context {
-    /// Starts a configuration job.
-    pub fn configure(&self) {
-        if self.has_ongoing() {
-            warn!(self, "There is already another ongoing process running.",);
-            return;
-        }
-        job_kill_action(self, job::Action::ConfigureImap);
-        job_add(self, job::Action::ConfigureImap, 0, Params::new(), 0);
+// connect
+pub fn configure(context: &Context) {
+    if context.has_ongoing() {
+        warn!(context, "There is already another ongoing process running.",);
+        return;
     }
+    job_kill_action(context, job::Action::ConfigureImap);
+    job_add(context, job::Action::ConfigureImap, 0, Params::new(), 0);
+}
 
-    /// Checks if the context is already configured.
-    pub fn is_configured(&self) -> bool {
-        self.sql.get_raw_config_bool(self, "configured")
-    }
+/// Check if the context is already configured.
+pub fn dc_is_configured(context: &Context) -> bool {
+    context.sql.get_raw_config_bool(context, "configured")
 }
 
 /*******************************************************************************
  * Configure JOB
  ******************************************************************************/
 #[allow(non_snake_case, unused_must_use, clippy::cognitive_complexity)]
-pub(crate) fn JobConfigureImap(context: &Context) -> job::Status {
+pub fn JobConfigureImap(context: &Context) -> job::Status {
     if !context.sql.is_open() {
         error!(context, "Cannot configure, database not opened.",);
         progress!(context, 0);
@@ -69,7 +66,6 @@ pub(crate) fn JobConfigureImap(context: &Context) -> job::Status {
     let mut smtp_connected_here = false;
 
     let mut param_autoconfig: Option<LoginParam> = None;
-    let was_configured_before = context.is_configured();
 
     context
         .inbox_thread
@@ -373,7 +369,7 @@ pub(crate) fn JobConfigureImap(context: &Context) -> job::Status {
                 let create_mvbox = context.get_config_bool(Config::MvboxWatch)
                     || context.get_config_bool(Config::MvboxMove);
                 let imap = &context.inbox_thread.read().unwrap().imap;
-                if let Err(err) = imap.configure_folders(context, create_mvbox) {
+                if let Err(err) = imap.ensure_configured_folders(context, create_mvbox) {
                     warn!(context, "configuring folders failed: {:?}", err);
                     false
                 } else {
@@ -442,15 +438,6 @@ pub(crate) fn JobConfigureImap(context: &Context) -> job::Status {
     }
 
     if let Some(provider) = provider::get_provider_info(&param.addr) {
-        if !was_configured_before {
-            if let Some(config_defaults) = &provider.config_defaults {
-                for def in config_defaults.iter() {
-                    info!(context, "apply config_defaults {}={}", def.key, def.value);
-                    context.set_config(def.key, Some(def.value));
-                }
-            }
-        }
-
         if !provider.after_login_hint.is_empty() {
             let mut msg = Message::new(Viewtype::Text);
             msg.text = Some(provider.after_login_hint.to_string());
@@ -662,28 +649,6 @@ fn try_smtp_one_param(context: &Context, param: &LoginParam) -> Option<bool> {
     }
 }
 
-#[derive(Debug, thiserror::Error)]
-pub enum Error {
-    #[error("Invalid email address: {0:?}")]
-    InvalidEmailAddress(String),
-
-    #[error("XML error at position {position}")]
-    InvalidXml {
-        position: usize,
-        #[source]
-        error: quick_xml::Error,
-    },
-
-    #[error("Bad or incomplete autoconfig")]
-    IncompleteAutoconfig(LoginParam),
-
-    #[error("Failed to get URL")]
-    ReadUrlError(#[from] self::read_url::Error),
-
-    #[error("Number of redirection is exceeded")]
-    RedirectionError,
-}
-
 #[cfg(test)]
 mod tests {
 

src/configure/read_url.rs

@@ -1,12 +1,14 @@
 use crate::context::Context;
 
-#[derive(Debug, thiserror::Error)]
+#[derive(Debug, Fail)]
 pub enum Error {
-    #[error("URL request error")]
-    GetError(#[from] reqwest::Error),
+    #[fail(display = "URL request error")]
+    GetError(#[cause] reqwest::Error),
 }
 
-pub fn read_url(context: &Context, url: &str) -> Result<String, Error> {
+pub type Result<T> = std::result::Result<T, Error>;
+
+pub fn read_url(context: &Context, url: &str) -> Result<String> {
     info!(context, "Requesting URL {}", url);
 
     match reqwest::blocking::Client::new()

src/constants.rs

@@ -3,7 +3,6 @@
 
 use deltachat_derive::*;
 use lazy_static::lazy_static;
-use serde::{Deserialize, Serialize};
 
 lazy_static! {
     pub static ref DC_VERSION_STR: String = env!("CARGO_PKG_VERSION").to_string();
@@ -16,20 +15,7 @@ const DC_SENTBOX_WATCH_DEFAULT: i32 = 1;
 const DC_MVBOX_WATCH_DEFAULT: i32 = 1;
 const DC_MVBOX_MOVE_DEFAULT: i32 = 1;
 
-#[derive(
-    Debug,
-    Display,
-    Clone,
-    Copy,
-    PartialEq,
-    Eq,
-    FromPrimitive,
-    ToPrimitive,
-    FromSql,
-    ToSql,
-    Serialize,
-    Deserialize,
-)]
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
 #[repr(u8)]
 pub enum Blocked {
     Not = 0,
@@ -57,32 +43,7 @@ impl Default for ShowEmails {
     }
 }
 
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
-#[repr(u8)]
-pub enum MediaQuality {
-    Balanced = 0,
-    Worse = 1,
-}
-
-impl Default for MediaQuality {
-    fn default() -> Self {
-        MediaQuality::Balanced // also change Config.MediaQuality props(default) on changes
-    }
-}
-
-#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
-#[repr(u8)]
-pub enum KeyGenType {
-    Default = 0,
-    Rsa2048 = 1,
-    Ed25519 = 2,
-}
-
-impl Default for KeyGenType {
-    fn default() -> Self {
-        KeyGenType::Default
-    }
-}
+pub const DC_IMAP_SEEN: u32 = 0x1;
 
 pub const DC_HANDSHAKE_CONTINUE_NORMAL_PROCESSING: i32 = 0x01;
 pub const DC_HANDSHAKE_STOP_NORMAL_PROCESSING: i32 = 0x02;
@@ -93,7 +54,6 @@ pub(crate) const DC_FROM_HANDSHAKE: i32 = 0x01;
 pub const DC_GCL_ARCHIVED_ONLY: usize = 0x01;
 pub const DC_GCL_NO_SPECIALS: usize = 0x02;
 pub const DC_GCL_ADD_ALLDONE_HINT: usize = 0x04;
-pub const DC_GCL_FOR_FORWARDING: usize = 0x08;
 
 pub const DC_GCM_ADDDAYMARKER: u32 = 0x01;
 
@@ -130,8 +90,6 @@ pub const DC_CHAT_ID_LAST_SPECIAL: u32 = 9;
     FromSql,
     ToSql,
     IntoStaticStr,
-    Serialize,
-    Deserialize,
 )]
 #[repr(u32)]
 pub enum Chattype {
@@ -210,13 +168,13 @@ pub const DC_LP_SMTP_SOCKET_SSL: usize = 0x20000;
 pub const DC_LP_SMTP_SOCKET_PLAIN: usize = 0x40000;
 
 /// if none of these flags are set, the default is chosen
-pub const DC_LP_AUTH_FLAGS: i32 = DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL;
+pub const DC_LP_AUTH_FLAGS: i32 = (DC_LP_AUTH_OAUTH2 | DC_LP_AUTH_NORMAL);
 /// if none of these flags are set, the default is chosen
 pub const DC_LP_IMAP_SOCKET_FLAGS: i32 =
-    DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_SSL | DC_LP_IMAP_SOCKET_PLAIN;
+    (DC_LP_IMAP_SOCKET_STARTTLS | DC_LP_IMAP_SOCKET_SSL | DC_LP_IMAP_SOCKET_PLAIN);
 /// if none of these flags are set, the default is chosen
 pub const DC_LP_SMTP_SOCKET_FLAGS: usize =
-    DC_LP_SMTP_SOCKET_STARTTLS | DC_LP_SMTP_SOCKET_SSL | DC_LP_SMTP_SOCKET_PLAIN;
+    (DC_LP_SMTP_SOCKET_STARTTLS | DC_LP_SMTP_SOCKET_SSL | DC_LP_SMTP_SOCKET_PLAIN);
 
 // QR code scanning (view from Bob, the joiner)
 pub const DC_VC_AUTH_REQUIRED: i32 = 2;
@@ -227,23 +185,7 @@ pub const DC_BOB_SUCCESS: i32 = 1;
 // max. width/height of an avatar
 pub const AVATAR_SIZE: u32 = 192;
 
-// this value can be increased if the folder configuration is changed and must be redone on next program start
-pub const DC_FOLDERS_CONFIGURED_VERSION: i32 = 3;
-
-#[derive(
-    Debug,
-    Display,
-    Clone,
-    Copy,
-    PartialEq,
-    Eq,
-    FromPrimitive,
-    ToPrimitive,
-    FromSql,
-    ToSql,
-    Serialize,
-    Deserialize,
-)]
+#[derive(Debug, Display, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
 #[repr(i32)]
 pub enum Viewtype {
     Unknown = 0,
@@ -327,6 +269,8 @@ const DC_STR_SELFNOTINGRP: usize = 21; // deprecated;
 const DC_STR_NOMESSAGES: usize = 1;
 const DC_STR_SELF: usize = 2;
 const DC_STR_DRAFT: usize = 3;
+const DC_STR_MEMBER: usize = 4;
+const DC_STR_CONTACT: usize = 6;
 const DC_STR_VOICEMESSAGE: usize = 7;
 const DC_STR_DEADDROP: usize = 8;
 const DC_STR_IMAGE: usize = 9;
@@ -358,6 +302,7 @@ const DC_STR_ARCHIVEDCHATS: usize = 40;
 const DC_STR_STARREDMSGS: usize = 41;
 const DC_STR_AC_SETUP_MSG_SUBJECT: usize = 42;
 const DC_STR_AC_SETUP_MSG_BODY: usize = 43;
+const DC_STR_SELFTALK_SUBTITLE: usize = 50;
 const DC_STR_CANNOT_LOGIN: usize = 60;
 const DC_STR_SERVER_RESPONSE: usize = 61;
 const DC_STR_MSGACTIONBYUSER: usize = 62;

src/contact.rs

@@ -4,6 +4,7 @@ use std::path::PathBuf;
 
 use deltachat_derive::*;
 use itertools::Itertools;
+use rusqlite;
 
 use crate::aheader::EncryptPreference;
 use crate::chat::ChatId;
@@ -11,9 +12,10 @@ use crate::config::Config;
 use crate::constants::*;
 use crate::context::Context;
 use crate::dc_tools::*;
-use crate::error::{bail, ensure, format_err, Result};
+use crate::e2ee;
+use crate::error::{Error, Result};
 use crate::events::Event;
-use crate::key::{DcKey, Key, SignedPublicKey};
+use crate::key::*;
 use crate::login_param::LoginParam;
 use crate::message::{MessageState, MsgId};
 use crate::mimeparser::AvatarAction;
@@ -22,6 +24,9 @@ use crate::peerstate::*;
 use crate::sql;
 use crate::stock::StockMessage;
 
+/// Contacts with at least this origin value are shown in the contact list.
+const DC_ORIGIN_MIN_CONTACT_LIST: i32 = 0x100;
+
 /// An object representing a single contact in memory.
 ///
 /// The contact object is not updated.
@@ -55,7 +60,7 @@ pub struct Contact {
     /// to access this field.
     authname: String,
 
-    /// E-Mail-Address of the contact. It is recommended to use `Contact::get_addr` to access this field.
+    /// E-Mail-Address of the contact. It is recommended to use `Contact::get_addr`` to access this field.
     addr: String,
 
     /// Blocked state. Use dc_contact_is_blocked to access this field.
@@ -89,7 +94,6 @@ pub enum Origin {
     UnhandledQrScan = 0x80,
 
     /// Reply-To: of incoming message of known sender
-    /// Contacts with at least this origin value are shown in the contact list.
     IncomingReplyTo = 0x100,
 
     /// Cc: of incoming message of known sender
@@ -114,7 +118,7 @@ pub enum Origin {
     Internal = 0x40000,
 
     /// address is in our address book
-    AddressBook = 0x80000,
+    AdressBook = 0x80000,
 
     /// set on Alice's side for contacts like Bob that have scanned the QR code offered by her. Only means the contact has once been established using the "securejoin" procedure in the past, getting the current key verification status requires calling dc_contact_is_verified() !
     SecurejoinInvited = 0x0100_0000,
@@ -142,7 +146,7 @@ impl Origin {
 }
 
 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
-pub(crate) enum Modifier {
+pub enum Modifier {
     None,
     Modified,
     Created,
@@ -270,7 +274,7 @@ impl Contact {
     ///
     /// To validate an e-mail address independently of the contact database
     /// use `dc_may_be_valid_addr()`.
-    pub fn lookup_id_by_addr(context: &Context, addr: impl AsRef<str>, min_origin: Origin) -> u32 {
+    pub fn lookup_id_by_addr(context: &Context, addr: impl AsRef<str>) -> u32 {
         if addr.as_ref().is_empty() {
             return 0;
         }
@@ -283,43 +287,22 @@ impl Contact {
         if addr_cmp(addr_normalized, addr_self) {
             return DC_CONTACT_ID_SELF;
         }
+
         context.sql.query_get_value(
             context,
             "SELECT id FROM contacts WHERE addr=?1 COLLATE NOCASE AND id>?2 AND origin>=?3 AND blocked=0;",
             params![
                 addr_normalized,
                 DC_CONTACT_ID_LAST_SPECIAL as i32,
-                min_origin as u32,
+                DC_ORIGIN_MIN_CONTACT_LIST,
             ],
         ).unwrap_or_default()
     }
 
     /// Lookup a contact and create it if it does not exist yet.
-    /// The contact is identified by the email-address, a name and an "origin" can be given.
-    ///
-    /// The "origin" is where the address comes from -
-    /// from-header, cc-header, addressbook, qr, manual-edit etc.
-    /// In general, "better" origins overwrite the names of "worse" origins -
-    /// Eg. if we got a name in cc-header and later in from-header, the name will change -
-    /// this does not happen the other way round.
-    ///
-    /// The "best" origin are manually created contacts -
-    /// names given manually can only be overwritten by further manual edits
-    /// (until they are set empty again or reset to the name seen in the From-header).
-    ///
-    /// These manually edited names are _never_ used for sending on the wire -
-    /// this should avoid sending sth. as "Mama" or "Daddy" to some 3rd party.
-    /// Instead, for the wire, we use so called "authnames"
-    /// that can only be set and updated by a From-header.
-    ///
-    /// The different names used in the function are:
-    /// - "name": name passed as function argument, belonging to the given origin
-    /// - "row_name": current name used in the database, typically set to "name"
-    /// - "row_authname": name as authorized from a contact, set only through a From-header
-    /// Depending on the origin, both, "row_name" and "row_authname" are updated from "name".
     ///
     /// Returns the contact_id and a `Modifier` value indicating if a modification occured.
-    pub(crate) fn add_or_lookup(
+    pub fn add_or_lookup(
         context: &Context,
         name: impl AsRef<str>,
         addr: impl AsRef<str>,
@@ -373,9 +356,7 @@ impl Contact {
 
                 if !name.as_ref().is_empty() {
                     if !row_name.is_empty() {
-                        if (origin >= row_origin || row_name == row_authname)
-                            && name.as_ref() != row_name
-                        {
+                        if origin >= row_origin && name.as_ref() != row_name {
                             update_name = true;
                         }
                     } else {
@@ -384,9 +365,6 @@ impl Contact {
                     if origin == Origin::IncomingUnknownFrom && name.as_ref() != row_authname {
                         update_authname = true;
                     }
-                } else if origin == Origin::ManuallyCreated && !row_authname.is_empty() {
-                    // no name given on manual edit, this will update the name to the authname
-                    update_name = true;
                 }
 
                 Ok((row_id, row_name, row_addr, row_origin, row_authname))
@@ -397,22 +375,16 @@ impl Contact {
                 update_addr = true;
             }
             if update_name || update_authname || update_addr || origin > row_origin {
-                let new_name = if update_name {
-                    if !name.as_ref().is_empty() {
-                        name.as_ref()
-                    } else {
-                        &row_authname
-                    }
-                } else {
-                    &row_name
-                };
-
                 sql::execute(
                     context,
                     &context.sql,
                     "UPDATE contacts SET name=?, addr=?, origin=?, authname=? WHERE id=?;",
                     params![
-                        new_name,
+                        if update_name {
+                            name.as_ref()
+                        } else {
+                            &row_name
+                        },
                         if update_addr { addr } else { &row_addr },
                         if origin > row_origin {
                             origin
@@ -430,13 +402,11 @@ impl Contact {
                 .ok();
 
                 if update_name {
-                    // Update the contact name also if it is used as a group name.
-                    // This is one of the few duplicated data, however, getting the chat list is easier this way.
                     sql::execute(
                     context,
                     &context.sql,
                     "UPDATE chats SET name=? WHERE type=? AND id IN(SELECT chat_id FROM chats_contacts WHERE contact_id=?);",
-                    params![new_name, Chattype::Single, row_id]
+                    params![name.as_ref(), Chattype::Single, row_id]
                 ).ok();
                 }
                 sth_modified = Modifier::Modified;
@@ -492,18 +462,9 @@ impl Contact {
 
         for (name, addr) in split_address_book(addr_book.as_ref()).into_iter() {
             let name = normalize_name(name);
-            match Contact::add_or_lookup(context, name, addr, Origin::AddressBook) {
-                Err(err) => {
-                    warn!(
-                        context,
-                        "Failed to add address {} from address book: {}", addr, err
-                    );
-                }
-                Ok((_, modified)) => {
-                    if modified != Modifier::None {
-                        modify_cnt += 1
-                    }
-                }
+            let (_, modified) = Contact::add_or_lookup(context, name, addr, Origin::AdressBook)?;
+            if modified != Modifier::None {
+                modify_cnt += 1
             }
         }
         if modify_cnt > 0 {
@@ -646,6 +607,8 @@ impl Contact {
             let peerstate = Peerstate::from_addr(context, &context.sql, &contact.addr);
             let loginparam = LoginParam::from_database(context, "configured_");
 
+            let mut self_key = Key::from_self_public(context, &loginparam.addr, &context.sql);
+
             if peerstate.is_some()
                 && peerstate
                     .as_ref()
@@ -660,11 +623,16 @@ impl Contact {
                         StockMessage::E2eAvailable
                     });
                 ret += &p;
-                let self_key = Key::from(SignedPublicKey::load_self(context)?);
+                if self_key.is_none() {
+                    e2ee::ensure_secret_key_exists(context)?;
+                    self_key = Key::from_self_public(context, &loginparam.addr, &context.sql);
+                }
                 let p = context.stock_str(StockMessage::FingerPrints);
                 ret += &format!(" {}:", p);
 
-                let fingerprint_self = self_key.formatted_fingerprint();
+                let fingerprint_self = self_key
+                    .map(|k| k.formatted_fingerprint())
+                    .unwrap_or_default();
                 let fingerprint_other_verified = peerstate
                     .peek_key(PeerstateVerifiedStatus::BidirectVerified)
                     .map(|k| k.formatted_fingerprint())
@@ -1016,7 +984,7 @@ fn set_block_contact(context: &Context, contact_id: u32, new_blocking: bool) {
     }
 }
 
-pub(crate) fn set_profile_image(
+pub fn set_profile_image(
     context: &Context,
     contact_id: u32,
     profile_image: &AvatarAction,
@@ -1033,6 +1001,7 @@ pub(crate) fn set_profile_image(
             contact.param.remove(Param::ProfileImage);
             true
         }
+        AvatarAction::None => false,
     };
     if changed {
         contact.update_param(context)?;
@@ -1055,7 +1024,7 @@ pub fn normalize_name(full_name: impl AsRef<str>) -> String {
     }
 
     let len = full_name.len();
-    if len > 1 {
+    if len > 0 {
         let firstchar = full_name.as_bytes()[0];
         let lastchar = full_name.as_bytes()[len - 1];
         if firstchar == b'\'' && lastchar == b'\''
@@ -1108,9 +1077,10 @@ fn cat_fingerprint(
 impl Context {
     /// determine whether the specified addr maps to the/a self addr
     pub fn is_self_addr(&self, addr: &str) -> Result<bool> {
-        let self_addr = self
-            .get_config(Config::ConfiguredAddr)
-            .ok_or_else(|| format_err!("Not configured"))?;
+        let self_addr = match self.get_config(Config::ConfiguredAddr) {
+            Some(s) => s,
+            None => return Err(Error::NotConfigured),
+        };
 
         Ok(addr_cmp(self_addr, addr))
     }
@@ -1163,10 +1133,6 @@ mod tests {
     fn test_normalize_name() {
         assert_eq!(&normalize_name("Doe, John"), "John Doe");
         assert_eq!(&normalize_name(" hello world   "), "hello world");
-        assert_eq!(&normalize_name("<"), "<");
-        assert_eq!(&normalize_name(">"), ">");
-        assert_eq!(&normalize_name("'"), "'");
-        assert_eq!(&normalize_name("\""), "\"");
     }
 
     #[test]
@@ -1229,7 +1195,6 @@ mod tests {
         let book = concat!(
             "  Name one  \n one@eins.org \n",
             "Name two\ntwo@deux.net\n",
-            "Invalid\n+1234567890\n", // invalid, should be ignored
             "\nthree@drei.sam\n",
             "Name two\ntwo@deux.net\n" // should not be added again
         );
@@ -1316,7 +1281,6 @@ mod tests {
     fn test_remote_authnames() {
         let t = dummy_context();
 
-        // incoming mail `From: bob1 <bob@example.org>` - this should init authname and name
         let (contact_id, sth_modified) = Contact::add_or_lookup(
             &t.ctx,
             "bob1",
@@ -1331,7 +1295,6 @@ mod tests {
         assert_eq!(contact.get_name(), "bob1");
         assert_eq!(contact.get_display_name(), "bob1");
 
-        // incoming mail `From: bob2 <bob@example.org>` - this should update authname and name
         let (contact_id, sth_modified) = Contact::add_or_lookup(
             &t.ctx,
             "bob2",
@@ -1346,15 +1309,16 @@ mod tests {
         assert_eq!(contact.get_name(), "bob2");
         assert_eq!(contact.get_display_name(), "bob2");
 
-        // manually edit name to "bob3" - authname should be still be "bob2" a given in `From:` above
-        let contact_id = Contact::create(&t.ctx, "bob3", "bob@example.org").unwrap();
+        let (contact_id, sth_modified) =
+            Contact::add_or_lookup(&t.ctx, "bob3", "bob@example.org", Origin::ManuallyCreated)
+                .unwrap();
         assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
+        assert_eq!(sth_modified, Modifier::Modified);
         let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
         assert_eq!(contact.get_authname(), "bob2");
         assert_eq!(contact.get_name(), "bob3");
         assert_eq!(contact.get_display_name(), "bob3");
 
-        // incoming mail `From: bob4 <bob@example.org>` - this should update authname, manually given name is still "bob3"
         let (contact_id, sth_modified) = Contact::add_or_lookup(
             &t.ctx,
             "bob4",
@@ -1370,81 +1334,6 @@ mod tests {
         assert_eq!(contact.get_display_name(), "bob3");
     }
 
-    #[test]
-    fn test_remote_authnames_create_empty() {
-        let t = dummy_context();
-
-        // manually create "claire@example.org" without a given name
-        let contact_id = Contact::create(&t.ctx, "", "claire@example.org").unwrap();
-        assert!(contact_id > DC_CONTACT_ID_LAST_SPECIAL);
-        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
-        assert_eq!(contact.get_authname(), "");
-        assert_eq!(contact.get_name(), "");
-        assert_eq!(contact.get_display_name(), "claire@example.org");
-
-        // incoming mail `From: claire1 <claire@example.org>` - this should update authname and name
-        let (contact_id_same, sth_modified) = Contact::add_or_lookup(
-            &t.ctx,
-            "claire1",
-            "claire@example.org",
-            Origin::IncomingUnknownFrom,
-        )
-        .unwrap();
-        assert_eq!(contact_id, contact_id_same);
-        assert_eq!(sth_modified, Modifier::Modified);
-        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
-        assert_eq!(contact.get_authname(), "claire1");
-        assert_eq!(contact.get_name(), "claire1");
-        assert_eq!(contact.get_display_name(), "claire1");
-
-        // incoming mail `From: claire2 <claire@example.org>` - this should update authname and name
-        let (contact_id_same, sth_modified) = Contact::add_or_lookup(
-            &t.ctx,
-            "claire2",
-            "claire@example.org",
-            Origin::IncomingUnknownFrom,
-        )
-        .unwrap();
-        assert_eq!(contact_id, contact_id_same);
-        assert_eq!(sth_modified, Modifier::Modified);
-        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
-        assert_eq!(contact.get_authname(), "claire2");
-        assert_eq!(contact.get_name(), "claire2");
-        assert_eq!(contact.get_display_name(), "claire2");
-    }
-
-    #[test]
-    fn test_remote_authnames_edit_empty() {
-        let t = dummy_context();
-
-        // manually create "dave@example.org"
-        let contact_id = Contact::create(&t.ctx, "dave1", "dave@example.org").unwrap();
-        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
-        assert_eq!(contact.get_authname(), "");
-        assert_eq!(contact.get_name(), "dave1");
-        assert_eq!(contact.get_display_name(), "dave1");
-
-        // incoming mail `From: dave2 <dave@example.org>` - this should update authname
-        Contact::add_or_lookup(
-            &t.ctx,
-            "dave2",
-            "dave@example.org",
-            Origin::IncomingUnknownFrom,
-        )
-        .unwrap();
-        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
-        assert_eq!(contact.get_authname(), "dave2");
-        assert_eq!(contact.get_name(), "dave1");
-        assert_eq!(contact.get_display_name(), "dave1");
-
-        // manually clear the name
-        Contact::create(&t.ctx, "", "dave@example.org").unwrap();
-        let contact = Contact::load_from_db(&t.ctx, contact_id).unwrap();
-        assert_eq!(contact.get_authname(), "dave2");
-        assert_eq!(contact.get_name(), "dave2");
-        assert_eq!(contact.get_display_name(), "dave2");
-    }
-
     #[test]
     fn test_addr_cmp() {
         assert!(addr_cmp("AA@AA.ORG", "aa@aa.ORG"));

src/context.rs

@@ -1,6 +1,6 @@
 //! Context module
 
-use std::collections::{BTreeMap, HashMap};
+use std::collections::HashMap;
 use std::ffi::OsString;
 use std::path::{Path, PathBuf};
 use std::sync::{Arc, Condvar, Mutex, RwLock};
@@ -9,20 +9,18 @@ use crate::chat::*;
 use crate::config::Config;
 use crate::constants::*;
 use crate::contact::*;
-use crate::dc_tools::duration_to_str;
 use crate::error::*;
 use crate::events::Event;
 use crate::imap::*;
 use crate::job::*;
 use crate::job_thread::JobThread;
-use crate::key::{DcKey, Key, SignedPublicKey};
+use crate::key::Key;
 use crate::login_param::LoginParam;
 use crate::lot::Lot;
 use crate::message::{self, Message, MessengerMessage, MsgId};
 use crate::param::Params;
 use crate::smtp::Smtp;
 use crate::sql::Sql;
-use std::time::SystemTime;
 
 /// Callback function type for [Context]
 ///
@@ -53,13 +51,12 @@ pub struct Context {
     cb: Box<ContextCallback>,
     pub os_name: Option<String>,
     pub cmdline_sel_chat_id: Arc<RwLock<ChatId>>,
-    pub(crate) bob: Arc<RwLock<BobStatus>>,
+    pub bob: Arc<RwLock<BobStatus>>,
     pub last_smeared_timestamp: RwLock<i64>,
     pub running_state: Arc<RwLock<RunningState>>,
     /// Mutex to avoid generating the key for the user more than once.
     pub generating_key_mutex: Mutex<()>,
     pub translated_stockstrings: RwLock<HashMap<usize, String>>,
-    creation_time: SystemTime,
 }
 
 #[derive(Debug, PartialEq, Eq)]
@@ -74,8 +71,8 @@ pub struct RunningState {
 /// actual keys and their values which will be present are not
 /// guaranteed.  Calling [Context::get_info] also includes information
 /// about the context on top of the information here.
-pub fn get_info() -> BTreeMap<&'static str, String> {
-    let mut res = BTreeMap::new();
+pub fn get_info() -> HashMap<&'static str, String> {
+    let mut res = HashMap::new();
     res.insert("deltachat_core_version", format!("v{}", &*DC_VERSION_STR));
     res.insert("sqlite_version", rusqlite::version().to_string());
     res.insert("arch", (std::mem::size_of::<usize>() * 8).to_string());
@@ -141,7 +138,6 @@ impl Context {
             perform_inbox_jobs_needed: Arc::new(RwLock::new(false)),
             generating_key_mutex: Mutex::new(()),
             translated_stockstrings: RwLock::new(HashMap::new()),
-            creation_time: std::time::SystemTime::now(),
         };
 
         ensure!(
@@ -226,7 +222,7 @@ impl Context {
      * UI chat/message related API
      ******************************************************************************/
 
-    pub fn get_info(&self) -> BTreeMap<&'static str, String> {
+    pub fn get_info(&self) -> HashMap<&'static str, String> {
         let unset = "0";
         let l = LoginParam::from_database(self, "");
         let l2 = LoginParam::from_database(self, "configured_");
@@ -240,10 +236,7 @@ impl Context {
             .sql
             .get_raw_config_int(self, "dbversion")
             .unwrap_or_default();
-        let journal_mode = self
-            .sql
-            .query_get_value(self, "PRAGMA journal_mode;", rusqlite::NO_PARAMS)
-            .unwrap_or_else(|| "unknown".to_string());
+
         let e2ee_enabled = self.get_config_int(Config::E2eeEnabled);
         let mdns_enabled = self.get_config_int(Config::MdnsEnabled);
         let bcc_self = self.get_config_int(Config::BccSelf);
@@ -258,9 +251,10 @@ impl Context {
             rusqlite::NO_PARAMS,
         );
 
-        let fingerprint_str = match SignedPublicKey::load_self(self) {
-            Ok(key) => Key::from(key).fingerprint(),
-            Err(err) => format!("<key failure: {}>", err),
+        let fingerprint_str = if let Some(key) = Key::from_self_public(self, &l2.addr, &self.sql) {
+            key.fingerprint()
+        } else {
+            "<Not yet calculated>".into()
         };
 
         let inbox_watch = self.get_config_int(Config::InboxWatch);
@@ -288,7 +282,6 @@ impl Context {
         res.insert("number_of_contacts", contacts.to_string());
         res.insert("database_dir", self.get_dbfile().display().to_string());
         res.insert("database_version", dbversion.to_string());
-        res.insert("journal_mode", journal_mode);
         res.insert("blobdir", self.get_blobdir().display().to_string());
         res.insert("display_name", displayname.unwrap_or_else(|| unset.into()));
         res.insert(
@@ -319,9 +312,6 @@ impl Context {
         );
         res.insert("fingerprint", fingerprint_str);
 
-        let elapsed = self.creation_time.elapsed();
-        res.insert("uptime", duration_to_str(elapsed.unwrap_or_default()));
-
         res
     }
 
@@ -488,14 +478,14 @@ impl Default for RunningState {
 }
 
 #[derive(Debug, Default)]
-pub(crate) struct BobStatus {
+pub struct BobStatus {
     pub expects: i32,
     pub status: i32,
     pub qr_scan: Option<Lot>,
 }
 
 #[derive(Debug, PartialEq)]
-pub(crate) enum PerformJobsNeeded {
+pub enum PerformJobsNeeded {
     Not,
     AtOnce,
     AvoidDos,
@@ -512,7 +502,7 @@ pub struct SmtpState {
     pub idle: bool,
     pub suspended: bool,
     pub doing_jobs: bool,
-    pub(crate) perform_jobs_needed: PerformJobsNeeded,
+    pub perform_jobs_needed: PerformJobsNeeded,
     pub probe_network: bool,
 }
 

src/dc_tools.rs

@@ -5,24 +5,24 @@ use core::cmp::{max, min};
 use std::borrow::Cow;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
-use std::time::{Duration, SystemTime};
+use std::time::SystemTime;
 use std::{fmt, fs};
 
 use chrono::{Local, TimeZone};
 use rand::{thread_rng, Rng};
 
 use crate::context::Context;
-use crate::error::{bail, Error};
+use crate::error::Error;
 use crate::events::Event;
 
 pub(crate) fn dc_exactly_one_bit_set(v: i32) -> bool {
     0 != v && 0 == v & (v - 1)
 }
 
-/// Shortens a string to a specified length and adds "[...]" to the
-/// end of the shortened string.
-pub(crate) fn dc_truncate(buf: &str, approx_chars: usize) -> Cow<str> {
-    let ellipse = "[...]";
+/// Shortens a string to a specified length and adds "..." or "[...]" to the end of
+/// the shortened string.
+pub(crate) fn dc_truncate(buf: &str, approx_chars: usize, do_unwrap: bool) -> Cow<str> {
+    let ellipse = if do_unwrap { "..." } else { "[...]" };
 
     let count = buf.chars().count();
     if approx_chars > 0 && count > approx_chars + ellipse.len() {
@@ -75,14 +75,6 @@ pub fn dc_timestamp_to_str(wanted: i64) -> String {
     ts.format("%Y.%m.%d %H:%M:%S").to_string()
 }
 
-pub fn duration_to_str(duration: Duration) -> String {
-    let secs = duration.as_secs();
-    let h = secs / 3600;
-    let m = (secs % 3600) / 60;
-    let s = (secs % 3600) % 60;
-    format!("{}h {}m {}s", h, m, s)
-}
-
 pub(crate) fn dc_gm2local_offset() -> i64 {
     /* returns the offset that must be _added_ to an UTC/GMT-time to create the localtime.
     the function may return negative values. */
@@ -460,23 +452,6 @@ pub(crate) fn time() -> i64 {
         .as_secs() as i64
 }
 
-/// An invalid email address was encountered
-#[derive(Debug, thiserror::Error)]
-#[error("Invalid email address: {message} ({addr})")]
-pub struct InvalidEmailError {
-    message: String,
-    addr: String,
-}
-
-impl InvalidEmailError {
-    fn new(msg: impl Into<String>, addr: impl Into<String>) -> InvalidEmailError {
-        InvalidEmailError {
-            message: msg.into(),
-            addr: addr.into(),
-        }
-    }
-}
-
 /// Very simple email address wrapper.
 ///
 /// Represents an email address, right now just the `name@domain` portion.
@@ -500,7 +475,7 @@ pub struct EmailAddress {
 }
 
 impl EmailAddress {
-    pub fn new(input: &str) -> Result<Self, InvalidEmailError> {
+    pub fn new(input: &str) -> Result<Self, Error> {
         input.parse::<EmailAddress>()
     }
 }
@@ -512,55 +487,31 @@ impl fmt::Display for EmailAddress {
 }
 
 impl FromStr for EmailAddress {
-    type Err = InvalidEmailError;
+    type Err = Error;
 
     /// Performs a dead-simple parse of an email address.
-    fn from_str(input: &str) -> Result<EmailAddress, InvalidEmailError> {
-        if input.is_empty() {
-            return Err(InvalidEmailError::new("empty string is not valid", input));
-        }
+    fn from_str(input: &str) -> Result<EmailAddress, Error> {
+        ensure!(!input.is_empty(), "empty string is not valid");
         let parts: Vec<&str> = input.rsplitn(2, '@').collect();
 
-        let err = |msg: &str| {
-            Err(InvalidEmailError {
-                message: msg.to_string(),
-                addr: input.to_string(),
-            })
-        };
-        match &parts[..] {
-            [domain, local] => {
-                if local.is_empty() {
-                    return err("empty string is not valid for local part");
-                }
-                if domain.len() <= 3 {
-                    return err("domain is too short");
-                }
-                let dot = domain.find('.');
-                match dot {
-                    None => {
-                        return err("invalid domain");
-                    }
-                    Some(dot_idx) => {
-                        if dot_idx >= domain.len() - 2 {
-                            return err("invalid domain");
-                        }
-                    }
-                }
-                Ok(EmailAddress {
-                    local: (*local).to_string(),
-                    domain: (*domain).to_string(),
-                })
-            }
-            _ => err("missing '@' character"),
-        }
-    }
-}
+        ensure!(parts.len() > 1, "missing '@' character");
+        let local = parts[1];
+        let domain = parts[0];
 
-impl rusqlite::types::ToSql for EmailAddress {
-    fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
-        let val = rusqlite::types::Value::Text(self.to_string());
-        let out = rusqlite::types::ToSqlOutput::Owned(val);
-        Ok(out)
+        ensure!(
+            !local.is_empty(),
+            "empty string is not valid for local part"
+        );
+        ensure!(domain.len() > 3, "domain is too short");
+
+        let dot = domain.find('.');
+        ensure!(dot.is_some(), "invalid domain");
+        ensure!(dot.unwrap() < domain.len() - 2, "invalid domain");
+
+        Ok(EmailAddress {
+            local: local.to_string(),
+            domain: domain.to_string(),
+        })
     }
 }
 
@@ -587,42 +538,54 @@ mod tests {
     #[test]
     fn test_dc_truncate_1() {
         let s = "this is a little test string";
-        assert_eq!(dc_truncate(s, 16), "this is a [...]");
+        assert_eq!(dc_truncate(s, 16, false), "this is a [...]");
+        assert_eq!(dc_truncate(s, 16, true), "this is a ...");
     }
 
     #[test]
     fn test_dc_truncate_2() {
-        assert_eq!(dc_truncate("1234", 2), "1234");
+        assert_eq!(dc_truncate("1234", 2, false), "1234");
+        assert_eq!(dc_truncate("1234", 2, true), "1234");
     }
 
     #[test]
     fn test_dc_truncate_3() {
-        assert_eq!(dc_truncate("1234567", 1), "1[...]");
+        assert_eq!(dc_truncate("1234567", 1, false), "1[...]");
+        assert_eq!(dc_truncate("1234567", 1, true), "1...");
     }
 
     #[test]
     fn test_dc_truncate_4() {
-        assert_eq!(dc_truncate("123456", 4), "123456");
+        assert_eq!(dc_truncate("123456", 4, false), "123456");
+        assert_eq!(dc_truncate("123456", 4, true), "123456");
     }
 
     #[test]
     fn test_dc_truncate_edge() {
-        assert_eq!(dc_truncate("", 4), "");
+        assert_eq!(dc_truncate("", 4, false), "");
+        assert_eq!(dc_truncate("", 4, true), "");
 
-        assert_eq!(dc_truncate("\n  hello \n world", 4), "\n  [...]");
+        assert_eq!(dc_truncate("\n  hello \n world", 4, false), "\n  [...]");
+        assert_eq!(dc_truncate("\n  hello \n world", 4, true), "\n  ...");
 
-        assert_eq!(dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 1), "𐠈[...]");
         assert_eq!(
-            dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 0),
+            dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 1, false),
+            "𐠈[...]"
+        );
+        assert_eq!(
+            dc_truncate("𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ", 0, false),
             "𐠈0Aᝮa𫝀®!ꫛa¡0A𐢧00𐹠®A  丽ⷐએ"
         );
 
         // 9 characters, so no truncation
-        assert_eq!(dc_truncate("𑒀ὐ￠🜀\u{1e01b}A a🟠", 6), "𑒀ὐ￠🜀\u{1e01b}A a🟠",);
+        assert_eq!(
+            dc_truncate("𑒀ὐ￠🜀\u{1e01b}A a🟠", 6, false),
+            "𑒀ὐ￠🜀\u{1e01b}A a🟠",
+        );
 
         // 12 characters, truncation
         assert_eq!(
-            dc_truncate("𑒀ὐ￠🜀\u{1e01b}A a🟠bcd", 6),
+            dc_truncate("𑒀ὐ￠🜀\u{1e01b}A a🟠bcd", 6, false),
             "𑒀ὐ￠🜀\u{1e01b}A[...]",
         );
     }
@@ -738,10 +701,11 @@ mod tests {
         #[test]
         fn test_dc_truncate(
             buf: String,
-            approx_chars in 0..10000usize
+            approx_chars in 0..10000usize,
+            do_unwrap: bool,
         ) {
-            let res = dc_truncate(&buf, approx_chars);
-            let el_len = 5;
+            let res = dc_truncate(&buf, approx_chars, do_unwrap);
+            let el_len = if do_unwrap { 3 } else { 5 };
             let l = res.chars().count();
             if approx_chars > 0 {
                 assert!(
@@ -755,7 +719,11 @@ mod tests {
 
             if approx_chars > 0 && buf.chars().count() > approx_chars + el_len {
                 let l = res.len();
-                assert_eq!(&res[l-5..l], "[...]", "missing ellipsis in {}", &res);
+                if do_unwrap {
+                    assert_eq!(&res[l-3..l], "...", "missing ellipsis in {}", &res);
+                } else {
+                    assert_eq!(&res[l-5..l], "[...]", "missing ellipsis in {}", &res);
+                }
             }
         }
     }
@@ -862,37 +830,4 @@ mod tests {
         let next = dc_smeared_time(&t.ctx);
         assert!((start + count - 1) < next);
     }
-
-    #[test]
-    fn test_duration_to_str() {
-        assert_eq!(duration_to_str(Duration::from_secs(0)), "0h 0m 0s");
-        assert_eq!(duration_to_str(Duration::from_secs(59)), "0h 0m 59s");
-        assert_eq!(duration_to_str(Duration::from_secs(60)), "0h 1m 0s");
-        assert_eq!(duration_to_str(Duration::from_secs(61)), "0h 1m 1s");
-        assert_eq!(duration_to_str(Duration::from_secs(59 * 60)), "0h 59m 0s");
-        assert_eq!(
-            duration_to_str(Duration::from_secs(59 * 60 + 59)),
-            "0h 59m 59s"
-        );
-        assert_eq!(
-            duration_to_str(Duration::from_secs(59 * 60 + 60)),
-            "1h 0m 0s"
-        );
-        assert_eq!(
-            duration_to_str(Duration::from_secs(2 * 60 * 60 + 59 * 60 + 59)),
-            "2h 59m 59s"
-        );
-        assert_eq!(
-            duration_to_str(Duration::from_secs(2 * 60 * 60 + 59 * 60 + 60)),
-            "3h 0m 0s"
-        );
-        assert_eq!(
-            duration_to_str(Duration::from_secs(3 * 60 * 60 + 59)),
-            "3h 0m 59s"
-        );
-        assert_eq!(
-            duration_to_str(Duration::from_secs(3 * 60 * 60 + 60)),
-            "3h 1m 0s"
-        );
-    }
 }

src/dehtml.rs

@@ -3,6 +3,7 @@
 //! A module to remove HTML tags from the email text
 
 use lazy_static::lazy_static;
+use quick_xml;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
 lazy_static! {
@@ -34,7 +35,6 @@ pub fn dehtml(buf: &str) -> String {
     };
 
     let mut reader = quick_xml::Reader::from_str(buf);
-    reader.check_end_names(false);
 
     let mut buf = Vec::new();
 
@@ -225,23 +225,4 @@ mod tests {
             "<>\"\'& äÄöÖüÜß fooÆçÇ \u{2666}\u{200e}\u{200f}\u{200c}&noent;\u{200d}"
         );
     }
-
-    #[test]
-    fn test_unclosed_tags() {
-        let input = r##"
-        <!DOCTYPE HTML PUBLIC '-//W3C//DTD HTML 4.01 Transitional//EN'
-        'http://www.w3.org/TR/html4/loose.dtd'>
-        <html>
-        <head>
-        <title>Hi</title>
-        <meta http-equiv='Content-Type' content='text/html; charset=iso-8859-1'>						
-        </head>
-        <body>
-        lots of text
-        </body>
-        </html>
-        "##;
-        let txt = dehtml(input);
-        assert_eq!(txt.trim(), "lots of text");
-    }
 }

src/e2ee.rs

@@ -1,6 +1,7 @@
 //! End-to-end encryption support.
 
 use std::collections::HashSet;
+use std::convert::TryFrom;
 
 use mailparse::ParsedMail;
 use num_traits::FromPrimitive;
@@ -8,10 +9,10 @@ use num_traits::FromPrimitive;
 use crate::aheader::*;
 use crate::config::Config;
 use crate::context::Context;
+use crate::dc_tools::EmailAddress;
 use crate::error::*;
-use crate::headerdef::HeaderDef;
-use crate::headerdef::HeaderDefMap;
-use crate::key::{DcKey, Key, SignedPublicKey, SignedSecretKey};
+use crate::headerdef::{HeaderDef, HeaderDefMap};
+use crate::key::{self, Key, KeyPairUse, SignedPublicKey};
 use crate::keyring::*;
 use crate::peerstate::*;
 use crate::pgp;
@@ -36,7 +37,7 @@ impl EncryptHelper {
             Some(addr) => addr,
         };
 
-        let public_key = SignedPublicKey::load_self(context)?;
+        let public_key = load_or_generate_self_public_key(context, &addr)?;
 
         Ok(EncryptHelper {
             prefer_encrypt,
@@ -106,7 +107,8 @@ impl EncryptHelper {
         }
         let public_key = Key::from(self.public_key.clone());
         keyring.add_ref(&public_key);
-        let sign_key = Key::from(SignedSecretKey::load_self(context)?);
+        let sign_key = Key::from_self_private(context, self.addr.clone(), &context.sql)
+            .ok_or_else(|| format_err!("missing own private key"))?;
 
         let raw_message = mail_to_encrypt.build().as_string().into_bytes();
 
@@ -123,8 +125,8 @@ pub fn try_decrypt(
 ) -> Result<(Option<Vec<u8>>, HashSet<String>)> {
     let from = mail
         .headers
-        .get_header(HeaderDef::From_)
-        .and_then(|from_addr| mailparse::addrparse_header(&from_addr).ok())
+        .get_header_value(HeaderDef::From_)?
+        .and_then(|from_addr| mailparse::addrparse(&from_addr).ok())
         .and_then(|from| from.extract_single_info())
         .map(|from| from.addr)
         .unwrap_or_default();
@@ -186,6 +188,43 @@ pub fn try_decrypt(
     Ok((out_mail, signatures))
 }
 
+/// Load public key from database or generate a new one.
+///
+/// This will load a public key from the database, generating and
+/// storing a new one when one doesn't exist yet.  Care is taken to
+/// only generate one key per context even when multiple threads call
+/// this function concurrently.
+fn load_or_generate_self_public_key(
+    context: &Context,
+    self_addr: impl AsRef<str>,
+) -> Result<SignedPublicKey> {
+    if let Some(key) = Key::from_self_public(context, &self_addr, &context.sql) {
+        return SignedPublicKey::try_from(key)
+            .map_err(|_| Error::Message("Not a public key".into()));
+    }
+    let _guard = context.generating_key_mutex.lock().unwrap();
+
+    // Check again in case the key was generated while we were waiting for the lock.
+    if let Some(key) = Key::from_self_public(context, &self_addr, &context.sql) {
+        return SignedPublicKey::try_from(key)
+            .map_err(|_| Error::Message("Not a public key".into()));
+    }
+
+    let start = std::time::Instant::now();
+    info!(
+        context,
+        "Generating keypair with {} bits, e={} ...", 2048, 65537,
+    );
+    let keypair = pgp::create_keypair(EmailAddress::new(self_addr.as_ref())?)?;
+    key::store_self_keypair(context, &keypair, KeyPairUse::Default)?;
+    info!(
+        context,
+        "Keypair generated in {:.3}s.",
+        start.elapsed().as_secs()
+    );
+    Ok(keypair.public)
+}
+
 /// Returns a reference to the encrypted payload and validates the autocrypt structure.
 fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Result<&'a ParsedMail<'b>> {
     ensure!(
@@ -307,7 +346,6 @@ fn contains_report(mail: &ParsedMail<'_>) -> bool {
 ///
 /// If this succeeds you are also guaranteed that the
 /// [Config::ConfiguredAddr] is configured, this address is returned.
-// TODO, remove this once deltachat::key::Key no longer exists.
 pub fn ensure_secret_key_exists(context: &Context) -> Result<String> {
     let self_addr = context.get_config(Config::ConfiguredAddr).ok_or_else(|| {
         format_err!(concat!(
@@ -315,7 +353,7 @@ pub fn ensure_secret_key_exists(context: &Context) -> Result<String> {
             "cannot ensure secret key if not configured."
         ))
     })?;
-    SignedPublicKey::load_self(context)?;
+    load_or_generate_self_public_key(context, &self_addr)?;
     Ok(self_addr)
 }
 
@@ -366,6 +404,49 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         );
     }
 
+    mod load_or_generate_self_public_key {
+        use super::*;
+
+        #[test]
+        fn test_existing() {
+            let t = dummy_context();
+            let addr = configure_alice_keypair(&t.ctx);
+            let key = load_or_generate_self_public_key(&t.ctx, addr);
+            assert!(key.is_ok());
+        }
+
+        #[test]
+        #[ignore] // generating keys is expensive
+        fn test_generate() {
+            let t = dummy_context();
+            let addr = "alice@example.org";
+            let key0 = load_or_generate_self_public_key(&t.ctx, addr);
+            assert!(key0.is_ok());
+            let key1 = load_or_generate_self_public_key(&t.ctx, addr);
+            assert!(key1.is_ok());
+            assert_eq!(key0.unwrap(), key1.unwrap());
+        }
+
+        #[test]
+        #[ignore]
+        fn test_generate_concurrent() {
+            use std::sync::Arc;
+            use std::thread;
+
+            let t = dummy_context();
+            let ctx = Arc::new(t.ctx);
+            let ctx0 = Arc::clone(&ctx);
+            let thr0 =
+                thread::spawn(move || load_or_generate_self_public_key(&ctx0, "alice@example.org"));
+            let ctx1 = Arc::clone(&ctx);
+            let thr1 =
+                thread::spawn(move || load_or_generate_self_public_key(&ctx1, "alice@example.org"));
+            let res0 = thr0.join().unwrap();
+            let res1 = thr1.join().unwrap();
+            assert_eq!(res0.unwrap(), res1.unwrap());
+        }
+    }
+
     #[test]
     fn test_has_decrypted_pgp_armor() {
         let data = b" -----BEGIN PGP MESSAGE-----";

src/error.rs

@@ -1,13 +1,189 @@
 //! # Error handling
 
-pub use anyhow::{bail, ensure, format_err, Error, Result};
+use lettre_email::mime;
 
-// #[fail(display = "Invalid Message ID.")]
-// InvalidMsgId,
-// #[fail(display = "Watch folder not found {:?}", _0)]
-// WatchFolderNotFound(String),
-// #[fail(display = "Not Configured")]
-// NotConfigured,
+#[derive(Debug, Fail)]
+pub enum Error {
+    #[fail(display = "{:?}", _0)]
+    Failure(failure::Error),
+
+    #[fail(display = "SQL error: {:?}", _0)]
+    SqlError(#[cause] crate::sql::Error),
+
+    #[fail(display = "{:?}", _0)]
+    Io(std::io::Error),
+
+    #[fail(display = "{:?}", _0)]
+    Message(String),
+
+    #[fail(display = "{:?}", _0)]
+    MessageWithCause(String, #[cause] failure::Error, failure::Backtrace),
+
+    #[fail(display = "{:?}", _0)]
+    Image(image_meta::ImageError),
+
+    #[fail(display = "{:?}", _0)]
+    Utf8(std::str::Utf8Error),
+
+    #[fail(display = "PGP: {:?}", _0)]
+    Pgp(pgp::errors::Error),
+
+    #[fail(display = "Base64Decode: {:?}", _0)]
+    Base64Decode(base64::DecodeError),
+
+    #[fail(display = "{:?}", _0)]
+    FromUtf8(std::string::FromUtf8Error),
+
+    #[fail(display = "{}", _0)]
+    BlobError(#[cause] crate::blob::BlobError),
+
+    #[fail(display = "Invalid Message ID.")]
+    InvalidMsgId,
+
+    #[fail(display = "Watch folder not found {:?}", _0)]
+    WatchFolderNotFound(String),
+
+    #[fail(display = "Invalid Email: {:?}", _0)]
+    MailParseError(#[cause] mailparse::MailParseError),
+
+    #[fail(display = "Building invalid Email: {:?}", _0)]
+    LettreError(#[cause] lettre_email::error::Error),
+
+    #[fail(display = "SMTP error: {:?}", _0)]
+    SmtpError(#[cause] async_smtp::error::Error),
+
+    #[fail(display = "FromStr error: {:?}", _0)]
+    FromStr(#[cause] mime::FromStrError),
+
+    #[fail(display = "Not Configured")]
+    NotConfigured,
+}
+
+pub type Result<T> = std::result::Result<T, Error>;
+
+impl From<crate::sql::Error> for Error {
+    fn from(err: crate::sql::Error) -> Error {
+        Error::SqlError(err)
+    }
+}
+
+impl From<base64::DecodeError> for Error {
+    fn from(err: base64::DecodeError) -> Error {
+        Error::Base64Decode(err)
+    }
+}
+
+impl From<failure::Error> for Error {
+    fn from(err: failure::Error) -> Error {
+        Error::Failure(err)
+    }
+}
+
+impl From<std::io::Error> for Error {
+    fn from(err: std::io::Error) -> Error {
+        Error::Io(err)
+    }
+}
+
+impl From<std::str::Utf8Error> for Error {
+    fn from(err: std::str::Utf8Error) -> Error {
+        Error::Utf8(err)
+    }
+}
+
+impl From<image_meta::ImageError> for Error {
+    fn from(err: image_meta::ImageError) -> Error {
+        Error::Image(err)
+    }
+}
+
+impl From<pgp::errors::Error> for Error {
+    fn from(err: pgp::errors::Error) -> Error {
+        Error::Pgp(err)
+    }
+}
+
+impl From<std::string::FromUtf8Error> for Error {
+    fn from(err: std::string::FromUtf8Error) -> Error {
+        Error::FromUtf8(err)
+    }
+}
+
+impl From<crate::blob::BlobError> for Error {
+    fn from(err: crate::blob::BlobError) -> Error {
+        Error::BlobError(err)
+    }
+}
+
+impl From<crate::message::InvalidMsgId> for Error {
+    fn from(_err: crate::message::InvalidMsgId) -> Error {
+        Error::InvalidMsgId
+    }
+}
+
+impl From<crate::key::SaveKeyError> for Error {
+    fn from(err: crate::key::SaveKeyError) -> Error {
+        Error::MessageWithCause(format!("{}", err), err.into(), failure::Backtrace::new())
+    }
+}
+
+impl From<crate::pgp::PgpKeygenError> for Error {
+    fn from(err: crate::pgp::PgpKeygenError) -> Error {
+        Error::MessageWithCause(format!("{}", err), err.into(), failure::Backtrace::new())
+    }
+}
+
+impl From<mailparse::MailParseError> for Error {
+    fn from(err: mailparse::MailParseError) -> Error {
+        Error::MailParseError(err)
+    }
+}
+
+impl From<lettre_email::error::Error> for Error {
+    fn from(err: lettre_email::error::Error) -> Error {
+        Error::LettreError(err)
+    }
+}
+
+impl From<mime::FromStrError> for Error {
+    fn from(err: mime::FromStrError) -> Error {
+        Error::FromStr(err)
+    }
+}
+
+#[macro_export]
+macro_rules! bail {
+    ($e:expr) => {
+        return Err($crate::error::Error::Message($e.to_string()));
+    };
+    ($fmt:expr, $($arg:tt)+) => {
+        return Err($crate::error::Error::Message(format!($fmt, $($arg)+)));
+    };
+}
+
+#[macro_export]
+macro_rules! format_err {
+    ($e:expr) => {
+        $crate::error::Error::Message($e.to_string());
+    };
+    ($fmt:expr, $($arg:tt)+) => {
+        $crate::error::Error::Message(format!($fmt, $($arg)+));
+    };
+}
+
+#[macro_export(local_inner_macros)]
+macro_rules! ensure {
+    ($cond:expr, $e:expr) => {
+        if !($cond) {
+            bail!($e);
+        }
+    };
+    ($cond:expr, $fmt:expr, $($arg:tt)+) => {
+        if !($cond) {
+            bail!($fmt, $($arg)+);
+        }
+    };
+}
 
 #[macro_export]
 macro_rules! ensure_eq {

src/events.rs

@@ -201,4 +201,10 @@ pub enum Event {
     ///     (Bob has verified alice and waits until Alice does the same for him)
     #[strum(props(id = "2061"))]
     SecurejoinJoinerProgress { contact_id: u32, progress: usize },
+
+    /// This event is sent out to the inviter when a joiner successfully joined a group.
+    /// @param data1 (int) chat_id
+    /// @param data2 (int) contact_id
+    #[strum(props(id = "2062"))]
+    SecurejoinMemberAdded { chat_id: ChatId, contact_id: u32 },
 }

src/headerdef.rs

@@ -1,5 +1,5 @@
 use crate::strum::AsStaticRef;
-use mailparse::{MailHeader, MailHeaderMap};
+use mailparse::{MailHeader, MailHeaderMap, MailParseError};
 
 #[derive(Debug, Display, Clone, PartialEq, Eq, EnumVariantNames, AsStaticStr)]
 #[strum(serialize_all = "kebab_case")]
@@ -26,6 +26,7 @@ pub enum HeaderDef {
     ChatGroupName,
     ChatGroupNameChanged,
     ChatVerified,
+    ChatGroupImage, // deprecated
     ChatGroupAvatar,
     ChatUserAvatar,
     ChatVoiceMessage,
@@ -52,17 +53,13 @@ impl HeaderDef {
 }
 
 pub trait HeaderDefMap {
-    fn get_header_value(&self, headerdef: HeaderDef) -> Option<String>;
-    fn get_header(&self, headerdef: HeaderDef) -> Option<&MailHeader>;
+    fn get_header_value(&self, headerdef: HeaderDef) -> Result<Option<String>, MailParseError>;
 }
 
 impl HeaderDefMap for [MailHeader<'_>] {
-    fn get_header_value(&self, headerdef: HeaderDef) -> Option<String> {
+    fn get_header_value(&self, headerdef: HeaderDef) -> Result<Option<String>, MailParseError> {
         self.get_first_value(headerdef.get_headername())
     }
-    fn get_header(&self, headerdef: HeaderDef) -> Option<&MailHeader> {
-        self.get_first_header(headerdef.get_headername())
-    }
 }
 
 #[cfg(test)]
@@ -83,13 +80,18 @@ mod tests {
         let (headers, _) =
             mailparse::parse_headers(b"fRoM: Bob\naUtoCryPt-SeTup-MessAge: v99").unwrap();
         assert_eq!(
-            headers.get_header_value(HeaderDef::AutocryptSetupMessage),
+            headers
+                .get_header_value(HeaderDef::AutocryptSetupMessage)
+                .unwrap(),
             Some("v99".to_string())
         );
         assert_eq!(
-            headers.get_header_value(HeaderDef::From_),
+            headers.get_header_value(HeaderDef::From_).unwrap(),
             Some("Bob".to_string())
         );
-        assert_eq!(headers.get_header_value(HeaderDef::Autocrypt), None);
+        assert_eq!(
+            headers.get_header_value(HeaderDef::Autocrypt).unwrap(),
+            None
+        );
     }
 }

src/imap/client.rs

@@ -1,104 +0,0 @@
-use async_imap::{
-    error::{Error as ImapError, Result as ImapResult},
-    Client as ImapClient,
-};
-use async_native_tls::TlsStream;
-use async_std::net::{self, TcpStream};
-
-use super::session::Session;
-use crate::login_param::{dc_build_tls, CertificateChecks};
-
-#[derive(Debug)]
-pub(crate) enum Client {
-    Secure(ImapClient<TlsStream<TcpStream>>),
-    Insecure(ImapClient<TcpStream>),
-}
-
-impl Client {
-    pub async fn connect_secure<A: net::ToSocketAddrs, S: AsRef<str>>(
-        addr: A,
-        domain: S,
-        certificate_checks: CertificateChecks,
-    ) -> ImapResult<Self> {
-        let stream = TcpStream::connect(addr).await?;
-        let tls = dc_build_tls(certificate_checks);
-        let tls_stream = tls.connect(domain.as_ref(), stream).await?;
-        let mut client = ImapClient::new(tls_stream);
-        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
-            client.debug = true;
-        }
-
-        let _greeting = client
-            .read_response()
-            .await
-            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
-
-        Ok(Client::Secure(client))
-    }
-
-    pub async fn connect_insecure<A: net::ToSocketAddrs>(addr: A) -> ImapResult<Self> {
-        let stream = TcpStream::connect(addr).await?;
-
-        let mut client = ImapClient::new(stream);
-        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
-            client.debug = true;
-        }
-        let _greeting = client
-            .read_response()
-            .await
-            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
-
-        Ok(Client::Insecure(client))
-    }
-
-    pub async fn secure<S: AsRef<str>>(
-        self,
-        domain: S,
-        certificate_checks: CertificateChecks,
-    ) -> ImapResult<Client> {
-        match self {
-            Client::Insecure(client) => {
-                let tls = dc_build_tls(certificate_checks);
-                let client_sec = client.secure(domain, tls).await?;
-
-                Ok(Client::Secure(client_sec))
-            }
-            // Nothing to do
-            Client::Secure(_) => Ok(self),
-        }
-    }
-
-    pub async fn authenticate<A: async_imap::Authenticator, S: AsRef<str>>(
-        self,
-        auth_type: S,
-        authenticator: &A,
-    ) -> Result<Session, (ImapError, Client)> {
-        match self {
-            Client::Secure(i) => match i.authenticate(auth_type, authenticator).await {
-                Ok(session) => Ok(Session::Secure(session)),
-                Err((err, c)) => Err((err, Client::Secure(c))),
-            },
-            Client::Insecure(i) => match i.authenticate(auth_type, authenticator).await {
-                Ok(session) => Ok(Session::Insecure(session)),
-                Err((err, c)) => Err((err, Client::Insecure(c))),
-            },
-        }
-    }
-
-    pub async fn login<U: AsRef<str>, P: AsRef<str>>(
-        self,
-        username: U,
-        password: P,
-    ) -> Result<Session, (ImapError, Client)> {
-        match self {
-            Client::Secure(i) => match i.login(username, password).await {
-                Ok(session) => Ok(Session::Secure(session)),
-                Err((err, c)) => Err((err, Client::Secure(c))),
-            },
-            Client::Insecure(i) => match i.login(username, password).await {
-                Ok(session) => Ok(Session::Insecure(session)),
-                Err((err, c)) => Err((err, Client::Insecure(c))),
-            },
-        }
-    }
-}

src/imap/idle.rs

@@ -1,56 +1,42 @@
 use super::Imap;
 
-use async_imap::extensions::idle::{Handle as ImapIdleHandle, IdleResponse};
-use async_native_tls::TlsStream;
-use async_std::net::TcpStream;
+use async_imap::extensions::idle::IdleResponse;
 use async_std::prelude::*;
 use async_std::task;
 use std::sync::atomic::Ordering;
 use std::time::{Duration, SystemTime};
 
 use crate::context::Context;
+use crate::imap_client::*;
 
 use super::select_folder;
-use super::session::Session;
 
 type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug, thiserror::Error)]
+#[derive(Debug, Fail)]
 pub enum Error {
-    #[error("IMAP IDLE protocol failed to init/complete")]
-    IdleProtocolFailed(#[from] async_imap::error::Error),
+    #[fail(display = "IMAP IDLE protocol failed to init/complete")]
+    IdleProtocolFailed(#[cause] async_imap::error::Error),
 
-    #[error("IMAP IDLE protocol timed out")]
-    IdleTimeout(#[from] async_std::future::TimeoutError),
+    #[fail(display = "IMAP IDLE protocol timed out")]
+    IdleTimeout(#[cause] async_std::future::TimeoutError),
 
-    #[error("IMAP server does not have IDLE capability")]
+    #[fail(display = "IMAP server does not have IDLE capability")]
     IdleAbilityMissing,
 
-    #[error("IMAP select folder error")]
-    SelectFolderError(#[from] select_folder::Error),
+    #[fail(display = "IMAP select folder error")]
+    SelectFolderError(#[cause] select_folder::Error),
 
-    #[error("Setup handle error")]
-    SetupHandleError(#[from] super::Error),
+    #[fail(display = "IMAP error")]
+    ImapError(#[cause] async_imap::error::Error),
+
+    #[fail(display = "Setup handle error")]
+    SetupHandleError(#[cause] super::Error),
 }
 
-#[derive(Debug)]
-pub(crate) enum IdleHandle {
-    Secure(ImapIdleHandle<TlsStream<TcpStream>>),
-    Insecure(ImapIdleHandle<TcpStream>),
-}
-
-impl Session {
-    pub fn idle(self) -> IdleHandle {
-        match self {
-            Session::Secure(i) => {
-                let h = i.idle();
-                IdleHandle::Secure(h)
-            }
-            Session::Insecure(i) => {
-                let h = i.idle();
-                IdleHandle::Insecure(h)
-            }
-        }
+impl From<select_folder::Error> for Error {
+    fn from(err: select_folder::Error) -> Error {
+        Error::SelectFolderError(err)
     }
 }
 
@@ -65,7 +51,9 @@ impl Imap {
                 return Err(Error::IdleAbilityMissing);
             }
 
-            self.setup_handle_if_needed(context).await?;
+            self.setup_handle_if_needed(context)
+                .await
+                .map_err(Error::SetupHandleError)?;
 
             self.select_folder(context, watch_folder.clone()).await?;
 
@@ -76,7 +64,9 @@ impl Imap {
                     // BEWARE: If you change the Secure branch you
                     // typically also need to change the Insecure branch.
                     IdleHandle::Secure(mut handle) => {
-                        handle.init().await?;
+                        if let Err(err) = handle.init().await {
+                            return Err(Error::IdleProtocolFailed(err));
+                        }
 
                         let (idle_wait, interrupt) = handle.wait_with_timeout(timeout);
                         *self.interrupt.lock().await = Some(interrupt);
@@ -131,7 +121,9 @@ impl Imap {
                         }
                     }
                     IdleHandle::Insecure(mut handle) => {
-                        handle.init().await?;
+                        if let Err(err) = handle.init().await {
+                            return Err(Error::IdleProtocolFailed(err));
+                        }
 
                         let (idle_wait, interrupt) = handle.wait_with_timeout(timeout);
                         *self.interrupt.lock().await = Some(interrupt);

src/imap/mod.rs

@@ -22,66 +22,93 @@ use crate::dc_receive_imf::{
 };
 use crate::events::Event;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
+use crate::imap_client::*;
 use crate::job::{job_add, Action};
 use crate::login_param::{CertificateChecks, LoginParam};
 use crate::message::{self, update_server_uid};
-use crate::mimeparser;
 use crate::oauth2::dc_get_oauth2_access_token;
 use crate::param::Params;
 use crate::stock::StockMessage;
 
-mod client;
 mod idle;
 pub mod select_folder;
-mod session;
 
-use client::Client;
-use session::Session;
+const DC_IMAP_SEEN: usize = 0x0001;
 
 type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug, thiserror::Error)]
+#[derive(Debug, Fail)]
 pub enum Error {
-    #[error("IMAP Connect without configured params")]
+    #[fail(display = "IMAP Connect without configured params")]
     ConnectWithoutConfigure,
 
-    #[error("IMAP Connection Failed params: {0}")]
+    #[fail(display = "IMAP Connection Failed params: {}", _0)]
     ConnectionFailed(String),
 
-    #[error("IMAP No Connection established")]
+    #[fail(display = "IMAP No Connection established")]
     NoConnection,
 
-    #[error("IMAP Could not get OAUTH token")]
+    #[fail(display = "IMAP Could not get OAUTH token")]
     OauthError,
 
-    #[error("IMAP Could not login as {0}")]
+    #[fail(display = "IMAP Could not login as {}", _0)]
     LoginFailed(String),
 
-    #[error("IMAP Could not fetch")]
-    FetchFailed(#[from] async_imap::error::Error),
+    #[fail(display = "IMAP Could not fetch")]
+    FetchFailed(#[cause] async_imap::error::Error),
 
-    #[error("IMAP operation attempted while it is torn down")]
+    #[fail(display = "IMAP operation attempted while it is torn down")]
     InTeardown,
 
-    #[error("IMAP operation attempted while it is torn down")]
-    SqlError(#[from] crate::sql::Error),
+    #[fail(display = "IMAP operation attempted while it is torn down")]
+    SqlError(#[cause] crate::sql::Error),
 
-    #[error("IMAP got error from elsewhere")]
-    WrappedError(#[from] crate::error::Error),
+    #[fail(display = "IMAP got error from elsewhere")]
+    WrappedError(#[cause] crate::error::Error),
 
-    #[error("IMAP select folder error")]
-    SelectFolderError(#[from] select_folder::Error),
+    #[fail(display = "IMAP select folder error")]
+    SelectFolderError(#[cause] select_folder::Error),
 
-    #[error("Mail parse error")]
-    MailParseError(#[from] mailparse::MailParseError),
+    #[fail(display = "Mail parse error")]
+    MailParseError(#[cause] mailparse::MailParseError),
 
-    #[error("No mailbox selected, folder: {0}")]
+    #[fail(display = "No mailbox selected, folder: {:?}", _0)]
     NoMailbox(String),
 
-    #[error("IMAP other error: {0}")]
+    #[fail(display = "IMAP other error: {:?}", _0)]
     Other(String),
 }
 
+impl From<crate::sql::Error> for Error {
+    fn from(err: crate::sql::Error) -> Error {
+        Error::SqlError(err)
+    }
+}
+
+impl From<crate::error::Error> for Error {
+    fn from(err: crate::error::Error) -> Error {
+        Error::WrappedError(err)
+    }
+}
+
+impl From<Error> for crate::error::Error {
+    fn from(err: Error) -> crate::error::Error {
+        crate::error::Error::Message(err.to_string())
+    }
+}
+
+impl From<select_folder::Error> for Error {
+    fn from(err: select_folder::Error) -> Error {
+        Error::SelectFolderError(err)
+    }
+}
+
+impl From<mailparse::MailParseError> for Error {
+    fn from(err: mailparse::MailParseError) -> Error {
+        Error::MailParseError(err)
+    }
+}
+
 #[derive(Debug, Display, Clone, Copy, PartialEq, Eq)]
 pub enum ImapActionResult {
     Failed,
@@ -135,7 +162,7 @@ impl async_imap::Authenticator for OAuth2 {
     }
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug)]
 enum FolderMeaning {
     Unknown,
     SentObjects,
@@ -155,10 +182,7 @@ struct ImapConfig {
     pub selected_mailbox: Option<Mailbox>,
     pub selected_folder_needs_expunge: bool,
     pub can_idle: bool,
-
-    /// True if the server has MOVE capability as defined in
-    /// https://tools.ietf.org/html/rfc6851
-    pub can_move: bool,
+    pub imap_delimiter: char,
 }
 
 impl Default for ImapConfig {
@@ -175,7 +199,7 @@ impl Default for ImapConfig {
             selected_mailbox: None,
             selected_folder_needs_expunge: false,
             can_idle: false,
-            can_move: false,
+            imap_delimiter: '.',
         }
     }
 }
@@ -269,7 +293,7 @@ impl Imap {
                     let imap_port = config.imap_port;
                     context.stock_string_repl_str2(
                         StockMessage::ServerResponse,
-                        format!("IMAP {}:{}", imap_server, imap_port),
+                        format!("{}:{}", imap_server, imap_port),
                         err.to_string(),
                     )
                 };
@@ -328,7 +352,6 @@ impl Imap {
         cfg.imap_port = 0;
 
         cfg.can_idle = false;
-        cfg.can_move = false;
     }
 
     /// Connects to imap account using already-configured parameters.
@@ -389,7 +412,6 @@ impl Imap {
                         true
                     } else {
                         let can_idle = caps.has_str("IDLE");
-                        let can_move = caps.has_str("MOVE");
                         let caps_list = caps.iter().fold(String::new(), |s, c| {
                             if let Capability::Atom(x) = c {
                                 s + &format!(" {}", x)
@@ -399,7 +421,6 @@ impl Imap {
                         });
 
                         self.config.write().await.can_idle = can_idle;
-                        self.config.write().await.can_move = can_move;
                         *self.connected.lock().await = true;
                         emit_event!(
                             context,
@@ -567,7 +588,7 @@ impl Imap {
 
         let mut list = if let Some(ref mut session) = &mut *self.session.lock().await {
             // fetch messages with larger UID than the last one seen
-            // `(UID FETCH lastseenuid+1:*)`, see RFC 4549
+            // (`UID FETCH lastseenuid+1:*)`, see RFC 4549
             let set = format!("{}:*", last_seen_uid + 1);
             match session.uid_fetch(set, PREFETCH_FLAGS).await {
                 Ok(list) => list,
@@ -605,12 +626,7 @@ impl Imap {
 
             let headers = get_fetch_headers(fetch)?;
             let message_id = prefetch_get_message_id(&headers).unwrap_or_default();
-            if let Ok(true) =
-                precheck_imf(context, &message_id, folder.as_ref(), cur_uid).map_err(|err| {
-                    warn!(context, "precheck_imf error: {}", err);
-                    err
-                })
-            {
+            if precheck_imf(context, &message_id, folder.as_ref(), cur_uid) {
                 // we know the message-id already or don't want the message otherwise.
                 info!(
                     context,
@@ -619,9 +635,6 @@ impl Imap {
                     folder.as_ref(),
                 );
             } else {
-                // we do not know the message-id
-                // or the message-id is missing (in this case, we create one in the further process)
-                // or some other error happened
                 let show = prefetch_should_download(context, &headers, show_emails)
                     .map_err(|err| {
                         warn!(context, "prefetch_should_download error: {}", err);
@@ -732,41 +745,54 @@ impl Imap {
             return Err(Error::Other("Could not get IMAP session".to_string()));
         };
 
-        if let Some(msg) = msgs.first() {
-            // XXX put flags into a set and pass them to dc_receive_imf
-            let is_deleted = msg.flags().any(|flag| flag == Flag::Deleted);
-            let is_seen = msg.flags().any(|flag| flag == Flag::Seen);
-
-            if !is_deleted && msg.body().is_some() {
-                let body = msg.body().unwrap_or_default();
-                if let Err(err) =
-                    dc_receive_imf(context, &body, folder.as_ref(), server_uid, is_seen)
-                {
-                    return Err(Error::Other(format!("dc_receive_imf error: {}", err)));
-                }
-            }
-        } else {
+        if msgs.is_empty() {
             warn!(
                 context,
                 "Message #{} does not exist in folder \"{}\".",
                 server_uid,
                 folder.as_ref()
             );
+        } else {
+            let msg = &msgs[0];
+
+            // XXX put flags into a set and pass them to dc_receive_imf
+            let is_deleted = msg.flags().any(|flag| match flag {
+                Flag::Deleted => true,
+                _ => false,
+            });
+            let is_seen = msg.flags().any(|flag| match flag {
+                Flag::Seen => true,
+                _ => false,
+            });
+
+            let flags = if is_seen { DC_IMAP_SEEN } else { 0 };
+
+            if !is_deleted && msg.body().is_some() {
+                let body = msg.body().unwrap_or_default();
+                if let Err(err) =
+                    dc_receive_imf(context, &body, folder.as_ref(), server_uid, flags as u32)
+                {
+                    warn!(
+                        context,
+                        "dc_receive_imf failed for imap-message {}/{}: {:?}",
+                        folder.as_ref(),
+                        server_uid,
+                        err
+                    );
+                }
+            }
         }
 
         Ok(())
     }
 
-    pub fn can_move(&self) -> bool {
-        task::block_on(async move { self.config.read().await.can_move })
-    }
-
     pub fn mv(
         &self,
         context: &Context,
         folder: &str,
         uid: u32,
         dest_folder: &str,
+        dest_uid: &mut u32,
     ) -> ImapActionResult {
         task::block_on(async move {
             if folder == dest_folder {
@@ -783,75 +809,71 @@ impl Imap {
                 return imapresult;
             }
             // we are connected, and the folder is selected
+
+            // XXX Rust-Imap provides no target uid on mv, so just set it to 0
+            *dest_uid = 0;
+
             let set = format!("{}", uid);
             let display_folder_id = format!("{}/{}", folder, uid);
-
-            if self.can_move() {
-                if let Some(ref mut session) = &mut *self.session.lock().await {
-                    match session.uid_mv(&set, &dest_folder).await {
-                        Ok(_) => {
-                            emit_event!(
-                                context,
-                                Event::ImapMessageMoved(format!(
-                                    "IMAP Message {} moved to {}",
-                                    display_folder_id, dest_folder
-                                ))
-                            );
-                            return ImapActionResult::Success;
-                        }
-                        Err(err) => {
-                            warn!(
-                                context,
-                                "Cannot move message, fallback to COPY/DELETE {}/{} to {}: {}",
-                                folder,
-                                uid,
-                                dest_folder,
-                                err
-                            );
-                        }
-                    }
-                } else {
-                    unreachable!();
-                };
-            } else {
-                info!(
-                    context,
-                    "Server does not support MOVE, fallback to COPY/DELETE {}/{} to {}",
-                    folder,
-                    uid,
-                    dest_folder
-                );
-            }
-
             if let Some(ref mut session) = &mut *self.session.lock().await {
-                if let Err(err) = session.uid_copy(&set, &dest_folder).await {
-                    warn!(context, "Could not copy message: {}", err);
-                    return ImapActionResult::Failed;
+                match session.uid_mv(&set, &dest_folder).await {
+                    Ok(_) => {
+                        emit_event!(
+                            context,
+                            Event::ImapMessageMoved(format!(
+                                "IMAP Message {} moved to {}",
+                                display_folder_id, dest_folder
+                            ))
+                        );
+                        return ImapActionResult::Success;
+                    }
+                    Err(err) => {
+                        warn!(
+                            context,
+                            "Cannot move message, fallback to COPY/DELETE {}/{} to {}: {}",
+                            folder,
+                            uid,
+                            dest_folder,
+                            err
+                        );
+                    }
                 }
             } else {
                 unreachable!();
-            }
+            };
 
-            if !self.add_flag_finalized(context, uid, "\\Deleted").await {
-                warn!(context, "Cannot mark {} as \"Deleted\" after copy.", uid);
-                emit_event!(
-                    context,
-                    Event::ImapMessageMoved(format!(
-                        "IMAP Message {} copied to {} (delete FAILED)",
-                        display_folder_id, dest_folder
-                    ))
-                );
-                ImapActionResult::Failed
+            if let Some(ref mut session) = &mut *self.session.lock().await {
+                match session.uid_copy(&set, &dest_folder).await {
+                    Ok(_) => {
+                        if !self.add_flag_finalized(context, uid, "\\Deleted").await {
+                            warn!(context, "Cannot mark {} as \"Deleted\" after copy.", uid);
+                            emit_event!(
+                                context,
+                                Event::ImapMessageMoved(format!(
+                                    "IMAP Message {} copied to {} (delete FAILED)",
+                                    display_folder_id, dest_folder
+                                ))
+                            );
+                            ImapActionResult::Failed
+                        } else {
+                            self.config.write().await.selected_folder_needs_expunge = true;
+                            emit_event!(
+                                context,
+                                Event::ImapMessageMoved(format!(
+                                    "IMAP Message {} copied to {} (delete successfull)",
+                                    display_folder_id, dest_folder
+                                ))
+                            );
+                            ImapActionResult::Success
+                        }
+                    }
+                    Err(err) => {
+                        warn!(context, "Could not copy message: {}", err);
+                        ImapActionResult::Failed
+                    }
+                }
             } else {
-                self.config.write().await.selected_folder_needs_expunge = true;
-                emit_event!(
-                    context,
-                    Event::ImapMessageMoved(format!(
-                        "IMAP Message {} copied to {} (delete successfull)",
-                        display_folder_id, dest_folder
-                    ))
-                );
-                ImapActionResult::Success
+                unreachable!();
             }
         })
     }
@@ -957,15 +979,16 @@ impl Imap {
         })
     }
 
+    // only returns 0 on connection problems; we should try later again in this case *
     pub fn delete_msg(
         &self,
         context: &Context,
         message_id: &str,
         folder: &str,
-        uid: u32,
+        uid: &mut u32,
     ) -> ImapActionResult {
         task::block_on(async move {
-            if let Some(imapresult) = self.prepare_imap_operation_on_msg(context, folder, uid) {
+            if let Some(imapresult) = self.prepare_imap_operation_on_msg(context, folder, *uid) {
                 return imapresult;
             }
             // we are connected, and the folder is selected
@@ -987,7 +1010,7 @@ impl Imap {
                                 display_imap_id,
                                 message_id,
                             );
-                            return ImapActionResult::AlreadyDone;
+                            return ImapActionResult::Failed;
                         };
 
                         let remote_message_id = get_fetch_headers(fetch)
@@ -1002,7 +1025,7 @@ impl Imap {
                                 remote_message_id,
                                 message_id,
                             );
-                            return ImapActionResult::Failed;
+                            *uid = 0;
                         }
                     }
                     Err(err) => {
@@ -1010,18 +1033,18 @@ impl Imap {
                             context,
                             "Cannot delete {} on IMAP: {}", display_imap_id, err
                         );
-                        return ImapActionResult::RetryLater;
+                        *uid = 0;
                     }
                 }
             }
 
             // mark the message for deletion
-            if !self.add_flag_finalized(context, uid, "\\Deleted").await {
+            if !self.add_flag_finalized(context, *uid, "\\Deleted").await {
                 warn!(
                     context,
                     "Cannot mark message {} as \"Deleted\".", display_imap_id
                 );
-                ImapActionResult::RetryLater
+                ImapActionResult::Failed
             } else {
                 emit_event!(
                     context,
@@ -1040,14 +1063,12 @@ impl Imap {
         let folders_configured = context
             .sql
             .get_raw_config_int(context, "folders_configured");
-        if folders_configured.unwrap_or_default() >= DC_FOLDERS_CONFIGURED_VERSION {
+        if folders_configured.unwrap_or_default() >= 3 {
+            // the "3" here we increase if we have future updates to
+            // to folder configuration
             return Ok(());
         }
 
-        self.configure_folders(context, create_mvbox)
-    }
-
-    pub fn configure_folders(&self, context: &Context, create_mvbox: bool) -> Result<()> {
         task::block_on(async move {
             if !self.is_connected().await {
                 return Err(Error::NoConnection);
@@ -1063,26 +1084,16 @@ impl Imap {
                     }
                 };
 
-                let sentbox_folder = folders
-                    .iter()
-                    .find(|folder| get_folder_meaning(folder) == FolderMeaning::SentObjects)
-                    .or_else(|| {
-                        info!(context, "can't find sentbox by attributes, checking names");
-                        folders.iter().find(|folder| {
-                            get_folder_meaning_by_name(folder) == FolderMeaning::SentObjects
-                        })
-                    });
+                let sentbox_folder =
+                    folders
+                        .iter()
+                        .find(|folder| match get_folder_meaning(folder) {
+                            FolderMeaning::SentObjects => true,
+                            _ => false,
+                        });
                 info!(context, "sentbox folder is {:?}", sentbox_folder);
 
-                let mut delimiter = ".";
-                if let Some(folder) = folders.first() {
-                    if let Some(d) = folder.delimiter() {
-                        if !d.is_empty() {
-                            delimiter = d;
-                        }
-                    }
-                }
-                info!(context, "Using \"{}\" as folder-delimiter.", delimiter);
+                let delimiter = self.config.read().await.imap_delimiter;
                 let fallback_folder = format!("INBOX{}DeltaChat", delimiter);
 
                 let mut mvbox_folder = folders
@@ -1146,11 +1157,9 @@ impl Imap {
                         Some(sentbox_folder.name()),
                     )?;
                 }
-                context.sql.set_raw_config_int(
-                    context,
-                    "folders_configured",
-                    DC_FOLDERS_CONFIGURED_VERSION,
-                )?;
+                context
+                    .sql
+                    .set_raw_config_int(context, "folders_configured", 3)?;
             }
             info!(context, "FINISHED configuring IMAP-folders.");
             Ok(())
@@ -1185,13 +1194,13 @@ impl Imap {
                 return;
             }
             if let Err(err) = self.setup_handle_if_needed(context).await {
-                error!(context, "could not setup imap connection: {}", err);
+                error!(context, "could not setup imap connection: {:?}", err);
                 return;
             }
             if let Err(err) = self.select_folder(context, Some(&folder)).await {
                 error!(
                     context,
-                    "Could not select {} for expunging: {}", folder, err
+                    "Could not select {} for expunging: {:?}", folder, err
                 );
                 return;
             }
@@ -1211,21 +1220,9 @@ impl Imap {
                     emit_event!(context, Event::ImapFolderEmptied(folder.to_string()));
                 }
                 Err(err) => {
-                    error!(context, "expunge failed {}: {}", folder, err);
+                    error!(context, "expunge failed {}: {:?}", folder, err);
                 }
             }
-
-            if let Err(err) = crate::sql::execute(
-                context,
-                &context.sql,
-                "UPDATE msgs SET server_folder='',server_uid=0 WHERE server_folder=?",
-                params![folder],
-            ) {
-                warn!(
-                    context,
-                    "Failed to reset server_uid and server_folder for deleted messages: {}", err
-                );
-            }
         });
     }
 }
@@ -1237,7 +1234,7 @@ impl Imap {
 // CAVE: if possible, take care not to add a name here that is "sent" in one language
 // but sth. different in others - a hard job.
 fn get_folder_meaning_by_name(folder_name: &Name) -> FolderMeaning {
-    let sent_names = vec!["sent", "sentmail", "sent objects", "gesendet"];
+    let sent_names = vec!["sent", "sent objects", "gesendet"];
     let lower = folder_name.name().to_lowercase();
 
     if sent_names.into_iter().any(|s| s == lower) {
@@ -1248,87 +1245,67 @@ fn get_folder_meaning_by_name(folder_name: &Name) -> FolderMeaning {
 }
 
 fn get_folder_meaning(folder_name: &Name) -> FolderMeaning {
+    if folder_name.attributes().is_empty() {
+        return FolderMeaning::Unknown;
+    }
+
+    let mut res = FolderMeaning::Unknown;
     let special_names = vec!["\\Spam", "\\Trash", "\\Drafts", "\\Junk"];
 
     for attr in folder_name.attributes() {
         if let NameAttribute::Custom(ref label) = attr {
             if special_names.iter().any(|s| *s == label) {
-                return FolderMeaning::Other;
+                res = FolderMeaning::Other;
             } else if label == "\\Sent" {
-                return FolderMeaning::SentObjects;
+                res = FolderMeaning::SentObjects
             }
         }
     }
-    FolderMeaning::Unknown
+
+    match res {
+        FolderMeaning::Unknown => get_folder_meaning_by_name(folder_name),
+        _ => res,
+    }
 }
 
-fn precheck_imf(
-    context: &Context,
-    rfc724_mid: &str,
-    server_folder: &str,
-    server_uid: u32,
-) -> Result<bool> {
-    if let Some((old_server_folder, old_server_uid, msg_id)) =
-        message::rfc724_mid_exists(context, &rfc724_mid)?
+fn precheck_imf(context: &Context, rfc724_mid: &str, server_folder: &str, server_uid: u32) -> bool {
+    if let Ok((old_server_folder, old_server_uid, msg_id)) =
+        message::rfc724_mid_exists(context, &rfc724_mid)
     {
         if old_server_folder.is_empty() && old_server_uid == 0 {
-            info!(
+            info!(context, "[move] detected bbc-self {}", rfc724_mid,);
+            context.do_heuristics_moves(server_folder.as_ref(), msg_id);
+            job_add(
                 context,
-                "[move] detected bcc-self {} as {}/{}", rfc724_mid, server_folder, server_uid
+                Action::MarkseenMsgOnImap,
+                msg_id.to_u32() as i32,
+                Params::new(),
+                0,
             );
-
-            let delete_server_after = context.get_config_delete_server_after();
-
-            if delete_server_after != Some(0) {
-                context.do_heuristics_moves(server_folder.as_ref(), msg_id);
-                job_add(
-                    context,
-                    Action::MarkseenMsgOnImap,
-                    msg_id.to_u32() as i32,
-                    Params::new(),
-                    0,
-                );
-            }
         } else if old_server_folder != server_folder {
-            info!(
-                context,
-                "[move] detected message {} moved by other device from {}/{} to {}/{}",
-                rfc724_mid,
-                old_server_folder,
-                old_server_uid,
-                server_folder,
-                server_uid
-            );
-        } else if old_server_uid == 0 {
-            info!(
-                context,
-                "[move] detected message {} moved by us from {}/{} to {}/{}",
-                rfc724_mid,
-                old_server_folder,
-                old_server_uid,
-                server_folder,
-                server_uid
-            );
-        } else if old_server_uid != server_uid {
-            warn!(
-                context,
-                "UID for message {} in folder {} changed from {} to {}",
-                rfc724_mid,
-                server_folder,
-                old_server_uid,
-                server_uid
-            );
+            info!(context, "[move] detected moved message {}", rfc724_mid,);
         }
 
         if old_server_folder != server_folder || old_server_uid != server_uid {
             update_server_uid(context, &rfc724_mid, server_folder, server_uid);
         }
-        Ok(true)
+        true
     } else {
-        Ok(false)
+        false
     }
 }
 
+fn parse_message_id(value: &str) -> crate::error::Result<String> {
+    let addrs = mailparse::addrparse(value)
+        .map_err(|err| format_err!("failed to parse message id {:?}", err))?;
+
+    if let Some(info) = addrs.extract_single_info() {
+        return Ok(info.addr);
+    }
+
+    bail!("could not parse message_id: {}", value);
+}
+
 fn get_fetch_headers(prefetch_msg: &Fetch) -> Result<Vec<mailparse::MailHeader>> {
     let header_bytes = match prefetch_msg.header() {
         Some(header_bytes) => header_bytes,
@@ -1339,27 +1316,30 @@ fn get_fetch_headers(prefetch_msg: &Fetch) -> Result<Vec<mailparse::MailHeader>>
 }
 
 fn prefetch_get_message_id(headers: &[mailparse::MailHeader]) -> Result<String> {
-    if let Some(message_id) = headers.get_header_value(HeaderDef::MessageId) {
-        Ok(crate::mimeparser::parse_message_id(&message_id)?)
+    if let Some(message_id) = headers.get_header_value(HeaderDef::MessageId)? {
+        Ok(parse_message_id(&message_id)?)
     } else {
         Err(Error::Other("prefetch: No message ID found".to_string()))
     }
 }
 
-fn prefetch_is_reply_to_chat_message(context: &Context, headers: &[mailparse::MailHeader]) -> bool {
-    if let Some(value) = headers.get_header_value(HeaderDef::InReplyTo) {
+fn prefetch_is_reply_to_chat_message(
+    context: &Context,
+    headers: &[mailparse::MailHeader],
+) -> Result<bool> {
+    if let Some(value) = headers.get_header_value(HeaderDef::InReplyTo)? {
         if is_msgrmsg_rfc724_mid_in_list(context, &value) {
-            return true;
+            return Ok(true);
         }
     }
 
-    if let Some(value) = headers.get_header_value(HeaderDef::References) {
+    if let Some(value) = headers.get_header_value(HeaderDef::References)? {
         if is_msgrmsg_rfc724_mid_in_list(context, &value) {
-            return true;
+            return Ok(true);
         }
     }
 
-    false
+    Ok(false)
 }
 
 fn prefetch_should_download(
@@ -1367,16 +1347,19 @@ fn prefetch_should_download(
     headers: &[mailparse::MailHeader],
     show_emails: ShowEmails,
 ) -> Result<bool> {
-    let is_chat_message = headers.get_header_value(HeaderDef::ChatVersion).is_some();
-    let is_reply_to_chat_message = prefetch_is_reply_to_chat_message(context, &headers);
+    let is_chat_message = headers.get_header_value(HeaderDef::ChatVersion)?.is_some();
+    let is_reply_to_chat_message = prefetch_is_reply_to_chat_message(context, &headers)?;
 
     // Autocrypt Setup Message should be shown even if it is from non-chat client.
     let is_autocrypt_setup_message = headers
-        .get_header_value(HeaderDef::AutocryptSetupMessage)
+        .get_header_value(HeaderDef::AutocryptSetupMessage)?
         .is_some();
 
-    let (_contact_id, blocked_contact, origin) =
-        from_field_to_contact_id(context, &mimeparser::get_from(headers))?;
+    let from_field = headers
+        .get_header_value(HeaderDef::From_)?
+        .unwrap_or_default();
+
+    let (_contact_id, blocked_contact, origin) = from_field_to_contact_id(context, &from_field)?;
     let accepted_contact = origin.is_known();
 
     let show = is_autocrypt_setup_message
@@ -1390,3 +1373,20 @@ fn prefetch_should_download(
     let show = show && !blocked_contact;
     Ok(show)
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_parse_message_id() {
+        assert_eq!(
+            parse_message_id("Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org").unwrap(),
+            "Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org"
+        );
+        assert_eq!(
+            parse_message_id("<Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org>").unwrap(),
+            "Mr.PRUe8HJBoaO.3whNvLCMFU0@testrun.org"
+        );
+    }
+}

src/imap/select_folder.rs

@@ -4,53 +4,25 @@ use crate::context::Context;
 
 type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug, thiserror::Error)]
+#[derive(Debug, Fail)]
 pub enum Error {
-    #[error("IMAP Could not obtain imap-session object.")]
+    #[fail(display = "IMAP Could not obtain imap-session object.")]
     NoSession,
 
-    #[error("IMAP Connection Lost or no connection established")]
+    #[fail(display = "IMAP Connection Lost or no connection established")]
     ConnectionLost,
 
-    #[error("IMAP Folder name invalid: {0}")]
+    #[fail(display = "IMAP Folder name invalid: {:?}", _0)]
     BadFolderName(String),
 
-    #[error("IMAP close/expunge failed")]
-    CloseExpungeFailed(#[from] async_imap::error::Error),
+    #[fail(display = "IMAP close/expunge failed: {}", _0)]
+    CloseExpungeFailed(#[cause] async_imap::error::Error),
 
-    #[error("IMAP other error: {0}")]
+    #[fail(display = "IMAP other error: {:?}", _0)]
     Other(String),
 }
 
 impl Imap {
-    /// Issues a CLOSE command to expunge selected folder.
-    ///
-    /// CLOSE is considerably faster than an EXPUNGE, see
-    /// https://tools.ietf.org/html/rfc3501#section-6.4.2
-    async fn close_folder(&self, context: &Context) -> Result<()> {
-        if let Some(ref folder) = self.config.read().await.selected_folder {
-            info!(context, "Expunge messages in \"{}\".", folder);
-
-            if let Some(ref mut session) = &mut *self.session.lock().await {
-                match session.close().await {
-                    Ok(_) => {
-                        info!(context, "close/expunge succeeded");
-                    }
-                    Err(err) => {
-                        self.trigger_reconnect();
-                        return Err(Error::CloseExpungeFailed(err));
-                    }
-                }
-            } else {
-                return Err(Error::NoSession);
-            }
-        }
-        let mut cfg = self.config.write().await;
-        cfg.selected_folder = None;
-        cfg.selected_folder_needs_expunge = false;
-        Ok(())
-    }
-
     /// select a folder, possibly update uid_validity and, if needed,
     /// expunge the folder to remove delete-marked messages.
     pub(super) async fn select_folder<S: AsRef<str>>(
@@ -66,14 +38,39 @@ impl Imap {
             return Err(Error::NoSession);
         }
 
-        let needs_expunge = self.config.read().await.selected_folder_needs_expunge;
-        if needs_expunge {
-            self.close_folder(context).await?;
+        // if there is a new folder and the new folder is equal to the selected one, there's nothing to do.
+        // if there is _no_ new folder, we continue as we might want to expunge below.
+        if let Some(ref folder) = folder {
+            if let Some(ref selected_folder) = self.config.read().await.selected_folder {
+                if folder.as_ref() == selected_folder {
+                    return Ok(());
+                }
+            }
         }
 
-        let folder_str: Option<&str> = folder.as_ref().map(|x| x.as_ref());
-        if self.config.read().await.selected_folder.as_deref() == folder_str {
-            return Ok(());
+        // deselect existing folder, if needed (it's also done implicitly by SELECT, however, without EXPUNGE then)
+        let needs_expunge = { self.config.read().await.selected_folder_needs_expunge };
+        if needs_expunge {
+            if let Some(ref folder) = self.config.read().await.selected_folder {
+                info!(context, "Expunge messages in \"{}\".", folder);
+
+                // A CLOSE-SELECT is considerably faster than an EXPUNGE-SELECT, see
+                // https://tools.ietf.org/html/rfc3501#section-6.4.2
+                if let Some(ref mut session) = &mut *self.session.lock().await {
+                    match session.close().await {
+                        Ok(_) => {
+                            info!(context, "close/expunge succeeded");
+                        }
+                        Err(err) => {
+                            self.trigger_reconnect();
+                            return Err(Error::CloseExpungeFailed(err));
+                        }
+                    }
+                } else {
+                    return Err(Error::NoSession);
+                }
+            }
+            self.config.write().await.selected_folder_needs_expunge = false;
         }
 
         // select new folder

src/imap_client.rs

@@ -1,18 +1,122 @@
 use async_imap::{
-    error::Result as ImapResult,
+    error::{Error as ImapError, Result as ImapResult},
+    extensions::idle::Handle as ImapIdleHandle,
     types::{Capabilities, Fetch, Mailbox, Name},
-    Session as ImapSession,
+    Client as ImapClient, Session as ImapSession,
 };
 use async_native_tls::TlsStream;
-use async_std::net::TcpStream;
+use async_std::net::{self, TcpStream};
 use async_std::prelude::*;
 
+use crate::login_param::{dc_build_tls, CertificateChecks};
+
+#[derive(Debug)]
+pub(crate) enum Client {
+    Secure(ImapClient<TlsStream<TcpStream>>),
+    Insecure(ImapClient<TcpStream>),
+}
+
 #[derive(Debug)]
 pub(crate) enum Session {
     Secure(ImapSession<TlsStream<TcpStream>>),
     Insecure(ImapSession<TcpStream>),
 }
 
+#[derive(Debug)]
+pub(crate) enum IdleHandle {
+    Secure(ImapIdleHandle<TlsStream<TcpStream>>),
+    Insecure(ImapIdleHandle<TcpStream>),
+}
+
+impl Client {
+    pub async fn connect_secure<A: net::ToSocketAddrs, S: AsRef<str>>(
+        addr: A,
+        domain: S,
+        certificate_checks: CertificateChecks,
+    ) -> ImapResult<Self> {
+        let stream = TcpStream::connect(addr).await?;
+        let tls = dc_build_tls(certificate_checks);
+        let tls_stream = tls.connect(domain.as_ref(), stream).await?;
+        let mut client = ImapClient::new(tls_stream);
+        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
+            client.debug = true;
+        }
+
+        let _greeting = client
+            .read_response()
+            .await
+            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
+
+        Ok(Client::Secure(client))
+    }
+
+    pub async fn connect_insecure<A: net::ToSocketAddrs>(addr: A) -> ImapResult<Self> {
+        let stream = TcpStream::connect(addr).await?;
+
+        let mut client = ImapClient::new(stream);
+        if std::env::var(crate::DCC_IMAP_DEBUG).is_ok() {
+            client.debug = true;
+        }
+        let _greeting = client
+            .read_response()
+            .await
+            .ok_or_else(|| ImapError::Bad("failed to read greeting".to_string()))?;
+
+        Ok(Client::Insecure(client))
+    }
+
+    pub async fn secure<S: AsRef<str>>(
+        self,
+        domain: S,
+        certificate_checks: CertificateChecks,
+    ) -> ImapResult<Client> {
+        match self {
+            Client::Insecure(client) => {
+                let tls = dc_build_tls(certificate_checks);
+                let client_sec = client.secure(domain, tls).await?;
+
+                Ok(Client::Secure(client_sec))
+            }
+            // Nothing to do
+            Client::Secure(_) => Ok(self),
+        }
+    }
+
+    pub async fn authenticate<A: async_imap::Authenticator, S: AsRef<str>>(
+        self,
+        auth_type: S,
+        authenticator: &A,
+    ) -> Result<Session, (ImapError, Client)> {
+        match self {
+            Client::Secure(i) => match i.authenticate(auth_type, authenticator).await {
+                Ok(session) => Ok(Session::Secure(session)),
+                Err((err, c)) => Err((err, Client::Secure(c))),
+            },
+            Client::Insecure(i) => match i.authenticate(auth_type, authenticator).await {
+                Ok(session) => Ok(Session::Insecure(session)),
+                Err((err, c)) => Err((err, Client::Insecure(c))),
+            },
+        }
+    }
+
+    pub async fn login<U: AsRef<str>, P: AsRef<str>>(
+        self,
+        username: U,
+        password: P,
+    ) -> Result<Session, (ImapError, Client)> {
+        match self {
+            Client::Secure(i) => match i.login(username, password).await {
+                Ok(session) => Ok(Session::Secure(session)),
+                Err((err, c)) => Err((err, Client::Secure(c))),
+            },
+            Client::Insecure(i) => match i.login(username, password).await {
+                Ok(session) => Ok(Session::Insecure(session)),
+                Err((err, c)) => Err((err, Client::Insecure(c))),
+            },
+        }
+    }
+}
+
 impl Session {
     pub async fn capabilities(&mut self) -> ImapResult<Capabilities> {
         let res = match self {
@@ -123,6 +227,19 @@ impl Session {
         Ok(res)
     }
 
+    pub fn idle(self) -> IdleHandle {
+        match self {
+            Session::Secure(i) => {
+                let h = i.idle();
+                IdleHandle::Secure(h)
+            }
+            Session::Insecure(i) => {
+                let h = i.idle();
+                IdleHandle::Insecure(h)
+            }
+        }
+    }
+
     pub async fn uid_store<S1, S2>(&mut self, uid_set: S1, query: S2) -> ImapResult<Vec<Fetch>>
     where
         S1: AsRef<str>,

src/imex.rs

@@ -10,6 +10,7 @@ use crate::blob::BlobObject;
 use crate::chat;
 use crate::chat::delete_and_reset_all_device_msgs;
 use crate::config::Config;
+use crate::configure::*;
 use crate::constants::*;
 use crate::context::Context;
 use crate::dc_tools::*;
@@ -17,7 +18,7 @@ use crate::e2ee;
 use crate::error::*;
 use crate::events::Event;
 use crate::job::*;
-use crate::key::{self, DcKey, Key, SignedSecretKey};
+use crate::key::{self, Key};
 use crate::message::{Message, MsgId};
 use crate::mimeparser::SystemMessage;
 use crate::param::*;
@@ -175,7 +176,9 @@ pub fn render_setup_file(context: &Context, passphrase: &str) -> Result<String>
         passphrase.len() >= 2,
         "Passphrase must be at least 2 chars long."
     );
-    let private_key = Key::from(SignedSecretKey::load_self(context)?);
+    let self_addr = e2ee::ensure_secret_key_exists(context)?;
+    let private_key = Key::from_self_private(context, self_addr, &context.sql)
+        .ok_or_else(|| format_err!("Failed to get private key."))?;
     let ac_headers = match context.get_config_bool(Config::E2eeEnabled) {
         false => None,
         true => Some(("Autocrypt-Prefer-Encrypt", "mutual")),
@@ -411,7 +414,7 @@ fn import_backup(context: &Context, backup_to_import: impl AsRef<Path>) -> Resul
     );
 
     ensure!(
-        !context.is_configured(),
+        !dc_is_configured(context),
         "Cannot import backups to accounts in use."
     );
     context.sql.close(&context);
@@ -754,6 +757,7 @@ mod tests {
         assert!(msg.contains("-----BEGIN PGP MESSAGE-----\r\n"));
         assert!(msg.contains("Passphrase-Format: numeric9x4\r\n"));
         assert!(msg.contains("Passphrase-Begin: he\n"));
+        assert!(msg.contains("==\n"));
         assert!(msg.contains("-----END PGP MESSAGE-----\n"));
     }
 

src/job.rs

@@ -9,9 +9,6 @@ use deltachat_derive::{FromSql, ToSql};
 use itertools::Itertools;
 use rand::{thread_rng, Rng};
 
-use async_smtp::smtp::response::Category;
-use async_smtp::smtp::response::Code;
-use async_smtp::smtp::response::Detail;
 use async_std::task;
 
 use crate::blob::BlobObject;
@@ -22,7 +19,7 @@ use crate::constants::*;
 use crate::contact::Contact;
 use crate::context::{Context, PerformJobsNeeded};
 use crate::dc_tools::*;
-use crate::error::{bail, ensure, format_err, Error, Result};
+use crate::error::{Error, Result};
 use crate::events::Event;
 use crate::imap::*;
 use crate::imex::*;
@@ -75,19 +72,7 @@ impl Default for Thread {
     }
 }
 
-#[derive(
-    Debug,
-    Display,
-    Copy,
-    Clone,
-    PartialEq,
-    Eq,
-    PartialOrd,
-    FromPrimitive,
-    ToPrimitive,
-    FromSql,
-    ToSql,
-)]
+#[derive(Debug, Display, Copy, Clone, PartialEq, Eq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
 #[repr(i32)]
 pub enum Action {
     Unknown = 0,
@@ -95,14 +80,10 @@ pub enum Action {
     // Jobs in the INBOX-thread, range from DC_IMAP_THREAD..DC_IMAP_THREAD+999
     Housekeeping = 105, // low priority ...
     EmptyServer = 107,
-    OldDeleteMsgOnImap = 110,
+    DeleteMsgOnImap = 110,
+    MarkseenMdnOnImap = 120,
     MarkseenMsgOnImap = 130,
-
-    // Moving message is prioritized lower than deletion so we don't
-    // bother moving message if it is already scheduled for deletion.
     MoveMsg = 200,
-
-    DeleteMsgOnImap = 210,
     ConfigureImap = 900,
     ImexImap = 910, // ... high priority
 
@@ -127,9 +108,9 @@ impl From<Action> for Thread {
             Unknown => Thread::Unknown,
 
             Housekeeping => Thread::Imap,
-            OldDeleteMsgOnImap => Thread::Imap,
             DeleteMsgOnImap => Thread::Imap,
             EmptyServer => Thread::Imap,
+            MarkseenMdnOnImap => Thread::Imap,
             MarkseenMsgOnImap => Thread::Imap,
             MoveMsg => Thread::Imap,
             ConfigureImap => Thread::Imap,
@@ -162,68 +143,30 @@ impl fmt::Display for Job {
 }
 
 impl Job {
-    fn new(action: Action, foreign_id: u32, param: Params, delay_seconds: i64) -> Self {
-        let timestamp = time();
-
-        Self {
-            job_id: 0,
-            action,
-            foreign_id,
-            desired_timestamp: timestamp + delay_seconds,
-            added_timestamp: timestamp,
-            tries: 0,
-            param,
-            pending_error: None,
-        }
-    }
-
     /// Deletes the job from the database.
     fn delete(&self, context: &Context) -> bool {
-        if self.job_id != 0 {
-            context
-                .sql
-                .execute("DELETE FROM jobs WHERE id=?;", params![self.job_id as i32])
-                .is_ok()
-        } else {
-            // Already deleted.
-            true
-        }
+        context
+            .sql
+            .execute("DELETE FROM jobs WHERE id=?;", params![self.job_id as i32])
+            .is_ok()
     }
 
-    /// Saves the job to the database, creating a new entry if necessary.
+    /// Updates the job already stored in the database.
     ///
-    /// The Job is consumed by this method.
-    fn save(self, context: &Context) -> bool {
-        let thread: Thread = self.action.into();
-
-        if self.job_id != 0 {
-            sql::execute(
-                context,
-                &context.sql,
-                "UPDATE jobs SET desired_timestamp=?, tries=?, param=? WHERE id=?;",
-                params![
-                    self.desired_timestamp,
-                    self.tries as i64,
-                    self.param.to_string(),
-                    self.job_id as i32,
-                ],
-            )
-            .is_ok()
-        } else {
-            sql::execute(
-                context,
-                &context.sql,
-                "INSERT INTO jobs (added_timestamp, thread, action, foreign_id, param, desired_timestamp) VALUES (?,?,?,?,?,?);",
-                params![
-                    self.added_timestamp,
-                    thread,
-                    self.action,
-                    self.foreign_id,
-                    self.param.to_string(),
-                    self.desired_timestamp
-                ]
-            ).is_ok()
-        }
+    /// To add a new job, use [job_add].
+    fn update(&self, context: &Context) -> bool {
+        sql::execute(
+            context,
+            &context.sql,
+            "UPDATE jobs SET desired_timestamp=?, tries=?, param=? WHERE id=?;",
+            params![
+                self.desired_timestamp,
+                self.tries as i64,
+                self.param.to_string(),
+                self.job_id as i32,
+            ],
+        )
+        .is_ok()
     }
 
     fn smtp_send<F>(
@@ -250,66 +193,15 @@ impl Job {
             Err(crate::smtp::send::Error::SendError(err)) => {
                 // Remote error, retry later.
                 warn!(context, "SMTP failed to send: {}", err);
-                self.pending_error = Some(err.to_string());
-
-                let res = match err {
-                    async_smtp::smtp::error::Error::Permanent(ref response) => {
-                        match response.code {
-                            // Sometimes servers send a permanent error when actually it is a temporary error
-                            // For documentation see https://tools.ietf.org/html/rfc3463
-
-                            // Code 5.5.0, see https://support.delta.chat/t/every-other-message-gets-stuck/877/2
-                            Code {
-                                category: Category::MailSystem,
-                                detail: Detail::Zero,
-                                ..
-                            } => Status::RetryLater,
-
-                            _ => {
-                                // If we do not retry, add an info message to the chat
-                                // Error 5.7.1 should definitely go here: Yandex sends 5.7.1 with a link when it thinks that the email is SPAM.
-                                match Message::load_from_db(context, MsgId::new(self.foreign_id)) {
-                                    Ok(message) => chat::add_info_msg(
-                                        context,
-                                        message.chat_id,
-                                        err.to_string(),
-                                    ),
-                                    Err(e) => warn!(
-                                        context,
-                                        "couldn't load chat_id to inform user about SMTP error: {}",
-                                        e
-                                    ),
-                                };
-
-                                Status::Finished(Err(format_err!("Permanent SMTP error: {}", err)))
-                            }
-                        }
-                    }
-                    async_smtp::smtp::error::Error::Transient(_) => {
-                        // We got a transient 4xx response from SMTP server.
-                        // Give some time until the server-side error maybe goes away.
-                        Status::RetryLater
-                    }
-                    _ => {
-                        if smtp.has_maybe_stale_connection() {
-                            info!(context, "stale connection? immediately reconnecting");
-                            Status::RetryNow
-                        } else {
-                            Status::RetryLater
-                        }
-                    }
-                };
-
-                // this clears last_success info
                 smtp.disconnect();
-
-                res
+                self.pending_error = Some(err.to_string());
+                Status::RetryLater
             }
             Err(crate::smtp::send::Error::EnvelopeError(err)) => {
                 // Local error, job is invalid, do not retry.
                 smtp.disconnect();
                 warn!(context, "SMTP job is invalid: {}", err);
-                Status::Finished(Err(err.into()))
+                Status::Finished(Err(Error::SmtpError(err)))
             }
             Err(crate::smtp::send::Error::NoTransport) => {
                 // Should never happen.
@@ -499,12 +391,18 @@ impl Job {
 
         if let Some(dest_folder) = dest_folder {
             let server_folder = msg.server_folder.as_ref().unwrap();
+            let mut dest_uid = 0;
 
-            match imap_inbox.mv(context, server_folder, msg.server_uid, &dest_folder) {
+            match imap_inbox.mv(
+                context,
+                server_folder,
+                msg.server_uid,
+                &dest_folder,
+                &mut dest_uid,
+            ) {
                 ImapActionResult::RetryLater => Status::RetryLater,
                 ImapActionResult::Success => {
-                    // XXX Rust-Imap provides no target uid on mv, so just set it to 0
-                    message::update_server_uid(context, &msg.rfc724_mid, &dest_folder, 0);
+                    message::update_server_uid(context, &msg.rfc724_mid, &dest_folder, dest_uid);
                     Status::Finished(Ok(()))
                 }
                 ImapActionResult::Failed => {
@@ -517,29 +415,14 @@ impl Job {
         }
     }
 
-    /// Deletes a message on the server.
-    ///
-    /// foreign_id is a MsgId pointing to a message in the trash chat
-    /// or a hidden message.
-    ///
-    /// This job removes the database record. If there are no more
-    /// records pointing to the same message on the server, the job
-    /// also removes the message on the server.
     #[allow(non_snake_case)]
     fn DeleteMsgOnImap(&mut self, context: &Context) -> Status {
         let imap_inbox = &context.inbox_thread.read().unwrap().imap;
 
-        let msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)));
+        let mut msg = job_try!(Message::load_from_db(context, MsgId::new(self.foreign_id)));
 
         if !msg.rfc724_mid.is_empty() {
-            let cnt = message::rfc724_mid_cnt(context, &msg.rfc724_mid);
-            info!(
-                context,
-                "Running delete job for message {} which has {} entries in the database",
-                &msg.rfc724_mid,
-                cnt
-            );
-            if cnt > 1 {
+            if message::rfc724_mid_cnt(context, &msg.rfc724_mid) > 1 {
                 info!(
                     context,
                     "The message is deleted from the server when all parts are deleted.",
@@ -549,47 +432,13 @@ impl Job {
                 we delete the message from the server */
                 let mid = msg.rfc724_mid;
                 let server_folder = msg.server_folder.as_ref().unwrap();
-                let res = if msg.server_uid == 0 {
-                    // Message is already deleted on IMAP server.
-                    ImapActionResult::AlreadyDone
-                } else {
-                    imap_inbox.delete_msg(context, &mid, server_folder, msg.server_uid)
-                };
-                match res {
-                    ImapActionResult::AlreadyDone | ImapActionResult::Success => {}
-                    ImapActionResult::RetryLater | ImapActionResult::Failed => {
-                        // If job has failed, for example due to some
-                        // IMAP bug, we postpone it instead of failing
-                        // immediately. This will prevent adding it
-                        // immediately again if user has enabled
-                        // automatic message deletion. Without this,
-                        // we might waste a lot of traffic constantly
-                        // retrying message deletion.
-                        return Status::RetryLater;
-                    }
+                let res = imap_inbox.delete_msg(context, &mid, server_folder, &mut msg.server_uid);
+                if res == ImapActionResult::RetryLater {
+                    // XXX RetryLater is converted to RetryNow here
+                    return Status::RetryNow;
                 }
             }
-            if msg.chat_id.is_trash() || msg.hidden {
-                // Messages are stored in trash chat only to keep
-                // their server UID and Message-ID. Once message is
-                // deleted from the server, database record can be
-                // removed as well.
-                //
-                // Hidden messages are similar to trashed, but are
-                // related to some chat. We also delete their
-                // database records.
-                job_try!(msg.id.delete_from_db(context))
-            } else {
-                // Remove server UID from the database record.
-                //
-                // We have either just removed the message from the
-                // server, in which case UID is not valid anymore, or
-                // we have more refernces to the same server UID, so
-                // we remove UID to reduce the number of messages
-                // pointing to the corresponding UID. Once the counter
-                // reaches zero, we will remove the message.
-                job_try!(msg.id.unlink(context));
-            }
+            Message::delete_from_db(context, msg.id);
             Status::Finished(Ok(()))
         } else {
             /* eg. device messages have no Message-ID */
@@ -641,6 +490,43 @@ impl Job {
             }
         }
     }
+
+    #[allow(non_snake_case)]
+    fn MarkseenMdnOnImap(&mut self, context: &Context) -> Status {
+        let folder = self
+            .param
+            .get(Param::ServerFolder)
+            .unwrap_or_default()
+            .to_string();
+        let uid = self.param.get_int(Param::ServerUid).unwrap_or_default() as u32;
+        let imap_inbox = &context.inbox_thread.read().unwrap().imap;
+        if imap_inbox.set_seen(context, &folder, uid) == ImapActionResult::RetryLater {
+            return Status::RetryLater;
+        }
+        if self.param.get_bool(Param::AlsoMove).unwrap_or_default() {
+            if let Err(err) = imap_inbox.ensure_configured_folders(context, true) {
+                warn!(context, "configuring folders failed: {:?}", err);
+                return Status::RetryLater;
+            }
+            let dest_folder = context
+                .sql
+                .get_raw_config(context, "configured_mvbox_folder");
+            if let Some(dest_folder) = dest_folder {
+                let mut dest_uid = 0;
+                if ImapActionResult::RetryLater
+                    == imap_inbox.mv(context, &folder, uid, &dest_folder, &mut dest_uid)
+                {
+                    Status::RetryLater
+                } else {
+                    Status::Finished(Ok(()))
+                }
+            } else {
+                Status::Finished(Err(format_err!("MVBOX is not configured")))
+            }
+        } else {
+            Status::Finished(Ok(()))
+        }
+    }
 }
 
 /* delete all pending jobs with the given action */
@@ -913,11 +799,7 @@ pub fn job_send_msg(context: &Context, msg_id: MsgId) -> Result<()> {
         .get_config(Config::ConfiguredAddr)
         .unwrap_or_default();
     let lowercase_from = from.to_lowercase();
-
-    // Send BCC to self if it is enabled and we are not going to
-    // delete it immediately.
     if context.get_config_bool(Config::BccSelf)
-        && context.get_config_delete_server_after() != Some(0)
         && !recipients
             .iter()
             .any(|x| x.to_lowercase() == lowercase_from)
@@ -993,36 +875,6 @@ pub fn job_send_msg(context: &Context, msg_id: MsgId) -> Result<()> {
     Ok(())
 }
 
-fn load_imap_deletion_msgid(context: &Context) -> sql::Result<Option<MsgId>> {
-    if let Some(delete_server_after) = context.get_config_delete_server_after() {
-        let threshold_timestamp = time() - delete_server_after;
-
-        context.sql.query_row_optional(
-            "SELECT id FROM msgs \
-             WHERE timestamp < ? \
-             AND server_uid != 0",
-            params![threshold_timestamp],
-            |row| row.get::<_, MsgId>(0),
-        )
-    } else {
-        Ok(None)
-    }
-}
-
-fn load_imap_deletion_job(context: &Context) -> sql::Result<Option<Job>> {
-    let res = if let Some(msg_id) = load_imap_deletion_msgid(context)? {
-        Some(Job::new(
-            Action::DeleteMsgOnImap,
-            msg_id.to_u32(),
-            Params::new(),
-            0,
-        ))
-    } else {
-        None
-    };
-    Ok(res)
-}
-
 pub fn perform_inbox_jobs(context: &Context) {
     info!(context, "dc_perform_inbox_jobs starting.",);
 
@@ -1043,17 +895,7 @@ pub fn perform_sentbox_jobs(context: &Context) {
 }
 
 fn job_perform(context: &Context, thread: Thread, probe_network: bool) {
-    let mut jobs_loaded = 0;
-
     while let Some(mut job) = load_next_job(context, thread, probe_network) {
-        jobs_loaded += 1;
-        if thread == Thread::Imap && jobs_loaded > 20 {
-            // Let the fetch run, but return back to the job afterwards.
-            info!(context, "postponing {}-job {} to run fetch...", thread, job);
-            *context.perform_inbox_jobs_needed.write().unwrap() = true;
-            break;
-        }
-
         info!(context, "{}-job {} started...", thread, job);
 
         // some configuration jobs are "exclusive":
@@ -1077,10 +919,52 @@ fn job_perform(context: &Context, thread: Thread, probe_network: bool) {
             suspend_smtp_thread(context, true);
         }
 
-        let try_res = match perform_job_action(context, &mut job, thread, 0) {
-            Status::RetryNow => perform_job_action(context, &mut job, thread, 1),
-            x => x,
-        };
+        let try_res = (0..2)
+            .map(|tries| {
+                info!(
+                    context,
+                    "{} performs immediate try {} of job {}", thread, tries, job
+                );
+
+                let try_res = match job.action {
+                    Action::Unknown => Status::Finished(Err(format_err!("Unknown job id found"))),
+                    Action::SendMsgToSmtp => job.SendMsgToSmtp(context),
+                    Action::EmptyServer => job.EmptyServer(context),
+                    Action::DeleteMsgOnImap => job.DeleteMsgOnImap(context),
+                    Action::MarkseenMsgOnImap => job.MarkseenMsgOnImap(context),
+                    Action::MarkseenMdnOnImap => job.MarkseenMdnOnImap(context),
+                    Action::MoveMsg => job.MoveMsg(context),
+                    Action::SendMdn => job.SendMdn(context),
+                    Action::ConfigureImap => JobConfigureImap(context),
+                    Action::ImexImap => match JobImexImap(context, &job) {
+                        Ok(()) => Status::Finished(Ok(())),
+                        Err(err) => {
+                            error!(context, "{}", err);
+                            Status::Finished(Err(err))
+                        }
+                    },
+                    Action::MaybeSendLocations => location::JobMaybeSendLocations(context, &job),
+                    Action::MaybeSendLocationsEnded => {
+                        location::JobMaybeSendLocationsEnded(context, &mut job)
+                    }
+                    Action::Housekeeping => {
+                        sql::housekeeping(context);
+                        Status::Finished(Ok(()))
+                    }
+                };
+
+                info!(
+                    context,
+                    "{} finished immediate try {} of job {}", thread, tries, job
+                );
+
+                try_res
+            })
+            .find(|try_res| match try_res {
+                Status::RetryNow => false,
+                _ => true,
+            })
+            .unwrap_or(Status::RetryNow);
 
         if Action::ConfigureImap == job.action || Action::ImexImap == job.action {
             context
@@ -1111,6 +995,7 @@ fn job_perform(context: &Context, thread: Thread, probe_network: bool) {
                     job.tries = tries;
                     let time_offset = get_backoff_time_offset(tries);
                     job.desired_timestamp = time() + time_offset;
+                    job.update(context);
                     info!(
                         context,
                         "{}-job #{} not succeeded on try #{}, retry in {} seconds.",
@@ -1119,7 +1004,6 @@ fn job_perform(context: &Context, thread: Thread, probe_network: bool) {
                         tries,
                         time_offset
                     );
-                    job.save(context);
                     if thread == Thread::Smtp && tries < JOB_RETRIES - 1 {
                         context
                             .smtp_state
@@ -1171,45 +1055,6 @@ fn job_perform(context: &Context, thread: Thread, probe_network: bool) {
     }
 }
 
-fn perform_job_action(context: &Context, mut job: &mut Job, thread: Thread, tries: u32) -> Status {
-    info!(
-        context,
-        "{} begin immediate try {} of job {}", thread, tries, job
-    );
-
-    let try_res = match job.action {
-        Action::Unknown => Status::Finished(Err(format_err!("Unknown job id found"))),
-        Action::SendMsgToSmtp => job.SendMsgToSmtp(context),
-        Action::EmptyServer => job.EmptyServer(context),
-        Action::OldDeleteMsgOnImap => job.DeleteMsgOnImap(context),
-        Action::DeleteMsgOnImap => job.DeleteMsgOnImap(context),
-        Action::MarkseenMsgOnImap => job.MarkseenMsgOnImap(context),
-        Action::MoveMsg => job.MoveMsg(context),
-        Action::SendMdn => job.SendMdn(context),
-        Action::ConfigureImap => JobConfigureImap(context),
-        Action::ImexImap => match JobImexImap(context, &job) {
-            Ok(()) => Status::Finished(Ok(())),
-            Err(err) => {
-                error!(context, "Import/export failed: {}", err);
-                Status::Finished(Err(err))
-            }
-        },
-        Action::MaybeSendLocations => location::JobMaybeSendLocations(context, &job),
-        Action::MaybeSendLocationsEnded => location::JobMaybeSendLocationsEnded(context, &mut job),
-        Action::Housekeeping => {
-            sql::housekeeping(context);
-            Status::Finished(Ok(()))
-        }
-    };
-
-    info!(
-        context,
-        "{} finished immediate try {} of job {}", thread, tries, job
-    );
-
-    try_res
-}
-
 fn get_backoff_time_offset(tries: u32) -> i64 {
     let n = 2_i32.pow(tries - 1) * 60;
     let mut rng = thread_rng();
@@ -1277,16 +1122,27 @@ pub fn job_add(
         return;
     }
 
-    let job = Job::new(action, foreign_id as u32, param, delay_seconds);
-    job.save(context);
+    let timestamp = time();
+    let thread: Thread = action.into();
 
-    if delay_seconds == 0 {
-        let thread: Thread = action.into();
-        match thread {
-            Thread::Imap => interrupt_inbox_idle(context),
-            Thread::Smtp => interrupt_smtp_idle(context),
-            Thread::Unknown => {}
-        }
+    sql::execute(
+        context,
+        &context.sql,
+        "INSERT INTO jobs (added_timestamp, thread, action, foreign_id, param, desired_timestamp) VALUES (?,?,?,?,?,?);",
+        params![
+            timestamp,
+            thread,
+            action,
+            foreign_id,
+            param.to_string(),
+            (timestamp + delay_seconds as i64)
+        ]
+    ).ok();
+
+    match thread {
+        Thread::Imap => interrupt_inbox_idle(context),
+        Thread::Smtp => interrupt_smtp_idle(context),
+        Thread::Unknown => {}
     }
 }
 
@@ -1330,7 +1186,7 @@ fn load_next_job(context: &Context, thread: Thread, probe_network: bool) -> Opti
         params_probe
     };
 
-    let job = context
+    context
         .sql
         .query_map(
             query,
@@ -1359,23 +1215,7 @@ fn load_next_job(context: &Context, thread: Thread, probe_network: bool) -> Opti
                 Ok(None)
             },
         )
-        .unwrap_or_default();
-
-    if thread == Thread::Imap {
-        if let Some(job) = job {
-            if job.action < Action::DeleteMsgOnImap {
-                load_imap_deletion_job(context)
-                    .unwrap_or_default()
-                    .or(Some(job))
-            } else {
-                Some(job)
-            }
-        } else {
-            load_imap_deletion_job(context).unwrap_or_default()
-        }
-    } else {
-        job
-    }
+        .unwrap_or_default()
 }
 
 #[cfg(test)]

src/job_thread.rs

@@ -1,7 +1,7 @@
 use std::sync::{Arc, Condvar, Mutex};
 
 use crate::context::Context;
-use crate::error::{format_err, Result};
+use crate::error::{Error, Result};
 use crate::imap::Imap;
 
 #[derive(Debug)]
@@ -99,21 +99,25 @@ impl JobThread {
 
     async fn connect_and_fetch(&mut self, context: &Context) -> Result<()> {
         let prefix = format!("{}-fetch", self.name);
-        self.imap.connect_configured(context)?;
-        if let Some(watch_folder) = self.get_watch_folder(context) {
-            let start = std::time::Instant::now();
-            info!(context, "{} started...", prefix);
-            let res = self
-                .imap
-                .fetch(context, &watch_folder)
-                .await
-                .map_err(Into::into);
-            let elapsed = start.elapsed().as_millis();
-            info!(context, "{} done in {:.3} ms.", prefix, elapsed);
+        match self.imap.connect_configured(context) {
+            Ok(()) => {
+                if let Some(watch_folder) = self.get_watch_folder(context) {
+                    let start = std::time::Instant::now();
+                    info!(context, "{} started...", prefix);
+                    let res = self
+                        .imap
+                        .fetch(context, &watch_folder)
+                        .await
+                        .map_err(Into::into);
+                    let elapsed = start.elapsed().as_millis();
+                    info!(context, "{} done in {:.3} ms.", prefix, elapsed);
 
-            res
-        } else {
-            Err(format_err!("WatchFolder not found: not-set"))
+                    res
+                } else {
+                    Err(Error::WatchFolderNotFound("not-set".to_string()))
+                }
+            }
+            Err(err) => Err(crate::error::Error::Message(err.to_string())),
         }
     }
 

src/key.rs

@@ -4,39 +4,38 @@ use std::collections::BTreeMap;
 use std::io::Cursor;
 use std::path::Path;
 
-use num_traits::FromPrimitive;
 use pgp::composed::Deserializable;
 use pgp::ser::Serialize;
 use pgp::types::{KeyTrait, SecretKeyTrait};
 
-use crate::config::Config;
 use crate::constants::*;
 use crate::context::Context;
-use crate::dc_tools::{dc_write_file, time, EmailAddress, InvalidEmailError};
-use crate::sql;
+use crate::dc_tools::*;
+use crate::sql::Sql;
 
 // Re-export key types
 pub use crate::pgp::KeyPair;
 pub use pgp::composed::{SignedPublicKey, SignedSecretKey};
 
 /// Error type for deltachat key handling.
-#[derive(Debug, thiserror::Error)]
-#[non_exhaustive]
+#[derive(Fail, Debug)]
 pub enum Error {
-    #[error("Could not decode base64")]
-    Base64Decode(#[from] base64::DecodeError),
-    #[error("rPGP error: {}", _0)]
-    Pgp(#[from] pgp::errors::Error),
-    #[error("Failed to generate PGP key: {}", _0)]
-    Keygen(#[from] crate::pgp::PgpKeygenError),
-    #[error("Failed to load key: {}", _0)]
-    LoadKey(#[from] sql::Error),
-    #[error("Failed to save generated key: {}", _0)]
-    StoreKey(#[from] SaveKeyError),
-    #[error("No address configured")]
-    NoConfiguredAddr,
-    #[error("Configured address is invalid: {}", _0)]
-    InvalidConfiguredAddr(#[from] InvalidEmailError),
+    #[fail(display = "Could not decode base64")]
+    Base64Decode(#[cause] base64::DecodeError, failure::Backtrace),
+    #[fail(display = "rPGP error: {}", _0)]
+    PgpError(#[cause] pgp::errors::Error, failure::Backtrace),
+}
+
+impl From<base64::DecodeError> for Error {
+    fn from(err: base64::DecodeError) -> Error {
+        Error::Base64Decode(err, failure::Backtrace::new())
+    }
+}
+
+impl From<pgp::errors::Error> for Error {
+    fn from(err: pgp::errors::Error) -> Error {
+        Error::PgpError(err, failure::Backtrace::new())
+    }
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -64,9 +63,6 @@ pub trait DcKey: Serialize + Deserializable {
         Self::from_slice(&bytes)
     }
 
-    /// Load the users' default key from the database.
-    fn load_self(context: &Context) -> Result<Self::KeyType>;
-
     /// Serialise the key to a base64 string.
     fn to_base64(&self) -> String {
         // Not using Serialize::to_bytes() to make clear *why* it is
@@ -81,91 +77,10 @@ pub trait DcKey: Serialize + Deserializable {
 
 impl DcKey for SignedPublicKey {
     type KeyType = SignedPublicKey;
-
-    fn load_self(context: &Context) -> Result<Self::KeyType> {
-        match context.sql.query_row(
-            r#"
-            SELECT public_key
-              FROM keypairs
-             WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
-               AND is_default=1;
-            "#,
-            params![],
-            |row| row.get::<_, Vec<u8>>(0),
-        ) {
-            Ok(bytes) => Self::from_slice(&bytes),
-            Err(sql::Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => {
-                let keypair = generate_keypair(context)?;
-                Ok(keypair.public)
-            }
-            Err(err) => Err(err.into()),
-        }
-    }
 }
 
 impl DcKey for SignedSecretKey {
     type KeyType = SignedSecretKey;
-
-    fn load_self(context: &Context) -> Result<Self::KeyType> {
-        match context.sql.query_row(
-            r#"
-            SELECT private_key
-              FROM keypairs
-             WHERE addr=(SELECT value FROM config WHERE keyname="configured_addr")
-               AND is_default=1;
-            "#,
-            params![],
-            |row| row.get::<_, Vec<u8>>(0),
-        ) {
-            Ok(bytes) => Self::from_slice(&bytes),
-            Err(sql::Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => {
-                let keypair = generate_keypair(context)?;
-                Ok(keypair.secret)
-            }
-            Err(err) => Err(err.into()),
-        }
-    }
-}
-
-fn generate_keypair(context: &Context) -> Result<KeyPair> {
-    let addr = context
-        .get_config(Config::ConfiguredAddr)
-        .ok_or_else(|| Error::NoConfiguredAddr)?;
-    let addr = EmailAddress::new(&addr)?;
-    let _guard = context.generating_key_mutex.lock().unwrap();
-
-    // Check if the key appeared while we were waiting on the lock.
-    match context.sql.query_row(
-        r#"
-        SELECT public_key, private_key
-          FROM keypairs
-         WHERE addr=?1
-           AND is_default=1;
-        "#,
-        params![addr],
-        |row| Ok((row.get::<_, Vec<u8>>(0)?, row.get::<_, Vec<u8>>(1)?)),
-    ) {
-        Ok((pub_bytes, sec_bytes)) => Ok(KeyPair {
-            addr,
-            public: SignedPublicKey::from_slice(&pub_bytes)?,
-            secret: SignedSecretKey::from_slice(&sec_bytes)?,
-        }),
-        Err(sql::Error::Sql(rusqlite::Error::QueryReturnedNoRows)) => {
-            let start = std::time::Instant::now();
-            let keytype = KeyGenType::from_i32(context.get_config_int(Config::KeyGenType))
-                .unwrap_or_default();
-            info!(context, "Generating keypair with type {}", keytype);
-            let keypair = crate::pgp::create_keypair(addr, keytype)?;
-            store_self_keypair(context, &keypair, KeyPairUse::Default)?;
-            info!(
-                context,
-                "Keypair generated in {:.3}s.",
-                start.elapsed().as_secs()
-            );
-            Ok(keypair)
-        }
-        Err(err) => Err(err.into()),
-    }
 }
 
 /// Cryptographic key
@@ -282,6 +197,34 @@ impl Key {
         }
     }
 
+    pub fn from_self_public(
+        context: &Context,
+        self_addr: impl AsRef<str>,
+        sql: &Sql,
+    ) -> Option<Self> {
+        let addr = self_addr.as_ref();
+
+        sql.query_get_value(
+            context,
+            "SELECT public_key FROM keypairs WHERE addr=? AND is_default=1;",
+            &[addr],
+        )
+        .and_then(|blob: Vec<u8>| Self::from_slice(&blob, KeyType::Public))
+    }
+
+    pub fn from_self_private(
+        context: &Context,
+        self_addr: impl AsRef<str>,
+        sql: &Sql,
+    ) -> Option<Self> {
+        sql.query_get_value(
+            context,
+            "SELECT private_key FROM keypairs WHERE addr=? AND is_default=1;",
+            &[self_addr.as_ref()],
+        )
+        .and_then(|blob: Vec<u8>| Self::from_slice(&blob, KeyType::Private))
+    }
+
     pub fn to_bytes(&self) -> Vec<u8> {
         match self {
             Key::Public(k) => k.to_bytes().unwrap_or_default(),
@@ -373,19 +316,21 @@ pub enum KeyPairUse {
 }
 
 /// Error saving a keypair to the database.
-#[derive(Debug, thiserror::Error)]
-#[error("SaveKeyError: {message}")]
+#[derive(Fail, Debug)]
+#[fail(display = "SaveKeyError: {}", message)]
 pub struct SaveKeyError {
     message: String,
-    #[source]
-    cause: anyhow::Error,
+    #[cause]
+    cause: failure::Error,
+    backtrace: failure::Backtrace,
 }
 
 impl SaveKeyError {
-    fn new(message: impl Into<String>, cause: impl Into<anyhow::Error>) -> Self {
+    fn new(message: impl Into<String>, cause: impl Into<failure::Error>) -> Self {
         Self {
             message: message.into(),
             cause: cause.into(),
+            backtrace: failure::Backtrace::new(),
         }
     }
 }
@@ -608,59 +553,6 @@ i8pcjGO+IZffvyZJVRWfVooBJmWWbPB1pueo3tx8w3+fcuzpxz+RLFKaPyqXO+dD
         }
     }
 
-    #[test]
-    fn test_load_self_existing() {
-        let alice = alice_keypair();
-        let t = dummy_context();
-        configure_alice_keypair(&t.ctx);
-        let pubkey = SignedPublicKey::load_self(&t.ctx).unwrap();
-        assert_eq!(alice.public, pubkey);
-        let seckey = SignedSecretKey::load_self(&t.ctx).unwrap();
-        assert_eq!(alice.secret, seckey);
-    }
-
-    #[test]
-    #[ignore] // generating keys is expensive
-    fn test_load_self_generate_public() {
-        let t = dummy_context();
-        t.ctx
-            .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
-            .unwrap();
-        let key = SignedPublicKey::load_self(&t.ctx);
-        assert!(key.is_ok());
-    }
-
-    #[test]
-    #[ignore] // generating keys is expensive
-    fn test_load_self_generate_secret() {
-        let t = dummy_context();
-        t.ctx
-            .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
-            .unwrap();
-        let key = SignedSecretKey::load_self(&t.ctx);
-        assert!(key.is_ok());
-    }
-
-    #[test]
-    #[ignore] // generating keys is expensive
-    fn test_load_self_generate_concurrent() {
-        use std::sync::Arc;
-        use std::thread;
-
-        let t = dummy_context();
-        t.ctx
-            .set_config(Config::ConfiguredAddr, Some("alice@example.com"))
-            .unwrap();
-        let ctx = Arc::new(t.ctx);
-        let ctx0 = Arc::clone(&ctx);
-        let thr0 = thread::spawn(move || SignedPublicKey::load_self(&ctx0));
-        let ctx1 = Arc::clone(&ctx);
-        let thr1 = thread::spawn(move || SignedPublicKey::load_self(&ctx1));
-        let res0 = thr0.join().unwrap();
-        let res1 = thr1.join().unwrap();
-        assert_eq!(res0.unwrap(), res1.unwrap());
-    }
-
     #[test]
     fn test_ascii_roundtrip() {
         let public_key = Key::from(KEYPAIR.public.clone());

src/lib.rs

@@ -2,6 +2,8 @@
 #![deny(clippy::correctness, missing_debug_implementations, clippy::all)]
 #![allow(clippy::match_bool)]
 
+#[macro_use]
+extern crate failure_derive;
 #[macro_use]
 extern crate num_derive;
 #[macro_use]
@@ -25,22 +27,23 @@ pub(crate) mod events;
 pub use events::*;
 
 mod aheader;
-mod blob;
+pub mod blob;
 pub mod chat;
 pub mod chatlist;
 pub mod config;
-mod configure;
+pub mod configure;
 pub mod constants;
 pub mod contact;
 pub mod context;
 mod e2ee;
 mod imap;
+mod imap_client;
 pub mod imex;
 #[macro_use]
 pub mod job;
 mod job_thread;
 pub mod key;
-mod keyring;
+pub mod keyring;
 pub mod location;
 mod login_param;
 pub mod lot;

src/location.rs

@@ -1,6 +1,7 @@
 //! Location handling
 
 use bitflags::bitflags;
+use quick_xml;
 use quick_xml::events::{BytesEnd, BytesStart, BytesText};
 
 use crate::chat::{self, ChatId};
@@ -8,7 +9,7 @@ use crate::config::Config;
 use crate::constants::*;
 use crate::context::*;
 use crate::dc_tools::*;
-use crate::error::{ensure, Error};
+use crate::error::Error;
 use crate::events::Event;
 use crate::job::{self, job_action_exists, job_add, Job};
 use crate::message::{Message, MsgId};
@@ -63,10 +64,11 @@ impl Kml {
         Default::default()
     }
 
-    pub fn parse(context: &Context, to_parse: &[u8]) -> Result<Self, Error> {
-        ensure!(to_parse.len() <= 1024 * 1024, "kml-file is too large");
+    pub fn parse(context: &Context, content: &[u8]) -> Result<Self, Error> {
+        ensure!(content.len() <= 1024 * 1024, "kml-file is too large");
 
-        let mut reader = quick_xml::Reader::from_reader(to_parse);
+        let to_parse = String::from_utf8_lossy(content);
+        let mut reader = quick_xml::Reader::from_str(&to_parse);
         reader.trim_text(true);
 
         let mut kml = Kml::new();
@@ -122,9 +124,9 @@ impl Kml {
                 }
             } else if self.tag.contains(KmlTag::COORDINATES) {
                 let parts = val.splitn(2, ',').collect::<Vec<_>>();
-                if let [longitude, latitude] = &parts[..] {
-                    self.curr.longitude = longitude.parse().unwrap_or_default();
-                    self.curr.latitude = latitude.parse().unwrap_or_default();
+                if parts.len() == 2 {
+                    self.curr.longitude = parts[0].parse().unwrap_or_default();
+                    self.curr.latitude = parts[1].parse().unwrap_or_default();
                 }
             }
         }
@@ -363,7 +365,6 @@ fn is_marker(txt: &str) -> bool {
     txt.len() == 1 && !txt.starts_with(' ')
 }
 
-/// Deletes all locations from the database.
 pub fn delete_all(context: &Context) -> Result<(), Error> {
     sql::execute(context, &context.sql, "DELETE FROM locations;", params![])?;
     context.call_cb(Event::LocationChanged(None));
@@ -547,7 +548,7 @@ pub fn save(
 }
 
 #[allow(non_snake_case)]
-pub(crate) fn JobMaybeSendLocations(context: &Context, _job: &Job) -> job::Status {
+pub fn JobMaybeSendLocations(context: &Context, _job: &Job) -> job::Status {
     let now = time();
     let mut continue_streaming = false;
     info!(
@@ -638,7 +639,7 @@ pub(crate) fn JobMaybeSendLocations(context: &Context, _job: &Job) -> job::Statu
 }
 
 #[allow(non_snake_case)]
-pub(crate) fn JobMaybeSendLocationsEnded(context: &Context, job: &mut Job) -> job::Status {
+pub fn JobMaybeSendLocationsEnded(context: &Context, job: &mut Job) -> job::Status {
     // this function is called when location-streaming _might_ have ended for a chat.
     // the function checks, if location-streaming is really ended;
     // if so, a device-message is added if not yet done.

src/lot.rs

@@ -40,11 +40,11 @@ impl Lot {
     }
 
     pub fn get_text1(&self) -> Option<&str> {
-        self.text1.as_deref()
+        self.text1.as_ref().map(|s| s.as_str())
     }
 
     pub fn get_text2(&self) -> Option<&str> {
-        self.text2.as_deref()
+        self.text2.as_ref().map(|s| s.as_str())
     }
 
     pub fn get_text1_meaning(&self) -> Meaning {

src/message.rs

@@ -3,15 +3,14 @@
 use std::path::{Path, PathBuf};
 
 use deltachat_derive::{FromSql, ToSql};
-use lazy_static::lazy_static;
-use serde::{Deserialize, Serialize};
+use failure::Fail;
 
 use crate::chat::{self, Chat, ChatId};
 use crate::constants::*;
 use crate::contact::*;
 use crate::context::*;
 use crate::dc_tools::*;
-use crate::error::{ensure, Error};
+use crate::error::Error;
 use crate::events::Event;
 use crate::job::*;
 use crate::lot::{Lot, LotState, Meaning};
@@ -21,10 +20,6 @@ use crate::pgp::*;
 use crate::sql;
 use crate::stock::StockMessage;
 
-lazy_static! {
-    static ref UNWRAP_RE: regex::Regex = regex::Regex::new(r"\s+").unwrap();
-}
-
 // In practice, the user additionally cuts the string themselves
 // pixel-accurate.
 const SUMMARY_CHARACTERS: usize = 160;
@@ -34,9 +29,7 @@ const SUMMARY_CHARACTERS: usize = 160;
 /// Some message IDs are reserved to identify special message types.
 /// This type can represent both the special as well as normal
 /// messages.
-#[derive(
-    Debug, Copy, Clone, Default, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize,
-)]
+#[derive(Debug, Copy, Clone, Default, PartialEq, Eq)]
 pub struct MsgId(u32);
 
 impl MsgId {
@@ -84,55 +77,6 @@ impl MsgId {
         self.0 == DC_MSG_ID_DAYMARKER
     }
 
-    /// Put message into trash chat and delete message text.
-    ///
-    /// It means the message is deleted locally, but not on the server
-    /// yet.
-    pub fn trash(self, context: &Context) -> crate::sql::Result<()> {
-        let chat_id = ChatId::new(DC_CHAT_ID_TRASH);
-        sql::execute(
-            context,
-            &context.sql,
-            "UPDATE msgs SET chat_id=?, txt='', txt_raw='' WHERE id=?",
-            params![chat_id, self],
-        )
-    }
-
-    /// Deletes a message and corresponding MDNs from the database.
-    pub fn delete_from_db(self, context: &Context) -> crate::sql::Result<()> {
-        // We don't use transactions yet, so remove MDNs first to make
-        // sure they are not left while the message is deleted.
-        sql::execute(
-            context,
-            &context.sql,
-            "DELETE FROM msgs_mdns WHERE msg_id=?;",
-            params![self],
-        )?;
-        sql::execute(
-            context,
-            &context.sql,
-            "DELETE FROM msgs WHERE id=?;",
-            params![self],
-        )?;
-        Ok(())
-    }
-
-    /// Removes IMAP server UID and folder from the database record.
-    ///
-    /// It is used to avoid trying to remove the message from the
-    /// server multiple times when there are multiple message records
-    /// pointing to the same server UID.
-    pub(crate) fn unlink(self, context: &Context) -> sql::Result<()> {
-        sql::execute(
-            context,
-            &context.sql,
-            "UPDATE msgs \
-             SET server_folder='', server_uid=0 \
-             WHERE id=?",
-            params![self],
-        )
-    }
-
     /// Bad evil escape hatch.
     ///
     /// Avoid using this, eventually types should be cleaned up enough
@@ -169,7 +113,7 @@ impl rusqlite::types::ToSql for MsgId {
     fn to_sql(&self) -> rusqlite::Result<rusqlite::types::ToSqlOutput> {
         if self.0 <= DC_MSG_ID_LAST_SPECIAL {
             return Err(rusqlite::Error::ToSqlConversionFailure(Box::new(
-                InvalidMsgId,
+                InvalidMsgId.compat(),
             )));
         }
         let val = rusqlite::types::Value::Integer(self.0 as i64);
@@ -197,24 +141,13 @@ impl rusqlite::types::FromSql for MsgId {
 /// This usually occurs when trying to use a message ID of
 /// [DC_MSG_ID_LAST_SPECIAL] or below in a situation where this is not
 /// possible.
-#[derive(Debug, thiserror::Error)]
-#[error("Invalid Message ID.")]
+#[derive(Debug, Fail)]
+#[fail(display = "Invalid Message ID.")]
 pub struct InvalidMsgId;
 
-#[derive(
-    Debug,
-    Copy,
-    Clone,
-    PartialEq,
-    FromPrimitive,
-    ToPrimitive,
-    FromSql,
-    ToSql,
-    Serialize,
-    Deserialize,
-)]
+#[derive(Debug, Copy, Clone, PartialEq, FromPrimitive, ToPrimitive, FromSql, ToSql)]
 #[repr(u8)]
-pub(crate) enum MessengerMessage {
+pub enum MessengerMessage {
     No = 0,
     Yes = 1,
 
@@ -235,7 +168,7 @@ impl Default for MessengerMessage {
 /// to check if a mail was sent, use dc_msg_is_sent()
 /// approx. max. length returned by dc_msg_get_text()
 /// approx. max. length returned by dc_get_msg_info()
-#[derive(Debug, Clone, Default, Serialize, Deserialize)]
+#[derive(Debug, Clone, Default)]
 pub struct Message {
     pub(crate) id: MsgId,
     pub(crate) from_id: u32,
@@ -353,6 +286,25 @@ impl Message {
             .map_err(Into::into)
     }
 
+    pub fn delete_from_db(context: &Context, msg_id: MsgId) {
+        if let Ok(msg) = Message::load_from_db(context, msg_id) {
+            sql::execute(
+                context,
+                &context.sql,
+                "DELETE FROM msgs WHERE id=?;",
+                params![msg.id],
+            )
+            .ok();
+            sql::execute(
+                context,
+                &context.sql,
+                "DELETE FROM msgs_mdns WHERE msg_id=?;",
+                params![msg.id],
+            )
+            .ok();
+        }
+    }
+
     pub fn get_filemime(&self) -> Option<String> {
         if let Some(m) = self.param.get(Param::MimeType) {
             return Some(m.to_string());
@@ -472,7 +424,7 @@ impl Message {
     pub fn get_text(&self) -> Option<String> {
         self.text
             .as_ref()
-            .map(|text| dc_truncate(text, 30000).to_string())
+            .map(|text| dc_truncate(text, 30000, false).to_string())
     }
 
     pub fn get_filename(&self) -> Option<String> {
@@ -655,19 +607,7 @@ impl Message {
     }
 }
 
-#[derive(
-    Debug,
-    Clone,
-    Copy,
-    PartialEq,
-    Eq,
-    FromPrimitive,
-    ToPrimitive,
-    ToSql,
-    FromSql,
-    Serialize,
-    Deserialize,
-)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, FromPrimitive, ToPrimitive, ToSql, FromSql)]
 #[repr(i32)]
 pub enum MessageState {
     Undefined = 0,
@@ -843,7 +783,7 @@ pub fn get_msg_info(context: &Context, msg_id: MsgId) -> String {
         return ret;
     }
     let rawtxt = rawtxt.unwrap_or_default();
-    let rawtxt = dc_truncate(rawtxt.trim(), 100_000);
+    let rawtxt = dc_truncate(rawtxt.trim(), 100_000, false);
 
     let fts = dc_timestamp_to_str(msg.get_timestamp());
     ret += &format!("Sent: {}", fts);
@@ -980,15 +920,13 @@ pub fn get_mime_headers(context: &Context, msg_id: MsgId) -> Option<String> {
 }
 
 pub fn delete_msgs(context: &Context, msg_ids: &[MsgId]) {
-    for msg_id in msg_ids {
+    for msg_id in msg_ids.iter() {
         if let Ok(msg) = Message::load_from_db(context, *msg_id) {
             if msg.location_id > 0 {
                 delete_poi_location(context, msg.location_id);
             }
         }
-        if let Err(err) = msg_id.trash(context) {
-            error!(context, "Unable to trash message {}: {}", msg_id, err);
-        }
+        update_msg_chat_id(context, *msg_id, ChatId::new(DC_CHAT_ID_TRASH));
         job_add(
             context,
             Action::DeleteMsgOnImap,
@@ -1008,6 +946,16 @@ pub fn delete_msgs(context: &Context, msg_ids: &[MsgId]) {
     };
 }
 
+fn update_msg_chat_id(context: &Context, msg_id: MsgId, chat_id: ChatId) -> bool {
+    sql::execute(
+        context,
+        &context.sql,
+        "UPDATE msgs SET chat_id=? WHERE id=?;",
+        params![chat_id, msg_id],
+    )
+    .is_ok()
+}
+
 fn delete_poi_location(context: &Context, location_id: u32) -> bool {
     sql::execute(
         context,
@@ -1167,20 +1115,18 @@ pub fn get_summarytext_by_raw(
         return prefix;
     }
 
-    let summary = if let Some(text) = text {
+    if let Some(text) = text {
         if text.as_ref().is_empty() {
             prefix
         } else if prefix.is_empty() {
-            dc_truncate(text.as_ref(), approx_characters).to_string()
+            dc_truncate(text.as_ref(), approx_characters, true).to_string()
         } else {
             let tmp = format!("{} – {}", prefix, text.as_ref());
-            dc_truncate(&tmp, approx_characters).to_string()
+            dc_truncate(&tmp, approx_characters, true).to_string()
         }
     } else {
         prefix
-    };
-
-    UNWRAP_RE.replace_all(&summary, " ").to_string()
+    }
 }
 
 // as we do not cut inside words, this results in about 32-42 characters.
@@ -1215,7 +1161,7 @@ pub fn set_msg_failed(context: &Context, msg_id: MsgId, error: Option<impl AsRef
         }
         if let Some(error) = error {
             msg.param.set(Param::Error, error.as_ref());
-            warn!(context, "Message failed: {}", error.as_ref());
+            error!(context, "{}", error.as_ref());
         }
 
         if sql::execute(
@@ -1368,55 +1314,10 @@ pub fn get_deaddrop_msg_cnt(context: &Context) -> usize {
     }
 }
 
-pub fn estimate_deletion_cnt(
-    context: &Context,
-    from_server: bool,
-    seconds: i64,
-) -> Result<usize, Error> {
-    let self_chat_id = chat::lookup_by_contact_id(context, DC_CONTACT_ID_SELF)
-        .unwrap_or_default()
-        .0;
-    let threshold_timestamp = time() - seconds;
-
-    let cnt: isize = if from_server {
-        context.sql.query_row(
-            "SELECT COUNT(*)
-             FROM msgs m
-             WHERE m.id > ?
-               AND timestamp < ?
-               AND chat_id != ?
-               AND server_uid != 0;",
-            params![DC_MSG_ID_LAST_SPECIAL, threshold_timestamp, self_chat_id],
-            |row| row.get(0),
-        )?
-    } else {
-        context.sql.query_row(
-            "SELECT COUNT(*)
-             FROM msgs m
-             WHERE m.id > ?
-               AND timestamp < ?
-               AND chat_id != ?
-               AND chat_id != ? AND hidden = 0;",
-            params![
-                DC_MSG_ID_LAST_SPECIAL,
-                threshold_timestamp,
-                self_chat_id,
-                ChatId::new(DC_CHAT_ID_TRASH)
-            ],
-            |row| row.get(0),
-        )?
-    };
-    Ok(cnt as usize)
-}
-
-/// Counts number of database records pointing to specified
-/// Message-ID.
-///
-/// Unlinked messages are excluded.
 pub fn rfc724_mid_cnt(context: &Context, rfc724_mid: &str) -> i32 {
     // check the number of messages with the same rfc724_mid
     match context.sql.query_row(
-        "SELECT COUNT(*) FROM msgs WHERE rfc724_mid=? AND NOT server_uid = 0",
+        "SELECT COUNT(*) FROM msgs WHERE rfc724_mid=?;",
         &[rfc724_mid],
         |row| row.get(0),
     ) {
@@ -1431,15 +1332,12 @@ pub fn rfc724_mid_cnt(context: &Context, rfc724_mid: &str) -> i32 {
 pub(crate) fn rfc724_mid_exists(
     context: &Context,
     rfc724_mid: &str,
-) -> Result<Option<(String, u32, MsgId)>, Error> {
-    if rfc724_mid.is_empty() {
-        warn!(context, "Empty rfc724_mid passed to rfc724_mid_exists");
-        return Ok(None);
-    }
+) -> Result<(String, u32, MsgId), Error> {
+    ensure!(!rfc724_mid.is_empty(), "empty rfc724_mid");
 
     context
         .sql
-        .query_row_optional(
+        .query_row(
             "SELECT server_folder, server_uid, id FROM msgs WHERE rfc724_mid=?",
             &[rfc724_mid],
             |row| {
@@ -1460,8 +1358,7 @@ pub fn update_server_uid(
     server_uid: u32,
 ) {
     match context.sql.execute(
-        "UPDATE msgs SET server_folder=?, server_uid=? \
-         WHERE rfc724_mid=?",
+        "UPDATE msgs SET server_folder=?, server_uid=? WHERE rfc724_mid=?;",
         params![server_folder.as_ref(), server_uid, rfc724_mid],
     ) {
         Ok(_) => {}

src/mimefactory.rs

@@ -9,13 +9,12 @@ use crate::contact::*;
 use crate::context::{get_version_str, Context};
 use crate::dc_tools::*;
 use crate::e2ee::*;
-use crate::error::{bail, ensure, format_err, Error};
+use crate::error::Error;
 use crate::location;
 use crate::message::{self, Message};
 use crate::mimeparser::SystemMessage;
 use crate::param::*;
 use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
-use crate::simplify::escape_message_footer_marks;
 use crate::stock::StockMessage;
 
 // attachments of 25 mb brutto should work on the majority of providers
@@ -69,7 +68,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
     pub fn from_msg(
         context: &'a Context,
         msg: &'b Message,
-        attach_selfavatar: bool,
+        add_selfavatar: bool,
     ) -> Result<MimeFactory<'a, 'b>, Error> {
         let chat = Chat::load_from_db(context, msg.chat_id)?;
 
@@ -107,6 +106,21 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
             let command = msg.param.get_cmd();
 
+            /* for added members, the list is just fine */
+            if command == SystemMessage::MemberRemovedFromGroup {
+                let email_to_remove = msg.param.get(Param::Arg).unwrap_or_default();
+
+                let self_addr = context
+                    .get_config(Config::ConfiguredAddr)
+                    .unwrap_or_default();
+
+                if !email_to_remove.is_empty()
+                    && !addr_cmp(email_to_remove, self_addr)
+                    && !recipients_contain_addr(&recipients, &email_to_remove)
+                {
+                    recipients.push(("".to_string(), email_to_remove.to_string()));
+                }
+            }
             if command != SystemMessage::AutocryptSetupMessage
                 && command != SystemMessage::SecurejoinMessage
                 && context.get_config_bool(Config::MdnsEnabled)
@@ -142,7 +156,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             references,
             req_mdn,
             last_added_location_id: 0,
-            attach_selfavatar,
+            attach_selfavatar: add_selfavatar,
             context,
         };
         Ok(factory)
@@ -354,7 +368,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             self.from_addr.clone(),
         );
 
-        let mut to = Vec::new();
+        let mut to = Vec::with_capacity(self.recipients.len());
         for (name, addr) in self.recipients.iter() {
             if name.is_empty() {
                 to.push(Address::new_mailbox(addr.clone()));
@@ -366,10 +380,6 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
             }
         }
 
-        if to.is_empty() {
-            to.push(from.clone());
-        }
-
         if !self.references.is_empty() {
             unprotected_headers.push(Header::new("References".into(), self.references.clone()));
         }
@@ -628,6 +638,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let command = self.msg.param.get_cmd();
         let mut placeholdertext = None;
         let mut meta_part = None;
+        let mut add_compatibility_header = false;
 
         if chat.typ == Chattype::VerifiedGroup {
             protected_headers.push(Header::new("Chat-Verified".to_string(), "1".to_string()));
@@ -670,6 +681,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                             "vg-member-added".to_string(),
                         ));
                     }
+                    add_compatibility_header = true;
                 }
                 SystemMessage::GroupNameChanged => {
                     let value_to_add = self.msg.param.get(Param::Arg).unwrap_or_default();
@@ -690,6 +702,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
                             "0".to_string(),
                         ));
                     }
+                    add_compatibility_header = true;
                 }
                 _ => {}
             }
@@ -757,7 +770,18 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
             let (mail, filename_as_sent) = build_body_file(context, &meta, "group-image")?;
             meta_part = Some(mail);
-            protected_headers.push(Header::new("Chat-Group-Avatar".into(), filename_as_sent));
+            protected_headers.push(Header::new(
+                "Chat-Group-Avatar".into(),
+                filename_as_sent.clone(),
+            ));
+
+            // add the old group-image headers for versions <=0.973 resp. <=beta.15 (december 2019)
+            // image deletion is not supported in the compatibility layer.
+            // this can be removed some time after releasing 1.0,
+            // grep for #DeprecatedAvatar to get the place where compatibility parsing takes place.
+            if add_compatibility_header {
+                protected_headers.push(Header::new("Chat-Group-Image".into(), filename_as_sent));
+            }
         }
 
         if self.msg.viewtype == Viewtype::Sticker {
@@ -808,7 +832,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
         let message_text = format!(
             "{}{}{}{}{}",
             fwdhint.unwrap_or_default(),
-            escape_message_footer_marks(final_text),
+            &final_text,
             if !final_text.is_empty() && !footer.is_empty() {
                 "\r\n\r\n"
             } else {
@@ -856,7 +880,7 @@ impl<'a, 'b> MimeFactory<'a, 'b> {
 
         if self.attach_selfavatar {
             match context.get_config(Config::Selfavatar) {
-                Some(path) => match build_selfavatar_file(context, &path) {
+                Some(path) => match build_selfavatar_file(context, path) {
                     Ok((part, filename)) => {
                         parts.push(part);
                         protected_headers.push(Header::new("Chat-User-Avatar".into(), filename))
@@ -1051,7 +1075,7 @@ fn build_body_file(
     Ok((mail, filename_to_send))
 }
 
-fn build_selfavatar_file(context: &Context, path: &str) -> Result<(PartBuilder, String), Error> {
+fn build_selfavatar_file(context: &Context, path: String) -> Result<(PartBuilder, String), Error> {
     let blob = BlobObject::from_path(context, path)?;
     let filename_to_send = match blob.suffix() {
         Some(suffix) => format!("avatar.{}", suffix),
@@ -1096,7 +1120,7 @@ fn is_file_size_okay(context: &Context, msg: &Message) -> bool {
 fn render_rfc724_mid(rfc724_mid: &str) -> String {
     let rfc724_mid = rfc724_mid.trim().to_string();
 
-    if rfc724_mid.chars().next().unwrap_or_default() == '<' {
+    if rfc724_mid.chars().nth(0).unwrap_or_default() == '<' {
         rfc724_mid
     } else {
         format!("<{}>", rfc724_mid)

src/param.rs

@@ -4,18 +4,15 @@ use std::path::PathBuf;
 use std::str;
 
 use num_traits::FromPrimitive;
-use serde::{Deserialize, Serialize};
 
 use crate::blob::{BlobError, BlobObject};
 use crate::context::Context;
-use crate::error::{self, bail, ensure};
+use crate::error;
 use crate::message::MsgId;
 use crate::mimeparser::SystemMessage;
 
 /// Available param keys.
-#[derive(
-    PartialEq, Eq, Debug, Clone, Copy, Hash, PartialOrd, Ord, FromPrimitive, Serialize, Deserialize,
-)]
+#[derive(PartialEq, Eq, Debug, Clone, Copy, Hash, PartialOrd, Ord, FromPrimitive)]
 #[repr(u8)]
 pub enum Param {
     /// For messages and jobs
@@ -88,6 +85,12 @@ pub enum Param {
     /// For Jobs
     SetLongitude = b'n',
 
+    /// For Jobs
+    ServerFolder = b'Z',
+
+    /// For Jobs
+    ServerUid = b'z',
+
     /// For Jobs
     AlsoMove = b'M',
 
@@ -132,7 +135,7 @@ pub enum ForcePlaintext {
 /// The structure is serialized by calling `to_string()` on it.
 ///
 /// Only for library-internal use.
-#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Eq, Default)]
 pub struct Params {
     inner: BTreeMap<Param, String>,
 }

src/peerstate.rs

@@ -316,25 +316,6 @@ impl<'a> Peerstate<'a> {
                 self.recalc_fingerprint();
                 self.to_save = Some(ToSave::All)
             }
-
-            // This is non-standard.
-            //
-            // According to Autocrypt 1.1.0 gossip headers SHOULD NOT
-            // contain encryption preference, but we include it into
-            // Autocrypt-Gossip and apply it one way (from
-            // "nopreference" to "mutual").
-            //
-            // This is compatible to standard clients, because they
-            // can't distinguish it from the case where we have
-            // contacted the client in the past and received this
-            // preference via Autocrypt header.
-            if self.last_seen_autocrypt == 0
-                && self.prefer_encrypt == EncryptPreference::NoPreference
-                && gossip_header.prefer_encrypt == EncryptPreference::Mutual
-            {
-                self.prefer_encrypt = EncryptPreference::Mutual;
-                self.to_save = Some(ToSave::All);
-            }
         };
     }
 
@@ -345,15 +326,7 @@ impl<'a> Peerstate<'a> {
             let header = Aheader::new(
                 self.addr.clone(),
                 public_key,
-                // Autocrypt 1.1.0 specification says that
-                // `prefer-encrypt` attribute SHOULD NOT be included,
-                // but we include it anyway to propagate encryption
-                // preference to new members in group chats.
-                if self.last_seen_autocrypt > 0 {
-                    self.prefer_encrypt
-                } else {
-                    EncryptPreference::NoPreference
-                },
+                EncryptPreference::NoPreference,
             );
             Some(header.to_string())
         } else {

src/pgp.rs

@@ -16,9 +16,8 @@ use pgp::types::{
 };
 use rand::{thread_rng, CryptoRng, Rng};
 
-use crate::constants::KeyGenType;
 use crate::dc_tools::EmailAddress;
-use crate::error::{bail, ensure, format_err, Result};
+use crate::error::Result;
 use crate::key::*;
 use crate::keyring::*;
 
@@ -117,19 +116,21 @@ pub fn split_armored_data(buf: &[u8]) -> Result<(BlockType, BTreeMap<String, Str
 ///
 /// Most of these are likely coding errors rather than user errors
 /// since all variability is hardcoded.
-#[derive(Debug, thiserror::Error)]
-#[error("PgpKeygenError: {message}")]
-pub struct PgpKeygenError {
+#[derive(Fail, Debug)]
+#[fail(display = "PgpKeygenError: {}", message)]
+pub(crate) struct PgpKeygenError {
     message: String,
-    #[source]
-    cause: anyhow::Error,
+    #[cause]
+    cause: failure::Error,
+    backtrace: failure::Backtrace,
 }
 
 impl PgpKeygenError {
-    fn new(message: impl Into<String>, cause: impl Into<anyhow::Error>) -> Self {
+    fn new(message: impl Into<String>, cause: impl Into<failure::Error>) -> Self {
         Self {
             message: message.into(),
             cause: cause.into(),
+            backtrace: failure::Backtrace::new(),
         }
     }
 }
@@ -146,18 +147,10 @@ pub struct KeyPair {
 }
 
 /// Create a new key pair.
-pub(crate) fn create_keypair(
-    addr: EmailAddress,
-    keygen_type: KeyGenType,
-) -> std::result::Result<KeyPair, PgpKeygenError> {
-    let (secret_key_type, public_key_type) = match keygen_type {
-        KeyGenType::Rsa2048 => (PgpKeyType::Rsa(2048), PgpKeyType::Rsa(2048)),
-        KeyGenType::Ed25519 | KeyGenType::Default => (PgpKeyType::EdDSA, PgpKeyType::ECDH),
-    };
-
+pub(crate) fn create_keypair(addr: EmailAddress) -> std::result::Result<KeyPair, PgpKeygenError> {
     let user_id = format!("<{}>", addr);
     let key_params = SecretKeyParamsBuilder::default()
-        .key_type(secret_key_type)
+        .key_type(PgpKeyType::Rsa(2048))
         .can_create_certificates(true)
         .can_sign(true)
         .primary_user_id(user_id)
@@ -180,14 +173,14 @@ pub(crate) fn create_keypair(
         ])
         .subkey(
             SubkeyParamsBuilder::default()
-                .key_type(public_key_type)
+                .key_type(PgpKeyType::Rsa(2048))
                 .can_encrypt(true)
                 .passphrase(None)
                 .build()
                 .unwrap(),
         )
         .build()
-        .map_err(|err| PgpKeygenError::new("invalid key params", format_err!(err)))?;
+        .map_err(|err| PgpKeygenError::new("invalid key params", failure::err_msg(err)))?;
     let key = key_params
         .generate()
         .map_err(|err| PgpKeygenError::new("invalid params", err))?;
@@ -392,17 +385,10 @@ mod tests {
     }
 
     #[test]
+    #[ignore] // is too expensive
     fn test_create_keypair() {
-        let keypair0 = create_keypair(
-            EmailAddress::new("foo@bar.de").unwrap(),
-            KeyGenType::Default,
-        )
-        .unwrap();
-        let keypair1 = create_keypair(
-            EmailAddress::new("two@zwo.de").unwrap(),
-            KeyGenType::Default,
-        )
-        .unwrap();
+        let keypair0 = create_keypair(EmailAddress::new("foo@bar.de").unwrap()).unwrap();
+        let keypair1 = create_keypair(EmailAddress::new("two@zwo.de").unwrap()).unwrap();
         assert_ne!(keypair0.public, keypair1.public);
     }
 

src/provider/data.rs

@@ -18,18 +18,6 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: STARTTLS, hostname: "newyear.aktivix.org", port: 143, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "newyear.aktivix.org", port: 25, username_pattern: EMAIL },
         ],
-        config_defaults: None,
-    };
-
-    // aol.md: aol.com
-    static ref P_AOL: Provider = Provider {
-        status: Status::PREPARATION,
-        before_login_hint: "To log in to AOL with Delta Chat, you need to set up an app password in the AOL web interface.",
-        after_login_hint: "",
-        overview_page: "https://providers.delta.chat/aol",
-        server: vec![
-        ],
-        config_defaults: None,
     };
 
     // autistici.org.md: autistici.org
@@ -42,7 +30,6 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "mail.autistici.org", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.autistici.org", port: 465, username_pattern: EMAIL },
         ],
-        config_defaults: None,
     };
 
     // bluewin.ch.md: bluewin.ch
@@ -55,7 +42,6 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imaps.bluewin.ch", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: SSL, hostname: "smtpauths.bluewin.ch", port: 465, username_pattern: EMAIL },
         ],
-        config_defaults: None,
     };
 
     // comcast.md: xfinity.com, comcast.net
@@ -77,18 +63,6 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.example.com", port: 1337, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.example.com", port: 1337, username_pattern: EMAIL },
         ],
-        config_defaults: None,
-    };
-
-    // fastmail.md: fastmail.com
-    static ref P_FASTMAIL: Provider = Provider {
-        status: Status::PREPARATION,
-        before_login_hint: "You must create an app-specific password for Delta Chat before you can log in.",
-        after_login_hint: "",
-        overview_page: "https://providers.delta.chat/fastmail",
-        server: vec![
-        ],
-        config_defaults: None,
     };
 
     // freenet.de.md: freenet.de
@@ -101,7 +75,6 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "mx.freenet.de", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "mx.freenet.de", port: 587, username_pattern: EMAIL },
         ],
-        config_defaults: None,
     };
 
     // gmail.md: gmail.com, googlemail.com
@@ -114,7 +87,6 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.gmail.com", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.gmail.com", port: 465, username_pattern: EMAIL },
         ],
-        config_defaults: None,
     };
 
     // gmx.net.md: gmx.net, gmx.de, gmx.at, gmx.ch, gmx.org, gmx.eu, gmx.info, gmx.biz, gmx.com
@@ -128,7 +100,6 @@ lazy_static::lazy_static! {
             Server { protocol: SMTP, socket: SSL, hostname: "mail.gmx.net", port: 465, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "mail.gmx.net", port: 587, username_pattern: EMAIL },
         ],
-        config_defaults: None,
     };
 
     // i.ua.md: i.ua
@@ -144,7 +115,6 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.mail.me.com", port: 993, username_pattern: EMAILLOCALPART },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.mail.me.com", port: 587, username_pattern: EMAIL },
         ],
-        config_defaults: None,
     };
 
     // kolst.com.md: kolst.com
@@ -169,15 +139,6 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: STARTTLS, hostname: "imap.nauta.cu", port: 143, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.nauta.cu", port: 25, username_pattern: EMAIL },
         ],
-        config_defaults: Some(vec![
-            ConfigDefault { key: Config::DeleteServerAfter, value: "1" },
-            ConfigDefault { key: Config::BccSelf, value: "0" },
-            ConfigDefault { key: Config::SentboxWatch, value: "0" },
-            ConfigDefault { key: Config::MvboxWatch, value: "0" },
-            ConfigDefault { key: Config::MvboxMove, value: "0" },
-            ConfigDefault { key: Config::E2eeEnabled, value: "0" },
-            ConfigDefault { key: Config::MediaQuality, value: "1" },
-        ]),
     };
 
     // outlook.com.md: hotmail.com, outlook.com, office365.com, outlook.com.tr, live.com
@@ -190,7 +151,6 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap-mail.outlook.com", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp-mail.outlook.com", port: 587, username_pattern: EMAIL },
         ],
-        config_defaults: None,
     };
 
     // posteo.md: posteo.de
@@ -203,18 +163,6 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: STARTTLS, hostname: "posteo.de", port: 143, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "posteo.de", port: 587, username_pattern: EMAIL },
         ],
-        config_defaults: None,
-    };
-
-    // protonmail.md: protonmail.com, protonmail.ch
-    static ref P_PROTONMAIL: Provider = Provider {
-        status: Status::BROKEN,
-        before_login_hint: "Protonmail does not offer the standard IMAP e-mail protocol, so you cannot log in with Delta Chat to Protonmail.",
-        after_login_hint: "To use Delta Chat with Protonmail, the IMAP bridge must be running in the background. If you have connectivity issues, double check whether it works as expected.",
-        overview_page: "https://providers.delta.chat/protonmail",
-        server: vec![
-        ],
-        config_defaults: None,
     };
 
     // riseup.net.md: riseup.net
@@ -223,34 +171,6 @@ lazy_static::lazy_static! {
     // rogers.com.md: rogers.com
     // - skipping provider with status OK and no special things to do
 
-    // systemli.org.md: systemli.org
-    // - skipping provider with status OK and no special things to do
-
-    // t-online.md: t-online.de, magenta.de
-    static ref P_T_ONLINE: Provider = Provider {
-        status: Status::PREPARATION,
-        before_login_hint: "To use Delta Chat with a T-Online email address, you need to create an app password in the web interface.",
-        after_login_hint: "",
-        overview_page: "https://providers.delta.chat/t-online",
-        server: vec![
-        ],
-        config_defaults: None,
-    };
-
-    // testrun.md: testrun.org
-    static ref P_TESTRUN: Provider = Provider {
-        status: Status::OK,
-        before_login_hint: "",
-        after_login_hint: "",
-        overview_page: "https://providers.delta.chat/testrun",
-        server: vec![
-            Server { protocol: IMAP, socket: SSL, hostname: "testrun.org", port: 993, username_pattern: EMAIL },
-            Server { protocol: IMAP, socket: STARTTLS, hostname: "testrun.org", port: 143, username_pattern: EMAIL },
-            Server { protocol: SMTP, socket: STARTTLS, hostname: "testrun.org", port: 587, username_pattern: EMAIL },
-        ],
-        config_defaults: None,
-    };
-
     // tiscali.it.md: tiscali.it
     static ref P_TISCALI_IT: Provider = Provider {
         status: Status::OK,
@@ -261,7 +181,6 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.tiscali.it", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.tiscali.it", port: 465, username_pattern: EMAIL },
         ],
-        config_defaults: None,
     };
 
     // ukr.net.md: ukr.net
@@ -281,7 +200,6 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: STARTTLS, hostname: "imap.web.de", port: 143, username_pattern: EMAILLOCALPART },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.web.de", port: 587, username_pattern: EMAILLOCALPART },
         ],
-        config_defaults: None,
     };
 
     // yahoo.md: yahoo.com, yahoo.de, yahoo.it, yahoo.fr, yahoo.es, yahoo.se, yahoo.co.uk, yahoo.co.nz, yahoo.com.au, yahoo.com.ar, yahoo.com.br, yahoo.com.mx, ymail.com, rocketmail.com, yahoodns.net
@@ -294,19 +212,10 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.mail.yahoo.com", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: SSL, hostname: "smtp.mail.yahoo.com", port: 465, username_pattern: EMAIL },
         ],
-        config_defaults: None,
     };
 
     // yandex.ru.md: yandex.ru, yandex.com
-    static ref P_YANDEX_RU: Provider = Provider {
-        status: Status::PREPARATION,
-        before_login_hint: "For Yandex accounts, you have to set IMAP protocol option turned on.",
-        after_login_hint: "",
-        overview_page: "https://providers.delta.chat/yandex-ru",
-        server: vec![
-        ],
-        config_defaults: None,
-    };
+    // - skipping provider with status OK and no special things to do
 
     // ziggo.nl.md: ziggo.nl
     static ref P_ZIGGO_NL: Provider = Provider {
@@ -318,17 +227,17 @@ lazy_static::lazy_static! {
             Server { protocol: IMAP, socket: SSL, hostname: "imap.ziggo.nl", port: 993, username_pattern: EMAIL },
             Server { protocol: SMTP, socket: STARTTLS, hostname: "smtp.ziggo.nl", port: 587, username_pattern: EMAIL },
         ],
-        config_defaults: None,
     };
 
+    // zoho.com.md: zoho.com
+    // - skipping provider with status OK and no special things to do
+
     pub static ref PROVIDER_DATA: HashMap<&'static str, &'static Provider> = [
         ("aktivix.org", &*P_AKTIVIX_ORG),
-        ("aol.com", &*P_AOL),
         ("autistici.org", &*P_AUTISTICI_ORG),
         ("bluewin.ch", &*P_BLUEWIN_CH),
         ("example.com", &*P_EXAMPLE_COM),
         ("example.org", &*P_EXAMPLE_COM),
-        ("fastmail.com", &*P_FASTMAIL),
         ("freenet.de", &*P_FREENET_DE),
         ("gmail.com", &*P_GMAIL),
         ("googlemail.com", &*P_GMAIL),
@@ -351,11 +260,6 @@ lazy_static::lazy_static! {
         ("outlook.com.tr", &*P_OUTLOOK_COM),
         ("live.com", &*P_OUTLOOK_COM),
         ("posteo.de", &*P_POSTEO),
-        ("protonmail.com", &*P_PROTONMAIL),
-        ("protonmail.ch", &*P_PROTONMAIL),
-        ("t-online.de", &*P_T_ONLINE),
-        ("magenta.de", &*P_T_ONLINE),
-        ("testrun.org", &*P_TESTRUN),
         ("tiscali.it", &*P_TISCALI_IT),
         ("web.de", &*P_WEB_DE),
         ("email.de", &*P_WEB_DE),
@@ -399,8 +303,6 @@ lazy_static::lazy_static! {
         ("ymail.com", &*P_YAHOO),
         ("rocketmail.com", &*P_YAHOO),
         ("yahoodns.net", &*P_YAHOO),
-        ("yandex.ru", &*P_YANDEX_RU),
-        ("yandex.com", &*P_YANDEX_RU),
         ("ziggo.nl", &*P_ZIGGO_NL),
     ].iter().copied().collect();
 }

src/provider/mod.rs

@@ -1,8 +1,5 @@
-//! [Provider database](https://providers.delta.chat/) module
-
 mod data;
 
-use crate::config::Config;
 use crate::dc_tools::EmailAddress;
 use crate::provider::data::PROVIDER_DATA;
 
@@ -58,12 +55,6 @@ impl Server {
     }
 }
 
-#[derive(Debug)]
-pub struct ConfigDefault {
-    pub key: Config,
-    pub value: &'static str,
-}
-
 #[derive(Debug)]
 pub struct Provider {
     pub status: Status,
@@ -71,7 +62,6 @@ pub struct Provider {
     pub after_login_hint: &'static str,
     pub overview_page: &'static str,
     pub server: Vec<Server>,
-    pub config_defaults: Option<Vec<ConfigDefault>>,
 }
 
 impl Provider {

src/provider/update.py

@@ -9,9 +9,6 @@ out_all = ""
 out_domains = ""
 domains_dict = {}
 
-def camel(name):
-    words = name.split("_")
-    return "".join(w.capitalize() for i, w in enumerate(words))
 
 def cleanstr(s):
     s = s.strip()
@@ -34,18 +31,6 @@ def file2url(f):
     return "https://providers.delta.chat/" + f
 
 
-def process_config_defaults(data):
-    if not "config_defaults" in data:
-        return "None"
-    defaults = "Some(vec![\n"
-    config_defaults = data.get("config_defaults", "")
-    for key in config_defaults:
-        value = str(config_defaults[key])
-        defaults += "            ConfigDefault { key: Config::" + camel(key) + ", value: \"" + value + "\" },\n"
-    defaults += "        ])"
-    return defaults
-
-
 def process_data(data, file):
     status = data.get("status", "")
     if status != "OK" and status != "PREPARATION" and status != "BROKEN":
@@ -98,8 +83,6 @@ def process_data(data, file):
             server += ("            Server { protocol: " + protocol + ", socket: " + socket + ", hostname: \""
             + hostname + "\", port: " + str(port) + ", username_pattern: " + username_pattern + " },\n")
 
-    config_defaults = process_config_defaults(data)
-
     provider = ""
     before_login_hint = cleanstr(data.get("before_login_hint", ""))
     after_login_hint = cleanstr(data.get("after_login_hint", ""))
@@ -110,7 +93,6 @@ def process_data(data, file):
         provider += "        after_login_hint: \"" + after_login_hint + "\",\n"
         provider += "        overview_page: \"" + file2url(file) + "\",\n"
         provider += "        server: vec![\n" + server + "        ],\n"
-        provider += "        config_defaults: " + config_defaults + ",\n"
         provider += "    };\n\n"
     else:
         raise TypeError("SMTP and IMAP must be specified together or left out both")
@@ -121,7 +103,7 @@ def process_data(data, file):
     # finally, add the provider
     global out_all, out_domains
     out_all += "    // " + file[file.rindex("/")+1:] + ": " + comment.strip(", ") + "\n"
-    if status == "OK" and before_login_hint == "" and after_login_hint == "" and server == "" and config_defaults == "None":
+    if status == "OK" and before_login_hint == "" and after_login_hint == "" and server == "":
         out_all += "    // - skipping provider with status OK and no special things to do\n\n"
     else:
         out_all += provider

src/qr.rs

@@ -2,20 +2,20 @@
 
 use lazy_static::lazy_static;
 use percent_encoding::percent_decode_str;
-use reqwest::Url;
-use serde::Deserialize;
 
 use crate::chat;
 use crate::config::*;
 use crate::constants::Blocked;
 use crate::contact::*;
 use crate::context::Context;
-use crate::error::{bail, ensure, format_err, Error};
+use crate::error::Error;
 use crate::key::dc_format_fingerprint;
 use crate::key::dc_normalize_fingerprint;
 use crate::lot::{Lot, LotState};
 use crate::param::*;
 use crate::peerstate::*;
+use reqwest::Url;
+use serde::Deserialize;
 
 const OPENPGP4FPR_SCHEME: &str = "OPENPGP4FPR:"; // yes: uppercase
 const DCACCOUNT_SCHEME: &str = "DCACCOUNT:";
@@ -37,10 +37,6 @@ impl Into<Lot> for Error {
     }
 }
 
-fn starts_with_ignore_case(string: &str, pattern: &str) -> bool {
-    string.to_lowercase().starts_with(&pattern.to_lowercase())
-}
-
 /// Check a scanned QR code.
 /// The function should be called after a QR code is scanned.
 /// The function takes the raw text scanned and checks what can be done with it.
@@ -49,9 +45,9 @@ pub fn check_qr(context: &Context, qr: impl AsRef<str>) -> Lot {
 
     info!(context, "Scanned QR code: {}", qr);
 
-    if starts_with_ignore_case(qr, OPENPGP4FPR_SCHEME) {
+    if qr.starts_with(OPENPGP4FPR_SCHEME) {
         decode_openpgp(context, qr)
-    } else if starts_with_ignore_case(qr, DCACCOUNT_SCHEME) {
+    } else if qr.starts_with(DCACCOUNT_SCHEME) {
         decode_account(context, qr)
     } else if qr.starts_with(MAILTO_SCHEME) {
         decode_mailto(context, qr)
@@ -225,20 +221,27 @@ pub fn set_config_from_qr(context: &Context, qr: &str) -> Result<(), Error> {
 
     let response = reqwest::blocking::Client::new().post(url_str).send();
     if response.is_err() {
-        bail!("Cannot create account, request to {} failed", url_str);
+        return Err(format_err!(
+            "Cannot create account, request to {} failed",
+            url_str
+        ));
     }
     let response = response.unwrap();
     if !response.status().is_success() {
-        bail!("Request to {} unsuccessful: {:?}", url_str, response);
+        return Err(format_err!(
+            "Request to {} unsuccessful: {:?}",
+            url_str,
+            response
+        ));
     }
 
     let parsed: reqwest::Result<CreateAccountResponse> = response.json();
     if parsed.is_err() {
-        bail!(
+        return Err(format_err!(
             "Failed to parse JSON response from {}: error: {:?}",
             url_str,
             parsed
-        );
+        ));
     }
     println!("response: {:?}", &parsed);
     let parsed = parsed.unwrap();
@@ -286,6 +289,7 @@ fn decode_smtp(context: &Context, qr: &str) -> Lot {
         Ok(addr) => addr,
         Err(err) => return err.into(),
     };
+
     let name = "".to_string();
     Lot::from_address(context, name, addr)
 }
@@ -521,17 +525,6 @@ mod tests {
         assert_ne!(res.get_id(), 0);
         assert_eq!(res.get_text1().unwrap(), "test ? test !");
 
-        // Test it again with lowercased "openpgp4fpr:" uri scheme
-        let res = check_qr(
-            &ctx.ctx,
-            "openpgp4fpr:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&g=test%20%3F+test%20%21&x=h-0oKQf2CDK&i=9JEXlxAqGM0&s=0V7LzL9cxRL"
-        );
-
-        println!("{:?}", res);
-        assert_eq!(res.get_state(), LotState::QrAskVerifyGroup);
-        assert_ne!(res.get_id(), 0);
-        assert_eq!(res.get_text1().unwrap(), "test ? test !");
-
         let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
         assert_eq!(contact.get_addr(), "cli@deltachat.de");
     }
@@ -549,16 +542,6 @@ mod tests {
         assert_eq!(res.get_state(), LotState::QrAskVerifyContact);
         assert_ne!(res.get_id(), 0);
 
-        // Test it again with lowercased "openpgp4fpr:" uri scheme
-        let res = check_qr(
-            &ctx.ctx,
-            "openpgp4fpr:79252762C34C5096AF57958F4FC3D21A81B0F0A7#a=cli%40deltachat.de&n=J%C3%B6rn%20P.+P.&i=TbnwJ6lSvD5&s=0ejvbdFSQxB"
-        );
-
-        println!("{:?}", res);
-        assert_eq!(res.get_state(), LotState::QrAskVerifyContact);
-        assert_ne!(res.get_id(), 0);
-
         let contact = Contact::get_by_id(&ctx.ctx, res.get_id()).unwrap();
         assert_eq!(contact.get_addr(), "cli@deltachat.de");
         assert_eq!(contact.get_name(), "Jörn P. P.");
@@ -579,19 +562,6 @@ mod tests {
         );
         assert_eq!(res.get_id(), 0);
 
-        // Test it again with lowercased "openpgp4fpr:" uri scheme
-
-        let res = check_qr(
-            &ctx.ctx,
-            "openpgp4fpr:1234567890123456789012345678901234567890",
-        );
-        assert_eq!(res.get_state(), LotState::QrFprWithoutAddr);
-        assert_eq!(
-            res.get_text1().unwrap(),
-            "1234 5678 9012 3456 7890\n1234 5678 9012 3456 7890"
-        );
-        assert_eq!(res.get_id(), 0);
-
         let res = check_qr(&ctx.ctx, "OPENPGP4FPR:12345678901234567890");
         assert_eq!(res.get_state(), LotState::QrError);
         assert_eq!(res.get_id(), 0);
@@ -607,14 +577,6 @@ mod tests {
         );
         assert_eq!(res.get_state(), LotState::QrAccount);
         assert_eq!(res.get_text1().unwrap(), "example.org");
-
-        // Test it again with lowercased "dcaccount:" uri scheme
-        let res = check_qr(
-            &ctx.ctx,
-            "dcaccount:https://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
-        );
-        assert_eq!(res.get_state(), LotState::QrAccount);
-        assert_eq!(res.get_text1().unwrap(), "example.org");
     }
 
     #[test]
@@ -627,13 +589,5 @@ mod tests {
         );
         assert_eq!(res.get_state(), LotState::QrError);
         assert!(res.get_text1().is_some());
-
-        // Test it again with lowercased "dcaccount:" uri scheme
-        let res = check_qr(
-            &ctx.ctx,
-            "dcaccount:http://example.org/new_email?t=1w_7wDjgjelxeX884x96v3",
-        );
-        assert_eq!(res.get_state(), LotState::QrError);
-        assert!(res.get_text1().is_some());
     }
 }

src/securejoin.rs

@@ -9,10 +9,10 @@ use crate::constants::*;
 use crate::contact::*;
 use crate::context::Context;
 use crate::e2ee::*;
-use crate::error::{bail, Error};
+use crate::error::Error;
 use crate::events::Event;
 use crate::headerdef::HeaderDef;
-use crate::key::{dc_normalize_fingerprint, DcKey, Key, SignedPublicKey};
+use crate::key::{dc_normalize_fingerprint, Key};
 use crate::lot::LotState;
 use crate::message::Message;
 use crate::mimeparser::*;
@@ -135,13 +135,12 @@ pub fn dc_get_securejoin_qr(context: &Context, group_chat_id: ChatId) -> Option<
 }
 
 fn get_self_fingerprint(context: &Context) -> Option<String> {
-    match SignedPublicKey::load_self(context) {
-        Ok(key) => Some(Key::from(key).fingerprint()),
-        Err(_) => {
-            warn!(context, "get_self_fingerprint(): failed to load key");
-            None
+    if let Some(self_addr) = context.get_config(Config::ConfiguredAddr) {
+        if let Some(key) = Key::from_self_public(context, self_addr, &context.sql) {
+            return Some(key.fingerprint());
         }
     }
+    None
 }
 
 /// Take a scanned QR-code and do the setup-contact/join-group handshake.
@@ -344,21 +343,24 @@ fn fingerprint_equals_sender(
     }
     false
 }
-#[derive(Debug, thiserror::Error)]
+#[derive(Fail, Debug)]
 pub(crate) enum HandshakeError {
-    #[error("Can not be called with special contact ID")]
+    #[fail(display = "Can not be called with special contact ID")]
     SpecialContactId,
-    #[error("Not a Secure-Join message")]
+    #[fail(display = "Not a Secure-Join message")]
     NotSecureJoinMsg,
-    #[error("Failed to look up or create chat for contact #{contact_id}")]
+    #[fail(
+        display = "Failed to look up or create chat for contact #{}",
+        contact_id
+    )]
     NoChat {
         contact_id: u32,
-        #[source]
+        #[cause]
         cause: Error,
     },
-    #[error("Chat for group {group} not found")]
+    #[fail(display = "Chat for group {} not found", group)]
     ChatNotFound { group: String },
-    #[error("No configured self address found")]
+    #[fail(display = "No configured self address found")]
     NoSelfAddr,
 }
 
@@ -391,6 +393,8 @@ pub(crate) fn handle_securejoin_handshake(
     mime_message: &MimeMessage,
     contact_id: u32,
 ) -> Result<HandshakeMessage, HandshakeError> {
+    let own_fingerprint: String;
+
     if contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
         return Err(HandshakeError::SpecialContactId);
     }
@@ -506,7 +510,7 @@ pub(crate) fn handle_securejoin_handshake(
                 return Ok(HandshakeMessage::Ignore);
             }
             info!(context, "Fingerprint verified.",);
-            let own_fingerprint = get_self_fingerprint(context).unwrap();
+            own_fingerprint = get_self_fingerprint(context).unwrap();
             joiner_progress!(context, contact_id, 400);
             context.bob.write().unwrap().expects = DC_VC_CONTACT_CONFIRM;
 
@@ -618,17 +622,10 @@ pub(crate) fn handle_securejoin_handshake(
                 }
             } else {
                 // Alice -> Bob
-                send_handshake_msg(
-                    context,
-                    contact_chat_id,
-                    "vc-contact-confirm",
-                    "",
-                    Some(fingerprint.clone()),
-                    "",
-                );
+                send_handshake_msg(context, contact_chat_id, "vc-contact-confirm", "", None, "");
                 inviter_progress!(context, contact_id, 1000);
             }
-            Ok(HandshakeMessage::Ignore) // "Done" would delete the message and break multi-device (the key from Autocrypt-header is needed)
+            Ok(HandshakeMessage::Done)
         }
         "vg-member-added" | "vc-contact-confirm" => {
             /*=======================================================
@@ -713,30 +710,26 @@ pub(crate) fn handle_securejoin_handshake(
             }
             secure_connection_established(context, contact_chat_id);
             context.bob.write().unwrap().expects = 0;
-
-            // Bob -> Alice
-            send_handshake_msg(
-                context,
-                contact_chat_id,
-                if join_vg {
-                    "vg-member-added-received"
-                } else {
-                    "vc-contact-confirm-received" // only for observe_securejoin_on_other_device()
-                },
-                "",
-                Some(scanned_fingerprint_of_alice),
-                "",
-            );
-
+            if join_vg {
+                // Bob -> Alice
+                send_handshake_msg(
+                    context,
+                    contact_chat_id,
+                    "vg-member-added-received",
+                    "",
+                    None,
+                    "",
+                );
+            }
             context.bob.write().unwrap().status = 1;
             context.stop_ongoing();
             Ok(if join_vg {
                 HandshakeMessage::Propagate
             } else {
-                HandshakeMessage::Ignore // "Done" deletes the message and breaks multi-device
+                HandshakeMessage::Done
             })
         }
-        "vg-member-added-received" | "vc-contact-confirm-received" => {
+        "vg-member-added-received" => {
             /*==========================================================
             ====              Alice - the inviter side              ====
             ====  Step 8 in "Out-of-band verified groups" protocol  ====
@@ -744,26 +737,29 @@ pub(crate) fn handle_securejoin_handshake(
 
             if let Ok(contact) = Contact::get_by_id(context, contact_id) {
                 if contact.is_verified(context) == VerifiedStatus::Unverified {
-                    warn!(context, "{} invalid.", step);
+                    warn!(context, "vg-member-added-received invalid.",);
                     return Ok(HandshakeMessage::Ignore);
                 }
-                if join_vg {
-                    inviter_progress!(context, contact_id, 800);
-                    inviter_progress!(context, contact_id, 1000);
-                    let field_grpid = mime_message
-                        .get(HeaderDef::SecureJoinGroup)
-                        .map(|s| s.as_str())
-                        .unwrap_or_else(|| "");
-                    if let Err(err) = chat::get_chat_id_by_grpid(context, &field_grpid) {
+                inviter_progress!(context, contact_id, 800);
+                inviter_progress!(context, contact_id, 1000);
+                let field_grpid = mime_message
+                    .get(HeaderDef::SecureJoinGroup)
+                    .map(|s| s.as_str())
+                    .unwrap_or_else(|| "");
+                let (group_chat_id, _, _) = chat::get_chat_id_by_grpid(context, &field_grpid)
+                    .map_err(|err| {
                         warn!(context, "Failed to lookup chat_id from grpid: {}", err);
-                        return Err(HandshakeError::ChatNotFound {
+                        HandshakeError::ChatNotFound {
                             group: field_grpid.to_string(),
-                        });
-                    }
-                }
-                Ok(HandshakeMessage::Ignore) // "Done" deletes the message and breaks multi-device
+                        }
+                    })?;
+                context.call_cb(Event::SecurejoinMemberAdded {
+                    chat_id: group_chat_id,
+                    contact_id,
+                });
+                Ok(HandshakeMessage::Done)
             } else {
-                warn!(context, "{} invalid.", step);
+                warn!(context, "vg-member-added-received invalid.",);
                 Ok(HandshakeMessage::Ignore)
             }
         }
@@ -774,98 +770,6 @@ pub(crate) fn handle_securejoin_handshake(
     }
 }
 
-/// observe_securejoin_on_other_device() must be called when a self-sent securejoin message is seen.
-///
-/// in a multi-device-setup, there may be other devices that "see" the handshake messages.
-/// if the seen messages seen are self-sent messages encrypted+signed correctly with our key,
-/// we can make some conclusions of it:
-///
-/// - if we see the self-sent-message vg-member-added/vc-contact-confirm,
-///   we know that we're an inviter-observer.
-///   the inviting device has marked a peer as verified on vg-request-with-auth/vc-request-with-auth
-///   before sending vg-member-added/vc-contact-confirm - so, if we observe vg-member-added/vc-contact-confirm,
-///   we can mark the peer as verified as well.
-///
-/// - if we see the self-sent-message vg-member-added-received
-///   we know that we're an joiner-observer.
-///   the joining device has marked the peer as verified on vg-member-added/vc-contact-confirm
-///   before sending vg-member-added-received - so, if we observe vg-member-added-received,
-///   we can mark the peer as verified as well.
-pub(crate) fn observe_securejoin_on_other_device(
-    context: &Context,
-    mime_message: &MimeMessage,
-    contact_id: u32,
-) -> Result<HandshakeMessage, HandshakeError> {
-    if contact_id <= DC_CONTACT_ID_LAST_SPECIAL {
-        return Err(HandshakeError::SpecialContactId);
-    }
-    let step = mime_message
-        .get(HeaderDef::SecureJoin)
-        .ok_or(HandshakeError::NotSecureJoinMsg)?;
-    info!(context, "observing secure-join message \'{}\'", step);
-
-    let contact_chat_id =
-        match chat::create_or_lookup_by_contact_id(context, contact_id, Blocked::Not) {
-            Ok((chat_id, blocked)) => {
-                if blocked != Blocked::Not {
-                    chat_id.unblock(context);
-                }
-                chat_id
-            }
-            Err(err) => {
-                return Err(HandshakeError::NoChat {
-                    contact_id,
-                    cause: err,
-                });
-            }
-        };
-
-    match step.as_str() {
-        "vg-member-added"
-        | "vc-contact-confirm"
-        | "vg-member-added-received"
-        | "vc-contact-confirm-received" => {
-            if !encrypted_and_signed(
-                context,
-                mime_message,
-                get_self_fingerprint(context).unwrap_or_default(),
-            ) {
-                could_not_establish_secure_connection(
-                    context,
-                    contact_chat_id,
-                    "Message not encrypted correctly.",
-                );
-                return Ok(HandshakeMessage::Ignore);
-            }
-            let fingerprint = match mime_message.get(HeaderDef::SecureJoinFingerprint) {
-                Some(fp) => fp,
-                None => {
-                    could_not_establish_secure_connection(
-                        context,
-                        contact_chat_id,
-                        "Fingerprint not provided, please update Delta Chat on all your devices.",
-                    );
-                    return Ok(HandshakeMessage::Ignore);
-                }
-            };
-            if mark_peer_as_verified(context, fingerprint).is_err() {
-                could_not_establish_secure_connection(
-                    context,
-                    contact_chat_id,
-                    format!("Fingerprint mismatch on observing {}.", step).as_ref(),
-                );
-                return Ok(HandshakeMessage::Ignore);
-            }
-            Ok(if step.as_str() == "vg-member-added" {
-                HandshakeMessage::Propagate
-            } else {
-                HandshakeMessage::Ignore
-            })
-        }
-        _ => Ok(HandshakeMessage::Ignore),
-    }
-}
-
 fn secure_connection_established(context: &Context, contact_chat_id: ChatId) {
     let contact_id: u32 = chat_id_2_contact_id(context, contact_chat_id);
     let contact = Contact::get_by_id(context, contact_id);

src/simplify.rs

@@ -1,41 +1,13 @@
-// protect lines starting with `--` against being treated as a footer.
-// for that, we insert a ZERO WIDTH SPACE (ZWSP, 0x200B);
-// this should be invisible on most systems and there is no need to unescape it again
-// (which won't be done by non-deltas anyway)
-//
-// this escapes a bit more than actually needed by delta (eg. also lines as "-- footer"),
-// but for non-delta-compatibility, that seems to be better.
-// (to be only compatible with delta, only "[\r\n|\n]-- {0,2}[\r\n|\n]" needs to be replaced)
-pub fn escape_message_footer_marks(text: &str) -> String {
-    if text.starts_with("--") {
-        "-\u{200B}-".to_string() + &text[2..].replace("\n--", "\n-\u{200B}-")
-    } else {
-        text.replace("\n--", "\n-\u{200B}-")
-    }
-}
-
 /// Remove standard (RFC 3676, §4.3) footer if it is found.
 fn remove_message_footer<'a>(lines: &'a [&str]) -> &'a [&'a str] {
-    let mut nearly_standard_footer = None;
     for (ix, &line) in lines.iter().enumerate() {
+        // quoted-printable may encode `-- ` to `-- =20` which is converted
+        // back to `--  `
         match line {
-            // some providers encode `-- ` to `-- =20` which results in `--  `
             "-- " | "--  " => return &lines[..ix],
-            // some providers encode `-- ` to `=2D-` which results in only `--`;
-            // use that only when no other footer is found
-            // and if the line before is empty and the line after is not empty
-            "--" => {
-                if (ix == 0 || lines[ix - 1] == "") && ix != lines.len() - 1 && lines[ix + 1] != ""
-                {
-                    nearly_standard_footer = Some(ix);
-                }
-            }
             _ => (),
         }
     }
-    if let Some(ix) = nearly_standard_footer {
-        return &lines[..ix];
-    }
     lines
 }
 
@@ -69,27 +41,25 @@ pub fn simplify(mut input: String, is_chat_message: bool) -> (String, bool) {
     let lines = split_lines(&input);
     let (lines, is_forwarded) = skip_forward_header(&lines);
 
-    let original_lines = &lines;
-
     let lines = remove_message_footer(lines);
-
-    let text = if is_chat_message {
-        render_message(lines, false, false)
+    let (lines, has_nonstandard_footer) = remove_nonstandard_footer(lines);
+    let (lines, has_bottom_quote) = if !is_chat_message {
+        remove_bottom_quote(lines)
     } else {
-        let (lines, has_nonstandard_footer) = remove_nonstandard_footer(lines);
-        let (lines, has_bottom_quote) = remove_bottom_quote(lines);
-        let (lines, has_top_quote) = remove_top_quote(lines);
-
-        if lines.iter().all(|it| it.trim().is_empty()) {
-            render_message(original_lines, false, false)
-        } else {
-            render_message(
-                lines,
-                has_top_quote,
-                has_nonstandard_footer || has_bottom_quote,
-            )
-        }
+        (lines, false)
     };
+    let (lines, has_top_quote) = if !is_chat_message {
+        remove_top_quote(lines)
+    } else {
+        (lines, false)
+    };
+
+    // re-create buffer from the remaining lines
+    let text = render_message(
+        lines,
+        has_top_quote,
+        has_nonstandard_footer || has_bottom_quote,
+    );
     (text, is_forwarded)
 }
 
@@ -97,13 +67,14 @@ pub fn simplify(mut input: String, is_chat_message: bool) -> (String, bool) {
 /// Returns message body lines and a boolean indicating whether
 /// a message is forwarded or not.
 fn skip_forward_header<'a>(lines: &'a [&str]) -> (&'a [&'a str], bool) {
-    match lines {
-        ["---------- Forwarded message ----------", first_line, "", rest @ ..]
-            if first_line.starts_with("From: ") =>
-        {
-            (rest, true)
-        }
-        _ => (lines, false),
+    if lines.len() >= 3
+        && lines[0] == "---------- Forwarded message ----------"
+        && lines[1].starts_with("From: ")
+        && lines[2].is_empty()
+    {
+        (&lines[3..], true)
+    } else {
+        (lines, false)
     }
 }
 
@@ -184,8 +155,7 @@ fn render_message(lines: &[&str], is_cut_at_begin: bool, is_cut_at_end: bool) ->
     if is_cut_at_end && (!is_cut_at_begin || !empty_body) {
         ret += " [...]";
     }
-    // redo escaping done by escape_message_footer_marks()
-    ret.replace("\u{200B}", "")
+    ret
 }
 
 /**
@@ -234,25 +204,6 @@ mod tests {
         }
     }
 
-    #[test]
-    fn test_dont_remove_whole_message() {
-        let input = "\n------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text".to_string();
-        let (plain, is_forwarded) = simplify(input, false);
-        assert_eq!(
-            plain,
-            "------\nFailed\n------\n\nUh-oh, this workflow did not succeed!\n\nlots of other text"
-        );
-        assert!(!is_forwarded);
-    }
-
-    #[test]
-    fn test_chat_message() {
-        let input = "Hi! How are you?\n\n---\n\nI am good.\n-- \nSent with my Delta Chat Messenger: https://delta.chat".to_string();
-        let (plain, is_forwarded) = simplify(input, true);
-        assert_eq!(plain, "Hi! How are you?\n\n---\n\nI am good.");
-        assert!(!is_forwarded);
-    }
-
     #[test]
     fn test_simplify_trim() {
         let input = "line1\n\r\r\rline2".to_string();
@@ -297,46 +248,4 @@ mod tests {
         assert_eq!(lines, &["not a quote", "> first", "> second"]);
         assert!(!has_top_quote);
     }
-
-    #[test]
-    fn test_escape_message_footer_marks() {
-        let esc = escape_message_footer_marks("--\n--text --in line");
-        assert_eq!(esc, "-\u{200B}-\n-\u{200B}-text --in line");
-
-        let esc = escape_message_footer_marks("--\r\n--text");
-        assert_eq!(esc, "-\u{200B}-\r\n-\u{200B}-text");
-    }
-
-    #[test]
-    fn test_remove_message_footer() {
-        let input = "text\n--\nno footer".to_string();
-        let (plain, _) = simplify(input, true);
-        assert_eq!(plain, "text\n--\nno footer");
-
-        let input = "text\n\n--\n\nno footer".to_string();
-        let (plain, _) = simplify(input, true);
-        assert_eq!(plain, "text\n\n--\n\nno footer");
-
-        let input = "text\n\n-- no footer\n\n".to_string();
-        let (plain, _) = simplify(input, true);
-        assert_eq!(plain, "text\n\n-- no footer");
-
-        let input = "text\n\n--\nno footer\n-- \nfooter".to_string();
-        let (plain, _) = simplify(input, true);
-        assert_eq!(plain, "text\n\n--\nno footer");
-
-        let input = "text\n\n--\ntreated as footer when unescaped".to_string();
-        let (plain, _) = simplify(input.clone(), true);
-        assert_eq!(plain, "text"); // see remove_message_footer() for some explanations
-        let escaped = escape_message_footer_marks(&input);
-        let (plain, _) = simplify(escaped, true);
-        assert_eq!(plain, "text\n\n--\ntreated as footer when unescaped");
-
-        let input = "--\ntreated as footer when unescaped".to_string();
-        let (plain, _) = simplify(input.clone(), true);
-        assert_eq!(plain, ""); // see remove_message_footer() for some explanations
-        let escaped = escape_message_footer_marks(&input);
-        let (plain, _) = simplify(escaped, true);
-        assert_eq!(plain, "--\ntreated as footer when unescaped");
-    }
 }