method to get busy webxdcs

Don't accept wexdc updates from members not in group

.github/mergeable.yml

@@ -16,7 +16,7 @@ mergeable:
                 required: ['CHANGELOG.md']
             - do: dependent
               changed:
-                file: 'deltachat-ffi/**'
+                file: 'deltachat-ffi/src/**'
                 required: ['CHANGELOG.md']
     fail:
       - do: checks

.github/workflows/ci.yml

@@ -45,7 +45,7 @@ jobs:
       - uses: actions-rs/clippy-check@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          args: --workspace --tests --examples --benches
+          args: --workspace --tests --examples --benches --features repl -- -D warnings
 
   docs:
     name: Rust doc comments
@@ -116,6 +116,11 @@ jobs:
         command: test
         args: --all
 
+    - name: test cargo vendor
+      uses: actions-rs/cargo@v1
+      with:
+        command: vendor
+
     - name: install python
       if: ${{ matrix.python }}
       uses: actions/setup-python@v4
@@ -131,7 +136,7 @@ jobs:
       uses: actions-rs/cargo@v1
       with:
         command: build
-        args: -p deltachat_ffi
+        args: -p deltachat_ffi --features jsonrpc
 
     - name: run python tests
       if: ${{ matrix.python }}

.github/workflows/jsonrpc.yml

@@ -0,0 +1,45 @@
+name: JSON-RPC API Test
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build_and_test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js 16.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 16.x
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - name: Add Rust cache
+        uses: Swatinem/rust-cache@v1.3.0
+      - name: npm install
+        run: |
+          cd deltachat-jsonrpc/typescript
+          npm install
+      - name: Build TypeScript, run Rust tests, generate bindings
+        run: |
+          cd deltachat-jsonrpc/typescript
+          npm run build
+      - name: Run integration tests
+        run: |
+          cd deltachat-jsonrpc/typescript
+          npm run test
+        env:
+          DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+      - name: Run linter
+        run: |
+          cd deltachat-jsonrpc/typescript
+          npm run prettier:check

.npmignore

@@ -40,3 +40,17 @@ node/old_docs.md
 .vscode/
 .github/
 node/.prettierrc.yml
+
+deltachat-jsonrpc/TODO.md
+deltachat-jsonrpc/README.MD
+deltachat-jsonrpc/.gitignore
+deltachat-jsonrpc/typescript/.gitignore
+deltachat-jsonrpc/typescript/.prettierignore
+deltachat-jsonrpc/typescript/accounts/
+deltachat-jsonrpc/typescript/index.html
+deltachat-jsonrpc/typescript/node-demo.js
+deltachat-jsonrpc/typescript/report_api_coverage.mjs
+deltachat-jsonrpc/typescript/test
+deltachat-jsonrpc/typescript/example.ts
+
+.DS_Store

CHANGELOG.md

@@ -2,9 +2,114 @@
 
 ## Unreleased
 
+### API-Changes
+- breaking change: replace `dc_accounts_event_emitter_t` with `dc_event_emitter_t` #3422
+
+  Type `dc_accounts_event_emitter_t` is removed.
+  `dc_accounts_get_event_emitter()` returns `dc_event_emitter_t` now, so
+  `dc_get_next_event()` should be used instead of `dc_accounts_get_next_event`
+  and `dc_event_emitter_unref()` should be used instead of
+  `dc_accounts_event_emitter_unref`.
+- add `dc_contact_was_seen_recently()` #3560
+- jsonrpc: add functions: #3586, #3587
+  - `deleteChat()`
+  - `getChatEncryptionInfo()`
+  - `getChatSecurejoinQrCodeSvg()`
+  - `leaveGroup()`
+  - `removeContactFromChat()`
+  - `addContactToChat()`
+  - `deleteMessages()`
+  - `getMessageInfo()`
+- jsonrpc: add `is_broadcast` property to `ChatListItemFetchResult` #3584
+- jsonrpc: add `was_seen_recently` property to `ChatListItemFetchResult`, `FullChat` and `Contact` #3584
+- jsonrpc: add `webxdc_info` property to `Message` #3588
+
 ### Changes
+- order contact lists by "last seen";
+  this affects `dc_get_chat_contacts()`, `dc_get_contacts()` and `dc_get_blocked_contacts()` #3562
+- add `internet_access` flag to `dc_msg_get_webxdc_info()` #3516
+- `DC_EVENT_WEBXDC_INSTANCE_DELETED` is emitted when a message containing a webxdc gets deleted #3105
+- `DC_EVENT_WEBXDC_BUSY_UPDATING` is emitted when a new update has to be sent by an webxdc #3320
+- `DC_EVENT_WEBXDC_UP_TO_DATE` is emitted when a webxdc has sent all updates #3320
 
 ### Fixes
+- do not emit notifications for blocked chats #3557
+- Show attached .eml files correctly #3561
+- Auto accept contact requests if `Config::Bot` is set for a client #3567 
+- Don't prepend the subject to chat messages in mailinglists
+- fix `set_core_version.py` script to also update version in `deltachat-jsonrpc/typescript/package.json` #3585
+- Reject webxcd-updates from contacts who are not group members #3568
+
+
+## 1.93.0
+
+### API-Changes
+- added a JSON RPC API, accessible through a WebSocket server, the CFFI bindings and the Node.js bindings #3463 #3554 #3542
+- JSON RPC methods in CFFI #3463:
+ - `dc_jsonrpc_instance_t* dc_jsonrpc_init(dc_accounts_t* account_manager);`
+ - `void dc_jsonrpc_unref(dc_jsonrpc_instance_t* jsonrpc_instance);`
+ - `void dc_jsonrpc_request(dc_jsonrpc_instance_t* jsonrpc_instance, char* request);`
+ - `char* dc_jsonrpc_next_response(dc_jsonrpc_instance_t* jsonrpc_instance);`
+- node: JSON RPC methods #3463:
+ - `AccountManager.prototype.startJsonRpcHandler(callback: ((response: string) => void)): void`
+ - `AccountManager.prototype.jsonRpcRequest(message: string): void`
+
+### Changes
+- use [pathlib](https://docs.python.org/3/library/pathlib.html) in provider update script #3543
+- `dc_get_chat_media()` can return media globally #3528
+- node: add `getMailinglistAddr()` #3524
+- avoid duplicate encoded-words package and test `cargo vendor` in ci #3549
+- python: don't raise an error if addr changes #3530
+- improve coverage script #3530
+
+### Fixes
+- improved error handling for account setup from qrcode #3474
+- python: enable certificate checks in cloned accounts #3443
+
+
+## 1.92.0
+
+### API-Changes
+- add `dc_chat_get_mailinglist_addr()` #3520
+
+
+## 1.91.0
+
+### Added
+- python bindings: extra method to get an account running
+
+### Changes
+- refactorings #3437
+
+### Fixes
+- mark "group image changed" as system message on receiver side #3517
+
+
+## 1.90.0
+
+### Changes
+- handle drafts from mailto links in scanned QR #3492
+- do not overflow ratelimiter leaky bucket #3496
+- (AEAP) Add device message after you changed your address #3505
+- (AEAP) Revert #3491, instead only replace contacts in verified groups #3510
+- improve python bindings and tests #3502 #3503
+
+### Fixes
+- don't squash text parts of NDN into attachments #3497
+- do not treat non-failed DSNs as NDNs #3506
+
+
+## 1.89.0
+
+### Changes
+
+- (AEAP) When one of your contacts changed their address, they are
+  only replaced in the chat where you got a message from them
+  for now #3491
+
+### Fixes
+- replace musl libc name resolution errors with a better message #3485
+- handle updates for not yet downloaded webxdc instances #3487
 
 
 ## 1.88.0

CMakeLists.txt

@@ -21,7 +21,7 @@ add_custom_command(
         PREFIX=${CMAKE_INSTALL_PREFIX}
         LIBDIR=${CMAKE_INSTALL_FULL_LIBDIR}
         INCLUDEDIR=${CMAKE_INSTALL_FULL_INCLUDEDIR}
-        ${CARGO} build --release --no-default-features
+        ${CARGO} build --release --no-default-features --features jsonrpc
 
 	# Build in `deltachat-ffi` directory instead of using
 	# `--package deltachat_ffi` to avoid feature resolver version

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.88.0"
+version = "1.93.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2021"
 license = "MPL-2.0"
@@ -32,11 +32,11 @@ bitflags = "1.3"
 chrono = { version = "0.4", default-features=false, features = ["clock", "std"] }
 dirs = { version = "4", optional=true }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
-encoded-words = "0.2"
+encoded-words = { git = "https://github.com/async-email/encoded-words", branch = "master" }
 escaper = "0.1"
 futures = "0.3"
 hex = "0.4.0"
-image = { version = "0.24.1", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
+image = { version = "0.24.3", default-features=false, features = ["gif", "jpeg", "ico", "png", "pnm", "webp", "bmp"] }
 kamadak-exif = "0.5"
 lettre_email = { git = "https://github.com/deltachat/lettre", branch = "master" }
 libc = "0.2"
@@ -46,7 +46,7 @@ native-tls = "0.2"
 num_cpus = "1.13"
 num-derive = "0.3"
 num-traits = "0.2"
-once_cell = "1.12.0"
+once_cell = "1.13.1"
 percent-encoding = "2.0"
 pgp = { version = "0.8", default-features = false }
 pretty_env_logger = { version = "0.4", optional = true }
@@ -54,10 +54,10 @@ quick-xml = "0.23"
 r2d2 = "0.8"
 r2d2_sqlite = "0.20"
 rand = "0.8"
-regex = "1.5"
+regex = "1.6"
 rusqlite = { version = "0.27", features = ["sqlcipher"] }
 rust-hsluv = "0.1"
-rustyline = { version = "9", optional = true }
+rustyline = { version = "10", optional = true }
 sanitize-filename = "0.4"
 serde_json = "1.0"
 serde = { version = "1.0", features = ["derive"] }
@@ -83,7 +83,7 @@ async_zip = { git = "https://github.com/dignifiedquire/rs-async-zip", branch = "
 
 [dev-dependencies]
 ansi_term = "0.12.0"
-criterion = { version = "0.3.4", features = ["async_tokio"] }
+criterion = { version = "0.3.6", features = ["async_tokio"] }
 futures-lite = "1.12"
 log = "0.4"
 pretty_env_logger = "0.4"
@@ -95,6 +95,7 @@ tokio = { version = "1", features = ["parking_lot", "rt-multi-thread", "macros"]
 members = [
   "deltachat-ffi",
   "deltachat_derive",
+  "deltachat-jsonrpc"
 ]
 
 [[example]]

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.88.0"
+version = "1.93.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
@@ -16,6 +16,7 @@ crate-type = ["cdylib", "staticlib"]
 
 [dependencies]
 deltachat = { path = "../", default-features = false }
+deltachat-jsonrpc = { path = "../deltachat-jsonrpc", optional = true }
 libc = "0.2"
 human-panic = "1"
 num-traits = "0.2"
@@ -24,10 +25,11 @@ tokio = { version = "1", features = ["rt-multi-thread"] }
 anyhow = "1"
 thiserror = "1"
 rand = "0.7"
-once_cell = "1.12.0"
+once_cell = "1.13.1"
 
 [features]
 default = ["vendored"]
 vendored = ["deltachat/vendored"]
 nightly = ["deltachat/nightly"]
+jsonrpc = ["deltachat-jsonrpc"]
 

deltachat-ffi/deltachat.h

@@ -22,8 +22,10 @@ typedef struct _dc_lot      dc_lot_t;
 typedef struct _dc_provider dc_provider_t;
 typedef struct _dc_event    dc_event_t;
 typedef struct _dc_event_emitter dc_event_emitter_t;
-typedef struct _dc_accounts_event_emitter dc_accounts_event_emitter_t;
+typedef struct _dc_jsonrpc_instance dc_jsonrpc_instance_t;
 
+// Alias for backwards compatibility, use dc_event_emitter_t instead.
+typedef struct _dc_event_emitter dc_accounts_event_emitter_t;
 
 /**
  * @mainpage Getting started
@@ -391,7 +393,8 @@ char*           dc_get_blobdir               (const dc_context_t* context);
  *                    If no type is prefixed, the videochat is handled completely in a browser.
  * - `bot`          = Set to "1" if this is a bot.
  *                    Prevents adding the "Device messages" and "Saved messages" chats,
- *                    adds Auto-Submitted header to outgoing messages.
+ *                    adds Auto-Submitted header to outgoing messages
+ *                    and accepts contact requests automatically (calling dc_accept_chat() is not needed for bots).
  * - `fetch_existing_msgs` = 1=fetch most recent existing messages on configure (default),
  *                    0=do not fetch existing messages on configure.
  *                    In both cases, existing recipients are added to the contact database.
@@ -1263,7 +1266,7 @@ void            dc_marknoticed_chat          (dc_context_t* context, uint32_t ch
 
 
 /**
- * Returns all message IDs of the given types in a chat.
+ * Returns all message IDs of the given types in a given chat or any chat.
  * Typically used to show a gallery.
  * The result must be dc_array_unref()'d
  *
@@ -1273,7 +1276,8 @@ void            dc_marknoticed_chat          (dc_context_t* context, uint32_t ch
  *
  * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
- * @param chat_id The chat ID to get all messages with media from.
+ * @param chat_id >0: get messages with media from this chat ID.
+ *    0: get messages with media from any chat of the currently used account.
  * @param msg_type Specify a message type to query here, one of the @ref DC_MSG constants.
  * @param msg_type2 Alternative message type to search for. 0 to skip.
  * @param msg_type3 Alternative message type to search for. 0 to skip.
@@ -1284,7 +1288,6 @@ dc_array_t*     dc_get_chat_media            (dc_context_t* context, uint32_t ch
 
 /**
  * Search next/previous message based on a given message and a list of types.
- * The
  * Typically used to implement the "next" and "previous" buttons
  * in a gallery or in a media player.
  *
@@ -1385,6 +1388,9 @@ void            dc_block_chat                (dc_context_t* context, uint32_t ch
  *
  * Use it to accept "contact request" chats as indicated by dc_chat_is_contact_request().
  *
+ * If the dc_set_config()-option `bot` is set,
+ * all chats are accepted automatically and calling this function has no effect.
+ *
  * @memberof dc_context_t
  * @param context The context object as returned from dc_context_new().
  * @param chat_id The ID of the chat to accept.
@@ -2292,7 +2298,7 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
  * - DC_QR_FPR_MISMATCH with dc_lot_t::id=Contact ID:
  *   scanned fingerprint does not match last seen fingerprint.
  *
- * - DC_QR_FPR_WITHOUT_ADDR with dc_lot_t::test1=Formatted fingerprint
+ * - DC_QR_FPR_WITHOUT_ADDR with dc_lot_t::text1=Formatted fingerprint
  *   the scanned QR code contains a fingerprint but no e-mail address;
  *   suggest the user to establish an encrypted connection first.
  *
@@ -2305,7 +2311,8 @@ void            dc_stop_ongoing_process      (dc_context_t* context);
  *   if so, call dc_set_config_from_qr().
  *
  * - DC_QR_ADDR with dc_lot_t::id=Contact ID:
- *   e-mail address scanned,
+ *   e-mail address scanned, optionally, a draft message could be set in
+ *   dc_lot_t::text1 in which case dc_lot_t::text1_meaning will be DC_TEXT1_DRAFT;
  *   ask the user if they want to start chatting;
  *   if so, call dc_create_chat_by_contact_id().
  *
@@ -2509,9 +2516,9 @@ int         dc_set_location                 (dc_context_t* context, double latit
  *     Must be given in number of seconds since 00:00 hours, Jan 1, 1970 UTC.
  *     0 for "all up to now".
  * @return An array of locations, NULL is never returned.
- *     The array is sorted decending;
+ *     The array is sorted descending;
  *     the first entry in the array is the location with the newest timestamp.
- *     Note that this is only realated to the recent postion of the user
+ *     Note that this is only related to the recent position of the user
  *     if dc_array_is_independent() returns 0.
  *     The returned array must be freed using dc_array_unref().
  *
@@ -2835,7 +2842,7 @@ void           dc_accounts_maybe_network_lost    (dc_accounts_t* accounts);
  *
  * The library will emit various @ref DC_EVENT events as "new message", "message read" etc.
  * To get these events, you have to create an event emitter using this function
- * and call dc_accounts_get_next_event() on the emitter.
+ * and call dc_get_next_event() on the emitter.
  *
  * This is similar to dc_get_event_emitter(), which, however,
  * must not be called for accounts handled by the account manager.
@@ -2843,13 +2850,13 @@ void           dc_accounts_maybe_network_lost    (dc_accounts_t* accounts);
  * @memberof dc_accounts_t
  * @param accounts The account manager as created by dc_accounts_new().
  * @return Returns the event emitter, NULL on errors.
- *     Must be freed using dc_accounts_event_emitter_unref() after usage.
+ *     Must be freed using dc_event_emitter_unref() after usage.
  *
  * Note: Use only one event emitter per account manager.
  * Having more than one event emitter running at the same time on the same account manager
  * will result in events randomly delivered to the one or to the other.
  */
-dc_accounts_event_emitter_t* dc_accounts_get_event_emitter (dc_accounts_t* accounts);
+dc_event_emitter_t* dc_accounts_get_event_emitter (dc_accounts_t* accounts);
 
 
 /**
@@ -3256,6 +3263,19 @@ uint32_t        dc_chat_get_id               (const dc_chat_t* chat);
 int             dc_chat_get_type             (const dc_chat_t* chat);
 
 
+/**
+ * Returns the address where messages are sent to if the chat is a mailing list.
+ * If you just want to know if a mailing list can be written to,
+ * use dc_chat_can_send() instead.
+ *
+ * @memberof dc_chat_t
+ * @param chat The chat object.
+ * @return The mailing list address. Must be released using dc_str_unref() after usage.
+ *     If there is no such address, an empty string is returned, NULL is never returned.
+ */
+char*           dc_chat_get_mailinglist_addr (const dc_chat_t* chat);
+
+
 /**
  * Get name of a chat. For one-to-one chats, this is the name of the contact.
  * For group chats, this is the name given e.g. to dc_create_group_chat() or
@@ -3743,6 +3763,11 @@ char*             dc_msg_get_webxdc_blob      (const dc_msg_t* msg, const char*
  *   URL where the source code of the Webxdc and other information can be found;
  *   defaults to an empty string.
  *   Implementations may offer an menu or a button to open this URL.
+ * - internet_access:
+ *   true if the Webxdc should get full internet access, including Webrtc.
+ *   currently, this is only true for encrypted Webxdc's in the self chat
+ *   that have requested internet access in the manifest.
+ *   this is useful for development and maybe for internal integrations at some point.
  *
  * @memberof dc_msg_t
  * @param msg The webxdc instance.
@@ -4624,6 +4649,22 @@ char*           dc_contact_get_status        (const dc_contact_t* contact);
  */
 int64_t         dc_contact_get_last_seen     (const dc_contact_t* contact);
 
+
+/**
+ * Check if the contact was seen recently.
+ *
+ * The UI may highlight these contacts,
+ * eg. draw a little green dot on the avatars of the users recently seen.
+ * DC_CONTACT_ID_SELF and other special contact IDs are defined as never seen recently (they should not get a dot).
+ * To get the time a contact was seen, use dc_contact_get_last_seen().
+ *
+ * @memberof dc_contact_t
+ * @param contact The contact object.
+ * @return 1=contact seen recently, 0=contact not seen recently.
+ */
+int             dc_contact_was_seen_recently (const dc_contact_t* contact);
+
+
 /**
  * Check if a contact is blocked.
  *
@@ -5178,13 +5219,61 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  */
 
 
+
+/**
+ * @class dc_jsonrpc_instance_t
+ *
+ * Opaque object for using the json rpc api from the cffi bindings.
+ */
+
+/**
+ * Create the jsonrpc instance that is used to call the jsonrpc.
+ *
+ * @memberof dc_accounts_t
+ * @param account_manager The accounts object as created by dc_accounts_new().
+ * @return Returns the jsonrpc instance, NULL on errors.
+ *     Must be freed using dc_jsonrpc_unref() after usage.
+ *
+ */
+dc_jsonrpc_instance_t* dc_jsonrpc_init(dc_accounts_t* account_manager);
+
+/**
+ * Free a jsonrpc instance.
+ *
+ * @memberof dc_jsonrpc_instance_t
+ * @param jsonrpc_instance jsonrpc instance as returned from dc_jsonrpc_init().
+ *     If NULL is given, nothing is done and an error is logged.
+ */
+void dc_jsonrpc_unref(dc_jsonrpc_instance_t* jsonrpc_instance);
+
+/**
+ * Makes an asynchronous jsonrpc request,
+ * returns immediately and once the result is ready it can be retrieved via dc_jsonrpc_next_response()
+ * the jsonrpc specification defines an invocation id that can then be used to match request and response.
+ *
+ * @memberof dc_jsonrpc_instance_t
+ * @param jsonrpc_instance jsonrpc instance as returned from dc_jsonrpc_init().
+ * @param request JSON-RPC request as string
+ */
+void dc_jsonrpc_request(dc_jsonrpc_instance_t* jsonrpc_instance, const char* request);
+
+/**
+ * Get the next json_rpc response, blocks until there is a new event, so call this in a loop from a thread.
+ *
+ * @memberof dc_jsonrpc_instance_t
+ * @param jsonrpc_instance jsonrpc instance as returned from dc_jsonrpc_init().
+ * @return JSON-RPC response as string, must be freed using dc_str_unref() after usage.
+ *     If NULL is returned, the accounts_t belonging to the jsonrpc instance is unref'd and no more events will come;
+ *     in this case, free the jsonrpc instance using dc_jsonrpc_unref().
+ */
+char* dc_jsonrpc_next_response(dc_jsonrpc_instance_t* jsonrpc_instance);
+
 /**
  * @class dc_event_emitter_t
  *
  * Opaque object that is used to get events from a single context.
- * You can get an event emitter from a context using dc_get_event_emitter().
- * If you are using the dc_accounts_t account manager,
- * dc_accounts_event_emitter_t must be used instead.
+ * You can get an event emitter from a context using dc_get_event_emitter()
+ * or dc_accounts_get_event_emitter().
  */
 
 /**
@@ -5200,6 +5289,8 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  */
 dc_event_t* dc_get_next_event(dc_event_emitter_t* emitter);
 
+// Alias for backwards compatibility, use dc_get_next_event instead.
+#define dc_accounts_get_next_event dc_get_next_event
 
 /**
  * Free a context event emitter object.
@@ -5210,39 +5301,8 @@ dc_event_t* dc_get_next_event(dc_event_emitter_t* emitter);
  */
 void  dc_event_emitter_unref(dc_event_emitter_t* emitter);
 
-
-/**
- * @class dc_accounts_event_emitter_t
- *
- * Opaque object that is used to get events from the dc_accounts_t account manager.
- * You get an event emitter from the account manager using dc_accounts_get_event_emitter().
- * If you are not using the dc_accounts_t account manager but just a single dc_context_t object,
- * dc_event_emitter_t must be used instead.
- */
-
-/**
- * Get the next event from an accounts event emitter object.
- *
- * @memberof dc_accounts_event_emitter_t
- * @param emitter Event emitter object as returned from dc_accounts_get_event_emitter().
- * @return An event as an dc_event_t object.
- *     You can query the event for information using dc_event_get_id(), dc_event_get_data1_int() and so on;
- *     if you are done with the event, you have to free the event using dc_event_unref().
- *     If NULL is returned, the contexts belonging to the event emitter are unref'd and no more events will come;
- *     in this case, free the event emitter using dc_accounts_event_emitter_unref().
- */
-dc_event_t* dc_accounts_get_next_event (dc_accounts_event_emitter_t* emitter);
-
-
-/**
- * Free an accounts event emitter object.
- *
- * @memberof dc_accounts_event_emitter_t
- * @param emitter Event emitter object as returned from dc_accounts_get_event_emitter().
- *     If NULL is given, nothing is done and an error is logged.
- */
-void dc_accounts_event_emitter_unref(dc_accounts_event_emitter_t* emitter);
-
+// Alias for backwards compatibility, use dc_event_emtitter_unref instead.
+#define dc_accounts_event_emitter_unref dc_event_emitter_unref
 
 /**
  * @class dc_event_t
@@ -5314,7 +5374,7 @@ char* dc_event_get_data2_str(dc_event_t* event);
  * To get the context object belonging to the event, use dc_accounts_get_account().
  *
  * @memberof dc_event_t
- * @param event The event object as returned from dc_accounts_get_next_event().
+ * @param event The event object as returned from dc_get_next_event().
  * @return The account ID belonging to the event, 0 for account manager errors.
  */
 uint32_t dc_event_get_account_id(dc_event_t* event);
@@ -5664,8 +5724,30 @@ void dc_event_unref(dc_event_t* event);
  * @param data1 (int) msg_id
  * @param data2 (int) status_update_serial - must not be used by UI implementations.
  */
-#define DC_EVENT_WEBXDC_STATUS_UPDATE                2120
+#define DC_EVENT_WEBXDC_STATUS_UPDATE             2120
 
+/**
+ * Message deleted which contained a webxdc instance.
+ *
+ * @param data1 (int) msg_id
+ */
+#define DC_EVENT_WEBXDC_INSTANCE_DELETED          2121
+
+
+/**
+ * Webxdc has some updates that need to be sent
+ *
+ * @param data1 (int) msg_id
+ */
+#define DC_EVENT_WEBXDC_BUSY_UPDATING             2122
+
+
+/**
+ * Webxdc has finished sending updates
+ *
+ * @param data1 (int) msg_id
+ */
+#define DC_EVENT_WEBXDC_UP_TO_DATE                2123
 
 /**
  * @}
@@ -6364,6 +6446,24 @@ void dc_event_unref(dc_event_t* event);
 /// Used as status in the connectivity view.
 #define DC_STR_NOT_CONNECTED              121
 
+/// %1$s changed their address from %2$s to %3$s"
+///
+/// Used as an info message to chats with contacts that changed their address.
+#define DC_STR_AEAP_ADDR_CHANGED          122
+
+/// "You changed your email address from %1$s to %2$s.
+/// If you now send a message to a group, contacts there will automatically
+/// replace the old with your new address.\n\nIt's highly advised to set up 
+/// your old email provider to forward all emails to your new email address. 
+/// Otherwise you might miss messages of contacts who did not get your new 
+/// address yet." + the link to the AEAP blog post
+/// 
+/// As soon as there is a post about AEAP, the UIs should add it:
+/// set_stock_translation(123, getString(aeap_explanation) + "\n\n" + AEAP_BLOG_LINK)
+///
+/// Used in a device message that explains AEAP.
+#define DC_STR_AEAP_EXPLANATION_AND_LINK  123
+
 /**
  * @}
  */

deltachat-ffi/src/lib.rs

@@ -19,6 +19,7 @@ use std::future::Future;
 use std::ops::Deref;
 use std::ptr;
 use std::str::FromStr;
+use std::sync::Arc;
 use std::time::{Duration, SystemTime};
 
 use anyhow::Context as _;
@@ -503,6 +504,9 @@ pub unsafe extern "C" fn dc_event_get_id(event: *mut dc_event_t) -> libc::c_int
         EventType::ConnectivityChanged => 2100,
         EventType::SelfavatarChanged => 2110,
         EventType::WebxdcStatusUpdate { .. } => 2120,
+        EventType::WebxdcInstanceDeleted { .. } => 2121,
+        EventType::WebxdcBusyUpdating { .. } => 2022,
+        EventType::WebxdcUpToDate { .. } => 2023,
     }
 }
 
@@ -549,6 +553,9 @@ pub unsafe extern "C" fn dc_event_get_data1_int(event: *mut dc_event_t) -> libc:
             contact_id.to_u32() as libc::c_int
         }
         EventType::WebxdcStatusUpdate { msg_id, .. } => msg_id.to_u32() as libc::c_int,
+        EventType::WebxdcInstanceDeleted { msg_id, .. } => msg_id.to_u32() as libc::c_int,
+        EventType::WebxdcBusyUpdating { msg_id } => msg_id.to_u32() as libc::c_int,
+        EventType::WebxdcUpToDate { msg_id } => msg_id.to_u32() as libc::c_int,
     }
 }
 
@@ -580,6 +587,9 @@ pub unsafe extern "C" fn dc_event_get_data2_int(event: *mut dc_event_t) -> libc:
         | EventType::ImexFileWritten(_)
         | EventType::MsgsNoticed(_)
         | EventType::ConnectivityChanged
+        | EventType::WebxdcInstanceDeleted { .. }
+        | EventType::WebxdcBusyUpdating { .. }
+        | EventType::WebxdcUpToDate { .. }
         | EventType::SelfavatarChanged => 0,
         EventType::ChatModified(_) => 0,
         EventType::MsgsChanged { msg_id, .. }
@@ -636,6 +646,9 @@ pub unsafe extern "C" fn dc_event_get_data2_str(event: *mut dc_event_t) -> *mut
         | EventType::ConnectivityChanged
         | EventType::SelfavatarChanged
         | EventType::WebxdcStatusUpdate { .. }
+        | EventType::WebxdcInstanceDeleted { .. }
+        | EventType::WebxdcBusyUpdating { .. }
+        | EventType::WebxdcUpToDate { .. }
         | EventType::ChatEphemeralTimerModified { .. } => ptr::null_mut(),
         EventType::ConfigureProgress { comment, .. } => {
             if let Some(comment) = comment {
@@ -1207,6 +1220,11 @@ pub unsafe extern "C" fn dc_get_chat_media(
         return ptr::null_mut();
     }
     let ctx = &*context;
+    let chat_id = if chat_id == 0 {
+        None
+    } else {
+        Some(ChatId::new(chat_id))
+    };
     let msg_type = from_prim(msg_type).expect(&format!("invalid msg_type = {}", msg_type));
     let or_msg_type2 =
         from_prim(or_msg_type2).expect(&format!("incorrect or_msg_type2 = {}", or_msg_type2));
@@ -1215,16 +1233,10 @@ pub unsafe extern "C" fn dc_get_chat_media(
 
     block_on(async move {
         Box::into_raw(Box::new(
-            chat::get_chat_media(
-                ctx,
-                ChatId::new(chat_id),
-                msg_type,
-                or_msg_type2,
-                or_msg_type3,
-            )
-            .await
-            .unwrap_or_log_default(ctx, "Failed get_chat_media")
-            .into(),
+            chat::get_chat_media(ctx, chat_id, msg_type, or_msg_type2, or_msg_type3)
+                .await
+                .unwrap_or_log_default(ctx, "Failed get_chat_media")
+                .into(),
         ))
     })
 }
@@ -2781,6 +2793,16 @@ pub unsafe extern "C" fn dc_chat_get_name(chat: *mut dc_chat_t) -> *mut libc::c_
     ffi_chat.chat.get_name().strdup()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_chat_get_mailinglist_addr(chat: *mut dc_chat_t) -> *mut libc::c_char {
+    if chat.is_null() {
+        eprintln!("ignoring careless call to dc_chat_get_mailinglist_addr()");
+        return "".strdup();
+    }
+    let ffi_chat = &*chat;
+    ffi_chat.chat.get_mailinglist_addr().strdup()
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_chat_get_profile_image(chat: *mut dc_chat_t) -> *mut libc::c_char {
     if chat.is_null() {
@@ -3849,6 +3871,16 @@ pub unsafe extern "C" fn dc_contact_get_last_seen(contact: *mut dc_contact_t) ->
     ffi_contact.contact.last_seen()
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn dc_contact_was_seen_recently(contact: *mut dc_contact_t) -> libc::c_int {
+    if contact.is_null() {
+        eprintln!("ignoring careless call to dc_contact_was_seen_recently()");
+        return 0;
+    }
+    let ffi_contact = &*contact;
+    ffi_contact.contact.was_seen_recently() as libc::c_int
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn dc_contact_is_blocked(contact: *mut dc_contact_t) -> libc::c_int {
     if contact.is_null() {
@@ -4108,11 +4140,11 @@ pub unsafe extern "C" fn dc_provider_unref(provider: *mut dc_provider_t) {
 /// Reader-writer lock wrapper for accounts manager to guarantee thread safety when using
 /// `dc_accounts_t` in multiple threads at once.
 pub struct AccountsWrapper {
-    inner: RwLock<Accounts>,
+    inner: Arc<RwLock<Accounts>>,
 }
 
 impl Deref for AccountsWrapper {
-    type Target = RwLock<Accounts>;
+    type Target = Arc<RwLock<Accounts>>;
 
     fn deref(&self) -> &Self::Target {
         &self.inner
@@ -4121,7 +4153,7 @@ impl Deref for AccountsWrapper {
 
 impl AccountsWrapper {
     fn new(accounts: Accounts) -> Self {
-        let inner = RwLock::new(accounts);
+        let inner = Arc::new(RwLock::new(accounts));
         Self { inner }
     }
 }
@@ -4396,12 +4428,10 @@ pub unsafe extern "C" fn dc_accounts_maybe_network_lost(accounts: *mut dc_accoun
     block_on(async move { accounts.write().await.maybe_network_lost().await });
 }
 
-pub type dc_accounts_event_emitter_t = EventEmitter;
-
 #[no_mangle]
 pub unsafe extern "C" fn dc_accounts_get_event_emitter(
     accounts: *mut dc_accounts_t,
-) -> *mut dc_accounts_event_emitter_t {
+) -> *mut dc_event_emitter_t {
     if accounts.is_null() {
         eprintln!("ignoring careless call to dc_accounts_get_event_emitter()");
         return ptr::null_mut();
@@ -4413,27 +4443,76 @@ pub unsafe extern "C" fn dc_accounts_get_event_emitter(
     Box::into_raw(Box::new(emitter))
 }
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_accounts_event_emitter_unref(
-    emitter: *mut dc_accounts_event_emitter_t,
-) {
-    if emitter.is_null() {
-        eprintln!("ignoring careless call to dc_accounts_event_emitter_unref()");
-        return;
-    }
-    let _ = Box::from_raw(emitter);
-}
+#[cfg(feature = "jsonrpc")]
+mod jsonrpc {
+    use super::*;
+    use deltachat_jsonrpc::api::CommandApi;
+    use deltachat_jsonrpc::yerpc::{OutReceiver, RpcClient, RpcSession};
 
-#[no_mangle]
-pub unsafe extern "C" fn dc_accounts_get_next_event(
-    emitter: *mut dc_accounts_event_emitter_t,
-) -> *mut dc_event_t {
-    if emitter.is_null() {
-        eprintln!("ignoring careless call to dc_accounts_get_next_event()");
-        return ptr::null_mut();
+    pub struct dc_jsonrpc_instance_t {
+        receiver: OutReceiver,
+        handle: RpcSession<CommandApi>,
+    }
+
+    #[no_mangle]
+    pub unsafe extern "C" fn dc_jsonrpc_init(
+        account_manager: *mut dc_accounts_t,
+    ) -> *mut dc_jsonrpc_instance_t {
+        if account_manager.is_null() {
+            eprintln!("ignoring careless call to dc_jsonrpc_init()");
+            return ptr::null_mut();
+        }
+
+        let cmd_api =
+            deltachat_jsonrpc::api::CommandApi::from_arc((*account_manager).inner.clone());
+
+        let (request_handle, receiver) = RpcClient::new();
+        let handle = RpcSession::new(request_handle, cmd_api);
+
+        let instance = dc_jsonrpc_instance_t { receiver, handle };
+
+        Box::into_raw(Box::new(instance))
+    }
+
+    #[no_mangle]
+    pub unsafe extern "C" fn dc_jsonrpc_unref(jsonrpc_instance: *mut dc_jsonrpc_instance_t) {
+        if jsonrpc_instance.is_null() {
+            eprintln!("ignoring careless call to dc_jsonrpc_unref()");
+            return;
+        }
+
+        Box::from_raw(jsonrpc_instance);
+    }
+
+    #[no_mangle]
+    pub unsafe extern "C" fn dc_jsonrpc_request(
+        jsonrpc_instance: *mut dc_jsonrpc_instance_t,
+        request: *const libc::c_char,
+    ) {
+        if jsonrpc_instance.is_null() || request.is_null() {
+            eprintln!("ignoring careless call to dc_jsonrpc_request()");
+            return;
+        }
+
+        let api = &*jsonrpc_instance;
+        let handle = &api.handle;
+        let request = to_string_lossy(request);
+        spawn(async move {
+            handle.handle_incoming(&request).await;
+        });
+    }
+
+    #[no_mangle]
+    pub unsafe extern "C" fn dc_jsonrpc_next_response(
+        jsonrpc_instance: *mut dc_jsonrpc_instance_t,
+    ) -> *mut libc::c_char {
+        if jsonrpc_instance.is_null() {
+            eprintln!("ignoring careless call to dc_jsonrpc_next_response()");
+            return ptr::null_mut();
+        }
+        let api = &*jsonrpc_instance;
+        block_on(api.receiver.recv())
+            .map(|result| serde_json::to_string(&result).unwrap_or_default().strdup())
+            .unwrap_or(ptr::null_mut())
     }
-    let emitter = &mut *emitter;
-    block_on(emitter.recv())
-        .map(|ev| Box::into_raw(Box::new(ev)))
-        .unwrap_or_else(ptr::null_mut)
 }

deltachat-ffi/src/lot.rs

@@ -51,7 +51,7 @@ impl Lot {
                 Qr::FprWithoutAddr { fingerprint, .. } => Some(fingerprint),
                 Qr::Account { domain } => Some(domain),
                 Qr::WebrtcInstance { domain, .. } => Some(domain),
-                Qr::Addr { .. } => None,
+                Qr::Addr { draft, .. } => draft.as_deref(),
                 Qr::Url { url } => Some(url),
                 Qr::Text { text } => Some(text),
                 Qr::WithdrawVerifyContact { .. } => None,
@@ -79,7 +79,13 @@ impl Lot {
                 Some(SummaryPrefix::Username(_username)) => Meaning::Text1Username,
                 Some(SummaryPrefix::Me(_text)) => Meaning::Text1Self,
             },
-            Self::Qr(_qr) => Meaning::None,
+            Self::Qr(qr) => match qr {
+                Qr::Addr {
+                    draft: Some(_draft),
+                    ..
+                } => Meaning::Text1Draft,
+                _ => Meaning::None,
+            },
             Self::Error(_err) => Meaning::None,
         }
     }
@@ -118,7 +124,7 @@ impl Lot {
                 Qr::FprWithoutAddr { .. } => Default::default(),
                 Qr::Account { .. } => Default::default(),
                 Qr::WebrtcInstance { .. } => Default::default(),
-                Qr::Addr { contact_id } => contact_id.to_u32(),
+                Qr::Addr { contact_id, .. } => contact_id.to_u32(),
                 Qr::Url { .. } => Default::default(),
                 Qr::Text { .. } => Default::default(),
                 Qr::WithdrawVerifyContact { contact_id, .. } => contact_id.to_u32(),

deltachat-jsonrpc/.gitignore

@@ -0,0 +1,3 @@
+accounts/
+
+.cargo

deltachat-jsonrpc/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+name = "deltachat-jsonrpc"
+version = "1.93.0"
+description = "DeltaChat JSON-RPC API"
+authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
+edition = "2021"
+default-run = "deltachat-jsonrpc-server"
+license = "MPL-2.0"
+
+[[bin]]
+name = "deltachat-jsonrpc-server"
+path = "src/webserver.rs"
+required-features = ["webserver"]
+
+[dependencies]
+anyhow = "1"
+deltachat = { path = ".." }
+num-traits = "0.2"
+serde = { version = "1.0", features = ["derive"] }
+tempfile = "3.3.0"
+log = "0.4"
+async-channel = { version = "1.6.1" }
+futures = { version = "0.3.24" }
+serde_json = "1.0.85"
+yerpc = { version = "^0.3.1", features = ["anyhow_expose"] }
+typescript-type-def = { version = "0.5.3", features = ["json_value"] }
+tokio = { version = "1.19.2" }
+
+# optional dependencies
+axum = { version = "0.5.9", optional = true, features = ["ws"] }
+env_logger = { version = "0.9.0", optional = true }
+
+[dev-dependencies]
+tokio = { version = "1.19.2", features = ["full", "rt-multi-thread"] }
+
+
+[features]
+default = []
+webserver = ["env_logger", "axum", "tokio/full", "yerpc/support-axum"]

deltachat-jsonrpc/README.md

@@ -0,0 +1,123 @@
+# deltachat-jsonrpc
+
+This crate provides a [JSON-RPC 2.0](https://www.jsonrpc.org/specification) interface to DeltaChat.
+
+The JSON-RPC API is exposed in two fashions:
+
+* A executable that exposes the JSON-RPC API through a [WebSocket](https://developer.mozilla.org/en-US/docs/Web/API/WebSockets_API) server running on localhost.
+* The JSON-RPC API can also be called through the [C FFI](../deltachat-ffi). The C FFI needs to be built with the `jsonrpc` feature. It will then expose the functions `dc_jsonrpc_init`, `dc_jsonrpc_request`, `dc_jsonrpc_next_response` and `dc_jsonrpc_unref`. See the docs in the [header file](../deltachat-ffi/deltachat.h) for details.
+
+We also include a JavaScript and TypeScript client for the JSON-RPC API. The source for this is in the [`typescript`](typescript) folder. The client can easily be used with the WebSocket server to build DeltaChat apps for web browsers or Node.js. See the [examples](typescript/example) for details.
+
+## Usage
+
+#### Running the WebSocket server
+
+From within this folder, you can start the WebSocket server with the following command:
+
+```sh
+cargo run --features webserver
+```
+
+If you want to use the server in a production setup, first build it in release mode:
+
+```sh
+cargo build --features webserver --release
+```
+You will then find the `deltachat-jsonrpc-server` executable in your `target/release` folder.
+
+The executable currently does not support any command-line arguments. By default, once started it will accept WebSocket connections on `ws://localhost:20808/ws`. It will store the persistent configuration and databases in a `./accounts` folder relative to the directory from where it is started.
+
+The server can be configured with environment variables:
+
+|variable|default|description|
+|-|-|-|
+|`DC_PORT`|`20808`|port to listen on|
+|`DC_ACCOUNTS_PATH`|`./accounts`|path to storage directory|
+
+If you are targetting other architectures (like KaiOS or Android), the webserver binary can be cross-compiled easily with [rust-cross](https://github.com/cross-rs/cross):
+
+```sh
+cross build --features=webserver --target armv7-linux-androideabi --release
+```
+
+#### Using the TypeScript/JavaScript client
+
+The package includes a JavaScript/TypeScript client which is partially auto-generated through the JSON-RPC library used by this crate ([yerpc](https://github.com/Frando/yerpc/)). Find the source in the [`typescript`](typescript) folder. 
+
+To use it locally, first install the dependencies and compile the TypeScript code to JavaScript:
+```sh
+cd typescript
+npm install
+npm run build
+```
+
+The JavaScript client is not yet published on NPM (but will likely be soon). Currently, it is recommended to vendor the bundled build. After running `npm run build` as documented above, there will be a file `dist/deltachat.bundle.js`. This is an ESM module containing all dependencies. Copy this file to your project and import the DeltaChat class.
+
+
+```typescript
+import { DeltaChat } from './deltachat.bundle.js'
+const dc = new DeltaChat('ws://localhost:20808/ws')
+const accounts = await dc.rpc.getAllAccounts()
+console.log('accounts', accounts)
+```
+
+A script is included to build autogenerated documentation, which includes all RPC methods:
+```sh
+cd typescript
+npm run docs
+```
+Then open the [`typescript/docs`](typescript/docs) folder in a web browser.
+
+## Development
+
+#### Running the example app
+
+We include a small demo web application that talks to the WebSocket server. It can be used for testing. Feel invited to expand this.
+
+```sh
+cd typescript
+npm run build
+npm run example:build
+npm run example:start
+```
+Then, open [`http://localhost:8080/example.html`](http://localhost:8080/example.html) in a web browser.
+
+Run `npm run example:dev` to live-rebuild the example app when files changes.
+
+### Testing
+
+The crate includes both a basic Rust smoke test and more featureful integration tests that use the TypeScript client.
+
+#### Rust tests
+
+To run the Rust test, use this command:
+
+```
+cargo test
+```
+
+#### TypeScript tests
+
+```
+cd typescript
+npm run test
+```
+
+This will build the `deltachat-jsonrpc-server` binary and then run a test suite against the WebSocket server.
+
+The test suite includes some tests that need online connectivity and a way to create test email accounts. To run these tests, talk to DeltaChat developers to get a token for the `testrun.org` service, or use a local instance of [`mailadm`](https://github.com/deltachat/docker-mailadm).
+
+Then, set the `DCC_NEW_TMP_EMAIL` environment variable to your mailadm token before running the tests.
+
+```
+DCC_NEW_TMP_EMAIL=https://testrun.org/new_email?t=yourtoken npm run test
+```
+
+#### Test Coverage
+
+Running `npm run test` will report test coverage. For the coverage to be accurate the online tests need to be run.
+
+> If you are offline and want to see the coverage results anyway (even though they are inaccurate), you can bypass the errors of the online tests by setting the `COVERAGE_OFFLINE=1` environment variable.
+
+A summary of the coverage will be reported in the terminal after the test run. Open `coverage/index.html` in a web browser for a detailed report.

deltachat-jsonrpc/TODO.md

@@ -0,0 +1,28 @@
+# TODO
+
+- [ ] different test type to simulate two devices: to test autocrypt_initiate_key_transfer & autocrypt_continue_key_transfer
+
+## MVP - Websocket server&client
+
+For kaiOS and other experiments, like a deltachat "web" over network from an android phone.
+
+- [ ] coverage for a majority of the API
+- [ ] Blobs served
+- [ ] Blob upload (for attachments, setting profile-picture, importing backup and so on)
+- [ ] other way blobs can be addressed when using websocket vs. jsonrpc over dc-node
+- [ ] Web push API? At least some kind of notification hook closure this lib can accept.
+
+### Other Ideas for the Websocket server
+
+- [ ] make sure there can only be one connection at a time to the ws
+  - why? , it could give problems if its commanded from multiple connections
+- [ ] encrypted connection?
+- [ ] authenticated connection?
+- [ ] Look into unit-testing for the proc macros?
+- [ ] proc macro taking over doc comments to generated typescript file
+
+## Desktop Apis
+
+Incomplete todo for desktop api porting, just some remainders for points that might need more work:
+
+- [ ] manual start/stop io functions in the api for context and accounts, so "not syncing all accounts" can still be done in desktop -> webserver should then not do start io on all accounts by default

deltachat-jsonrpc/src/api/events.rs

@@ -0,0 +1,166 @@
+use deltachat::{Event, EventType};
+use serde::Serialize;
+use serde_json::{json, Value};
+use typescript_type_def::TypeDef;
+
+pub fn event_to_json_rpc_notification(event: Event) -> Value {
+    let (field1, field2): (Value, Value) = match &event.typ {
+        // events with a single string in field1
+        EventType::Info(txt)
+        | EventType::SmtpConnected(txt)
+        | EventType::ImapConnected(txt)
+        | EventType::SmtpMessageSent(txt)
+        | EventType::ImapMessageDeleted(txt)
+        | EventType::ImapMessageMoved(txt)
+        | EventType::NewBlobFile(txt)
+        | EventType::DeletedBlobFile(txt)
+        | EventType::Warning(txt)
+        | EventType::Error(txt)
+        | EventType::ErrorSelfNotInGroup(txt) => (json!(txt), Value::Null),
+        EventType::ImexFileWritten(path) => (json!(path.to_str()), Value::Null),
+        // single number
+        EventType::MsgsNoticed(chat_id) | EventType::ChatModified(chat_id) => {
+            (json!(chat_id), Value::Null)
+        }
+        EventType::ImexProgress(progress) => (json!(progress), Value::Null),
+        // both fields contain numbers
+        EventType::MsgsChanged { chat_id, msg_id }
+        | EventType::IncomingMsg { chat_id, msg_id }
+        | EventType::MsgDelivered { chat_id, msg_id }
+        | EventType::MsgFailed { chat_id, msg_id }
+        | EventType::MsgRead { chat_id, msg_id } => (json!(chat_id), json!(msg_id)),
+        EventType::ChatEphemeralTimerModified { chat_id, timer } => (json!(chat_id), json!(timer)),
+        EventType::SecurejoinInviterProgress {
+            contact_id,
+            progress,
+        }
+        | EventType::SecurejoinJoinerProgress {
+            contact_id,
+            progress,
+        } => (json!(contact_id), json!(progress)),
+        // field 1 number or null
+        EventType::ContactsChanged(maybe_number) | EventType::LocationChanged(maybe_number) => (
+            match maybe_number {
+                Some(number) => json!(number),
+                None => Value::Null,
+            },
+            Value::Null,
+        ),
+        // number and maybe string
+        EventType::ConfigureProgress { progress, comment } => (
+            json!(progress),
+            match comment {
+                Some(content) => json!(content),
+                None => Value::Null,
+            },
+        ),
+        EventType::ConnectivityChanged => (Value::Null, Value::Null),
+        EventType::SelfavatarChanged => (Value::Null, Value::Null),
+        EventType::WebxdcStatusUpdate {
+            msg_id,
+            status_update_serial,
+        } => (json!(msg_id), json!(status_update_serial)),
+        EventType::WebxdcInstanceDeleted { msg_id } => (json!(msg_id), Value::Null),
+        EventType::WebxdcBusyUpdating { msg_id } => (json!(msg_id), Value::Null),
+        EventType::WebxdcUpToDate { msg_id } => (json!(msg_id), Value::Null),
+    };
+
+    let id: EventTypeName = event.typ.into();
+    json!({
+        "id": id,
+        "contextId": event.id,
+        "field1": field1,
+        "field2": field2
+    })
+}
+
+#[derive(Serialize, TypeDef)]
+pub enum EventTypeName {
+    Info,
+    SmtpConnected,
+    ImapConnected,
+    SmtpMessageSent,
+    ImapMessageDeleted,
+    ImapMessageMoved,
+    NewBlobFile,
+    DeletedBlobFile,
+    Warning,
+    Error,
+    ErrorSelfNotInGroup,
+    MsgsChanged,
+    IncomingMsg,
+    MsgsNoticed,
+    MsgDelivered,
+    MsgFailed,
+    MsgRead,
+    ChatModified,
+    ChatEphemeralTimerModified,
+    ContactsChanged,
+    LocationChanged,
+    ConfigureProgress,
+    ImexProgress,
+    ImexFileWritten,
+    SecurejoinInviterProgress,
+    SecurejoinJoinerProgress,
+    ConnectivityChanged,
+    SelfavatarChanged,
+    WebxdcStatusUpdate,
+    WebxdcInstanceDeleted,
+    WebxdcBusyUpdating,
+    WebxdcUpToDate,
+}
+
+impl From<EventType> for EventTypeName {
+    fn from(event: EventType) -> Self {
+        use EventTypeName::*;
+        match event {
+            EventType::Info(_) => Info,
+            EventType::SmtpConnected(_) => SmtpConnected,
+            EventType::ImapConnected(_) => ImapConnected,
+            EventType::SmtpMessageSent(_) => SmtpMessageSent,
+            EventType::ImapMessageDeleted(_) => ImapMessageDeleted,
+            EventType::ImapMessageMoved(_) => ImapMessageMoved,
+            EventType::NewBlobFile(_) => NewBlobFile,
+            EventType::DeletedBlobFile(_) => DeletedBlobFile,
+            EventType::Warning(_) => Warning,
+            EventType::Error(_) => Error,
+            EventType::ErrorSelfNotInGroup(_) => ErrorSelfNotInGroup,
+            EventType::MsgsChanged { .. } => MsgsChanged,
+            EventType::IncomingMsg { .. } => IncomingMsg,
+            EventType::MsgsNoticed(_) => MsgsNoticed,
+            EventType::MsgDelivered { .. } => MsgDelivered,
+            EventType::MsgFailed { .. } => MsgFailed,
+            EventType::MsgRead { .. } => MsgRead,
+            EventType::ChatModified(_) => ChatModified,
+            EventType::ChatEphemeralTimerModified { .. } => ChatEphemeralTimerModified,
+            EventType::ContactsChanged(_) => ContactsChanged,
+            EventType::LocationChanged(_) => LocationChanged,
+            EventType::ConfigureProgress { .. } => ConfigureProgress,
+            EventType::ImexProgress(_) => ImexProgress,
+            EventType::ImexFileWritten(_) => ImexFileWritten,
+            EventType::SecurejoinInviterProgress { .. } => SecurejoinInviterProgress,
+            EventType::SecurejoinJoinerProgress { .. } => SecurejoinJoinerProgress,
+            EventType::ConnectivityChanged => ConnectivityChanged,
+            EventType::SelfavatarChanged => SelfavatarChanged,
+            EventType::WebxdcStatusUpdate { .. } => WebxdcStatusUpdate,
+            EventType::WebxdcInstanceDeleted { .. } => WebxdcInstanceDeleted,
+            EventType::WebxdcBusyUpdating { .. } => WebxdcBusyUpdating,
+            EventType::WebxdcUpToDate { .. } => WebxdcUpToDate,
+        }
+    }
+}
+
+#[cfg(test)]
+#[test]
+fn generate_events_ts_types_definition() {
+    let events = {
+        let mut buf = Vec::new();
+        let options = typescript_type_def::DefinitionFileOptions {
+            root_namespace: None,
+            ..typescript_type_def::DefinitionFileOptions::default()
+        };
+        typescript_type_def::write_definition_file::<_, EventTypeName>(&mut buf, options).unwrap();
+        String::from_utf8(buf).unwrap()
+    };
+    std::fs::write("typescript/generated/events.ts", events).unwrap();
+}

deltachat-jsonrpc/src/api/mod.rs

@@ -0,0 +1,819 @@
+use anyhow::{anyhow, bail, Context, Result};
+use deltachat::{
+    chat::{add_contact_to_chat, get_chat_media, get_chat_msgs, remove_contact_from_chat, ChatId},
+    chatlist::Chatlist,
+    config::Config,
+    contact::{may_be_valid_addr, Contact, ContactId},
+    context::get_info,
+    message::{delete_msgs, get_msg_info, Message, MsgId, Viewtype},
+    provider::get_provider_info,
+    qr,
+    qr_code_generator::get_securejoin_qr_svg,
+    securejoin,
+    webxdc::StatusUpdateSerial,
+};
+use std::collections::BTreeMap;
+use std::sync::Arc;
+use std::{collections::HashMap, str::FromStr};
+use tokio::sync::RwLock;
+use yerpc::rpc;
+
+pub use deltachat::accounts::Accounts;
+
+pub mod events;
+pub mod types;
+
+use crate::api::types::chat_list::{get_chat_list_item_by_id, ChatListItemFetchResult};
+use crate::api::types::QrObject;
+
+use types::account::Account;
+use types::chat::FullChat;
+use types::chat_list::ChatListEntry;
+use types::contact::ContactObject;
+use types::message::MessageObject;
+use types::provider_info::ProviderInfo;
+use types::webxdc::WebxdcMessageInfo;
+
+use self::types::message::MessageViewtype;
+
+#[derive(Clone, Debug)]
+pub struct CommandApi {
+    pub(crate) accounts: Arc<RwLock<Accounts>>,
+}
+
+impl CommandApi {
+    pub fn new(accounts: Accounts) -> Self {
+        CommandApi {
+            accounts: Arc::new(RwLock::new(accounts)),
+        }
+    }
+
+    #[allow(dead_code)]
+    pub fn from_arc(accounts: Arc<RwLock<Accounts>>) -> Self {
+        CommandApi { accounts }
+    }
+
+    async fn get_context(&self, id: u32) -> Result<deltachat::context::Context> {
+        let sc = self
+            .accounts
+            .read()
+            .await
+            .get_account(id)
+            .await
+            .ok_or_else(|| anyhow!("account with id {} not found", id))?;
+        Ok(sc)
+    }
+}
+
+#[rpc(all_positional, ts_outdir = "typescript/generated")]
+impl CommandApi {
+    // ---------------------------------------------
+    //  Misc top level functions
+    // ---------------------------------------------
+
+    /// Check if an email address is valid.
+    async fn check_email_validity(&self, email: String) -> bool {
+        may_be_valid_addr(&email)
+    }
+
+    /// Get general system info.
+    async fn get_system_info(&self) -> BTreeMap<&'static str, String> {
+        get_info()
+    }
+
+    // ---------------------------------------------
+    // Account Management
+    // ---------------------------------------------
+
+    async fn add_account(&self) -> Result<u32> {
+        self.accounts.write().await.add_account().await
+    }
+
+    async fn remove_account(&self, account_id: u32) -> Result<()> {
+        self.accounts.write().await.remove_account(account_id).await
+    }
+
+    async fn get_all_account_ids(&self) -> Vec<u32> {
+        self.accounts.read().await.get_all().await
+    }
+
+    /// Select account id for internally selected state.
+    /// TODO: Likely this is deprecated as all methods take an account id now.
+    async fn select_account(&self, id: u32) -> Result<()> {
+        self.accounts.write().await.select_account(id).await
+    }
+
+    /// Get the selected account id of the internal state..
+    /// TODO: Likely this is deprecated as all methods take an account id now.
+    async fn get_selected_account_id(&self) -> Option<u32> {
+        self.accounts.read().await.get_selected_account_id().await
+    }
+
+    /// Get a list of all configured accounts.
+    async fn get_all_accounts(&self) -> Result<Vec<Account>> {
+        let mut accounts = Vec::new();
+        for id in self.accounts.read().await.get_all().await {
+            let context_option = self.accounts.read().await.get_account(id).await;
+            if let Some(ctx) = context_option {
+                accounts.push(Account::from_context(&ctx, id).await?)
+            } else {
+                println!("account with id {} doesn't exist anymore", id);
+            }
+        }
+        Ok(accounts)
+    }
+
+    // ---------------------------------------------
+    // Methods that work on individual accounts
+    // ---------------------------------------------
+
+    /// Get top-level info for an account.
+    async fn get_account_info(&self, account_id: u32) -> Result<Account> {
+        let context_option = self.accounts.read().await.get_account(account_id).await;
+        if let Some(ctx) = context_option {
+            Ok(Account::from_context(&ctx, account_id).await?)
+        } else {
+            Err(anyhow!(
+                "account with id {} doesn't exist anymore",
+                account_id
+            ))
+        }
+    }
+
+    /// Returns provider for the given domain.
+    ///
+    /// This function looks up domain in offline database.
+    ///
+    /// For compatibility, email address can be passed to this function
+    /// instead of the domain.
+    async fn get_provider_info(
+        &self,
+        account_id: u32,
+        email: String,
+    ) -> Result<Option<ProviderInfo>> {
+        let ctx = self.get_context(account_id).await?;
+
+        let socks5_enabled = ctx
+            .get_config_bool(deltachat::config::Config::Socks5Enabled)
+            .await?;
+
+        let provider_info =
+            get_provider_info(&ctx, email.split('@').last().unwrap_or(""), socks5_enabled).await;
+        Ok(ProviderInfo::from_dc_type(provider_info))
+    }
+
+    /// Checks if the context is already configured.
+    async fn is_configured(&self, account_id: u32) -> Result<bool> {
+        let ctx = self.get_context(account_id).await?;
+        ctx.is_configured().await
+    }
+
+    /// Get system info for an account.
+    async fn get_info(&self, account_id: u32) -> Result<BTreeMap<&'static str, String>> {
+        let ctx = self.get_context(account_id).await?;
+        ctx.get_info().await
+    }
+
+    async fn set_config(&self, account_id: u32, key: String, value: Option<String>) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        set_config(&ctx, &key, value.as_deref()).await
+    }
+
+    async fn batch_set_config(
+        &self,
+        account_id: u32,
+        config: HashMap<String, Option<String>>,
+    ) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        for (key, value) in config.into_iter() {
+            set_config(&ctx, &key, value.as_deref())
+                .await
+                .with_context(|| format!("Can't set {} to {:?}", key, value))?;
+        }
+        Ok(())
+    }
+
+    /// Set configuration values from a QR code. (technically from the URI that is stored in the qrcode)
+    /// Before this function is called, `checkQr()` should confirm the type of the
+    /// QR code is `account` or `webrtcInstance`.
+    ///
+    /// Internally, the function will call dc_set_config() with the appropriate keys,
+    async fn set_config_from_qr(&self, account_id: u32, qr_content: String) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        qr::set_config_from_qr(&ctx, &qr_content).await
+    }
+
+    async fn check_qr(&self, account_id: u32, qr_content: String) -> Result<QrObject> {
+        let ctx = self.get_context(account_id).await?;
+        let qr = qr::check_qr(&ctx, &qr_content).await?;
+        let qr_object = QrObject::from(qr);
+        Ok(qr_object)
+    }
+
+    async fn get_config(&self, account_id: u32, key: String) -> Result<Option<String>> {
+        let ctx = self.get_context(account_id).await?;
+        get_config(&ctx, &key).await
+    }
+
+    async fn batch_get_config(
+        &self,
+        account_id: u32,
+        keys: Vec<String>,
+    ) -> Result<HashMap<String, Option<String>>> {
+        let ctx = self.get_context(account_id).await?;
+        let mut result: HashMap<String, Option<String>> = HashMap::new();
+        for key in keys {
+            result.insert(key.clone(), get_config(&ctx, &key).await?);
+        }
+        Ok(result)
+    }
+
+    /// Configures this account with the currently set parameters.
+    /// Setup the credential config before calling this.
+    async fn configure(&self, account_id: u32) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        ctx.stop_io().await;
+        let result = ctx.configure().await;
+        if result.is_err() {
+            if let Ok(true) = ctx.is_configured().await {
+                ctx.start_io().await;
+            }
+            return result;
+        }
+        ctx.start_io().await;
+        Ok(())
+    }
+
+    /// Signal an ongoing process to stop.
+    async fn stop_ongoing_process(&self, account_id: u32) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        ctx.stop_ongoing().await;
+        Ok(())
+    }
+
+    /// Returns the message IDs of all _fresh_ messages of any chat.
+    /// Typically used for implementing notification summaries
+    /// or badge counters e.g. on the app icon.
+    /// The list is already sorted and starts with the most recent fresh message.
+    ///
+    /// Messages belonging to muted chats or to the contact requests are not returned;
+    /// these messages should not be notified
+    /// and also badge counters should not include these messages.
+    ///
+    /// To get the number of fresh messages for a single chat, muted or not,
+    /// use `get_fresh_msg_cnt()`.
+    async fn get_fresh_msgs(&self, account_id: u32) -> Result<Vec<u32>> {
+        let ctx = self.get_context(account_id).await?;
+        Ok(ctx
+            .get_fresh_msgs()
+            .await?
+            .iter()
+            .map(|msg_id| msg_id.to_u32())
+            .collect())
+    }
+
+    /// Get the number of _fresh_ messages in a chat.
+    /// Typically used to implement a badge with a number in the chatlist.
+    ///
+    /// If the specified chat is muted,
+    /// the UI should show the badge counter "less obtrusive",
+    /// e.g. using "gray" instead of "red" color.
+    async fn get_fresh_msg_cnt(&self, account_id: u32, chat_id: u32) -> Result<usize> {
+        let ctx = self.get_context(account_id).await?;
+        ChatId::new(chat_id).get_fresh_msg_cnt(&ctx).await
+    }
+
+    // ---------------------------------------------
+    //  autocrypt
+    // ---------------------------------------------
+
+    async fn autocrypt_initiate_key_transfer(&self, account_id: u32) -> Result<String> {
+        let ctx = self.get_context(account_id).await?;
+        deltachat::imex::initiate_key_transfer(&ctx).await
+    }
+
+    async fn autocrypt_continue_key_transfer(
+        &self,
+        account_id: u32,
+        message_id: u32,
+        setup_code: String,
+    ) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        deltachat::imex::continue_key_transfer(&ctx, MsgId::new(message_id), &setup_code).await
+    }
+
+    // ---------------------------------------------
+    //   chat list
+    // ---------------------------------------------
+
+    async fn get_chatlist_entries(
+        &self,
+        account_id: u32,
+        list_flags: Option<u32>,
+        query_string: Option<String>,
+        query_contact_id: Option<u32>,
+    ) -> Result<Vec<ChatListEntry>> {
+        let ctx = self.get_context(account_id).await?;
+        let list = Chatlist::try_load(
+            &ctx,
+            list_flags.unwrap_or(0) as usize,
+            query_string.as_deref(),
+            query_contact_id.map(ContactId::new),
+        )
+        .await?;
+        let mut l: Vec<ChatListEntry> = Vec::with_capacity(list.len());
+        for i in 0..list.len() {
+            l.push(ChatListEntry(
+                list.get_chat_id(i)?.to_u32(),
+                list.get_msg_id(i)?.unwrap_or_default().to_u32(),
+            ));
+        }
+        Ok(l)
+    }
+
+    async fn get_chatlist_items_by_entries(
+        &self,
+        account_id: u32,
+        entries: Vec<ChatListEntry>,
+    ) -> Result<HashMap<u32, ChatListItemFetchResult>> {
+        // todo custom json deserializer for ChatListEntry?
+        let ctx = self.get_context(account_id).await?;
+        let mut result: HashMap<u32, ChatListItemFetchResult> =
+            HashMap::with_capacity(entries.len());
+        for entry in entries.iter() {
+            result.insert(
+                entry.0,
+                match get_chat_list_item_by_id(&ctx, entry).await {
+                    Ok(res) => res,
+                    Err(err) => ChatListItemFetchResult::Error {
+                        id: entry.0,
+                        error: format!("{:?}", err),
+                    },
+                },
+            );
+        }
+        Ok(result)
+    }
+
+    // ---------------------------------------------
+    //  chat
+    // ---------------------------------------------
+
+    async fn chatlist_get_full_chat_by_id(
+        &self,
+        account_id: u32,
+        chat_id: u32,
+    ) -> Result<FullChat> {
+        let ctx = self.get_context(account_id).await?;
+        FullChat::try_from_dc_chat_id(&ctx, chat_id).await
+    }
+
+    async fn accept_chat(&self, account_id: u32, chat_id: u32) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        ChatId::new(chat_id).accept(&ctx).await
+    }
+
+    async fn block_chat(&self, account_id: u32, chat_id: u32) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        ChatId::new(chat_id).block(&ctx).await
+    }
+
+    /// Delete a chat.
+    ///
+    /// Messages are deleted from the device and the chat database entry is deleted.
+    /// After that, the event #DC_EVENT_MSGS_CHANGED is posted.
+    ///
+    /// Things that are _not done_ implicitly:
+    ///
+    /// - Messages are **not deleted from the server**.
+    /// - The chat or the contact is **not blocked**, so new messages from the user/the group may appear as a contact request
+    ///   and the user may create the chat again.
+    /// - **Groups are not left** - this would
+    ///   be unexpected as (1) deleting a normal chat also does not prevent new mails
+    ///   from arriving, (2) leaving a group requires sending a message to
+    ///   all group members - especially for groups not used for a longer time, this is
+    ///   really unexpected when deletion results in contacting all members again,
+    ///   (3) only leaving groups is also a valid usecase.
+    ///
+    /// To leave a chat explicitly, use dc_remove_contact_from_chat() with
+    /// chat_id=DC_CONTACT_ID_SELF)
+    // TODO fix doc comment after adding dc_remove_contact_from_chat
+    async fn delete_chat(&self, account_id: u32, chat_id: u32) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        ChatId::new(chat_id).delete(&ctx).await
+    }
+
+    /// Get encryption info for a chat.
+    /// Get a multi-line encryption info, containing encryption preferences of all members.
+    /// Can be used to find out why messages sent to group are not encrypted.
+    ///
+    /// returns Multi-line text
+    async fn get_chat_encryption_info(&self, account_id: u32, chat_id: u32) -> Result<String> {
+        let ctx = self.get_context(account_id).await?;
+        ChatId::new(chat_id).get_encryption_info(&ctx).await
+    }
+
+    /// Get QR code (text and SVG) that will offer an Setup-Contact or Verified-Group invitation.
+    /// The QR code is compatible to the OPENPGP4FPR format
+    /// so that a basic fingerprint comparison also works e.g. with OpenKeychain.
+    ///
+    /// The scanning device will pass the scanned content to `checkQr()` then;
+    /// if `checkQr()` returns `askVerifyContact` or `askVerifyGroup`
+    /// an out-of-band-verification can be joined using dc_join_securejoin()
+    ///
+    /// chat_id: If set to a group-chat-id,
+    ///     the Verified-Group-Invite protocol is offered in the QR code;
+    ///     works for protected groups as well as for normal groups.
+    ///     If not set, the Setup-Contact protocol is offered in the QR code.
+    ///     See https://countermitm.readthedocs.io/en/latest/new.html
+    ///     for details about both protocols.
+    // TODO fix doc comment after adding dc_join_securejoin
+    async fn get_chat_securejoin_qr_code_svg(
+        &self,
+        account_id: u32,
+        chat_id: Option<u32>,
+    ) -> Result<(String, String)> {
+        let ctx = self.get_context(account_id).await?;
+        let chat = chat_id.map(ChatId::new);
+        Ok((
+            securejoin::get_securejoin_qr(&ctx, chat).await?,
+            get_securejoin_qr_svg(&ctx, chat).await?,
+        ))
+    }
+
+    async fn leave_group(&self, account_id: u32, chat_id: u32) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        remove_contact_from_chat(&ctx, ChatId::new(chat_id), ContactId::SELF).await
+    }
+
+    /// Remove a member from a group.
+    ///
+    /// If the group is already _promoted_ (any message was sent to the group),
+    /// all group members are informed by a special status message that is sent automatically by this function.
+    ///
+    /// Sends out #DC_EVENT_CHAT_MODIFIED and #DC_EVENT_MSGS_CHANGED if a status message was sent.
+    async fn remove_contact_from_chat(
+        &self,
+        account_id: u32,
+        chat_id: u32,
+        contact_id: u32,
+    ) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        remove_contact_from_chat(&ctx, ChatId::new(chat_id), ContactId::new(contact_id)).await
+    }
+
+    /// Add a member to a group.
+    ///
+    /// If the group is already _promoted_ (any message was sent to the group),
+    /// all group members are informed by a special status message that is sent automatically by this function.
+    ///
+    /// If the group has group protection enabled, only verified contacts can be added to the group.
+    ///
+    /// Sends out #DC_EVENT_CHAT_MODIFIED and #DC_EVENT_MSGS_CHANGED if a status message was sent.
+    async fn add_contact_to_chat(
+        &self,
+        account_id: u32,
+        chat_id: u32,
+        contact_id: u32,
+    ) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        add_contact_to_chat(&ctx, ChatId::new(chat_id), ContactId::new(contact_id)).await
+    }
+
+    // for now only text messages, because we only used text messages in desktop thusfar
+    async fn add_device_message(
+        &self,
+        account_id: u32,
+        label: String,
+        text: String,
+    ) -> Result<u32> {
+        let ctx = self.get_context(account_id).await?;
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some(text));
+        let message_id =
+            deltachat::chat::add_device_msg(&ctx, Some(&label), Some(&mut msg)).await?;
+        Ok(message_id.to_u32())
+    }
+
+    // ---------------------------------------------
+    // message list
+    // ---------------------------------------------
+
+    async fn message_list_get_message_ids(
+        &self,
+        account_id: u32,
+        chat_id: u32,
+        flags: u32,
+    ) -> Result<Vec<u32>> {
+        let ctx = self.get_context(account_id).await?;
+        let msg = get_chat_msgs(&ctx, ChatId::new(chat_id), flags).await?;
+        Ok(msg
+            .iter()
+            .filter_map(|chat_item| match chat_item {
+                deltachat::chat::ChatItem::Message { msg_id } => Some(msg_id.to_u32()),
+                _ => None,
+            })
+            .collect())
+    }
+
+    async fn message_get_message(&self, account_id: u32, message_id: u32) -> Result<MessageObject> {
+        let ctx = self.get_context(account_id).await?;
+        MessageObject::from_message_id(&ctx, message_id).await
+    }
+
+    async fn message_get_messages(
+        &self,
+        account_id: u32,
+        message_ids: Vec<u32>,
+    ) -> Result<HashMap<u32, MessageObject>> {
+        let ctx = self.get_context(account_id).await?;
+        let mut messages: HashMap<u32, MessageObject> = HashMap::new();
+        for message_id in message_ids {
+            messages.insert(
+                message_id,
+                MessageObject::from_message_id(&ctx, message_id).await?,
+            );
+        }
+        Ok(messages)
+    }
+
+    /// Delete messages. The messages are deleted on the current device and
+    /// on the IMAP server.
+    async fn delete_messages(&self, account_id: u32, message_ids: Vec<u32>) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        let msgs: Vec<MsgId> = message_ids.into_iter().map(MsgId::new).collect();
+        delete_msgs(&ctx, &msgs).await
+    }
+
+    /// Get an informational text for a single message. The text is multiline and may
+    /// contain e.g. the raw text of the message.
+    ///
+    /// The max. text returned is typically longer (about 100000 characters) than the
+    /// max. text returned by dc_msg_get_text() (about 30000 characters).
+    async fn get_message_info(&self, account_id: u32, message_id: u32) -> Result<String> {
+        let ctx = self.get_context(account_id).await?;
+        get_msg_info(&ctx, MsgId::new(message_id)).await
+    }
+
+    // ---------------------------------------------
+    //  contact
+    // ---------------------------------------------
+
+    /// Get a single contact options by ID.
+    async fn contacts_get_contact(
+        &self,
+        account_id: u32,
+        contact_id: u32,
+    ) -> Result<ContactObject> {
+        let ctx = self.get_context(account_id).await?;
+        let contact_id = ContactId::new(contact_id);
+
+        ContactObject::try_from_dc_contact(
+            &ctx,
+            deltachat::contact::Contact::get_by_id(&ctx, contact_id).await?,
+        )
+        .await
+    }
+
+    /// Add a single contact as a result of an explicit user action.
+    ///
+    /// Returns contact id of the created or existing contact
+    async fn contacts_create_contact(
+        &self,
+        account_id: u32,
+        email: String,
+        name: Option<String>,
+    ) -> Result<u32> {
+        let ctx = self.get_context(account_id).await?;
+        if !may_be_valid_addr(&email) {
+            bail!(anyhow!(
+                "provided email address is not a valid email address"
+            ))
+        }
+        let contact_id = Contact::create(&ctx, &name.unwrap_or_default(), &email).await?;
+        Ok(contact_id.to_u32())
+    }
+
+    /// Returns contact id of the created or existing DM chat with that contact
+    async fn contacts_create_chat_by_contact_id(
+        &self,
+        account_id: u32,
+        contact_id: u32,
+    ) -> Result<u32> {
+        let ctx = self.get_context(account_id).await?;
+        let contact = Contact::get_by_id(&ctx, ContactId::new(contact_id)).await?;
+        ChatId::create_for_contact(&ctx, contact.id)
+            .await
+            .map(|id| id.to_u32())
+    }
+
+    async fn contacts_block(&self, account_id: u32, contact_id: u32) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        Contact::block(&ctx, ContactId::new(contact_id)).await
+    }
+
+    async fn contacts_unblock(&self, account_id: u32, contact_id: u32) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        Contact::unblock(&ctx, ContactId::new(contact_id)).await
+    }
+
+    async fn contacts_get_blocked(&self, account_id: u32) -> Result<Vec<ContactObject>> {
+        let ctx = self.get_context(account_id).await?;
+        let blocked_ids = Contact::get_all_blocked(&ctx).await?;
+        let mut contacts: Vec<ContactObject> = Vec::with_capacity(blocked_ids.len());
+        for id in blocked_ids {
+            contacts.push(
+                ContactObject::try_from_dc_contact(
+                    &ctx,
+                    deltachat::contact::Contact::get_by_id(&ctx, id).await?,
+                )
+                .await?,
+            );
+        }
+        Ok(contacts)
+    }
+
+    async fn contacts_get_contact_ids(
+        &self,
+        account_id: u32,
+        list_flags: u32,
+        query: Option<String>,
+    ) -> Result<Vec<u32>> {
+        let ctx = self.get_context(account_id).await?;
+        let contacts = Contact::get_all(&ctx, list_flags, query.as_deref()).await?;
+        Ok(contacts.into_iter().map(|c| c.to_u32()).collect())
+    }
+
+    /// Get a list of contacts.
+    /// (formerly called getContacts2 in desktop)
+    async fn contacts_get_contacts(
+        &self,
+        account_id: u32,
+        list_flags: u32,
+        query: Option<String>,
+    ) -> Result<Vec<ContactObject>> {
+        let ctx = self.get_context(account_id).await?;
+        let contact_ids = Contact::get_all(&ctx, list_flags, query.as_deref()).await?;
+        let mut contacts: Vec<ContactObject> = Vec::with_capacity(contact_ids.len());
+        for id in contact_ids {
+            contacts.push(
+                ContactObject::try_from_dc_contact(
+                    &ctx,
+                    deltachat::contact::Contact::get_by_id(&ctx, id).await?,
+                )
+                .await?,
+            );
+        }
+        Ok(contacts)
+    }
+
+    async fn contacts_get_contacts_by_ids(
+        &self,
+        account_id: u32,
+        ids: Vec<u32>,
+    ) -> Result<HashMap<u32, ContactObject>> {
+        let ctx = self.get_context(account_id).await?;
+
+        let mut contacts = HashMap::with_capacity(ids.len());
+        for id in ids {
+            contacts.insert(
+                id,
+                ContactObject::try_from_dc_contact(
+                    &ctx,
+                    deltachat::contact::Contact::get_by_id(&ctx, ContactId::new(id)).await?,
+                )
+                .await?,
+            );
+        }
+        Ok(contacts)
+    }
+    // ---------------------------------------------
+    //                   chat
+    // ---------------------------------------------
+
+    /// Returns all message IDs of the given types in a chat.
+    /// Typically used to show a gallery.
+    ///
+    /// The list is already sorted and starts with the oldest message.
+    /// Clients should not try to re-sort the list as this would be an expensive action
+    /// and would result in inconsistencies between clients.
+    ///
+    /// Setting `chat_id` to `None` (`null` in typescript) means get messages with media
+    /// from any chat of the currently used account.
+    async fn chat_get_media(
+        &self,
+        account_id: u32,
+        chat_id: Option<u32>,
+        message_type: MessageViewtype,
+        or_message_type2: Option<MessageViewtype>,
+        or_message_type3: Option<MessageViewtype>,
+    ) -> Result<Vec<u32>> {
+        let ctx = self.get_context(account_id).await?;
+
+        let chat_id = match chat_id {
+            None | Some(0) => None,
+            Some(id) => Some(ChatId::new(id)),
+        };
+        let msg_type = message_type.into();
+        let or_msg_type2 = or_message_type2.map_or(Viewtype::Unknown, |v| v.into());
+        let or_msg_type3 = or_message_type3.map_or(Viewtype::Unknown, |v| v.into());
+
+        let media = get_chat_media(&ctx, chat_id, msg_type, or_msg_type2, or_msg_type3).await?;
+        Ok(media.iter().map(|msg_id| msg_id.to_u32()).collect())
+    }
+
+    // ---------------------------------------------
+    //                   webxdc
+    // ---------------------------------------------
+
+    async fn webxdc_send_status_update(
+        &self,
+        account_id: u32,
+        instance_msg_id: u32,
+        update_str: String,
+        description: String,
+    ) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        ctx.send_webxdc_status_update(MsgId::new(instance_msg_id), &update_str, &description)
+            .await
+    }
+
+    async fn webxdc_get_status_updates(
+        &self,
+        account_id: u32,
+        instance_msg_id: u32,
+        last_known_serial: u32,
+    ) -> Result<String> {
+        let ctx = self.get_context(account_id).await?;
+        ctx.get_webxdc_status_updates(
+            MsgId::new(instance_msg_id),
+            StatusUpdateSerial::new(last_known_serial),
+        )
+        .await
+    }
+
+    /// Get info from a webxdc message
+    async fn message_get_webxdc_info(
+        &self,
+        account_id: u32,
+        instance_msg_id: u32,
+    ) -> Result<WebxdcMessageInfo> {
+        let ctx = self.get_context(account_id).await?;
+        WebxdcMessageInfo::get_for_message(&ctx, MsgId::new(instance_msg_id)).await
+    }
+
+    // ---------------------------------------------
+    //           misc prototyping functions
+    //       that might get removed later again
+    // ---------------------------------------------
+
+    /// Returns the messageid of the sent message
+    async fn misc_send_text_message(
+        &self,
+        account_id: u32,
+        text: String,
+        chat_id: u32,
+    ) -> Result<u32> {
+        let ctx = self.get_context(account_id).await?;
+
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some(text));
+
+        let message_id = deltachat::chat::send_msg(&ctx, ChatId::new(chat_id), &mut msg).await?;
+        Ok(message_id.to_u32())
+    }
+}
+
+// Helper functions (to prevent code duplication)
+async fn set_config(
+    ctx: &deltachat::context::Context,
+    key: &str,
+    value: Option<&str>,
+) -> Result<(), anyhow::Error> {
+    if key.starts_with("ui.") {
+        ctx.set_ui_config(key, value).await?;
+    } else {
+        ctx.set_config(Config::from_str(key).context("unknown key")?, value)
+            .await?;
+
+        match key {
+            "sentbox_watch" | "mvbox_move" | "only_fetch_mvbox" => {
+                ctx.restart_io_if_running().await;
+            }
+            _ => {}
+        }
+    }
+    Ok(())
+}
+
+async fn get_config(
+    ctx: &deltachat::context::Context,
+    key: &str,
+) -> Result<Option<String>, anyhow::Error> {
+    if key.starts_with("ui.") {
+        ctx.get_ui_config(key).await
+    } else {
+        ctx.get_config(Config::from_str(key).context("unknown key")?)
+            .await
+    }
+}

deltachat-jsonrpc/src/api/types/account.rs

@@ -0,0 +1,45 @@
+use anyhow::Result;
+use deltachat::config::Config;
+use deltachat::contact::{Contact, ContactId};
+use serde::Serialize;
+use typescript_type_def::TypeDef;
+
+use super::color_int_to_hex_string;
+
+#[derive(Serialize, TypeDef)]
+#[serde(tag = "type")]
+pub enum Account {
+    #[serde(rename_all = "camelCase")]
+    Configured {
+        id: u32,
+        display_name: Option<String>,
+        addr: Option<String>,
+        // size: u32,
+        profile_image: Option<String>, // TODO: This needs to be converted to work with blob http server.
+        color: String,
+    },
+    #[serde(rename_all = "camelCase")]
+    Unconfigured { id: u32 },
+}
+
+impl Account {
+    pub async fn from_context(ctx: &deltachat::context::Context, id: u32) -> Result<Self> {
+        if ctx.is_configured().await? {
+            let display_name = ctx.get_config(Config::Displayname).await?;
+            let addr = ctx.get_config(Config::Addr).await?;
+            let profile_image = ctx.get_config(Config::Selfavatar).await?;
+            let color = color_int_to_hex_string(
+                Contact::get_by_id(ctx, ContactId::SELF).await?.get_color(),
+            );
+            Ok(Account::Configured {
+                id,
+                display_name,
+                addr,
+                profile_image,
+                color,
+            })
+        } else {
+            Ok(Account::Unconfigured { id })
+        }
+    }
+}

deltachat-jsonrpc/src/api/types/chat.rs

@@ -0,0 +1,106 @@
+use anyhow::{anyhow, Result};
+use deltachat::chat::get_chat_contacts;
+use deltachat::chat::{Chat, ChatId};
+use deltachat::constants::Chattype;
+use deltachat::contact::{Contact, ContactId};
+use deltachat::context::Context;
+use num_traits::cast::ToPrimitive;
+use serde::Serialize;
+use typescript_type_def::TypeDef;
+
+use super::color_int_to_hex_string;
+use super::contact::ContactObject;
+
+#[derive(Serialize, TypeDef)]
+#[serde(rename_all = "camelCase")]
+pub struct FullChat {
+    id: u32,
+    name: String,
+    is_protected: bool,
+    profile_image: Option<String>, //BLOBS ?
+    archived: bool,
+    // subtitle  - will be moved to frontend because it uses translation functions
+    chat_type: u32,
+    is_unpromoted: bool,
+    is_self_talk: bool,
+    contacts: Vec<ContactObject>,
+    contact_ids: Vec<u32>,
+    color: String,
+    fresh_message_counter: usize,
+    // is_group - please check over chat.type in frontend instead
+    is_contact_request: bool,
+    is_device_chat: bool,
+    self_in_group: bool,
+    is_muted: bool,
+    ephemeral_timer: u32, //TODO look if there are more important properties in newer core versions
+    can_send: bool,
+    was_seen_recently: bool,
+}
+
+impl FullChat {
+    pub async fn try_from_dc_chat_id(context: &Context, chat_id: u32) -> Result<Self> {
+        let rust_chat_id = ChatId::new(chat_id);
+        let chat = Chat::load_from_db(context, rust_chat_id).await?;
+
+        let contact_ids = get_chat_contacts(context, rust_chat_id).await?;
+
+        let mut contacts = Vec::with_capacity(contact_ids.len());
+
+        for contact_id in &contact_ids {
+            contacts.push(
+                ContactObject::try_from_dc_contact(
+                    context,
+                    Contact::load_from_db(context, *contact_id).await?,
+                )
+                .await?,
+            )
+        }
+
+        let profile_image = match chat.get_profile_image(context).await? {
+            Some(path_buf) => path_buf.to_str().map(|s| s.to_owned()),
+            None => None,
+        };
+
+        let color = color_int_to_hex_string(chat.get_color(context).await?);
+        let fresh_message_counter = rust_chat_id.get_fresh_msg_cnt(context).await?;
+        let ephemeral_timer = rust_chat_id.get_ephemeral_timer(context).await?.to_u32();
+
+        let can_send = chat.can_send(context).await?;
+
+        let was_seen_recently = if chat.get_type() == Chattype::Single {
+            match contact_ids.get(0) {
+                Some(contact) => Contact::load_from_db(context, *contact)
+                    .await?
+                    .was_seen_recently(),
+                None => false,
+            }
+        } else {
+            false
+        };
+
+        Ok(FullChat {
+            id: chat_id,
+            name: chat.name.clone(),
+            is_protected: chat.is_protected(),
+            profile_image, //BLOBS ?
+            archived: chat.get_visibility() == deltachat::chat::ChatVisibility::Archived,
+            chat_type: chat
+                .get_type()
+                .to_u32()
+                .ok_or_else(|| anyhow!("unknown chat type id"))?, // TODO get rid of this unwrap?
+            is_unpromoted: chat.is_unpromoted(),
+            is_self_talk: chat.is_self_talk(),
+            contacts,
+            contact_ids: contact_ids.iter().map(|id| id.to_u32()).collect(),
+            color,
+            fresh_message_counter,
+            is_contact_request: chat.is_contact_request(),
+            is_device_chat: chat.is_device_talk(),
+            self_in_group: contact_ids.contains(&ContactId::SELF),
+            is_muted: chat.is_muted(),
+            ephemeral_timer,
+            can_send,
+            was_seen_recently,
+        })
+    }
+}

deltachat-jsonrpc/src/api/types/chat_list.rs

@@ -0,0 +1,141 @@
+use anyhow::Result;
+use deltachat::constants::*;
+use deltachat::contact::{Contact, ContactId};
+use deltachat::{
+    chat::{get_chat_contacts, ChatVisibility},
+    chatlist::Chatlist,
+};
+use deltachat::{
+    chat::{Chat, ChatId},
+    message::MsgId,
+};
+use num_traits::cast::ToPrimitive;
+use serde::{Deserialize, Serialize};
+use typescript_type_def::TypeDef;
+
+use super::color_int_to_hex_string;
+
+#[derive(Deserialize, Serialize, TypeDef)]
+pub struct ChatListEntry(pub u32, pub u32);
+
+#[derive(Serialize, TypeDef)]
+#[serde(tag = "type")]
+pub enum ChatListItemFetchResult {
+    #[serde(rename_all = "camelCase")]
+    ChatListItem {
+        id: u32,
+        name: String,
+        avatar_path: Option<String>,
+        color: String,
+        last_updated: Option<i64>,
+        summary_text1: String,
+        summary_text2: String,
+        summary_status: u32,
+        is_protected: bool,
+        is_group: bool,
+        fresh_message_counter: usize,
+        is_self_talk: bool,
+        is_device_talk: bool,
+        is_sending_location: bool,
+        is_self_in_group: bool,
+        is_archived: bool,
+        is_pinned: bool,
+        is_muted: bool,
+        is_contact_request: bool,
+        /// true when chat is a broadcastlist
+        is_broadcast: bool,
+        /// contact id if this is a dm chat (for view profile entry in context menu)
+        dm_chat_contact: Option<u32>,
+        was_seen_recently: bool,
+    },
+    ArchiveLink,
+    #[serde(rename_all = "camelCase")]
+    Error {
+        id: u32,
+        error: String,
+    },
+}
+
+pub(crate) async fn get_chat_list_item_by_id(
+    ctx: &deltachat::context::Context,
+    entry: &ChatListEntry,
+) -> Result<ChatListItemFetchResult> {
+    let chat_id = ChatId::new(entry.0);
+    let last_msgid = match entry.1 {
+        0 => None,
+        _ => Some(MsgId::new(entry.1)),
+    };
+
+    if chat_id.is_archived_link() {
+        return Ok(ChatListItemFetchResult::ArchiveLink);
+    }
+
+    let chat = Chat::load_from_db(ctx, chat_id).await?;
+    let summary = Chatlist::get_summary2(ctx, chat_id, last_msgid, Some(&chat)).await?;
+
+    let summary_text1 = summary.prefix.map_or_else(String::new, |s| s.to_string());
+    let summary_text2 = summary.text.to_owned();
+
+    let visibility = chat.get_visibility();
+
+    let avatar_path = chat
+        .get_profile_image(ctx)
+        .await?
+        .map(|path| path.to_str().unwrap_or("invalid/path").to_owned());
+
+    let last_updated = match last_msgid {
+        Some(id) => {
+            let last_message = deltachat::message::Message::load_from_db(ctx, id).await?;
+            Some(last_message.get_timestamp() * 1000)
+        }
+        None => None,
+    };
+
+    let chat_contacts = get_chat_contacts(ctx, chat_id).await?;
+
+    let self_in_group = chat_contacts.contains(&ContactId::SELF);
+
+    let (dm_chat_contact, was_seen_recently) = if chat.get_type() == Chattype::Single {
+        let contact = chat_contacts.get(0);
+        let was_seen_recently = match contact {
+            Some(contact) => Contact::load_from_db(ctx, *contact)
+                .await?
+                .was_seen_recently(),
+            None => false,
+        };
+        (
+            contact.map(|contact_id| contact_id.to_u32()),
+            was_seen_recently,
+        )
+    } else {
+        (None, false)
+    };
+
+    let fresh_message_counter = chat_id.get_fresh_msg_cnt(ctx).await?;
+    let color = color_int_to_hex_string(chat.get_color(ctx).await?);
+
+    Ok(ChatListItemFetchResult::ChatListItem {
+        id: chat_id.to_u32(),
+        name: chat.get_name().to_owned(),
+        avatar_path,
+        color,
+        last_updated,
+        summary_text1,
+        summary_text2,
+        summary_status: summary.state.to_u32().expect("impossible"), // idea and a function to transform the constant to strings? or return string enum
+        is_protected: chat.is_protected(),
+        is_group: chat.get_type() == Chattype::Group,
+        fresh_message_counter,
+        is_self_talk: chat.is_self_talk(),
+        is_device_talk: chat.is_device_talk(),
+        is_self_in_group: self_in_group,
+        is_sending_location: chat.is_sending_locations(),
+        is_archived: visibility == ChatVisibility::Archived,
+        is_pinned: visibility == ChatVisibility::Pinned,
+        is_muted: chat.is_muted(),
+        is_contact_request: chat.is_contact_request(),
+        is_broadcast: chat.get_type() == Chattype::Broadcast,
+        dm_chat_contact,
+        was_seen_recently,
+    })
+}

deltachat-jsonrpc/src/api/types/contact.rs

@@ -0,0 +1,52 @@
+use anyhow::Result;
+use deltachat::contact::VerifiedStatus;
+use deltachat::context::Context;
+use serde::Serialize;
+use typescript_type_def::TypeDef;
+
+use super::color_int_to_hex_string;
+
+#[derive(Serialize, TypeDef)]
+#[serde(rename = "Contact", rename_all = "camelCase")]
+pub struct ContactObject {
+    address: String,
+    color: String,
+    auth_name: String,
+    status: String,
+    display_name: String,
+    id: u32,
+    name: String,
+    profile_image: Option<String>, // BLOBS
+    name_and_addr: String,
+    is_blocked: bool,
+    is_verified: bool,
+    was_seen_recently: bool,
+}
+
+impl ContactObject {
+    pub async fn try_from_dc_contact(
+        context: &Context,
+        contact: deltachat::contact::Contact,
+    ) -> Result<Self> {
+        let profile_image = match contact.get_profile_image(context).await? {
+            Some(path_buf) => path_buf.to_str().map(|s| s.to_owned()),
+            None => None,
+        };
+        let is_verified = contact.is_verified(context).await? == VerifiedStatus::BidirectVerified;
+
+        Ok(ContactObject {
+            address: contact.get_addr().to_owned(),
+            color: color_int_to_hex_string(contact.get_color()),
+            auth_name: contact.get_authname().to_owned(),
+            status: contact.get_status().to_owned(),
+            display_name: contact.get_display_name().to_owned(),
+            id: contact.id.to_u32(),
+            name: contact.get_name().to_owned(),
+            profile_image, //BLOBS
+            name_and_addr: contact.get_name_n_addr(),
+            is_blocked: contact.is_blocked(),
+            is_verified,
+            was_seen_recently: contact.was_seen_recently(),
+        })
+    }
+}

deltachat-jsonrpc/src/api/types/message.rs

@@ -0,0 +1,212 @@
+use anyhow::{anyhow, Result};
+use deltachat::contact::Contact;
+use deltachat::context::Context;
+use deltachat::message::Message;
+use deltachat::message::MsgId;
+use deltachat::message::Viewtype;
+use num_traits::cast::ToPrimitive;
+use serde::Deserialize;
+use serde::Serialize;
+use typescript_type_def::TypeDef;
+
+use super::contact::ContactObject;
+use super::webxdc::WebxdcMessageInfo;
+
+#[derive(Serialize, TypeDef)]
+#[serde(rename = "Message", rename_all = "camelCase")]
+pub struct MessageObject {
+    id: u32,
+    chat_id: u32,
+    from_id: u32,
+    quoted_text: Option<String>,
+    quoted_message_id: Option<u32>,
+    text: Option<String>,
+    has_location: bool,
+    has_html: bool,
+    view_type: MessageViewtype,
+    state: u32,
+
+    timestamp: i64,
+    sort_timestamp: i64,
+    received_timestamp: i64,
+    has_deviating_timestamp: bool,
+
+    // summary - use/create another function if you need it
+    subject: String,
+    show_padlock: bool,
+    is_setupmessage: bool,
+    is_info: bool,
+    is_forwarded: bool,
+
+    duration: i32,
+    dimensions_height: i32,
+    dimensions_width: i32,
+
+    videochat_type: Option<u32>,
+    videochat_url: Option<String>,
+
+    override_sender_name: Option<String>,
+    sender: ContactObject,
+
+    setup_code_begin: Option<String>,
+
+    file: Option<String>,
+    file_mime: Option<String>,
+    file_bytes: u64,
+    file_name: Option<String>,
+
+    webxdc_info: Option<WebxdcMessageInfo>,
+}
+
+impl MessageObject {
+    pub async fn from_message_id(context: &Context, message_id: u32) -> Result<Self> {
+        let msg_id = MsgId::new(message_id);
+        let message = Message::load_from_db(context, msg_id).await?;
+
+        let quoted_message_id = message
+            .quoted_message(context)
+            .await?
+            .map(|m| m.get_id().to_u32());
+
+        let sender_contact = Contact::load_from_db(context, message.get_from_id()).await?;
+        let sender = ContactObject::try_from_dc_contact(context, sender_contact).await?;
+        let file_bytes = message.get_filebytes(context).await;
+        let override_sender_name = message.get_override_sender_name();
+
+        let webxdc_info = if message.get_viewtype() == Viewtype::Webxdc {
+            Some(WebxdcMessageInfo::get_for_message(context, msg_id).await?)
+        } else {
+            None
+        };
+
+        Ok(MessageObject {
+            id: message_id,
+            chat_id: message.get_chat_id().to_u32(),
+            from_id: message.get_from_id().to_u32(),
+            quoted_text: message.quoted_text(),
+            quoted_message_id,
+            text: message.get_text(),
+            has_location: message.has_location(),
+            has_html: message.has_html(),
+            view_type: message.get_viewtype().into(),
+            state: message
+                .get_state()
+                .to_u32()
+                .ok_or_else(|| anyhow!("state conversion to number failed"))?,
+
+            timestamp: message.get_timestamp(),
+            sort_timestamp: message.get_sort_timestamp(),
+            received_timestamp: message.get_received_timestamp(),
+            has_deviating_timestamp: message.has_deviating_timestamp(),
+
+            subject: message.get_subject().to_owned(),
+            show_padlock: message.get_showpadlock(),
+            is_setupmessage: message.is_setupmessage(),
+            is_info: message.is_info(),
+            is_forwarded: message.is_forwarded(),
+
+            duration: message.get_duration(),
+            dimensions_height: message.get_height(),
+            dimensions_width: message.get_width(),
+
+            videochat_type: match message.get_videochat_type() {
+                Some(vct) => Some(
+                    vct.to_u32()
+                        .ok_or_else(|| anyhow!("state conversion to number failed"))?,
+                ),
+                None => None,
+            },
+            videochat_url: message.get_videochat_url(),
+
+            override_sender_name,
+            sender,
+
+            setup_code_begin: message.get_setupcodebegin(context).await,
+
+            file: match message.get_file(context) {
+                Some(path_buf) => path_buf.to_str().map(|s| s.to_owned()),
+                None => None,
+            }, //BLOBS
+            file_mime: message.get_filemime(),
+            file_bytes,
+            file_name: message.get_filename(),
+            webxdc_info,
+        })
+    }
+}
+
+#[derive(Serialize, Deserialize, TypeDef)]
+#[serde(rename = "Viewtype")]
+pub enum MessageViewtype {
+    Unknown,
+
+    /// Text message.
+    Text,
+
+    /// Image message.
+    /// If the image is an animated GIF, the type `Viewtype.Gif` should be used.
+    Image,
+
+    /// Animated GIF message.
+    Gif,
+
+    /// Message containing a sticker, similar to image.
+    /// If possible, the ui should display the image without borders in a transparent way.
+    /// A click on a sticker will offer to install the sticker set in some future.
+    Sticker,
+
+    /// Message containing an Audio file.
+    Audio,
+
+    /// A voice message that was directly recorded by the user.
+    /// For all other audio messages, the type `Viewtype.Audio` should be used.
+    Voice,
+
+    /// Video messages.
+    Video,
+
+    /// Message containing any file, eg. a PDF.
+    File,
+
+    /// Message is an invitation to a videochat.
+    VideochatInvitation,
+
+    /// Message is an webxdc instance.
+    Webxdc,
+}
+
+impl From<Viewtype> for MessageViewtype {
+    fn from(viewtype: Viewtype) -> Self {
+        match viewtype {
+            Viewtype::Unknown => MessageViewtype::Unknown,
+            Viewtype::Text => MessageViewtype::Text,
+            Viewtype::Image => MessageViewtype::Image,
+            Viewtype::Gif => MessageViewtype::Gif,
+            Viewtype::Sticker => MessageViewtype::Sticker,
+            Viewtype::Audio => MessageViewtype::Audio,
+            Viewtype::Voice => MessageViewtype::Voice,
+            Viewtype::Video => MessageViewtype::Video,
+            Viewtype::File => MessageViewtype::File,
+            Viewtype::VideochatInvitation => MessageViewtype::VideochatInvitation,
+            Viewtype::Webxdc => MessageViewtype::Webxdc,
+        }
+    }
+}
+
+impl From<MessageViewtype> for Viewtype {
+    fn from(viewtype: MessageViewtype) -> Self {
+        match viewtype {
+            MessageViewtype::Unknown => Viewtype::Unknown,
+            MessageViewtype::Text => Viewtype::Text,
+            MessageViewtype::Image => Viewtype::Image,
+            MessageViewtype::Gif => Viewtype::Gif,
+            MessageViewtype::Sticker => Viewtype::Sticker,
+            MessageViewtype::Audio => Viewtype::Audio,
+            MessageViewtype::Voice => Viewtype::Voice,
+            MessageViewtype::Video => Viewtype::Video,
+            MessageViewtype::File => Viewtype::File,
+            MessageViewtype::VideochatInvitation => Viewtype::VideochatInvitation,
+            MessageViewtype::Webxdc => Viewtype::Webxdc,
+        }
+    }
+}

deltachat-jsonrpc/src/api/types/mod.rs

@@ -0,0 +1,229 @@
+use deltachat::qr::Qr;
+use serde::Serialize;
+use typescript_type_def::TypeDef;
+
+pub mod account;
+pub mod chat;
+pub mod chat_list;
+pub mod contact;
+pub mod message;
+pub mod provider_info;
+pub mod webxdc;
+
+pub fn color_int_to_hex_string(color: u32) -> String {
+    format!("{:#08x}", color).replace("0x", "#")
+}
+
+fn maybe_empty_string_to_option(string: String) -> Option<String> {
+    if string.is_empty() {
+        None
+    } else {
+        Some(string)
+    }
+}
+
+#[derive(Serialize, TypeDef)]
+#[serde(rename = "Qr", rename_all = "camelCase")]
+#[serde(tag = "type")]
+pub enum QrObject {
+    AskVerifyContact {
+        contact_id: u32,
+        fingerprint: String,
+        invitenumber: String,
+        authcode: String,
+    },
+    AskVerifyGroup {
+        grpname: String,
+        grpid: String,
+        contact_id: u32,
+        fingerprint: String,
+        invitenumber: String,
+        authcode: String,
+    },
+    FprOk {
+        contact_id: u32,
+    },
+    FprMismatch {
+        contact_id: Option<u32>,
+    },
+    FprWithoutAddr {
+        fingerprint: String,
+    },
+    Account {
+        domain: String,
+    },
+    WebrtcInstance {
+        domain: String,
+        instance_pattern: String,
+    },
+    Addr {
+        contact_id: u32,
+        draft: Option<String>,
+    },
+    Url {
+        url: String,
+    },
+    Text {
+        text: String,
+    },
+    WithdrawVerifyContact {
+        contact_id: u32,
+        fingerprint: String,
+        invitenumber: String,
+        authcode: String,
+    },
+    WithdrawVerifyGroup {
+        grpname: String,
+        grpid: String,
+        contact_id: u32,
+        fingerprint: String,
+        invitenumber: String,
+        authcode: String,
+    },
+    ReviveVerifyContact {
+        contact_id: u32,
+        fingerprint: String,
+        invitenumber: String,
+        authcode: String,
+    },
+    ReviveVerifyGroup {
+        grpname: String,
+        grpid: String,
+        contact_id: u32,
+        fingerprint: String,
+        invitenumber: String,
+        authcode: String,
+    },
+}
+
+impl From<Qr> for QrObject {
+    fn from(qr: Qr) -> Self {
+        match qr {
+            Qr::AskVerifyContact {
+                contact_id,
+                fingerprint,
+                invitenumber,
+                authcode,
+            } => {
+                let contact_id = contact_id.to_u32();
+                let fingerprint = fingerprint.to_string();
+                QrObject::AskVerifyContact {
+                    contact_id,
+                    fingerprint,
+                    invitenumber,
+                    authcode,
+                }
+            }
+            Qr::AskVerifyGroup {
+                grpname,
+                grpid,
+                contact_id,
+                fingerprint,
+                invitenumber,
+                authcode,
+            } => {
+                let contact_id = contact_id.to_u32();
+                let fingerprint = fingerprint.to_string();
+                QrObject::AskVerifyGroup {
+                    grpname,
+                    grpid,
+                    contact_id,
+                    fingerprint,
+                    invitenumber,
+                    authcode,
+                }
+            }
+            Qr::FprOk { contact_id } => {
+                let contact_id = contact_id.to_u32();
+                QrObject::FprOk { contact_id }
+            }
+            Qr::FprMismatch { contact_id } => {
+                let contact_id = contact_id.map(|contact_id| contact_id.to_u32());
+                QrObject::FprMismatch { contact_id }
+            }
+            Qr::FprWithoutAddr { fingerprint } => QrObject::FprWithoutAddr { fingerprint },
+            Qr::Account { domain } => QrObject::Account { domain },
+            Qr::WebrtcInstance {
+                domain,
+                instance_pattern,
+            } => QrObject::WebrtcInstance {
+                domain,
+                instance_pattern,
+            },
+            Qr::Addr { contact_id, draft } => {
+                let contact_id = contact_id.to_u32();
+                QrObject::Addr { contact_id, draft }
+            }
+            Qr::Url { url } => QrObject::Url { url },
+            Qr::Text { text } => QrObject::Text { text },
+            Qr::WithdrawVerifyContact {
+                contact_id,
+                fingerprint,
+                invitenumber,
+                authcode,
+            } => {
+                let contact_id = contact_id.to_u32();
+                let fingerprint = fingerprint.to_string();
+                QrObject::WithdrawVerifyContact {
+                    contact_id,
+                    fingerprint,
+                    invitenumber,
+                    authcode,
+                }
+            }
+            Qr::WithdrawVerifyGroup {
+                grpname,
+                grpid,
+                contact_id,
+                fingerprint,
+                invitenumber,
+                authcode,
+            } => {
+                let contact_id = contact_id.to_u32();
+                let fingerprint = fingerprint.to_string();
+                QrObject::WithdrawVerifyGroup {
+                    grpname,
+                    grpid,
+                    contact_id,
+                    fingerprint,
+                    invitenumber,
+                    authcode,
+                }
+            }
+            Qr::ReviveVerifyContact {
+                contact_id,
+                fingerprint,
+                invitenumber,
+                authcode,
+            } => {
+                let contact_id = contact_id.to_u32();
+                let fingerprint = fingerprint.to_string();
+                QrObject::ReviveVerifyContact {
+                    contact_id,
+                    fingerprint,
+                    invitenumber,
+                    authcode,
+                }
+            }
+            Qr::ReviveVerifyGroup {
+                grpname,
+                grpid,
+                contact_id,
+                fingerprint,
+                invitenumber,
+                authcode,
+            } => {
+                let contact_id = contact_id.to_u32();
+                let fingerprint = fingerprint.to_string();
+                QrObject::ReviveVerifyGroup {
+                    grpname,
+                    grpid,
+                    contact_id,
+                    fingerprint,
+                    invitenumber,
+                    authcode,
+                }
+            }
+        }
+    }
+}

deltachat-jsonrpc/src/api/types/provider_info.rs

@@ -0,0 +1,22 @@
+use deltachat::provider::Provider;
+use num_traits::cast::ToPrimitive;
+use serde::Serialize;
+use typescript_type_def::TypeDef;
+
+#[derive(Serialize, TypeDef)]
+#[serde(rename_all = "camelCase")]
+pub struct ProviderInfo {
+    pub before_login_hint: String,
+    pub overview_page: String,
+    pub status: u32, // in reality this is an enum, but for simlicity and because it gets converted into a number anyway, we use an u32 here.
+}
+
+impl ProviderInfo {
+    pub fn from_dc_type(provider: Option<&Provider>) -> Option<Self> {
+        provider.map(|p| ProviderInfo {
+            before_login_hint: p.before_login_hint.to_owned(),
+            overview_page: p.overview_page.to_owned(),
+            status: p.status.to_u32().unwrap(),
+        })
+    }
+}

deltachat-jsonrpc/src/api/types/webxdc.rs

@@ -0,0 +1,64 @@
+use deltachat::{
+    context::Context,
+    message::{Message, MsgId},
+    webxdc::WebxdcInfo,
+};
+use serde::Serialize;
+use typescript_type_def::TypeDef;
+
+use super::maybe_empty_string_to_option;
+
+#[derive(Serialize, TypeDef)]
+#[serde(rename = "WebxdcMessageInfo", rename_all = "camelCase")]
+pub struct WebxdcMessageInfo {
+    /// The name of the app.
+    ///
+    /// Defaults to the filename if not set in the manifest.
+    name: String,
+    /// App icon file name.
+    /// Defaults to an standard icon if nothing is set in the manifest.
+    ///
+    /// To get the file, use dc_msg_get_webxdc_blob(). (not yet in jsonrpc, use rust api or cffi for it)
+    ///
+    /// App icons should should be square,
+    /// the implementations will add round corners etc. as needed.
+    icon: String,
+    /// if the Webxdc represents a document, then this is the name of the document
+    document: Option<String>,
+    /// short string describing the state of the app,
+    /// sth. as "2 votes", "Highscore: 123",
+    /// can be changed by the apps
+    summary: Option<String>,
+    /// URL where the source code of the Webxdc and other information can be found;
+    /// defaults to an empty string.
+    /// Implementations may offer an menu or a button to open this URL.
+    source_code_url: Option<String>,
+    /// True if full internet access should be granted to the app.
+    internet_access: bool,
+}
+
+impl WebxdcMessageInfo {
+    pub async fn get_for_message(
+        context: &Context,
+        instance_message_id: MsgId,
+    ) -> anyhow::Result<Self> {
+        let message = Message::load_from_db(context, instance_message_id).await?;
+        let WebxdcInfo {
+            name,
+            icon,
+            document,
+            summary,
+            source_code_url,
+            internet_access,
+        } = message.get_webxdc_info(context).await?;
+
+        Ok(Self {
+            name,
+            icon,
+            document: maybe_empty_string_to_option(document),
+            summary: maybe_empty_string_to_option(summary),
+            source_code_url: maybe_empty_string_to_option(source_code_url),
+            internet_access,
+        })
+    }
+}

deltachat-jsonrpc/src/lib.rs

@@ -0,0 +1,92 @@
+pub mod api;
+pub use api::events;
+pub use yerpc;
+
+#[cfg(test)]
+mod tests {
+    use super::api::{Accounts, CommandApi};
+    use async_channel::unbounded;
+    use futures::StreamExt;
+    use tempfile::TempDir;
+    use yerpc::{RpcClient, RpcSession};
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn basic_json_rpc_functionality() -> anyhow::Result<()> {
+        let tmp_dir = TempDir::new().unwrap().path().into();
+        let accounts = Accounts::new(tmp_dir).await?;
+        let api = CommandApi::new(accounts);
+
+        let (sender, mut receiver) = unbounded::<String>();
+
+        let (client, mut rx) = RpcClient::new();
+        let session = RpcSession::new(client, api);
+        tokio::spawn({
+            async move {
+                while let Some(message) = rx.next().await {
+                    let message = serde_json::to_string(&message)?;
+                    sender.send(message).await?;
+                }
+                let res: Result<(), anyhow::Error> = Ok(());
+                res
+            }
+        });
+
+        {
+            let request = r#"{"jsonrpc":"2.0","method":"add_account","params":[],"id":1}"#;
+            let response = r#"{"jsonrpc":"2.0","id":1,"result":1}"#;
+            session.handle_incoming(request).await;
+            let result = receiver.next().await;
+            println!("{:?}", result);
+            assert_eq!(result, Some(response.to_owned()));
+        }
+        {
+            let request = r#"{"jsonrpc":"2.0","method":"get_all_account_ids","params":[],"id":2}"#;
+            let response = r#"{"jsonrpc":"2.0","id":2,"result":[1]}"#;
+            session.handle_incoming(request).await;
+            let result = receiver.next().await;
+            println!("{:?}", result);
+            assert_eq!(result, Some(response.to_owned()));
+        }
+
+        Ok(())
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn test_batch_set_config() -> anyhow::Result<()> {
+        let tmp_dir = TempDir::new().unwrap().path().into();
+        let accounts = Accounts::new(tmp_dir).await?;
+        let api = CommandApi::new(accounts);
+
+        let (sender, mut receiver) = unbounded::<String>();
+
+        let (client, mut rx) = RpcClient::new();
+        let session = RpcSession::new(client, api);
+        tokio::spawn({
+            async move {
+                while let Some(message) = rx.next().await {
+                    let message = serde_json::to_string(&message)?;
+                    sender.send(message).await?;
+                }
+                let res: Result<(), anyhow::Error> = Ok(());
+                res
+            }
+        });
+
+        {
+            let request = r#"{"jsonrpc":"2.0","method":"add_account","params":[],"id":1}"#;
+            let response = r#"{"jsonrpc":"2.0","id":1,"result":1}"#;
+            session.handle_incoming(request).await;
+            let result = receiver.next().await;
+            assert_eq!(result, Some(response.to_owned()));
+        }
+        {
+            let request = r#"{"jsonrpc":"2.0","method":"batch_set_config","id":2,"params":[1,{"addr":"","mail_user":"","mail_pw":"","mail_server":"","mail_port":"","mail_security":"","imap_certificate_checks":"","send_user":"","send_pw":"","send_server":"","send_port":"","send_security":"","smtp_certificate_checks":"","socks5_enabled":"0","socks5_host":"","socks5_port":"","socks5_user":"","socks5_password":""}]}"#;
+            let response = r#"{"jsonrpc":"2.0","id":2,"result":null}"#;
+            session.handle_incoming(request).await;
+            let result = receiver.next().await;
+            assert_eq!(result, Some(response.to_owned()));
+        }
+
+        Ok(())
+    }
+}

deltachat-jsonrpc/src/webserver.rs

@@ -0,0 +1,54 @@
+use axum::{extract::ws::WebSocketUpgrade, response::Response, routing::get, Extension, Router};
+use std::net::SocketAddr;
+use std::path::PathBuf;
+use yerpc::axum::handle_ws_rpc;
+use yerpc::{RpcClient, RpcSession};
+
+mod api;
+use api::events::event_to_json_rpc_notification;
+use api::{Accounts, CommandApi};
+
+const DEFAULT_PORT: u16 = 20808;
+
+#[tokio::main(flavor = "multi_thread")]
+async fn main() -> Result<(), std::io::Error> {
+    env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init();
+
+    let path = std::env::var("DC_ACCOUNTS_PATH").unwrap_or_else(|_| "./accounts".to_string());
+    let port = std::env::var("DC_PORT")
+        .map(|port| port.parse::<u16>().expect("DC_PORT must be a number"))
+        .unwrap_or(DEFAULT_PORT);
+    log::info!("Starting with accounts directory `{path}`.");
+    let accounts = Accounts::new(PathBuf::from(&path)).await.unwrap();
+    let state = CommandApi::new(accounts);
+
+    let app = Router::new()
+        .route("/ws", get(handler))
+        .layer(Extension(state.clone()));
+
+    tokio::spawn(async move {
+        state.accounts.read().await.start_io().await;
+    });
+
+    let addr = SocketAddr::from(([127, 0, 0, 1], port));
+    log::info!("JSON-RPC WebSocket server listening on {}", addr);
+    axum::Server::bind(&addr)
+        .serve(app.into_make_service())
+        .await
+        .unwrap();
+
+    Ok(())
+}
+
+async fn handler(ws: WebSocketUpgrade, Extension(api): Extension<CommandApi>) -> Response {
+    let (client, out_receiver) = RpcClient::new();
+    let session = RpcSession::new(client.clone(), api.clone());
+    tokio::spawn(async move {
+        let events = api.accounts.read().await.get_event_emitter().await;
+        while let Some(event) = events.recv().await {
+            let event = event_to_json_rpc_notification(event);
+            client.send_notification("event", Some(event)).await.ok();
+        }
+    });
+    handle_ws_rpc(ws, out_receiver, session).await
+}

deltachat-jsonrpc/typescript/.gitignore

@@ -0,0 +1,8 @@
+node_modules
+dist
+test_dist
+coverage
+yarn.lock
+package-lock.json
+docs
+accounts

deltachat-jsonrpc/typescript/.npmignore

@@ -0,0 +1,6 @@
+node_modules
+accounts
+docs
+coverage
+yarn*
+package-lock.json

deltachat-jsonrpc/typescript/.prettierignore

@@ -0,0 +1,3 @@
+coverage
+dist
+generated

deltachat-jsonrpc/typescript/deltachat.ts

@@ -0,0 +1 @@
+export * from "./src/lib.js";

deltachat-jsonrpc/typescript/example.html

@@ -0,0 +1,56 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <title>DeltaChat JSON-RPC example</title>
+    <style>
+      body {
+        font-family: monospace;
+        background: black;
+        color: grey;
+      }
+      .grid {
+        display: grid;
+        grid-template-columns: 3fr 1fr;
+        grid-template-areas: "a a" "b c";
+      }
+      .message {
+        color: red;
+      }
+      #header {
+        grid-area: a;
+        color: white;
+        font-size: 1.2rem;
+      }
+      #header a {
+        color: white;
+        font-weight: bold;
+      }
+      #main {
+        grid-area: b;
+        color: green;
+      }
+      #main h2,
+      #main h3 {
+        color: blue;
+      }
+      #side {
+        grid-area: c;
+        color: #777;
+        overflow-y: auto;
+      }
+    </style>
+    <script type="module" src="dist/example.bundle.js"></script>
+  </head>
+  <body>
+    <h1>DeltaChat JSON-RPC example</h1>
+    <div class="grid">
+      <div id="header"></div>
+      <div id="main"></div>
+      <div id="side"><h2>log</h2></div>
+    </div>
+    <p>
+      Tip: open the dev console and use the client with
+      <code>window.client</code>
+    </p>
+  </body>
+</html>

deltachat-jsonrpc/typescript/example/example.ts

@@ -0,0 +1,109 @@
+import { DeltaChat, DeltaChatEvent } from "../deltachat.js";
+
+var SELECTED_ACCOUNT = 0;
+
+window.addEventListener("DOMContentLoaded", (_event) => {
+  (window as any).selectDeltaAccount = (id: string) => {
+    SELECTED_ACCOUNT = Number(id);
+    window.dispatchEvent(new Event("account-changed"));
+  };
+  console.log('launch run script...')
+  run().catch((err) => console.error("run failed", err));
+});
+
+async function run() {
+  const $main = document.getElementById("main")!;
+  const $side = document.getElementById("side")!;
+  const $head = document.getElementById("header")!;
+
+  const client = new DeltaChat('ws://localhost:20808/ws')
+
+  ;(window as any).client = client.rpc;
+
+  client.on("ALL", event => {
+    onIncomingEvent(event)
+  })
+
+  window.addEventListener("account-changed", async (_event: Event) => {
+    listChatsForSelectedAccount();
+  });
+
+  await Promise.all([loadAccountsInHeader(), listChatsForSelectedAccount()]);
+
+  async function loadAccountsInHeader() {
+    console.log('load accounts')
+    const accounts = await client.rpc.getAllAccounts();
+    console.log('accounts loaded', accounts)
+    for (const account of accounts) {
+      if (account.type === "Configured") {
+        write(
+          $head,
+          `<a href="#" onclick="selectDeltaAccount(${account.id})">
+          ${account.id}: ${account.addr!}
+          </a>&nbsp;`
+        );
+      } else {
+        write(
+          $head,
+          `<a href="#">
+          ${account.id}: (unconfigured)
+          </a>&nbsp;`
+        )
+      }
+    }
+  }
+
+  async function listChatsForSelectedAccount() {
+    clear($main);
+    const selectedAccount = SELECTED_ACCOUNT
+    const info = await client.rpc.getAccountInfo(selectedAccount);
+    if (info.type !== "Configured") {
+      return write($main, "Account is not configured");
+    }
+    write($main, `<h2>${info.addr!}</h2>`);
+    const chats = await client.rpc.getChatlistEntries(
+      selectedAccount,
+      0,
+      null,
+      null
+    );
+    for (const [chatId, _messageId] of chats) {
+      const chat = await client.rpc.chatlistGetFullChatById(
+        selectedAccount,
+        chatId
+      );
+      write($main, `<h3>${chat.name}</h3>`);
+      const messageIds = await client.rpc.messageListGetMessageIds(
+        selectedAccount,
+        chatId,
+        0
+      );
+      const messages = await client.rpc.messageGetMessages(
+        selectedAccount,
+        messageIds
+      );
+      for (const [_messageId, message] of Object.entries(messages)) {
+        write($main, `<p>${message.text}</p>`);
+      }
+    }
+  }
+
+  function onIncomingEvent(event: DeltaChatEvent) {
+    write(
+      $side,
+      `
+        <p class="message">
+          [<strong>${event.id}</strong> on account ${event.contextId}]<br>
+          <em>f1:</em> ${JSON.stringify(event.field1)}<br>
+          <em>f2:</em> ${JSON.stringify(event.field2)}
+        </p>`
+    );
+  }
+}
+
+function write(el: HTMLElement, html: string) {
+  el.innerHTML += html;
+}
+function clear(el: HTMLElement) {
+  el.innerHTML = "";
+}

deltachat-jsonrpc/typescript/example/node-add-account.js

@@ -0,0 +1,26 @@
+import { DeltaChat } from "../dist/deltachat.js";
+
+run().catch(console.error);
+
+async function run() {
+  const delta = new DeltaChat('ws://localhost:20808/ws');
+  delta.on("event", (event) => {
+    console.log("event", event.data);
+  });
+
+  const email = process.argv[2]
+  const password = process.argv[3]
+  if (!email || !password) throw new Error('USAGE: node node-add-account.js <EMAILADDRESS> <PASSWORD>')
+  console.log(`creating acccount for ${email}`)
+  const id = await delta.rpc.addAccount()
+  console.log(`created account id ${id}`)
+  await delta.rpc.setConfig(id, "addr", email);
+  await delta.rpc.setConfig(id, "mail_pw", password);
+  console.log('configuration updated')
+  await delta.rpc.configure(id)
+  console.log('account configured!')
+
+  const accounts = await delta.rpc.getAllAccounts();
+  console.log("accounts", accounts);
+  console.log("waiting for events...")
+}

deltachat-jsonrpc/typescript/example/node-demo.js

@@ -0,0 +1,14 @@
+import { DeltaChat } from "../dist/deltachat.js";
+
+run().catch(console.error);
+
+async function run() {
+  const delta = new DeltaChat();
+  delta.on("event", (event) => {
+    console.log("event", event.data);
+  });
+
+  const accounts = await delta.rpc.getAllAccounts();
+  console.log("accounts", accounts);
+  console.log("waiting for events...")
+}

deltachat-jsonrpc/typescript/generated/client.ts

@@ -0,0 +1,439 @@
+// AUTO-GENERATED by yerpc-derive
+
+import * as T from "./types.js"
+import * as RPC from "./jsonrpc.js"
+
+type RequestMethod = (method: string, params?: RPC.Params) => Promise<unknown>;
+type NotificationMethod = (method: string, params?: RPC.Params) => void;
+
+interface Transport {
+  request: RequestMethod,
+  notification: NotificationMethod
+}
+
+export class RawClient {
+  constructor(private _transport: Transport) {}
+
+  /**
+   * Check if an email address is valid.
+   */
+  public checkEmailValidity(email: string): Promise<boolean> {
+    return (this._transport.request('check_email_validity', [email] as RPC.Params)) as Promise<boolean>;
+  }
+
+  /**
+   * Get general system info.
+   */
+  public getSystemInfo(): Promise<Record<string,string>> {
+    return (this._transport.request('get_system_info', [] as RPC.Params)) as Promise<Record<string,string>>;
+  }
+
+
+  public addAccount(): Promise<T.U32> {
+    return (this._transport.request('add_account', [] as RPC.Params)) as Promise<T.U32>;
+  }
+
+
+  public removeAccount(accountId: T.U32): Promise<null> {
+    return (this._transport.request('remove_account', [accountId] as RPC.Params)) as Promise<null>;
+  }
+
+
+  public getAllAccountIds(): Promise<(T.U32)[]> {
+    return (this._transport.request('get_all_account_ids', [] as RPC.Params)) as Promise<(T.U32)[]>;
+  }
+
+  /**
+   * Select account id for internally selected state.
+   * TODO: Likely this is deprecated as all methods take an account id now.
+   */
+  public selectAccount(id: T.U32): Promise<null> {
+    return (this._transport.request('select_account', [id] as RPC.Params)) as Promise<null>;
+  }
+
+  /**
+   * Get the selected account id of the internal state..
+   * TODO: Likely this is deprecated as all methods take an account id now.
+   */
+  public getSelectedAccountId(): Promise<(T.U32|null)> {
+    return (this._transport.request('get_selected_account_id', [] as RPC.Params)) as Promise<(T.U32|null)>;
+  }
+
+  /**
+   * Get a list of all configured accounts.
+   */
+  public getAllAccounts(): Promise<(T.Account)[]> {
+    return (this._transport.request('get_all_accounts', [] as RPC.Params)) as Promise<(T.Account)[]>;
+  }
+
+  /**
+   * Get top-level info for an account.
+   */
+  public getAccountInfo(accountId: T.U32): Promise<T.Account> {
+    return (this._transport.request('get_account_info', [accountId] as RPC.Params)) as Promise<T.Account>;
+  }
+
+  /**
+   * Returns provider for the given domain.
+   *
+   * This function looks up domain in offline database.
+   *
+   * For compatibility, email address can be passed to this function
+   * instead of the domain.
+   */
+  public getProviderInfo(accountId: T.U32, email: string): Promise<(T.ProviderInfo|null)> {
+    return (this._transport.request('get_provider_info', [accountId, email] as RPC.Params)) as Promise<(T.ProviderInfo|null)>;
+  }
+
+  /**
+   * Checks if the context is already configured.
+   */
+  public isConfigured(accountId: T.U32): Promise<boolean> {
+    return (this._transport.request('is_configured', [accountId] as RPC.Params)) as Promise<boolean>;
+  }
+
+  /**
+   * Get system info for an account.
+   */
+  public getInfo(accountId: T.U32): Promise<Record<string,string>> {
+    return (this._transport.request('get_info', [accountId] as RPC.Params)) as Promise<Record<string,string>>;
+  }
+
+
+  public setConfig(accountId: T.U32, key: string, value: (string|null)): Promise<null> {
+    return (this._transport.request('set_config', [accountId, key, value] as RPC.Params)) as Promise<null>;
+  }
+
+
+  public batchSetConfig(accountId: T.U32, config: Record<string,(string|null)>): Promise<null> {
+    return (this._transport.request('batch_set_config', [accountId, config] as RPC.Params)) as Promise<null>;
+  }
+
+  /**
+   * Set configuration values from a QR code. (technically from the URI that is stored in the qrcode)
+   * Before this function is called, `checkQr()` should confirm the type of the
+   * QR code is `account` or `webrtcInstance`.
+   *
+   * Internally, the function will call dc_set_config() with the appropriate keys,
+   */
+  public setConfigFromQr(accountId: T.U32, qrContent: string): Promise<null> {
+    return (this._transport.request('set_config_from_qr', [accountId, qrContent] as RPC.Params)) as Promise<null>;
+  }
+
+
+  public checkQr(accountId: T.U32, qrContent: string): Promise<T.Qr> {
+    return (this._transport.request('check_qr', [accountId, qrContent] as RPC.Params)) as Promise<T.Qr>;
+  }
+
+
+  public getConfig(accountId: T.U32, key: string): Promise<(string|null)> {
+    return (this._transport.request('get_config', [accountId, key] as RPC.Params)) as Promise<(string|null)>;
+  }
+
+
+  public batchGetConfig(accountId: T.U32, keys: (string)[]): Promise<Record<string,(string|null)>> {
+    return (this._transport.request('batch_get_config', [accountId, keys] as RPC.Params)) as Promise<Record<string,(string|null)>>;
+  }
+
+  /**
+   * Configures this account with the currently set parameters.
+   * Setup the credential config before calling this.
+   */
+  public configure(accountId: T.U32): Promise<null> {
+    return (this._transport.request('configure', [accountId] as RPC.Params)) as Promise<null>;
+  }
+
+  /**
+   * Signal an ongoing process to stop.
+   */
+  public stopOngoingProcess(accountId: T.U32): Promise<null> {
+    return (this._transport.request('stop_ongoing_process', [accountId] as RPC.Params)) as Promise<null>;
+  }
+
+  /**
+   * Returns the message IDs of all _fresh_ messages of any chat.
+   * Typically used for implementing notification summaries
+   * or badge counters e.g. on the app icon.
+   * The list is already sorted and starts with the most recent fresh message.
+   *
+   * Messages belonging to muted chats or to the contact requests are not returned;
+   * these messages should not be notified
+   * and also badge counters should not include these messages.
+   *
+   * To get the number of fresh messages for a single chat, muted or not,
+   * use `get_fresh_msg_cnt()`.
+   */
+  public getFreshMsgs(accountId: T.U32): Promise<(T.U32)[]> {
+    return (this._transport.request('get_fresh_msgs', [accountId] as RPC.Params)) as Promise<(T.U32)[]>;
+  }
+
+  /**
+   * Get the number of _fresh_ messages in a chat.
+   * Typically used to implement a badge with a number in the chatlist.
+   *
+   * If the specified chat is muted,
+   * the UI should show the badge counter "less obtrusive",
+   * e.g. using "gray" instead of "red" color.
+   */
+  public getFreshMsgCnt(accountId: T.U32, chatId: T.U32): Promise<T.Usize> {
+    return (this._transport.request('get_fresh_msg_cnt', [accountId, chatId] as RPC.Params)) as Promise<T.Usize>;
+  }
+
+
+  public autocryptInitiateKeyTransfer(accountId: T.U32): Promise<string> {
+    return (this._transport.request('autocrypt_initiate_key_transfer', [accountId] as RPC.Params)) as Promise<string>;
+  }
+
+
+  public autocryptContinueKeyTransfer(accountId: T.U32, messageId: T.U32, setupCode: string): Promise<null> {
+    return (this._transport.request('autocrypt_continue_key_transfer', [accountId, messageId, setupCode] as RPC.Params)) as Promise<null>;
+  }
+
+
+  public getChatlistEntries(accountId: T.U32, listFlags: (T.U32|null), queryString: (string|null), queryContactId: (T.U32|null)): Promise<(T.ChatListEntry)[]> {
+    return (this._transport.request('get_chatlist_entries', [accountId, listFlags, queryString, queryContactId] as RPC.Params)) as Promise<(T.ChatListEntry)[]>;
+  }
+
+
+  public getChatlistItemsByEntries(accountId: T.U32, entries: (T.ChatListEntry)[]): Promise<Record<T.U32,T.ChatListItemFetchResult>> {
+    return (this._transport.request('get_chatlist_items_by_entries', [accountId, entries] as RPC.Params)) as Promise<Record<T.U32,T.ChatListItemFetchResult>>;
+  }
+
+
+  public chatlistGetFullChatById(accountId: T.U32, chatId: T.U32): Promise<T.FullChat> {
+    return (this._transport.request('chatlist_get_full_chat_by_id', [accountId, chatId] as RPC.Params)) as Promise<T.FullChat>;
+  }
+
+
+  public acceptChat(accountId: T.U32, chatId: T.U32): Promise<null> {
+    return (this._transport.request('accept_chat', [accountId, chatId] as RPC.Params)) as Promise<null>;
+  }
+
+
+  public blockChat(accountId: T.U32, chatId: T.U32): Promise<null> {
+    return (this._transport.request('block_chat', [accountId, chatId] as RPC.Params)) as Promise<null>;
+  }
+
+  /**
+   * Delete a chat.
+   *
+   * Messages are deleted from the device and the chat database entry is deleted.
+   * After that, the event #DC_EVENT_MSGS_CHANGED is posted.
+   *
+   * Things that are _not done_ implicitly:
+   *
+   * - Messages are **not deleted from the server**.
+   * - The chat or the contact is **not blocked**, so new messages from the user/the group may appear as a contact request
+   *   and the user may create the chat again.
+   * - **Groups are not left** - this would
+   *   be unexpected as (1) deleting a normal chat also does not prevent new mails
+   *   from arriving, (2) leaving a group requires sending a message to
+   *   all group members - especially for groups not used for a longer time, this is
+   *   really unexpected when deletion results in contacting all members again,
+   *   (3) only leaving groups is also a valid usecase.
+   *
+   * To leave a chat explicitly, use dc_remove_contact_from_chat() with
+   * chat_id=DC_CONTACT_ID_SELF)
+   */
+  public deleteChat(accountId: T.U32, chatId: T.U32): Promise<null> {
+    return (this._transport.request('delete_chat', [accountId, chatId] as RPC.Params)) as Promise<null>;
+  }
+
+  /**
+   * Get encryption info for a chat.
+   * Get a multi-line encryption info, containing encryption preferences of all members.
+   * Can be used to find out why messages sent to group are not encrypted.
+   *
+   * returns Multi-line text
+   */
+  public getChatEncryptionInfo(accountId: T.U32, chatId: T.U32): Promise<string> {
+    return (this._transport.request('get_chat_encryption_info', [accountId, chatId] as RPC.Params)) as Promise<string>;
+  }
+
+  /**
+   * Get QR code (text and SVG) that will offer an Setup-Contact or Verified-Group invitation.
+   * The QR code is compatible to the OPENPGP4FPR format
+   * so that a basic fingerprint comparison also works e.g. with OpenKeychain.
+   *
+   * The scanning device will pass the scanned content to `checkQr()` then;
+   * if `checkQr()` returns `askVerifyContact` or `askVerifyGroup`
+   * an out-of-band-verification can be joined using dc_join_securejoin()
+   *
+   * chat_id: If set to a group-chat-id,
+   *     the Verified-Group-Invite protocol is offered in the QR code;
+   *     works for protected groups as well as for normal groups.
+   *     If not set, the Setup-Contact protocol is offered in the QR code.
+   *     See https://countermitm.readthedocs.io/en/latest/new.html
+   *     for details about both protocols.
+   */
+  public getChatSecurejoinQrCodeSvg(accountId: T.U32, chatId: (T.U32|null)): Promise<[string,string]> {
+    return (this._transport.request('get_chat_securejoin_qr_code_svg', [accountId, chatId] as RPC.Params)) as Promise<[string,string]>;
+  }
+
+
+  public leaveGroup(accountId: T.U32, chatId: T.U32): Promise<null> {
+    return (this._transport.request('leave_group', [accountId, chatId] as RPC.Params)) as Promise<null>;
+  }
+
+  /**
+   * Remove a member from a group.
+   *
+   * If the group is already _promoted_ (any message was sent to the group),
+   * all group members are informed by a special status message that is sent automatically by this function.
+   *
+   * Sends out #DC_EVENT_CHAT_MODIFIED and #DC_EVENT_MSGS_CHANGED if a status message was sent.
+   */
+  public removeContactFromChat(accountId: T.U32, chatId: T.U32, contactId: T.U32): Promise<null> {
+    return (this._transport.request('remove_contact_from_chat', [accountId, chatId, contactId] as RPC.Params)) as Promise<null>;
+  }
+
+  /**
+   * Add a member to a group.
+   *
+   * If the group is already _promoted_ (any message was sent to the group),
+   * all group members are informed by a special status message that is sent automatically by this function.
+   *
+   * If the group has group protection enabled, only verified contacts can be added to the group.
+   *
+   * Sends out #DC_EVENT_CHAT_MODIFIED and #DC_EVENT_MSGS_CHANGED if a status message was sent.
+   */
+  public addContactToChat(accountId: T.U32, chatId: T.U32, contactId: T.U32): Promise<null> {
+    return (this._transport.request('add_contact_to_chat', [accountId, chatId, contactId] as RPC.Params)) as Promise<null>;
+  }
+
+
+  public addDeviceMessage(accountId: T.U32, label: string, text: string): Promise<T.U32> {
+    return (this._transport.request('add_device_message', [accountId, label, text] as RPC.Params)) as Promise<T.U32>;
+  }
+
+
+  public messageListGetMessageIds(accountId: T.U32, chatId: T.U32, flags: T.U32): Promise<(T.U32)[]> {
+    return (this._transport.request('message_list_get_message_ids', [accountId, chatId, flags] as RPC.Params)) as Promise<(T.U32)[]>;
+  }
+
+
+  public messageGetMessage(accountId: T.U32, messageId: T.U32): Promise<T.Message> {
+    return (this._transport.request('message_get_message', [accountId, messageId] as RPC.Params)) as Promise<T.Message>;
+  }
+
+
+  public messageGetMessages(accountId: T.U32, messageIds: (T.U32)[]): Promise<Record<T.U32,T.Message>> {
+    return (this._transport.request('message_get_messages', [accountId, messageIds] as RPC.Params)) as Promise<Record<T.U32,T.Message>>;
+  }
+
+  /**
+   * Delete messages. The messages are deleted on the current device and
+   * on the IMAP server.
+   */
+  public deleteMessages(accountId: T.U32, messageIds: (T.U32)[]): Promise<null> {
+    return (this._transport.request('delete_messages', [accountId, messageIds] as RPC.Params)) as Promise<null>;
+  }
+
+  /**
+   * Get an informational text for a single message. The text is multiline and may
+   * contain e.g. the raw text of the message.
+   *
+   * The max. text returned is typically longer (about 100000 characters) than the
+   * max. text returned by dc_msg_get_text() (about 30000 characters).
+   */
+  public getMessageInfo(accountId: T.U32, messageId: T.U32): Promise<string> {
+    return (this._transport.request('get_message_info', [accountId, messageId] as RPC.Params)) as Promise<string>;
+  }
+
+  /**
+   * Get a single contact options by ID.
+   */
+  public contactsGetContact(accountId: T.U32, contactId: T.U32): Promise<T.Contact> {
+    return (this._transport.request('contacts_get_contact', [accountId, contactId] as RPC.Params)) as Promise<T.Contact>;
+  }
+
+  /**
+   * Add a single contact as a result of an explicit user action.
+   *
+   * Returns contact id of the created or existing contact
+   */
+  public contactsCreateContact(accountId: T.U32, email: string, name: (string|null)): Promise<T.U32> {
+    return (this._transport.request('contacts_create_contact', [accountId, email, name] as RPC.Params)) as Promise<T.U32>;
+  }
+
+  /**
+   * Returns contact id of the created or existing DM chat with that contact
+   */
+  public contactsCreateChatByContactId(accountId: T.U32, contactId: T.U32): Promise<T.U32> {
+    return (this._transport.request('contacts_create_chat_by_contact_id', [accountId, contactId] as RPC.Params)) as Promise<T.U32>;
+  }
+
+
+  public contactsBlock(accountId: T.U32, contactId: T.U32): Promise<null> {
+    return (this._transport.request('contacts_block', [accountId, contactId] as RPC.Params)) as Promise<null>;
+  }
+
+
+  public contactsUnblock(accountId: T.U32, contactId: T.U32): Promise<null> {
+    return (this._transport.request('contacts_unblock', [accountId, contactId] as RPC.Params)) as Promise<null>;
+  }
+
+
+  public contactsGetBlocked(accountId: T.U32): Promise<(T.Contact)[]> {
+    return (this._transport.request('contacts_get_blocked', [accountId] as RPC.Params)) as Promise<(T.Contact)[]>;
+  }
+
+
+  public contactsGetContactIds(accountId: T.U32, listFlags: T.U32, query: (string|null)): Promise<(T.U32)[]> {
+    return (this._transport.request('contacts_get_contact_ids', [accountId, listFlags, query] as RPC.Params)) as Promise<(T.U32)[]>;
+  }
+
+  /**
+   * Get a list of contacts.
+   * (formerly called getContacts2 in desktop)
+   */
+  public contactsGetContacts(accountId: T.U32, listFlags: T.U32, query: (string|null)): Promise<(T.Contact)[]> {
+    return (this._transport.request('contacts_get_contacts', [accountId, listFlags, query] as RPC.Params)) as Promise<(T.Contact)[]>;
+  }
+
+
+  public contactsGetContactsByIds(accountId: T.U32, ids: (T.U32)[]): Promise<Record<T.U32,T.Contact>> {
+    return (this._transport.request('contacts_get_contacts_by_ids', [accountId, ids] as RPC.Params)) as Promise<Record<T.U32,T.Contact>>;
+  }
+
+  /**
+   * Returns all message IDs of the given types in a chat.
+   * Typically used to show a gallery.
+   *
+   * The list is already sorted and starts with the oldest message.
+   * Clients should not try to re-sort the list as this would be an expensive action
+   * and would result in inconsistencies between clients.
+   *
+   * Setting `chat_id` to `None` (`null` in typescript) means get messages with media
+   * from any chat of the currently used account.
+   */
+  public chatGetMedia(accountId: T.U32, chatId: (T.U32|null), messageType: T.Viewtype, orMessageType2: (T.Viewtype|null), orMessageType3: (T.Viewtype|null)): Promise<(T.U32)[]> {
+    return (this._transport.request('chat_get_media', [accountId, chatId, messageType, orMessageType2, orMessageType3] as RPC.Params)) as Promise<(T.U32)[]>;
+  }
+
+
+  public webxdcSendStatusUpdate(accountId: T.U32, instanceMsgId: T.U32, updateStr: string, description: string): Promise<null> {
+    return (this._transport.request('webxdc_send_status_update', [accountId, instanceMsgId, updateStr, description] as RPC.Params)) as Promise<null>;
+  }
+
+
+  public webxdcGetStatusUpdates(accountId: T.U32, instanceMsgId: T.U32, lastKnownSerial: T.U32): Promise<string> {
+    return (this._transport.request('webxdc_get_status_updates', [accountId, instanceMsgId, lastKnownSerial] as RPC.Params)) as Promise<string>;
+  }
+
+  /**
+   * Get info from a webxdc message
+   */
+  public messageGetWebxdcInfo(accountId: T.U32, instanceMsgId: T.U32): Promise<T.WebxdcMessageInfo> {
+    return (this._transport.request('message_get_webxdc_info', [accountId, instanceMsgId] as RPC.Params)) as Promise<T.WebxdcMessageInfo>;
+  }
+
+  /**
+   * Returns the messageid of the sent message
+   */
+  public miscSendTextMessage(accountId: T.U32, text: string, chatId: T.U32): Promise<T.U32> {
+    return (this._transport.request('misc_send_text_message', [accountId, text, chatId] as RPC.Params)) as Promise<T.U32>;
+  }
+
+
+}

deltachat-jsonrpc/typescript/generated/events.ts

@@ -0,0 +1,3 @@
+// AUTO-GENERATED by typescript-type-def
+
+export type EventTypeName=("Info"|"SmtpConnected"|"ImapConnected"|"SmtpMessageSent"|"ImapMessageDeleted"|"ImapMessageMoved"|"NewBlobFile"|"DeletedBlobFile"|"Warning"|"Error"|"ErrorSelfNotInGroup"|"MsgsChanged"|"IncomingMsg"|"MsgsNoticed"|"MsgDelivered"|"MsgFailed"|"MsgRead"|"ChatModified"|"ChatEphemeralTimerModified"|"ContactsChanged"|"LocationChanged"|"ConfigureProgress"|"ImexProgress"|"ImexFileWritten"|"SecurejoinInviterProgress"|"SecurejoinJoinerProgress"|"ConnectivityChanged"|"SelfavatarChanged"|"WebxdcStatusUpdate"|"WebXdInstanceDeleted");

deltachat-jsonrpc/typescript/generated/jsonrpc.ts

@@ -0,0 +1,10 @@
+// AUTO-GENERATED by typescript-type-def
+
+export type JSONValue=(null|boolean|number|string|(JSONValue)[]|{[key:string]:JSONValue;});
+export type Params=((JSONValue)[]|Record<string,JSONValue>);
+export type U32=number;
+export type Request={"jsonrpc":"2.0";"method":string;"params"?:Params;"id"?:U32;};
+export type I32=number;
+export type Error={"code":I32;"message":string;"data"?:JSONValue;};
+export type Response={"jsonrpc":"2.0";"id":(U32|null);"result"?:JSONValue;"error"?:Error;};
+export type Message=(Request|Response);

deltachat-jsonrpc/typescript/generated/types.ts

@@ -0,0 +1,106 @@
+// AUTO-GENERATED by typescript-type-def
+
+export type U32=number;
+export type Account=(({"type":"Configured";}&{"id":U32;"displayName":(string|null);"addr":(string|null);"profileImage":(string|null);"color":string;})|({"type":"Unconfigured";}&{"id":U32;}));
+export type ProviderInfo={"beforeLoginHint":string;"overviewPage":string;"status":U32;};
+export type Qr=(({"type":"askVerifyContact";}&{"contact_id":U32;"fingerprint":string;"invitenumber":string;"authcode":string;})|({"type":"askVerifyGroup";}&{"grpname":string;"grpid":string;"contact_id":U32;"fingerprint":string;"invitenumber":string;"authcode":string;})|({"type":"fprOk";}&{"contact_id":U32;})|({"type":"fprMismatch";}&{"contact_id":(U32|null);})|({"type":"fprWithoutAddr";}&{"fingerprint":string;})|({"type":"account";}&{"domain":string;})|({"type":"webrtcInstance";}&{"domain":string;"instance_pattern":string;})|({"type":"addr";}&{"contact_id":U32;"draft":(string|null);})|({"type":"url";}&{"url":string;})|({"type":"text";}&{"text":string;})|({"type":"withdrawVerifyContact";}&{"contact_id":U32;"fingerprint":string;"invitenumber":string;"authcode":string;})|({"type":"withdrawVerifyGroup";}&{"grpname":string;"grpid":string;"contact_id":U32;"fingerprint":string;"invitenumber":string;"authcode":string;})|({"type":"reviveVerifyContact";}&{"contact_id":U32;"fingerprint":string;"invitenumber":string;"authcode":string;})|({"type":"reviveVerifyGroup";}&{"grpname":string;"grpid":string;"contact_id":U32;"fingerprint":string;"invitenumber":string;"authcode":string;}));
+export type Usize=number;
+export type ChatListEntry=[U32,U32];
+export type I64=number;
+export type ChatListItemFetchResult=(({"type":"ChatListItem";}&{"id":U32;"name":string;"avatarPath":(string|null);"color":string;"lastUpdated":(I64|null);"summaryText1":string;"summaryText2":string;"summaryStatus":U32;"isProtected":boolean;"isGroup":boolean;"freshMessageCounter":Usize;"isSelfTalk":boolean;"isDeviceTalk":boolean;"isSendingLocation":boolean;"isSelfInGroup":boolean;"isArchived":boolean;"isPinned":boolean;"isMuted":boolean;"isContactRequest":boolean;
+/**
+ * true when chat is a broadcastlist
+ */
+"isBroadcast":boolean;
+/**
+ * contact id if this is a dm chat (for view profile entry in context menu)
+ */
+"dmChatContact":(U32|null);"wasSeenRecently":boolean;})|{"type":"ArchiveLink";}|({"type":"Error";}&{"id":U32;"error":string;}));
+export type Contact={"address":string;"color":string;"authName":string;"status":string;"displayName":string;"id":U32;"name":string;"profileImage":(string|null);"nameAndAddr":string;"isBlocked":boolean;"isVerified":boolean;"wasSeenRecently":boolean;};
+export type FullChat={"id":U32;"name":string;"isProtected":boolean;"profileImage":(string|null);"archived":boolean;"chatType":U32;"isUnpromoted":boolean;"isSelfTalk":boolean;"contacts":(Contact)[];"contactIds":(U32)[];"color":string;"freshMessageCounter":Usize;"isContactRequest":boolean;"isDeviceChat":boolean;"selfInGroup":boolean;"isMuted":boolean;"ephemeralTimer":U32;"canSend":boolean;"wasSeenRecently":boolean;};
+export type Viewtype=("Unknown"|
+/**
+ * Text message.
+ */
+"Text"|
+/**
+ * Image message.
+ * If the image is an animated GIF, the type `Viewtype.Gif` should be used.
+ */
+"Image"|
+/**
+ * Animated GIF message.
+ */
+"Gif"|
+/**
+ * Message containing a sticker, similar to image.
+ * If possible, the ui should display the image without borders in a transparent way.
+ * A click on a sticker will offer to install the sticker set in some future.
+ */
+"Sticker"|
+/**
+ * Message containing an Audio file.
+ */
+"Audio"|
+/**
+ * A voice message that was directly recorded by the user.
+ * For all other audio messages, the type `Viewtype.Audio` should be used.
+ */
+"Voice"|
+/**
+ * Video messages.
+ */
+"Video"|
+/**
+ * Message containing any file, eg. a PDF.
+ */
+"File"|
+/**
+ * Message is an invitation to a videochat.
+ */
+"VideochatInvitation"|
+/**
+ * Message is an webxdc instance.
+ */
+"Webxdc");
+export type I32=number;
+export type U64=number;
+export type WebxdcMessageInfo={
+/**
+ * The name of the app.
+ * 
+ * Defaults to the filename if not set in the manifest.
+ */
+"name":string;
+/**
+ * App icon file name.
+ * Defaults to an standard icon if nothing is set in the manifest.
+ * 
+ * To get the file, use dc_msg_get_webxdc_blob(). (not yet in jsonrpc, use rust api or cffi for it)
+ * 
+ * App icons should should be square,
+ * the implementations will add round corners etc. as needed.
+ */
+"icon":string;
+/**
+ * if the Webxdc represents a document, then this is the name of the document
+ */
+"document":(string|null);
+/**
+ * short string describing the state of the app,
+ * sth. as "2 votes", "Highscore: 123",
+ * can be changed by the apps
+ */
+"summary":(string|null);
+/**
+ * URL where the source code of the Webxdc and other information can be found;
+ * defaults to an empty string.
+ * Implementations may offer an menu or a button to open this URL.
+ */
+"sourceCodeUrl":(string|null);
+/**
+ * True if full internet access should be granted to the app.
+ */
+"internetAccess":boolean;};
+export type Message={"id":U32;"chatId":U32;"fromId":U32;"quotedText":(string|null);"quotedMessageId":(U32|null);"text":(string|null);"hasLocation":boolean;"hasHtml":boolean;"viewType":Viewtype;"state":U32;"timestamp":I64;"sortTimestamp":I64;"receivedTimestamp":I64;"hasDeviatingTimestamp":boolean;"subject":string;"showPadlock":boolean;"isSetupmessage":boolean;"isInfo":boolean;"isForwarded":boolean;"duration":I32;"dimensionsHeight":I32;"dimensionsWidth":I32;"videochatType":(U32|null);"videochatUrl":(string|null);"overrideSenderName":(string|null);"sender":Contact;"setupCodeBegin":(string|null);"file":(string|null);"fileMime":(string|null);"fileBytes":U64;"fileName":(string|null);"webxdcInfo":(WebxdcMessageInfo|null);};
+export type __AllTyps=[string,boolean,Record<string,string>,U32,U32,null,(U32)[],U32,null,(U32|null),(Account)[],U32,Account,U32,string,(ProviderInfo|null),U32,boolean,U32,Record<string,string>,U32,string,(string|null),null,U32,Record<string,(string|null)>,null,U32,string,null,U32,string,Qr,U32,string,(string|null),U32,(string)[],Record<string,(string|null)>,U32,null,U32,null,U32,(U32)[],U32,U32,Usize,U32,string,U32,U32,string,null,U32,(U32|null),(string|null),(U32|null),(ChatListEntry)[],U32,(ChatListEntry)[],Record<U32,ChatListItemFetchResult>,U32,U32,FullChat,U32,U32,null,U32,U32,null,U32,U32,null,U32,U32,string,U32,(U32|null),[string,string],U32,U32,null,U32,U32,U32,null,U32,U32,U32,null,U32,string,string,U32,U32,U32,U32,(U32)[],U32,U32,Message,U32,(U32)[],Record<U32,Message>,U32,(U32)[],null,U32,U32,string,U32,U32,Contact,U32,string,(string|null),U32,U32,U32,U32,U32,U32,null,U32,U32,null,U32,(Contact)[],U32,U32,(string|null),(U32)[],U32,U32,(string|null),(Contact)[],U32,(U32)[],Record<U32,Contact>,U32,(U32|null),Viewtype,(Viewtype|null),(Viewtype|null),(U32)[],U32,U32,string,string,null,U32,U32,U32,string,U32,U32,WebxdcMessageInfo,U32,string,U32,U32];

deltachat-jsonrpc/typescript/package.json

@@ -0,0 +1,51 @@
+{
+  "author": "Delta Chat Developers (ML) <delta@codespeak.net>",
+  "dependencies": {
+    "isomorphic-ws": "^4.0.1",
+    "tiny-emitter": "git+https://github.com/Simon-Laux/tiny-emitter.git",
+    "yerpc": "^0.3.3"
+  },
+  "devDependencies": {
+    "@types/chai": "^4.2.21",
+    "@types/chai-as-promised": "^7.1.5",
+    "@types/mocha": "^9.0.0",
+    "@types/node-fetch": "^2.5.7",
+    "@types/ws": "^7.2.4",
+    "c8": "^7.10.0",
+    "chai": "^4.3.4",
+    "chai-as-promised": "^7.1.1",
+    "esbuild": "^0.14.11",
+    "http-server": "^14.1.1",
+    "mocha": "^9.1.1",
+    "node-fetch": "^2.6.1",
+    "npm-run-all": "^4.1.5",
+    "prettier": "^2.6.2",
+    "typedoc": "^0.23.2",
+    "typescript": "^4.5.5",
+    "ws": "^8.5.0"
+  },
+  "license": "MPL-2.0",
+  "main": "dist/deltachat.js",
+  "name": "deltachat-jsonrpc-client",
+  "scripts": {
+    "build": "run-s generate-bindings build:tsc build:bundle",
+    "build:bundle": "esbuild --format=esm --bundle dist/deltachat.js --outfile=dist/deltachat.bundle.js",
+    "build:tsc": "tsc",
+    "docs": "typedoc --out docs deltachat.ts",
+    "example": "run-s build example:build example:start",
+    "example:build": "esbuild --bundle dist/example/example.js --outfile=dist/example.bundle.js",
+    "example:dev": "esbuild example/example.ts --bundle --outfile=dist/example.bundle.js --servedir=.",
+    "example:start": "http-server .",
+    "generate-bindings": "cargo test",
+    "prettier:check": "prettier --check **.ts",
+    "prettier:fix": "prettier --write **.ts",
+    "test": "run-s test:prepare test:run-coverage test:report-coverage",
+    "test:prepare": "cargo build --features webserver --bin deltachat-jsonrpc-server",
+    "test:report-coverage": "node report_api_coverage.mjs",
+    "test:run": "mocha dist/test",
+    "test:run-coverage": "COVERAGE=1 NODE_OPTIONS=--enable-source-maps c8 --include 'dist/*' -r text -r html -r json mocha dist/test"
+  },
+  "type": "module",
+  "types": "dist/deltachat.d.ts",
+  "version": "1.93.0"
+}

deltachat-jsonrpc/typescript/report_api_coverage.mjs

@@ -0,0 +1,28 @@
+import { readFileSync } from "fs";
+// only checks for the coverge of the api functions in bindings.ts for now
+const generatedFile = "typescript/generated/client.ts";
+const json = JSON.parse(readFileSync("./coverage/coverage-final.json"));
+const jsonCoverage =
+  json[Object.keys(json).find((k) => k.includes(generatedFile))];
+const fnMap = Object.keys(jsonCoverage.fnMap).map(
+  (key) => jsonCoverage.fnMap[key]
+);
+const htmlCoverage = readFileSync(
+  "./coverage/" + generatedFile + ".html",
+  "utf8"
+);
+const uncoveredLines = htmlCoverage
+  .split("\n")
+  .filter((line) => line.includes(`"function not covered"`));
+const uncoveredFunctions = uncoveredLines.map(
+  (line) => />([\w_]+)\(/.exec(line)[1]
+);
+console.log(
+  "\nUncovered api functions:\n" +
+    uncoveredFunctions
+      .map((uF) => fnMap.find(({ name }) => name === uF))
+      .map(
+        ({ name, line }) => `.${name.padEnd(40)}  (${generatedFile}:${line})`
+      )
+      .join("\n")
+);

deltachat-jsonrpc/typescript/src/client.ts

@@ -0,0 +1,77 @@
+import * as T from "../generated/types.js";
+import * as RPC from "../generated/jsonrpc.js";
+import { RawClient } from "../generated/client.js";
+import { EventTypeName } from "../generated/events.js";
+import { WebsocketTransport, BaseTransport, Request } from "yerpc";
+import { TinyEmitter } from "tiny-emitter";
+
+export type DeltaChatEvent = {
+  id: EventTypeName;
+  contextId: number;
+  field1: any;
+  field2: any;
+};
+export type Events = Record<
+  EventTypeName | "ALL",
+  (event: DeltaChatEvent) => void
+>;
+
+export class BaseDeltaChat<
+  Transport extends BaseTransport<any>
+> extends TinyEmitter<Events> {
+  rpc: RawClient;
+  account?: T.Account;
+  private contextEmitters: TinyEmitter<Events>[] = [];
+  constructor(public transport: Transport) {
+    super();
+    this.rpc = new RawClient(this.transport);
+    this.transport.on("request", (request: Request) => {
+      const method = request.method;
+      if (method === "event") {
+        const event = request.params! as DeltaChatEvent;
+        this.emit(event.id, event);
+        this.emit("ALL", event);
+
+        if (this.contextEmitters[event.contextId]) {
+          this.contextEmitters[event.contextId].emit(event.id, event);
+          this.contextEmitters[event.contextId].emit("ALL", event);
+        }
+      }
+    });
+  }
+
+  async listAccounts(): Promise<T.Account[]> {
+    return await this.rpc.getAllAccounts();
+  }
+
+  getContextEvents(account_id: number) {
+    if (this.contextEmitters[account_id]) {
+      return this.contextEmitters[account_id];
+    } else {
+      this.contextEmitters[account_id] = new TinyEmitter();
+      return this.contextEmitters[account_id];
+    }
+  }
+}
+
+export type Opts = {
+  url: string;
+};
+
+export const DEFAULT_OPTS: Opts = {
+  url: "ws://localhost:20808/ws",
+};
+export class DeltaChat extends BaseDeltaChat<WebsocketTransport> {
+  opts: Opts;
+  close() {
+    this.transport.close();
+  }
+  constructor(opts?: Opts | string) {
+    if (typeof opts === "string") opts = { url: opts };
+    if (opts) opts = { ...DEFAULT_OPTS, ...opts };
+    else opts = { ...DEFAULT_OPTS };
+    const transport = new WebsocketTransport(opts.url)
+    super(transport);
+    this.opts = opts;
+  }
+}

deltachat-jsonrpc/typescript/src/lib.ts

@@ -0,0 +1,6 @@
+export * as RPC from "../generated/jsonrpc.js";
+export * as T from "../generated/types.js";
+export * from "../generated/events.js";
+export { RawClient } from "../generated/client.js";
+export * from "./client.js";
+export * as yerpc from "yerpc";

deltachat-jsonrpc/typescript/test/basic.ts

@@ -0,0 +1,154 @@
+import { strictEqual } from "assert";
+import chai, { assert, expect } from "chai";
+import chaiAsPromised from "chai-as-promised";
+chai.use(chaiAsPromised);
+import { DeltaChat } from "../deltachat.js";
+
+import {
+  RpcServerHandle,
+  startServer,
+} from "./test_base.js";
+
+describe("basic tests", () => {
+  let serverHandle: RpcServerHandle;
+  let dc: DeltaChat;
+
+  before(async () => {
+    serverHandle = await startServer();
+    // make sure server is up by the time we continue
+    await new Promise((res) => setTimeout(res, 100));
+    dc = new DeltaChat(serverHandle.url)
+    // dc.on("ALL", (event) => {
+      //console.log("event", event);
+    // });
+  });
+
+  after(async () => {
+    dc && dc.close();
+    await serverHandle.close();
+  });
+
+  it("check email address validity", async () => {
+    const validAddresses = [
+      "email@example.com",
+      "36aa165ae3406424e0c61af17700f397cad3fe8ab83d682d0bddf3338a5dd52e@yggmail@yggmail",
+    ];
+    const invalidAddresses = ["email@", "example.com", "emai221"];
+
+    expect(
+      await Promise.all(
+        validAddresses.map((email) => dc.rpc.checkEmailValidity(email))
+      )
+    ).to.not.contain(false);
+
+    expect(
+      await Promise.all(
+        invalidAddresses.map((email) => dc.rpc.checkEmailValidity(email))
+      )
+    ).to.not.contain(true);
+  });
+
+  it("system info", async () => {
+    const systemInfo = await dc.rpc.getSystemInfo();
+    expect(systemInfo).to.contain.keys([
+      "arch",
+      "num_cpus",
+      "deltachat_core_version",
+      "sqlite_version",
+    ]);
+  });
+
+  describe("account managment", () => {
+    it("should create account", async () => {
+      const res = await dc.rpc.addAccount();
+      assert((await dc.rpc.getAllAccountIds()).length === 1);
+    });
+
+    it("should remove the account again", async () => {
+      await dc.rpc.removeAccount((await dc.rpc.getAllAccountIds())[0]);
+      assert((await dc.rpc.getAllAccountIds()).length === 0);
+    });
+
+    it("should create multiple accounts", async () => {
+      await dc.rpc.addAccount();
+      await dc.rpc.addAccount();
+      await dc.rpc.addAccount();
+      await dc.rpc.addAccount();
+      assert((await dc.rpc.getAllAccountIds()).length === 4);
+    });
+  });
+
+  describe("contact managment", function () {
+    let accountId: number;
+    before(async () => {
+      accountId = await dc.rpc.addAccount();
+    });
+    it("should block and unblock contact", async function () {
+      const contactId = await dc.rpc.contactsCreateContact(
+        accountId,
+        "example@delta.chat",
+        null
+      );
+      expect((await dc.rpc.contactsGetContact(accountId, contactId)).isBlocked).to.be
+        .false;
+      await dc.rpc.contactsBlock(accountId, contactId);
+      expect((await dc.rpc.contactsGetContact(accountId, contactId)).isBlocked).to.be
+        .true;
+      expect(await dc.rpc.contactsGetBlocked(accountId)).to.have.length(1);
+      await dc.rpc.contactsUnblock(accountId, contactId);
+      expect((await dc.rpc.contactsGetContact(accountId, contactId)).isBlocked).to.be
+        .false;
+      expect(await dc.rpc.contactsGetBlocked(accountId)).to.have.length(0);
+    });
+  });
+
+  describe("configuration", function () {
+    let accountId: number;
+    before(async () => {
+      accountId = await dc.rpc.addAccount();
+    });
+
+    it("set and retrive", async function () {
+      await dc.rpc.setConfig(accountId, "addr", "valid@email");
+      assert((await dc.rpc.getConfig(accountId, "addr")) == "valid@email");
+    });
+    it("set invalid key should throw", async function () {
+      await expect(dc.rpc.setConfig(accountId, "invalid_key", "some value")).to.be
+        .eventually.rejected;
+    });
+    it("get invalid key should throw", async function () {
+      await expect(dc.rpc.getConfig(accountId, "invalid_key")).to.be.eventually
+        .rejected;
+    });
+    it("set and retrive ui.*", async function () {
+      await dc.rpc.setConfig(accountId, "ui.chat_bg", "color:red");
+      assert((await dc.rpc.getConfig(accountId, "ui.chat_bg")) == "color:red");
+    });
+    it("set and retrive (batch)", async function () {
+      const config = { addr: "valid@email", mail_pw: "1234" };
+      await dc.rpc.batchSetConfig(accountId, config);
+      const retrieved = await dc.rpc.batchGetConfig(accountId, Object.keys(config));
+      expect(retrieved).to.deep.equal(config);
+    });
+    it("set and retrive ui.* (batch)", async function () {
+      const config = {
+        "ui.chat_bg": "color:green",
+        "ui.enter_key_sends": "true",
+      };
+      await dc.rpc.batchSetConfig(accountId, config);
+      const retrieved = await dc.rpc.batchGetConfig(accountId, Object.keys(config));
+      expect(retrieved).to.deep.equal(config);
+    });
+    it("set and retrive mixed(ui and core) (batch)", async function () {
+      const config = {
+        "ui.chat_bg": "color:yellow",
+        "ui.enter_key_sends": "false",
+        addr: "valid2@email",
+        mail_pw: "123456",
+      };
+      await dc.rpc.batchSetConfig(accountId, config);
+      const retrieved = await dc.rpc.batchGetConfig(accountId, Object.keys(config));
+      expect(retrieved).to.deep.equal(config);
+    });
+  });
+});

deltachat-jsonrpc/typescript/test/online.ts

@@ -0,0 +1,202 @@
+import { assert, expect } from "chai";
+import { DeltaChat, DeltaChatEvent, EventTypeName } from "../deltachat.js";
+import {
+  RpcServerHandle,
+  createTempUser,
+  startServer,
+} from "./test_base.js";
+
+const EVENT_TIMEOUT = 20000
+
+describe("online tests", function () {
+  let serverHandle: RpcServerHandle;
+  let dc: DeltaChat;
+  let account1: { email: string; password: string };
+  let account2: { email: string; password: string };
+  let accountId1: number, accountId2: number;
+
+  before(async function () {
+    this.timeout(12000)
+    if (!process.env.DCC_NEW_TMP_EMAIL) {
+      if (process.env.COVERAGE && !process.env.COVERAGE_OFFLINE) {
+        console.error(
+          "CAN NOT RUN COVERAGE correctly: Missing DCC_NEW_TMP_EMAIL environment variable!\n\n",
+          "You can set COVERAGE_OFFLINE=1 to circumvent this check and skip the online tests, but those coverage results will be wrong, because some functions can only be tested in the online test"
+        );
+        process.exit(1);
+      }
+      console.log(
+        "Missing DCC_NEW_TMP_EMAIL environment variable!, skip intergration tests"
+      );
+      this.skip();
+    }
+    serverHandle = await startServer();
+    dc = new DeltaChat(serverHandle.url)
+
+    dc.on("ALL", ({ id, contextId }) => {
+      if (id !== "Info") console.log(contextId, id);
+    });
+
+    account1 = await createTempUser(process.env.DCC_NEW_TMP_EMAIL);
+    if (!account1 || !account1.email || !account1.password) {
+      console.log(
+        "We didn't got back an account from the api, skip intergration tests"
+      );
+      this.skip();
+    }
+
+    account2 = await createTempUser(process.env.DCC_NEW_TMP_EMAIL);
+    if (!account2 || !account2.email || !account2.password) {
+      console.log(
+        "We didn't got back an account2 from the api, skip intergration tests"
+      );
+      this.skip();
+    }
+  });
+
+  after(async () => {
+    dc && dc.close();
+    serverHandle && (await serverHandle.close());
+  });
+
+  let accountsConfigured = false;
+
+  it("configure test accounts", async function () {
+    this.timeout(40000);
+
+    accountId1 = await dc.rpc.addAccount();
+    await dc.rpc.setConfig(accountId1, "addr", account1.email);
+    await dc.rpc.setConfig(accountId1, "mail_pw", account1.password);
+    await dc.rpc.configure(accountId1);
+
+    accountId2 = await dc.rpc.addAccount();
+    await dc.rpc.batchSetConfig(accountId2, {
+      addr: account2.email,
+      mail_pw: account2.password,
+    });
+    await dc.rpc.configure(accountId2)
+    accountsConfigured = true;
+  });
+
+  it("send and recieve text message", async function () {
+    if (!accountsConfigured) {
+      this.skip();
+    }
+    this.timeout(15000);
+
+    const contactId = await dc.rpc.contactsCreateContact(
+      accountId1,
+      account2.email,
+      null
+    );
+    const chatId = await dc.rpc.contactsCreateChatByContactId(accountId1, contactId);
+    const eventPromise = Promise.race([
+      waitForEvent(dc, "MsgsChanged", accountId2),
+      waitForEvent(dc, "IncomingMsg", accountId2),
+    ]);
+
+    await dc.rpc.miscSendTextMessage(accountId1, "Hello", chatId);
+    const { field1: chatIdOnAccountB } = await eventPromise;
+    await dc.rpc.acceptChat(accountId2, chatIdOnAccountB);
+    const messageList = await dc.rpc.messageListGetMessageIds(
+      accountId2,
+      chatIdOnAccountB,
+      0
+    );
+
+    expect(messageList).have.length(1);
+    const message = await dc.rpc.messageGetMessage(accountId2, messageList[0]);
+    expect(message.text).equal("Hello");
+  });
+
+  it("send and recieve text message roundtrip, encrypted on answer onwards", async function () {
+    if (!accountsConfigured) {
+      this.skip();
+    }
+    this.timeout(10000);
+
+    // send message from A to B
+    const contactId = await dc.rpc.contactsCreateContact(
+      accountId1,
+      account2.email,
+      null
+    );
+    const chatId = await dc.rpc.contactsCreateChatByContactId(accountId1, contactId);
+    const eventPromise = Promise.race([
+      waitForEvent(dc, "MsgsChanged", accountId2),
+      waitForEvent(dc, "IncomingMsg", accountId2),
+    ]);
+    dc.rpc.miscSendTextMessage(accountId1, "Hello2", chatId);
+    // wait for message from A
+    console.log("wait for message from A");
+
+    const event = await eventPromise;
+    const { field1: chatIdOnAccountB } = event;
+
+    await dc.rpc.acceptChat(accountId2, chatIdOnAccountB);
+    const messageList = await dc.rpc.messageListGetMessageIds(
+      accountId2,
+      chatIdOnAccountB,
+      0
+    );
+    const message = await dc.rpc.messageGetMessage(
+      accountId2,
+      messageList.reverse()[0]
+    );
+    expect(message.text).equal("Hello2");
+    // Send message back from B to A
+    const eventPromise2 = Promise.race([
+      waitForEvent(dc, "MsgsChanged", accountId1),
+      waitForEvent(dc, "IncomingMsg", accountId1),
+    ]);
+    dc.rpc.miscSendTextMessage(accountId2, "super secret message", chatId);
+    // Check if answer arives at A and if it is encrypted
+    await eventPromise2;
+
+    const messageId = (
+      await dc.rpc.messageListGetMessageIds(accountId1, chatId, 0)
+    ).reverse()[0];
+    const message2 = await dc.rpc.messageGetMessage(accountId1, messageId);
+    expect(message2.text).equal("super secret message");
+    expect(message2.showPadlock).equal(true);
+  });
+
+  it("get provider info for example.com", async () => {
+    const acc = await dc.rpc.addAccount();
+    const info = await dc.rpc.getProviderInfo(acc, "example.com");
+    expect(info).to.be.not.null;
+    expect(info?.overviewPage).to.equal(
+      "https://providers.delta.chat/example-com"
+    );
+    expect(info?.status).to.equal(3);
+  });
+
+  it("get provider info - domain and email should give same result", async () => {
+    const acc = await dc.rpc.addAccount();
+    const info_domain = await dc.rpc.getProviderInfo(acc, "example.com");
+    const info_email = await dc.rpc.getProviderInfo(acc, "hi@example.com");
+    expect(info_email).to.deep.equal(info_domain);
+  });
+});
+
+async function waitForEvent(
+  dc: DeltaChat,
+  eventType: EventTypeName,
+  accountId: number,
+  timeout: number = EVENT_TIMEOUT
+): Promise<DeltaChatEvent> {
+  return new Promise((resolve, reject) => {
+    const rejectTimeout = setTimeout(
+      () => reject(new Error('Timeout reached before event came in')),
+      timeout
+    )
+    const callback = (event: DeltaChatEvent) => {
+      if (event.contextId == accountId) {
+        dc.off(eventType, callback);
+        clearTimeout(rejectTimeout)
+        resolve(event);
+      }
+    };
+    dc.on(eventType, callback);
+  });
+}

deltachat-jsonrpc/typescript/test/test_base.ts

@@ -0,0 +1,94 @@
+import { tmpdir } from "os";
+import { join, resolve } from "path";
+import { mkdtemp, rm } from "fs/promises";
+import { existsSync } from "fs";
+import { spawn, exec } from "child_process";
+import fetch from "node-fetch";
+
+export const RPC_SERVER_PORT = 20808;
+
+export type RpcServerHandle = {
+  url: string,
+  close: () => Promise<void>
+}
+
+export async function startServer(port: number = RPC_SERVER_PORT): Promise<RpcServerHandle> {
+  const tmpDir = await mkdtemp(join(tmpdir(), "deltachat-jsonrpc-test"));
+
+  const pathToServerBinary = resolve(join(await getTargetDir(), "debug/deltachat-jsonrpc-server"));
+  console.log('using server binary: ' + pathToServerBinary);
+
+  if (!existsSync(pathToServerBinary)) {
+    throw new Error(
+      "server executable does not exist, you need to build it first" +
+        "\nserver executable not found at " +
+        pathToServerBinary
+    );
+  }
+
+  const server = spawn(pathToServerBinary, {
+    cwd: tmpDir,
+    env: {
+      RUST_LOG: process.env.RUST_LOG || "info",
+      DC_PORT: '' + port
+    },
+  });
+  let shouldClose = false;
+
+  server.on("exit", () => {
+    if (shouldClose) {
+      return;
+    }
+    throw new Error("Server quit");
+  });
+
+  server.stderr.pipe(process.stderr);
+  server.stdout.pipe(process.stdout)
+
+  const url = `ws://localhost:${port}/ws`
+
+  return {
+    url,
+    close: async () => {
+      shouldClose = true;
+      if (!server.kill()) {
+        console.log("server termination failed");
+      }
+      await rm(tmpDir, { recursive: true });
+    },
+  };
+}
+
+export async function createTempUser(url: string) {
+  const response = await fetch(url, {
+    method: "POST", 
+    headers: {
+      "cache-control": "no-cache",
+    },
+  });
+  if (!response.ok) throw new Error('Received invalid response')
+  return response.json(); 
+}
+
+function getTargetDir(): Promise<string> {
+  return new Promise((resolve, reject) => {
+    exec(
+      "cargo metadata --no-deps --format-version 1",
+      (error, stdout, _stderr) => {
+        if (error) {
+          console.log("error", error);
+          reject(error);
+        } else {
+          try {
+            const json = JSON.parse(stdout);
+            resolve(json.target_directory);
+          } catch (error) {
+            console.log("json error", error);
+            reject(error);
+          }
+        }
+      }
+    );
+  });
+}
+

deltachat-jsonrpc/typescript/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "compilerOptions": {
+    "alwaysStrict": true,
+    "strict": true,
+    "sourceMap": true,
+    "strictNullChecks": true,
+    "rootDir": ".",
+    "outDir": "dist",
+    "lib": ["ES2017", "dom"],
+    "target": "ES2017",
+    "module": "es2020",
+    "declaration": true,
+    "esModuleInterop": true,
+    "moduleResolution": "node",
+    "noImplicitAny": true,
+    "isolatedModules": true
+  },
+  "include": ["*.ts", "example/*.ts", "test/*.ts"],
+  "compileOnSave": false
+}

examples/repl/cmdline.rs

@@ -1,3 +1,4 @@
+#![allow(clippy::format_push_string)]
 extern crate dirs;
 
 use std::path::Path;
@@ -982,11 +983,9 @@ pub async fn cmdline(context: Context, line: &str, chat_id: &mut ChatId) -> Resu
             chat::add_device_msg(&context, None, Some(&mut msg)).await?;
         }
         "listmedia" => {
-            ensure!(sel_chat.is_some(), "No chat selected.");
-
             let images = chat::get_chat_media(
                 &context,
-                sel_chat.as_ref().unwrap().get_id(),
+                sel_chat.map(|c| c.id),
                 Viewtype::Image,
                 Viewtype::Gif,
                 Viewtype::Video,

examples/repl/main.rs

@@ -23,7 +23,6 @@ use deltachat::securejoin::*;
 use deltachat::{EventType, Events};
 use log::{error, info, warn};
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
-use rustyline::config::OutputStreamType;
 use rustyline::error::ReadlineError;
 use rustyline::highlight::{Highlighter, MatchingBracketHighlighter};
 use rustyline::hint::{Hinter, HistoryHinter};
@@ -314,7 +313,6 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         .history_ignore_space(true)
         .completion_type(CompletionType::List)
         .edit_mode(EditMode::Emacs)
-        .output_stream(OutputStreamType::Stdout)
         .build();
     let mut selected_chat = ChatId::default();
 
@@ -325,7 +323,7 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
             highlighter: MatchingBracketHighlighter::new(),
             hinter: HistoryHinter {},
         };
-        let mut rl = Editor::with_config(config);
+        let mut rl = Editor::with_config(config)?;
         rl.set_helper(Some(h));
         rl.bind_sequence(KeyEvent::alt('N'), Cmd::HistorySearchForward);
         rl.bind_sequence(KeyEvent::alt('P'), Cmd::HistorySearchBackward);

node/README.md

@@ -234,7 +234,7 @@ We have the following scripts for building, testing and coverage:
 The following steps are needed to make a release:
 
 1. Wait until `pack-module` github action is completed
-2. Run `npm publish https://download.delta.chat/node/deltachat-node-v1.x.x.tar.gz` to publish it to npm. You probably need write rights to npm.
+2. Run `npm publish https://download.delta.chat/node/deltachat-node-1.x.x.tar.gz` to publish it to npm. You probably need write rights to npm.
 
 ## License
 

node/lib/chat.ts

@@ -39,6 +39,10 @@ export class Chat {
     return binding.dcn_chat_get_name(this.dc_chat)
   }
 
+  getMailinglistAddr(): string {
+    return binding.dcn_chat_get_mailinglist_addr(this.dc_chat)
+  }
+
   getProfileImage(): string {
     return binding.dcn_chat_get_profile_image(this.dc_chat)
   }
@@ -92,6 +96,7 @@ export class Chat {
       color: this.color,
       id: this.getId(),
       name: this.getName(),
+      mailinglistAddr: this.getMailinglistAddr(),
       profileImage: this.getProfileImage(),
       type: this.getType(),
       isSelfTalk: this.isSelfTalk(),

node/lib/contact.ts

@@ -72,6 +72,10 @@ export class Contact {
     return binding.dcn_contact_get_last_seen(this.dc_contact)
   }
 
+  wasSeenRecently() {
+    return Boolean(binding.dcn_contact_was_seen_recently(this.dc_contact))
+  }
+
   getName(): string {
     return binding.dcn_contact_get_name(this.dc_contact)
   }

node/lib/deltachat.ts

@@ -19,10 +19,11 @@ interface NativeAccount {}
 export class AccountManager extends EventEmitter {
   dcn_accounts: NativeAccount
   accountDir: string
+  jsonRpcStarted = false
 
   constructor(cwd: string, os = 'deltachat-node') {
-    debug('DeltaChat constructor')
     super()
+    debug('DeltaChat constructor')
 
     this.accountDir = cwd
     this.dcn_accounts = binding.dcn_accounts_new(os, this.accountDir)
@@ -114,6 +115,31 @@ export class AccountManager extends EventEmitter {
     debug('Started event handler')
   }
 
+  startJsonRpcHandler(callback: ((response: string) => void) | null) {
+    if (this.dcn_accounts === null) {
+      throw new Error('dcn_account is null')
+    }
+    if (!callback) {
+      throw new Error('no callback set')
+    }
+    if (this.jsonRpcStarted) {
+      throw new Error('jsonrpc was started already')
+    }
+
+    binding.dcn_accounts_start_jsonrpc(this.dcn_accounts, callback.bind(this))
+    debug('Started JSON-RPC handler')
+    this.jsonRpcStarted = true
+  }
+
+  jsonRpcRequest(message: string) {
+    if (!this.jsonRpcStarted) {
+      throw new Error(
+        'jsonrpc is not active, start it with startJsonRpcHandler first'
+      )
+    }
+    binding.dcn_json_rpc_request(this.dcn_accounts, message)
+  }
+
   startIO() {
     binding.dcn_accounts_start_io(this.dcn_accounts)
   }

node/lib/types.ts

@@ -12,6 +12,7 @@ export interface ChatJSON {
   color: string
   id: number
   name: string
+  mailinglistAddr: string
   profileImage: string
   type: number
   isSelfTalk: boolean

node/scripts/rebuild-core.js

@@ -9,7 +9,7 @@ const buildArgs = [
   'build',
   '--release',
   '--features',
-  'vendored',
+  'vendored,jsonrpc',
   '-p',
   'deltachat_ffi'
 ]

node/src/module.c

@@ -34,6 +34,9 @@ typedef struct dcn_accounts_t {
   dc_accounts_t* dc_accounts;
   napi_threadsafe_function threadsafe_event_handler;
   uv_thread_t event_handler_thread;
+  napi_threadsafe_function threadsafe_jsonrpc_handler;
+  uv_thread_t jsonrpc_thread;
+  dc_jsonrpc_instance_t* jsonrpc_instance;
   int gc;
 } dcn_accounts_t;
 
@@ -1625,6 +1628,18 @@ NAPI_METHOD(dcn_chat_get_name) {
   NAPI_RETURN_AND_UNREF_STRING(name);
 }
 
+NAPI_METHOD(dcn_chat_get_mailinglist_addr) {
+  NAPI_ARGV(1);
+  NAPI_DC_CHAT();
+
+  //TRACE("calling..");
+  char* addr = dc_chat_get_mailinglist_addr(dc_chat);
+  //TRACE("result %s", name);
+
+  NAPI_RETURN_AND_UNREF_STRING(addr);
+}
+
+
 NAPI_METHOD(dcn_chat_get_profile_image) {
   NAPI_ARGV(1);
   NAPI_DC_CHAT();
@@ -1915,6 +1930,13 @@ NAPI_METHOD(dcn_contact_get_last_seen) {
   NAPI_RETURN_INT64(timestamp);
 }
 
+NAPI_METHOD(dcn_contact_was_seen_recently) {
+  NAPI_ARGV(1);
+  NAPI_DC_CONTACT();
+  int seen_recently = dc_contact_was_seen_recently(dc_contact);
+  NAPI_RETURN_UINT32(seen_recently);
+}
+
 NAPI_METHOD(dcn_contact_is_blocked) {
   NAPI_ARGV(1);
   NAPI_DC_CONTACT();
@@ -2924,6 +2946,12 @@ NAPI_METHOD(dcn_accounts_unref) {
     uv_thread_join(&dcn_accounts->event_handler_thread);
     dcn_accounts->event_handler_thread = 0;
   }
+  if (dcn_accounts->jsonrpc_instance) {
+    dc_jsonrpc_request(dcn_accounts->jsonrpc_instance, "{}");
+    uv_thread_join(&dcn_accounts->jsonrpc_thread);
+    dc_jsonrpc_unref(dcn_accounts->jsonrpc_instance);
+    dcn_accounts->jsonrpc_instance = NULL;
+  }
   dc_accounts_unref(dcn_accounts->dc_accounts);
   dcn_accounts->dc_accounts = NULL;
 
@@ -3082,18 +3110,16 @@ static void accounts_event_handler_thread_func(void* arg)
 {
   dcn_accounts_t* dcn_accounts = (dcn_accounts_t*)arg;
 
-
-
   TRACE("event_handler_thread_func starting");
 
-  dc_accounts_event_emitter_t * dc_accounts_event_emitter = dc_accounts_get_event_emitter(dcn_accounts->dc_accounts);
+  dc_event_emitter_t * dc_event_emitter = dc_accounts_get_event_emitter(dcn_accounts->dc_accounts);
   dc_event_t* event;
   while (true) {
-    if (dc_accounts_event_emitter == NULL) {
+    if (dc_event_emitter == NULL) {
       TRACE("event emitter is null, bailing");
       break;
     }
-    event = dc_accounts_get_next_event(dc_accounts_event_emitter);
+    event = dc_get_next_event(dc_event_emitter);
     if (event == NULL) {
       TRACE("no more events");
       break;
@@ -3119,7 +3145,7 @@ static void accounts_event_handler_thread_func(void* arg)
     }
   }
 
-  dc_accounts_event_emitter_unref(dc_accounts_event_emitter);
+  dc_event_emitter_unref(dc_event_emitter);
 
   TRACE("event_handler_thread_func ended");
 
@@ -3173,7 +3199,7 @@ static void call_accounts_js_event_handler(napi_env env, napi_value js_callback,
     if (status != napi_ok) {
       napi_throw_error(env, NULL, "Unable to create argv[3] for event_handler arguments");
     }
-    free(data2_string);
+    dc_str_unref(data2_string);
   } else {
     status = napi_create_int32(env, dc_event_get_data2_int(dc_event), &argv[3]);
     if (status != napi_ok) {
@@ -3234,6 +3260,124 @@ NAPI_METHOD(dcn_accounts_start_event_handler) {
   NAPI_RETURN_UNDEFINED();
 }
 
+// JSON RPC
+
+static void accounts_jsonrpc_thread_func(void* arg)
+{
+  dcn_accounts_t* dcn_accounts = (dcn_accounts_t*)arg;
+  TRACE("accounts_jsonrpc_thread_func starting");
+  char* response;
+  while (true) {
+    response = dc_jsonrpc_next_response(dcn_accounts->jsonrpc_instance);
+    if (response == NULL) {
+      // done or broken
+      break;
+    }
+
+    if (!dcn_accounts->threadsafe_jsonrpc_handler) {
+      TRACE("threadsafe_jsonrpc_handler not set, bailing");
+      break;
+    }
+    // Don't process events if we're being garbage collected!
+    if (dcn_accounts->gc == 1) {
+      TRACE("dc_accounts has been destroyed, bailing");
+      break;
+    }
+
+    napi_status status = napi_call_threadsafe_function(dcn_accounts->threadsafe_jsonrpc_handler, response, napi_tsfn_blocking);
+
+    if (status == napi_closing) {
+      TRACE("JS function got released, bailing");
+      break;
+    }
+  }
+  TRACE("accounts_jsonrpc_thread_func ended");
+  napi_release_threadsafe_function(dcn_accounts->threadsafe_jsonrpc_handler, napi_tsfn_release);
+}
+
+static void call_accounts_js_jsonrpc_handler(napi_env env, napi_value js_callback, void* _context, void* data)
+{
+  char* response = (char*)data;
+  napi_value global;
+  napi_status status = napi_get_global(env, &global);
+  if (status != napi_ok) {
+    napi_throw_error(env, NULL, "Unable to get global");
+  }
+
+  napi_value argv[1];
+  if (response != 0) {
+    status = napi_create_string_utf8(env, response, NAPI_AUTO_LENGTH, &argv[0]);
+  } else {
+    status = napi_create_string_utf8(env, "", NAPI_AUTO_LENGTH, &argv[0]);
+  }
+  if (status != napi_ok) {
+    napi_throw_error(env, NULL, "Unable to create argv for js jsonrpc_handler arguments");
+  } 
+  dc_str_unref(response);
+
+  TRACE("calling back into js");
+  napi_value result;
+  status = napi_call_function(
+    env,
+    global,
+    js_callback,
+    1,
+    argv,
+    &result);
+  if (status != napi_ok) {
+    TRACE("Unable to call jsonrpc_handler callback2");
+    const napi_extended_error_info* error_result;
+    NAPI_STATUS_THROWS(napi_get_last_error_info(env, &error_result));
+  }
+}
+
+NAPI_METHOD(dcn_accounts_start_jsonrpc) {
+  NAPI_ARGV(2);
+  NAPI_DCN_ACCOUNTS();
+  napi_value callback = argv[1];
+
+  TRACE("calling..");
+  napi_value async_resource_name;
+  NAPI_STATUS_THROWS(napi_create_string_utf8(env, "dc_accounts_jsonrpc_callback", NAPI_AUTO_LENGTH, &async_resource_name));
+
+  TRACE("creating threadsafe function..");
+
+  NAPI_STATUS_THROWS(napi_create_threadsafe_function(
+    env,
+    callback,
+    0,
+    async_resource_name,
+    1000, // max_queue_size
+    1,
+    NULL,
+    NULL,
+    NULL,
+    call_accounts_js_jsonrpc_handler,
+    &dcn_accounts->threadsafe_jsonrpc_handler));
+  TRACE("done");
+
+  dcn_accounts->gc = 0;
+  dcn_accounts->jsonrpc_instance = dc_jsonrpc_init(dcn_accounts->dc_accounts);
+
+  TRACE("creating uv thread..");
+  uv_thread_create(&dcn_accounts->jsonrpc_thread, accounts_jsonrpc_thread_func, dcn_accounts);
+
+  NAPI_RETURN_UNDEFINED();
+}
+
+NAPI_METHOD(dcn_json_rpc_request) {
+  NAPI_ARGV(2);
+  NAPI_DCN_ACCOUNTS();
+  if (!dcn_accounts->jsonrpc_instance) {
+    const char* msg = "dcn_accounts->jsonrpc_instance is null, have you called dcn_accounts_start_jsonrpc()?";
+    NAPI_STATUS_THROWS(napi_throw_type_error(env, NULL, msg));
+  }
+  NAPI_ARGV_UTF8_MALLOC(request, 1);
+  dc_jsonrpc_request(dcn_accounts->jsonrpc_instance, request);
+  free(request);
+  NAPI_RETURN_UNDEFINED();
+}
+
 
 NAPI_INIT() {
   /**
@@ -3370,6 +3514,7 @@ NAPI_INIT() {
   NAPI_EXPORT_FUNCTION(dcn_chat_get_visibility);
   NAPI_EXPORT_FUNCTION(dcn_chat_get_id);
   NAPI_EXPORT_FUNCTION(dcn_chat_get_name);
+  NAPI_EXPORT_FUNCTION(dcn_chat_get_mailinglist_addr);
   NAPI_EXPORT_FUNCTION(dcn_chat_get_profile_image);
   NAPI_EXPORT_FUNCTION(dcn_chat_get_type);
   NAPI_EXPORT_FUNCTION(dcn_chat_is_self_talk);
@@ -3504,4 +3649,9 @@ NAPI_INIT() {
   NAPI_EXPORT_FUNCTION(dcn_send_webxdc_status_update);
   NAPI_EXPORT_FUNCTION(dcn_get_webxdc_status_updates);
   NAPI_EXPORT_FUNCTION(dcn_msg_get_webxdc_blob);
+
+
+  /** jsonrpc **/
+  NAPI_EXPORT_FUNCTION(dcn_accounts_start_jsonrpc);
+  NAPI_EXPORT_FUNCTION(dcn_json_rpc_request);
 }

node/src/napi-macros-extensions.h

@@ -23,7 +23,7 @@
   dcn_accounts_t* dcn_accounts; \
   NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], (void**)&dcn_accounts)); \
   if (!dcn_accounts) { \
-    const char* msg = "Provided dnc_acounts is null"; \
+    const char* msg = "Provided dcn_acounts is null"; \
     NAPI_STATUS_THROWS(napi_throw_type_error(env, NULL, msg)); \
   } \
   if (!dcn_accounts->dc_accounts) { \

node/test/test.js

@@ -2,7 +2,7 @@
 import DeltaChat, { Message } from '../dist'
 import binding from '../binding'
 
-import { strictEqual } from 'assert'
+import { deepEqual, deepStrictEqual, strictEqual } from 'assert'
 import chai, { expect } from 'chai'
 import chaiAsPromised from 'chai-as-promised'
 import { EventId2EventName, C } from '../dist/constants'
@@ -84,6 +84,95 @@ describe('static tests', function () {
   })
 })
 
+describe('JSON RPC', function () {
+  it('smoketest', async function () {
+    const { dc } = DeltaChat.newTemporary()
+    let promise_resolve
+    const promise = new Promise((res, _rej) => {
+      promise_resolve = res
+    })
+    dc.startJsonRpcHandler(promise_resolve)
+    dc.jsonRpcRequest(
+      JSON.stringify({
+        jsonrpc: '2.0',
+        method: 'get_all_account_ids',
+        params: [],
+        id: 2,
+      })
+    )
+    deepStrictEqual(
+      {
+        jsonrpc: '2.0',
+        id: 2,
+        result: [1],
+      },
+      JSON.parse(await promise)
+    )
+    dc.close()
+  })
+
+  it('basic test', async function () {
+    const { dc } = DeltaChat.newTemporary()
+
+    const promises = {}
+    dc.startJsonRpcHandler((msg) => {
+      const response = JSON.parse(msg)
+      promises[response.id](response)
+      delete promises[response.id]
+    })
+    const call = (request) => {
+      dc.jsonRpcRequest(JSON.stringify(request))
+      return new Promise((res, _rej) => {
+        promises[request.id] = res
+      })
+    }
+
+    deepStrictEqual(
+      {
+        jsonrpc: '2.0',
+        id: 2,
+        result: [1],
+      },
+      await call({
+        jsonrpc: '2.0',
+        method: 'get_all_account_ids',
+        params: [],
+        id: 2,
+      })
+    )
+
+    deepStrictEqual(
+      {
+        jsonrpc: '2.0',
+        id: 3,
+        result: 2,
+      },
+      await call({
+        jsonrpc: '2.0',
+        method: 'add_account',
+        params: [],
+        id: 3,
+      })
+    )
+
+    deepStrictEqual(
+      {
+        jsonrpc: '2.0',
+        id: 4,
+        result: [1, 2],
+      },
+      await call({
+        jsonrpc: '2.0',
+        method: 'get_all_account_ids',
+        params: [],
+        id: 4,
+      })
+    )
+
+    dc.close()
+  })
+})
+
 describe('Basic offline Tests', function () {
   it('opens a context', async function () {
     const { dc, context } = DeltaChat.newTemporary()

package.json

@@ -1,5 +1,6 @@
 {
   "dependencies": {
+    "@deltachat/jsonrpc-client": "file:deltachat-jsonrpc/typescript",
     "debug": "^4.1.1",
     "napi-macros": "^2.0.0",
     "node-gyp-build": "^4.1.0"
@@ -60,5 +61,5 @@
     "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.88.0"
+  "version": "1.93.0"
 }

python/README.rst

@@ -1,84 +1,77 @@
 =========================
-deltachat python bindings
+DeltaChat Python bindings
 =========================
 
-This package provides bindings to the deltachat-core_ Rust -library
-which implements IMAP/SMTP/MIME/PGP e-mail standards and offers
+This package provides `Python bindings`_ to the `deltachat-core library`_
+which implements IMAP/SMTP/MIME/OpenPGP e-mail standards and offers
 a low-level Chat/Contact/Message API to user interfaces and bots.
 
+.. _`deltachat-core library`: https://github.com/deltachat/deltachat-core-rust
+.. _`Python bindings`: https://py.delta.chat/
 
 Installing pre-built packages (Linux-only)
-========================================================
+==========================================
 
 If you have a Linux system you may try to install the ``deltachat`` binary "wheel" packages
 without any "build-from-source" steps.
-Otherwise you need to `compile the Delta Chat bindings yourself <#sourceinstall>`_.
+Otherwise you need to `compile the Delta Chat bindings yourself`__.
+
+__ sourceinstall_
 
 We recommend to first `install virtualenv <https://virtualenv.pypa.io/en/stable/installation.html>`_,
 then create a fresh Python virtual environment and activate it in your shell::
 
-        virtualenv venv  # or: python -m venv
-        source venv/bin/activate
+        virtualenv env  # or: python -m venv
+        source env/bin/activate
 
 Afterwards, invoking ``python`` or ``pip install`` only
-modifies files in your ``venv`` directory and leaves
+modifies files in your ``env`` directory and leaves
 your system installation alone.
 
-For Linux, we automatically build wheels for all github PR branches
-and push them to a python package index. To install the latest
-github ``master`` branch::
+For Linux we build wheels for all releases and push them to a python package
+index. To install the latest release::
 
-    pip install --pre -i https://m.devpi.net/dc/master deltachat
+    pip install deltachat
 
 To verify it worked::
 
     python -c "import deltachat"
 
-.. note::
-
-    If you can help to automate the building of wheels for Mac or Windows,
-    that'd be much appreciated! please then get
-    `in contact with us <https://delta.chat/en/contribute>`_.
-
-
 Running tests
 =============
 
-After successful binding installation you can install a few more
-Python packages before running the tests::
+Recommended way to run tests is using `tox <https://tox.wiki>`_.
+After successful binding installation you can install tox
+and run the tests::
 
-    python -m pip install pytest pytest-xdist pytest-timeout pytest-rerunfailures requests
-    pytest -v tests
+    pip install tox
+    tox -e py3
 
 This will run all "offline" tests and skip all functional
 end-to-end tests that require accounts on real e-mail servers.
 
 .. _livetests:
 
-running "live" tests with temporary accounts
----------------------------------------------
+Running "live" tests with temporary accounts
+--------------------------------------------
 
-If you want to run live functional tests you can set ``DCC_NEW_TMP_EMAIL`` to a URL that creates e-mail accounts.  Most developers use https://testrun.org URLS created and managed by [mailadm](https://mailadm.readthedocs.io/en/latest/).
+If you want to run live functional tests you can set ``DCC_NEW_TMP_EMAIL`` to a URL that creates e-mail accounts.  Most developers use https://testrun.org URLs created and managed by `mailadm <https://mailadm.readthedocs.io/>`_.
 
-Please feel free to contact us through a github issue or by e-mail and we'll send you a URL that you can then use for functional tests like this:
+Please feel free to contact us through a github issue or by e-mail and we'll send you a URL that you can then use for functional tests like this::
 
     export DCC_NEW_TMP_EMAIL=<URL you got from us>
 
 With this account-creation setting, pytest runs create ephemeral e-mail accounts on the http://testrun.org server.  These accounts exists only for one hour and then are removed completely.
-One hour is enough to invoke pytest and run all offline and online tests:
+One hour is enough to invoke pytest and run all offline and online tests::
 
-    pytest
-
-    # or if you have installed pytest-xdist for parallel test execution
-    pytest -n6
+    tox -e py3
 
 Each test run creates new accounts.
 
-
 .. _sourceinstall:
 
-Installing bindings from source (Updated: July 2020)
-=========================================================
+Installing bindings from source
+===============================
 
 Install Rust and Cargo first.
 The easiest is probably to use `rustup <https://rustup.rs/>`_.
@@ -97,74 +90,42 @@ E.g. on Debian-based systems `apt install python3 python3-pip
 python3-venv` should give you a usable python installation.
 
 Ensure you are in the deltachat-core-rust/python directory, create the
-virtual environment and activate it in your shell::
+virtual environment with dependencies using tox
+and activate it in your shell::
 
    cd python
-   python3 -m venv venv  # or: virtualenv venv
-   source venv/bin/activate
+   tox --devenv env
+   source env/bin/activate
 
 You should now be able to build the python bindings using the supplied script::
 
-   python install_python_bindings.py
+   python3 install_python_bindings.py
 
 The core compilation and bindings building might take a while,
 depending on the speed of your machine.
-The bindings will be installed in release mode but with debug symbols.
-The release mode is currently necessary because some tests generate RSA keys
-which is prohibitively slow in non-release mode.
-
-
-Code examples
-=============
-
-You may look at `examples <https://py.delta.chat/examples.html>`_.
-
-
-.. _`deltachat-core-rust github repository`: https://github.com/deltachat/deltachat-core-rust
-.. _`deltachat-core`: https://github.com/deltachat/deltachat-core-rust
-
 
 Building manylinux based wheels
-====================================
+===============================
 
 Building portable manylinux wheels which come with libdeltachat.so
-can be done with docker-tooling.
+can be done with Docker_ or Podman_.
 
-using docker pull / premade images
-------------------------------------
+.. _Docker: https://www.docker.com/
+.. _Podman: https://podman.io/
 
-We publish a build environment under the ``deltachat/coredeps`` tag so
-that you can pull it from the ``hub.docker.com`` site's "deltachat"
-organization::
+If you want to build your own wheels, build container image first::
 
-    $ docker pull deltachat/coredeps
+   $ cd deltachat-core-rust # cd to deltachat-core-rust working tree
+   $ docker build -t deltachat/coredeps scripts/coredeps
 
-This docker image can be used to run tests and build Python wheels for all interpreters::
-
-    $ docker run -e DCC_NEW_TMP_EMAIL \
-       --rm -it -v \$(pwd):/mnt -w /mnt \
-       deltachat/coredeps scripts/run_all.sh
-
-
-Optionally build your own docker image
---------------------------------------
-
-If you want to build your own custom docker image you can do this::
-
-   $ cd deltachat-core # cd to deltachat-core checkout directory
-   $ docker build -t deltachat/coredeps scripts/docker_coredeps
-
-This will use the ``scripts/docker_coredeps/Dockerfile`` to build
-up docker image called ``deltachat/coredeps``.  You can afterwards
+This will use the ``scripts/coredeps/Dockerfile`` to build
+container image called ``deltachat/coredeps``.  You can afterwards
 find it with::
 
    $ docker images
 
+This docker image can be used to run tests and build Python wheels for all interpreters::
 
-Troubleshooting
----------------
-
-On more recent systems running the docker image may crash.  You can
-fix this by adding ``vsyscall=emulate`` to the Linux kernel boot
-arguments commandline.  E.g. on Debian you'd add this to
-``GRUB_CMDLINE_LINUX_DEFAULT`` in ``/etc/default/grub``.
+    $ docker run -e DCC_NEW_TMP_EMAIL \
+       --rm -it -v $(pwd):/mnt -w /mnt \
+       deltachat/coredeps scripts/run_all.sh

python/fail_test.py

@@ -1,7 +0,0 @@
-from __future__ import print_function
-from deltachat import capi
-from deltachat.capi import ffi, lib
-
-if __name__ == "__main__":
-    ctx = capi.lib.dc_context_new(ffi.NULL, ffi.NULL)
-    lib.dc_stop_io(ctx)

python/install_python_bindings.py

@@ -16,7 +16,7 @@ if __name__ == "__main__":
         dn = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
         os.environ["DCC_RS_DEV"] = dn
 
-    cmd = ["cargo", "build", "-p", "deltachat_ffi"]
+    cmd = ["cargo", "build", "-p", "deltachat_ffi", "--features", "jsonrpc"]
 
     if target == "release":
         os.environ["CARGO_PROFILE_RELEASE_LTO"] = "on"

python/pyproject.toml

@@ -1,7 +1,41 @@
 [build-system]
-requires = ["setuptools>=45", "wheel", "setuptools_scm>=6.2", "cffi>=1.0.0", "pkgconfig"]
+requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.2", "cffi>=1.0.0", "pkgconfig"]
 build-backend = "setuptools.build_meta"
 
+[project]
+name = "deltachat"
+description = "Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat"
+readme = "README.rst"
+requires-python = ">=3.7"
+authors = [
+    { name = "holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors" },
+]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
+    "Programming Language :: Python :: 3",
+    "Topic :: Communications :: Email",
+    "Topic :: Software Development :: Libraries",
+]
+dependencies = [
+    "cffi>=1.0.0",
+    "imap-tools",
+    "pluggy",
+    "requests",
+]
+dynamic = [
+    "version"
+]
+
+[project.urls]
+"Home" = "https://github.com/deltachat/deltachat-core-rust/"
+"Bug Tracker" = "https://github.com/deltachat/deltachat-core-rust/issues"
+"Documentation" = "https://py.delta.chat/"
+
+[project.entry-points.pytest11]
+"deltachat.testplugin" = "deltachat.testplugin"
+
 [tool.setuptools_scm]
 root = ".."
 tag_regex = '^(?P<prefix>py-)?(?P<version>[^\+]+)(?P<suffix>.*)?$'

python/setup.cfg

@@ -1,4 +0,0 @@
-[devpi:upload]
-formats = sdist.tgz
-no-vcs = 1
-

python/setup.py

@@ -1,40 +1,4 @@
-import os
-import re
-
-import setuptools
-
-
-def main():
-    with open("README.rst") as f:
-        long_description = f.read()
-    setuptools.setup(
-        name="deltachat",
-        description="Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat",
-        long_description=long_description,
-        author="holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors",
-        install_requires=["cffi>=1.0.0", "pluggy", "imap-tools", "requests"],
-        setup_requires=[
-            "setuptools_scm",  # required for compatibility with `python3 setup.py sdist`
-            "pkgconfig",
-        ],
-        packages=setuptools.find_packages("src"),
-        package_dir={"": "src"},
-        cffi_modules=["src/deltachat/_build.py:ffibuilder"],
-        entry_points={
-            "pytest11": [
-                "deltachat.testplugin = deltachat.testplugin",
-            ],
-        },
-        classifiers=[
-            "Development Status :: 4 - Beta",
-            "Intended Audience :: Developers",
-            "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
-            "Programming Language :: Python :: 3",
-            "Topic :: Communications :: Email",
-            "Topic :: Software Development :: Libraries",
-        ],
-    )
-
+from setuptools import setup
 
 if __name__ == "__main__":
-    main()
+    setup(cffi_modules=["src/deltachat/_build.py:ffibuilder"])

python/src/deltachat/__init__.py

@@ -60,29 +60,7 @@ def run_cmdline(argv=None, account_plugins=None):
 
     ac = Account(args.db)
 
-    if args.show_ffi:
-        ac.set_config("displayname", "bot")
-        log = events.FFIEventLogger(ac)
-        ac.add_account_plugin(log)
-
-    for plugin in account_plugins or []:
-        print("adding plugin", plugin)
-        ac.add_account_plugin(plugin)
-
-    if not ac.is_configured():
-        assert (
-            args.email and args.password
-        ), "you must specify --email and --password once to configure this database/account"
-        ac.set_config("addr", args.email)
-        ac.set_config("mail_pw", args.password)
-        ac.set_config("mvbox_move", "0")
-        ac.set_config("sentbox_watch", "0")
-        ac.set_config("bot", "1")
-        configtracker = ac.configure()
-        configtracker.wait_finish()
-
-    # start IO threads and configure if neccessary
-    ac.start_io()
+    ac.run_account(addr=args.email, password=args.password, account_plugins=account_plugins, show_ffi=args.show_ffi)
 
     print("{}: waiting for message".format(ac.get_config("addr")))
 

python/src/deltachat/account.py

@@ -20,7 +20,7 @@ from .cutil import (
     from_optional_dc_charpointer,
     iter_array,
 )
-from .events import EventThread
+from .events import EventThread, FFIEventLogger
 from .message import Message
 from .tracker import ConfigureTracker, ImexTracker
 
@@ -169,8 +169,6 @@ class Account(object):
         """
         self._check_config_key(name)
         namebytes = name.encode("utf8")
-        if namebytes == b"addr" and self.is_configured():
-            raise ValueError("can not change 'addr' after account is configured.")
         if isinstance(value, (int, bool)):
             value = str(int(value))
         if value is not None:
@@ -598,6 +596,36 @@ class Account(object):
     # meta API for start/stop and event based processing
     #
 
+    def run_account(self, addr=None, password=None, account_plugins=None, show_ffi=False):
+        """get the account running, configure it if necessary. add plugins if provided.
+
+        :param addr: the email address of the account
+        :param password: the password of the account
+        :param account_plugins: a list of plugins to add
+        :param show_ffi: show low level ffi events
+        """
+        if show_ffi:
+            self.set_config("displayname", "bot")
+            log = FFIEventLogger(self)
+            self.add_account_plugin(log)
+
+        for plugin in account_plugins or []:
+            print("adding plugin", plugin)
+            self.add_account_plugin(plugin)
+
+        if not self.is_configured():
+            assert addr and password, "you must specify email and password once to configure this database/account"
+            self.set_config("addr", addr)
+            self.set_config("mail_pw", password)
+            self.set_config("mvbox_move", "0")
+            self.set_config("sentbox_watch", "0")
+            self.set_config("bot", "1")
+            configtracker = self.configure()
+            configtracker.wait_finish()
+
+        # start IO threads and configure if neccessary
+        self.start_io()
+
     def add_account_plugin(self, plugin, name=None):
         """add an account plugin which implements one or more of
         the :class:`deltachat.hookspec.PerAccount` hooks.
@@ -680,8 +708,9 @@ class Account(object):
         """Start configuration process and return a Configtracker instance
         on which you can block with wait_finish() to get a True/False success
         value for the configuration process.
+
+        :param reconfigure: deprecated, doesn't need to be checked anymore.
         """
-        assert self.is_configured() == reconfigure
         if not self.get_config("addr") or not self.get_config("mail_pw"):
             raise MissingCredentials("addr or mail_pwd not set in config")
         configtracker = ConfigureTracker(self)

python/src/deltachat/testplugin.py

@@ -497,6 +497,8 @@ class ACFactory:
             configdict = dict(
                 addr=cloned_from.get_config("addr"),
                 mail_pw=cloned_from.get_config("mail_pw"),
+                imap_certificate_checks=cloned_from.get_config("imap_certificate_checks"),
+                smtp_certificate_checks=cloned_from.get_config("smtp_certificate_checks"),
             )
         configdict.update(kwargs)
         ac = self._get_cached_account(addr=configdict["addr"]) if cache else None

python/tests/package_wheels.py

@@ -1,13 +0,0 @@
-import os
-import subprocess
-import sys
-
-if __name__ == "__main__":
-    assert len(sys.argv) == 2
-    wheelhousedir = sys.argv[1]
-    # pip wheel will build in an isolated tmp dir that does not have git
-    # history so setuptools_scm can not automatically determine a
-    # version there. So pass in the version through an env var.
-    version = subprocess.check_output(["python", "setup.py", "--version"]).strip().split(b"\n")[-1]
-    os.environ["SETUPTOOLS_SCM_PRETEND_VERSION"] = version.decode("ascii")
-    subprocess.check_call(("pip wheel . -w %s" % wheelhousedir).split())

python/tests/test_1_online.py

@@ -1581,8 +1581,9 @@ def test_set_get_group_image(acfactory, data, lp):
 
     lp.sec("ac2: wait for receiving message from ac1")
     msg1 = ac2._evtracker.wait_next_incoming_message()
+    assert msg1.is_system_message()  # Member added
     msg2 = ac2._evtracker.wait_next_incoming_message()
-    assert msg1.text == "hi" or msg2.text == "hi"
+    assert msg2.text == "hi"
     assert msg1.chat.id == msg2.chat.id
 
     lp.sec("ac2: see if chat now has got the profile image")
@@ -1596,6 +1597,8 @@ def test_set_get_group_image(acfactory, data, lp):
     lp.sec("ac2: delete profile image from chat")
     msg1.chat.remove_profile_image()
     msg_back = ac1._evtracker.wait_next_incoming_message()
+    assert msg_back.text == "Group image deleted by {}.".format(ac2.get_config("addr"))
+    assert msg_back.is_system_message()
     assert msg_back.chat == chat
     assert chat.get_profile_image() is None
 
@@ -1854,7 +1857,7 @@ def test_configure_error_msgs_invalid_server(acfactory):
     # Can't connect so it probably should say something about "internet"
     # again, should not repeat itself
     # If this fails then probably `e.msg.to_lowercase().contains("could not resolve")`
-    # in configure/mod.rs returned false because the error message was changed
+    # in configure.rs returned false because the error message was changed
     # (i.e. did not contain "could not resolve" anymore)
     assert (ev.data2.count("internet") + ev.data2.count("network")) == 1
     # Should mention that it can't connect:
@@ -2052,6 +2055,47 @@ def test_delete_deltachat_folder(acfactory):
     assert "DeltaChat" in ac1.direct_imap.list_folders()
 
 
+def test_aeap_flow_verified(acfactory, lp):
+    """Test that a new address is added to a contact when it changes its address."""
+    ac1, ac2, ac1new = acfactory.get_online_accounts(3)
+
+    lp.sec("ac1: create verified-group QR, ac2 scans and joins")
+    chat = ac1.create_group_chat("hello", verified=True)
+    assert chat.is_protected()
+    qr = chat.get_join_qr()
+    lp.sec("ac2: start QR-code based join-group protocol")
+    chat2 = ac2.qr_join_chat(qr)
+    assert chat2.id >= 10
+    ac1._evtracker.wait_securejoin_inviter_progress(1000)
+
+    lp.sec("sending first message")
+    msg_out = chat.send_text("old address")
+
+    lp.sec("receiving first message")
+    ac2._evtracker.wait_next_incoming_message()  # member added message
+    msg_in_1 = ac2._evtracker.wait_next_incoming_message()
+    assert msg_in_1.text == msg_out.text
+
+    lp.sec("changing email account")
+    ac1.set_config("addr", ac1new.get_config("addr"))
+    ac1.set_config("mail_pw", ac1new.get_config("mail_pw"))
+    ac1.stop_io()
+    configtracker = ac1.configure()
+    configtracker.wait_finish()
+    ac1.start_io()
+
+    lp.sec("sending second message")
+    msg_out = chat.send_text("changed address")
+
+    lp.sec("receiving second message")
+    msg_in_2 = ac2._evtracker.wait_next_incoming_message()
+    assert msg_in_2.text == msg_out.text
+    assert msg_in_2.chat.id == msg_in_1.chat.id
+    assert msg_in_2.get_sender_contact().addr == ac1new.get_config("addr")
+    assert len(msg_in_2.chat.get_contacts()) == 2
+    assert ac1new.get_config("addr") in [contact.addr for contact in msg_in_2.chat.get_contacts()]
+
+
 class TestOnlineConfigureFails:
     def test_invalid_password(self, acfactory):
         configdict = acfactory.get_next_liveconfig()

python/tests/test_3_offline.py

@@ -497,12 +497,6 @@ class TestOfflineChat:
         contact = msg.get_sender_contact()
         assert contact == ac1.get_self_contact()
 
-    def test_set_config_after_configure_is_forbidden(self, ac1):
-        assert ac1.get_config("mail_pw")
-        assert ac1.is_configured()
-        with pytest.raises(ValueError):
-            ac1.set_config("addr", "123@example.org")
-
     def test_import_export_on_unencrypted_acct(self, acfactory, tmpdir):
         backupdir = tmpdir.mkdir("backup")
         ac1 = acfactory.get_pseudo_configured_account()

python/tox.ini

@@ -9,9 +9,8 @@ envlist =
 [testenv]
 commands = 
     pytest -n6 --extra-info --reruns 2 --reruns-delay 5 -v -rsXx --ignored --strict-tls {posargs: tests examples}
-    python tests/package_wheels.py {toxworkdir}/wheelhouse
+    pip wheel . -w {toxworkdir}/wheelhouse --no-deps
 passenv = 
-    TRAVIS 
     DCC_RS_DEV
     DCC_RS_TARGET
     DCC_NEW_TMP_EMAIL 

scripts/README.md

@@ -14,18 +14,15 @@ and an own build machine.
 - `remote_tests_rust.sh` rsyncs to the build machine and runs
   `run-rust-test.sh` remotely on the build machine. 
 
-- `doxygen/Dockerfile` specifies an image that contains
-  the doxygen tool which is used by `run-doxygen.sh` 
-  to generate C-docs which are then uploaded 
-  via `ci_upload.sh` to `https://c.delta.chat/_unofficial_unreleased_docs/<BRANCH>`
-  (and the master branch is linked to https://c.delta.chat proper). 
+- `run-doxygen.sh` generates C-docs which are then uploaded to https://c.delta.chat/
 
+- `run_all.sh` builds Python wheels
 
 ## Triggering runs on the build machine locally (fast!)
 
 There is experimental support for triggering a remote Python or Rust test run 
 from your local checkout/branch. You will need to be authorized to login to 
-the build machine (ask your friendly sysadmin on #deltachat Libera Chat) to type::
+the build machine (ask your friendly sysadmin on #deltachat Libera Chat) to type:
 
     scripts/manual_remote_tests.sh rust
     scripts/manual_remote_tests.sh python
@@ -33,19 +30,18 @@ the build machine (ask your friendly sysadmin on #deltachat Libera Chat) to type
 This will **rsync** your current checkout to the remote build machine 
 (no need to commit before) and then run either rust or python tests. 
 
-# Outdated files (for later re-use)
+# coredeps Dockerfile
 
 `coredeps/Dockerfile` specifies an image that contains all 
-of Delta Chat's core dependencies. It used to run 
-python tests and build wheels (binary packages for Python)
+of Delta Chat's core dependencies. It is used to
+build python wheels (binary packages for Python).
 
 You can build the docker images yourself locally
-to avoid the relatively large download:: 
+to avoid the relatively large download:
  
     cd scripts  # where all CI things are 
-    docker build -t deltachat/coredeps docker-coredeps
-    docker build -t deltachat/doxygen docker-doxygen 
+    docker build -t deltachat/coredeps coredeps
 
-Additionally, you can install qemu and build arm64 docker image:
+Additionally, you can install qemu and build arm64 docker image on x86\_64 machine:
     apt-get install qemu binfmt-support qemu-user-static
-    docker build -t deltachat/coredeps-arm64 docker-coredeps-arm64
+    docker build -t deltachat/coredeps-arm64 --build-arg BASEIMAGE=quay.io/pypa/manylinux2014_aarch64 coredeps

scripts/concourse/docs_wheels.yml

@@ -29,18 +29,19 @@ jobs:
         - name: c-docs
       image_resource:
         source:
-          repository: hrektts/doxygen
+          repository: alpine
         type: registry-image
       platform: linux
       run:
-        path: bash
+        path: sh
         args:
-        - -exc
+        - -ec
         - |
+          apk add --no-cache doxygen git
           cd deltachat-core-rust
-          bash scripts/run-doxygen.sh
+          scripts/run-doxygen.sh
           cd ..
-          cp -av deltachat-core-rust/deltachat-ffi/{html,xml} c-docs/
+          cp -av deltachat-core-rust/deltachat-ffi/html deltachat-core-rust/deltachat-ffi/xml c-docs/
 
   - task: upload-c-docs
     config:
@@ -302,3 +303,73 @@ jobs:
           devpi use https://m.devpi.net/dc/master
           devpi login ((devpi.login)) --password ((devpi.password))
           devpi upload py-wheels/*musllinux_1_1_x86_64*
+
+- name: python-musl-aarch64
+  plan:
+  - get: deltachat-core-rust
+  - get: deltachat-core-rust-release
+    trigger: true
+
+  # Build manylinux image with additional dependencies
+  - task: build-coredeps
+    privileged: true
+    config:
+      inputs:
+        # Building the latest, not tagged coredeps
+        - name: deltachat-core-rust
+      image_resource:
+        source:
+          repository: concourse/oci-build-task
+        type: registry-image
+      outputs:
+        - name: coredeps-image
+          path: image
+      params:
+        CONTEXT: deltachat-core-rust/scripts/coredeps
+        UNPACK_ROOTFS: "true"
+        BUILD_ARG_BASEIMAGE: quay.io/pypa/musllinux_1_1_aarch64
+      platform: linux
+      caches:
+      - path: cache
+      run:
+        path: build
+
+  # Use built image to build python wheels
+  - task: build-wheels
+    image: coredeps-image
+    config:
+      inputs:
+        - name: deltachat-core-rust-release
+          path: .
+      outputs:
+        - name: py-wheels
+          path: ./python/.docker-tox/wheelhouse/
+      platform: linux
+      run:
+        path: bash
+        args:
+        - -exc
+        - |
+          scripts/run_all.sh
+
+  # Upload musl aarch64 wheels
+  - task: upload-wheels
+    config:
+      inputs:
+        - name: py-wheels
+      image_resource:
+        type: registry-image
+        source:
+          repository: debian
+      platform: linux
+      run:
+        path: sh
+        args:
+        - -ec
+        - |
+          apt-get update -y
+          apt-get install -y --no-install-recommends python3-pip python3-setuptools
+          pip3 install devpi
+          devpi use https://m.devpi.net/dc/master
+          devpi login ((devpi.login)) --password ((devpi.password))
+          devpi upload py-wheels/*musllinux_1_1_aarch64*

scripts/coverage.sh

@@ -17,7 +17,7 @@ export RUSTC_BOOTSTRAP=1
 # [2] https://github.com/mozilla/grcov/issues/595
 
 export CARGO_INCREMENTAL=0
-export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort"
+export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort -Cdebuginfo=2"
 export RUSTDOCFLAGS="-Cpanic=abort"
 cargo clean
 cargo build

scripts/docker-doxygen/Dockerfile

@@ -1,5 +0,0 @@
-FROM debian:stable
-
-# this is tagged as deltachat/doxygen 
-
-RUN apt-get update && apt-get install -y doxygen 

scripts/run-doxygen.sh

@@ -1,6 +1,5 @@
-#!/usr/bin/env bash
-
-set -ex
+#!/bin/sh
+set -e
 
 cd deltachat-ffi 
 PROJECT_NUMBER=$(git log -1 --format="%h (%cd)") doxygen

scripts/run_all.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 #
 # Build the Delta Chat Core Rust library, Python wheels and docs 
 
@@ -8,21 +8,16 @@ set -e -x
 
 # compile core lib
 
-export PATH=/root/.cargo/bin:$PATH
-cargo build --release -p deltachat_ffi
-# cargo test --all --all-features
+cargo build --release -p deltachat_ffi --features jsonrpc
 
 # Statically link against libdeltachat.a.
-export DCC_RS_DEV=$(pwd)
+export DCC_RS_DEV="$PWD"
 export DCC_RS_TARGET=release
 
-# Configure access to a base python and to several python interpreters
-# needed by tox below.
-export PATH=$PATH:/opt/python/cp37-cp37m/bin
 export PYTHONDONTWRITEBYTECODE=1
+cd python
 
 TOXWORKDIR=.docker-tox
-pushd python
 # prepare a clean tox run
 rm -rf tests/__pycache__
 rm -rf src/deltachat/__pycache__
@@ -33,11 +28,13 @@ mkdir -p $TOXWORKDIR
 # Note that the independent remote_tests_python step does all kinds of
 # live-testing already. 
 unset DCC_NEW_TMP_EMAIL
-tox --workdir "$TOXWORKDIR" -e py37,py38,py39,py310,pypy37,pypy38,pypy39,auditwheels
-popd
+
+# Try to build wheels for a range of interpreters, but don't fail if they are not available.
+# E.g. musllinux_1_1 does not have PyPy interpreters as of 2022-07-10
+tox --workdir "$TOXWORKDIR" -e py37,py38,py39,py310,pypy37,pypy38,pypy39,auditwheels --skip-missing-interpreters true
 
 
 echo -----------------------
 echo generating python docs
 echo -----------------------
-(cd python && tox --workdir "$TOXWORKDIR" -e doc)
+tox --workdir "$TOXWORKDIR" -e doc

scripts/set_core_version.py

@@ -42,15 +42,15 @@ def replace_toml_version(relpath, newversion):
 
 
 def read_json_version(relpath):
-    p = pathlib.Path("package.json")
+    p = pathlib.Path(relpath)
     assert p.exists()
     with open(p, "r") as f:
         json_data = json.loads(f.read())
     return json_data["version"]
 
 
-def update_package_json(newversion):
-    p = pathlib.Path("package.json")
+def update_package_json(relpath, newversion):
+    p = pathlib.Path(relpath)
     assert p.exists()
     with open(p, "r") as f:
         json_data = json.loads(f.read())
@@ -63,14 +63,16 @@ def main():
     parser = ArgumentParser(prog="set_core_version")
     parser.add_argument("newversion")
 
-    toml_list = ["Cargo.toml", "deltachat-ffi/Cargo.toml"]
+    toml_list = ["Cargo.toml", "deltachat-ffi/Cargo.toml",  "deltachat-jsonrpc/Cargo.toml"]
+    json_list = ["package.json",  "deltachat-jsonrpc/typescript/package.json"]
     try:
         opts = parser.parse_args()
     except SystemExit:
         print()
         for x in toml_list:
             print("{}: {}".format(x, read_toml_version(x)))
-        print("package.json:", str(read_json_version("package.json")))
+        for x in json_list:
+            print("{}: {}".format(x, read_json_version(x)))
         print()
         raise SystemExit("need argument: new version, example: 1.25.0")
 
@@ -91,9 +93,12 @@ def main():
         else:
             raise SystemExit("CHANGELOG.md contains no entry for version: {}".format(newversion))
 
-    replace_toml_version("Cargo.toml", newversion)
-    replace_toml_version("deltachat-ffi/Cargo.toml", newversion)
-    update_package_json(newversion)
+    for toml_filename in toml_list:
+        replace_toml_version(toml_filename, newversion)
+    
+    for json_filename in json_list:
+        update_package_json(json_filename, newversion)
+
 
     print("running cargo check")
     subprocess.call(["cargo", "check"])

src/accounts.rs

@@ -33,7 +33,7 @@ impl Accounts {
     }
 
     /// Creates a new default structure.
-    pub async fn create(dir: &PathBuf) -> Result<()> {
+    pub async fn create(dir: &Path) -> Result<()> {
         fs::create_dir_all(dir)
             .await
             .context("failed to create folder")?;
@@ -580,7 +580,7 @@ mod tests {
             let ids = accounts.get_all().await;
             assert_eq!(ids.len(), 1);
 
-            let id0 = *ids.get(0).unwrap();
+            let id0 = *ids.first().unwrap();
             let ctx = accounts.get_account(id0).await.unwrap();
             ctx.set_config(crate::config::Config::Addr, Some("one@example.org"))
                 .await?;
@@ -620,7 +620,7 @@ mod tests {
             let ids = accounts.get_all().await;
             assert_eq!(ids.len(), 3);
 
-            let id0 = *ids.get(0).unwrap();
+            let id0 = *ids.first().unwrap();
             let ctx = accounts.get_account(id0).await.unwrap();
             assert_eq!(
                 ctx.get_config(crate::config::Config::Addr).await?,

src/blob.rs

@@ -738,7 +738,7 @@ mod tests {
         check_image_size(avatar_src, 1000, 1000);
         check_image_size(&avatar_blob, BALANCED_AVATAR_SIZE, BALANCED_AVATAR_SIZE);
 
-        async fn file_size(path_buf: &PathBuf) -> u64 {
+        async fn file_size(path_buf: &Path) -> u64 {
             let file = File::open(path_buf).await.unwrap();
             file.metadata().await.unwrap().len()
         }

src/chat.rs

@@ -40,7 +40,7 @@ use crate::webxdc::WEBXDC_SUFFIX;
 use crate::{location, sql};
 
 /// An chat item, such as a message or a marker.
-#[derive(Debug, Copy, Clone, PartialEq)]
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
 pub enum ChatItem {
     Message {
         msg_id: MsgId,
@@ -1129,6 +1129,11 @@ impl Chat {
         &self.name
     }
 
+    /// Returns mailing list address where messages are sent to.
+    pub fn get_mailinglist_addr(&self) -> &str {
+        self.param.get(Param::ListPost).unwrap_or_default()
+    }
+
     /// Returns profile image path for the chat.
     pub async fn get_profile_image(&self, context: &Context) -> Result<Option<PathBuf>> {
         if let Some(image_rel) = self.param.get(Param::ProfileImage) {
@@ -2418,7 +2423,7 @@ pub(crate) async fn mark_old_messages_as_noticed(
 
 pub async fn get_chat_media(
     context: &Context,
-    chat_id: ChatId,
+    chat_id: Option<ChatId>,
     msg_type: Viewtype,
     msg_type2: Viewtype,
     msg_type3: Viewtype,
@@ -2429,11 +2434,13 @@ pub async fn get_chat_media(
         .query_map(
             "SELECT id
                FROM msgs
-              WHERE chat_id=?
+              WHERE (1=? OR chat_id=?)
                 AND (type=? OR type=? OR type=?)
+                AND hidden=0
               ORDER BY timestamp, id;",
             paramsv![
-                chat_id,
+                if chat_id.is_none() { 1i32 } else { 0i32 },
+                chat_id.unwrap_or_else(|| ChatId::new(0)),
                 msg_type,
                 if msg_type2 != Viewtype::Unknown {
                     msg_type2
@@ -2474,7 +2481,7 @@ pub async fn get_next_media(
     if let Ok(msg) = Message::load_from_db(context, curr_msg_id).await {
         let list: Vec<MsgId> = get_chat_media(
             context,
-            msg.chat_id,
+            Some(msg.chat_id),
             if msg_type != Viewtype::Unknown {
                 msg_type
             } else {
@@ -2518,7 +2525,7 @@ pub async fn get_chat_contacts(context: &Context, chat_id: ChatId) -> Result<Vec
                LEFT JOIN contacts c
                       ON c.id=cc.contact_id
               WHERE cc.chat_id=?
-              ORDER BY c.id=1, LOWER(c.name||c.addr), c.id;",
+              ORDER BY c.id=1, c.last_seen DESC, c.id DESC;",
             paramsv![chat_id],
             |row| row.get::<_, ContactId>(0),
             |ids| ids.collect::<Result<Vec<_>, _>>().map_err(Into::into),
@@ -2781,7 +2788,7 @@ pub(crate) async fn shall_attach_selfavatar(context: &Context, chat_id: ChatId)
     Ok(needs_attach)
 }
 
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 pub enum MuteDuration {
     NotMuted,
     Forever,
@@ -2856,7 +2863,7 @@ pub async fn remove_contact_from_chat(
     let mut success = false;
 
     /* we do not check if "contact_id" exists but just delete all records with the id from chats_contacts */
-    /* this allows to delete pending references to deleted contacts.  Of course, this should _not_ happen. */
+    /* this allows to delete pending references to deleted contacts. Of course, this should _not_ happen. */
     if let Ok(chat) = Chat::load_from_db(context, chat_id).await {
         if chat.typ == Chattype::Group || chat.typ == Chattype::Broadcast {
             if !chat.is_self_in_chat(context).await? {
@@ -5449,8 +5456,8 @@ mod tests {
         assert_eq!(
             chat_id.get_encryption_info(&alice).await?,
             "No encryption:\n\
-            bob@example.net\n\
-            fiona@example.net"
+            fiona@example.net\n\
+            bob@example.net"
         );
 
         let direct_chat = bob.create_chat(&alice).await;
@@ -5481,4 +5488,158 @@ mod tests {
 
         Ok(())
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_get_chat_media() -> Result<()> {
+        let t = TestContext::new_alice().await;
+        let chat_id1 = create_group_chat(&t, ProtectionStatus::Unprotected, "foo").await?;
+        let chat_id2 = create_group_chat(&t, ProtectionStatus::Unprotected, "bar").await?;
+
+        assert_eq!(
+            get_chat_media(
+                &t,
+                Some(chat_id1),
+                Viewtype::Image,
+                Viewtype::Sticker,
+                Viewtype::Unknown
+            )
+            .await?
+            .len(),
+            0
+        );
+
+        async fn send_media(
+            t: &TestContext,
+            chat_id: ChatId,
+            msg_type: Viewtype,
+            name: &str,
+            bytes: &[u8],
+        ) -> Result<MsgId> {
+            let file = t.get_blobdir().join(name);
+            tokio::fs::write(&file, bytes).await?;
+            let mut msg = Message::new(msg_type);
+            msg.set_file(file.to_str().unwrap(), None);
+            send_msg(t, chat_id, &mut msg).await
+        }
+
+        send_media(
+            &t,
+            chat_id1,
+            Viewtype::Image,
+            "a.jpg",
+            include_bytes!("../test-data/image/rectangle200x180-rotated.jpg"),
+        )
+        .await?;
+        send_media(
+            &t,
+            chat_id1,
+            Viewtype::Sticker,
+            "b.png",
+            include_bytes!("../test-data/image/avatar64x64.png"),
+        )
+        .await?;
+        send_media(
+            &t,
+            chat_id2,
+            Viewtype::Image,
+            "c.jpg",
+            include_bytes!("../test-data/image/avatar64x64.png"),
+        )
+        .await?;
+        send_media(
+            &t,
+            chat_id2,
+            Viewtype::Webxdc,
+            "d.xdc",
+            include_bytes!("../test-data/webxdc/minimal.xdc"),
+        )
+        .await?;
+
+        assert_eq!(
+            get_chat_media(
+                &t,
+                Some(chat_id1),
+                Viewtype::Image,
+                Viewtype::Unknown,
+                Viewtype::Unknown,
+            )
+            .await?
+            .len(),
+            1
+        );
+        assert_eq!(
+            get_chat_media(
+                &t,
+                Some(chat_id1),
+                Viewtype::Sticker,
+                Viewtype::Unknown,
+                Viewtype::Unknown,
+            )
+            .await?
+            .len(),
+            1
+        );
+        assert_eq!(
+            get_chat_media(
+                &t,
+                Some(chat_id1),
+                Viewtype::Sticker,
+                Viewtype::Image,
+                Viewtype::Unknown,
+            )
+            .await?
+            .len(),
+            2
+        );
+        assert_eq!(
+            get_chat_media(
+                &t,
+                Some(chat_id2),
+                Viewtype::Webxdc,
+                Viewtype::Unknown,
+                Viewtype::Unknown,
+            )
+            .await?
+            .len(),
+            1
+        );
+        assert_eq!(
+            get_chat_media(
+                &t,
+                None,
+                Viewtype::Image,
+                Viewtype::Unknown,
+                Viewtype::Unknown,
+            )
+            .await?
+            .len(),
+            2
+        );
+        assert_eq!(
+            get_chat_media(
+                &t,
+                None,
+                Viewtype::Image,
+                Viewtype::Sticker,
+                Viewtype::Unknown,
+            )
+            .await?
+            .len(),
+            3
+        );
+        assert_eq!(
+            get_chat_media(
+                &t,
+                None,
+                Viewtype::Image,
+                Viewtype::Sticker,
+                Viewtype::Webxdc,
+            )
+            .await?
+            .len(),
+            4
+        );
+
+        Ok(())
+    }
 }

src/color.rs

@@ -11,7 +11,7 @@ use sha1::{Digest, Sha1};
 fn str_to_angle(s: &str) -> f64 {
     let bytes = s.as_bytes();
     let result = Sha1::digest(bytes);
-    let checksum: u16 = result.get(0).map_or(0, |&x| u16::from(x))
+    let checksum: u16 = result.first().map_or(0, |&x| u16::from(x))
         + 256 * result.get(1).map_or(0, |&x| u16::from(x));
     f64::from(checksum) / 65536.0 * 360.0
 }

src/config.rs

@@ -1,7 +1,7 @@
 //! # Key-value configuration management.
 
 use anyhow::{ensure, Context as _, Result};
-use strum::{EnumProperty, IntoEnumIterator};
+use strum::{EnumProperty as EnumPropertyTrait, IntoEnumIterator};
 use strum_macros::{AsRefStr, Display, EnumIter, EnumProperty, EnumString};
 
 use crate::blob::BlobObject;

src/configure.rs

@@ -12,9 +12,11 @@ use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 use tokio::task;
 
 use crate::config::Config;
+use crate::contact::addr_cmp;
 use crate::context::Context;
 use crate::imap::Imap;
 use crate::job;
+use crate::log::LogExt;
 use crate::login_param::{CertificateChecks, LoginParam, ServerLoginParam, Socks5Config};
 use crate::message::{Message, Viewtype};
 use crate::oauth2::get_oauth2_addr;
@@ -101,35 +103,11 @@ impl Context {
         info!(self, "Configure ...");
 
         let mut param = LoginParam::load_candidate_params(self).await?;
+        let old_addr = self.get_config(Config::ConfiguredAddr).await?;
         let success = configure(self, &mut param).await;
         self.set_config(Config::NotifyAboutWrongPw, None).await?;
 
-        if let Some(provider) = param.provider {
-            if let Some(config_defaults) = &provider.config_defaults {
-                for def in config_defaults.iter() {
-                    if !self.config_exists(def.key).await? {
-                        info!(self, "apply config_defaults {}={}", def.key, def.value);
-                        self.set_config(def.key, Some(def.value)).await?;
-                    } else {
-                        info!(
-                            self,
-                            "skip already set config_defaults {}={}", def.key, def.value
-                        );
-                    }
-                }
-            }
-
-            if !provider.after_login_hint.is_empty() {
-                let mut msg = Message::new(Viewtype::Text);
-                msg.text = Some(provider.after_login_hint.to_string());
-                if chat::add_device_msg(self, Some("core-provider-info"), Some(&mut msg))
-                    .await
-                    .is_err()
-                {
-                    warn!(self, "cannot add after_login_hint as core-provider-info");
-                }
-            }
-        }
+        on_configure_completed(self, param, old_addr).await?;
 
         success?;
         self.set_config(Config::NotifyAboutWrongPw, Some("1"))
@@ -138,6 +116,54 @@ impl Context {
     }
 }
 
+async fn on_configure_completed(
+    context: &Context,
+    param: LoginParam,
+    old_addr: Option<String>,
+) -> Result<()> {
+    if let Some(provider) = param.provider {
+        if let Some(config_defaults) = &provider.config_defaults {
+            for def in config_defaults.iter() {
+                if !context.config_exists(def.key).await? {
+                    info!(context, "apply config_defaults {}={}", def.key, def.value);
+                    context.set_config(def.key, Some(def.value)).await?;
+                } else {
+                    info!(
+                        context,
+                        "skip already set config_defaults {}={}", def.key, def.value
+                    );
+                }
+            }
+        }
+
+        if !provider.after_login_hint.is_empty() {
+            let mut msg = Message::new(Viewtype::Text);
+            msg.text = Some(provider.after_login_hint.to_string());
+            if chat::add_device_msg(context, Some("core-provider-info"), Some(&mut msg))
+                .await
+                .is_err()
+            {
+                warn!(context, "cannot add after_login_hint as core-provider-info");
+            }
+        }
+    }
+
+    if let Some(new_addr) = context.get_config(Config::ConfiguredAddr).await? {
+        if let Some(old_addr) = old_addr {
+            if !addr_cmp(&new_addr, &old_addr) {
+                let mut msg = Message::new(Viewtype::Text);
+                msg.text =
+                    Some(stock_str::aeap_explanation_and_link(context, old_addr, new_addr).await);
+                chat::add_device_msg(context, None, Some(&mut msg))
+                    .await
+                    .ok_or_log_msg(context, "Cannot add AEAP explanation");
+            }
+        }
+    }
+
+    Ok(())
+}
+
 async fn configure(ctx: &Context, param: &mut LoginParam) -> Result<()> {
     progress!(ctx, 1);
 
@@ -642,6 +668,9 @@ async fn nicer_configuration_error(context: &Context, errors: Vec<ConfigurationE
                 .to_lowercase()
                 .contains("temporary failure in name resolution")
             || e.msg.to_lowercase().contains("name or service not known")
+            || e.msg
+                .to_lowercase()
+                .contains("failed to lookup address information")
     }) {
         return stock_str::error_no_network(context).await;
     }

src/contact.rs

@@ -24,9 +24,12 @@ use crate::mimeparser::AvatarAction;
 use crate::param::{Param, Params};
 use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
 use crate::sql::{self, params_iter};
-use crate::tools::{get_abs_path, improve_single_line_input, EmailAddress};
+use crate::tools::{get_abs_path, improve_single_line_input, time, EmailAddress};
 use crate::{chat, stock_str};
 
+/// Time during which a contact is considered as seen recently.
+const SEEN_RECENTLY_SECONDS: i64 = 600;
+
 /// Contact ID, including reserved IDs.
 ///
 /// Some contact IDs are reserved to identify special contacts.  This
@@ -323,6 +326,11 @@ impl Contact {
         self.last_seen
     }
 
+    /// Returns `true` if this contact was seen recently.
+    pub fn was_seen_recently(&self) -> bool {
+        time() - self.last_seen <= SEEN_RECENTLY_SECONDS
+    }
+
     /// Check if a contact is blocked.
     pub async fn is_blocked_load(context: &Context, id: ContactId) -> Result<bool> {
         let blocked = Self::load_from_db(context, id).await?.blocked;
@@ -708,7 +716,7 @@ impl Contact {
                  AND c.blocked=0 \
                  AND (iif(c.name='',c.authname,c.name) LIKE ? OR c.addr LIKE ?) \
                  AND (1=? OR LENGTH(ps.verified_key_fingerprint)!=0)  \
-                 ORDER BY LOWER(iif(c.name='',c.authname,c.name)||c.addr),c.id;",
+                 ORDER BY c.last_seen DESC, c.id DESC;",
                         sql::repeat_vars(self_addrs.len())
                     ),
                     rusqlite::params_from_iter(params_iter(&self_addrs).chain(params_iterv![
@@ -760,7 +768,7 @@ impl Contact {
                  AND id>?
                  AND origin>=?
                  AND blocked=0
-                 ORDER BY LOWER(iif(name='',authname,name)||addr),id;",
+                 ORDER BY last_seen DESC, id DESC;",
                         sql::repeat_vars(self_addrs.len())
                     ),
                     rusqlite::params_from_iter(params_iter(&self_addrs).chain(params_iterv![
@@ -849,7 +857,7 @@ impl Contact {
         let list = context
             .sql
             .query_map(
-                "SELECT id FROM contacts WHERE id>? AND blocked!=0 ORDER BY LOWER(iif(name='',authname,name)||addr),id;",
+                "SELECT id FROM contacts WHERE id>? AND blocked!=0 ORDER BY last_seen DESC, id DESC;",
                 paramsv![ContactId::LAST_SPECIAL],
                 |row| row.get::<_, ContactId>(0),
                 |ids| {
@@ -1429,7 +1437,7 @@ fn split_address_book(book: &str) -> Vec<(&str, &str)> {
         .chunks(2)
         .into_iter()
         .filter_map(|chunk| {
-            let name = chunk.get(0)?;
+            let name = chunk.first()?;
             let addr = chunk.get(1)?;
             Some((*name, *addr))
         })
@@ -1444,7 +1452,7 @@ mod tests {
     use crate::chatlist::Chatlist;
     use crate::message::Message;
     use crate::receive_imf::receive_imf;
-    use crate::test_utils::{self, TestContext};
+    use crate::test_utils::{self, TestContext, TestContextManager};
 
     #[test]
     fn test_contact_id_values() {
@@ -2270,4 +2278,28 @@ Hi."#;
 
         Ok(())
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_was_seen_recently() -> Result<()> {
+        let mut tcm = TestContextManager::new().await;
+        let alice = tcm.alice().await;
+        let bob = tcm.bob().await;
+
+        let chat = alice.create_chat(&bob).await;
+        let sent_msg = alice.send_text(chat.id, "moin").await;
+
+        let chat = bob.create_chat(&alice).await;
+        let contacts = chat::get_chat_contacts(&bob, chat.id).await?;
+        let contact = Contact::get_by_id(&bob, *contacts.first().unwrap()).await?;
+        assert!(!contact.was_seen_recently());
+
+        bob.recv_msg(&sent_msg).await;
+        let contact = Contact::get_by_id(&bob, *contacts.first().unwrap()).await?;
+        assert!(contact.was_seen_recently());
+
+        let self_contact = Contact::get_by_id(&bob, ContactId::SELF).await?;
+        assert!(!self_contact.was_seen_recently());
+
+        Ok(())
+    }
 }

src/context.rs

@@ -239,6 +239,17 @@ impl Context {
         }
     }
 
+    /// Restarts the IO scheduler if it was running before
+    /// when it is not running this is an no-op
+    pub async fn restart_io_if_running(&self) {
+        info!(self, "restarting IO");
+        let is_running = { self.inner.scheduler.read().await.is_some() };
+        if is_running {
+            self.stop_io().await;
+            self.start_io().await;
+        }
+    }
+
     /// Returns a reference to the underlying SQL instance.
     ///
     /// Warning: this is only here for testing, not part of the public API.
@@ -594,7 +605,7 @@ impl Context {
 
         let list = if let Some(chat_id) = chat_id {
             do_query(
-                "SELECT m.id AS id, m.timestamp AS timestamp
+                "SELECT m.id AS id
                  FROM msgs m
                  LEFT JOIN contacts ct
                         ON m.from_id=ct.id
@@ -618,7 +629,7 @@ impl Context {
             // According to some tests, this limit speeds up eg. 2 character searches by factor 10.
             // The limit is documented and UI may add a hint when getting 1000 results.
             do_query(
-                "SELECT m.id AS id, m.timestamp AS timestamp
+                "SELECT m.id AS id
                  FROM msgs m
                  LEFT JOIN contacts ct
                         ON m.from_id=ct.id

src/decrypt.rs

@@ -0,0 +1,375 @@
+//! End-to-end decryption support.
+
+use std::collections::HashSet;
+
+use anyhow::{Context as _, Result};
+use mailparse::ParsedMail;
+
+use crate::aheader::Aheader;
+use crate::contact::addr_cmp;
+use crate::context::Context;
+use crate::headerdef::HeaderDef;
+use crate::headerdef::HeaderDefMap;
+use crate::key::{DcKey, Fingerprint, SignedPublicKey, SignedSecretKey};
+use crate::keyring::Keyring;
+use crate::log::LogExt;
+use crate::peerstate::Peerstate;
+use crate::pgp;
+
+/// Tries to decrypt a message, but only if it is structured as an
+/// Autocrypt message.
+///
+/// Returns decrypted body and a set of valid signature fingerprints
+/// if successful.
+///
+/// If the message is wrongly signed, this will still return the decrypted
+/// message but the HashSet will be empty.
+pub async fn try_decrypt(
+    context: &Context,
+    mail: &ParsedMail<'_>,
+    decryption_info: &DecryptionInfo,
+) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
+    // Possibly perform decryption
+    let public_keyring_for_validate = keyring_from_peerstate(&decryption_info.peerstate);
+
+    let encrypted_data_part = match get_autocrypt_mime(mail)
+        .or_else(|| get_mixed_up_mime(mail))
+        .or_else(|| get_attachment_mime(mail))
+    {
+        None => {
+            // not an autocrypt mime message, abort and ignore
+            return Ok(None);
+        }
+        Some(res) => res,
+    };
+    info!(context, "Detected Autocrypt-mime message");
+    let private_keyring: Keyring<SignedSecretKey> = Keyring::new_self(context)
+        .await
+        .context("failed to get own keyring")?;
+
+    decrypt_part(
+        encrypted_data_part,
+        private_keyring,
+        public_keyring_for_validate,
+    )
+    .await
+}
+
+pub async fn create_decryption_info(
+    context: &Context,
+    mail: &ParsedMail<'_>,
+    message_time: i64,
+) -> Result<DecryptionInfo> {
+    let from = mail
+        .headers
+        .get_header(HeaderDef::From_)
+        .and_then(|from_addr| mailparse::addrparse_header(from_addr).ok())
+        .and_then(|from| from.extract_single_info())
+        .map(|from| from.addr)
+        .unwrap_or_default();
+
+    let autocrypt_header = Aheader::from_headers(&from, &mail.headers)
+        .ok_or_log_msg(context, "Failed to parse Autocrypt header")
+        .flatten();
+
+    let peerstate =
+        get_autocrypt_peerstate(context, &from, autocrypt_header.as_ref(), message_time).await?;
+
+    Ok(DecryptionInfo {
+        from,
+        autocrypt_header,
+        peerstate,
+        message_time,
+    })
+}
+
+#[derive(Debug)]
+pub struct DecryptionInfo {
+    /// The From address. This is the address from the unnencrypted, outer
+    /// From header.
+    pub from: String,
+    pub autocrypt_header: Option<Aheader>,
+    /// The peerstate that will be used to validate the signatures
+    pub peerstate: Option<Peerstate>,
+    /// The timestamp when the message was sent.
+    /// If this is older than the peerstate's last_seen, this probably
+    /// means out-of-order message arrival, We don't modify the
+    /// peerstate in this case.
+    pub message_time: i64,
+}
+
+/// Returns a reference to the encrypted payload of a ["Mixed
+/// Up"][pgpmime-message-mangling] message.
+///
+/// According to [RFC 3156] encrypted messages should have
+/// `multipart/encrypted` MIME type and two parts, but Microsoft
+/// Exchange and ProtonMail IMAP/SMTP Bridge are known to mangle this
+/// structure by changing the type to `multipart/mixed` and prepending
+/// an empty part at the start.
+///
+/// ProtonMail IMAP/SMTP Bridge prepends a part literally saying
+/// "Empty Message", so we don't check its contents at all, checking
+/// only for `text/plain` type.
+///
+/// Returns `None` if the message is not a "Mixed Up" message.
+///
+/// [RFC 3156]: https://www.rfc-editor.org/info/rfc3156
+/// [pgpmime-message-mangling]: https://tools.ietf.org/id/draft-dkg-openpgp-pgpmime-message-mangling-00.html
+fn get_mixed_up_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Option<&'a ParsedMail<'b>> {
+    if mail.ctype.mimetype != "multipart/mixed" {
+        return None;
+    }
+    if let [first_part, second_part, third_part] = &mail.subparts[..] {
+        if first_part.ctype.mimetype == "text/plain"
+            && second_part.ctype.mimetype == "application/pgp-encrypted"
+            && third_part.ctype.mimetype == "application/octet-stream"
+        {
+            Some(third_part)
+        } else {
+            None
+        }
+    } else {
+        None
+    }
+}
+
+/// Returns a reference to the encrypted payload of a message turned into attachment.
+///
+/// Google Workspace has an option "Append footer" which appends standard footer defined
+/// by administrator to all outgoing messages. However, there is no plain text part in
+/// encrypted messages sent by Delta Chat, so Google Workspace turns the message into
+/// multipart/mixed MIME, where the first part is an empty plaintext part with a footer
+/// and the second part is the original encrypted message.
+fn get_attachment_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Option<&'a ParsedMail<'b>> {
+    if mail.ctype.mimetype != "multipart/mixed" {
+        return None;
+    }
+    if let [first_part, second_part] = &mail.subparts[..] {
+        if first_part.ctype.mimetype == "text/plain"
+            && second_part.ctype.mimetype == "multipart/encrypted"
+        {
+            get_autocrypt_mime(second_part)
+        } else {
+            None
+        }
+    } else {
+        None
+    }
+}
+
+/// Returns a reference to the encrypted payload of a valid PGP/MIME message.
+///
+/// Returns `None` if the message is not a valid PGP/MIME message.
+fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Option<&'a ParsedMail<'b>> {
+    if mail.ctype.mimetype != "multipart/encrypted" {
+        return None;
+    }
+    if let [first_part, second_part] = &mail.subparts[..] {
+        if first_part.ctype.mimetype == "application/pgp-encrypted"
+            && second_part.ctype.mimetype == "application/octet-stream"
+        {
+            Some(second_part)
+        } else {
+            None
+        }
+    } else {
+        None
+    }
+}
+
+/// Returns Ok(None) if nothing encrypted was found.
+async fn decrypt_part(
+    mail: &ParsedMail<'_>,
+    private_keyring: Keyring<SignedSecretKey>,
+    public_keyring_for_validate: Keyring<SignedPublicKey>,
+) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
+    let data = mail.get_body_raw()?;
+
+    if has_decrypted_pgp_armor(&data) {
+        let (plain, ret_valid_signatures) =
+            pgp::pk_decrypt(data, private_keyring, &public_keyring_for_validate).await?;
+
+        // Check for detached signatures.
+        // If decrypted part is a multipart/signed, then there is a detached signature.
+        let decrypted_part = mailparse::parse_mail(&plain)?;
+        if let Some((content, valid_detached_signatures)) =
+            validate_detached_signature(&decrypted_part, &public_keyring_for_validate)?
+        {
+            return Ok(Some((content, valid_detached_signatures)));
+        } else {
+            // If the message was wrongly or not signed, still return the plain text.
+            // The caller has to check if the signatures set is empty then.
+
+            return Ok(Some((plain, ret_valid_signatures)));
+        }
+    }
+
+    Ok(None)
+}
+
+#[allow(clippy::indexing_slicing)]
+fn has_decrypted_pgp_armor(input: &[u8]) -> bool {
+    if let Some(index) = input.iter().position(|b| *b > b' ') {
+        if input.len() - index > 26 {
+            let start = index;
+            let end = start + 27;
+
+            return &input[start..end] == b"-----BEGIN PGP MESSAGE-----";
+        }
+    }
+
+    false
+}
+
+/// Validates signatures of Multipart/Signed message part, as defined in RFC 1847.
+///
+/// Returns `None` if the part is not a Multipart/Signed part, otherwise retruns the set of key
+/// fingerprints for which there is a valid signature.
+fn validate_detached_signature(
+    mail: &ParsedMail<'_>,
+    public_keyring_for_validate: &Keyring<SignedPublicKey>,
+) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
+    if mail.ctype.mimetype != "multipart/signed" {
+        return Ok(None);
+    }
+
+    if let [first_part, second_part] = &mail.subparts[..] {
+        // First part is the content, second part is the signature.
+        let content = first_part.raw_bytes;
+        let signature = second_part.get_body_raw()?;
+        let ret_valid_signatures =
+            pgp::pk_validate(content, &signature, public_keyring_for_validate)?;
+
+        Ok(Some((content.to_vec(), ret_valid_signatures)))
+    } else {
+        Ok(None)
+    }
+}
+
+fn keyring_from_peerstate(peerstate: &Option<Peerstate>) -> Keyring<SignedPublicKey> {
+    let mut public_keyring_for_validate: Keyring<SignedPublicKey> = Keyring::new();
+    if let Some(ref peerstate) = *peerstate {
+        if let Some(key) = &peerstate.public_key {
+            public_keyring_for_validate.add(key.clone());
+        } else if let Some(key) = &peerstate.gossip_key {
+            public_keyring_for_validate.add(key.clone());
+        }
+    }
+    public_keyring_for_validate
+}
+
+/// Applies Autocrypt header to Autocrypt peer state and saves it into the database.
+///
+/// If we already know this fingerprint from another contact's peerstate, return that
+/// peerstate in order to make AEAP work, but don't save it into the db yet.
+///
+/// Returns updated peerstate.
+pub(crate) async fn get_autocrypt_peerstate(
+    context: &Context,
+    from: &str,
+    autocrypt_header: Option<&Aheader>,
+    message_time: i64,
+) -> Result<Option<Peerstate>> {
+    let mut peerstate;
+
+    // Apply Autocrypt header
+    if let Some(header) = autocrypt_header {
+        // The "from_verified_fingerprint" part is for AEAP:
+        // If we know this fingerprint from another addr,
+        // we may want to do a transition from this other addr
+        // (and keep its peerstate)
+        // For security reasons, for now, we only do a transition
+        // if the fingerprint is verified.
+        peerstate = Peerstate::from_verified_fingerprint_or_addr(
+            context,
+            &header.public_key.fingerprint(),
+            from,
+        )
+        .await?;
+
+        if let Some(ref mut peerstate) = peerstate {
+            if addr_cmp(&peerstate.addr, from) {
+                peerstate.apply_header(header, message_time);
+                peerstate.save_to_db(&context.sql, false).await?;
+            }
+            // If `peerstate.addr` and `from` differ, this means that
+            // someone is using the same key but a different addr, probably
+            // because they made an AEAP transition.
+            // But we don't know if that's legit until we checked the
+            // signatures, so wait until then with writing anything
+            // to the database.
+        } else {
+            let p = Peerstate::from_header(header, message_time);
+            p.save_to_db(&context.sql, true).await?;
+            peerstate = Some(p);
+        }
+    } else {
+        peerstate = Peerstate::from_addr(context, from).await?;
+    }
+
+    Ok(peerstate)
+}
+
+#[cfg(test)]
+mod tests {
+    use crate::receive_imf::receive_imf;
+    use crate::test_utils::TestContext;
+
+    use super::*;
+
+    #[test]
+    fn test_has_decrypted_pgp_armor() {
+        let data = b" -----BEGIN PGP MESSAGE-----";
+        assert_eq!(has_decrypted_pgp_armor(data), true);
+
+        let data = b"    \n-----BEGIN PGP MESSAGE-----";
+        assert_eq!(has_decrypted_pgp_armor(data), true);
+
+        let data = b"    -----BEGIN PGP MESSAGE---";
+        assert_eq!(has_decrypted_pgp_armor(data), false);
+
+        let data = b" -----BEGIN PGP MESSAGE-----";
+        assert_eq!(has_decrypted_pgp_armor(data), true);
+
+        let data = b"blas";
+        assert_eq!(has_decrypted_pgp_armor(data), false);
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_mixed_up_mime() -> Result<()> {
+        // "Mixed Up" mail as received when sending an encrypted
+        // message using Delta Chat Desktop via ProtonMail IMAP/SMTP
+        // Bridge.
+        let mixed_up_mime = include_bytes!("../test-data/message/protonmail-mixed-up.eml");
+        let mail = mailparse::parse_mail(mixed_up_mime)?;
+        assert!(get_autocrypt_mime(&mail).is_none());
+        assert!(get_mixed_up_mime(&mail).is_some());
+        assert!(get_attachment_mime(&mail).is_none());
+
+        // Same "Mixed Up" mail repaired by Thunderbird 78.9.0.
+        //
+        // It added `X-Enigmail-Info: Fixed broken PGP/MIME message`
+        // header although the repairing is done by the built-in
+        // OpenPGP support, not Enigmail.
+        let repaired_mime = include_bytes!("../test-data/message/protonmail-repaired.eml");
+        let mail = mailparse::parse_mail(repaired_mime)?;
+        assert!(get_autocrypt_mime(&mail).is_some());
+        assert!(get_mixed_up_mime(&mail).is_none());
+        assert!(get_attachment_mime(&mail).is_none());
+
+        // Another form of "Mixed Up" mail created by Google Workspace,
+        // where original message is turned into attachment to empty plaintext message.
+        let attachment_mime = include_bytes!("../test-data/message/google-workspace-mixed-up.eml");
+        let mail = mailparse::parse_mail(attachment_mime)?;
+        assert!(get_autocrypt_mime(&mail).is_none());
+        assert!(get_mixed_up_mime(&mail).is_none());
+        assert!(get_attachment_mime(&mail).is_some());
+
+        let bob = TestContext::new_bob().await;
+        receive_imf(&bob, attachment_mime, false).await?;
+        let msg = bob.get_last_msg().await;
+        assert_eq!(msg.text.as_deref(), Some("Hello from Thunderbird!"));
+
+        Ok(())
+    }
+}

src/download.rs

@@ -11,7 +11,7 @@ use crate::imap::{Imap, ImapActionResult};
 use crate::job::{self, Action, Job, Status};
 use crate::message::{Message, MsgId, Viewtype};
 use crate::mimeparser::{MimeMessage, Part};
-use crate::param::Params;
+use crate::param::{Param, Params};
 use crate::tools::time;
 use crate::{job_try, stock_str, EventType};
 use std::cmp::max;
@@ -69,6 +69,42 @@ impl Context {
             Ok(Some(max(MIN_DOWNLOAD_LIMIT, download_limit as u32)))
         }
     }
+
+    // Merges the two messages to `placeholder_msg_id`;
+    // `full_msg_id` is no longer used afterwards.
+    pub(crate) async fn merge_messages(
+        &self,
+        full_msg_id: MsgId,
+        placeholder_msg_id: MsgId,
+    ) -> Result<()> {
+        let placeholder = Message::load_from_db(self, placeholder_msg_id).await?;
+        self.sql
+            .transaction(move |transaction| {
+                transaction
+                    .execute("DELETE FROM msgs WHERE id=?;", paramsv![placeholder_msg_id])?;
+                transaction.execute(
+                    "UPDATE msgs SET id=? WHERE id=?",
+                    paramsv![placeholder_msg_id, full_msg_id],
+                )?;
+                Ok(())
+            })
+            .await?;
+        let mut full = Message::load_from_db(self, placeholder_msg_id).await?;
+
+        for key in [
+            Param::WebxdcSummary,
+            Param::WebxdcSummaryTimestamp,
+            Param::WebxdcDocument,
+            Param::WebxdcDocumentTimestamp,
+        ] {
+            if let Some(value) = placeholder.param.get(key) {
+                full.param.set(key, value);
+            }
+        }
+        full.update_param(self).await?;
+
+        Ok(())
+    }
 }
 
 impl MsgId {
@@ -256,7 +292,7 @@ impl MimeMessage {
 mod tests {
     use num_traits::FromPrimitive;
 
-    use crate::chat::send_msg;
+    use crate::chat::{get_chat_msgs, send_msg};
     use crate::ephemeral::Timer;
     use crate::message::Viewtype;
     use crate::receive_imf::receive_imf_inner;
@@ -410,4 +446,119 @@ mod tests {
 
         Ok(())
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_status_update_expands_to_nothing() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+        let bob = TestContext::new_bob().await;
+        let chat_id = alice.create_chat(&bob).await.id;
+
+        let file = alice.get_blobdir().join("minimal.xdc");
+        tokio::fs::write(&file, include_bytes!("../test-data/webxdc/minimal.xdc")).await?;
+        let mut instance = Message::new(Viewtype::File);
+        instance.set_file(file.to_str().unwrap(), None);
+        let _sent1 = alice.send_msg(chat_id, &mut instance).await;
+
+        alice
+            .send_webxdc_status_update(instance.id, r#"{"payload":7}"#, "d")
+            .await?;
+        alice.flush_status_updates().await?;
+        let sent2 = alice.pop_sent_msg().await;
+        let sent2_rfc742_mid = Message::load_from_db(&alice, sent2.sender_msg_id)
+            .await?
+            .rfc724_mid;
+
+        // not downloading the status update results in an placeholder
+        receive_imf_inner(
+            &bob,
+            &sent2_rfc742_mid,
+            sent2.payload().as_bytes(),
+            false,
+            Some(sent2.payload().len() as u32),
+            false,
+        )
+        .await?;
+        let msg = bob.get_last_msg().await;
+        let chat_id = msg.chat_id;
+        assert_eq!(get_chat_msgs(&bob, chat_id, 0).await?.len(), 1);
+        assert_eq!(msg.download_state(), DownloadState::Available);
+
+        // downloading the status update afterwards expands to nothing and moves the placeholder to trash-chat
+        // (usually status updates are too small for not being downloaded directly)
+        receive_imf_inner(
+            &bob,
+            &sent2_rfc742_mid,
+            sent2.payload().as_bytes(),
+            false,
+            None,
+            false,
+        )
+        .await?;
+        assert_eq!(get_chat_msgs(&bob, chat_id, 0).await?.len(), 0);
+        assert!(Message::load_from_db(&bob, msg.id)
+            .await?
+            .chat_id
+            .is_trash());
+
+        Ok(())
+    }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_mdn_expands_to_nothing() -> Result<()> {
+        let bob = TestContext::new_bob().await;
+        let raw = b"Subject: Message opened\n\
+            Date: Mon, 10 Jan 2020 00:00:00 +0000\n\
+            Chat-Version: 1.0\n\
+            Message-ID: <bar@example.org>\n\
+            To: Alice <alice@example.org>\n\
+            From: Bob <bob@example.org>\n\
+            Content-Type: multipart/report; report-type=disposition-notification;\n\t\
+            boundary=\"kJBbU58X1xeWNHgBtTbMk80M5qnV4N\"\n\
+            \n\
+            \n\
+            --kJBbU58X1xeWNHgBtTbMk80M5qnV4N\n\
+            Content-Type: text/plain; charset=utf-8\n\
+            \n\
+            bla\n\
+            \n\
+            \n\
+            --kJBbU58X1xeWNHgBtTbMk80M5qnV4N\n\
+            Content-Type: message/disposition-notification\n\
+            \n\
+            Reporting-UA: Delta Chat 1.88.0\n\
+            Original-Recipient: rfc822;bob@example.org\n\
+            Final-Recipient: rfc822;bob@example.org\n\
+            Original-Message-ID: <foo@example.org>\n\
+            Disposition: manual-action/MDN-sent-automatically; displayed\n\
+            \n\
+            \n\
+            --kJBbU58X1xeWNHgBtTbMk80M5qnV4N--\n\
+            ";
+
+        // not downloading the mdn results in an placeholder
+        receive_imf_inner(
+            &bob,
+            "bar@example.org",
+            raw,
+            false,
+            Some(raw.len() as u32),
+            false,
+        )
+        .await?;
+        let msg = bob.get_last_msg().await;
+        let chat_id = msg.chat_id;
+        assert_eq!(get_chat_msgs(&bob, chat_id, 0).await?.len(), 1);
+        assert_eq!(msg.download_state(), DownloadState::Available);
+
+        // downloading the mdn afterwards expands to nothing and deletes the placeholder directly
+        // (usually mdn are too small for not being downloaded directly)
+        receive_imf_inner(&bob, "bar@example.org", raw, false, None, false).await?;
+        assert_eq!(get_chat_msgs(&bob, chat_id, 0).await?.len(), 0);
+        assert!(Message::load_from_db(&bob, msg.id)
+            .await?
+            .chat_id
+            .is_trash());
+
+        Ok(())
+    }
 }

src/e2ee.rs

@@ -1,20 +1,13 @@
 //! End-to-end encryption support.
 
-use std::collections::HashSet;
-
 use anyhow::{format_err, Context as _, Result};
-use mailparse::ParsedMail;
 use num_traits::FromPrimitive;
 
 use crate::aheader::{Aheader, EncryptPreference};
 use crate::config::Config;
-use crate::contact::addr_cmp;
 use crate::context::Context;
-use crate::headerdef::HeaderDef;
-use crate::headerdef::HeaderDefMap;
-use crate::key::{DcKey, Fingerprint, SignedPublicKey, SignedSecretKey};
+use crate::key::{DcKey, SignedPublicKey, SignedSecretKey};
 use crate::keyring::Keyring;
-use crate::log::LogExt;
 use crate::peerstate::{Peerstate, PeerstateVerifiedStatus};
 use crate::pgp;
 
@@ -133,299 +126,6 @@ impl EncryptHelper {
     }
 }
 
-/// Applies Autocrypt header to Autocrypt peer state and saves it into the database.
-///
-/// If we already know this fingerprint from another contact's peerstate, return that
-/// peerstate in order to make AEAP work, but don't save it into the db yet.
-///
-/// Returns updated peerstate.
-pub(crate) async fn get_autocrypt_peerstate(
-    context: &Context,
-    from: &str,
-    autocrypt_header: Option<&Aheader>,
-    message_time: i64,
-) -> Result<Option<Peerstate>> {
-    let mut peerstate;
-
-    // Apply Autocrypt header
-    if let Some(header) = autocrypt_header {
-        // The "from_nongossiped_fingerprint" part is for AEAP:
-        // If we know this fingerprint from another addr,
-        // we may want to do a transition from this other addr
-        // (and keep its peerstate)
-        peerstate = Peerstate::from_nongossiped_fingerprint_or_addr(
-            context,
-            &header.public_key.fingerprint(),
-            from,
-        )
-        .await?;
-
-        if let Some(ref mut peerstate) = peerstate {
-            if addr_cmp(&peerstate.addr, from) {
-                peerstate.apply_header(header, message_time);
-                peerstate.save_to_db(&context.sql, false).await?;
-            }
-            // If `peerstate.addr` and `from` differ, this means that
-            // someone is using the same key but a different addr, probably
-            // because they made an AEAP transition.
-            // But we don't know if that's legit until we checked the
-            // signatures, so wait until then with writing anything
-            // to the database.
-        } else {
-            let p = Peerstate::from_header(header, message_time);
-            p.save_to_db(&context.sql, true).await?;
-            peerstate = Some(p);
-        }
-    } else {
-        peerstate = Peerstate::from_addr(context, from).await?;
-    }
-
-    Ok(peerstate)
-}
-
-/// Tries to decrypt a message, but only if it is structured as an
-/// Autocrypt message.
-///
-/// Returns decrypted body and a set of valid signature fingerprints
-/// if successful.
-///
-/// If the message is wrongly signed, this will still return the decrypted
-/// message but the HashSet will be empty.
-pub async fn try_decrypt(
-    context: &Context,
-    mail: &ParsedMail<'_>,
-    decryption_info: &DecryptionInfo,
-) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
-    // Possibly perform decryption
-    let public_keyring_for_validate = keyring_from_peerstate(&decryption_info.peerstate);
-
-    let context = context;
-    let encrypted_data_part = match get_autocrypt_mime(mail)
-        .or_else(|| get_mixed_up_mime(mail))
-        .or_else(|| get_attachment_mime(mail))
-    {
-        None => {
-            // not an autocrypt mime message, abort and ignore
-            return Ok(None);
-        }
-        Some(res) => res,
-    };
-    info!(context, "Detected Autocrypt-mime message");
-    let private_keyring: Keyring<SignedSecretKey> = Keyring::new_self(context)
-        .await
-        .context("failed to get own keyring")?;
-
-    decrypt_part(
-        encrypted_data_part,
-        private_keyring,
-        public_keyring_for_validate,
-    )
-    .await
-}
-
-pub async fn create_decryption_info(
-    context: &Context,
-    mail: &ParsedMail<'_>,
-    message_time: i64,
-) -> Result<DecryptionInfo> {
-    let from = mail
-        .headers
-        .get_header(HeaderDef::From_)
-        .and_then(|from_addr| mailparse::addrparse_header(from_addr).ok())
-        .and_then(|from| from.extract_single_info())
-        .map(|from| from.addr)
-        .unwrap_or_default();
-
-    let autocrypt_header = Aheader::from_headers(&from, &mail.headers)
-        .ok_or_log_msg(context, "Failed to parse Autocrypt header")
-        .flatten();
-
-    let peerstate =
-        get_autocrypt_peerstate(context, &from, autocrypt_header.as_ref(), message_time).await?;
-
-    Ok(DecryptionInfo {
-        from,
-        autocrypt_header,
-        peerstate,
-        message_time,
-    })
-}
-
-#[derive(Debug)]
-pub struct DecryptionInfo {
-    /// The From address. This is the address from the unnencrypted, outer
-    /// From header.
-    pub from: String,
-    pub autocrypt_header: Option<Aheader>,
-    /// The peerstate that will be used to validate the signatures
-    pub peerstate: Option<Peerstate>,
-    /// The timestamp when the message was sent.
-    /// If this is older than the peerstate's last_seen, this probably
-    /// means out-of-order message arrival, We don't modify the
-    /// peerstate in this case.
-    pub message_time: i64,
-}
-
-/// Returns a reference to the encrypted payload of a valid PGP/MIME message.
-///
-/// Returns `None` if the message is not a valid PGP/MIME message.
-fn get_autocrypt_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Option<&'a ParsedMail<'b>> {
-    if mail.ctype.mimetype != "multipart/encrypted" {
-        return None;
-    }
-    if let [first_part, second_part] = &mail.subparts[..] {
-        if first_part.ctype.mimetype == "application/pgp-encrypted"
-            && second_part.ctype.mimetype == "application/octet-stream"
-        {
-            Some(second_part)
-        } else {
-            None
-        }
-    } else {
-        None
-    }
-}
-
-/// Returns a reference to the encrypted payload of a ["Mixed
-/// Up"][pgpmime-message-mangling] message.
-///
-/// According to [RFC 3156] encrypted messages should have
-/// `multipart/encrypted` MIME type and two parts, but Microsoft
-/// Exchange and ProtonMail IMAP/SMTP Bridge are known to mangle this
-/// structure by changing the type to `multipart/mixed` and prepending
-/// an empty part at the start.
-///
-/// ProtonMail IMAP/SMTP Bridge prepends a part literally saying
-/// "Empty Message", so we don't check its contents at all, checking
-/// only for `text/plain` type.
-///
-/// Returns `None` if the message is not a "Mixed Up" message.
-///
-/// [RFC 3156]: https://www.rfc-editor.org/info/rfc3156
-/// [pgpmime-message-mangling]: https://tools.ietf.org/id/draft-dkg-openpgp-pgpmime-message-mangling-00.html
-fn get_mixed_up_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Option<&'a ParsedMail<'b>> {
-    if mail.ctype.mimetype != "multipart/mixed" {
-        return None;
-    }
-    if let [first_part, second_part, third_part] = &mail.subparts[..] {
-        if first_part.ctype.mimetype == "text/plain"
-            && second_part.ctype.mimetype == "application/pgp-encrypted"
-            && third_part.ctype.mimetype == "application/octet-stream"
-        {
-            Some(third_part)
-        } else {
-            None
-        }
-    } else {
-        None
-    }
-}
-
-/// Returns a reference to the encrypted payload of a message turned into attachment.
-///
-/// Google Workspace has an option "Append footer" which appends standard footer defined
-/// by administrator to all outgoing messages. However, there is no plain text part in
-/// encrypted messages sent by Delta Chat, so Google Workspace turns the message into
-/// multipart/mixed MIME, where the first part is an empty plaintext part with a footer
-/// and the second part is the original encrypted message.
-fn get_attachment_mime<'a, 'b>(mail: &'a ParsedMail<'b>) -> Option<&'a ParsedMail<'b>> {
-    if mail.ctype.mimetype != "multipart/mixed" {
-        return None;
-    }
-    if let [first_part, second_part] = &mail.subparts[..] {
-        if first_part.ctype.mimetype == "text/plain"
-            && second_part.ctype.mimetype == "multipart/encrypted"
-        {
-            get_autocrypt_mime(second_part)
-        } else {
-            None
-        }
-    } else {
-        None
-    }
-}
-
-fn keyring_from_peerstate(peerstate: &Option<Peerstate>) -> Keyring<SignedPublicKey> {
-    let mut public_keyring_for_validate: Keyring<SignedPublicKey> = Keyring::new();
-    if let Some(ref peerstate) = *peerstate {
-        if let Some(key) = &peerstate.public_key {
-            public_keyring_for_validate.add(key.clone());
-        } else if let Some(key) = &peerstate.gossip_key {
-            public_keyring_for_validate.add(key.clone());
-        }
-    }
-    public_keyring_for_validate
-}
-
-/// Validates signatures of Multipart/Signed message part, as defined in RFC 1847.
-///
-/// Returns `None` if the part is not a Multipart/Signed part, otherwise retruns the set of key
-/// fingerprints for which there is a valid signature.
-fn validate_detached_signature(
-    mail: &ParsedMail<'_>,
-    public_keyring_for_validate: &Keyring<SignedPublicKey>,
-) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
-    if mail.ctype.mimetype != "multipart/signed" {
-        return Ok(None);
-    }
-
-    if let [first_part, second_part] = &mail.subparts[..] {
-        // First part is the content, second part is the signature.
-        let content = first_part.raw_bytes;
-        let signature = second_part.get_body_raw()?;
-        let ret_valid_signatures =
-            pgp::pk_validate(content, &signature, public_keyring_for_validate)?;
-
-        Ok(Some((content.to_vec(), ret_valid_signatures)))
-    } else {
-        Ok(None)
-    }
-}
-
-/// Returns Ok(None) if nothing encrypted was found.
-async fn decrypt_part(
-    mail: &ParsedMail<'_>,
-    private_keyring: Keyring<SignedSecretKey>,
-    public_keyring_for_validate: Keyring<SignedPublicKey>,
-) -> Result<Option<(Vec<u8>, HashSet<Fingerprint>)>> {
-    let data = mail.get_body_raw()?;
-
-    if has_decrypted_pgp_armor(&data) {
-        let (plain, ret_valid_signatures) =
-            pgp::pk_decrypt(data, private_keyring, &public_keyring_for_validate).await?;
-
-        // Check for detached signatures.
-        // If decrypted part is a multipart/signed, then there is a detached signature.
-        let decrypted_part = mailparse::parse_mail(&plain)?;
-        if let Some((content, valid_detached_signatures)) =
-            validate_detached_signature(&decrypted_part, &public_keyring_for_validate)?
-        {
-            return Ok(Some((content, valid_detached_signatures)));
-        } else {
-            // If the message was wrongly or not signed, still return the plain text.
-            // The caller has to check if the signatures set is empty then.
-
-            return Ok(Some((plain, ret_valid_signatures)));
-        }
-    }
-
-    Ok(None)
-}
-
-#[allow(clippy::indexing_slicing)]
-fn has_decrypted_pgp_armor(input: &[u8]) -> bool {
-    if let Some(index) = input.iter().position(|b| *b > b' ') {
-        if input.len() - index > 26 {
-            let start = index;
-            let end = start + 27;
-
-            return &input[start..end] == b"-----BEGIN PGP MESSAGE-----";
-        }
-    }
-
-    false
-}
-
 /// Ensures a private key exists for the configured user.
 ///
 /// Normally the private key is generated when the first message is
@@ -448,7 +148,6 @@ mod tests {
     use crate::message::{Message, Viewtype};
     use crate::param::Param;
     use crate::peerstate::ToSave;
-    use crate::receive_imf::receive_imf;
     use crate::test_utils::{bob_keypair, TestContext};
 
     use super::*;
@@ -496,24 +195,6 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         );
     }
 
-    #[test]
-    fn test_has_decrypted_pgp_armor() {
-        let data = b" -----BEGIN PGP MESSAGE-----";
-        assert_eq!(has_decrypted_pgp_armor(data), true);
-
-        let data = b"    \n-----BEGIN PGP MESSAGE-----";
-        assert_eq!(has_decrypted_pgp_armor(data), true);
-
-        let data = b"    -----BEGIN PGP MESSAGE---";
-        assert_eq!(has_decrypted_pgp_armor(data), false);
-
-        let data = b" -----BEGIN PGP MESSAGE-----";
-        assert_eq!(has_decrypted_pgp_armor(data), true);
-
-        let data = b"blas";
-        assert_eq!(has_decrypted_pgp_armor(data), false);
-    }
-
     #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
     async fn test_encrypted_no_autocrypt() -> anyhow::Result<()> {
         let alice = TestContext::new_alice().await;
@@ -648,42 +329,4 @@ Sent with my Delta Chat Messenger: https://delta.chat";
         assert!(encrypt_helper.should_encrypt(&t, true, &ps).is_err());
         assert!(!encrypt_helper.should_encrypt(&t, false, &ps).unwrap());
     }
-
-    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
-    async fn test_mixed_up_mime() -> Result<()> {
-        // "Mixed Up" mail as received when sending an encrypted
-        // message using Delta Chat Desktop via ProtonMail IMAP/SMTP
-        // Bridge.
-        let mixed_up_mime = include_bytes!("../test-data/message/protonmail-mixed-up.eml");
-        let mail = mailparse::parse_mail(mixed_up_mime)?;
-        assert!(get_autocrypt_mime(&mail).is_none());
-        assert!(get_mixed_up_mime(&mail).is_some());
-        assert!(get_attachment_mime(&mail).is_none());
-
-        // Same "Mixed Up" mail repaired by Thunderbird 78.9.0.
-        //
-        // It added `X-Enigmail-Info: Fixed broken PGP/MIME message`
-        // header although the repairing is done by the built-in
-        // OpenPGP support, not Enigmail.
-        let repaired_mime = include_bytes!("../test-data/message/protonmail-repaired.eml");
-        let mail = mailparse::parse_mail(repaired_mime)?;
-        assert!(get_autocrypt_mime(&mail).is_some());
-        assert!(get_mixed_up_mime(&mail).is_none());
-        assert!(get_attachment_mime(&mail).is_none());
-
-        // Another form of "Mixed Up" mail created by Google Workspace,
-        // where original message is turned into attachment to empty plaintext message.
-        let attachment_mime = include_bytes!("../test-data/message/google-workspace-mixed-up.eml");
-        let mail = mailparse::parse_mail(attachment_mime)?;
-        assert!(get_autocrypt_mime(&mail).is_none());
-        assert!(get_mixed_up_mime(&mail).is_none());
-        assert!(get_attachment_mime(&mail).is_some());
-
-        let bob = TestContext::new_bob().await;
-        receive_imf(&bob, attachment_mime, false).await?;
-        let msg = bob.get_last_msg().await;
-        assert_eq!(msg.text.as_deref(), Some("Hello from Thunderbird!"));
-
-        Ok(())
-    }
 }

src/events.rs

@@ -302,4 +302,17 @@ pub enum EventType {
         msg_id: MsgId,
         status_update_serial: StatusUpdateSerial,
     },
+
+    /// Inform that a message containing a webxdc instance has been deleted
+    WebxdcInstanceDeleted {
+        msg_id: MsgId,
+    },
+
+    WebxdcBusyUpdating {
+        msg_id: MsgId,
+    },
+
+    WebxdcUpToDate {
+        msg_id: MsgId,
+    },
 }

src/job.rs

@@ -69,7 +69,7 @@ pub enum Action {
     ResyncFolders = 300,
 }
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 pub struct Job {
     pub job_id: u32,
     pub action: Action,

src/lib.rs

@@ -15,7 +15,7 @@
 )]
 #![allow(
     clippy::match_bool,
-    clippy::eval_order_dependence,
+    clippy::mixed_read_write_in_expression,
     clippy::bool_assert_comparison,
     clippy::manual_split_once,
     clippy::format_push_string
@@ -58,6 +58,7 @@ mod configure;
 pub mod constants;
 pub mod contact;
 pub mod context;
+mod decrypt;
 pub mod download;
 mod e2ee;
 pub mod ephemeral;

src/login_param.rs

@@ -39,7 +39,7 @@ impl Default for CertificateChecks {
 }
 
 /// Login parameters for a single server, either IMAP or SMTP
-#[derive(Default, Debug, Clone, PartialEq)]
+#[derive(Default, Debug, Clone, PartialEq, Eq)]
 pub struct ServerLoginParam {
     pub server: String,
     pub user: String,
@@ -53,7 +53,7 @@ pub struct ServerLoginParam {
     pub certificate_checks: CertificateChecks,
 }
 
-#[derive(Default, Debug, Clone, PartialEq)]
+#[derive(Default, Debug, Clone, PartialEq, Eq)]
 pub struct Socks5Config {
     pub host: String,
     pub port: u16,
@@ -128,7 +128,7 @@ impl fmt::Display for Socks5Config {
     }
 }
 
-#[derive(Default, Debug, Clone, PartialEq)]
+#[derive(Default, Debug, Clone, PartialEq, Eq)]
 pub struct LoginParam {
     pub addr: String,
     pub imap: ServerLoginParam,

src/message.rs

@@ -19,7 +19,7 @@ use crate::download::DownloadState;
 use crate::ephemeral::{start_ephemeral_timers_msgids, Timer as EphemeralTimer};
 use crate::events::EventType;
 use crate::imap::markseen_on_imap_table;
-use crate::mimeparser::{parse_message_id, FailureReport, SystemMessage};
+use crate::mimeparser::{parse_message_id, DeliveryReport, SystemMessage};
 use crate::param::{Param, Params};
 use crate::pgp::split_armored_data;
 use crate::scheduler::InterruptInfo;
@@ -1236,6 +1236,11 @@ pub async fn delete_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> {
             .trash(context)
             .await
             .with_context(|| format!("Unable to trash message {}", msg_id))?;
+
+        if msg.viewtype == Viewtype::Webxdc {
+            context.emit_event(EventType::WebxdcInstanceDeleted { msg_id: *msg_id });
+        }
+
         context
             .sql
             .execute(
@@ -1532,7 +1537,7 @@ pub async fn handle_mdn(
 /// Where appropriate, also adds an info message telling the user which of the recipients of a group message failed.
 pub(crate) async fn handle_ndn(
     context: &Context,
-    failed: &FailureReport,
+    failed: &DeliveryReport,
     error: Option<String>,
 ) -> Result<()> {
     if failed.rfc724_mid.is_empty() {
@@ -1588,7 +1593,7 @@ pub(crate) async fn handle_ndn(
 
 async fn ndn_maybe_add_info_msg(
     context: &Context,
-    failed: &FailureReport,
+    failed: &DeliveryReport,
     chat_id: ChatId,
     chat_type: Chattype,
 ) -> Result<()> {

src/mimefactory.rs

@@ -445,6 +445,8 @@ impl<'a> MimeFactory<'a> {
             .collect()
     }
 
+    /// Consumes a `MimeFactory` and renders it into a message which is then stored in
+    /// `smtp`-table to be used by the SMTP loop
     pub async fn render(mut self, context: &Context) -> Result<RenderedEmail> {
         let mut headers: MessageHeaders = Default::default();
 

src/mimeparser.rs

@@ -15,8 +15,8 @@ use crate::blob::BlobObject;
 use crate::constants::{DC_DESIRED_TEXT_LEN, DC_ELLIPSIS};
 use crate::contact::{addr_cmp, addr_normalize, ContactId};
 use crate::context::Context;
+use crate::decrypt::{create_decryption_info, try_decrypt};
 use crate::dehtml::dehtml;
-use crate::e2ee;
 use crate::events::EventType;
 use crate::format_flowed::unformat_flowed;
 use crate::headerdef::{HeaderDef, HeaderDefMap};
@@ -73,7 +73,7 @@ pub struct MimeMessage {
     pub(crate) user_avatar: Option<AvatarAction>,
     pub(crate) group_avatar: Option<AvatarAction>,
     pub(crate) mdn_reports: Vec<Report>,
-    pub(crate) failure_report: Option<FailureReport>,
+    pub(crate) delivery_report: Option<DeliveryReport>,
 
     /// Standard USENET signature, if any.
     pub(crate) footer: Option<String>,
@@ -220,12 +220,11 @@ impl MimeMessage {
         let mut mail_raw = Vec::new();
         let mut gossiped_addr = Default::default();
         let mut from_is_signed = false;
-        let mut decryption_info =
-            e2ee::create_decryption_info(context, &mail, message_time).await?;
+        let mut decryption_info = create_decryption_info(context, &mail, message_time).await?;
 
         // `signatures` is non-empty exactly if the message was encrypted and correctly signed.
         let (mail, signatures, warn_empty_signature) =
-            match e2ee::try_decrypt(context, &mail, &decryption_info).await {
+            match try_decrypt(context, &mail, &decryption_info).await {
                 Ok(Some((raw, signatures))) => {
                     // Encrypted, but maybe unsigned message. Only if
                     // `signatures` set is non-empty, it is a valid
@@ -332,7 +331,7 @@ impl MimeMessage {
             webxdc_status_update: None,
             user_avatar: None,
             group_avatar: None,
-            failure_report: None,
+            delivery_report: None,
             footer: None,
             is_mime_modified: false,
             decoded_data: Vec::new(),
@@ -417,6 +416,8 @@ impl MimeMessage {
                 self.is_system_message = SystemMessage::ChatProtectionEnabled;
             } else if value == "protection-disabled" {
                 self.is_system_message = SystemMessage::ChatProtectionDisabled;
+            } else if value == "group-avatar-changed" {
+                self.is_system_message = SystemMessage::GroupImageChanged;
             }
         } else if self.get_header(HeaderDef::ChatGroupMemberRemoved).is_some() {
             self.is_system_message = SystemMessage::MemberRemovedFromGroup;
@@ -424,10 +425,6 @@ impl MimeMessage {
             self.is_system_message = SystemMessage::MemberAddedToGroup;
         } else if self.get_header(HeaderDef::ChatGroupNameChanged).is_some() {
             self.is_system_message = SystemMessage::GroupNameChanged;
-        } else if let Some(value) = self.get_header(HeaderDef::ChatContent) {
-            if value == "group-avatar-changed" {
-                self.is_system_message = SystemMessage::GroupImageChanged;
-            }
         }
     }
 
@@ -535,7 +532,9 @@ impl MimeMessage {
         self.parse_system_message_headers(context);
         self.parse_avatar_headers(context).await;
         self.parse_videochat_headers();
-        self.squash_attachment_parts();
+        if self.delivery_report.is_none() {
+            self.squash_attachment_parts();
+        }
 
         if let Some(ref subject) = self.get_subject() {
             let mut prepend_subject = true;
@@ -552,7 +551,7 @@ impl MimeMessage {
 
             // For mailing lists, always add the subject because sometimes there are different topics
             // and otherwise it might be hard to keep track:
-            if self.is_mailinglist_message() {
+            if self.is_mailinglist_message() && !self.has_chat_version() {
                 prepend_subject = true;
             }
 
@@ -747,7 +746,7 @@ impl MimeMessage {
                     MimeS::Single
                 }
             } else if mimetype.starts_with("message") {
-                if mimetype == "message/rfc822" {
+                if mimetype == "message/rfc822" && !is_attachment_disposition(mail) {
                     MimeS::Message
                 } else {
                     MimeS::Single
@@ -867,7 +866,7 @@ impl MimeMessage {
                         // Some providers, e.g. Tiscali, forget to set the report-type. So, if it's None, assume that it might be delivery-status
                         Some("delivery-status") | None => {
                             if let Some(report) = self.process_delivery_status(context, mail)? {
-                                self.failure_report = Some(report);
+                                self.delivery_report = Some(report);
                             }
 
                             // Add all parts (we need another part, preferably text/plain, to show as an error message)
@@ -1278,9 +1277,46 @@ impl MimeMessage {
         &self,
         context: &Context,
         report: &mailparse::ParsedMail<'_>,
-    ) -> Result<Option<FailureReport>> {
+    ) -> Result<Option<DeliveryReport>> {
+        // Assume failure.
+        let mut failure = true;
+
+        if let Some(status_part) = report.subparts.get(1) {
+            // RFC 3464 defines `message/delivery-status`
+            // RFC 6533 defines `message/global-delivery-status`
+            if status_part.ctype.mimetype != "message/delivery-status"
+                && status_part.ctype.mimetype != "message/global-delivery-status"
+            {
+                warn!(context, "Second part of Delivery Status Notification is not message/delivery-status or message/global-delivery-status, ignoring");
+                return Ok(None);
+            }
+
+            let status_body = status_part.get_body_raw()?;
+
+            // Skip per-message fields.
+            let (_, sz) = mailparse::parse_headers(&status_body)?;
+
+            // Parse first set of per-recipient fields
+            if let Some(status_body) = status_body.get(sz..) {
+                let (status_fields, _) = mailparse::parse_headers(status_body)?;
+                if let Some(action) = status_fields.get_first_value("action") {
+                    if action != "failed" {
+                        info!(context, "DSN with {:?} action", action);
+                        failure = false;
+                    }
+                } else {
+                    warn!(context, "DSN without action");
+                }
+            } else {
+                warn!(context, "DSN without per-recipient fields");
+            }
+        } else {
+            // No message/delivery-status part.
+            return Ok(None);
+        }
+
         // parse as mailheaders
-        if let Some(original_msg) = report.subparts.iter().find(|p| {
+        if let Some(original_msg) = report.subparts.get(2).filter(|p| {
             p.ctype.mimetype.contains("rfc822")
                 || p.ctype.mimetype == "message/global"
                 || p.ctype.mimetype == "message/global-headers"
@@ -1301,9 +1337,10 @@ impl MimeMessage {
                     None // We do not know which recipient failed
                 };
 
-                return Ok(Some(FailureReport {
+                return Ok(Some(DeliveryReport {
                     rfc724_mid: original_message_id,
                     failed_recipient: to.map(|s| s.addr),
+                    failure,
                 }));
             }
 
@@ -1384,7 +1421,7 @@ impl MimeMessage {
         } else {
             false
         };
-        if maybe_ndn && self.failure_report.is_none() {
+        if maybe_ndn && self.delivery_report.is_none() {
             static RE: Lazy<regex::Regex> =
                 Lazy::new(|| regex::Regex::new(r"Message-ID:(.*)").unwrap());
             for captures in self
@@ -1398,9 +1435,10 @@ impl MimeMessage {
                     if let Ok(Some(_)) =
                         message::rfc724_mid_exists(context, &original_message_id).await
                     {
-                        self.failure_report = Some(FailureReport {
+                        self.delivery_report = Some(DeliveryReport {
                             rfc724_mid: original_message_id,
                             failed_recipient: None,
+                            failure: true,
                         })
                     }
                 }
@@ -1438,13 +1476,15 @@ impl MimeMessage {
             }
         }
 
-        if let Some(failure_report) = &self.failure_report {
-            let error = parts
-                .iter()
-                .find(|p| p.typ == Viewtype::Text)
-                .map(|p| p.msg.clone());
-            if let Err(e) = message::handle_ndn(context, failure_report, error).await {
-                warn!(context, "Could not handle ndn: {}", e);
+        if let Some(delivery_report) = &self.delivery_report {
+            if delivery_report.failure {
+                let error = parts
+                    .iter()
+                    .find(|p| p.typ == Viewtype::Text)
+                    .map(|p| p.msg.clone());
+                if let Err(e) = message::handle_ndn(context, delivery_report, error).await {
+                    warn!(context, "Could not handle ndn: {}", e);
+                }
             }
         }
     }
@@ -1524,6 +1564,7 @@ async fn update_gossip_peerstates(
     Ok(gossiped_addr)
 }
 
+/// Message Disposition Notification (RFC 8098)
 #[derive(Debug)]
 pub(crate) struct Report {
     /// Original-Message-ID header
@@ -1535,10 +1576,12 @@ pub(crate) struct Report {
     additional_message_ids: Vec<String>,
 }
 
+/// Delivery Status Notification (RFC 3464, RFC 6533)
 #[derive(Debug)]
-pub(crate) struct FailureReport {
+pub(crate) struct DeliveryReport {
     pub rfc724_mid: String,
     pub failed_recipient: Option<String>,
+    pub failure: bool,
 }
 
 #[allow(clippy::indexing_slicing)]
@@ -1631,14 +1674,18 @@ fn get_mime_type(mail: &mailparse::ParsedMail<'_>) -> Result<(Mime, Viewtype)> {
         mime::VIDEO => Viewtype::Video,
         mime::MULTIPART => Viewtype::Unknown,
         mime::MESSAGE => {
-            // Enacapsulated messages, see <https://www.w3.org/Protocols/rfc1341/7_3_Message.html>
-            // Also used as part "message/disposition-notification" of "multipart/report", which, however, will
-            // be handled separatedly.
-            // I've not seen any messages using this, so we do not attach these parts (maybe they're used to attach replies,
-            // which are unwanted at all).
-            // For now, we skip these parts at all; if desired, we could return DcMimeType::File/DC_MSG_File
-            // for selected and known subparts.
-            Viewtype::Unknown
+            if is_attachment_disposition(mail) {
+                Viewtype::File
+            } else {
+                // Enacapsulated messages, see <https://www.w3.org/Protocols/rfc1341/7_3_Message.html>
+                // Also used as part "message/disposition-notification" of "multipart/report", which, however, will
+                // be handled separatedly.
+                // I've not seen any messages using this, so we do not attach these parts (maybe they're used to attach replies,
+                // which are unwanted at all).
+                // For now, we skip these parts at all; if desired, we could return DcMimeType::File/DC_MSG_File
+                // for selected and known subparts.
+                Viewtype::Unknown
+            }
         }
         mime::APPLICATION => Viewtype::File,
         _ => Viewtype::Unknown,
@@ -3256,4 +3303,36 @@ Message.
 
         Ok(())
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+    async fn test_receive_eml() -> Result<()> {
+        let alice = TestContext::new_alice().await;
+
+        let mime_message = MimeMessage::from_bytes(
+            &alice,
+            include_bytes!("../test-data/message/attached-eml.eml"),
+        )
+        .await?;
+
+        assert_eq!(mime_message.parts.len(), 1);
+        assert_eq!(mime_message.parts[0].typ, Viewtype::File);
+        assert_eq!(
+            mime_message.parts[0].mimetype,
+            Some("message/rfc822".parse().unwrap(),)
+        );
+        assert_eq!(
+            mime_message.parts[0].msg,
+            "this is a classic email – I attached the .EML file".to_string()
+        );
+        assert_eq!(
+            mime_message.parts[0].param.get(Param::File),
+            Some("$BLOBDIR/.eml")
+        );
+
+        assert_eq!(mime_message.parts[0].org_filename, Some(".eml".to_string()));
+
+        dbg!(mime_message);
+
+        Ok(())
+    }
 }

src/param.rs

@@ -163,7 +163,7 @@ pub enum Param {
     /// For Chats: timestamp of group name update.
     GroupNameTimestamp = b'g',
 
-    /// For Chats: timestamp of group name update.
+    /// For Chats: timestamp of member list update.
     MemberListTimestamp = b'k',
 
     /// For Chats: timestamp of protection settings update.

src/peerstate.rs

@@ -9,6 +9,7 @@ use crate::chatlist::Chatlist;
 use crate::constants::Chattype;
 use crate::contact::{addr_cmp, Contact, Origin};
 use crate::context::Context;
+use crate::decrypt::DecryptionInfo;
 use crate::events::EventType;
 use crate::key::{DcKey, Fingerprint, SignedPublicKey};
 use crate::message::Message;
@@ -165,7 +166,7 @@ impl Peerstate {
         Self::from_stmt(context, query, paramsv![fp, fp, fp]).await
     }
 
-    pub async fn from_nongossiped_fingerprint_or_addr(
+    pub async fn from_verified_fingerprint_or_addr(
         context: &Context,
         fingerprint: &Fingerprint,
         addr: &str,
@@ -174,9 +175,9 @@ impl Peerstate {
                      gossip_timestamp, gossip_key, public_key_fingerprint, gossip_key_fingerprint, \
                      verified_key, verified_key_fingerprint \
                      FROM acpeerstates  \
-                     WHERE public_key_fingerprint=? \
+                     WHERE verified_key_fingerprint=? \
                      OR addr=? COLLATE NOCASE \
-                     ORDER BY public_key_fingerprint=? DESC, last_seen DESC LIMIT 1;";
+                     ORDER BY verified_key_fingerprint=? DESC, last_seen DESC LIMIT 1;";
         let fp = fingerprint.hex();
         Self::from_stmt(context, query, paramsv![fp, addr, fp]).await
     }
@@ -516,22 +517,22 @@ impl Peerstate {
             .with_context(|| format!("contact with peerstate.addr {:?} not found", &self.addr))?;
 
         let chats = Chatlist::try_load(context, 0, None, Some(contact_id)).await?;
+        let msg = match &change {
+            PeerstateChange::FingerprintChange => {
+                stock_str::contact_setup_changed(context, self.addr.clone()).await
+            }
+            PeerstateChange::Aeap(new_addr) => {
+                let old_contact = Contact::load_from_db(context, contact_id).await?;
+                stock_str::aeap_addr_changed(
+                    context,
+                    old_contact.get_display_name(),
+                    &self.addr,
+                    new_addr,
+                )
+                .await
+            }
+        };
         for (chat_id, msg_id) in chats.iter() {
-            let msg = match &change {
-                PeerstateChange::FingerprintChange => {
-                    stock_str::contact_setup_changed(context, self.addr.clone()).await
-                }
-                PeerstateChange::Aeap(new_addr) => {
-                    let old_contact = Contact::load_from_db(context, contact_id).await?;
-                    stock_str::aeap_addr_changed(
-                        context,
-                        old_contact.get_display_name(),
-                        &self.addr,
-                        new_addr,
-                    )
-                    .await
-                }
-            };
             let timestamp_sort = if let Some(msg_id) = msg_id {
                 let lastmsg = Message::load_from_db(context, *msg_id).await?;
                 lastmsg.timestamp_sort
@@ -545,21 +546,23 @@ impl Peerstate {
                     .await?
                     .unwrap_or(0)
             };
-            chat::add_info_msg_with_cmd(
-                context,
-                *chat_id,
-                &msg,
-                SystemMessage::Unknown,
-                timestamp_sort,
-                Some(timestamp),
-                None,
-                None,
-            )
-            .await?;
 
             if let PeerstateChange::Aeap(new_addr) = &change {
                 let chat = Chat::load_from_db(context, *chat_id).await?;
-                if chat.typ == Chattype::Group || chat.typ == Chattype::Broadcast {
+
+                if chat.typ == Chattype::Group && !chat.is_protected() {
+                    // Don't add an info_msg to the group, in order not to make the user think
+                    // that the address was automatically replaced in the group.
+                    continue;
+                }
+
+                // For security reasons, for now, we only do the AEAP transition if the fingerprint
+                // is verified (that's what from_verified_fingerprint_or_addr() does).
+                // In order to not have inconsistent group membership state, we then only do the
+                // transition in verified groups and in broadcast lists.
+                if (chat.typ == Chattype::Group && chat.is_protected())
+                    || chat.typ == Chattype::Broadcast
+                {
                     chat::remove_from_chat_contacts_table(context, *chat_id, contact_id).await?;
 
                     let (new_contact_id, _) =
@@ -572,6 +575,18 @@ impl Peerstate {
                     context.emit_event(EventType::ChatModified(*chat_id));
                 }
             }
+
+            chat::add_info_msg_with_cmd(
+                context,
+                *chat_id,
+                &msg,
+                SystemMessage::Unknown,
+                timestamp_sort,
+                Some(timestamp),
+                None,
+                None,
+            )
+            .await?;
         }
 
         Ok(())
@@ -597,7 +612,7 @@ impl Peerstate {
 /// In `drafts/aeap_mvp.md` there is a "big picture" overview over AEAP.
 pub async fn maybe_do_aeap_transition(
     context: &Context,
-    info: &mut crate::e2ee::DecryptionInfo,
+    info: &mut DecryptionInfo,
     mime_parser: &crate::mimeparser::MimeMessage,
 ) -> Result<()> {
     if let Some(peerstate) = &mut info.peerstate {
@@ -621,7 +636,7 @@ pub async fn maybe_do_aeap_transition(
                 && mime_parser.from_is_signed
                 && info.message_time > peerstate.last_seen
             {
-                // Add an info messages to all chats with this contact
+                // Add info messages to chats with this (verified) contact
                 //
                 peerstate
                     .handle_setup_change(