improve naming

that dc-node has everything it needs to provide @deltachat/jsonrpc-client
without an extra ts compile step

.github/mergeable.yml

@@ -22,5 +22,5 @@ mergeable:
       - do: checks
         status: 'action_required' 
         payload:
-          title: Changlog might need an update
+          title: Changelog might need an update
           summary: "Check if CHANGELOG.md needs an update or add #skip-changelog to the PR description."

.github/workflows/ci.yml

@@ -77,10 +77,10 @@ jobs:
         include:
           # Currently used Rust version, same as in `rust-toolchain` file.
           - os: ubuntu-latest
-            rust: 1.60.0
+            rust: 1.61.0
             python: 3.9
           - os: windows-latest
-            rust: 1.60.0
+            rust: 1.61.0
             python: false # Python bindings compilation on Windows is not supported.
 
           # Minimum Supported Rust Version = 1.56.0

.github/workflows/jsonrpc_api.yml

@@ -0,0 +1,66 @@
+name: JSON-RPC API Test
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: 1.56.0
+          override: true
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v1.3.0
+      - name: Build
+        run: cargo build --verbose --features webserver -p deltachat-jsonrpc
+      - name: Run tests
+        run: cargo test --verbose --features webserver -p deltachat-jsonrpc
+
+  ts_bindings:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js 16.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 16.x
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: 1.56.0
+          override: true
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v1.3.0
+      - name: npm i
+        run: |
+          cd deltachat-jsonrpc/typescript
+          npm i
+      - name: npm run generate-bindings
+        run: |
+          cd deltachat-jsonrpc/typescript
+          npm run generate-bindings
+      - name: npm run check ts
+        run: |
+          cd deltachat-jsonrpc/typescript
+          npx tsc --noEmit
+      - name: run integration tests
+        run: |
+          cd deltachat-jsonrpc/typescript
+          npm run build
+          cargo build --features webserver
+          npm run test:integration
+      - name: run prettier
+        run: |
+          cd deltachat-jsonrpc/typescript
+          npm run prettier:check

.github/workflows/node-delete-preview.yml

@@ -29,4 +29,4 @@ jobs:
         host: "download.delta.chat"
         port: 22
         local: "deltachat-node-${{ steps.getid.outputs.prid }}.tar.gz"
-        remote: "/var/www/html/download/node/"
+        remote: "/var/www/html/download/node/preview/"

.github/workflows/node-package.yml

@@ -1,14 +1,15 @@
-name: 'node.js'
+name: 'node.js build'
 on:
   pull_request:
   push:
     tags:
     - '*'
+    - '!py-*'
 
 
 jobs:
   prebuild:
-    name: 'Tests & Prebuild'
+    name: 'prebuild'
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -50,20 +51,6 @@ jobs:
           cd node
           npm install --verbose
 
-      - name: Test
-        if: runner.os != 'Windows'
-        run: |
-          cd node
-          npm run test
-        env:
-          DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
-      - name: Run tests on Windows, except lint
-        if: runner.os == 'Windows'
-        run: |
-          cd node
-          npm run test:mocha
-        env:
-          DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
       - name: Build Prebuild
         run: |
           cd node
@@ -97,8 +84,9 @@ jobs:
         run: |
           tag=${{ steps.tag.outputs.tag }}
           if [ -z "$tag" ]; then
-            node -e "console.log('::set-output name=prid::' + '${{ github.ref }}'.split('/')[2])"
+            node -e "console.log('DELTACHAT_NODE_TAR_GZ=deltachat-node-' + '${{ github.ref }}'.split('/')[2] + '.tar.gz')" >> $GITHUB_ENV
           else
+            echo "DELTACHAT_NODE_TAR_GZ=deltachat-node-${{ steps.tag.outputs.tag }}.tar.gz" >> $GITHUB_ENV
             echo "No preview will be uploaded this time, but the $tag release"
           fi
       - name: System info
@@ -108,6 +96,7 @@ jobs:
           cargo -vV
           npm --version
           node --version
+          echo $DELTACHAT_NODE_TAR_GZ
       - name: Download ubuntu prebuild
         uses: actions/download-artifact@v1
         with:
@@ -122,46 +111,48 @@ jobs:
           name: windows-latest
       - shell: bash
         run: |
-          mkdir prebuilds
-          tar -xvzf ubuntu-18.04/ubuntu-18.04.tar.gz -C prebuilds
-          tar -xvzf macos-latest/macos-latest.tar.gz -C prebuilds
-          tar -xvzf windows-latest/windows-latest.tar.gz -C prebuilds
-          tree prebuilds
+          mkdir node/prebuilds
+          tar -xvzf ubuntu-18.04/ubuntu-18.04.tar.gz -C node/prebuilds
+          tar -xvzf macos-latest/macos-latest.tar.gz -C node/prebuilds
+          tar -xvzf windows-latest/windows-latest.tar.gz -C node/prebuilds
+          tree node/prebuilds
+          rm -rf ubuntu-18.04 macos-latest windows-latest
       - name: install dependencies without running scripts
         run: |
           npm install --ignore-scripts
+      - name: build constants
+        run: |
+          npm run build:core:constants
       - name: build typescript part
         run: |
           npm run build:bindings:ts
       - name: package
         shell: bash
         run: |
+          mv node/README.md README.md
           npm pack .
           ls -lah
-          mv $(find deltachat-node-*) deltachat-node-${{ steps.prepare.outputs.prid }}.tar.gz
+          mv $(find deltachat-node-*) $DELTACHAT_NODE_TAR_GZ
       - name: Upload Prebuild
         uses: actions/upload-artifact@v1
         with:
           name: deltachat-node.tgz
-          path: deltachat-node-${{ steps.prepare.outputs.prid }}.tar.gz
+          path: ${{ env.DELTACHAT_NODE_TAR_GZ }}
       # Upload to download.delta.chat/node/preview/
       - name: Upload deltachat-node preview to download.delta.chat/node/preview/
+        if: ${{ ! steps.tag.outputs.tag }}
         id: upload-preview
         shell: bash
         run: |
           echo -e "${{ secrets.SSH_KEY }}" >__TEMP_INPUT_KEY_FILE
           chmod 600 __TEMP_INPUT_KEY_FILE
-          if [[ -z "${{ steps.prepare.outputs.prid }}" ]]
-          then
-            exit 1
-          fi
-          scp -o StrictHostKeyChecking=no -v -i __TEMP_INPUT_KEY_FILE -P "22" -r deltachat-node-${{ steps.prepare.outputs.prid }}.tar.gz "${{ secrets.USERNAME }}"@"download.delta.chat":"/var/www/html/download/node/preview/"
+          scp -o StrictHostKeyChecking=no -v -i __TEMP_INPUT_KEY_FILE -P "22" -r $DELTACHAT_NODE_TAR_GZ "${{ secrets.USERNAME }}"@"download.delta.chat":"/var/www/html/download/node/preview/"
         continue-on-error: true
       - name: "Post links to details"
         if: steps.upload-preview.outcome == 'success'
         run: node ./node/scripts/postLinksToDetails.js
         env:
-         URL: preview/deltachat-node-${{ steps.prepare.outputs.prid }}
+         URL: preview/${{ env.DELTACHAT_NODE_TAR_GZ }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       # Upload to download.delta.chat/node/
       - name: Upload deltachat-node build to download.delta.chat/node/
@@ -171,5 +162,4 @@ jobs:
         run: |
           echo -e "${{ secrets.SSH_KEY }}" >__TEMP_INPUT_KEY_FILE
           chmod 600 __TEMP_INPUT_KEY_FILE
-          mv deltachat-node-${{ steps.prepare.outputs.prid }}.tar.gz deltachat-node-${{ steps.tag.outputs.tag }}.tar.gz
-          scp -o StrictHostKeyChecking=no -v -i __TEMP_INPUT_KEY_FILE -P "22" -r deltachat-node-${{ steps.tag.outputs.tag }}.tar.gz "${{ secrets.USERNAME }}"@"download.delta.chat":"/var/www/html/download/node/"
+          scp -o StrictHostKeyChecking=no -v -i __TEMP_INPUT_KEY_FILE -P "22" -r $DELTACHAT_NODE_TAR_GZ "${{ secrets.USERNAME }}"@"download.delta.chat":"/var/www/html/download/node/"

.github/workflows/node-tests.yml

@@ -0,0 +1,67 @@
+name: 'node.js tests'
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+      - staging
+      - trying
+
+jobs:
+  tests:
+    name: 'tests'
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-18.04, macos-latest, windows-latest]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '16'
+      - name: System info
+        run: |
+          rustc -vV
+          rustup -vV
+          cargo -vV
+          npm --version
+          node --version
+
+      - name: Cache node modules
+        uses: actions/cache@v2
+        with:
+          path: |
+            ${{ env.APPDATA }}/npm-cache
+            ~/.npm
+          key: ${{ matrix.os }}-node-${{ hashFiles('**/package.json') }}
+
+      - name: Cache cargo index
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cargo/registry/
+            ~/.cargo/git
+            target
+          key: ${{ matrix.os }}-cargo-index-${{ hashFiles('**/Cargo.lock') }}-2
+
+      - name: Install dependencies & build
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          cd node
+          npm install --verbose
+
+      - name: Test
+        if: runner.os != 'Windows'
+        run: |
+          cd node
+          npm run test
+        env:
+          DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+      - name: Run tests on Windows, except lint
+        if: runner.os == 'Windows'
+        run: |
+          cd node
+          npm run test:mocha
+        env:
+          DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}

.npmignore

@@ -40,3 +40,17 @@ node/old_docs.md
 .vscode/
 .github/
 node/.prettierrc.yml
+
+deltachat-jsonrpc/TODO.md
+deltachat-jsonrpc/README.MD
+deltachat-jsonrpc/.gitignore
+deltachat-jsonrpc/typescript/.gitignore
+deltachat-jsonrpc/typescript/.prettierignore
+deltachat-jsonrpc/typescript/accounts/
+deltachat-jsonrpc/typescript/index.html
+deltachat-jsonrpc/typescript/node-demo.js
+deltachat-jsonrpc/typescript/report_api_coverage.mjs
+deltachat-jsonrpc/typescript/test
+deltachat-jsonrpc/typescript/example.ts
+
+.DS_Store

CHANGELOG.md

@@ -1,9 +1,101 @@
 # Changelog
 
+## Unreleased
+
+### Changes
+- limit the rate of MDN sending #3402
+- ignore ratelimits for bots #3439
+- remove `msgs_mdns` references to deleted messages during housekeeping #3387
+- format message lines starting with `>` as quotes #3434
+- node: remove `split2` dependency #3418
+- node: add git installation info to readme #3418
+
+### Fixes
+- set a default error if NDN does not provide an error
+- python: avoid exceptions when messages/contacts/chats are compared with `None`
+- node: wait for the event loop to stop before destroying contexts #3431 #3451
+- emit configuration errors via event on failure #3433
+- report configure and imex success/failure after freeing ongoing process #3442
+
+### API-Changes
+- python: added `Message.get_status_updates()`  #3416
+- python: added `Message.send_status_update()`  #3416
+- python: added `Message.is_webxdc()`  #3416
+- python: added `Message.is_videochat_invitation()`  #3416
+- python: added support for "videochat" and "webxdc" view types to `Message.new_empty()`  #3416
+
+## 1.86.0
+
+### API-Changes
+- python: added optional `closed` parameter to `Account` constructor #3394
+- python: added optional `passphrase` parameter to `Account.export_all()` and `Account.import_all()` #3394
+- python: added `Account.open()` #3394
+- python: added `Chat.is_single()` #3394
+- python: added `Chat.is_mailinglist()` #3394
+- python: added `Chat.is_broadcast()` #3394
+- python: added `Chat.is_multiuser()` #3394
+- python: added `Chat.is_self_talk()` #3394
+- python: added `Chat.is_device_talk()` #3394
+- python: added `Chat.is_pinned()` #3394
+- python: added `Chat.pin()` #3394
+- python: added `Chat.unpin()` #3394
+- python: added `Chat.archive()` #3394
+- python: added `Chat.unarchive()` #3394
+- python: added `Message.get_summarytext()` #3394
+- python: added optional `closed` parameter to `ACFactory.get_unconfigured_account()` (pytest plugin) #3394
+- python: added optional `passphrase` parameter to `ACFactory.get_pseudo_configured_account()` (pytest plugin) #3394
+
+### Changes
+- clean up series of webxdc info messages;
+  `DC_EVENT_MSGS_CHANGED` is emitted on changes of existing info messages #3395
+- update provider database #3399
+- refactorings #3375 #3403 #3398 #3404
+
+### Fixes
+- do not reset our database if imported backup cannot be decrypted #3397
+- node: remove `npx` from build script, this broke flathub build #3396
+
+
+## 1.85.0
+
+### Changes
+- refactorings #3373 #3345 #3380 #3382
+- node: move split2 to devDependencies
+- python: build Python 3.10 wheels #3392
+- update Rust dependencies
+
+### Fixes
+- delete outgoing MDNs found in the Sent folder on Gmail #3372
+- fix searching one-to-one chats #3377
+- do not add legacy info-messages on resending webxdc #3389
+
+
+## 1.84.0
+
+### Changes
+- refactorings #3354 #3347 #3353 #3346
+
+### Fixes
+- do not unnecessarily SELECT folders if there are no operations planned on
+  them #3333
+- trim chat encryption info #3350
+- fix failure to decrypt first message to self after key synchronization
+  via Autocrypt Setup Message #3352
+- Keep pgp key when you change your own email address #3351
+- Do not ignore Sent and Spam folders on Gmail #3369
+- handle decryption errors explicitly and don't get confused by encrypted mail attachments #3374
+
+
+## 1.83.0
+
+### Fixes
+- fix node prebuild & package ci #3337
+
+
 ## 1.82.0
 
 ### API-Changes
-- re-add removed DC_MSG_ID_MARKER1 as in use on iOS #3330
+- re-add removed `DC_MSG_ID_MARKER1` as in use on iOS #3330
 
 ### Changes
 - refactorings #3328

CMakeLists.txt

@@ -21,7 +21,7 @@ add_custom_command(
         PREFIX=${CMAKE_INSTALL_PREFIX}
         LIBDIR=${CMAKE_INSTALL_FULL_LIBDIR}
         INCLUDEDIR=${CMAKE_INSTALL_FULL_INCLUDEDIR}
-        ${CARGO} build --release --no-default-features
+        ${CARGO} build --release --no-default-features --features jsonrpc
 
 	# Build in `deltachat-ffi` directory instead of using
 	# `--package deltachat_ffi` to avoid feature resolver version

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat"
-version = "1.82.0"
+version = "1.86.0"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2021"
 license = "MPL-2.0"
@@ -25,11 +25,10 @@ async-smtp = { git = "https://github.com/async-email/async-smtp", branch="master
 async-std-resolver = "0.21"
 async-std = { version = "1" }
 async-tar = { version = "0.4", default-features=false }
-async-trait = "0.1"
 backtrace = "0.3"
 base64 = "0.13"
 bitflags = "1.3"
-chrono = "0.4"
+chrono = { version = "0.4", default-features=false, features = ["clock", "std"] }
 dirs = { version = "4", optional=true }
 email = { git = "https://github.com/deltachat/rust-email", branch = "master" }
 encoded-words = { git = "https://github.com/async-email/encoded-words", branch="master" }
@@ -46,11 +45,11 @@ native-tls = "0.2"
 num_cpus = "1.13"
 num-derive = "0.3"
 num-traits = "0.2"
-once_cell = "1.10.0"
+once_cell = "1.12.0"
 percent-encoding = "2.0"
 pgp = { version = "0.7", default-features = false }
 pretty_env_logger = { version = "0.4", optional = true }
-quick-xml = "0.22"
+quick-xml = "0.23"
 r2d2 = "0.8"
 r2d2_sqlite = "0.20"
 rand = "0.7"
@@ -58,7 +57,7 @@ regex = "1.5"
 rusqlite = { version = "0.27", features = ["sqlcipher"] }
 rust-hsluv = "0.1"
 rustyline = { version = "9", optional = true }
-sanitize-filename = "0.3"
+sanitize-filename = "0.4"
 serde_json = "1.0"
 serde = { version = "1.0", features = ["derive"] }
 sha-1 = "0.10"
@@ -70,7 +69,7 @@ surf = { version = "2.3", default-features = false, features = ["h1-client"] }
 thiserror = "1"
 toml = "0.5"
 url = "2"
-uuid = { version = "0.8", features = ["serde", "v4"] }
+uuid = { version = "1", features = ["serde", "v4"] }
 fast-socks5 = "0.4"
 humansize = "1"
 qrcodegen = "1.7.0"
@@ -92,6 +91,7 @@ tempfile = "3"
 members = [
   "deltachat-ffi",
   "deltachat_derive",
+  "deltachat-jsonrpc"
 ]
 
 [[example]]

README.md

@@ -102,9 +102,6 @@ $ cargo build -p deltachat_ffi --release
 
 ## Debugging environment variables 
 
-- `DCC_IMAP_DEBUG`: if set IMAP protocol commands and responses will be
-  printed
-
 - `DCC_MIME_DEBUG`: if set outgoing and incoming message will be printed 
 
 - `RUST_LOG=repl=info,async_imap=trace,async_smtp=trace`: enable IMAP and

benches/contacts.rs

@@ -2,13 +2,16 @@ use async_std::task::block_on;
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use deltachat::contact::Contact;
 use deltachat::context::Context;
+use deltachat::Events;
 use tempfile::tempdir;
 
 async fn address_book_benchmark(n: u32, read_count: u32) {
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
+    let context = Context::new(dbfile.into(), id, Events::new())
+        .await
+        .unwrap();
 
     let book = (0..n)
         .map(|i| format!("Name {}\naddr{}@example.org\n", i, i))

benches/get_chat_msgs.rs

@@ -6,10 +6,13 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use deltachat::chat::{self, ChatId};
 use deltachat::chatlist::Chatlist;
 use deltachat::context::Context;
+use deltachat::Events;
 
 async fn get_chat_msgs_benchmark(dbfile: &Path, chats: &[ChatId]) {
     let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
+    let context = Context::new(dbfile.into(), id, Events::new())
+        .await
+        .unwrap();
 
     for c in chats.iter().take(10) {
         black_box(chat::get_chat_msgs(&context, *c, 0).await.ok());
@@ -21,7 +24,9 @@ fn criterion_benchmark(c: &mut Criterion) {
     // messages, such as your primary account.
     if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
         let chats: Vec<_> = async_std::task::block_on(async {
-            let context = Context::new((&path).into(), 100).await.unwrap();
+            let context = Context::new((&path).into(), 100, Events::new())
+                .await
+                .unwrap();
             let chatlist = Chatlist::try_load(&context, 0, None, None).await.unwrap();
             let len = chatlist.len();
             (0..len).map(|i| chatlist.get_chat_id(i).unwrap()).collect()

benches/get_chatlist.rs

@@ -3,6 +3,7 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion};
 
 use deltachat::chatlist::Chatlist;
 use deltachat::context::Context;
+use deltachat::Events;
 
 async fn get_chat_list_benchmark(context: &Context) {
     Chatlist::try_load(context, 0, None, None).await.unwrap();
@@ -12,8 +13,9 @@ fn criterion_benchmark(c: &mut Criterion) {
     // To enable this benchmark, set `DELTACHAT_BENCHMARK_DATABASE` to some large database with many
     // messages, such as your primary account.
     if let Ok(path) = std::env::var("DELTACHAT_BENCHMARK_DATABASE") {
-        let context =
-            async_std::task::block_on(async { Context::new(path.into(), 100).await.unwrap() });
+        let context = async_std::task::block_on(async {
+            Context::new(path.into(), 100, Events::new()).await.unwrap()
+        });
         c.bench_function("chatlist:try_load (Get Chatlist)", |b| {
             b.to_async(AsyncStdExecutor)
                 .iter(|| get_chat_list_benchmark(black_box(&context)))

benches/receive_emails.rs

@@ -8,6 +8,7 @@ use deltachat::{
     context::Context,
     dc_receive_imf::dc_receive_imf,
     imex::{imex, ImexMode},
+    Events,
 };
 use tempfile::tempdir;
 
@@ -42,7 +43,9 @@ async fn create_context() -> Context {
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
+    let context = Context::new(dbfile.into(), id, Events::new())
+        .await
+        .unwrap();
 
     let backup: PathBuf = std::env::current_dir()
         .unwrap()

benches/search_msgs.rs

@@ -1,12 +1,15 @@
 use async_std::task::block_on;
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use deltachat::context::Context;
+use deltachat::Events;
 use std::path::Path;
 
 async fn search_benchmark(path: impl AsRef<Path>) {
     let dbfile = path.as_ref();
     let id = 100;
-    let context = Context::new(dbfile.into(), id).await.unwrap();
+    let context = Context::new(dbfile.into(), id, Events::new())
+        .await
+        .unwrap();
 
     for _ in 0..10u32 {
         context.search_msgs(None, "hello").await.unwrap();

deltachat-ffi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltachat_ffi"
-version = "1.82.0"
+version = "1.86.0"
 description = "Deltachat FFI"
 authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
 edition = "2018"
@@ -16,6 +16,7 @@ crate-type = ["cdylib", "staticlib"]
 
 [dependencies]
 deltachat = { path = "../", default-features = false }
+deltachat-jsonrpc = { path = "../deltachat-jsonrpc", optional = true }
 libc = "0.2"
 human-panic = "1"
 num-traits = "0.2"
@@ -29,4 +30,5 @@ rand = "0.7"
 default = ["vendored"]
 vendored = ["deltachat/vendored"]
 nightly = ["deltachat/nightly"]
+jsonrpc = ["deltachat-jsonrpc"]
 

deltachat-ffi/deltachat.h

@@ -23,7 +23,7 @@ typedef struct _dc_provider dc_provider_t;
 typedef struct _dc_event    dc_event_t;
 typedef struct _dc_event_emitter dc_event_emitter_t;
 typedef struct _dc_accounts_event_emitter dc_accounts_event_emitter_t;
-
+typedef struct _dc_jsonrpc_instance dc_jsonrpc_instance_t;
 
 /**
  * @mainpage Getting started
@@ -1791,6 +1791,11 @@ void            dc_delete_msgs               (dc_context_t* context, const uint3
 /**
  * Forward messages to another chat.
  *
+ * All types of messages can be forwarded,
+ * however, they will be flagged as such (dc_msg_is_forwarded() is set).
+ *
+ * Original sender, info-state and webxdc updates are not forwarded on purpose.
+ *
  * @memberof dc_context_t
  * @param context The context object.
  * @param msg_ids An array of uint32_t containing all message IDs that should be forwarded.
@@ -3979,7 +3984,7 @@ int             dc_msg_is_sent                (const dc_msg_t* msg);
  *
  * For privacy reasons, we do not provide the name or the e-mail address of the
  * original author (in a typical GUI, you select the messages text and click on
- * "forwared"; you won't expect other data to be send to the new recipient,
+ * "forwarded"; you won't expect other data to be send to the new recipient,
  * esp. as the new recipient may not be in any relationship to the original author)
  *
  * @memberof dc_msg_t
@@ -5173,6 +5178,55 @@ int64_t          dc_lot_get_timestamp     (const dc_lot_t* lot);
  */
 
 
+
+/**
+ * @class dc_jsonrpc_instance_t
+ *
+ * Opaque object for using the json rpc api from the cffi bindings.
+ */
+
+/**
+ * Create the jsonrpc instance that is used to call the jsonrpc.
+ *
+ * @memberof dc_accounts_t
+ * @param account_manager The accounts object as created by dc_accounts_new().
+ * @return Returns the jsonrpc instance, NULL on errors.
+ *     Must be freed using dc_jsonrpc_unref() after usage.
+ *
+ */
+dc_jsonrpc_instance_t* dc_jsonrpc_init(dc_accounts_t* account_manager);
+
+/**
+ * Free a jsonrpc instance.
+ *
+ * @memberof dc_jsonrpc_instance_t
+ * @param jsonrpc_instance jsonrpc instance as returned from dc_jsonrpc_init().
+ *     If NULL is given, nothing is done and an error is logged.
+ */
+void dc_jsonrpc_unref(dc_jsonrpc_instance_t* jsonrpc_instance);
+
+/**
+ * Makes an asynchronous jsonrpc request,
+ * returns immediately and once the result is ready it can be retrieved via dc_jsonrpc_next_response()
+ * the jsonrpc specification defines an invocation id that can then be used to match request and response.
+ *
+ * @memberof dc_jsonrpc_instance_t
+ * @param jsonrpc_instance jsonrpc instance as returned from dc_jsonrpc_init().
+ * @param request JSON-RPC request as string
+ */
+void dc_jsonrpc_request(dc_jsonrpc_instance_t* jsonrpc_instance, char* request);
+
+/**
+ * Get the next json_rpc response, blocks until there is a new event, so call this in a loop from a thread.
+ *
+ * @memberof dc_jsonrpc_instance_t
+ * @param jsonrpc_instance jsonrpc instance as returned from dc_jsonrpc_init().
+ * @return JSON-RPC response as string
+ *     If NULL is returned, the accounts_t belonging to the jsonrpc instance is unref'd and no more events will come;
+ *     in this case, free the jsonrpc instance using dc_jsonrpc_unref().
+ */
+char* dc_jsonrpc_next_response(dc_jsonrpc_instance_t* jsonrpc_instance);
+
 /**
  * @class dc_event_emitter_t
  *

deltachat-ffi/src/lib.rs

@@ -1,4 +1,4 @@
-#![deny(clippy::all)]
+#![deny(unused, clippy::all)]
 #![allow(
     non_camel_case_types,
     non_snake_case,
@@ -11,8 +11,6 @@
 
 #[macro_use]
 extern crate human_panic;
-extern crate num_traits;
-extern crate serde_json;
 
 use std::collections::BTreeMap;
 use std::convert::TryFrom;
@@ -20,6 +18,7 @@ use std::fmt::Write;
 use std::ops::Deref;
 use std::ptr;
 use std::str::FromStr;
+use std::sync::Arc;
 use std::time::{Duration, SystemTime};
 
 use anyhow::Context as _;
@@ -79,7 +78,11 @@ pub unsafe extern "C" fn dc_context_new(
     let ctx = if blobdir.is_null() || *blobdir == 0 {
         // generate random ID as this functionality is not yet available on the C-api.
         let id = rand::thread_rng().gen();
-        block_on(Context::new(as_path(dbfile).to_path_buf().into(), id))
+        block_on(Context::new(
+            as_path(dbfile).to_path_buf().into(),
+            id,
+            Events::new(),
+        ))
     } else {
         eprintln!("blobdir can not be defined explicitly anymore");
         return ptr::null_mut();
@@ -106,6 +109,7 @@ pub unsafe extern "C" fn dc_context_new_closed(dbfile: *const libc::c_char) -> *
     match block_on(Context::new_closed(
         as_path(dbfile).to_path_buf().into(),
         id,
+        Events::new(),
     )) {
         Ok(context) => Box::into_raw(Box::new(context)),
         Err(err) => {
@@ -458,7 +462,37 @@ pub unsafe extern "C" fn dc_event_get_id(event: *mut dc_event_t) -> libc::c_int
     }
 
     let event = &*event;
-    event.as_id()
+    match event.typ {
+        EventType::Info(_) => 100,
+        EventType::SmtpConnected(_) => 101,
+        EventType::ImapConnected(_) => 102,
+        EventType::SmtpMessageSent(_) => 103,
+        EventType::ImapMessageDeleted(_) => 104,
+        EventType::ImapMessageMoved(_) => 105,
+        EventType::NewBlobFile(_) => 150,
+        EventType::DeletedBlobFile(_) => 151,
+        EventType::Warning(_) => 300,
+        EventType::Error(_) => 400,
+        EventType::ErrorSelfNotInGroup(_) => 410,
+        EventType::MsgsChanged { .. } => 2000,
+        EventType::IncomingMsg { .. } => 2005,
+        EventType::MsgsNoticed { .. } => 2008,
+        EventType::MsgDelivered { .. } => 2010,
+        EventType::MsgFailed { .. } => 2012,
+        EventType::MsgRead { .. } => 2015,
+        EventType::ChatModified(_) => 2020,
+        EventType::ChatEphemeralTimerModified { .. } => 2021,
+        EventType::ContactsChanged(_) => 2030,
+        EventType::LocationChanged(_) => 2035,
+        EventType::ConfigureProgress { .. } => 2041,
+        EventType::ImexProgress(_) => 2051,
+        EventType::ImexFileWritten(_) => 2052,
+        EventType::SecurejoinInviterProgress { .. } => 2060,
+        EventType::SecurejoinJoinerProgress { .. } => 2061,
+        EventType::ConnectivityChanged => 2100,
+        EventType::SelfavatarChanged => 2110,
+        EventType::WebxdcStatusUpdate { .. } => 2120,
+    }
 }
 
 #[no_mangle]
@@ -3546,7 +3580,8 @@ pub unsafe extern "C" fn dc_msg_latefiling_mediasize(
         ffi_msg
             .message
             .latefiling_mediasize(ctx, width, height, duration)
-    });
+    })
+    .ok_or_log_msg(ctx, "Cannot set media size");
 }
 
 #[no_mangle]
@@ -4059,11 +4094,11 @@ pub unsafe extern "C" fn dc_provider_unref(provider: *mut dc_provider_t) {
 /// Reader-writer lock wrapper for accounts manager to guarantee thread safety when using
 /// `dc_accounts_t` in multiple threads at once.
 pub struct AccountsWrapper {
-    inner: RwLock<Accounts>,
+    inner: Arc<RwLock<Accounts>>,
 }
 
 impl Deref for AccountsWrapper {
-    type Target = RwLock<Accounts>;
+    type Target = Arc<RwLock<Accounts>>;
 
     fn deref(&self) -> &Self::Target {
         &self.inner
@@ -4072,7 +4107,7 @@ impl Deref for AccountsWrapper {
 
 impl AccountsWrapper {
     fn new(accounts: Accounts) -> Self {
-        let inner = RwLock::new(accounts);
+        let inner = Arc::new(RwLock::new(accounts));
         Self { inner }
     }
 }
@@ -4347,7 +4382,7 @@ pub unsafe extern "C" fn dc_accounts_maybe_network_lost(accounts: *mut dc_accoun
     block_on(async move { accounts.write().await.maybe_network_lost().await });
 }
 
-pub type dc_accounts_event_emitter_t = deltachat::accounts::EventEmitter;
+pub type dc_accounts_event_emitter_t = EventEmitter;
 
 #[no_mangle]
 pub unsafe extern "C" fn dc_accounts_get_event_emitter(
@@ -4390,3 +4425,77 @@ pub unsafe extern "C" fn dc_accounts_get_next_event(
         .map(|ev| Box::into_raw(Box::new(ev)))
         .unwrap_or_else(ptr::null_mut)
 }
+
+#[cfg(feature = "jsonrpc")]
+mod jsonrpc {
+    use super::*;
+    use deltachat_jsonrpc::api::CommandApi;
+    use deltachat_jsonrpc::yerpc::{MessageHandle, RpcHandle};
+
+    pub struct dc_jsonrpc_instance_t {
+        receiver: async_std::channel::Receiver<deltachat_jsonrpc::yerpc::Message>,
+        handle: MessageHandle<CommandApi>,
+    }
+
+    #[no_mangle]
+    pub unsafe extern "C" fn dc_jsonrpc_init(
+        account_manager: *mut dc_accounts_t,
+    ) -> *mut dc_jsonrpc_instance_t {
+        if account_manager.is_null() {
+            eprintln!("ignoring careless call to dc_jsonrpc_init()");
+            return ptr::null_mut();
+        }
+
+        let cmd_api =
+            deltachat_jsonrpc::api::CommandApi::new_from_arc((*account_manager).inner.clone());
+
+        let (request_handle, receiver) = RpcHandle::new();
+        let handle = MessageHandle::new(request_handle, cmd_api);
+
+        let instance = dc_jsonrpc_instance_t { receiver, handle };
+
+        Box::into_raw(Box::new(instance))
+    }
+
+    #[no_mangle]
+    pub unsafe extern "C" fn dc_jsonrpc_unref(jsonrpc_instance: *mut dc_jsonrpc_instance_t) {
+        if jsonrpc_instance.is_null() {
+            eprintln!("ignoring careless call to dc_jsonrpc_unref()");
+            return;
+        }
+
+        Box::from_raw(jsonrpc_instance);
+    }
+
+    #[no_mangle]
+    pub unsafe extern "C" fn dc_jsonrpc_request(
+        jsonrpc_instance: *mut dc_jsonrpc_instance_t,
+        request: *const libc::c_char,
+    ) {
+        if jsonrpc_instance.is_null() || request.is_null() {
+            eprintln!("ignoring careless call to dc_jsonrpc_request()");
+            return;
+        }
+
+        let api = &*jsonrpc_instance;
+        let handle = &api.handle;
+        let request = to_string_lossy(request);
+        async_std::task::spawn(async move {
+            handle.handle_message(&request).await;
+        });
+    }
+
+    #[no_mangle]
+    pub unsafe extern "C" fn dc_jsonrpc_next_response(
+        jsonrpc_instance: *mut dc_jsonrpc_instance_t,
+    ) -> *mut libc::c_char {
+        if jsonrpc_instance.is_null() {
+            eprintln!("ignoring careless call to dc_jsonrpc_next_response()");
+            return ptr::null_mut();
+        }
+        let api = &*jsonrpc_instance;
+        async_std::task::block_on(api.receiver.recv())
+            .map(|result| serde_json::to_string(&result).unwrap_or_default().strdup())
+            .unwrap_or(ptr::null_mut())
+    }
+}

deltachat-jsonrpc/.gitignore

@@ -0,0 +1,3 @@
+accounts/
+
+.cargo

deltachat-jsonrpc/Cargo.toml

@@ -0,0 +1,39 @@
+[package]
+name = "deltachat-jsonrpc"
+version = "1.86.0"
+authors = ["Delta Chat Developers (ML) <delta@codespeak.net>"]
+edition = "2021"
+default-run = "webserver"
+license = "MPL-2.0"
+
+[[bin]]
+name = "webserver"
+path = "src/webserver.rs"
+required-features = ["webserver"]
+
+[dependencies]
+anyhow = "1"
+async-std = { version = "1", features = ["attributes"] }
+deltachat = { path = ".." }
+num-traits = "0.2"
+serde = { version = "1.0", features = ["derive"] }
+tempfile = "3.3.0"
+log = "0.4"
+async-channel = { version = "1.6.1" }
+futures = { version = "0.3.19" }
+serde_json = "1.0.75"
+yerpc = { git = "https://github.com/Frando/yerpc", features = ["anyhow"] }
+typescript-type-def = { git = "https://github.com/Frando/rust-typescript-type-def", branch = "yerpc", features = ["json_value"] }
+# optional, depended on features
+env_logger = { version = "0.9.0", optional = true }
+tide = { version = "0.16.0", optional = true }
+tide-websockets = { version = "0.4.0", optional = true }
+yerpc-tide = { git = "https://github.com/Frando/yerpc", optional = true }
+
+
+[features]
+default = []
+webserver = ["env_logger", "tide", "tide-websockets", "yerpc-tide"]
+
+[profile.release]
+lto = true

deltachat-jsonrpc/README.MD

@@ -0,0 +1,75 @@
+# deltachat-jsonrpc
+
+## Build Requirements
+
+- Linux or Mac, scrips make use of features like `>` pipes and `&&` (maybe the newer versions of powershell support them, but I didn't try that.)
+- rust (installed via rustup)
+
+## Start the webserver
+
+The webserver is an example usage. Goal of it is to be usable both as example and as base for deltachat-kaiOS.
+
+```sh
+RUST_LOG=info cargo run --features webserver
+```
+
+## Generate Typescript Bindings
+
+```sh
+cd typescript
+npm i
+npm run build
+```
+
+## Run the development example
+
+Mac
+
+```sh
+alias firefox=/Applications/Firefox.app/Contents/MacOS/firefox
+npm run example:build && firefox --devtools $(pwd)/example/browser-example.html
+```
+
+Linux:
+
+```sh
+npm run example:run
+```
+
+## Compiling server for kaiOS or android:
+
+```sh
+cross build --features=webserver --target armv7-linux-androideabi --release
+```
+
+## Run the tests
+
+### Rust tests
+
+```
+cargo test --features=webserver
+```
+
+### Typescript
+
+```
+cd typescript
+npm run test
+```
+
+For the online tests to run you need a test account token for a mailadm instance,
+you can use docker to spin up a local instance: https://github.com/deltachat/docker-mailadm
+
+> set the env var `DCC_NEW_TMP_EMAIL` to your mailadm token: example:
+> `DCC_NEW_TMP_EMAIL=https://testrun.org/new_email?t=1h_195dksa6544`
+
+If your test fail with server shutdown at the start, then you might have a process from a last run still running probably and you need to kill that process manually to continue.
+
+#### Test Coverage
+
+You can test coverage with `npm run coverage`, but you need to have `DCC_NEW_TMP_EMAIL` set, otherwise the result will be useless because some functions can only be tested with the online tests.
+
+> If you are offline and want to see the coverage results anyway (even though they are NOT correct), you can bypass the error with `COVERAGE_OFFLINE=1 npm run coverage`
+
+Open `coverage/index.html` for a detailed report.
+`bindings.ts` is probably the most interesting file for coverage, because it describes the api functions.

deltachat-jsonrpc/TODO.md

@@ -0,0 +1,347 @@
+## Core system
+
+- [X] Base structure of JSON API code 
+- [X] Implement the first methods for testing + the code that should later be generated by the proc macro
+- [X] Create the proc macro
+  - [X] json api
+  - [X] ts types
+    - [X] arguments (no args, one argument, multiple args)
+    - [X] return type
+    - [X] custom types as type aliases that ts file looks prettier
+
+
+## Pre - MVP
+
+- [X] Web socket server
+- [WIP] Web socket client (ts)
+  - [X] backend connection state changed events
+  - [X] Reconnect on connection loss / connection state
+  - [ ] find a way to type the event emitter callback functions
+- [X] Events
+
+## MVP
+
+- [X] mocha integration test for ts api
+ - [X] basic tests
+ - [X] advanced / "online tests" (mailadm for burner accounts)
+- [ ] coverage for a majority of the API
+- [ ] Blobs served
+- [ ] Blob upload (for attachments, setting profile-picture, importing backup and so on)
+- [ ] Web push API? At least some kind of notification hook closure this lib can accept.
+
+## Other Ideas
+
+- [ ] make sure there can only be one connection at a time to the ws 
+  - why? , it could give problems if its commanded from multiple connections
+- [ ] encrypted connection?
+- [ ] authenticated connection?
+- [ ] Look into unit-testing for the proc macros?
+- [ ] proc macro taking over doc comments to generated typescript file
+- [X] GH action for tests (rust and typescript)
+  - [X] rust test
+  - [X] rust fmt
+  - [X] rust clippy
+  - [X] tsc check
+  - [X] prettier
+  - [X] mocha
+- [X] scripts to check&fix prettier formatting
+
+
+
+## Apis
+
+replicate desktop api feature set:
+
+(this feature set is based on desktop version `1.20`, needs to be updated in the future)
+
+```rs
+
+
+struct sendMessageParams {
+  text: Option<String>,
+  filename: Option<String>, // TODO we need to think about blobs some more
+  location: Option<(u32,u32)>,
+  quote_message_id: Option<u32>,
+}
+
+struct QrCodeResponse = {
+  state: u32 // also enum in reality, for simlicity u32 here
+  id: u32
+  text1: String
+}
+
+impl Api {
+
+// root ---------------------------------------------------------------
+
+// NEEDS_THE_BLOB_QUESTION_ANSWERED_EVENTUALLY
+async fn sc_set_profile_picture(&self, new_image: String) -> Result<()> {}
+
+// NEEDS_THE_BLOB_QUESTION_ANSWERED_EVENTUALLY
+// 'getProfilePicture' equals to `dc.getContact(C.DC_CONTACT_ID_SELF).getProfileImage()` or `dc.get_config("selfavatar")`
+
+async fn sc_join_secure_join(&self, qrCode: String) -> Result<u32> {}
+async fn sc_stop_ongoing_process(&self) -> Result<u32> {}
+async fn sc_check_qr_code(&self, qrCode: String) -> Result<QrCodeResponse> {}
+
+// login ----------------------------------------------------
+
+// INFO: login functions need to call stop&start io where applicable
+
+// login.newLogin:
+// do instead in frontend:
+//   1. call `add_account`
+//   2. call `select_account`
+//   3. set credentials via set config
+//   4. call `sc_configure`
+
+// login.getLogins - is already implemented: `get_all_accounts`
+
+// login.loadAccount - Basically `select_account`
+
+// login.logout -> TODO: unselect account, which isn't implemented in the core yet
+
+// login.forgetAccount -> `remove_account`
+
+// login.getLastLoggedInAccount -> `get_selected_account_id`
+
+// login.updateCredentials -> do instead: set config then call `sc_configure`
+
+// backup -------------------------------------------------------------
+
+// INFO: backup functions need to call stop&start io
+
+// NEEDS_THE_BLOB_QUESTION_ANSWERED_EVENTUALLY
+async fn sc_backup_export(&self, out_dir: String) -> Result<()> {}
+// NEEDS_THE_BLOB_QUESTION_ANSWERED_EVENTUALLY
+async fn sc_backup_import(&self, file: String) -> Result<()> {} // will not return the same as in desktop because this function imports backup to the current context unlike it was in desktop
+
+// chatList -----------------------------------------------------------
+
+// chatList.selectChat - will be removed from desktop
+// chatList.getSelectedChatId - will be removed from desktop
+// chatList.onChatModified - will be removed from desktop
+
+async fn sc_chatlist_get_general_fresh_message_counter(&self) -> Result<u32> // this method might be used for a favicon badge counter
+
+// contacts ------------------------------------------------------------
+
+async fn sc_contacts_change_nickname(&self, contact_id: u32, new_name: String) -> Result<()>
+
+
+// contacts.getChatIdByContactId - very similar to sc_contacts_create_chat_by_contact_id
+// contacts.getDMChatId -  very similar to sc_contacts_create_chat_by_contact_id
+
+async fn sc_contacts_get_encryption_info(&self, contact_id: u32) -> Result<String>
+
+async fn sc_contacts_lookup_contact_id_by_addr(&self, email: String) -> Result<u32>
+
+
+}
+
+```
+```ts
+
+class DeltaRemote {
+  // chat ---------------------------------------------------------------
+  call(
+    fnName: 'chat.getChatMedia',
+    chatId: number,
+    msgType1: number,
+    msgType2: number
+  ): Promise<MessageType[]>
+  call(fnName: 'chat.getEncryptionInfo', chatId: number): Promise<string>
+  call(fnName: 'chat.getQrCode', chatId?: number): Promise<string>
+  call(fnName: 'chat.leaveGroup', chatId: number): Promise<void>
+  call(fnName: 'chat.setName', chatId: number, name: string): Promise<boolean>
+  call(
+    fnName: 'chat.modifyGroup',
+    chatId: number,
+    name: string,
+    image: string,
+    remove: number[],
+    add: number[]
+  ): Promise<boolean>
+  call(
+    fnName: 'chat.addContactToChat',
+    chatId: number,
+    contactId: number
+  ): Promise<boolean>
+  call(
+    fnName: 'chat.setProfileImage',
+    chatId: number,
+    newImage: string
+  ): Promise<boolean>
+  call(
+    fnName: 'chat.setMuteDuration',
+    chatId: number,
+    duration: MuteDuration
+  ): Promise<boolean>
+  call(
+    fnName: 'chat.createGroupChat',
+    verified: boolean,
+    name: string
+  ): Promise<number>
+  call(fnName: 'chat.delete', chatId: number): Promise<void>
+  call(
+    fnName: 'chat.setVisibility',
+    chatId: number,
+    visibility:
+      | C.DC_CERTCK_AUTO
+      | C.DC_CERTCK_STRICT
+      | C.DC_CHAT_VISIBILITY_PINNED
+  ): Promise<void>
+  call(fnName: 'chat.getChatContacts', chatId: number): Promise<number[]>
+  call(fnName: 'chat.markNoticedChat', chatId: number): Promise<void>
+  call(fnName: 'chat.getChatEphemeralTimer', chatId: number): Promise<number>
+  call(
+    fnName: 'chat.setChatEphemeralTimer',
+    chatId: number,
+    ephemeralTimer: number
+  ): Promise<void>
+  call(fnName: 'chat.sendVideoChatInvitation', chatId: number): Promise<number>
+  call(
+    fnName: 'chat.decideOnContactRequest',
+    messageId: number,
+    decision:
+      | C.DC_DECISION_START_CHAT
+      | C.DC_DECISION_NOT_NOW
+      | C.DC_DECISION_BLOCK
+  ): Promise<number>
+  // locations ----------------------------------------------------------
+  call(
+    fnName: 'locations.setLocation',
+    latitude: number,
+    longitude: number,
+    accuracy: number
+  ): Promise<void>
+  call(
+    fnName: 'locations.getLocations',
+    chatId: number,
+    contactId: number,
+    timestampFrom: number,
+    timestampTo: number
+  ): Promise<JsonLocations>
+
+  // NOTHING HERE that is called directly from the frontend, yet
+  // messageList --------------------------------------------------------
+  call(
+    fnName: 'messageList.sendMessage',
+    chatId: number,
+    params: sendMessageParams
+  ): Promise<[number, MessageType | null]>
+  call(
+    fnName: 'messageList.sendSticker',
+    chatId: number,
+    stickerPath: string
+  ): Promise<void>
+  call(fnName: 'messageList.deleteMessage', id: number): Promise<void>
+  call(fnName: 'messageList.getMessageInfo', msgId: number): Promise<string>
+  call(
+    fnName: 'messageList.getDraft',
+    chatId: number
+  ): Promise<MessageType | null>
+  call(
+    fnName: 'messageList.setDraft',
+    chatId: number,
+    {
+      text,
+      file,
+      quotedMessageId,
+    }: { text?: string; file?: string; quotedMessageId?: number }
+  ): Promise<void>
+  call(
+    fnName: 'messageList.messageIdToJson',
+    id: number
+  ): Promise<{ msg: null } | MessageType>
+  call(
+    fnName: 'messageList.forwardMessage',
+    msgId: number,
+    chatId: number
+  ): Promise<void>
+  call(
+    fnName: 'messageList.searchMessages',
+    query: string,
+    chatId?: number
+  ): Promise<number[]>
+  call(
+    fnName: 'messageList.msgIds2SearchResultItems',
+    msgIds: number[]
+  ): Promise<{ [id: number]: MessageSearchResult }>
+  call(
+    fnName: 'messageList.saveMessageHTML2Disk',
+    messageId: number
+  ): Promise<string>
+  // settings -----------------------------------------------------------
+  call(fnName: 'settings.keysImport', directory: string): Promise<void>
+  call(fnName: 'settings.keysExport', directory: string): Promise<void>
+  call(
+    fnName: 'settings.serverFlags',
+    {
+      mail_security,
+      send_security,
+    }: {
+      mail_security?: string
+      send_security?: string
+    }
+  ): Promise<number | ''>
+  call(
+    fnName: 'settings.setDesktopSetting',
+    key: keyof DesktopSettings,
+    value: string | number | boolean
+  ): Promise<boolean>
+  call(fnName: 'settings.getDesktopSettings'): Promise<DesktopSettings>
+  call(
+    fnName: 'settings.saveBackgroundImage',
+    file: string,
+    isDefaultPicture: boolean
+  ): Promise<string>
+  call(
+    fnName: 'settings.estimateAutodeleteCount',
+    fromServer: boolean,
+    seconds: number
+  ): Promise<number>
+  // stickers -----------------------------------------------------------
+  call(
+    fnName: 'stickers.getStickers'
+  ): Promise<{
+    [key: string]: string[]
+  }> // todo move to extras? because its not directly elated to core
+  // context ------------------------------------------------------------
+  call(fnName: 'context.maybeNetwork'): Promise<void>
+  // burner accounts ------------------------------------------------------------
+  call(
+    fnName: 'burnerAccounts.create',
+    url: string
+  ): Promise<{ email: string; password: string }> // think about how to improve that api - probably use core api instead
+  // extras -------------------------------------------------------------
+  call(fnName: 'extras.getLocaleData', locale: string): Promise<LocaleData>
+  call(fnName: 'extras.setLocale', locale: string): Promise<void>
+  call(
+    fnName: 'extras.getActiveTheme'
+  ): Promise<{
+    theme: Theme
+    data: string
+  } | null>
+  call(fnName: 'extras.setThemeFilePath', address: string): void
+  call(fnName: 'extras.getAvailableThemes'): Promise<Theme[]>
+  call(fnName: 'extras.setTheme', address: string): Promise<boolean>
+  // catchall: ----------------------------------------------------------
+  call(fnName: string): Promise<any>
+  call(fnName: string, ...args: any[]): Promise<any> {
+    return _callDcMethodAsync(fnName, ...args)
+  }
+}
+
+export const DeltaBackend = new DeltaRemote()
+```
+
+
+after that, or while doing it adjust api to be more complete
+
+
+
+
+TODO different test to simulate two devices:
+
+to test autocrypt_initiate_key_transfer & autocrypt_continue_key_transfer

deltachat-jsonrpc/src/api/events.rs

@@ -0,0 +1,154 @@
+use deltachat::{Event, EventType};
+use serde::Serialize;
+use serde_json::{json, Value};
+use typescript_type_def::TypeDef;
+
+pub fn event_to_json_rpc_notification(event: Event) -> Value {
+    let (field1, field2): (Value, Value) = match &event.typ {
+        // events with a single string in field1
+        EventType::Info(txt)
+        | EventType::SmtpConnected(txt)
+        | EventType::ImapConnected(txt)
+        | EventType::SmtpMessageSent(txt)
+        | EventType::ImapMessageDeleted(txt)
+        | EventType::ImapMessageMoved(txt)
+        | EventType::NewBlobFile(txt)
+        | EventType::DeletedBlobFile(txt)
+        | EventType::Warning(txt)
+        | EventType::Error(txt)
+        | EventType::ErrorSelfNotInGroup(txt) => (json!(txt), Value::Null),
+        EventType::ImexFileWritten(path) => (json!(path.to_str()), Value::Null),
+        // single number
+        EventType::MsgsNoticed(chat_id) | EventType::ChatModified(chat_id) => {
+            (json!(chat_id), Value::Null)
+        }
+        EventType::ImexProgress(progress) => (json!(progress), Value::Null),
+        // both fields contain numbers
+        EventType::MsgsChanged { chat_id, msg_id }
+        | EventType::IncomingMsg { chat_id, msg_id }
+        | EventType::MsgDelivered { chat_id, msg_id }
+        | EventType::MsgFailed { chat_id, msg_id }
+        | EventType::MsgRead { chat_id, msg_id } => (json!(chat_id), json!(msg_id)),
+        EventType::ChatEphemeralTimerModified { chat_id, timer } => (json!(chat_id), json!(timer)),
+        EventType::SecurejoinInviterProgress {
+            contact_id,
+            progress,
+        }
+        | EventType::SecurejoinJoinerProgress {
+            contact_id,
+            progress,
+        } => (json!(contact_id), json!(progress)),
+        // field 1 number or null
+        EventType::ContactsChanged(maybe_number) | EventType::LocationChanged(maybe_number) => (
+            match maybe_number {
+                Some(number) => json!(number),
+                None => Value::Null,
+            },
+            Value::Null,
+        ),
+        // number and maybe string
+        EventType::ConfigureProgress { progress, comment } => (
+            json!(progress),
+            match comment {
+                Some(content) => json!(content),
+                None => Value::Null,
+            },
+        ),
+        EventType::ConnectivityChanged => (Value::Null, Value::Null),
+        EventType::SelfavatarChanged => (Value::Null, Value::Null),
+        EventType::WebxdcStatusUpdate {
+            msg_id,
+            status_update_serial,
+        } => (json!(msg_id), json!(status_update_serial)),
+    };
+
+    json!({
+        "id": event_type_to_string(event.typ),
+        "contextId": event.id,
+        "field1": field1,
+        "field2": field2
+    })
+}
+
+#[derive(Serialize, TypeDef)]
+pub enum EventTypeName {
+    Info,
+    SmtpConnected,
+    ImapConnected,
+    SmtpMessageSent,
+    ImapMessageDeleted,
+    ImapMessageMoved,
+    NewBlobFile,
+    DeletedBlobFile,
+    Warning,
+    Error,
+    ErrorSelfNotInGroup,
+    MsgsChanged,
+    IncomingMsg,
+    MsgsNoticed,
+    MsgDelivered,
+    MsgFailed,
+    MsgRead,
+    ChatModified,
+    ChatEphemeralTimerModified,
+    ContactsChanged,
+    LocationChanged,
+    ConfigureProgress,
+    ImexProgress,
+    ImexFileWritten,
+    SecurejoinInviterProgress,
+    SecurejoinJoinerProgress,
+    ConnectivityChanged,
+    SelfavatarChanged,
+    WebxdcStatusUpdate,
+}
+
+fn event_type_to_string(event: EventType) -> EventTypeName {
+    use EventTypeName::*;
+    match event {
+        EventType::Info(_) => Info,
+        EventType::SmtpConnected(_) => SmtpConnected,
+        EventType::ImapConnected(_) => ImapConnected,
+        EventType::SmtpMessageSent(_) => SmtpMessageSent,
+        EventType::ImapMessageDeleted(_) => ImapMessageDeleted,
+        EventType::ImapMessageMoved(_) => ImapMessageMoved,
+        EventType::NewBlobFile(_) => NewBlobFile,
+        EventType::DeletedBlobFile(_) => DeletedBlobFile,
+        EventType::Warning(_) => Warning,
+        EventType::Error(_) => Error,
+        EventType::ErrorSelfNotInGroup(_) => ErrorSelfNotInGroup,
+        EventType::MsgsChanged { .. } => MsgsChanged,
+        EventType::IncomingMsg { .. } => IncomingMsg,
+        EventType::MsgsNoticed(_) => MsgsNoticed,
+        EventType::MsgDelivered { .. } => MsgDelivered,
+        EventType::MsgFailed { .. } => MsgFailed,
+        EventType::MsgRead { .. } => MsgRead,
+        EventType::ChatModified(_) => ChatModified,
+        EventType::ChatEphemeralTimerModified { .. } => ChatEphemeralTimerModified,
+        EventType::ContactsChanged(_) => ContactsChanged,
+        EventType::LocationChanged(_) => LocationChanged,
+        EventType::ConfigureProgress { .. } => ConfigureProgress,
+        EventType::ImexProgress(_) => ImexProgress,
+        EventType::ImexFileWritten(_) => ImexFileWritten,
+        EventType::SecurejoinInviterProgress { .. } => SecurejoinInviterProgress,
+        EventType::SecurejoinJoinerProgress { .. } => SecurejoinJoinerProgress,
+        EventType::ConnectivityChanged => ConnectivityChanged,
+        EventType::SelfavatarChanged => SelfavatarChanged,
+        EventType::WebxdcStatusUpdate { .. } => WebxdcStatusUpdate,
+    }
+}
+
+#[cfg(test)]
+#[test]
+fn generate_events_ts_types_definition() {
+    let events = {
+        let mut buf = Vec::new();
+        let options = typescript_type_def::DefinitionFileOptions {
+            root_namespace: None,
+            ..typescript_type_def::DefinitionFileOptions::default()
+        };
+        typescript_type_def::write_definition_file::<_, EventTypeName>(&mut buf, options).unwrap();
+        String::from_utf8(buf).unwrap()
+    };
+    std::fs::write("typescript/generated/events.ts", events).unwrap();
+}

deltachat-jsonrpc/src/api/mod.rs

@@ -0,0 +1,536 @@
+use anyhow::{anyhow, bail, Context, Result};
+use async_std::sync::{Arc, RwLock};
+use deltachat::{
+    chat::{get_chat_msgs, ChatId},
+    chatlist::Chatlist,
+    config::Config,
+    contact::{may_be_valid_addr, Contact, ContactId},
+    context::get_info,
+    message::{Message, MsgId, Viewtype},
+    provider::get_provider_info,
+};
+use std::collections::BTreeMap;
+use std::{collections::HashMap, str::FromStr};
+use yerpc::rpc;
+
+pub use deltachat::accounts::Accounts;
+
+pub mod events;
+pub mod types;
+
+use crate::api::types::chat_list::{get_chat_list_item_by_id, ChatListItemFetchResult};
+
+use types::account::Account;
+use types::chat::FullChat;
+use types::chat_list::ChatListEntry;
+use types::contact::ContactObject;
+use types::message::MessageObject;
+use types::provider_info::ProviderInfo;
+
+#[derive(Clone, Debug)]
+pub struct CommandApi {
+    pub(crate) accounts: Arc<RwLock<Accounts>>,
+}
+
+impl CommandApi {
+    pub fn new(accounts: Accounts) -> Self {
+        CommandApi {
+            accounts: Arc::new(RwLock::new(accounts)),
+        }
+    }
+
+    pub fn new_from_arc(accounts: Arc<RwLock<Accounts>>) -> Self {
+        CommandApi { accounts }
+    }
+
+    async fn get_context(&self, id: u32) -> Result<deltachat::context::Context> {
+        let sc = self
+            .accounts
+            .read()
+            .await
+            .get_account(id)
+            .await
+            .ok_or_else(|| anyhow!("account with id {} not found", id))?;
+        Ok(sc)
+    }
+}
+
+#[rpc(all_positional, ts_outdir = "typescript/generated")]
+impl CommandApi {
+    // ---------------------------------------------
+    //  Misc top level functions
+    // ---------------------------------------------
+
+    /// Check if an email address is valid.
+    async fn check_email_validity(&self, email: String) -> bool {
+        may_be_valid_addr(&email)
+    }
+
+    /// Get general system info.
+    async fn get_system_info(&self) -> BTreeMap<&'static str, String> {
+        get_info()
+    }
+
+    // ---------------------------------------------
+    // Account Management
+    // ---------------------------------------------
+
+    async fn add_account(&self) -> Result<u32> {
+        self.accounts.write().await.add_account().await
+    }
+
+    async fn remove_account(&self, account_id: u32) -> Result<()> {
+        self.accounts.write().await.remove_account(account_id).await
+    }
+
+    async fn get_all_account_ids(&self) -> Vec<u32> {
+        self.accounts.read().await.get_all().await
+    }
+
+    /// Select account id for internally selected state.
+    /// TODO: Likely this is deprecated as all methods take an account id now.
+    async fn select_account(&self, id: u32) -> Result<()> {
+        self.accounts.write().await.select_account(id).await
+    }
+
+    /// Get the selected account id of the internal state..
+    /// TODO: Likely this is deprecated as all methods take an account id now.
+    async fn get_selected_account_id(&self) -> Option<u32> {
+        self.accounts.read().await.get_selected_account_id().await
+    }
+
+    /// Get a list of all configured accounts.
+    async fn get_all_accounts(&self) -> Result<Vec<Account>> {
+        let mut accounts = Vec::new();
+        for id in self.accounts.read().await.get_all().await {
+            let context_option = self.accounts.read().await.get_account(id).await;
+            if let Some(ctx) = context_option {
+                accounts.push(Account::from_context(&ctx, id).await?)
+            } else {
+                println!("account with id {} doesn't exist anymore", id);
+            }
+        }
+        Ok(accounts)
+    }
+
+    // ---------------------------------------------
+    // Methods that work on individual accounts
+    // ---------------------------------------------
+
+    /// Get top-level info for an account.
+    async fn get_account_info(&self, account_id: u32) -> Result<Account> {
+        let context_option = self.accounts.read().await.get_account(account_id).await;
+        if let Some(ctx) = context_option {
+            Ok(Account::from_context(&ctx, account_id).await?)
+        } else {
+            Err(anyhow!(
+                "account with id {} doesn't exist anymore",
+                account_id
+            ))
+        }
+    }
+
+    /// Returns provider for the given domain.
+    ///
+    /// This function looks up domain in offline database.
+    ///
+    /// For compatibility, email address can be passed to this function
+    /// instead of the domain.
+    async fn get_provider_info(
+        &self,
+        account_id: u32,
+        email: String,
+    ) -> Result<Option<ProviderInfo>> {
+        let ctx = self.get_context(account_id).await?;
+
+        let socks5_enabled = ctx
+            .get_config_bool(deltachat::config::Config::Socks5Enabled)
+            .await?;
+
+        let provider_info =
+            get_provider_info(&ctx, email.split('@').last().unwrap_or(""), socks5_enabled).await;
+        Ok(ProviderInfo::from_dc_type(provider_info))
+    }
+
+    /// Checks if the context is already configured.
+    async fn is_configured(&self, account_id: u32) -> Result<bool> {
+        let ctx = self.get_context(account_id).await?;
+        ctx.is_configured().await
+    }
+
+    /// Get system info for an account.
+    async fn get_info(&self, account_id: u32) -> Result<BTreeMap<&'static str, String>> {
+        let ctx = self.get_context(account_id).await?;
+        ctx.get_info().await
+    }
+
+    async fn set_config(&self, account_id: u32, key: String, value: Option<String>) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        set_config(&ctx, &key, value.as_deref()).await
+    }
+
+    async fn batch_set_config(
+        &self,
+        account_id: u32,
+        config: HashMap<String, Option<String>>,
+    ) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        for (key, value) in config.into_iter() {
+            set_config(&ctx, &key, value.as_deref())
+                .await
+                .with_context(|| format!("Can't set {} to {:?}", key, value))?;
+        }
+        Ok(())
+    }
+
+    async fn get_config(&self, account_id: u32, key: String) -> Result<Option<String>> {
+        let ctx = self.get_context(account_id).await?;
+        get_config(&ctx, &key).await
+    }
+
+    async fn batch_get_config(
+        &self,
+        account_id: u32,
+        keys: Vec<String>,
+    ) -> Result<HashMap<String, Option<String>>> {
+        let ctx = self.get_context(account_id).await?;
+        let mut result: HashMap<String, Option<String>> = HashMap::new();
+        for key in keys {
+            result.insert(key.clone(), get_config(&ctx, &key).await?);
+        }
+        Ok(result)
+    }
+
+    /// Configures this account with the currently set parameters.
+    /// Setup the credential config before calling this.
+    async fn configure(&self, account_id: u32) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        ctx.stop_io().await;
+        ctx.configure().await?;
+        ctx.start_io().await;
+        Ok(())
+    }
+
+    /// Signal an ongoing process to stop.
+    async fn stop_ongoing_process(&self, account_id: u32) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        ctx.stop_ongoing().await;
+        Ok(())
+    }
+
+    // ---------------------------------------------
+    //  autocrypt
+    // ---------------------------------------------
+
+    async fn autocrypt_initiate_key_transfer(&self, account_id: u32) -> Result<String> {
+        let ctx = self.get_context(account_id).await?;
+        deltachat::imex::initiate_key_transfer(&ctx).await
+    }
+
+    async fn autocrypt_continue_key_transfer(
+        &self,
+        account_id: u32,
+        message_id: u32,
+        setup_code: String,
+    ) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        deltachat::imex::continue_key_transfer(&ctx, MsgId::new(message_id), &setup_code).await
+    }
+
+    // ---------------------------------------------
+    //   chat list
+    // ---------------------------------------------
+
+    async fn get_chatlist_entries(
+        &self,
+        account_id: u32,
+        list_flags: Option<u32>,
+        query_string: Option<String>,
+        query_contact_id: Option<u32>,
+    ) -> Result<Vec<ChatListEntry>> {
+        let ctx = self.get_context(account_id).await?;
+        let list = Chatlist::try_load(
+            &ctx,
+            list_flags.unwrap_or(0) as usize,
+            query_string.as_deref(),
+            query_contact_id.map(ContactId::new),
+        )
+        .await?;
+        let mut l: Vec<ChatListEntry> = Vec::new();
+        for i in 0..list.len() {
+            l.push(ChatListEntry(
+                list.get_chat_id(i)?.to_u32(),
+                list.get_msg_id(i)?.unwrap_or_default().to_u32(),
+            ));
+        }
+        Ok(l)
+    }
+
+    async fn get_chatlist_items_by_entries(
+        &self,
+        account_id: u32,
+        entries: Vec<ChatListEntry>,
+    ) -> Result<HashMap<u32, ChatListItemFetchResult>> {
+        // todo custom json deserializer for ChatListEntry?
+        let ctx = self.get_context(account_id).await?;
+        let mut result: HashMap<u32, ChatListItemFetchResult> = HashMap::new();
+        for (_i, entry) in entries.iter().enumerate() {
+            result.insert(
+                entry.0,
+                match get_chat_list_item_by_id(&ctx, entry).await {
+                    Ok(res) => res,
+                    Err(err) => ChatListItemFetchResult::Error {
+                        id: entry.0,
+                        error: format!("{:?}", err),
+                    },
+                },
+            );
+        }
+        Ok(result)
+    }
+
+    // ---------------------------------------------
+    //  chat
+    // ---------------------------------------------
+
+    async fn chatlist_get_full_chat_by_id(
+        &self,
+        account_id: u32,
+        chat_id: u32,
+    ) -> Result<FullChat> {
+        let ctx = self.get_context(account_id).await?;
+        FullChat::from_dc_chat_id(&ctx, chat_id).await
+    }
+
+    async fn accept_chat(&self, account_id: u32, chat_id: u32) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        ChatId::new(chat_id).accept(&ctx).await
+    }
+
+    async fn block_chat(&self, account_id: u32, chat_id: u32) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        ChatId::new(chat_id).block(&ctx).await
+    }
+
+    // ---------------------------------------------
+    // message list
+    // ---------------------------------------------
+
+    async fn message_list_get_message_ids(
+        &self,
+        account_id: u32,
+        chat_id: u32,
+        flags: u32,
+    ) -> Result<Vec<u32>> {
+        let ctx = self.get_context(account_id).await?;
+        let msg = get_chat_msgs(&ctx, ChatId::new(chat_id), flags).await?;
+        Ok(msg
+            .iter()
+            .filter_map(|chat_item| match chat_item {
+                deltachat::chat::ChatItem::Message { msg_id } => Some(msg_id.to_u32()),
+                _ => None,
+            })
+            .collect())
+    }
+
+    async fn message_get_message(&self, account_id: u32, message_id: u32) -> Result<MessageObject> {
+        let ctx = self.get_context(account_id).await?;
+        MessageObject::from_message_id(&ctx, message_id).await
+    }
+
+    async fn message_get_messages(
+        &self,
+        account_id: u32,
+        message_ids: Vec<u32>,
+    ) -> Result<HashMap<u32, MessageObject>> {
+        let ctx = self.get_context(account_id).await?;
+        let mut messages: HashMap<u32, MessageObject> = HashMap::new();
+        for message_id in message_ids {
+            messages.insert(
+                message_id,
+                MessageObject::from_message_id(&ctx, message_id).await?,
+            );
+        }
+        Ok(messages)
+    }
+
+    // ---------------------------------------------
+    //  contact
+    // ---------------------------------------------
+
+    /// Get a single contact options by ID.
+    async fn contacts_get_contact(
+        &self,
+        account_id: u32,
+        contact_id: u32,
+    ) -> Result<ContactObject> {
+        let ctx = self.get_context(account_id).await?;
+        let contact_id = ContactId::new(contact_id);
+
+        ContactObject::from_dc_contact(
+            &ctx,
+            deltachat::contact::Contact::get_by_id(&ctx, contact_id).await?,
+        )
+        .await
+    }
+
+    /// Add a single contact as a result of an explicit user action.
+    ///
+    /// Returns contact id of the created or existing contact
+    async fn contacts_create_contact(
+        &self,
+        account_id: u32,
+        email: String,
+        name: Option<String>,
+    ) -> Result<u32> {
+        let ctx = self.get_context(account_id).await?;
+        if !may_be_valid_addr(&email) {
+            bail!(anyhow!(
+                "provided email address is not a valid email address"
+            ))
+        }
+        let contact_id = Contact::create(&ctx, &name.unwrap_or_default(), &email).await?;
+        Ok(contact_id.to_u32())
+    }
+
+    /// Returns contact id of the created or existing DM chat with that contact
+    async fn contacts_create_chat_by_contact_id(
+        &self,
+        account_id: u32,
+        contact_id: u32,
+    ) -> Result<u32> {
+        let ctx = self.get_context(account_id).await?;
+        let contact = Contact::get_by_id(&ctx, ContactId::new(contact_id)).await?;
+        ChatId::create_for_contact(&ctx, contact.id)
+            .await
+            .map(|id| id.to_u32())
+    }
+
+    async fn contacts_block(&self, account_id: u32, contact_id: u32) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        Contact::block(&ctx, ContactId::new(contact_id)).await
+    }
+
+    async fn contacts_unblock(&self, account_id: u32, contact_id: u32) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        Contact::unblock(&ctx, ContactId::new(contact_id)).await
+    }
+
+    async fn contacts_get_blocked(&self, account_id: u32) -> Result<Vec<ContactObject>> {
+        let ctx = self.get_context(account_id).await?;
+        let blocked_ids = Contact::get_all_blocked(&ctx).await?;
+        let mut contacts: Vec<ContactObject> = Vec::with_capacity(blocked_ids.len());
+        for id in blocked_ids {
+            contacts.push(
+                ContactObject::from_dc_contact(
+                    &ctx,
+                    deltachat::contact::Contact::get_by_id(&ctx, id).await?,
+                )
+                .await?,
+            );
+        }
+        Ok(contacts)
+    }
+
+    async fn contacts_get_contact_ids(
+        &self,
+        account_id: u32,
+        list_flags: u32,
+        query: Option<String>,
+    ) -> Result<Vec<u32>> {
+        let ctx = self.get_context(account_id).await?;
+        let contacts = Contact::get_all(&ctx, list_flags, query.as_deref()).await?;
+        Ok(contacts.into_iter().map(|c| c.to_u32()).collect())
+    }
+
+    /// Get a list of contacts.
+    /// (formerly called getContacts2 in desktop)
+    async fn contacts_get_contacts(
+        &self,
+        account_id: u32,
+        list_flags: u32,
+        query: Option<String>,
+    ) -> Result<Vec<ContactObject>> {
+        let ctx = self.get_context(account_id).await?;
+        let contact_ids = Contact::get_all(&ctx, list_flags, query.as_deref()).await?;
+        let mut contacts: Vec<ContactObject> = Vec::with_capacity(contact_ids.len());
+        for id in contact_ids {
+            contacts.push(
+                ContactObject::from_dc_contact(
+                    &ctx,
+                    deltachat::contact::Contact::get_by_id(&ctx, id).await?,
+                )
+                .await?,
+            );
+        }
+        Ok(contacts)
+    }
+
+    async fn contacts_get_contacts_by_ids(
+        &self,
+        account_id: u32,
+        ids: Vec<u32>,
+    ) -> Result<HashMap<u32, ContactObject>> {
+        let ctx = self.get_context(account_id).await?;
+
+        let mut contacts = HashMap::with_capacity(ids.len());
+        for id in ids {
+            contacts.insert(
+                id,
+                ContactObject::from_dc_contact(
+                    &ctx,
+                    deltachat::contact::Contact::get_by_id(&ctx, ContactId::new(id)).await?,
+                )
+                .await?,
+            );
+        }
+        Ok(contacts)
+    }
+
+    // ---------------------------------------------
+    //           misc prototyping functions
+    //       that might get removed later again
+    // ---------------------------------------------
+
+    /// Returns the messageid of the sent message
+    async fn misc_send_text_message(
+        &self,
+        account_id: u32,
+        text: String,
+        chat_id: u32,
+    ) -> Result<u32> {
+        let ctx = self.get_context(account_id).await?;
+
+        let mut msg = Message::new(Viewtype::Text);
+        msg.set_text(Some(text));
+
+        let message_id = deltachat::chat::send_msg(&ctx, ChatId::new(chat_id), &mut msg).await?;
+        Ok(message_id.to_u32())
+    }
+}
+
+// Helper functions (to prevent code duplication)
+async fn set_config(
+    ctx: &deltachat::context::Context,
+    key: &str,
+    value: Option<&str>,
+) -> Result<(), anyhow::Error> {
+    if key.starts_with("ui.") {
+        ctx.set_ui_config(key, value).await
+    } else {
+        ctx.set_config(Config::from_str(key).context("unknown key")?, value)
+            .await
+    }
+}
+
+async fn get_config(
+    ctx: &deltachat::context::Context,
+    key: &str,
+) -> Result<Option<String>, anyhow::Error> {
+    if key.starts_with("ui.") {
+        ctx.get_ui_config(key).await
+    } else {
+        ctx.get_config(Config::from_str(key).context("unknown key")?)
+            .await
+    }
+}

deltachat-jsonrpc/src/api/types/account.rs

@@ -0,0 +1,46 @@
+use anyhow::Result;
+use deltachat::config::Config;
+use deltachat::contact::{Contact, ContactId};
+use serde::Serialize;
+use typescript_type_def::TypeDef;
+
+use super::color_int_to_hex_string;
+
+#[derive(Serialize, TypeDef)]
+#[serde(tag = "type")]
+pub enum Account {
+    //#[serde(rename_all = "camelCase")]
+    Configured {
+        id: u32,
+        display_name: Option<String>,
+        addr: Option<String>,
+        // size: u32,
+        profile_image: Option<String>, // TODO: This needs to be converted to work with blob http server.
+        color: String,
+    },
+    Unconfigured {
+        id: u32,
+    },
+}
+
+impl Account {
+    pub async fn from_context(ctx: &deltachat::context::Context, id: u32) -> Result<Self> {
+        if ctx.is_configured().await? {
+            let display_name = ctx.get_config(Config::Displayname).await?;
+            let addr = ctx.get_config(Config::Addr).await?;
+            let profile_image = ctx.get_config(Config::Selfavatar).await?;
+            let color = color_int_to_hex_string(
+                Contact::get_by_id(ctx, ContactId::SELF).await?.get_color(),
+            );
+            Ok(Account::Configured {
+                id,
+                display_name,
+                addr,
+                profile_image,
+                color,
+            })
+        } else {
+            Ok(Account::Unconfigured { id })
+        }
+    }
+}

deltachat-jsonrpc/src/api/types/chat.rs

@@ -0,0 +1,91 @@
+use anyhow::{anyhow, Result};
+use deltachat::chat::get_chat_contacts;
+use deltachat::chat::{Chat, ChatId};
+use deltachat::contact::{Contact, ContactId};
+use deltachat::context::Context;
+use num_traits::cast::ToPrimitive;
+use serde::Serialize;
+use typescript_type_def::TypeDef;
+
+use super::color_int_to_hex_string;
+use super::contact::ContactObject;
+
+#[derive(Serialize, TypeDef)]
+pub struct FullChat {
+    id: u32,
+    name: String,
+    is_protected: bool,
+    profile_image: Option<String>, //BLOBS ?
+    archived: bool,
+    // subtitle  - will be moved to frontend because it uses translation functions
+    chat_type: u32,
+    is_unpromoted: bool,
+    is_self_talk: bool,
+    contacts: Vec<ContactObject>,
+    contact_ids: Vec<u32>,
+    color: String,
+    fresh_message_counter: usize,
+    // is_group - please check over chat.type in frontend instead
+    is_contact_request: bool,
+    is_device_chat: bool,
+    self_in_group: bool,
+    is_muted: bool,
+    ephemeral_timer: u32, //TODO look if there are more important properties in newer core versions
+    can_send: bool,
+}
+
+impl FullChat {
+    pub async fn from_dc_chat_id(context: &Context, chat_id: u32) -> Result<Self> {
+        let rust_chat_id = ChatId::new(chat_id);
+        let chat = Chat::load_from_db(context, rust_chat_id).await?;
+
+        let contact_ids = get_chat_contacts(context, rust_chat_id).await?;
+
+        let mut contacts = Vec::new();
+
+        for contact_id in &contact_ids {
+            contacts.push(
+                ContactObject::from_dc_contact(
+                    context,
+                    Contact::load_from_db(context, *contact_id).await?,
+                )
+                .await?,
+            )
+        }
+
+        let profile_image = match chat.get_profile_image(context).await? {
+            Some(path_buf) => path_buf.to_str().map(|s| s.to_owned()),
+            None => None,
+        };
+
+        let color = color_int_to_hex_string(chat.get_color(context).await?);
+        let fresh_message_counter = rust_chat_id.get_fresh_msg_cnt(context).await?;
+        let ephemeral_timer = rust_chat_id.get_ephemeral_timer(context).await?.to_u32();
+
+        let can_send = chat.can_send(context).await?;
+
+        Ok(FullChat {
+            id: chat_id,
+            name: chat.name.clone(),
+            is_protected: chat.is_protected(),
+            profile_image, //BLOBS ?
+            archived: chat.get_visibility() == deltachat::chat::ChatVisibility::Archived,
+            chat_type: chat
+                .get_type()
+                .to_u32()
+                .ok_or_else(|| anyhow!("unknown chat type id"))?, // TODO get rid of this unwrap?
+            is_unpromoted: chat.is_unpromoted(),
+            is_self_talk: chat.is_self_talk(),
+            contacts,
+            contact_ids: contact_ids.iter().map(|id| id.to_u32()).collect(),
+            color,
+            fresh_message_counter,
+            is_contact_request: chat.is_contact_request(),
+            is_device_chat: chat.is_device_talk(),
+            self_in_group: contact_ids.contains(&ContactId::SELF),
+            is_muted: chat.is_muted(),
+            ephemeral_timer,
+            can_send,
+        })
+    }
+}

deltachat-jsonrpc/src/api/types/chat_list.rs

@@ -0,0 +1,117 @@
+use anyhow::Result;
+use deltachat::constants::*;
+use deltachat::contact::ContactId;
+use deltachat::{
+    chat::{get_chat_contacts, ChatVisibility},
+    chatlist::Chatlist,
+};
+use deltachat::{
+    chat::{Chat, ChatId},
+    message::MsgId,
+};
+use num_traits::cast::ToPrimitive;
+use serde::{Deserialize, Serialize};
+use typescript_type_def::TypeDef;
+
+use super::color_int_to_hex_string;
+
+#[derive(Deserialize, Serialize, TypeDef)]
+pub struct ChatListEntry(pub u32, pub u32);
+
+#[derive(Serialize, TypeDef)]
+#[serde(tag = "type")]
+pub enum ChatListItemFetchResult {
+    #[serde(rename_all = "camelCase")]
+    ChatListItem {
+        id: u32,
+        name: String,
+        avatar_path: Option<String>,
+        color: String,
+        last_updated: Option<i64>,
+        summary_text1: String,
+        summary_text2: String,
+        summary_status: u32,
+        is_protected: bool,
+        is_group: bool,
+        fresh_message_counter: usize,
+        is_self_talk: bool,
+        is_device_talk: bool,
+        is_sending_location: bool,
+        is_self_in_group: bool,
+        is_archived: bool,
+        is_pinned: bool,
+        is_muted: bool,
+        is_contact_request: bool,
+    },
+    ArchiveLink,
+    #[serde(rename_all = "camelCase")]
+    Error {
+        id: u32,
+        error: String,
+    },
+}
+
+pub(crate) async fn get_chat_list_item_by_id(
+    ctx: &deltachat::context::Context,
+    entry: &ChatListEntry,
+) -> Result<ChatListItemFetchResult> {
+    let chat_id = ChatId::new(entry.0);
+    let last_msgid = match entry.1 {
+        0 => None,
+        _ => Some(MsgId::new(entry.1)),
+    };
+
+    if chat_id.is_archived_link() {
+        return Ok(ChatListItemFetchResult::ArchiveLink);
+    }
+
+    let chat = Chat::load_from_db(ctx, chat_id).await?;
+    let summary = Chatlist::get_summary2(ctx, chat_id, last_msgid, Some(&chat)).await?;
+
+    let summary_text1 = summary.prefix.map_or_else(String::new, |s| s.to_string());
+    let summary_text2 = summary.text.to_owned();
+
+    let visibility = chat.get_visibility();
+
+    let avatar_path = chat
+        .get_profile_image(ctx)
+        .await?
+        .map(|path| path.to_str().unwrap_or("invalid/path").to_owned());
+
+    let last_updated = match last_msgid {
+        Some(id) => {
+            let last_message = deltachat::message::Message::load_from_db(ctx, id).await?;
+            Some(last_message.get_timestamp() * 1000)
+        }
+        None => None,
+    };
+
+    let self_in_group = get_chat_contacts(ctx, chat_id)
+        .await?
+        .contains(&ContactId::SELF);
+
+    let fresh_message_counter = chat_id.get_fresh_msg_cnt(ctx).await?;
+    let color = color_int_to_hex_string(chat.get_color(ctx).await?);
+
+    Ok(ChatListItemFetchResult::ChatListItem {
+        id: chat_id.to_u32(),
+        name: chat.get_name().to_owned(),
+        avatar_path,
+        color,
+        last_updated,
+        summary_text1,
+        summary_text2,
+        summary_status: summary.state.to_u32().expect("impossible"), // idea and a function to transform the constant to strings? or return string enum
+        is_protected: chat.is_protected(),
+        is_group: chat.get_type() == Chattype::Group,
+        fresh_message_counter,
+        is_self_talk: chat.is_self_talk(),
+        is_device_talk: chat.is_device_talk(),
+        is_self_in_group: self_in_group,
+        is_sending_location: chat.is_sending_locations(),
+        is_archived: visibility == ChatVisibility::Archived,
+        is_pinned: visibility == ChatVisibility::Pinned,
+        is_muted: chat.is_muted(),
+        is_contact_request: chat.is_contact_request(),
+    })
+}

deltachat-jsonrpc/src/api/types/contact.rs

@@ -0,0 +1,50 @@
+use anyhow::Result;
+use deltachat::contact::VerifiedStatus;
+use deltachat::context::Context;
+use serde::Serialize;
+use typescript_type_def::TypeDef;
+
+use super::color_int_to_hex_string;
+
+#[derive(Serialize, TypeDef)]
+#[serde(rename = "Contact")]
+pub struct ContactObject {
+    address: String,
+    color: String,
+    auth_name: String,
+    status: String,
+    display_name: String,
+    id: u32,
+    name: String,
+    profile_image: Option<String>, // BLOBS
+    name_and_addr: String,
+    is_blocked: bool,
+    is_verified: bool,
+}
+
+impl ContactObject {
+    pub async fn from_dc_contact(
+        context: &Context,
+        contact: deltachat::contact::Contact,
+    ) -> Result<Self> {
+        let profile_image = match contact.get_profile_image(context).await? {
+            Some(path_buf) => path_buf.to_str().map(|s| s.to_owned()),
+            None => None,
+        };
+        let is_verified = contact.is_verified(context).await? == VerifiedStatus::BidirectVerified;
+
+        Ok(ContactObject {
+            address: contact.get_addr().to_owned(),
+            color: color_int_to_hex_string(contact.get_color()),
+            auth_name: contact.get_authname().to_owned(),
+            status: contact.get_status().to_owned(),
+            display_name: contact.get_display_name().to_owned(),
+            id: contact.id.to_u32(),
+            name: contact.get_name().to_owned(),
+            profile_image, //BLOBS
+            name_and_addr: contact.get_name_n_addr(),
+            is_blocked: contact.is_blocked(),
+            is_verified,
+        })
+    }
+}

deltachat-jsonrpc/src/api/types/message.rs

@@ -0,0 +1,127 @@
+use anyhow::{anyhow, Result};
+use deltachat::contact::Contact;
+use deltachat::context::Context;
+use deltachat::message::Message;
+use deltachat::message::MsgId;
+use num_traits::cast::ToPrimitive;
+use serde::Serialize;
+use typescript_type_def::TypeDef;
+
+use super::contact::ContactObject;
+
+#[derive(Serialize, TypeDef)]
+#[serde(rename = "Message")]
+pub struct MessageObject {
+    id: u32,
+    chat_id: u32,
+    from_id: u32,
+    quoted_text: Option<String>,
+    quoted_message_id: Option<u32>,
+    text: Option<String>,
+    has_location: bool,
+    has_html: bool,
+    view_type: u32,
+    state: u32,
+
+    timestamp: i64,
+    sort_timestamp: i64,
+    received_timestamp: i64,
+    has_deviating_timestamp: bool,
+
+    // summary - use/create another function if you need it
+    subject: String,
+    show_padlock: bool,
+    is_setupmessage: bool,
+    is_info: bool,
+    is_forwarded: bool,
+
+    duration: i32,
+    dimensions_height: i32,
+    dimensions_width: i32,
+
+    videochat_type: Option<u32>,
+    videochat_url: Option<String>,
+
+    override_sender_name: Option<String>,
+    sender: ContactObject,
+
+    setup_code_begin: Option<String>,
+
+    file: Option<String>,
+    file_mime: Option<String>,
+    file_bytes: u64,
+    file_name: Option<String>,
+}
+
+impl MessageObject {
+    pub async fn from_message_id(context: &Context, message_id: u32) -> Result<Self> {
+        let msg_id = MsgId::new(message_id);
+        let message = Message::load_from_db(context, msg_id).await?;
+
+        let quoted_message_id = message
+            .quoted_message(context)
+            .await?
+            .map(|m| m.get_id().to_u32());
+
+        let sender_contact = Contact::load_from_db(context, message.get_from_id()).await?;
+        let sender = ContactObject::from_dc_contact(context, sender_contact).await?;
+        let file_bytes = message.get_filebytes(context).await;
+        let override_sender_name = message.get_override_sender_name();
+
+        Ok(MessageObject {
+            id: message_id,
+            chat_id: message.get_chat_id().to_u32(),
+            from_id: message.get_from_id().to_u32(),
+            quoted_text: message.quoted_text(),
+            quoted_message_id,
+            text: message.get_text(),
+            has_location: message.has_location(),
+            has_html: message.has_html(),
+            view_type: message
+                .get_viewtype()
+                .to_u32()
+                .ok_or_else(|| anyhow!("viewtype conversion to number failed"))?,
+            state: message
+                .get_state()
+                .to_u32()
+                .ok_or_else(|| anyhow!("state conversion to number failed"))?,
+
+            timestamp: message.get_timestamp(),
+            sort_timestamp: message.get_sort_timestamp(),
+            received_timestamp: message.get_received_timestamp(),
+            has_deviating_timestamp: message.has_deviating_timestamp(),
+
+            subject: message.get_subject().to_owned(),
+            show_padlock: message.get_showpadlock(),
+            is_setupmessage: message.is_setupmessage(),
+            is_info: message.is_info(),
+            is_forwarded: message.is_forwarded(),
+
+            duration: message.get_duration(),
+            dimensions_height: message.get_height(),
+            dimensions_width: message.get_width(),
+
+            videochat_type: match message.get_videochat_type() {
+                Some(vct) => Some(
+                    vct.to_u32()
+                        .ok_or_else(|| anyhow!("state conversion to number failed"))?,
+                ),
+                None => None,
+            },
+            videochat_url: message.get_videochat_url(),
+
+            override_sender_name,
+            sender,
+
+            setup_code_begin: message.get_setupcodebegin(context).await,
+
+            file: match message.get_file(context) {
+                Some(path_buf) => path_buf.to_str().map(|s| s.to_owned()),
+                None => None,
+            }, //BLOBS
+            file_mime: message.get_filemime(),
+            file_bytes,
+            file_name: message.get_filename(),
+        })
+    }
+}

deltachat-jsonrpc/src/api/types/mod.rs

@@ -0,0 +1,10 @@
+pub mod account;
+pub mod chat;
+pub mod chat_list;
+pub mod contact;
+pub mod message;
+pub mod provider_info;
+
+pub fn color_int_to_hex_string(color: u32) -> String {
+    format!("{:#08x}", color).replace("0x", "#")
+}

deltachat-jsonrpc/src/api/types/provider_info.rs

@@ -0,0 +1,21 @@
+use deltachat::provider::Provider;
+use num_traits::cast::ToPrimitive;
+use serde::Serialize;
+use typescript_type_def::TypeDef;
+
+#[derive(Serialize, TypeDef)]
+pub struct ProviderInfo {
+    pub before_login_hint: String,
+    pub overview_page: String,
+    pub status: u32, // in reality this is an enum, but for simlicity and because it gets converted into a number anyway, we use an u32 here.
+}
+
+impl ProviderInfo {
+    pub fn from_dc_type(provider: Option<&Provider>) -> Option<Self> {
+        provider.map(|p| ProviderInfo {
+            before_login_hint: p.before_login_hint.to_owned(),
+            overview_page: p.overview_page.to_owned(),
+            status: p.status.to_u32().unwrap(),
+        })
+    }
+}

deltachat-jsonrpc/src/lib.rs

@@ -0,0 +1,60 @@
+pub mod api;
+pub use api::events;
+
+pub use yerpc;
+
+#[cfg(test)]
+mod tests {
+    use super::api::{Accounts, CommandApi};
+    use async_channel::unbounded;
+    use async_std::task;
+    use futures::StreamExt;
+    use tempfile::TempDir;
+    use yerpc::{MessageHandle, RpcHandle};
+
+    #[async_std::test]
+    async fn basic_json_rpc_functionality() -> anyhow::Result<()> {
+        // println!("{}", "");
+        let tmp_dir = TempDir::new().unwrap().path().into();
+        println!("tmp_dir: {:?}", tmp_dir);
+
+        let accounts = Accounts::new(tmp_dir).await?;
+        let cmd_api = CommandApi::new(accounts);
+
+        let (sender, mut receiver) = unbounded::<String>();
+
+        let (request_handle, mut rx) = RpcHandle::new();
+        let session = cmd_api;
+        let handle = MessageHandle::new(request_handle, session);
+        task::spawn({
+            async move {
+                while let Some(message) = rx.next().await {
+                    let message = serde_json::to_string(&message)?;
+                    // Abort serialization on error.
+                    sender.send(message).await?;
+                }
+                let res: Result<(), anyhow::Error> = Ok(());
+                res
+            }
+        });
+
+        {
+            let request = r#"{"jsonrpc":"2.0","method":"add_account","params":[],"id":1}"#;
+            let response = r#"{"jsonrpc":"2.0","id":1,"result":1}"#;
+            handle.handle_message(request).await;
+            let result = receiver.next().await;
+            println!("{:?}", result);
+            assert_eq!(result, Some(response.to_owned()));
+        }
+        {
+            let request = r#"{"jsonrpc":"2.0","method":"get_all_account_ids","params":[],"id":2}"#;
+            let response = r#"{"jsonrpc":"2.0","id":2,"result":[1]}"#;
+            handle.handle_message(request).await;
+            let result = receiver.next().await;
+            println!("{:?}", result);
+            assert_eq!(result, Some(response.to_owned()));
+        }
+
+        Ok(())
+    }
+}

deltachat-jsonrpc/src/webserver.rs

@@ -0,0 +1,44 @@
+use async_std::path::PathBuf;
+use async_std::task;
+use tide::Request;
+use yerpc::RpcHandle;
+use yerpc_tide::yerpc_handler;
+
+mod api;
+use api::events::event_to_json_rpc_notification;
+use api::{Accounts, CommandApi};
+
+#[async_std::main]
+async fn main() -> Result<(), std::io::Error> {
+    env_logger::init();
+    log::info!("Starting");
+
+    let accounts = Accounts::new(PathBuf::from("./accounts")).await.unwrap();
+    let state = CommandApi::new(accounts);
+
+    let mut app = tide::with_state(state.clone());
+    app.at("/ws").get(yerpc_handler(request_handler));
+
+    state.accounts.read().await.start_io().await;
+    app.listen("127.0.0.1:20808").await?;
+
+    Ok(())
+}
+async fn request_handler(
+    request: Request<CommandApi>,
+    rpc: RpcHandle,
+) -> anyhow::Result<CommandApi> {
+    let state = request.state().clone();
+    task::spawn(event_loop(state.clone(), rpc));
+    Ok(state)
+}
+
+async fn event_loop(state: CommandApi, rpc: RpcHandle) -> anyhow::Result<()> {
+    let events = state.accounts.read().await.get_event_emitter().await;
+    while let Some(event) = events.recv().await {
+        // log::debug!("event {:?}", event);
+        let event = event_to_json_rpc_notification(event);
+        rpc.notify("event", Some(event)).await?;
+    }
+    Ok(())
+}

deltachat-jsonrpc/typescript/.gitignore

@@ -0,0 +1,6 @@
+node_modules
+dist
+test_dist
+coverage
+yarn.lock
+package-lock.json

deltachat-jsonrpc/typescript/.prettierignore

@@ -0,0 +1,3 @@
+coverage
+dist
+generated

deltachat-jsonrpc/typescript/deltachat.ts

@@ -0,0 +1 @@
+export * from "./src/lib.js";

deltachat-jsonrpc/typescript/example.ts

@@ -0,0 +1,107 @@
+import { RawClient, RPC } from "./src/lib";
+import { WebsocketTransport, Request } from "yerpc";
+
+type DeltaEvent = { id: string; contextId: number; field1: any; field2: any };
+var selectedAccount = 0;
+window.addEventListener("DOMContentLoaded", (_event) => {
+  (window as any).selectDeltaAccount = (id: string) => {
+    selectedAccount = Number(id);
+    window.dispatchEvent(new Event("account-changed"));
+  };
+  run().catch((err) => console.error("run failed", err));
+});
+
+async function run() {
+  const $main = document.getElementById("main")!;
+  const $side = document.getElementById("side")!;
+  const $head = document.getElementById("header")!;
+
+  const transport = new WebsocketTransport("ws://localhost:20808/ws");
+  const client = new RawClient(transport);
+
+  (window as any).client = client;
+
+  transport.on("request", (request: Request) => {
+    const method = request.method;
+    if (method === "event") {
+      const params = request.params! as DeltaEvent;
+      onIncomingEvent(params, params.id);
+    }
+  });
+
+  window.addEventListener("account-changed", async (_event: Event) => {
+    await client.selectAccount(selectedAccount);
+    listChatsForSelectedAccount();
+  });
+
+  await Promise.all([loadAccountsInHeader(), listChatsForSelectedAccount()]);
+
+  async function loadAccountsInHeader() {
+    const accounts = await client.getAllAccounts();
+    for (const account of accounts) {
+      if (account.type === "Configured") {
+        write(
+          $head,
+          `<a href="#" onclick="selectDeltaAccount(${account.id})">
+          ${account.addr!}
+          </a>&nbsp;`
+        );
+      }
+    }
+  }
+
+  async function listChatsForSelectedAccount() {
+    clear($main);
+    const selectedAccount = await client.getSelectedAccountId();
+    if (!selectedAccount) return write($main, "No account selected");
+    const info = await client.getAccountInfo(selectedAccount);
+    if (info.type !== "Configured") {
+      return write($main, "Account is not configured");
+    }
+    write($main, `<h2>${info.addr!}</h2>`);
+    const chats = await client.getChatlistEntries(
+      selectedAccount,
+      0,
+      null,
+      null
+    );
+    for (const [chatId, _messageId] of chats) {
+      const chat = await client.chatlistGetFullChatById(
+        selectedAccount,
+        chatId
+      );
+      write($main, `<h3>${chat.name}</h3>`);
+      const messageIds = await client.messageListGetMessageIds(
+        selectedAccount,
+        chatId,
+        0
+      );
+      const messages = await client.messageGetMessages(
+        selectedAccount,
+        messageIds
+      );
+      for (const [_messageId, message] of Object.entries(messages)) {
+        write($main, `<p>${message.text}</p>`);
+      }
+    }
+  }
+
+  function onIncomingEvent(event: DeltaEvent, name: string) {
+    write(
+      $side,
+      `
+        <p class="message">
+          [<strong>${name}</strong> on account ${event.contextId}]<br>
+          <em>f1:</em> ${JSON.stringify(event.field1)}<br>
+          <em>f2:</em> ${JSON.stringify(event.field2)}
+        </p>`
+    );
+  }
+}
+
+function write(el: HTMLElement, html: string) {
+  el.innerHTML += html;
+}
+function clear(el: HTMLElement) {
+  el.innerHTML = "";
+}

deltachat-jsonrpc/typescript/generated/client.ts

@@ -0,0 +1,251 @@
+// AUTO-GENERATED by yerpc-derive
+
+import * as T from "./types.js"
+import * as RPC from "./jsonrpc.js"
+
+type RequestMethod = (method: string, params?: RPC.Params) => Promise<unknown>;
+type NotificationMethod = (method: string, params?: RPC.Params) => void;
+
+interface Transport {
+  request: RequestMethod,
+  notification: NotificationMethod
+}
+
+export class RawClient {
+  constructor(private _transport: Transport) {}
+
+  /**
+   * Check if an email address is valid.
+   */
+  public checkEmailValidity(email: string): Promise<boolean> {
+    return (this._transport.request('check_email_validity', [email] as RPC.Params)) as Promise<boolean>;
+  }
+
+  /**
+   * Get general system info.
+   */
+  public getSystemInfo(): Promise<Record<string,string>> {
+    return (this._transport.request('get_system_info', [] as RPC.Params)) as Promise<Record<string,string>>;
+  }
+
+
+  public addAccount(): Promise<T.U32> {
+    return (this._transport.request('add_account', [] as RPC.Params)) as Promise<T.U32>;
+  }
+
+
+  public removeAccount(accountId: T.U32): Promise<null> {
+    return (this._transport.request('remove_account', [accountId] as RPC.Params)) as Promise<null>;
+  }
+
+
+  public getAllAccountIds(): Promise<(T.U32)[]> {
+    return (this._transport.request('get_all_account_ids', [] as RPC.Params)) as Promise<(T.U32)[]>;
+  }
+
+  /**
+   * Select account id for internally selected state.
+   * TODO: Likely this is deprecated as all methods take an account id now.
+   */
+  public selectAccount(id: T.U32): Promise<null> {
+    return (this._transport.request('select_account', [id] as RPC.Params)) as Promise<null>;
+  }
+
+  /**
+   * Get the selected account id of the internal state..
+   * TODO: Likely this is deprecated as all methods take an account id now.
+   */
+  public getSelectedAccountId(): Promise<(T.U32|null)> {
+    return (this._transport.request('get_selected_account_id', [] as RPC.Params)) as Promise<(T.U32|null)>;
+  }
+
+  /**
+   * Get a list of all configured accounts.
+   */
+  public getAllAccounts(): Promise<(T.Account)[]> {
+    return (this._transport.request('get_all_accounts', [] as RPC.Params)) as Promise<(T.Account)[]>;
+  }
+
+  /**
+   * Get top-level info for an account.
+   */
+  public getAccountInfo(accountId: T.U32): Promise<T.Account> {
+    return (this._transport.request('get_account_info', [accountId] as RPC.Params)) as Promise<T.Account>;
+  }
+
+  /**
+   * Returns provider for the given domain.
+   *
+   * This function looks up domain in offline database.
+   *
+   * For compatibility, email address can be passed to this function
+   * instead of the domain.
+   */
+  public getProviderInfo(accountId: T.U32, email: string): Promise<(T.ProviderInfo|null)> {
+    return (this._transport.request('get_provider_info', [accountId, email] as RPC.Params)) as Promise<(T.ProviderInfo|null)>;
+  }
+
+  /**
+   * Checks if the context is already configured.
+   */
+  public isConfigured(accountId: T.U32): Promise<boolean> {
+    return (this._transport.request('is_configured', [accountId] as RPC.Params)) as Promise<boolean>;
+  }
+
+  /**
+   * Get system info for an account.
+   */
+  public getInfo(accountId: T.U32): Promise<Record<string,string>> {
+    return (this._transport.request('get_info', [accountId] as RPC.Params)) as Promise<Record<string,string>>;
+  }
+
+
+  public setConfig(accountId: T.U32, key: string, value: (string|null)): Promise<null> {
+    return (this._transport.request('set_config', [accountId, key, value] as RPC.Params)) as Promise<null>;
+  }
+
+
+  public batchSetConfig(accountId: T.U32, config: Record<string,(string|null)>): Promise<null> {
+    return (this._transport.request('batch_set_config', [accountId, config] as RPC.Params)) as Promise<null>;
+  }
+
+
+  public getConfig(accountId: T.U32, key: string): Promise<(string|null)> {
+    return (this._transport.request('get_config', [accountId, key] as RPC.Params)) as Promise<(string|null)>;
+  }
+
+
+  public batchGetConfig(accountId: T.U32, keys: (string)[]): Promise<Record<string,(string|null)>> {
+    return (this._transport.request('batch_get_config', [accountId, keys] as RPC.Params)) as Promise<Record<string,(string|null)>>;
+  }
+
+  /**
+   * Configures this account with the currently set parameters.
+   * Setup the credential config before calling this.
+   */
+  public configure(accountId: T.U32): Promise<null> {
+    return (this._transport.request('configure', [accountId] as RPC.Params)) as Promise<null>;
+  }
+
+  /**
+   * Signal an ongoing process to stop.
+   */
+  public stopOngoingProcess(accountId: T.U32): Promise<null> {
+    return (this._transport.request('stop_ongoing_process', [accountId] as RPC.Params)) as Promise<null>;
+  }
+
+
+  public autocryptInitiateKeyTransfer(accountId: T.U32): Promise<string> {
+    return (this._transport.request('autocrypt_initiate_key_transfer', [accountId] as RPC.Params)) as Promise<string>;
+  }
+
+
+  public autocryptContinueKeyTransfer(accountId: T.U32, messageId: T.U32, setupCode: string): Promise<null> {
+    return (this._transport.request('autocrypt_continue_key_transfer', [accountId, messageId, setupCode] as RPC.Params)) as Promise<null>;
+  }
+
+
+  public getChatlistEntries(accountId: T.U32, listFlags: (T.U32|null), queryString: (string|null), queryContactId: (T.U32|null)): Promise<(T.ChatListEntry)[]> {
+    return (this._transport.request('get_chatlist_entries', [accountId, listFlags, queryString, queryContactId] as RPC.Params)) as Promise<(T.ChatListEntry)[]>;
+  }
+
+
+  public getChatlistItemsByEntries(accountId: T.U32, entries: (T.ChatListEntry)[]): Promise<Record<T.U32,T.ChatListItemFetchResult>> {
+    return (this._transport.request('get_chatlist_items_by_entries', [accountId, entries] as RPC.Params)) as Promise<Record<T.U32,T.ChatListItemFetchResult>>;
+  }
+
+
+  public chatlistGetFullChatById(accountId: T.U32, chatId: T.U32): Promise<T.FullChat> {
+    return (this._transport.request('chatlist_get_full_chat_by_id', [accountId, chatId] as RPC.Params)) as Promise<T.FullChat>;
+  }
+
+
+  public acceptChat(accountId: T.U32, chatId: T.U32): Promise<null> {
+    return (this._transport.request('accept_chat', [accountId, chatId] as RPC.Params)) as Promise<null>;
+  }
+
+
+  public blockChat(accountId: T.U32, chatId: T.U32): Promise<null> {
+    return (this._transport.request('block_chat', [accountId, chatId] as RPC.Params)) as Promise<null>;
+  }
+
+
+  public messageListGetMessageIds(accountId: T.U32, chatId: T.U32, flags: T.U32): Promise<(T.U32)[]> {
+    return (this._transport.request('message_list_get_message_ids', [accountId, chatId, flags] as RPC.Params)) as Promise<(T.U32)[]>;
+  }
+
+
+  public messageGetMessage(accountId: T.U32, messageId: T.U32): Promise<T.Message> {
+    return (this._transport.request('message_get_message', [accountId, messageId] as RPC.Params)) as Promise<T.Message>;
+  }
+
+
+  public messageGetMessages(accountId: T.U32, messageIds: (T.U32)[]): Promise<Record<T.U32,T.Message>> {
+    return (this._transport.request('message_get_messages', [accountId, messageIds] as RPC.Params)) as Promise<Record<T.U32,T.Message>>;
+  }
+
+  /**
+   * Get a single contact options by ID.
+   */
+  public contactsGetContact(accountId: T.U32, contactId: T.U32): Promise<T.Contact> {
+    return (this._transport.request('contacts_get_contact', [accountId, contactId] as RPC.Params)) as Promise<T.Contact>;
+  }
+
+  /**
+   * Add a single contact as a result of an explicit user action.
+   *
+   * Returns contact id of the created or existing contact
+   */
+  public contactsCreateContact(accountId: T.U32, email: string, name: (string|null)): Promise<T.U32> {
+    return (this._transport.request('contacts_create_contact', [accountId, email, name] as RPC.Params)) as Promise<T.U32>;
+  }
+
+  /**
+   * Returns contact id of the created or existing DM chat with that contact
+   */
+  public contactsCreateChatByContactId(accountId: T.U32, contactId: T.U32): Promise<T.U32> {
+    return (this._transport.request('contacts_create_chat_by_contact_id', [accountId, contactId] as RPC.Params)) as Promise<T.U32>;
+  }
+
+
+  public contactsBlock(accountId: T.U32, contactId: T.U32): Promise<null> {
+    return (this._transport.request('contacts_block', [accountId, contactId] as RPC.Params)) as Promise<null>;
+  }
+
+
+  public contactsUnblock(accountId: T.U32, contactId: T.U32): Promise<null> {
+    return (this._transport.request('contacts_unblock', [accountId, contactId] as RPC.Params)) as Promise<null>;
+  }
+
+
+  public contactsGetBlocked(accountId: T.U32): Promise<(T.Contact)[]> {
+    return (this._transport.request('contacts_get_blocked', [accountId] as RPC.Params)) as Promise<(T.Contact)[]>;
+  }
+
+
+  public contactsGetContactIds(accountId: T.U32, listFlags: T.U32, query: (string|null)): Promise<(T.U32)[]> {
+    return (this._transport.request('contacts_get_contact_ids', [accountId, listFlags, query] as RPC.Params)) as Promise<(T.U32)[]>;
+  }
+
+  /**
+   * Get a list of contacts.
+   * (formerly called getContacts2 in desktop)
+   */
+  public contactsGetContacts(accountId: T.U32, listFlags: T.U32, query: (string|null)): Promise<(T.Contact)[]> {
+    return (this._transport.request('contacts_get_contacts', [accountId, listFlags, query] as RPC.Params)) as Promise<(T.Contact)[]>;
+  }
+
+
+  public contactsGetContactsByIds(accountId: T.U32, ids: (T.U32)[]): Promise<Record<T.U32,T.Contact>> {
+    return (this._transport.request('contacts_get_contacts_by_ids', [accountId, ids] as RPC.Params)) as Promise<Record<T.U32,T.Contact>>;
+  }
+
+  /**
+   * Returns the messageid of the sent message
+   */
+  public miscSendTextMessage(accountId: T.U32, text: string, chatId: T.U32): Promise<T.U32> {
+    return (this._transport.request('misc_send_text_message', [accountId, text, chatId] as RPC.Params)) as Promise<T.U32>;
+  }
+
+
+}

deltachat-jsonrpc/typescript/generated/events.ts

@@ -0,0 +1,3 @@
+// AUTO-GENERATED by typescript-type-def
+
+export type EventTypeName=("Info"|"SmtpConnected"|"ImapConnected"|"SmtpMessageSent"|"ImapMessageDeleted"|"ImapMessageMoved"|"NewBlobFile"|"DeletedBlobFile"|"Warning"|"Error"|"ErrorSelfNotInGroup"|"MsgsChanged"|"IncomingMsg"|"MsgsNoticed"|"MsgDelivered"|"MsgFailed"|"MsgRead"|"ChatModified"|"ChatEphemeralTimerModified"|"ContactsChanged"|"LocationChanged"|"ConfigureProgress"|"ImexProgress"|"ImexFileWritten"|"SecurejoinInviterProgress"|"SecurejoinJoinerProgress"|"ConnectivityChanged"|"SelfavatarChanged"|"WebxdcStatusUpdate");

deltachat-jsonrpc/typescript/generated/jsonrpc.ts

@@ -0,0 +1,10 @@
+// AUTO-GENERATED by typescript-type-def
+
+export type JSONValue=(null|boolean|number|string|(JSONValue)[]|{[key:string]:JSONValue;});
+export type Params=((JSONValue)[]|Record<string,JSONValue>);
+export type U32=number;
+export type Request={"jsonrpc":"2.0";"method":string;"params"?:Params;"id"?:U32;};
+export type I32=number;
+export type Error={"code":I32;"message":string;"data"?:JSONValue;};
+export type Response={"jsonrpc":"2.0";"id":(U32|null);"result"?:JSONValue;"error"?:Error;};
+export type Message=(Request|Response);

deltachat-jsonrpc/typescript/generated/types.ts

@@ -0,0 +1,15 @@
+// AUTO-GENERATED by typescript-type-def
+
+export type U32=number;
+export type Account=(({"type":"Configured";}&{"id":U32;"display_name":(string|null);"addr":(string|null);"profile_image":(string|null);"color":string;})|({"type":"Unconfigured";}&{"id":U32;}));
+export type ProviderInfo={"before_login_hint":string;"overview_page":string;"status":U32;};
+export type ChatListEntry=[U32,U32];
+export type I64=number;
+export type Usize=number;
+export type ChatListItemFetchResult=(({"type":"ChatListItem";}&{"id":U32;"name":string;"avatarPath":(string|null);"color":string;"lastUpdated":(I64|null);"summaryText1":string;"summaryText2":string;"summaryStatus":U32;"isProtected":boolean;"isGroup":boolean;"freshMessageCounter":Usize;"isSelfTalk":boolean;"isDeviceTalk":boolean;"isSendingLocation":boolean;"isSelfInGroup":boolean;"isArchived":boolean;"isPinned":boolean;"isMuted":boolean;"isContactRequest":boolean;})|{"type":"ArchiveLink";}|({"type":"Error";}&{"id":U32;"error":string;}));
+export type Contact={"address":string;"color":string;"auth_name":string;"status":string;"display_name":string;"id":U32;"name":string;"profile_image":(string|null);"name_and_addr":string;"is_blocked":boolean;"is_verified":boolean;};
+export type FullChat={"id":U32;"name":string;"is_protected":boolean;"profile_image":(string|null);"archived":boolean;"chat_type":U32;"is_unpromoted":boolean;"is_self_talk":boolean;"contacts":(Contact)[];"contact_ids":(U32)[];"color":string;"fresh_message_counter":Usize;"is_contact_request":boolean;"is_device_chat":boolean;"self_in_group":boolean;"is_muted":boolean;"ephemeral_timer":U32;"can_send":boolean;};
+export type I32=number;
+export type U64=number;
+export type Message={"id":U32;"chat_id":U32;"from_id":U32;"quoted_text":(string|null);"quoted_message_id":(U32|null);"text":(string|null);"has_location":boolean;"has_html":boolean;"view_type":U32;"state":U32;"timestamp":I64;"sort_timestamp":I64;"received_timestamp":I64;"has_deviating_timestamp":boolean;"subject":string;"show_padlock":boolean;"is_setupmessage":boolean;"is_info":boolean;"is_forwarded":boolean;"duration":I32;"dimensions_height":I32;"dimensions_width":I32;"videochat_type":(U32|null);"videochat_url":(string|null);"override_sender_name":(string|null);"sender":Contact;"setup_code_begin":(string|null);"file":(string|null);"file_mime":(string|null);"file_bytes":U64;"file_name":(string|null);};
+export type __AllTyps=[string,boolean,Record<string,string>,U32,U32,null,(U32)[],U32,null,(U32|null),(Account)[],U32,Account,U32,string,(ProviderInfo|null),U32,boolean,U32,Record<string,string>,U32,string,(string|null),null,U32,Record<string,(string|null)>,null,U32,string,(string|null),U32,(string)[],Record<string,(string|null)>,U32,null,U32,null,U32,string,U32,U32,string,null,U32,(U32|null),(string|null),(U32|null),(ChatListEntry)[],U32,(ChatListEntry)[],Record<U32,ChatListItemFetchResult>,U32,U32,FullChat,U32,U32,null,U32,U32,null,U32,U32,U32,(U32)[],U32,U32,Message,U32,(U32)[],Record<U32,Message>,U32,U32,Contact,U32,string,(string|null),U32,U32,U32,U32,U32,U32,null,U32,U32,null,U32,(Contact)[],U32,U32,(string|null),(U32)[],U32,U32,(string|null),(Contact)[],U32,(U32)[],Record<U32,Contact>,U32,string,U32,U32];

deltachat-jsonrpc/typescript/index.html

@@ -0,0 +1,54 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <style>
+      body {
+        font-family: monospace;
+        background: black;
+        color: grey;
+      }
+      .grid {
+        display: grid;
+        grid-template-columns: 3fr 1fr;
+        grid-template-areas: "a a" "b c";
+      }
+      .message {
+        color: red;
+      }
+      #header {
+        grid-area: a;
+        color: white;
+        font-size: 1.2rem;
+      }
+      #header a {
+        color: white;
+        font-weight: bold;
+      }
+      #main {
+        grid-area: b;
+        color: green;
+      }
+      #main h2,
+      #main h3 {
+        color: blue;
+      }
+      #side {
+        grid-area: c;
+        color: #777;
+        overflow-y: auto;
+      }
+    </style>
+    <script type="module" src="dist/example.bundle.js"></script>
+  </head>
+  <body>
+    <div class="grid">
+      <div id="header"></div>
+      <div id="main"></div>
+      <div id="side"><h2>log</h2></div>
+    </div>
+    <p>
+      Tip: open the dev console and use the client with
+      <code>window.client</code>
+    </p>
+  </body>
+</html>

deltachat-jsonrpc/typescript/node-demo.js

@@ -0,0 +1,13 @@
+import { Deltachat } from "./dist/deltachat.js";
+
+run().catch(console.error);
+
+async function run() {
+  const delta = new Deltachat();
+  delta.addEventListener("event", (event) => {
+    console.log("event", event.data);
+  });
+
+  const accounts = await delta.rpc.getAllAccounts();
+  console.log("accounts", accounts);
+}

deltachat-jsonrpc/typescript/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "@deltachat/jsonrpc-client",
+  "version": "0.1.0",
+  "main": "dist/deltachat.js",
+  "types": "dist/deltachat.d.ts",
+  "type": "module",
+  "author": "Delta Chat Developers (ML) <delta@codespeak.net>",
+  "license": "MPL-2.0",
+  "scripts": {
+    "prettier:check": "prettier --check **.ts",
+    "prettier:fix": "prettier --write **.ts",
+    "build": "npm run generate-bindings && tsc",
+    "bundle": "npm run build && esbuild --bundle dist/deltachat.js --outfile=dist/deltachat.bundle.js",
+    "generate-bindings": "cargo test",
+    "example:build": "tsc && esbuild --bundle dist/example.js --outfile=dist/example.bundle.js",
+    "example:dev": "esbuild example.ts --bundle --outdir=dist --servedir=.",
+    "coverage": "tsc -b test && COVERAGE=1 NODE_OPTIONS=--enable-source-maps c8 --include \"dist/*\" -r text -r html -r json mocha test_dist && node report_api_coverage.mjs",
+    "test": "rm -rf dist && npm run build && npm run coverage && npm run prettier:check"
+  },
+  "dependencies": {
+    "isomorphic-ws": "^4.0.1",
+    "tiny-emitter": "git+https://github.com/Simon-Laux/tiny-emitter.git",
+    "yerpc": "^0.2.3"
+  },
+  "devDependencies": {
+    "prettier": "^2.6.2",
+    "chai-as-promised": "^7.1.1",
+    "@types/chai": "^4.2.21",
+    "@types/chai-as-promised": "^7.1.5",
+    "@types/mocha": "^9.0.0",
+    "@types/node-fetch": "^2.5.7",
+    "@types/ws": "^7.2.4",
+    "c8": "^7.10.0",
+    "chai": "^4.3.4",
+    "esbuild": "^0.14.11",
+    "mocha": "^9.1.1",
+    "node-fetch": "^2.6.1",
+    "typescript": "^4.5.5",
+    "ws": "^8.5.0"
+  }
+}

deltachat-jsonrpc/typescript/report_api_coverage.mjs

@@ -0,0 +1,28 @@
+import { readFileSync } from "fs";
+// only checks for the coverge of the api functions in bindings.ts for now
+const generated_file = "typescript/generated/client.ts";
+const json = JSON.parse(readFileSync("./coverage/coverage-final.json"));
+const jsonCoverage =
+  json[Object.keys(json).find((k) => k.includes(generated_file))];
+const fnMap = Object.keys(jsonCoverage.fnMap).map(
+  (key) => jsonCoverage.fnMap[key]
+);
+const htmlCoverage = readFileSync(
+  "./coverage/" + generated_file + ".html",
+  "utf8"
+);
+const uncoveredLines = htmlCoverage
+  .split("\n")
+  .filter((line) => line.includes(`"function not covered"`));
+const uncoveredFunctions = uncoveredLines.map(
+  (line) => />([\w_]+)\(/.exec(line)[1]
+);
+console.log(
+  "\nUncovered api functions:\n" +
+    uncoveredFunctions
+      .map((uF) => fnMap.find(({ name }) => name === uF))
+      .map(
+        ({ name, line }) => `.${name.padEnd(40)}  (${generated_file}:${line})`
+      )
+      .join("\n")
+);

deltachat-jsonrpc/typescript/src/client.ts

@@ -0,0 +1,77 @@
+import * as T from "../generated/types.js";
+import * as RPC from "../generated/jsonrpc.js";
+import { RawClient } from "../generated/client.js";
+import { EventTypeName } from "../generated/events.js";
+import { WebsocketTransport, BaseTransport, Request } from "yerpc";
+import { TinyEmitter } from "tiny-emitter";
+
+export type DeltachatEvent = {
+  id: EventTypeName;
+  contextId: number;
+  field1: any;
+  field2: any;
+};
+export type Events = Record<
+  EventTypeName | "ALL",
+  (event: DeltachatEvent) => void
+>;
+
+export class BaseDeltachat<
+  Transport extends BaseTransport
+> extends TinyEmitter<Events> {
+  rpc: RawClient;
+  account?: T.Account;
+  constructor(protected transport: Transport) {
+    super();
+    this.rpc = new RawClient(this.transport);
+    this.transport.on("request", (request: Request) => {
+      const method = request.method;
+      if (method === "event") {
+        const event = request.params! as DeltachatEvent;
+        this.emit(event.id, event);
+        this.emit("ALL", event);
+
+        if (this.contextEmitters[event.contextId]) {
+          this.contextEmitters[event.contextId].emit(event.id, event);
+          this.contextEmitters[event.contextId].emit("ALL", event);
+        }
+      }
+    });
+  }
+
+  async listAccounts(): Promise<T.Account[]> {
+    return await this.rpc.getAllAccounts();
+  }
+
+  private contextEmitters: TinyEmitter<Events>[] = [];
+
+  getContextEvents(account_id: number) {
+    if (this.contextEmitters[account_id]) {
+      return this.contextEmitters[account_id];
+    } else {
+      this.contextEmitters[account_id] = new TinyEmitter();
+      return this.contextEmitters[account_id];
+    }
+  }
+}
+
+export type Opts = {
+  url: string;
+};
+
+export const DEFAULT_OPTS: Opts = {
+  url: "ws://localhost:20808/ws",
+};
+export class Deltachat extends BaseDeltachat<WebsocketTransport> {
+  opts: Opts;
+  close() {
+    this.transport._socket.close();
+  }
+  constructor(opts: Opts | string | undefined) {
+    if (typeof opts === "string") opts = { url: opts };
+    if (opts) opts = { ...DEFAULT_OPTS, ...opts };
+    else opts = { ...DEFAULT_OPTS };
+    super(new WebsocketTransport(opts.url));
+    this.opts = opts;
+  }
+}

deltachat-jsonrpc/typescript/src/lib.ts

@@ -0,0 +1,6 @@
+export * as RPC from "../generated/jsonrpc.js";
+export * as T from "../generated/types.js";
+export * from "../generated/events.js";
+export { RawClient } from "../generated/client.js";
+export * from "./client.js";
+export * as yerpc from "yerpc";

deltachat-jsonrpc/typescript/test/README.md

@@ -0,0 +1 @@
+# tests need to be ported to new API

deltachat-jsonrpc/typescript/test/basic.ts

@@ -0,0 +1,158 @@
+import { strictEqual } from "assert";
+import chai, { assert, expect } from "chai";
+import chaiAsPromised from "chai-as-promised";
+chai.use(chaiAsPromised);
+import { Deltachat } from "../dist/deltachat.js";
+
+import {
+  CMD_API_Server_Handle,
+  CMD_API_SERVER_PORT,
+  startCMD_API_Server,
+} from "./test_base.js";
+
+describe("basic tests", () => {
+  let server_handle: CMD_API_Server_Handle;
+  let dc: Deltachat;
+
+  before(async () => {
+    server_handle = await startCMD_API_Server(CMD_API_SERVER_PORT);
+    // make sure server is up by the time we continue
+    await new Promise((res) => setTimeout(res, 100));
+
+    dc = new Deltachat({
+      url: "ws://localhost:" + CMD_API_SERVER_PORT + "/ws",
+    });
+    dc.on("ALL", (event) => {
+      //console.log("event", event);
+    });
+  });
+
+  after(async () => {
+    dc && dc.close();
+    await server_handle.close();
+  });
+
+  it("check email", async () => {
+    const positive_test_cases = [
+      "email@example.com",
+      "36aa165ae3406424e0c61af17700f397cad3fe8ab83d682d0bddf3338a5dd52e@yggmail@yggmail",
+    ];
+    const negative_test_cases = ["email@", "example.com", "emai221"];
+
+    expect(
+      await Promise.all(
+        positive_test_cases.map((email) => dc.rpc.checkEmailValidity(email))
+      )
+    ).to.not.contain(false);
+
+    expect(
+      await Promise.all(
+        negative_test_cases.map((email) => dc.rpc.checkEmailValidity(email))
+      )
+    ).to.not.contain(true);
+  });
+
+  it("system info", async () => {
+    const system_info = await dc.rpc.getSystemInfo();
+    expect(system_info).to.contain.keys([
+      "arch",
+      "num_cpus",
+      "deltachat_core_version",
+      "sqlite_version",
+    ]);
+  });
+
+  describe("account managment", () => {
+    it("should create account", async () => {
+      await dc.rpc.addAccount();
+      assert((await dc.rpc.getAllAccountIds()).length === 1);
+    });
+
+    it("should remove the account again", async () => {
+      await dc.rpc.removeAccount((await dc.rpc.getAllAccountIds())[0]);
+      assert((await dc.rpc.getAllAccountIds()).length === 0);
+    });
+
+    it("should create multiple accounts", async () => {
+      await dc.rpc.addAccount();
+      await dc.rpc.addAccount();
+      await dc.rpc.addAccount();
+      await dc.rpc.addAccount();
+      assert((await dc.rpc.getAllAccountIds()).length === 4);
+    });
+  });
+
+  describe("contact managment", function () {
+    let acc: number;
+    before(async () => {
+      acc = await dc.rpc.addAccount();
+    });
+    it("block and unblock contact", async function () {
+      const contactId = await dc.rpc.contactsCreateContact(
+        acc,
+        "example@delta.chat",
+        null
+      );
+      expect((await dc.rpc.contactsGetContact(acc, contactId)).is_blocked).to.be
+        .false;
+      await dc.rpc.contactsBlock(acc, contactId);
+      expect((await dc.rpc.contactsGetContact(acc, contactId)).is_blocked).to.be
+        .true;
+      expect(await dc.rpc.contactsGetBlocked(acc)).to.have.length(1);
+      await dc.rpc.contactsUnblock(acc, contactId);
+      expect((await dc.rpc.contactsGetContact(acc, contactId)).is_blocked).to.be
+        .false;
+      expect(await dc.rpc.contactsGetBlocked(acc)).to.have.length(0);
+    });
+  });
+
+  describe("configuration", function () {
+    let acc: number;
+    before(async () => {
+      acc = await dc.rpc.addAccount();
+    });
+
+    it("set and retrive", async function () {
+      await dc.rpc.setConfig(acc, "addr", "valid@email");
+      assert((await dc.rpc.getConfig(acc, "addr")) == "valid@email");
+    });
+    it("set invalid key should throw", async function () {
+      await expect(dc.rpc.setConfig(acc, "invalid_key", "some value")).to.be
+        .eventually.rejected;
+    });
+    it("get invalid key should throw", async function () {
+      await expect(dc.rpc.getConfig(acc, "invalid_key")).to.be.eventually
+        .rejected;
+    });
+    it("set and retrive ui.*", async function () {
+      await dc.rpc.setConfig(acc, "ui.chat_bg", "color:red");
+      assert((await dc.rpc.getConfig(acc, "ui.chat_bg")) == "color:red");
+    });
+    it("set and retrive (batch)", async function () {
+      const config = { addr: "valid@email", mail_pw: "1234" };
+      await dc.rpc.batchSetConfig(acc, config);
+      const retrieved = await dc.rpc.batchGetConfig(acc, Object.keys(config));
+      expect(retrieved).to.deep.equal(config);
+    });
+    it("set and retrive ui.* (batch)", async function () {
+      const config = {
+        "ui.chat_bg": "color:green",
+        "ui.enter_key_sends": "true",
+      };
+      await dc.rpc.batchSetConfig(acc, config);
+      const retrieved = await dc.rpc.batchGetConfig(acc, Object.keys(config));
+      expect(retrieved).to.deep.equal(config);
+    });
+    it("set and retrive mixed(ui and core) (batch)", async function () {
+      const config = {
+        "ui.chat_bg": "color:yellow",
+        "ui.enter_key_sends": "false",
+        addr: "valid2@email",
+        mail_pw: "123456",
+      };
+      await dc.rpc.batchSetConfig(acc, config);
+      const retrieved = await dc.rpc.batchGetConfig(acc, Object.keys(config));
+      expect(retrieved).to.deep.equal(config);
+    });
+  });
+});

deltachat-jsonrpc/typescript/test/online.ts

@@ -0,0 +1,203 @@
+import { assert, expect } from "chai";
+import { Deltachat, DeltachatEvent, EventTypeName } from "../dist/deltachat.js";
+import {
+  CMD_API_Server_Handle,
+  CMD_API_SERVER_PORT,
+  createTempUser,
+  startCMD_API_Server,
+} from "./test_base.js";
+
+describe("online tests", function () {
+  let server_handle: CMD_API_Server_Handle;
+  let dc: Deltachat;
+  let account: { email: string; password: string };
+  let account2: { email: string; password: string };
+  let acc1: number, acc2: number;
+
+  before(async function () {
+    this.timeout(12000)
+    if (!process.env.DCC_NEW_TMP_EMAIL) {
+      if (process.env.COVERAGE && !process.env.COVERAGE_OFFLINE) {
+        console.error(
+          "CAN NOT RUN COVERAGE correctly: Missing DCC_NEW_TMP_EMAIL environment variable!\n\n",
+          "You can set COVERAGE_OFFLINE=1 to circumvent this check and skip the online tests, but those coverage results will be wrong, because some functions can only be tested in the online test"
+        );
+        process.exit(1);
+      }
+      console.log(
+        "Missing DCC_NEW_TMP_EMAIL environment variable!, skip intergration tests"
+      );
+      this.skip();
+    }
+    server_handle = await startCMD_API_Server(CMD_API_SERVER_PORT);
+    dc = new Deltachat({
+      url: "ws://localhost:" + CMD_API_SERVER_PORT + "/ws",
+    });
+
+    dc.on("ALL", ({ id, contextId }) => {
+      if (id !== "Info") console.log(contextId, id);
+    });
+
+    account = await createTempUser(process.env.DCC_NEW_TMP_EMAIL);
+    if (!account || !account.email || !account.password) {
+      console.log(
+        "We didn't got back an account from the api, skip intergration tests"
+      );
+      this.skip();
+    }
+
+    account2 = await createTempUser(process.env.DCC_NEW_TMP_EMAIL);
+    if (!account2 || !account2.email || !account2.password) {
+      console.log(
+        "We didn't got back an account2 from the api, skip intergration tests"
+      );
+      this.skip();
+    }
+  });
+
+  after(async () => {
+    dc && dc.close();
+    server_handle && (await server_handle.close());
+  });
+
+  let are_configured = false;
+
+  it("configure test accounts", async function () {
+    this.timeout(20000);
+
+    acc1 = await dc.rpc.addAccount();
+    await dc.rpc.setConfig(acc1, "addr", account.email);
+    await dc.rpc.setConfig(acc1, "mail_pw", account.password);
+    let configure_promise = dc.rpc.configure(acc1);
+
+    acc2 = await dc.rpc.addAccount();
+    await dc.rpc.batchSetConfig(acc2, {
+      addr: account2.email,
+      mail_pw: account2.password,
+    });
+
+    await Promise.all([configure_promise, dc.rpc.configure(acc2)]);
+    are_configured = true;
+  });
+
+  it("send and recieve text message", async function () {
+    if (!are_configured) {
+      this.skip();
+    }
+    this.timeout(15000);
+
+    const contactId = await dc.rpc.contactsCreateContact(
+      acc1,
+      account2.email,
+      null
+    );
+    const chatId = await dc.rpc.contactsCreateChatByContactId(acc1, contactId);
+    const eventPromise = Promise.race([
+      waitForEvent(dc, "MsgsChanged", acc2),
+      waitForEvent(dc, "IncomingMsg", acc2),
+    ]);
+
+    dc.rpc.miscSendTextMessage(acc1, "Hello", chatId);
+    const { field1: chatIdOnAccountB } = await eventPromise;
+    await dc.rpc.acceptChat(acc2, chatIdOnAccountB);
+    const messageList = await dc.rpc.messageListGetMessageIds(
+      acc2,
+      chatIdOnAccountB,
+      0
+    );
+
+    expect(messageList).have.length(1);
+    const message = await dc.rpc.messageGetMessage(acc2, messageList[0]);
+    expect(message.text).equal("Hello");
+  });
+
+  it("send and recieve text message roundtrip, encrypted on answer onwards", async function () {
+    if (!are_configured) {
+      this.skip();
+    }
+    this.timeout(10000);
+
+    // send message from A to B
+    const contactId = await dc.rpc.contactsCreateContact(
+      acc1,
+      account2.email,
+      null
+    );
+    const chatId = await dc.rpc.contactsCreateChatByContactId(acc1, contactId);
+    const eventPromise = Promise.race([
+      waitForEvent(dc, "MsgsChanged", acc2),
+      waitForEvent(dc, "IncomingMsg", acc2),
+    ]);
+    dc.rpc.miscSendTextMessage(acc1, "Hello2", chatId);
+    // wait for message from A
+    console.log("wait for message from A");
+
+    const event = await eventPromise;
+    const { field1: chatIdOnAccountB } = event;
+
+    await dc.rpc.acceptChat(acc2, chatIdOnAccountB);
+    const messageList = await dc.rpc.messageListGetMessageIds(
+      acc2,
+      chatIdOnAccountB,
+      0
+    );
+    const message = await dc.rpc.messageGetMessage(
+      acc2,
+      messageList.reverse()[0]
+    );
+    expect(message.text).equal("Hello2");
+    // Send message back from B to A
+    const eventPromise2 = Promise.race([
+      waitForEvent(dc, "MsgsChanged", acc1),
+      waitForEvent(dc, "IncomingMsg", acc1),
+    ]);
+    dc.rpc.miscSendTextMessage(acc2, "super secret message", chatId);
+    // Check if answer arives at A and if it is encrypted
+    await eventPromise2;
+
+    const messageId = (
+      await dc.rpc.messageListGetMessageIds(acc1, chatId, 0)
+    ).reverse()[0];
+    const message2 = await dc.rpc.messageGetMessage(acc1, messageId);
+    expect(message2.text).equal("super secret message");
+    expect(message2.show_padlock).equal(true);
+  });
+
+  it("get provider info for example.com", async () => {
+    const acc = await dc.rpc.addAccount();
+    const info = await dc.rpc.getProviderInfo(acc, "example.com");
+    expect(info).to.be.not.null;
+    expect(info?.overview_page).to.equal(
+      "https://providers.delta.chat/example-com"
+    );
+    expect(info?.status).to.equal(3);
+  });
+
+  it("get provider info - domain and email should give same result", async () => {
+    const acc = await dc.rpc.addAccount();
+    const info_domain = await dc.rpc.getProviderInfo(acc, "example.com");
+    const info_email = await dc.rpc.getProviderInfo(acc, "hi@example.com");
+    expect(info_email).to.deep.equal(info_domain);
+  });
+});
+
+type event_data = {
+  contextId: number;
+  id: EventTypeName;
+  [key: string]: any;
+};
+async function waitForEvent(
+  dc: Deltachat,
+  event: EventTypeName,
+  accountId: number
+): Promise<event_data> {
+  return new Promise((res, rej) => {
+    const callback = (ev: DeltachatEvent) => {
+      if (ev.contextId == accountId) {
+        dc.off(event, callback);
+        res(ev);
+      }
+    };
+    dc.on(event, callback);
+  });
+}

deltachat-jsonrpc/typescript/test/test_base.ts

@@ -0,0 +1,95 @@
+import { tmpdir } from "os";
+import { join } from "path";
+import { mkdtemp, rm } from "fs/promises";
+import { existsSync } from "fs";
+import { spawn, exec } from "child_process";
+import { unwrapPromise } from "./ts_helpers.js";
+import fetch from "node-fetch";
+/* port is not configurable yet */
+
+
+function getTargetDir(): Promise<string> {
+  return new Promise((res, rej) => {
+    exec(
+      "cargo metadata --no-deps --format-version 1",
+      (error, stdout, stderr) => {
+        if (error) {
+          console.log("error", error);
+          rej(error);
+        } else {
+          try {
+            const json = JSON.parse(stdout);
+            res(json.target_directory);
+          } catch (error) {
+            console.log("json error", error);
+            rej(error);
+          }
+        }
+      }
+    );
+  });
+}
+
+export const CMD_API_SERVER_PORT = 20808;
+export async function startCMD_API_Server(port: typeof CMD_API_SERVER_PORT) {
+  const tmp_dir = await mkdtemp(join(tmpdir(), "test_prefix"));
+
+  const path_of_server = join(await getTargetDir(), "debug/webserver");
+  console.log(path_of_server);
+
+  if (!existsSync(path_of_server)) {
+    throw new Error(
+      "server executable does not exist, you need to build it first" +
+        "\nserver executable not found at " +
+        path_of_server
+    );
+  }
+
+  const server = spawn(path_of_server, {
+    cwd: tmp_dir,
+    env: {
+      RUST_LOG: "info",
+    },
+  });
+  let should_close = false;
+
+  server.on("exit", () => {
+    if (should_close) {
+      return;
+    }
+    throw new Error("Server quit");
+  });
+
+  server.stderr.pipe(process.stderr);
+
+  //server.stdout.pipe(process.stdout)
+
+  return {
+    close: async () => {
+      should_close = true;
+      if (!server.kill(9)) {
+        console.log("server termination failed");
+      }
+      await rm(tmp_dir, { recursive: true });
+    },
+  };
+}
+
+export type CMD_API_Server_Handle = unwrapPromise<
+  ReturnType<typeof startCMD_API_Server>
+>;
+
+export async function createTempUser(url: string) {
+  async function postData(url = "") {
+    // Default options are marked with *
+    const response = await fetch(url, {
+      method: "POST", // *GET, POST, PUT, DELETE, etc.
+      headers: {
+        "cache-control": "no-cache",
+      },
+    });
+    return response.json(); // parses JSON response into native JavaScript objects
+  }
+
+  return await postData(url);
+}

deltachat-jsonrpc/typescript/test/ts_helpers.ts

@@ -0,0 +1 @@
+export type unwrapPromise<T> = T extends Promise<infer U> ? U : never;

deltachat-jsonrpc/typescript/test/tsconfig.json

@@ -0,0 +1,17 @@
+{
+  "compilerOptions": {
+    "rootDir": ".",
+    "outDir": "../test_dist",
+    "target": "ES2020",
+    "module": "es2020",
+    "moduleResolution": "node",
+    "declaration": false,
+    "esModuleInterop": true,
+    "noImplicitAny": true,
+    "isolatedModules": true,
+    "strictNullChecks": true,
+    "strict": true,
+    "sourceMap": true
+  },
+  "compileOnSave": true
+}

deltachat-jsonrpc/typescript/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "compilerOptions": {
+    "alwaysStrict": true,
+    "strict": true,
+    "sourceMap": true,
+    "strictNullChecks": true,
+    "rootDir": ".",
+    "outDir": "dist",
+    "lib": ["ES2017", "dom"],
+    "target": "ES2017",
+    "module": "es2015",
+    "declaration": true,
+    "esModuleInterop": true,
+    "moduleResolution": "node",
+    "noImplicitAny": true,
+    "isolatedModules": true
+  },
+  "include": ["*.ts"],
+  "compileOnSave": false
+}

draft/aeap-mvp.md

@@ -0,0 +1,99 @@
+AEAP MVP
+========
+
+Changes to the UIs
+------------------
+
+- The secondary self addresses (see below) are shown in the UI, but not editable.
+
+- When the user changed the email address in the configure screen, show a dialog to the user, either directly explaining things or with a link to the FAQ (see "Other" below)
+
+Changes in the core
+-------------------
+
+- DONE: We have one primary self address and any number of secondary self addresses. `is_self_addr()` checks all of them.
+
+- DONE: If the user does a reconfigure and changes the email address, the previous address is added as a secondary self address.
+
+  - don't forget to deduplicate secondary self addresses in case the user switches back and forth between addresses).
+
+  - The key stays the same.
+
+- No changes for 1:1 chats, there simply is a new one. (This works since, contrary to group messages, messages sent to a 1:1 chat are not assigned to the group chat but always to the 1:1 chat with the sender. So it's not a problem that the new messages might be put into the old chat if they are a reply to a message there.)
+
+- When sending a message: If any of the secondary self addrs is in the chat's member list, remove it locally (because we just transitioned away from it). We add a log message for this (alternatively, a system message in the chat would be more visible).
+
+- When receiving a message: If the key exists, but belongs to another address (we may want to benchmark this)
+  AND there is a `Chat-Version` header\
+  AND the message timestamp is newer than the contact's `lastseen` (to prevent changing the address back when messages arrive out of order) (this condition is not that important since we will have eventual consistency even without it):
+
+  Replace the contact in _all_ groups, possibly deduplicate the members list, and add a system message to all of these chats.
+  
+  - Note that we can't simply compare the keys byte-by-byte, since the UID may have changed, or the sender may have rotated the key and signed the new key with the old one.
+
+### Notes:
+  
+- We treat protected and non-protected chats the same
+- We leave the aeap transition statement away since it seems not to be needed, makes things harder on the sending side, wastes some network traffic, and is worse for privacy (since more pepole know what old addresses you had).
+- As soon as we encrypt read receipts, sending a read receipt will be enough to tell a lot of people that you transitioned
+- AEAP will make the problem of inconsistent group state worse, both because it doesn't work if the message is unencrypted (even if the design allowed it, it would be problematic security-wise) and because some chat partners may have gotten the transition and some not. We should do something against this at some point in the future, like asking the user whether they want to add/remove the members to restore consistent group state.
+
+#### Downsides of this design:
+- Inconsistent group state: Suppose Alice does an AEAP transition and sends a 1:1 message to Bob, so Bob rewrites Alice's contact. Alice, Bob and Charlie are together in a group. Before Alice writes to this group, Bob and Charlie will have different membership lists, and Bob will send messages to Alice's new address, while Charlie will send them to her old address.
+
+#### Upsides:
+- With this approach, it's easy to switch to a model where the info about the transition is encoded in the PGP key. Since the key is gossiped, the information about the transition will spread virally.
+- Faster transation: If you send a message to e.g. "Delta Chat Dev", all members of the "sub-group" "delta android" will know of your transition.
+
+### Alternatives and old discussions/plans:
+
+- Change the contact instead of rewriting the group member lists. This seems to call for more trouble since we will end up with multiple contacts having the same email address.
+
+- If needed, we could add a header a) indicating that the sender did an address transition or b) listing all the secondary (old) addresses.  For now, there is no big enough benefit to warrant introducing another header and its processing on the receiver side (including all the neccessary checks and handling of error cases). Instead, we only check for the `Chat-Version` header to prevent accidental transitions when an MUA user sends a message from another email address with the same key.
+
+- The condition for a transition temporarily was:
+
+  > When receiving a message: If we are going to assign a message to a chat, but the sender is not a member of this chat\
+  > AND the signing key is the same as the direct (non-gossiped) key of one of the chat members\
+  > AND ...
+
+  However, this would mean that in 1:1 messages can't trigger a transition, since we don't assign private messages to the parent chat, but always to the 1:1 chat with the sender.
+  
+<details>
+<summary>Some previous state of the discussion, which temporarily lived in an issue description</summary>
+Summarizing the discussions from https://github.com/deltachat/deltachat-core-rust/pull/2896, mostly quoting @hpk42:
+
+1. (DONE) At the time of configure we push the current primary to become a secondary. 
+
+2. When a message is sent out to a chat, and the message is encrypted, and we have secondary addresses, then we 
+  a) add a protected "AEAP-Replacement" header that contains all secondary addresses 
+  b) if any of the secondary addresses is in the chat's member list, we remove it and leave a system message that we did so
+3. When an encrypted message with a replacement header is received, replace the e-mail address of all secondary contacts (if they exist) with the new primary and drop a sysmessage in all chats the secondary is member off.  This might (in edge cases) result in chats that have two or more contacts with the same e-mail address.  We might ignore this for a first release and just log a warning.  Let's maybe not get hung up on this case before everything else works. 
+
+Notes: 
+- for now we will send out aeap replacement headers forever, there is no termination condition other than lack of secondary addresses.  I think that's fine for now.  Later on we might introduce options to remove secondary addresses but i wouldn't do this for a first release/PR. 
+- the design is resilient against changing e-mail providers from A to B to C and then back to A, with partially updated chats and diverging views from recipients/contacts on this transition.  In the end, you will have a primary and some secondaries, and when you start sending out messages everybody will eventually synchronize when they receive the current state of primaries/secondaries. 
+- of course on incoming message for need to check for each stated secondary address in the replacement header that it uses the same signature as the signature we verified as valid with the incoming message  **-->  Also we have to somehow make sure that the signing key was not just gossiped from some random other person in some group.**
+- there are no extra flags/columns in the database needed (i hope) 
+
+#### Downsides of the chosen approach:
+- Inconsistent group state: Suppose Alice does an AEAP transition and sends a 1:1 message to Bob, so Bob rewrites Alice's contact. Alice, Bob and Charlie are together in a group. Before Alice writes to this group, Bob and Charlie will have different membership lists, and Bob will send messages to Alice's new address, while Charlie will send them to her old address.
+- There will be multiple contacts with the same address in the database. We will have to do something against this at some point.
+
+The most obvious alternative would be to create a new contact with the new address and replace the old contact in the groups.
+
+#### Upsides:
+- With this approach, it's easier to switch to a model where the info about the transition is encoded in the PGP key. Since the key is gossiped, the information about the transition will spread virally.
+- (Also, less important: Slightly faster transation: If you send a message to e.g. "Delta Chat Dev", all members of the "sub-group" "delta android" will know of your transition.)
+- It's easier to implement (if too many problems turn up, we can still switch to another approach and didn't wast that much development time.)
+
+[full messages](https://github.com/deltachat/deltachat-core-rust/pull/2896#discussion_r852002161)
+  
+_end of the previous state of the discussion_  
+
+</details>
+  
+Other
+-----
+
+- The user is responsible that messages to the old address arrive at the new address, for example by configuring the old provider to forward all emails to the new one.

draft/aeap-mvp.rst

@@ -1,33 +0,0 @@
-AEAP MVP
-========
-
-Changes to the UIs
-------------------
-
-- The secondary self addresses (see below) are shown in the UI, but not editable.
-
-- When the user changed the email address in the configure screen, show a dialog to the user, either directly explaining things or with a link to the FAQ (see "Other" below)
-
-Changes in the core
--------------------
-
-- We have one primary self address and any number of secondary self addresses. `is_self_addr()` checks all of them.
-
-- If the user does a reconfigure and changes the email address, the previous address is added as a secondary self address.
-
-  - don't forget to deduplicate secondary self addresses in case the user switches back and forth between addresses).
-
-  - The key stays the same.
-
-- No changes for 1:1 chats, there simply is a new one
-
-- When we send a message to a group, and the primary address is not a member of a group, but a secondary address is:
-  
-  Add Chat-Group-Member-Removed=<old address> and Chat-Group-Member-Added=<new address> headers to this message
-
-  - On the receiving side, make sure that we accept this (even in verified groups) if the message is signed and the key stayed the same
-
-Other
------
-
-- The user is responsible that messages to the old address arrive at the new address, for example by configuring the old provider to forward all emails to the new one.

draft/webxdc-dev-reference.md

@@ -1,6 +1,12 @@
 # Webxdc Developer Reference
 
-(This document may eventually be merged with the [webxdc guidebook](https://deltachat.github.io/webxdc_docs/), where you may currently find other useful information.)
+This document gives a quick overview about the Webxdc specification,
+It is meant for both, developing Webxdc apps
+and developing Webxdc implementations.
+
+The [Webxdc guidebook](https://deltachat.github.io/webxdc_docs/) shows more detailed information
+when developing Webxdc apps.
+
 
 ## Webxdc File Format
 
@@ -34,8 +40,9 @@ To get a shared state, the peers use `sendUpdate()` to send updates to each othe
 - `update`: an object with the following properties:  
     - `update.payload`: any javascript primitive, array or object.
     - `update.info`: optional, short, informational message that will be added to the chat,
-       eg. "Alice voted" or "Bob scored 123 in MyGame";
-       usually only one line of text is shown,
+       eg. "Alice voted" or "Bob scored 123 in MyGame".
+       usually only one line of text is shown
+       and if there are series of info messages, older ones may be dropped.
        use this option sparingly to not spam the chat.
     - `update.document`: optional, name of the document in edit,
        must not be used eg. in games where the Webxdc does not create documents
@@ -137,9 +144,37 @@ round corners etc. will be added by the implementations as needed.
 If no icon is set, a default icon will be used.
 
 
+## Other APIs and Tags Usage Hints
+
+- `localStorage`, `sessionStorage`, `indexedDB` are okay to be used
+- `visibilitychange`-events are okay to be used
+- `window.navigator.language` is okay to be used, on desktop it is the system language
+- `<a href="localfile.html">` and other internal links are okay to be used
+- `<a href="mailto:addr@example.org?body=...">`- mailto links are okay to be used
+- `<meta name="viewport" ...>` usage is okay to be used
+  and useful esp. different webviews have different defaults
+
+
+### Discouraged Things
+
+- `document.cookie` is known not to work on desktop and iOS
+  use `localStorage` instead
+- `unload`-, `beforeunload`- and `pagehide`-events are known not to work on iOS and are flaky on other systems
+  (also partly discouraged by [mozilla](https://developer.mozilla.org/en-US/docs/Web/API/Window/unload_event))
+  use `visibilitychange` instead
+- `<title>` and `document.title` is ignored by Webxdc;
+  use the `name` property from `manifest.toml` instead
+- newest js features may not work on all webviews,
+  you may want to transpile your code down to an older js version
+  eg. with <https://babeljs.io>
+- `<a href="https://example.org/foo">` and other external links are blocked by definition;
+  instead, embed content or use `mailto:` link to offer a way for contact
+- `<input type="file">` is discouraged currently; this may change in future
+
+
 ## Webxdc Examples
 
-The following example shows an input field and  every input is show on all peers.
+The following example shows an input field and every input is show on all peers.
 
 ```html
 <!DOCTYPE html>
@@ -169,30 +204,10 @@ The following example shows an input field and  every input is show on all peers
 </html>
 ```
 
-[Webxdc Development Tool](https://github.com/deltachat/webxdc-dev)
-offers an **Webxdc Simulator** that can be used in many browsers without any installation needed.
+More examples at [github.com/webxdc](https://github.com/webxdc) and
+[topic #webxdc](https://github.com/topics/webxdc)
+
+[github.com/webxdc/hello](https://github.com/webxdc/hello)
+offers an **Webxdc Tool** that can be used in many browsers without any installation needed.
 You can also use that repository as a template for your own Webxdc -
 just clone and start adapting things to your need.
-
-
-### Advanced Examples
-
-- [2048](https://github.com/adbenitez/2048.xdc)
-- [Draw](https://github.com/adbenitez/draw.xdc)
-- [Poll](https://github.com/r10s/webxdc-poll/)
-- [Tic Tac Toe](https://github.com/Simon-Laux/tictactoe.xdc)
-- Even more with [Topic #webxdc on Github](https://github.com/topics/webxdc) or in the [webxdc GitHub organization](https://github.com/webxdc)
-
-
-## Closing Remarks
-
-- older devices might not have the newest js features in their webview,
-  you may want to transpile your code down to an older js version eg. with https://babeljs.io
-- viewport and scaling features are implementation specific,
-  if you want to have an explicit behavior, you can add eg.
-  `<meta name="viewport" content="initial-scale=1; user-scalable=no">` to your Webxdc
-- the `<title>` tag should not be used and its content is usually not displayed;
-  instead, use the `name` property from `manifest.toml`
-- there are tons of ideas for enhancements of the API and the file format,
-  eg. in the future, we will may define icon- and manifest-files,
-  allow to aggregate the state or add metadata.

examples/repl/main.rs

@@ -19,7 +19,7 @@ use deltachat::config;
 use deltachat::context::*;
 use deltachat::oauth2::*;
 use deltachat::securejoin::*;
-use deltachat::EventType;
+use deltachat::{EventType, Events};
 use log::{error, info, warn};
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
 use rustyline::config::OutputStreamType;
@@ -298,7 +298,7 @@ async fn start(args: Vec<String>) -> Result<(), Error> {
         println!("Error: Bad arguments, expected [db-name].");
         bail!("No db-name specified");
     }
-    let context = Context::new(Path::new(&args[1]).to_path_buf(), 0).await?;
+    let context = Context::new(Path::new(&args[1]).to_path_buf(), 0, Events::new()).await?;
 
     let events = context.get_event_emitter();
     async_std::task::spawn(async move {

examples/simple.rs

@@ -6,7 +6,7 @@ use deltachat::config;
 use deltachat::contact::*;
 use deltachat::context::*;
 use deltachat::message::Message;
-use deltachat::EventType;
+use deltachat::{EventType, Events};
 
 fn cb(event: EventType) {
     match event {
@@ -36,7 +36,7 @@ async fn main() {
     let dir = tempdir().unwrap();
     let dbfile = dir.path().join("db.sqlite");
     log::info!("creating database {:?}", dbfile);
-    let ctx = Context::new(dbfile.into(), 0)
+    let ctx = Context::new(dbfile.into(), 0, Events::new())
         .await
         .expect("Failed to create context");
     let info = ctx.get_info().await;

node/README.md

@@ -60,6 +60,19 @@ building from source or clone this repository and follow this steps:
 > not inside this folder. (We need this in order to include the rust source
 > code in the npm package.)
 
+### Use a git branch in deltachat-desktop
+
+You can directly install a core branch, but make sure:
+- that you have typescript in your project dependencies, as it is likely required
+- you know that there are **no prebuilds** and so core is built during installation which is why it takes so long
+
+```
+npm install https://github.com/deltachat/deltachat-core-rust.git#branch
+```
+
+If you want prebuilds for a branch that has a core pr, you might find an npm tar.gz package for that branch at <https://download.delta.chat/node/preview/>.
+The github ci also posts a link to it in the checks for each pr.
+
 ### Use build-from-source in deltachat-desktop
 
 If you want to use the manually built node bindings in the desktop client (for

node/lib/deltachat.ts

@@ -19,10 +19,11 @@ interface NativeAccount {}
 export class AccountManager extends EventEmitter {
   dcn_accounts: NativeAccount
   accountDir: string
+  jsonRpcStarted = false
 
   constructor(cwd: string, os = 'deltachat-node') {
-    debug('DeltaChat constructor')
     super()
+    debug('DeltaChat constructor')
 
     this.accountDir = cwd
     this.dcn_accounts = binding.dcn_accounts_new(os, this.accountDir)
@@ -114,6 +115,31 @@ export class AccountManager extends EventEmitter {
     debug('Started event handler')
   }
 
+  startJSONRPCHandler(callback: ((response: string) => void) | null) {
+    if (this.dcn_accounts === null) {
+      throw new Error('dcn_account is null')
+    }
+    if (!callback) {
+      throw new Error('no callback set')
+    }
+    if (this.jsonRpcStarted) {
+      throw new Error('jsonrpc was started already')
+    }
+
+    binding.dcn_accounts_start_jsonrpc(this.dcn_accounts, callback.bind(this))
+    debug('Started jsonrpc handler')
+    this.jsonRpcStarted = true
+  }
+
+  jsonRPCRequest(message: string) {
+    if (!this.jsonRpcStarted) {
+      throw new Error(
+        'jsonrpc is not active, start it with startJSONRPCHandler first'
+      )
+    }
+    binding.dcn_json_rpc_request(this.dcn_accounts, message)
+  }
+
   startIO() {
     binding.dcn_accounts_start_io(this.dcn_accounts)
   }

node/scripts/generate-constants.js

@@ -1,73 +1,65 @@
 #!/usr/bin/env node
-
 const fs = require('fs')
 const path = require('path')
-const split = require('split2')
 
 const data = []
-const regex = /^#define\s+(\w+)\s+(\w+)/i
-const header = path.resolve(
-  __dirname,
-  '../../deltachat-ffi/deltachat.h'
-)
+const header = path.resolve(__dirname, '../../deltachat-ffi/deltachat.h')
 
 console.log('Generating constants...')
 
-fs.createReadStream(header)
-  .pipe(split())
-  .on('data', (line) => {
-    const match = regex.exec(line)
-    if (match) {
-      const key = match[1]
-      const value = parseInt(match[2])
-      if (isNaN(value)) return
+const header_data = fs.readFileSync(header, 'UTF-8')
+const regex = /^#define\s+(\w+)\s+(\w+)/gm
+while (null != (match = regex.exec(header_data))) {
+  const key = match[1]
+  const value = parseInt(match[2])
+  if (!isNaN(value)) {
+    data.push({ key, value })
+  }
+}
 
-      data.push({ key, value })
-    }
+delete header_data
+
+const constants = data
+  .filter(
+    ({ key }) => key.toUpperCase()[0] === key[0] // check if define name is uppercase
+  )
+  .sort((lhs, rhs) => {
+    if (lhs.key < rhs.key) return -1
+    else if (lhs.key > rhs.key) return 1
+    return 0
   })
-  .on('end', () => {
-    const constants = data
-      .filter(
-        ({ key }) => key.toUpperCase()[0] === key[0] // check if define name is uppercase
-      )
-      .sort((lhs, rhs) => {
-        if (lhs.key < rhs.key) return -1
-        else if (lhs.key > rhs.key) return 1
-        return 0
-      })
-      .map((row) => {
-        return `  ${row.key}: ${row.value}`
-      })
-      .join(',\n')
+  .map((row) => {
+    return `  ${row.key}: ${row.value}`
+  })
+  .join(',\n')
 
-    const events = data
-      .sort((lhs, rhs) => {
-        if (lhs.value < rhs.value) return -1
-        else if (lhs.value > rhs.value) return 1
-        return 0
-      })
-      .filter((i) => {
-        return i.key.startsWith('DC_EVENT_')
-      })
-      .map((i) => {
-        return `  ${i.value}: '${i.key}'`
-      })
-      .join(',\n')
+const events = data
+  .sort((lhs, rhs) => {
+    if (lhs.value < rhs.value) return -1
+    else if (lhs.value > rhs.value) return 1
+    return 0
+  })
+  .filter((i) => {
+    return i.key.startsWith('DC_EVENT_')
+  })
+  .map((i) => {
+    return `  ${i.value}: '${i.key}'`
+  })
+  .join(',\n')
 
-    // backwards compat
-    fs.writeFileSync(
-      path.resolve(__dirname, '../constants.js'),
-      `// Generated!\n\nmodule.exports = {\n${constants}\n}\n`
-    )
-    // backwards compat
-    fs.writeFileSync(
-      path.resolve(__dirname, '../events.js'),
-      `/* eslint-disable quotes */\n// Generated!\n\nmodule.exports = {\n${events}\n}\n`
-    )
+// backwards compat
+fs.writeFileSync(
+  path.resolve(__dirname, '../constants.js'),
+  `// Generated!\n\nmodule.exports = {\n${constants}\n}\n`
+)
+// backwards compat
+fs.writeFileSync(
+  path.resolve(__dirname, '../events.js'),
+  `/* eslint-disable quotes */\n// Generated!\n\nmodule.exports = {\n${events}\n}\n`
+)
 
-    fs.writeFileSync(
-      path.resolve(__dirname, '../lib/constants.ts'),
-      `// Generated!\n\nexport enum C {\n${constants.replace(/:/g, ' =')},\n}\n
+fs.writeFileSync(
+  path.resolve(__dirname, '../lib/constants.ts'),
+  `// Generated!\n\nexport enum C {\n${constants.replace(/:/g, ' =')},\n}\n
 // Generated!\n\nexport const EventId2EventName: { [key: number]: string } = {\n${events},\n}\n`
-    )
-  })
+)

node/scripts/postLinksToDetails.js

@@ -17,7 +17,7 @@ const STATUS_DATA = {
   state: 'success',
   description: '⏩ Click on "Details" to download →',
   context: 'Download the node-bindings.tar.gz',
-  target_url: base_url + file_url + '.tar.gz',
+  target_url: base_url + file_url,
 }
 
 const http = require('https')

node/scripts/rebuild-core.js

@@ -9,7 +9,7 @@ const buildArgs = [
   'build',
   '--release',
   '--features',
-  'vendored',
+  'vendored,jsonrpc',
   '-p',
   'deltachat_ffi'
 ]

node/src/module.c

@@ -34,6 +34,9 @@ typedef struct dcn_accounts_t {
   dc_accounts_t* dc_accounts;
   napi_threadsafe_function threadsafe_event_handler;
   uv_thread_t event_handler_thread;
+  napi_threadsafe_function threadsafe_jsonrpc_handler;
+  uv_thread_t jsonrpc_thread;
+  dc_jsonrpc_instance_t* jsonrpc_instance;
   int gc;
 } dcn_accounts_t;
 
@@ -326,7 +329,7 @@ static void call_js_event_handler(napi_env env, napi_value js_callback, void* _c
 
   if (status != napi_ok) {
     TRACE("Unable to call event_handler callback2");
-    napi_extended_error_info* error_result;
+    const napi_extended_error_info* error_result;
     NAPI_STATUS_THROWS(napi_get_last_error_info(env, &error_result));
   }
 }
@@ -348,7 +351,7 @@ NAPI_METHOD(dcn_start_event_handler) {
     callback,
     0,
     async_resource_name,
-    1,
+    1000, // max_queue_size
     1,
     NULL,
     NULL,
@@ -371,6 +374,11 @@ NAPI_METHOD(dcn_context_unref) {
 
   TRACE("Unrefing dc_context");
   dcn_context->gc = 1;
+  if (dcn_context->event_handler_thread != 0) {
+    dc_stop_io(dcn_context->dc_context);
+    uv_thread_join(&dcn_context->event_handler_thread);
+    dcn_context->event_handler_thread = 0;
+  }
   dc_context_unref(dcn_context->dc_context);
   dcn_context->dc_context = NULL;
 
@@ -2922,6 +2930,16 @@ NAPI_METHOD(dcn_accounts_unref) {
 
   TRACE("Unrefing dc_accounts");
   dcn_accounts->gc = 1;
+  if (dcn_accounts->event_handler_thread != 0) {
+    dc_accounts_stop_io(dcn_accounts->dc_accounts);
+    uv_thread_join(&dcn_accounts->event_handler_thread);
+    dcn_accounts->event_handler_thread = 0;
+  }
+  if (dcn_accounts->jsonrpc_instance) {
+    dc_jsonrpc_request(dcn_accounts->jsonrpc_instance, "{}");
+    uv_thread_join(&dcn_accounts->jsonrpc_thread);
+    dcn_accounts->jsonrpc_instance = NULL;
+  }
   dc_accounts_unref(dcn_accounts->dc_accounts);
   dcn_accounts->dc_accounts = NULL;
 
@@ -3080,8 +3098,6 @@ static void accounts_event_handler_thread_func(void* arg)
 {
   dcn_accounts_t* dcn_accounts = (dcn_accounts_t*)arg;
 
-
-
   TRACE("event_handler_thread_func starting");
 
   dc_accounts_event_emitter_t * dc_accounts_event_emitter = dc_accounts_get_event_emitter(dcn_accounts->dc_accounts);
@@ -3093,8 +3109,8 @@ static void accounts_event_handler_thread_func(void* arg)
     }
     event = dc_accounts_get_next_event(dc_accounts_event_emitter);
     if (event == NULL) {
-      //TRACE("received NULL event, skipping");
-      continue;
+      TRACE("no more events");
+      break;
     }
 
     if (!dcn_accounts->threadsafe_event_handler) {
@@ -3195,7 +3211,7 @@ static void call_accounts_js_event_handler(napi_env env, napi_value js_callback,
 
   if (status != napi_ok) {
     TRACE("Unable to call event_handler callback2");
-    napi_extended_error_info* error_result;
+    const napi_extended_error_info* error_result;
     NAPI_STATUS_THROWS(napi_get_last_error_info(env, &error_result));
   }
 }
@@ -3216,7 +3232,7 @@ NAPI_METHOD(dcn_accounts_start_event_handler) {
     callback,
     0,
     async_resource_name,
-    1,
+    1000, // max_queue_size
     1,
     NULL,
     NULL,
@@ -3232,6 +3248,125 @@ NAPI_METHOD(dcn_accounts_start_event_handler) {
   NAPI_RETURN_UNDEFINED();
 }
 
+// JSON RPC
+
+static void accounts_jsonrpc_thread_func(void* arg)
+{
+  dcn_accounts_t* dcn_accounts = (dcn_accounts_t*)arg;
+  TRACE("accounts_jsonrpc_thread_func starting");
+  char* response;
+  while (true) {
+    response = dc_jsonrpc_next_response(dcn_accounts->jsonrpc_instance);
+    if (response == NULL) {
+      // done or broken
+      break;
+    }
+
+    if (!dcn_accounts->threadsafe_jsonrpc_handler) {
+      TRACE("threadsafe_jsonrpc_handler not set, bailing");
+      break;
+    }
+    // Don't process events if we're being garbage collected!
+    if (dcn_accounts->gc == 1) {
+      TRACE("dc_accounts has been destroyed, bailing");
+      break;
+    }
+
+    napi_status status = napi_call_threadsafe_function(dcn_accounts->threadsafe_jsonrpc_handler, response, napi_tsfn_blocking);
+
+    if (status == napi_closing) {
+      TRACE("JS function got released, bailing");
+      break;
+    }
+  }
+  dc_jsonrpc_unref(dcn_accounts->jsonrpc_instance);
+  dcn_accounts->jsonrpc_instance = NULL;
+  TRACE("accounts_jsonrpc_thread_func ended");
+  napi_release_threadsafe_function(dcn_accounts->threadsafe_jsonrpc_handler, napi_tsfn_release);
+}
+
+static void call_accounts_js_jsonrpc_handler(napi_env env, napi_value js_callback, void* _context, void* data)
+{
+  char* response = (char*)data;
+  napi_value global;
+  napi_status status = napi_get_global(env, &global);
+  if (status != napi_ok) {
+    napi_throw_error(env, NULL, "Unable to get global");
+  }
+
+  napi_value argv[1];
+  if (response != 0) {
+    status = napi_create_string_utf8(env, response, NAPI_AUTO_LENGTH, &argv[0]);
+  } else {
+    status = napi_create_string_utf8(env, "", NAPI_AUTO_LENGTH, &argv[0]);
+  }
+  if (status != napi_ok) {
+    napi_throw_error(env, NULL, "Unable to create argv for js jsonrpc_handler arguments");
+  } 
+  free(response);
+
+  TRACE("calling back into js");
+  napi_value result;
+  status = napi_call_function(
+    env,
+    global,
+    js_callback,
+    1,
+    argv,
+    &result);
+  if (status != napi_ok) {
+    TRACE("Unable to call jsonrpc_handler callback2");
+    const napi_extended_error_info* error_result;
+    NAPI_STATUS_THROWS(napi_get_last_error_info(env, &error_result));
+  }
+}
+
+NAPI_METHOD(dcn_accounts_start_jsonrpc) {
+  NAPI_ARGV(2);
+  NAPI_DCN_ACCOUNTS();
+  napi_value callback = argv[1];
+
+  TRACE("calling..");
+  napi_value async_resource_name;
+  NAPI_STATUS_THROWS(napi_create_string_utf8(env, "dc_accounts_jsonrpc_callback", NAPI_AUTO_LENGTH, &async_resource_name));
+
+  TRACE("creating threadsafe function..");
+
+  NAPI_STATUS_THROWS(napi_create_threadsafe_function(
+    env,
+    callback,
+    0,
+    async_resource_name,
+    1,
+    1,
+    NULL,
+    NULL,
+    dcn_accounts,
+    call_accounts_js_jsonrpc_handler,
+    &dcn_accounts->threadsafe_jsonrpc_handler));
+  TRACE("done");
+
+  dcn_accounts->gc = 0;
+  dcn_accounts->jsonrpc_instance = dc_jsonrpc_init(dcn_accounts->dc_accounts);
+
+  TRACE("creating uv thread..");
+  uv_thread_create(&dcn_accounts->jsonrpc_thread, accounts_jsonrpc_thread_func, dcn_accounts);
+
+  NAPI_RETURN_UNDEFINED();
+}
+
+NAPI_METHOD(dcn_json_rpc_request) {
+  NAPI_ARGV(2);
+  NAPI_DCN_ACCOUNTS();
+  if (!dcn_accounts->jsonrpc_instance) {
+    const char* msg = "dcn_accounts->jsonrpc_instance is null, have you called dcn_accounts_start_jsonrpc()?";
+    NAPI_STATUS_THROWS(napi_throw_type_error(env, NULL, msg));
+  }
+  NAPI_ARGV_UTF8_MALLOC(request, 1);
+  dc_jsonrpc_request(dcn_accounts->jsonrpc_instance, request);
+  free(request);
+}
+
 
 NAPI_INIT() {
   /**
@@ -3502,4 +3637,9 @@ NAPI_INIT() {
   NAPI_EXPORT_FUNCTION(dcn_send_webxdc_status_update);
   NAPI_EXPORT_FUNCTION(dcn_get_webxdc_status_updates);
   NAPI_EXPORT_FUNCTION(dcn_msg_get_webxdc_blob);
+
+
+  /** jsonrpc **/
+  NAPI_EXPORT_FUNCTION(dcn_accounts_start_jsonrpc);
+  NAPI_EXPORT_FUNCTION(dcn_json_rpc_request);
 }

node/src/napi-macros-extensions.h

@@ -23,7 +23,7 @@
   dcn_accounts_t* dcn_accounts; \
   NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], (void**)&dcn_accounts)); \
   if (!dcn_accounts) { \
-    const char* msg = "Provided dnc_acounts is null"; \
+    const char* msg = "Provided dcn_acounts is null"; \
     NAPI_STATUS_THROWS(napi_throw_type_error(env, NULL, msg)); \
   } \
   if (!dcn_accounts->dc_accounts) { \

node/test/test.js

@@ -2,7 +2,7 @@
 import DeltaChat, { Message } from '../dist'
 import binding from '../binding'
 
-import { strictEqual } from 'assert'
+import { deepEqual, deepStrictEqual, strictEqual } from 'assert'
 import chai, { expect } from 'chai'
 import chaiAsPromised from 'chai-as-promised'
 import { EventId2EventName, C } from '../dist/constants'
@@ -84,6 +84,95 @@ describe('static tests', function () {
   })
 })
 
+describe('JSON RPC', function () {
+  it('smoketest', async function () {
+    const { dc } = DeltaChat.newTemporary()
+    let promise_resolve
+    const promise = new Promise((res, _rej) => {
+      promise_resolve = res
+    })
+    dc.startJSONRPCHandler(promise_resolve)
+    dc.jsonRPCRequest(
+      JSON.stringify({
+        jsonrpc: '2.0',
+        method: 'get_all_account_ids',
+        params: [],
+        id: 2,
+      })
+    )
+    deepStrictEqual(
+      {
+        jsonrpc: '2.0',
+        id: 2,
+        result: [1],
+      },
+      JSON.parse(await promise)
+    )
+    dc.close()
+  })
+
+  it('basic test', async function () {
+    const { dc } = DeltaChat.newTemporary()
+
+    const promises = {}
+    dc.startJSONRPCHandler((msg) => {
+      const response = JSON.parse(msg)
+      promises[response.id](response)
+      delete promises[response.id]
+    })
+    const call = (request) => {
+      dc.jsonRPCRequest(JSON.stringify(request))
+      return new Promise((res, _rej) => {
+        promises[request.id] = res
+      })
+    }
+
+    deepStrictEqual(
+      {
+        jsonrpc: '2.0',
+        id: 2,
+        result: [1],
+      },
+      await call({
+        jsonrpc: '2.0',
+        method: 'get_all_account_ids',
+        params: [],
+        id: 2,
+      })
+    )
+
+    deepStrictEqual(
+      {
+        jsonrpc: '2.0',
+        id: 3,
+        result: 2,
+      },
+      await call({
+        jsonrpc: '2.0',
+        method: 'add_account',
+        params: [],
+        id: 3,
+      })
+    )
+
+    deepStrictEqual(
+      {
+        jsonrpc: '2.0',
+        id: 4,
+        result: [1, 2],
+      },
+      await call({
+        jsonrpc: '2.0',
+        method: 'get_all_account_ids',
+        params: [],
+        id: 4,
+      })
+    )
+
+    dc.close()
+  })
+})
+
 describe('Basic offline Tests', function () {
   it('opens a context', async function () {
     const { dc, context } = DeltaChat.newTemporary()
@@ -128,7 +217,7 @@ describe('Basic offline Tests', function () {
 
     await expect(
       context.configure({ addr: 'delta1@delta.localhost' })
-    ).to.eventually.be.rejectedWith('Please enter a password.')
+    ).to.eventually.be.rejectedWith('Missing (IMAP) password.')
     await expect(context.configure({ mailPw: 'delta1' })).to.eventually.be
       .rejected
 
@@ -601,8 +690,8 @@ describe('Offline Tests with unconfigured account', function () {
     strictEqual(chatList.getCount(), 1, 'only one archived')
   })
 
-  it('Remove qoute from (draft) message', function () {
-    context.addDeviceMessage('test_qoute', 'test')
+  it('Remove quote from (draft) message', function () {
+    context.addDeviceMessage('test_quote', 'test')
     const msgId = context.getChatMessages(10, 0, 0)[0]
     const msg = context.messageNew()
 

package.json

@@ -1,9 +1,9 @@
 {
   "dependencies": {
+    "@deltachat/jsonrpc-client": "file:deltachat-jsonrpc/typescript",
     "debug": "^4.1.1",
     "napi-macros": "^2.0.0",
-    "node-gyp-build": "^4.1.0",
-    "split2": "^3.1.1"
+    "node-gyp-build": "^4.1.0"
   },
   "description": "node.js bindings for deltachat-core",
   "devDependencies": {
@@ -42,7 +42,7 @@
     "build": "npm run build:core && npm run build:bindings",
     "build:bindings": "npm run build:bindings:c && npm run build:bindings:ts",
     "build:bindings:c": "npm run build:bindings:c:c && npm run build:bindings:c:postinstall",
-    "build:bindings:c:c": "cd node && npx node-gyp rebuild",
+    "build:bindings:c:c": "cd node && node-gyp rebuild",
     "build:bindings:c:postinstall": "node node/scripts/postinstall.js",
     "build:bindings:ts": "cd node && tsc",
     "build:core": "npm run build:core:rust && npm run build:core:constants",
@@ -58,8 +58,8 @@
     "prebuildify": "cd node && prebuildify -t 16.13.0 --napi --strip --postinstall \"node scripts/postinstall.js --prebuild\"",
     "test": "npm run test:lint && npm run test:mocha",
     "test:lint": "npm run lint",
-    "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec"
+    "test:mocha": "mocha -r esm node/test/test.js --growl --reporter=spec --bail"
   },
   "types": "node/dist/index.d.ts",
-  "version": "1.82.0"
-}
+  "version": "1.86.0"
+}

python/doc/_templates/globaltoc.html

@@ -11,8 +11,7 @@
 <ul>
   <li><a href="https://github.com/deltachat/deltachat-core-rust">github repository</a></li>
   <li><a href="https://pypi.python.org/pypi/deltachat">pypi: deltachat</a></li>
+  <li><a href="https://web.libera.chat/#deltachat">#deltachat</a></li>
 </ul>
 
-<b>#deltachat [freenode]</b>
-
 </div>

python/examples/echo_and_quit.py

@@ -1,4 +1,3 @@
-
 # content of echo_and_quit.py
 
 from deltachat import account_hookimpl, run_cmdline

python/examples/group_tracking.py

@@ -1,4 +1,3 @@
-
 # content of group_tracking.py
 
 from deltachat import account_hookimpl, run_cmdline
@@ -33,15 +32,21 @@ class GroupTrackingPlugin:
 
     @account_hookimpl
     def ac_member_added(self, chat, contact, actor, message):
-        print("ac_member_added {} to chat {} from {}".format(
-            contact.addr, chat.id, actor or message.get_sender_contact().addr))
+        print(
+            "ac_member_added {} to chat {} from {}".format(
+                contact.addr, chat.id, actor or message.get_sender_contact().addr
+            )
+        )
         for member in chat.get_contacts():
             print("chat member: {}".format(member.addr))
 
     @account_hookimpl
     def ac_member_removed(self, chat, contact, actor, message):
-        print("ac_member_removed {} from chat {} by {}".format(
-            contact.addr, chat.id, actor or message.get_sender_contact().addr))
+        print(
+            "ac_member_removed {} from chat {} by {}".format(
+                contact.addr, chat.id, actor or message.get_sender_contact().addr
+            )
+        )
 
 
 def main(argv=None):

python/examples/test_examples.py

@@ -1,20 +1,20 @@
-
-import pytest
-import py
 import echo_and_quit
 import group_tracking
+import py
+import pytest
+
 from deltachat.events import FFIEventLogger
 
 
-@pytest.fixture(scope='session')
+@pytest.fixture(scope="session")
 def datadir():
     """The py.path.local object of the test-data/ directory."""
     for path in reversed(py.path.local(__file__).parts()):
-        datadir = path.join('test-data')
+        datadir = path.join("test-data")
         if datadir.isdir():
             return datadir
     else:
-        pytest.skip('test-data directory not found')
+        pytest.skip("test-data directory not found")
 
 
 def test_echo_quit_plugin(acfactory, lp):
@@ -22,7 +22,7 @@ def test_echo_quit_plugin(acfactory, lp):
     botproc = acfactory.run_bot_process(echo_and_quit)
 
     lp.sec("creating a temp account to contact the bot")
-    ac1, = acfactory.get_online_accounts(1)
+    (ac1,) = acfactory.get_online_accounts(1)
 
     lp.sec("sending a message to the bot")
     bot_contact = ac1.create_contact(botproc.addr)
@@ -44,9 +44,11 @@ def test_group_tracking_plugin(acfactory, lp):
 
     ac1, ac2 = acfactory.get_online_accounts(2)
 
-    botproc.fnmatch_lines("""
+    botproc.fnmatch_lines(
+        """
         *ac_configure_completed*
-    """)
+    """
+    )
     ac1.add_account_plugin(FFIEventLogger(ac1))
     ac2.add_account_plugin(FFIEventLogger(ac2))
 
@@ -56,9 +58,11 @@ def test_group_tracking_plugin(acfactory, lp):
     ch.add_contact(bot_contact)
     ch.send_text("hello")
 
-    botproc.fnmatch_lines("""
+    botproc.fnmatch_lines(
+        """
         *ac_chat_modified*bot test group*
-    """)
+    """
+    )
 
     lp.sec("adding third member {}".format(ac2.get_config("addr")))
     contact3 = ac1.create_contact(ac2.get_config("addr"))
@@ -68,12 +72,20 @@ def test_group_tracking_plugin(acfactory, lp):
     assert "hello" in reply.text
 
     lp.sec("now looking at what the bot received")
-    botproc.fnmatch_lines("""
+    botproc.fnmatch_lines(
+        """
         *ac_member_added {}*from*{}*
-    """.format(contact3.addr, ac1.get_config("addr")))
+    """.format(
+            contact3.addr, ac1.get_config("addr")
+        )
+    )
 
     lp.sec("contact successfully added, now removing")
     ch.remove_contact(contact3)
-    botproc.fnmatch_lines("""
+    botproc.fnmatch_lines(
+        """
         *ac_member_removed {}*from*{}*
-    """.format(contact3.addr, ac1.get_config("addr")))
+    """.format(
+            contact3.addr, ac1.get_config("addr")
+        )
+    )

python/install_python_bindings.py

@@ -18,15 +18,13 @@ if __name__ == "__main__":
 
     cmd = ["cargo", "build", "-p", "deltachat_ffi"]
 
-    if target == 'release':
+    if target == "release":
         os.environ["CARGO_PROFILE_RELEASE_LTO"] = "on"
         cmd.append("--release")
 
     print("running:", " ".join(cmd))
     subprocess.check_call(cmd)
-    subprocess.check_call("rm -rf build/ src/deltachat/*.so src/deltachat/*.dylib src/deltachat/*.dll" , shell=True)
+    subprocess.check_call("rm -rf build/ src/deltachat/*.so src/deltachat/*.dylib src/deltachat/*.dll", shell=True)
 
     if len(sys.argv) <= 1 or sys.argv[1] != "onlybuild":
-        subprocess.check_call([
-            sys.executable, "-m", "pip", "install", "-e", "."
-        ])
+        subprocess.check_call([sys.executable, "-m", "pip", "install", "-e", "."])

python/pyproject.toml

@@ -6,3 +6,9 @@ build-backend = "setuptools.build_meta"
 root = ".."
 tag_regex = '^(?P<prefix>py-)?(?P<version>[^\+]+)(?P<suffix>.*)?$'
 git_describe_command = "git describe --dirty --tags --long --match py-*.*"
+
+[tool.black]
+line-length = 120
+
+[tool.isort]
+profile = "black"

python/setup.py

@@ -1,36 +1,37 @@
-import setuptools
 import os
 import re
 
+import setuptools
+
 
 def main():
     with open("README.rst") as f:
         long_description = f.read()
     setuptools.setup(
-        name='deltachat',
-        description='Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat',
+        name="deltachat",
+        description="Python bindings for the Delta Chat Core library using CFFI against the Rust-implemented libdeltachat",
         long_description=long_description,
-        author='holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors',
-        install_requires=['cffi>=1.0.0', 'pluggy', 'imap-tools', 'requests'],
+        author="holger krekel, Floris Bruynooghe, Bjoern Petersen and contributors",
+        install_requires=["cffi>=1.0.0", "pluggy", "imap-tools", "requests"],
         setup_requires=[
-            'setuptools_scm', # required for compatibility with `python3 setup.py sdist`
-            'pkgconfig',
+            "setuptools_scm",  # required for compatibility with `python3 setup.py sdist`
+            "pkgconfig",
         ],
-        packages=setuptools.find_packages('src'),
-        package_dir={'': 'src'},
-        cffi_modules=['src/deltachat/_build.py:ffibuilder'],
-        entry_points = {
-            'pytest11': [
-                'deltachat.testplugin = deltachat.testplugin',
+        packages=setuptools.find_packages("src"),
+        package_dir={"": "src"},
+        cffi_modules=["src/deltachat/_build.py:ffibuilder"],
+        entry_points={
+            "pytest11": [
+                "deltachat.testplugin = deltachat.testplugin",
             ],
         },
         classifiers=[
-            'Development Status :: 4 - Beta',
-            'Intended Audience :: Developers',
-            'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
-            'Programming Language :: Python :: 3',
-            'Topic :: Communications :: Email',
-            'Topic :: Software Development :: Libraries',
+            "Development Status :: 4 - Beta",
+            "Intended Audience :: Developers",
+            "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
+            "Programming Language :: Python :: 3",
+            "Topic :: Communications :: Email",
+            "Topic :: Software Development :: Libraries",
         ],
     )
 

python/src/deltachat/__init__.py

@@ -1,15 +1,15 @@
 import sys
 
-from . import capi, const, hookspec # noqa
-from .capi import ffi  # noqa
-from .account import Account, get_core_info  # noqa
-from .message import Message  # noqa
-from .contact import Contact  # noqa
-from .chat import Chat        # noqa
-from .hookspec import account_hookimpl, global_hookimpl  # noqa
-from . import events
+from pkg_resources import DistributionNotFound, get_distribution
+
+from . import capi, const, events, hookspec  # noqa
+from .account import Account, get_core_info  # noqa
+from .capi import ffi  # noqa
+from .chat import Chat  # noqa
+from .contact import Contact  # noqa
+from .hookspec import account_hookimpl, global_hookimpl  # noqa
+from .message import Message  # noqa
 
-from pkg_resources import get_distribution, DistributionNotFound
 try:
     __version__ = get_distribution(__name__).version
 except DistributionNotFound:
@@ -26,7 +26,7 @@ def get_dc_event_name(integer, _DC_EVENTNAME_MAP={}):
 
 
 def register_global_plugin(plugin):
-    """ Register a global plugin which implements one or more
+    """Register a global plugin which implements one or more
     of the :class:`deltachat.hookspec.Global` hooks.
     """
     gm = hookspec.Global._get_plugin_manager()
@@ -43,9 +43,10 @@ register_global_plugin(events)
 
 
 def run_cmdline(argv=None, account_plugins=None):
-    """ Run a simple default command line app, registering the specified
-    account plugins. """
+    """Run a simple default command line app, registering the specified
+    account plugins."""
     import argparse
+
     if argv is None:
         argv = sys.argv
 
@@ -69,9 +70,9 @@ def run_cmdline(argv=None, account_plugins=None):
         ac.add_account_plugin(plugin)
 
     if not ac.is_configured():
-        assert args.email and args.password, (
-            "you must specify --email and --password once to configure this database/account"
-        )
+        assert (
+            args.email and args.password
+        ), "you must specify --email and --password once to configure this database/account"
         ac.set_config("addr", args.email)
         ac.set_config("mail_pw", args.password)
         ac.set_config("mvbox_move", "0")

python/src/deltachat/_build.py

@@ -20,30 +20,33 @@ def local_build_flags(projdir, target):
     :param target: The rust build target, `debug` or `release`.
     """
     flags = {}
-    if platform.system() == 'Darwin':
-        flags['libraries'] = ['resolv', 'dl']
-        flags['extra_link_args'] = [
-                '-framework', 'CoreFoundation',
-                '-framework', 'CoreServices',
-                '-framework', 'Security',
+    if platform.system() == "Darwin":
+        flags["libraries"] = ["resolv", "dl"]
+        flags["extra_link_args"] = [
+            "-framework",
+            "CoreFoundation",
+            "-framework",
+            "CoreServices",
+            "-framework",
+            "Security",
         ]
-    elif platform.system() == 'Linux':
-        flags['libraries'] = ['rt', 'dl', 'm']
-        flags['extra_link_args'] = []
+    elif platform.system() == "Linux":
+        flags["libraries"] = ["rt", "dl", "m"]
+        flags["extra_link_args"] = []
     else:
         raise NotImplementedError("Compilation not supported yet on Windows, can you help?")
     target_dir = os.environ.get("CARGO_TARGET_DIR")
     if target_dir is None:
-        target_dir = os.path.join(projdir, 'target')
-    flags['extra_objects'] = [os.path.join(target_dir, target, 'libdeltachat.a')]
-    assert os.path.exists(flags['extra_objects'][0]), flags['extra_objects']
-    flags['include_dirs'] = [os.path.join(projdir, 'deltachat-ffi')]
+        target_dir = os.path.join(projdir, "target")
+    flags["extra_objects"] = [os.path.join(target_dir, target, "libdeltachat.a")]
+    assert os.path.exists(flags["extra_objects"][0]), flags["extra_objects"]
+    flags["include_dirs"] = [os.path.join(projdir, "deltachat-ffi")]
     return flags
 
 
 def system_build_flags():
     """Construct build flags for building against an installed libdeltachat."""
-    return pkgconfig.parse('deltachat')
+    return pkgconfig.parse("deltachat")
 
 
 def extract_functions(flags):
@@ -61,11 +64,13 @@ def extract_functions(flags):
         src_name = os.path.join(tmpdir, "include.h")
         dst_name = os.path.join(tmpdir, "expanded.h")
         with open(src_name, "w") as src_fp:
-            src_fp.write('#include <deltachat.h>')
-        cc.preprocess(source=src_name,
-                      output_file=dst_name,
-                      include_dirs=flags['include_dirs'],
-                      macros=[('PY_CFFI', '1')])
+            src_fp.write("#include <deltachat.h>")
+        cc.preprocess(
+            source=src_name,
+            output_file=dst_name,
+            include_dirs=flags["include_dirs"],
+            macros=[("PY_CFFI", "1")],
+        )
         with open(dst_name, "r") as dst_fp:
             return dst_fp.read()
     finally:
@@ -87,7 +92,9 @@ def find_header(flags):
         obj_name = os.path.join(tmpdir, "where.o")
         dst_name = os.path.join(tmpdir, "where")
         with open(src_name, "w") as src_fp:
-            src_fp.write(textwrap.dedent("""
+            src_fp.write(
+                textwrap.dedent(
+                    """
                 #include <stdio.h>
                 #include <deltachat.h>
 
@@ -95,18 +102,20 @@ def find_header(flags):
                     printf("%s", _dc_header_file_location());
                     return 0;
                 }
-            """))
+            """
+                )
+            )
         cwd = os.getcwd()
         try:
             os.chdir(tmpdir)
-            cc.compile(sources=["where.c"],
-                       include_dirs=flags['include_dirs'],
-                       macros=[("PY_CFFI_INC", "1")])
+            cc.compile(
+                sources=["where.c"],
+                include_dirs=flags["include_dirs"],
+                macros=[("PY_CFFI_INC", "1")],
+            )
         finally:
             os.chdir(cwd)
-        cc.link_executable(objects=[obj_name],
-                           output_progname="where",
-                           output_dir=tmpdir)
+        cc.link_executable(objects=[obj_name], output_progname="where", output_dir=tmpdir)
         return subprocess.check_output(dst_name)
     finally:
         shutil.rmtree(tmpdir)
@@ -123,7 +132,8 @@ def extract_defines(flags):
     cdef() method.
     """
     header = find_header(flags)
-    defines_re = re.compile(r"""
+    defines_re = re.compile(
+        r"""
         \#define\s+   # The start of a define.
         (             # Begin capturing group which captures the define name.
             (?:       # A nested group which is not captured, this allows us
@@ -151,26 +161,28 @@ def extract_defines(flags):
         )             # Close the capturing group, this contains
                       # the entire name e.g. DC_MSG_TEXT.
         \s+\S+        # Ensure there is whitespace followed by a value.
-    """, re.VERBOSE)
+    """,
+        re.VERBOSE,
+    )
     defines = []
     with open(header) as fp:
         for line in fp:
             match = defines_re.match(line)
             if match:
                 defines.append(match.group(1))
-    return '\n'.join('#define {} ...'.format(d) for d in defines)
+    return "\n".join("#define {} ...".format(d) for d in defines)
 
 
 def ffibuilder():
-    projdir = os.environ.get('DCC_RS_DEV')
+    projdir = os.environ.get("DCC_RS_DEV")
     if projdir:
-        target = os.environ.get('DCC_RS_TARGET', 'release')
+        target = os.environ.get("DCC_RS_TARGET", "release")
         flags = local_build_flags(projdir, target)
     else:
         flags = system_build_flags()
     builder = cffi.FFI()
     builder.set_source(
-        'deltachat.capi',
+        "deltachat.capi",
         """
             #include <deltachat.h>
             int dc_event_has_string_data(int e)
@@ -180,11 +192,13 @@ def ffibuilder():
         """,
         **flags,
     )
-    builder.cdef("""
+    builder.cdef(
+        """
         typedef int... time_t;
         void free(void *ptr);
         extern int dc_event_has_string_data(int);
-    """)
+    """
+    )
     function_defs = extract_functions(flags)
     defines = extract_defines(flags)
     builder.cdef(function_defs)
@@ -192,8 +206,9 @@ def ffibuilder():
     return builder
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     import os.path
-    pkgdir = os.path.join(os.path.dirname(__file__), '..')
+
+    pkgdir = os.path.join(os.path.dirname(__file__), "..")
     builder = ffibuilder()
     builder.compile(tmpdir=pkgdir, verbose=True)

python/src/deltachat/account.py

@@ -1,37 +1,46 @@
 """ Account class implementation. """
 
 from __future__ import print_function
+
+import os
+from array import array
 from contextlib import contextmanager
 from email.utils import parseaddr
 from threading import Event
-import os
-from array import array
-from . import const
+from typing import Any, Dict, Generator, List, Optional, Union
+
+from . import const, hookspec
 from .capi import ffi, lib
-from .cutil import as_dc_charpointer, from_dc_charpointer, from_optional_dc_charpointer, iter_array, DCLot
 from .chat import Chat
-from .message import Message
 from .contact import Contact
-from .tracker import ImexTracker, ConfigureTracker
-from . import hookspec
+from .cutil import (
+    DCLot,
+    as_dc_charpointer,
+    from_dc_charpointer,
+    from_optional_dc_charpointer,
+    iter_array,
+)
 from .events import EventThread
-from typing import Union, Any, Dict, Optional, List, Generator
+from .message import Message
+from .tracker import ConfigureTracker, ImexTracker
 
 
 class MissingCredentials(ValueError):
-    """ Account is missing `addr` and `mail_pw` config values. """
+    """Account is missing `addr` and `mail_pw` config values."""
 
 
 def get_core_info():
-    """ get some system info. """
+    """get some system info."""
     from tempfile import NamedTemporaryFile
 
     with NamedTemporaryFile() as path:
         path.close()
-        return get_dc_info_as_dict(ffi.gc(
-            lib.dc_context_new(as_dc_charpointer(""), as_dc_charpointer(path.name), ffi.NULL),
-            lib.dc_context_unref,
-        ))
+        return get_dc_info_as_dict(
+            ffi.gc(
+                lib.dc_context_new(as_dc_charpointer(""), as_dc_charpointer(path.name), ffi.NULL),
+                lib.dc_context_unref,
+            )
+        )
 
 
 def get_dc_info_as_dict(dc_context):
@@ -46,18 +55,22 @@ def get_dc_info_as_dict(dc_context):
 
 
 class Account(object):
-    """ Each account is tied to a sqlite database file which is fully managed
+    """Each account is tied to a sqlite database file which is fully managed
     by the underlying deltachat core library.  All public Account methods are
     meant to be memory-safe and return memory-safe objects.
     """
+
     MissingCredentials = MissingCredentials
 
-    def __init__(self, db_path, os_name=None, logging=True) -> None:
-        """ initialize account object.
+    def __init__(self, db_path, os_name=None, logging=True, closed=False) -> None:
+        """initialize account object.
 
         :param db_path: a path to the account database. The database
                         will be created if it doesn't exist.
-        :param os_name: this will be put to the X-Mailer header in outgoing messages
+        :param os_name: [Deprecated]
+        :param logging: enable logging for this account
+        :param closed: set to True to avoid automatically opening the account
+                       after creation.
         """
         # initialize per-account plugin system
         self._pm = hookspec.PerAccount._make_plugin_manager()
@@ -70,7 +83,7 @@ class Account(object):
             db_path = db_path.encode("utf8")
 
         self._dc_context = ffi.gc(
-            lib.dc_context_new(as_dc_charpointer(os_name), db_path, ffi.NULL),
+            lib.dc_context_new_closed(db_path) if closed else lib.dc_context_new(ffi.NULL, db_path, ffi.NULL),
             lib.dc_context_unref,
         )
         if self._dc_context == ffi.NULL:
@@ -82,12 +95,23 @@ class Account(object):
         hook = hookspec.Global._get_plugin_manager().hook
         hook.dc_account_init(account=self)
 
+    def open(self, passphrase: Optional[str] = None) -> bool:
+        """Open the account's database with the given passphrase.
+        This can only be used on a closed account. If the account is new, this
+        operation sets the database passphrase. For existing databases the passphrase
+        should be the one used to encrypt the database the first time.
+
+        :returns: True if the database is opened with this passphrase, False if the
+                  passphrase is incorrect or an error occurred.
+        """
+        return bool(lib.dc_context_open(self._dc_context, as_dc_charpointer(passphrase)))
+
     def disable_logging(self) -> None:
-        """ disable logging. """
+        """disable logging."""
         self._logging = False
 
     def enable_logging(self) -> None:
-        """ re-enable logging. """
+        """re-enable logging."""
         self._logging = True
 
     def __repr__(self):
@@ -102,11 +126,10 @@ class Account(object):
 
     def _check_config_key(self, name: str) -> None:
         if name not in self._configkeys:
-            raise KeyError("{!r} not a valid config key, existing keys: {!r}".format(
-                           name, self._configkeys))
+            raise KeyError("{!r} not a valid config key, existing keys: {!r}".format(name, self._configkeys))
 
     def get_info(self) -> Dict[str, str]:
-        """ return dictionary of built config parameters. """
+        """return dictionary of built config parameters."""
         return get_dc_info_as_dict(self._dc_context)
 
     def dump_account_info(self, logfile):
@@ -126,7 +149,7 @@ class Account(object):
         log("")
 
     def set_stock_translation(self, id: int, string: str) -> None:
-        """ set stock translation string.
+        """set stock translation string.
 
         :param id: id of stock string (const.DC_STR_*)
         :param value: string to set as new transalation
@@ -138,7 +161,7 @@ class Account(object):
             raise ValueError("could not set translation string")
 
     def set_config(self, name: str, value: Optional[str]) -> None:
-        """ set configuration values.
+        """set configuration values.
 
         :param name: config key name (unicode)
         :param value: value to set (unicode)
@@ -157,7 +180,7 @@ class Account(object):
         lib.dc_set_config(self._dc_context, namebytes, valuebytes)
 
     def get_config(self, name: str) -> str:
-        """ return unicode string value.
+        """return unicode string value.
 
         :param name: configuration key to lookup (eg "addr" or "mail_pw")
         :returns: unicode value
@@ -175,15 +198,17 @@ class Account(object):
 
         In other words, you don't need this.
         """
-        res = lib.dc_preconfigure_keypair(self._dc_context,
-                                          as_dc_charpointer(addr),
-                                          as_dc_charpointer(public),
-                                          as_dc_charpointer(secret))
+        res = lib.dc_preconfigure_keypair(
+            self._dc_context,
+            as_dc_charpointer(addr),
+            as_dc_charpointer(public),
+            as_dc_charpointer(secret),
+        )
         if res == 0:
             raise Exception("Failed to set key")
 
     def update_config(self, kwargs: Dict[str, Any]) -> None:
-        """ update config values.
+        """update config values.
 
         :param kwargs: name=value config settings for this account.
                        values need to be unicode.
@@ -193,18 +218,18 @@ class Account(object):
             self.set_config(key, value)
 
     def is_configured(self) -> bool:
-        """ determine if the account is configured already; an initial connection
+        """determine if the account is configured already; an initial connection
         to SMTP/IMAP has been verified.
 
         :returns: True if account is configured.
         """
-        return True if lib.dc_is_configured(self._dc_context) else False
+        return bool(lib.dc_is_configured(self._dc_context))
 
     def is_open(self) -> bool:
         """Determine if account is open
 
         :returns True if account is open."""
-        return True if lib.dc_context_is_open(self._dc_context) else False
+        return bool(lib.dc_context_is_open(self._dc_context))
 
     def set_avatar(self, img_path: Optional[str]) -> None:
         """Set self avatar.
@@ -219,18 +244,17 @@ class Account(object):
             self.set_config("selfavatar", img_path)
 
     def check_is_configured(self) -> None:
-        """ Raise ValueError if this account is not configured. """
+        """Raise ValueError if this account is not configured."""
         if not self.is_configured():
             raise ValueError("need to configure first")
 
     def get_latest_backupfile(self, backupdir) -> Optional[str]:
-        """ return the latest backup file in a given directory.
-        """
+        """return the latest backup file in a given directory."""
         res = lib.dc_imex_has_backup(self._dc_context, as_dc_charpointer(backupdir))
         return from_optional_dc_charpointer(res)
 
     def get_blobdir(self) -> str:
-        """ return the directory for files.
+        """return the directory for files.
 
         All sent files are copied to this directory if necessary.
         Place files there directly to avoid copying.
@@ -238,7 +262,7 @@ class Account(object):
         return from_dc_charpointer(lib.dc_get_blobdir(self._dc_context))
 
     def get_self_contact(self) -> Contact:
-        """ return this account's identity as a :class:`deltachat.contact.Contact`.
+        """return this account's identity as a :class:`deltachat.contact.Contact`.
 
         :returns: :class:`deltachat.contact.Contact`
         """
@@ -280,14 +304,14 @@ class Account(object):
         elif isinstance(obj, str):
             displayname, addr = parseaddr(obj)
         else:
-            raise TypeError("don't know how to create chat for %r" % (obj, ))
+            raise TypeError("don't know how to create chat for %r" % (obj,))
 
         if name is None and displayname:
             name = displayname
         return (name, addr)
 
     def delete_contact(self, contact: Contact) -> bool:
-        """ delete a Contact.
+        """delete a Contact.
 
         :param contact: contact object obtained
         :returns: True if deletion succeeded (contact was deleted)
@@ -298,7 +322,7 @@ class Account(object):
         return bool(lib.dc_delete_contact(self._dc_context, contact_id))
 
     def get_contact_by_addr(self, email: str) -> Optional[Contact]:
-        """ get a contact for the email address or None if it's blocked or doesn't exist. """
+        """get a contact for the email address or None if it's blocked or doesn't exist."""
         _, addr = parseaddr(email)
         addr = as_dc_charpointer(addr)
         contact_id = lib.dc_lookup_contact_id_by_addr(self._dc_context, addr)
@@ -307,21 +331,18 @@ class Account(object):
         return None
 
     def get_contact_by_id(self, contact_id: int) -> Contact:
-        """ return Contact instance or raise an exception.
+        """return Contact instance or raise an exception.
         :param contact_id: integer id of this contact.
         :returns: :class:`deltachat.contact.Contact` instance.
         """
         return Contact(self, contact_id)
 
     def get_blocked_contacts(self) -> List[Contact]:
-        """ return a list of all blocked contacts.
+        """return a list of all blocked contacts.
 
         :returns: list of :class:`deltachat.contact.Contact` objects.
         """
-        dc_array = ffi.gc(
-            lib.dc_get_blocked_contacts(self._dc_context),
-            lib.dc_array_unref
-        )
+        dc_array = ffi.gc(lib.dc_get_blocked_contacts(self._dc_context), lib.dc_array_unref)
         return list(iter_array(dc_array, lambda x: Contact(self, x)))
 
     def get_contacts(
@@ -344,22 +365,16 @@ class Account(object):
             flags |= const.DC_GCL_VERIFIED_ONLY
         if with_self:
             flags |= const.DC_GCL_ADD_SELF
-        dc_array = ffi.gc(
-            lib.dc_get_contacts(self._dc_context, flags, query),
-            lib.dc_array_unref
-        )
+        dc_array = ffi.gc(lib.dc_get_contacts(self._dc_context, flags, query), lib.dc_array_unref)
         return list(iter_array(dc_array, lambda x: Contact(self, x)))
 
     def get_fresh_messages(self) -> Generator[Message, None, None]:
-        """ yield all fresh messages from all chats. """
-        dc_array = ffi.gc(
-            lib.dc_get_fresh_msgs(self._dc_context),
-            lib.dc_array_unref
-        )
+        """yield all fresh messages from all chats."""
+        dc_array = ffi.gc(lib.dc_get_fresh_msgs(self._dc_context), lib.dc_array_unref)
         yield from iter_array(dc_array, lambda x: Message.from_db(self, x))
 
     def create_chat(self, obj) -> Chat:
-        """ Create a 1:1 chat with Account, Contact or e-mail address. """
+        """Create a 1:1 chat with Account, Contact or e-mail address."""
         return self.create_contact(obj).create_chat()
 
     def create_group_chat(
@@ -385,14 +400,11 @@ class Account(object):
         return chat
 
     def get_chats(self) -> List[Chat]:
-        """ return list of chats.
+        """return list of chats.
 
         :returns: a list of :class:`deltachat.chat.Chat` objects.
         """
-        dc_chatlist = ffi.gc(
-            lib.dc_get_chatlist(self._dc_context, 0, ffi.NULL, 0),
-            lib.dc_chatlist_unref
-        )
+        dc_chatlist = ffi.gc(lib.dc_get_chatlist(self._dc_context, 0, ffi.NULL, 0), lib.dc_chatlist_unref)
 
         assert dc_chatlist != ffi.NULL
         chatlist = []
@@ -405,14 +417,14 @@ class Account(object):
         return Contact(self, const.DC_CONTACT_ID_DEVICE).create_chat()
 
     def get_message_by_id(self, msg_id: int) -> Message:
-        """ return Message instance.
+        """return Message instance.
         :param msg_id: integer id of this message.
         :returns: :class:`deltachat.message.Message` instance.
         """
         return Message.from_db(self, msg_id)
 
     def get_chat_by_id(self, chat_id: int) -> Chat:
-        """ return Chat instance.
+        """return Chat instance.
         :param chat_id: integer id of this chat.
         :returns: :class:`deltachat.chat.Chat` instance.
         :raises: ValueError if chat does not exist.
@@ -424,7 +436,7 @@ class Account(object):
         return Chat(self, chat_id)
 
     def mark_seen_messages(self, messages: List[Union[int, Message]]) -> None:
-        """ mark the given set of messages as seen.
+        """mark the given set of messages as seen.
 
         :param messages: a list of message ids or Message instances.
         """
@@ -438,7 +450,7 @@ class Account(object):
         lib.dc_markseen_msgs(self._dc_context, msg_ids, len(messages))
 
     def forward_messages(self, messages: List[Message], chat: Chat) -> None:
-        """ Forward list of messages to a chat.
+        """Forward list of messages to a chat.
 
         :param messages: list of :class:`deltachat.message.Message` object.
         :param chat: :class:`deltachat.chat.Chat` object.
@@ -448,7 +460,7 @@ class Account(object):
         lib.dc_forward_msgs(self._dc_context, msg_ids, len(msg_ids), chat.id)
 
     def delete_messages(self, messages: List[Message]) -> None:
-        """ delete messages (local and remote).
+        """delete messages (local and remote).
 
         :param messages: list of :class:`deltachat.message.Message` object.
         :returns: None
@@ -457,31 +469,34 @@ class Account(object):
         lib.dc_delete_msgs(self._dc_context, msg_ids, len(msg_ids))
 
     def export_self_keys(self, path):
-        """ export public and private keys to the specified directory.
+        """export public and private keys to the specified directory.
 
         Note that the account does not have to be started.
         """
         return self._export(path, imex_cmd=const.DC_IMEX_EXPORT_SELF_KEYS)
 
-    def export_all(self, path):
-        """return new file containing a backup of all database state
-        (chats, contacts, keys, media, ...). The file is created in the
-        the `path` directory.
+    def export_all(self, path: str, passphrase: Optional[str] = None) -> str:
+        """Export a backup of all database state (chats, contacts, keys, media, ...).
+
+        :param path: the directory where the backup will be stored.
+        :param passphrase: the backup will be encrypted with the passphrase, if it is
+                           None or empty string, the backup is not encrypted.
+        :returns: path to the created backup.
 
         Note that the account has to be stopped; call stop_io() if necessary.
         """
-        export_files = self._export(path, const.DC_IMEX_EXPORT_BACKUP)
+        export_files = self._export(path, const.DC_IMEX_EXPORT_BACKUP, passphrase)
         if len(export_files) != 1:
             raise RuntimeError("found more than one new file")
         return export_files[0]
 
-    def _export(self, path, imex_cmd):
+    def _export(self, path: str, imex_cmd: int, passphrase: Optional[str] = None) -> list:
         with self.temp_plugin(ImexTracker()) as imex_tracker:
-            self.imex(path, imex_cmd)
+            self.imex(path, imex_cmd, passphrase)
             return imex_tracker.wait_finish()
 
     def import_self_keys(self, path):
-        """ Import private keys found in the `path` directory.
+        """Import private keys found in the `path` directory.
         The last imported key is made the default keys unless its name
         contains the string legacy. Public keys are not imported.
 
@@ -489,21 +504,23 @@ class Account(object):
         """
         self._import(path, imex_cmd=const.DC_IMEX_IMPORT_SELF_KEYS)
 
-    def import_all(self, path):
-        """import delta chat state from the specified backup `path` (a file).
-
+    def import_all(self, path: str, passphrase: Optional[str] = None) -> None:
+        """Import Delta Chat state from the specified backup file.
         The account must be in unconfigured state for import to attempted.
+
+        :param path: path to the backup file.
+        :param passphrase: if not None or empty, the backup will be opened with the passphrase.
         """
         assert not self.is_configured(), "cannot import into configured account"
-        self._import(path, imex_cmd=const.DC_IMEX_IMPORT_BACKUP)
+        self._import(path, imex_cmd=const.DC_IMEX_IMPORT_BACKUP, passphrase=passphrase)
 
-    def _import(self, path, imex_cmd):
+    def _import(self, path: str, imex_cmd: int, passphrase: Optional[str] = None) -> None:
         with self.temp_plugin(ImexTracker()) as imex_tracker:
-            self.imex(path, imex_cmd)
+            self.imex(path, imex_cmd, passphrase)
             imex_tracker.wait_finish()
 
-    def imex(self, path, imex_cmd):
-        lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), ffi.NULL)
+    def imex(self, path: str, imex_cmd: int, passphrase: Optional[str] = None) -> None:
+        lib.dc_imex(self._dc_context, imex_cmd, as_dc_charpointer(path), as_dc_charpointer(passphrase))
 
     def initiate_key_transfer(self) -> str:
         """return setup code after a Autocrypt setup message
@@ -517,7 +534,7 @@ class Account(object):
         return from_dc_charpointer(res)
 
     def get_setup_contact_qr(self) -> str:
-        """ get/create Setup-Contact QR Code as ascii-string.
+        """get/create Setup-Contact QR Code as ascii-string.
 
         this string needs to be transferred to another DC account
         in a second channel (typically used by mobiles with QRcode-show + scan UX)
@@ -527,18 +544,15 @@ class Account(object):
         return from_dc_charpointer(res)
 
     def check_qr(self, qr):
-        """ check qr code and return :class:`ScannedQRCode` instance representing the result"""
-        res = ffi.gc(
-            lib.dc_check_qr(self._dc_context, as_dc_charpointer(qr)),
-            lib.dc_lot_unref
-        )
+        """check qr code and return :class:`ScannedQRCode` instance representing the result"""
+        res = ffi.gc(lib.dc_check_qr(self._dc_context, as_dc_charpointer(qr)), lib.dc_lot_unref)
         lot = DCLot(res)
         if lot.state() == const.DC_QR_ERROR:
             raise ValueError("invalid or unknown QR code: {}".format(lot.text1()))
         return ScannedQRCode(lot)
 
     def qr_setup_contact(self, qr):
-        """ setup contact and return a Chat after contact is established.
+        """setup contact and return a Chat after contact is established.
 
         Note that this function may block for a long time as messages are exchanged
         with the emitter of the QR code.  On success a :class:`deltachat.chat.Chat` instance
@@ -552,7 +566,7 @@ class Account(object):
         return Chat(self, chat_id)
 
     def qr_join_chat(self, qr):
-        """ join a chat group through a QR code.
+        """join a chat group through a QR code.
 
         Note that this function may block for a long time as messages are exchanged
         with the emitter of the QR code.  On success a :class:`deltachat.chat.Chat` instance
@@ -566,9 +580,7 @@ class Account(object):
             raise ValueError("could not join group")
         return Chat(self, chat_id)
 
-    def set_location(
-        self, latitude: float = 0.0, longitude: float = 0.0, accuracy: float = 0.0
-    ) -> None:
+    def set_location(self, latitude: float = 0.0, longitude: float = 0.0, accuracy: float = 0.0) -> None:
         """set a new location. It effects all chats where we currently
         have enabled location streaming.
 
@@ -587,7 +599,7 @@ class Account(object):
     #
 
     def add_account_plugin(self, plugin, name=None):
-        """ add an account plugin which implements one or more of
+        """add an account plugin which implements one or more of
         the :class:`deltachat.hookspec.PerAccount` hooks.
         """
         if name and self._pm.has_plugin(name=name):
@@ -597,18 +609,18 @@ class Account(object):
         return plugin
 
     def remove_account_plugin(self, plugin, name=None):
-        """ remove an account plugin. """
+        """remove an account plugin."""
         self._pm.unregister(plugin, name=name)
 
     @contextmanager
     def temp_plugin(self, plugin):
-        """ run a with-block with the given plugin temporarily registered. """
+        """run a with-block with the given plugin temporarily registered."""
         self._pm.register(plugin)
         yield plugin
         self._pm.unregister(plugin)
 
     def stop_ongoing(self):
-        """ Stop ongoing securejoin, configuration or other core jobs. """
+        """Stop ongoing securejoin, configuration or other core jobs."""
         lib.dc_stop_ongoing_process(self._dc_context)
 
     def get_connectivity(self):
@@ -621,7 +633,7 @@ class Account(object):
         return lib.dc_all_work_done(self._dc_context)
 
     def start_io(self):
-        """ start this account's IO scheduling (Rust-core async scheduler)
+        """start this account's IO scheduling (Rust-core async scheduler)
 
         If this account is not configured an Exception is raised.
         You need to call account.configure() and account.wait_configure_finish()
@@ -665,7 +677,7 @@ class Account(object):
         lib.dc_maybe_network(self._dc_context)
 
     def configure(self, reconfigure: bool = False) -> ConfigureTracker:
-        """ Start configuration process and return a Configtracker instance
+        """Start configuration process and return a Configtracker instance
         on which you can block with wait_finish() to get a True/False success
         value for the configuration process.
         """
@@ -678,11 +690,11 @@ class Account(object):
         return configtracker
 
     def wait_shutdown(self) -> None:
-        """ wait until shutdown of this account has completed. """
+        """wait until shutdown of this account has completed."""
         self._shutdown_event.wait()
 
     def stop_io(self) -> None:
-        """ stop core IO scheduler if it is running. """
+        """stop core IO scheduler if it is running."""
         self.log("stop_ongoing")
         self.stop_ongoing()
 
@@ -690,7 +702,7 @@ class Account(object):
         lib.dc_stop_io(self._dc_context)
 
     def shutdown(self) -> None:
-        """ shutdown and destroy account (stop callback thread, close and remove
+        """shutdown and destroy account (stop callback thread, close and remove
         underlying dc_context)."""
         if self._dc_context is None:
             return

python/src/deltachat/chat.py

@@ -1,32 +1,40 @@
 """ Chat and Location related API. """
 
-import mimetypes
 import calendar
 import json
-from datetime import datetime, timezone
+import mimetypes
 import os
-from .cutil import as_dc_charpointer, from_dc_charpointer, from_optional_dc_charpointer, iter_array
-from .capi import lib, ffi
-from . import const
-from .message import Message
+from datetime import datetime, timezone
 from typing import Optional
 
+from . import const
+from .capi import ffi, lib
+from .cutil import (
+    as_dc_charpointer,
+    from_dc_charpointer,
+    from_optional_dc_charpointer,
+    iter_array,
+)
+from .message import Message
+
 
 class Chat(object):
-    """ Chat object which manages members and through which you can send and retrieve messages.
+    """Chat object which manages members and through which you can send and retrieve messages.
 
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
 
     def __init__(self, account, id) -> None:
         from .account import Account
+
         assert isinstance(account, Account), repr(account)
         self.account = account
         self.id = id
 
     def __eq__(self, other) -> bool:
-        return self.id == getattr(other, "id", None) and \
-               self.account._dc_context == other.account._dc_context
+        if other is None:
+            return False
+        return self.id == getattr(other, "id", None) and self.account._dc_context == other.account._dc_context
 
     def __ne__(self, other) -> bool:
         return not (self == other)
@@ -36,10 +44,7 @@ class Chat(object):
 
     @property
     def _dc_chat(self):
-        return ffi.gc(
-            lib.dc_get_chat(self.account._dc_context, self.id),
-            lib.dc_chat_unref
-        )
+        return ffi.gc(lib.dc_get_chat(self.account._dc_context, self.id), lib.dc_chat_unref)
 
     def delete(self) -> None:
         """Delete this chat and all its messages.
@@ -62,28 +67,66 @@ class Chat(object):
     # ------  chat status/metadata API ------------------------------
 
     def is_group(self) -> bool:
-        """ return true if this chat is a group chat.
+        """Return True if this chat is a group chat.
 
-        :returns: True if chat is a group-chat, false if it's a contact 1:1 chat.
+        :returns: True if chat is a group-chat, False otherwise
         """
         return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_GROUP
 
+    def is_single(self) -> bool:
+        """Return True if this chat is a single/direct chat, False otherwise"""
+        return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_SINGLE
+
+    def is_mailinglist(self) -> bool:
+        """Return True if this chat is a mailing list, False otherwise"""
+        return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_MAILINGLIST
+
+    def is_broadcast(self) -> bool:
+        """Return True if this chat is a broadcast list, False otherwise"""
+        return lib.dc_chat_get_type(self._dc_chat) == const.DC_CHAT_TYPE_BROADCAST
+
+    def is_multiuser(self) -> bool:
+        """Return True if this chat is a multi-user chat (group, mailing list or broadcast), False otherwise"""
+        return lib.dc_chat_get_type(self._dc_chat) in (
+            const.DC_CHAT_TYPE_GROUP,
+            const.DC_CHAT_TYPE_MAILINGLIST,
+            const.DC_CHAT_TYPE_BROADCAST,
+        )
+
+    def is_self_talk(self) -> bool:
+        """Return True if this chat is the self-chat (a.k.a. "Saved Messages"), False otherwise"""
+        return bool(lib.dc_chat_is_self_talk(self._dc_chat))
+
+    def is_device_talk(self) -> bool:
+        """Returns True if this chat is the "Device Messages" chat, False otherwise"""
+        return bool(lib.dc_chat_is_device_talk(self._dc_chat))
+
     def is_muted(self) -> bool:
-        """ return true if this chat is muted.
+        """return true if this chat is muted.
 
         :returns: True if chat is muted, False otherwise.
         """
-        return lib.dc_chat_is_muted(self._dc_chat)
+        return bool(lib.dc_chat_is_muted(self._dc_chat))
 
-    def is_contact_request(self):
-        """ return True if this chat is a contact request chat.
+    def is_pinned(self) -> bool:
+        """Return True if this chat is pinned, False otherwise"""
+        return lib.dc_chat_get_visibility(self._dc_chat) == const.DC_CHAT_VISIBILITY_PINNED
+
+    def is_archived(self) -> bool:
+        """Return True if this chat is archived, False otherwise.
+        :returns: True if archived, False otherwise
+        """
+        return lib.dc_chat_get_visibility(self._dc_chat) == const.DC_CHAT_VISIBILITY_ARCHIVED
+
+    def is_contact_request(self) -> bool:
+        """return True if this chat is a contact request chat.
 
         :returns: True if chat is a contact request chat, False otherwise.
         """
-        return lib.dc_chat_is_contact_request(self._dc_chat)
+        return bool(lib.dc_chat_is_contact_request(self._dc_chat))
 
-    def is_promoted(self):
-        """ return True if this chat is promoted, i.e.
+    def is_promoted(self) -> bool:
+        """return True if this chat is promoted, i.e.
         the member contacts are aware of their membership,
         have been sent messages.
 
@@ -97,24 +140,24 @@ class Chat(object):
 
         :returns: True if the chat is writable, False otherwise
         """
-        return lib.dc_chat_can_send(self._dc_chat)
+        return bool(lib.dc_chat_can_send(self._dc_chat))
 
     def is_protected(self) -> bool:
-        """ return True if this chat is a protected chat.
+        """return True if this chat is a protected chat.
 
         :returns: True if chat is protected, False otherwise.
         """
-        return lib.dc_chat_is_protected(self._dc_chat)
+        return bool(lib.dc_chat_is_protected(self._dc_chat))
 
     def get_name(self) -> Optional[str]:
-        """ return name of this chat.
+        """return name of this chat.
 
         :returns: unicode name
         """
         return from_dc_charpointer(lib.dc_chat_get_name(self._dc_chat))
 
     def set_name(self, name: str) -> bool:
-        """ set name of this chat.
+        """set name of this chat.
 
         :param name: as a unicode string.
         :returns: True on success, False otherwise
@@ -122,8 +165,20 @@ class Chat(object):
         name = as_dc_charpointer(name)
         return bool(lib.dc_set_chat_name(self.account._dc_context, self.id, name))
 
+    def get_color(self):
+        """return the color of the chat.
+        :returns: color as 0x00rrggbb
+        """
+        return lib.dc_chat_get_color(self._dc_chat)
+
+    def get_summary(self):
+        """return dictionary with summary information."""
+        dc_res = lib.dc_chat_get_info_json(self.account._dc_context, self.id)
+        s = from_dc_charpointer(dc_res)
+        return json.loads(s)
+
     def mute(self, duration: Optional[int] = None) -> None:
-        """ mutes the chat
+        """mutes the chat
 
         :param duration: Number of seconds to mute the chat for. None to mute until unmuted again.
         :returns: None
@@ -137,7 +192,7 @@ class Chat(object):
             raise ValueError("Call to dc_set_chat_mute_duration failed")
 
     def unmute(self) -> None:
-        """ unmutes the chat
+        """unmutes the chat
 
         :returns: None
         """
@@ -145,8 +200,26 @@ class Chat(object):
         if not bool(ret):
             raise ValueError("Failed to unmute chat")
 
+    def pin(self) -> None:
+        """Pin the chat."""
+        lib.dc_set_chat_visibility(self.account._dc_context, self.id, const.DC_CHAT_VISIBILITY_PINNED)
+
+    def unpin(self) -> None:
+        """Unpin the chat."""
+        if self.is_pinned():
+            lib.dc_set_chat_visibility(self.account._dc_context, self.id, const.DC_CHAT_VISIBILITY_NORMAL)
+
+    def archive(self) -> None:
+        """Archive the chat."""
+        lib.dc_set_chat_visibility(self.account._dc_context, self.id, const.DC_CHAT_VISIBILITY_ARCHIVED)
+
+    def unarchive(self) -> None:
+        """Unarchive the chat."""
+        if self.is_archived():
+            lib.dc_set_chat_visibility(self.account._dc_context, self.id, const.DC_CHAT_VISIBILITY_NORMAL)
+
     def get_mute_duration(self) -> int:
-        """ Returns the number of seconds until the mute of this chat is lifted.
+        """Returns the number of seconds until the mute of this chat is lifted.
 
         :param duration:
         :returns: Returns the number of seconds the chat is still muted for. (0 for not muted, -1 forever muted)
@@ -154,14 +227,14 @@ class Chat(object):
         return lib.dc_chat_get_remaining_mute_duration(self._dc_chat)
 
     def get_ephemeral_timer(self) -> int:
-        """ get ephemeral timer.
+        """get ephemeral timer.
 
         :returns: ephemeral timer value in seconds
         """
         return lib.dc_get_chat_ephemeral_timer(self.account._dc_context, self.id)
 
     def set_ephemeral_timer(self, timer: int) -> bool:
-        """ set ephemeral timer.
+        """set ephemeral timer.
 
         :param: timer value in seconds
 
@@ -170,7 +243,7 @@ class Chat(object):
         return bool(lib.dc_set_chat_ephemeral_timer(self.account._dc_context, self.id, timer))
 
     def get_type(self) -> int:
-        """ (deprecated) return type of this chat.
+        """(deprecated) return type of this chat.
 
         :returns: one of const.DC_CHAT_TYPE_*
         """
@@ -184,7 +257,7 @@ class Chat(object):
         return from_dc_charpointer(res)
 
     def get_join_qr(self) -> Optional[str]:
-        """ get/create Join-Group QR Code as ascii-string.
+        """get/create Join-Group QR Code as ascii-string.
 
         this string needs to be transferred to another DC account
         in a second channel (typically used by mobiles with QRcode-show + scan UX)
@@ -220,7 +293,7 @@ class Chat(object):
         return msg
 
     def send_text(self, text):
-        """ send a text message and return the resulting Message instance.
+        """send a text message and return the resulting Message instance.
 
         :param msg: unicode text
         :raises ValueError: if message can not be send/chat does not exist.
@@ -233,7 +306,7 @@ class Chat(object):
         return Message.from_db(self.account, msg_id)
 
     def send_file(self, path, mime_type="application/octet-stream"):
-        """ send a file and return the resulting Message instance.
+        """send a file and return the resulting Message instance.
 
         :param path: path to the file.
         :param mime_type: the mime-type of this file, defaults to application/octet-stream.
@@ -248,7 +321,7 @@ class Chat(object):
         return Message.from_db(self.account, sent_id)
 
     def send_image(self, path):
-        """ send an image message and return the resulting Message instance.
+        """send an image message and return the resulting Message instance.
 
         :param path: path to an image file.
         :raises ValueError: if message can not be send/chat does not exist.
@@ -263,7 +336,7 @@ class Chat(object):
         return Message.from_db(self.account, sent_id)
 
     def prepare_message(self, msg):
-        """ prepare a message for sending.
+        """prepare a message for sending.
 
         :param msg: the message to be prepared.
         :returns: a :class:`deltachat.message.Message` instance.
@@ -278,7 +351,7 @@ class Chat(object):
         return msg
 
     def prepare_message_file(self, path, mime_type=None, view_type="file"):
-        """ prepare a message for sending and return the resulting Message instance.
+        """prepare a message for sending and return the resulting Message instance.
 
         To actually send the message, call :meth:`send_prepared`.
         The file must be inside the blob directory.
@@ -294,7 +367,7 @@ class Chat(object):
         return self.prepare_message(msg)
 
     def send_prepared(self, message):
-        """ send a previously prepared message.
+        """send a previously prepared message.
 
         :param message: a :class:`Message` instance previously returned by
                         :meth:`prepare_file`.
@@ -314,7 +387,7 @@ class Chat(object):
         msg._dc_msg = Message.from_db(self.account, sent_id)._dc_msg
 
     def set_draft(self, message):
-        """ set message as draft.
+        """set message as draft.
 
         :param message: a :class:`Message` instance
         :returns: None
@@ -325,7 +398,7 @@ class Chat(object):
             lib.dc_set_draft(self.account._dc_context, self.id, message._dc_msg)
 
     def get_draft(self):
-        """ get draft message for this chat.
+        """get draft message for this chat.
 
         :param message: a :class:`Message` instance
         :returns: Message object or None (if no draft available)
@@ -337,40 +410,34 @@ class Chat(object):
         return Message(self.account, dc_msg)
 
     def get_messages(self):
-        """ return list of messages in this chat.
+        """return list of messages in this chat.
 
         :returns: list of :class:`deltachat.message.Message` objects for this chat.
         """
         dc_array = ffi.gc(
             lib.dc_get_chat_msgs(self.account._dc_context, self.id, 0, 0),
-            lib.dc_array_unref
+            lib.dc_array_unref,
         )
         return list(iter_array(dc_array, lambda x: Message.from_db(self.account, x)))
 
     def count_fresh_messages(self):
-        """ return number of fresh messages in this chat.
+        """return number of fresh messages in this chat.
 
         :returns: number of fresh messages
         """
         return lib.dc_get_fresh_msg_cnt(self.account._dc_context, self.id)
 
     def mark_noticed(self):
-        """ mark all messages in this chat as noticed.
+        """mark all messages in this chat as noticed.
 
         Noticed messages are no longer fresh.
         """
         return lib.dc_marknoticed_chat(self.account._dc_context, self.id)
 
-    def get_summary(self):
-        """ return dictionary with summary information. """
-        dc_res = lib.dc_chat_get_info_json(self.account._dc_context, self.id)
-        s = from_dc_charpointer(dc_res)
-        return json.loads(s)
-
     # ------  group management API ------------------------------
 
     def add_contact(self, obj):
-        """ add a contact to this chat.
+        """add a contact to this chat.
 
         :params obj: Contact, Account or e-mail address.
         :raises ValueError: if contact could not be added
@@ -383,7 +450,7 @@ class Chat(object):
         return contact
 
     def remove_contact(self, obj):
-        """ remove a contact from this chat.
+        """remove a contact from this chat.
 
         :params obj: Contact, Account or e-mail address.
         :raises ValueError: if contact could not be removed
@@ -395,23 +462,22 @@ class Chat(object):
             raise ValueError("could not remove contact {!r} from chat".format(contact))
 
     def get_contacts(self):
-        """ get all contacts for this chat.
+        """get all contacts for this chat.
         :returns: list of :class:`deltachat.contact.Contact` objects for this chat
         """
         from .contact import Contact
+
         dc_array = ffi.gc(
             lib.dc_get_chat_contacts(self.account._dc_context, self.id),
-            lib.dc_array_unref
-        )
-        return list(iter_array(
-            dc_array, lambda id: Contact(self.account, id))
+            lib.dc_array_unref,
         )
+        return list(iter_array(dc_array, lambda id: Contact(self.account, id)))
 
     def num_contacts(self):
-        """ return number of contacts in this chat. """
+        """return number of contacts in this chat."""
         dc_array = ffi.gc(
             lib.dc_get_chat_contacts(self.account._dc_context, self.id),
-            lib.dc_array_unref
+            lib.dc_array_unref,
         )
         return lib.dc_array_get_cnt(dc_array)
 
@@ -458,12 +524,6 @@ class Chat(object):
             return None
         return from_dc_charpointer(dc_res)
 
-    def get_color(self):
-        """return the color of the chat.
-        :returns: color as 0x00rrggbb
-        """
-        return lib.dc_chat_get_color(self._dc_chat)
-
     # ------  location streaming API ------------------------------
 
     def is_sending_locations(self):
@@ -472,12 +532,6 @@ class Chat(object):
         """
         return lib.dc_is_sending_locations_to_chat(self.account._dc_context, self.id)
 
-    def is_archived(self):
-        """return True if this chat is archived.
-        :returns: True if archived.
-        """
-        return lib.dc_chat_get_visibility(self._dc_chat) == const.DC_CHAT_VISIBILITY_ARCHIVED
-
     def enable_sending_locations(self, seconds):
         """enable sending locations for this chat.
 
@@ -513,10 +567,7 @@ class Chat(object):
                 latitude=lib.dc_array_get_latitude(dc_array, i),
                 longitude=lib.dc_array_get_longitude(dc_array, i),
                 accuracy=lib.dc_array_get_accuracy(dc_array, i),
-                timestamp=datetime.fromtimestamp(
-                    lib.dc_array_get_timestamp(dc_array, i),
-                    timezone.utc
-                ),
+                timestamp=datetime.fromtimestamp(lib.dc_array_get_timestamp(dc_array, i), timezone.utc),
                 marker=from_optional_dc_charpointer(lib.dc_array_get_marker(dc_array, i)),
             )
             for i in range(lib.dc_array_get_cnt(dc_array))

python/src/deltachat/contact.py

@@ -10,17 +10,21 @@ from .cutil import from_dc_charpointer, from_optional_dc_charpointer
 
 
 class Contact(object):
-    """ Delta-Chat Contact.
+    """Delta-Chat Contact.
 
     You obtain instances of it through :class:`deltachat.account.Account`.
     """
+
     def __init__(self, account, id):
         from .account import Account
+
         assert isinstance(account, Account), repr(account)
         self.account = account
         self.id = id
 
-    def __eq__(self, other):
+    def __eq__(self, other) -> bool:
+        if other is None:
+            return False
         return self.account._dc_context == other.account._dc_context and self.id == other.id
 
     def __ne__(self, other):
@@ -31,19 +35,16 @@ class Contact(object):
 
     @property
     def _dc_contact(self):
-        return ffi.gc(
-            lib.dc_get_contact(self.account._dc_context, self.id),
-            lib.dc_contact_unref
-        )
+        return ffi.gc(lib.dc_get_contact(self.account._dc_context, self.id), lib.dc_contact_unref)
 
     @props.with_doc
     def addr(self) -> str:
-        """ normalized e-mail address for this account. """
+        """normalized e-mail address for this account."""
         return from_dc_charpointer(lib.dc_contact_get_addr(self._dc_contact))
 
     @props.with_doc
     def name(self) -> str:
-        """ display name for this contact. """
+        """display name for this contact."""
         return from_dc_charpointer(lib.dc_contact_get_display_name(self._dc_contact))
 
     # deprecated alias
@@ -52,28 +53,26 @@ class Contact(object):
     @props.with_doc
     def last_seen(self) -> date:
         """Last seen timestamp."""
-        return datetime.fromtimestamp(
-            lib.dc_contact_get_last_seen(self._dc_contact), timezone.utc
-        )
+        return datetime.fromtimestamp(lib.dc_contact_get_last_seen(self._dc_contact), timezone.utc)
 
     def is_blocked(self):
-        """ Return True if the contact is blocked. """
+        """Return True if the contact is blocked."""
         return lib.dc_contact_is_blocked(self._dc_contact)
 
     def set_blocked(self, block=True):
-        """ [Deprecated, use block/unblock methods] Block or unblock a contact. """
+        """[Deprecated, use block/unblock methods] Block or unblock a contact."""
         return lib.dc_block_contact(self.account._dc_context, self.id, block)
 
     def block(self):
-        """ Block this contact. Message will not be seen/retrieved from this contact. """
+        """Block this contact. Message will not be seen/retrieved from this contact."""
         return lib.dc_block_contact(self.account._dc_context, self.id, True)
 
     def unblock(self):
-        """ Unblock this contact. Messages from this contact will be retrieved (again)."""
+        """Unblock this contact. Messages from this contact will be retrieved (again)."""
         return lib.dc_block_contact(self.account._dc_context, self.id, False)
 
     def is_verified(self):
-        """ Return True if the contact is verified. """
+        """Return True if the contact is verified."""
         return lib.dc_contact_is_verified(self._dc_contact)
 
     def get_profile_image(self) -> Optional[str]:
@@ -93,7 +92,7 @@ class Contact(object):
         return from_dc_charpointer(lib.dc_contact_get_status(self._dc_contact))
 
     def create_chat(self):
-        """ create or get an existing 1:1 chat object for the specified contact or contact id.
+        """create or get an existing 1:1 chat object for the specified contact or contact id.
 
         :param contact: chat_id (int) or contact object.
         :returns: a :class:`deltachat.chat.Chat` object.

python/src/deltachat/cutil.py

@@ -1,9 +1,9 @@
-from .capi import lib
-from .capi import ffi
 from datetime import datetime, timezone
-from typing import Optional, TypeVar, Generator, Callable
+from typing import Callable, Generator, Optional, TypeVar
 
-T = TypeVar('T')
+from .capi import ffi, lib
+
+T = TypeVar("T")
 
 
 def as_dc_charpointer(obj):

python/src/deltachat/direct_imap.py

@@ -3,18 +3,27 @@ Internal Python-level IMAP handling used by the testplugin
 and for cleaning up inbox/mvbox for each test function run.
 """
 
-import io
-import ssl
-import pathlib
-from contextlib import contextmanager
-from imap_tools import MailBox, MailBoxTls, errors, AND, Header, MailMessageFlags, MailMessage
 import imaplib
-from deltachat import const, Account
+import io
+import pathlib
+import ssl
+from contextlib import contextmanager
 from typing import List
 
+from imap_tools import (
+    AND,
+    Header,
+    MailBox,
+    MailBoxTls,
+    MailMessage,
+    MailMessageFlags,
+    errors,
+)
 
-FLAGS = b'FLAGS'
-FETCH = b'FETCH'
+from deltachat import Account, const
+
+FLAGS = b"FLAGS"
+FETCH = b"FETCH"
 ALL = "1:*"
 
 
@@ -69,8 +78,8 @@ class DirectImap:
         return self.conn.folder.set(foldername)
 
     def select_config_folder(self, config_name: str):
-        """ Return info about selected folder if it is
-        configured, otherwise None. """
+        """Return info about selected folder if it is
+        configured, otherwise None."""
         if "_" not in config_name:
             config_name = "configured_{}_folder".format(config_name)
         foldername = self.account.get_config(config_name)
@@ -78,17 +87,17 @@ class DirectImap:
             return self.select_folder(foldername)
 
     def list_folders(self) -> List[str]:
-        """ return list of all existing folder names"""
+        """return list of all existing folder names"""
         assert not self._idling
         return [folder.name for folder in self.conn.folder.list()]
 
     def delete(self, uid_list: str, expunge=True):
-        """ delete a range of messages (imap-syntax).
+        """delete a range of messages (imap-syntax).
         If expunge is true, perform the expunge-operation
         to make sure the messages are really gone and not
         just flagged as deleted.
         """
-        self.conn.client.uid('STORE', uid_list, '+FLAGS', r'(\Deleted)')
+        self.conn.client.uid("STORE", uid_list, "+FLAGS", r"(\Deleted)")
         if expunge:
             self.conn.expunge()
 
@@ -141,7 +150,13 @@ class DirectImap:
                 fn = path.joinpath(str(msg.uid))
                 fn.write_bytes(body)
                 log("Message", msg.uid, fn)
-                log("Message", msg.uid, msg.flags, "Message-Id:", msg.obj.get("Message-Id"))
+                log(
+                    "Message",
+                    msg.uid,
+                    msg.flags,
+                    "Message-Id:",
+                    msg.obj.get("Message-Id"),
+                )
 
         if empty_folders:
             log("--------- EMPTY FOLDERS:", empty_folders)
@@ -150,7 +165,7 @@ class DirectImap:
 
     @contextmanager
     def idle(self):
-        """ return Idle ContextManager. """
+        """return Idle ContextManager."""
         idle_manager = IdleManager(self)
         try:
             yield idle_manager
@@ -163,11 +178,11 @@ class DirectImap:
         """
         if msg.startswith("\n"):
             msg = msg[1:]
-        msg = '\n'.join([s.lstrip() for s in msg.splitlines()])
-        self.conn.append(bytes(msg, encoding='ascii'), folder)
+        msg = "\n".join([s.lstrip() for s in msg.splitlines()])
+        self.conn.append(bytes(msg, encoding="ascii"), folder)
 
     def get_uid_by_message_id(self, message_id) -> str:
-        msgs = [msg.uid for msg in self.conn.fetch(AND(header=Header('MESSAGE-ID', message_id)))]
+        msgs = [msg.uid for msg in self.conn.fetch(AND(header=Header("MESSAGE-ID", message_id)))]
         if len(msgs) == 0:
             raise Exception("Did not find message " + message_id + ", maybe you forgot to select the correct folder?")
         return msgs[0]
@@ -183,7 +198,7 @@ class IdleManager:
         self.direct_imap.conn.idle.start()
 
     def check(self, timeout=None) -> List[bytes]:
-        """ (blocking) wait for next idle message from server. """
+        """(blocking) wait for next idle message from server."""
         self.log("imap-direct: calling idle_check")
         res = self.direct_imap.conn.idle.poll(timeout=timeout)
         self.log("imap-direct: idle_check returned {!r}".format(res))
@@ -192,20 +207,19 @@ class IdleManager:
     def wait_for_new_message(self, timeout=None) -> bytes:
         while 1:
             for item in self.check(timeout=timeout):
-                if b'EXISTS' in item or b'RECENT' in item:
+                if b"EXISTS" in item or b"RECENT" in item:
                     return item
 
     def wait_for_seen(self, timeout=None) -> int:
-        """ Return first message with SEEN flag from a running idle-stream.
-        """
+        """Return first message with SEEN flag from a running idle-stream."""
         while 1:
             for item in self.check(timeout=timeout):
                 if FETCH in item:
                     self.log(str(item))
-                    if FLAGS in item and rb'\Seen' in item:
-                        return int(item.split(b' ')[1])
+                    if FLAGS in item and rb"\Seen" in item:
+                        return int(item.split(b" ")[1])
 
     def done(self):
-        """ send idle-done to server if we are currently in idle mode. """
+        """send idle-done to server if we are currently in idle mode."""
         res = self.direct_imap.conn.idle.stop()
         return res

python/src/deltachat/events.py

@@ -1,18 +1,19 @@
-import threading
-import sys
-import traceback
-import time
 import io
-import re
 import os
-from queue import Queue, Empty
+import re
+import sys
+import threading
+import time
+import traceback
+from contextlib import contextmanager
+from queue import Empty, Queue
 
 import deltachat
-from .hookspec import account_hookimpl
-from contextlib import contextmanager
+
 from .capi import ffi, lib
-from .message import map_system_message
 from .cutil import from_optional_dc_charpointer
+from .hookspec import account_hookimpl
+from .message import map_system_message
 
 
 class FFIEvent:
@@ -26,9 +27,10 @@ class FFIEvent:
 
 
 class FFIEventLogger:
-    """ If you register an instance of this logger with an Account
+    """If you register an instance of this logger with an Account
     you'll get all ffi-events printed.
     """
+
     # to prevent garbled logging
     _loglock = threading.RLock()
 
@@ -56,9 +58,9 @@ class FFIEventLogger:
         s = "{:2.2f} [{}] {}".format(elapsed, locname, message)
 
         if os.name == "posix":
-            WARN = '\033[93m'
-            ERROR = '\033[91m'
-            ENDC = '\033[0m'
+            WARN = "\033[93m"
+            ERROR = "\033[91m"
+            ENDC = "\033[0m"
             if message.startswith("DC_EVENT_WARNING"):
                 s = WARN + s + ENDC
             if message.startswith("DC_EVENT_ERROR"):
@@ -171,12 +173,12 @@ class FFIEventTracker:
         self.get_info_contains("INBOX: Idle entering")
 
     def wait_next_incoming_message(self):
-        """ wait for and return next incoming message. """
+        """wait for and return next incoming message."""
         ev = self.get_matching("DC_EVENT_INCOMING_MSG")
         return self.account.get_message_by_id(ev.data2)
 
     def wait_next_messages_changed(self):
-        """ wait for and return next message-changed message or None
+        """wait for and return next message-changed message or None
         if the event contains no msgid"""
         ev = self.get_matching("DC_EVENT_MSGS_CHANGED")
         if ev.data2 > 0:
@@ -191,10 +193,11 @@ class FFIEventTracker:
 
 
 class EventThread(threading.Thread):
-    """ Event Thread for an account.
+    """Event Thread for an account.
 
     With each Account init this callback thread is started.
     """
+
     def __init__(self, account) -> None:
         self.account = account
         super(EventThread, self).__init__(name="events")
@@ -219,7 +222,7 @@ class EventThread(threading.Thread):
         self.join(timeout=timeout)
 
     def run(self) -> None:
-        """ get and run events until shutdown. """
+        """get and run events until shutdown."""
         with self.log_execution("EVENT THREAD"):
             self._inner_run()
 
@@ -259,8 +262,7 @@ class EventThread(threading.Thread):
         except Exception as ex:
             logfile = io.StringIO()
             traceback.print_exception(*sys.exc_info(), file=logfile)
-            self.account.log("{}\nException {}\nTraceback:\n{}"
-                             .format(info, ex, logfile.getvalue()))
+            self.account.log("{}\nException {}\nTraceback:\n{}".format(info, ex, logfile.getvalue()))
 
     def _map_ffi_event(self, ffi_event: FFIEvent):
         name = ffi_event.name
@@ -269,7 +271,8 @@ class EventThread(threading.Thread):
             data1 = ffi_event.data1
             if data1 == 0 or data1 == 1000:
                 success = data1 == 1000
-                yield "ac_configure_completed", dict(success=success)
+                comment = ffi_event.data2
+                yield "ac_configure_completed", dict(success=success, comment=comment)
         elif name == "DC_EVENT_INCOMING_MSG":
             msg = account.get_message_by_id(ffi_event.data2)
             yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))
@@ -282,7 +285,10 @@ class EventThread(threading.Thread):
                         yield res
                     yield "ac_outgoing_message", dict(message=msg)
                 elif msg.is_in_fresh():
-                    yield map_system_message(msg) or ("ac_incoming_message", dict(message=msg))
+                    yield map_system_message(msg) or (
+                        "ac_incoming_message",
+                        dict(message=msg),
+                    )
         elif name == "DC_EVENT_MSG_DELIVERED":
             msg = account.get_message_by_id(ffi_event.data2)
             yield "ac_message_delivered", dict(message=msg)

python/src/deltachat/hookspec.py

@@ -2,7 +2,6 @@
 
 import pluggy
 
-
 account_spec_name = "deltachat-account"
 account_hookspec = pluggy.HookspecMarker(account_spec_name)
 account_hookimpl = pluggy.HookimplMarker(account_spec_name)
@@ -13,12 +12,13 @@ global_hookimpl = pluggy.HookimplMarker(global_spec_name)
 
 
 class PerAccount:
-    """ per-Account-instance hook specifications.
+    """per-Account-instance hook specifications.
 
     All hooks are executed in a dedicated Event thread.
     Hooks are generally not allowed to block/last long as this
     blocks overall event processing on the python side.
     """
+
     @classmethod
     def _make_plugin_manager(cls):
         pm = pluggy.PluginManager(account_spec_name)
@@ -27,7 +27,7 @@ class PerAccount:
 
     @account_hookspec
     def ac_process_ffi_event(self, ffi_event):
-        """ process a CFFI low level events for a given account.
+        """process a CFFI low level events for a given account.
 
         ffi_event has "name", "data1", "data2" values as specified
         with `DC_EVENT_* <https://c.delta.chat/group__DC__EVENT.html>`_.
@@ -35,37 +35,37 @@ class PerAccount:
 
     @account_hookspec
     def ac_log_line(self, message):
-        """ log a message related to the account. """
+        """log a message related to the account."""
 
     @account_hookspec
-    def ac_configure_completed(self, success):
-        """ Called after a configure process completed. """
+    def ac_configure_completed(self, success, comment):
+        """Called after a configure process completed."""
 
     @account_hookspec
     def ac_incoming_message(self, message):
-        """ Called on any incoming message (both existing chats and contact requests). """
+        """Called on any incoming message (both existing chats and contact requests)."""
 
     @account_hookspec
     def ac_outgoing_message(self, message):
-        """ Called on each outgoing message (both system and "normal")."""
+        """Called on each outgoing message (both system and "normal")."""
 
     @account_hookspec
     def ac_message_delivered(self, message):
-        """ Called when an outgoing message has been delivered to SMTP.
+        """Called when an outgoing message has been delivered to SMTP.
 
         :param message: Message that was just delivered.
         """
 
     @account_hookspec
     def ac_chat_modified(self, chat):
-        """ Chat was created or modified regarding membership, avatar, title.
+        """Chat was created or modified regarding membership, avatar, title.
 
         :param chat: Chat which was modified.
         """
 
     @account_hookspec
     def ac_member_added(self, chat, contact, actor, message):
-        """ Called for each contact added to an accepted chat.
+        """Called for each contact added to an accepted chat.
 
         :param chat: Chat where contact was added.
         :param contact: Contact that was added.
@@ -75,7 +75,7 @@ class PerAccount:
 
     @account_hookspec
     def ac_member_removed(self, chat, contact, actor, message):
-        """ Called for each contact removed from a chat.
+        """Called for each contact removed from a chat.
 
         :param chat: Chat where contact was removed.
         :param contact: Contact that was removed.
@@ -85,10 +85,11 @@ class PerAccount:
 
 
 class Global:
-    """ global hook specifications using a per-process singleton
+    """global hook specifications using a per-process singleton
     plugin manager instance.
 
     """
+
     _plugin_manager = None
 
     @classmethod
@@ -100,11 +101,11 @@ class Global:
 
     @global_hookspec
     def dc_account_init(self, account):
-        """ called when `Account::__init__()` function starts executing. """
+        """called when `Account::__init__()` function starts executing."""
 
     @global_hookspec
     def dc_account_extra_configure(self, account):
-        """ Called when account configuration successfully finished.
+        """Called when account configuration successfully finished.
 
         This hook can be used to perform extra work before
         ac_configure_completed is called.
@@ -112,4 +113,4 @@ class Global:
 
     @global_hookspec
     def dc_account_after_shutdown(self, account):
-        """ Called after the account has been shutdown. """
+        """Called after the account has been shutdown."""

python/src/deltachat/message.py

@@ -1,21 +1,23 @@
 """ The Message object. """
 
+import json
 import os
 import re
-from . import props
-from .cutil import from_dc_charpointer, from_optional_dc_charpointer, as_dc_charpointer
-from .capi import lib, ffi
-from . import const
 from datetime import datetime, timezone
-from typing import Optional
+from typing import Optional, Union
+
+from . import const, props
+from .capi import ffi, lib
+from .cutil import as_dc_charpointer, from_dc_charpointer, from_optional_dc_charpointer
 
 
 class Message(object):
-    """ Message object.
+    """Message object.
 
     You obtain instances of it through :class:`deltachat.account.Account` or
     :class:`deltachat.chat.Chat`.
     """
+
     def __init__(self, account, dc_msg):
         self.account = account
         assert isinstance(self.account._dc_context, ffi.CData)
@@ -25,42 +27,48 @@ class Message(object):
         msg_id = self.id
         assert msg_id is not None and msg_id >= 0, repr(msg_id)
 
-    def __eq__(self, other):
+    def __eq__(self, other) -> bool:
+        if other is None:
+            return False
         return self.account == other.account and self.id == other.id
 
     def __repr__(self):
         c = self.get_sender_contact()
         typ = "outgoing" if self.is_outgoing() else "incoming"
         return "<Message {} sys={} {} id={} sender={}/{} chat={}/{}>".format(
-            typ, self.is_system_message(), repr(self.text[:10]),
-            self.id, c.id, c.addr, self.chat.id, self.chat.get_name())
+            typ,
+            self.is_system_message(),
+            repr(self.text[:10]),
+            self.id,
+            c.id,
+            c.addr,
+            self.chat.id,
+            self.chat.get_name(),
+        )
 
     @classmethod
     def from_db(cls, account, id):
         assert id > 0
-        return cls(account, ffi.gc(
-                lib.dc_get_msg(account._dc_context, id),
-                lib.dc_msg_unref
-        ))
+        return cls(account, ffi.gc(lib.dc_get_msg(account._dc_context, id), lib.dc_msg_unref))
 
     @classmethod
     def new_empty(cls, account, view_type):
-        """ create a non-persistent message.
+        """create a non-persistent message.
 
-        :param: view_type is the message type code or one of the strings:
-           "text", "audio", "video", "file", "sticker"
+        :param view_type: the message type code or one of the strings:
+           "text", "audio", "video", "file", "sticker", "videochat", "webxdc"
         """
         if isinstance(view_type, int):
             view_type_code = view_type
         else:
             view_type_code = get_viewtype_code_from_name(view_type)
-        return Message(account, ffi.gc(
-            lib.dc_msg_new(account._dc_context, view_type_code),
-            lib.dc_msg_unref
-        ))
+        return Message(
+            account,
+            ffi.gc(lib.dc_msg_new(account._dc_context, view_type_code), lib.dc_msg_unref),
+        )
 
     def create_chat(self):
-        """ create or get an existing chat (group) object for this message.
+        """create or get an existing chat (group) object for this message.
 
         If the message is a contact request
         the sender will become an accepted contact.
@@ -72,23 +80,22 @@ class Message(object):
 
     @props.with_doc
     def id(self):
-        """id of this message. """
+        """id of this message."""
         return lib.dc_msg_get_id(self._dc_msg)
 
     @props.with_doc
     def text(self) -> str:
-        """unicode text of this messages (might be empty if not a text message). """
+        """unicode text of this messages (might be empty if not a text message)."""
         return from_dc_charpointer(lib.dc_msg_get_text(self._dc_msg))
 
     def set_text(self, text):
-        """set text of this message. """
+        """set text of this message."""
         lib.dc_msg_set_text(self._dc_msg, as_dc_charpointer(text))
 
     @props.with_doc
     def html(self) -> str:
-        """html text of this messages (might be empty if not an html message). """
-        return from_optional_dc_charpointer(
-            lib.dc_get_msg_html(self.account._dc_context, self.id)) or ""
+        """html text of this messages (might be empty if not an html message)."""
+        return from_optional_dc_charpointer(lib.dc_get_msg_html(self.account._dc_context, self.id)) or ""
 
     def has_html(self):
         """return True if this message has an html part, False otherwise."""
@@ -103,11 +110,11 @@ class Message(object):
 
     @props.with_doc
     def filename(self):
-        """filename if there was an attachment, otherwise empty string. """
+        """filename if there was an attachment, otherwise empty string."""
         return from_dc_charpointer(lib.dc_msg_get_file(self._dc_msg))
 
     def set_file(self, path, mime_type=None):
-        """set file for this message from path and mime_type. """
+        """set file for this message from path and mime_type."""
         mtype = ffi.NULL if mime_type is None else as_dc_charpointer(mime_type)
         if not os.path.exists(path):
             raise ValueError("path does not exist: {!r}".format(path))
@@ -115,7 +122,7 @@ class Message(object):
 
     @props.with_doc
     def basename(self) -> str:
-        """basename of the attachment if it exists, otherwise empty string. """
+        """basename of the attachment if it exists, otherwise empty string."""
         # FIXME, it does not return basename
         return from_dc_charpointer(lib.dc_msg_get_filename(self._dc_msg))
 
@@ -124,44 +131,74 @@ class Message(object):
         """mime type of the file (if it exists)"""
         return from_dc_charpointer(lib.dc_msg_get_filemime(self._dc_msg))
 
+    def get_status_updates(self, serial: int = 0) -> list:
+        """Get the status updates of this webxdc message.
+
+        The status updates may be sent by yourself or by other members.
+        If this message doesn't have a webxdc instance, an empty list is returned.
+
+        :param serial: The last known serial. Pass 0 if there are no known serials to receive all updates.
+        """
+        return json.loads(
+            from_dc_charpointer(lib.dc_get_webxdc_status_updates(self.account._dc_context, self.id, serial))
+        )
+
+    def send_status_update(self, json_data: Union[str, dict], description: str) -> bool:
+        """Send an status update for the webxdc instance of this message.
+
+        If the webxdc instance is a draft, the update is not sent immediately.
+        Instead, the updates are collected and sent out in a batch when the instance is actually sent.
+
+        :param json_data: program-readable data, the actual payload.
+        :param description: The user-visible description of JSON data
+        :returns: True on success, False otherwise
+        """
+        if isinstance(json_data, dict):
+            json_data = json.dumps(json_data, default=str)
+        return bool(
+            lib.dc_send_webxdc_status_update(
+                self.account._dc_context, self.id, as_dc_charpointer(json_data), as_dc_charpointer(description)
+            )
+        )
+
     def is_system_message(self):
-        """ return True if this message is a system/info message. """
+        """return True if this message is a system/info message."""
         return bool(lib.dc_msg_is_info(self._dc_msg))
 
     def is_setup_message(self):
-        """ return True if this message is a setup message. """
+        """return True if this message is a setup message."""
         return lib.dc_msg_is_setupmessage(self._dc_msg)
 
     def get_setupcodebegin(self) -> str:
-        """ return the first characters of a setup code in a setup message. """
+        """return the first characters of a setup code in a setup message."""
         return from_dc_charpointer(lib.dc_msg_get_setupcodebegin(self._dc_msg))
 
     def is_encrypted(self):
-        """ return True if this message was encrypted. """
+        """return True if this message was encrypted."""
         return bool(lib.dc_msg_get_showpadlock(self._dc_msg))
 
     def is_bot(self):
-        """ return True if this message is submitted automatically. """
+        """return True if this message is submitted automatically."""
         return bool(lib.dc_msg_is_bot(self._dc_msg))
 
     def is_forwarded(self):
-        """ return True if this message was forwarded. """
+        """return True if this message was forwarded."""
         return bool(lib.dc_msg_is_forwarded(self._dc_msg))
 
     def get_message_info(self) -> str:
-        """ Return informational text for a single message.
+        """Return informational text for a single message.
 
         The text is multiline and may contain eg. the raw text of the message.
         """
         return from_dc_charpointer(lib.dc_get_msg_info(self.account._dc_context, self.id))
 
+    def get_summarytext(self, width: int) -> str:
+        """Get a message summary as a single line of text. Typically used for notifications."""
+        return from_dc_charpointer(lib.dc_msg_get_summarytext(self._dc_msg, width))
+
     def continue_key_transfer(self, setup_code):
-        """ extract key and use it as primary key for this account. """
-        res = lib.dc_continue_key_transfer(
-                self.account._dc_context,
-                self.id,
-                as_dc_charpointer(setup_code)
-        )
+        """extract key and use it as primary key for this account."""
+        res = lib.dc_continue_key_transfer(self.account._dc_context, self.id, as_dc_charpointer(setup_code))
         if res == 0:
             raise ValueError("could not decrypt")
 
@@ -230,7 +267,7 @@ class Message(object):
         lib.dc_msg_force_plaintext(self._dc_msg)
 
     def get_mime_headers(self):
-        """ return mime-header object for an incoming message.
+        """return mime-header object for an incoming message.
 
         This only returns a non-None object if ``save_mime_headers``
         config option was set and the message is incoming.
@@ -238,6 +275,7 @@ class Message(object):
         :returns: email-mime message object (with headers only, no body).
         """
         import email.parser
+
         mime_headers = lib.dc_get_mime_headers(self.account._dc_context, self.id)
         if mime_headers:
             s = ffi.string(ffi.gc(mime_headers, lib.dc_str_unref))
@@ -257,6 +295,7 @@ class Message(object):
         :returns: :class:`deltachat.chat.Chat` object
         """
         from .chat import Chat
+
         chat_id = lib.dc_msg_get_chat_id(self._dc_msg)
         return Chat(self.account, chat_id)
 
@@ -266,13 +305,11 @@ class Message(object):
 
         Usually used to impersonate someone else.
         """
-        return from_optional_dc_charpointer(
-            lib.dc_msg_get_override_sender_name(self._dc_msg))
+        return from_optional_dc_charpointer(lib.dc_msg_get_override_sender_name(self._dc_msg))
 
     def set_override_sender_name(self, name):
-        """set different sender name for a message. """
-        lib.dc_msg_set_override_sender_name(
-            self._dc_msg, as_dc_charpointer(name))
+        """set different sender name for a message."""
+        lib.dc_msg_set_override_sender_name(self._dc_msg, as_dc_charpointer(name))
 
     def get_sender_chat(self):
         """return the 1:1 chat with the sender of this message.
@@ -287,6 +324,7 @@ class Message(object):
         :returns: :class:`deltachat.chat.Contact` instance
         """
         from .contact import Contact
+
         contact_id = lib.dc_msg_get_from_id(self._dc_msg)
         return Contact(self.account, contact_id)
 
@@ -299,14 +337,11 @@ class Message(object):
             dc_msg = self._dc_msg
         else:
             # load message from db to get a fresh/current state
-            dc_msg = ffi.gc(
-                lib.dc_get_msg(self.account._dc_context, self.id),
-                lib.dc_msg_unref
-            )
+            dc_msg = ffi.gc(lib.dc_get_msg(self.account._dc_context, self.id), lib.dc_msg_unref)
         return lib.dc_msg_get_state(dc_msg)
 
     def is_in_fresh(self):
-        """ return True if Message is incoming fresh message (un-noticed).
+        """return True if Message is incoming fresh message (un-noticed).
 
         Fresh messages are not noticed nor seen and are typically
         shown in notifications.
@@ -330,25 +365,25 @@ class Message(object):
         return self._msgstate == const.DC_STATE_IN_SEEN
 
     def is_outgoing(self):
-        """Return True if Message is outgoing. """
+        """Return True if Message is outgoing."""
         return self._msgstate in (
-            const.DC_STATE_OUT_PREPARING, const.DC_STATE_OUT_PENDING,
-            const.DC_STATE_OUT_FAILED, const.DC_STATE_OUT_MDN_RCVD,
-            const.DC_STATE_OUT_DELIVERED)
+            const.DC_STATE_OUT_PREPARING,
+            const.DC_STATE_OUT_PENDING,
+            const.DC_STATE_OUT_FAILED,
+            const.DC_STATE_OUT_MDN_RCVD,
+            const.DC_STATE_OUT_DELIVERED,
+        )
 
     def is_out_preparing(self):
-        """Return True if Message is outgoing, but its file is being prepared.
-        """
+        """Return True if Message is outgoing, but its file is being prepared."""
         return self._msgstate == const.DC_STATE_OUT_PREPARING
 
     def is_out_pending(self):
-        """Return True if Message is outgoing, but is pending (no single checkmark).
-        """
+        """Return True if Message is outgoing, but is pending (no single checkmark)."""
         return self._msgstate == const.DC_STATE_OUT_PENDING
 
     def is_out_failed(self):
-        """Return True if Message is unrecoverably failed.
-        """
+        """Return True if Message is unrecoverably failed."""
         return self._msgstate == const.DC_STATE_OUT_FAILED
 
     def is_out_delivered(self):
@@ -375,48 +410,58 @@ class Message(object):
         return lib.dc_msg_get_viewtype(self._dc_msg)
 
     def is_text(self):
-        """ return True if it's a text message. """
+        """return True if it's a text message."""
         return self._view_type == const.DC_MSG_TEXT
 
     def is_image(self):
-        """ return True if it's an image message. """
+        """return True if it's an image message."""
         return self._view_type == const.DC_MSG_IMAGE
 
     def is_gif(self):
-        """ return True if it's a gif message. """
+        """return True if it's a gif message."""
         return self._view_type == const.DC_MSG_GIF
 
     def is_sticker(self):
-        """ return True if it's a sticker message. """
+        """return True if it's a sticker message."""
         return self._view_type == const.DC_MSG_STICKER
 
     def is_audio(self):
-        """ return True if it's an audio message. """
+        """return True if it's an audio message."""
         return self._view_type == const.DC_MSG_AUDIO
 
     def is_video(self):
-        """ return True if it's a video message. """
+        """return True if it's a video message."""
         return self._view_type == const.DC_MSG_VIDEO
 
+    def is_videochat_invitation(self):
+        """return True if it's a videochat invitation message."""
+        return self._view_type == const.DC_MSG_VIDEOCHAT_INVITATION
+
+    def is_webxdc(self):
+        """return True if it's a Webxdc message."""
+        return self._view_type == const.DC_MSG_WEBXDC
+
     def is_file(self):
-        """ return True if it's a file message. """
+        """return True if it's a file message."""
         return self._view_type == const.DC_MSG_FILE
 
     def mark_seen(self):
-        """ mark this message as seen. """
+        """mark this message as seen."""
         self.account.mark_seen_messages([self.id])
 
 
 # some code for handling DC_MSG_* view types
 
 _view_type_mapping = {
-    'text': const.DC_MSG_TEXT,
-    'image': const.DC_MSG_IMAGE,
-    'gif': const.DC_MSG_GIF,
-    'audio': const.DC_MSG_AUDIO,
-    'video': const.DC_MSG_VIDEO,
-    'file': const.DC_MSG_FILE,
-    'sticker': const.DC_MSG_STICKER,
+    "text": const.DC_MSG_TEXT,
+    "image": const.DC_MSG_IMAGE,
+    "gif": const.DC_MSG_GIF,
+    "audio": const.DC_MSG_AUDIO,
+    "video": const.DC_MSG_VIDEO,
+    "file": const.DC_MSG_FILE,
+    "sticker": const.DC_MSG_STICKER,
+    "videochat": const.DC_MSG_VIDEOCHAT_INVITATION,
+    "webxdc": const.DC_MSG_WEBXDC,
 }
 
 
@@ -424,14 +469,16 @@ def get_viewtype_code_from_name(view_type_name):
     code = _view_type_mapping.get(view_type_name)
     if code is not None:
         return code
-    raise ValueError("message typecode not found for {!r}, "
-                     "available {!r}".format(view_type_name, list(_view_type_mapping.keys())))
+    raise ValueError(
+        "message typecode not found for {!r}, " "available {!r}".format(view_type_name, list(_view_type_mapping.keys()))
+    )
 
 
 #
 # some helper code for turning system messages into hook events
 #
 
+
 def map_system_message(msg):
     if msg.is_system_message():
         res = parse_system_add_remove(msg.text)
@@ -448,7 +495,7 @@ def map_system_message(msg):
 
 
 def extract_addr(text):
-    m = re.match(r'.*\((.+@.+)\)', text)
+    m = re.match(r".*\((.+@.+)\)", text)
     if m:
         text = m.group(1)
     text = text.rstrip(".")
@@ -456,9 +503,9 @@ def extract_addr(text):
 
 
 def parse_system_add_remove(text):
-    """ return add/remove info from parsing the given system message text.
+    """return add/remove info from parsing the given system message text.
 
-    returns a (action, affected, actor) triple """
+    returns a (action, affected, actor) triple"""
 
     # Member Me (x@y) removed by a@b.
     # Member x@y added by a@b
@@ -467,7 +514,7 @@ def parse_system_add_remove(text):
     # Group left by some one (tmp1@x.org).
     # Group left by tmp1@x.org.
     text = text.lower()
-    m = re.match(r'member (.+) (removed|added) by (.+)', text)
+    m = re.match(r"member (.+) (removed|added) by (.+)", text)
     if m:
         affected, action, actor = m.groups()
         return action, extract_addr(affected), extract_addr(actor)

python/src/deltachat/props.py

@@ -9,6 +9,7 @@ def with_doc(f):
 # https://github.com/devpi/devpi/blob/master/common/devpi_common/types.py
 def cached(f):
     """returns a cached property that is calculated by function f"""
+
     def get(self):
         try:
             return self._property_cache[f]

python/src/deltachat/provider.py

@@ -26,14 +26,12 @@ class Provider(object):
     @property
     def overview_page(self) -> str:
         """URL to the overview page of the provider on providers.delta.chat."""
-        return from_dc_charpointer(
-            lib.dc_provider_get_overview_page(self._provider))
+        return from_dc_charpointer(lib.dc_provider_get_overview_page(self._provider))
 
     @property
     def get_before_login_hints(self) -> str:
         """Should be shown to the user on login."""
-        return from_dc_charpointer(
-            lib.dc_provider_get_before_login_hint(self._provider))
+        return from_dc_charpointer(lib.dc_provider_get_before_login_hint(self._provider))
 
     @property
     def status(self) -> int:

python/src/deltachat/testplugin.py

@@ -1,56 +1,62 @@
 from __future__ import print_function
-import os
-import sys
-import io
-import subprocess
-import queue
-import threading
+
 import fnmatch
+import io
+import os
+import pathlib
+import queue
+import subprocess
+import sys
+import threading
 import time
 import weakref
 from queue import Queue
-from typing import List, Callable
+from typing import Callable, List, Optional
 
 import pytest
 import requests
-import pathlib
-
-from . import Account, const, account_hookimpl, get_core_info
-from .events import FFIEventLogger, FFIEventTracker
 from _pytest._code import Source
 
 import deltachat
 
+from . import Account, account_hookimpl, const, get_core_info
+from .events import FFIEventLogger, FFIEventTracker
+
 
 def pytest_addoption(parser):
     group = parser.getgroup("deltachat testplugin options")
     group.addoption(
-        "--liveconfig", action="store", default=None,
-        help="a file with >=2 lines where each line "
-             "contains NAME=VALUE config settings for one account"
+        "--liveconfig",
+        action="store",
+        default=None,
+        help="a file with >=2 lines where each line " "contains NAME=VALUE config settings for one account",
     )
     group.addoption(
-        "--ignored", action="store_true",
+        "--ignored",
+        action="store_true",
         help="Also run tests marked with the ignored marker",
     )
     group.addoption(
-        "--strict-tls", action="store_true",
+        "--strict-tls",
+        action="store_true",
         help="Never accept invalid TLS certificates for test accounts",
     )
     group.addoption(
-        "--extra-info", action="store_true",
-        help="show more info on failures (imap server state, config)"
+        "--extra-info",
+        action="store_true",
+        help="show more info on failures (imap server state, config)",
     )
     group.addoption(
-        "--debug-setup", action="store_true",
-        help="show events during configure and start io phases of online accounts"
+        "--debug-setup",
+        action="store_true",
+        help="show events during configure and start io phases of online accounts",
     )
 
 
 def pytest_configure(config):
-    cfg = config.getoption('--liveconfig')
+    cfg = config.getoption("--liveconfig")
     if not cfg:
-        cfg = os.getenv('DCC_NEW_TMP_EMAIL')
+        cfg = os.getenv("DCC_NEW_TMP_EMAIL")
         if cfg:
             config.option.liveconfig = cfg
 
@@ -113,19 +119,21 @@ def pytest_configure(config):
 
 def pytest_report_header(config, startdir):
     info = get_core_info()
-    summary = ['Deltachat core={} sqlite={} journal_mode={}'.format(
-        info['deltachat_core_version'],
-        info['sqlite_version'],
-        info['journal_mode'],
-    )]
+    summary = [
+        "Deltachat core={} sqlite={} journal_mode={}".format(
+            info["deltachat_core_version"],
+            info["sqlite_version"],
+            info["journal_mode"],
+        )
+    ]
 
     cfg = config.option.liveconfig
     if cfg:
         if "?" in cfg:
             url, token = cfg.split("?", 1)
-            summary.append('Liveconfig provider: {}?<token ommitted>'.format(url))
+            summary.append("Liveconfig provider: {}?<token ommitted>".format(url))
         else:
-            summary.append('Liveconfig file: {}'.format(cfg))
+            summary.append("Liveconfig file: {}".format(cfg))
     return summary
 
 
@@ -135,15 +143,15 @@ def testprocess(request):
 
 
 class TestProcess:
-    """ A pytest session-scoped instance to help with managing "live" account configurations.
-    """
+    """A pytest session-scoped instance to help with managing "live" account configurations."""
+
     def __init__(self, pytestconfig):
         self.pytestconfig = pytestconfig
         self._addr2files = {}
         self._configlist = []
 
     def get_liveconfig_producer(self):
-        """ provide live account configs, cached on a per-test-process scope
+        """provide live account configs, cached on a per-test-process scope
         so that test functions can re-use already known live configs.
         Depending on the --liveconfig option this comes from
         a HTTP provider or a file with a line specifying each accounts config.
@@ -154,7 +162,7 @@ class TestProcess:
 
         if not liveconfig_opt.startswith("http"):
             for line in open(liveconfig_opt):
-                if line.strip() and not line.strip().startswith('#'):
+                if line.strip() and not line.strip().startswith("#"):
                     d = {}
                     for part in line.split():
                         name, value = part.split("=")
@@ -170,8 +178,7 @@ class TestProcess:
                 except IndexError:
                     res = requests.post(liveconfig_opt)
                     if res.status_code != 200:
-                        pytest.fail("newtmpuser count={} code={}: '{}'".format(
-                                    index, res.status_code, res.text))
+                        pytest.fail("newtmpuser count={} code={}: '{}'".format(index, res.status_code, res.text))
                     d = res.json()
                     config = dict(addr=d["email"], mail_pw=d["password"])
                     print("newtmpuser {}: addr={}".format(index, config["addr"]))
@@ -230,13 +237,17 @@ def data(request):
             # because we are run from a dev-setup with pytest direct,
             # through tox, and then maybe also from deltachat-binding
             # users like "deltabot".
-            self.paths = [os.path.normpath(x) for x in [
-                os.path.join(os.path.dirname(request.fspath.strpath), "data"),
-                os.path.join(os.path.dirname(__file__), "..", "..", "..", "test-data")
-            ]]
+            self.paths = [
+                os.path.normpath(x)
+                for x in [
+                    os.path.join(os.path.dirname(request.fspath.strpath), "data"),
+                    os.path.join(os.path.dirname(request.fspath.strpath), "..", "..", "test-data"),
+                    os.path.join(os.path.dirname(__file__), "..", "..", "..", "test-data"),
+                ]
+            ]
 
         def get_path(self, bn):
-            """ return path of file or None if it doesn't exist. """
+            """return path of file or None if it doesn't exist."""
             for path in self.paths:
                 fn = os.path.join(path, *bn.split("/"))
                 if os.path.exists(fn):
@@ -253,10 +264,11 @@ def data(request):
 
 
 class ACSetup:
-    """ accounts setup helper to deal with multiple configure-process
+    """accounts setup helper to deal with multiple configure-process
     and io & imap initialization phases. From tests, use the higher level
     public ACFactory methods instead of its private helper class.
     """
+
     CONFIGURING = "CONFIGURING"
     CONFIGURED = "CONFIGURED"
     IDLEREADY = "IDLEREADY"
@@ -272,17 +284,18 @@ class ACSetup:
         print("[acsetup]", "{:.3f}".format(time.time() - self.init_time), *args)
 
     def add_configured(self, account):
-        """ add an already configured account. """
+        """add an already configured account."""
         assert account.is_configured()
         self._account2state[account] = self.CONFIGURED
         self.log("added already configured account", account, account.get_config("addr"))
 
     def start_configure(self, account, reconfigure=False):
-        """ add an account and start its configure process. """
+        """add an account and start its configure process."""
+
         class PendingTracker:
             @account_hookimpl
-            def ac_configure_completed(this, success):
-                self._configured_events.put((account, success))
+            def ac_configure_completed(this, success: bool, comment: Optional[str]) -> None:
+                self._configured_events.put((account, success, comment))
 
         account.add_account_plugin(PendingTracker(), name="pending_tracker")
         self._account2state[account] = self.CONFIGURING
@@ -290,7 +303,7 @@ class ACSetup:
         self.log("started configure on", account)
 
     def wait_one_configured(self, account):
-        """ wait until this account has successfully configured. """
+        """wait until this account has successfully configured."""
         if self._account2state[account] == self.CONFIGURING:
             while 1:
                 acc = self._pop_config_success()
@@ -301,7 +314,7 @@ class ACSetup:
             acc._evtracker.consume_events()
 
     def bring_online(self):
-        """ Wait for all accounts to become ready to receive messages.
+        """Wait for all accounts to become ready to receive messages.
 
         This will initialize logging, start IO and the direct_imap attribute
         for each account which either is CONFIGURED already or which is CONFIGURING
@@ -320,9 +333,9 @@ class ACSetup:
         print("finished, account2state", self._account2state)
 
     def _pop_config_success(self):
-        acc, success = self._configured_events.get()
+        acc, success, comment = self._configured_events.get()
         if not success:
-            pytest.fail("configuring online account failed: {}".format(acc))
+            pytest.fail("configuring online account {} failed: {}".format(acc, comment))
         self._account2state[acc] = self.CONFIGURED
         return acc
 
@@ -336,12 +349,12 @@ class ACSetup:
         acc.log("inbox IDLE ready")
 
     def init_logging(self, acc):
-        """ idempotent function for initializing logging (will replace existing logger). """
+        """idempotent function for initializing logging (will replace existing logger)."""
         logger = FFIEventLogger(acc, logid=acc._logid, init_time=self.init_time)
         acc.add_account_plugin(logger, name="logger-" + acc._logid)
 
     def init_imap(self, acc):
-        """ initialize direct_imap and cleanup server state. """
+        """initialize direct_imap and cleanup server state."""
         from deltachat.direct_imap import DirectImap
 
         assert acc.is_configured()
@@ -375,8 +388,7 @@ class ACFactory:
         self._finalizers = []
         self._accounts = []
         self._acsetup = ACSetup(testprocess, self.init_time)
-        self._preconfigured_keys = ["alice", "bob", "charlie",
-                                    "dom", "elena", "fiona"]
+        self._preconfigured_keys = ["alice", "bob", "charlie", "dom", "elena", "fiona"]
         self.set_logging_default(False)
         request.addfinalizer(self.finalize)
 
@@ -399,7 +411,7 @@ class ACFactory:
                 acc.disable_logging()
 
     def get_next_liveconfig(self):
-        """ Base function to get functional online configurations
+        """Base function to get functional online configurations
         where we can make valid SMTP and IMAP connections with.
         """
         configdict = next(self._liveconfig_producer).copy()
@@ -418,16 +430,16 @@ class ACFactory:
         if addr in self.testprocess._addr2files:
             return self._getaccount(addr)
 
-    def get_unconfigured_account(self):
-        return self._getaccount()
+    def get_unconfigured_account(self, closed=False):
+        return self._getaccount(closed=closed)
 
-    def _getaccount(self, try_cache_addr=None):
+    def _getaccount(self, try_cache_addr=None, closed=False):
         logid = "ac{}".format(len(self._accounts) + 1)
         # we need to use fixed database basename for maybe_cache_* functions to work
         path = self.tmpdir.mkdir(logid).join("dc.db")
         if try_cache_addr:
             self.testprocess.cache_maybe_retrieve_configured_db_files(try_cache_addr, path)
-        ac = Account(path.strpath, logging=self._logging)
+        ac = Account(path.strpath, logging=self._logging, closed=closed)
         ac._logid = logid  # later instantiated FFIEventLogger needs this
         ac._evtracker = ac.add_account_plugin(FFIEventTracker(ac))
         if self.pytestconfig.getoption("--debug-setup"):
@@ -456,16 +468,23 @@ class ACFactory:
             else:
                 print("WARN: could not use preconfigured keys for {!r}".format(addr))
 
-    def get_pseudo_configured_account(self):
+    def get_pseudo_configured_account(self, passphrase: Optional[str] = None) -> Account:
         # do a pseudo-configured account
-        ac = self.get_unconfigured_account()
+        ac = self.get_unconfigured_account(closed=bool(passphrase))
+        if passphrase:
+            ac.open(passphrase)
         acname = ac._logid
         addr = "{}@offline.org".format(acname)
-        ac.update_config(dict(
-            addr=addr, displayname=acname, mail_pw="123",
-            configured_addr=addr, configured_mail_pw="123",
-            configured="1",
-        ))
+        ac.update_config(
+            dict(
+                addr=addr,
+                displayname=acname,
+                mail_pw="123",
+                configured_addr=addr,
+                configured_mail_pw="123",
+                configured="1",
+            )
+        )
         self._preconfigure_key(ac, addr)
         self._acsetup.init_logging(ac)
         return ac
@@ -501,7 +520,7 @@ class ACFactory:
         return ac
 
     def wait_configured(self, account):
-        """ Wait until the specified account has successfully completed configure. """
+        """Wait until the specified account has successfully completed configure."""
         self._acsetup.wait_one_configured(account)
 
     def bring_accounts_online(self):
@@ -531,8 +550,10 @@ class ACFactory:
             sys.executable,
             "-u",
             fn,
-            "--email", bot_cfg["addr"],
-            "--password", bot_cfg["mail_pw"],
+            "--email",
+            bot_cfg["addr"],
+            "--password",
+            bot_cfg["mail_pw"],
             bot_ac.db_path,
         ]
         if ffi:
@@ -543,9 +564,9 @@ class ACFactory:
             stdin=subprocess.DEVNULL,
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,  # combine stdout/stderr in one stream
-            bufsize=0,                 # line buffering
-            close_fds=True,            # close all FDs other than 0/1/2
-            universal_newlines=True    # give back text
+            bufsize=0,  # line buffering
+            close_fds=True,  # close all FDs other than 0/1/2
+            universal_newlines=True,  # give back text
         )
         bot = BotProcess(popen, addr=bot_cfg["addr"])
         self._finalizers.append(bot.kill)
@@ -565,7 +586,7 @@ class ACFactory:
     def introduce_each_other(self, accounts, sending=True):
         to_wait = []
         for i, acc in enumerate(accounts):
-            for acc2 in accounts[i + 1:]:
+            for acc2 in accounts[i + 1 :]:
                 chat = self.get_accepted_chat(acc, acc2)
                 if sending:
                     chat.send_text("hi")

python/src/deltachat/tracker.py

@@ -1,12 +1,11 @@
-
 from queue import Queue
 from threading import Event
 
-from .hookspec import account_hookimpl, Global
+from .hookspec import Global, account_hookimpl
 
 
 class ImexFailed(RuntimeError):
-    """ Exception for signalling that import/export operations failed."""
+    """Exception for signalling that import/export operations failed."""
 
 
 class ImexTracker:
@@ -24,14 +23,15 @@ class ImexTracker:
         while True:
             ev = self._imex_events.get(timeout=progress_timeout)
             if isinstance(ev, int) and ev >= target_progress:
-                assert ev <= progress_upper_limit, \
+                assert ev <= progress_upper_limit, (
                     str(ev) + " exceeded upper progress limit " + str(progress_upper_limit)
+                )
                 return ev
             if ev == 0:
                 return None
 
     def wait_finish(self, progress_timeout=60):
-        """ Return list of written files, raise ValueError if ExportFailed. """
+        """Return list of written files, raise ValueError if ExportFailed."""
         files_written = []
         while True:
             ev = self._imex_events.get(timeout=progress_timeout)
@@ -44,7 +44,7 @@ class ImexTracker:
 
 
 class ConfigureFailed(RuntimeError):
-    """ Exception for signalling that configuration failed."""
+    """Exception for signalling that configuration failed."""
 
 
 class ConfigureTracker:
@@ -77,11 +77,11 @@ class ConfigureTracker:
         self.account.remove_account_plugin(self)
 
     def wait_smtp_connected(self):
-        """ wait until smtp is configured. """
+        """wait until smtp is configured."""
         self._smtp_finished.wait()
 
     def wait_imap_connected(self):
-        """ wait until smtp is configured. """
+        """wait until smtp is configured."""
         self._imap_finished.wait()
 
     def wait_progress(self, data1=None):
@@ -91,7 +91,7 @@ class ConfigureTracker:
                 break
 
     def wait_finish(self, timeout=None):
-        """ wait until configure is completed.
+        """wait until configure is completed.
 
         Raise Exception if Configure failed
         """

python/tests/auditwheels.py

@@ -1,17 +1,19 @@
-
 import os
-import platform
 import subprocess
 import sys
 
-
 if __name__ == "__main__":
     assert len(sys.argv) == 2
     workspacedir = sys.argv[1]
-    arch = platform.machine()
     for relpath in os.listdir(workspacedir):
         if relpath.startswith("deltachat"):
             p = os.path.join(workspacedir, relpath)
             subprocess.check_call(
-                ["auditwheel", "repair", p, "-w", workspacedir,
-                 "--plat", "manylinux2014_" + arch])
+                [
+                    "auditwheel",
+                    "repair",
+                    p,
+                    "-w",
+                    workspacedir,
+                ]
+            )

python/tests/package_wheels.py

@@ -1,8 +1,6 @@
-
 import os
-import sys
 import subprocess
-
+import sys
 
 if __name__ == "__main__":
     assert len(sys.argv) == 2

python/tests/stress_test_db.py

@@ -1,9 +1,9 @@
-
-import time
-import threading
-import pytest
 import os
-from queue import Queue, Empty
+import threading
+import time
+from queue import Empty, Queue
+
+import pytest
 
 import deltachat
 
@@ -24,7 +24,7 @@ def test_db_busy_error(acfactory, tmpdir):
     for acc in accounts:
         acc.bigfile = os.path.join(acc.get_blobdir(), "bigfile")
         with open(acc.bigfile, "wb") as f:
-            f.write(b"01234567890"*1000_000)
+            f.write(b"01234567890" * 1000_000)
     log("created %s bigfiles" % len(accounts))
 
     contact_addrs = [acc.get_self_contact().addr for acc in accounts]
@@ -63,9 +63,7 @@ def test_db_busy_error(acfactory, tmpdir):
         elif report_type == ReportType.message_echo:
             continue
         else:
-            raise ValueError("{} unknown report type {}, args={}".format(
-                addr, report_type, report_args
-            ))
+            raise ValueError("{} unknown report type {}, args={}".format(addr, report_type, report_args))
         alive_count -= 1
         replier.log("shutting down")
         replier.account.shutdown()
@@ -88,10 +86,7 @@ class AutoReplier:
         self.current_sent = 0
         self.addr = self.account.get_self_contact().addr
 
-        self._thread = threading.Thread(
-            name="Stats{}".format(self.account),
-            target=self.thread_stats
-        )
+        self._thread = threading.Thread(name="Stats{}".format(self.account), target=self.thread_stats)
         self._thread.setDaemon(True)
         self._thread.start()
 

python/tests/test_0_complex_or_slow.py

@@ -1,4 +1,5 @@
 import sys
+
 import pytest
 
 
@@ -277,7 +278,7 @@ def test_fetch_existing_msgs_group_and_single(acfactory, lp):
     assert len(chats) == 4  # two newly created chats + self-chat + device-chat
     group_chat = [c for c in chats if c.get_name() == "group name"][0]
     assert group_chat.is_group()
-    private_chat, = [c for c in chats if c.get_name() == ac1_ac2_chat.get_name()]
+    (private_chat,) = [c for c in chats if c.get_name() == ac1_ac2_chat.get_name()]
     assert not private_chat.is_group()
 
     group_messages = group_chat.get_messages()
@@ -378,7 +379,7 @@ def test_ephemeral_timer(acfactory, lp):
     lp.sec("ac1: check that ephemeral timer is set for chat")
     assert chat1.get_ephemeral_timer() == 60
     chat1_summary = chat1.get_summary()
-    assert chat1_summary["ephemeral_timer"] == {'Enabled': {'duration': 60}}
+    assert chat1_summary["ephemeral_timer"] == {"Enabled": {"duration": 60}}
 
     lp.sec("ac2: receive system message about ephemeral timer modification")
     ac2._evtracker.get_matching("DC_EVENT_CHAT_EPHEMERAL_TIMER_MODIFIED")

python/tests/test_1_online.py

@@ -1,10 +1,11 @@
 import os
-import sys
 import queue
+import sys
 from datetime import datetime, timezone
-from imap_tools import AND, U
 
 import pytest
+from imap_tools import AND, U
+
 from deltachat import const
 from deltachat.hookspec import account_hookimpl
 from deltachat.message import Message
@@ -78,7 +79,7 @@ def test_export_import_self_keys(acfactory, tmpdir, lp):
     assert len(export_files) == 2
     for x in export_files:
         assert x.startswith(dir.strpath)
-    key_id, = ac1._evtracker.get_info_regex_groups(r".*xporting.*KeyId\((.*)\).*")
+    (key_id,) = ac1._evtracker.get_info_regex_groups(r".*xporting.*KeyId\((.*)\).*")
     ac1._evtracker.consume_events()
 
     lp.sec("exported keys (private and public)")
@@ -86,8 +87,7 @@ def test_export_import_self_keys(acfactory, tmpdir, lp):
         lp.indent(dir.strpath + os.sep + name)
     lp.sec("importing into existing account")
     ac2.import_self_keys(dir.strpath)
-    key_id2, = ac2._evtracker.get_info_regex_groups(
-        r".*stored.*KeyId\((.*)\).*", check_error=False)
+    (key_id2,) = ac2._evtracker.get_info_regex_groups(r".*stored.*KeyId\((.*)\).*", check_error=False)
     assert key_id2 == key_id
 
 
@@ -238,6 +238,70 @@ def test_html_message(acfactory, lp):
     assert html_text in msg2.html
 
 
+def test_videochat_invitation_message(acfactory, lp):
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    chat = acfactory.get_accepted_chat(ac1, ac2)
+    text = "You are invited to a video chat, click https://meet.jit.si/WxEGad0gGzX to join."
+
+    lp.sec("ac1: prepare and send text message to ac2")
+    msg1 = chat.send_text("message0")
+    assert not msg1.is_videochat_invitation()
+
+    lp.sec("wait for ac2 to receive message")
+    msg2 = ac2._evtracker.wait_next_incoming_message()
+    assert msg2.text == "message0"
+    assert not msg2.is_videochat_invitation()
+
+    lp.sec("ac1: prepare and send videochat invitation to ac2")
+    msg1 = Message.new_empty(ac1, "videochat")
+    msg1.set_text(text)
+    msg1 = chat.send_msg(msg1)
+    assert msg1.is_videochat_invitation()
+
+    lp.sec("wait for ac2 to receive message")
+    msg2 = ac2._evtracker.wait_next_incoming_message()
+    assert msg2.text == text
+    assert msg2.is_videochat_invitation()
+
+
+def test_webxdc_message(acfactory, data, lp):
+    ac1, ac2 = acfactory.get_online_accounts(2)
+    chat = acfactory.get_accepted_chat(ac1, ac2)
+
+    lp.sec("ac1: prepare and send text message to ac2")
+    msg1 = chat.send_text("message0")
+    assert not msg1.is_webxdc()
+    assert not msg1.send_status_update({"payload": "not an webxdc"}, "invalid")
+    assert not msg1.get_status_updates()
+
+    lp.sec("wait for ac2 to receive message")
+    msg2 = ac2._evtracker.wait_next_incoming_message()
+    assert msg2.text == "message0"
+    assert not msg2.is_webxdc()
+    assert not msg1.get_status_updates()
+
+    lp.sec("ac1: prepare and send webxdc instance to ac2")
+    msg1 = Message.new_empty(ac1, "webxdc")
+    msg1.set_text("message1")
+    msg1.set_file(data.get_path("webxdc/minimal.xdc"))
+    msg1 = chat.send_msg(msg1)
+    assert msg1.is_webxdc()
+    assert msg1.filename
+
+    assert msg1.send_status_update({"payload": "test1"}, "some test data")
+    assert msg1.send_status_update({"payload": "test2"}, "more test data")
+    assert len(msg1.get_status_updates()) == 2
+    update1 = msg1.get_status_updates()[0]
+    assert update1["payload"] == "test1"
+    assert len(msg1.get_status_updates(update1["serial"])) == 1
+
+    lp.sec("wait for ac2 to receive message")
+    msg2 = ac2._evtracker.wait_next_incoming_message()
+    assert msg2.text == "message1"
+    assert msg2.is_webxdc()
+    assert msg2.filename
+
+
 def test_mvbox_sentbox_threads(acfactory, lp):
     lp.sec("ac1: start with mvbox thread")
     ac1 = acfactory.new_online_configuring_account(mvbox_move=True, sentbox_watch=True)
@@ -689,7 +753,7 @@ def test_gossip_encryption_preference(acfactory, lp):
     msg = ac1._evtracker.wait_next_incoming_message()
     assert msg.text == "first message"
     assert not msg.is_encrypted()
-    res = "End-to-end encryption preferred:\n{}\n".format(ac2.get_config('addr'))
+    res = "End-to-end encryption preferred:\n{}".format(ac2.get_config("addr"))
     assert msg.chat.get_encryption_info() == res
     lp.sec("ac2 learns that ac3 prefers encryption")
     ac2.create_chat(ac3)
@@ -701,7 +765,7 @@ def test_gossip_encryption_preference(acfactory, lp):
     lp.sec("ac3 does not know that ac1 prefers encryption")
     ac1.create_chat(ac3)
     chat = ac3.create_chat(ac1)
-    res = "No encryption:\n{}\n".format(ac1.get_config('addr'))
+    res = "No encryption:\n{}".format(ac1.get_config("addr"))
     assert chat.get_encryption_info() == res
     msg = chat.send_text("not encrypted")
     msg = ac1._evtracker.wait_next_incoming_message()
@@ -712,7 +776,7 @@ def test_gossip_encryption_preference(acfactory, lp):
     group_chat = ac1.create_group_chat("hello")
     group_chat.add_contact(ac2)
     encryption_info = group_chat.get_encryption_info()
-    res = "End-to-end encryption preferred:\n{}\n".format(ac2.get_config("addr"))
+    res = "End-to-end encryption preferred:\n{}".format(ac2.get_config("addr"))
     assert encryption_info == res
     msg = group_chat.send_text("hi")
 
@@ -766,7 +830,7 @@ def test_send_first_message_as_long_unicode_with_cr(acfactory, lp):
 
 def test_no_draft_if_cant_send(acfactory):
     """Tests that no quote can be set if the user can't send to this chat"""
-    ac1, = acfactory.get_online_accounts(1)
+    (ac1,) = acfactory.get_online_accounts(1)
     device_chat = ac1.get_device_chat()
     msg = Message.new_empty(ac1, "text")
     device_chat.set_draft(msg)
@@ -795,7 +859,9 @@ def test_dont_show_emails(acfactory, lp):
     acfactory.bring_accounts_online()
     ac1.stop_io()
 
-    ac1.direct_imap.append("Drafts", """
+    ac1.direct_imap.append(
+        "Drafts",
+        """
         From: ac1 <{}>
         Subject: subj
         To: alice@example.org
@@ -803,8 +869,13 @@ def test_dont_show_emails(acfactory, lp):
         Content-Type: text/plain; charset=utf-8
 
         message in Drafts that is moved to Sent later
-    """.format(ac1.get_config("configured_addr")))
-    ac1.direct_imap.append("Sent", """
+    """.format(
+            ac1.get_config("configured_addr")
+        ),
+    )
+    ac1.direct_imap.append(
+        "Sent",
+        """
         From: ac1 <{}>
         Subject: subj
         To: alice@example.org
@@ -812,8 +883,13 @@ def test_dont_show_emails(acfactory, lp):
         Content-Type: text/plain; charset=utf-8
 
         message in Sent
-    """.format(ac1.get_config("configured_addr")))
-    ac1.direct_imap.append("Spam", """
+    """.format(
+            ac1.get_config("configured_addr")
+        ),
+    )
+    ac1.direct_imap.append(
+        "Spam",
+        """
         From: unknown.address@junk.org
         Subject: subj
         To: {}
@@ -821,8 +897,13 @@ def test_dont_show_emails(acfactory, lp):
         Content-Type: text/plain; charset=utf-8
 
         Unknown message in Spam
-    """.format(ac1.get_config("configured_addr")))
-    ac1.direct_imap.append("Junk", """
+    """.format(
+            ac1.get_config("configured_addr")
+        ),
+    )
+    ac1.direct_imap.append(
+        "Junk",
+        """
         From: unknown.address@junk.org
         Subject: subj
         To: {}
@@ -830,7 +911,10 @@ def test_dont_show_emails(acfactory, lp):
         Content-Type: text/plain; charset=utf-8
 
         Unknown message in Junk
-    """.format(ac1.get_config("configured_addr")))
+    """.format(
+            ac1.get_config("configured_addr")
+        ),
+    )
 
     ac1.set_config("scan_all_folders_debounce_secs", "0")
     lp.sec("All prepared, now let DC find the message")
@@ -1154,7 +1238,7 @@ def test_send_and_receive_image(acfactory, lp, data):
 
 
 def test_import_export_online_all(acfactory, tmpdir, data, lp):
-    ac1, = acfactory.get_online_accounts(1)
+    (ac1,) = acfactory.get_online_accounts(1)
 
     lp.sec("create some chat content")
     chat1 = ac1.create_contact("some1@example.org", name="some1").create_chat()
@@ -1349,8 +1433,9 @@ def test_set_get_contact_avatar(acfactory, data, lp):
 
 
 def test_add_remove_member_remote_events(acfactory, lp):
-    ac1, ac2 = acfactory.get_online_accounts(2)
+    ac1, ac2, ac3 = acfactory.get_online_accounts(3)
     ac1_addr = ac1.get_config("addr")
+    ac3_addr = ac3.get_config("addr")
     # activate local plugin for ac2
     in_list = queue.Queue()
 
@@ -1387,13 +1472,12 @@ def test_add_remove_member_remote_events(acfactory, lp):
     ev = in_list.get()
     assert ev.action == "chat-modified"
     assert chat.is_promoted()
-    assert sorted(x.addr for x in chat.get_contacts()) == \
-        sorted(x.addr for x in ev.chat.get_contacts())
+    assert sorted(x.addr for x in chat.get_contacts()) == sorted(x.addr for x in ev.chat.get_contacts())
 
     lp.sec("ac1: add address2")
     # note that if the above create_chat() would not
     # happen we would not receive a proper member_added event
-    contact2 = chat.add_contact("devnull@testrun.org")
+    contact2 = chat.add_contact(ac3_addr)
     ev = in_list.get()
     assert ev.action == "chat-modified"
     ev = in_list.get()
@@ -1401,7 +1485,7 @@ def test_add_remove_member_remote_events(acfactory, lp):
     ev = in_list.get()
     assert ev.action == "added"
     assert ev.message.get_sender_contact().addr == ac1_addr
-    assert ev.contact.addr == "devnull@testrun.org"
+    assert ev.contact.addr == ac3_addr
 
     lp.sec("ac1: remove address2")
     chat.remove_contact(contact2)
@@ -1530,8 +1614,10 @@ def test_connectivity(acfactory, lp):
     ac1._evtracker.wait_for_connectivity(const.DC_CONNECTIVITY_CONNECTING)
     ac1._evtracker.wait_for_connectivity_change(const.DC_CONNECTIVITY_CONNECTING, const.DC_CONNECTIVITY_CONNECTED)
 
-    lp.sec("Test that after calling start_io(), maybe_network() and waiting for `all_work_done()`, " +
-           "all messages are fetched")
+    lp.sec(
+        "Test that after calling start_io(), maybe_network() and waiting for `all_work_done()`, "
+        + "all messages are fetched"
+    )
 
     ac1.direct_imap.select_config_folder("inbox")
     with ac1.direct_imap.idle() as idle1:
@@ -1594,10 +1680,12 @@ def test_fetch_deleted_msg(acfactory, lp):
 
     See https://github.com/deltachat/deltachat-core-rust/issues/2429.
     """
-    ac1, = acfactory.get_online_accounts(1)
+    (ac1,) = acfactory.get_online_accounts(1)
     ac1.stop_io()
 
-    ac1.direct_imap.append("INBOX", """
+    ac1.direct_imap.append(
+        "INBOX",
+        """
         From: alice <alice@example.org>
         Subject: subj
         To: bob@example.com
@@ -1606,7 +1694,8 @@ def test_fetch_deleted_msg(acfactory, lp):
         Content-Type: text/plain; charset=utf-8
 
         Deleted message
-    """)
+    """,
+    )
     ac1.direct_imap.delete("1:*", expunge=False)
     ac1.start_io()
 
@@ -1888,11 +1977,26 @@ def test_group_quote(acfactory, lp):
     assert received_reply.quote.id == out_msg.id
 
 
-@pytest.mark.parametrize("folder,move,expected_destination,", [
-    ("xyz", False, "xyz"),  # Test that emails are recognized in a random folder but not moved
-    ("xyz", True, "DeltaChat"),  # ...emails are found in a random folder and moved to DeltaChat
-    ("Spam", False, "INBOX"),  # ...emails are moved from the spam folder to the Inbox
-])
+@pytest.mark.parametrize(
+    "folder,move,expected_destination,",
+    [
+        (
+            "xyz",
+            False,
+            "xyz",
+        ),  # Test that emails are recognized in a random folder but not moved
+        (
+            "xyz",
+            True,
+            "DeltaChat",
+        ),  # ...emails are found in a random folder and moved to DeltaChat
+        (
+            "Spam",
+            False,
+            "INBOX",
+        ),  # ...emails are moved from the spam folder to the Inbox
+    ],
+)
 # Testrun.org does not support the CREATE-SPECIAL-USE capability, which means that we can't create a folder with
 # the "\Junk" flag (see https://tools.ietf.org/html/rfc6154). So, we can't test spam folder detection by flag.
 def test_scan_folders(acfactory, lp, folder, move, expected_destination):

python/tests/test_2_increation.py

@@ -2,13 +2,13 @@ from __future__ import print_function
 
 import os.path
 import shutil
+from filecmp import cmp
 
 import pytest
-from filecmp import cmp
 
 
 def wait_msg_delivered(account, msg_list):
-    """ wait for one or more MSG_DELIVERED events to match msg_list contents. """
+    """wait for one or more MSG_DELIVERED events to match msg_list contents."""
     msg_list = list(msg_list)
     while msg_list:
         ev = account._evtracker.get_matching("DC_EVENT_MSG_DELIVERED")
@@ -16,7 +16,7 @@ def wait_msg_delivered(account, msg_list):
 
 
 def wait_msgs_changed(account, msgs_list):
-    """ wait for one or more MSGS_CHANGED events to match msgs_list contents. """
+    """wait for one or more MSGS_CHANGED events to match msgs_list contents."""
     account.log("waiting for msgs_list={}".format(msgs_list))
     msgs_list = list(msgs_list)
     while msgs_list:
@@ -38,7 +38,7 @@ class TestOnlineInCreation:
 
         lp.sec("Creating in-creation file outside of blobdir")
         assert tmpdir.strpath != ac1.get_blobdir()
-        src = tmpdir.join('file.txt').ensure(file=1)
+        src = tmpdir.join("file.txt").ensure(file=1)
         with pytest.raises(Exception):
             chat.prepare_message_file(src.strpath)
 
@@ -48,11 +48,11 @@ class TestOnlineInCreation:
 
         lp.sec("Creating file outside of blobdir")
         assert tmpdir.strpath != ac1.get_blobdir()
-        src = tmpdir.join('file.txt')
+        src = tmpdir.join("file.txt")
         src.write("hello there\n")
         chat.send_file(src.strpath)
 
-        blob_src = os.path.join(ac1.get_blobdir(), 'file.txt')
+        blob_src = os.path.join(ac1.get_blobdir(), "file.txt")
         assert os.path.exists(blob_src), "file.txt not copied to blobdir"
 
     def test_forward_increation(self, acfactory, data, lp):
@@ -63,7 +63,7 @@ class TestOnlineInCreation:
 
         lp.sec("create a message with a file in creation")
         orig = data.get_path("d.png")
-        path = os.path.join(ac1.get_blobdir(), 'd.png')
+        path = os.path.join(ac1.get_blobdir(), "d.png")
         with open(path, "x") as fp:
             fp.write("preparing")
         prepared_original = chat.prepare_message_file(path)
@@ -94,10 +94,7 @@ class TestOnlineInCreation:
         assert fwd_msg.is_out_pending() or fwd_msg.is_out_delivered()
 
         lp.sec("wait for both messages to be delivered to SMTP")
-        wait_msg_delivered(ac1, [
-            (chat2.id, forwarded_id),
-            (chat.id, prepared_original.id)
-        ])
+        wait_msg_delivered(ac1, [(chat2.id, forwarded_id), (chat.id, prepared_original.id)])
 
         lp.sec("wait1 for original or forwarded messages to arrive")
         received_original = ac2._evtracker.wait_next_incoming_message()

python/tests/test_3_offline.py

@@ -1,32 +1,51 @@
 from __future__ import print_function
-import pytest
+
 import os
 import time
-from deltachat import const, Account
-from deltachat.message import Message
-from deltachat.hookspec import account_hookimpl
-from deltachat.capi import ffi, lib
-from deltachat.cutil import iter_array
 from datetime import datetime, timedelta, timezone
 
+import pytest
 
-@pytest.mark.parametrize("msgtext,res", [
-    ("Member Me (tmp1@x.org) removed by tmp2@x.org.",
-        ("removed", "tmp1@x.org", "tmp2@x.org")),
-    ("Member With space (tmp1@x.org) removed by tmp2@x.org.",
-        ("removed", "tmp1@x.org", "tmp2@x.org")),
-    ("Member With space (tmp1@x.org) removed by Another member (tmp2@x.org).",
-        ("removed", "tmp1@x.org", "tmp2@x.org")),
-    ("Member With space (tmp1@x.org) removed by me",
-        ("removed", "tmp1@x.org", "me")),
-    ("Group left by some one (tmp1@x.org).",
-        ("removed", "tmp1@x.org", "tmp1@x.org")),
-    ("Group left by tmp1@x.org.",
-        ("removed", "tmp1@x.org", "tmp1@x.org")),
-    ("Member tmp1@x.org added by tmp2@x.org.", ("added", "tmp1@x.org", "tmp2@x.org")),
-    ("Member nothing bla bla", None),
-    ("Another unknown system message", None),
-])
+from deltachat import Account, const
+from deltachat.capi import ffi, lib
+from deltachat.cutil import iter_array
+from deltachat.hookspec import account_hookimpl
+from deltachat.message import Message
+from deltachat.tracker import ImexFailed
+
+
+@pytest.mark.parametrize(
+    "msgtext,res",
+    [
+        (
+            "Member Me (tmp1@x.org) removed by tmp2@x.org.",
+            ("removed", "tmp1@x.org", "tmp2@x.org"),
+        ),
+        (
+            "Member With space (tmp1@x.org) removed by tmp2@x.org.",
+            ("removed", "tmp1@x.org", "tmp2@x.org"),
+        ),
+        (
+            "Member With space (tmp1@x.org) removed by Another member (tmp2@x.org).",
+            ("removed", "tmp1@x.org", "tmp2@x.org"),
+        ),
+        (
+            "Member With space (tmp1@x.org) removed by me",
+            ("removed", "tmp1@x.org", "me"),
+        ),
+        (
+            "Group left by some one (tmp1@x.org).",
+            ("removed", "tmp1@x.org", "tmp1@x.org"),
+        ),
+        ("Group left by tmp1@x.org.", ("removed", "tmp1@x.org", "tmp1@x.org")),
+        (
+            "Member tmp1@x.org added by tmp2@x.org.",
+            ("added", "tmp1@x.org", "tmp2@x.org"),
+        ),
+        ("Member nothing bla bla", None),
+        ("Another unknown system message", None),
+    ],
+)
 def test_parse_system_add_remove(msgtext, res):
     from deltachat.message import parse_system_add_remove
 
@@ -136,9 +155,11 @@ class TestOfflineContact:
         ac1 = acfactory.get_pseudo_configured_account()
         contact1 = ac1.create_contact("some1@example.org", name="some1")
         contact2 = ac1.create_contact("some1@example.org", name="some1")
+        contact3 = None
         str(contact1)
         repr(contact1)
         assert contact1 == contact2
+        assert contact1 != contact3
         assert contact1.id
         assert contact1.addr == "some1@example.org"
         assert contact1.display_name == "some1"
@@ -232,10 +253,12 @@ class TestOfflineChat:
     def test_chat_idempotent(self, chat1, ac1):
         contact1 = chat1.get_contacts()[0]
         chat2 = contact1.create_chat()
+        chat3 = None
         assert chat2.id == chat1.id
         assert chat2.get_name() == chat1.get_name()
         assert chat1 == chat2
         assert not (chat1 != chat2)
+        assert chat1 != chat3
 
         for ichat in ac1.get_chats():
             if ichat.id == chat1.id:
@@ -387,6 +410,8 @@ class TestOfflineChat:
 
     def test_message_eq_contains(self, chat1):
         msg = chat1.send_text("msg1")
+        msg2 = None
+        assert msg != msg2
         assert msg in chat1.get_messages()
         assert not (msg not in chat1.get_messages())
         str(msg)
@@ -424,11 +449,14 @@ class TestOfflineChat:
         assert os.path.exists(msg.filename)
         assert msg.filemime == "image/png"
 
-    @pytest.mark.parametrize("typein,typeout", [
+    @pytest.mark.parametrize(
+        "typein,typeout",
+        [
             (None, "application/octet-stream"),
             ("text/plain", "text/plain"),
             ("image/png", "image/png"),
-    ])
+        ],
+    )
     def test_message_file(self, ac1, chat1, data, lp, typein, typeout):
         lp.sec("sending file")
         fn = data.get_path("r.txt")
@@ -475,7 +503,7 @@ class TestOfflineChat:
         with pytest.raises(ValueError):
             ac1.set_config("addr", "123@example.org")
 
-    def test_import_export_one_contact(self, acfactory, tmpdir):
+    def test_import_export_on_unencrypted_acct(self, acfactory, tmpdir):
         backupdir = tmpdir.mkdir("backup")
         ac1 = acfactory.get_pseudo_configured_account()
         chat = ac1.create_contact("some1 <some1@example.org>").create_chat()
@@ -504,6 +532,162 @@ class TestOfflineChat:
         assert messages[0].text == "msg1"
         assert os.path.exists(messages[1].filename)
 
+    def test_import_export_on_encrypted_acct(self, acfactory, tmpdir):
+        passphrase1 = "passphrase1"
+        passphrase2 = "passphrase2"
+        backupdir = tmpdir.mkdir("backup")
+        ac1 = acfactory.get_pseudo_configured_account(passphrase=passphrase1)
+
+        chat = ac1.create_contact("some1 <some1@example.org>").create_chat()
+        # send a text message
+        msg = chat.send_text("msg1")
+        # send a binary file
+        bin = tmpdir.join("some.bin")
+        with bin.open("w") as f:
+            f.write("\00123" * 10000)
+        msg = chat.send_file(bin.strpath)
+        contact = msg.get_sender_contact()
+        assert contact == ac1.get_self_contact()
+
+        assert not backupdir.listdir()
+        ac1.stop_io()
+
+        path = ac1.export_all(backupdir.strpath)
+        assert os.path.exists(path)
+
+        ac2 = acfactory.get_unconfigured_account(closed=True)
+        ac2.open(passphrase2)
+        ac2.import_all(path)
+
+        # check data integrity
+        contacts = ac2.get_contacts(query="some1")
+        assert len(contacts) == 1
+        contact2 = contacts[0]
+        assert contact2.addr == "some1@example.org"
+        chat2 = contact2.create_chat()
+        messages = chat2.get_messages()
+        assert len(messages) == 2
+        assert messages[0].text == "msg1"
+        assert os.path.exists(messages[1].filename)
+
+        ac2.shutdown()
+
+        # check that passphrase is not lost after import:
+        ac2 = Account(ac2.db_path, logging=ac2._logging, closed=True)
+        ac2.open(passphrase2)
+
+        # check data integrity
+        contacts = ac2.get_contacts(query="some1")
+        assert len(contacts) == 1
+        contact2 = contacts[0]
+        assert contact2.addr == "some1@example.org"
+        chat2 = contact2.create_chat()
+        messages = chat2.get_messages()
+        assert len(messages) == 2
+        assert messages[0].text == "msg1"
+        assert os.path.exists(messages[1].filename)
+
+    def test_import_export_with_passphrase(self, acfactory, tmpdir):
+        passphrase = "test_passphrase"
+        wrong_passphrase = "wrong_passprase"
+        backupdir = tmpdir.mkdir("backup")
+        ac1 = acfactory.get_pseudo_configured_account()
+
+        chat = ac1.create_contact("some1 <some1@example.org>").create_chat()
+        # send a text message
+        msg = chat.send_text("msg1")
+        # send a binary file
+        bin = tmpdir.join("some.bin")
+        with bin.open("w") as f:
+            f.write("\00123" * 10000)
+        msg = chat.send_file(bin.strpath)
+        contact = msg.get_sender_contact()
+        assert contact == ac1.get_self_contact()
+
+        assert not backupdir.listdir()
+        ac1.stop_io()
+
+        path = ac1.export_all(backupdir.strpath, passphrase)
+        assert os.path.exists(path)
+
+        ac2 = acfactory.get_unconfigured_account()
+        with pytest.raises(ImexFailed):
+            ac2.import_all(path, wrong_passphrase)
+        ac2.import_all(path, passphrase)
+
+        # check data integrity
+        contacts = ac2.get_contacts(query="some1")
+        assert len(contacts) == 1
+        contact2 = contacts[0]
+        assert contact2.addr == "some1@example.org"
+        chat2 = contact2.create_chat()
+        messages = chat2.get_messages()
+        assert len(messages) == 2
+        assert messages[0].text == "msg1"
+        assert os.path.exists(messages[1].filename)
+
+    def test_import_encrypted_bak_into_encrypted_acct(self, acfactory, tmpdir):
+        """
+        Test that account passphrase isn't lost if backup failed to be imported.
+        See https://github.com/deltachat/deltachat-core-rust/issues/3379
+        """
+        acct_passphrase = "passphrase1"
+        bak_passphrase = "passphrase2"
+        wrong_passphrase = "wrong_passprase"
+        backupdir = tmpdir.mkdir("backup")
+
+        ac1 = acfactory.get_pseudo_configured_account()
+        chat = ac1.create_contact("some1 <some1@example.org>").create_chat()
+        # send a text message
+        msg = chat.send_text("msg1")
+        # send a binary file
+        bin = tmpdir.join("some.bin")
+        with bin.open("w") as f:
+            f.write("\00123" * 10000)
+        msg = chat.send_file(bin.strpath)
+        contact = msg.get_sender_contact()
+        assert contact == ac1.get_self_contact()
+
+        assert not backupdir.listdir()
+        ac1.stop_io()
+
+        path = ac1.export_all(backupdir.strpath, bak_passphrase)
+        assert os.path.exists(path)
+
+        ac2 = acfactory.get_unconfigured_account(closed=True)
+        ac2.open(acct_passphrase)
+        with pytest.raises(ImexFailed):
+            ac2.import_all(path, wrong_passphrase)
+        ac2.import_all(path, bak_passphrase)
+
+        # check data integrity
+        contacts = ac2.get_contacts(query="some1")
+        assert len(contacts) == 1
+        contact2 = contacts[0]
+        assert contact2.addr == "some1@example.org"
+        chat2 = contact2.create_chat()
+        messages = chat2.get_messages()
+        assert len(messages) == 2
+        assert messages[0].text == "msg1"
+        assert os.path.exists(messages[1].filename)
+
+        ac2.shutdown()
+
+        # check that passphrase is not lost after import
+        ac2 = Account(ac2.db_path, logging=ac2._logging, closed=True)
+        ac2.open(acct_passphrase)
+
+        # check data integrity
+        contacts = ac2.get_contacts(query="some1")
+        assert len(contacts) == 1
+        contact2 = contacts[0]
+        assert contact2.addr == "some1@example.org"
+        chat2 = contact2.create_chat()
+        messages = chat2.get_messages()
+        assert len(messages) == 2
+        assert messages[0].text == "msg1"
+        assert os.path.exists(messages[1].filename)
+
     def test_set_get_draft(self, chat1):
         msg = Message.new_empty(chat1.account, "text")
         msg1 = chat1.prepare_message(msg)
@@ -629,6 +813,6 @@ class TestOfflineChat:
         lp.sec("check message count of only system messages (without daymarkers)")
         dc_array = ffi.gc(
             lib.dc_get_chat_msgs(ac1._dc_context, chat.id, const.DC_GCM_INFO_ONLY, 0),
-            lib.dc_array_unref
+            lib.dc_array_unref,
         )
         assert len(list(iter_array(dc_array, lambda x: x))) == 2

python/tests/test_4_lowlevel.py

@@ -1,24 +1,25 @@
-
 import os
-
 from queue import Queue
-from deltachat import capi, cutil, const
-from deltachat import register_global_plugin
+
+from deltachat import capi, const, cutil, register_global_plugin
+from deltachat.capi import ffi, lib
 from deltachat.hookspec import global_hookimpl
-from deltachat.capi import ffi
-from deltachat.capi import lib
-from deltachat.testplugin import ACSetup, create_dict_from_files_in_path, write_dict_to_dir
+from deltachat.testplugin import (
+    ACSetup,
+    create_dict_from_files_in_path,
+    write_dict_to_dir,
+)
+
 # from deltachat.account import EventLogger
 
 
 class TestACSetup:
-
     def test_cache_writing(self, tmp_path):
         base = tmp_path.joinpath("hello")
         base.mkdir()
         d1 = base.joinpath("dir1")
         d1.mkdir()
-        d1.joinpath("file1").write_bytes(b'content1')
+        d1.joinpath("file1").write_bytes(b"content1")
         d2 = d1.joinpath("dir2")
         d2.mkdir()
         d2.joinpath("file2").write_bytes(b"123")
@@ -34,7 +35,7 @@ class TestACSetup:
         monkeypatch.setattr(acc, "configure", lambda **kwargs: None)
         pc.start_configure(acc)
         assert pc._account2state[acc] == pc.CONFIGURING
-        pc._configured_events.put((acc, True))
+        pc._configured_events.put((acc, True, None))
         monkeypatch.setattr(pc, "init_imap", lambda *args, **kwargs: None)
         pc.wait_one_configured(acc)
         assert pc._account2state[acc] == pc.CONFIGURED
@@ -54,11 +55,11 @@ class TestACSetup:
         pc.start_configure(ac2)
         assert pc._account2state[ac1] == pc.CONFIGURING
         assert pc._account2state[ac2] == pc.CONFIGURING
-        pc._configured_events.put((ac1, True))
+        pc._configured_events.put((ac1, True, None))
         pc.wait_one_configured(ac1)
         assert pc._account2state[ac1] == pc.CONFIGURED
         assert pc._account2state[ac2] == pc.CONFIGURING
-        pc._configured_events.put((ac2, True))
+        pc._configured_events.put((ac2, True, None))
         pc.bring_online()
         assert pc._account2state[ac1] == pc.IDLEREADY
         assert pc._account2state[ac2] == pc.IDLEREADY
@@ -103,6 +104,7 @@ def test_dc_close_events(tmpdir, acfactory):
         def dc_account_after_shutdown(self, account):
             assert account._dc_context is None
             shutdowns.put(account)
+
     register_global_plugin(ShutdownPlugin())
     assert hasattr(ac1, "_dc_context")
     ac1.shutdown()
@@ -178,8 +180,8 @@ def test_get_info_open(tmpdir):
         lib.dc_context_unref,
     )
     info = cutil.from_dc_charpointer(lib.dc_get_info(ctx))
-    assert 'deltachat_core_version' in info
-    assert 'database_dir' in info
+    assert "deltachat_core_version" in info
+    assert "database_dir" in info
 
 
 def test_logged_hook_failure(acfactory):
@@ -187,7 +189,7 @@ def test_logged_hook_failure(acfactory):
     cap = []
     ac1.log = cap.append
     with ac1._event_thread.swallow_and_log_exception("some"):
-        0/0
+        0 / 0
     assert cap
     assert "some" in str(cap)
     assert "ZeroDivisionError" in str(cap)
@@ -202,7 +204,7 @@ def test_logged_ac_process_ffi_failure(acfactory):
     class FailPlugin:
         @account_hookimpl
         def ac_process_ffi_event(ffi_event):
-            0/0
+            0 / 0
 
     cap = Queue()
     ac1.log = cap.put

python/tox.ini

@@ -28,6 +28,10 @@ deps =
 [testenv:auditwheels]
 skipsdist = True
 deps = auditwheel
+passenv =
+    AUDITWHEEL_ARCH
+    AUDITWHEEL_PLAT
+    AUDITWHEEL_POLICY
 commands = 
     python tests/auditwheels.py {toxworkdir}/wheelhouse
 
@@ -36,10 +40,14 @@ skipsdist = True
 skip_install = True
 deps =
     flake8
+    isort
+    black
     # pygments required by rst-lint
     pygments
     restructuredtext_lint 
 commands =
+    isort --check setup.py install_python_bindings.py src/deltachat examples/ tests/
+    black --check setup.py install_python_bindings.py src/deltachat examples/ tests/
     flake8 src/deltachat
     flake8 tests/ examples/
     rst-lint --encoding 'utf-8' README.rst
@@ -85,3 +93,4 @@ markers =
 
 [flake8]
 max-line-length = 120
+ignore = E203, E266, E501, W503