Integrate JSON-RPC API in core (#3463)

* integrate json-rpc repo https://github.com/deltachat/deltachat-jsonrpc * get target dir from cargo * fix clippy * use node 16 in ci use `npm i` instead of `npm ci` try fix ci script and fix a doc comment * fix get_provider_info docs * refactor function name * fix formatting make test pass fix clippy * update .gitignore * change now returns event names as id directly, no conversion method or number ids anymore also longer timeout for requesting test accounts from mailadm * fix compile after rebase * add json api to cffi and expose it in dc node * add some files to npm ignore that don't need to be in the npm package * add jsonrpc crate to set_core_version * add jsonrpc feature flag * call a jsonrpc function in segfault example * break loop on empty response * fix closing segfault thanks again to link2xt for figguring this out * activate other tests again * remove selectAccount from highlevel client * put jsonrpc stuff in own module * disable jsonrpc by default * add @deltachat/jsonrpc-client to make sure its dependencies are installed, too whwn installing dc-node * commit types.ts that dc-node has everything it needs to provide @deltachat/jsonrpc-client without an extra ts compile step * improve naming * Changes for tokio compat, upgrade to yerpc 0.3 This also changes the webserver binary to use axum in place of tide. * Improvements to typescript package * Improve docs. * improve docs, fix example * Fix CFFI for JSON-RPC changes * use stable toolchain not 1.56.0 * fix ci * try to fix ci * remove emtpy file allow unused code for new_from_arc * expose anyhow errors feature name was wrong * use multi-threaded runtime in JSON-RPC webserver * improve test setup and code style * don't wait for IO on webserver start * Bump yerpc to 0.3.1 with fix for axum server * update todo document remove specific api stuff for now, we now have the an incremental aproach on moving not the all at-once effort I though it would be * remove debug logs * changelog entry about the jsonrpc * Fix method name casings and cleanups * Improve JSON-RPC CI, no need to build things multiple times * Naming consistency: Use DeltaChat not Deltachat * Improve documentation * fix docs * adress dig's comments - description in cargo.toml - impl From<EventType> for EventTypeName - rename `CommandApi::new_from_arc` -> `CommandApi::from_arc` - pre-allocate if we know the entry count already - remove unused enumerate - remove unused serde attribute comment - rename `FullChat::from_dc_chat_id` -> `FullChat::try_from_dc_chat_id` * make it more idiomatic: rename `ContactObject::from_dc_contact -> `ContactObject::try_from_dc_contact` * apply link2xt's suggestions: - unref jsonrpc_instance in same thread it was created in - increase `max_queue_size` from 1 to 1000 * reintroduce segfault test script * remove unneeded context thanks to link2xt for pointing that out * Update deltachat-ffi/deltachat.h Co-authored-by: bjoern <r10s@b44t.com> * Update deltachat-ffi/deltachat.h Co-authored-by: bjoern <r10s@b44t.com> * make sure to use dc_str_unref instead of free on cstrings returned/owned by rust * Increase online test timeouts for CI * fix the typos thanks to ralphtheninja for finding them * restore same configure behaviour as desktop: make configure restart io with the old configuration if it had one on error * found another segfault: this time in batch_set_config * remove print from test * make dcn_json_rpc_request return undefined instead of not returning this might have been the cause for the second segfault * add set_config_from_qr to jsonrpc * add `add_device_message` to jsonrpc * jsonrpc: add `get_fresh_msgs` and `get_fresh_msg_cnt` * jsonrpc: add dm_chat_contact to ChatListItemFetchResult * add webxdc methods to jsonrpc: - `webxdc_send_status_update` - `webxdc_get_status_updates` - `message_get_webxdc_info` * add `chat_get_media` to jsonrpc also add viewtype wrapper enum and use it in `MessageObject`, additionally to using it in `chat_get_media` * use camelCase in all js object properties * Add check_qr function to jsonrpc * Fixed clippy errors and formatting * Fixed formatting * fix changelog ordering after rebase * fix compile after merging in master branch Co-authored-by: Simon Laux <mobile.info@simonlaux.de> Co-authored-by: Simon Laux <Simon-Laux@users.noreply.github.com> Co-authored-by: bjoern <r10s@b44t.com> Co-authored-by: flipsimon <28535045+flipsimon@users.noreply.github.com>
2026-05-03 05:16:28 +03:00 · 2022-08-04 18:56:37 +02:00
parent 142c02b425
commit 0887acf1bf
55 changed files with 4499 additions and 22 deletions
--- a/node/lib/deltachat.ts
+++ b/node/lib/deltachat.ts
@@ -19,10 +19,11 @@ interface NativeAccount {}
 export class AccountManager extends EventEmitter {
  dcn_accounts: NativeAccount
  accountDir: string
+  jsonRpcStarted = false

  constructor(cwd: string, os = 'deltachat-node') {
-    debug('DeltaChat constructor')
    super()
+    debug('DeltaChat constructor')

    this.accountDir = cwd
    this.dcn_accounts = binding.dcn_accounts_new(os, this.accountDir)
@@ -114,6 +115,31 @@ export class AccountManager extends EventEmitter {
    debug('Started event handler')
  }

+  startJsonRpcHandler(callback: ((response: string) => void) | null) {
+    if (this.dcn_accounts === null) {
+      throw new Error('dcn_account is null')
+    }
+    if (!callback) {
+      throw new Error('no callback set')
+    }
+    if (this.jsonRpcStarted) {
+      throw new Error('jsonrpc was started already')
+    }
+
+    binding.dcn_accounts_start_jsonrpc(this.dcn_accounts, callback.bind(this))
+    debug('Started JSON-RPC handler')
+    this.jsonRpcStarted = true
+  }
+
+  jsonRpcRequest(message: string) {
+    if (!this.jsonRpcStarted) {
+      throw new Error(
+        'jsonrpc is not active, start it with startJsonRpcHandler first'
+      )
+    }
+    binding.dcn_json_rpc_request(this.dcn_accounts, message)
+  }
+
  startIO() {
    binding.dcn_accounts_start_io(this.dcn_accounts)
  }
--- a/node/scripts/rebuild-core.js
+++ b/node/scripts/rebuild-core.js
@@ -9,7 +9,7 @@ const buildArgs = [
  'build',
  '--release',
  '--features',
-  'vendored',
+  'vendored,jsonrpc',
  '-p',
  'deltachat_ffi'
 ]
--- a/node/segfault.js
+++ b/node/segfault.js
@@ -0,0 +1,28 @@
+const { default: dc } = require('./dist')
+
+const ac = new dc('test1233490')
+
+console.log("[1]");
+
+ac.startJsonRpcHandler(console.log)
+
+console.log("[2]");
+console.log(
+  ac.jsonRpcRequest(
+    JSON.stringify({
+      jsonrpc: '2.0',
+      method: 'get_all_account_ids',
+      params: [],
+      id: 2,
+    })
+  )
+)
+
+console.log("[3]");
+
+setTimeout(() => {
+    console.log("[4]");
+  ac.close() // This segfaults -> TODO Findout why?
+
+  console.log('still living')
+}, 1000)
--- a/node/segfault2.js
+++ b/node/segfault2.js
@@ -0,0 +1,50 @@
+const { default: dc } = require('./dist')
+
+const ac = new dc('test1233490')
+
+console.log('[1]')
+
+ac.startJsonRpcHandler(console.log)
+
+console.log('[2]')
+console.log(
+  ac.jsonRpcRequest(
+    JSON.stringify({
+      jsonrpc: '2.0',
+      method: 'batch_set_config',
+      id: 3,
+      params: [
+        69,
+        {
+          addr: '',
+          mail_user: '',
+          mail_pw: '',
+          mail_server: '',
+          mail_port: '',
+          mail_security: '',
+          imap_certificate_checks: '',
+          send_user: '',
+          send_pw: '',
+          send_server: '',
+          send_port: '',
+          send_security: '',
+          smtp_certificate_checks: '',
+          socks5_enabled: '0',
+          socks5_host: '',
+          socks5_port: '',
+          socks5_user: '',
+          socks5_password: '',
+        },
+      ],
+    })
+  )
+)
+
+console.log('[3]')
+
+setTimeout(() => {
+  console.log('[4]')
+  ac.close() // This segfaults -> TODO Findout why?
+
+  console.log('still living')
+}, 1000)
--- a/node/src/module.c
+++ b/node/src/module.c
@@ -34,6 +34,9 @@ typedef struct dcn_accounts_t {
  dc_accounts_t* dc_accounts;
  napi_threadsafe_function threadsafe_event_handler;
  uv_thread_t event_handler_thread;
+  napi_threadsafe_function threadsafe_jsonrpc_handler;
+  uv_thread_t jsonrpc_thread;
+  dc_jsonrpc_instance_t* jsonrpc_instance;
  int gc;
 } dcn_accounts_t;

@@ -2936,6 +2939,12 @@ NAPI_METHOD(dcn_accounts_unref) {
    uv_thread_join(&dcn_accounts->event_handler_thread);
    dcn_accounts->event_handler_thread = 0;
  }
+  if (dcn_accounts->jsonrpc_instance) {
+    dc_jsonrpc_request(dcn_accounts->jsonrpc_instance, "{}");
+    uv_thread_join(&dcn_accounts->jsonrpc_thread);
+    dc_jsonrpc_unref(dcn_accounts->jsonrpc_instance);
+    dcn_accounts->jsonrpc_instance = NULL;
+  }
  dc_accounts_unref(dcn_accounts->dc_accounts);
  dcn_accounts->dc_accounts = NULL;

@@ -3094,8 +3103,6 @@ static void accounts_event_handler_thread_func(void* arg)
 {
  dcn_accounts_t* dcn_accounts = (dcn_accounts_t*)arg;

-
-
  TRACE("event_handler_thread_func starting");

  dc_accounts_event_emitter_t * dc_accounts_event_emitter = dc_accounts_get_event_emitter(dcn_accounts->dc_accounts);
@@ -3185,7 +3192,7 @@ static void call_accounts_js_event_handler(napi_env env, napi_value js_callback,
    if (status != napi_ok) {
      napi_throw_error(env, NULL, "Unable to create argv[3] for event_handler arguments");
    }
-    free(data2_string);
+    dc_str_unref(data2_string);
  } else {
    status = napi_create_int32(env, dc_event_get_data2_int(dc_event), &argv[3]);
    if (status != napi_ok) {
@@ -3246,6 +3253,124 @@ NAPI_METHOD(dcn_accounts_start_event_handler) {
  NAPI_RETURN_UNDEFINED();
 }

+// JSON RPC
+
+static void accounts_jsonrpc_thread_func(void* arg)
+{
+  dcn_accounts_t* dcn_accounts = (dcn_accounts_t*)arg;
+  TRACE("accounts_jsonrpc_thread_func starting");
+  char* response;
+  while (true) {
+    response = dc_jsonrpc_next_response(dcn_accounts->jsonrpc_instance);
+    if (response == NULL) {
+      // done or broken
+      break;
+    }
+
+    if (!dcn_accounts->threadsafe_jsonrpc_handler) {
+      TRACE("threadsafe_jsonrpc_handler not set, bailing");
+      break;
+    }
+    // Don't process events if we're being garbage collected!
+    if (dcn_accounts->gc == 1) {
+      TRACE("dc_accounts has been destroyed, bailing");
+      break;
+    }
+
+    napi_status status = napi_call_threadsafe_function(dcn_accounts->threadsafe_jsonrpc_handler, response, napi_tsfn_blocking);
+
+    if (status == napi_closing) {
+      TRACE("JS function got released, bailing");
+      break;
+    }
+  }
+  TRACE("accounts_jsonrpc_thread_func ended");
+  napi_release_threadsafe_function(dcn_accounts->threadsafe_jsonrpc_handler, napi_tsfn_release);
+}
+
+static void call_accounts_js_jsonrpc_handler(napi_env env, napi_value js_callback, void* _context, void* data)
+{
+  char* response = (char*)data;
+  napi_value global;
+  napi_status status = napi_get_global(env, &global);
+  if (status != napi_ok) {
+    napi_throw_error(env, NULL, "Unable to get global");
+  }
+
+  napi_value argv[1];
+  if (response != 0) {
+    status = napi_create_string_utf8(env, response, NAPI_AUTO_LENGTH, &argv[0]);
+  } else {
+    status = napi_create_string_utf8(env, "", NAPI_AUTO_LENGTH, &argv[0]);
+  }
+  if (status != napi_ok) {
+    napi_throw_error(env, NULL, "Unable to create argv for js jsonrpc_handler arguments");
+  } 
+  dc_str_unref(response);
+
+  TRACE("calling back into js");
+  napi_value result;
+  status = napi_call_function(
+    env,
+    global,
+    js_callback,
+    1,
+    argv,
+    &result);
+  if (status != napi_ok) {
+    TRACE("Unable to call jsonrpc_handler callback2");
+    const napi_extended_error_info* error_result;
+    NAPI_STATUS_THROWS(napi_get_last_error_info(env, &error_result));
+  }
+}
+
+NAPI_METHOD(dcn_accounts_start_jsonrpc) {
+  NAPI_ARGV(2);
+  NAPI_DCN_ACCOUNTS();
+  napi_value callback = argv[1];
+
+  TRACE("calling..");
+  napi_value async_resource_name;
+  NAPI_STATUS_THROWS(napi_create_string_utf8(env, "dc_accounts_jsonrpc_callback", NAPI_AUTO_LENGTH, &async_resource_name));
+
+  TRACE("creating threadsafe function..");
+
+  NAPI_STATUS_THROWS(napi_create_threadsafe_function(
+    env,
+    callback,
+    0,
+    async_resource_name,
+    1000, // max_queue_size
+    1,
+    NULL,
+    NULL,
+    NULL,
+    call_accounts_js_jsonrpc_handler,
+    &dcn_accounts->threadsafe_jsonrpc_handler));
+  TRACE("done");
+
+  dcn_accounts->gc = 0;
+  dcn_accounts->jsonrpc_instance = dc_jsonrpc_init(dcn_accounts->dc_accounts);
+
+  TRACE("creating uv thread..");
+  uv_thread_create(&dcn_accounts->jsonrpc_thread, accounts_jsonrpc_thread_func, dcn_accounts);
+
+  NAPI_RETURN_UNDEFINED();
+}
+
+NAPI_METHOD(dcn_json_rpc_request) {
+  NAPI_ARGV(2);
+  NAPI_DCN_ACCOUNTS();
+  if (!dcn_accounts->jsonrpc_instance) {
+    const char* msg = "dcn_accounts->jsonrpc_instance is null, have you called dcn_accounts_start_jsonrpc()?";
+    NAPI_STATUS_THROWS(napi_throw_type_error(env, NULL, msg));
+  }
+  NAPI_ARGV_UTF8_MALLOC(request, 1);
+  dc_jsonrpc_request(dcn_accounts->jsonrpc_instance, request);
+  free(request);
+  NAPI_RETURN_UNDEFINED();
+}
+

 NAPI_INIT() {
  /**
@@ -3517,4 +3642,9 @@ NAPI_INIT() {
  NAPI_EXPORT_FUNCTION(dcn_send_webxdc_status_update);
  NAPI_EXPORT_FUNCTION(dcn_get_webxdc_status_updates);
  NAPI_EXPORT_FUNCTION(dcn_msg_get_webxdc_blob);
+
+
+  /** jsonrpc **/
+  NAPI_EXPORT_FUNCTION(dcn_accounts_start_jsonrpc);
+  NAPI_EXPORT_FUNCTION(dcn_json_rpc_request);
 }
--- a/node/src/napi-macros-extensions.h
+++ b/node/src/napi-macros-extensions.h
@@ -23,7 +23,7 @@
  dcn_accounts_t* dcn_accounts; \
  NAPI_STATUS_THROWS(napi_get_value_external(env, argv[0], (void**)&dcn_accounts)); \
  if (!dcn_accounts) { \
-    const char* msg = "Provided dnc_acounts is null"; \
+    const char* msg = "Provided dcn_acounts is null"; \
    NAPI_STATUS_THROWS(napi_throw_type_error(env, NULL, msg)); \
  } \
  if (!dcn_accounts->dc_accounts) { \
--- a/node/test/test.js
+++ b/node/test/test.js
@@ -2,7 +2,7 @@
 import DeltaChat, { Message } from '../dist'
 import binding from '../binding'

-import { strictEqual } from 'assert'
+import { deepEqual, deepStrictEqual, strictEqual } from 'assert'
 import chai, { expect } from 'chai'
 import chaiAsPromised from 'chai-as-promised'
 import { EventId2EventName, C } from '../dist/constants'
@@ -84,6 +84,95 @@ describe('static tests', function () {
  })
 })

+describe('JSON RPC', function () {
+  it('smoketest', async function () {
+    const { dc } = DeltaChat.newTemporary()
+    let promise_resolve
+    const promise = new Promise((res, _rej) => {
+      promise_resolve = res
+    })
+    dc.startJsonRpcHandler(promise_resolve)
+    dc.jsonRpcRequest(
+      JSON.stringify({
+        jsonrpc: '2.0',
+        method: 'get_all_account_ids',
+        params: [],
+        id: 2,
+      })
+    )
+    deepStrictEqual(
+      {
+        jsonrpc: '2.0',
+        id: 2,
+        result: [1],
+      },
+      JSON.parse(await promise)
+    )
+    dc.close()
+  })
+
+  it('basic test', async function () {
+    const { dc } = DeltaChat.newTemporary()
+
+    const promises = {}
+    dc.startJsonRpcHandler((msg) => {
+      const response = JSON.parse(msg)
+      promises[response.id](response)
+      delete promises[response.id]
+    })
+    const call = (request) => {
+      dc.jsonRpcRequest(JSON.stringify(request))
+      return new Promise((res, _rej) => {
+        promises[request.id] = res
+      })
+    }
+
+    deepStrictEqual(
+      {
+        jsonrpc: '2.0',
+        id: 2,
+        result: [1],
+      },
+      await call({
+        jsonrpc: '2.0',
+        method: 'get_all_account_ids',
+        params: [],
+        id: 2,
+      })
+    )
+
+    deepStrictEqual(
+      {
+        jsonrpc: '2.0',
+        id: 3,
+        result: 2,
+      },
+      await call({
+        jsonrpc: '2.0',
+        method: 'add_account',
+        params: [],
+        id: 3,
+      })
+    )
+
+    deepStrictEqual(
+      {
+        jsonrpc: '2.0',
+        id: 4,
+        result: [1, 2],
+      },
+      await call({
+        jsonrpc: '2.0',
+        method: 'get_all_account_ids',
+        params: [],
+        id: 4,
+      })
+    )
+
+    dc.close()
+  })
+})
+
 describe('Basic offline Tests', function () {
  it('opens a context', async function () {
    const { dc, context } = DeltaChat.newTemporary()