feat: generate jsonrpc headers at build time (#8350)

Discussion in https://github.com/chatmail/yerpc/issues/77. The necessary
yerpc changes for this PR are in
https://github.com/chatmail/yerpc/pull/78

This is also a preparation for #8330
This commit is contained in:
d2weber
2026-09-09 16:09:00 +02:00
committed by GitHub
parent 4bf42c0af9
commit 187c5f4949
26 changed files with 51 additions and 17 deletions

View File

@@ -0,0 +1,16 @@
[package]
name = "deltachat-jsonrpc-bindings"
version = "2.60.0-dev"
description = "Autogenerate DeltaChat JSON-RPC API bindings at build time"
edition = "2024"
license = "MPL-2.0"
repository = "https://github.com/chatmail/core"
[build-dependencies]
deltachat-jsonrpc = { workspace = true }
[dependencies]
[features]
default = ["vendored"]
vendored = ["deltachat-jsonrpc/vendored"]

View File

@@ -0,0 +1,6 @@
use deltachat_jsonrpc::api::write_ts_bindings;
use std::path::Path;
fn main() {
write_ts_bindings(Path::new("typescript/generated"));
}

View File

@@ -0,0 +1 @@

View File

@@ -0,0 +1,9 @@
node_modules
dist
test_dist
coverage
yarn.lock
package-lock.json
docs
accounts
generated

View File

@@ -0,0 +1,12 @@
node_modules
accounts
docs
coverage
yarn*
package-lock.json
.prettierignore
example.html
report_api_coverage.mjs
scripts
dist/example
dist/test

View File

@@ -0,0 +1,3 @@
coverage
dist
generated

View File

@@ -0,0 +1 @@
export * from "./src/lib.js";

View File

@@ -0,0 +1,58 @@
{
"author": "Delta Chat Developers (ML) <delta@codespeak.net>",
"dependencies": {
"@deltachat/tiny-emitter": "3.0.0",
"isomorphic-ws": "^5.0.0",
"yerpc": "^0.6.2"
},
"devDependencies": {
"@types/chai": "^4.3.10",
"@types/chai-as-promised": "^7.1.8",
"@types/mocha": "^10.0.4",
"@types/ws": "^8.5.9",
"c8": "^8.0.1",
"chai": "^4.3.4",
"chai-as-promised": "^7.1.1",
"esbuild": "^0.25.5",
"http-server": "^14.1.1",
"mocha": "^10.2.0",
"npm-run-all": "^4.1.5",
"prettier": "^3.5.3",
"typedoc": "^0.28.5",
"typescript": "^5.8.3",
"ws": "^8.5.0"
},
"exports": {
".": {
"import": "./dist/deltachat.js",
"require": "./dist/deltachat.cjs",
"types": "./dist/deltachat.d.ts"
}
},
"license": "MPL-2.0",
"main": "dist/deltachat.js",
"name": "@deltachat/jsonrpc-client",
"repository": {
"type": "git",
"url": "https://github.com/chatmail/core.git"
},
"scripts": {
"build": "run-s generate-bindings extract-constants build:tsc build:bundle build:cjs",
"build:bundle": "esbuild --format=esm --bundle dist/deltachat.js --outfile=dist/deltachat.bundle.js",
"build:cjs": "esbuild --format=cjs --bundle --packages=external dist/deltachat.js --outfile=dist/deltachat.cjs",
"build:tsc": "tsc",
"docs": "typedoc --out docs deltachat.ts",
"extract-constants": "node ./scripts/generate-constants.js",
"generate-bindings": "cargo test",
"prettier:check": "prettier --check .",
"prettier:fix": "prettier --write .",
"test": "run-s test:prepare test:run-coverage test:report-coverage",
"test:prepare": "cargo build --package deltachat-rpc-server --bin deltachat-rpc-server",
"test:report-coverage": "node report_api_coverage.mjs",
"test:run": "mocha dist/test",
"test:run-coverage": "COVERAGE=1 NODE_OPTIONS=--enable-source-maps c8 --include 'dist/*' -r text -r html -r json mocha dist/test"
},
"type": "module",
"types": "dist/deltachat.d.ts",
"version": "2.60.0-dev"
}

View File

@@ -0,0 +1,28 @@
import { readFileSync } from "fs";
// only checks for the coverage of the api functions in bindings.ts for now
const generatedFile = "typescript/generated/client.ts";
const json = JSON.parse(readFileSync("./coverage/coverage-final.json"));
const jsonCoverage =
json[Object.keys(json).find((k) => k.includes(generatedFile))];
const fnMap = Object.keys(jsonCoverage.fnMap).map(
(key) => jsonCoverage.fnMap[key],
);
const htmlCoverage = readFileSync(
"./coverage/" + generatedFile + ".html",
"utf8",
);
const uncoveredLines = htmlCoverage
.split("\n")
.filter((line) => line.includes(`"function not covered"`));
const uncoveredFunctions = uncoveredLines.map(
(line) => />([\w_]+)\(/.exec(line)[1],
);
console.log(
"\nUncovered api functions:\n" +
uncoveredFunctions
.map((uF) => fnMap.find(({ name }) => name === uF))
.map(
({ name, line }) => `.${name.padEnd(40)} (${generatedFile}:${line})`,
)
.join("\n"),
);

View File

@@ -0,0 +1,73 @@
#!/usr/bin/env node
import { readFileSync, writeFileSync } from "fs";
import { resolve } from "path";
import { fileURLToPath } from "url";
import { dirname } from "path";
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const data = [];
const header = resolve(__dirname, "../../../deltachat-ffi/deltachat.h");
console.log("Generating constants...");
const header_data = readFileSync(header, "UTF-8");
const regex = /^#define\s+(\w+)\s+(\w+)/gm;
let match;
while (null != (match = regex.exec(header_data))) {
const key = match[1];
const value = parseInt(match[2]);
if (!isNaN(value)) {
data.push({ key, value });
}
}
const constants = data
.filter(
({ key }) => key.toUpperCase()[0] === key[0], // check if define name is uppercase
)
.sort((lhs, rhs) => {
if (lhs.key < rhs.key) return -1;
else if (lhs.key > rhs.key) return 1;
return 0;
})
.filter(({ key }) => {
// filter out what we don't need it
return !(
key.startsWith("DC_EVENT_") ||
key.startsWith("DC_IMEX_") ||
key.startsWith("DC_CHAT_VISIBILITY") ||
key.startsWith("DC_DOWNLOAD") ||
key.startsWith("DC_INFO_") ||
(key.startsWith("DC_MSG") && !key.startsWith("DC_MSG_ID")) ||
key.startsWith("DC_QR_") ||
key.startsWith("DC_CERTCK_") ||
key.startsWith("DC_SOCKET_") ||
key.startsWith("DC_LP_AUTH_") ||
key.startsWith("DC_TEXT1_") ||
key.startsWith("DC_CHAT_TYPE")
);
})
.map((row) => {
return ` export const ${row.key} = ${row.value};`;
})
.join("\n");
writeFileSync(
resolve(__dirname, "../generated/constants.ts"),
`// Generated!
export namespace C {
${constants}
/** @deprecated 10-8-2025 compare string directly with \`== "Group"\` */
export const DC_CHAT_TYPE_GROUP = "Group";
/** @deprecated 10-8-2025 compare string directly with \`== "InBroadcast"\`*/
export const DC_CHAT_TYPE_IN_BROADCAST = "InBroadcast";
/** @deprecated 10-8-2025 compare string directly with \`== "Mailinglist"\` */
export const DC_CHAT_TYPE_MAILINGLIST = "Mailinglist";
/** @deprecated 10-8-2025 compare string directly with \`== "OutBroadcast"\` */
export const DC_CHAT_TYPE_OUT_BROADCAST = "OutBroadcast";
/** @deprecated 10-8-2025 compare string directly with \`== "Single"\` */
export const DC_CHAT_TYPE_SINGLE = "Single";
}\n`,
);

View File

@@ -0,0 +1,126 @@
import * as T from "../generated/types.js";
import { EventType } from "../generated/types.js";
import * as RPC from "../generated/jsonrpc.js";
import { RawClient } from "../generated/client.js";
import { BaseTransport, Request } from "yerpc";
import { TinyEmitter } from "@deltachat/tiny-emitter";
type Events = { ALL: (accountId: number, event: EventType) => void } & {
[Property in EventType["kind"]]: (
accountId: number,
event: Extract<EventType, { kind: Property }>,
) => void;
};
type ContextEvents = { ALL: (event: EventType) => void } & {
[Property in EventType["kind"]]: (
event: Extract<EventType, { kind: Property }>,
) => void;
};
export type DcEvent = EventType;
export type DcEventType<T extends EventType["kind"]> = Extract<
EventType,
{ kind: T }
>;
export class BaseDeltaChat<
Transport extends BaseTransport<any>,
> extends TinyEmitter<Events> {
rpc: RawClient;
private contextEmitters: { [key: number]: TinyEmitter<ContextEvents> } = {};
//@ts-ignore
private eventTask: Promise<void>;
constructor(
public transport: Transport,
/**
* Whether to start calling {@linkcode RawClient.getNextEvent}
* and emitting the respective events on this class.
*/
startEventLoop: boolean,
) {
super();
this.rpc = new RawClient(this.transport);
if (startEventLoop) {
this.eventTask = this.eventLoop();
}
}
/**
* @see the constructor's `startEventLoop`
*/
async eventLoop(): Promise<void> {
while (true) {
for (const event of await this.rpc.getNextEventBatch()) {
//@ts-ignore
this.emit(event.event.kind, event.contextId, event.event);
this.emit("ALL", event.contextId, event.event);
if (this.contextEmitters[event.contextId]) {
this.contextEmitters[event.contextId].emit(
event.event.kind,
//@ts-ignore
event.event as any,
);
this.contextEmitters[event.contextId].emit("ALL", event.event as any);
}
}
}
}
/**
* @deprecated use {@linkcode BaseDeltaChat.rpc.getAllAccounts} instead.
*/
async listAccounts(): Promise<T.Account[]> {
return await this.rpc.getAllAccounts();
}
/**
* A convenience function to listen on events binned by `account_id`
* (see {@linkcode RawClient.getAllAccounts}).
*/
getContextEvents(account_id: number) {
if (this.contextEmitters[account_id]) {
return this.contextEmitters[account_id];
} else {
this.contextEmitters[account_id] = new TinyEmitter();
return this.contextEmitters[account_id];
}
}
}
export class StdioDeltaChat extends BaseDeltaChat<StdioTransport> {
close() {}
constructor(input: any, output: any, startEventLoop: boolean) {
const transport = new StdioTransport(input, output);
super(transport, startEventLoop);
}
}
export class StdioTransport extends BaseTransport {
constructor(
public input: any,
public output: any,
) {
super();
var buffer = "";
this.output.on("data", (data: any) => {
buffer += data.toString();
while (buffer.includes("\n")) {
const n = buffer.indexOf("\n");
const line = buffer.substring(0, n);
const message = JSON.parse(line);
this._onmessage(message);
buffer = buffer.substring(n + 1);
}
});
}
_send(message: any): void {
const serialized = JSON.stringify(message);
this.input.write(serialized + "\n");
}
}

View File

@@ -0,0 +1,6 @@
export * as RPC from "../generated/jsonrpc.js";
export * as T from "../generated/types.js";
export { RawClient } from "../generated/client.js";
export * from "./client.js";
export * as yerpc from "yerpc";
export { C } from "../generated/constants.js";

View File

@@ -0,0 +1,160 @@
import chai, { assert, expect } from "chai";
import chaiAsPromised from "chai-as-promised";
chai.use(chaiAsPromised);
import { StdioDeltaChat as DeltaChat } from "../deltachat.js";
import { RpcServerHandle, startServer } from "./test_base.js";
describe("basic tests", () => {
let serverHandle: RpcServerHandle;
let dc: DeltaChat;
before(async () => {
serverHandle = await startServer();
dc = new DeltaChat(serverHandle.stdin, serverHandle.stdout, true);
// dc.on("ALL", (event) => {
//console.log("event", event);
// });
});
after(async () => {
dc && dc.close();
await serverHandle.close();
});
it("check email address validity", async () => {
const validAddresses = [
"email@example.com",
"36aa165ae3406424e0c61af17700f397cad3fe8ab83d682d0bddf3338a5dd52e@yggmail@yggmail",
];
const invalidAddresses = ["email@", "example.com", "emai221"];
expect(
await Promise.all(
validAddresses.map((email) => dc.rpc.checkEmailValidity(email)),
),
).to.not.contain(false);
expect(
await Promise.all(
invalidAddresses.map((email) => dc.rpc.checkEmailValidity(email)),
),
).to.not.contain(true);
});
it("system info", async () => {
const systemInfo = await dc.rpc.getSystemInfo();
expect(systemInfo).to.contain.keys([
"arch",
"num_cpus",
"deltachat_core_version",
"sqlite_version",
]);
});
describe("account management", () => {
it("should create account", async () => {
const res = await dc.rpc.addAccount();
assert((await dc.rpc.getAllAccountIds()).length === 1);
});
it("should remove the account again", async () => {
await dc.rpc.removeAccount((await dc.rpc.getAllAccountIds())[0]);
assert((await dc.rpc.getAllAccountIds()).length === 0);
});
it("should create multiple accounts", async () => {
await dc.rpc.addAccount();
await dc.rpc.addAccount();
await dc.rpc.addAccount();
await dc.rpc.addAccount();
assert((await dc.rpc.getAllAccountIds()).length === 4);
});
});
describe("contact management", function () {
let accountId: number;
before(async () => {
accountId = await dc.rpc.addAccount();
});
it("should block and unblock contact", async function () {
// Cannot send sync messages to self as we do not have a self address.
await dc.rpc.setConfig(accountId, "sync_msgs", "0");
const contactId = await dc.rpc.createContact(
accountId,
"example@delta.chat",
null,
);
expect((await dc.rpc.getContact(accountId, contactId)).isBlocked).to.be
.false;
await dc.rpc.blockContact(accountId, contactId);
expect((await dc.rpc.getContact(accountId, contactId)).isBlocked).to.be
.true;
expect(await dc.rpc.getBlockedContacts(accountId)).to.have.length(1);
await dc.rpc.unblockContact(accountId, contactId);
expect((await dc.rpc.getContact(accountId, contactId)).isBlocked).to.be
.false;
expect(await dc.rpc.getBlockedContacts(accountId)).to.have.length(0);
});
});
describe("configuration", function () {
let accountId: number;
before(async () => {
accountId = await dc.rpc.addAccount();
});
it("set and retrieve", async function () {
await dc.rpc.setConfig(accountId, "addr", "valid@email");
assert((await dc.rpc.getConfig(accountId, "addr")) == "valid@email");
});
it("set invalid key should throw", async function () {
await expect(dc.rpc.setConfig(accountId, "invalid_key", "some value")).to
.be.eventually.rejected;
});
it("get invalid key should throw", async function () {
await expect(dc.rpc.getConfig(accountId, "invalid_key")).to.be.eventually
.rejected;
});
it("set and retrieve ui.*", async function () {
await dc.rpc.setConfig(accountId, "ui.chat_bg", "color:red");
assert((await dc.rpc.getConfig(accountId, "ui.chat_bg")) == "color:red");
});
it("set and retrieve (batch)", async function () {
const config = { addr: "valid@email", mail_pw: "1234" };
await dc.rpc.batchSetConfig(accountId, config);
const retrieved = await dc.rpc.batchGetConfig(
accountId,
Object.keys(config),
);
expect(retrieved).to.deep.equal(config);
});
it("set and retrieve ui.* (batch)", async function () {
const config = {
"ui.chat_bg": "color:green",
"ui.enter_key_sends": "true",
};
await dc.rpc.batchSetConfig(accountId, config);
const retrieved = await dc.rpc.batchGetConfig(
accountId,
Object.keys(config),
);
expect(retrieved).to.deep.equal(config);
});
it("set and retrieve mixed(ui and core) (batch)", async function () {
const config = {
"ui.chat_bg": "color:yellow",
"ui.enter_key_sends": "false",
addr: "valid2@email",
mail_pw: "123456",
};
await dc.rpc.batchSetConfig(accountId, config);
const retrieved = await dc.rpc.batchGetConfig(
accountId,
Object.keys(config),
);
expect(retrieved).to.deep.equal(config);
});
});
});

View File

@@ -0,0 +1,173 @@
import { assert, expect } from "chai";
import { StdioDeltaChat as DeltaChat, DcEvent, C } from "../deltachat.js";
import { RpcServerHandle, createTempUser, startServer } from "./test_base.js";
const EVENT_TIMEOUT = 20000;
describe("online tests", function () {
let serverHandle: RpcServerHandle;
let dc: DeltaChat;
let account1: { email: string; password: string };
let account2: { email: string; password: string };
let accountId1: number, accountId2: number;
before(async function () {
this.timeout(60000);
if (!process.env.CHATMAIL_DOMAIN) {
if (process.env.COVERAGE && !process.env.COVERAGE_OFFLINE) {
console.error(
"CAN NOT RUN COVERAGE correctly: Missing CHATMAIL_DOMAIN environment variable!\n\n",
"You can set COVERAGE_OFFLINE=1 to circumvent this check and skip the online tests, but those coverage results will be wrong, because some functions can only be tested in the online test",
);
process.exit(1);
}
console.log(
"Missing CHATMAIL_DOMAIN environment variable!, skip integration tests",
);
this.skip();
}
serverHandle = await startServer();
dc = new DeltaChat(serverHandle.stdin, serverHandle.stdout, true);
dc.on("ALL", (contextId, { kind }) => {
if (kind !== "Info") console.log(contextId, kind);
});
account1 = createTempUser(process.env.CHATMAIL_DOMAIN);
if (!account1 || !account1.email || !account1.password) {
console.log(
"We didn't got back an account from the api, skip integration tests",
);
this.skip();
}
account2 = createTempUser(process.env.CHATMAIL_DOMAIN);
if (!account2 || !account2.email || !account2.password) {
console.log(
"We didn't got back an account2 from the api, skip integration tests",
);
this.skip();
}
});
after(async () => {
dc && dc.close();
serverHandle && (await serverHandle.close());
});
let accountsConfigured = false;
it("configure test accounts", async function () {
this.timeout(40000);
accountId1 = await dc.rpc.addAccount();
await dc.rpc.setConfig(accountId1, "addr", account1.email);
await dc.rpc.setConfig(accountId1, "mail_pw", account1.password);
await dc.rpc.configure(accountId1);
await waitForEvent(dc, "ImapInboxIdle", accountId1);
accountId2 = await dc.rpc.addAccount();
await dc.rpc.batchSetConfig(accountId2, {
addr: account2.email,
mail_pw: account2.password,
});
await dc.rpc.configure(accountId2);
await waitForEvent(dc, "ImapInboxIdle", accountId2);
accountsConfigured = true;
});
it("send and receive text message", async function () {
if (!accountsConfigured) {
this.skip();
}
this.timeout(15000);
const vcard = await dc.rpc.makeVcard(accountId2, [C.DC_CONTACT_ID_SELF]);
const contactId = (await dc.rpc.importVcardContents(accountId1, vcard))[0];
const chatId = await dc.rpc.createChatByContactId(accountId1, contactId);
const eventPromise = waitForEvent(dc, "IncomingMsg", accountId2);
await dc.rpc.miscSendTextMessage(accountId1, chatId, "Hello");
const { chatId: chatIdOnAccountB } = await eventPromise;
await dc.rpc.acceptChat(accountId2, chatIdOnAccountB);
const messageList = await dc.rpc.getMessageIds(
accountId2,
chatIdOnAccountB,
false,
false,
);
// There are 2 messages in the chat:
// 'Messages are end-to-end encrypted' (info message) and 'Hello'
expect(messageList).have.length(2);
const message = await dc.rpc.getMessage(accountId2, messageList[1]);
expect(message.text).equal("Hello");
expect(message.showPadlock).equal(true);
});
it("send and receive text message roundtrip", async function () {
if (!accountsConfigured) {
this.skip();
}
this.timeout(10000);
// send message from A to B
const vcard = await dc.rpc.makeVcard(accountId2, [C.DC_CONTACT_ID_SELF]);
const contactId = (await dc.rpc.importVcardContents(accountId1, vcard))[0];
const chatId = await dc.rpc.createChatByContactId(accountId1, contactId);
const eventPromise = waitForEvent(dc, "IncomingMsg", accountId2);
dc.rpc.miscSendTextMessage(accountId1, chatId, "Hello2");
// wait for message from A
console.log("wait for message from A");
const event = await eventPromise;
const { chatId: chatIdOnAccountB } = event;
await dc.rpc.acceptChat(accountId2, chatIdOnAccountB);
const messageList = await dc.rpc.getMessageIds(
accountId2,
chatIdOnAccountB,
false,
false,
);
const message = await dc.rpc.getMessage(
accountId2,
messageList.reverse()[0],
);
expect(message.text).equal("Hello2");
// Send message back from B to A
const eventPromise2 = waitForEvent(dc, "IncomingMsg", accountId1);
dc.rpc.miscSendTextMessage(accountId2, chatId, "super secret message");
// Check if answer arrives at A and if it is encrypted
await eventPromise2;
const messageId = (
await dc.rpc.getMessageIds(accountId1, chatId, false, false)
).reverse()[0];
const message2 = await dc.rpc.getMessage(accountId1, messageId);
expect(message2.text).equal("super secret message");
expect(message2.showPadlock).equal(true);
});
});
async function waitForEvent<T extends DcEvent["kind"]>(
dc: DeltaChat,
eventType: T,
accountId: number,
timeout: number = EVENT_TIMEOUT,
): Promise<Extract<DcEvent, { kind: T }>> {
return new Promise((resolve, reject) => {
const rejectTimeout = setTimeout(
() => reject(new Error("Timeout reached before event came in")),
timeout,
);
const callback = (contextId: number, event: DcEvent) => {
if (contextId == accountId) {
dc.off(eventType, callback);
clearTimeout(rejectTimeout);
resolve(event as any);
}
};
dc.on(eventType, callback);
});
}

View File

@@ -0,0 +1,89 @@
import { tmpdir } from "os";
import { join, resolve } from "path";
import { mkdtemp, rm } from "fs/promises";
import { spawn, exec } from "child_process";
import { Readable, Writable } from "node:stream";
export type RpcServerHandle = {
stdin: Writable;
stdout: Readable;
close: () => Promise<void>;
};
export async function startServer(): Promise<RpcServerHandle> {
const tmpDir = await mkdtemp(join(tmpdir(), "deltachat-jsonrpc-test"));
const pathToServerBinary = resolve(
join(await getTargetDir(), "debug/deltachat-rpc-server"),
);
const server = spawn(pathToServerBinary, {
cwd: tmpDir,
env: {
RUST_LOG: process.env.RUST_LOG || "info",
RUST_MIN_STACK: "8388608",
},
});
server.on("error", (err) => {
throw new Error(
"Failed to start server executable " +
pathToServerBinary +
", make sure you built it first.",
);
});
let shouldClose = false;
server.on("exit", () => {
if (shouldClose) {
return;
}
throw new Error("Server quit");
});
server.stderr.pipe(process.stderr);
return {
stdin: server.stdin,
stdout: server.stdout,
close: async () => {
shouldClose = true;
if (!server.kill()) {
console.log("server termination failed");
}
await rm(tmpDir, { recursive: true });
},
};
}
export function createTempUser(chatmailDomain: String) {
const charset = "2345789acdefghjkmnpqrstuvwxyz";
let user = "ci-";
for (let i = 0; i < 6; i++) {
user += charset[Math.floor(Math.random() * charset.length)];
}
const email = user + "@" + chatmailDomain;
return { email: email, password: user + "$" + user };
}
function getTargetDir(): Promise<string> {
return new Promise((resolve, reject) => {
exec(
"cargo metadata --no-deps --format-version 1",
(error, stdout, _stderr) => {
if (error) {
console.log("error", error);
reject(error);
} else {
try {
const json = JSON.parse(stdout);
resolve(json.target_directory);
} catch (error) {
console.log("json error", error);
reject(error);
}
}
},
);
});
}

View File

@@ -0,0 +1,20 @@
{
"compilerOptions": {
"alwaysStrict": true,
"strict": true,
"sourceMap": true,
"strictNullChecks": true,
"rootDir": ".",
"outDir": "dist",
"lib": ["ES2017", "dom"],
"target": "ES2017",
"module": "es2020",
"declaration": true,
"esModuleInterop": true,
"moduleResolution": "node",
"noImplicitAny": true,
"isolatedModules": true
},
"include": ["*.ts", "test/*.ts"],
"compileOnSave": false
}