Merge async JSON-RPC client

PR: https://github.com/deltachat/deltachat-core-rust/pull/3734

.github/workflows/ci.yml

@@ -147,6 +147,20 @@ jobs:
       working-directory: python
       run: tox -e lint,mypy,doc,py3
 
+    - name: build deltachat-rpc-server
+      if: ${{ matrix.python }}
+      uses: actions-rs/cargo@v1
+      with:
+        command: build
+        args: -p deltachat-rpc-server
+
+    - name: run deltachat-rpc-client tests
+      if: ${{ matrix.python }}
+      env:
+        DCC_NEW_TMP_EMAIL: ${{ secrets.DCC_NEW_TMP_EMAIL }}
+      working-directory: deltachat-rpc-client
+      run: tox -e py3
+
     - name: install pypy
       if: ${{ matrix.python }}
       uses: actions/setup-python@v4

.gitignore

@@ -12,8 +12,8 @@ include
 *.db
 *.db-blobs
 
+.tox
 python/.eggs
-python/.tox
 *.egg-info
 __pycache__
 python/src/deltachat/capi*.so

CHANGELOG.md

@@ -9,6 +9,7 @@
 ### API-Changes
 - Add Python API to send reactions #3762
 - jsonrpc: add message errors to MessageObject #3788
+- jsonrpc: Add async Python client #3734
 
 ### Fixes
 - Make sure malformed messsages will never block receiving further messages anymore #3771

deltachat-rpc-client/README.md

@@ -0,0 +1,41 @@
+# Delta Chat RPC python client
+
+RPC client connects to standalone Delta Chat RPC server `deltachat-rpc-server`
+and provides asynchronous interface to it.
+
+## Getting started
+
+To use Delta Chat RPC client, first build a `deltachat-rpc-server` with `cargo build -p deltachat-rpc-server`.
+Install it anywhere in your `PATH`.
+
+## Testing
+
+1. Build `deltachat-rpc-server` with `cargo build -p deltachat-rpc-server`.
+2. Run `tox`.
+
+Additional arguments to `tox` are passed to pytest, e.g. `tox -- -s` does not capture test output.
+
+## Using in REPL
+
+Setup a development environment:
+```
+$ tox --devenv env
+$ . env/bin/activate
+```
+
+It is recommended to use IPython, because it supports using `await` directly
+from the REPL.
+
+```
+$ pip install ipython
+$ PATH="../target/debug:$PATH" ipython
+...
+In [1]: from deltachat_rpc_client import *
+In [2]: rpc = Rpc()
+In [3]: await rpc.start()
+In [4]: dc = DeltaChat(rpc)
+In [5]: system_info = await dc.get_system_info()
+In [6]: system_info["level"]
+Out[6]: 'awesome'
+In [7]: await rpc.close()
+```

deltachat-rpc-client/examples/echobot.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+import asyncio
+import logging
+import sys
+
+import deltachat_rpc_client as dc
+
+
+async def main():
+    async with dc.Rpc() as rpc:
+        deltachat = dc.DeltaChat(rpc)
+        system_info = await deltachat.get_system_info()
+        logging.info("Running deltachat core %s", system_info["deltachat_core_version"])
+
+        accounts = await deltachat.get_all_accounts()
+        account = accounts[0] if accounts else await deltachat.add_account()
+
+        await account.set_config("bot", "1")
+        if not await account.is_configured():
+            logging.info("Account is not configured, configuring")
+            await account.set_config("addr", sys.argv[1])
+            await account.set_config("mail_pw", sys.argv[2])
+            await account.configure()
+            logging.info("Configured")
+        else:
+            logging.info("Account is already configured")
+            await deltachat.start_io()
+
+        async def process_messages():
+            for message in await account.get_fresh_messages_in_arrival_order():
+                snapshot = await message.get_snapshot()
+                if not snapshot.is_info:
+                    await snapshot.chat.send_text(snapshot.text)
+                await snapshot.message.mark_seen()
+
+        # Process old messages.
+        await process_messages()
+
+        while True:
+            event = await account.wait_for_event()
+            if event["type"] == "Info":
+                logging.info("%s", event["msg"])
+            elif event["type"] == "Warning":
+                logging.warning("%s", event["msg"])
+            elif event["type"] == "Error":
+                logging.error("%s", event["msg"])
+            elif event["type"] == "IncomingMsg":
+                logging.info("Got an incoming message")
+                await process_messages()
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    asyncio.run(main())

deltachat-rpc-client/pyproject.toml

@@ -0,0 +1,29 @@
+[build-system]
+requires = ["setuptools>=45", "setuptools_scm[toml]>=6.2"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "deltachat-rpc-client"
+description = "Python client for Delta Chat core JSON-RPC interface"
+dependencies = [
+    "aiohttp",
+    "aiodns"
+]
+dynamic = [
+    "version"
+]
+
+[tool.setuptools]
+# We declare the package not-zip-safe so that our type hints are also available
+# when checking client code that uses our (installed) package.
+# Ref:
+# https://mypy.readthedocs.io/en/stable/installed_packages.html?highlight=zip#using-installed-packages-with-mypy-pep-561
+zip-safe = false
+
+[tool.setuptools.package-data]
+deltachat_rpc_client = [
+    "py.typed"
+]
+
+[project.entry-points.pytest11]
+"deltachat_rpc_client.pytestplugin" = "deltachat_rpc_client.pytestplugin"

deltachat-rpc-client/src/deltachat_rpc_client/__init__.py

@@ -0,0 +1,5 @@
+from .account import Account
+from .contact import Contact
+from .deltachat import DeltaChat
+from .message import Message
+from .rpc import Rpc

deltachat-rpc-client/src/deltachat_rpc_client/account.py

@@ -0,0 +1,79 @@
+from typing import List, Optional
+
+from .chat import Chat
+from .contact import Contact
+from .message import Message
+
+
+class Account:
+    def __init__(self, rpc, account_id) -> None:
+        self._rpc = rpc
+        self.account_id = account_id
+
+    def __repr__(self) -> str:
+        return f"<Account id={self.account_id}>"
+
+    async def wait_for_event(self) -> dict:
+        """Wait until the next event and return it."""
+        return await self._rpc.wait_for_event(self.account_id)
+
+    async def remove(self) -> None:
+        """Remove the account."""
+        await self._rpc.remove_account(self.account_id)
+
+    async def start_io(self) -> None:
+        """Start the account I/O."""
+        await self._rpc.start_io(self.account_id)
+
+    async def stop_io(self) -> None:
+        """Stop the account I/O."""
+        await self._rpc.stop_io(self.account_id)
+
+    async def get_info(self) -> dict:
+        return await self._rpc.get_info(self.account_id)
+
+    async def get_file_size(self) -> int:
+        return await self._rpc.get_account_file_size(self.account_id)
+
+    async def is_configured(self) -> bool:
+        """Return True for configured accounts."""
+        return await self._rpc.is_configured(self.account_id)
+
+    async def set_config(self, key: str, value: Optional[str] = None) -> None:
+        """Set the configuration value key pair."""
+        await self._rpc.set_config(self.account_id, key, value)
+
+    async def get_config(self, key: str) -> Optional[str]:
+        """Get the configuration value."""
+        return await self._rpc.get_config(self.account_id, key)
+
+    async def configure(self) -> None:
+        """Configure an account."""
+        await self._rpc.configure(self.account_id)
+
+    async def create_contact(self, address: str, name: Optional[str] = None) -> Contact:
+        """Create a contact with the given address and, optionally, a name."""
+        return Contact(
+            self._rpc,
+            self.account_id,
+            await self._rpc.create_contact(self.account_id, address, name),
+        )
+
+    async def secure_join(self, qrdata: str) -> Chat:
+        chat_id = await self._rpc.secure_join(self.account_id, qrdata)
+        return Chat(self._rpc, self.account_id, chat_id)
+
+    async def get_fresh_messages(self) -> List[Message]:
+        """Return the list of fresh messages, newest messages first.
+
+        This call is intended for displaying notifications.
+        If you are writing a bot, use get_fresh_messages_in_arrival_order instead,
+        to process oldest messages first.
+        """
+        fresh_msg_ids = await self._rpc.get_fresh_msgs(self.account_id)
+        return [Message(self._rpc, self.account_id, msg_id) for msg_id in fresh_msg_ids]
+
+    async def get_fresh_messages_in_arrival_order(self) -> List[Message]:
+        """Return fresh messages list sorted in the order of their arrival, with ascending IDs."""
+        fresh_msg_ids = sorted(await self._rpc.get_fresh_msgs(self.account_id))
+        return [Message(self._rpc, self.account_id, msg_id) for msg_id in fresh_msg_ids]

deltachat-rpc-client/src/deltachat_rpc_client/chat.py

@@ -0,0 +1,41 @@
+from typing import TYPE_CHECKING
+
+from .rpc import Rpc
+
+if TYPE_CHECKING:
+    from .message import Message
+
+
+class Chat:
+    def __init__(self, rpc: Rpc, account_id: int, chat_id: int) -> None:
+        self._rpc = rpc
+        self.account_id = account_id
+        self.chat_id = chat_id
+
+    async def block(self) -> None:
+        """Block the chat."""
+        await self._rpc.block_chat(self.account_id, self.chat_id)
+
+    async def accept(self) -> None:
+        """Accept the contact request."""
+        await self._rpc.accept_chat(self.account_id, self.chat_id)
+
+    async def delete(self) -> None:
+        await self._rpc.delete_chat(self.account_id, self.chat_id)
+
+    async def get_encryption_info(self) -> str:
+        return await self._rpc.get_chat_encryption_info(self.account_id, self.chat_id)
+
+    async def send_text(self, text: str) -> "Message":
+        from .message import Message
+
+        msg_id = await self._rpc.misc_send_text_message(
+            self.account_id, self.chat_id, text
+        )
+        return Message(self._rpc, self.account_id, msg_id)
+
+    async def leave(self) -> None:
+        await self._rpc.leave_group(self.account_id, self.chat_id)
+
+    async def get_fresh_message_count(self) -> int:
+        return await self._rpc.get_fresh_msg_cnt(self.account_id, self.chat_id)

deltachat-rpc-client/src/deltachat_rpc_client/contact.py

@@ -0,0 +1,52 @@
+from typing import TYPE_CHECKING
+
+from .rpc import Rpc
+
+if TYPE_CHECKING:
+    from .chat import Chat
+
+
+class Contact:
+    """
+    Contact API.
+
+    Essentially a wrapper for RPC, account ID and a contact ID.
+    """
+
+    def __init__(self, rpc: Rpc, account_id: int, contact_id: int) -> None:
+        self._rpc = rpc
+        self.account_id = account_id
+        self.contact_id = contact_id
+
+    async def block(self) -> None:
+        """Block contact."""
+        await self._rpc.block_contact(self.account_id, self.contact_id)
+
+    async def unblock(self) -> None:
+        """Unblock contact."""
+        await self._rpc.unblock_contact(self.account_id, self.contact_id)
+
+    async def delete(self) -> None:
+        """Delete contact."""
+        await self._rpc.delete_contact(self.account_id, self.contact_id)
+
+    async def change_name(self, name: str) -> None:
+        await self._rpc.change_contact_name(self.account_id, self.contact_id, name)
+
+    async def get_encryption_info(self) -> str:
+        return await self._rpc.get_contact_encryption_info(
+            self.account_id, self.contact_id
+        )
+
+    async def get_dictionary(self) -> dict:
+        """Return a dictionary with a snapshot of all contact properties."""
+        return await self._rpc.get_contact(self.account_id, self.contact_id)
+
+    async def create_chat(self) -> "Chat":
+        from .chat import Chat
+
+        return Chat(
+            self._rpc,
+            self.account_id,
+            await self._rpc.create_chat_by_contact_id(self.account_id, self.contact_id),
+        )

deltachat-rpc-client/src/deltachat_rpc_client/deltachat.py

@@ -0,0 +1,34 @@
+from typing import List
+
+from .account import Account
+from .rpc import Rpc
+
+
+class DeltaChat:
+    """
+    Delta Chat account manager.
+    This is the root of the object oriented API.
+    """
+
+    def __init__(self, rpc: Rpc) -> None:
+        self.rpc = rpc
+
+    async def add_account(self) -> Account:
+        account_id = await self.rpc.add_account()
+        return Account(self.rpc, account_id)
+
+    async def get_all_accounts(self) -> List[Account]:
+        account_ids = await self.rpc.get_all_account_ids()
+        return [Account(self.rpc, account_id) for account_id in account_ids]
+
+    async def start_io(self) -> None:
+        await self.rpc.start_io_for_all_accounts()
+
+    async def stop_io(self) -> None:
+        await self.rpc.stop_io_for_all_accounts()
+
+    async def maybe_network(self) -> None:
+        await self.rpc.maybe_network()
+
+    async def get_system_info(self) -> dict:
+        return await self.rpc.get_system_info()

deltachat-rpc-client/src/deltachat_rpc_client/message.py

@@ -0,0 +1,42 @@
+from dataclasses import dataclass
+from typing import Optional
+
+from .chat import Chat
+from .contact import Contact
+from .rpc import Rpc
+
+
+class Message:
+    def __init__(self, rpc: Rpc, account_id: int, msg_id: int) -> None:
+        self._rpc = rpc
+        self.account_id = account_id
+        self.msg_id = msg_id
+
+    async def send_reaction(self, reactions: str) -> "Message":
+        msg_id = await self._rpc.send_reaction(self.account_id, self.msg_id, reactions)
+        return Message(self._rpc, self.account_id, msg_id)
+
+    async def get_snapshot(self) -> "MessageSnapshot":
+        message_object = await self._rpc.get_message(self.account_id, self.msg_id)
+        return MessageSnapshot(
+            message=self,
+            chat=Chat(self._rpc, self.account_id, message_object["chatId"]),
+            sender=Contact(self._rpc, self.account_id, message_object["fromId"]),
+            text=message_object["text"],
+            error=message_object.get("error"),
+            is_info=message_object["isInfo"],
+        )
+
+    async def mark_seen(self) -> None:
+        """Mark the message as seen."""
+        await self._rpc.markseen_msgs(self.account_id, [self.msg_id])
+
+
+@dataclass
+class MessageSnapshot:
+    message: Message
+    chat: Chat
+    sender: Contact
+    text: str
+    error: Optional[str]
+    is_info: bool

deltachat-rpc-client/src/deltachat_rpc_client/py.typed

@@ -0,0 +1 @@
+# PEP 561 marker file. See https://peps.python.org/pep-0561/

deltachat-rpc-client/src/deltachat_rpc_client/pytestplugin.py

@@ -0,0 +1,52 @@
+import asyncio
+import json
+import os
+from typing import AsyncGenerator, List
+
+import aiohttp
+import pytest_asyncio
+
+from .account import Account
+from .deltachat import DeltaChat
+from .rpc import Rpc
+
+
+async def get_temp_credentials() -> dict:
+    url = os.getenv("DCC_NEW_TMP_EMAIL")
+    assert url, "Failed to get online account, DCC_NEW_TMP_EMAIL is not set"
+    async with aiohttp.ClientSession() as session:
+        async with session.post(url) as response:
+            return json.loads(await response.text())
+
+
+class ACFactory:
+    def __init__(self, deltachat: DeltaChat) -> None:
+        self.deltachat = deltachat
+
+    async def new_configured_account(self) -> Account:
+        credentials = await get_temp_credentials()
+        account = await self.deltachat.add_account()
+        assert not await account.is_configured()
+        await account.set_config("addr", credentials["email"])
+        await account.set_config("mail_pw", credentials["password"])
+        await account.configure()
+        assert await account.is_configured()
+        return account
+
+    async def get_online_accounts(self, num: int) -> List[Account]:
+        accounts = [await self.new_configured_account() for _ in range(num)]
+        await self.deltachat.start_io()
+        return accounts
+
+
+@pytest_asyncio.fixture
+async def rpc(tmp_path) -> AsyncGenerator:
+    env = {**os.environ, "DC_ACCOUNTS_PATH": str(tmp_path / "accounts")}
+    rpc_server = Rpc(env=env)
+    async with rpc_server as rpc:
+        yield rpc
+
+
+@pytest_asyncio.fixture
+async def acfactory(rpc) -> AsyncGenerator:
+    yield ACFactory(DeltaChat(rpc))

deltachat-rpc-client/src/deltachat_rpc_client/rpc.py

@@ -0,0 +1,92 @@
+import asyncio
+import json
+from typing import Any, AsyncGenerator, Dict, Optional
+
+
+class JsonRpcError(Exception):
+    pass
+
+
+class Rpc:
+    def __init__(self, *args, **kwargs):
+        """The given arguments will be passed to asyncio.create_subprocess_exec()"""
+        self._args = args
+        self._kwargs = kwargs
+
+    async def start(self) -> None:
+        self.process = await asyncio.create_subprocess_exec(
+            "deltachat-rpc-server",
+            stdin=asyncio.subprocess.PIPE,
+            stdout=asyncio.subprocess.PIPE,
+            *self._args,
+            **self._kwargs
+        )
+        self.event_queues: Dict[int, asyncio.Queue] = {}
+        self.id = 0
+        self.reader_task = asyncio.create_task(self.reader_loop())
+
+        # Map from request ID to `asyncio.Future` returning the response.
+        self.request_events: Dict[int, asyncio.Future] = {}
+
+    async def close(self) -> None:
+        """Terminate RPC server process and wait until the reader loop finishes."""
+        self.process.terminate()
+        await self.reader_task
+
+    async def __aenter__(self):
+        await self.start()
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb):
+        await self.close()
+
+    async def reader_loop(self) -> None:
+        while True:
+            line = await self.process.stdout.readline()
+            if not line:  # EOF
+                break
+            response = json.loads(line)
+            if "id" in response:
+                fut = self.request_events.pop(response["id"])
+                fut.set_result(response)
+            elif response["method"] == "event":
+                # An event notification.
+                params = response["params"]
+                account_id = params["contextId"]
+                if account_id not in self.event_queues:
+                    self.event_queues[account_id] = asyncio.Queue()
+                await self.event_queues[account_id].put(params["event"])
+            else:
+                print(response)
+
+    async def wait_for_event(self, account_id: int) -> Optional[dict]:
+        """Waits for the next event from the given account and returns it."""
+        if account_id in self.event_queues:
+            return await self.event_queues[account_id].get()
+        return None
+
+    def __getattr__(self, attr: str):
+        async def method(*args, **kwargs) -> Any:
+            self.id += 1
+            request_id = self.id
+
+            assert not (args and kwargs), "Mixing positional and keyword arguments"
+
+            request = {
+                "jsonrpc": "2.0",
+                "method": attr,
+                "params": kwargs or args,
+                "id": self.id,
+            }
+            data = (json.dumps(request) + "\n").encode()
+            self.process.stdin.write(data)
+            loop = asyncio.get_running_loop()
+            fut = loop.create_future()
+            self.request_events[request_id] = fut
+            response = await fut
+            if "error" in response:
+                raise JsonRpcError(response["error"])
+            if "result" in response:
+                return response["result"]
+
+        return method

deltachat-rpc-client/tests/test_something.py

@@ -0,0 +1,60 @@
+import pytest
+
+
+@pytest.mark.asyncio
+async def test_system_info(rpc) -> None:
+    system_info = await rpc.get_system_info()
+    assert "arch" in system_info
+    assert "deltachat_core_version" in system_info
+
+
+@pytest.mark.asyncio
+async def test_email_address_validity(rpc) -> None:
+    valid_addresses = [
+        "email@example.com",
+        "36aa165ae3406424e0c61af17700f397cad3fe8ab83d682d0bddf3338a5dd52e@yggmail@yggmail",
+    ]
+    invalid_addresses = ["email@", "example.com", "emai221"]
+
+    for addr in valid_addresses:
+        assert await rpc.check_email_validity(addr)
+    for addr in invalid_addresses:
+        assert not await rpc.check_email_validity(addr)
+
+
+@pytest.mark.asyncio
+async def test_acfactory(acfactory) -> None:
+    account = await acfactory.new_configured_account()
+    while True:
+        event = await account.wait_for_event()
+        if event["type"] == "ConfigureProgress":
+            # Progress 0 indicates error.
+            assert event["progress"] != 0
+
+            if event["progress"] == 1000:
+                # Success.
+                break
+        else:
+            print(event)
+    print("Successful configuration")
+
+
+@pytest.mark.asyncio
+async def test_object_account(acfactory) -> None:
+    alice, bob = await acfactory.get_online_accounts(2)
+
+    alice_contact_bob = await alice.create_contact(await bob.get_config("addr"), "Bob")
+    alice_chat_bob = await alice_contact_bob.create_chat()
+    await alice_chat_bob.send_text("Hello!")
+
+    while True:
+        event = await bob.wait_for_event()
+        if event["type"] == "IncomingMsg":
+            chat_id = event["chatId"]
+            msg_id = event["msgId"]
+            break
+
+    rpc = acfactory.deltachat.rpc
+    message = await rpc.get_message(bob.account_id, msg_id)
+    assert message["chatId"] == chat_id
+    assert message["text"] == "Hello!"

deltachat-rpc-client/tox.ini

@@ -0,0 +1,20 @@
+[tox]
+isolated_build = true
+envlist =
+    py3
+
+[testenv]
+commands =
+    pytest {posargs}
+setenv =
+# Avoid stack overflow when Rust core is built without optimizations.
+    RUST_MIN_STACK=8388608
+    PATH = {env:PATH}{:}{toxinidir}/../target/debug
+passenv =
+    DCC_NEW_TMP_EMAIL
+deps =
+    pytest
+    pytest-async
+    pytest-asyncio
+    aiohttp
+    aiodns