add another test case

It only contained proxy
that some users ran in Termux
to look at IMAP traffic.

The same can be achieved with `socat`, e.g.:
  socat -v TCP-LISTEN:9999,bind=127.0.0.1 OPENSSL:nine.testrun.org:993

Cargo.lock

@@ -783,9 +783,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cfg_aliases"
-version = "0.1.1"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e"
+checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
 
 [[package]]
 name = "chacha20"
@@ -3880,9 +3880,9 @@ dependencies = [
 
 [[package]]
 name = "nix"
-version = "0.28.0"
+version = "0.29.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4"
+checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46"
 dependencies = [
  "bitflags 2.6.0",
  "cfg-if",
@@ -5472,9 +5472,9 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4"
 
 [[package]]
 name = "rustyline"
-version = "14.0.0"
+version = "15.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7803e8936da37efd9b6d4478277f4b2b9bb5cdb37a113e8d63222e58da647e63"
+checksum = "2ee1e066dc922e513bda599c6ccb5f3bb2b0ea5870a579448f2622993f0a9a2f"
 dependencies = [
  "bitflags 2.6.0",
  "cfg-if",
@@ -5484,12 +5484,12 @@ dependencies = [
  "libc",
  "log",
  "memchr",
- "nix 0.28.0",
+ "nix 0.29.0",
  "radix_trie",
  "unicode-segmentation",
- "unicode-width",
+ "unicode-width 0.2.0",
  "utf8parse",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -6273,7 +6273,7 @@ checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9"
 dependencies = [
  "smawk",
  "unicode-linebreak",
- "unicode-width",
+ "unicode-width 0.1.11",
 ]
 
 [[package]]
@@ -6779,6 +6779,12 @@ version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85"
 
+[[package]]
+name = "unicode-width"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd"
+
 [[package]]
 name = "unicode-xid"
 version = "0.2.4"

contrib/proxy.py

@@ -1,80 +0,0 @@
-#!/usr/bin/env python3
-# Examples:
-#
-# Original server that doesn't use SSL:
-# ./proxy.py 8080 imap.nauta.cu 143
-# ./proxy.py 8081 smtp.nauta.cu 25
-#
-# Original server that uses SSL:
-# ./proxy.py 8080 testrun.org 993 --ssl
-# ./proxy.py 8081 testrun.org 465 --ssl
-
-from datetime import datetime
-import argparse
-import selectors
-import ssl
-import socket
-import socketserver
-
-
-class Proxy(socketserver.ThreadingTCPServer):
-    allow_reuse_address = True
-
-    def __init__(self, proxy_host, proxy_port, real_host, real_port, use_ssl):
-        self.real_host = real_host
-        self.real_port = real_port
-        self.use_ssl = use_ssl
-        super().__init__((proxy_host, proxy_port), RequestHandler)
-
-
-class RequestHandler(socketserver.BaseRequestHandler):
-
-    def handle(self):
-        print('{} - {} CONNECTED.'.format(datetime.now(), self.client_address))
-
-        total = 0
-        real_server = (self.server.real_host, self.server.real_port)
-        with socket.create_connection(real_server) as sock:
-            if self.server.use_ssl:
-                context = ssl.create_default_context()
-                sock = context.wrap_socket(
-                    sock, server_hostname=real_server[0])
-
-            forward = {self.request: sock, sock: self.request}
-
-            sel = selectors.DefaultSelector()
-            sel.register(self.request, selectors.EVENT_READ,
-                         self.client_address)
-            sel.register(sock, selectors.EVENT_READ, real_server)
-
-            active = True
-            while active:
-                events = sel.select()
-                for key, mask in events:
-                    print('\n{} - {} wrote:'.format(datetime.now(), key.data))
-                    data = key.fileobj.recv(1024)
-                    received = len(data)
-                    total += received
-                    print(data)
-                    print('{} Bytes\nTotal: {} Bytes'.format(received, total))
-                    if data:
-                        forward[key.fileobj].sendall(data)
-                    else:
-                        print('\nCLOSING CONNECTION.\n\n')
-                        forward[key.fileobj].close()
-                        key.fileobj.close()
-                        active = False
-
-
-if __name__ == '__main__':
-    p = argparse.ArgumentParser(description='Simple Python Proxy')
-    p.add_argument(
-        "proxy_port", help="the port where the proxy will listen", type=int)
-    p.add_argument('host', help="the real host")
-    p.add_argument('port', help="the port of the real host", type=int)
-    p.add_argument("--ssl", help="use ssl to connect to the real host",
-                   action="store_true")
-    args = p.parse_args()
-
-    with Proxy('', args.proxy_port, args.host, args.port, args.ssl) as proxy:
-        proxy.serve_forever()

deltachat-repl/Cargo.toml

@@ -13,7 +13,7 @@ log = { workspace = true }
 nu-ansi-term = { workspace = true }
 qr2term = "0.3.3"
 rusqlite = { workspace = true }
-rustyline = "14"
+rustyline = "15"
 tokio = { workspace = true, features = ["fs", "rt-multi-thread", "macros"] }
 tracing-subscriber = { workspace = true, features = ["env-filter"] }
 

deltachat-repl/src/main.rs

@@ -22,7 +22,7 @@ use log::{error, info, warn};
 use nu_ansi_term::Color;
 use rustyline::completion::{Completer, FilenameCompleter, Pair};
 use rustyline::error::ReadlineError;
-use rustyline::highlight::{Highlighter, MatchingBracketHighlighter};
+use rustyline::highlight::{CmdKind as HighlightCmdKind, Highlighter, MatchingBracketHighlighter};
 use rustyline::hint::{Hinter, HistoryHinter};
 use rustyline::validate::Validator;
 use rustyline::{
@@ -298,8 +298,8 @@ impl Highlighter for DcHelper {
         self.highlighter.highlight(line, pos)
     }
 
-    fn highlight_char(&self, line: &str, pos: usize, forced: bool) -> bool {
-        self.highlighter.highlight_char(line, pos, forced)
+    fn highlight_char(&self, line: &str, pos: usize, kind: HighlightCmdKind) -> bool {
+        self.highlighter.highlight_char(line, pos, kind)
     }
 }
 

deny.toml

@@ -52,6 +52,7 @@ skip = [
      { name = "sync_wrapper", version = "0.1.2" },
      { name = "syn", version = "1.0.109" },
      { name = "time", version = "<0.3" },
+     { name = "unicode-width", version = "0.1.11" },
      { name = "wasi", version = "<0.11" },
      { name = "windows_aarch64_gnullvm", version = "<0.52" },
      { name = "windows_aarch64_msvc", version = "<0.52" },

python/tests/test_5_simulation.py

@@ -0,0 +1,229 @@
+import pytest
+
+
+@pytest.fixture
+def relay():
+    return Relay()
+
+
+class Relay:
+    def __init__(self):
+        self.peers = {}
+
+    def make_peers(self, num):
+        for i in range(num):
+            newpeer = Peer(relay=self, num=i)
+            self.peers[newpeer.id] = newpeer
+        return self.peers.values()
+
+    def dump(self, title):
+        print()
+        print(f"# {title}")
+        for peer_id, peer in self.peers.items():
+            pending = sum(len(x) for x in peer.from2mailbox.values())
+            members = ",".join(peer.members)
+            print(f"{peer_id} clock={peer.current_clock} members={members} pending={pending}")
+        print()
+
+    def receive_all(self, peers=None):
+        peers = peers if peers is not None else list(self.peers.values())
+        for peer in peers:
+            # drain peer mailbox by reading messages from each sender separately
+            for from_peer in self.peers.values():
+                pending = peer.from2mailbox.pop(from_peer, [])
+                if from_peer.id != peer.id:
+                    for msg in pending:
+                        msg.receive(peer)
+
+    def assert_group_consistency(self):
+        peers = list(self.peers.values())
+        for peer1, peer2 in zip(peers, peers[1:]):
+            assert peer1.members == peer2.members
+            assert peer1.current_clock == peer2.current_clock
+            nums = ",".join(sorted(peer1.members))
+            print(f"{peer1.id} and {peer2.id} have same members {nums}")
+
+
+class Message:
+    def __init__(self, sender, **payload):
+        self.sender = sender
+        self.payload = payload
+        self.recipients = set(sender.members)
+        # we increment clock on AddMemberMessage and DelMemberMessage
+        sender.current_clock += self.clock_inc
+        self.clock = sender.current_clock
+        self.send()
+
+    def __repr__(self):
+        nums = ",".join(self.recipients)
+        return f"<{self.__class__.__name__} clock={self.clock} {self.sender.id}->{nums} {self.payload}>"
+
+    def send(self):
+        print(f"sending {self}")
+        for peer_id in self.sender.members:
+            peer = self.sender.relay.peers[peer_id]
+            peer.from2mailbox.setdefault(self.sender, []).append(self)
+
+
+class AddMemberMessage(Message):
+    clock_inc = 1
+
+    def __init__(self, sender, member):
+        sender.members.add(member)
+        super().__init__(sender, member=member)
+
+    def receive(self, peer):
+        if not peer.members:  # create group
+            peer.members = self.recipients.copy()
+            peer.current_clock = self.clock
+            return
+
+        peer.members.add(self.payload["member"])
+        if peer.current_clock < self.clock:
+            peer.members.update(self.recipients)
+            peer.current_clock = self.clock
+        elif peer.current_clock == self.clock:
+            if peer.members != self.recipients:
+                peer.current_clock += 1
+
+
+class DelMemberMessage(Message):
+    clock_inc = 1
+
+    def send(self):
+        super().send()
+        self.sender.members.remove(self.payload["member"])
+
+    def receive(self, peer):
+        member = self.payload["member"]
+        if member in peer.members:
+            if peer.current_clock <= self.clock:
+                peer.members.remove(member)
+            peer.current_clock = self.clock
+
+
+class ChatMessage(Message):
+    clock_inc = 0
+
+    def receive(self, peer):
+        print(f"receive {peer.id} clock={peer.current_clock} msgclock={self.clock}")
+        if peer.current_clock < self.clock:
+            print(f"{peer.id} is outdated, using incoming memberslist")
+            peer.members = set(self.recipients)
+            peer.current_clock = self.clock
+            print(f"-> NEWCLOCK: {peer.current_clock}")
+        elif peer.current_clock == self.clock:
+            if peer.members != set(self.recipients):
+                print(f"{peer.id} has different members than incoming same-clock message")
+                print(f"{peer.id} resetting to incoming recipients, and increase own clock")
+                peer.members = set(self.recipients)
+                peer.current_clock = self.clock + 1
+        else:
+            print(f"{peer.id} has newer clock than incoming message")
+
+
+class Peer:
+    """A peer in a group"""
+
+    def __init__(self, relay, num):
+        self.relay = relay
+        self.id = f"p{num}"
+        self.members = set()
+        self.from2mailbox = {}
+        self.current_clock = 0
+
+    def __eq__(self, other):
+        return self.id == other.id
+
+    def __hash__(self):
+        return int(self.id[1:])
+
+    def __repr__(self):
+        clock = self.current_clock
+        return f"<Peer {self.id} members={','.join(self.members)} clock={clock}>"
+
+    def immediate_create_group(self, peers):
+        assert not self.members
+        self.members.add(self.id)
+        for peer in peers:
+            AddMemberMessage(self, member=peer.id)
+        self.relay.receive_all()
+
+
+### Tests
+
+
+def test_add_and_remove(relay):
+    p0, p1, p2, p3 = relay.make_peers(4)
+
+    # create group
+    p0.immediate_create_group([p1])
+    assert p0.members == p1.members == set([p0.id, p1.id])
+
+    # add members
+    AddMemberMessage(p0, member=p2.id)
+    AddMemberMessage(p0, member=p3.id)
+    relay.receive_all()
+    relay.assert_group_consistency()
+
+    DelMemberMessage(p3, member=p0.id)
+    relay.receive_all()
+    relay.assert_group_consistency()
+
+
+def test_concurrent_add(relay):
+    p0, p1, p2, p3 = relay.make_peers(4)
+
+    p0.immediate_create_group([p1])
+    # concurrent adding and then let base set send a chat message
+    AddMemberMessage(p1, member=p2.id)
+    AddMemberMessage(p0, member=p3.id)
+    relay.receive_all()
+
+    relay.dump("after concurrent add")
+    # only now do p0 and p1 know of each others additions
+    # so p0 or p1 needs to send another message to get consistent membership
+    ChatMessage(p0)
+    relay.receive_all()
+    relay.dump("after chatmessage")
+    relay.assert_group_consistency()
+
+
+def test_add_remove_and_stale_member_sends_chatmessage(relay):
+    p0, p1, p2, p3 = relay.make_peers(4)
+
+    p0.immediate_create_group([p1, p2, p3])
+
+    # p3 is offline and p0 deletes p2
+    DelMemberMessage(p0, member=p2.id)
+    relay.receive_all([p0, p1, p2])
+
+    # p3 sends a message with old memberlist and goes online
+    ChatMessage(p3)
+    relay.receive_all()
+    relay.assert_group_consistency()
+    ChatMessage(p0)
+    relay.receive_all()
+    assert p0.members == set(["p0", "p1", "p3"])
+
+
+def test_add_remove_and_stale_member_sends_addition(relay):
+    p0, p1, p2, p3, p4 = relay.make_peers(5)
+
+    p0.immediate_create_group([p1, p2, p3])
+
+    # p3 is offline and p0 deletes p2
+    DelMemberMessage(p0, member=p2.id)
+    relay.receive_all([p0, p1, p2])
+
+    # p3 sends a message with member addition and goes online
+    AddMemberMessage(p3, member=p4.id)
+    relay.receive_all()
+    relay.dump("after p3 is online")
+
+    # we need a chat message from a higher clock state to heal consistency
+    ChatMessage(p0)
+    relay.receive_all()
+    relay.dump("after p0 sent chatmessage")
+    relay.assert_group_consistency()
+    assert p0.members == set(["p0", "p1", "p3", "p4"])