fix: (linux-only) make sure large attachments return memory to kernel

The test fails on main with ~150MB memory remaining allocated
after sending/receiving two 20MB messages,
and stays well below when explicitely setting libc's M_MMAP_THRESHOLD.

Cargo.lock

@@ -1485,6 +1485,7 @@ dependencies = [
  "deltachat",
  "deltachat-jsonrpc",
  "futures-lite",
+ "libc",
  "log",
  "serde",
  "serde_json",

deltachat-rpc-client/tests/test_memory.py

@@ -0,0 +1,33 @@
+import os
+import sys
+
+import pytest
+
+
+def anonymous_mib(pid):
+    with open(f"/proc/{pid}/smaps_rollup") as f:
+        for line in f:
+            if line.startswith("Anonymous:"):
+                return int(line.split()[1]) // 1024
+    raise LookupError("Anonymous")
+
+
+@pytest.mark.skipif(sys.platform != "linux", reason="reads /proc")
+def test_attachment_memory_is_returned(acf, rpc, tmp_path):
+    # See also comments for `tune_malloc` in `deltachat-rpc-server/src/main.rs`
+    ac1, ac2 = acf.get_online_accounts(2)
+    chat1 = acf.get_accepted_chat(ac1, ac2)
+    chat2 = ac2.create_chat(ac1)
+    blob = tmp_path / "blob.bin"
+    blob.write_bytes(os.urandom(20 << 20))
+    before = anonymous_mib(rpc.process.pid)
+
+    for sender_chat, receiver in ((chat1, ac2), (chat2, ac1)):
+        sender_chat.send_file(str(blob))
+        event = receiver.wait_for_incoming_msg_event()
+        assert receiver.get_message_by_id(event.msg_id).get_snapshot().file_bytes == 20 << 20
+    for ac in (ac1, ac2):
+        rpc.wait_for_all_work_done(ac.id)
+
+    grown = anonymous_mib(rpc.process.pid) - before
+    assert grown < 64, f"the server kept {grown} MiB after two 20 MiB attachments"

deltachat-rpc-server/Cargo.toml

@@ -14,6 +14,7 @@ deltachat = { workspace = true }
 
 anyhow = { workspace = true }
 futures-lite = { workspace = true }
+libc = { workspace = true }
 log = { workspace = true }
 serde_json = { workspace = true }
 serde = { workspace = true, features = ["derive"] }

deltachat-rpc-server/src/main.rs

@@ -22,8 +22,32 @@ use tokio::task::JoinHandle;
 use tokio_util::sync::CancellationToken;
 use yerpc::{RpcClient, RpcSession};
 
+/// Pins Linux glibc's mmap threshold so that freed message buffers go back to the kernel.
+///
+/// See M_MMAP_THRESHOLD in <https://man7.org/linux/man-pages/man3/mallopt.3.html>:
+/// glibc by default starts with a M_MMAP_THRESHOLD threshold of 128 KiB
+/// but raises it to the size of every freed block that exceeds it,
+/// up to 32 MiB on 64-bit systems,
+/// and trims the heap only from its top end once twice that much is free.
+/// Fixating the threshold disables the adjustment: allocations at or above it
+/// that the free list cannot satisfy are mmapped and unmapped on free,
+/// at the price of the kernel zeroing each such buffer after unmap.
+/// Large message processing (allocations above 128KiB) very slightly slows
+/// down to the kernel zeroing the buffers, but it's hardly measurable,
+/// while overall process memory allocation behaviour significantly improves.
+#[cfg(all(target_os = "linux", target_env = "gnu"))]
+fn tune_malloc() {
+    unsafe {
+        libc::mallopt(libc::M_MMAP_THRESHOLD, 128 * 1024);
+    }
+}
+
+#[cfg(not(all(target_os = "linux", target_env = "gnu")))]
+fn tune_malloc() {}
+
 #[tokio::main(flavor = "multi_thread")]
 async fn main() {
+    tune_malloc();
     // Logs from `log` crate and traces from `tracing` crate
     // are configurable with `RUST_LOG` environment variable
     // and go to stderr to avoid interfering with JSON-RPC using stdout.