From f809e84b536a9eedf653abd5bc30fecebe246e26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mart=C3=ADn=20Lucas=20Golini?= Date: Wed, 14 Oct 2026 18:12:23 -0300 Subject: [PATCH] test(a11y): stabilize inactive performance benchmark and CI coverage - Warm the notification workload before timing one million calls - Collect ten fixed enabled/disabled pairs with alternating order - Preserve the 1.01x ratio-of-medians guard without conditional retries - Validate reported measurements and retain JSON diagnostics on failure - Prevent D-Bus service output from contaminating benchmark artifacts - Run native AT-SPI checks after timing failures without masking CI failure - Add deterministic runner tests and document the measurement protocol --- .github/workflows/eepp-linux-build-check.yml | 18 +- .../articles/ui_accessibility_architecture.md | 8 + .../benchmark_accessibility_inactive.py | 97 +++++----- .../test_benchmark_accessibility_inactive.py | 173 ++++++++++++++++++ .../ui_accessibility/ui_accessibility.cpp | 29 ++- 5 files changed, 277 insertions(+), 48 deletions(-) create mode 100644 projects/scripts/tests/test_benchmark_accessibility_inactive.py diff --git a/.github/workflows/eepp-linux-build-check.yml b/.github/workflows/eepp-linux-build-check.yml index c00cb6338..e382edb74 100644 --- a/.github/workflows/eepp-linux-build-check.yml +++ b/.github/workflows/eepp-linux-build-check.yml @@ -28,6 +28,7 @@ jobs: wget https://cdn.ensoft.dev/eepp-assets/premake-5.0.0-beta6-linux.tar.gz tar xvzf premake-5.0.0-beta6-linux.tar.gz - name: Build + id: build run: | ./premake5 --disable-static-build --with-debug-symbols gmake cd make/linux @@ -88,18 +89,28 @@ jobs: name: linux-eepp-sdk-x86_64 path: projects/linux/sdk/eepp-sdk-*.tar.gz - name: Unit Tests + id: unit_tests run: | cd projects/scripts bash ./run_gdb_tests.sh + - name: Inactive Benchmark Runner Tests + run: | + /usr/bin/python3 -m unittest discover -s projects/scripts/tests \ + -p test_benchmark_accessibility_inactive.py - name: Inactive Accessibility Benchmark timeout-minutes: 2 run: | mkdir -p bin/unit_tests/output + benchmark_exit_code=0 + # Redirect only Python's JSON, not stdout inherited by activated D-Bus services. env -u AT_SPI_BUS_ADDRESS dbus-run-session -- projects/scripts/xvfb-run-eepp \ - /usr/bin/python3 projects/scripts/benchmark_accessibility_inactive.py --json \ - > bin/unit_tests/output/accessibility-inactive-benchmark.json + bash -c '/usr/bin/python3 projects/scripts/benchmark_accessibility_inactive.py --json \ + > bin/unit_tests/output/accessibility-inactive-benchmark.json' || benchmark_exit_code=$? cat bin/unit_tests/output/accessibility-inactive-benchmark.json + exit "$benchmark_exit_code" - name: Native AT-SPI Regression Tests + # A noisy performance failure must not skip correctness coverage or turn the job green. + if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.unit_tests.outcome == 'success' }} timeout-minutes: 2 run: | mkdir -p bin/unit_tests/output @@ -108,6 +119,7 @@ jobs: > bin/unit_tests/output/atspi-regressions.log cat bin/unit_tests/output/atspi-regressions.log - name: Native AT-SPI Secondary Window Tests + if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.unit_tests.outcome == 'success' }} # Closes a secondary window while the client stays connected; the primary must keep answering. timeout-minutes: 2 run: | @@ -117,6 +129,7 @@ jobs: > bin/unit_tests/output/atspi-multi-window.log cat bin/unit_tests/output/atspi-multi-window.log - name: Native AT-SPI Deferred Policy Tests + if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.unit_tests.outcome == 'success' }} # Changes policy from inside a native text request; teardown must wait for its reply. timeout-minutes: 2 run: | @@ -126,6 +139,7 @@ jobs: > bin/unit_tests/output/atspi-deferred-policy.log cat bin/unit_tests/output/atspi-deferred-policy.log - name: AT-SPI Benchmark + if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.unit_tests.outcome == 'success' }} timeout-minutes: 5 run: | mkdir -p bin/unit_tests/output diff --git a/docs/articles/ui_accessibility_architecture.md b/docs/articles/ui_accessibility_architecture.md index dc671c656..e22c60cb6 100644 --- a/docs/articles/ui_accessibility_architecture.md +++ b/docs/articles/ui_accessibility_architecture.md @@ -563,6 +563,14 @@ or wrong. | UIA end to end | `src/tests/windows_accessibility/main.cpp` (`eepp-windows-accessibility-tests`) | Windows CI, including the provider regressions. Under Wine it builds and finds the window, but Wine's UIA client lacks several APIs, so most end-to-end checks report `E_NOTIMPL`. | | NSAccessibility end to end | `projects/scripts/test_macos_accessibility.swift`; `benchmark_macos_accessibility.swift` locally | macOS CI. | +The inactive benchmark compares enabled and disabled accessibility in the **same Release build**, +not a before/after branch comparison. Each process warms 100,000 `setText()` calls before timing +1,000,000 calls (roughly one second on the CI runner). It collects ten adjacent sample pairs with +alternating order, regardless of the result, and retains the ratio-of-medians notification guard +of **1.01×**. Raw samples, warmup times and failure reasons are included in JSON even when a guard +fails. Linux native correctness checks still run after a timing failure; the failed guard still +fails the CI job. + Useful tools: - the runtime inspector's *Accessibility* tab shows exactly what a widget projects, and its *Audit* diff --git a/projects/scripts/benchmark_accessibility_inactive.py b/projects/scripts/benchmark_accessibility_inactive.py index d6b382d74..6f1e9969f 100644 --- a/projects/scripts/benchmark_accessibility_inactive.py +++ b/projects/scripts/benchmark_accessibility_inactive.py @@ -2,6 +2,7 @@ import argparse import json +import math import os from pathlib import Path import statistics @@ -36,6 +37,8 @@ def run_sample(args, disabled): "--benchmark-inactive", "--benchmark-iterations", str(args.iterations), + "--benchmark-warmup-iterations", + str(args.warmup_iterations), ], check=True, capture_output=True, @@ -45,9 +48,22 @@ def run_sample(args, disabled): ) wall_ms = (time.perf_counter() - start) * 1000 result = json.loads(completed.stdout) + check( + result.get("iterations") == args.iterations + and result.get("warmup_iterations") == args.warmup_iterations, + "executable did not report the requested measurement and warmup iterations", + ) + for key in ("initialization_us", "readiness_us", "warmup_us", "notifications_us"): + value = result.get(key) + check( + isinstance(value, (int, float)) and math.isfinite(value) and value >= 0, + f"invalid or missing {key} measurement", + ) + check(result["notifications_us"] > 0, "notification measurement was zero") result["wall_ms"] = wall_ms result["initialization_ms"] = result.pop("initialization_us") / 1000 result["readiness_ms"] = result.pop("readiness_us") / 1000 + result["warmup_ms"] = result.pop("warmup_us") / 1000 result["notifications_ms"] = result.pop("notifications_us") / 1000 return result @@ -59,8 +75,8 @@ def median_sample(samples): } -def collect_samples(args, first_index, count, enabled_samples, disabled_samples): - for index in range(first_index, first_index + count): +def collect_samples(args, enabled_samples, disabled_samples): + for index in range(args.samples): # Reverse each adjacent pair so neither mode consistently benefits from running second. modes = (True, False) if index % 2 == 0 else (False, True) for disabled in modes: @@ -83,61 +99,58 @@ def summarize(enabled_samples, disabled_samples): } -def main(): +def create_parser(): parser = argparse.ArgumentParser(description="Benchmark inactive accessibility overhead") parser.add_argument("--executable", default="bin/eepp-ui-accessibility") - parser.add_argument("--iterations", type=int, default=100000) - parser.add_argument("--samples", type=int, default=5) - parser.add_argument("--confirmation-samples", type=int, default=10) + parser.add_argument("--iterations", type=int, default=1000000, + help="measured setText calls per process (roughly one second on the CI runner)") + parser.add_argument("--warmup-iterations", type=int, default=100000, + help="untimed setText calls before each measurement") + parser.add_argument("--samples", type=int, default=10, + help="fixed number of adjacent enabled/disabled sample pairs") parser.add_argument("--timeout", type=float, default=10) parser.add_argument("--max-initialization-ms", type=float, default=250) parser.add_argument("--max-notification-ratio", type=float, default=1.01) parser.add_argument("--max-wall-ratio", type=float, default=1.25) parser.add_argument("--json", action="store_true") - args = parser.parse_args() + return parser + + +def main(argv=None): + args = create_parser().parse_args(argv) check(args.iterations > 0 and args.samples > 0, "iterations and samples must be positive") - check(args.confirmation_samples >= 0, "confirmation samples must not be negative") + check(args.warmup_iterations >= 0, "warmup iterations must not be negative") + for key in ("timeout", "max_initialization_ms", "max_notification_ratio", "max_wall_ratio"): + value = getattr(args, key) + check(math.isfinite(value) and value > 0, f"{key} must be finite and positive") enabled_samples = [] disabled_samples = [] - collect_samples(args, 0, args.samples, enabled_samples, disabled_samples) + # The sample count is independent of the result: no early pass or retry-until-green. + collect_samples(args, enabled_samples, disabled_samples) output = summarize(enabled_samples, disabled_samples) - if ( - output["notification_ratio"] > args.max_notification_ratio - or output["wall_ratio"] > args.max_wall_ratio - ) and args.confirmation_samples: - collect_samples( - args, - args.samples, - args.confirmation_samples, - enabled_samples, - disabled_samples, - ) - output = summarize(enabled_samples, disabled_samples) + output["sample_pairs"] = args.samples enabled = output["enabled"] - failed = ( - enabled["initialization_ms"] > args.max_initialization_ms - or output["notification_ratio"] > args.max_notification_ratio - or output["wall_ratio"] > args.max_wall_ratio - ) - if failed: - print(json.dumps(output, sort_keys=True), file=sys.stderr) - check( - enabled["initialization_ms"] <= args.max_initialization_ms, - "inactive native backend initialization exceeded threshold", - ) - check( - output["notification_ratio"] <= args.max_notification_ratio, - "inactive accessibility notification overhead exceeded threshold", - ) - check( - output["wall_ratio"] <= args.max_wall_ratio, - "inactive accessibility wall-time overhead exceeded threshold", - ) + failures = [] + for value, limit, message in ( + (enabled["initialization_ms"], args.max_initialization_ms, + "inactive native backend initialization exceeded threshold"), + (output["notification_ratio"], args.max_notification_ratio, + "inactive accessibility notification overhead exceeded threshold"), + (output["wall_ratio"], args.max_wall_ratio, + "inactive accessibility wall-time overhead exceeded threshold"), + ): + if value > limit: + failures.append(message) + output["failures"] = failures + # Preserve the complete JSON artifact even when a guard rejects this run. if args.json: - print(json.dumps(output, sort_keys=True)) - else: + print(json.dumps(output, sort_keys=True), flush=True) + elif failures: + print(json.dumps(output, sort_keys=True), file=sys.stderr) + check(not failures, "; ".join(failures)) + if not args.json: print( "Inactive accessibility benchmark passed: " f"initialization {enabled['initialization_ms']:.3f}ms, " diff --git a/projects/scripts/tests/test_benchmark_accessibility_inactive.py b/projects/scripts/tests/test_benchmark_accessibility_inactive.py new file mode 100644 index 000000000..7d14b7e3f --- /dev/null +++ b/projects/scripts/tests/test_benchmark_accessibility_inactive.py @@ -0,0 +1,173 @@ +"""Deterministic runner tests; no graphics, D-Bus or benchmark timings are required.""" + +import contextlib +import importlib.util +import io +import json +from pathlib import Path +import subprocess +import unittest +from unittest import mock + + +SCRIPT = Path(__file__).resolve().parents[1] / "benchmark_accessibility_inactive.py" +SPEC = importlib.util.spec_from_file_location("benchmark_accessibility_inactive", SCRIPT) +benchmark = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(benchmark) + + +def sample(notifications_ms=1000): + return { + "initialization_ms": 0.02, + "readiness_ms": 1, + "warmup_ms": 100, + "notifications_ms": notifications_ms, + "wall_ms": notifications_ms + 100, + "iterations": 1000000, + "warmup_iterations": 100000, + } + + +def executable_report(): + return { + "initialization_us": 20, + "readiness_us": 1000, + "warmup_us": 100000, + "notifications_us": 1000000, + "iterations": 1000000, + "warmup_iterations": 100000, + } + + +class InactiveBenchmarkTests(unittest.TestCase): + def test_defaults_preserve_strict_guard_with_long_fixed_workload(self): + args = benchmark.create_parser().parse_args([]) + self.assertEqual(args.iterations, 1000000) + self.assertEqual(args.warmup_iterations, 100000) + self.assertEqual(args.samples, 10) + self.assertEqual(args.max_notification_ratio, 1.01) + self.assertFalse(hasattr(args, "confirmation_samples")) + + def test_pairs_reverse_order_without_losing_pair_alignment(self): + args = benchmark.create_parser().parse_args(["--samples", "4"]) + enabled, disabled, calls = [], [], [] + + def run(args, is_disabled): + calls.append(is_disabled) + return {"pair": (len(calls) - 1) // 2} + + with mock.patch.object(benchmark, "run_sample", side_effect=run): + benchmark.collect_samples(args, enabled, disabled) + self.assertEqual(calls, [True, False, False, True, True, False, False, True]) + self.assertEqual(enabled, [{"pair": n} for n in range(4)]) + self.assertEqual(disabled, enabled) + + def test_guard_still_uses_ratio_of_medians(self): + enabled = [sample(v) for v in (102, 99, 100)] + disabled = [sample(v) for v in (100, 101, 140)] + output = benchmark.summarize(enabled, disabled) + self.assertEqual(output["notification_ratio"], 100 / 101) + + def test_fixed_count_on_both_pass_and_failure_and_failure_json_survives(self): + for ratio in (1.0, 1.02): + with self.subTest(ratio=ratio): + output = io.StringIO() + + def run(args, disabled): + return sample(1000 if disabled else 1000 * ratio) + + with mock.patch.object(benchmark, "run_sample", side_effect=run) as runner: + with contextlib.redirect_stdout(output): + if ratio > 1.01: + with self.assertRaisesRegex(RuntimeError, "notification overhead"): + benchmark.main(["--json"]) + else: + benchmark.main(["--json"]) + self.assertEqual(runner.call_count, 20) + report = json.loads(output.getvalue()) + self.assertEqual(report["sample_pairs"], 10) + self.assertEqual(len(report["raw_enabled"]), 10) + self.assertEqual(len(report["raw_disabled"]), 10) + self.assertEqual(bool(report["failures"]), ratio > 1.01) + self.assertEqual(report["notification_ratio"], ratio) + + def test_exact_threshold_is_allowed(self): + def run(args, disabled): + return sample(1000 if disabled else 1010) + + with mock.patch.object(benchmark, "run_sample", side_effect=run): + with contextlib.redirect_stdout(io.StringIO()): + benchmark.main(["--json", "--samples", "2"]) + + def test_all_guards_remain_failures(self): + def run(args, disabled): + result = sample(1000 if disabled else 1500) + if not disabled: + result["initialization_ms"] = 251 + return result + + output = io.StringIO() + with mock.patch.object(benchmark, "run_sample", side_effect=run): + with contextlib.redirect_stdout(output): + with self.assertRaises(RuntimeError): + benchmark.main(["--json", "--samples", "2"]) + self.assertEqual(len(json.loads(output.getvalue())["failures"]), 3) + + def test_warmup_argument_and_disabled_environment_are_forwarded(self): + args = benchmark.create_parser().parse_args([]) + completed = subprocess.CompletedProcess([], 0, json.dumps(executable_report()), "") + for disabled in (False, True): + with self.subTest(disabled=disabled): + with mock.patch.dict(benchmark.os.environ, {"EEPP_DISABLE_ACCESSIBILITY": "1"}): + with mock.patch.object(benchmark.subprocess, "run", return_value=completed) as run: + with mock.patch.object(benchmark.time, "perf_counter", side_effect=(10, 11.25)): + result = benchmark.run_sample(args, disabled) + command = run.call_args.args[0] + self.assertEqual(command[-2:], ["--benchmark-warmup-iterations", "100000"]) + self.assertEqual("EEPP_DISABLE_ACCESSIBILITY" in run.call_args.kwargs["env"], disabled) + self.assertTrue(run.call_args.kwargs["check"]) + self.assertEqual(result["warmup_ms"], 100) + self.assertEqual(result["wall_ms"], 1250) + + def test_stale_binary_or_invalid_duration_is_rejected(self): + args = benchmark.create_parser().parse_args([]) + cases = [ + ("warmup_iterations", None), + ("iterations", 100000), + ("warmup_us", None), + ("notifications_us", 0), + ("notifications_us", float("nan")), + ("initialization_us", float("inf")), + ("readiness_us", -1), + ] + for key, value in cases: + with self.subTest(key=key, value=value): + report = executable_report() + report[key] = value + completed = subprocess.CompletedProcess([], 0, json.dumps(report), "") + with mock.patch.object(benchmark.subprocess, "run", return_value=completed): + with self.assertRaises(RuntimeError): + benchmark.run_sample(args, False) + + def test_invalid_arguments_do_not_launch_a_process(self): + for option, value in ( + ("--iterations", "0"), ("--samples", "0"), + ("--warmup-iterations", "-1"), ("--timeout", "0"), + ("--max-notification-ratio", "nan"), ("--max-wall-ratio", "inf"), + ): + with self.subTest(option=option): + with mock.patch.object(benchmark, "run_sample") as run: + with self.assertRaises(RuntimeError): + benchmark.main([option, value]) + run.assert_not_called() + + def test_failed_executable_is_not_turned_into_a_timing_result(self): + args = benchmark.create_parser().parse_args([]) + with mock.patch.object(benchmark.subprocess, "run", + side_effect=subprocess.CalledProcessError(1, "benchmark")): + with self.assertRaises(subprocess.CalledProcessError): + benchmark.run_sample(args, False) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/examples/ui_accessibility/ui_accessibility.cpp b/src/examples/ui_accessibility/ui_accessibility.cpp index 22dcd67db..8046d8aec 100644 --- a/src/examples/ui_accessibility/ui_accessibility.cpp +++ b/src/examples/ui_accessibility/ui_accessibility.cpp @@ -19,7 +19,8 @@ EE_MAIN_FUNC int main( int argc, char** argv ) { bool testPasswordMode = false; bool testHTML = false; size_t benchmarkItems = 1000; - size_t benchmarkIterations = 100000; + size_t benchmarkIterations = 1000000; + size_t benchmarkWarmupIterations = 100000; for ( int i = 1; i < argc; ++i ) { multiWindow |= std::string_view( argv[i] ) == "--multi-window"; closePrimary |= std::string_view( argv[i] ) == "--close-primary"; @@ -35,6 +36,8 @@ EE_MAIN_FUNC int main( int argc, char** argv ) { benchmarkItems = std::max( 1, std::strtoull( argv[++i], nullptr, 10 ) ); if ( std::string_view( argv[i] ) == "--benchmark-iterations" && i + 1 < argc ) benchmarkIterations = std::max( 1, std::strtoull( argv[++i], nullptr, 10 ) ); + if ( std::string_view( argv[i] ) == "--benchmark-warmup-iterations" && i + 1 < argc ) + benchmarkWarmupIterations = std::strtoull( argv[++i], nullptr, 10 ); } UIApplication::Settings settings; settings.threadPool = ThreadPool::createShared( 1 ); @@ -163,18 +166,36 @@ EE_MAIN_FUNC int main( int argc, char** argv ) { } } const auto readinessUs = clock.getElapsedTime().asMicroseconds(); + if ( !manager->isBackendInitializationComplete() ) { + std::cerr << "Inactive accessibility benchmark backend initialization timed out\n"; + return EXIT_FAILURE; + } if ( manager->hasActiveNativeClients() ) { std::cerr << "Inactive accessibility benchmark requires no attached native clients\n"; return EXIT_FAILURE; } + // Warm the same text/layout/event path before measuring it, in both modes. No drawing + // or event-loop pumping is added to the timed workload. + const auto runNotifications = [status]( size_t iterations ) { + for ( size_t i = 0; i < iterations; ++i ) + status->setText( i & 1 ? "Ready" : "Idle" ); + }; clock.restart(); - for ( size_t i = 0; i < benchmarkIterations; ++i ) - status->setText( i & 1 ? "Ready" : "Idle" ); + runNotifications( benchmarkWarmupIterations ); + const auto warmupUs = clock.getElapsedTime().asMicroseconds(); + clock.restart(); + runNotifications( benchmarkIterations ); const auto notificationsUs = clock.getElapsedTime().asMicroseconds(); + if ( manager->hasActiveNativeClients() ) { + std::cerr << "A native client connected during the inactive accessibility benchmark\n"; + return EXIT_FAILURE; + } std::cout << "{\"initialization_us\":" << initializationUs << ",\"readiness_us\":" << readinessUs << ",\"iterations\":" << benchmarkIterations - << ",\"notifications_us\":" << notificationsUs << "}\n"; + << ",\"warmup_iterations\":" << benchmarkWarmupIterations + << ",\"warmup_us\":" << warmupUs << ",\"notifications_us\":" << notificationsUs + << "}\n"; return EXIT_SUCCESS; } content->find( "save" )->onClick(