Mirrors/eepp
1
0
Fork 0
mirror of https://github.com/SpartanJ/eepp.git synced 2026-10-18 11:33:23 +03:00
Code Issues Packages Projects Releases Wiki Activity

test(a11y): stabilize inactive performance benchmark and CI coverage

Browse Source 
- Warm the notification workload before timing one million calls
- Collect ten fixed enabled/disabled pairs with alternating order
- Preserve the 1.01x ratio-of-medians guard without conditional retries
- Validate reported measurements and retain JSON diagnostics on failure
- Prevent D-Bus service output from contaminating benchmark artifacts
- Run native AT-SPI checks after timing failures without masking CI failure
- Add deterministic runner tests and document the measurement protocol
This commit is contained in:
Martín Lucas Golini
2026-10-14 18:12:23 -03:00
parent 110a9d46f0
commit f809e84b53
5 changed files with 277 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/eepp-linux-build-check.yml
+16 -2
View File
@@ -28,6 +28,7 @@ jobs:
wget https://cdn.ensoft.dev/eepp-assets/premake-5.0.0-beta6-linux.tar.gz
tar xvzf premake-5.0.0-beta6-linux.tar.gz
- name: Build
id: build
run: |
./premake5 --disable-static-build --with-debug-symbols gmake
cd make/linux
@@ -88,18 +89,28 @@ jobs:
name: linux-eepp-sdk-x86_64
path: projects/linux/sdk/eepp-sdk-*.tar.gz
- name: Unit Tests
id: unit_tests
run: |
cd projects/scripts
bash ./run_gdb_tests.sh
- name: Inactive Benchmark Runner Tests
run: |
/usr/bin/python3 -m unittest discover -s projects/scripts/tests \
-p test_benchmark_accessibility_inactive.py
- name: Inactive Accessibility Benchmark
timeout-minutes: 2
run: |
mkdir -p bin/unit_tests/output
benchmark_exit_code=0
# Redirect only Python's JSON, not stdout inherited by activated D-Bus services.
env -u AT_SPI_BUS_ADDRESS dbus-run-session -- projects/scripts/xvfb-run-eepp \
/usr/bin/python3 projects/scripts/benchmark_accessibility_inactive.py --json \
> bin/unit_tests/output/accessibility-inactive-benchmark.json
bash -c '/usr/bin/python3 projects/scripts/benchmark_accessibility_inactive.py --json \
> bin/unit_tests/output/accessibility-inactive-benchmark.json' || benchmark_exit_code=$?
cat bin/unit_tests/output/accessibility-inactive-benchmark.json
exit "$benchmark_exit_code"
- name: Native AT-SPI Regression Tests
# A noisy performance failure must not skip correctness coverage or turn the job green.
if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.unit_tests.outcome == 'success' }}
timeout-minutes: 2
run: |
mkdir -p bin/unit_tests/output
@@ -108,6 +119,7 @@ jobs:
> bin/unit_tests/output/atspi-regressions.log
cat bin/unit_tests/output/atspi-regressions.log
- name: Native AT-SPI Secondary Window Tests
if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.unit_tests.outcome == 'success' }}
# Closes a secondary window while the client stays connected; the primary must keep answering.
timeout-minutes: 2
run: |
@@ -117,6 +129,7 @@ jobs:
> bin/unit_tests/output/atspi-multi-window.log
cat bin/unit_tests/output/atspi-multi-window.log
- name: Native AT-SPI Deferred Policy Tests
if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.unit_tests.outcome == 'success' }}
# Changes policy from inside a native text request; teardown must wait for its reply.
timeout-minutes: 2
run: |
@@ -126,6 +139,7 @@ jobs:
> bin/unit_tests/output/atspi-deferred-policy.log
cat bin/unit_tests/output/atspi-deferred-policy.log
- name: AT-SPI Benchmark
if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.unit_tests.outcome == 'success' }}
timeout-minutes: 5
run: |
mkdir -p bin/unit_tests/output
docs/articles/ui_accessibility_architecture.md
+8
View File
@@ -563,6 +563,14 @@ or wrong.
| UIA end to end | `src/tests/windows_accessibility/main.cpp` (`eepp-windows-accessibility-tests`) | Windows CI, including the provider regressions. Under Wine it builds and finds the window, but Wine's UIA client lacks several APIs, so most end-to-end checks report `E_NOTIMPL`. |
| NSAccessibility end to end | `projects/scripts/test_macos_accessibility.swift`; `benchmark_macos_accessibility.swift` locally | macOS CI. |
The inactive benchmark compares enabled and disabled accessibility in the **same Release build**,
not a before/after branch comparison. Each process warms 100,000 `setText()` calls before timing
1,000,000 calls (roughly one second on the CI runner). It collects ten adjacent sample pairs with
alternating order, regardless of the result, and retains the ratio-of-medians notification guard
of **1.01×**. Raw samples, warmup times and failure reasons are included in JSON even when a guard
fails. Linux native correctness checks still run after a timing failure; the failed guard still
fails the CI job.
Useful tools:
- the runtime inspector's *Accessibility* tab shows exactly what a widget projects, and its *Audit*
projects/scripts/benchmark_accessibility_inactive.py
+55 -42
View File
@@ -2,6 +2,7 @@
import argparse
import json
import math
import os
from pathlib import Path
import statistics
@@ -36,6 +37,8 @@ def run_sample(args, disabled):
"--benchmark-inactive",
"--benchmark-iterations",
str(args.iterations),
"--benchmark-warmup-iterations",
str(args.warmup_iterations),
],
check=True,
capture_output=True,
@@ -45,9 +48,22 @@ def run_sample(args, disabled):
)
wall_ms = (time.perf_counter() - start) * 1000
result = json.loads(completed.stdout)
check(
result.get("iterations") == args.iterations
and result.get("warmup_iterations") == args.warmup_iterations,
"executable did not report the requested measurement and warmup iterations",
)
for key in ("initialization_us", "readiness_us", "warmup_us", "notifications_us"):
value = result.get(key)
check(
isinstance(value, (int, float)) and math.isfinite(value) and value >= 0,
f"invalid or missing {key} measurement",
)
check(result["notifications_us"] > 0, "notification measurement was zero")
result["wall_ms"] = wall_ms
result["initialization_ms"] = result.pop("initialization_us") / 1000
result["readiness_ms"] = result.pop("readiness_us") / 1000
result["warmup_ms"] = result.pop("warmup_us") / 1000
result["notifications_ms"] = result.pop("notifications_us") / 1000
return result
@@ -59,8 +75,8 @@ def median_sample(samples):
}
def collect_samples(args, first_index, count, enabled_samples, disabled_samples):
for index in range(first_index, first_index + count):
def collect_samples(args, enabled_samples, disabled_samples):
for index in range(args.samples):
# Reverse each adjacent pair so neither mode consistently benefits from running second.
modes = (True, False) if index % 2 == 0 else (False, True)
for disabled in modes:
@@ -83,61 +99,58 @@ def summarize(enabled_samples, disabled_samples):
}
def main():
def create_parser():
parser = argparse.ArgumentParser(description="Benchmark inactive accessibility overhead")
parser.add_argument("--executable", default="bin/eepp-ui-accessibility")
parser.add_argument("--iterations", type=int, default=100000)
parser.add_argument("--samples", type=int, default=5)
parser.add_argument("--confirmation-samples", type=int, default=10)
parser.add_argument("--iterations", type=int, default=1000000,
help="measured setText calls per process (roughly one second on the CI runner)")
parser.add_argument("--warmup-iterations", type=int, default=100000,
help="untimed setText calls before each measurement")
parser.add_argument("--samples", type=int, default=10,
help="fixed number of adjacent enabled/disabled sample pairs")
parser.add_argument("--timeout", type=float, default=10)
parser.add_argument("--max-initialization-ms", type=float, default=250)
parser.add_argument("--max-notification-ratio", type=float, default=1.01)
parser.add_argument("--max-wall-ratio", type=float, default=1.25)
parser.add_argument("--json", action="store_true")
args = parser.parse_args()
return parser
def main(argv=None):
args = create_parser().parse_args(argv)
check(args.iterations > 0 and args.samples > 0, "iterations and samples must be positive")
check(args.confirmation_samples >= 0, "confirmation samples must not be negative")
check(args.warmup_iterations >= 0, "warmup iterations must not be negative")
for key in ("timeout", "max_initialization_ms", "max_notification_ratio", "max_wall_ratio"):
value = getattr(args, key)
check(math.isfinite(value) and value > 0, f"{key} must be finite and positive")
enabled_samples = []
disabled_samples = []
collect_samples(args, 0, args.samples, enabled_samples, disabled_samples)
# The sample count is independent of the result: no early pass or retry-until-green.
collect_samples(args, enabled_samples, disabled_samples)
output = summarize(enabled_samples, disabled_samples)
if (
output["notification_ratio"] > args.max_notification_ratio
or output["wall_ratio"] > args.max_wall_ratio
) and args.confirmation_samples:
collect_samples(
args,
args.samples,
args.confirmation_samples,
enabled_samples,
disabled_samples,
)
output = summarize(enabled_samples, disabled_samples)
output["sample_pairs"] = args.samples
enabled = output["enabled"]
failed = (
enabled["initialization_ms"] > args.max_initialization_ms
or output["notification_ratio"] > args.max_notification_ratio
or output["wall_ratio"] > args.max_wall_ratio
)
if failed:
print(json.dumps(output, sort_keys=True), file=sys.stderr)
check(
enabled["initialization_ms"] <= args.max_initialization_ms,
"inactive native backend initialization exceeded threshold",
)
check(
output["notification_ratio"] <= args.max_notification_ratio,
"inactive accessibility notification overhead exceeded threshold",
)
check(
output["wall_ratio"] <= args.max_wall_ratio,
"inactive accessibility wall-time overhead exceeded threshold",
)
failures = []
for value, limit, message in (
(enabled["initialization_ms"], args.max_initialization_ms,
"inactive native backend initialization exceeded threshold"),
(output["notification_ratio"], args.max_notification_ratio,
"inactive accessibility notification overhead exceeded threshold"),
(output["wall_ratio"], args.max_wall_ratio,
"inactive accessibility wall-time overhead exceeded threshold"),
):
if value > limit:
failures.append(message)
output["failures"] = failures
# Preserve the complete JSON artifact even when a guard rejects this run.
if args.json:
print(json.dumps(output, sort_keys=True))
else:
print(json.dumps(output, sort_keys=True), flush=True)
elif failures:
print(json.dumps(output, sort_keys=True), file=sys.stderr)
check(not failures, "; ".join(failures))
if not args.json:
print(
"Inactive accessibility benchmark passed: "
f"initialization {enabled['initialization_ms']:.3f}ms, "
projects/scripts/tests/test_benchmark_accessibility_inactive.py
+173
View File
@@ -0,0 +1,173 @@
"""Deterministic runner tests; no graphics, D-Bus or benchmark timings are required."""
import contextlib
import importlib.util
import io
import json
from pathlib import Path
import subprocess
import unittest
from unittest import mock
SCRIPT = Path(__file__).resolve().parents[1] / "benchmark_accessibility_inactive.py"
SPEC = importlib.util.spec_from_file_location("benchmark_accessibility_inactive", SCRIPT)
benchmark = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(benchmark)
def sample(notifications_ms=1000):
return {
"initialization_ms": 0.02,
"readiness_ms": 1,
"warmup_ms": 100,
"notifications_ms": notifications_ms,
"wall_ms": notifications_ms + 100,
"iterations": 1000000,
"warmup_iterations": 100000,
}
def executable_report():
return {
"initialization_us": 20,
"readiness_us": 1000,
"warmup_us": 100000,
"notifications_us": 1000000,
"iterations": 1000000,
"warmup_iterations": 100000,
}
class InactiveBenchmarkTests(unittest.TestCase):
def test_defaults_preserve_strict_guard_with_long_fixed_workload(self):
args = benchmark.create_parser().parse_args([])
self.assertEqual(args.iterations, 1000000)
self.assertEqual(args.warmup_iterations, 100000)
self.assertEqual(args.samples, 10)
self.assertEqual(args.max_notification_ratio, 1.01)
self.assertFalse(hasattr(args, "confirmation_samples"))
def test_pairs_reverse_order_without_losing_pair_alignment(self):
args = benchmark.create_parser().parse_args(["--samples", "4"])
enabled, disabled, calls = [], [], []
def run(args, is_disabled):
calls.append(is_disabled)
return {"pair": (len(calls) - 1) // 2}
with mock.patch.object(benchmark, "run_sample", side_effect=run):
benchmark.collect_samples(args, enabled, disabled)
self.assertEqual(calls, [True, False, False, True, True, False, False, True])
self.assertEqual(enabled, [{"pair": n} for n in range(4)])
self.assertEqual(disabled, enabled)
def test_guard_still_uses_ratio_of_medians(self):
enabled = [sample(v) for v in (102, 99, 100)]
disabled = [sample(v) for v in (100, 101, 140)]
output = benchmark.summarize(enabled, disabled)
self.assertEqual(output["notification_ratio"], 100 / 101)
def test_fixed_count_on_both_pass_and_failure_and_failure_json_survives(self):
for ratio in (1.0, 1.02):
with self.subTest(ratio=ratio):
output = io.StringIO()
def run(args, disabled):
return sample(1000 if disabled else 1000 * ratio)
with mock.patch.object(benchmark, "run_sample", side_effect=run) as runner:
with contextlib.redirect_stdout(output):
if ratio > 1.01:
with self.assertRaisesRegex(RuntimeError, "notification overhead"):
benchmark.main(["--json"])
else:
benchmark.main(["--json"])
self.assertEqual(runner.call_count, 20)
report = json.loads(output.getvalue())
self.assertEqual(report["sample_pairs"], 10)
self.assertEqual(len(report["raw_enabled"]), 10)
self.assertEqual(len(report["raw_disabled"]), 10)
self.assertEqual(bool(report["failures"]), ratio > 1.01)
self.assertEqual(report["notification_ratio"], ratio)
def test_exact_threshold_is_allowed(self):
def run(args, disabled):
return sample(1000 if disabled else 1010)
with mock.patch.object(benchmark, "run_sample", side_effect=run):
with contextlib.redirect_stdout(io.StringIO()):
benchmark.main(["--json", "--samples", "2"])
def test_all_guards_remain_failures(self):
def run(args, disabled):
result = sample(1000 if disabled else 1500)
if not disabled:
result["initialization_ms"] = 251
return result
output = io.StringIO()
with mock.patch.object(benchmark, "run_sample", side_effect=run):
with contextlib.redirect_stdout(output):
with self.assertRaises(RuntimeError):
benchmark.main(["--json", "--samples", "2"])
self.assertEqual(len(json.loads(output.getvalue())["failures"]), 3)
def test_warmup_argument_and_disabled_environment_are_forwarded(self):
args = benchmark.create_parser().parse_args([])
completed = subprocess.CompletedProcess([], 0, json.dumps(executable_report()), "")
for disabled in (False, True):
with self.subTest(disabled=disabled):
with mock.patch.dict(benchmark.os.environ, {"EEPP_DISABLE_ACCESSIBILITY": "1"}):
with mock.patch.object(benchmark.subprocess, "run", return_value=completed) as run:
with mock.patch.object(benchmark.time, "perf_counter", side_effect=(10, 11.25)):
result = benchmark.run_sample(args, disabled)
command = run.call_args.args[0]
self.assertEqual(command[-2:], ["--benchmark-warmup-iterations", "100000"])
self.assertEqual("EEPP_DISABLE_ACCESSIBILITY" in run.call_args.kwargs["env"], disabled)
self.assertTrue(run.call_args.kwargs["check"])
self.assertEqual(result["warmup_ms"], 100)
self.assertEqual(result["wall_ms"], 1250)
def test_stale_binary_or_invalid_duration_is_rejected(self):
args = benchmark.create_parser().parse_args([])
cases = [
("warmup_iterations", None),
("iterations", 100000),
("warmup_us", None),
("notifications_us", 0),
("notifications_us", float("nan")),
("initialization_us", float("inf")),
("readiness_us", -1),
]
for key, value in cases:
with self.subTest(key=key, value=value):
report = executable_report()
report[key] = value
completed = subprocess.CompletedProcess([], 0, json.dumps(report), "")
with mock.patch.object(benchmark.subprocess, "run", return_value=completed):
with self.assertRaises(RuntimeError):
benchmark.run_sample(args, False)
def test_invalid_arguments_do_not_launch_a_process(self):
for option, value in (
("--iterations", "0"), ("--samples", "0"),
("--warmup-iterations", "-1"), ("--timeout", "0"),
("--max-notification-ratio", "nan"), ("--max-wall-ratio", "inf"),
):
with self.subTest(option=option):
with mock.patch.object(benchmark, "run_sample") as run:
with self.assertRaises(RuntimeError):
benchmark.main([option, value])
run.assert_not_called()
def test_failed_executable_is_not_turned_into_a_timing_result(self):
args = benchmark.create_parser().parse_args([])
with mock.patch.object(benchmark.subprocess, "run",
side_effect=subprocess.CalledProcessError(1, "benchmark")):
with self.assertRaises(subprocess.CalledProcessError):
benchmark.run_sample(args, False)
if __name__ == "__main__":
unittest.main()
src/examples/ui_accessibility/ui_accessibility.cpp
+25 -4
View File
@@ -19,7 +19,8 @@ EE_MAIN_FUNC int main( int argc, char** argv ) {
bool testPasswordMode = false;
bool testHTML = false;
size_t benchmarkItems = 1000;
size_t benchmarkIterations = 100000;
size_t benchmarkIterations = 1000000;
size_t benchmarkWarmupIterations = 100000;
for ( int i = 1; i < argc; ++i ) {
multiWindow |= std::string_view( argv[i] ) == "--multi-window";
closePrimary |= std::string_view( argv[i] ) == "--close-primary";
@@ -35,6 +36,8 @@ EE_MAIN_FUNC int main( int argc, char** argv ) {
benchmarkItems = std::max<size_t>( 1, std::strtoull( argv[++i], nullptr, 10 ) );
if ( std::string_view( argv[i] ) == "--benchmark-iterations" && i + 1 < argc )
benchmarkIterations = std::max<size_t>( 1, std::strtoull( argv[++i], nullptr, 10 ) );
if ( std::string_view( argv[i] ) == "--benchmark-warmup-iterations" && i + 1 < argc )
benchmarkWarmupIterations = std::strtoull( argv[++i], nullptr, 10 );
}
UIApplication::Settings settings;
settings.threadPool = ThreadPool::createShared( 1 );
@@ -163,18 +166,36 @@ EE_MAIN_FUNC int main( int argc, char** argv ) {
}
}
const auto readinessUs = clock.getElapsedTime().asMicroseconds();
if ( !manager->isBackendInitializationComplete() ) {
std::cerr << "Inactive accessibility benchmark backend initialization timed out\n";
return EXIT_FAILURE;
}
if ( manager->hasActiveNativeClients() ) {
std::cerr << "Inactive accessibility benchmark requires no attached native clients\n";
return EXIT_FAILURE;
}
// Warm the same text/layout/event path before measuring it, in both modes. No drawing
// or event-loop pumping is added to the timed workload.
const auto runNotifications = [status]( size_t iterations ) {
for ( size_t i = 0; i < iterations; ++i )
status->setText( i & 1 ? "Ready" : "Idle" );
};
clock.restart();
for ( size_t i = 0; i < benchmarkIterations; ++i )
status->setText( i & 1 ? "Ready" : "Idle" );
runNotifications( benchmarkWarmupIterations );
const auto warmupUs = clock.getElapsedTime().asMicroseconds();
clock.restart();
runNotifications( benchmarkIterations );
const auto notificationsUs = clock.getElapsedTime().asMicroseconds();
if ( manager->hasActiveNativeClients() ) {
std::cerr << "A native client connected during the inactive accessibility benchmark\n";
return EXIT_FAILURE;
}
std::cout << "{\"initialization_us\":" << initializationUs
<< ",\"readiness_us\":" << readinessUs
<< ",\"iterations\":" << benchmarkIterations
<< ",\"notifications_us\":" << notificationsUs << "}\n";
<< ",\"warmup_iterations\":" << benchmarkWarmupIterations
<< ",\"warmup_us\":" << warmupUs << ",\"notifications_us\":" << notificationsUs
<< "}\n";
return EXIT_SUCCESS;
}
content->find<UIPushButton>( "save" )->onClick(