test(a11y): stabilize inactive performance benchmark and CI coverage

- Warm the notification workload before timing one million calls
- Collect ten fixed enabled/disabled pairs with alternating order
- Preserve the 1.01x ratio-of-medians guard without conditional retries
- Validate reported measurements and retain JSON diagnostics on failure
- Prevent D-Bus service output from contaminating benchmark artifacts
- Run native AT-SPI checks after timing failures without masking CI failure
- Add deterministic runner tests and document the measurement protocol
This commit is contained in:
Martín Lucas Golini
2026-10-14 18:12:23 -03:00
parent 110a9d46f0
commit f809e84b53
5 changed files with 277 additions and 48 deletions
+16 -2
View File
@@ -28,6 +28,7 @@ jobs:
wget https://cdn.ensoft.dev/eepp-assets/premake-5.0.0-beta6-linux.tar.gz
tar xvzf premake-5.0.0-beta6-linux.tar.gz
- name: Build
id: build
run: |
./premake5 --disable-static-build --with-debug-symbols gmake
cd make/linux
@@ -88,18 +89,28 @@ jobs:
name: linux-eepp-sdk-x86_64
path: projects/linux/sdk/eepp-sdk-*.tar.gz
- name: Unit Tests
id: unit_tests
run: |
cd projects/scripts
bash ./run_gdb_tests.sh
- name: Inactive Benchmark Runner Tests
run: |
/usr/bin/python3 -m unittest discover -s projects/scripts/tests \
-p test_benchmark_accessibility_inactive.py
- name: Inactive Accessibility Benchmark
timeout-minutes: 2
run: |
mkdir -p bin/unit_tests/output
benchmark_exit_code=0
# Redirect only Python's JSON, not stdout inherited by activated D-Bus services.
env -u AT_SPI_BUS_ADDRESS dbus-run-session -- projects/scripts/xvfb-run-eepp \
/usr/bin/python3 projects/scripts/benchmark_accessibility_inactive.py --json \
> bin/unit_tests/output/accessibility-inactive-benchmark.json
bash -c '/usr/bin/python3 projects/scripts/benchmark_accessibility_inactive.py --json \
> bin/unit_tests/output/accessibility-inactive-benchmark.json' || benchmark_exit_code=$?
cat bin/unit_tests/output/accessibility-inactive-benchmark.json
exit "$benchmark_exit_code"
- name: Native AT-SPI Regression Tests
# A noisy performance failure must not skip correctness coverage or turn the job green.
if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.unit_tests.outcome == 'success' }}
timeout-minutes: 2
run: |
mkdir -p bin/unit_tests/output
@@ -108,6 +119,7 @@ jobs:
> bin/unit_tests/output/atspi-regressions.log
cat bin/unit_tests/output/atspi-regressions.log
- name: Native AT-SPI Secondary Window Tests
if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.unit_tests.outcome == 'success' }}
# Closes a secondary window while the client stays connected; the primary must keep answering.
timeout-minutes: 2
run: |
@@ -117,6 +129,7 @@ jobs:
> bin/unit_tests/output/atspi-multi-window.log
cat bin/unit_tests/output/atspi-multi-window.log
- name: Native AT-SPI Deferred Policy Tests
if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.unit_tests.outcome == 'success' }}
# Changes policy from inside a native text request; teardown must wait for its reply.
timeout-minutes: 2
run: |
@@ -126,6 +139,7 @@ jobs:
> bin/unit_tests/output/atspi-deferred-policy.log
cat bin/unit_tests/output/atspi-deferred-policy.log
- name: AT-SPI Benchmark
if: ${{ !cancelled() && steps.build.outcome == 'success' && steps.unit_tests.outcome == 'success' }}
timeout-minutes: 5
run: |
mkdir -p bin/unit_tests/output