From 3a3c36ac7142e8b225dc495e8cb2b04bb2f0c59d Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Tue, 18 Aug 2026 11:14:02 +0200 Subject: [PATCH] tests: replace CLI-dependent xfail/skipif conditions --- .../esp_event/test_apps/pytest_esp_event.py | 16 +++--- .../contribute/esp-idf-tests-with-pytest.rst | 24 ++++++++- .../contribute/esp-idf-tests-with-pytest.rst | 24 ++++++++- .../esp_http_client/pytest_esp_http_client.py | 2 +- tools/test_apps/system/panic/pytest_panic.py | 52 +++++++++++-------- 5 files changed, 80 insertions(+), 38 deletions(-) diff --git a/components/esp_event/test_apps/pytest_esp_event.py b/components/esp_event/test_apps/pytest_esp_event.py index 8b1a6dd60cd..19a4cc752c9 100644 --- a/components/esp_event/test_apps/pytest_esp_event.py +++ b/components/esp_event/test_apps/pytest_esp_event.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 import pytest from pytest_embedded import Dut @@ -20,18 +20,14 @@ def test_esp_event(dut: Dut) -> None: @pytest.mark.qemu -@pytest.mark.xfail( - 'config.getvalue("target") == "esp32c3"', - reason='Unstable on QEMU, needs investigation', -) -@pytest.mark.parametrize( - 'config', +@idf_parametrize( + 'config,target,markers', [ - 'defaults', + ('defaults', 'esp32'), + ('defaults', 'esp32c3', (pytest.mark.xfail(reason='Unstable on QEMU, needs investigation'),)), ], - indirect=True, + indirect=['config', 'target'], ) -@idf_parametrize('target', ['esp32', 'esp32c3'], indirect=['target']) def test_esp_event_qemu(dut: Dut) -> None: for case in dut.test_menu: if 'qemu-ignore' not in case.groups and not case.is_ignored and case.type == 'normal': diff --git a/docs/en/contribute/esp-idf-tests-with-pytest.rst b/docs/en/contribute/esp-idf-tests-with-pytest.rst index 75ab632e7e8..420ee704636 100644 --- a/docs/en/contribute/esp-idf-tests-with-pytest.rst +++ b/docs/en/contribute/esp-idf-tests-with-pytest.rst @@ -750,15 +750,35 @@ Sometimes, a test can consistently fail for the following reasons: Now you may mark this test case with marker `xfail `__ with a user-friendly readable reason. +.. attention:: + + Avoid using a string condition, e.g., ``@pytest.mark.xfail('config.getvalue("target") == "esp32s2"', ...)``. This condition is evaluated against the session-level CLI ``--target`` option, not against the actual target resolved for a given parametrized test instance, so it can silently mismatch when the same test is parametrized over multiple targets and produce unreliable results. + + Instead, attach the ``xfail`` marker directly to the specific parametrized value passed to ``idf_parametrize`` (see :ref:`Same App With Different Running Environments `), so the marker is tied to that value at collection time. + This code example is taken from :idf_file:`pytest_panic.py ` .. code-block:: python - @pytest.mark.xfail('config.getvalue("target") == "esp32s2"', reason='raised IllegalInstruction instead') - def test_cache_error(dut: PanicTestDut, config: str, test_func_name: str) -> None: + @pytest.mark.generic + @idf_parametrize( + 'config,target,markers', + [ + ( + 'memprot_esp32s2', + 'esp32s2', + pytest.mark.xfail(reason='Incorrect panic reason may be observed', run=False), + ), + ('memprot_esp32c3', 'esp32c3'), + ], + indirect=['config', 'target'], + ) + def test_cache_error(dut: PanicTestDut, test_func_name: str) -> None: This marker means that test is a known failure on the ESP32-S2. +If a test is only ever parametrized with a single target, there is no ambiguity, and you can use an unconditional ``@pytest.mark.xfail(reason=..., run=False)`` instead. + Mark Nightly Run Test Cases --------------------------- diff --git a/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst b/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst index af9b59e8740..7eb608d34cd 100644 --- a/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst +++ b/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst @@ -750,15 +750,35 @@ Pytest 使用技巧 可使用 `xfail `__ marker 来标记此测试用例，并写出原因。 +.. attention:: + + 请避免使用字符串条件，例如 ``@pytest.mark.xfail('config.getvalue("target") == "esp32s2"', ...)``。该条件是在运行时根据 session 级别的 CLI ``--target`` 参数求值的，而不是针对某个具体参数化测试实例实际解析出的 target 值，因此当同一测试针对多个 target 参数化时，该条件可能被错误匹配，导致结果不可靠。 + + 推荐做法是通过 ``idf_parametrize`` 传入的具体参数化取值直接附加 ``xfail`` marker，更复杂的示例请参阅 :ref:`在不同运行环境中运行相同的应用程序 ` 小节，这样该 marker 在 collection 阶段就与该取值绑定。 + 以下代码来自 :idf_file:`pytest_panic.py `。 .. code-block:: python - @pytest.mark.xfail('config.getvalue("target") == "esp32s2"', reason='raised IllegalInstruction instead') - def test_cache_error(dut: PanicTestDut, config: str, test_func_name: str) -> None: + @pytest.mark.generic + @idf_parametrize( + 'config,target,markers', + [ + ( + 'memprot_esp32s2', + 'esp32s2', + pytest.mark.xfail(reason='Incorrect panic reason may be observed', run=False), + ), + ('memprot_esp32c3', 'esp32c3'), + ], + indirect=['config', 'target'], + ) + def test_cache_error(dut: PanicTestDut, test_func_name: str) -> None: 这一 marker 表示该测试在 ESP32-S2 上是一个已知失败。 +如果一个测试始终只针对单个 target 参数化，则不存在歧义，此时可直接使用无条件的 ``@pytest.mark.xfail(reason=..., run=False)``。 + 标记夜间运行的测试用例 --------------------------- diff --git a/examples/protocols/esp_http_client/pytest_esp_http_client.py b/examples/protocols/esp_http_client/pytest_esp_http_client.py index f5d69b26180..2f6a86c93fb 100644 --- a/examples/protocols/esp_http_client/pytest_esp_http_client.py +++ b/examples/protocols/esp_http_client/pytest_esp_http_client.py @@ -108,7 +108,7 @@ def test_examples_protocol_esp_http_client_dynamic_buffer(dut: Dut) -> None: # Currently we are just testing the build for esp_http_client on Linux target. So skipping the test run. # Later we will enable the test run for Linux target as well. -@pytest.mark.skipif('config.getvalue("target") == "linux"', reason='Do not run on Linux') +@pytest.mark.skip(reason='Do not run on Linux') @pytest.mark.parametrize( 'config', [ diff --git a/tools/test_apps/system/panic/pytest_panic.py b/tools/test_apps/system/panic/pytest_panic.py index 2d275831f0f..aa8a9832541 100644 --- a/tools/test_apps/system/panic/pytest_panic.py +++ b/tools/test_apps/system/panic/pytest_panic.py @@ -1623,7 +1623,7 @@ def test_dcache_read_violation(dut: PanicTestDut, test_func_name: str) -> None: # TODO: IDF-6820: ESP32-S2 -> Fix multiple panic reasons in different runs @pytest.mark.generic -@pytest.mark.xfail('config.getvalue("target") == "esp32s2"', reason='Incorrect panic reason may be observed', run=False) +@pytest.mark.xfail(reason='Incorrect panic reason may be observed', run=False) @idf_parametrize('config', ['memprot_esp32s2'], indirect=['config']) @idf_parametrize('target', ['esp32s2'], indirect=['target']) def test_dcache_write_violation(dut: PanicTestDut, test_func_name: str) -> None: @@ -1744,13 +1744,16 @@ def test_iram_reg3_write_violation(dut: PanicTestDut, test_func_name: str) -> No dut.expect_cpu_reset() -# TODO: IDF-6820: ESP32-S2 -> Fix incorrect panic reason: Unhandled debug exception @pytest.mark.generic -@pytest.mark.xfail('config.getvalue("target") == "esp32s2"', reason='Incorrect panic reason may be observed', run=False) @idf_parametrize( - 'config,target', + 'config,target,markers', [ - ('memprot_esp32s2', 'esp32s2'), + ( + 'memprot_esp32s2', + 'esp32s2', + # TODO: IDF-6820: ESP32-S2 -> Fix incorrect panic reason: Unhandled debug exception + pytest.mark.xfail(reason='Incorrect panic reason may be observed', run=False), + ), ('memprot_esp32c3', 'esp32c3'), ('memprot_esp32c2', 'esp32c2'), ('memprot_esp32c5', 'esp32c5'), @@ -1784,15 +1787,16 @@ def test_iram_reg4_write_violation(dut: PanicTestDut, test_func_name: str) -> No dut.expect_cpu_reset() -# TODO: IDF-6820: ESP32-S2 -> Fix multiple panic reasons in different runs @pytest.mark.generic -@pytest.mark.xfail( - 'config.getvalue("target") == "esp32s2"', reason='Multiple panic reasons for the same test may surface', run=False -) @idf_parametrize( - 'config,target', + 'config,target,markers', [ - ('memprot_esp32s2', 'esp32s2'), + ( + 'memprot_esp32s2', + 'esp32s2', + # TODO: IDF-6820: ESP32-S2 -> Fix multiple panic reasons in different runs + pytest.mark.xfail(reason='Multiple panic reasons for the same test may surface', run=False), + ), ('memprot_esp32c3', 'esp32c3'), ('memprot_esp32c2', 'esp32c2'), ('memprot_esp32c5', 'esp32c5'), @@ -1819,15 +1823,16 @@ def test_dram_reg1_execute_violation(dut: PanicTestDut, test_func_name: str) -> dut.expect_cpu_reset() -# TODO: IDF-6820: ESP32-S2 -> Fix multiple panic reasons in different runs @pytest.mark.generic -@pytest.mark.xfail( - 'config.getvalue("target") == "esp32s2"', reason='Multiple panic reasons for the same test may surface', run=False -) @idf_parametrize( - 'config,target', + 'config,target,markers', [ - ('memprot_esp32s2', 'esp32s2'), + ( + 'memprot_esp32s2', + 'esp32s2', + # TODO: IDF-6820: ESP32-S2 -> Fix multiple panic reasons in different runs + pytest.mark.xfail(reason='Multiple panic reasons for the same test may surface', run=False), + ), ('memprot_esp32c3', 'esp32c3'), ('memprot_esp32c2', 'esp32c2'), ('memprot_esp32c5', 'esp32c5'), @@ -1907,15 +1912,16 @@ def test_rtc_fast_reg2_execute_violation(dut: PanicTestDut, test_func_name: str) dut.expect_cpu_reset() -# TODO: IDF-6820: ESP32-S2 -> Fix multiple panic reasons in different runs @pytest.mark.generic -@pytest.mark.xfail( - 'config.getvalue("target") == "esp32s2"', reason='Multiple panic reasons for the same test may surface', run=False -) @idf_parametrize( - 'config,target', + 'config,target,markers', [ - ('memprot_esp32s2', 'esp32s2'), + ( + 'memprot_esp32s2', + 'esp32s2', + # TODO: IDF-6820: ESP32-S2 -> Fix multiple panic reasons in different runs + pytest.mark.xfail(reason='Multiple panic reasons for the same test may surface', run=False), + ), ('memprot_esp32c3', 'esp32c3'), ('memprot_esp32c5', 'esp32c5'), ('memprot_esp32c6', 'esp32c6'),