tests: replace CLI-dependent xfail/skipif conditions

2026-09-22 13:01:16 +03:00 · 2026-08-18 11:14:02 +02:00
parent e962d4e60f
commit 3a3c36ac71
5 changed files with 80 additions and 38 deletions
--- a/components/esp_event/test_apps/pytest_esp_event.py
+++ b/components/esp_event/test_apps/pytest_esp_event.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: CC0-1.0
 import pytest
 from pytest_embedded import Dut
@@ -20,18 +20,14 @@ def test_esp_event(dut: Dut) -> None:


@pytest.mark.qemu
-@pytest.mark.xfail(
-    'config.getvalue("target") == "esp32c3"',
-    reason='Unstable on QEMU, needs investigation',
-)
-@pytest.mark.parametrize(
-    'config',
+@idf_parametrize(
+    'config,target,markers',
    [
-        'defaults',
+        ('defaults', 'esp32'),
+        ('defaults', 'esp32c3', (pytest.mark.xfail(reason='Unstable on QEMU, needs investigation'),)),
    ],
-    indirect=True,
+    indirect=['config', 'target'],
 )
-@idf_parametrize('target', ['esp32', 'esp32c3'], indirect=['target'])
 def test_esp_event_qemu(dut: Dut) -> None:
    for case in dut.test_menu:
        if 'qemu-ignore' not in case.groups and not case.is_ignored and case.type == 'normal':
--- a/docs/en/contribute/esp-idf-tests-with-pytest.rst
+++ b/docs/en/contribute/esp-idf-tests-with-pytest.rst
@@ -750,15 +750,35 @@ Sometimes, a test can consistently fail for the following reasons:

 Now you may mark this test case with marker `xfail <https://docs.pytest.org/en/latest/how-to/skipping.html#xfail-mark-test-functions-as-expected-to-fail>`__ with a user-friendly readable reason.

+.. attention::
+
+    Avoid using a string condition, e.g., ``@pytest.mark.xfail('config.getvalue("target") == "esp32s2"', ...)``. This condition is evaluated against the session-level CLI ``--target`` option, not against the actual target resolved for a given parametrized test instance, so it can silently mismatch when the same test is parametrized over multiple targets and produce unreliable results.
+
+    Instead, attach the ``xfail`` marker directly to the specific parametrized value passed to ``idf_parametrize`` (see :ref:`Same App With Different Running Environments <pytest-same-app-different-running-environments>`), so the marker is tied to that value at collection time.
+
 This code example is taken from :idf_file:`pytest_panic.py <tools/test_apps/system/panic/pytest_panic.py>`

 .. code-block:: python

-    @pytest.mark.xfail('config.getvalue("target") == "esp32s2"', reason='raised IllegalInstruction instead')
-    def test_cache_error(dut: PanicTestDut, config: str, test_func_name: str) -> None:
+    @pytest.mark.generic
+    @idf_parametrize(
+        'config,target,markers',
+        [
+            (
+                'memprot_esp32s2',
+                'esp32s2',
+                pytest.mark.xfail(reason='Incorrect panic reason may be observed', run=False),
+            ),
+            ('memprot_esp32c3', 'esp32c3'),
+        ],
+        indirect=['config', 'target'],
+    )
+    def test_cache_error(dut: PanicTestDut, test_func_name: str) -> None:

 This marker means that test is a known failure on the ESP32-S2.

+If a test is only ever parametrized with a single target, there is no ambiguity, and you can use an unconditional ``@pytest.mark.xfail(reason=..., run=False)`` instead.
+
 Mark Nightly Run Test Cases
 ---------------------------

--- a/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst
+++ b/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst
@@ -750,15 +750,35 @@ Pytest 使用技巧

 可使用 `xfail <https://docs.pytest.org/en/latest/how-to/skipping.html#xfail-mark-test-functions-as-expected-to-fail>`__ marker 来标记此测试用例，并写出原因。

+.. attention::
+
+    请避免使用字符串条件，例如 ``@pytest.mark.xfail('config.getvalue("target") == "esp32s2"', ...)``。该条件是在运行时根据 session 级别的 CLI ``--target`` 参数求值的，而不是针对某个具体参数化测试实例实际解析出的 target 值，因此当同一测试针对多个 target 参数化时，该条件可能被错误匹配，导致结果不可靠。
+
+    推荐做法是通过 ``idf_parametrize`` 传入的具体参数化取值直接附加 ``xfail`` marker，更复杂的示例请参阅 :ref:`在不同运行环境中运行相同的应用程序 <pytest-same-app-different-running-environments>` 小节，这样该 marker 在 collection 阶段就与该取值绑定。
+
 以下代码来自 :idf_file:`pytest_panic.py <tools/test_apps/system/panic/pytest_panic.py>`。

 .. code-block:: python

-    @pytest.mark.xfail('config.getvalue("target") == "esp32s2"', reason='raised IllegalInstruction instead')
-    def test_cache_error(dut: PanicTestDut, config: str, test_func_name: str) -> None:
+    @pytest.mark.generic
+    @idf_parametrize(
+        'config,target,markers',
+        [
+            (
+                'memprot_esp32s2',
+                'esp32s2',
+                pytest.mark.xfail(reason='Incorrect panic reason may be observed', run=False),
+            ),
+            ('memprot_esp32c3', 'esp32c3'),
+        ],
+        indirect=['config', 'target'],
+    )
+    def test_cache_error(dut: PanicTestDut, test_func_name: str) -> None:

 这一 marker 表示该测试在 ESP32-S2 上是一个已知失败。

+如果一个测试始终只针对单个 target 参数化，则不存在歧义，此时可直接使用无条件的 ``@pytest.mark.xfail(reason=..., run=False)``。
+
 标记夜间运行的测试用例
 ---------------------------

--- a/examples/protocols/esp_http_client/pytest_esp_http_client.py
+++ b/examples/protocols/esp_http_client/pytest_esp_http_client.py
@@ -108,7 +108,7 @@ def test_examples_protocol_esp_http_client_dynamic_buffer(dut: Dut) -> None:

 # Currently we are just testing the build for esp_http_client on Linux target. So skipping the test run.
 # Later we will enable the test run for Linux target as well.
-@pytest.mark.skipif('config.getvalue("target") == "linux"', reason='Do not run on Linux')
+@pytest.mark.skip(reason='Do not run on Linux')
@pytest.mark.parametrize(
    'config',
    [
--- a/tools/test_apps/system/panic/pytest_panic.py
+++ b/tools/test_apps/system/panic/pytest_panic.py
@@ -1623,7 +1623,7 @@ def test_dcache_read_violation(dut: PanicTestDut, test_func_name: str) -> None:

 # TODO: IDF-6820: ESP32-S2 -> Fix multiple panic reasons in different runs
@pytest.mark.generic
-@pytest.mark.xfail('config.getvalue("target") == "esp32s2"', reason='Incorrect panic reason may be observed', run=False)
+@pytest.mark.xfail(reason='Incorrect panic reason may be observed', run=False)
@idf_parametrize('config', ['memprot_esp32s2'], indirect=['config'])
@idf_parametrize('target', ['esp32s2'], indirect=['target'])
 def test_dcache_write_violation(dut: PanicTestDut, test_func_name: str) -> None:
@@ -1744,13 +1744,16 @@ def test_iram_reg3_write_violation(dut: PanicTestDut, test_func_name: str) -> No
    dut.expect_cpu_reset()


-# TODO: IDF-6820: ESP32-S2 -> Fix incorrect panic reason: Unhandled debug exception
@pytest.mark.generic
-@pytest.mark.xfail('config.getvalue("target") == "esp32s2"', reason='Incorrect panic reason may be observed', run=False)
@idf_parametrize(
-    'config,target',
+    'config,target,markers',
    [
-        ('memprot_esp32s2', 'esp32s2'),
+        (
+            'memprot_esp32s2',
+            'esp32s2',
+            # TODO: IDF-6820: ESP32-S2 -> Fix incorrect panic reason: Unhandled debug exception
+            pytest.mark.xfail(reason='Incorrect panic reason may be observed', run=False),
+        ),
        ('memprot_esp32c3', 'esp32c3'),
        ('memprot_esp32c2', 'esp32c2'),
        ('memprot_esp32c5', 'esp32c5'),
@@ -1784,15 +1787,16 @@ def test_iram_reg4_write_violation(dut: PanicTestDut, test_func_name: str) -> No
    dut.expect_cpu_reset()


-# TODO: IDF-6820: ESP32-S2 -> Fix multiple panic reasons in different runs
@pytest.mark.generic
-@pytest.mark.xfail(
-    'config.getvalue("target") == "esp32s2"', reason='Multiple panic reasons for the same test may surface', run=False
-)
@idf_parametrize(
-    'config,target',
+    'config,target,markers',
    [
-        ('memprot_esp32s2', 'esp32s2'),
+        (
+            'memprot_esp32s2',
+            'esp32s2',
+            # TODO: IDF-6820: ESP32-S2 -> Fix multiple panic reasons in different runs
+            pytest.mark.xfail(reason='Multiple panic reasons for the same test may surface', run=False),
+        ),
        ('memprot_esp32c3', 'esp32c3'),
        ('memprot_esp32c2', 'esp32c2'),
        ('memprot_esp32c5', 'esp32c5'),
@@ -1819,15 +1823,16 @@ def test_dram_reg1_execute_violation(dut: PanicTestDut, test_func_name: str) ->
    dut.expect_cpu_reset()


-# TODO: IDF-6820: ESP32-S2 -> Fix multiple panic reasons in different runs
@pytest.mark.generic
-@pytest.mark.xfail(
-    'config.getvalue("target") == "esp32s2"', reason='Multiple panic reasons for the same test may surface', run=False
-)
@idf_parametrize(
-    'config,target',
+    'config,target,markers',
    [
-        ('memprot_esp32s2', 'esp32s2'),
+        (
+            'memprot_esp32s2',
+            'esp32s2',
+            # TODO: IDF-6820: ESP32-S2 -> Fix multiple panic reasons in different runs
+            pytest.mark.xfail(reason='Multiple panic reasons for the same test may surface', run=False),
+        ),
        ('memprot_esp32c3', 'esp32c3'),
        ('memprot_esp32c2', 'esp32c2'),
        ('memprot_esp32c5', 'esp32c5'),
@@ -1907,15 +1912,16 @@ def test_rtc_fast_reg2_execute_violation(dut: PanicTestDut, test_func_name: str)
    dut.expect_cpu_reset()


-# TODO: IDF-6820: ESP32-S2 -> Fix multiple panic reasons in different runs
@pytest.mark.generic
-@pytest.mark.xfail(
-    'config.getvalue("target") == "esp32s2"', reason='Multiple panic reasons for the same test may surface', run=False
-)
@idf_parametrize(
-    'config,target',
+    'config,target,markers',
    [
-        ('memprot_esp32s2', 'esp32s2'),
+        (
+            'memprot_esp32s2',
+            'esp32s2',
+            # TODO: IDF-6820: ESP32-S2 -> Fix multiple panic reasons in different runs
+            pytest.mark.xfail(reason='Multiple panic reasons for the same test may surface', run=False),
+        ),
        ('memprot_esp32c3', 'esp32c3'),
        ('memprot_esp32c5', 'esp32c5'),
        ('memprot_esp32c6', 'esp32c6'),