diff --git a/.gitlab/ci/README.md b/.gitlab/ci/README.md index de9562aa2ba..204732c8937 100644 --- a/.gitlab/ci/README.md +++ b/.gitlab/ci/README.md @@ -23,12 +23,6 @@ - [Grammar](#grammar) - [Special Rules](#special-rules) - [Upload/Download Artifacts to Internal Minio Server](#uploaddownload-artifacts-to-internal-minio-server) - - [Users Without Access to Minio](#users-without-access-to-minio) - - [Users With Access to Minio](#users-with-access-to-minio) - - [Env Vars for Minio](#env-vars-for-minio) - - [Artifacts Types and File Patterns](#artifacts-types-and-file-patterns) - - [Upload](#upload) - - [Download](#download) - [Revision Marker Naming Rules](#revision-marker-naming-rules) ## General Workflow @@ -263,3 +257,5 @@ Please refer to the documentation [here](https://docs.espressif.com/projects/idf When specifying chip revisions for testing markers, configurations or filenames, strictly follow the format `esp32_rev` (or similarly `_rev` when brief). > e.g., `esp32_rev3`, `esp32c2_rev2`, `esp32c3_rev1`, `esp32p4_rev1`, `esp32c5_rev1` + +This convention replaces the legacy `eco` naming format (e.g., `esp32c2eco4`). diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index e827e42c52e..8a39499730d 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -276,7 +276,11 @@ variables: - source tools/ci/utils.sh - source tools/ci/configure_ci_environment.sh - *show_ccache_statistics - - *upload_failed_job_log_artifacts + - run_cmd idf-ci gitlab upload-artifacts --type junit + - | + if [ $CI_JOB_STATUS = "failed" ]; then + run_cmd idf-ci gitlab upload-artifacts --type log + fi .brew-macos-settings: variables: @@ -289,6 +293,46 @@ variables: after_script: [] # ccache now is disabled for macos brew runners timeout: 30m +.brew-windows-settings: + image: windows_11_idf_v6_0 + tags: [windows-vm, brew] + variables: + GIT_STRATEGY: fetch + LC_ALL: C.UTF-8 + # git clean at the final cleanup stage stuck on Windows VM + # a new VM for each job anyway, so we can skip cleaning the workspace + GIT_CLEAN_FLAGS: none +# IDF_CCACHE_ENABLE: "1" +# CCACHE_DIR: "C:/cache/idf_ccache" + before_script: + - if ($env:IDF_DONT_USE_MIRRORS) { $env:IDF_MIRROR_PREFIX_MAP = '' } + - ./install.ps1 --enable-ci + - ./export.ps1 + - $env:PYTHONPATH = "$env:PYTHONPATH;$env:IDF_PATH\tools;$env:IDF_PATH\tools\esp_app_trace;$env:IDF_PATH\components\partition_table;$env:IDF_PATH\tools\ci\python_packages" + - python "${SUBMODULE_FETCH_TOOL}" -s "${SUBMODULES_TO_FETCH}" + # Sync VM clock. net start/stop exit 2 when the service is already in that state; do not fail CI. + - | + function Invoke-NetService($Action, $Name) { + $output = net $Action $Name 2>&1 + if ($LASTEXITCODE -eq 2) { + Write-Host "net $Action $Name already in requested state: $output" + $global:LASTEXITCODE = 0 + return + } + if ($LASTEXITCODE -ne 0) { + Write-Host $output + exit $LASTEXITCODE + } + } + Invoke-NetService start w32time + w32tm /config /manualpeerlist:"time.google.com,0x9 pool.ntp.org,0x9" /syncfromflags:manual /reliable:yes /update + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + Invoke-NetService stop w32time + Invoke-NetService start w32time + w32tm /resync + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + after_script: [] + ############# # `default` # ############# diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index 43b7e14af23..98a187b9db2 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -359,7 +359,6 @@ test_pytest_qemu: - run_cmd pytest --target $IDF_TARGET -m qemu - --embedded-services idf,qemu --junitxml=XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} --qemu-extra-args \"-global driver=timer.$IDF_TARGET.timg,property=wdt_disable,value=true\" @@ -384,7 +383,7 @@ test_pytest_linux: - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} - run_cmd pytest --target linux - --embedded-services idf + -m \"not macos\" --junitxml=XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} diff --git a/.gitlab/ci/pre_commit.yml b/.gitlab/ci/pre_commit.yml index 49b6dfbd4d2..da07c9bbdbb 100644 --- a/.gitlab/ci/pre_commit.yml +++ b/.gitlab/ci/pre_commit.yml @@ -2,7 +2,7 @@ extends: - .before_script:minimal stage: pre_check - image: "${CI_DOCKER_REGISTRY}/esp-idf-pre-commit:2" + image: "${CI_REGISTRY}/ci/images/pre-commit-idf-v6.0:1" tags: [pre-commit] variables: # Both shiny and brew runners can pick this job diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 7cd939739f7..9cd0a901737 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -65,6 +65,7 @@ - "tools/ci/check_esp_memory_utils_headers.sh" - "tools/ci/check_blobs.sh" - "tools/ci/check_public_headers.py" + - "tools/ci/check_public_headers_exceptions.txt" - "tools/ci/check_register_rw_half_word.cmake" - "tools/ci/check_register_rw_half_word.py" - "examples/build_system/**/*" @@ -99,6 +100,7 @@ - "tools/idf_tools.py" - "tools/test_idf_tools/**/*" - "tools/install_util.py" + - "tools/idf_extra_components.yml" - "tools/export_utils/utils.py" - "tools/export_utils/shell_types.py" diff --git a/.gitlab/ci/test-win.yml b/.gitlab/ci/test-win.yml index bf2a116e91a..0bb3c7986ca 100644 --- a/.gitlab/ci/test-win.yml +++ b/.gitlab/ci/test-win.yml @@ -1,9 +1,8 @@ -# Host tests .host_test_win_template: - extends: .rules:test:host_test + extends: + - .brew-windows-settings + - .rules:test:host_test stage: host_test - image: $ESP_ENV_IMAGE - tags: [windows-build, brew] dependencies: # set dependencies to null to avoid missing artifacts issue # run host_test jobs immediately, only after upload cache needs: @@ -13,13 +12,8 @@ - job: upload-submodules-cache optional: true artifacts: false - variables: - GIT_STRATEGY: fetch # use brew local mirror first - before_script: - - if ($env:IDF_DONT_USE_MIRRORS) { - $env:IDF_MIRROR_PREFIX_MAP = "" - } - after_script: [] + - job: manual_gate + optional: true test_cli_installer_win: rules: @@ -34,15 +28,12 @@ test_cli_installer_win: - tools/tools.new.json - tools/test_idf_tools/test_python_env_logs.txt expire_in: 1 week - variables: - IDF_PATH: "$CI_PROJECT_DIR" timeout: 3h script: # Tools must be downloaded for testing # cmake@version that is supported - - python ${IDF_PATH}\tools\idf_tools.py download required qemu-riscv32 qemu-xtensa cmake cmake@3.22.1 - - cd ${IDF_PATH}\tools\test_idf_tools - - python -m pip install jsonschema + - python tools\idf_tools.py download required qemu-riscv32 qemu-xtensa cmake cmake@3.22.1 + - cd tools\test_idf_tools - python .\test_idf_tools.py - python .\test_idf_tools_python_env.py @@ -53,39 +44,31 @@ test_tools_win: parallel: 4 artifacts: paths: - - ${IDF_PATH}/*.out - - ${IDF_PATH}/XUNIT_*.xml + - "*.out" + - "XUNIT_*.xml" reports: - junit: ${IDF_PATH}/XUNIT_*.xml + junit: "XUNIT_*.xml" expire_in: 1 week when: always - variables: - LC_ALL: C.UTF-8 - PYTHONPATH: "$PYTHONPATH;$IDF_PATH\\tools;$IDF_PATH\\tools\\esp_app_trace;$IDF_PATH\\components\\partition_table;$IDF_PATH\\tools\\ci\\python_packages" script: - - python -m pip install jsonschema - - .\install.ps1 --enable-ci - - .\export.ps1 - - python "${SUBMODULE_FETCH_TOOL}" -s "all" - - cd ${IDF_PATH}/tools/test_idf_py + - cd tools\test_idf_py - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} - - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=${IDF_PATH}/XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} + - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} # Build tests .test_build_system_template_win: - stage: host_test - variables: - # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings. - IDF_CCACHE_ENABLE: "1" - PYTHONPATH: "$PYTHONPATH;$IDF_PATH\\tools;$IDF_PATH\\tools\\esp_app_trace;$IDF_PATH\\components\\partition_table;$IDF_PATH\\tools\\ci\\python_packages" - before_script: [] - after_script: [] + extends: .host_test_win_template timeout: 4 hours + artifacts: + paths: + - XUNIT_RESULT.xml + - test_build_system + expire_in: 2 days + when: always + reports: + junit: XUNIT_RESULT.xml script: - - .\install.ps1 --enable-ci - - . .\export.ps1 - - python "${SUBMODULE_FETCH_TOOL}" -s "all" - - cd ${IDF_PATH}\tools\test_build_system + - cd tools\test_build_system - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} @@ -126,9 +109,7 @@ pytest_build_system_win_minimal_cmake: variables: MINIMAL_CMAKE_VERSION: "3.22.1" script: - - .\install.ps1 --enable-ci - - . .\export.ps1 - - python ${IDF_PATH}\tools\idf_tools.py install cmake@${MINIMAL_CMAKE_VERSION} + - python tools\idf_tools.py install cmake@${MINIMAL_CMAKE_VERSION} - $Env:PATH = "$Env:USERPROFILE\.espressif\tools\cmake\${MINIMAL_CMAKE_VERSION}\bin;$Env:PATH" - | $actualVersion = (& cmake --version).Split()[2] @@ -136,8 +117,7 @@ pytest_build_system_win_minimal_cmake: Write-Error "ERROR: Wrong CMake version! Detected: $actualVersion, but expected: $Env:MINIMAL_CMAKE_VERSION" exit 1 } - - python "${SUBMODULE_FETCH_TOOL}" -s "all" - - cd ${IDF_PATH}\tools\test_build_system + - cd tools\test_build_system - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} - pytest -k cmake --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} @@ -159,10 +139,7 @@ pytest_buildv2_system_win: junit: XUNIT_RESULT.xml when: always script: - - .\install.ps1 --enable-ci - - . .\export.ps1 - - python "${SUBMODULE_FETCH_TOOL}" -s "all" - - cd ${IDF_PATH}\tools\test_build_system + - cd tools\test_build_system - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} - pytest --buildv2 diff --git a/.idf_build_apps.toml b/.idf_build_apps.toml index 9583f9ff588..75282d94be0 100644 --- a/.idf_build_apps.toml +++ b/.idf_build_apps.toml @@ -31,8 +31,8 @@ additional_build_targets = [ ] # collect -collect_app_info_filename = "app_info_${CI_JOB_NAME_SLUG}.txt" -junitxml = "build_summary_${CI_JOB_NAME_SLUG}.xml" +collect_app_info_filename = "app_info_${CI_JOB_ID}.txt" +junitxml = "build_summary_${CI_JOB_ID}.xml" # manifest check_manifest_rules = true diff --git a/.idf_ci.toml b/.idf_ci.toml index 32f75f3f5d5..18db33e7590 100644 --- a/.idf_ci.toml +++ b/.idf_ci.toml @@ -51,37 +51,44 @@ include: - tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml """ -[gitlab.artifacts.s3.debug] +[gitlab.artifacts.s3] +enable = true + +[gitlab.artifacts.s3.configs.debug] bucket = "idf-artifacts" +zip_first = true +build_dir_pattern = "**/build*/" patterns = [ - '**/build*/bootloader/*.map', - '**/build*/bootloader/*.elf', - '**/build*/*.map', - '**/build*/*.elf', + 'bootloader/*.map', + 'bootloader/*.elf', + '*.map', + '*.elf', # customized - '**/build*/esp_tee/*.map', - '**/build*/esp_tee/*.elf', - '**/build*/gdbinit/*', + 'esp_tee/*.map', + 'esp_tee/*.elf', + 'gdbinit/*', ] if_clause = 'CI_JOB_GROUP_NAME != "build_non_test_related_apps"' -[gitlab.artifacts.s3.flash] +[gitlab.artifacts.s3.configs.flash] bucket = "idf-artifacts" +zip_first = true +build_dir_pattern = "**/build*/" patterns = [ - '**/build*/bootloader/*.bin', - '**/build*/*.bin', - '**/build*/partition_table/*.bin', - '**/build*/flasher_args.json', - '**/build*/flash_project_args', - '**/build*/config/sdkconfig.json', - '**/build*/sdkconfig', - '**/build*/project_description.json', + 'bootloader/*.bin', + '*.bin', + 'partition_table/*.bin', + 'flasher_args.json', + 'flash_project_args', + 'config/sdkconfig.json', + 'sdkconfig', + 'project_description.json', # customized - '**/build*/esp_tee/*.bin', + 'esp_tee/*.bin', ] if_clause = 'CI_JOB_GROUP_NAME != "build_non_test_related_apps"' -[gitlab.artifacts.s3.app_extra] +[gitlab.artifacts.s3.configs.app_extra] bucket = "idf-artifacts" # App-dir files needed at target-test time but outside build_* (uploaded without --build-dir). # Add patterns here as needed, e.g. coverage info produced during build. @@ -90,26 +97,26 @@ patterns = [ ] if_clause = 'CI_JOB_GROUP_NAME != "build_non_test_related_apps"' -[gitlab.artifacts.s3.log] +[gitlab.artifacts.s3.configs.log] bucket = "idf-artifacts" patterns = [ '**/build*/build_log.txt', '**/build*/size*.json', ] -[gitlab.artifacts.s3.junit] +[gitlab.artifacts.s3.configs.junit] bucket = "idf-artifacts" patterns = [ '**/XUNIT_RESULT_*.xml', ] -[gitlab.artifacts.s3.env] +[gitlab.artifacts.s3.configs.env] bucket = "idf-artifacts" patterns = [ '**/pipeline.env', ] -[gitlab.artifacts.s3.longterm] +[gitlab.artifacts.s3.configs.longterm] bucket = "longterm" if_clause = 'CI_COMMIT_REF_NAME == "master"' patterns = [ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 82000208fcc..5feed811a63 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -233,7 +233,7 @@ repos: - id: check-kconfig-files - id: check-deprecated-kconfig-options - repo: https://github.com/espressif/idf-ci - rev: v1.0.0b4 + rev: v1.0.0 hooks: - id: check-tests-missing-config files: 'pytest_.*\.py$|sdkconfig(\..*)?$' diff --git a/components/bootloader_support/test_apps/bootloader_support/pytest_bootloader_support.py b/components/bootloader_support/test_apps/bootloader_support/pytest_bootloader_support.py index 59e3d4cf6a1..7bb24e96908 100644 --- a/components/bootloader_support/test_apps/bootloader_support/pytest_bootloader_support.py +++ b/components/bootloader_support/test_apps/bootloader_support/pytest_bootloader_support.py @@ -11,14 +11,12 @@ def test_bootloader_support(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.host_test @pytest.mark.qemu @idf_parametrize('target', ['esp32'], indirect=['target']) def test_bootloader_support_qemu_esp32(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.host_test @pytest.mark.qemu @idf_parametrize('target', ['esp32c3'], indirect=['target']) def test_bootloader_support_qemu_esp32c3(dut: Dut) -> None: diff --git a/components/console/test_apps/console/pytest_console.py b/components/console/test_apps/console/pytest_console.py index 45ffbd5441d..084772e388f 100644 --- a/components/console/test_apps/console/pytest_console.py +++ b/components/console/test_apps/console/pytest_console.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 import pytest from pytest_embedded import Dut @@ -50,63 +50,59 @@ def do_test_help_quit(dut: Dut) -> None: dut.expect(r'quit\s+Quit REPL environment\s+esp>') -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console(dut: Dut, test_on: str) -> None: +def test_console(dut: Dut) -> None: dut.run_all_single_board_cases(group='!ignore', timeout=120) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_repl(dut: Dut, test_on: str) -> None: +def test_console_repl(dut: Dut) -> None: do_test_quit(dut) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_sorted_registration(dut: Dut, test_on: str) -> None: +def test_console_help_sorted_registration(dut: Dut) -> None: do_test_help_generic(dut, 'sorted') -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_reverse_registration(dut: Dut, test_on: str) -> None: +def test_console_help_reverse_registration(dut: Dut) -> None: dut.expect_exact('Press ENTER to see the list of tests') dut.confirm_write('"esp console help command - reverse registration"', expect_str='esp>') @@ -131,45 +127,41 @@ def test_console_help_reverse_registration(dut: Dut, test_on: str) -> None: @idf_parametrize('config', ['sorted'], indirect=['config']) @idf_parametrize('target', ['linux'], indirect=['target']) -@idf_parametrize('test_on,markers', [('host', (pytest.mark.host_test,))]) -def test_console_sorted_help_sorted_registration(dut: Dut, test_on: str) -> None: +def test_console_sorted_help_sorted_registration(dut: Dut) -> None: do_test_help_generic(dut, 'sorted') @idf_parametrize('config', ['sorted'], indirect=['config']) @idf_parametrize('target', ['linux'], indirect=['target']) -@idf_parametrize('test_on,markers', [('host', (pytest.mark.host_test,))]) -def test_console_sorted_help_reverse_registration(dut: Dut, test_on: str) -> None: +def test_console_sorted_help_reverse_registration(dut: Dut) -> None: do_test_help_generic(dut, 'reverse') -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_quit(dut: Dut, test_on: str) -> None: +def test_console_help_quit(dut: Dut) -> None: do_test_help_quit(dut) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_verbose_level_0(dut: Dut, test_on: str) -> None: +def test_console_help_verbose_level_0(dut: Dut) -> None: help_verbose_info = 'Print the summary of all registered commands if no arguments are given,' dut.expect_exact('Press ENTER to see the list of tests') dut.confirm_write('"esp console help command - set verbose level = 0"', expect_str='esp>') @@ -180,18 +172,17 @@ def test_console_help_verbose_level_0(dut: Dut, test_on: str) -> None: dut.expect_exact('help', not_matching=help_verbose_info) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_verbose_level_1(dut: Dut, test_on: str) -> None: +def test_console_help_verbose_level_1(dut: Dut) -> None: help_verbose_info = 'Print the summary of all registered commands if no arguments are given,' dut.expect_exact('Press ENTER to see the list of tests') dut.confirm_write('"esp console help command - set verbose level = 1"', expect_str='esp>') @@ -201,18 +192,17 @@ def test_console_help_verbose_level_1(dut: Dut, test_on: str) -> None: dut.expect_exact(help_verbose_info) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_verbose_subcommand(dut: Dut, test_on: str) -> None: +def test_console_help_verbose_subcommand(dut: Dut) -> None: help_verbose_info = 'Print the summary of all registered commands if no arguments are given,' dut.expect_exact('Press ENTER to see the list of tests') dut.confirm_write('"esp console help command - --verbose sub command"', expect_str='esp>') @@ -227,18 +217,17 @@ def test_console_help_verbose_subcommand(dut: Dut, test_on: str) -> None: dut.expect_exact(help_verbose_info) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_deregister(dut: Dut, test_on: str) -> None: +def test_console_help_deregister(dut: Dut) -> None: dut.expect_exact('Press ENTER to see the list of tests') dut.confirm_write('"esp console deregister commands"', expect_str='esp>') @@ -250,18 +239,17 @@ def test_console_help_deregister(dut: Dut, test_on: str) -> None: dut.expect_exact(cmd_z_description, not_matching=cmd_a_description) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_re_register(dut: Dut, test_on: str) -> None: +def test_console_help_re_register(dut: Dut) -> None: dut.expect_exact('Press ENTER to see the list of tests') dut.confirm_write('"esp console re-register commands"', expect_str='esp>') @@ -271,17 +259,16 @@ def test_console_help_re_register(dut: Dut, test_on: str) -> None: dut.expect_exact('should appear first in help') -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_custom_uart_repl(dut: Dut, test_on: str) -> None: +def test_console_custom_uart_repl(dut: Dut) -> None: dut.expect_exact('Press ENTER to see the list of tests') dut.confirm_write('"esp console repl custom_uart test"', expect_str='Running repl on UART1') diff --git a/components/efuse/test_apps/pytest_efuse.py b/components/efuse/test_apps/pytest_efuse.py index 6d823e62cb0..f75dff43793 100644 --- a/components/efuse/test_apps/pytest_efuse.py +++ b/components/efuse/test_apps/pytest_efuse.py @@ -15,13 +15,11 @@ def test_efuse(dut: Dut) -> None: @pytest.mark.qemu -@pytest.mark.host_test @idf_parametrize('target', ['esp32'], indirect=['target']) def test_efuse_qemu(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_efuse_linux(dut: Dut) -> None: dut.run_all_single_board_cases(timeout=60) diff --git a/components/esp_app_format/test_apps/pytest_esp_app_format.py b/components/esp_app_format/test_apps/pytest_esp_app_format.py index 843ceb825a5..d8f85371c19 100644 --- a/components/esp_app_format/test_apps/pytest_esp_app_format.py +++ b/components/esp_app_format/test_apps/pytest_esp_app_format.py @@ -11,7 +11,6 @@ def test_esp_app_format(dut: Dut) -> None: dut.expect_unity_test_output() -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_esp_app_format_linux(dut: Dut) -> None: dut.expect_unity_test_output() diff --git a/components/esp_event/host_test/esp_event_unit_test/pytest_esp_event_linux.py b/components/esp_event/host_test/esp_event_unit_test/pytest_esp_event_linux.py index 79b2c2ebbb7..70888df072d 100644 --- a/components/esp_event/host_test/esp_event_unit_test/pytest_esp_event_linux.py +++ b/components/esp_event/host_test/esp_event_unit_test/pytest_esp_event_linux.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 -import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_esp_event_linux(dut: Dut) -> None: dut.expect_exact('All tests passed', timeout=5) diff --git a/components/esp_event/test_apps/pytest_esp_event.py b/components/esp_event/test_apps/pytest_esp_event.py index 20d59bb2b9e..205ccb19a9f 100644 --- a/components/esp_event/test_apps/pytest_esp_event.py +++ b/components/esp_event/test_apps/pytest_esp_event.py @@ -19,7 +19,6 @@ def test_esp_event(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.host_test @pytest.mark.qemu @pytest.mark.xfail('config.getvalue("target") == "esp32c3"', reason='Unstable on QEMU, needs investigation') @pytest.mark.parametrize( @@ -36,7 +35,6 @@ def test_esp_event_qemu(dut: Dut) -> None: dut._run_normal_case(case) -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) @pytest.mark.parametrize( 'config', diff --git a/components/esp_gdbstub/test_gdbstub_host/rv_decode/pytest_gdbstub_host.py b/components/esp_gdbstub/test_gdbstub_host/rv_decode/pytest_gdbstub_host.py index a5723bfa3c4..86c374f624e 100644 --- a/components/esp_gdbstub/test_gdbstub_host/rv_decode/pytest_gdbstub_host.py +++ b/components/esp_gdbstub/test_gdbstub_host/rv_decode/pytest_gdbstub_host.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 -import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_rv_decode(dut: Dut) -> None: dut.expect_exact('All tests passed', timeout=180) diff --git a/components/esp_hw_support/test_apps/host_test_linux/pytest_esp_hw_support_linux.py b/components/esp_hw_support/test_apps/host_test_linux/pytest_esp_hw_support_linux.py index 9c3bd552736..1c55f0ec994 100644 --- a/components/esp_hw_support/test_apps/host_test_linux/pytest_esp_hw_support_linux.py +++ b/components/esp_hw_support/test_apps/host_test_linux/pytest_esp_hw_support_linux.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 -import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_esp_hw_support_linux(dut: Dut) -> None: dut.run_all_single_board_cases(group='!ignore', timeout=120) diff --git a/components/esp_partition/host_test/partition_api_test/pytest_esp_partition_linux.py b/components/esp_partition/host_test/partition_api_test/pytest_esp_partition_linux.py index 7f2b68dc356..5543592d595 100644 --- a/components/esp_partition/host_test/partition_api_test/pytest_esp_partition_linux.py +++ b/components/esp_partition/host_test/partition_api_test/pytest_esp_partition_linux.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 -import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_esp_partition_linux(dut: Dut) -> None: dut.expect_unity_test_output(timeout=5) diff --git a/components/esp_partition/host_test/partition_bdl_test/pytest_esp_partition_bdl.py b/components/esp_partition/host_test/partition_bdl_test/pytest_esp_partition_bdl.py index af0bd34a132..3cd42cc763a 100644 --- a/components/esp_partition/host_test/partition_bdl_test/pytest_esp_partition_bdl.py +++ b/components/esp_partition/host_test/partition_bdl_test/pytest_esp_partition_bdl.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 -import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_esp_partition_bdl(dut: Dut) -> None: dut.expect_unity_test_output(timeout=5) diff --git a/components/esp_ringbuf/test_apps/pytest_esp_ringbuf.py b/components/esp_ringbuf/test_apps/pytest_esp_ringbuf.py index ca44dcfa16c..730414977ec 100644 --- a/components/esp_ringbuf/test_apps/pytest_esp_ringbuf.py +++ b/components/esp_ringbuf/test_apps/pytest_esp_ringbuf.py @@ -24,7 +24,6 @@ def test_esp_ringbuf(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.host_test @pytest.mark.qemu @pytest.mark.parametrize( 'target', @@ -46,7 +45,6 @@ def test_esp_ringbuf_qemu(dut: Dut) -> None: dut._run_normal_case(case) -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_esp_ringbuf_posix_simulator(dut: Dut) -> None: dut.run_all_single_board_cases(group='linux') diff --git a/components/esp_rom/test_apps/linux_rom_apis/pytest_esp_rom_linux.py b/components/esp_rom/test_apps/linux_rom_apis/pytest_esp_rom_linux.py index 0d2a4b48fdd..60b73d1bacf 100644 --- a/components/esp_rom/test_apps/linux_rom_apis/pytest_esp_rom_linux.py +++ b/components/esp_rom/test_apps/linux_rom_apis/pytest_esp_rom_linux.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 -import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_esp_rom_linux(dut: Dut) -> None: dut.expect_exact('All tests passed', timeout=5) diff --git a/components/esp_rom/test_apps/rom_tests/pytest_esp_rom.py b/components/esp_rom/test_apps/rom_tests/pytest_esp_rom.py index 0bc7bfef1a0..b3e41e01af8 100644 --- a/components/esp_rom/test_apps/rom_tests/pytest_esp_rom.py +++ b/components/esp_rom/test_apps/rom_tests/pytest_esp_rom.py @@ -15,7 +15,6 @@ def test_esp_rom(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.host_test @pytest.mark.qemu @idf_parametrize('target', ['esp32c3'], indirect=['target']) def test_esp_rom_qemu(dut: Dut) -> None: diff --git a/components/esp_system/test_apps/linux_apis/pytest_esp_system_linux.py b/components/esp_system/test_apps/linux_apis/pytest_esp_system_linux.py index 279dc5b2370..74c209d1f3c 100644 --- a/components/esp_system/test_apps/linux_apis/pytest_esp_system_linux.py +++ b/components/esp_system/test_apps/linux_apis/pytest_esp_system_linux.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 -import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_esp_system_linux(dut: Dut) -> None: dut.run_all_single_board_cases(timeout=60) diff --git a/components/fatfs/host_test/pytest_fatfs_linux.py b/components/fatfs/host_test/pytest_fatfs_linux.py index e14e927a349..142cf63a35f 100644 --- a/components/fatfs/host_test/pytest_fatfs_linux.py +++ b/components/fatfs/host_test/pytest_fatfs_linux.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 -import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_fatfs_linux(dut: Dut) -> None: dut.expect_exact('All tests passed', timeout=120) diff --git a/components/hal/test_apps/hal_utils/pytest_hal_utils.py b/components/hal/test_apps/hal_utils/pytest_hal_utils.py index 16d1a2b39fe..e277d02471e 100644 --- a/components/hal/test_apps/hal_utils/pytest_hal_utils.py +++ b/components/hal/test_apps/hal_utils/pytest_hal_utils.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 -import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_hal_utils(dut: Dut) -> None: dut.run_all_single_board_cases() diff --git a/components/heap/test_apps/heap_tests/pytest_heap.py b/components/heap/test_apps/heap_tests/pytest_heap.py index 879755c64f6..ab1bb25b12b 100644 --- a/components/heap/test_apps/heap_tests/pytest_heap.py +++ b/components/heap/test_apps/heap_tests/pytest_heap.py @@ -21,7 +21,6 @@ def test_heap_poisoning(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.host_test @pytest.mark.qemu @pytest.mark.parametrize( 'config, embedded_services', diff --git a/components/heap/test_apps/host_test_linux/pytest_heap_linux.py b/components/heap/test_apps/host_test_linux/pytest_heap_linux.py index 582cbf2f56b..16c7f15c72c 100644 --- a/components/heap/test_apps/host_test_linux/pytest_heap_linux.py +++ b/components/heap/test_apps/host_test_linux/pytest_heap_linux.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 -import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_heap_linux(dut: Dut) -> None: dut.run_all_single_board_cases(timeout=60) diff --git a/components/heap/test_multi_heap_host/pytest_multi_heap_linux.py b/components/heap/test_multi_heap_host/pytest_multi_heap_linux.py index 0883d9ff1d6..add65636d3a 100644 --- a/components/heap/test_multi_heap_host/pytest_multi_heap_linux.py +++ b/components/heap/test_multi_heap_host/pytest_multi_heap_linux.py @@ -5,7 +5,6 @@ from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @pytest.mark.parametrize( 'config', [ diff --git a/components/linux/test_apps/linux_test/pytest_linux_test.py b/components/linux/test_apps/linux_test/pytest_linux_test.py index 15cf390d9dc..459cf3d475c 100644 --- a/components/linux/test_apps/linux_test/pytest_linux_test.py +++ b/components/linux/test_apps/linux_test/pytest_linux_test.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 -import pytest from pytest_embedded_idf.dut import IdfDut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_linux_component(dut: IdfDut) -> None: dut.expect_exact('Press ENTER to see the list of tests.') diff --git a/components/log/host_test/log_test/pytest_log_linux.py b/components/log/host_test/log_test/pytest_log_linux.py index ffd06d72fbf..2fad7387e2c 100644 --- a/components/log/host_test/log_test/pytest_log_linux.py +++ b/components/log/host_test/log_test/pytest_log_linux.py @@ -5,7 +5,6 @@ from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @pytest.mark.parametrize( 'config', [ diff --git a/components/nvs_flash/host_test/nvs_host_test/pytest_nvs_host_linux.py b/components/nvs_flash/host_test/nvs_host_test/pytest_nvs_host_linux.py index df792a53783..c4dd2b375a0 100644 --- a/components/nvs_flash/host_test/nvs_host_test/pytest_nvs_host_linux.py +++ b/components/nvs_flash/host_test/nvs_host_test/pytest_nvs_host_linux.py @@ -5,7 +5,6 @@ from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @pytest.mark.parametrize( 'config', [ diff --git a/components/nvs_flash/host_test/nvs_page_test/pytest_nvs_page_linux.py b/components/nvs_flash/host_test/nvs_page_test/pytest_nvs_page_linux.py index 0e30e463bf6..9baefd3960a 100644 --- a/components/nvs_flash/host_test/nvs_page_test/pytest_nvs_page_linux.py +++ b/components/nvs_flash/host_test/nvs_page_test/pytest_nvs_page_linux.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 -import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_nvs_page_linux(dut: Dut) -> None: dut.expect_unity_test_output(timeout=120) diff --git a/components/pthread/test_apps/pthread_unity_tests/pytest_pthread_unity_tests.py b/components/pthread/test_apps/pthread_unity_tests/pytest_pthread_unity_tests.py index 2cf9732abdb..38806448ef2 100644 --- a/components/pthread/test_apps/pthread_unity_tests/pytest_pthread_unity_tests.py +++ b/components/pthread/test_apps/pthread_unity_tests/pytest_pthread_unity_tests.py @@ -47,7 +47,6 @@ def test_pthread_single_core_tls(dut: Dut) -> None: dut.run_all_single_board_cases(group='thread-specific', timeout=300) -@pytest.mark.host_test @pytest.mark.qemu @idf_parametrize('target', ['esp32'], indirect=['target']) def test_pthread_qemu(dut: Dut) -> None: @@ -56,7 +55,6 @@ def test_pthread_qemu(dut: Dut) -> None: dut._run_normal_case(case, timeout=75) -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_pthread_linux(dut: Dut) -> None: dut.run_all_single_board_cases(timeout=120) diff --git a/components/spiffs/host_test/pytest_spiffs_linux.py b/components/spiffs/host_test/pytest_spiffs_linux.py index 2d1ddfe6f71..64ab28e6625 100644 --- a/components/spiffs/host_test/pytest_spiffs_linux.py +++ b/components/spiffs/host_test/pytest_spiffs_linux.py @@ -5,7 +5,6 @@ from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @pytest.mark.parametrize('config', ['erase_check', 'no_erase_check']) @idf_parametrize('target', ['linux'], indirect=['target']) def test_spiffs_linux(dut: Dut) -> None: diff --git a/components/unity/test_apps/pytest_unit_test.py b/components/unity/test_apps/pytest_unit_test.py index bb592778a11..12762b80849 100644 --- a/components/unity/test_apps/pytest_unit_test.py +++ b/components/unity/test_apps/pytest_unit_test.py @@ -20,7 +20,6 @@ def test_unit_test_order(dut: Dut) -> None: verify_test_order(dut) -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_unit_test_order_linux(dut: Dut) -> None: verify_test_order(dut) diff --git a/components/wear_levelling/host_test/pytest_wear_levelling_linux.py b/components/wear_levelling/host_test/pytest_wear_levelling_linux.py index 6310ba3b65b..29a902b0021 100644 --- a/components/wear_levelling/host_test/pytest_wear_levelling_linux.py +++ b/components/wear_levelling/host_test/pytest_wear_levelling_linux.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 -import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_wear_levelling_linux(dut: Dut) -> None: dut.expect_exact('All tests passed', timeout=180) diff --git a/conftest.py b/conftest.py index 480dc06e739..6789cc602fa 100644 --- a/conftest.py +++ b/conftest.py @@ -44,6 +44,7 @@ from idf_pytest.plugin import IDF_LOCAL_PLUGIN_KEY from idf_pytest.plugin import IdfLocalPlugin from idf_pytest.plugin import requires_elf_or_map from idf_pytest.utils import format_case_id +from pytest_embedded.plugin import _request_param_or_config_option_or_default from pytest_embedded.plugin import multi_dut_fixture from pytest_embedded.utils import to_bytes from pytest_embedded.utils import to_str @@ -127,7 +128,7 @@ class AppDownloader: self.commit_sha = commit_sha self.pipeline_id = pipeline_id - def download_app(self, app_build_path: str, artifact_type: str | None = None) -> None: + def download_app(self, app_dir: str, build_dir: str, artifact_type: str | None = None) -> None: args = [ 'idf-ci', 'gitlab', @@ -137,15 +138,26 @@ class AppDownloader: ] if artifact_type: args.extend(['--type', artifact_type]) + if self.pipeline_id: args.extend(['--pipeline-id', self.pipeline_id]) - args.append(app_build_path) - subprocess.run( - args, - stdout=sys.stdout, - stderr=sys.stderr, + args.extend( + [ + app_dir, + '--build-dir', + build_dir, + ] ) + result = subprocess.run( + args, + capture_output=True, + text=True, + cwd=IDF_PATH, + ) + logging.info(result.stdout) + if result.stderr: + logging.info(result.stderr) def download_app_extra(self, app_dir: str) -> None: """Download app-dir artifacts defined under app_extra in .idf_ci.toml.""" @@ -350,12 +362,13 @@ def build_dir( downloader = app_downloader if downloader: + app_dir = idf_relpath(app_path) + build_dir = f'build_{target}_{config}' # somehow hardcoded... - app_build_path = os.path.join(idf_relpath(app_path), f'build_{target}_{config}') if requires_elf_or_map(case): - downloader.download_app(app_build_path) + downloader.download_app(app_dir, build_dir) else: - downloader.download_app(app_build_path, 'flash') + downloader.download_app(app_dir, build_dir, 'flash') check_dirs = [f'build_{target}_{config}'] else: check_dirs = [] @@ -539,12 +552,12 @@ def log_minimum_free_heap_size(dut: IdfDut, config: str, idf_path: str) -> t.Cal @pytest.fixture(scope='session') def dev_password(request: FixtureRequest) -> str: - return request.config.getoption('dev_passwd') or '' + return _request_param_or_config_option_or_default(request, 'dev_password', '') # type: ignore @pytest.fixture(scope='session') def dev_user(request: FixtureRequest) -> str: - return request.config.getoption('dev_user') or '' + return _request_param_or_config_option_or_default(request, 'dev_user', '') # type: ignore ################## diff --git a/docs/en/api-guides/tools/idf-tools.rst b/docs/en/api-guides/tools/idf-tools.rst index 6c167a1fd3d..992f359f342 100644 --- a/docs/en/api-guides/tools/idf-tools.rst +++ b/docs/en/api-guides/tools/idf-tools.rst @@ -45,6 +45,8 @@ Inside the ``IDF_TOOLS_PATH`` directory, the tools installation scripts create t - ``python_env`` — not related to the tools; virtual Python environments are installed in the sub-directories. Note that the Python environment directory can be placed elsewhere by setting the ``IDF_PYTHON_ENV_PATH`` environment variable. - ``idf_version.txt`` — located within each specific Python environment sub-directory under ``python_env``, this file records the ESP-IDF version corresponding to that environment. The version is stored in a format like ``5.3`` to represent ESP-IDF version ``v5.3``. + +- ``root_managed_components`` — directory managed by ``idf-component-manager`` for components installed globally. - ``espidf.constraints.*.txt`` — one constraint file for each ESP-IDF release containing Python package version requirements. GitHub Assets Mirror diff --git a/docs/en/contribute/esp-idf-tests-with-pytest.rst b/docs/en/contribute/esp-idf-tests-with-pytest.rst index f5d92ce8c59..9b04d8b2962 100644 --- a/docs/en/contribute/esp-idf-tests-with-pytest.rst +++ b/docs/en/contribute/esp-idf-tests-with-pytest.rst @@ -111,6 +111,52 @@ Next is the environment marker. The ``@pytest.mark.generic`` marker indicates th Finally, we have the test function. With a ``dut`` fixture. In single-dut test cases, the ``dut`` fixture is an instance of ``IdfDut`` class, for multi-dut test cases, it is a tuple of ``IdfDut`` instances. For more details regarding the ``IdfDut`` class, please refer to `pytest-embedded IdfDut API reference `__. +Running Tests on Linux +^^^^^^^^^^^^^^^^^^^^^^ + +To execute a pytest case on the Linux host, set ``target`` to ``linux``. + +.. code-block:: python + + @idf_parametrize('target', ['linux'], indirect=['target']) + def test_hello_world_linux(dut) -> None: + dut.expect('Hello world!') + +This is the simplest way to run the same test flow on the Linux host instead of on physical hardware. + +For a simple Linux-only test, setting ``target`` to ``linux`` is enough and the ``idf`` embedded services will be selected automatically. The ``pytest.mark.host_test`` marker is not required. + +For a mixed environment matrix, specify ``embedded_services`` manually for each case. See :ref:`Same App With Different Running Environments ` for a more complex example. + +.. only:: TARGET_SUPPORT_QEMU + + Running Tests in QEMU + ^^^^^^^^^^^^^^^^^^^^^ + + To execute a pytest case in QEMU, add the ``@pytest.mark.qemu`` marker to the test function. + + .. code-block:: python + + @pytest.mark.qemu + @idf_parametrize('target', ['esp32', 'esp32c3'], indirect=['target']) + def test_hello_world_qemu(dut) -> None: + dut.expect('Hello world!') + + This is the simplest way to run the same test flow in QEMU instead of on physical hardware. + + For a simple QEMU-only test, adding ``pytest.mark.qemu`` is enough and the ``idf,qemu`` embedded services will be selected automatically. + + For a mixed environment matrix, specify ``embedded_services`` manually for each case. See the later section in this guide for a more complex example. + + For QEMU installation and setup, refer to page :doc:`/api-guides/tools/qemu`. + +Deprecation of ``pytest.mark.host_test`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``pytest.mark.host_test`` is no longer needed and should not be added to new test cases. + +For Linux target test cases and QEMU test cases, the required behavior is handled dynamically by the test framework. In particular, the embedded services are selected automatically for simple Linux-only and QEMU-only cases. + Same App With Different sdkconfig Files ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -190,6 +236,49 @@ Now this test function would be replicated to 2 test cases (represented as test * ``esp32.foo.test_foo_bar`` * ``esp32s2.bar.test_foo_bar`` +.. _pytest-same-app-different-running-environments: + +Same App With Different Running Environments +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Sometimes the same app should be validated in different running environments, for example on the host with the Linux target, on real hardware, or in QEMU. If a single ``@pytest.mark.qemu`` test is not enough, combine ``target``, ``config``, and ``embedded_services`` in a single ``idf_parametrize`` decorator, and attach the required marker for each case. + +The following example is adapted from :idf_file:`components/console/test_apps/console/pytest_console.py`: + +.. code-block:: python + + @idf_parametrize( + 'target,config,embedded_services,markers', + [ + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), + ], + indirect=['target', 'config', 'embedded_services'], + ) + def test_console_repl(dut) -> None: + dut.expect_exact('Press ENTER to see the list of tests') + +This creates four test cases for the same app: + +* Linux host execution with the ``idf`` service +* ESP32 hardware execution with the ``esp,idf`` services +* ESP32-C3 hardware execution with the ``esp,idf`` services +* ESP32 execution in QEMU with the ``idf,qemu`` services + +When running locally, you can select only the environment you want: + +.. code-block:: shell + + $ pytest --target linux + $ pytest -m qemu + $ pytest -m qemu --target esp32 + +``pytest --target linux`` selects Linux target cases only. ``pytest -m qemu`` selects all QEMU-marked cases. ``pytest -m qemu --target esp32`` further limits the selection to QEMU cases for the ESP32 target. + +Use this pattern when the test logic is the same but the execution environment changes. + Testing Serial Output (Expecting) --------------------------------- diff --git a/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst b/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst index 3b99aacf6a6..a6999f9a3f7 100644 --- a/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst +++ b/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst @@ -111,6 +111,52 @@ ESP-IDF 在主机端使用 pytest 框架(以及一些 pytest 插件)来自 关于测试函数,使用了一个 ``dut`` fixture。在单一 DUT 测试用例中,``dut`` fixture 是 ``IdfDut`` 类的一个实例,对于多个 DUT 测试用例,它是 ``IdfDut`` 实例的一个元组。有关 ``IdfDut`` 类的更多详细信息,请参阅 `pytest-embedded IdfDut API 参考 `__。 +在 Linux 上运行测试 +^^^^^^^^^^^^^^^^^^^^ + +要在 Linux 主机上执行 pytest 测试用例,请将 ``target`` 设置为 ``linux``。 + +.. code-block:: python + + @idf_parametrize('target', ['linux'], indirect=['target']) + def test_hello_world_linux(dut) -> None: + dut.expect('Hello world!') + +这是在 Linux 主机上运行与物理硬件相同测试流程的最简单方式。 + +对于简单的纯 Linux 测试,只需将 ``target`` 设置为 ``linux``,系统会自动选择 ``idf`` 对应的 embedded services。``pytest.mark.host_test`` marker 不再需要。 + +对于混合运行环境矩阵,则需要为每个用例手动指定 ``embedded_services``。更复杂的示例请参阅 :ref:`在不同运行环境中运行相同的应用程序 ` 小节。 + +.. only:: TARGET_SUPPORT_QEMU + + 在 QEMU 中运行测试 + ^^^^^^^^^^^^^^^^^^^^^ + + 要在 QEMU 中执行 pytest 测试用例,请将 ``@pytest.mark.qemu`` 添加到测试函数上。 + + .. code-block:: python + + @pytest.mark.qemu + @idf_parametrize('target', ['esp32', 'esp32c3'], indirect=['target']) + def test_hello_world_qemu(dut) -> None: + dut.expect('Hello world!') + + 这是在 QEMU 中运行与物理硬件相同测试流程的最简单方式。 + + 对于简单的纯 QEMU 测试,只需添加 ``pytest.mark.qemu``,系统会自动选择 ``idf,qemu`` 对应的 embedded services。 + + 对于混合运行环境矩阵,则需要为每个用例手动指定 ``embedded_services``。更复杂的示例请参阅本指南后面的对应小节。 + + 有关 QEMU 的安装和配置,请参阅页面 :doc:`../api-guides/tools/qemu`。 + +``pytest.mark.host_test`` 的弃用说明 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``pytest.mark.host_test`` 已不再需要,也不应再添加到新的测试用例中。 + +对于 Linux target 测试用例和 QEMU 测试用例,相关行为会由测试框架动态处理。尤其是在简单的纯 Linux 或纯 QEMU 场景下,embedded services 会被自动选择。 + 使用不同的 sdkconfig 文件运行相同的应用程序 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -190,6 +236,49 @@ ESP-IDF 在主机端使用 pytest 框架(以及一些 pytest 插件)来自 * ``esp32.foo.test_foo_bar`` * ``esp32s2.bar.test_foo_bar`` +.. _pytest-same-app-different-running-environments: + +在不同运行环境中运行相同的应用程序 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +有时,同一个应用程序需要在不同的运行环境中进行验证,例如在 Linux target 的主机上、真实硬件上,或在 QEMU 中运行。如果单独使用 ``@pytest.mark.qemu`` 测试还不够,可以在一个 ``idf_parametrize`` 装饰器中组合 ``target``、``config`` 和 ``embedded_services``,并为每种情况附加所需的 marker。 + +下面的示例改编自 :idf_file:`components/console/test_apps/console/pytest_console.py`: + +.. code-block:: python + + @idf_parametrize( + 'target,config,embedded_services,markers', + [ + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), + ], + indirect=['target', 'config', 'embedded_services'], + ) + def test_console_repl(dut) -> None: + dut.expect_exact('Press ENTER to see the list of tests') + +这会为同一个应用程序生成 4 个测试用例: + +* 在 Linux 主机上使用 ``idf`` service 运行 +* 在 ESP32 硬件上使用 ``esp,idf`` services 运行 +* 在 ESP32-C3 硬件上使用 ``esp,idf`` services 运行 +* 在 QEMU 中以 ESP32 为目标,使用 ``idf,qemu`` services 运行 + +在本地运行时,可以按需只选择某一种运行环境: + +.. code-block:: shell + + $ pytest --target linux + $ pytest -m qemu + $ pytest -m qemu --target esp32 + +``pytest --target linux`` 只选择 Linux target 的测试用例。``pytest -m qemu`` 选择所有带有 QEMU marker 的测试用例。``pytest -m qemu --target esp32`` 会进一步把范围限制为目标芯片为 ESP32 的 QEMU 测试用例。 + +当测试逻辑相同,但执行环境不同的时候,可使用此模式。 + 测试串行输出 ^^^^^^^^^^^^^^^^ diff --git a/examples/build_system/cmake/import_lib/pytest_import_lib.py b/examples/build_system/cmake/import_lib/pytest_import_lib.py index 95f86b0807f..4bf5203c624 100644 --- a/examples/build_system/cmake/import_lib/pytest_import_lib.py +++ b/examples/build_system/cmake/import_lib/pytest_import_lib.py @@ -5,7 +5,6 @@ from pytest_embedded_idf.utils import idf_parametrize from pytest_embedded_qemu.dut import QemuDut -@pytest.mark.host_test @pytest.mark.qemu @idf_parametrize('target', ['esp32', 'esp32c3'], indirect=['target']) def test_pytest_host(dut: QemuDut) -> None: diff --git a/examples/get-started/hello_world/pytest_hello_world.py b/examples/get-started/hello_world/pytest_hello_world.py index 64f6bb8f5fb..65d4b05ff34 100644 --- a/examples/get-started/hello_world/pytest_hello_world.py +++ b/examples/get-started/hello_world/pytest_hello_world.py @@ -1,8 +1,8 @@ -# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 import hashlib import logging -from typing import Callable +from collections.abc import Callable import pytest from pytest_embedded_idf.dut import IdfDut @@ -18,13 +18,11 @@ def test_hello_world(dut: IdfDut, log_minimum_free_heap_size: Callable[..., None log_minimum_free_heap_size() -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_hello_world_linux(dut: IdfDut) -> None: dut.expect('Hello world!') -@pytest.mark.host_test @pytest.mark.macos @idf_parametrize('target', ['linux'], indirect=['target']) def test_hello_world_macos(dut: IdfDut) -> None: @@ -45,7 +43,6 @@ def verify_elf_sha256_embedding(app: QemuApp, sha256_reported: str) -> None: raise ValueError('ELF file SHA256 mismatch') -@pytest.mark.host_test @pytest.mark.qemu @idf_parametrize('target', ['esp32', 'esp32c3'], indirect=['target']) def test_hello_world_host(app: QemuApp, dut: QemuDut) -> None: diff --git a/examples/network/bridge/pytest_example_bridge.py b/examples/network/bridge/pytest_example_bridge.py index 50f5d16e290..eddcf6f447c 100644 --- a/examples/network/bridge/pytest_example_bridge.py +++ b/examples/network/bridge/pytest_example_bridge.py @@ -25,6 +25,8 @@ from pytest_embedded_idf.utils import idf_parametrize # Testbed configuration ETHVM_ENDNODE_USER = 'ci.ethvm' +ETHERNET_TEST_USER = os.getenv('ETHERNET_TEST_USER') +ETHERNET_TEST_PASSWORD = os.getenv('ETHERNET_TEST_PASSWORD') BR_PORTS_NUM = 2 IPERF_BW_LIM = 6 @@ -632,5 +634,10 @@ def setup_test_environment() -> Generator[None, None, None]: indirect=True, ) @idf_parametrize('target', ['esp32'], indirect=['target']) +@pytest.mark.parametrize( + 'dev_user, dev_password', + [(ETHERNET_TEST_USER, ETHERNET_TEST_PASSWORD)], + indirect=True, +) def test_esp_eth_bridge(dut: Dut, dev_user: str, dev_password: str) -> None: eth_bridge_test(dut, dev_user, dev_password) diff --git a/examples/protocols/esp_http_client/pytest_esp_http_client.py b/examples/protocols/esp_http_client/pytest_esp_http_client.py index 4d3404394a7..51a05aafe4f 100644 --- a/examples/protocols/esp_http_client/pytest_esp_http_client.py +++ b/examples/protocols/esp_http_client/pytest_esp_http_client.py @@ -111,7 +111,6 @@ def test_examples_protocol_esp_http_client_dynamic_buffer(dut: Dut) -> None: dut.expect('Finish http example') -@pytest.mark.host_test @pytest.mark.parametrize( 'config', [ diff --git a/examples/protocols/esp_http_client_mutual_auth/pytest_esp_http_client_mutual_auth.py b/examples/protocols/esp_http_client_mutual_auth/pytest_esp_http_client_mutual_auth.py index 8b49a6cd58b..ba7058441a2 100644 --- a/examples/protocols/esp_http_client_mutual_auth/pytest_esp_http_client_mutual_auth.py +++ b/examples/protocols/esp_http_client_mutual_auth/pytest_esp_http_client_mutual_auth.py @@ -186,7 +186,6 @@ def _run_mtls_test( # --------------------------------------------------------------------------- -@pytest.mark.host_test @pytest.mark.qemu @pytest.mark.parametrize('config', ['default'], indirect=True) @idf_parametrize('target', ['esp32c3'], indirect=['target']) @@ -203,7 +202,6 @@ def test_mutual_auth_software_keys(app: QemuApp, dut: QemuDut) -> None: # --------------------------------------------------------------------------- -@pytest.mark.host_test @pytest.mark.qemu @pytest.mark.parametrize('config', ['qemu_ds'], indirect=True) @idf_parametrize('target', ['esp32c3'], indirect=['target']) @@ -216,7 +214,6 @@ def test_mutual_auth_ds(app: QemuApp, dut: QemuDut) -> None: _run_mtls_test(dut, server_port=8070, client_ca=client_ca, expect_success=True) -@pytest.mark.host_test @pytest.mark.qemu @pytest.mark.parametrize('config', ['qemu_ds'], indirect=True) @idf_parametrize('target', ['esp32c3'], indirect=['target']) diff --git a/examples/protocols/https_x509_bundle/pytest_https_x509_bundle.py b/examples/protocols/https_x509_bundle/pytest_https_x509_bundle.py index 8d3e8fc5384..84db16ed903 100644 --- a/examples/protocols/https_x509_bundle/pytest_https_x509_bundle.py +++ b/examples/protocols/https_x509_bundle/pytest_https_x509_bundle.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 import logging import os @@ -52,7 +52,6 @@ def test_examples_protocol_https_x509_bundle_dynamic_buffer(dut: Dut) -> None: @pytest.mark.qemu -@pytest.mark.host_test @pytest.mark.parametrize( 'config', [ diff --git a/examples/system/efuse/pytest_system_efuse_example.py b/examples/system/efuse/pytest_system_efuse_example.py index 55d091f7499..9f5fa4a3a1d 100644 --- a/examples/system/efuse/pytest_system_efuse_example.py +++ b/examples/system/efuse/pytest_system_efuse_example.py @@ -43,7 +43,6 @@ def test_examples_efuse(dut: Dut) -> None: basic_efuse_example(dut) -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_examples_efuse_linux(dut: Dut) -> None: basic_efuse_example(dut) @@ -900,7 +899,6 @@ def test_examples_efuse_with_virt_sb_v2_and_fe(dut: Dut) -> None: dut.expect('example: Done') -@pytest.mark.host_test @pytest.mark.qemu @pytest.mark.parametrize( 'config', diff --git a/examples/system/ota/advanced_https_ota/pytest_advanced_ota.py b/examples/system/ota/advanced_https_ota/pytest_advanced_ota.py index 5c47ebd212a..7f820e46d64 100644 --- a/examples/system/ota/advanced_https_ota/pytest_advanced_ota.py +++ b/examples/system/ota/advanced_https_ota/pytest_advanced_ota.py @@ -1129,7 +1129,6 @@ def test_examples_protocol_advanced_https_ota_example_openssl_aligned_bin(dut: D @pytest.mark.qemu @pytest.mark.nightly_run -@pytest.mark.host_test @pytest.mark.parametrize( 'qemu_extra_args', [ @@ -1185,7 +1184,6 @@ def test_examples_protocol_advanced_https_ota_example_verify_min_chip_revision(d @pytest.mark.qemu @pytest.mark.nightly_run -@pytest.mark.host_test @pytest.mark.parametrize( 'qemu_extra_args', [ diff --git a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml index 0e50de03a34..f5fc1cbbea6 100644 --- a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml +++ b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml @@ -52,7 +52,7 @@ # set while generating the pipeline nodes: "" INSTALL_EXTRA_TOOLS: "xtensa-esp-elf-gdb riscv32-esp-elf-gdb openocd-esp32 esp-rom-elfs" - PYTEST_EXTRA_FLAGS: "--dev-passwd ${ETHERNET_TEST_PASSWORD} --dev-user ${ETHERNET_TEST_USER} --capture=fd --verbosity=0 --unity-test-report-mode merge" + PYTEST_EXTRA_FLAGS: "--capture=fd --verbosity=0 --unity-test-report-mode merge" needs: - pipeline: $PARENT_PIPELINE_ID job: pipeline_variables @@ -77,7 +77,6 @@ - run_cmd python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs # CI specific options start from "--known-failure-cases-file xxx". could ignore when running locally - run_cmd pytest $nodes - --pipeline-id $PARENT_PIPELINE_ID --junitxml=XUNIT_RESULT_${CI_JOB_ID}.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} --parallel-count ${CI_NODE_TOTAL:-1} diff --git a/tools/ci/idf_ci_local/app.py b/tools/ci/idf_ci_local/app.py index ad65e8f7a08..2f5b2a63c5f 100644 --- a/tools/ci/idf_ci_local/app.py +++ b/tools/ci/idf_ci_local/app.py @@ -8,6 +8,7 @@ import typing as t from dynamic_pipelines.constants import BINARY_SIZE_METRIC_NAME from idf_build_apps import App from idf_build_apps import CMakeApp +from idf_build_apps.constants import BuildStatus from idf_build_apps.utils import rmdir from idf_ci_utils import APP_EXTRA_S3_ARTIFACT_TYPE from idf_ci_utils import idf_relpath @@ -36,6 +37,8 @@ class IdfCMakeApp(CMakeApp): 'gitlab', 'upload-artifacts', self.app_dir, + '--build-dir', + self.build_dir, ], [ 'idf-ci', @@ -48,11 +51,14 @@ class IdfCMakeApp(CMakeApp): ] for command in upload_commands: - subprocess.run( + result = subprocess.run( command, stdout=sys.stdout, stderr=sys.stderr, ) + if result.returncode != 0: + self.build_status = BuildStatus.FAILED + self.build_comment = 'Failed to upload artifacts' rmdir( self.build_path, diff --git a/tools/ci/idf_pytest/plugin.py b/tools/ci/idf_pytest/plugin.py index 051beb18e10..f84d2dc6820 100644 --- a/tools/ci/idf_pytest/plugin.py +++ b/tools/ci/idf_pytest/plugin.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import os import typing as t @@ -9,6 +9,7 @@ import pytest import yaml from _pytest.config import Config from _pytest.python import Function +from _pytest.python import Metafunc from _pytest.runner import CallInfo from dynamic_pipelines.constants import KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH from idf_ci import IdfPytestPlugin @@ -114,6 +115,55 @@ class IdfLocalPlugin: return item.callspec.params.get(key, default) or default + @staticmethod + def _has_parametrized_arg(metafunc: Metafunc, arg_name: str) -> bool: + for marker in metafunc.definition.iter_markers(name='parametrize'): + if not marker.args: + continue + + argnames = marker.args[0] + if isinstance(argnames, str): + names = [name.strip() for name in argnames.split(',')] + else: + names = list(argnames) + + if arg_name in names: + return True + + for callspec in getattr(metafunc, '_calls', []): + if arg_name in callspec.params: + return True + + return False + + @staticmethod + def _is_linux_target_run(config: Config) -> bool: + target = config.getoption('target') + if not target: + return False + + if isinstance(target, str): + targets = [_t.strip() for _t in target.split(',')] + else: + targets = [str(_t).strip() for _t in target] + + return 'linux' in targets + + @pytest.hookimpl(trylast=True) + def pytest_generate_tests(self, metafunc: Metafunc) -> None: + if 'embedded_services' not in metafunc.fixturenames: + return + + if self._has_parametrized_arg(metafunc, 'embedded_services'): + return + + if metafunc.definition.get_closest_marker('qemu') is not None: + metafunc.parametrize('embedded_services', ['idf,qemu'], indirect=True) + return + + if self._is_linux_target_run(metafunc.config): + metafunc.parametrize('embedded_services', ['idf'], indirect=True) + @pytest.hookimpl(wrapper=True) def pytest_collection_modifyitems(self, config: Config, items: list[Function]) -> t.Generator[None, None, None]: yield # throw it back to idf-ci diff --git a/tools/requirements/requirements.ci.txt b/tools/requirements/requirements.ci.txt index 2865254683f..7874fbabdd4 100644 --- a/tools/requirements/requirements.ci.txt +++ b/tools/requirements/requirements.ci.txt @@ -6,7 +6,7 @@ # https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/tools/idf-tools.html # ci -idf-ci>=0.3,<1 +idf-ci coverage jsonschema diff --git a/tools/test_apps/linux_compatible/linux_freertos/pytest_linux_freertos.py b/tools/test_apps/linux_compatible/linux_freertos/pytest_linux_freertos.py index d4eb999dbdf..2bbd57d2bbd 100644 --- a/tools/test_apps/linux_compatible/linux_freertos/pytest_linux_freertos.py +++ b/tools/test_apps/linux_compatible/linux_freertos/pytest_linux_freertos.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 -import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_linux_freertos_SMP(dut: Dut) -> None: dut.expect_exact('Press ENTER to see the list of tests.') diff --git a/tools/test_apps/security/secure_boot/pytest_secure_boot.py b/tools/test_apps/security/secure_boot/pytest_secure_boot.py index f8f841aaf78..a1a233d7663 100644 --- a/tools/test_apps/security/secure_boot/pytest_secure_boot.py +++ b/tools/test_apps/security/secure_boot/pytest_secure_boot.py @@ -164,7 +164,6 @@ def test_examples_security_secure_boot_ecdsa(dut: Dut) -> None: # Test secure boot flow. # Correctly signed bootloader + correctly signed app should work -@pytest.mark.host_test @pytest.mark.qemu @pytest.mark.parametrize( 'qemu_extra_args', diff --git a/tools/test_apps/security/signed_app_no_secure_boot/pytest_signed_app_no_secure_boot.py b/tools/test_apps/security/signed_app_no_secure_boot/pytest_signed_app_no_secure_boot.py index 9dde2e391a5..101f971a65d 100644 --- a/tools/test_apps/security/signed_app_no_secure_boot/pytest_signed_app_no_secure_boot.py +++ b/tools/test_apps/security/signed_app_no_secure_boot/pytest_signed_app_no_secure_boot.py @@ -6,7 +6,6 @@ from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @pytest.mark.qemu @pytest.mark.parametrize('config', ['secure_update_with_fe'], indirect=True) @idf_parametrize('target', ['esp32c3'], indirect=['target']) diff --git a/tools/test_apps/storage/std_filesystem/pytest_std_filesystem.py b/tools/test_apps/storage/std_filesystem/pytest_std_filesystem.py index 75676116044..86031bc508f 100644 --- a/tools/test_apps/storage/std_filesystem/pytest_std_filesystem.py +++ b/tools/test_apps/storage/std_filesystem/pytest_std_filesystem.py @@ -6,7 +6,6 @@ from pytest_embedded_idf.utils import idf_parametrize @pytest.mark.qemu -@pytest.mark.host_test @idf_parametrize('target', ['esp32', 'esp32c3'], indirect=['target']) def test_std_filesystem(dut: Dut) -> None: dut.expect_exact('All tests passed', timeout=200) diff --git a/tools/test_apps/system/esp_intr_dump/pytest_esp_intr_dump.py b/tools/test_apps/system/esp_intr_dump/pytest_esp_intr_dump.py index cccfdbedeb9..e03da57cc20 100644 --- a/tools/test_apps/system/esp_intr_dump/pytest_esp_intr_dump.py +++ b/tools/test_apps/system/esp_intr_dump/pytest_esp_intr_dump.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 import os @@ -10,7 +10,6 @@ PROMPT = 'test_intr_dump>' @pytest.mark.qemu -@pytest.mark.host_test @idf_parametrize('target', ['esp32'], indirect=['target']) def test_esp_intr_dump_nonshared(dut: Dut) -> None: dut.expect_exact(PROMPT, timeout=30) @@ -24,7 +23,6 @@ def test_esp_intr_dump_nonshared(dut: Dut) -> None: @pytest.mark.qemu -@pytest.mark.host_test @idf_parametrize('target', ['esp32'], indirect=['target']) def test_esp_intr_dump_shared(dut: Dut) -> None: dut.expect_exact(PROMPT, timeout=30) @@ -54,5 +52,5 @@ def test_esp_intr_dump_expected_output(dut: Dut) -> None: dut.expect_exact(PROMPT, timeout=30) dut.write('intr_dump\n') exp_out_file = os.path.join(os.path.dirname(__file__), 'expected_output', f'{dut.target}.txt') - for line in open(exp_out_file, 'r').readlines(): + for line in open(exp_out_file).readlines(): dut.expect_exact(line.strip()) diff --git a/tools/test_apps/system/mmu_page_size/pytest_mmu_page_size.py b/tools/test_apps/system/mmu_page_size/pytest_mmu_page_size.py index 09d1b94ab59..88492e046f2 100644 --- a/tools/test_apps/system/mmu_page_size/pytest_mmu_page_size.py +++ b/tools/test_apps/system/mmu_page_size/pytest_mmu_page_size.py @@ -1,9 +1,8 @@ -# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2024-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 import os import pytest -from idf_ci_utils import IDF_PATH from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize @@ -16,15 +15,12 @@ def test_app_mmu_page_size_32k_and_bootloader_mmu_page_size_64k(dut: Dut, app_do assert '32K' in config app_config = config.replace('32K', '64K') + build_dir = f'build_{dut.target}_{app_config}' - path_to_mmu_page_size_64k_build = os.path.join(dut.app.app_path, f'build_{dut.target}_{app_config}') if app_downloader: - app_downloader.download_app( - os.path.relpath(path_to_mmu_page_size_64k_build, IDF_PATH), - 'flash', - ) + app_downloader.download_app(dut.app.app_path, build_dir, 'flash') - dut.serial.bootloader_flash(path_to_mmu_page_size_64k_build) + dut.serial.bootloader_flash(os.path.join(dut.app.app_path, build_dir)) dut.expect('MMU page size mismatch') dut.expect('App is running') dut.expect('Partition test done') @@ -39,15 +35,12 @@ def test_app_mmu_page_size_64k_and_bootloader_mmu_page_size_32k(dut: Dut, app_do assert '64K' in config app_config = config.replace('64K', '32K') + build_dir = f'build_{dut.target}_{app_config}' - path_to_mmu_page_size_32k_build = os.path.join(dut.app.app_path, f'build_{dut.target}_{app_config}') if app_downloader: - app_downloader.download_app( - os.path.relpath(path_to_mmu_page_size_32k_build, IDF_PATH), - 'flash', - ) + app_downloader.download_app(dut.app.app_path, build_dir, 'flash') - dut.serial.bootloader_flash(path_to_mmu_page_size_32k_build) + dut.serial.bootloader_flash(os.path.join(dut.app.app_path, build_dir)) dut.expect('MMU page size mismatch') dut.expect('App is running') dut.expect('Partition test done') diff --git a/tools/test_apps/system/unicore_bootloader/pytest_unicore_bootloader.py b/tools/test_apps/system/unicore_bootloader/pytest_unicore_bootloader.py index 0fde23758a0..de5222c79cc 100644 --- a/tools/test_apps/system/unicore_bootloader/pytest_unicore_bootloader.py +++ b/tools/test_apps/system/unicore_bootloader/pytest_unicore_bootloader.py @@ -1,10 +1,9 @@ -# SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 import os import re import pytest -from idf_ci_utils import IDF_PATH from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize from pytest_embedded_idf.utils import soc_filtered_targets @@ -20,15 +19,12 @@ def test_multicore_app_and_unicore_bootloader(dut: Dut, app_downloader, config) assert 'multicore' in config app_config = config.replace('multicore', 'unicore') + build_dir = f'build_{dut.target}_{app_config}' - path_to_unicore_build = os.path.join(dut.app.app_path, f'build_{dut.target}_{app_config}') if app_downloader: - app_downloader.download_app( - os.path.relpath(path_to_unicore_build, IDF_PATH), - 'flash', - ) + app_downloader.download_app(dut.app.app_path, build_dir, 'flash') - dut.serial.bootloader_flash(path_to_unicore_build) + dut.serial.bootloader_flash(os.path.join(dut.app.app_path, build_dir)) dut.expect('Unicore bootloader') dut.expect('Multicore app') if 'psram' in config: @@ -47,15 +43,12 @@ def test_unicore_app_and_multicore_bootloader(dut: Dut, app_downloader, config) assert 'unicore' in config app_config = config.replace('unicore', 'multicore') + build_dir = f'build_{dut.target}_{app_config}' - path_to_multicore_build = os.path.join(dut.app.app_path, f'build_{dut.target}_{app_config}') if app_downloader: - app_downloader.download_app( - os.path.relpath(path_to_multicore_build, IDF_PATH), - 'flash', - ) + app_downloader.download_app(dut.app.app_path, build_dir, 'flash') - dut.serial.bootloader_flash(path_to_multicore_build) + dut.serial.bootloader_flash(os.path.join(dut.app.app_path, build_dir)) dut.expect('Multicore bootloader') dut.expect('Unicore app') if 'psram' in config: diff --git a/tools/test_build_system/README.md b/tools/test_build_system/README.md index 0bdf25cc358..c41d92613a4 100644 --- a/tools/test_build_system/README.md +++ b/tools/test_build_system/README.md @@ -149,6 +149,20 @@ def test_target_guessing() def test_target_guessing() ``` +### `pytest.mark.revert_later` Marker + +This marker reverts all files to their original state after the test is finished. should pass a list of file paths (absolute or relative to `IDF_PATH`) to the marker. The files will be reverted even if the test fails. + +```python +@pytest.mark.revert_later(['tools/idf_extra_components.yml']) +def test_modify_file(idf_copy): + path = os.path.join(os.getenv('IDF_PATH'), 'tools', 'idf_extra_components.yml') + with open(path, 'a') as f: + f.write('# some changes\n') + # The changes to idf_extra_components.yml will be reverted after the test +``` + + ### Build snapshots `get_snapshot(list_of_globs)` function takes a list of glob expressions, finds the files matching these expressions, and returns a `Snapshot` instance. `Snapshot` instances record file names and their modification timestamps. Two `Snapshot` instances can be compared using `assert_same` and `assert_different` methods: diff --git a/tools/test_build_system/pytest.ini b/tools/test_build_system/pytest.ini index 0e6a45443be..7da2e164f9c 100644 --- a/tools/test_build_system/pytest.ini +++ b/tools/test_build_system/pytest.ini @@ -14,6 +14,9 @@ junit_family = xunit1 junit_logging = stdout junit_log_passing_tests = False +filterwarnings = + ignore::pytest.PytestExperimentalApiWarning + ## !! When adding new markers, don't forget to update also the tools\test_build_system\README.md !! markers = test_app_copy: specify relative path of the app to copy, and the prefix of the destination directory name diff --git a/tools/test_idf_py/test_hints.py b/tools/test_idf_py/test_hints.py index 110d5327a07..4459c6e499e 100755 --- a/tools/test_idf_py/test_hints.py +++ b/tools/test_idf_py/test_hints.py @@ -33,7 +33,8 @@ def safe_cleanup_tmpdir(tmpdir: tempfile.TemporaryDirectory) -> None: """Safely cleanup temporary directory, handling specific errors on Windows.""" try: tmpdir.cleanup() - except (PermissionError, NotADirectoryError): + except (PermissionError, NotADirectoryError, OSError): + # OSError: The directory is not empty warnings.warn( f'Failed to cleanup temporary directory {tmpdir.name}. ' 'This is common on Windows when files are still in use.', diff --git a/tools/test_idf_py/test_idf_py.py b/tools/test_idf_py/test_idf_py.py index f48d890821e..bfd8625b0a3 100755 --- a/tools/test_idf_py/test_idf_py.py +++ b/tools/test_idf_py/test_idf_py.py @@ -27,9 +27,9 @@ except ImportError: import idf current_dir = os.path.dirname(os.path.realpath(__file__)) -idf_py_path = os.path.join(current_dir, '..', 'idf.py') +idf_py_path = os.path.normpath(os.path.join(current_dir, '..', 'idf.py')) extension_path = os.path.join(current_dir, 'test_idf_extensions', 'test_ext') -py_actions_path = os.path.join(current_dir, '..', 'idf_py_actions') +py_actions_path = os.path.normpath(os.path.join(current_dir, '..', 'idf_py_actions')) link_path = os.path.join(py_actions_path, 'test_ext')