From b47f9850c7103b269d7cf58b69e2bb2a9cfb1f99 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Mon, 17 Aug 2026 14:13:15 +0200 Subject: [PATCH] ci: apply common-scripts CI refactor Squashes these original commits for release/v5.4 backport traceability: - 9b289dc5ad6 ci: apply idf-ci 1.x - 381df980d57 ci: remove pip-cache and other unused jobs - 32375a0a15e ci: apply common-scripts CI refactor --- .gitlab-ci.yml | 65 +- .gitlab/ci/README.md | 82 +- .gitlab/ci/build.yml | 370 ++------- .gitlab/ci/common.yml | 347 +------- .gitlab/ci/default-build-test-rules.yml | 20 - .gitlab/ci/dependencies/dependencies.yml | 19 - .gitlab/ci/deploy.yml | 48 -- .gitlab/ci/host-test.yml | 421 +++++----- .gitlab/ci/post_deploy.yml | 13 - .gitlab/ci/pre_check.yml | 108 +-- .gitlab/ci/pre_commit.yml | 48 +- .gitlab/ci/pre_deploy.yml | 24 + .gitlab/ci/retry_failed_jobs.yml | 15 - .gitlab/ci/rules.yml | 107 +-- .gitlab/ci/static-code-analysis.yml | 111 +-- .gitlab/ci/test-win.yml | 103 +-- .gitlab/ci/upload_cache.yml | 20 - .idf_build_apps.toml | 63 ++ .idf_ci.toml | 116 +++ .pre-commit-config.yaml | 52 +- .../pytest_bootloader_support.py | 2 - .../test_apps/console/pytest_console.py | 166 ++-- components/efuse/test_apps/pytest_efuse.py | 5 +- .../test_apps/pytest_esp_app_format.py | 1 - .../pytest_external_coex_function.py | 16 +- ...l26m => sdkconfig.ci.esp32c2_rev2_xtal26m} | 0 ....esp32c3eco7 => sdkconfig.ci.esp32c3_rev1} | 0 .../pytest_esp_event_linux.py | 2 - .../esp_event/test_apps/pytest_esp_event.py | 7 +- .../pytest_esp_hw_support_linux.py | 2 - .../sleep_retention/pytest_retention.py | 24 - ...onfig.ci.defaults => sdkconfig.ci.default} | 0 components/esp_mm/test_apps/mm/pytest_mmap.py | 15 +- .../pytest_esp_partition_linux.py | 2 - .../test_apps/pytest_esp_ringbuf.py | 2 - .../linux_rom_apis/pytest_esp_rom_linux.py | 2 - .../test_apps/rom_tests/pytest_esp_rom.py | 12 +- .../pytest_esp_system_console_tests.py | 9 - .../linux_apis/pytest_esp_system_linux.py | 2 - .../test_apps/pytest_esp_timer_ut.py | 13 - .../wifi_connect/pytest_wifi_connect.py | 12 +- ...l26m => sdkconfig.ci.esp32c2_rev2_xtal26m} | 0 ....esp32c3eco7 => sdkconfig.ci.esp32c3_rev1} | 0 .../wifi_function/pytest_wifi_function.py | 13 +- ...l26m => sdkconfig.ci.esp32c2_rev2_xtal26m} | 0 ....esp32c3eco7 => sdkconfig.ci.esp32c3_rev1} | 0 .../fatfs/host_test/pytest_fatfs_linux.py | 2 - .../test_apps/freertos/pytest_freertos.py | 25 - .../test_apps/hal_utils/pytest_hal_utils.py | 2 - .../heap/test_apps/heap_tests/pytest_heap.py | 7 +- .../host_test_linux/pytest_heap_linux.py | 2 - .../test_apps/linux_test/pytest_linux_test.py | 2 - .../host_test/log_test/pytest_log_linux.py | 4 +- .../nvs_host_test/pytest_nvs_host_linux.py | 11 - .../nvs_page_test/pytest_nvs_page_linux.py | 2 - .../nvs_flash/test_apps/pytest_nvs_flash.py | 5 - .../pytest_pthread_unity_tests.py | 6 +- .../spiffs/host_test/pytest_spiffs_linux.py | 1 - .../test_apps/pytest_tcp_transport_ut.py | 5 - .../usb/test_apps/hcd/pytest_usb_hcd.py | 5 - .../host_test/pytest_wear_levelling_linux.py | 2 - .../test_apps/pytest_wpa_supplicant_ut.py | 12 +- ...l26m => sdkconfig.ci.esp32c2_rev2_xtal26m} | 0 ....esp32c3eco7 => sdkconfig.ci.esp32c3_rev1} | 0 conftest.py | 285 +++---- .../contribute/esp-idf-tests-with-pytest.rst | 97 ++- .../contribute/esp-idf-tests-with-pytest.rst | 97 ++- ...l26m => sdkconfig.ci.esp32c2_rev2_xtal26m} | 0 ....esp32c3eco7 => sdkconfig.ci.esp32c3_rev1} | 0 examples/bluetooth/nimble/blemesh/README.md | 2 +- ...l26m => sdkconfig.ci.esp32c2_rev2_xtal26m} | 0 ....esp32c3eco7 => sdkconfig.ci.esp32c3_rev1} | 0 .../bluetooth/nimble/pytest_nimble_test.py | 30 +- .../cmake/import_lib/pytest_import_lib.py | 1 - .../hello_world/pytest_hello_world.py | 9 +- .../network/bridge/pytest_example_bridge.py | 38 +- examples/openthread/pytest_otbr.py | 5 +- .../usb/host/msc/pytest_usb_host_msc.py | 5 - .../esp_http_client/pytest_esp_http_client.py | 5 +- .../pytest_https_x509_bundle.py | 17 +- examples/protocols/sockets/README.md | 8 +- .../system/deep_sleep/pytest_deep_sleep.py | 26 +- .../efuse/pytest_system_efuse_example.py | 150 +++- ...2eco3_efuses.bin => esp32_rev3_efuses.bin} | Bin .../pytest_wifi_getting_started.py | 13 +- ...l26m => sdkconfig.ci.esp32c2_rev2_xtal26m} | 0 ...ig.ci.c3eco7 => sdkconfig.ci.esp32c3_rev1} | 0 ...l26m => sdkconfig.ci.esp32c2_rev2_xtal26m} | 0 ...ig.ci.c3eco7 => sdkconfig.ci.esp32c3_rev1} | 0 .../wifi/power_save/pytest_wifi_power_save.py | 12 +- ...l26m => sdkconfig.ci.esp32c2_rev2_xtal26m} | 0 ...ig.ci.c3eco7 => sdkconfig.ci.esp32c3_rev1} | 0 pytest.ini | 97 ++- ruff.toml | 48 ++ sonar-project.properties | 24 - tools/ci/build_template_app.sh | 18 +- tools/ci/check_build_test_rules.py | 666 +++++++--------- tools/ci/ci_get_mr_info.py | 165 ---- tools/ci/deploy_docs.py | 217 ----- tools/ci/dynamic_pipelines/constants.py | 42 +- tools/ci/dynamic_pipelines/models.py | 226 ------ tools/ci/dynamic_pipelines/report.py | 739 ------------------ .../scripts/child_pipeline_build_apps.py | 74 -- .../scripts/generate_build_child_pipeline.py | 257 ------ .../scripts/generate_report.py | 101 --- .../generate_target_test_child_pipeline.py | 203 ++--- .../templates/.dynamic_jobs.yml | 67 +- .../templates/fast_pipeline.yml | 16 - .../templates/generate_target_test_report.yml | 18 - ..._generate_test_child_pipeline_warnings.yml | 24 +- .../templates/report.template.html | 132 ---- .../templates/retry-jobs.png | Bin 17552 -> 0 bytes .../templates/test_child_pipeline.yml | 39 +- tools/ci/dynamic_pipelines/tests/__init__.py | 0 .../tests/test_report_generator/__init__.py | 0 .../reports_sample_data/XUNIT_REPORT.xml | 200 ----- .../expected_job_report.html | 170 ---- .../expected_target_test_report.html | 335 -------- .../failure_rate_jobs_response.json | 7 - .../jobs_api_response.json | 212 ----- .../test_report_generator.py | 152 ---- tools/ci/dynamic_pipelines/utils.py | 218 ------ tools/ci/exclude_check_tools_files.txt | 4 - tools/ci/executable-list.txt | 1 - tools/ci/generate_rules.py | 123 ++- tools/ci/get_all_test_results.py | 84 -- tools/ci/gitlab_yaml_linter.py | 42 +- tools/ci/idf_ci_local/app.py | 55 +- tools/ci/idf_ci_local/uploader.py | 169 ---- tools/ci/idf_ci_utils.py | 234 +++--- tools/ci/idf_pytest/constants.py | 348 +-------- tools/ci/idf_pytest/plugin.py | 404 ++++------ tools/ci/idf_pytest/pytest.ini | 3 - tools/ci/idf_pytest/script.py | 236 ------ tools/ci/idf_pytest/utils.py | 41 +- tools/ci/previous_stage_job_status.py | 36 - tools/ci/python_packages/gitlab_api.py | 75 +- tools/ci/utils.sh | 63 +- tools/esp_prov/README.md | 2 +- tools/install_util.py | 14 +- tools/requirements.json | 6 - tools/requirements/requirements.ci.txt | 7 +- tools/requirements/requirements.pytest.txt | 21 - .../pytest_ld_non_contiguous_memory.py | 3 - .../linux_freertos/pytest_linux_freertos.py | 2 - .../phy/phy_tsens/pytest_phy_tsens.py | 12 +- ...g.ci.c2_eco4 => sdkconfig.ci.esp32c2_rev2} | 0 ...g.ci.c3_eco7 => sdkconfig.ci.esp32c3_rev1} | 0 .../test_apps/security/secure_boot/README.md | 2 +- .../std_filesystem/pytest_std_filesystem.py | 1 - .../esp_intr_dump/pytest_esp_intr_dump.py | 4 +- .../mmu_page_size/pytest_mmu_page_size.py | 23 +- tools/test_apps/system/panic/pytest_panic.py | 161 +--- .../pytest_unicore_bootloader.py | 18 +- tools/test_build_system/README.md | 2 +- tools/test_build_system/pytest.ini | 3 + 156 files changed, 2376 insertions(+), 7642 deletions(-) delete mode 100644 .gitlab/ci/default-build-test-rules.yml delete mode 100644 .gitlab/ci/deploy.yml delete mode 100644 .gitlab/ci/post_deploy.yml create mode 100644 .gitlab/ci/pre_deploy.yml delete mode 100644 .gitlab/ci/retry_failed_jobs.yml create mode 100644 .idf_build_apps.toml create mode 100644 .idf_ci.toml rename components/esp_coex/test_apps/external_coex_function/{sdkconfig.ci.esp32c2eco4_xtal26m => sdkconfig.ci.esp32c2_rev2_xtal26m} (100%) rename components/esp_coex/test_apps/external_coex_function/{sdkconfig.ci.esp32c3eco7 => sdkconfig.ci.esp32c3_rev1} (100%) rename components/esp_hw_support/test_apps/sleep_retention/{sdkconfig.ci.defaults => sdkconfig.ci.default} (100%) rename components/esp_wifi/test_apps/wifi_connect/{sdkconfig.ci.esp32c2eco4_xtal26m => sdkconfig.ci.esp32c2_rev2_xtal26m} (100%) rename components/esp_wifi/test_apps/wifi_connect/{sdkconfig.ci.esp32c3eco7 => sdkconfig.ci.esp32c3_rev1} (100%) rename components/esp_wifi/test_apps/wifi_function/{sdkconfig.ci.esp32c2eco4_xtal26m => sdkconfig.ci.esp32c2_rev2_xtal26m} (100%) rename components/esp_wifi/test_apps/wifi_function/{sdkconfig.ci.esp32c3eco7 => sdkconfig.ci.esp32c3_rev1} (100%) delete mode 100644 components/nvs_flash/host_test/nvs_host_test/pytest_nvs_host_linux.py rename components/wpa_supplicant/test_apps/{sdkconfig.ci.esp32c2eco4_xtal26m => sdkconfig.ci.esp32c2_rev2_xtal26m} (100%) rename components/wpa_supplicant/test_apps/{sdkconfig.ci.esp32c3eco7 => sdkconfig.ci.esp32c3_rev1} (100%) rename examples/bluetooth/nimble/blecent/{sdkconfig.ci.esp32c2eco4_xtal26m => sdkconfig.ci.esp32c2_rev2_xtal26m} (100%) rename examples/bluetooth/nimble/blecent/{sdkconfig.ci.esp32c3eco7 => sdkconfig.ci.esp32c3_rev1} (100%) rename examples/bluetooth/nimble/power_save/{sdkconfig.ci.esp32c2eco4_xtal26m => sdkconfig.ci.esp32c2_rev2_xtal26m} (100%) rename examples/bluetooth/nimble/power_save/{sdkconfig.ci.esp32c3eco7 => sdkconfig.ci.esp32c3_rev1} (100%) rename examples/system/efuse/test/{esp32eco3_efuses.bin => esp32_rev3_efuses.bin} (100%) rename examples/wifi/getting_started/softAP/{sdkconfig.ci.esp32c2eco4_xtal26m => sdkconfig.ci.esp32c2_rev2_xtal26m} (100%) rename examples/wifi/getting_started/softAP/{sdkconfig.ci.c3eco7 => sdkconfig.ci.esp32c3_rev1} (100%) rename examples/wifi/getting_started/station/{sdkconfig.ci.esp32c2eco4_xtal26m => sdkconfig.ci.esp32c2_rev2_xtal26m} (100%) rename examples/wifi/getting_started/station/{sdkconfig.ci.c3eco7 => sdkconfig.ci.esp32c3_rev1} (100%) rename examples/wifi/power_save/{sdkconfig.ci.c2eco4_xtal26m => sdkconfig.ci.esp32c2_rev2_xtal26m} (100%) rename examples/wifi/power_save/{sdkconfig.ci.c3eco7 => sdkconfig.ci.esp32c3_rev1} (100%) create mode 100644 ruff.toml delete mode 100644 sonar-project.properties delete mode 100644 tools/ci/ci_get_mr_info.py delete mode 100755 tools/ci/deploy_docs.py delete mode 100644 tools/ci/dynamic_pipelines/models.py delete mode 100644 tools/ci/dynamic_pipelines/report.py delete mode 100644 tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py delete mode 100644 tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py delete mode 100644 tools/ci/dynamic_pipelines/scripts/generate_report.py delete mode 100644 tools/ci/dynamic_pipelines/templates/fast_pipeline.yml delete mode 100644 tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml delete mode 100644 tools/ci/dynamic_pipelines/templates/report.template.html delete mode 100644 tools/ci/dynamic_pipelines/templates/retry-jobs.png delete mode 100644 tools/ci/dynamic_pipelines/tests/__init__.py delete mode 100644 tools/ci/dynamic_pipelines/tests/test_report_generator/__init__.py delete mode 100644 tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/XUNIT_REPORT.xml delete mode 100644 tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_job_report.html delete mode 100644 tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html delete mode 100644 tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/failure_rate_jobs_response.json delete mode 100644 tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/jobs_api_response.json delete mode 100644 tools/ci/dynamic_pipelines/tests/test_report_generator/test_report_generator.py delete mode 100644 tools/ci/dynamic_pipelines/utils.py delete mode 100644 tools/ci/get_all_test_results.py delete mode 100644 tools/ci/idf_ci_local/uploader.py delete mode 100644 tools/ci/idf_pytest/pytest.ini delete mode 100644 tools/ci/idf_pytest/script.py delete mode 100644 tools/ci/previous_stage_job_status.py delete mode 100644 tools/requirements/requirements.pytest.txt rename tools/test_apps/phy/phy_tsens/{sdkconfig.ci.c2_eco4 => sdkconfig.ci.esp32c2_rev2} (100%) rename tools/test_apps/phy/phy_tsens/{sdkconfig.ci.c3_eco7 => sdkconfig.ci.esp32c3_rev1} (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ca6b8ae1cf3..409d1985a6c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,44 +1,26 @@ -workflow: - rules: - # Disable those non-protected push triggered pipelines - - if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && $CI_PIPELINE_SOURCE == "push"' - when: never - # when running merged result pipelines, CI_COMMIT_SHA represents the temp commit it created. - # Please use PIPELINE_COMMIT_SHA at all places that require a commit sha of the original commit. - - if: $CI_OPEN_MERGE_REQUESTS != null - variables: - PIPELINE_COMMIT_SHA: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA - IS_MR_PIPELINE: 1 - - if: $CI_OPEN_MERGE_REQUESTS == null - variables: - PIPELINE_COMMIT_SHA: $CI_COMMIT_SHA - IS_MR_PIPELINE: 0 - - when: always - # Place the default settings in `.gitlab/ci/common.yml` instead include: - project: "ci/actions/common" + ref: "v2" file: - - "templates/idf/integration-test.yml" - - '.gitlab/ci/danger.yml' - - '.gitlab/ci/common.yml' - - '.gitlab/ci/rules.yml' - - ".gitlab/ci/manual_gate.yml" - - '.gitlab/ci/upload_cache.yml' - - '.gitlab/ci/static-code-analysis.yml' - - '.gitlab/ci/pre_commit.yml' - - '.gitlab/ci/pre_check.yml' - - '.gitlab/ci/build.yml' - - '.gitlab/ci/host-test.yml' - - '.gitlab/ci/deploy.yml' - - '.gitlab/ci/post_deploy.yml' - - '.gitlab/ci/retry_failed_jobs.yml' - - '.gitlab/ci/test-win.yml' - - - project: "ci/actions/common" - file: + - "templates/stable/default-workflow.yml" + - "templates/idf/pre_check.yml" + - "templates/idf/build.yml" + - "templates/idf/deploy-github.yml" - "templates/idf/deploy-docs.yml" + - "templates/idf/integration-test.yml" + - project: "ci/actions/common" + ref: "v2" + file: "templates/idf/sync-files.yml" + inputs: + job_name: "sync-pre-commit-config" + remote_repo_url: "https://__token__:${CI_SYNC_REPO_TOKEN}@${CI_SERVER_FQDN}/ci/images.git" + ref_name: "ci/sync_idf_pre_commit_config" + file_mappings: + - ".pre-commit-config.yaml:pre-commit/idf/.pre-commit-config.v5.4.yaml" + changes: + - ".pre-commit-config.yaml" - project: "ci/actions/common" file: "templates/idf/build-docs.yml" inputs: @@ -50,3 +32,16 @@ include: DOCTGT: "esp32" - DOCLANG: "zh_CN" DOCTGT: "esp32p4" + - ".gitlab/ci/danger.yml" + - ".gitlab/ci/common.yml" # has to be placed after danger.yml, since danger.yml defined stages. + - ".gitlab/ci/rules.yml" + # by stages + - ".gitlab/ci/manual_gate.yml" + - ".gitlab/ci/upload_cache.yml" + - ".gitlab/ci/pre_check.yml" + - ".gitlab/ci/pre_deploy.yml" + - ".gitlab/ci/static-code-analysis.yml" + - ".gitlab/ci/pre_commit.yml" + - ".gitlab/ci/build.yml" + - ".gitlab/ci/host-test.yml" + - ".gitlab/ci/test-win.yml" diff --git a/.gitlab/ci/README.md b/.gitlab/ci/README.md index ab138c09433..7b109d30689 100644 --- a/.gitlab/ci/README.md +++ b/.gitlab/ci/README.md @@ -23,12 +23,7 @@ - [Grammar](#grammar) - [Special Rules](#special-rules) - [Upload/Download Artifacts to Internal Minio Server](#uploaddownload-artifacts-to-internal-minio-server) - - [Users Without Access to Minio](#users-without-access-to-minio) - - [Users With Access to Minio](#users-with-access-to-minio) - - [Env Vars for Minio](#env-vars-for-minio) - - [Artifacts Types and File Patterns](#artifacts-types-and-file-patterns) - - [Upload](#upload) - - [Download](#download) + - [Revision Marker Naming Rules](#revision-marker-naming-rules) ## General Workflow @@ -145,11 +140,11 @@ check if there's a suitable `.if-` anchor 1. if there is, create a rule following [`rules` Template Naming Rules](#rules-template-naming-rules).For detail information, please refer to [GitLab Documentation `rules-if`](https://docs.gitlab.com/ee/ci/yaml/README.html#rulesif). Here's an example. ```yaml - .rules:patterns:python-files: + .rules:patterns:clang_tidy: rules: - <<: *if-protected - <<: *if-dev-push - changes: *patterns-python-files + changes: *patterns-c-files ``` 2. if there isn't @@ -250,76 +245,15 @@ We're using the latest version of [idf-build-apps][idf-build-apps]. Please refer In ESP-IDF CI, there's a few more special rules are additionally supported to disable the check app dependencies feature: - Add MR labels `BUILD_AND_TEST_ALL_APPS` +- Pipeline variable `IDF_CI_SELECT_ALL_PYTEST_CASES=1` - Run in protected branches ## Upload/Download Artifacts to Internal Minio Server -### Users Without Access to Minio +Please refer to the documentation [here](https://docs.espressif.com/projects/idf-ci/en/latest/guides/cli.html) -If you don't have access to the internal Minio server, you can still download the artifacts from the shared link in the job log. +## Revision Marker Naming Rules -The log will look like this: +When specifying chip revisions for testing markers, configurations or filenames, strictly follow the format `esp32_rev` (or similarly `_rev` when brief). -```shell -Pipeline ID : 587355 -Job name : build_clang_test_apps_esp32 -Job ID : 40272275 -Created archive file: 40272275.zip, uploading as 587355/build_dir_without_map_and_elf_files/build_clang_test_apps_esp32/40272275.zip -Please download the archive file includes build_dir_without_map_and_elf_files from [INTERNAL_URL] -``` - -### Users With Access to Minio - -#### Env Vars for Minio - -Minio takes these env vars to connect to the server: - -- `IDF_S3_SERVER` -- `IDF_S3_ACCESS_KEY` -- `IDF_S3_SECRET_KEY` -- `IDF_S3_BUCKET` - -#### Artifacts Types and File Patterns - -The artifacts types and corresponding file patterns are defined in tools/ci/artifacts_handler.py, inside `ArtifactType` and `TYPE_PATTERNS_DICT`. - -#### Upload - -```shell -python tools/ci/artifacts_handler.py upload -``` - - will upload the files that match the file patterns to minio object storage with name: - -`///.zip` - -For example, job 39043328 will upload these four files: - -- `575500/map_and_elf_files/build_pytest_examples_esp32/39043328.zip` -- `575500/build_dir_without_map_and_elf_files/build_pytest_examples_esp32/39043328.zip` -- `575500/logs/build_pytest_examples_esp32/39043328.zip` -- `575500/size_reports/build_pytest_examples_esp32/39043328.zip` - -#### Download - -You may run - -```shell -python tools/ci/artifacts_handler.py download --pipeline_id -``` - -to download all files of the pipeline, or - -```shell -python tools/ci/artifacts_handler.py download --pipeline_id --job_name -``` - -to download all files with the specified job name or pattern, or - -```shell -python tools/ci/artifacts_handler.py download --pipeline_id --job_name --type ... -``` - -to download all files with the specified job name or pattern and artifact type(s). - -You may check all detailed documentation with `python tools/ci/artifacts_handler.py download -h` +> e.g., `esp32_rev3`, `esp32c2_rev2`, `esp32c3_rev1`, `esp32p4_rev1`, `esp32c5_rev1` diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index a24e1187e36..ae4d74ad4c9 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -1,55 +1,13 @@ .build_template: stage: build extends: - - .after_script:build:ccache-show-stats:upload-failed-job-logs + - .after_script:build image: $ESP_ENV_IMAGE - tags: - - build - # build only on shiny servers since shiny storage server is at the same location - - shiny + tags: [build, shiny] variables: # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings. IDF_CCACHE_ENABLE: "1" - dependencies: [] - -.build_cmake_clang_template: - extends: - - .build_template - - .before_script:build - - .after_script:build:ccache-show-stats dependencies: # set dependencies to null to avoid missing artifacts issue - needs: - - job: fast_template_app - artifacts: false - - pipeline_variables - artifacts: - paths: - # The other artifacts patterns are defined under tools/ci/artifacts_handler.py - # Now we're uploading/downloading the binary files from our internal storage server - # - # keep the log file to help debug - - "**/build*/build_log.txt" - # keep the size info to help track the binary size - - size_info.txt - - "**/build*/size.json" - expire_in: 1 week - when: always - variables: - IDF_TOOLCHAIN: clang - TEST_BUILD_OPTS_EXTRA: "" - TEST_DIR: tools/test_apps/system/clang_build_test - PYTEST_IGNORE_COLLECT_IMPORT_ERROR: "1" - script: - # CI specific options start from "--parallel-count xxx". could ignore when running locally - - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v - -t $IDF_TARGET - --copy-sdkconfig - --parallel-count ${CI_NODE_TOTAL:-1} - --parallel-index ${CI_NODE_INDEX:-1} - --modified-components ${MR_MODIFIED_COMPONENTS} - --modified-files ${MR_MODIFIED_FILES} - $TEST_BUILD_OPTS_EXTRA - - python tools/ci/artifacts_handler.py upload ###################### # build_template_app # @@ -57,18 +15,11 @@ .build_template_app_template: extends: - .build_template - - .before_script:build - variables: - LOG_PATH: "${CI_PROJECT_DIR}/log_template_app" - BUILD_PATH: "${CI_PROJECT_DIR}/build_template_app" - BUILD_DIR: "${BUILD_PATH}/@t/@w" - BUILD_LOG_CMAKE: "${LOG_PATH}/cmake_@t_@w.txt" - BUILD_COMMAND_ARGS: "" artifacts: paths: - log_template_app/* - size_info.txt - - build_template_app/**/size.json + - build_template_app/**/size*.json expire_in: 1 week when: always script: @@ -82,224 +33,15 @@ # Only do the default cmake build for each target, remaining part are done in the build_template_app job - tools/ci/build_template_app.sh ${BUILD_COMMAND_ARGS} -# build-related-pre-check-jobs ------------------------------------------------ # Build at least one project for each target at earliest stage to reduce build cost for obvious failing commits fast_template_app: extends: - .build_template_app_template - - .rules:build:target_test + - .rules:build stage: pre_check tags: [fast_run, shiny] variables: BUILD_COMMAND_ARGS: "-p" -#------------------------------------------------------------------------------ - -####################### -# gnu_static_analyzer # -####################### -gcc_static_analyzer: - extends: - - .build_template_app_template - - .rules:build:target_test - stage: pre_check - tags: [build, shiny] - variables: - CI_CCACHE_DISABLE: 1 - ANALYZING_APP: "examples/get-started/hello_world" - script: - - echo "CONFIG_COMPILER_STATIC_ANALYZER=y" >> ${ANALYZING_APP}/sdkconfig.defaults - - python -m idf_build_apps build -v -p ${ANALYZING_APP} -t all - -######################################## -# Clang Build Apps Without Tests Cases # -######################################## -build_clang_test_apps_esp32: - extends: - - .build_cmake_clang_template - - .rules:build - variables: - IDF_TARGET: esp32 - -build_clang_test_apps_esp32s2: - extends: - - .build_cmake_clang_template - - .rules:build - variables: - IDF_TARGET: esp32s2 - -build_clang_test_apps_esp32s3: - extends: - - .build_cmake_clang_template - - .rules:build - variables: - IDF_TARGET: esp32s3 - -.build_clang_test_apps_riscv: - extends: - - .build_cmake_clang_template - variables: - # https://reviews.llvm.org/D90108. - # GNU 'as' lets .weak override .globl since binutils-gdb - # https://github.com/bminor/binutils-gdb/commit/5ca547dc2399a0a5d9f20626d4bf5547c3ccfddd (1996) - # while MC lets the last directive win (PR38921). - # For RISCV chips we use integrated assembler by default, so suppress this warning to pass CI pipeline. - TEST_BUILD_OPTS_EXTRA: "--ignore-warning-str 'changed binding to STB_WEAK'" - -build_clang_test_apps_esp32c3: - extends: - - .build_clang_test_apps_riscv - - .rules:build - variables: - IDF_TARGET: esp32c3 - -build_clang_test_apps_esp32c2: - extends: - - .build_clang_test_apps_riscv - - .rules:build - variables: - IDF_TARGET: esp32c2 - -build_clang_test_apps_esp32c6: - extends: - - .build_clang_test_apps_riscv - - .rules:build - variables: - IDF_TARGET: esp32c6 - -build_clang_test_apps_esp32c5: - extends: - - .build_clang_test_apps_riscv - - .rules:build - variables: - IDF_TARGET: esp32c5 - -build_clang_test_apps_esp32h2: - extends: - - .build_clang_test_apps_riscv - - .rules:build - variables: - IDF_TARGET: esp32h2 - -build_clang_test_apps_esp32p4: - extends: - - .build_clang_test_apps_riscv - - .rules:build - variables: - IDF_TARGET: esp32p4 - -###################### -# Build System Tests # -###################### -.test_build_system_template: - stage: host_test - extends: - - .build_template - - .rules:build:check - dependencies: # set dependencies to null to avoid missing artifacts issue - needs: - - job: fast_template_app - artifacts: false - optional: true - artifacts: - reports: - junit: XUNIT_RESULT.xml - paths: - - XUNIT_RESULT.xml - - test_build_system - expire_in: 1 week - when: always - script: - - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh - - cd ${IDF_PATH}/tools/test_build_system - - python ${IDF_PATH}/tools/ci/get_known_failure_cases_file.py - - pytest - --cleanup-idf-copy - --parallel-count ${CI_NODE_TOTAL:-1} - --parallel-index ${CI_NODE_INDEX:-1} - --work-dir ${CI_PROJECT_DIR}/test_build_system - --junitxml ${CI_PROJECT_DIR}/XUNIT_RESULT.xml - --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} - -pytest_build_system: - extends: .test_build_system_template - parallel: 3 - -pytest_build_system_macos: - extends: - - .test_build_system_template - - .before_script:build:macos - - .after_script:build:macos:upload-failed-job-logs:ccache-show-stats - - .rules:build:macos - tags: - - macos_shell - parallel: 3 - variables: - PYENV_VERSION: "3.8" - # CCACHE_DIR: "/cache/idf_ccache". On macOS, you cannot write to this folder due to insufficient permissions. - CCACHE_DIR: "" # ccache will use "$HOME/Library/Caches/ccache". - -.test_build_system_minimal_cmake_template: - extends: .test_build_system_template - variables: - INSTALL_EXTRA_TOOLS: cmake@3.16.3 - script: - - MINIMAL_SUPPORTED_CMAKE_VERSION=$(echo "${INSTALL_EXTRA_TOOLS}" | sed -n 's/.*cmake@\([0-9.]*\).*/\1/p') - - export PATH=$(echo "$PATH" | sed -E "s|/tools/cmake/[0-9.]+|/tools/cmake/${MINIMAL_SUPPORTED_CMAKE_VERSION}|") - - ACTUAL_CMAKE_VERSION=$(cmake --version | head -n1 | awk '{print $3}') - - | - if [ "${ACTUAL_CMAKE_VERSION}" != "${MINIMAL_SUPPORTED_CMAKE_VERSION}" ]; then - echo "ERROR: Wrong minimal CMake version! Detected: ${ACTUAL_CMAKE_VERSION}, but should be: ${MINIMAL_SUPPORTED_CMAKE_VERSION}" - exit 1 - fi - - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh - - cd ${IDF_PATH}/tools/test_build_system - - python ${IDF_PATH}/tools/ci/get_known_failure_cases_file.py - - pytest - -k cmake - --cleanup-idf-copy - --work-dir ${CI_PROJECT_DIR}/test_build_system - --junitxml ${CI_PROJECT_DIR}/XUNIT_RESULT.xml - --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} - -pytest_build_system_minimal_cmake: - extends: .test_build_system_minimal_cmake_template - -pytest_build_system_macos_minimal_cmake: - extends: - - .test_build_system_minimal_cmake_template - - .before_script:build:macos - - .after_script:build:macos:upload-failed-job-logs:ccache-show-stats - - .rules:build:macos - tags: - - macos_shell - variables: - PYENV_VERSION: "3.8" - # CCACHE_DIR: "/cache/idf_ccache". On macOS, you cannot write to this folder due to insufficient permissions. - CCACHE_DIR: "" # ccache will use "$HOME/Library/Caches/ccache". - INSTALL_EXTRA_TOOLS: ninja cmake@3.16.3 - -build_docker: - extends: - - .before_script:minimal - - .rules:build:docker - stage: host_test - needs: - - job: manual_gate - optional: true - image: espressif/docker-builder:1 - tags: [shiny, dind] - variables: - DOCKER_TMP_IMAGE_NAME: "idf_tmp_image" - script: - # use different image tag for different concurrent job - - export DOCKER_TMP_IMAGE="${DOCKER_TMP_IMAGE_NAME}:${CI_CONCURRENT_ID:-latest}" - - export DOCKER_BUILD_ARGS="--build-arg IDF_CLONE_URL=${CI_REPOSITORY_URL} --build-arg IDF_CLONE_BRANCH_OR_TAG=${CI_COMMIT_REF_NAME} --build-arg IDF_CHECKOUT_REF=${CI_COMMIT_TAG:-$CI_COMMIT_SHA} --build-arg IDF_CLONE_SHALLOW=1 ${EXTRA_DOCKER_BUILD_ARGS:-} " - - time docker build --tag ${DOCKER_TMP_IMAGE} ${DOCKER_BUILD_ARGS} tools/docker/ - # We can't mount $PWD/examples/get-started/blink into the container, see https://gitlab.com/gitlab-org/gitlab-ce/issues/41227. - # The workaround mentioned there works, but leaves around directories which need to be cleaned up manually. - # Therefore, build a copy of the example located inside the container. - - docker run --rm --workdir /opt/esp/idf/examples/get-started/blink ${DOCKER_TMP_IMAGE} idf.py build - # This job builds template app with permutations of targets and optimization levels build_template_app: @@ -307,57 +49,67 @@ build_template_app: - .build_template_app_template - .rules:build stage: host_test - dependencies: # set dependencies to null to avoid missing artifacts issue needs: - job: fast_template_app artifacts: false +######################################## +# Clang Build Apps Without Tests Cases # +######################################## +.build_cmake_clang_template: + extends: + - .build_template + - .rules:build + needs: + - job: fast_template_app + artifacts: false + - pipeline_variables + artifacts: + paths: + # The other artifacts patterns are defined under .idf_ci.toml + # Now we're uploading/downloading the binary files from our internal storage server + # + # keep the log file to help debug + - "**/build*/build_log.txt" + - "**/build*/size*.json" + expire_in: 1 week + when: always + variables: + IDF_TOOLCHAIN: clang + TEST_BUILD_OPTS_EXTRA: "" + script: + # CI specific options start from "--parallel-count xxx". could ignore when running locally + - run_cmd idf-build-apps build + -p tools/test_apps/system/clang_build_test + -t $IDF_TARGET + --parallel-count ${CI_NODE_TOTAL:-1} + --parallel-index ${CI_NODE_INDEX:-1} + --modified-components ${MR_MODIFIED_COMPONENTS} + --modified-files ${MR_MODIFIED_FILES} + $TEST_BUILD_OPTS_EXTRA + +build_clang_test_apps_xtensa: + extends: .build_cmake_clang_template + parallel: + matrix: + - IDF_TARGET: [esp32, esp32s2, esp32s3] + +build_clang_test_apps_riscv: + extends: .build_cmake_clang_template + variables: + # https://reviews.llvm.org/D90108. + # GNU 'as' lets .weak override .globl since binutils-gdb + # https://github.com/bminor/binutils-gdb/commit/5ca547dc2399a0a5d9f20626d4bf5547c3ccfddd (1996) + # while MC lets the last directive win (PR38921). + # For RISCV chips we use integrated assembler by default, so suppress this warning to pass CI pipeline. + TEST_BUILD_OPTS_EXTRA: "--ignore-warning-str 'changed binding to STB_WEAK'" + parallel: + matrix: + - IDF_TARGET: [esp32c3, esp32c2, esp32c6, esp32c5, esp32h2, esp32p4] + #################### # Dynamic Pipeline # #################### -generate_build_child_pipeline: - extends: - - .build_template - tags: [fast_run, shiny] - dependencies: # set dependencies to null to avoid missing artifacts issue - needs: - - pipeline_variables - - job: baseline_manifest_sha - optional: true - artifacts: - paths: - - build_child_pipeline.yml - - test_related_apps.txt - - non_test_related_apps.txt - expire_in: 1 week - when: always - variables: - PYTEST_IGNORE_COLLECT_IMPORT_ERROR: "1" - script: - # requires basic pytest dependencies - - run_cmd bash install.sh --enable-pytest - - run_cmd python tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py - -build_child_pipeline: - stage: build - needs: - - job: fast_template_app - optional: true - artifacts: false - - pipeline_variables - - generate_build_child_pipeline - variables: - IS_MR_PIPELINE: $IS_MR_PIPELINE - MR_MODIFIED_COMPONENTS: $MR_MODIFIED_COMPONENTS - MR_MODIFIED_FILES: $MR_MODIFIED_FILES - PARENT_PIPELINE_ID: $CI_PIPELINE_ID - BUILD_AND_TEST_ALL_APPS: $BUILD_AND_TEST_ALL_APPS - REPORT_EXIT_CODE: $REPORT_EXIT_CODE - # https://gitlab.com/gitlab-org/gitlab/-/issues/214340 - inherit: - variables: false - trigger: - include: - - artifact: build_child_pipeline.yml - job: generate_build_child_pipeline - strategy: depend +# !!! +# configurations moved to ci/actions/common/idf/ +# !!! diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index b687c670d09..c7ca4ad5eb8 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -11,9 +11,18 @@ stages: - target_test - host_test - test_deploy + - pre_deploy - deploy - post_deploy - - retry_failed_jobs + +include: + - project: "ci/actions/common" + ref: "v2" + file: "templates/idf/common-scripts.yml" + inputs: + ci_python_constraint_file: "espidf.constraints.v5.4.txt" + macos_image: "registry.hub.docker.com/espressif/macos-idf:sonoma-v5.4" + windows_image: "windows_11_idf_v5_4" variables: # System environment @@ -35,16 +44,12 @@ variables: # - set GIT_STRATEGY: "clone" to shiny runners # - set GIT_STRATEGY: "fetch" to brew runners GIT_STRATEGY: clone - # we will download archive for each submodule instead of clone. - # we don't do "recursive" when fetch submodule as they're not used in CI now. - GIT_SUBMODULE_STRATEGY: none - # since we're using merged-result pipelines, the last commit should work for most cases GIT_DEPTH: 1 - # --no-recurse-submodules: we use cache for submodules + GIT_SUBMODULE_STRATEGY: none # here we use cache for submodules, so we don't need to fetch them every time + # since we're using merged-result pipelines, the last commit should work for most cases # --prune --prune-tags: in case remote branch or tag is force pushed GIT_FETCH_EXTRA_FLAGS: "--no-recurse-submodules --prune --prune-tags" - # we're using .cache folder for caches - GIT_CLEAN_FLAGS: -ffdx -e .cache/ + LATEST_GIT_TAG: v5.4.4 SUBMODULE_FETCH_TOOL: "tools/ci/ci_fetch_submodule.py" @@ -63,40 +68,36 @@ variables: ESP_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v5.4:1" ESP_IDF_DOC_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v5.4-docs:1" TARGET_TEST_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v5.4-target-test:1" - SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:5" - PRE_COMMIT_IMAGE: "${CI_DOCKER_REGISTRY}/esp-idf-pre-commit:1" - - # cache python dependencies - PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" # Set this variable to the branch of idf-constraints repo in order to test a custom Python constraint file. The # branch name must be without the remote part ("origin/"). Keep the variable empty in order to use the constraint # file from https://dl.espressif.com/dl/esp-idf. - CI_PYTHON_CONSTRAINT_BRANCH: "" + # CI_PYTHON_CONSTRAINT_BRANCH could be defined as component input `ci_python_constraint_branch` in common-scripts above # Update the filename for a specific ESP-IDF release. It is used only with CI_PYTHON_CONSTRAINT_BRANCH. - CI_PYTHON_CONSTRAINT_FILE: "espidf.constraints.v5.4.txt" + # CI_PYTHON_CONSTRAINT_FILE is defined as component input `ci_python_constraint_file` in common-scripts above # Set this variable to repository name of a Python tool you wish to install and test in the context of ESP-IDF CI. # Keep the variable empty when not used. - CI_PYTHON_TOOL_REPO: "" + # CI_PYTHON_TOOL_REPO could be defined as component input `ci_python_tool_repo` in common-scripts above # Set this variable to the branch of a Python tool repo specified in CI_PYTHON_TOOL_REPO. The # branch name must be without the remote part ("origin/"). Keep the variable empty when not used. # This is used only if CI_PYTHON_TOOL_REPO is not empty. - CI_PYTHON_TOOL_BRANCH: "" + # CI_PYTHON_TOOL_BRANCH could be defined as component input `ci_python_tool_branch` in common-scripts above # Set this variable to Clang toolchain distro URL to be used. # NOTE: We have separate toolchains for Xtensa and RISCV, therefore jobs for one arch will fail. # This is OK as far as we use CI_CLANG_DISTRO_URL for pre-release tests purposes only. # Keep the variable empty when not used. - CI_CLANG_DISTRO_URL: "" + # CI_CLANG_DISTRO_URL could be defined as component input `ci_clang_distro_url` in common-scripts above # Set this variable to specify the file name for the known failure cases. KNOWN_FAILURE_CASES_FILE_NAME: "5.4.txt" IDF_CI_BUILD: 1 + PYTHONWARNINGS: "ignore" # ccache settings # some settings need to set in .gitlab-ci.yml as it takes effect while start-up the job # https://ccache.dev/manual/latest.html#_configuring_ccache @@ -106,295 +107,23 @@ variables: CCACHE_MAXSIZE: "50G" FF_USE_NEW_BASH_EVAL_STRATEGY: "true" + FORCE_COLOR: "1" # rich print with color ################################################ # `before_script` and `after_script` Templates # ################################################ -.common_before_scripts: &common-before_scripts | - source tools/ci/utils.sh - - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS - - if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then - export IDF_MIRROR_PREFIX_MAP= - fi - - if echo "$CI_MERGE_REQUEST_LABELS" | egrep "(^|,)include_nightly_run(,|$)"; then - export INCLUDE_NIGHTLY_RUN="1" - export NIGHTLY_RUN="1" - fi - - # configure cmake related flags - source tools/ci/configure_ci_environment.sh - - if [[ "$CI_CCACHE_STATS" == 1 ]] && command -v ccache >/dev/null 2>&1 && [[ -n "$CCACHE_STATSLOG" ]]; then - mkdir -p "$(dirname "$CCACHE_STATSLOG")" - rm -f "$CCACHE_STATSLOG" - fi - - # add extra python packages - export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH" - -.setup_tools_and_idf_python_venv: &setup_tools_and_idf_python_venv | - # must use after setup_tools_except_target_test - # otherwise the export.sh won't work properly - - # download constraint file for dev - if [[ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]]; then - wget -O /tmp/constraint.txt --header="Authorization:Bearer ${ESPCI_TOKEN}" ${GITLAB_HTTP_SERVER}/api/v4/projects/2581/repository/files/${CI_PYTHON_CONSTRAINT_FILE}/raw?ref=${CI_PYTHON_CONSTRAINT_BRANCH} - mkdir -p "${IDF_TOOLS_PATH:-$HOME/.espressif}" - mv /tmp/constraint.txt "${IDF_TOOLS_PATH:-$HOME/.espressif}/${CI_PYTHON_CONSTRAINT_FILE}" - fi - - # Mirror - if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then - export IDF_MIRROR_PREFIX_MAP= - fi - - if [[ "${CI_JOB_STAGE}" != "target_test" ]]; then - section_start "running_install_sh" "Running install.sh" - if [[ "${CI_JOB_STAGE}" == "build_doc" || "${CI_JOB_GROUP_NAME}" == *docs* ]]; then - run_cmd bash install.sh --enable-ci --enable-docs - elif [[ "${CI_JOB_STAGE}" == "build" ]]; then - run_cmd bash install.sh --enable-ci - else - if ! echo "${CI_JOB_NAME}" | egrep ".*pytest.*"; then - run_cmd bash install.sh --enable-ci - else - run_cmd bash install.sh --enable-ci --enable-pytest --enable-test-specific - fi - fi - section_end "running_install_sh" - else - section_start "install_python_env" "Install Python environment" - run_cmd python tools/idf_tools.py install-python-env --features ci,pytest,test-specific - section_end "install_python_env" - fi - - if [[ ! -z "$INSTALL_EXTRA_TOOLS" ]]; then - section_start "installing_optional_tools" "Install optional tools ${INSTALL_EXTRA_TOOLS}" - $IDF_PATH/tools/idf_tools.py --non-interactive install $INSTALL_EXTRA_TOOLS - section_end "installing_optional_tools" - fi - - # Install esp-clang if necessary (esp-clang is separately installed) - if [[ "$IDF_TOOLCHAIN" == "clang" && -z "$CI_CLANG_DISTRO_URL" ]]; then - $IDF_PATH/tools/idf_tools.py --non-interactive install esp-clang - fi - - if [[ "${CI_JOB_STAGE}" == "target_test" ]]; then - section_start "IDF_SKIP_TOOLS_CHECK" "Skip required tools check" - export IDF_SKIP_TOOLS_CHECK=1 - section_end "IDF_SKIP_TOOLS_CHECK" - fi - section_start "source_export" "Source export.sh" - source ./export.sh - section_end "source_export" - - # we need the latest DEV release for esptool to be installed in every job without bumping the minimal requirement in - # the constraint file. - section_start "esptool_upgrade" "Upgrade esptool to the latest development version" - pip install --upgrade --pre esptool -c "${IDF_TOOLS_PATH:-$HOME/.espressif}/${CI_PYTHON_CONSTRAINT_FILE}" - section_end "esptool_upgrade" - - # Custom clang toolchain - if [[ "$IDF_TOOLCHAIN" == "clang" && ! -z "$CI_CLANG_DISTRO_URL" ]]; then - echo "Using custom clang from ${CI_CLANG_DISTRO_URL}" - wget $CI_CLANG_DISTRO_URL - ARCH_NAME=$(basename $CI_CLANG_DISTRO_URL) - tar -x -f $ARCH_NAME - export PATH=$PWD/esp-clang/bin:$PATH - fi - - # Custom OpenOCD - if [[ "$CI_JOB_STAGE" == "target_test" ]]; then - machine="$(uname -m)" - if [[ "$machine" == "armv7l" ]] ; then - OOCD_DISTRO_URL="$OOCD_DISTRO_URL_ARMHF" - elif [[ "$machine" == "aarch64" ]] ; then - OOCD_DISTRO_URL="$OOCD_DISTRO_URL_ARM64" - fi - if [[ ! -z "$OOCD_DISTRO_URL" ]]; then - echo "Using custom OpenOCD from ${OOCD_DISTRO_URL}" - wget $OOCD_DISTRO_URL - ARCH_NAME=$(basename $OOCD_DISTRO_URL) - tar -x -f $ARCH_NAME - export OPENOCD_SCRIPTS=$PWD/openocd-esp32/share/openocd/scripts - export PATH=$PWD/openocd-esp32/bin:$PATH - fi - fi - - if [[ -n "$CI_PYTHON_TOOL_REPO" ]]; then - git clone --quiet --depth=1 -b ${CI_PYTHON_TOOL_BRANCH} https://gitlab-ci-token:${ESPCI_TOKEN}@${GITLAB_HTTPS_HOST}/espressif/${CI_PYTHON_TOOL_REPO}.git - pip install ./${CI_PYTHON_TOOL_REPO} - rm -rf ${CI_PYTHON_TOOL_REPO} - fi - - info "setup tools and python venv done" - -.show_ccache_statistics: &show_ccache_statistics | - # Show ccache statistics if enabled globally - section_start "ccache_show_stats" "Show ccache statistics" - if [[ "$CI_CCACHE_STATS" == 1 ]] && command -v ccache >/dev/null 2>&1; then - if ccache --help 2>/dev/null | grep -q -- '--show-log-stats'; then - if [[ -n "$CCACHE_STATSLOG" && -f "$CCACHE_STATSLOG" ]]; then - ccache --show-log-stats -vv - else - echo "INFO: No per-job ccache statistics were recorded" - fi - else - ccache --show-stats -vv - fi - fi || true - section_end "ccache_show_stats" - -.upload_failed_job_log_artifacts: &upload_failed_job_log_artifacts | - if [ $CI_JOB_STATUS = "failed" ]; then - python tools/ci/artifacts_handler.py upload --type logs - fi - .before_script:minimal: before_script: - - *common-before_scripts + - !reference [.common_before_scripts, minimal] -.before_script:build:macos: - before_script: - # macos is running shell executor, which means it would use - # the system installed /usr/local/bin/python3 by default. - # Ensure pyenv and PYENV_VERSION installed - - eval "$(pyenv init -)" - - *common-before_scripts - # On macOS, these tools need to be installed - - export IDF_TOOLS_PATH="${HOME}/.espressif_runner_${CI_RUNNER_ID}_${CI_CONCURRENT_ID}" - # remove idf-env.json, since it may contains enabled "features" - - rm -f $IDF_TOOLS_PATH/idf-env.json - # This adds tools (compilers) and the version-specific Python environment to PATH - - *setup_tools_and_idf_python_venv - - fetch_submodules - variables: - INSTALL_EXTRA_TOOLS: cmake ninja - -.after_script:build:macos:upload-failed-job-logs:ccache-show-stats: +.after_script:build: after_script: - # macos is running shell executor, which means it would use - # the system installed /usr/local/bin/python3 by default. - # Ensure pyenv and PYENV_VERSION installed - - eval "$(pyenv init -)" - - *upload_failed_job_log_artifacts - - *show_ccache_statistics - -.before_script:build: - before_script: - - *common-before_scripts - - *setup_tools_and_idf_python_venv - - add_gitlab_ssh_keys - - fetch_submodules - - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS} - - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS} - -.after_script:build:ccache-show-stats: - after_script: - - *show_ccache_statistics - -.after_script:build:ccache-show-stats:upload-failed-job-logs: - after_script: - - *show_ccache_statistics - - *upload_failed_job_log_artifacts - -############################## -# Git Strategy Job Templates # -############################## -.git_init: &git_init | - mkdir -p "${CI_PROJECT_DIR}" - cd "${CI_PROJECT_DIR}" - git init - -.git_fetch_from_mirror_url_if_exists: &git_fetch_from_mirror_url_if_exists | - # check if set mirror - if [ -n "${LOCAL_GITLAB_HTTPS_HOST:-}" ] && [ -n "${ESPCI_TOKEN:-}" ]; then - MIRROR_REPO_URL="https://bot:${ESPCI_TOKEN}@${LOCAL_GITLAB_HTTPS_HOST}/${CI_PROJECT_PATH}" - elif [ -n "${LOCAL_GIT_MIRROR:-}" ]; then - MIRROR_REPO_URL="${LOCAL_GIT_MIRROR}/${CI_PROJECT_PATH}" - fi - - # fetch from mirror first if set - if [ -n "${MIRROR_REPO_URL:-}" ]; then - if git remote -v | grep origin; then - git remote set-url origin "${MIRROR_REPO_URL}" - else - git remote add origin "${MIRROR_REPO_URL}" - fi - # mirror url may fail with authentication issue - git fetch origin --no-recurse-submodules || true - fi - - # set remote url to CI_REPOSITORY_URL - if git remote -v | grep origin; then - git remote set-url origin "${CI_REPOSITORY_URL}" - else - git remote add origin "${CI_REPOSITORY_URL}" - fi - -.git_checkout_ci_commit_sha: &git_checkout_ci_commit_sha | - git checkout $CI_COMMIT_SHA - git clean ${GIT_CLEAN_FLAGS} - -# git diff requires two commits, with different CI env var -# -# By default, we use git strategy "clone" with depth 1 to speed up the clone process. -# But for jobs requires running `git diff`, we need to fetch more commits to get the correct diffs. -# -# Since there's no way to get the correct git_depth before the job starts, -# we can't set `GIT_DEPTH` in the job definition. -# -# Set git strategy to "none" and fetch manually instead. -.before_script:fetch:git_diff: - variables: - GIT_STRATEGY: none - before_script: - - *git_init - - *git_fetch_from_mirror_url_if_exists + - !reference [.common_after_scripts, ccache] + - run_cmd idf-ci gitlab upload-artifacts --type junit - | - # merged results pipelines, by default - if [[ -n $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ]]; then - git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA) - git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - # merge request pipelines, when the mr got conflicts - elif [[ -n $CI_MERGE_REQUEST_DIFF_BASE_SHA ]]; then - git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_COMMIT_SHA) - # other pipelines, like the protected branches pipelines - elif [[ "$CI_COMMIT_BEFORE_SHA" != "0000000000000000000000000000000000000000" ]]; then - git fetch origin $CI_COMMIT_BEFORE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA) - else - # pipeline source could be web, scheduler, etc. - git fetch origin $CI_COMMIT_SHA --depth=2 ${GIT_FETCH_EXTRA_FLAGS} - export GIT_DIFF_OUTPUT=$(git diff --name-only $CI_COMMIT_SHA~1 $CI_COMMIT_SHA) + if [ $CI_JOB_STATUS = "failed" ]; then + run_cmd idf-ci gitlab upload-artifacts --type log fi - - *git_checkout_ci_commit_sha - - *common-before_scripts - - *setup_tools_and_idf_python_venv - - add_gitlab_ssh_keys - -# target test runners may locate in different places -# for runners set git mirror, we fetch from the mirror first, then fetch the HEAD commit -.before_script:fetch:target_test: - variables: - GIT_STRATEGY: none - before_script: - - *git_init - - *git_fetch_from_mirror_url_if_exists - - git fetch origin "${CI_COMMIT_SHA}" --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - - *git_checkout_ci_commit_sha - - *common-before_scripts - - *setup_tools_and_idf_python_venv - - add_gitlab_ssh_keys - # no submodules ############# # `default` # @@ -403,12 +132,6 @@ default: cache: # pull only for most of the use cases since it's cache dir. # Only set "push" policy for "upload_cache" stage jobs - - key: pip-cache-${LATEST_GIT_TAG} - fallback_keys: - - pip-cache - paths: - - .cache/pip - policy: pull - key: submodule-cache-${LATEST_GIT_TAG} fallback_keys: - submodule-cache @@ -416,20 +139,8 @@ default: - .cache/submodule_archives policy: pull before_script: - - *common-before_scripts - - *setup_tools_and_idf_python_venv + - !reference [.common_before_scripts, with_tools] - add_gitlab_ssh_keys - fetch_submodules -# gitlab bug, setting them here doesn't work -# - expire_in: https://gitlab.com/gitlab-org/gitlab/-/issues/404563 -# - when: https://gitlab.com/gitlab-org/gitlab/-/issues/440672 -# artifacts: -# expire_in: 1 week -# when: always - retry: - max: 2 - when: - # In case of a runner failure we could hop to another one, or a network error could go away. - - runner_system_failure - # Job execution timeout may be caused by a network issue. - - job_execution_timeout + - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS} + - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS} diff --git a/.gitlab/ci/default-build-test-rules.yml b/.gitlab/ci/default-build-test-rules.yml deleted file mode 100644 index b10ffeedb05..00000000000 --- a/.gitlab/ci/default-build-test-rules.yml +++ /dev/null @@ -1,20 +0,0 @@ -# this file support two keywords: -# - extra_default_build_targets: -# besides of the SUPPORTED_TARGETS in IDF, -# enable build for the specified targets by default as well. -# - bypass_check_test_targets: -# suppress the check_build_test_rules check-test-script warnings for the specified targets -# -# This file should ONLY be used during bringup. Should be reset to empty after the bringup process -extra_default_build_targets: - - -bypass_check_test_targets: - - esp32c5 - - esp32c61 - -# -# These lines would -# - enable the README.md check for esp32c6. Don't forget to add the build jobs in .gitlab/ci/build.yml -# - disable the test script check with the manifest file. -# diff --git a/.gitlab/ci/dependencies/dependencies.yml b/.gitlab/ci/dependencies/dependencies.yml index 4f87a9ab3b4..8f7fc782dc7 100644 --- a/.gitlab/ci/dependencies/dependencies.yml +++ b/.gitlab/ci/dependencies/dependencies.yml @@ -1,19 +1,3 @@ -.all_targets: &all_targets - - esp32 - - esp32s2 - - esp32s3 - - esp32c3 - - esp32c2 - - esp32c6 - - esp32c5 - - esp32h2 - - esp32p4 - -.target_test: &target_test - - example_test - - custom_test - - component_ut - ############## # Build Jobs # ############## @@ -25,7 +9,6 @@ - build_system - downloadable-tools included_in: - - build:target_test - build:check # ------------------- @@ -57,8 +40,6 @@ "patterns:template-app": patterns: - build_template-app - included_in: - - build:target_test "patterns:build-check": patterns: diff --git a/.gitlab/ci/deploy.yml b/.gitlab/ci/deploy.yml deleted file mode 100644 index c9b1b2f3711..00000000000 --- a/.gitlab/ci/deploy.yml +++ /dev/null @@ -1,48 +0,0 @@ -.deploy_job_template: - stage: deploy - image: $ESP_ENV_IMAGE - tags: [ deploy ] - -# Check this before push_to_github -check_submodule_sync: - extends: - - .deploy_job_template - - .rules:test:submodule - stage: test_deploy - tags: [ brew, github_sync ] - retry: 2 - variables: - # for brew runners, we always set GIT_STRATEGY to fetch - GIT_STRATEGY: fetch - SUBMODULES_TO_FETCH: "none" - PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git" - dependencies: [] - script: - - git submodule deinit --force . - - rm -rf .git/modules # remove all the cached metadata - # setting the default remote URL to the public one, to resolve relative location URLs - - git config remote.origin.url ${PUBLIC_IDF_URL} - # check if all submodules are correctly synced to public repository - - git submodule init - - git config --get-regexp '^submodule\..*\.url$' || true - - git submodule update --recursive - - echo "IDF was cloned from ${PUBLIC_IDF_URL} completely" - -push_to_github: - extends: - - .deploy_job_template - - .before_script:minimal - - .rules:push_to_github - needs: - - check_submodule_sync - tags: [ brew, github_sync ] - variables: - # for brew runners, we always set GIT_STRATEGY to fetch - GIT_STRATEGY: fetch - # github also need full record of commits - GIT_DEPTH: 0 - script: - - add_github_ssh_keys - - git remote remove github &>/dev/null || true - - git remote add github git@github.com:espressif/esp-idf.git - - tools/ci/push_to_github.sh diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index b21f6c6c631..611d6dd7341 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -2,18 +2,17 @@ extends: .rules:test:host_test stage: host_test image: $ESP_ENV_IMAGE - tags: - - host_test + tags: [build, shiny] dependencies: # set dependencies to null to avoid missing artifacts issue # run host_test jobs immediately, only after upload cache needs: - - job: upload-pip-cache - optional: true - artifacts: false + - pipeline_variables - job: upload-submodules-cache optional: true artifacts: false - - pipeline_variables + - job: fast_template_app + optional: true + artifacts: false artifacts: expire_in: 1 week when: always @@ -23,16 +22,28 @@ check_public_headers: - .host_test_template - .rules:build:check script: - - IDF_TARGET=esp32 python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32-elf- - - IDF_TARGET=esp32s2 python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32s2-elf- - - IDF_TARGET=esp32s3 python tools/ci/check_public_headers.py --jobs 4 --prefix xtensa-esp32s3-elf- - - IDF_TARGET=esp32c3 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf- - - IDF_TARGET=esp32c2 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf- - - IDF_TARGET=esp32c6 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf- - - IDF_TARGET=esp32c5 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf- - - IDF_TARGET=esp32h2 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf- - - IDF_TARGET=esp32p4 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf- - - IDF_TARGET=esp32c61 python tools/ci/check_public_headers.py --jobs 4 --prefix riscv32-esp-elf- + - | + targets="esp32 \ + esp32s2 \ + esp32s3 \ + esp32c3 \ + esp32c2 \ + esp32c6 \ + esp32c5 \ + esp32h2 \ + esp32p4 \ + esp32c61" + for target in ${targets}; do + case "${target}" in + esp32|esp32s2|esp32s3) + PREFIX="xtensa-${target}-elf-" + ;; + *) + PREFIX="riscv32-esp-elf-" + ;; + esac + IDF_TARGET=${target} python tools/ci/check_public_headers.py --jobs 4 --prefix ${PREFIX} + done test_nvs_coverage: extends: @@ -51,14 +62,15 @@ test_partition_table_on_host: extends: .host_test_template script: - cd components/partition_table/test_gen_esp32part_host - - ./gen_esp32part_tests.py + - pytest_for_ut ./gen_esp32part_tests.py test_ldgen_on_host: - extends: .host_test_template + extends: + - .host_test_template script: - cd tools/ldgen/test - export PYTHONPATH=$PYTHONPATH:.. - - python -m unittest + - pytest_for_ut . variables: LC_ALL: C.UTF-8 @@ -66,15 +78,13 @@ test_spiffs_on_host: extends: .host_test_template script: - cd components/spiffs/test_spiffsgen/ - - ./test_spiffsgen.py + - pytest_for_ut ./test_spiffsgen.py test_fatfsgen_on_host: extends: .host_test_template script: - cd components/fatfs/test_fatfsgen/ - - ./test_fatfsgen.py - - ./test_wl_fatfsgen.py - - ./test_fatfsparse.py + - pytest_for_ut ./test_fatfsgen.py ./test_wl_fatfsgen.py ./test_fatfsparse.py test_multi_heap_on_host: extends: .host_test_template @@ -86,7 +96,7 @@ test_certificate_bundle_on_host: extends: .host_test_template script: - cd components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/ - - ./test_gen_crt_bundle.py + - pytest_for_ut ./test_gen_crt_bundle.py test_gdbstub_on_host: extends: .host_test_template @@ -115,15 +125,17 @@ test_cli_installer: - python3 ${IDF_PATH}/tools/idf_tools.py download required qemu-riscv32 qemu-xtensa cmake cmake@3.16.3 - cd ${IDF_PATH}/tools/test_idf_tools - python3 -m pip install jsonschema --break-system-packages + # Testing with system python3, so don't use any third-party packages - python3 ./test_idf_tools.py -v - python3 ./test_idf_tools_python_env.py # It runs at the end because it modifies dependencies - IDF_TEST_MAY_BREAK_DEPENDENCIES=1 python3 ./test_idf_tools.py -v TestSystemDependencies.test_commands_when_nodeps -.test_efuse_table_on_host_template: +test_efuse_table_on_host: extends: .host_test_template - variables: - IDF_TARGET: "esp32" + parallel: + matrix: + - IDF_TARGET: [esp32, esp32s2, esp32c3, esp32s3, esp32c6, esp32h2] artifacts: when: on_failure paths: @@ -133,35 +145,7 @@ test_cli_installer: - ./efuse_table_gen.py -t "${IDF_TARGET}" ${IDF_PATH}/components/efuse/${IDF_TARGET}/esp_efuse_table.csv - git diff --exit-code -- ${IDF_TARGET}/esp_efuse_table.c || { echo 'Differences found for ${IDF_TARGET} target. Please run idf.py efuse-common-table and commit the changes.'; exit 1; } - cd ${IDF_PATH}/components/efuse/test_efuse_host - - ./efuse_tests.py - -test_efuse_table_on_host_esp32: - extends: .test_efuse_table_on_host_template - -test_efuse_table_on_host_esp32s2: - extends: .test_efuse_table_on_host_template - variables: - IDF_TARGET: esp32s2 - -test_efuse_table_on_host_esp32s3: - extends: .test_efuse_table_on_host_template - variables: - IDF_TARGET: esp32s3 - -test_efuse_table_on_host_esp32c3: - extends: .test_efuse_table_on_host_template - variables: - IDF_TARGET: esp32c3 - -test_efuse_table_on_host_esp32h2: - extends: .test_efuse_table_on_host_template - variables: - IDF_TARGET: esp32h2 - -test_efuse_table_on_host_esp32c6: - extends: .test_efuse_table_on_host_template - variables: - IDF_TARGET: esp32c6 + - pytest_for_ut ./efuse_tests.py test_logtrace_proc: extends: .host_test_template @@ -199,16 +183,17 @@ test_tools: INSTALL_EXTRA_TOOLS: "qemu-xtensa qemu-riscv32" # for test_idf_qemu.py script: - stat=0 + - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} - cd ${IDF_PATH}/tools/ci/test_autocomplete - - pytest --noconftest test_autocomplete.py --junitxml=${IDF_PATH}/XUNIT_AUTOCOMP.xml || stat=1 + - run_cmd pytest --noconftest test_autocomplete.py --junitxml=${IDF_PATH}/XUNIT_AUTOCOMP.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 - cd ${IDF_PATH}/tools/test_idf_py - - pytest --noconftest test_idf_py.py --junitxml=${IDF_PATH}/XUNIT_IDF_PY.xml || stat=1 - - pytest --noconftest test_hints.py --junitxml=${IDF_PATH}/XUNIT_HINTS.xml || stat=1 - - pytest --noconftest test_idf_qemu.py --junitxml=${IDF_PATH}/XUNIT_IDF_PY_QEMU.xml || stat=1 + - run_cmd pytest --noconftest test_idf_py.py --junitxml=${IDF_PATH}/XUNIT_IDF_PY.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 + - run_cmd pytest --noconftest test_hints.py --junitxml=${IDF_PATH}/XUNIT_HINTS.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 + - run_cmd pytest --noconftest test_idf_qemu.py --junitxml=${IDF_PATH}/XUNIT_IDF_PY_QEMU.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 - cd ${IDF_PATH}/tools/test_mkdfu - - pytest --noconftest test_mkdfu.py --junitxml=${IDF_PATH}/XUNIT_MKDFU.xml || stat=1 + - run_cmd pytest --noconftest test_mkdfu.py --junitxml=${IDF_PATH}/XUNIT_MKDFU.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 - cd ${IDF_PATH}/tools/test_idf_size - - pytest --noconftest test_idf_size.py --junitxml=${IDF_PATH}/XUNIT_IDF_SIZE.xml || stat=1 + - run_cmd pytest --noconftest test_idf_size.py --junitxml=${IDF_PATH}/XUNIT_IDF_SIZE.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 - cd ${IDF_PATH} - shellcheck -s sh tools/detect_python.sh || stat=1 - shellcheck -s bash tools/detect_python.sh || stat=1 @@ -223,12 +208,15 @@ test_split_path_by_spaces: extends: .host_test_template script: - cd ${IDF_PATH}/tools - - python -m unittest split_paths_by_spaces.py + - pytest_for_ut ./split_paths_by_spaces.py test_mqtt_on_host: extends: .host_test_template script: - cd ${IDF_PATH}/components/mqtt/esp-mqtt/host_test + # no -Werror + - export EXTRA_CFLAGS= + - export EXTRA_CXXFLAGS= - idf.py build - LSAN_OPTIONS=verbosity=1:log_threads=1 build/host_mqtt_client_test.elf @@ -269,122 +257,13 @@ test_gen_soc_caps_kconfig: extends: .host_test_template script: - cd ${IDF_PATH}/tools/gen_soc_caps_kconfig/ - - ./test/test_gen_soc_caps_kconfig.py - -test_pytest_qemu: - extends: - - .host_test_template - - .before_script:build - artifacts: - paths: - - XUNIT_RESULT.xml - - pytest-embedded/ - - "**/build*/*.bin" - reports: - junit: XUNIT_RESULT.xml - parallel: - matrix: - - IDF_TARGET: "esp32" - INSTALL_EXTRA_TOOLS: "qemu-xtensa" - - IDF_TARGET: "esp32c3" - INSTALL_EXTRA_TOOLS: "qemu-riscv32" - script: - - run_cmd python tools/ci/ci_build_apps.py . -v - --target $IDF_TARGET - --pytest-apps - -m qemu - --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" - --modified-components ${MR_MODIFIED_COMPONENTS} - --modified-files ${MR_MODIFIED_FILES} - - python tools/ci/get_known_failure_cases_file.py - - run_cmd pytest - --target $IDF_TARGET - --log-cli-level DEBUG - -m qemu - --embedded-services idf,qemu - --junitxml=XUNIT_RESULT.xml - --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} - --app-info-filepattern \"list_job_*.txt\" - --qemu-extra-args \"-global driver=timer.$IDF_TARGET.timg,property=wdt_disable,value=true\" - -test_pytest_linux: - extends: - - .host_test_template - - .before_script:build - artifacts: - paths: - - XUNIT_RESULT.xml - - pytest-embedded/ - - "**/build*/build_log.txt" - reports: - junit: XUNIT_RESULT.xml - script: - - run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -v - --target linux - --pytest-apps - -m host_test - --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" - --modified-components ${MR_MODIFIED_COMPONENTS} - --modified-files ${MR_MODIFIED_FILES} - - python tools/ci/get_known_failure_cases_file.py - - run_cmd pytest - --target linux - -m host_test - --embedded-services idf - --junitxml=XUNIT_RESULT.xml - --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} - --app-info-filepattern \"list_job_*.txt\" - -test_pytest_macos: - extends: - - .host_test_template - - .before_script:build:macos - tags: - - macos_shell - artifacts: - paths: - - XUNIT_RESULT.xml - - pytest-embedded/ - - "**/build*/build_log.txt" - reports: - junit: XUNIT_RESULT.xml - variables: - PYTEST_IGNORE_COLLECT_IMPORT_ERROR: "1" - script: - - run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -v - --target linux - --pytest-apps - -m \"host_test and macos_shell\" - --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" - --modified-components ${MR_MODIFIED_COMPONENTS} - --modified-files ${MR_MODIFIED_FILES} - - python tools/ci/get_known_failure_cases_file.py - - run_cmd pytest - --target linux - -m \"host_test and macos_shell\" - --junitxml=XUNIT_RESULT.xml - --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} - --app-info-filepattern \"list_job_*.txt\" - -test_idf_pytest_plugin: - extends: - - .host_test_template - - .rules:patterns:idf-pytest-plugin - variables: - SUBMODULES_TO_FETCH: "none" - artifacts: - reports: - junit: XUNIT_RESULT.xml - script: - - cd ${IDF_PATH}/tools/ci/dynamic_pipelines/tests/test_report_generator - - python -m unittest test_report_generator.py - - cd ${IDF_PATH}/tools/ci/idf_pytest - - pytest --junitxml=${CI_PROJECT_DIR}/XUNIT_RESULT.xml + - pytest_for_ut ./test/test_gen_soc_caps_kconfig.py test_idf_build_apps_load_soc_caps: extends: .host_test_template script: - - python tools/ci/check_soc_headers_load_in_idf_build_apps.py + - cd tools/ci + - pytest_for_ut ./check_soc_headers_load_in_idf_build_apps.py test_nvs_gen_check: extends: .host_test_template @@ -398,7 +277,7 @@ test_nvs_gen_check: LC_ALL: C.UTF-8 script: - cd ${IDF_PATH}/components/nvs_flash/nvs_partition_tool - - pytest --noconftest test_nvs_gen_check.py --junitxml=XUNIT_RESULT.xml + - pytest_for_ut test_nvs_gen_check.py test_esp_rom: extends: .host_test_template @@ -409,7 +288,7 @@ test_esp_rom: junit: XUNIT_RESULT.xml script: - cd ${IDF_PATH}/components/esp_rom/ - - pytest --noconftest test_esp_rom.py --junitxml=XUNIT_RESULT.xml + - pytest_for_ut test_esp_rom.py make_sure_soc_caps_compatible_in_idf_build_apps: extends: @@ -428,3 +307,191 @@ make_sure_soc_caps_compatible_in_idf_build_apps: - git checkout $CI_COMMIT_SHA -- tools/ci/idf_build_apps_dump_soc_caps.py - python tools/ci/idf_build_apps_dump_soc_caps.py base.json - diff new.json base.json + +build_docker: + extends: + - .before_script:minimal + - .rules:build:docker + stage: host_test + needs: + - job: manual_gate + optional: true + image: espressif/docker-builder:1 + tags: [shiny, dind] + variables: + DOCKER_TMP_IMAGE_NAME: "idf_tmp_image" + script: + # use different image tag for different concurrent job + - export DOCKER_TMP_IMAGE="${DOCKER_TMP_IMAGE_NAME}:${CI_CONCURRENT_ID:-latest}" + - export DOCKER_BUILD_ARGS="--build-arg IDF_CLONE_URL=${CI_REPOSITORY_URL} --build-arg IDF_CLONE_BRANCH_OR_TAG=${CI_COMMIT_REF_NAME} --build-arg IDF_CHECKOUT_REF=${CI_COMMIT_TAG:-$CI_COMMIT_SHA} --build-arg IDF_CLONE_SHALLOW=1 ${EXTRA_DOCKER_BUILD_ARGS:-} " + - time docker build --tag ${DOCKER_TMP_IMAGE} ${DOCKER_BUILD_ARGS} tools/docker/ + # We can't mount $PWD/examples/get-started/blink into the container, see https://gitlab.com/gitlab-org/gitlab-ce/issues/41227. + # The workaround mentioned there works, but leaves around directories which need to be cleaned up manually. + # Therefore, build a copy of the example located inside the container. + - docker run --rm --workdir /opt/esp/idf/examples/get-started/blink ${DOCKER_TMP_IMAGE} idf.py build + +############################ +# Host test with test apps # +############################ +test_pytest_qemu: + extends: + - .host_test_template + artifacts: + paths: + - XUNIT_RESULT.xml + - pytest-embedded/ + - "**/build*/*.bin" + reports: + junit: XUNIT_RESULT.xml + parallel: + matrix: + - IDF_TARGET: "esp32" + INSTALL_EXTRA_TOOLS: "qemu-xtensa" + # Skip Clang + Xtensa tests due to bootloader size issue + IDF_TOOLCHAIN: [gcc] + - IDF_TARGET: "esp32c3" + INSTALL_EXTRA_TOOLS: "qemu-riscv32" + IDF_TOOLCHAIN: [gcc, clang] + script: + - run_cmd idf-ci build run + --build-system cmake + --target $IDF_TARGET + -m qemu + --modified-files ${MR_MODIFIED_FILES} + - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} + - run_cmd pytest + --target $IDF_TARGET + -m qemu + --junitxml=XUNIT_RESULT.xml + --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} + --qemu-extra-args \"-global driver=timer.$IDF_TARGET.timg,property=wdt_disable,value=true\" + +test_pytest_linux: + extends: + - .host_test_template + artifacts: + paths: + - XUNIT_RESULT.xml + - pytest-embedded/ + - "**/build*/build_log.txt" + reports: + junit: XUNIT_RESULT.xml + script: + - run_cmd idf-ci build run + --build-system cmake + -p components -p examples -p tools/test_apps + --target linux + --only-test-related + --modified-files ${MR_MODIFIED_FILES} + - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} + - run_cmd pytest + --target linux + -m \"not macos\" + --junitxml=XUNIT_RESULT.xml + --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} + +test_pytest_macos: + extends: + - .host_test_template + - .macos-settings + artifacts: + paths: + - XUNIT_RESULT.xml + - pytest-embedded/ + - "**/build*/build_log.txt" + reports: + junit: XUNIT_RESULT.xml + script: + - run_cmd idf-ci build run + -p components -p examples -p tools/test_apps + --build-system cmake + --target linux + --only-test-related + -m macos + --modified-files ${MR_MODIFIED_FILES} + - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} + - run_cmd pytest + --target linux + -m macos + --junitxml=XUNIT_RESULT.xml + --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} + +###################### +# Build System Tests # +###################### +.test_build_system_template: + stage: host_test + extends: + - .build_template + - .rules:build:check + dependencies: # set dependencies to null to avoid missing artifacts issue + needs: + - job: fast_template_app + artifacts: false + optional: true + artifacts: + reports: + junit: XUNIT_RESULT.xml + paths: + - XUNIT_RESULT.xml + - test_build_system + expire_in: 1 week + when: always + script: + - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh + - cd ${IDF_PATH}/tools/test_build_system + - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} + - pytest + --cleanup-idf-copy + --parallel-count ${CI_NODE_TOTAL:-1} + --parallel-index ${CI_NODE_INDEX:-1} + --work-dir ${CI_PROJECT_DIR}/test_build_system + --junitxml ${CI_PROJECT_DIR}/XUNIT_RESULT.xml + --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} + +pytest_build_system: + extends: .test_build_system_template + parallel: 3 + +pytest_build_system_macos: + extends: + - .test_build_system_template + - .macos-settings + - .rules:build:macos + parallel: 3 + +.test_build_system_minimal_cmake_template: + extends: .test_build_system_template + variables: + INSTALL_EXTRA_TOOLS: cmake@3.16.3 + script: + - MINIMAL_SUPPORTED_CMAKE_VERSION=$(echo "${INSTALL_EXTRA_TOOLS}" | sed -n 's/.*cmake@\([0-9.]*\).*/\1/p') + - export PATH=$(echo "$PATH" | sed -E "s|/tools/cmake/[0-9.]+|/tools/cmake/${MINIMAL_SUPPORTED_CMAKE_VERSION}|") + - ACTUAL_CMAKE_VERSION=$(cmake --version | head -n1 | awk '{print $3}') + - | + if [ "${ACTUAL_CMAKE_VERSION}" != "${MINIMAL_SUPPORTED_CMAKE_VERSION}" ]; then + echo "ERROR: Wrong minimal CMake version! Detected: ${ACTUAL_CMAKE_VERSION}, but should be: ${MINIMAL_SUPPORTED_CMAKE_VERSION}" + exit 1 + fi + - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh + - cd ${IDF_PATH}/tools/test_build_system + - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} + - pytest + -k cmake + --cleanup-idf-copy + --parallel-count ${CI_NODE_TOTAL:-1} + --parallel-index ${CI_NODE_INDEX:-1} + --work-dir ${CI_PROJECT_DIR}/test_build_system + --junitxml ${CI_PROJECT_DIR}/XUNIT_RESULT.xml + --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} + +pytest_build_system_minimal_cmake: + extends: .test_build_system_minimal_cmake_template + +pytest_build_system_macos_minimal_cmake: + extends: + - .test_build_system_minimal_cmake_template + - .macos-settings + - .rules:build:macos + variables: + INSTALL_EXTRA_TOOLS: ninja cmake@3.16.3 diff --git a/.gitlab/ci/post_deploy.yml b/.gitlab/ci/post_deploy.yml deleted file mode 100644 index fe1ea90f834..00000000000 --- a/.gitlab/ci/post_deploy.yml +++ /dev/null @@ -1,13 +0,0 @@ -generate_failed_jobs_report: - stage: post_deploy - tags: [build, shiny] - image: $ESP_ENV_IMAGE - when: always - dependencies: [] # Do not download artifacts from the previous stages - artifacts: - expire_in: 1 week - when: always - paths: - - job_report.html - script: - - python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type job diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index f70f7872b65..7ce0757843f 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -1,16 +1,9 @@ -.pre_check_template: - stage: pre_check - image: $ESP_ENV_IMAGE - tags: - - host_test - dependencies: # set dependencies to null to avoid missing artifacts issue - check_version: # Don't run this for feature/bugfix branches, so that it is possible to modify # esp_idf_version.h in a branch before tagging the next version. extends: - .pre_check_template - - .rules:protected + - .rules:protected:check tags: [ brew, github_sync ] variables: # need a full clone to get the latest tag @@ -116,108 +109,23 @@ check_version_tag: check_test_scripts_build_test_rules: extends: - .pre_check_template - - .before_script:build - variables: - PYTEST_IGNORE_COLLECT_IMPORT_ERROR: "1" script: # requires basic pytest dependencies - - run_cmd bash install.sh --enable-pytest - python tools/ci/check_build_test_rules.py check-test-scripts examples/ tools/test_apps components -check_configure_ci_environment_parsing: +check_gitlab_yaml: extends: - .pre_check_template - - .before_script:build - - .rules:build script: - - cd tools/ci - - python -m unittest ci_build_apps.py + - python tools/ci/gitlab_yaml_linter.py -pipeline_variables: - extends: - - .pre_check_template - - .before_script:fetch:git_diff - tags: [fast_run, shiny] - script: - # MODIFIED_FILES is a list of files that changed, could be used everywhere - - MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | xargs) - - echo "MODIFIED_FILES=$MODIFIED_FILES" >> pipeline.env - - echo "REPORT_EXIT_CODE=0" >> pipeline.env - # MR_MODIFIED_FILES and MR_MODIFIED_COMPONENTS are semicolon separated lists that is used in MR only - # for non MR pipeline, these are empty lists - - | - if [ $IS_MR_PIPELINE == "0" ]; then - echo "MR_MODIFIED_FILES=\"\"" >> pipeline.env - echo "MR_MODIFIED_COMPONENTS=\"\"" >> pipeline.env - else - MR_MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | tr '\n' ';') - echo "MR_MODIFIED_FILES=\"$MR_MODIFIED_FILES\"" >> pipeline.env - - MR_MODIFIED_COMPONENTS=$(run_cmd python tools/ci/ci_get_mr_info.py components --modified-files $MODIFIED_FILES | tr '\n' ';') - echo "MR_MODIFIED_COMPONENTS=\"$MR_MODIFIED_COMPONENTS\"" >> pipeline.env - fi - - | - if echo "$CI_MERGE_REQUEST_LABELS" | egrep "(^|,)BUILD_AND_TEST_ALL_APPS(,|$)"; then - echo "BUILD_AND_TEST_ALL_APPS=1" >> pipeline.env - fi - # run full pipeline if testing constraint branch - - | - if [ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]; then - echo "BUILD_AND_TEST_ALL_APPS=1" >> pipeline.env - fi - - echo "OOCD_DISTRO_URL_ARMHF=$OOCD_DISTRO_URL_ARMHF" >> pipeline.env - - echo "OOCD_DISTRO_URL_ARM64=$OOCD_DISTRO_URL_ARM64" >> pipeline.env - - python tools/ci/ci_process_description.py - - cat pipeline.env - - python tools/ci/artifacts_handler.py upload --type modified_files_and_components_report - artifacts: - reports: - dotenv: pipeline.env - paths: - - pipeline.env - expire_in: 1 week - when: always - -baseline_manifest_sha: +retry_failed_jobs: extends: - .pre_check_template - .rules:dev-push - tags: [fast_run, shiny] - script: - - | - # merged results pipelines, by default - # diff between target-branch-head and merged-result-head - if [ -n "$CI_MERGE_REQUEST_TARGET_BRANCH_SHA" ]; then - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA --depth=1 - git checkout FETCH_HEAD - idf-build-apps dump-manifest-sha \ - --manifest-files $(find . -name ".build-test-rules.yml" | xargs) \ - --output .manifest_sha - # merge request pipelines, when the mr got conflicts - # diff between diff-base-sha and merge-request-head - elif [ -n "$CI_MERGE_REQUEST_DIFF_BASE_SHA" ]; then - git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 - git checkout FETCH_HEAD - idf-build-apps dump-manifest-sha \ - --manifest-files $(find . -name ".build-test-rules.yml" | xargs) \ - --output .manifest_sha - # other pipelines, like the protected branches pipelines - # not triggered in this job - fi - artifacts: - paths: - - .manifest_sha - expire_in: 1 week - when: always - -redundant_pass_job: - stage: pre_check tags: [shiny, fast_run] - image: $ESP_ENV_IMAGE - dependencies: null - before_script: [] - cache: [] - extends: [] + allow_failure: true script: - - echo "This job is redundant to ensure the 'retry_failed_jobs' job can exist and not be skipped" - when: always + - echo "Retrieving and retrying all failed jobs for the pipeline..." + - python tools/ci/python_packages/gitlab_api.py retry_failed_jobs $CI_MERGE_REQUEST_PROJECT_ID --pipeline_id $CI_PIPELINE_ID + when: manual diff --git a/.gitlab/ci/pre_commit.yml b/.gitlab/ci/pre_commit.yml index 49449106e73..44898754ac9 100644 --- a/.gitlab/ci/pre_commit.yml +++ b/.gitlab/ci/pre_commit.yml @@ -2,33 +2,20 @@ extends: - .before_script:minimal stage: pre_check - image: $PRE_COMMIT_IMAGE - tags: [cache, shiny] + image: "${CI_REGISTRY}/ci/images/pre-commit-idf-v5.4:1" + tags: [pre-commit] + variables: + # Both shiny and brew runners can pick this job + GIT_STRATEGY: fetch + GIT_DEPTH: 1 + SUBMODULES_TO_FETCH: "all" needs: - pipeline_variables - variables: - # cache pre_commit - PRE_COMMIT_HOME: "$CI_PROJECT_DIR/.cache/pre-commit" script: - fetch_submodules - pre-commit run --files $MODIFIED_FILES - pre-commit run --hook-stage post-commit validate-sbom-manifest -check_pre_commit_upload_cache: - extends: - - .check_pre_commit_template - rules: - - if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/) && $CI_PIPELINE_SOURCE == "push"' - cache: - - key: pre_commit-cache-${LATEST_GIT_TAG} - paths: - - .cache/pre-commit - policy: pull-push - - key: submodule-cache-${LATEST_GIT_TAG} - paths: - - .cache/submodule_archives - policy: pull - check_pre_commit: extends: - .check_pre_commit_template @@ -37,10 +24,6 @@ check_pre_commit: when: never - when: on_success cache: - - key: pre_commit-cache-${LATEST_GIT_TAG} - paths: - - .cache/pre-commit - policy: pull - key: submodule-cache-${LATEST_GIT_TAG} paths: - .cache/submodule_archives @@ -56,25 +39,20 @@ check_powershell: tags: - dind - amd64 + - brew # faster "apk add" needs: - pipeline_variables variables: - # cache pre_commit - PRE_COMMIT_HOME: "$CI_PROJECT_DIR/.cache/pre-commit" + # brew runners always use fetch + GIT_STRATEGY: fetch + GIT_DEPTH: 1 + SUBMODULES_TO_FETCH: "none" rules: - changes: - "*.ps1" + - ".gitlab/ci/pre_commit.yml" script: - apk add python3 - apk add py3-pip - pip install pre-commit --break-system-packages - pre-commit run --hook-stage manual check-powershell-scripts --files $MODIFIED_FILES - cache: - - key: pre_commit-cache-${LATEST_GIT_TAG} - paths: - - .cache/pre-commit - policy: pull - - key: submodule-cache-${LATEST_GIT_TAG} - paths: - - .cache/submodule_archives - policy: pull diff --git a/.gitlab/ci/pre_deploy.yml b/.gitlab/ci/pre_deploy.yml new file mode 100644 index 00000000000..df8525dcd24 --- /dev/null +++ b/.gitlab/ci/pre_deploy.yml @@ -0,0 +1,24 @@ +.pre_deploy_template: + stage: pre_deploy + image: $ESP_ENV_IMAGE + +check_submodule_sync: + extends: + - .pre_deploy_template + - .rules:test:submodule + tags: [ brew, github_sync ] + retry: 2 + variables: + GIT_STRATEGY: fetch # use brew local mirror first + SUBMODULES_TO_FETCH: "none" + PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git" + script: + - git submodule deinit --force . + - rm -rf .git/modules # remove all the cached metadata + # setting the default remote URL to the public one, to resolve relative location URLs + - git config remote.origin.url ${PUBLIC_IDF_URL} + # check if all submodules are correctly synced to public repository + - git submodule init + - git config --get-regexp '^submodule\..*\.url$' || true + - git submodule update --recursive + - echo "IDF was cloned from ${PUBLIC_IDF_URL} completely" diff --git a/.gitlab/ci/retry_failed_jobs.yml b/.gitlab/ci/retry_failed_jobs.yml deleted file mode 100644 index 8eff2ed6b4b..00000000000 --- a/.gitlab/ci/retry_failed_jobs.yml +++ /dev/null @@ -1,15 +0,0 @@ -retry_failed_jobs: - stage: retry_failed_jobs - tags: [shiny, fast_run] - allow_failure: true - image: $ESP_ENV_IMAGE - dependencies: null - before_script: [] - cache: [] - extends: [] - script: - - echo "Retrieving and retrying all failed jobs for the pipeline..." - - python tools/ci/python_packages/gitlab_api.py retry_failed_jobs $CI_MERGE_REQUEST_PROJECT_ID --pipeline_id $CI_PIPELINE_ID - when: manual - needs: - - redundant_pass_job diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index a3fedf7a7fd..73ecca2da7c 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -9,19 +9,10 @@ - "**/*.{c,C,cpp}" - "**/*.{h,H,hpp}" + - "Kconfig" - "components/**/Kconfig" - - "components/**/CMakeList.txt" + - "components/**/CMakeLists.txt" -.patterns-python-cache: &patterns-python-cache - - "tools/requirements.json" - - "tools/requirements/requirements.*.txt" - -.patterns-python-files: &patterns-python-files - - ".gitlab/ci/static-code-analysis.yml" - - "**/*.py" - -.patterns-sonarqube-files: &patterns-sonarqube-files - - "tools/ci/sonar_exclude_list.txt" .patterns-build_components: &patterns-build_components # components files except "test*/" "host*/" folders @@ -35,6 +26,7 @@ # Add folders excluded by "???[!t]" and "??[!s]?" # pre-commit: tools/ci/check_rules_components_patterns.py - "components/bt/host/**/*" + - "components/esp_psram/system_layer/*" .patterns-downloadable-tools: &patterns-downloadable-tools - "tools/idf_tools.py" @@ -66,8 +58,10 @@ - "tools/ci/check_esp_memory_utils_headers.sh" - "tools/ci/check_blobs.sh" - "tools/ci/check_public_headers.py" + - "tools/ci/check_public_headers_exceptions.txt" - "tools/ci/check_register_rw_half_word.cmake" - "tools/ci/check_register_rw_half_word.py" + - "examples/build_system/**/*" .patterns-host_test: &patterns-host_test - ".gitlab/ci/host-test.yml" @@ -97,6 +91,7 @@ - "tools/idf_tools.py" - "tools/test_idf_tools/**/*" - "tools/install_util.py" + - "tools/idf_extra_components.yml" - "tools/export_utils/utils.py" - "tools/export_utils/shell_types.py" @@ -160,34 +155,22 @@ ############## # if anchors # ############## -.if-ref-master: &if-ref-master - if: '$CI_COMMIT_REF_NAME == "master"' +.if-master-push: &if-master-push + if: '$CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "push"' -.if-tag-release: &if-tag-release +.if-release-tag: &if-release-tag if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/' -.if-protected: &if-protected +.if-protected-check: &if-protected-check if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) || $CI_COMMIT_TAG =~ /^qa-test/' -.if-protected-deploy: &if-protected-deploy - if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "api")' - -.if-protected-no_label: &if-protected-no_label - if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && $BOT_TRIGGER_WITH_LABEL == null' - -.if-protected-ref-push: &if-protected-ref-push +.if-protected-branch-push: &if-protected-branch-push # rules:changes always evaluates to true for new branch pipelines or when there is no Git push event if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/) && $CI_PIPELINE_SOURCE == "push"' -.if-qa-test-tag: &if-qa-test-tag - if: '$CI_COMMIT_TAG =~ /^qa-test/' - .if-dev-push: &if-dev-push if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")' -.if-schedule: &if-schedule - if: '$CI_PIPELINE_SOURCE == "schedule"' - .if-schedule-nightly: &if-schedule-nightly if: '$CI_PIPELINE_SOURCE == "schedule" && $INCLUDE_NIGHTLY_RUN == "1"' @@ -207,39 +190,26 @@ # Rules # ######### ### Branches ### -.rules:protected: +.rules:protected:check: rules: - - <<: *if-protected + - <<: *if-protected-check -.rules:push_to_github: +.rules:master:push: rules: - - <<: *if-qa-test-tag - when: never - - <<: *if-protected-no_label + - <<: *if-master-push .rules:tag:release: rules: - - <<: *if-tag-release + - <<: *if-release-tag .rules:dev-push: rules: - <<: *if-dev-push -# Do not upload caches on dev branches by default -.rules:upload-python-cache: - rules: - - <<: *if-tag-release - - <<: *if-schedule-nightly - - <<: *if-protected-ref-push - changes: *patterns-python-cache - - <<: *if-label-upload_cache - when: manual - .rules:upload-submodule-cache: rules: - # Needn't upload submodule cache in schedule pipeline - - <<: *if-tag-release - - <<: *if-protected-ref-push + - <<: *if-release-tag + - <<: *if-protected-branch-push changes: *patterns-submodule - <<: *if-label-upload_cache when: manual @@ -247,16 +217,10 @@ ### Patterns ### .rules:patterns:clang_tidy: rules: - - <<: *if-protected + - <<: *if-protected-check - <<: *if-dev-push changes: *patterns-c-files -.rules:patterns:python-files: - rules: - - <<: *if-protected - - <<: *if-dev-push - changes: *patterns-python-files - #.rules:patterns:static-code-analysis-preview: # rules: # - <<: *if-dev-push @@ -266,12 +230,6 @@ # - <<: *if-dev-push # changes: *patterns-sonarqube-files -.rules:patterns:idf-pytest-plugin: - rules: - - <<: *if-protected - - <<: *if-dev-push - changes: *patterns-idf-pytest-plugin - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # DO NOT place comments or maintain any code from this line # @@ -312,7 +270,7 @@ rules: - <<: *if-revert-branch when: never - - <<: *if-protected + - <<: *if-protected-check - <<: *if-label-build - <<: *if-dev-push changes: *patterns-build_components @@ -325,7 +283,7 @@ rules: - <<: *if-revert-branch when: never - - <<: *if-protected + - <<: *if-protected-check - <<: *if-label-build - <<: *if-dev-push changes: *patterns-build_check @@ -340,7 +298,7 @@ rules: - <<: *if-revert-branch when: never - - <<: *if-protected + - <<: *if-protected-check - <<: *if-label-build - <<: *if-label-docker - <<: *if-dev-push @@ -356,7 +314,7 @@ rules: - <<: *if-revert-branch when: never - - <<: *if-protected + - <<: *if-protected-check - <<: *if-label-build - <<: *if-label-macos - <<: *if-label-macos_test @@ -367,21 +325,6 @@ - <<: *if-dev-push changes: *patterns-downloadable-tools -.rules:build:target_test: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-build_template-app - - <<: *if-dev-push - changes: *patterns-downloadable-tools - .rules:labels:nvs_coverage: rules: - <<: *if-revert-branch @@ -399,7 +342,7 @@ rules: - <<: *if-revert-branch when: never - - <<: *if-protected + - <<: *if-protected-check - <<: *if-label-build-only when: never - <<: *if-label-host_test @@ -410,7 +353,7 @@ rules: - <<: *if-revert-branch when: never - - <<: *if-protected + - <<: *if-protected-check - <<: *if-label-build-only when: never - <<: *if-label-submodule diff --git a/.gitlab/ci/static-code-analysis.yml b/.gitlab/ci/static-code-analysis.yml index 2773d2cbf7f..69640fc2f7e 100644 --- a/.gitlab/ci/static-code-analysis.yml +++ b/.gitlab/ci/static-code-analysis.yml @@ -16,106 +16,17 @@ clang_tidy_check: --limit-file tools/ci/static-analysis-rules.yml --xtensa-include-dir -check_pylint: +gcc_static_analyzer: extends: - .pre_check_template - - .rules:patterns:python-files - needs: - - pipeline_variables - artifacts: - reports: - codequality: pylint.json - paths: - - pylint.json - expire_in: 1 week - when: always + - .rules:patterns:clang_tidy + variables: + CI_CCACHE_DISABLE: 1 + ANALYZING_APP: "examples/get-started/hello_world" script: - - | - if [ -n "$CI_MERGE_REQUEST_IID" ]; then - export files=$(echo "$GIT_DIFF_OUTPUT" | grep ".py$" | xargs); - else - export files=$(git ls-files "*.py" | xargs); - fi - - if [ -z "$files" ]; then echo "No python files found"; exit 0; fi - - run_cmd pylint --exit-zero --load-plugins=pylint_gitlab --output-format=gitlab-codeclimate:pylint.json $files -# -## build stage -## Sonarqube related jobs put here for this reason: -## Here we have two jobs. code_quality_check and code_quality_report. -## -## code_quality_check will analyze the code changes between your MR and -## code repo stored in sonarqube server. The analysis result is only shown in -## the comments under this MR and won't be transferred to the server. -## -## code_quality_report will analyze and transfer both of the newly added code -## and the analysis result to the server. -## -## Put in the front to ensure that the newly merged code can be stored in -## sonarqube server ASAP, in order to avoid reporting unrelated code issues -#.sonar_scan_template: -# stage: build -# extends: .pre_check_template -# # full clone since this image does not support fetch --shallow-since-cutoff -# # shiny runners are used for full clone -# tags: [build, shiny] -# image: $SONARQUBE_SCANNER_IMAGE -# before_script: -# - source tools/ci/utils.sh -# - export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH" -# - fetch_submodules -# # Exclude the submodules, all paths ends with /** -# - submodules=$(get_all_submodules) -# # get all exclude paths specified in tools/ci/sonar_exclude_list.txt | ignore lines start with # | xargs | replace all to -# - custom_excludes=$(cat $CI_PROJECT_DIR/tools/ci/sonar_exclude_list.txt | grep -v '^#' | xargs | sed -e 's/ /,/g') -# # Exclude the report dir as well -# - export EXCLUSIONS="$custom_excludes,$submodules" -# - export SONAR_SCANNER_OPTS="-Xmx2048m" -# variables: -# GIT_DEPTH: 0 -# REPORT_PATTERN: clang_tidy_reports/**/*.txt -# artifacts: -# paths: -# - $REPORT_PATTERN -# expire_in: 1 week -# when: always -# dependencies: # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here. -# - clang_tidy_check -# -#code_quality_check: -# extends: -# - .sonar_scan_template -# - .rules:patterns:static-code-analysis-preview -# allow_failure: true # it's using exit code to indicate the code analysis result, -# # we don't want to block ci when critical issues founded -# script: -# - export CI_MERGE_REQUEST_COMMITS=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py commits --src-branch ${CI_COMMIT_REF_NAME} | tr '\n' ',') -# # test if this branch have merge request, if not, exit 0 -# - test -n "$CI_MERGE_REQUEST_IID" || exit 0 -# - test -n "$CI_MERGE_REQUEST_COMMITS" || exit 0 -# - sonar-scanner -# -Dsonar.analysis.mode=preview -# -Dsonar.branch.name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME -# -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN -# -Dsonar.exclusions=$EXCLUSIONS -# -Dsonar.gitlab.ci_merge_request_iid=$CI_MERGE_REQUEST_IID -# -Dsonar.gitlab.commit_sha=$CI_MERGE_REQUEST_COMMITS -# -Dsonar.gitlab.merge_request_discussion=true -# -Dsonar.gitlab.ref_name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME -# -Dsonar.host.url=$SONAR_HOST_URL -# -Dsonar.login=$SONAR_LOGIN -# -#code_quality_report: -# extends: -# - .sonar_scan_template -# - .rules:protected -# allow_failure: true # it's using exit code to indicate the code analysis result, -# # we don't want to block ci when critical issues founded -# script: -# - sonar-scanner -# -Dsonar.branch.name=$CI_COMMIT_REF_NAME -# -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN -# -Dsonar.exclusions=$EXCLUSIONS -# -Dsonar.gitlab.commit_sha=$PIPELINE_COMMIT_SHA -# -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME -# -Dsonar.host.url=$SONAR_HOST_URL -# -Dsonar.login=$SONAR_LOGIN + - echo "CONFIG_COMPILER_STATIC_ANALYZER=y" >> ${ANALYZING_APP}/sdkconfig.defaults + - sed -i 's/.*MINIMAL_BUILD.*//g' ${ANALYZING_APP}/CMakeLists.txt + # no -Werror + - export EXTRA_CFLAGS= + - export EXTRA_CXXFLAGS= + - idf-build-apps build -p ${ANALYZING_APP} --no-check-warnings diff --git a/.gitlab/ci/test-win.yml b/.gitlab/ci/test-win.yml index 71e0af5ebd5..6ad27791513 100644 --- a/.gitlab/ci/test-win.yml +++ b/.gitlab/ci/test-win.yml @@ -1,21 +1,16 @@ -# Host tests .host_test_win_template: - extends: .rules:test:host_test + extends: + - .windows-settings + - .rules:test:host_test stage: host_test - image: $ESP_ENV_IMAGE - tags: - - windows-build dependencies: # set dependencies to null to avoid missing artifacts issue # run host_test jobs immediately, only after upload cache needs: - - job: upload-pip-cache - optional: true - artifacts: false - job: upload-submodules-cache optional: true artifacts: false - before_script: [] - after_script: [] + - job: manual_gate + optional: true test_cli_installer_win: rules: @@ -30,14 +25,11 @@ test_cli_installer_win: - tools/tools.new.json - tools/test_idf_tools/test_python_env_logs.txt expire_in: 1 week - variables: - IDF_PATH: "$CI_PROJECT_DIR" script: # Tools must be downloaded for testing # cmake@version that is supported - python ${IDF_PATH}\tools\idf_tools.py download required qemu-riscv32 qemu-xtensa cmake cmake@3.16.3 - - cd ${IDF_PATH}\tools\test_idf_tools - - python -m pip install jsonschema + - cd tools\test_idf_tools - python .\test_idf_tools.py - python .\test_idf_tools_python_env.py @@ -48,84 +40,48 @@ test_tools_win: parallel: 4 artifacts: paths: - - ${IDF_PATH}/*.out - - ${IDF_PATH}/XUNIT_*.xml + - "*.out" + - "XUNIT_*.xml" reports: - junit: ${IDF_PATH}/XUNIT_*.xml + junit: "XUNIT_*.xml" expire_in: 1 week when: always - variables: - LC_ALL: C.UTF-8 - PYTHONPATH: "$PYTHONPATH;$IDF_PATH\\tools;$IDF_PATH\\tools\\esp_app_trace;$IDF_PATH\\components\\partition_table;$IDF_PATH\\tools\\ci\\python_packages" script: - - python -m pip install jsonschema - - .\install.ps1 --enable-ci --enable-pytest - - .\export.ps1 - - python "${SUBMODULE_FETCH_TOOL}" -s "all" - - cd ${IDF_PATH}/tools/test_idf_py - - pytest --noconftest test_idf_py.py --junitxml=${IDF_PATH}/XUNIT_IDF_PY.xml - - pytest --noconftest test_hints.py --junitxml=${IDF_PATH}/XUNIT_HINTS.xml + - cd tools\test_idf_py + - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} + - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml ${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} # Build tests .test_build_system_template_win: - stage: host_test - variables: - # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings. - IDF_CCACHE_ENABLE: "1" - PYTHONPATH: "$PYTHONPATH;$IDF_PATH\\tools;$IDF_PATH\\tools\\esp_app_trace;$IDF_PATH\\components\\partition_table;$IDF_PATH\\tools\\ci\\python_packages" - before_script: [] - after_script: [] + extends: .host_test_win_template timeout: 4 hours + artifacts: + paths: + - XUNIT_RESULT.xml + - test_build_system + expire_in: 2 days + when: always + reports: + junit: XUNIT_RESULT.xml script: - - .\install.ps1 --enable-ci --enable-pytest - - . .\export.ps1 - - python "${SUBMODULE_FETCH_TOOL}" -s "all" - - cd ${IDF_PATH}\tools\test_build_system - - python ${IDF_PATH}\tools\ci\get_known_failure_cases_file.py - - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} + - cd tools\test_build_system + - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} + - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml ${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} pytest_build_system_win: extends: - .test_build_system_template_win - .rules:labels:windows_pytest_build_system - parallel: 2 - tags: - - windows-build - needs: - - job: manual_gate - optional: true - artifacts: - paths: - - XUNIT_RESULT.xml - - test_build_system - expire_in: 2 days - reports: - junit: XUNIT_RESULT.xml - when: always + parallel: 6 pytest_build_system_win_minimal_cmake: extends: - .test_build_system_template_win - .rules:labels:windows_pytest_build_system - needs: - - job: manual_gate - optional: true - tags: - - windows-build - artifacts: - paths: - - XUNIT_RESULT.xml - - test_build_system - expire_in: 2 days - reports: - junit: XUNIT_RESULT.xml - when: always variables: MINIMAL_CMAKE_VERSION: "3.16.3" script: - - .\install.ps1 --enable-ci --enable-pytest - - . .\export.ps1 - - python ${IDF_PATH}\tools\idf_tools.py install cmake@${MINIMAL_CMAKE_VERSION} + - python tools\idf_tools.py install cmake@${MINIMAL_CMAKE_VERSION} - $Env:PATH = "$Env:USERPROFILE\.espressif\tools\cmake\${MINIMAL_CMAKE_VERSION}\bin;$Env:PATH" - | $actualVersion = (& cmake --version).Split()[2] @@ -133,7 +89,6 @@ pytest_build_system_win_minimal_cmake: Write-Error "ERROR: Wrong CMake version! Detected: $actualVersion, but expected: $Env:MINIMAL_CMAKE_VERSION" exit 1 } - - python "${SUBMODULE_FETCH_TOOL}" -s "all" - - cd ${IDF_PATH}\tools\test_build_system - - python ${IDF_PATH}\tools\ci\get_known_failure_cases_file.py - - pytest -k cmake --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} + - cd tools\test_build_system + - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} + - pytest -k cmake --junitxml ${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} diff --git a/.gitlab/ci/upload_cache.yml b/.gitlab/ci/upload_cache.yml index 173b63224b4..320c0cd4244 100644 --- a/.gitlab/ci/upload_cache.yml +++ b/.gitlab/ci/upload_cache.yml @@ -6,26 +6,6 @@ stage: upload_cache image: $ESP_ENV_IMAGE -upload-pip-cache: - extends: - - .upload_cache_template - - .before_script:minimal - - .rules:upload-python-cache - tags: - - $GEO - - cache - cache: - key: pip-cache-${LATEST_GIT_TAG} - paths: - - .cache/pip - policy: push - script: - - rm -rf .cache/pip # clear old packages - - bash install.sh --enable-ci --enable-pytest - parallel: - matrix: - - GEO: [ 'shiny', 'brew' ] - upload-submodules-cache: extends: - .upload_cache_template diff --git a/.idf_build_apps.toml b/.idf_build_apps.toml new file mode 100644 index 00000000000..8072a45c0c1 --- /dev/null +++ b/.idf_build_apps.toml @@ -0,0 +1,63 @@ +config_rules = [ + 'sdkconfig.ci=default', + 'sdkconfig.ci.*=', + '=default', +] + +extra_pythonpaths = [ + '$IDF_PATH/tools/ci/python_packages', + '$IDF_PATH/tools/ci', + '$IDF_PATH/tools', +] +build_system = "idf_ci_local.app:IdfCMakeApp" + +recursive = true +check_warnings = true +keep_going = true +copy_sdkconfig = true +ignore_warning_files = [ + '$IDF_PATH/tools/ci/ignore_build_warnings.txt', +] + +build_dir = "build_@t_@w" +build_log_filename = "build_log.txt" +size_json_filename = "size_${CI_JOB_ID}.json" + +verbose = 1 # INFO + +# collect +collect_app_info_filename = "app_info_${CI_JOB_ID}.txt" +junitxml = "build_summary_${CI_JOB_ID}.xml" + +# manifest +check_manifest_rules = true +manifest_rootpath = "$IDF_PATH" +manifest_filepatterns = [ + '**/.build-test-rules.yml', +] + +# dependency-driven build +deactivate_dependency_driven_build_by_components = [ + 'cxx', + 'esp_common', + 'esp_hw_support', + 'esp_rom', + 'esp_system', + 'esp_timer', + 'freertos', + 'hal', + 'heap', + 'log', + 'newlib', + 'riscv', + 'soc', + 'xtensa', +] + +deactivate_dependency_driven_build_by_filepatterns = [ + # tools + 'tools/cmake/**/*', + 'tools/tools.json', + # ci + 'tools/ci/ignore_build_warnings.txt', +] diff --git a/.idf_ci.toml b/.idf_ci.toml new file mode 100644 index 00000000000..2ae7546d00e --- /dev/null +++ b/.idf_ci.toml @@ -0,0 +1,116 @@ +preserve_non_test_related_apps = false +filter_non_test_related_apps_by_modified_files = true +filter_apps_by_component_target = true +exclude_dirs = [ + 'tools/test_mkdfu', + 'tools/test_idf_size', + 'tools/test_idf_py', + 'tools/ci/test_autocomplete', + 'tools/test_build_system', + 'tools/bt/ble_log_console', +] + +[local_runtime_envs] +EXTRA_CFLAGS = "-Werror -Werror=deprecated-declarations -Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function -Wstrict-prototypes" +EXTRA_CXXFLAGS = "-Werror -Werror=deprecated-declarations -Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function" +LDGEN_CHECK_MAPPING = "1" +IDF_CI_BUILD = "1" + +[gitlab] + +[gitlab.build_pipeline] +workflow_name = "build_child_pipeline" +runs_per_job = 30 + +job_tags = ['build', 'shiny'] +job_template_name = '.dynamic_build_template' +job_template_jinja = '' # write in tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml +pre_yaml_jinja = """ +include: + - .gitlab/ci/common.yml + - tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml + - tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml +""" +yaml_jinja = """ +{{ settings.gitlab.build_pipeline.pre_yaml_jinja }} + +workflow: + name: {{ settings.gitlab.build_pipeline.workflow_name }} + rules: + - when: always + +{{ jobs }} +""" # simplified since we included the tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml + +[gitlab.test_pipeline] +job_template_name = '.dynamic_target_test_template' +job_template_jinja = '' # write in tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml +pre_yaml_jinja = """ +include: + - .gitlab/ci/common.yml + - tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml +""" + +[gitlab.artifacts.s3] +enable = true + +[gitlab.artifacts.s3.configs.debug] +bucket = "idf-artifacts" +zip_first = true +build_dir_pattern = "**/build*/" +patterns = [ + 'bootloader/*.map', + 'bootloader/*.elf', + '*.map', + '*.elf', + # customized + 'esp_tee/*.map', + 'esp_tee/*.elf', + 'gdbinit/*', +] +if_clause = 'CI_JOB_GROUP_NAME != "build_non_test_related_apps"' + +[gitlab.artifacts.s3.configs.flash] +bucket = "idf-artifacts" +zip_first = true +build_dir_pattern = "**/build*/" +patterns = [ + 'bootloader/*.bin', + '*.bin', + 'partition_table/*.bin', + 'flasher_args.json', + 'flash_project_args', + 'config/sdkconfig.json', + 'sdkconfig', + 'project_description.json', + # customized + 'esp_tee/*.bin', +] +if_clause = 'CI_JOB_GROUP_NAME != "build_non_test_related_apps"' + +[gitlab.artifacts.s3.configs.log] +bucket = "idf-artifacts" +patterns = [ + '**/build*/build_log.txt', + '**/build*/size*.json', +] + +[gitlab.artifacts.s3.configs.junit] +bucket = "idf-artifacts" +patterns = [ + '**/XUNIT_RESULT_*.xml', + '**/build_summary_*.xml', +] + +[gitlab.artifacts.s3.configs.env] +bucket = "idf-artifacts" +patterns = [ + '**/pipeline.env', +] + +[gitlab.artifacts.s3.configs.longterm] +bucket = "longterm" +if_clause = 'CI_COMMIT_REF_NAME == "master"' +patterns = [ + '**/build*/size*.json', +] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6728f0b9916..32a23c959b1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,9 +1,15 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks -default_stages: [commit] +default_stages: [pre-commit] repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: "v0.9.7" + hooks: + - id: ruff-format + - id: ruff + args: [ "--fix" ] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: @@ -14,6 +20,7 @@ repos: # 2 - any file matching *test*/*expected* (for host tests, if possible use this naming pattern always) # 3 - any directory named 'testdata' # 4 - protobuf auto-generated files + # 5 - COPYING files exclude: &whitespace_excludes | (?x)^( .+\.(md|rst|map|bin)| @@ -41,21 +48,6 @@ repos: - id: no-commit-to-branch name: Do not use uppercase letters in the branch name args: ['--pattern', '^[^A-Z]*[A-Z]'] - - repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 - hooks: - - id: flake8 - args: ['--config=.flake8', '--tee', '--benchmark'] - - repo: https://github.com/asottile/reorder-python-imports - rev: v3.12.0 - hooks: - - id: reorder-python-imports - name: Reorder Python imports - args: [--py38-plus] - exclude: > - (?x)^( - .*_pb2.py - )$ - repo: https://github.com/codespell-project/codespell rev: v2.3.0 hooks: @@ -126,7 +118,7 @@ repos: name: Check tools dir files patterns entry: tools/ci/check_tools_files_patterns.py language: python - files: '^tools/.+' + files: '^(tools/.+|.gitlab/ci/rules\.yml)' additional_dependencies: - PyYAML == 5.3.1 pass_filenames: false @@ -150,11 +142,11 @@ repos: name: Check if all apps readme files match given .build-test-rules.yml files. Modify the supported target tables entry: tools/ci/check_build_test_rules.py check-readmes language: python - files: 'tools/test_apps/.+|examples/.+|components/.+|tools/idf_py_actions/constants.py|tools/ci/check_build_test_rules.py' + files: 'tools/.+|examples/.+|components/.+' require_serial: true additional_dependencies: - PyYAML == 5.3.1 - - idf-build-apps~=2.5 + - idf-build-apps~=2.16 - id: sort-yaml-files name: sort yaml files entry: tools/ci/sort_yaml.py @@ -169,29 +161,12 @@ repos: files: 'tools/ci/sort_yaml\.py$' additional_dependencies: - ruamel.yaml - - id: check-build-test-rules-path-exists - name: check path in .build-test-rules.yml exists - entry: tools/ci/check_build_test_rules.py check-exist - language: python - additional_dependencies: - - PyYAML == 5.3.1 - always_run: true - pass_filenames: false - require_serial: true - id: cleanup-ignore-lists name: Remove non-existing patterns from ignore lists entry: tools/ci/cleanup_ignore_lists.py language: python always_run: true require_serial: true - - id: gitlab-yaml-linter - name: Check gitlab yaml files - entry: tools/ci/gitlab_yaml_linter.py - language: python - files: '\.gitlab-ci\.yml|\.gitlab/ci/.+\.yml|\.gitmodules' - pass_filenames: false - additional_dependencies: - - PyYAML == 5.3.1 - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.0.1 hooks: @@ -245,3 +220,8 @@ repos: hooks: - id: check-kconfig-files - id: check-deprecated-kconfig-options + - repo: https://github.com/espressif/idf-ci + rev: v1.2.1 + hooks: + - id: check-tests-missing-config + files: 'pytest_.*\.py$|sdkconfig(\..*)?$' diff --git a/components/bootloader_support/test_apps/bootloader_support/pytest_bootloader_support.py b/components/bootloader_support/test_apps/bootloader_support/pytest_bootloader_support.py index 1a7397fd605..8f49283b710 100644 --- a/components/bootloader_support/test_apps/bootloader_support/pytest_bootloader_support.py +++ b/components/bootloader_support/test_apps/bootloader_support/pytest_bootloader_support.py @@ -11,14 +11,12 @@ def test_bootloader_support(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.host_test @pytest.mark.qemu @idf_parametrize('target', ['esp32'], indirect=['target']) def test_bootloader_support_qemu_esp32(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.host_test @pytest.mark.qemu @idf_parametrize('target', ['esp32c3'], indirect=['target']) def test_bootloader_support_qemu_esp32c3(dut: Dut) -> None: diff --git a/components/console/test_apps/console/pytest_console.py b/components/console/test_apps/console/pytest_console.py index 3d6243391f4..672d016de2e 100644 --- a/components/console/test_apps/console/pytest_console.py +++ b/components/console/test_apps/console/pytest_console.py @@ -14,7 +14,10 @@ def do_test_quit(dut: Dut) -> None: def do_test_help_generic(dut: Dut, registration_order: str) -> None: dut.expect_exact('Press ENTER to see the list of tests') - dut.confirm_write('"esp console help command - {} registration"'.format(registration_order), expect_str='esp>') + dut.confirm_write( + f'"esp console help command - {registration_order} registration"', + expect_str='esp>', + ) dut.confirm_write('help', expect_str='aaa') @@ -45,63 +48,59 @@ def do_test_help_quit(dut: Dut) -> None: dut.expect(r'quit\s+Quit REPL environment\s+esp>') -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console(dut: Dut, test_on: str) -> None: +def test_console(dut: Dut) -> None: dut.run_all_single_board_cases(group='!ignore', timeout=120) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_repl(dut: Dut, test_on: str) -> None: +def test_console_repl(dut: Dut) -> None: do_test_quit(dut) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_sorted_registration(dut: Dut, test_on: str) -> None: +def test_console_help_sorted_registration(dut: Dut) -> None: do_test_help_generic(dut, 'sorted') -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_reverse_registration(dut: Dut, test_on: str) -> None: +def test_console_help_reverse_registration(dut: Dut) -> None: dut.expect_exact('Press ENTER to see the list of tests') dut.confirm_write('"esp console help command - reverse registration"', expect_str='esp>') @@ -126,45 +125,41 @@ def test_console_help_reverse_registration(dut: Dut, test_on: str) -> None: @idf_parametrize('config', ['sorted'], indirect=['config']) @idf_parametrize('target', ['linux'], indirect=['target']) -@idf_parametrize('test_on,markers', [('host', (pytest.mark.host_test,))]) -def test_console_sorted_help_sorted_registration(dut: Dut, test_on: str) -> None: +def test_console_sorted_help_sorted_registration(dut: Dut) -> None: do_test_help_generic(dut, 'sorted') @idf_parametrize('config', ['sorted'], indirect=['config']) @idf_parametrize('target', ['linux'], indirect=['target']) -@idf_parametrize('test_on,markers', [('host', (pytest.mark.host_test,))]) -def test_console_sorted_help_reverse_registration(dut: Dut, test_on: str) -> None: +def test_console_sorted_help_reverse_registration(dut: Dut) -> None: do_test_help_generic(dut, 'reverse') -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_quit(dut: Dut, test_on: str) -> None: +def test_console_help_quit(dut: Dut) -> None: do_test_help_quit(dut) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_verbose_level_0(dut: Dut, test_on: str) -> None: +def test_console_help_verbose_level_0(dut: Dut) -> None: help_verbose_info = 'Print the summary of all registered commands if no arguments are given,' dut.expect_exact('Press ENTER to see the list of tests') dut.confirm_write('"esp console help command - set verbose level = 0"', expect_str='esp>') @@ -175,18 +170,17 @@ def test_console_help_verbose_level_0(dut: Dut, test_on: str) -> None: dut.expect_exact('help', not_matching=help_verbose_info) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_verbose_level_1(dut: Dut, test_on: str) -> None: +def test_console_help_verbose_level_1(dut: Dut) -> None: help_verbose_info = 'Print the summary of all registered commands if no arguments are given,' dut.expect_exact('Press ENTER to see the list of tests') dut.confirm_write('"esp console help command - set verbose level = 1"', expect_str='esp>') @@ -196,18 +190,17 @@ def test_console_help_verbose_level_1(dut: Dut, test_on: str) -> None: dut.expect_exact(help_verbose_info) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_verbose_subcommand(dut: Dut, test_on: str) -> None: +def test_console_help_verbose_subcommand(dut: Dut) -> None: help_verbose_info = 'Print the summary of all registered commands if no arguments are given,' dut.expect_exact('Press ENTER to see the list of tests') dut.confirm_write('"esp console help command - --verbose sub command"', expect_str='esp>') @@ -222,18 +215,17 @@ def test_console_help_verbose_subcommand(dut: Dut, test_on: str) -> None: dut.expect_exact(help_verbose_info) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_deregister(dut: Dut, test_on: str) -> None: +def test_console_help_deregister(dut: Dut) -> None: dut.expect_exact('Press ENTER to see the list of tests') dut.confirm_write('"esp console deregister commands"', expect_str='esp>') @@ -241,22 +233,24 @@ def test_console_help_deregister(dut: Dut, test_on: str) -> None: # in the test sequence, command a is registered before registering command z, then # command a is deregistered; therefore, console shall not print command a's description - cmd_a_description, cmd_z_description = 'should appear first in help', 'should appear last in help' + cmd_a_description, cmd_z_description = ( + 'should appear first in help', + 'should appear last in help', + ) dut.expect_exact(cmd_z_description, not_matching=cmd_a_description) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_re_register(dut: Dut, test_on: str) -> None: +def test_console_help_re_register(dut: Dut) -> None: dut.expect_exact('Press ENTER to see the list of tests') dut.confirm_write('"esp console re-register commands"', expect_str='esp>') diff --git a/components/efuse/test_apps/pytest_efuse.py b/components/efuse/test_apps/pytest_efuse.py index 4ca945ab5b7..4351b347b53 100644 --- a/components/efuse/test_apps/pytest_efuse.py +++ b/components/efuse/test_apps/pytest_efuse.py @@ -6,7 +6,8 @@ from pytest_embedded_idf.utils import idf_parametrize @pytest.mark.temp_skip_ci( - targets=['esp32s2', 'esp32s3'], reason='eFuse for S2 and S3 is similar to the C3 chip, so testing on C3 is enough' + targets=['esp32s2', 'esp32s3'], + reason='eFuse for S2 and S3 is similar to the C3 chip, so testing on C3 is enough', ) @pytest.mark.generic @idf_parametrize('target', ['supported_targets'], indirect=['target']) @@ -15,13 +16,11 @@ def test_efuse(dut: Dut) -> None: @pytest.mark.qemu -@pytest.mark.host_test @idf_parametrize('target', ['esp32'], indirect=['target']) def test_efuse_qemu(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_efuse_linux(dut: Dut) -> None: dut.run_all_single_board_cases(timeout=60) diff --git a/components/esp_app_format/test_apps/pytest_esp_app_format.py b/components/esp_app_format/test_apps/pytest_esp_app_format.py index a0901831957..78bb1022b43 100644 --- a/components/esp_app_format/test_apps/pytest_esp_app_format.py +++ b/components/esp_app_format/test_apps/pytest_esp_app_format.py @@ -11,7 +11,6 @@ def test_esp_app_format(dut: Dut) -> None: dut.expect_unity_test_output() -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_esp_app_format_linux(dut: Dut) -> None: dut.expect_unity_test_output() diff --git a/components/esp_coex/test_apps/external_coex_function/pytest_external_coex_function.py b/components/esp_coex/test_apps/external_coex_function/pytest_external_coex_function.py index 4808b449746..27589a49d41 100644 --- a/components/esp_coex/test_apps/external_coex_function/pytest_external_coex_function.py +++ b/components/esp_coex/test_apps/external_coex_function/pytest_external_coex_function.py @@ -6,9 +6,7 @@ from pytest_embedded_idf.utils import idf_parametrize @pytest.mark.generic -@idf_parametrize( - 'target', ['esp32h2', 'esp32c3', 'esp32s2', 'esp32s3', 'esp32c6'], indirect=['target'] -) +@idf_parametrize('target', ['esp32h2', 'esp32c3', 'esp32s2', 'esp32s3', 'esp32c6'], indirect=['target']) def test_external_coex_unit_test(dut: Dut) -> None: dut.run_all_single_board_cases() @@ -27,30 +25,30 @@ def test_external_coex_unit_test_esp32c2_xtal26m(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.esp32c2eco4 +@pytest.mark.esp32c2_rev2 @pytest.mark.xtal_26mhz @pytest.mark.generic @pytest.mark.parametrize( 'config, baud', [ - ('esp32c2eco4_xtal26m', '74880'), + ('esp32c2_rev2_xtal26m', '74880'), ], indirect=True, ) @idf_parametrize('target', ['esp32c2'], indirect=['target']) -def test_external_coex_unit_test_esp32c2eco4_xtal26m(dut: Dut) -> None: +def test_external_coex_unit_test_esp32c2_rev2_xtal26m(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.esp32c3eco7 +@pytest.mark.esp32c3_rev1 @pytest.mark.generic @pytest.mark.parametrize( 'config', [ - 'esp32c3eco7', + 'esp32c3_rev1', ], indirect=True, ) @idf_parametrize('target', ['esp32c3'], indirect=['target']) -def test_external_coex_unit_test_esp32c3eco7(dut: Dut) -> None: +def test_external_coex_unit_test_esp32c3_rev1(dut: Dut) -> None: dut.run_all_single_board_cases() diff --git a/components/esp_coex/test_apps/external_coex_function/sdkconfig.ci.esp32c2eco4_xtal26m b/components/esp_coex/test_apps/external_coex_function/sdkconfig.ci.esp32c2_rev2_xtal26m similarity index 100% rename from components/esp_coex/test_apps/external_coex_function/sdkconfig.ci.esp32c2eco4_xtal26m rename to components/esp_coex/test_apps/external_coex_function/sdkconfig.ci.esp32c2_rev2_xtal26m diff --git a/components/esp_coex/test_apps/external_coex_function/sdkconfig.ci.esp32c3eco7 b/components/esp_coex/test_apps/external_coex_function/sdkconfig.ci.esp32c3_rev1 similarity index 100% rename from components/esp_coex/test_apps/external_coex_function/sdkconfig.ci.esp32c3eco7 rename to components/esp_coex/test_apps/external_coex_function/sdkconfig.ci.esp32c3_rev1 diff --git a/components/esp_event/host_test/esp_event_unit_test/pytest_esp_event_linux.py b/components/esp_event/host_test/esp_event_unit_test/pytest_esp_event_linux.py index fd4b56e68a1..41f73b7aa98 100644 --- a/components/esp_event/host_test/esp_event_unit_test/pytest_esp_event_linux.py +++ b/components/esp_event/host_test/esp_event_unit_test/pytest_esp_event_linux.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 -import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_esp_event_linux(dut: Dut) -> None: dut.expect_exact('All tests passed', timeout=5) diff --git a/components/esp_event/test_apps/pytest_esp_event.py b/components/esp_event/test_apps/pytest_esp_event.py index 397e1dc01f0..8b1a6dd60cd 100644 --- a/components/esp_event/test_apps/pytest_esp_event.py +++ b/components/esp_event/test_apps/pytest_esp_event.py @@ -19,9 +19,11 @@ def test_esp_event(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.host_test @pytest.mark.qemu -@pytest.mark.xfail('config.getvalue("target") == "esp32c3"', reason='Unstable on QEMU, needs investigation') +@pytest.mark.xfail( + 'config.getvalue("target") == "esp32c3"', + reason='Unstable on QEMU, needs investigation', +) @pytest.mark.parametrize( 'config', [ @@ -36,7 +38,6 @@ def test_esp_event_qemu(dut: Dut) -> None: dut._run_normal_case(case) -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) @pytest.mark.parametrize( 'config', diff --git a/components/esp_hw_support/test_apps/host_test_linux/pytest_esp_hw_support_linux.py b/components/esp_hw_support/test_apps/host_test_linux/pytest_esp_hw_support_linux.py index aa9da8d2a53..54068bc00a6 100644 --- a/components/esp_hw_support/test_apps/host_test_linux/pytest_esp_hw_support_linux.py +++ b/components/esp_hw_support/test_apps/host_test_linux/pytest_esp_hw_support_linux.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 -import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_esp_hw_support_linux(dut: Dut) -> None: dut.run_all_single_board_cases(group='!ignore', timeout=120) diff --git a/components/esp_hw_support/test_apps/sleep_retention/pytest_retention.py b/components/esp_hw_support/test_apps/sleep_retention/pytest_retention.py index db1f258bb9a..95456b85dd8 100644 --- a/components/esp_hw_support/test_apps/sleep_retention/pytest_retention.py +++ b/components/esp_hw_support/test_apps/sleep_retention/pytest_retention.py @@ -1,34 +1,10 @@ # SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 -import functools -from typing import Callable -from typing import Dict -from typing import List - import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -def target_list(targets: List[str]) -> Callable: - def decorator(func: Callable) -> Callable: - @functools.wraps(func) - def wrapper(*args: List, **kwargs: Dict) -> Callable: - return func(*args, **kwargs) # type: ignore - - for target in targets: - wrapper = pytest.mark.__getattr__(target)(wrapper) - - return wrapper - - return decorator - - -# SOC_PAU_SUPPORTED == 1 -retention_targets = ['esp32c6', 'esp32h2', 'esp32p4', 'esp32c5', 'esp32c61'] - - -@target_list(retention_targets) @pytest.mark.generic @idf_parametrize('target', ['esp32c61', 'esp32c5', 'esp32p4', 'esp32h2', 'esp32c6'], indirect=['target']) def test_sleep_retention(dut: Dut) -> None: diff --git a/components/esp_hw_support/test_apps/sleep_retention/sdkconfig.ci.defaults b/components/esp_hw_support/test_apps/sleep_retention/sdkconfig.ci.default similarity index 100% rename from components/esp_hw_support/test_apps/sleep_retention/sdkconfig.ci.defaults rename to components/esp_hw_support/test_apps/sleep_retention/sdkconfig.ci.default diff --git a/components/esp_mm/test_apps/mm/pytest_mmap.py b/components/esp_mm/test_apps/mm/pytest_mmap.py index 12413ed9cd8..f82c2c9e72a 100644 --- a/components/esp_mm/test_apps/mm/pytest_mmap.py +++ b/components/esp_mm/test_apps/mm/pytest_mmap.py @@ -3,6 +3,7 @@ import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize + # normal mmu tests @@ -20,14 +21,6 @@ def test_mmap(dut: Dut) -> None: # mmu tests with psram enabled -PSRAM_RELEASE_CONFIGS = [ - pytest.param('psram_release_esp32', marks=[pytest.mark.esp32]), - pytest.param('psram_release_esp32s2', marks=[pytest.mark.esp32s2]), - pytest.param('psram_release_esp32s3', marks=[pytest.mark.esp32s3]), - pytest.param('psram_release_esp32p4', marks=[pytest.mark.esp32p4]), -] - - @pytest.mark.generic @idf_parametrize( 'config,target', @@ -44,12 +37,6 @@ def test_mmap_psram(dut: Dut) -> None: # mmu tests with xip_psram -XIP_CONFIGS = [ - pytest.param('xip_psram_esp32s2', marks=[pytest.mark.esp32s2]), - pytest.param('xip_psram_esp32s3', marks=[pytest.mark.esp32s3]), -] - - @pytest.mark.generic @idf_parametrize( 'config,target', [('xip_psram_esp32s2', 'esp32s2'), ('xip_psram_esp32s3', 'esp32s3')], indirect=['config', 'target'] diff --git a/components/esp_partition/host_test/partition_api_test/pytest_esp_partition_linux.py b/components/esp_partition/host_test/partition_api_test/pytest_esp_partition_linux.py index 211df105eef..f0be1be4e99 100644 --- a/components/esp_partition/host_test/partition_api_test/pytest_esp_partition_linux.py +++ b/components/esp_partition/host_test/partition_api_test/pytest_esp_partition_linux.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 -import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_esp_partition_linux(dut: Dut) -> None: dut.expect_unity_test_output(timeout=5) diff --git a/components/esp_ringbuf/test_apps/pytest_esp_ringbuf.py b/components/esp_ringbuf/test_apps/pytest_esp_ringbuf.py index d3d91c3e7f0..892c599bed1 100644 --- a/components/esp_ringbuf/test_apps/pytest_esp_ringbuf.py +++ b/components/esp_ringbuf/test_apps/pytest_esp_ringbuf.py @@ -24,7 +24,6 @@ def test_esp_ringbuf(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.host_test @pytest.mark.qemu @pytest.mark.parametrize( 'target', @@ -46,7 +45,6 @@ def test_esp_ringbuf_qemu(dut: Dut) -> None: dut._run_normal_case(case) -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_esp_ringbuf_posix_simulator(dut: Dut) -> None: dut.run_all_single_board_cases(group='linux') diff --git a/components/esp_rom/test_apps/linux_rom_apis/pytest_esp_rom_linux.py b/components/esp_rom/test_apps/linux_rom_apis/pytest_esp_rom_linux.py index 14ac9488fae..2fd21501a61 100644 --- a/components/esp_rom/test_apps/linux_rom_apis/pytest_esp_rom_linux.py +++ b/components/esp_rom/test_apps/linux_rom_apis/pytest_esp_rom_linux.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 -import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_esp_rom_linux(dut: Dut) -> None: dut.expect_exact('All tests passed', timeout=5) diff --git a/components/esp_rom/test_apps/rom_tests/pytest_esp_rom.py b/components/esp_rom/test_apps/rom_tests/pytest_esp_rom.py index 679f7762c79..68d7963a905 100644 --- a/components/esp_rom/test_apps/rom_tests/pytest_esp_rom.py +++ b/components/esp_rom/test_apps/rom_tests/pytest_esp_rom.py @@ -8,14 +8,22 @@ from pytest_embedded_idf.utils import idf_parametrize @pytest.mark.generic @idf_parametrize( 'target', - ['esp32c3', 'esp32s2', 'esp32s3', 'esp32c5', 'esp32c6', 'esp32c61', 'esp32h2', 'esp32p4'], + [ + 'esp32c3', + 'esp32s2', + 'esp32s3', + 'esp32c5', + 'esp32c6', + 'esp32c61', + 'esp32h2', + 'esp32p4', + ], indirect=['target'], ) def test_esp_rom(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.host_test @pytest.mark.qemu @idf_parametrize('target', ['esp32c3'], indirect=['target']) def test_esp_rom_qemu(dut: Dut) -> None: diff --git a/components/esp_system/test_apps/console/pytest_esp_system_console_tests.py b/components/esp_system/test_apps/console/pytest_esp_system_console_tests.py index fa8a49cdfc7..2ac8d8d59e9 100644 --- a/components/esp_system/test_apps/console/pytest_esp_system_console_tests.py +++ b/components/esp_system/test_apps/console/pytest_esp_system_console_tests.py @@ -15,15 +15,6 @@ def not_expect(dut: Dut, output_regex: str) -> None: raise RuntimeError(f'Found not_expect output {output_regex}') -JTAG_SERIAL_MARKS = [ - pytest.mark.esp32s3, - pytest.mark.esp32c2, - pytest.mark.esp32c3, - pytest.mark.esp32c6, - pytest.mark.esp32h2, -] - - @pytest.mark.generic @idf_parametrize('config', ['console_none'], indirect=['config']) @idf_parametrize('target', ['supported_targets'], indirect=['target']) diff --git a/components/esp_system/test_apps/linux_apis/pytest_esp_system_linux.py b/components/esp_system/test_apps/linux_apis/pytest_esp_system_linux.py index a2ada0ea611..c4855aa3c6b 100644 --- a/components/esp_system/test_apps/linux_apis/pytest_esp_system_linux.py +++ b/components/esp_system/test_apps/linux_apis/pytest_esp_system_linux.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 -import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_esp_system_linux(dut: Dut) -> None: dut.run_all_single_board_cases(timeout=60) diff --git a/components/esp_timer/test_apps/pytest_esp_timer_ut.py b/components/esp_timer/test_apps/pytest_esp_timer_ut.py index c20b5e871ff..cb53b96de7e 100644 --- a/components/esp_timer/test_apps/pytest_esp_timer_ut.py +++ b/components/esp_timer/test_apps/pytest_esp_timer_ut.py @@ -4,19 +4,6 @@ import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -CONFIGS = [ - pytest.param('general', marks=[pytest.mark.supported_targets]), - pytest.param('release', marks=[pytest.mark.supported_targets]), - pytest.param('single_core', marks=[pytest.mark.esp32]), - pytest.param('freertos_compliance', marks=[pytest.mark.esp32]), - pytest.param('isr_dispatch_esp32', marks=[pytest.mark.esp32]), - pytest.param('isr_dispatch_esp32c3', marks=[pytest.mark.esp32c3]), - pytest.param('cpu1_esp32', marks=[pytest.mark.esp32]), - pytest.param('any_cpu_esp32', marks=[pytest.mark.esp32]), - pytest.param('cpu1_esp32s3', marks=[pytest.mark.esp32s3]), - pytest.param('any_cpu_esp32s3', marks=[pytest.mark.esp32s3]), -] - @pytest.mark.generic @idf_parametrize( diff --git a/components/esp_wifi/test_apps/wifi_connect/pytest_wifi_connect.py b/components/esp_wifi/test_apps/wifi_connect/pytest_wifi_connect.py index 1486043502b..847807807bf 100644 --- a/components/esp_wifi/test_apps/wifi_connect/pytest_wifi_connect.py +++ b/components/esp_wifi/test_apps/wifi_connect/pytest_wifi_connect.py @@ -36,28 +36,28 @@ def test_wifi_connect_cases_esp32c2_xtal26m(case_tester: CaseTester) -> None: case_tester.run_all_cases() -@pytest.mark.esp32c2eco4 +@pytest.mark.esp32c2_rev2 @pytest.mark.two_duts @pytest.mark.xtal_26mhz @pytest.mark.parametrize( 'count, config, target, baud', [ - (2, 'esp32c2eco4_xtal26m', 'esp32c2', '74880'), + (2, 'esp32c2_rev2_xtal26m', 'esp32c2', '74880'), ], indirect=True, ) -def test_wifi_connect_cases_esp32c2eco4_xtal26m(case_tester: CaseTester) -> None: +def test_wifi_connect_cases_esp32c2_rev2_xtal26m(case_tester: CaseTester) -> None: case_tester.run_all_cases() @pytest.mark.two_duts -@pytest.mark.esp32c3eco7 +@pytest.mark.esp32c3_rev1 @pytest.mark.parametrize( 'count, config, target', [ - (2, 'esp32c3eco7', 'esp32c3'), + (2, 'esp32c3_rev1', 'esp32c3'), ], indirect=True, ) -def test_wifi_connect_cases_esp32c3eco7(case_tester: CaseTester) -> None: +def test_wifi_connect_cases_esp32c3_rev1(case_tester: CaseTester) -> None: case_tester.run_all_cases() diff --git a/components/esp_wifi/test_apps/wifi_connect/sdkconfig.ci.esp32c2eco4_xtal26m b/components/esp_wifi/test_apps/wifi_connect/sdkconfig.ci.esp32c2_rev2_xtal26m similarity index 100% rename from components/esp_wifi/test_apps/wifi_connect/sdkconfig.ci.esp32c2eco4_xtal26m rename to components/esp_wifi/test_apps/wifi_connect/sdkconfig.ci.esp32c2_rev2_xtal26m diff --git a/components/esp_wifi/test_apps/wifi_connect/sdkconfig.ci.esp32c3eco7 b/components/esp_wifi/test_apps/wifi_connect/sdkconfig.ci.esp32c3_rev1 similarity index 100% rename from components/esp_wifi/test_apps/wifi_connect/sdkconfig.ci.esp32c3eco7 rename to components/esp_wifi/test_apps/wifi_connect/sdkconfig.ci.esp32c3_rev1 diff --git a/components/esp_wifi/test_apps/wifi_function/pytest_wifi_function.py b/components/esp_wifi/test_apps/wifi_function/pytest_wifi_function.py index ec28081ac20..e4b1a9e22ac 100644 --- a/components/esp_wifi/test_apps/wifi_function/pytest_wifi_function.py +++ b/components/esp_wifi/test_apps/wifi_function/pytest_wifi_function.py @@ -3,6 +3,7 @@ import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize + # @pytest.mark.esp32c2 # esp32c2 are using xtal_26mhz @@ -28,26 +29,26 @@ def test_wifi_unit_test_esp32c2_xtal26m(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.esp32c2eco4 +@pytest.mark.esp32c2_rev2 @pytest.mark.xtal_26mhz @pytest.mark.generic @pytest.mark.parametrize( 'config, target, baud', [ - ('esp32c2eco4_xtal26m', 'esp32c2', '74880'), + ('esp32c2_rev2_xtal26m', 'esp32c2', '74880'), ], indirect=True, ) -def test_wifi_unit_test_esp32c2eco4_xtal26m(dut: Dut) -> None: +def test_wifi_unit_test_esp32c2_rev2_xtal26m(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.esp32c3eco7 +@pytest.mark.esp32c3_rev1 @pytest.mark.generic @pytest.mark.parametrize( 'config, target', - [('esp32c3eco7', 'esp32c3')], + [('esp32c3_rev1', 'esp32c3')], indirect=True, ) -def test_wifi_unit_test_esp32c3eco7(dut: Dut) -> None: +def test_wifi_unit_test_esp32c3_rev1(dut: Dut) -> None: dut.run_all_single_board_cases() diff --git a/components/esp_wifi/test_apps/wifi_function/sdkconfig.ci.esp32c2eco4_xtal26m b/components/esp_wifi/test_apps/wifi_function/sdkconfig.ci.esp32c2_rev2_xtal26m similarity index 100% rename from components/esp_wifi/test_apps/wifi_function/sdkconfig.ci.esp32c2eco4_xtal26m rename to components/esp_wifi/test_apps/wifi_function/sdkconfig.ci.esp32c2_rev2_xtal26m diff --git a/components/esp_wifi/test_apps/wifi_function/sdkconfig.ci.esp32c3eco7 b/components/esp_wifi/test_apps/wifi_function/sdkconfig.ci.esp32c3_rev1 similarity index 100% rename from components/esp_wifi/test_apps/wifi_function/sdkconfig.ci.esp32c3eco7 rename to components/esp_wifi/test_apps/wifi_function/sdkconfig.ci.esp32c3_rev1 diff --git a/components/fatfs/host_test/pytest_fatfs_linux.py b/components/fatfs/host_test/pytest_fatfs_linux.py index 96cdd40207f..404989452bc 100644 --- a/components/fatfs/host_test/pytest_fatfs_linux.py +++ b/components/fatfs/host_test/pytest_fatfs_linux.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 -import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_fatfs_linux(dut: Dut) -> None: dut.expect_exact('All tests passed', timeout=120) diff --git a/components/freertos/test_apps/freertos/pytest_freertos.py b/components/freertos/test_apps/freertos/pytest_freertos.py index b9090c91ba4..324011e51ca 100644 --- a/components/freertos/test_apps/freertos/pytest_freertos.py +++ b/components/freertos/test_apps/freertos/pytest_freertos.py @@ -4,28 +4,6 @@ import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -CONFIGS = [ - pytest.param( - 'default', - marks=[ - pytest.mark.supported_targets, - ], - ), - pytest.param('freertos_options', marks=[pytest.mark.supported_targets]), - pytest.param('psram', marks=[pytest.mark.esp32, pytest.mark.esp32s3, pytest.mark.esp32p4, pytest.mark.esp32c5]), - pytest.param('single_core', marks=[pytest.mark.esp32, pytest.mark.esp32p4]), - # TODO: [ESP32C5] IDF-10335 - # TODO: [ESP32C61] IDF-11146 - pytest.param( - 'smp', - marks=[ - pytest.mark.supported_targets, - pytest.mark.temp_skip_ci(targets=['esp32p4', 'esp32c5', 'esp32c61'], reason='test failed/TBD IDF-8113'), - ], - ), - pytest.param('tickless_idle', marks=[pytest.mark.supported_targets]), -] - @pytest.mark.generic @idf_parametrize( @@ -34,9 +12,6 @@ CONFIGS = [ ('default', 'supported_targets'), ('freertos_options', 'supported_targets'), ('psram', 'esp32'), - ('psram', 'esp32c5'), - ('psram', 'esp32p4'), - ('psram', 'esp32s3'), ('single_core', 'esp32'), ('single_core', 'esp32p4'), ( diff --git a/components/hal/test_apps/hal_utils/pytest_hal_utils.py b/components/hal/test_apps/hal_utils/pytest_hal_utils.py index c49b80b0908..94ae238393c 100644 --- a/components/hal/test_apps/hal_utils/pytest_hal_utils.py +++ b/components/hal/test_apps/hal_utils/pytest_hal_utils.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 -import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_hal_utils(dut: Dut) -> None: dut.run_all_single_board_cases() diff --git a/components/heap/test_apps/heap_tests/pytest_heap.py b/components/heap/test_apps/heap_tests/pytest_heap.py index 9d53f81392e..2d891f0380d 100644 --- a/components/heap/test_apps/heap_tests/pytest_heap.py +++ b/components/heap/test_apps/heap_tests/pytest_heap.py @@ -20,11 +20,14 @@ def test_heap_poisoning(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.host_test @pytest.mark.qemu @pytest.mark.parametrize( 'config, embedded_services', - [('no_poisoning', 'idf,qemu'), ('light_poisoning', 'idf,qemu'), ('comprehensive_poisoning', 'idf,qemu')], + [ + ('no_poisoning', 'idf,qemu'), + ('light_poisoning', 'idf,qemu'), + ('comprehensive_poisoning', 'idf,qemu'), + ], ) @idf_parametrize('target', ['esp32', 'esp32c3'], indirect=['target']) def test_heap_poisoning_qemu(dut: Dut) -> None: diff --git a/components/heap/test_apps/host_test_linux/pytest_heap_linux.py b/components/heap/test_apps/host_test_linux/pytest_heap_linux.py index 2eea0388805..ecb228e4a91 100644 --- a/components/heap/test_apps/host_test_linux/pytest_heap_linux.py +++ b/components/heap/test_apps/host_test_linux/pytest_heap_linux.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 -import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_heap_linux(dut: Dut) -> None: dut.run_all_single_board_cases(timeout=60) diff --git a/components/linux/test_apps/linux_test/pytest_linux_test.py b/components/linux/test_apps/linux_test/pytest_linux_test.py index 1d6d342758e..ece04715156 100644 --- a/components/linux/test_apps/linux_test/pytest_linux_test.py +++ b/components/linux/test_apps/linux_test/pytest_linux_test.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: 2024-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 -import pytest from pytest_embedded_idf.dut import IdfDut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_linux_component(dut: IdfDut) -> None: dut.expect_exact('Press ENTER to see the list of tests.') diff --git a/components/log/host_test/log_test/pytest_log_linux.py b/components/log/host_test/log_test/pytest_log_linux.py index 5f5ffd10514..e2bd7d63e99 100644 --- a/components/log/host_test/log_test/pytest_log_linux.py +++ b/components/log/host_test/log_test/pytest_log_linux.py @@ -5,12 +5,12 @@ from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @pytest.mark.parametrize( 'config', [ 'default', - 'system_timestamptag_level_linked_list', + 'system_timestamp', + 'tag_level_linked_list', 'tag_level_linked_list_and_array_cache', 'tag_level_none', ], diff --git a/components/nvs_flash/host_test/nvs_host_test/pytest_nvs_host_linux.py b/components/nvs_flash/host_test/nvs_host_test/pytest_nvs_host_linux.py deleted file mode 100644 index 195d7cee559..00000000000 --- a/components/nvs_flash/host_test/nvs_host_test/pytest_nvs_host_linux.py +++ /dev/null @@ -1,11 +0,0 @@ -# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Unlicense OR CC0-1.0 -import pytest -from pytest_embedded import Dut -from pytest_embedded_idf.utils import idf_parametrize - - -@pytest.mark.host_test -@idf_parametrize('target', ['linux'], indirect=['target']) -def test_nvs_host_linux(dut: Dut) -> None: - dut.expect_exact('All tests passed', timeout=60) diff --git a/components/nvs_flash/host_test/nvs_page_test/pytest_nvs_page_linux.py b/components/nvs_flash/host_test/nvs_page_test/pytest_nvs_page_linux.py index c916e3b904f..902094c543a 100644 --- a/components/nvs_flash/host_test/nvs_page_test/pytest_nvs_page_linux.py +++ b/components/nvs_flash/host_test/nvs_page_test/pytest_nvs_page_linux.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 -import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_nvs_page_linux(dut: Dut) -> None: dut.expect_unity_test_output(timeout=120) diff --git a/components/nvs_flash/test_apps/pytest_nvs_flash.py b/components/nvs_flash/test_apps/pytest_nvs_flash.py index fe236968d33..19ce3903f1b 100644 --- a/components/nvs_flash/test_apps/pytest_nvs_flash.py +++ b/components/nvs_flash/test_apps/pytest_nvs_flash.py @@ -4,11 +4,6 @@ import pytest from pytest_embedded_idf.dut import IdfDut from pytest_embedded_idf.utils import idf_parametrize -CONFIGS_NVS_ENCR_FLASH_ENC = [ - pytest.param('nvs_encr_flash_enc_esp32', marks=[pytest.mark.esp32]), - pytest.param('nvs_encr_flash_enc_esp32c3', marks=[pytest.mark.esp32c3]), -] - @pytest.mark.generic @pytest.mark.parametrize('config', ['default'], indirect=True) diff --git a/components/pthread/test_apps/pthread_unity_tests/pytest_pthread_unity_tests.py b/components/pthread/test_apps/pthread_unity_tests/pytest_pthread_unity_tests.py index 1fca0040950..768d99d5138 100644 --- a/components/pthread/test_apps/pthread_unity_tests/pytest_pthread_unity_tests.py +++ b/components/pthread/test_apps/pthread_unity_tests/pytest_pthread_unity_tests.py @@ -20,7 +20,9 @@ def test_pthread(dut: Dut) -> None: @pytest.mark.generic @idf_parametrize( - 'config,target', [('single_core_esp32', 'esp32'), ('single_core_esp32s3', 'esp32s3')], indirect=['config', 'target'] + 'config,target', + [('single_core_esp32', 'esp32'), ('single_core_esp32s3', 'esp32s3')], + indirect=['config', 'target'], ) def test_pthread_single_core(dut: Dut) -> None: dut.run_all_single_board_cases(group='!thread-specific', timeout=300) @@ -46,7 +48,6 @@ def test_pthread_single_core_tls(dut: Dut) -> None: dut.run_all_single_board_cases(group='thread-specific', timeout=300) -@pytest.mark.host_test @pytest.mark.qemu @idf_parametrize('target', ['esp32'], indirect=['target']) def test_pthread_qemu(dut: Dut) -> None: @@ -55,7 +56,6 @@ def test_pthread_qemu(dut: Dut) -> None: dut._run_normal_case(case, timeout=75) -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_pthread_linux(dut: Dut) -> None: dut.run_all_single_board_cases(timeout=120) diff --git a/components/spiffs/host_test/pytest_spiffs_linux.py b/components/spiffs/host_test/pytest_spiffs_linux.py index 46e5e062fc7..0c9746fe492 100644 --- a/components/spiffs/host_test/pytest_spiffs_linux.py +++ b/components/spiffs/host_test/pytest_spiffs_linux.py @@ -5,7 +5,6 @@ from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @pytest.mark.parametrize('config', ['erase_check', 'no_erase_check']) @idf_parametrize('target', ['linux'], indirect=['target']) def test_spiffs_linux(dut: Dut) -> None: diff --git a/components/tcp_transport/test_apps/pytest_tcp_transport_ut.py b/components/tcp_transport/test_apps/pytest_tcp_transport_ut.py index 39652cc5e3f..b7c123959ea 100644 --- a/components/tcp_transport/test_apps/pytest_tcp_transport_ut.py +++ b/components/tcp_transport/test_apps/pytest_tcp_transport_ut.py @@ -4,11 +4,6 @@ import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -CONFIGS = [ - pytest.param('default', marks=[pytest.mark.esp32, pytest.mark.esp32c3]), - pytest.param('psram_esp32', marks=[pytest.mark.esp32]), -] - @pytest.mark.generic @idf_parametrize( diff --git a/components/usb/test_apps/hcd/pytest_usb_hcd.py b/components/usb/test_apps/hcd/pytest_usb_hcd.py index 5276b05f39c..6b2686cb479 100644 --- a/components/usb/test_apps/hcd/pytest_usb_hcd.py +++ b/components/usb/test_apps/hcd/pytest_usb_hcd.py @@ -4,11 +4,6 @@ import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -CONFIGS = [ - pytest.param('default', marks=[pytest.mark.esp32s2, pytest.mark.esp32s3, pytest.mark.esp32p4]), - pytest.param('esp32p4_psram', marks=[pytest.mark.esp32p4]), -] - @pytest.mark.usb_host_flash_disk @pytest.mark.temp_skip_ci(targets=['esp32s2'], reason='lack of runners with usb_host_flash_disk tag') diff --git a/components/wear_levelling/host_test/pytest_wear_levelling_linux.py b/components/wear_levelling/host_test/pytest_wear_levelling_linux.py index b25fb724e28..fdfa0b37b65 100644 --- a/components/wear_levelling/host_test/pytest_wear_levelling_linux.py +++ b/components/wear_levelling/host_test/pytest_wear_levelling_linux.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 -import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_wear_levelling_linux(dut: Dut) -> None: dut.expect_exact('All tests passed', timeout=120) diff --git a/components/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.py b/components/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.py index 67736ec77c8..aed8b0f0287 100644 --- a/components/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.py +++ b/components/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.py @@ -34,40 +34,40 @@ def test_wpa_supplicant_ut_offchan(case_tester: CaseTester) -> None: @pytest.mark.two_duts -@pytest.mark.esp32c2eco4 +@pytest.mark.esp32c2_rev2 @pytest.mark.xtal_26mhz @pytest.mark.parametrize( 'count, config, target, baud', [ ( 2, - 'esp32c2eco4_xtal26m', + 'esp32c2_rev2_xtal26m', 'esp32c2', '74880', ), ], indirect=True, ) -def test_wpa_supplicant_esp32c2eco4_xtal26mhz(case_tester: CaseTester) -> None: +def test_wpa_supplicant_esp32c2_rev2_xtal26mhz(case_tester: CaseTester) -> None: for case in case_tester.test_menu: if case.attributes.get('test_env') == 'two_duts': case_tester.run_multi_dev_case(case=case, reset=True) @pytest.mark.two_duts -@pytest.mark.esp32c3eco7 +@pytest.mark.esp32c3_rev1 @pytest.mark.parametrize( 'count, config, target', [ ( 2, - 'esp32c3eco7', + 'esp32c3_rev1', 'esp32c3', ), ], indirect=True, ) -def test_wpa_supplicant_esp32c3eco7(case_tester: CaseTester) -> None: +def test_wpa_supplicant_esp32c3_rev1(case_tester: CaseTester) -> None: for case in case_tester.test_menu: if case.attributes.get('test_env') == 'two_duts': case_tester.run_multi_dev_case(case=case, reset=True) diff --git a/components/wpa_supplicant/test_apps/sdkconfig.ci.esp32c2eco4_xtal26m b/components/wpa_supplicant/test_apps/sdkconfig.ci.esp32c2_rev2_xtal26m similarity index 100% rename from components/wpa_supplicant/test_apps/sdkconfig.ci.esp32c2eco4_xtal26m rename to components/wpa_supplicant/test_apps/sdkconfig.ci.esp32c2_rev2_xtal26m diff --git a/components/wpa_supplicant/test_apps/sdkconfig.ci.esp32c3eco7 b/components/wpa_supplicant/test_apps/sdkconfig.ci.esp32c3_rev1 similarity index 100% rename from components/wpa_supplicant/test_apps/sdkconfig.ci.esp32c3eco7 rename to components/wpa_supplicant/test_apps/sdkconfig.ci.esp32c3_rev1 diff --git a/conftest.py b/conftest.py index 16747203861..431ee44116f 100644 --- a/conftest.py +++ b/conftest.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # pylint: disable=W0621 # redefined-outer-name # @@ -9,6 +9,7 @@ # please report to https://github.com/espressif/pytest-embedded/issues # or discuss at https://github.com/espressif/pytest-embedded/discussions import os +import subprocess import sys if os.path.join(os.path.dirname(__file__), 'tools', 'ci') not in sys.path: @@ -17,8 +18,6 @@ if os.path.join(os.path.dirname(__file__), 'tools', 'ci') not in sys.path: if os.path.join(os.path.dirname(__file__), 'tools', 'ci', 'python_packages') not in sys.path: sys.path.append(os.path.join(os.path.dirname(__file__), 'tools', 'ci', 'python_packages')) -import glob -import io import json import logging import os @@ -26,7 +25,6 @@ import re import signal import time import typing as t -import zipfile from copy import deepcopy from urllib.parse import quote @@ -34,20 +32,19 @@ import common_test_methods # noqa: F401 import gitlab_api import pexpect import pytest -import requests -import yaml from _pytest.config import Config from _pytest.fixtures import FixtureRequest -from artifacts_handler import ArtifactType -from dynamic_pipelines.constants import TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME -from idf_ci_local.app import import_apps_from_txt -from idf_ci_local.uploader import AppDownloader, AppUploader -from idf_ci_utils import IDF_PATH, idf_relpath -from idf_pytest.constants import DEFAULT_SDKCONFIG, ENV_MARKERS, SPECIAL_MARKERS, TARGET_MARKERS, PytestCase, \ - DEFAULT_LOGDIR -from idf_pytest.plugin import IDF_PYTEST_EMBEDDED_KEY, ITEM_PYTEST_CASE_KEY, IdfPytestEmbedded +from idf_ci import PytestCase +from idf_ci.idf_pytest import IDF_CI_PYTEST_CASE_KEY +from idf_ci_utils import IDF_PATH +from idf_ci_utils import idf_relpath +from idf_pytest.constants import DEFAULT_LOGDIR +from idf_pytest.plugin import IDF_LOCAL_PLUGIN_KEY +from idf_pytest.plugin import IdfLocalPlugin +from idf_pytest.plugin import requires_elf_or_map from idf_pytest.utils import format_case_id -from pytest_embedded.plugin import multi_dut_argument, multi_dut_fixture +from pytest_embedded.plugin import _request_param_or_config_option_or_default +from pytest_embedded.plugin import multi_dut_fixture from pytest_embedded.utils import to_bytes from pytest_embedded.utils import to_str from pytest_embedded_idf.dut import IdfDut @@ -74,23 +71,6 @@ def case_tester(unity_tester: CaseTester) -> CaseTester: return unity_tester -@pytest.fixture -@multi_dut_argument -def config(request: FixtureRequest) -> str: - return getattr(request, 'param', None) or DEFAULT_SDKCONFIG # type: ignore - - -@pytest.fixture -@multi_dut_fixture -def target(request: FixtureRequest, dut_total: int, dut_index: int) -> str: - plugin = request.config.stash[IDF_PYTEST_EMBEDDED_KEY] - - if dut_total == 1: - return plugin.target[0] # type: ignore - - return plugin.target[dut_index] # type: ignore - - @pytest.fixture def test_func_name(request: FixtureRequest) -> str: return request.node.function.__name__ # type: ignore @@ -117,33 +97,69 @@ def pipeline_id(request: FixtureRequest) -> t.Optional[str]: return request.config.getoption('pipeline_id', None) or os.getenv('PARENT_PIPELINE_ID', None) # type: ignore -class BuildReportDownloader(AppDownloader): - def __init__(self, presigned_url_yaml: str) -> None: - self.app_presigned_urls_dict: t.Dict[str, t.Dict[str, str]] = yaml.safe_load(presigned_url_yaml) +def get_pipeline_commit_sha_by_pipeline_id(pipeline_id: str) -> t.Optional[str]: + gl = gitlab_api.Gitlab(os.getenv('CI_PROJECT_ID', 'espressif/esp-idf')) + pipeline = gl.project.pipelines.get(pipeline_id) + if not pipeline: + return None - def _download_app(self, app_build_path: str, artifact_type: ArtifactType) -> None: - url = self.app_presigned_urls_dict[app_build_path][artifact_type.value] + commit = gl.project.commits.get(pipeline.sha) + if not commit or not commit.parent_ids: + return None - logging.info('Downloading app from %s', url) - with io.BytesIO() as f: - for chunk in requests.get(url).iter_content(chunk_size=1024 * 1024): - if chunk: - f.write(chunk) + if len(commit.parent_ids) == 1: + return commit.parent_ids[0] # type: ignore - f.seek(0) + for parent_id in commit.parent_ids: + parent_commit = gl.project.commits.get(parent_id) + if parent_commit.parent_ids and len(parent_commit.parent_ids) == 1: + return parent_id # type: ignore - with zipfile.ZipFile(f) as zip_ref: - zip_ref.extractall(IDF_PATH) + return None - def download_app(self, app_build_path: str, artifact_type: t.Optional[ArtifactType] = None) -> None: - if app_build_path not in self.app_presigned_urls_dict: - raise ValueError( - f'No presigned url found for {app_build_path}. ' - f'Usually this should not happen, please re-trigger a pipeline.' - f'If this happens again, please report this bug to the CI channel.' - ) - super().download_app(app_build_path, artifact_type) +class AppDownloader: + def __init__( + self, + commit_sha: str, + pipeline_id: t.Optional[str] = None, + ) -> None: + self.commit_sha = commit_sha + self.pipeline_id = pipeline_id + + def download_app(self, app_dir: str, build_dir: str, artifact_type: t.Optional[str] = None) -> None: + args = [ + 'idf-ci', + 'gitlab', + 'download-artifacts', + '--commit-sha', + self.commit_sha, + ] + if artifact_type: + args.extend(['--type', artifact_type]) + + if self.pipeline_id: + args.extend(['--pipeline-id', self.pipeline_id]) + + args.extend( + [ + app_dir, + '--build-dir', + build_dir, + ] + ) + result = subprocess.run( + args, + capture_output=True, + text=True, + cwd=IDF_PATH, + ) + logging.info(result.stdout) + if result.stderr: + logging.info(result.stderr) + + +PRESIGNED_JSON = 'presigned.json' class OpenOCD: @@ -166,7 +182,7 @@ class OpenOCD: desc_path = os.path.join(self.dut.app.binary_path, 'project_description.json') try: - with open(desc_path, 'r') as f: + with open(desc_path) as f: project_desc = json.load(f) except FileNotFoundError: logging.error('Project description file not found at %s', desc_path) @@ -184,7 +200,7 @@ class OpenOCD: ocd_env = os.environ.copy() ocd_env['LIBUSB_DEBUG'] = '1' - for _ in range(1, self.MAX_RETRIES + 1): + for attempt in range(1, self.MAX_RETRIES + 1): try: self.proc = pexpect.spawn( command='openocd', @@ -197,10 +213,12 @@ class OpenOCD: if self.proc and self.proc.isalive(): self.proc.expect_exact('Info : Listening on port 3333 for gdb connections', timeout=5) self.connect_telnet() - self.write('log_output {}'.format(self.log_file)) + self.write(f'log_output {self.log_file}') return self except (pexpect.exceptions.EOF, pexpect.exceptions.TIMEOUT, ConnectionRefusedError) as e: - logging.error('Error running OpenOCD: %s', str(e)) + logging.error( + 'OpenOCD connection attempt %d/%d failed. Error: %s', attempt, self.MAX_RETRIES, type(e).__name__ + ) self.kill() time.sleep(self.RETRY_DELAY) @@ -255,38 +273,22 @@ def openocd_dut(dut: IdfDut) -> OpenOCD: @pytest.fixture(scope='session') def app_downloader(pipeline_id: t.Optional[str]) -> t.Optional[AppDownloader]: + if commit_sha := os.getenv('PIPELINE_COMMIT_SHA'): + logging.debug('pipeline commit sha from CI env is %s', commit_sha) + return AppDownloader(commit_sha, None) + if not pipeline_id: return None - if ( - 'IDF_S3_BUCKET' in os.environ - and 'IDF_S3_ACCESS_KEY' in os.environ - and 'IDF_S3_SECRET_KEY' in os.environ - and 'IDF_S3_SERVER' in os.environ - and 'IDF_S3_BUCKET' in os.environ - ): - return AppUploader(pipeline_id) + commit_sha = get_pipeline_commit_sha_by_pipeline_id(pipeline_id) + if not commit_sha: + raise ValueError( + 'commit sha cannot be found for pipeline id %s. Please check the pipeline id. ' + 'If you think this is a bug, please report it to CI team', + ) + logging.debug('pipeline commit sha of pipeline %s is %s', pipeline_id, commit_sha) - logging.info('Downloading build report from the build pipeline %s', pipeline_id) - test_app_presigned_urls_file = None - - gl = gitlab_api.Gitlab(os.getenv('CI_PROJECT_ID', 'espressif/esp-idf')) - - for child_pipeline in gl.project.pipelines.get(pipeline_id, lazy=True).bridges.list(iterator=True): - if child_pipeline.name == 'build_child_pipeline': - for job in gl.project.pipelines.get(child_pipeline.downstream_pipeline['id'], lazy=True).jobs.list( - iterator=True - ): - if job.name == 'generate_pytest_build_report': - test_app_presigned_urls_file = gl.download_artifact( - job.id, [TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME] - )[0] - break - - if test_app_presigned_urls_file: - return BuildReportDownloader(test_app_presigned_urls_file) - - return None + return AppDownloader(commit_sha, pipeline_id) @pytest.fixture @@ -310,14 +312,21 @@ def build_dir( valid build directory """ # download from minio on CI - case: PytestCase = request._pyfuncitem.stash[ITEM_PYTEST_CASE_KEY] - if app_downloader: + case: PytestCase = request.node.stash[IDF_CI_PYTEST_CASE_KEY] + if 'skip_app_downloader' in case.all_markers: + logging.debug('skip_app_downloader marker found, skip downloading app') + downloader = None + else: + downloader = app_downloader + + if downloader: + app_dir = idf_relpath(app_path) + build_dir = f'build_{target}_{config}' # somehow hardcoded... - app_build_path = os.path.join(idf_relpath(app_path), f'build_{target}_{config}') - if case.requires_elf_or_map: - app_downloader.download_app(app_build_path) + if requires_elf_or_map(case): + downloader.download_app(app_dir, build_dir) else: - app_downloader.download_app(app_build_path, ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES) + downloader.download_app(app_dir, build_dir, 'flash') check_dirs = [f'build_{target}_{config}'] else: check_dirs = [] @@ -438,7 +447,7 @@ def check_performance(idf_path: str) -> t.Callable[[str, float, str], None]: def _find_perf_item(operator: str, path: str) -> float: with open(path, encoding='utf-8') as f: data = f.read() - match = re.search(fr'#define\s+IDF_PERFORMANCE_{operator}_{item.upper()}\s+([\d.]+)', data) + match = re.search(rf'#define\s+IDF_PERFORMANCE_{operator}_{item.upper()}\s+([\d.]+)', data) return float(match.group(1)) # type: ignore def _check_perf(operator: str, standard_value: float) -> None: @@ -477,16 +486,18 @@ def check_performance(idf_path: str) -> t.Callable[[str, float, str], None]: @pytest.fixture -def log_minimum_free_heap_size(dut: IdfDut, config: str) -> t.Callable[..., None]: +def log_minimum_free_heap_size(dut: IdfDut, config: str, idf_path: str) -> t.Callable[..., None]: def real_func() -> None: res = dut.expect(r'Minimum free heap size: (\d+) bytes') logging.info( '\n------ heap size info ------\n' + '[app_path] {}\n' '[app_name] {}\n' '[config_name] {}\n' '[target] {}\n' '[minimum_free_heap_size] {} Bytes\n' '------ heap size end ------'.format( + dut.app.app_path.replace(idf_path, '').lstrip('/\\'), os.path.basename(dut.app.app_path), config, dut.target, @@ -499,12 +510,12 @@ def log_minimum_free_heap_size(dut: IdfDut, config: str) -> t.Callable[..., None @pytest.fixture(scope='session') def dev_password(request: FixtureRequest) -> str: - return request.config.getoption('dev_passwd') or '' + return _request_param_or_config_option_or_default(request, 'dev_password', '') # type: ignore @pytest.fixture(scope='session') def dev_user(request: FixtureRequest) -> str: - return request.config.getoption('dev_user') or '' + return _request_param_or_config_option_or_default(request, 'dev_user', '') # type: ignore ################## @@ -512,10 +523,6 @@ def dev_user(request: FixtureRequest) -> str: ################## def pytest_addoption(parser: pytest.Parser) -> None: idf_group = parser.getgroup('idf') - idf_group.addoption( - '--sdkconfig', - help='sdkconfig postfix, like sdkconfig.ci.. (Default: None, which would build all found apps)', - ) idf_group.addoption( '--dev-user', help='user name associated with some specific device/service used during the test execution', @@ -524,80 +531,34 @@ def pytest_addoption(parser: pytest.Parser) -> None: '--dev-passwd', help='password associated with some specific device/service used during the test execution', ) - idf_group.addoption( - '--app-info-filepattern', - help='glob pattern to specify the files that include built app info generated by ' - '`idf-build-apps --collect-app-info ...`. will not raise ValueError when binary ' - 'paths not exist in local file system if not listed recorded in the app info.', - ) idf_group.addoption( '--pipeline-id', - help='main pipeline id, not the child pipeline id. Specify this option to download the artifacts ' - 'from the minio server for debugging purpose.', + help='For users without s3 access. main pipeline id, not the child pipeline id. ' + 'Specify this option to download the artifacts from the minio server for debugging purpose.', ) def pytest_configure(config: Config) -> None: - # cli option "--target" - target = [_t.strip().lower() for _t in (config.getoption('target', '') or '').split(',') if _t.strip()] + from idf_pytest.constants import PREVIEW_TARGETS + from idf_pytest.constants import SUPPORTED_TARGETS + from pytest_embedded_idf.utils import preview_targets + from pytest_embedded_idf.utils import supported_targets - # add markers based on idf_pytest/constants.py - for name, description in { - **TARGET_MARKERS, - **ENV_MARKERS, - **SPECIAL_MARKERS, - }.items(): - config.addinivalue_line('markers', f'{name}: {description}') + supported_targets.set(SUPPORTED_TARGETS) + preview_targets.set(PREVIEW_TARGETS) - help_commands = ['--help', '--fixtures', '--markers', '--version'] - for cmd in help_commands: - if cmd in config.invocation_params.args: - target = ['unneeded'] - break - - markexpr = config.getoption('markexpr') or '' - # check marker expr set via "pytest -m" - if not target and markexpr: - # we use `-m "esp32 and generic"` in our CI to filter the test cases - # this doesn't cover all use cases, but fit what we do in CI. - for marker in markexpr.split('and'): - marker = marker.strip() - if marker in TARGET_MARKERS: - target.append(marker) - - # "--target" must be set - if not target: - raise SystemExit( - """Pass `--target TARGET[,TARGET...]` to specify all targets the test cases are using. - - for single DUT, we run with `pytest --target esp32` - - for multi DUT, we run with `pytest --target esp32,esp32,esp32s2` to indicate all DUTs -""" - ) - - apps = None - app_info_filepattern = config.getoption('app_info_filepattern') - if app_info_filepattern: - apps = [] - for f in glob.glob(os.path.join(IDF_PATH, app_info_filepattern)): - apps.extend(import_apps_from_txt(f)) - - if '--collect-only' not in config.invocation_params.args: - config.stash[IDF_PYTEST_EMBEDDED_KEY] = IdfPytestEmbedded( - config_name=config.getoption('sdkconfig'), - target=target, - apps=apps, - ) - config.pluginmanager.register(config.stash[IDF_PYTEST_EMBEDDED_KEY]) + config.stash[IDF_LOCAL_PLUGIN_KEY] = IdfLocalPlugin() + config.pluginmanager.register(config.stash[IDF_LOCAL_PLUGIN_KEY]) def pytest_unconfigure(config: Config) -> None: - _pytest_embedded = config.stash.get(IDF_PYTEST_EMBEDDED_KEY, None) - if _pytest_embedded: - del config.stash[IDF_PYTEST_EMBEDDED_KEY] - config.pluginmanager.unregister(_pytest_embedded) + idf_local_plugin = config.stash.get(IDF_LOCAL_PLUGIN_KEY, None) + if idf_local_plugin: + del config.stash[IDF_LOCAL_PLUGIN_KEY] + config.pluginmanager.unregister(idf_local_plugin) -dut_artifacts_url = [] +dut_artifacts_url: t.List[str] = [] @pytest.hookimpl(hookwrapper=True) @@ -620,10 +581,10 @@ def pytest_runtest_makereport(item, call): # type: ignore if isinstance(_dut, list): logs_files.extend([template.format(get_path(d.logfile)) for d in _dut]) - dut_artifacts_url.append('{}:'.format(_dut[0].test_case_name)) + dut_artifacts_url.append(f'{_dut[0].test_case_name}:') else: logs_files.append(template.format(get_path(_dut.logfile))) - dut_artifacts_url.append('{}:'.format(_dut.test_case_name)) + dut_artifacts_url.append(f'{_dut.test_case_name}:') for file in logs_files: dut_artifacts_url.append(' - {}'.format(quote(file, safe=':/'))) diff --git a/docs/en/contribute/esp-idf-tests-with-pytest.rst b/docs/en/contribute/esp-idf-tests-with-pytest.rst index 9210de1d8e8..75ab632e7e8 100644 --- a/docs/en/contribute/esp-idf-tests-with-pytest.rst +++ b/docs/en/contribute/esp-idf-tests-with-pytest.rst @@ -33,13 +33,13 @@ On the host side, ESP-IDF employs the pytest framework (alongside certain pytest Installation ============ -All basic dependencies could be installed by running the ESP-IDF install script with the ``--enable-pytest`` argument: +All basic dependencies could be installed by running the ESP-IDF install script with the ``--enable-ci`` argument: .. code-block:: bash - $ install.sh --enable-pytest + $ install.sh --enable-ci -Additional test script specific dependencies could be installed separately by running the ESP-IDF install script with the ``--enable-pytest-specific`` argument: +Additional test script specific dependencies could be installed separately by running the ESP-IDF install script with the ``--enable-test-specific`` argument: .. code-block:: bash @@ -120,6 +120,52 @@ Next, we have the environment marker: Finally, we have the test function. With a ``dut`` fixture. In single-dut test cases, the ``dut`` fixture is an instance of ``IdfDut`` class, for multi-dut test cases, it is a tuple of ``IdfDut`` instances. For more details regarding the ``IdfDut`` class, please refer to `pytest-embedded IdfDut API reference `__. +Running Tests on Linux +^^^^^^^^^^^^^^^^^^^^^^ + +To execute a pytest case on the Linux host, set ``target`` to ``linux``. + +.. code-block:: python + + @idf_parametrize('target', ['linux'], indirect=['target']) + def test_hello_world_linux(dut) -> None: + dut.expect('Hello world!') + +This is the simplest way to run the same test flow on the Linux host instead of on physical hardware. + +For a simple Linux-only test, setting ``target`` to ``linux`` is enough and the ``idf`` embedded services will be selected automatically. The ``pytest.mark.host_test`` marker is not required. + +For a mixed environment matrix, specify ``embedded_services`` manually for each case. See :ref:`Same App With Different Running Environments ` for a more complex example. + +.. only:: TARGET_SUPPORT_QEMU + + Running Tests in QEMU + ^^^^^^^^^^^^^^^^^^^^^ + + To execute a pytest case in QEMU, add the ``@pytest.mark.qemu`` marker to the test function. + + .. code-block:: python + + @pytest.mark.qemu + @idf_parametrize('target', ['esp32', 'esp32c3'], indirect=['target']) + def test_hello_world_qemu(dut) -> None: + dut.expect('Hello world!') + + This is the simplest way to run the same test flow in QEMU instead of on physical hardware. + + For a simple QEMU-only test, adding ``pytest.mark.qemu`` is enough and the ``idf,qemu`` embedded services will be selected automatically. + + For a mixed environment matrix, specify ``embedded_services`` manually for each case. See the later section in this guide for a more complex example. + + For QEMU installation and setup, refer to page :doc:`/api-guides/tools/qemu`. + +Deprecation of ``pytest.mark.host_test`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``pytest.mark.host_test`` is no longer needed and should not be added to new test cases. + +For Linux target test cases and QEMU test cases, the required behavior is handled dynamically by the test framework. In particular, the embedded services are selected automatically for simple Linux-only and QEMU-only cases. + Same App With Different sdkconfig Files ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -208,6 +254,49 @@ Now this test function would be replicated to 2 test cases (represented as test * ``esp32.foo.test_foo_bar`` * ``esp32s2.bar.test_foo_bar`` +.. _pytest-same-app-different-running-environments: + +Same App With Different Running Environments +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Sometimes the same app should be validated in different running environments, for example on the host with the Linux target, on real hardware, or in QEMU. If a single ``@pytest.mark.qemu`` test is not enough, combine ``target``, ``config``, and ``embedded_services`` in a single ``idf_parametrize`` decorator, and attach the required marker for each case. + +The following example is adapted from :idf_file:`components/console/test_apps/console/pytest_console.py`: + +.. code-block:: python + + @idf_parametrize( + 'target,config,embedded_services,markers', + [ + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), + ], + indirect=['target', 'config', 'embedded_services'], + ) + def test_console_repl(dut) -> None: + dut.expect_exact('Press ENTER to see the list of tests') + +This creates four test cases for the same app: + +* Linux host execution with the ``idf`` service +* ESP32 hardware execution with the ``esp,idf`` services +* ESP32-C3 hardware execution with the ``esp,idf`` services +* ESP32 execution in QEMU with the ``idf,qemu`` services + +When running locally, you can select only the environment you want: + +.. code-block:: shell + + $ pytest --target linux + $ pytest -m qemu + $ pytest -m qemu --target esp32 + +``pytest --target linux`` selects Linux target cases only. ``pytest -m qemu`` selects all QEMU-marked cases. ``pytest -m qemu --target esp32`` further limits the selection to QEMU cases for the ESP32 target. + +Use this pattern when the test logic is the same but the execution environment changes. + Testing Serial Output (Expecting) --------------------------------- @@ -492,7 +581,7 @@ First you need to install ESP-IDF with additional Python requirements: .. code-block:: shell $ cd $IDF_PATH - $ bash install.sh --enable-ci --enable-pytest + $ bash install.sh --enable-ci $ . ./export.sh Build Directories diff --git a/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst b/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst index dc3642fb58b..af9b59e8740 100644 --- a/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst +++ b/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst @@ -33,13 +33,13 @@ ESP-IDF 在主机端使用 pytest 框架（以及一些 pytest 插件）来自 安装 ============ -基础依赖项可以通过执行 ESP-IDF 安装脚本 ``--enable-pytest`` 进行安装： +基础依赖项可以通过执行 ESP-IDF 安装脚本 ``--enable-ci`` 进行安装： .. code-block:: bash - $ install.sh --enable-pytest + $ install.sh --enable-ci -额外的测试脚本依赖项可以通过执行 ESP-IDF 安装脚本 ``--enable-pytest-specific`` 进行安装： +额外的测试脚本依赖项可以通过执行 ESP-IDF 安装脚本 ``--enable-test-specific`` 进行安装： .. code-block:: bash @@ -120,6 +120,52 @@ ESP-IDF 在主机端使用 pytest 框架（以及一些 pytest 插件）来自 关于测试函数，使用了一个 ``dut`` fixture。在单一 DUT 测试用例中，``dut`` fixture 是 ``IdfDut`` 类的一个实例，对于多个 DUT 测试用例，它是 ``IdfDut`` 实例的一个元组。有关 ``IdfDut`` 类的更多详细信息，请参阅 `pytest-embedded IdfDut API 参考 `__。 +在 Linux 上运行测试 +^^^^^^^^^^^^^^^^^^^^ + +要在 Linux 主机上执行 pytest 测试用例，请将 ``target`` 设置为 ``linux``。 + +.. code-block:: python + + @idf_parametrize('target', ['linux'], indirect=['target']) + def test_hello_world_linux(dut) -> None: + dut.expect('Hello world!') + +这是在 Linux 主机上运行与物理硬件相同测试流程的最简单方式。 + +对于简单的纯 Linux 测试，只需将 ``target`` 设置为 ``linux``，系统会自动选择 ``idf`` 对应的 embedded services。``pytest.mark.host_test`` marker 不再需要。 + +对于混合运行环境矩阵，则需要为每个用例手动指定 ``embedded_services``。更复杂的示例请参阅 :ref:`在不同运行环境中运行相同的应用程序 ` 小节。 + +.. only:: TARGET_SUPPORT_QEMU + + 在 QEMU 中运行测试 + ^^^^^^^^^^^^^^^^^^^^^ + + 要在 QEMU 中执行 pytest 测试用例，请将 ``@pytest.mark.qemu`` 添加到测试函数上。 + + .. code-block:: python + + @pytest.mark.qemu + @idf_parametrize('target', ['esp32', 'esp32c3'], indirect=['target']) + def test_hello_world_qemu(dut) -> None: + dut.expect('Hello world!') + + 这是在 QEMU 中运行与物理硬件相同测试流程的最简单方式。 + + 对于简单的纯 QEMU 测试，只需添加 ``pytest.mark.qemu``，系统会自动选择 ``idf,qemu`` 对应的 embedded services。 + + 对于混合运行环境矩阵，则需要为每个用例手动指定 ``embedded_services``。更复杂的示例请参阅本指南后面的对应小节。 + + 有关 QEMU 的安装和配置，请参阅页面 :doc:`../api-guides/tools/qemu`。 + +``pytest.mark.host_test`` 的弃用说明 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``pytest.mark.host_test`` 已不再需要，也不应再添加到新的测试用例中。 + +对于 Linux target 测试用例和 QEMU 测试用例，相关行为会由测试框架动态处理。尤其是在简单的纯 Linux 或纯 QEMU 场景下，embedded services 会被自动选择。 + 使用不同的 sdkconfig 文件运行相同的应用程序 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -208,6 +254,49 @@ ESP-IDF 在主机端使用 pytest 框架（以及一些 pytest 插件）来自 * ``esp32.foo.test_foo_bar`` * ``esp32s2.bar.test_foo_bar`` +.. _pytest-same-app-different-running-environments: + +在不同运行环境中运行相同的应用程序 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +有时，同一个应用程序需要在不同的运行环境中进行验证，例如在 Linux target 的主机上、真实硬件上，或在 QEMU 中运行。如果单独使用 ``@pytest.mark.qemu`` 测试还不够，可以在一个 ``idf_parametrize`` 装饰器中组合 ``target``、``config`` 和 ``embedded_services``，并为每种情况附加所需的 marker。 + +下面的示例改编自 :idf_file:`components/console/test_apps/console/pytest_console.py`： + +.. code-block:: python + + @idf_parametrize( + 'target,config,embedded_services,markers', + [ + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), + ], + indirect=['target', 'config', 'embedded_services'], + ) + def test_console_repl(dut) -> None: + dut.expect_exact('Press ENTER to see the list of tests') + +这会为同一个应用程序生成 4 个测试用例： + +* 在 Linux 主机上使用 ``idf`` service 运行 +* 在 ESP32 硬件上使用 ``esp,idf`` services 运行 +* 在 ESP32-C3 硬件上使用 ``esp,idf`` services 运行 +* 在 QEMU 中以 ESP32 为目标，使用 ``idf,qemu`` services 运行 + +在本地运行时，可以按需只选择某一种运行环境： + +.. code-block:: shell + + $ pytest --target linux + $ pytest -m qemu + $ pytest -m qemu --target esp32 + +``pytest --target linux`` 只选择 Linux target 的测试用例。``pytest -m qemu`` 选择所有带有 QEMU marker 的测试用例。``pytest -m qemu --target esp32`` 会进一步把范围限制为目标芯片为 ESP32 的 QEMU 测试用例。 + +当测试逻辑相同，但执行环境不同的时候，可使用此模式。 + 测试串行输出 ^^^^^^^^^^^^^^^^ @@ -492,7 +581,7 @@ CI 的工作流程如下所示： .. code-block:: shell $ cd $IDF_PATH - $ bash install.sh --enable-ci --enable-pytest + $ bash install.sh --enable-ci $ . ./export.sh 编译目录 diff --git a/examples/bluetooth/nimble/blecent/sdkconfig.ci.esp32c2eco4_xtal26m b/examples/bluetooth/nimble/blecent/sdkconfig.ci.esp32c2_rev2_xtal26m similarity index 100% rename from examples/bluetooth/nimble/blecent/sdkconfig.ci.esp32c2eco4_xtal26m rename to examples/bluetooth/nimble/blecent/sdkconfig.ci.esp32c2_rev2_xtal26m diff --git a/examples/bluetooth/nimble/blecent/sdkconfig.ci.esp32c3eco7 b/examples/bluetooth/nimble/blecent/sdkconfig.ci.esp32c3_rev1 similarity index 100% rename from examples/bluetooth/nimble/blecent/sdkconfig.ci.esp32c3eco7 rename to examples/bluetooth/nimble/blecent/sdkconfig.ci.esp32c3_rev1 diff --git a/examples/bluetooth/nimble/blemesh/README.md b/examples/bluetooth/nimble/blemesh/README.md index 604906480b6..64d4c59201a 100644 --- a/examples/bluetooth/nimble/blemesh/README.md +++ b/examples/bluetooth/nimble/blemesh/README.md @@ -11,7 +11,7 @@ https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-guides/esp-ble-m (See the README.md file in the upper level 'examples' directory for more information about examples.) This example implements Bluetooth Mesh node that supports On/Off and Level models. -It has suport for both Advertising Bearer and GATT Bearer. +It has support for both Advertising Bearer and GATT Bearer. For more information on NimBLE MESH, please visit [NimBLE_MESH](https://mynewt.apache.org/latest/network/mesh/index.html#bluetooth-mesh). diff --git a/examples/bluetooth/nimble/power_save/sdkconfig.ci.esp32c2eco4_xtal26m b/examples/bluetooth/nimble/power_save/sdkconfig.ci.esp32c2_rev2_xtal26m similarity index 100% rename from examples/bluetooth/nimble/power_save/sdkconfig.ci.esp32c2eco4_xtal26m rename to examples/bluetooth/nimble/power_save/sdkconfig.ci.esp32c2_rev2_xtal26m diff --git a/examples/bluetooth/nimble/power_save/sdkconfig.ci.esp32c3eco7 b/examples/bluetooth/nimble/power_save/sdkconfig.ci.esp32c3_rev1 similarity index 100% rename from examples/bluetooth/nimble/power_save/sdkconfig.ci.esp32c3eco7 rename to examples/bluetooth/nimble/power_save/sdkconfig.ci.esp32c3_rev1 diff --git a/examples/bluetooth/nimble/pytest_nimble_test.py b/examples/bluetooth/nimble/pytest_nimble_test.py index fd2b5608902..adf8a9e336e 100644 --- a/examples/bluetooth/nimble/pytest_nimble_test.py +++ b/examples/bluetooth/nimble/pytest_nimble_test.py @@ -1,22 +1,24 @@ -# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 -import os.path +from pathlib import Path from typing import Tuple import pexpect import pytest from pytest_embedded_idf.dut import IdfDut from pytest_embedded_idf.utils import idf_parametrize + +CUR_DIR = Path(__file__).parent.resolve() + + # Case 1: BLE power save test - - @pytest.mark.two_duts @pytest.mark.parametrize( 'count, app_path', [ ( 2, - f'{os.path.join(os.path.dirname(__file__), "power_save")}|{os.path.join(os.path.dirname(__file__), "blecent")}', + f'{str(CUR_DIR / "power_save")}|{str(CUR_DIR / "blecent")}', ), ], indirect=True, @@ -47,7 +49,7 @@ def test_power_save_conn(app_path: str, dut: Tuple[IdfDut, IdfDut]) -> None: ( 'esp32c2_xtal26m', 2, - f'{os.path.join(os.path.dirname(__file__), "power_save")}|{os.path.join(os.path.dirname(__file__), "blecent")}', + f'{str(CUR_DIR / "power_save")}|{str(CUR_DIR / "blecent")}', '74880', ), ], @@ -71,21 +73,21 @@ def test_power_save_conn_esp32c2_26mhz(dut: Tuple[IdfDut, IdfDut]) -> None: # Case 2: BLE power save test for ESP32C2ECO4 @pytest.mark.two_duts @pytest.mark.xtal_26mhz -@pytest.mark.esp32c2eco4 +@pytest.mark.esp32c2_rev2 @pytest.mark.parametrize( 'config, count, target, app_path, baud', [ ( - 'esp32c2eco4_xtal26m', + 'esp32c2_rev2_xtal26m', 2, 'esp32c2', - f'{os.path.join(os.path.dirname(__file__), "power_save")}|{os.path.join(os.path.dirname(__file__), "blecent")}', + f'{str(CUR_DIR / "power_save")}|{str(CUR_DIR / "blecent")}', '74880', ), ], indirect=True, ) -def test_power_save_conn_esp32c2eco4(dut: Tuple[IdfDut, IdfDut]) -> None: +def test_power_save_conn_esp32c2_rev2(dut: Tuple[IdfDut, IdfDut]) -> None: peripheral = dut[0] central = dut[1] @@ -101,20 +103,20 @@ def test_power_save_conn_esp32c2eco4(dut: Tuple[IdfDut, IdfDut]) -> None: # Case 2: BLE power save test for ESP32C3ECO7 @pytest.mark.two_duts -@pytest.mark.esp32c3eco7 +@pytest.mark.esp32c3_rev1 @pytest.mark.parametrize( 'config, count, target, app_path', [ ( - 'esp32c3eco7', + 'esp32c3_rev1', 2, 'esp32c3', - f'{os.path.join(os.path.dirname(__file__), "power_save")}|{os.path.join(os.path.dirname(__file__), "blecent")}', + f'{str(CUR_DIR / "power_save")}|{str(CUR_DIR / "blecent")}', ), ], indirect=True, ) -def test_power_save_conn_esp32c3eco7(dut: Tuple[IdfDut, IdfDut]) -> None: +def test_power_save_conn_esp32c3_rev1(dut: Tuple[IdfDut, IdfDut]) -> None: peripheral = dut[0] central = dut[1] diff --git a/examples/build_system/cmake/import_lib/pytest_import_lib.py b/examples/build_system/cmake/import_lib/pytest_import_lib.py index ab7f82cb568..6fa8ca4b991 100644 --- a/examples/build_system/cmake/import_lib/pytest_import_lib.py +++ b/examples/build_system/cmake/import_lib/pytest_import_lib.py @@ -5,7 +5,6 @@ from pytest_embedded_idf.utils import idf_parametrize from pytest_embedded_qemu.dut import QemuDut -@pytest.mark.host_test @pytest.mark.qemu @idf_parametrize('target', ['esp32', 'esp32c3'], indirect=['target']) def test_pytest_host(dut: QemuDut) -> None: diff --git a/examples/get-started/hello_world/pytest_hello_world.py b/examples/get-started/hello_world/pytest_hello_world.py index 196c9605b33..7b0f19ba717 100644 --- a/examples/get-started/hello_world/pytest_hello_world.py +++ b/examples/get-started/hello_world/pytest_hello_world.py @@ -1,8 +1,10 @@ # SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 +from __future__ import annotations + import hashlib import logging -from typing import Callable +from collections.abc import Callable import pytest from pytest_embedded_idf.dut import IdfDut @@ -18,14 +20,12 @@ def test_hello_world(dut: IdfDut, log_minimum_free_heap_size: Callable[..., None log_minimum_free_heap_size() -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_hello_world_linux(dut: IdfDut) -> None: dut.expect('Hello world!') -@pytest.mark.host_test -@pytest.mark.macos_shell +@pytest.mark.macos @idf_parametrize('target', ['linux'], indirect=['target']) def test_hello_world_macos(dut: IdfDut) -> None: dut.expect('Hello world!') @@ -45,7 +45,6 @@ def verify_elf_sha256_embedding(app: QemuApp, sha256_reported: str) -> None: raise ValueError('ELF file SHA256 mismatch') -@pytest.mark.host_test @pytest.mark.qemu @idf_parametrize('target', ['esp32'], indirect=['target']) def test_hello_world_host(app: QemuApp, dut: QemuDut) -> None: diff --git a/examples/network/bridge/pytest_example_bridge.py b/examples/network/bridge/pytest_example_bridge.py index d407464bc50..52a0506d57d 100644 --- a/examples/network/bridge/pytest_example_bridge.py +++ b/examples/network/bridge/pytest_example_bridge.py @@ -1,5 +1,7 @@ # SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 +from __future__ import annotations + import base64 import io import ipaddress @@ -9,14 +11,10 @@ import re import socket import subprocess import time +from collections.abc import Generator from concurrent.futures import Future from concurrent.futures import ThreadPoolExecutor -from typing import Generator -from typing import List -from typing import Match -from typing import Optional -from typing import Tuple -from typing import Union +from re import Match import netifaces import paramiko # type: ignore @@ -25,9 +23,12 @@ from common_test_methods import get_host_ip_by_interface from netmiko import ConnectHandler from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize + # Testbed configuration ETHVM_ENDNODE_USER = 'ci.ethvm' +ETHERNET_TEST_USER = os.getenv('ETHERNET_TEST_USER') +ETHERNET_TEST_PASSWORD = os.getenv('ETHERNET_TEST_PASSWORD') BR_PORTS_NUM = 2 IPERF_BW_LIM = 6 @@ -36,7 +37,7 @@ MIN_TCP_THROUGHPUT = 4 class EndnodeSsh: - def __init__(self, host_ip: str, usr: str, passwd: Optional[str] = None): + def __init__(self, host_ip: str, usr: str, passwd: str | None = None): key_string = os.getenv('CI_ETHVM_KEY') key = None if key_string: @@ -93,7 +94,7 @@ class SwitchSsh: } self.ssh_client = ConnectHandler(**edgeSwitch) - def exec_cmd(self, cmd: Union[str, List[str]]) -> str: + def exec_cmd(self, cmd: str | list[str]) -> str: if self.type == self.EDGE_SWITCH_5XP: _, stdout, stderr = self.ssh_client.exec_command(cmd) @@ -192,7 +193,17 @@ def run_iperf( if ipaddress.ip_address(server_ip).is_multicast: # Configure Multicast Server server_proc = subprocess.Popen( - ['iperf', '-u', '-s', '-i', '1', '-t', str(interval), '-B', f'{server_ip}%{server_if}'], + [ + 'iperf', + '-u', + '-s', + '-i', + '1', + '-t', + str(interval), + '-B', + f'{server_ip}%{server_if}', + ], text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -284,14 +295,14 @@ def send_brcast_msg_endnode_to_host(endnode: EndnodeSsh, host_brcast_ip: str, po return nc_host_out -def get_legacy_host_name_match() -> Optional[Match[str]]: +def get_legacy_host_name_match() -> Match[str] | None: host_name = socket.gethostname() regex = r'ethVM-(\d+)-(\d+)' host_name_match = re.search(regex, host_name, re.DOTALL) return host_name_match -def get_host_info() -> Tuple[int, int]: +def get_host_info() -> tuple[int, int]: # Get switch configuration info from the hostname (legacy runners) sw_info = get_legacy_host_name_match() if sw_info is not None: @@ -635,5 +646,10 @@ def setup_test_environment() -> Generator[None, None, None]: indirect=True, ) @idf_parametrize('target', ['esp32'], indirect=['target']) +@pytest.mark.parametrize( + 'dev_user, dev_password', + [(ETHERNET_TEST_USER, ETHERNET_TEST_PASSWORD)], + indirect=True, +) def test_esp_eth_bridge(dut: Dut, dev_user: str, dev_password: str) -> None: eth_bridge_test(dut, dev_user, dev_password) diff --git a/examples/openthread/pytest_otbr.py b/examples/openthread/pytest_otbr.py index 225b4adce8e..185891ffb93 100644 --- a/examples/openthread/pytest_otbr.py +++ b/examples/openthread/pytest_otbr.py @@ -8,16 +8,19 @@ import random import re import secrets import subprocess +import sys import threading import time from typing import Generator from typing import Optional from typing import Tuple -import ot_ci_function as ocf +sys.path.append(os.path.dirname(os.path.abspath(__file__))) +import ot_ci_function as ocf # noqa: E402 import pexpect import pytest from pytest_embedded_idf.dut import IdfDut + # This file contains the test scripts for Thread: # Case 1: Thread network formation and attaching # A Thread Border Router forms a Thread network, Thread devices attach to it, then test ping diff --git a/examples/peripherals/usb/host/msc/pytest_usb_host_msc.py b/examples/peripherals/usb/host/msc/pytest_usb_host_msc.py index 5209a072243..1cc86b138bd 100644 --- a/examples/peripherals/usb/host/msc/pytest_usb_host_msc.py +++ b/examples/peripherals/usb/host/msc/pytest_usb_host_msc.py @@ -4,11 +4,6 @@ import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -CONFIGS = [ - pytest.param('default', marks=[pytest.mark.esp32s2, pytest.mark.esp32s3, pytest.mark.esp32p4]), - pytest.param('esp32p4_psram', marks=[pytest.mark.esp32p4]), -] - @pytest.mark.usb_host_flash_disk @pytest.mark.temp_skip_ci(targets=['esp32s2'], reason='lack of runners with usb_host_flash_disk tag') diff --git a/examples/protocols/esp_http_client/pytest_esp_http_client.py b/examples/protocols/esp_http_client/pytest_esp_http_client.py index 1e1d485bf9e..f5d69b26180 100644 --- a/examples/protocols/esp_http_client/pytest_esp_http_client.py +++ b/examples/protocols/esp_http_client/pytest_esp_http_client.py @@ -18,7 +18,7 @@ def test_examples_protocol_esp_http_client(dut: Dut) -> None: """ binary_file = os.path.join(dut.app.binary_path, 'esp_http_client_example.bin') bin_size = os.path.getsize(binary_file) - logging.info('esp_http_client_bin_size : {}KB'.format(bin_size // 1024)) + logging.info(f'esp_http_client_bin_size : {bin_size // 1024}KB') # start test dut.expect('Connected to AP, begin http example', timeout=30) dut.expect(r'HTTP GET Status = 200, content_length = (\d)') @@ -70,7 +70,7 @@ def test_examples_protocol_esp_http_client_dynamic_buffer(dut: Dut) -> None: # check and log bin size binary_file = os.path.join(dut.app.binary_path, 'esp_http_client_example.bin') bin_size = os.path.getsize(binary_file) - logging.info('esp_http_client_bin_size : {}KB'.format(bin_size // 1024)) + logging.info(f'esp_http_client_bin_size : {bin_size // 1024}KB') dut.expect('Connected to AP, begin http example', timeout=30) dut.expect(r'HTTP GET Status = 200, content_length = (\d)') @@ -106,7 +106,6 @@ def test_examples_protocol_esp_http_client_dynamic_buffer(dut: Dut) -> None: dut.expect('Finish http example') -@pytest.mark.host_test # Currently we are just testing the build for esp_http_client on Linux target. So skipping the test run. # Later we will enable the test run for Linux target as well. @pytest.mark.skipif('config.getvalue("target") == "linux"', reason='Do not run on Linux') diff --git a/examples/protocols/https_x509_bundle/pytest_https_x509_bundle.py b/examples/protocols/https_x509_bundle/pytest_https_x509_bundle.py index e172d4f24c0..078c0f6c2f8 100644 --- a/examples/protocols/https_x509_bundle/pytest_https_x509_bundle.py +++ b/examples/protocols/https_x509_bundle/pytest_https_x509_bundle.py @@ -20,13 +20,13 @@ def test_examples_protocol_https_x509_bundle(dut: Dut) -> None: # check and log bin size binary_file = os.path.join(dut.app.binary_path, 'https_x509_bundle.bin') bin_size = os.path.getsize(binary_file) - logging.info('https_x509_bundle_bin_size : {}KB'.format(bin_size // 1024)) + logging.info(f'https_x509_bundle_bin_size : {bin_size // 1024}KB') dut.expect(r'IPv4 address: (\d+\.\d+\.\d+\.\d+)[^\d]', timeout=30) # start test num_URLS = int(dut.expect(r'Connecting to (\d+) URLs', timeout=30)[1].decode()) for _ in range(num_URLS): dut.expect(r'Connection established to ([\s\S]*)', timeout=30) - dut.expect('Completed {} connections'.format(num_URLS), timeout=60) + dut.expect(f'Completed {num_URLS} connections', timeout=60) @pytest.mark.ethernet @@ -43,16 +43,15 @@ def test_examples_protocol_https_x509_bundle_dynamic_buffer(dut: Dut) -> None: # check and log bin size binary_file = os.path.join(dut.app.binary_path, 'https_x509_bundle.bin') bin_size = os.path.getsize(binary_file) - logging.info('https_x509_bundle_bin_size : {}KB'.format(bin_size // 1024)) + logging.info(f'https_x509_bundle_bin_size : {bin_size // 1024}KB') dut.expect(r'IPv4 address: (\d+\.\d+\.\d+\.\d+)[^\d]', timeout=30) # start test num_URLS = int(dut.expect(r'Connecting to (\d+) URLs', timeout=30)[1].decode()) dut.expect(r'Connection established to ([\s\S]*)', timeout=30) - dut.expect('Completed {} connections'.format(num_URLS), timeout=60) + dut.expect(f'Completed {num_URLS} connections', timeout=60) @pytest.mark.qemu -@pytest.mark.host_test @pytest.mark.parametrize( 'config', [ @@ -61,14 +60,16 @@ def test_examples_protocol_https_x509_bundle_dynamic_buffer(dut: Dut) -> None: indirect=True, ) @idf_parametrize('target', ['esp32'], indirect=['target']) -def test_examples_protocol_https_x509_bundle_default_crt_bundle_stress_test(dut: Dut) -> None: +def test_examples_protocol_https_x509_bundle_default_crt_bundle_stress_test( + dut: Dut, +) -> None: # check and log bin size binary_file = os.path.join(dut.app.binary_path, 'https_x509_bundle.bin') bin_size = os.path.getsize(binary_file) - logging.info('https_x509_bundle_bin_size : {}KB'.format(bin_size // 1024)) + logging.info(f'https_x509_bundle_bin_size : {bin_size // 1024}KB') dut.expect(r'IPv4 address: (\d+\.\d+\.\d+\.\d+)[^\d]', timeout=30) # start test num_URLS = int(dut.expect(r'Connecting to (\d+) URLs', timeout=30)[1].decode()) for _ in range(num_URLS): dut.expect(r'Connection established to ([\s\S]*)', timeout=60) - dut.expect('Completed {} connections'.format(num_URLS), timeout=180) + dut.expect(f'Completed {num_URLS} connections', timeout=180) diff --git a/examples/protocols/sockets/README.md b/examples/protocols/sockets/README.md index 771d642a7ca..1fa56aff053 100644 --- a/examples/protocols/sockets/README.md +++ b/examples/protocols/sockets/README.md @@ -2,8 +2,8 @@ # BSD Socket API Examples This directory contains simple examples demonstrating BSD Socket API. -Each example, contains README.md file with mode detailed informations about that particular example. -For more general informations about all examples, see the README.md file in the upper level 'examples' directory. +Each example, contains README.md file with mode detailed information about that particular example. +For more general information about all examples, see the README.md file in the upper level 'examples' directory. Examples: * UDP Client - The application creates UDP socket and sends message to the predefined port and IP address. After the server's reply, the application prints received reply as ASCII text, waits for 2 seconds and sends another message. @@ -56,7 +56,7 @@ They can also be run locally. Ref: [ESP-IDF Tests with Pytest Guide](https://doc Example: ```bash $ cd $IDF_PATH -$ bash install.sh --enable-pytest +$ bash install.sh --enable-ci $ . ./export.sh $ cd examples/protocols/sockets/tcp_client $ python $IDF_PATH/tools/ci/ci_build_apps.py . --target esp32 -vv --pytest-apps @@ -112,7 +112,7 @@ Please make sure that when using the Local Link address, an interface id is incl * On the host - Interface name suffix is present when passing the address as a string, for example `fe80::260a:XXX:XXX:XXX%en0` - - The interface id is present when passing the endpoint as tupple, for example `socket.connect(('fd00::260a:XXXX:XXXX:XXXX', 3333, 0, 3))` + - The interface id is present when passing the endpoint as tuple, for example `socket.connect(('fd00::260a:XXXX:XXXX:XXXX', 3333, 0, 3))` ## Hardware Required diff --git a/examples/system/deep_sleep/pytest_deep_sleep.py b/examples/system/deep_sleep/pytest_deep_sleep.py index 320abeaf6e0..5b8741d32aa 100644 --- a/examples/system/deep_sleep/pytest_deep_sleep.py +++ b/examples/system/deep_sleep/pytest_deep_sleep.py @@ -10,26 +10,6 @@ from pytest_embedded_idf.utils import idf_parametrize touch_wake_up_support = ['esp32', 'esp32s2'] -CONFIGS = [ - pytest.param('esp32_singlecore', marks=[pytest.mark.esp32]), - pytest.param( - 'basic', - marks=[ - pytest.mark.esp32, - pytest.mark.esp32s2, - pytest.mark.esp32s3, - pytest.mark.esp32c3, - pytest.mark.esp32c5, - pytest.mark.esp32c6, - pytest.mark.esp32c61, - pytest.mark.esp32h2, - pytest.mark.esp32p4, - pytest.mark.esp32c2, - ], - ), -] - - @pytest.mark.generic @idf_parametrize( 'config,target', [('esp32_singlecore', 'esp32'), ('basic', 'supported_targets')], indirect=['config', 'target'] @@ -42,11 +22,11 @@ def test_deep_sleep(dut: Dut) -> None: 'esp32s2': [9], }[dut.target] - logging.info('Expecting to see wakeup configured on pad(s): {}'.format(wake_pads)) + logging.info(f'Expecting to see wakeup configured on pad(s): {wake_pads}') expect_items = ['Enabling timer wakeup, 20s'] for pad in wake_pads: - expect_items += [r'Touch pad #{} average: \d+, wakeup threshold set to \d+.'.format(pad)] + expect_items += [rf'Touch pad #{pad} average: \d+, wakeup threshold set to \d+.'] expect_items += ['Enabling touch pad wakeup'] for exp in expect_items: @@ -69,7 +49,7 @@ def test_deep_sleep(dut: Dut) -> None: dut.expect_exact('boot: ESP-IDF') # first output that's the same on all chips sleep_time = time.time() - start_sleep - logging.info('Host measured sleep time at {:.2f}s'.format(sleep_time)) + logging.info(f'Host measured sleep time at {sleep_time:.2f}s') assert 18 < sleep_time < 22 # note: high tolerance as measuring time on the host may have some timing skew dut.expect_exact('boot: Fast booting app from partition', timeout=2) diff --git a/examples/system/efuse/pytest_system_efuse_example.py b/examples/system/efuse/pytest_system_efuse_example.py index 962b181463d..c37ed829e2f 100644 --- a/examples/system/efuse/pytest_system_efuse_example.py +++ b/examples/system/efuse/pytest_system_efuse_example.py @@ -10,7 +10,10 @@ from pytest_embedded_qemu.dut import QemuDut def basic_efuse_example(dut: Dut) -> None: - dut.expect(r'example: Coding Scheme (3/4)|(NONE)|(REPEAT)|(RS \(Reed-Solomon coding\))', timeout=20) + dut.expect( + r'example: Coding Scheme (3/4)|(NONE)|(REPEAT)|(RS \(Reed-Solomon coding\))', + timeout=20, + ) dut.expect( [ 'example: read efuse fields', @@ -43,7 +46,6 @@ def test_examples_efuse(dut: Dut) -> None: basic_efuse_example(dut) -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_examples_efuse_linux(dut: Dut) -> None: basic_efuse_example(dut) @@ -60,14 +62,24 @@ def test_examples_efuse_linux(dut: Dut) -> None: @pytest.mark.parametrize('skip_autoflash', ['y'], indirect=True) @idf_parametrize( 'target', - ['esp32', 'esp32c2', 'esp32c3', 'esp32c5', 'esp32c61', 'esp32c6', 'esp32h2', 'esp32s2', 'esp32s3'], + [ + 'esp32', + 'esp32c2', + 'esp32c3', + 'esp32c5', + 'esp32c61', + 'esp32c6', + 'esp32h2', + 'esp32s2', + 'esp32s3', + ], indirect=['target'], ) def test_examples_efuse_with_virt_flash_enc(dut: Dut) -> None: # check and log bin size binary_file = os.path.join(dut.app.binary_path, 'bootloader', 'bootloader.bin') bin_size = os.path.getsize(binary_file) - logging.info('{}_bootloader_virt_flash_enc_bin_size: {}KB'.format(dut.app.target, bin_size // 1024)) + logging.info(f'{dut.app.target}_bootloader_virt_flash_enc_bin_size: {bin_size // 1024}KB') print(' - Erase flash') dut.serial.erase_flash() @@ -126,7 +138,7 @@ def test_examples_efuse_with_virt_flash_enc_aes_256(dut: Dut) -> None: # check and log bin size binary_file = os.path.join(dut.app.binary_path, 'bootloader', 'bootloader.bin') bin_size = os.path.getsize(binary_file) - logging.info('{}_bootloader_virt_flash_enc_aes_256_bin_size: {}KB'.format(dut.app.target, bin_size // 1024)) + logging.info(f'{dut.app.target}_bootloader_virt_flash_enc_aes_256_bin_size: {bin_size // 1024}KB') print(' - Erase flash') dut.serial.erase_flash() @@ -169,7 +181,17 @@ def test_examples_efuse_with_virt_flash_enc_aes_256(dut: Dut) -> None: @pytest.mark.parametrize('skip_autoflash', ['y'], indirect=True) @idf_parametrize( 'target', - ['esp32', 'esp32c2', 'esp32c3', 'esp32c5', 'esp32c61', 'esp32c6', 'esp32h2', 'esp32s2', 'esp32s3'], + [ + 'esp32', + 'esp32c2', + 'esp32c3', + 'esp32c5', + 'esp32c61', + 'esp32c6', + 'esp32h2', + 'esp32s2', + 'esp32s3', + ], indirect=['target'], ) def test_examples_efuse_with_virt_flash_enc_pre_loaded(dut: Dut) -> None: @@ -239,14 +261,24 @@ def test_examples_efuse_with_virt_flash_enc_pre_loaded(dut: Dut) -> None: @pytest.mark.parametrize('skip_autoflash', ['y'], indirect=True) @idf_parametrize( 'target', - ['esp32', 'esp32c2', 'esp32c3', 'esp32c5', 'esp32c61', 'esp32c6', 'esp32h2', 'esp32s2', 'esp32s3'], + [ + 'esp32', + 'esp32c2', + 'esp32c3', + 'esp32c5', + 'esp32c61', + 'esp32c6', + 'esp32h2', + 'esp32s2', + 'esp32s3', + ], indirect=['target'], ) def test_examples_efuse_with_virt_flash_enc_release(dut: Dut) -> None: # check and log bin size binary_file = os.path.join(dut.app.binary_path, 'bootloader', 'bootloader.bin') bin_size = os.path.getsize(binary_file) - logging.info('{}_bootloader_virt_flash_enc_release_bin_size: {}KB'.format(dut.app.target, bin_size // 1024)) + logging.info(f'{dut.app.target}_bootloader_virt_flash_enc_release_bin_size: {bin_size // 1024}KB') dut.serial.erase_flash() @@ -302,7 +334,7 @@ def test_examples_efuse_with_virt_secure_boot_v1(dut: Dut) -> None: # check and log bin size binary_file = os.path.join(dut.app.binary_path, 'bootloader', 'bootloader.bin') bin_size = os.path.getsize(binary_file) - logging.info('{}_bootloader_virt_secure_boot_v1_bin_size: {}KB'.format(dut.app.target, bin_size // 1024)) + logging.info(f'{dut.app.target}_bootloader_virt_secure_boot_v1_bin_size: {bin_size // 1024}KB') print(' - Erase flash') dut.serial.erase_flash() @@ -400,7 +432,7 @@ def test_examples_efuse_with_virt_secure_boot_v1_pre_loaded(dut: Dut) -> None: dut.expect('example: Done') -@pytest.mark.esp32eco3 +@pytest.mark.esp32_rev3 @pytest.mark.parametrize( 'config', [ @@ -414,7 +446,7 @@ def test_examples_efuse_with_virt_secure_boot_v2(dut: Dut) -> None: # only for ESP32 ECO3 binary_file = os.path.join(dut.app.binary_path, 'bootloader', 'bootloader.bin') bin_size = os.path.getsize(binary_file) - logging.info('{}_bootloader_secure_boot_v2_bin_size: {}KB'.format(dut.app.target, bin_size // 1024)) + logging.info(f'{dut.app.target}_bootloader_secure_boot_v2_bin_size: {bin_size // 1024}KB') print(' - Erase flash') dut.serial.erase_flash() @@ -520,7 +552,7 @@ def test_examples_efuse_with_virt_secure_boot_v2(dut: Dut) -> None: dut.expect('example: Done') -@pytest.mark.esp32eco3 +@pytest.mark.esp32_rev3 @pytest.mark.parametrize( 'config', [ @@ -593,14 +625,24 @@ def test_examples_efuse_with_virt_secure_boot_v2_pre_loaded(dut: Dut) -> None: @pytest.mark.parametrize('skip_autoflash', ['y'], indirect=True) @idf_parametrize( 'target', - ['esp32c3', 'esp32c2', 'esp32c5', 'esp32c6', 'esp32c61', 'esp32h2', 'esp32p4', 'esp32s2', 'esp32s3'], + [ + 'esp32c3', + 'esp32c2', + 'esp32c5', + 'esp32c6', + 'esp32c61', + 'esp32h2', + 'esp32p4', + 'esp32s2', + 'esp32s3', + ], indirect=['target'], ) def test_examples_efuse_with_virt_secure_boot_v2_esp32xx(dut: Dut) -> None: # check and log bin size binary_file = os.path.join(dut.app.binary_path, 'bootloader', 'bootloader.bin') bin_size = os.path.getsize(binary_file) - logging.info('{}_bootloader_virt_secure_boot_v2_bin_size: {}KB'.format(dut.app.target, bin_size // 1024)) + logging.info(f'{dut.app.target}_bootloader_virt_secure_boot_v2_bin_size: {bin_size // 1024}KB') print(' - Erase flash') dut.serial.erase_flash() @@ -616,13 +658,13 @@ def test_examples_efuse_with_virt_secure_boot_v2_esp32xx(dut: Dut) -> None: signed_scheme = 'ECDSA' else: signed_scheme = 'RSA-PSS' - dut.expect('secure_boot_v2: Verifying with %s...' % signed_scheme) + dut.expect(f'secure_boot_v2: Verifying with {signed_scheme}...') dut.expect('secure_boot_v2: Signature verified successfully!') dut.expect('secure_boot_v2: enabling secure boot v2...') dut.expect('Verifying image signature...') dut.expect('secure_boot_v2: Secure boot V2 is not enabled yet and eFuse digest keys are not set') - dut.expect('secure_boot_v2: Verifying with %s...' % signed_scheme) + dut.expect(f'secure_boot_v2: Verifying with {signed_scheme}...') dut.expect('secure_boot_v2: Signature verified successfully!') dut.expect('secure_boot_v2: Secure boot digests absent, generating..') dut.expect('secure_boot_v2: Digests successfully calculated, 1 valid signatures') @@ -651,7 +693,7 @@ def test_examples_efuse_with_virt_secure_boot_v2_esp32xx(dut: Dut) -> None: dut.serial.hard_reset() dut.expect('Loading virtual efuse blocks from flash') dut.expect('Verifying image signature...') - dut.expect('secure_boot_v2: Verifying with %s...' % signed_scheme) + dut.expect(f'secure_boot_v2: Verifying with {signed_scheme}...') dut.expect('secure_boot_v2: Signature verified successfully!') dut.expect('secure_boot_v2: enabling secure boot v2...') dut.expect('secure_boot_v2: secure boot v2 is already enabled, continuing..') @@ -664,7 +706,17 @@ def test_examples_efuse_with_virt_secure_boot_v2_esp32xx(dut: Dut) -> None: @pytest.mark.parametrize('skip_autoflash', ['y'], indirect=True) @idf_parametrize( 'target', - ['esp32c3', 'esp32c2', 'esp32c5', 'esp32c6', 'esp32c61', 'esp32h2', 'esp32p4', 'esp32s2', 'esp32s3'], + [ + 'esp32c3', + 'esp32c2', + 'esp32c5', + 'esp32c6', + 'esp32c61', + 'esp32h2', + 'esp32p4', + 'esp32s2', + 'esp32s3', + ], indirect=['target'], ) def test_example_efuse_with_virt_secure_boot_v2_esp32xx_pre_loaded(dut: Dut) -> None: @@ -685,12 +737,14 @@ def test_example_efuse_with_virt_secure_boot_v2_esp32xx_pre_loaded(dut: Dut) -> # Resets eFuse, which enables Secure boot feature # Resets eFuses, which control digest slots if dut.app.sdkconfig.get('SOC_EFUSE_REVOKE_BOOT_KEY_DIGESTS'): - dut.serial.erase_field_on_emul_efuse_by_name([ - 'SECURE_BOOT_EN', - 'SECURE_BOOT_KEY_REVOKE0', - 'SECURE_BOOT_KEY_REVOKE1', - 'SECURE_BOOT_KEY_REVOKE2', - ]) + dut.serial.erase_field_on_emul_efuse_by_name( + [ + 'SECURE_BOOT_EN', + 'SECURE_BOOT_KEY_REVOKE0', + 'SECURE_BOOT_KEY_REVOKE1', + 'SECURE_BOOT_KEY_REVOKE2', + ] + ) else: dut.serial.erase_field_on_emul_efuse_by_name(['SECURE_BOOT_EN']) @@ -704,7 +758,7 @@ def test_example_efuse_with_virt_secure_boot_v2_esp32xx_pre_loaded(dut: Dut) -> else: signed_scheme = 'RSA-PSS' - dut.expect('secure_boot_v2: Verifying with %s...' % signed_scheme) + dut.expect(f'secure_boot_v2: Verifying with {signed_scheme}...') dut.expect('secure_boot_v2: Signature verified successfully!') dut.expect('secure_boot_v2: Secure boot digests already present') dut.expect('secure_boot_v2: Using pre-loaded public key digest in eFuse') @@ -727,7 +781,7 @@ def test_example_efuse_with_virt_secure_boot_v2_esp32xx_pre_loaded(dut: Dut) -> dut.serial.hard_reset() dut.expect('Loading virtual efuse blocks from flash') dut.expect('Verifying image signature...') - dut.expect('secure_boot_v2: Verifying with %s...' % signed_scheme) + dut.expect(f'secure_boot_v2: Verifying with {signed_scheme}...') dut.expect('secure_boot_v2: Signature verified successfully!') dut.expect('secure_boot_v2: enabling secure boot v2...') dut.expect('secure_boot_v2: secure boot v2 is already enabled, continuing..') @@ -749,7 +803,7 @@ def test_examples_efuse_with_virt_sb_v1_and_fe(dut: Dut) -> None: # check and log bin size binary_file = os.path.join(dut.app.binary_path, 'bootloader', 'bootloader.bin') bin_size = os.path.getsize(binary_file) - logging.info('{}_bootloader_virt_sb_v1_and_fe_bin_size: {}KB'.format(dut.app.target, bin_size // 1024)) + logging.info(f'{dut.app.target}_bootloader_virt_sb_v1_and_fe_bin_size: {bin_size // 1024}KB') print(' - Erase flash') dut.serial.erase_flash() @@ -806,7 +860,7 @@ def test_examples_efuse_with_virt_sb_v1_and_fe(dut: Dut) -> None: dut.expect('example: Done') -@pytest.mark.esp32eco3 +@pytest.mark.esp32_rev3 @pytest.mark.parametrize( 'config', [ @@ -820,7 +874,7 @@ def test_examples_efuse_with_virt_sb_v2_and_fe(dut: Dut) -> None: # check and log bin size binary_file = os.path.join(dut.app.binary_path, 'bootloader', 'bootloader.bin') bin_size = os.path.getsize(binary_file) - logging.info('{}_bootloader_virt_sb_v2_and_fe_bin_size: {}KB'.format(dut.app.target, bin_size // 1024)) + logging.info(f'{dut.app.target}_bootloader_virt_sb_v2_and_fe_bin_size: {bin_size // 1024}KB') print(' - Erase flash') dut.serial.erase_flash() @@ -894,7 +948,6 @@ def test_examples_efuse_with_virt_sb_v2_and_fe(dut: Dut) -> None: dut.expect('example: Done') -@pytest.mark.host_test @pytest.mark.qemu @pytest.mark.parametrize( 'config', @@ -906,7 +959,8 @@ def test_examples_efuse_with_virt_sb_v2_and_fe(dut: Dut) -> None: @pytest.mark.parametrize( 'qemu_extra_args', [ - f'-drive file={os.path.join(os.path.dirname(__file__), "test", "esp32eco3_efuses.bin")},if=none,format=raw,id=efuse ' + f'-drive file={os.path.join(os.path.dirname(__file__), "test", "esp32_rev3_efuses.bin")},' + 'if=none,format=raw,id=efuse ' '-global driver=nvram.esp32.efuse,property=drive,value=efuse ' '-global driver=timer.esp32.timg,property=wdt_disable,value=true', ], @@ -976,11 +1030,14 @@ def test_examples_efuse_with_virt_sb_v2_and_fe_qemu(dut: QemuDut) -> None: finally: # the above example test burns the efuses, and hence the efuses file which the - # qemu uses to emulate the efuses, "test/esp32eco3_efuses.bin", gets modified. + # qemu uses to emulate the efuses, "test/esp32_rev3_efuses.bin", gets modified. # Thus, restore the efuses file values back to the default ESP32-ECO3 efuses values. - with open(os.path.join(os.path.dirname(__file__), 'test', 'esp32eco3_efuses.bin'), 'wb') as efuse_file: - esp32eco3_efuses = '0' * 26 + '8' + '0' * 17 + '1' + '0' * 203 - efuse_file.write(bytearray.fromhex(esp32eco3_efuses)) + with open( + os.path.join(os.path.dirname(__file__), 'test', 'esp32_rev3_efuses.bin'), + 'wb', + ) as efuse_file: + esp32_rev3_efuses = '0' * 26 + '8' + '0' * 17 + '1' + '0' * 203 + efuse_file.write(bytearray.fromhex(esp32_rev3_efuses)) @pytest.mark.generic @@ -988,14 +1045,23 @@ def test_examples_efuse_with_virt_sb_v2_and_fe_qemu(dut: QemuDut) -> None: @pytest.mark.parametrize('config', ['virt_sb_v2_and_fe'], indirect=True) @idf_parametrize( 'target', - ['esp32c3', 'esp32c2', 'esp32c5', 'esp32c61', 'esp32c6', 'esp32h2', 'esp32s2', 'esp32s3'], + [ + 'esp32c3', + 'esp32c2', + 'esp32c5', + 'esp32c61', + 'esp32c6', + 'esp32h2', + 'esp32s2', + 'esp32s3', + ], indirect=['target'], ) def test_examples_efuse_with_virt_sb_v2_and_fe_esp32xx(dut: Dut) -> None: # check and log bin size binary_file = os.path.join(dut.app.binary_path, 'bootloader', 'bootloader.bin') bin_size = os.path.getsize(binary_file) - logging.info('{}_bootloader_virt_sb_v2_and_fe_bin_size: {}KB'.format(dut.app.target, bin_size // 1024)) + logging.info(f'{dut.app.target}_bootloader_virt_sb_v2_and_fe_bin_size: {bin_size // 1024}KB') dut.serial.erase_flash() @@ -1012,13 +1078,13 @@ def test_examples_efuse_with_virt_sb_v2_and_fe_esp32xx(dut: Dut) -> None: signed_scheme = 'ECDSA' else: signed_scheme = 'RSA-PSS' - dut.expect('secure_boot_v2: Verifying with %s...' % signed_scheme) + dut.expect(f'secure_boot_v2: Verifying with {signed_scheme}...') dut.expect('secure_boot_v2: Signature verified successfully!') dut.expect('secure_boot_v2: enabling secure boot v2...') dut.expect('Verifying image signature...') dut.expect('secure_boot_v2: Secure boot V2 is not enabled yet and eFuse digest keys are not set') - dut.expect('secure_boot_v2: Verifying with %s...' % signed_scheme) + dut.expect(f'secure_boot_v2: Verifying with {signed_scheme}...') dut.expect('secure_boot_v2: Signature verified successfully!') dut.expect('secure_boot_v2: Secure boot digests absent, generating..') dut.expect('secure_boot_v2: Digests successfully calculated, 1 valid signatures') @@ -1057,20 +1123,20 @@ def test_examples_efuse_with_virt_sb_v2_and_fe_esp32xx(dut: Dut) -> None: dut.expect('boot: Secure boot permanently enabled') dut.expect('Verifying image signature...') - dut.expect('secure_boot_v2: Verifying with %s...' % signed_scheme) + dut.expect(f'secure_boot_v2: Verifying with {signed_scheme}...') dut.expect('secure_boot_v2: Signature verified successfully!') dut.expect('flash_encrypt: bootloader encrypted successfully') dut.expect('flash_encrypt: partition table encrypted and loaded successfully') dut.expect('Verifying image signature...') - dut.expect('secure_boot_v2: Verifying with %s...' % signed_scheme) + dut.expect(f'secure_boot_v2: Verifying with {signed_scheme}...') dut.expect('secure_boot_v2: Signature verified successfully!') dut.expect('flash_encrypt: Flash encryption completed', timeout=90) dut.expect('Resetting with flash encryption enabled...') dut.expect('Loading virtual efuse blocks from flash') dut.expect('Verifying image signature...') - dut.expect('secure_boot_v2: Verifying with %s...' % signed_scheme) + dut.expect(f'secure_boot_v2: Verifying with {signed_scheme}...') dut.expect('secure_boot_v2: Signature verified successfully!') dut.expect('secure_boot_v2: enabling secure boot v2...') dut.expect('secure_boot_v2: secure boot v2 is already enabled, continuing..') diff --git a/examples/system/efuse/test/esp32eco3_efuses.bin b/examples/system/efuse/test/esp32_rev3_efuses.bin similarity index 100% rename from examples/system/efuse/test/esp32eco3_efuses.bin rename to examples/system/efuse/test/esp32_rev3_efuses.bin diff --git a/examples/wifi/getting_started/pytest_wifi_getting_started.py b/examples/wifi/getting_started/pytest_wifi_getting_started.py index 42622d5ba0d..0e98d1538b9 100644 --- a/examples/wifi/getting_started/pytest_wifi_getting_started.py +++ b/examples/wifi/getting_started/pytest_wifi_getting_started.py @@ -6,6 +6,7 @@ from typing import Tuple import pytest from pytest_embedded_idf.dut import IdfDut from pytest_embedded_idf.utils import idf_parametrize + # @pytest.mark.supported_targets # This test should support all targets, even between different target types # For now our CI only support multi dut with esp32 @@ -82,13 +83,13 @@ def test_wifi_getting_started_esp32c2_xtal_26mhz(dut: Tuple[IdfDut, IdfDut]) -> @pytest.mark.two_duts @pytest.mark.xtal_26mhz -@pytest.mark.esp32c2eco4 +@pytest.mark.esp32c2_rev2 @pytest.mark.parametrize( 'count, config, target, baud, app_path', [ ( 2, - 'esp32c2eco4_xtal26m', + 'esp32c2_rev2_xtal26m', 'esp32c2', '74880', f'{os.path.join(os.path.dirname(__file__), "softAP")}|{os.path.join(os.path.dirname(__file__), "station")}', @@ -96,7 +97,7 @@ def test_wifi_getting_started_esp32c2_xtal_26mhz(dut: Tuple[IdfDut, IdfDut]) -> ], indirect=True, ) -def test_wifi_getting_started_esp32c2eco4_xtal_26mhz(dut: Tuple[IdfDut, IdfDut]) -> None: +def test_wifi_getting_started_esp32c2_rev2_xtal_26mhz(dut: Tuple[IdfDut, IdfDut]) -> None: softap = dut[0] station = dut[1] @@ -113,20 +114,20 @@ def test_wifi_getting_started_esp32c2eco4_xtal_26mhz(dut: Tuple[IdfDut, IdfDut]) @pytest.mark.two_duts -@pytest.mark.esp32c3eco7 +@pytest.mark.esp32c3_rev1 @pytest.mark.parametrize( 'count, config, target, app_path', [ ( 2, - 'esp32c3eco7', + 'esp32c3_rev1', 'esp32c3', f'{os.path.join(os.path.dirname(__file__), "softAP")}|{os.path.join(os.path.dirname(__file__), "station")}', ), ], indirect=True, ) -def test_wifi_getting_started_esp32c3eco7(dut: Tuple[IdfDut, IdfDut]) -> None: +def test_wifi_getting_started_esp32c3_rev1(dut: Tuple[IdfDut, IdfDut]) -> None: softap = dut[0] station = dut[1] diff --git a/examples/wifi/getting_started/softAP/sdkconfig.ci.esp32c2eco4_xtal26m b/examples/wifi/getting_started/softAP/sdkconfig.ci.esp32c2_rev2_xtal26m similarity index 100% rename from examples/wifi/getting_started/softAP/sdkconfig.ci.esp32c2eco4_xtal26m rename to examples/wifi/getting_started/softAP/sdkconfig.ci.esp32c2_rev2_xtal26m diff --git a/examples/wifi/getting_started/softAP/sdkconfig.ci.c3eco7 b/examples/wifi/getting_started/softAP/sdkconfig.ci.esp32c3_rev1 similarity index 100% rename from examples/wifi/getting_started/softAP/sdkconfig.ci.c3eco7 rename to examples/wifi/getting_started/softAP/sdkconfig.ci.esp32c3_rev1 diff --git a/examples/wifi/getting_started/station/sdkconfig.ci.esp32c2eco4_xtal26m b/examples/wifi/getting_started/station/sdkconfig.ci.esp32c2_rev2_xtal26m similarity index 100% rename from examples/wifi/getting_started/station/sdkconfig.ci.esp32c2eco4_xtal26m rename to examples/wifi/getting_started/station/sdkconfig.ci.esp32c2_rev2_xtal26m diff --git a/examples/wifi/getting_started/station/sdkconfig.ci.c3eco7 b/examples/wifi/getting_started/station/sdkconfig.ci.esp32c3_rev1 similarity index 100% rename from examples/wifi/getting_started/station/sdkconfig.ci.c3eco7 rename to examples/wifi/getting_started/station/sdkconfig.ci.esp32c3_rev1 diff --git a/examples/wifi/power_save/pytest_wifi_power_save.py b/examples/wifi/power_save/pytest_wifi_power_save.py index 97d52261617..765afa53edc 100644 --- a/examples/wifi/power_save/pytest_wifi_power_save.py +++ b/examples/wifi/power_save/pytest_wifi_power_save.py @@ -93,24 +93,24 @@ def test_wifi_power_save_esp32c2_26mhz(dut: Dut) -> None: @pytest.mark.wifi_ap @pytest.mark.xtal_26mhz -@pytest.mark.esp32c2eco4 +@pytest.mark.esp32c2_rev2 @pytest.mark.parametrize( 'config, baud, target', [ - ('c2eco4_xtal26m', '74880', 'esp32c2'), + ('esp32c2_rev2_xtal26m', '74880', 'esp32c2'), ], indirect=True, ) -def test_wifi_power_save_esp32c2eco4_26mhz(dut: Dut) -> None: +def test_wifi_power_save_esp32c2_rev2_26mhz(dut: Dut) -> None: _run_test(dut) @pytest.mark.wifi_ap -@pytest.mark.esp32c3eco7 +@pytest.mark.esp32c3_rev1 @pytest.mark.parametrize( 'config, target', - [('c3eco7', 'esp32c3')], + [('esp32c3_rev1', 'esp32c3')], indirect=True, ) -def test_wifi_power_save_esp32c3eco7(dut: Dut) -> None: +def test_wifi_power_save_esp32c3_rev1(dut: Dut) -> None: _run_test(dut) diff --git a/examples/wifi/power_save/sdkconfig.ci.c2eco4_xtal26m b/examples/wifi/power_save/sdkconfig.ci.esp32c2_rev2_xtal26m similarity index 100% rename from examples/wifi/power_save/sdkconfig.ci.c2eco4_xtal26m rename to examples/wifi/power_save/sdkconfig.ci.esp32c2_rev2_xtal26m diff --git a/examples/wifi/power_save/sdkconfig.ci.c3eco7 b/examples/wifi/power_save/sdkconfig.ci.esp32c3_rev1 similarity index 100% rename from examples/wifi/power_save/sdkconfig.ci.c3eco7 rename to examples/wifi/power_save/sdkconfig.ci.esp32c3_rev1 diff --git a/pytest.ini b/pytest.ini index 1fe5e78cc60..fa1e8da49fa 100644 --- a/pytest.ini +++ b/pytest.ini @@ -7,7 +7,6 @@ python_files = pytest_*.py addopts = -s -vv --embedded-services esp,idf - -p no:idf-ci --tb short --strict-markers --skip-check-coredump y @@ -15,12 +14,14 @@ addopts = --check-duplicates y --ignore-glob */managed_components/* --ignore pytest-embedded + --unity-test-report-mode merge + --ignore-no-tests-collected-error # ignore DeprecationWarning filterwarnings = ignore::DeprecationWarning:matplotlib.*: ignore::DeprecationWarning:google.protobuf.*: - ignore::_pytest.warning_types.PytestExperimentalApiWarning + ignore::FutureWarning # log related log_cli = True @@ -34,3 +35,95 @@ junit_family = xunit1 ## log all to `system-out` when case fail junit_logging = stdout junit_log_passing_tests = False + +markers = + temp_skip_ci: mark test to be skipped in CI + temp_skip: mark test to be skipped in CI and locally + skip_app_downloader: mark test required apps built locally, not downloaded from CI + require_elf: mark test to be skipped if no elf file is found + +env_markers = + qemu: build and test using qemu, not real target + macos: tests should be run on macos hosts + generic: tests should be run on generic runners + flash_suspend: support flash suspend feature + eth_ip101: connected via IP101 ethernet transceiver + eth_lan8720: connected via LAN8720 ethernet transceiver + eth_rtl8201: connected via RTL8201 ethernet transceiver + eth_ksz8041: connected via KSZ8041 ethernet transceiver + eth_dp83848: connected via DP83848 ethernet transceiver + eth_w5500: SPI Ethernet module with two W5500 + eth_ksz8851snl: SPI Ethernet module with two KSZ8851SNL + eth_dm9051: SPI Ethernet module with two DM9051 + quad_psram: runners with quad psram + octal_psram: runners with octal psram + usb_host_flash_disk: usb host runners with USB flash disk attached + usb_device: usb device runners + ethernet_ota: ethernet OTA runners + flash_encryption: Flash Encryption runners + flash_encryption_f4r8: Flash Encryption runners with 4-line flash and 8-line psram + flash_encryption_f8r8: Flash Encryption runners with 8-line flash and 8-line psram + flash_encryption_ota: Flash Encryption runners with ethernet OTA support with 4mb flash size + flash_multi: Multiple flash chips tests + psram: Chip has 4-line psram + ir_transceiver: runners with a pair of IR transmitter and receiver + flash_encryption_wifi_high_traffic: Flash Encryption runners with wifi high traffic support + ethernet: ethernet runner + ethernet_flash_8m: ethernet runner with 8mb flash + ethernet_router: both the runner and dut connect to the same router through ethernet NIC + ethernet_vlan: ethernet runner GARM-32-SH-1-R16S5N3 + wifi_ap: a wifi AP in the environment + wifi_router: both the runner and dut connect to the same wifi router + wifi_high_traffic: wifi high traffic runners + wifi_wlan: wifi runner with a wireless NIC + wifi_iperf: the AP and ESP dut were placed in a shielded box - for iperf test + Example_ShieldBox: multiple shielded APs connected to shielded ESP DUT via RF cable with programmable attenuator + xtal_26mhz: runner with 26MHz xtal on board + xtal_40mhz: runner with 40MHz xtal on board + external_flash: external flash memory connected via VSPI (FSPI) + sdcard_sdmode: sdcard running in SD mode, to be removed after test migration + sdcard_spimode: sdcard running in SPI mode + emmc: eMMC card + sdcard: sdcard runner + MSPI_F8R8: runner with Octal Flash and Octal PSRAM + MSPI_F4R8: runner with Quad Flash and Octal PSRAM + MSPI_F4R4: runner with Quad Flash and Quad PSRAM + jtag: runner where the chip is accessible through JTAG as well + usb_serial_jtag: runner where the chip is accessible through builtin JTAG as well + xtal32k: Runner with external 32k crystal connected + no32kXtal: Runner with no external 32k crystal connected + psramv0: Runner with PSRAM version 0 + ecdsa_efuse: Runner with test ECDSA private keys programmed in efuse + ccs811: Runner with CCS811 connected + nvs_encr_hmac: Runner with test HMAC key programmed in efuse + i2c_oled: Runner with ssd1306 I2C oled connected + httpbin: runner for tests that need to access the httpbin service + flash_4mb: C2 runners with 4 MB flash + jtag_re_enable: Runner to re-enable jtag which is softly disabled by burning bit SOFT_DIS_JTAG on eFuse + es8311: Development board that carries es8311 codec + multi_dut_modbus_rs485: a pair of runners connected by RS485 bus + ieee802154: ieee802154 related tests should run on ieee802154 runners. + openthread_br: openthread BR basic and multicast cases. + openthread_br_connect: openthread BR attach and connectivity cases. + openthread_br_service: openthread BR service discovery cases. + openthread_br_nat64: openthread BR NAT64 cases without internet. + openthread_bbr: openthread BR cases with internet access. + openthread_sleep: tests should be used for openthread sleepy device. + zigbee_multi_dut: zigbee runner which have multiple duts. + two_duts: the runner has two duts with antenna available. + generic_multi_device: generic multiple devices whose corresponding gpio pins are connected to each other. + twai_network: multiple runners form a TWAI network. + sdio_master_slave: Test sdio multi board, esp32+esp32 + sdio_multidev_32_c6: Test sdio multi board, esp32+esp32c6 + usj_device: Test usb_serial_jtag and usb_serial_jtag is used as serial only (not console) + twai_adapter: runner with multiple twai_std + lp_i2s: lp_i2s runner tested with hp_i2s + ram_app: ram_app runners + + # rev markers + rev_default: Runner with default revision connected + esp32_rev3: Runner with ESP32 rev 3.x connected + esp32c2_rev2: Runner with ESP32-C2 rev 2.x connected + esp32c3_rev1: Runner with ESP32-C3 rev 1.x connected + esp32c5_rev1: Runner with ESP32-C5 rev 1.x connected + esp32p4_rev1: Runner with ESP32-P4 rev 1.x connected diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 00000000000..f9e74bca033 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,48 @@ +line-length = 120 +target-version = "py38" + +[format] + quote-style = "single" + docstring-code-format = true + +[lint] + select = [ + "A", # flake8-builtins + "E", # pycodestyle + "F", # pyflakes + "W", # pycodestyle + "I", # reorder-imports + "UP", # pyupgrade + ] + ignore = [ + "E203", # whitespace before ':' + "E221", # multiple spaces before operator + "E225", # missing whitespace around operator + "E231", # missing whitespace after ',', ';', or ':' + "E241", # multiple spaces after ',' + ] + + # Exclude a variety of commonly ignored directories. + exclude = [ + ".git", + "__pycache__", + # submodules + "components/bootloader/subproject/components/micro-ecc/micro-ecc", + "components/bt/host/nimble/nimble", + "components/cmock/CMock", + "components/JSON/cJSON", + "components/mbedtls/mbedtls", + "components/openthread/openthread", + "components/unity/unity", + "components/spiffs/spiffs", + # autogenerated scripts + "components/protocomm/python/constants_pb2.py", + "components/protocomm/python/sec0_pb2.py", + "components/protocomm/python/sec1_pb2.py", + "components/protocomm/python/sec2_pb2.py", + "components/protocomm/python/session_pb2.py", + "components/esp_local_ctrl/python/esp_local_ctrl_pb2.py", + ] + +[lint.isort] + force-single-line = true diff --git a/sonar-project.properties b/sonar-project.properties deleted file mode 100644 index 1c9d5882fb3..00000000000 --- a/sonar-project.properties +++ /dev/null @@ -1,24 +0,0 @@ -sonar.projectKey=esp-idf -sonar.projectName=Espressif IoT Development Framework -sonar.projectVersion=4.4 - -sonar.sources=. -sonar.sourceEncoding=UTF-8 - -sonar.inclusions=\ - **/*.c,**/*.cpp,**/*.h,**/*.hpp,\ - **/*.py - -#################### -# sonar-cxx-plugin # -#################### - -sonar.cxx.includeDirectories=/usr/include -sonar.cxx.file.suffixes=.cxx,.cpp,.cc,.c,.hxx,.hpp,.hh,.h - -####################### -# sonar-gitlab-plugin # -####################### - -sonar.gitlab.failure_notification_mode=exit-code -sonar.gitlab.project_id=103 diff --git a/tools/ci/build_template_app.sh b/tools/ci/build_template_app.sh index 224967aa681..6f69794c129 100755 --- a/tools/ci/build_template_app.sh +++ b/tools/ci/build_template_app.sh @@ -11,6 +11,13 @@ set -euo pipefail TEMPLATE_APP_PATH="esp-idf-template" +# Set default paths for template app builds +BUILD_PATH="${IDF_PATH}/build_template_app" +BUILD_DIR="${BUILD_PATH}/@t/@w" + +LOG_PATH="${IDF_PATH}/log_template_app" +BUILD_LOG_CMAKE="${LOG_PATH}/cmake_@t_@w.txt" + gen_configs() { # CONFIG_COMPILER_OPTIMIZATION_NONE with flag -O0 echo "CONFIG_COMPILER_OPTIMIZATION_NONE=y" > ${TEMPLATE_APP_PATH}/sdkconfig.ci.O0 @@ -54,30 +61,23 @@ build_stage2() { # Override EXTRA_CFLAGS and EXTRA_CXXFLAGS in the environment export EXTRA_CFLAGS=${PEDANTIC_CFLAGS/-Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function/} export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS/-Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function/} - python -m idf_build_apps build -v \ + idf-build-apps build \ -p ${TEMPLATE_APP_PATH} \ - -t all \ ${CONFIG_STR} \ --work-dir ${BUILD_PATH}/cmake \ --build-dir ${BUILD_DIR} \ --build-log ${BUILD_LOG_CMAKE} \ - --size-file size.json \ - --keep-going \ - --collect-size-info size_info.txt \ --default-build-targets esp32 esp32s2 esp32s3 esp32c2 esp32c3 esp32c5 esp32c6 esp32h2 esp32p4 } build_stage1() { CONFIG_STR=$(get_config_str sdkconfig.ci2.*=) - python -m idf_build_apps build -v \ + idf-build-apps build \ -p ${TEMPLATE_APP_PATH} \ - -t all \ ${CONFIG_STR} \ --work-dir ${BUILD_PATH}/cmake \ --build-dir ${BUILD_DIR} \ --build-log ${BUILD_LOG_CMAKE} \ - --size-file size.json \ - --collect-size-info size_info.txt \ --default-build-targets esp32 esp32s2 esp32s3 esp32c2 esp32c3 esp32c5 esp32c6 esp32h2 esp32p4 } diff --git a/tools/ci/check_build_test_rules.py b/tools/ci/check_build_test_rules.py index 093c0af3375..537b0d83e54 100755 --- a/tools/ci/check_build_test_rules.py +++ b/tools/ci/check_build_test_rules.py @@ -1,33 +1,27 @@ #!/usr/bin/env python # SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + import argparse -import inspect import os import re import sys +from collections import defaultdict from pathlib import Path -from typing import Dict -from typing import List -from typing import Optional -from typing import Tuple -import yaml -from idf_ci_utils import get_all_manifest_files +from idf_build_apps import App +from idf_build_apps import find_apps +from idf_build_apps.constants import SUPPORTED_TARGETS from idf_ci_utils import IDF_PATH - -# Copy of the constant from idf_pytest.constants, to avoid installing -# additional dependencies (pytest, pytest-embedded) in pre-commit hooks -DEFAULT_CONFIG_RULES_STR = ['sdkconfig.ci=default', 'sdkconfig.ci.*=', '=default'] - -YES = u'\u2713' -NO = u'\u2717' +from idf_ci_utils import idf_relpath +from rich.console import Console +from rich.table import Table # | Supported Target | ... | # | ---------------- | --- | -SUPPORTED_TARGETS_TABLE_REGEX = re.compile( - r'^\|\s*Supported Targets.+$\n^\|(?:\s*|-).+$\n?', re.MULTILINE -) +SUPPORTED_TARGETS_TABLE_REGEX = re.compile(r'^\|\s*Supported Targets.+$\n^\|(?:\s*|-).+$\n?', re.MULTILINE) USUAL_TO_FORMAL = { 'esp32': 'ESP32', @@ -43,350 +37,265 @@ USUAL_TO_FORMAL = { 'linux': 'Linux', } -FORMAL_TO_USUAL = { - 'ESP32': 'esp32', - 'ESP32-S2': 'esp32s2', - 'ESP32-S3': 'esp32s3', - 'ESP32-C3': 'esp32c3', - 'ESP32-C2': 'esp32c2', - 'ESP32-C6': 'esp32c6', - 'ESP32-C5': 'esp32c5', - 'ESP32-H2': 'esp32h2', - 'ESP32-P4': 'esp32p4', - 'ESP32-C61': 'esp32c61', - 'Linux': 'linux', -} +FORMAL_TO_USUAL = {v: k for k, v in USUAL_TO_FORMAL.items()} + +# Targets under bringup: suppress check-test-scripts warnings for these. +# Reset to [] once bringup is complete. +BYPASS_CHECK_TEST_TARGETS: list[str] = [ + 'esp32c5', + 'esp32c61', +] -def doublequote(s: str) -> str: - if s.startswith('"') and s.endswith('"'): - return s +def print_diff_table( + list1: list[str], + list2: list[str], + title1: str, + title2: str, + excluded: list[str] | None = None, +) -> None: + left_set = set(list1) + right_set = set(list2) - return f'"{s}"' + if left_set == right_set: + return + + all_items = sorted(left_set.union(right_set)) + left_column = [] + right_column = [] + + for item in all_items: + if item in left_set and item not in right_set: + if excluded and item in excluded: + left_column.append(f'[red]{item}[/red] (excluded)') + else: + left_column.append(f'[red]{item}[/red]') + right_column.append('') + elif item in right_set and item not in left_set: + if excluded and item in excluded: + right_column.append(f'[green]{item}[/green] (excluded)') + else: + right_column.append(f'[green]{item}[/green]') + left_column.append('') + else: + left_column.append(item) + right_column.append(item) + + table = Table(show_header=True, header_style='bold magenta') + table.add_column(title1.strip(), justify='left') + table.add_column(title2.strip(), justify='left') + + for l_col, r_col in zip(left_column, right_column): + table.add_row(l_col, r_col) + + console = Console() + console.print(table) + + +def get_readme_path(app_dir: str) -> str | None: + readme_path = os.path.join(app_dir, 'README.md') + + if not os.path.isfile(readme_path): + readme_path = os.path.join(app_dir, '..', 'README.md') + + if not os.path.isfile(readme_path): + readme_path = None # type: ignore + + return readme_path + + +def generate_new_support_table(targets: list[str]) -> str: + # extra space here + table_headers = [f'{USUAL_TO_FORMAL[target]}' for target in targets] + table_headers = ['Supported Targets'] + table_headers + + res = '| ' + ' | '.join(table_headers) + ' |\n' + res += '| ' + ' | '.join(['-' * len(item) for item in table_headers]) + ' |' + + return res + + +def parse_existing_table(app_dir: str) -> tuple[str | None, list[str]]: + readme_path = get_readme_path(app_dir) + if not readme_path: + return None, SUPPORTED_TARGETS + + with open(readme_path, encoding='utf8') as _fr: + readme_str = _fr.read() + + support_string = SUPPORTED_TARGETS_TABLE_REGEX.findall(readme_str) + if not support_string: + return None, SUPPORTED_TARGETS + + # old style + parts = [part.strip() for part in support_string[0].split('\n', 1)[0].split('|') if part.strip()] + return support_string[0].strip(), [FORMAL_TO_USUAL[part] for part in parts[1:] if part in FORMAL_TO_USUAL] + + +def get_grouped_apps( + paths: list[str], + exclude_dirs: list[str] | None = None, +) -> dict[str, list[App]]: + apps = sorted( + find_apps( + paths, + 'all', + exclude_list=exclude_dirs or [], + ) + ) + + grouped_apps: dict[str, list[App]] = defaultdict(list) + for app in apps: + grouped_apps[app.app_dir].append(app) + + return grouped_apps def check_readme( - paths: List[str], - exclude_dirs: Optional[List[str]] = None, - extra_default_build_targets: Optional[List[str]] = None, + paths: list[str], + exclude_dirs: list[str] | None = None, ) -> None: - from idf_build_apps import App, find_apps - from idf_build_apps.constants import SUPPORTED_TARGETS + grouped_apps = get_grouped_apps(paths, exclude_dirs) + exit_code = 0 - def app_build_targets(_app: App) -> List[str]: - return sorted( + for app_dir, apps in grouped_apps.items(): + old_table, old_targets = parse_existing_table(app_dir) + manifest_targets = sorted( { target + for app in apps for target in ( - App.MANIFEST.enable_build_targets(_app.app_dir) - + App.MANIFEST.enable_build_targets(_app.app_dir, config_name=_app.config_name) + App.MANIFEST.enable_build_targets(app_dir) + + App.MANIFEST.enable_build_targets(app_dir, config_name=app.config_name) ) } ) - def get_readme_path(_app: App) -> Optional[str]: - _readme_path = os.path.join(_app.app_dir, 'README.md') - - if not os.path.isfile(_readme_path): - _readme_path = os.path.join(_app.app_dir, '..', 'README.md') - - if not os.path.isfile(_readme_path): - _readme_path = None # type: ignore - - return _readme_path - - def _generate_new_support_table_str(_app: App) -> str: - manifest_targets = app_build_targets(_app) - # extra space here - table_headers = [ - f'{USUAL_TO_FORMAL[target]}' for target in manifest_targets - ] - table_headers = ['Supported Targets'] + table_headers - - res = '| ' + ' | '.join(table_headers) + ' |\n' - res += '| ' + ' | '.join(['-' * len(item) for item in table_headers]) + ' |' - - return res - - def _parse_existing_support_table_str(_app: App) -> Tuple[Optional[str], List[str]]: - _readme_path = get_readme_path(_app) - if not _readme_path: - return None, SUPPORTED_TARGETS - - with open(_readme_path, encoding='utf8') as _fr: - _readme_str = _fr.read() - - support_string = SUPPORTED_TARGETS_TABLE_REGEX.findall(_readme_str) - if not support_string: - return None, SUPPORTED_TARGETS - - # old style - parts = [ - part.strip() - for part in support_string[0].split('\n', 1)[0].split('|') - if part.strip() - ] - return support_string[0].strip(), [FORMAL_TO_USUAL[part] for part in parts[1:] if part in FORMAL_TO_USUAL] - - def check_enable_build(_app: App, _old_supported_targets: List[str]) -> bool: - manifest_targets = app_build_targets(_app) - - if manifest_targets == sorted(_old_supported_targets): - return True - - _readme_path = get_readme_path(_app) - if_clause = f'IDF_TARGET in [{", ".join([doublequote(target) for target in sorted(_old_supported_targets)])}]' - - print( - inspect.cleandoc( - f''' - {_app.app_dir}: - - enable build targets according to the manifest file: {manifest_targets} - - enable build targets according to the old Supported Targets table under readme "{_readme_path}": {_old_supported_targets} - - If you want to disable some targets, please use the following snippet: - - # Please combine this with the original one - # - # Notes: - # - please keep in mind to avoid duplicated folders as yaml keys - # - please use parentheses to group conditions, the "or" and "and" operators could only accept two operands - {_app.app_dir}: - enable: - - if: {if_clause} - temporary: true - reason: - - ''' - ) - ) - - return False - - apps = sorted( - find_apps( - paths, - 'all', - recursive=True, - exclude_list=exclude_dirs or [], - manifest_files=get_all_manifest_files(), - default_build_targets=SUPPORTED_TARGETS + extra_default_build_targets, - config_rules_str=DEFAULT_CONFIG_RULES_STR, - ) - ) - exit_code = 0 - - checked_app_dirs = set() - for app in apps: - if app.app_dir not in checked_app_dirs: - checked_app_dirs.add(app.app_dir) - else: - continue - - replace_str, old_supported_targets = _parse_existing_support_table_str(app) - success = check_enable_build(app, old_supported_targets) - if not success: - print(f'check_enable_build failed for app: {app}') - print('-' * 80) + if sorted(manifest_targets) != sorted(old_targets): + print(f'Build target MISMATCH!!!: {app_dir}') + print_diff_table(manifest_targets, old_targets, 'Manifest', 'README') + print(f'Corresponding manifest file: {App.MANIFEST.most_suitable_rule(app_dir).by_manifest_file}') exit_code = 1 - readme_path = get_readme_path(app) + readme_path = get_readme_path(app_dir) + new_table = generate_new_support_table(manifest_targets) # no readme, create a new file if not readme_path: - with open(os.path.join(app.app_dir, 'README.md'), 'w') as fw: - fw.write(_generate_new_support_table_str(app) + '\n') - print(f'Added new README file: {os.path.join(app.app_dir, "README.md")}') - print('-' * 80) + with open(os.path.join(app_dir, 'README.md'), 'w') as fw: + fw.write(new_table + '\n') + print(f'Auto-fixing: Added new README: {os.path.join(app_dir, "README.md")}') exit_code = 1 # has old table, but different string - elif replace_str and replace_str != _generate_new_support_table_str(app): + elif old_table and old_table != new_table: with open(readme_path) as fr: readme_str = fr.read() with open(readme_path, 'w') as fw: - fw.write( - readme_str.replace( - replace_str, _generate_new_support_table_str(app) - ) - ) - print(f'Modified README file: {readme_path}') - print('-' * 80) + fw.write(readme_str.replace(old_table, new_table)) + print(f'Auto-fixing: Updated README: {readme_path}') exit_code = 1 # does not have old table - elif not replace_str: + elif not old_table: with open(readme_path) as fr: readme_str = fr.read() with open(readme_path, 'w') as fw: - fw.write( - _generate_new_support_table_str(app) + '\n\n' + readme_str - ) # extra new line - - print(f'Modified README file: {readme_path}') - print('-' * 80) + fw.write(new_table + '\n\n' + readme_str) # extra new line + print(f'Auto-fixing: Modified README: {readme_path}') exit_code = 1 + if exit_code != 0: + print('') # extra new line for readability + print('Related documentation:') + print( + '\t- https://docs.espressif.com/projects/idf-build-apps/en/latest/references/manifest.html#enable-disable-rules' + ) sys.exit(exit_code) +def get_grouped_cases(paths: list[str]) -> dict[str, dict[str, set[str]]]: + """ + return something like this: + { + app_dir: { + 'script_paths': {'path/to/script1', 'path/to/script2'}, + 'targets': {'esp32', 'esp32s2', 'esp32s3', 'esp32c3', 'esp32c2', 'linux'}, + } + } + """ + from idf_ci import get_pytest_cases + + pytest_cases = get_pytest_cases( + paths=paths, + marker_expr=None, # don't filter host_test + ) + + grouped_cases = {} + for case in pytest_cases: + for pytest_app in case.apps: + app_dir = idf_relpath(pytest_app.path) + if app_dir not in grouped_cases: + grouped_cases[app_dir] = { + 'script_paths': {case.path}, + 'targets': {pytest_app.target}, + } + else: + grouped_cases[app_dir]['script_paths'].add(case.path) + grouped_cases[app_dir]['targets'].add(pytest_app.target) + + return grouped_cases + + def check_test_scripts( - paths: List[str], - exclude_dirs: Optional[List[str]] = None, - bypass_check_test_targets: Optional[List[str]] = None, - extra_default_build_targets: Optional[List[str]] = None, + paths: list[str], + exclude_dirs: list[str] | None = None, + *, + bypass_targets: list[str] | None = None, ) -> None: - from idf_build_apps import App, find_apps - from idf_build_apps.constants import SUPPORTED_TARGETS - from idf_pytest.script import get_pytest_cases - # takes long time, run only in CI - # dict: - # { - # app_dir: { - # 'script_path': 'path/to/script', - # 'targets': ['esp32', 'esp32s2', 'esp32s3', 'esp32c3', 'esp32c2', 'linux'], - # } - # } - def check_enable_test( - _app: App, - _pytest_app_dir_targets_dict: Dict[str, Dict[str, str]], - ) -> bool: - if _app.app_dir in _pytest_app_dir_targets_dict: - test_script_path = _pytest_app_dir_targets_dict[_app.app_dir]['script_path'] - actual_verified_targets = sorted( - set(_pytest_app_dir_targets_dict[_app.app_dir]['targets']) - ) - else: - return True # no test case + grouped_apps = get_grouped_apps(paths, exclude_dirs) + grouped_cases = get_grouped_cases(paths) + exit_code = 0 - manifest_targets = sorted({ - target for target in ( - App.MANIFEST.enable_test_targets(_app.app_dir) - + App.MANIFEST.enable_test_targets(_app.app_dir, config_name=_app.config_name) - ) - }) + for app_dir, apps in grouped_apps.items(): + if app_dir not in grouped_cases: + continue - actual_extra_tested_targets = set(actual_verified_targets) - set(manifest_targets) - if actual_extra_tested_targets - set(bypass_check_test_targets or []): - print( - inspect.cleandoc( - f''' - {_app.app_dir}: - - enable test targets according to the manifest file: {manifest_targets} - - enable test targets according to the test scripts: {actual_verified_targets} - - test scripts enabled targets should be a subset of the manifest file declared ones. - Please check the test script: {test_script_path}. - - ''' + # manifest defined ones + manifest_targets = sorted( + { + target + for app in apps + for target in ( + App.MANIFEST.enable_test_targets(app_dir) + + App.MANIFEST.enable_test_targets(app_dir, config_name=app.config_name) ) - ) - return False + } + ) + actual_targets = sorted(grouped_cases[app_dir]['targets']) - if manifest_targets == actual_verified_targets: - return True - if not ((set(manifest_targets) ^ set(actual_verified_targets)) - set(bypass_check_test_targets or [])): - print(f'WARNING: bypass test script check on {_app.app_dir} for targets {bypass_check_test_targets} ') - return True + if manifest_targets == actual_targets: + continue + if not ((set(manifest_targets) ^ set(actual_targets)) - set(bypass_targets or [])): + continue - if_clause = f'IDF_TARGET in [{", ".join([doublequote(target) for target in sorted(set(manifest_targets) - set(actual_verified_targets))])}]' + print(f'Test target MISMATCH!!!: {app_dir}') + print_diff_table(manifest_targets, actual_targets, 'Manifest', 'Test Scripts', bypass_targets) + print(f'Corresponding manifest file: {App.MANIFEST.most_suitable_rule(app_dir).by_manifest_file}') + print('Corresponding test scripts:') + for script_path in grouped_cases[app_dir]['script_paths']: + print(' - ' + script_path) + exit_code = 1 + if exit_code != 0: + print('') # extra new line for readability + print('Related documentation:') print( - inspect.cleandoc( - f''' - {_app.app_dir}: - - enable test targets according to the manifest file: {manifest_targets} - - enable test targets according to the test scripts: {actual_verified_targets} - - the test scripts enabled test targets should be the same with the manifest file enabled ones. Please check - the test script manually: {test_script_path}. - - If you want to enable test targets in the pytest test scripts, please add `@pytest.mark.MISSING_TARGET` - marker above the test case function. - - If you want to disable the test targets in the manifest file, please modify your manifest file with - the following code snippet: - - # Please combine this with the original one - # - # Notes: - # - please keep in mind to avoid duplicated folders as yaml keys - # - please use parentheses to group conditions, the "or" and "and" operators could only accept two operands - {_app.app_dir}: - disable_test: - - if: {if_clause} - temporary: true - reason: - - ''' - ) + '\t- https://docs.espressif.com/projects/idf-build-apps/en/latest/references/manifest.html#enable-disable-rules' ) - return False - - apps = sorted( - find_apps( - paths, - 'all', - recursive=True, - exclude_list=exclude_dirs or [], - manifest_files=get_all_manifest_files(), - default_build_targets=SUPPORTED_TARGETS + extra_default_build_targets, - config_rules_str=DEFAULT_CONFIG_RULES_STR, + print( + '\t- https://docs.espressif.com/projects/esp-idf/en/latest/esp32/contribute/esp-idf-tests-with-pytest.html' ) - ) - exit_code = 0 - - pytest_cases = get_pytest_cases(paths) - - pytest_app_dir_targets_dict = {} - for case in pytest_cases: - for pytest_app in case.apps: - app_dir = os.path.relpath(pytest_app.path, IDF_PATH) - if app_dir not in pytest_app_dir_targets_dict: - pytest_app_dir_targets_dict[app_dir] = { - 'script_path': case.path, - 'targets': [pytest_app.target], - } - else: - pytest_app_dir_targets_dict[app_dir]['targets'].append( - pytest_app.target - ) - - checked_app_dirs = set() - for app in apps: - if app.app_dir not in checked_app_dirs: - checked_app_dirs.add(app.app_dir) - else: - continue - - success = check_enable_test( - app, pytest_app_dir_targets_dict - ) - if not success: - print(f'check_enable_test failed for app: {app}') - print('-' * 80) - exit_code = 1 - continue - - sys.exit(exit_code) - - -def check_exist() -> None: - exit_code = 0 - - config_files = get_all_manifest_files() - for file in config_files: - if 'managed_components' in Path(file).parts: - continue - - with open(file) as fr: - configs = yaml.safe_load(fr) - for path in configs.keys(): - if path.startswith('.'): - continue - if not os.path.isdir(path): - print(f'Path \'{path}\' referred in \'{file}\' does not exist!') - exit_code = 1 sys.exit(exit_code) @@ -398,92 +307,57 @@ if __name__ == '__main__': parser = argparse.ArgumentParser(description='ESP-IDF apps build/test checker') action = parser.add_subparsers(dest='action') - _check_readme = action.add_parser('check-readmes') - _check_readme.add_argument('paths', nargs='+', help='check under paths') - _check_readme.add_argument( - '-c', - '--config', - default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'), - help='default build test rules config file', - ) - - _check_test_scripts = action.add_parser('check-test-scripts') - _check_test_scripts.add_argument('paths', nargs='+', help='check under paths') - _check_test_scripts.add_argument( - '-c', - '--config', - default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'), - help='default build test rules config file', - ) - - _check_exist = action.add_parser('check-exist') + readme_parser = action.add_parser('check-readmes') + readme_parser.add_argument('paths', nargs='+', help='check under paths') + test_parser = action.add_parser('check-test-scripts') + test_parser.add_argument('paths', nargs='+', help='check under paths') arg = parser.parse_args() - # Since this script is executed from the pre-commit hook environment, make sure IDF_PATH is set - os.environ['IDF_PATH'] = os.path.realpath( - os.path.join(os.path.dirname(__file__), '..', '..') + check_dirs = set() + + # check if *_caps.h files changed + check_all = False + soc_caps_header_files = list((Path(IDF_PATH) / 'components' / 'soc').glob('**/*_caps.h')) + for p in arg.paths: + if Path(p).resolve() in soc_caps_header_files: + check_all = True + break + + if os.path.isfile(p): + check_dirs.add(os.path.dirname(p)) + else: + check_dirs.add(p) + + if 'tools/idf_py_actions/constants.py' in arg.paths or 'tools/ci/check_build_test_rules.py' in arg.paths: + check_all = True + + if check_all: + check_dirs = {IDF_PATH} + + _exclude_dirs = [ + os.path.join(IDF_PATH, 'tools', 'unit-test-app'), + os.path.join(IDF_PATH, 'tools', 'test_build_system'), + os.path.join(IDF_PATH, 'tools', 'templates', 'sample_project'), + os.path.join(IDF_PATH, 'tools', 'cmakev2', 'test'), + ] + + os.environ.update( + { + 'IDF_PATH': IDF_PATH, + 'INCLUDE_NIGHTLY_RUN': '1', + 'NIGHTLY_RUN': '1', + } ) - if arg.action == 'check-exist': - check_exist() - else: - check_dirs = set() - - # check if *_caps.h files changed - check_all = False - soc_caps_header_files = list( - (Path(IDF_PATH) / 'components' / 'soc').glob('**/*_caps.h') + if arg.action == 'check-readmes': + check_readme( + list(check_dirs), + _exclude_dirs, + ) + elif arg.action == 'check-test-scripts': + check_test_scripts( + list(check_dirs), + _exclude_dirs, + bypass_targets=BYPASS_CHECK_TEST_TARGETS, ) - for p in arg.paths: - if Path(p).resolve() in soc_caps_header_files: - check_all = True - break - - if os.path.isfile(p): - check_dirs.add(os.path.dirname(p)) - else: - check_dirs.add(p) - - if 'tools/idf_py_actions/constants.py' in arg.paths or 'tools/ci/check_build_test_rules.py' in arg.paths: - check_all = True - - if check_all: - check_dirs = {IDF_PATH} - _exclude_dirs = [os.path.join(IDF_PATH, 'tools', 'unit-test-app'), - os.path.join(IDF_PATH, 'tools', 'test_build_system', 'build_test_app'), - os.path.join(IDF_PATH, 'tools', 'templates', 'sample_project')] - else: - _exclude_dirs = [os.path.join(IDF_PATH, 'tools', 'templates', 'sample_project')] - - extra_default_build_targets_list: List[str] = [] - bypass_check_test_targets_list: List[str] = [] - if arg.config: - with open(arg.config) as fr: - configs = yaml.safe_load(fr) - - if configs: - extra_default_build_targets_list = ( - configs.get('extra_default_build_targets') or [] - ) - bypass_check_test_targets_list = ( - configs.get('bypass_check_test_targets') or [] - ) - - if arg.action == 'check-readmes': - os.environ['INCLUDE_NIGHTLY_RUN'] = '1' - os.environ['NIGHTLY_RUN'] = '1' - check_readme( - list(check_dirs), - exclude_dirs=_exclude_dirs, - extra_default_build_targets=extra_default_build_targets_list, - ) - elif arg.action == 'check-test-scripts': - os.environ['INCLUDE_NIGHTLY_RUN'] = '1' - os.environ['NIGHTLY_RUN'] = '1' - check_test_scripts( - list(check_dirs), - exclude_dirs=_exclude_dirs, - bypass_check_test_targets=bypass_check_test_targets_list, - extra_default_build_targets=extra_default_build_targets_list, - ) diff --git a/tools/ci/ci_get_mr_info.py b/tools/ci/ci_get_mr_info.py deleted file mode 100644 index 686775adc78..00000000000 --- a/tools/ci/ci_get_mr_info.py +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env python -# -# internal use only for CI -# get latest MR information by source branch -# -# SPDX-FileCopyrightText: 2020-2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -# -import argparse -import logging -import os -import subprocess -import typing as t -from pathlib import Path - -from gitlab_api import Gitlab -from idf_ci_utils import IDF_PATH - -if t.TYPE_CHECKING: - from gitlab.v4.objects import ProjectCommit, ProjectMergeRequest - - -def _get_mr_obj(source_branch: str) -> t.Optional['ProjectMergeRequest']: - gl = Gitlab(os.getenv('CI_PROJECT_ID', 'espressif/esp-idf')) - if not gl.project: - return None - - mrs = gl.project.mergerequests.list(state='opened', source_branch=source_branch) - if mrs: - return mrs[0] # one source branch can only have one opened MR at one moment - else: - return None - - -def get_mr_iid(source_branch: str) -> str: - mr = _get_mr_obj(source_branch) - if not mr: - return '' - else: - return str(mr.iid) - - -def get_mr_changed_files(source_branch: str) -> t.List[str]: - mr = _get_mr_obj(source_branch) - if not mr: - return [] - - git_output = subprocess.check_output( - ['git', 'diff', '--name-only', '--diff-filter=d', f'origin/{mr.target_branch}...origin/{source_branch}'] - ).decode('utf8') - - return [line.strip() for line in git_output.splitlines() if line.strip()] - - -def get_mr_commits(source_branch: str) -> t.List['ProjectCommit']: - mr = _get_mr_obj(source_branch) - if not mr: - return [] - - return list(mr.commits()) - - -_COMPONENT_NAME_DIR_RECORDS = {} - - -def get_modified_component(filepath: str) -> t.Optional[str]: - """Return the component name if the file is in a component directory, otherwise None.""" - try: - f_path = Path(filepath).resolve().relative_to(IDF_PATH) - except ValueError: # not in IDF_PATH - return None - - # skip md files, etc. - if f_path.suffix in ['.md', '.yml']: - return None - - # skip test_apps files - if 'test_apps' in f_path.parts: - return None - - component_parent_dirs = [f_path.parts[0]] - for part in f_path.parts[1:]: - if component_parent_dirs[-1] == 'components' or component_parent_dirs[-1].endswith('common_components'): - if part not in _COMPONENT_NAME_DIR_RECORDS: - logging.debug('Found component "%s" in path "%s"' % (part, component_parent_dirs)) - _COMPONENT_NAME_DIR_RECORDS[part] = component_parent_dirs - elif _COMPONENT_NAME_DIR_RECORDS.get(part) != component_parent_dirs: - logging.debug( - 'WARNING!!! Found component "%s" in path "%s" and "%s"' - % (part, component_parent_dirs, _COMPONENT_NAME_DIR_RECORDS.get(part)) - ) - - return part - - component_parent_dirs.append(part) - - return None - - -def get_mr_components( - source_branch: t.Optional[str] = None, modified_files: t.Optional[t.List[str]] = None -) -> t.List[str]: - components: t.Set[str] = set() - if modified_files is None: - if not source_branch: - raise RuntimeError('--src-branch is required if --modified-files is not provided') - - modified_files = get_mr_changed_files(source_branch) - - for f in modified_files: - modified_component = get_modified_component(f) - if modified_component: - components.add(modified_component) - - return list(components) - - -def get_target_in_tags(tags: str) -> str: - from idf_pytest.constants import TARGET_MARKERS - - for x in tags.split(','): - if x in TARGET_MARKERS: - return x - - raise RuntimeError(f'No target marker found in {tags}') - - -def _print_list(_list: t.List[str], separator: str = '\n') -> None: - print(separator.join(_list)) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Get the latest merge request info by pipeline') - actions = parser.add_subparsers(dest='action', help='info type', required=True) - - common_args = argparse.ArgumentParser(add_help=False) - common_args.add_argument('--src-branch', help='source branch') - common_args.add_argument( - '--modified-files', - nargs='+', - help='space-separated list specifies the modified files. will be detected by --src-branch if not provided', - ) - - actions.add_parser('id', parents=[common_args]) - actions.add_parser('commits', parents=[common_args]) - actions.add_parser('components', parents=[common_args]) - target = actions.add_parser('target_in_tags') - target.add_argument('tags', help='comma separated tags, e.g., esp32,generic') - - args = parser.parse_args() - - if args.action == 'id': - if not args.src_branch: - raise RuntimeError('--src-branch is required') - print(get_mr_iid(args.src_branch)) - elif args.action == 'commits': - if not args.src_branch: - raise RuntimeError('--src-branch is required') - _print_list([commit.id for commit in get_mr_commits(args.src_branch)]) - elif args.action == 'components': - _print_list(get_mr_components(args.src_branch, args.modified_files)) - elif args.action == 'target_in_tags': - print(get_target_in_tags(args.tags)) - else: - raise NotImplementedError('not possible to get here') diff --git a/tools/ci/deploy_docs.py b/tools/ci/deploy_docs.py deleted file mode 100755 index 954958e738d..00000000000 --- a/tools/ci/deploy_docs.py +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/env python3 -# -# CI script to deploy docs to a webserver. Not useful outside of CI environment -# -# -# SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -# -import glob -import os -import os.path -import re -import stat -import subprocess -import sys -import tarfile -from typing import Any, List, Tuple - -import packaging.version - - -def env(variable: str, default: str=None) -> str: - """ Shortcut to return the expanded version of an environment variable """ - return os.path.expandvars(os.environ.get(variable, default) if default else os.environ[variable]) - - -# import sanitize_version from the docs directory, shared with here -sys.path.append(os.path.join(env('IDF_PATH'), 'docs')) -from sanitize_version import sanitize_version # noqa - - -def main() -> None: - # if you get KeyErrors on the following lines, it's probably because you're not running in Gitlab CI - git_ver = env('GIT_VER') # output of git describe --always - ci_ver = env('CI_COMMIT_REF_NAME', git_ver) # branch or tag we're building for (used for 'release' & URL) - - version = sanitize_version(ci_ver) - print('Git version: {}'.format(git_ver)) - print('CI Version: {}'.format(ci_ver)) - print('Deployment version: {}'.format(version)) - - if not version: - raise RuntimeError('A version is needed to deploy') - - build_dir = env('DOCS_BUILD_DIR') # top-level local build dir, where docs have already been built - - if not build_dir: - raise RuntimeError('Valid DOCS_BUILD_DIR is needed to deploy') - - url_base = env('DOCS_DEPLOY_URL_BASE') # base for HTTP URLs, used to print the URL to the log after deploying - - docs_server = env('DOCS_DEPLOY_SERVER') # ssh server to deploy to - docs_user = env('DOCS_DEPLOY_SERVER_USER') - docs_path = env('DOCS_DEPLOY_PATH') # filesystem path on DOCS_SERVER - - if not docs_server: - raise RuntimeError('Valid DOCS_DEPLOY_SERVER is needed to deploy') - - if not docs_user: - raise RuntimeError('Valid DOCS_DEPLOY_SERVER_USER is needed to deploy') - - docs_server = '{}@{}'.format(docs_user, docs_server) - - if not docs_path: - raise RuntimeError('Valid DOCS_DEPLOY_PATH is needed to deploy') - - print('DOCS_DEPLOY_SERVER {} DOCS_DEPLOY_PATH {}'.format(docs_server, docs_path)) - - tarball_path, version_urls = build_doc_tarball(version, git_ver, build_dir) - - deploy(version, tarball_path, docs_path, docs_server) - - print('Docs URLs:') - doc_deploy_type = os.getenv('TYPE') - for vurl in version_urls: - language, _, target = vurl.split('/') - tag = '{}_{}'.format(language, target) - url = '{}/{}/index.html'.format(url_base, vurl) # (index.html needed for the preview server) - url = re.sub(r'([^:])//', r'\1/', url) # get rid of any // that isn't in the https:// part - print('[document {}][{}] {}'.format(doc_deploy_type, tag, url)) - - # note: it would be neater to use symlinks for stable, but because of the directory order - # (language first) it's kind of a pain to do on a remote server, so we just repeat the - # process but call the version 'stable' this time - if is_stable_version(version): - print('Deploying again as stable version...') - tarball_path, version_urls = build_doc_tarball('stable', git_ver, build_dir) - deploy('stable', tarball_path, docs_path, docs_server) - - -def deploy(version: str, tarball_path: str, docs_path: str, docs_server: str) -> None: - def run_ssh(commands: List) -> None: - """ Log into docs_server and run a sequence of commands using ssh """ - print('Running ssh: {}'.format(commands)) - subprocess.run(['ssh', '-o', 'BatchMode=yes', docs_server, '-x', ' && '.join(commands)], check=True) - - # copy the version tarball to the server - run_ssh(['mkdir -p {}'.format(docs_path)]) - print('Running scp {} to {}'.format(tarball_path, '{}:{}'.format(docs_server, docs_path))) - subprocess.run(['scp', '-B', tarball_path, '{}:{}'.format(docs_server, docs_path)], check=True) - - tarball_name = os.path.basename(tarball_path) - - run_ssh(['cd {}'.format(docs_path), - 'rm -rf ./*/{}'.format(version), # remove any pre-existing docs matching this version - 'tar -zxvf {}'.format(tarball_name), # untar the archive with the new docs - 'rm {}'.format(tarball_name)]) - - # Note: deleting and then extracting the archive is a bit awkward for updating stable/latest/etc - # as the version will be invalid for a window of time. Better to do it atomically, but this is - # another thing made much more complex by the directory structure putting language before version... - - -def build_doc_tarball(version: str, git_ver: str, build_dir: str) -> Tuple[str, List]: - """ Make a tar.gz archive of the docs, in the directory structure used to deploy as - the given version """ - version_paths = [] - tarball_path = '{}/{}.tar.gz'.format(build_dir, version) - - # find all the 'html/' directories under build_dir - html_dirs = glob.glob('{}/**/html/'.format(build_dir), recursive=True) - print('Found %d html directories' % len(html_dirs)) - - pdfs = glob.glob('{}/**/latex/build/*.pdf'.format(build_dir), recursive=True) - print('Found %d PDFs in latex directories' % len(pdfs)) - - # add symlink for stable and latest and adds them to PDF blob - symlinks = create_and_add_symlinks(version, git_ver, pdfs) - - def not_sources_dir(ti: Any) -> Any: - print(type(ti)) - """ Filter the _sources directories out of the tarballs """ - if ti.name.endswith('/_sources'): - return None - - ti.mode |= stat.S_IWGRP # make everything group-writeable - return ti - - try: - os.remove(tarball_path) - except OSError: - pass - - with tarfile.open(tarball_path, 'w:gz') as tarball: - for html_dir in html_dirs: - # html_dir has the form '///html/' - target_dirname = os.path.dirname(os.path.dirname(html_dir)) - target = os.path.basename(target_dirname) - language = os.path.basename(os.path.dirname(target_dirname)) - - # when deploying, we want the top-level directory layout 'language/version/target' - archive_path = '{}/{}/{}'.format(language, version, target) - print("Archiving '{}' as '{}'...".format(html_dir, archive_path)) - tarball.add(html_dir, archive_path, filter=not_sources_dir) - version_paths.append(archive_path) - - for pdf_path in pdfs: - # pdf_path has the form '///latex/build' - latex_dirname = os.path.dirname(pdf_path) - pdf_filename = os.path.basename(pdf_path) - target_dirname = os.path.dirname(os.path.dirname(latex_dirname)) - target = os.path.basename(target_dirname) - language = os.path.basename(os.path.dirname(target_dirname)) - - # when deploying, we want the layout 'language/version/target/pdf' - archive_path = '{}/{}/{}/{}'.format(language, version, target, pdf_filename) - print("Archiving '{}' as '{}'...".format(pdf_path, archive_path)) - tarball.add(pdf_path, archive_path) - - for symlink in symlinks: - os.unlink(symlink) - - return (os.path.abspath(tarball_path), version_paths) - - -def create_and_add_symlinks(version: str, git_ver: str, pdfs: List) -> List: - """ Create symbolic links for PDFs for 'latest' and 'stable' releases """ - - symlinks = [] - if 'stable' in version or 'latest' in version: - for pdf_path in pdfs: - symlink_path = pdf_path.replace(git_ver, version) - os.symlink(pdf_path, symlink_path) - symlinks.append(symlink_path) - - pdfs.extend(symlinks) - print('Found %d PDFs in latex directories after adding symlink' % len(pdfs)) - - return symlinks - - -def is_stable_version(version: str) -> bool: - """ Heuristic for whether this is the latest stable release """ - if not version.startswith('v'): - return False # branch name - if '-' in version: - return False # prerelease tag - - git_out = subprocess.check_output(['git', 'tag', '-l']).decode('utf-8') - - versions = [v.strip() for v in git_out.split('\n')] - versions = [v for v in versions if re.match(r'^v[\d\.]+$', v.strip())] # include vX.Y.Z only - - versions_pack = [packaging.version.parse(v) for v in versions] - - max_version = max(versions_pack) - - if max_version.public != version[1:]: - print('Stable version is v{}. This version is {}.'.format(max_version.public, version)) - return False - else: - print('This version {} is the stable version'.format(version)) - return True - - -if __name__ == '__main__': - main() diff --git a/tools/ci/dynamic_pipelines/constants.py b/tools/ci/dynamic_pipelines/constants.py index c0c90c4d270..2f64e515e73 100644 --- a/tools/ci/dynamic_pipelines/constants.py +++ b/tools/ci/dynamic_pipelines/constants.py @@ -1,49 +1,9 @@ -# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2024-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import os from idf_ci_utils import IDF_PATH -# use relative path to avoid absolute path in pipeline -DEFAULT_TEST_PATHS = [ - 'examples', - os.path.join('tools', 'test_apps'), - 'components', -] - -DEFAULT_APPS_BUILD_PER_JOB = 60 -DEFAULT_CASES_TEST_PER_JOB = 30 - -DEFAULT_BUILD_CHILD_PIPELINE_FILEPATH = os.path.join(IDF_PATH, 'build_child_pipeline.yml') -DEFAULT_TARGET_TEST_CHILD_PIPELINE_FILEPATH = os.path.join(IDF_PATH, 'target_test_child_pipeline.yml') - -DEFAULT_BUILD_CHILD_PIPELINE_NAME = 'Build Child Pipeline' -DEFAULT_TARGET_TEST_CHILD_PIPELINE_NAME = 'Target Test Child Pipeline' - -TEST_RELATED_BUILD_JOB_NAME = 'build_test_related_apps' -NON_TEST_RELATED_BUILD_JOB_NAME = 'build_non_test_related_apps' - -COMMENT_START_MARKER = '### Dynamic Pipeline Report' -TEST_RELATED_APPS_FILENAME = 'test_related_apps.txt' -NON_TEST_RELATED_APPS_FILENAME = 'non_test_related_apps.txt' - -TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME = 'test_related_apps_download_urls.yml' -REPORT_TEMPLATE_FILEPATH = os.path.join( - IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'report.template.html' -) - -RETRY_JOB_PICTURE_PATH = 'tools/ci/dynamic_pipelines/templates/retry-jobs.png' -RETRY_JOB_TITLE = '\n\nRetry failed jobs with with help of "retry_failed_jobs" stage of the pipeline:' -RETRY_JOB_PICTURE_LINK = '![Retry Jobs Image]({pic_url})' - -BUILD_ONLY_LABEL = 'For Maintainers: Only Build Tests' - KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH = os.path.join( IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'known_generate_test_child_pipeline_warnings.yml' ) - -CI_JOB_TOKEN = os.getenv('CI_JOB_TOKEN', '') -CI_DASHBOARD_API = os.getenv('CI_DASHBOARD_API', '') -CI_PAGES_URL = os.getenv('CI_PAGES_URL', '') -CI_PROJECT_URL = os.getenv('CI_PROJECT_URL', '') -CI_MERGE_REQUEST_SOURCE_BRANCH_SHA = os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA', '') diff --git a/tools/ci/dynamic_pipelines/models.py b/tools/ci/dynamic_pipelines/models.py deleted file mode 100644 index 55659179401..00000000000 --- a/tools/ci/dynamic_pipelines/models.py +++ /dev/null @@ -1,226 +0,0 @@ -# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import inspect -import os -import typing as t -import urllib.parse -from dataclasses import dataclass -from xml.etree.ElementTree import Element - -import yaml - - -class Job: - def __init__( - self, - *, - name: str, - extends: t.Optional[t.List[str]] = None, - tags: t.Optional[t.List[str]] = None, - stage: t.Optional[str] = None, - parallel: int = 1, - variables: t.Optional[t.Dict[str, str]] = None, - script: t.Optional[t.List[str]] = None, - before_script: t.Optional[t.List[str]] = None, - after_script: t.Optional[t.List[str]] = None, - needs: t.Optional[t.List[str]] = None, - **kwargs: t.Any, - ) -> None: - self.name = name - self.extends = extends - self.tags = tags - self.stage = stage - self.parallel = parallel - self.variables = variables or {} - self.script = script - self.before_script = before_script - self.after_script = after_script - self.needs = needs - - for k, v in kwargs.items(): - setattr(self, k, v) - - def __str__(self) -> str: - return yaml.dump(self.to_dict()) # type: ignore - - def set_variable(self, key: str, value: str) -> None: - self.variables[key] = value - - def to_dict(self) -> t.Dict[str, t.Any]: - res = {} - for k, v in inspect.getmembers(self): - if k.startswith('_'): - continue - - # name is the dict key - if k == 'name': - continue - - # parallel 1 is not allowed - if k == 'parallel' and v == 1: - continue - - if v is None: - continue - - if inspect.ismethod(v) or inspect.isfunction(v): - continue - - res[k] = v - - return {self.name: res} - - -class EmptyJob(Job): - def __init__( - self, - *, - name: t.Optional[str] = None, - tags: t.Optional[t.List[str]] = None, - stage: t.Optional[str] = None, - before_script: t.Optional[t.List[str]] = None, - after_script: t.Optional[t.List[str]] = None, - **kwargs: t.Any, - ) -> None: - super().__init__( - name=name or 'fake_pass_job', - tags=tags or ['fast_run', 'shiny'], - stage=stage or 'build', - script=['echo "This is a fake job to pass the pipeline"'], - before_script=before_script or [], - after_script=after_script or [], - **kwargs, - ) - - -class BuildJob(Job): - def __init__( - self, - *, - extends: t.Optional[t.List[str]] = None, - tags: t.Optional[t.List[str]] = None, - stage: t.Optional[str] = None, - **kwargs: t.Any, - ) -> None: - super().__init__( - extends=extends or ['.dynamic_build_template'], - tags=tags or ['build', 'shiny'], - stage=stage or 'build', - **kwargs, - ) - - -class TargetTestJob(Job): - def __init__( - self, - *, - extends: t.Optional[t.List[str]] = None, - stage: t.Optional[str] = None, - **kwargs: t.Any, - ) -> None: - super().__init__( - extends=extends or ['.dynamic_target_test_template'], - stage=stage or 'target_test', - **kwargs, - ) - - -@dataclass -class TestCase: - name: str - file: str - time: float - failure: t.Optional[str] = None - skipped: t.Optional[str] = None - ci_job_url: t.Optional[str] = None - ci_dashboard_url: t.Optional[str] = None - dut_log_url: t.Optional[str] = None - latest_total_count: int = 0 - latest_failed_count: int = 0 - - @property - def is_failure(self) -> bool: - return self.failure is not None - - @property - def is_skipped(self) -> bool: - return self.skipped is not None - - @property - def is_success(self) -> bool: - return not self.is_failure and not self.is_skipped - - @classmethod - def from_test_case_node(cls, node: Element) -> t.Optional['TestCase']: - if 'name' not in node.attrib: - print('WARNING: Node Invalid: ', node) - return None - - # url to test cases dashboard - grafana_base_url = urllib.parse.urljoin(os.getenv('CI_DASHBOARD_HOST', ''), '/d/Ucg477Fnz/case-list') - encoded_params = urllib.parse.urlencode({'var-case_id': node.attrib['name']}, quote_via=urllib.parse.quote) - - kwargs = { - 'name': node.attrib['name'], - 'file': node.attrib.get('file'), - 'time': float(node.attrib.get('time') or 0), - 'ci_job_url': node.attrib.get('ci_job_url') or 'Not found', - 'ci_dashboard_url': f'{grafana_base_url}?{encoded_params}', - 'dut_log_url': node.attrib.get('dut_log_url') or 'Not found', - } - - failure_node = node.find('failure') - # bool(failure_node) is False, so compare with None - if failure_node is None: - failure_node = node.find('error') - if failure_node is not None: - message = failure_node.attrib.get('message', '') - kwargs['failure'] = message - - skipped_node = node.find('skipped') - if skipped_node is not None: - kwargs['skipped'] = skipped_node.attrib['message'] - - return cls(**kwargs) # type: ignore - - -@dataclass -class GitlabJob: - id: int - name: str - stage: str - status: str - url: str - ci_dashboard_url: str - failure_reason: t.Optional[str] = None - failure_log: t.Optional[str] = None - latest_total_count: int = 0 - latest_failed_count: int = 0 - - @property - def is_failed(self) -> bool: - return self.status == 'failed' - - @property - def is_success(self) -> bool: - return self.status == 'success' - - @classmethod - def from_json_data(cls, job_data: dict, failure_data: dict) -> t.Optional['GitlabJob']: - grafana_base_url = urllib.parse.urljoin(os.getenv('CI_DASHBOARD_HOST', ''), '/d/LoUa-qLWz/job-list') - encoded_params = urllib.parse.urlencode({'var-job_name': job_data['name']}, quote_via=urllib.parse.quote) - - kwargs = { - 'id': job_data['id'], - 'name': job_data['name'], - 'stage': job_data['stage'], - 'status': job_data['status'], - 'url': job_data['url'], - 'ci_dashboard_url': f'{grafana_base_url}?{encoded_params}', - 'failure_reason': job_data['failure_reason'], - 'failure_log': job_data['failure_log'], - 'latest_total_count': failure_data.get('total_count', 0), - 'latest_failed_count': failure_data.get('failed_count', 0), - } - - return cls(**kwargs) # type: ignore diff --git a/tools/ci/dynamic_pipelines/report.py b/tools/ci/dynamic_pipelines/report.py deleted file mode 100644 index 153f1071c38..00000000000 --- a/tools/ci/dynamic_pipelines/report.py +++ /dev/null @@ -1,739 +0,0 @@ -# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import abc -import copy -import fnmatch -import html -import os -import re -import typing as t -from textwrap import dedent - -import yaml -from artifacts_handler import ArtifactType -from gitlab import GitlabUpdateError -from gitlab_api import Gitlab -from idf_build_apps import App -from idf_build_apps.constants import BuildStatus -from idf_ci_local.uploader import AppUploader -from prettytable import PrettyTable - -from .constants import COMMENT_START_MARKER -from .constants import REPORT_TEMPLATE_FILEPATH -from .constants import RETRY_JOB_PICTURE_LINK -from .constants import RETRY_JOB_PICTURE_PATH -from .constants import RETRY_JOB_TITLE -from .constants import TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME -from .models import GitlabJob -from .models import TestCase -from .utils import fetch_failed_testcases_failure_ratio -from .utils import format_permalink -from .utils import get_artifacts_url -from .utils import get_repository_file_url -from .utils import is_url -from .utils import load_known_failure_cases - - -class ReportGenerator: - REGEX_PATTERN = r'#### {}\n[\s\S]*?(?=\n#### |$)' - - def __init__(self, project_id: int, mr_iid: int, pipeline_id: int, job_id: int, commit_id: str, *, title: str): - gl_project = Gitlab(project_id).project - if mr_iid is not None: - self.mr = gl_project.mergerequests.get(mr_iid) - else: - self.mr = None - self.pipeline_id = pipeline_id - self.job_id = job_id - self.commit_id = commit_id - - self.title = title - self.output_filepath = self.title.lower().replace(' ', '_') + '.html' - self.additional_info = '' - - @staticmethod - def get_download_link_for_url(url: str) -> str: - if url: - return f'Download' - - return '' - - def write_report_to_file(self, report_str: str, job_id: int, output_filepath: str) -> t.Optional[str]: - """ - Writes the report to a file and constructs a modified URL based on environment settings. - - :param report_str: The report content to be written to the file. - :param job_id: The job identifier used to construct the URL. - :param output_filepath: The path to the output file. - :return: The modified URL pointing to the job's artifacts. - """ - if not report_str: - return None - with open(output_filepath, 'w') as file: - file.write(report_str) - - # for example, {URL}/-/esp-idf/-/jobs/{id}/artifacts/list_job_84.txt - # CI_PAGES_URL is {URL}/esp-idf, which missed one `-` - report_url: str = get_artifacts_url(job_id, output_filepath) - return report_url - - def generate_html_report(self, table_str: str) -> str: - # we're using bootstrap table - table_str = table_str.replace( - '', - '
', - ) - with open(REPORT_TEMPLATE_FILEPATH) as fr: - template = fr.read() - - return template.replace('{{title}}', self.title).replace('{{table}}', table_str) - - @staticmethod - def table_to_html_str(table: PrettyTable) -> str: - return html.unescape(table.get_html_string()) # type: ignore - - def create_table_section( - self, - title: str, - items: list, - headers: list, - row_attrs: list, - value_functions: t.Optional[list] = None, - ) -> t.List: - """ - Appends a formatted section to a report based on the provided items. This section includes - a header and a table constructed from the items list with specified headers and attributes. - - :param title: Title for the report section. This title is used as a header above the table. - :param items: List of item objects to include in the table. Each item should have attributes - that correspond to the row_attrs and value_functions specified. - :param headers: List of strings that will serve as the column headers in the generated table. - :param row_attrs: List of attributes to include from each item for the table rows. These - should be attributes or keys that exist on the items in the 'items' list. - :param value_functions: Optional list of tuples containing additional header and corresponding - value function. Each tuple should specify a header (as a string) and - a function that takes an item and returns a string. This is used for - generating dynamic columns based on item data. - - :return: List with appended HTML sections. - """ - if not items: - return [] - - report_sections = [ - f"""

{title}

""", - self._create_table_for_items( - items=items, headers=headers, row_attrs=row_attrs, value_functions=value_functions or [] - ), - ] - return report_sections - - @staticmethod - def generate_additional_info_section(title: str, count: int, report_url: t.Optional[str] = None) -> str: - """ - Generate a section for the additional info string. - - :param title: The title of the section. - :param count: The count of test cases. - :param report_url: The URL of the report. If count = 0, only the count will be included. - :return: The formatted additional info section string. - """ - if count != 0 and report_url: - return f'- **{title}:** [{count}]({report_url}/#{format_permalink(title)})\n' - else: - return f'- **{title}:** {count}\n' - - def _create_table_for_items( - self, - items: t.Union[t.List[TestCase], t.List[GitlabJob]], - headers: t.List[str], - row_attrs: t.List[str], - value_functions: t.Optional[t.List[t.Tuple[str, t.Callable[[t.Union[TestCase, GitlabJob]], str]]]] = None, - ) -> str: - """ - Create a PrettyTable and convert it to an HTML string for the provided test cases. - :param items: List of item objects to include in the table. - :param headers: List of strings for the table headers. - :param row_attrs: List of attributes to include in each row. - :param value_functions: List of tuples containing additional header and corresponding value function. - :return: HTML table string. - """ - table = PrettyTable() - table.field_names = headers - - # Create a mapping of header names to their corresponding index in the headers list - header_index_map = {header: i for i, header in enumerate(headers)} - - for item in items: - row = [] - for attr in row_attrs: - value = str(getattr(item, attr, '')) - if is_url(value): - link = f'link' - row.append(link) - else: - row.append(value) - - # Insert values computed by value functions at the correct column position based on their headers - if value_functions: - for header, func in value_functions: - index = header_index_map.get(header) - if index is not None: - computed_value = func(item) - row.insert(index, computed_value) - - table.add_row(row) - - return self.table_to_html_str(table) - - @staticmethod - def _filter_items( - items: t.Union[t.List[TestCase], t.List[GitlabJob]], condition: t.Callable[[t.Union[TestCase, GitlabJob]], bool] - ) -> t.List[TestCase]: - """ - Filter items s based on a given condition. - - :param items: List of items to filter by given condition. - :param condition: A function that evaluates to True or False for each items. - :return: List of filtered instances. - """ - return [item for item in items if condition(item)] - - @staticmethod - def _sort_items( - items: t.List[t.Union[TestCase, GitlabJob]], - key: t.Union[str, t.Callable[[t.Union[TestCase, GitlabJob]], t.Any]], - order: str = 'asc', - ) -> t.List[t.Union[TestCase, GitlabJob]]: - """ - Sort items based on a given key and order. - - :param items: List of items to sort. - :param key: A string representing the attribute name or a function to extract the sorting key. - :param order: Order of sorting ('asc' for ascending, 'desc' for descending). - :return: List of sorted instances. - """ - key_func = None - if isinstance(key, str): - - def key_func(item: t.Any) -> t.Any: - return getattr(item, key) - - if key_func is not None: - try: - items = sorted(items, key=key_func, reverse=(order == 'desc')) - except TypeError: - print(f'Comparison for the key {key} is not supported') - return items - - @abc.abstractmethod - def _get_report_str(self) -> str: - raise NotImplementedError - - def _generate_comment(self, print_report_path: bool) -> str: - # Report in HTML format to avoid exceeding length limits - comment = f'#### {self.title}\n' - report_str = self._get_report_str() - - if self.additional_info: - comment += f'{self.additional_info}\n' - - report_url_path = self.write_report_to_file(report_str, self.job_id, self.output_filepath) - if print_report_path and report_url_path: - comment += dedent( - f""" - Full {self.title} here: {report_url_path} (with commit {self.commit_id[:8]}) - - """ - ) - return comment - - def _update_mr_comment(self, comment: str, print_retry_jobs_message: bool) -> None: - retry_job_picture_comment = (f'{RETRY_JOB_TITLE}\n\n' f'{RETRY_JOB_PICTURE_LINK}').format( - pic_url=get_repository_file_url(RETRY_JOB_PICTURE_PATH) - ) - del_retry_job_pic_pattern = re.escape(RETRY_JOB_TITLE) + r'.*?' + re.escape(f'{RETRY_JOB_PICTURE_PATH})') - - new_comment = f'{COMMENT_START_MARKER}\n\n{comment}' - if print_retry_jobs_message: - new_comment += retry_job_picture_comment - - for note in self.mr.notes.list(iterator=True): - if note.body.startswith(COMMENT_START_MARKER): - updated_str = self._get_updated_comment(note.body, comment) - - # Add retry job message only if any job has failed - if print_retry_jobs_message: - updated_str = re.sub(del_retry_job_pic_pattern, '', updated_str, flags=re.DOTALL) - updated_str += retry_job_picture_comment - - note.body = updated_str - try: - note.save() - except GitlabUpdateError: - print('Failed to update MR comment, Creating a new comment') - self.mr.notes.create({'body': new_comment}) - break - else: - self.mr.notes.create({'body': new_comment}) - - def _get_updated_comment(self, existing_comment: str, new_comment: str) -> str: - updated_str = re.sub(self.REGEX_PATTERN.format(self.title), new_comment, existing_comment) - if updated_str == existing_comment: - updated_str = f'{existing_comment.strip()}\n\n{new_comment}' - return updated_str - - def post_report(self, print_report_path: bool = True, print_retry_jobs_message: bool = False) -> None: - comment = self._generate_comment(print_report_path) - - print(comment) - - if self.mr is None: - print('No MR found, skip posting comment') - return - - self._update_mr_comment(comment, print_retry_jobs_message=print_retry_jobs_message) - - -class BuildReportGenerator(ReportGenerator): - def __init__( - self, - project_id: int, - mr_iid: int, - pipeline_id: int, - job_id: int, - commit_id: str, - *, - title: str = 'Build Report', - apps: t.List[App], - ): - super().__init__(project_id, mr_iid, pipeline_id, job_id, commit_id, title=title) - self.apps = apps - - self.apps_presigned_url_filepath = TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME - - def _get_report_str(self) -> str: - if not self.apps: - print('No apps found, skip generating build report') - return 'No Apps Built' - - uploader = AppUploader(self.pipeline_id) - - table_str = '' - - failed_apps = [app for app in self.apps if app.build_status == BuildStatus.FAILED] - if failed_apps: - table_str += '

Failed Apps

' - - failed_apps_table = PrettyTable() - failed_apps_table.field_names = [ - 'App Dir', - 'Build Dir', - 'Failed Reason', - 'Build Log', - ] - for app in failed_apps: - failed_apps_table.add_row( - [ - app.app_dir, - app.build_dir, - app.build_comment or '', - self.get_download_link_for_url(uploader.get_app_presigned_url(app, ArtifactType.LOGS)), - ] - ) - - table_str += self.table_to_html_str(failed_apps_table) - - built_test_related_apps = [app for app in self.apps if app.build_status == BuildStatus.SUCCESS and app.preserve] - if built_test_related_apps: - table_str += '

Built Apps (Test Related)

' - - built_apps_table = PrettyTable() - built_apps_table.field_names = [ - 'App Dir', - 'Build Dir', - 'Bin Files with Build Log (without map and elf)', - 'Map and Elf Files', - ] - app_presigned_urls_dict: t.Dict[str, t.Dict[str, str]] = {} - for app in built_test_related_apps: - _d = { - ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES.value: uploader.get_app_presigned_url( - app, ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES - ), - ArtifactType.MAP_AND_ELF_FILES.value: uploader.get_app_presigned_url( - app, ArtifactType.MAP_AND_ELF_FILES - ), - } - - built_apps_table.add_row( - [ - app.app_dir, - app.build_dir, - self.get_download_link_for_url(_d[ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES]), - self.get_download_link_for_url(_d[ArtifactType.MAP_AND_ELF_FILES]), - ] - ) - - app_presigned_urls_dict[app.build_path] = _d - - # also generate a yaml file that includes the apps and the presigned urls - # for helping debugging locally - with open(self.apps_presigned_url_filepath, 'w') as fw: - yaml.dump(app_presigned_urls_dict, fw) - - table_str += self.table_to_html_str(built_apps_table) - - built_non_test_related_apps = [ - app for app in self.apps if app.build_status == BuildStatus.SUCCESS and not app.preserve - ] - if built_non_test_related_apps: - table_str += '

Built Apps (Non Test Related)

' - - built_apps_table = PrettyTable() - built_apps_table.field_names = [ - 'App Dir', - 'Build Dir', - 'Build Log', - ] - for app in built_non_test_related_apps: - built_apps_table.add_row( - [ - app.app_dir, - app.build_dir, - self.get_download_link_for_url(uploader.get_app_presigned_url(app, ArtifactType.LOGS)), - ] - ) - - table_str += self.table_to_html_str(built_apps_table) - - skipped_apps = [app for app in self.apps if app.build_status == BuildStatus.SKIPPED] - if skipped_apps: - table_str += '

Skipped Apps

' - - skipped_apps_table = PrettyTable() - skipped_apps_table.field_names = ['App Dir', 'Build Dir', 'Skipped Reason', 'Build Log'] - for app in skipped_apps: - skipped_apps_table.add_row( - [ - app.app_dir, - app.build_dir, - app.build_comment or '', - self.get_download_link_for_url(uploader.get_app_presigned_url(app, ArtifactType.LOGS)), - ] - ) - - table_str += self.table_to_html_str(skipped_apps_table) - - return self.generate_html_report(table_str) - - -class TargetTestReportGenerator(ReportGenerator): - def __init__( - self, - project_id: int, - mr_iid: int, - pipeline_id: int, - job_id: int, - commit_id: str, - *, - title: str = 'Target Test Report', - test_cases: t.List[TestCase], - ): - super().__init__(project_id, mr_iid, pipeline_id, job_id, commit_id, title=title) - - self.test_cases = test_cases - self._known_failure_cases_set = None - self.report_titles_map = { - 'failed_yours': 'Failed Test Cases on Your branch (Excludes Known Failure Cases)', - 'failed_others': 'Failed Test Cases on Other branches (Excludes Known Failure Cases)', - 'failed_known': 'Known Failure Cases', - 'skipped': 'Skipped Test Cases', - 'succeeded': 'Succeeded Test Cases', - } - self.skipped_test_cases_report_file = 'skipped_cases.html' - self.succeeded_cases_report_file = 'succeeded_cases.html' - self.failed_cases_report_file = 'failed_cases.html' - - @property - def known_failure_cases_set(self) -> t.Optional[t.Set[str]]: - if self._known_failure_cases_set is None: - self._known_failure_cases_set = load_known_failure_cases() - - return self._known_failure_cases_set - - def get_known_failure_cases(self) -> t.List[TestCase]: - """ - Retrieve the known failure test cases. - :return: A list of known failure test cases. - """ - if self.known_failure_cases_set is None: - return [] - matched_cases = [ - testcase - for testcase in self.test_cases - if any(fnmatch.fnmatch(testcase.name, pattern) for pattern in self.known_failure_cases_set) - and testcase.is_failure - ] - return matched_cases - - @staticmethod - def filter_test_cases( - cur_branch_failures: t.List[TestCase], - other_branch_failures: t.List[TestCase], - ) -> t.Tuple[t.List[TestCase], t.List[TestCase]]: - """ - Filter the test cases into current branch failures and other branch failures. - - :param cur_branch_failures: List of failed test cases on the current branch. - :param other_branch_failures: List of failed test cases on other branches. - :return: A tuple containing two lists: - - failed_test_cases_cur_branch_only: Test cases that have failed only on the current branch. - - failed_test_cases_other_branch_exclude_cur_branch: Test cases that have failed on other branches - excluding the current branch. - """ - cur_branch_unique_failures = [] - other_branch_failure_map = {tc.name: tc for tc in other_branch_failures} - - for cur_tc in cur_branch_failures: - if cur_tc.latest_failed_count > 0 and ( - cur_tc.name not in other_branch_failure_map - or other_branch_failure_map[cur_tc.name].latest_failed_count == 0 - ): - cur_branch_unique_failures.append(cur_tc) - uniq_fail_names = {cur_tc.name for cur_tc in cur_branch_unique_failures} - other_branch_exclusive_failures = [tc for tc in other_branch_failures if tc.name not in uniq_fail_names] - - return cur_branch_unique_failures, other_branch_exclusive_failures - - def get_failed_cases_report_parts(self) -> t.List[str]: - """ - Generate the report parts for failed test cases and update the additional info section. - :return: A list of strings representing the table sections for the failed test cases. - """ - known_failures = self.get_known_failure_cases() - failed_test_cases = self._filter_items( - self.test_cases, lambda tc: tc.is_failure and tc.name not in {case.name for case in known_failures} - ) - failed_test_cases_cur_branch = self._sort_items( - fetch_failed_testcases_failure_ratio( - copy.deepcopy(failed_test_cases), - branches_filter={'include_branches': [os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_NAME', '')]}, - ), - key='latest_failed_count', - ) - failed_test_cases_other_branch = self._sort_items( - fetch_failed_testcases_failure_ratio( - copy.deepcopy(failed_test_cases), - branches_filter={'exclude_branches': [os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_NAME', '')]}, - ), - key='latest_failed_count', - ) - failed_test_cases_cur_branch, failed_test_cases_other_branch = self.filter_test_cases( - failed_test_cases_cur_branch, failed_test_cases_other_branch - ) - cur_branch_cases_table_section = self.create_table_section( - title=self.report_titles_map['failed_yours'], - items=failed_test_cases_cur_branch, - headers=[ - 'Test Case', - 'Test Script File Path', - 'Failure Reason', - f'Failures on your branch (40 latest testcases)', - 'Dut Log URL', - 'Job URL', - 'Grafana URL', - ], - row_attrs=['name', 'file', 'failure', 'dut_log_url', 'ci_job_url', 'ci_dashboard_url'], - value_functions=[ - ( - 'Failures on your branch (40 latest testcases)', - lambda item: f"{getattr(item, 'latest_failed_count', '')} / {getattr(item, 'latest_total_count', '')}", - ) - ], - ) - other_branch_cases_table_section = self.create_table_section( - title=self.report_titles_map['failed_others'], - items=failed_test_cases_other_branch, - headers=[ - 'Test Case', - 'Test Script File Path', - 'Failure Reason', - 'Cases that failed in other branches as well (40 latest testcases)', - 'Dut Log URL', - 'Job URL', - 'Grafana URL', - ], - row_attrs=['name', 'file', 'failure', 'dut_log_url', 'ci_job_url', 'ci_dashboard_url'], - value_functions=[ - ( - 'Cases that failed in other branches as well (40 latest testcases)', - lambda item: f"{getattr(item, 'latest_failed_count', '')} / {getattr(item, 'latest_total_count', '')}", - ) - ], - ) - known_failures_cases_table_section = self.create_table_section( - title=self.report_titles_map['failed_known'], - items=known_failures, - headers=['Test Case', 'Test Script File Path', 'Failure Reason', 'Job URL', 'Grafana URL'], - row_attrs=['name', 'file', 'failure', 'ci_job_url', 'ci_dashboard_url'], - ) - failed_cases_report_url = self.write_report_to_file( - self.generate_html_report( - ''.join( - cur_branch_cases_table_section - + other_branch_cases_table_section - + known_failures_cases_table_section - ) - ), - self.job_id, - self.failed_cases_report_file, - ) - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['failed_yours'], len(failed_test_cases_cur_branch), failed_cases_report_url - ) - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['failed_others'], len(failed_test_cases_other_branch), failed_cases_report_url - ) - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['failed_known'], len(known_failures), failed_cases_report_url - ) - return cur_branch_cases_table_section + other_branch_cases_table_section + known_failures_cases_table_section - - def get_skipped_cases_report_parts(self) -> t.List[str]: - """ - Generate the report parts for skipped test cases and update the additional info section. - :return: A list of strings representing the table sections for the skipped test cases. - """ - skipped_test_cases = self._filter_items(self.test_cases, lambda tc: tc.is_skipped) - skipped_cases_table_section = self.create_table_section( - title=self.report_titles_map['skipped'], - items=skipped_test_cases, - headers=['Test Case', 'Test Script File Path', 'Skipped Reason', 'Grafana URL'], - row_attrs=['name', 'file', 'skipped', 'ci_dashboard_url'], - ) - skipped_cases_report_url = self.write_report_to_file( - self.generate_html_report(''.join(skipped_cases_table_section)), - self.job_id, - self.skipped_test_cases_report_file, - ) - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['skipped'], len(skipped_test_cases), skipped_cases_report_url - ) - return skipped_cases_table_section - - def get_succeeded_cases_report_parts(self) -> t.List[str]: - """ - Generate the report parts for succeeded test cases and update the additional info section. - :return: A list of strings representing the table sections for the succeeded test cases. - """ - succeeded_test_cases = self._filter_items(self.test_cases, lambda tc: tc.is_success) - succeeded_cases_table_section = self.create_table_section( - title=self.report_titles_map['succeeded'], - items=succeeded_test_cases, - headers=['Test Case', 'Test Script File Path', 'Job URL', 'Grafana URL'], - row_attrs=['name', 'file', 'ci_job_url', 'ci_dashboard_url'], - ) - succeeded_cases_report_url = self.write_report_to_file( - self.generate_html_report(''.join(succeeded_cases_table_section)), - self.job_id, - self.succeeded_cases_report_file, - ) - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['succeeded'], len(succeeded_test_cases), succeeded_cases_report_url - ) - self.additional_info += '\n' - return succeeded_cases_table_section - - def _get_report_str(self) -> str: - """ - Generate a complete HTML report string by processing test cases. - :return: Complete HTML report string. - """ - self.additional_info = f'**Test Case Summary (with commit {self.commit_id[:8]}):**\n' - failed_cases_report_parts = self.get_failed_cases_report_parts() - skipped_cases_report_parts = self.get_skipped_cases_report_parts() - succeeded_cases_report_parts = self.get_succeeded_cases_report_parts() - - return self.generate_html_report( - ''.join(failed_cases_report_parts + skipped_cases_report_parts + succeeded_cases_report_parts) - ) - - -class JobReportGenerator(ReportGenerator): - def __init__( - self, - project_id: int, - mr_iid: int, - pipeline_id: int, - job_id: int, - commit_id: str, - *, - title: str = 'Job Report', - jobs: t.List[GitlabJob], - ): - super().__init__(project_id, mr_iid, pipeline_id, job_id, commit_id, title=title) - self.jobs = jobs - self.report_titles_map = { - 'failed_jobs': 'Failed Jobs (Excludes "integration_test" and "target_test" jobs)', - 'succeeded': 'Succeeded Jobs', - } - self.failed_jobs_report_file = 'job_report.html' - - def _get_report_str(self) -> str: - """ - Generate a complete HTML report string by processing jobs. - :return: Complete HTML report string. - """ - report_str: str = '' - - if not self.jobs: - print('No jobs found, skip generating job report') - return 'No Job Found' - - relevant_failed_jobs = self._sort_items( - self._filter_items( - self.jobs, lambda job: job.is_failed and job.stage not in ['integration_test', 'target_test'] - ), - key='latest_failed_count', - ) - succeeded_jobs = self._filter_items(self.jobs, lambda job: job.is_success) - - self.additional_info = f'**Job Summary (with commit {self.commit_id[:8]}):**\n' - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['succeeded'], len(succeeded_jobs) - ) - - if not relevant_failed_jobs: - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['failed_jobs'], len(relevant_failed_jobs) - ) - return report_str - - report_sections = self.create_table_section( - title='Failed Jobs (Excludes "integration_test" and "target_test" jobs)', - items=relevant_failed_jobs, - headers=[ - 'Job Name', - 'Failure Reason', - 'Failure Log', - 'Failures across all other branches (10 latest jobs)', - 'URL', - 'CI Dashboard URL', - ], - row_attrs=['name', 'failure_reason', 'failure_log', 'url', 'ci_dashboard_url'], - value_functions=[ - ( - 'Failures across all other branches (10 latest jobs)', - lambda item: f"{getattr(item, 'latest_failed_count', '')} / {getattr(item, 'latest_total_count', '')}", - ) - ], - ) - relevant_failed_jobs_report_url = get_artifacts_url(self.job_id, self.failed_jobs_report_file) - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['failed_jobs'], len(relevant_failed_jobs), relevant_failed_jobs_report_url - ) - - report_str = self.generate_html_report(''.join(report_sections)) - return report_str diff --git a/tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py b/tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py deleted file mode 100644 index 8a247e8f51b..00000000000 --- a/tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py +++ /dev/null @@ -1,74 +0,0 @@ -# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import argparse -import sys - -import __init__ # noqa: F401 # inject the system path -from dynamic_pipelines.constants import TEST_RELATED_APPS_FILENAME -from idf_build_apps import build_apps -from idf_build_apps import setup_logging -from idf_build_apps.utils import semicolon_separated_str_to_list -from idf_ci_local.app import import_apps_from_txt -from idf_pytest.constants import DEFAULT_IGNORE_WARNING_FILEPATH - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Build Apps for Dynamic Pipeline') - parser.add_argument('app_list_file', default=TEST_RELATED_APPS_FILENAME, help='List of apps to build') - parser.add_argument( - '--build-verbose', - action='store_true', - help='Enable verbose output from build system.', - ) - parser.add_argument('--parallel-count', default=1, type=int, help='Number of parallel build jobs.') - parser.add_argument( - '--parallel-index', - default=1, - type=int, - help='Index (1-based) of the job, out of the number specified by --parallel-count.', - ) - parser.add_argument( - '--ignore-warning-file', - default=DEFAULT_IGNORE_WARNING_FILEPATH, - type=argparse.FileType('r'), - help='Ignore the warning strings in the specified file. Each line should be a regex string.', - ) - parser.add_argument( - '--modified-components', - type=semicolon_separated_str_to_list, - help='semicolon-separated string which specifies the modified components. ' - 'app with `depends_components` set in the corresponding manifest files would only be built ' - 'if depends on any of the specified components. ' - 'If set to "", the value would be considered as None. ' - 'If set to ";", the value would be considered as an empty list', - ) - parser.add_argument( - '--collect-app-info', - default='list_job_@p.txt', - help='If specified, the test case name and app info json will be written to this file', - ) - parser.add_argument( - '--junitxml', - default='build_summary_@p.xml', - help='Path to the junitxml file. If specified, the junitxml file will be generated', - ) - - args = parser.parse_args() - - setup_logging(verbose=1) - - sys.exit( - build_apps( - import_apps_from_txt(args.app_list_file), - build_verbose=args.build_verbose, - keep_going=True, - ignore_warning_file=args.ignore_warning_file, - modified_components=args.modified_components, - check_app_dependencies=True, - parallel_count=args.parallel_count, - parallel_index=args.parallel_index, - collect_size_info='size_info.txt', - collect_app_info=args.collect_app_info, - junitxml=args.junitxml, - copy_sdkconfig=True, - ) - ) diff --git a/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py b/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py deleted file mode 100644 index f7bb2c793d3..00000000000 --- a/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py +++ /dev/null @@ -1,257 +0,0 @@ -# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -"""This file is used for generating the child pipeline for build jobs.""" -import argparse -import os -import typing as t - -import __init__ # noqa: F401 # inject the system path -import yaml -from dynamic_pipelines.constants import DEFAULT_APPS_BUILD_PER_JOB -from dynamic_pipelines.constants import DEFAULT_BUILD_CHILD_PIPELINE_FILEPATH -from dynamic_pipelines.constants import DEFAULT_BUILD_CHILD_PIPELINE_NAME -from dynamic_pipelines.constants import DEFAULT_TEST_PATHS -from dynamic_pipelines.constants import NON_TEST_RELATED_APPS_FILENAME -from dynamic_pipelines.constants import NON_TEST_RELATED_BUILD_JOB_NAME -from dynamic_pipelines.constants import TEST_RELATED_APPS_FILENAME -from dynamic_pipelines.constants import TEST_RELATED_BUILD_JOB_NAME -from dynamic_pipelines.models import BuildJob -from dynamic_pipelines.models import EmptyJob -from dynamic_pipelines.utils import dump_jobs_to_yaml -from idf_build_apps.utils import semicolon_separated_str_to_list -from idf_ci_local.app import dump_apps_to_txt -from idf_ci_utils import IDF_PATH -from idf_pytest.constants import CollectMode -from idf_pytest.constants import DEFAULT_CONFIG_RULES_STR -from idf_pytest.constants import DEFAULT_FULL_BUILD_TEST_COMPONENTS -from idf_pytest.constants import DEFAULT_FULL_BUILD_TEST_FILEPATTERNS -from idf_pytest.script import get_all_apps - - -def _separate_str_to_list(s: str) -> t.List[str]: - """ - Gitlab env file will escape the doublequotes in the env file, so we need to remove them - - For example, - - in pipeline.env file we have - - MR_MODIFIED_COMPONENTS="app1;app2" - MR_MODIFIED_FILES="main/app1.c;main/app2.c" - - gitlab will load the doublequotes as well, so we need to remove the doublequotes - """ - return semicolon_separated_str_to_list(s.strip('"')) # type: ignore - - -def main(arguments: argparse.Namespace) -> None: - # load from default build test rules config file - extra_default_build_targets: t.List[str] = [] - if arguments.default_build_test_rules: - with open(arguments.default_build_test_rules) as fr: - configs = yaml.safe_load(fr) - - if configs: - extra_default_build_targets = configs.get('extra_default_build_targets') or [] - - build_jobs = [] - ########################################### - # special case with -k, ignore other args # - ########################################### - if arguments.filter_expr: - # build only test related apps - test_related_apps, _ = get_all_apps( - arguments.paths, - target=CollectMode.ALL, - config_rules_str=DEFAULT_CONFIG_RULES_STR, - filter_expr=arguments.filter_expr, - marker_expr='not host_test', - extra_default_build_targets=extra_default_build_targets, - ) - dump_apps_to_txt(sorted(test_related_apps), TEST_RELATED_APPS_FILENAME) - print(f'Generate test related apps file {TEST_RELATED_APPS_FILENAME} with {len(test_related_apps)} apps') - - test_apps_build_job = BuildJob( - name=TEST_RELATED_BUILD_JOB_NAME, - parallel=len(test_related_apps) // DEFAULT_APPS_BUILD_PER_JOB + 1, - variables={ - 'APP_LIST_FILE': TEST_RELATED_APPS_FILENAME, - }, - ) - - build_jobs.append(test_apps_build_job) - else: - ############# - # all cases # - ############# - test_related_apps, non_test_related_apps = get_all_apps( - arguments.paths, - CollectMode.ALL, - marker_expr='not host_test', - config_rules_str=DEFAULT_CONFIG_RULES_STR, - extra_default_build_targets=extra_default_build_targets, - compare_manifest_sha_filepath=arguments.compare_manifest_sha_filepath, - modified_components=arguments.modified_components, - modified_files=arguments.modified_files, - ignore_app_dependencies_components=arguments.ignore_app_dependencies_components, - ignore_app_dependencies_filepatterns=arguments.ignore_app_dependencies_filepatterns, - ) - - dump_apps_to_txt(sorted(test_related_apps), TEST_RELATED_APPS_FILENAME) - print(f'Generate test related apps file {TEST_RELATED_APPS_FILENAME} with {len(test_related_apps)} apps') - dump_apps_to_txt(sorted(non_test_related_apps), NON_TEST_RELATED_APPS_FILENAME) - print( - f'Generate non-test related apps file {NON_TEST_RELATED_APPS_FILENAME} with {len(non_test_related_apps)} apps' - ) - - if test_related_apps: - test_apps_build_job = BuildJob( - name=TEST_RELATED_BUILD_JOB_NAME, - parallel=len(test_related_apps) // DEFAULT_APPS_BUILD_PER_JOB + 1, - variables={ - 'APP_LIST_FILE': TEST_RELATED_APPS_FILENAME, - }, - ) - build_jobs.append(test_apps_build_job) - - if non_test_related_apps: - non_test_apps_build_job = BuildJob( - name=NON_TEST_RELATED_BUILD_JOB_NAME, - parallel=len(non_test_related_apps) // DEFAULT_APPS_BUILD_PER_JOB + 1, - variables={ - 'APP_LIST_FILE': NON_TEST_RELATED_APPS_FILENAME, - }, - ) - build_jobs.append(non_test_apps_build_job) - - if mr_labels := os.getenv('CI_MERGE_REQUEST_LABELS'): - print(f'MR labels: {mr_labels}') - - # check if there's no jobs - if not build_jobs: - print('No apps need to be built. Create one empty job instead') - build_jobs.append(EmptyJob()) - extra_include_yml = [] - else: - extra_include_yml = ['tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml'] - - dump_jobs_to_yaml(build_jobs, arguments.yaml_output, DEFAULT_BUILD_CHILD_PIPELINE_NAME, extra_include_yml) - print(f'Generate child pipeline yaml file {arguments.yaml_output} with {sum(j.parallel for j in build_jobs)} jobs') - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description='Generate build child pipeline', - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - parser.add_argument( - '-o', - '--yaml-output', - default=DEFAULT_BUILD_CHILD_PIPELINE_FILEPATH, - help='Output YAML path', - ) - parser.add_argument( - '-p', - '--paths', - nargs='+', - default=DEFAULT_TEST_PATHS, - help='Paths to the apps to build.', - ) - parser.add_argument( - '-k', - '--filter-expr', - help='only build tests matching given filter expression. For example: -k "test_hello_world". Works only' - 'for pytest', - ) - parser.add_argument( - '--default-build-test-rules', - default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'), - help='default build test rules config file', - ) - parser.add_argument( - '--compare-manifest-sha-filepath', - default=os.path.join(IDF_PATH, '.manifest_sha'), - help='Path to the recorded manifest sha file generated by `idf-build-apps dump-manifest-sha`' - ) - parser.add_argument( - '--modified-components', - type=_separate_str_to_list, - default=os.getenv('MR_MODIFIED_COMPONENTS'), - help='semicolon-separated string which specifies the modified components. ' - 'app with `depends_components` set in the corresponding manifest files would only be built ' - 'if depends on any of the specified components. ' - 'If set to "", the value would be considered as None. ' - 'If set to ";", the value would be considered as an empty list', - ) - parser.add_argument( - '--modified-files', - type=_separate_str_to_list, - default=os.getenv('MR_MODIFIED_FILES'), - help='semicolon-separated string which specifies the modified files. ' - 'app with `depends_filepatterns` set in the corresponding manifest files would only be built ' - 'if any of the specified file pattern matches any of the specified modified files. ' - 'If set to "", the value would be considered as None. ' - 'If set to ";", the value would be considered as an empty list', - ) - parser.add_argument( - '-ic', - '--ignore-app-dependencies-components', - type=_separate_str_to_list, - help='semicolon-separated string which specifies the modified components used for ' - 'ignoring checking the app dependencies. ' - 'The `depends_components` and `depends_filepatterns` set in the manifest files will be ignored ' - 'when any of the specified components matches any of the modified components. ' - 'Must be used together with --modified-components. ' - 'If set to "", the value would be considered as None. ' - 'If set to ";", the value would be considered as an empty list', - ) - parser.add_argument( - '-if', - '--ignore-app-dependencies-filepatterns', - type=_separate_str_to_list, - help='semicolon-separated string which specifies the file patterns used for ' - 'ignoring checking the app dependencies. ' - 'The `depends_components` and `depends_filepatterns` set in the manifest files will be ignored ' - 'when any of the specified file patterns matches any of the modified files. ' - 'Must be used together with --modified-files. ' - 'If set to "", the value would be considered as None. ' - 'If set to ";", the value would be considered as an empty list', - ) - - args = parser.parse_args() - - if test_case_filters := os.getenv('TEST_CASE_FILTERS', None): - args.filter_expr = test_case_filters - - if os.getenv('IS_MR_PIPELINE') == '0' or os.getenv('BUILD_AND_TEST_ALL_APPS') == '1': - print('Build and run all test cases, and compile all cmake apps') - args.modified_components = None - args.modified_files = None - args.ignore_app_dependencies_components = None - args.ignore_app_dependencies_filepatterns = None - elif args.filter_expr is not None: - print('Build and run only test cases matching "%s"' % args.filter_expr) - args.modified_components = None - args.modified_files = None - args.ignore_app_dependencies_components = None - args.ignore_app_dependencies_filepatterns = None - else: - print( - f'Build and run only test cases matching:\n' - f'- modified components: {args.modified_components}\n' - f'- modified files: {args.modified_files}' - ) - - if args.modified_components is not None and not args.ignore_app_dependencies_components: - # setting default values - args.ignore_app_dependencies_components = DEFAULT_FULL_BUILD_TEST_COMPONENTS - - if args.modified_files is not None and not args.ignore_app_dependencies_filepatterns: - # setting default values - args.ignore_app_dependencies_filepatterns = DEFAULT_FULL_BUILD_TEST_FILEPATTERNS - - if not os.path.isfile(args.compare_manifest_sha_filepath): - # ignore if the file does not exist - args.compare_manifest_sha_filepath = None - - main(args) diff --git a/tools/ci/dynamic_pipelines/scripts/generate_report.py b/tools/ci/dynamic_pipelines/scripts/generate_report.py deleted file mode 100644 index b769c93481b..00000000000 --- a/tools/ci/dynamic_pipelines/scripts/generate_report.py +++ /dev/null @@ -1,101 +0,0 @@ -# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import argparse -import glob -import os -import typing as t - -import __init__ # noqa: F401 # inject the system path -from dynamic_pipelines.report import BuildReportGenerator -from dynamic_pipelines.report import JobReportGenerator -from dynamic_pipelines.report import TargetTestReportGenerator -from dynamic_pipelines.utils import fetch_failed_jobs -from dynamic_pipelines.utils import parse_testcases_from_filepattern -from idf_ci_local.app import import_apps_from_txt - - -def main() -> None: - parser: argparse.ArgumentParser = setup_argument_parser() - args: argparse.Namespace = parser.parse_args() - - report_actions: t.Dict[str, t.Callable[[argparse.Namespace], None]] = { - 'build': generate_build_report, - 'target_test': generate_target_test_report, - 'job': generate_jobs_report, - } - - report_action = report_actions.get(args.report_type) - if report_action is None: - raise ValueError('Unknown report type is requested to be generated.') - - report_action(args) - - -def setup_argument_parser() -> argparse.ArgumentParser: - report_type_parser: argparse.ArgumentParser = argparse.ArgumentParser(add_help=False) - report_type_parser.add_argument( - '--report-type', choices=['build', 'target_test', 'job'], required=True, help='Type of report to generate' - ) - report_type_args: argparse.Namespace - remaining_args: t.List[str] - report_type_args, remaining_args = report_type_parser.parse_known_args() - - parser: argparse.ArgumentParser = argparse.ArgumentParser( - description='Update reports in MR pipelines based on the selected report type', - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - parents=[report_type_parser], - ) - - common_arguments(parser) - conditional_arguments(report_type_args, parser) - - return parser - - -def common_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument('--project-id', type=int, default=os.getenv('CI_PROJECT_ID'), help='Project ID') - parser.add_argument('--mr-iid', type=int, default=os.getenv('CI_MERGE_REQUEST_IID'), help='Merge Request IID') - parser.add_argument('--pipeline-id', type=int, default=os.getenv('PARENT_PIPELINE_ID'), help='Pipeline ID') - parser.add_argument('--job-id', type=int, default=os.getenv('CI_JOB_ID'), help='Job ID') - parser.add_argument('--commit-id', default=os.getenv('CI_COMMIT_SHA'), help='MR commit ID') - - -def conditional_arguments(report_type_args: argparse.Namespace, parser: argparse.ArgumentParser) -> None: - if report_type_args.report_type == 'build': - parser.add_argument('--app-list-filepattern', default='list_job_*.txt', help='Pattern to match app list files') - elif report_type_args.report_type == 'target_test': - parser.add_argument( - '--junit-report-filepattern', default='XUNIT_RESULT*.xml', help='Pattern to match JUnit report files' - ) - - -def generate_build_report(args: argparse.Namespace) -> None: - apps: t.List[t.Any] = [ - app for file_name in glob.glob(args.app_list_filepattern) for app in import_apps_from_txt(file_name) - ] - report_generator = BuildReportGenerator( - args.project_id, args.mr_iid, args.pipeline_id, args.job_id, args.commit_id, apps=apps - ) - report_generator.post_report() - - -def generate_target_test_report(args: argparse.Namespace) -> None: - test_cases: t.List[t.Any] = parse_testcases_from_filepattern(args.junit_report_filepattern) - report_generator = TargetTestReportGenerator( - args.project_id, args.mr_iid, args.pipeline_id, args.job_id, args.commit_id, test_cases=test_cases - ) - report_generator.post_report(print_report_path=False) - - -def generate_jobs_report(args: argparse.Namespace) -> None: - jobs: t.List[t.Any] = fetch_failed_jobs(args.commit_id) - - if not jobs: - return - - report_generator = JobReportGenerator(args.project_id, args.mr_iid, args.pipeline_id, args.job_id, args.commit_id, jobs=jobs) - report_generator.post_report(print_report_path=False, print_retry_jobs_message=any(job.is_failed for job in jobs)) - - -if __name__ == '__main__': - main() diff --git a/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py b/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py index 46e934f8a4d..d7af5e0e794 100644 --- a/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py +++ b/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2024-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 """This file is used for generating the child pipeline for target test jobs. @@ -6,121 +6,47 @@ 2. Post the Build Report if it's running in an MR pipeline. 3. Generate the child pipeline for target test jobs. """ + +from __future__ import annotations + import argparse -import glob import os import typing as t -from collections import Counter -from collections import defaultdict import __init__ # noqa: F401 # inject the system path import yaml -from dynamic_pipelines.constants import BUILD_ONLY_LABEL -from dynamic_pipelines.constants import DEFAULT_CASES_TEST_PER_JOB -from dynamic_pipelines.constants import DEFAULT_TARGET_TEST_CHILD_PIPELINE_FILEPATH -from dynamic_pipelines.constants import DEFAULT_TARGET_TEST_CHILD_PIPELINE_NAME -from dynamic_pipelines.constants import DEFAULT_TEST_PATHS -from dynamic_pipelines.constants import ( - KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH, -) -from dynamic_pipelines.models import EmptyJob -from dynamic_pipelines.models import Job -from dynamic_pipelines.models import TargetTestJob -from dynamic_pipelines.utils import dump_jobs_to_yaml -from idf_build_apps import App -from idf_ci_local.app import import_apps_from_txt -from idf_pytest.script import get_pytest_cases +from idf_ci import get_pytest_cases +from idf_ci.idf_gitlab import test_child_pipeline +from idf_ci.idf_pytest import GroupedPytestCases +from idf_ci.idf_pytest.models import GroupKey +from idf_ci_utils import IDF_PATH +from idf_pytest.constants import TIMEOUT_4H_MARKERS + +from dynamic_pipelines.constants import KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH + +BUILD_ONLY_LABEL = 'For Maintainers: Only Build Tests' +TIMEOUT_4H_TEMPLATE_NAME = '.timeout_4h_template' +TARGET_TEST_CHILD_PIPELINE_FILEPATH = os.path.join(IDF_PATH, 'target_test_child_pipeline.yml') -def get_tags_with_amount(s: str) -> t.List[str]: - c: Counter = Counter() - for _t in s.split(','): - c[_t] += 1 - - res = set() - for target, amount in c.items(): - if amount > 1: - res.add(f'{target}_{amount}') - else: - res.add(target) - - return sorted(res) - - -def get_target_test_jobs( - paths: str, apps: t.List[App], exclude_runner_tags: t.Set[str], exclude_runner_tags_matching: t.List[t.Set] -) -> t.Tuple[t.List[Job], t.List[str], t.List[str]]: - """ - Return the target test jobs and the extra yaml files to include - """ +def main(output_filepath: str) -> None: if mr_labels := os.getenv('CI_MERGE_REQUEST_LABELS'): print(f'MR labels: {mr_labels}') if BUILD_ONLY_LABEL in mr_labels.split(','): print('MR has build only label, skip generating target test child pipeline') - return [EmptyJob()], [], [] + test_child_pipeline( + output_filepath, + cases=GroupedPytestCases([]), + ) + return - pytest_cases = get_pytest_cases( - paths, - apps=apps, - marker_expr='not host_test', # since it's generating target-test child pipeline - ) + cases = GroupedPytestCases(get_pytest_cases()) - no_env_marker_test_cases: t.List[str] = [] - res = defaultdict(list) - for case in pytest_cases: - if not case.env_markers: - no_env_marker_test_cases.append(case.item.nodeid) - continue - - res[(case.target_selector, tuple(sorted(case.env_markers)))].append(case) - - target_test_jobs: t.List[Job] = [] - for (target_selector, env_markers), cases in res.items(): - runner_tags = get_tags_with_amount(target_selector) + list(env_markers) - if ','.join(runner_tags) in exclude_runner_tags: - print('WARNING: excluding test cases with runner tags:', runner_tags) - continue - - if any(set(runner_tags) & group for group in exclude_runner_tags_matching): - print(f'WARNING: Excluding test cases with runner tags (wildcard match): {runner_tags}') - continue - - target_test_job = TargetTestJob( - name=f'{target_selector} - {",".join(env_markers)}', - tags=runner_tags, - parallel=len(cases) // DEFAULT_CASES_TEST_PER_JOB + 1, - ) - target_test_job.set_variable('TARGET_SELECTOR', f"'{target_selector}'") - target_test_job.set_variable('ENV_MARKERS', "'" + ' and '.join(env_markers) + "'") - target_test_job.set_variable('PYTEST_NODES', ' '.join([f"'{case.item.nodeid}'" for case in cases])) - - target_test_jobs.append(target_test_job) - - extra_include_yml: t.List[str] = [] - if not target_test_jobs: - print('No target test cases required, create one empty job instead') - target_test_jobs.append(EmptyJob()) - else: - extra_include_yml = ['tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml'] - - fast_pipeline_flag = int(os.getenv('REPORT_EXIT_CODE', 0)) == 30 - if fast_pipeline_flag: - extra_include_yml = ['tools/ci/dynamic_pipelines/templates/fast_pipeline.yml'] - - no_env_marker_test_cases.sort() - return target_test_jobs, extra_include_yml, no_env_marker_test_cases - - -def generate_target_test_child_pipeline( - paths: str, - apps: t.List[App], - output_filepath: str, -) -> None: with open(KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH) as fr: known_warnings_dict = yaml.safe_load(fr) or dict() - def _process_match_group(runner_tags: str) -> t.Union[t.Set, None]: + def _process_match_group(runner_tags: str) -> set | None: match_group = {_el for _el in runner_tags.split(',') if _el != '*'} if len(match_group) == 0: print('WARNING: wildcard exclusion requires at least one specific tag — skipped') @@ -131,7 +57,7 @@ def generate_target_test_child_pipeline( exclude_runner_tags_matching = [] for _tag in known_warnings_dict.get('no_runner_tags', []): if '*' not in _tag: - exclude_runner_tags_set.add(_tag) + exclude_runner_tags_set.add(frozenset(_tag.split(','))) else: if res := _process_match_group(_tag): exclude_runner_tags_matching.append(res) @@ -142,43 +68,44 @@ def generate_target_test_child_pipeline( if exclude_runner_tags := os.getenv('EXCLUDE_RUNNER_TAGS'): for _tag in exclude_runner_tags.split(';'): if '*' not in _tag: - exclude_runner_tags_set.add(_tag) + exclude_runner_tags_set.add(frozenset(_tag.split(','))) else: if res := _process_match_group(_tag): exclude_runner_tags_matching.append(res) - target_test_jobs, extra_include_yml, no_env_marker_test_cases = get_target_test_jobs( - paths=paths, - apps=apps, - exclude_runner_tags=exclude_runner_tags_set, - exclude_runner_tags_matching=exclude_runner_tags_matching, - ) + flattened_cases = [] + additional_dict: dict[GroupKey, dict[str, t.Any]] = {} + for key, grouped_cases in cases.grouped_cases.items(): + # skip test cases with no runner tags + if frozenset(key.runner_tags) in exclude_runner_tags_set: + print(f'WARNING: excluding test cases with runner tags: {key.runner_tags}') + continue - known_no_env_marker_test_cases = set(known_warnings_dict.get('no_env_marker_test_cases', [])) - no_env_marker_test_cases_set = set(no_env_marker_test_cases) + print(f'INFO: key.runner tags: {key.runner_tags}') + if any(set(key.runner_tags) & group for group in exclude_runner_tags_matching): + print(f'WARNING: Excluding test cases with runner tags (wildcard match): {key.runner_tags}') + continue - no_env_marker_test_cases_fail = False - if no_env_marker_test_cases_set - known_no_env_marker_test_cases: - print('ERROR: NEW "no_env_marker_test_cases" detected:') - for case in no_env_marker_test_cases_set - known_no_env_marker_test_cases: - print(f' - {case}') - no_env_marker_test_cases_fail = True + flattened_cases.extend(grouped_cases) - print( - 'Please add at least one environment markers to the test cases listed above. ' - 'You may check all the env markers here: tools/ci/idf_pytest/constants.py' - ) + for case in grouped_cases: + for timeout_4h_marker in TIMEOUT_4H_MARKERS: + if timeout_4h_marker in case.env_markers: + if key not in additional_dict: + additional_dict[key] = { + 'extra_extends': [], + } - if no_env_marker_test_cases_fail: - raise SystemExit('Failed to generate target test child pipeline.') + if TIMEOUT_4H_TEMPLATE_NAME not in additional_dict[key]['extra_extends']: + additional_dict[key]['extra_extends'].append(TIMEOUT_4H_TEMPLATE_NAME) - dump_jobs_to_yaml( - target_test_jobs, + test_child_pipeline( output_filepath, - DEFAULT_TARGET_TEST_CHILD_PIPELINE_NAME, - extra_include_yml, + cases=GroupedPytestCases( + cases=flattened_cases, + additional_dict=additional_dict, + ), ) - print(f'Generate child pipeline yaml file {output_filepath} with {sum(j.parallel for j in target_test_jobs)} jobs') if __name__ == '__main__': @@ -186,35 +113,13 @@ if __name__ == '__main__': description='Generate Target Test Child Pipeline. Update Build Report in MR pipelines', formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - parser.add_argument( - '-p', - '--paths', - nargs='+', - default=DEFAULT_TEST_PATHS, - help='Paths to the apps to build.', - ) parser.add_argument( '-o', '--output', - default=DEFAULT_TARGET_TEST_CHILD_PIPELINE_FILEPATH, + default=TARGET_TEST_CHILD_PIPELINE_FILEPATH, help='Output child pipeline file path', ) - parser.add_argument( - '--app-info-filepattern', - default='list_job_*.txt', - help='glob pattern to specify the files that include built app info generated by ' - '`idf-build-apps --collect-app-info ...`. will not raise ValueError when binary ' - 'paths not exist in local file system if not listed recorded in the app info.', - ) args = parser.parse_args() - apps = [] - for f in glob.glob(args.app_info_filepattern): - apps.extend(import_apps_from_txt(f)) - - generate_target_test_child_pipeline( - paths=args.paths, - apps=apps, - output_filepath=args.output, - ) + main(args.output) diff --git a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml index 7f0f5ebd661..a145097f312 100644 --- a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml +++ b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml @@ -6,94 +6,87 @@ ######################## .dynamic_build_template: extends: - - .before_script:build - - .after_script:build:ccache-show-stats:upload-failed-job-logs + - .after_script:build image: $ESP_ENV_IMAGE + tags: [build, shiny] stage: build - timeout: 1 hour + timeout: 2 hour variables: # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings. IDF_CCACHE_ENABLE: "1" # Since the version 3.21 CMake passes source files and include dirs to ninja using absolute paths. # Needed for pytest junit reports. INSTALL_EXTRA_TOOLS: cmake + # Suppress Python warning messages during CI builds to keep logs clean and focused on errors. + # This prevents non-critical warnings (e.g., deprecation or resource warnings) from cluttering output. + PYTHONWARNINGS: "ignore" needs: - pipeline: $PARENT_PIPELINE_ID job: generate_build_child_pipeline + - pipeline: $PARENT_PIPELINE_ID + job: pipeline_variables artifacts: paths: - # The other artifacts patterns are defined under tools/ci/artifacts_handler.py + # The other artifacts patterns are defined under .idf_ci.toml # Now we're uploading/downloading the binary files from our internal storage server - # + # keep the log file to help debug - "**/build*/build_log.txt" # build spec files - build_summary_*.xml # list of built apps - - list_job_*.txt + - app_info_*.txt when: always expire_in: 1 week script: # CI specific options start from "--parallel-count xxx". could ignore when running locally - - run_cmd python tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py $APP_LIST_FILE + - run_cmd idf-ci build run --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1} - --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" - --modified-components ${MR_MODIFIED_COMPONENTS} - --junitxml "build_summary_${CI_JOB_NAME_SLUG}.xml" - - run_cmd python tools/ci/artifacts_handler.py upload --type size_reports .dynamic_target_test_template: - extends: - - .before_script:fetch:target_test image: $TARGET_TEST_ENV_IMAGE stage: target_test timeout: 1 hour variables: + GIT_STRATEGY: "fetch" SUBMODULES_TO_FETCH: "none" # set while generating the pipeline - PYTEST_NODES: "" - TARGET_SELECTOR: "" - ENV_MARKERS: "" + nodes: "" INSTALL_EXTRA_TOOLS: "xtensa-esp-elf-gdb riscv32-esp-elf-gdb openocd-esp32 esp-rom-elfs" - PYTEST_EXTRA_FLAGS: "--dev-passwd ${ETHERNET_TEST_PASSWORD} --dev-user ${ETHERNET_TEST_USER} --capture=fd --verbosity=0" + PYTEST_EXTRA_FLAGS: "--capture=fd --verbosity=0 --unity-test-report-mode merge" needs: - pipeline: $PARENT_PIPELINE_ID job: pipeline_variables - cache: - # Usually do not need submodule-cache in target_test - - key: pip-cache-${LATEST_GIT_TAG} - paths: - - .cache/pip - policy: pull + - pipeline: $PARENT_PIPELINE_ID + job: snapshot_known_failure_cases artifacts: paths: - XUNIT_RESULT*.xml - pytest-embedded/ -# Child pipeline reports won't be collected in the main one -# https://gitlab.com/groups/gitlab-org/-/epics/8205 -# reports: -# junit: XUNIT_RESULT.xml + reports: + junit: XUNIT_RESULT*.xml when: always expire_in: 1 week script: - # get known failure cases - - run_cmd python tools/ci/get_known_failure_cases_file.py + # Merge the current known failure cases file with snapshot_known_failure_cases file captured at pipeline start + # to avoid retry failures caused by later updates to the known failure cases file. + - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} + - printf '\n' >> ${KNOWN_FAILURE_CASES_FILE_NAME} + - cat artifacts/${KNOWN_FAILURE_CASES_FILE_NAME}.freeze >> ${KNOWN_FAILURE_CASES_FILE_NAME} # get runner env config file - retry_failed git clone $TEST_ENV_CONFIG_REPO - run_cmd python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs # CI specific options start from "--known-failure-cases-file xxx". could ignore when running locally - - run_cmd pytest ${PYTEST_NODES} - --target ${TARGET_SELECTOR} - -m ${ENV_MARKERS} - --pipeline-id $PARENT_PIPELINE_ID - --junitxml=XUNIT_RESULT_${CI_JOB_NAME_SLUG}.xml + - run_cmd pytest $nodes + --junitxml=XUNIT_RESULT_${CI_JOB_ID}.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1} ${PYTEST_EXTRA_FLAGS} after_script: - source tools/ci/utils.sh - - section_start "upload_junit_reports" - - run_cmd python tools/ci/artifacts_handler.py upload --type logs junit_reports - - section_end "upload_junit_reports" + - run_cmd idf-ci gitlab upload-artifacts --type junit + +.timeout_4h_template: + timeout: 4 hours diff --git a/tools/ci/dynamic_pipelines/templates/fast_pipeline.yml b/tools/ci/dynamic_pipelines/templates/fast_pipeline.yml deleted file mode 100644 index ea23b84abb3..00000000000 --- a/tools/ci/dynamic_pipelines/templates/fast_pipeline.yml +++ /dev/null @@ -1,16 +0,0 @@ -.generate_pytest_report_base: - stage: .post - tags: [build, shiny] - image: $ESP_ENV_IMAGE - artifacts: - paths: - - target_test_report.html - expire_in: 1 week - when: always - -fast_pipeline:pipeline_ended:always_failed: - when: on_success - extends: .generate_pytest_report_base - script: - - python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type target_test - - exit 30 diff --git a/tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml b/tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml deleted file mode 100644 index 1631f949f4f..00000000000 --- a/tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml +++ /dev/null @@ -1,18 +0,0 @@ -generate_pytest_report: - stage: .post - tags: [build, shiny] - image: $ESP_ENV_IMAGE - when: always - artifacts: - paths: - - target_test_report.html - - failed_cases.html - - skipped_cases.html - - succeeded_cases.html - expire_in: 1 week - when: always - - script: - - python tools/ci/get_known_failure_cases_file.py - - python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type target_test - - python tools/ci/previous_stage_job_status.py --stage target_test diff --git a/tools/ci/dynamic_pipelines/templates/known_generate_test_child_pipeline_warnings.yml b/tools/ci/dynamic_pipelines/templates/known_generate_test_child_pipeline_warnings.yml index 2d6e2178f56..ed78af49934 100644 --- a/tools/ci/dynamic_pipelines/templates/known_generate_test_child_pipeline_warnings.yml +++ b/tools/ci/dynamic_pipelines/templates/known_generate_test_child_pipeline_warnings.yml @@ -2,7 +2,11 @@ # no_env_marker_test_cases: List of test cases that do not have environment markers. # each item shall be the test node id, you may check the error message to get the node id. no_env_marker_test_cases: - - dummy_test_case + - test_examples_security_secure_boot + - test_examples_security_secure_boot_corrupt_app_sig + - test_examples_security_secure_boot_corrupt_bl_sig + - test_examples_security_secure_boot_key_combo + - test_examples_security_secure_boot_key_revoke # no_runner_tags: List of runner tags that has no test runner set. # each item shall be a comma separated list of runner tags. @@ -27,14 +31,14 @@ no_runner_tags: - esp32c61,generic - esp32c61,jtag - esp32h2,jtag - - esp32p4,esp32p4_eco4,flash_encryption - - esp32p4,esp32p4_eco4,lp_i2s - - esp32p4,esp32p4_eco4,ram_app - - esp32p4,esp32p4_eco4,sdcard - - esp32p4,esp32p4_eco4,sdcard_spimode - - esp32p4,esp32p4_eco4,twai_adapter - - esp32p4,esp32p4_eco4,usb_device - - esp32p4,esp32p4_eco4,usb_host_flash_disk + - esp32p4,esp32p4_rev1,flash_encryption + - esp32p4,esp32p4_rev1,lp_i2s + - esp32p4,esp32p4_rev1,ram_app + - esp32p4,esp32p4_rev1,sdcard + - esp32p4,esp32p4_rev1,sdcard_spimode + - esp32p4,esp32p4_rev1,twai_adapter + - esp32p4,esp32p4_rev1,usb_device + - esp32p4,esp32p4_rev1,usb_host_flash_disk - esp32p4,jtag - - esp32p4_2,generic_multi_device + - esp32p4_2,esp32p4_rev1,generic_multi_device - esp32s2,usb_host_flash_disk diff --git a/tools/ci/dynamic_pipelines/templates/report.template.html b/tools/ci/dynamic_pipelines/templates/report.template.html deleted file mode 100644 index cefa904b462..00000000000 --- a/tools/ci/dynamic_pipelines/templates/report.template.html +++ /dev/null @@ -1,132 +0,0 @@ - - - - - {{title}} - - - - - - - -
{{table}}
- - - - - - - - diff --git a/tools/ci/dynamic_pipelines/templates/retry-jobs.png b/tools/ci/dynamic_pipelines/templates/retry-jobs.png deleted file mode 100644 index a8a60112c01546ea8c8ff06fc5b8d30f83ed4bcd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 17552 zcmV*+Kr_FIP)PyA07*naRCr$PT?JrN$J75^+&x5q;BLXaxO;GlyOlz5DBdDPi@UoNN{bb5aa!CV zXhPgw@B3!ny?5^}7XyO)xotx(_x7#K?)-LUc6L@VWcwi{DAB)4;iK@ULiMHk7kw*b z;SUNxp%mWLd|mSk&Z6Y(VqSS&QvFix)D(4FbQi8_uDndC@mlYS{S*79UAC-V^+$ET zbUqf#R4CMR(@|}@?`HHsXV+EqmcJ){mDPH)noo;f@6?C%UJ~1&YuvQok@b(V3rYT> z0!q~bo`wnq6zPtw)_baw)MZKo3K~-Bd#OBi3$&Z5P`{v_-`v!+?JHha0t6L3H0F#PAT$WA$t;?WGY_>)C;;3>L^p`b z+M{MPp00?E)Jh+iWLdbn!CO@-1C^I5V`a6mK;0UV3FRQ5dMb(h+#yv)E>n}* zY4WKUIAdlM%1jTqbR&uIty0#fwBs%grfQa?(HGSytyGE~(Y0b~p}A1AssWj@sRn(s zoLh@vnF3B0R$=*qm8p~?XDpc-ptJpQ2GyXkHr|4073_+%K$|_ zBs$xuEp4cpmW~jTp7Nij=rq!S7FecvF6mRcRD^VZfj~jRB{?w223=tw(1^y$z(6%U zHEBjwD+Y@ygG|(A*3+6{Mg|6Ow6cJMr3GxvO<`haD6}r4iKbzs6pQEUp5I_!M+M=)QnGUU9Q(b$X z4(Wz$dF|B?9-UGO}=SDzC1d~smB2sCA zBpAuX3*j?M3o4t1YIRo(-Xa(kHCHYH7@VvuQMW`vd{e9-OpH_#fqq2v6wp)Pv!j3r z2*yT6Shj2xVq#;^woNO9hK1w7vo|nvFOCN<0+GZjwB};Y(vnIjR1DgDAgwh{2{@XY zqF%w=DBxrdQWx||PXRpz1PYLJY-?)+8ylO?h9n{&I5^nj;lsyRyl5!~4D5?irAlJc zrY#5v3PY5_21(X=kj&;V(Z)q!pjMz1gEkQ$Xr^4&mgruy0*bmie>NS~H$+c?&xHah zDJk&t3xKt?C2VbNKbK}@*p8T(SQr|c;MnEIIQg$PQd8yDewA^SfI$ujqA{^cwQ{T` z80sa1FJiFujr{l&NKQ_HkB<-X=g;@?D}Rp~qhb@V?ZhqIe&(mL8i<9dKtTiqap$Il zyEBG0r~zZW&EP%Au7CAwpn$jca}+96@Qdr}>ws{q`0Wb9qvBQmO)_5!0t8`;P6=Z} z0}QBJ73DnKzqn578?2|m$ELuS3J3{_$=Gu22L5~eRw9|W)2SG^alc?gP%4qv(GKHU z)YI=MeC(N}ukVYbz?TXLMC^-qUSaF88v@-`J_+oYG;req=?G(`0(sD{ZWS10?0xY? z4xYZzdJ24K3Vg|c@ZxPK=59N$B8mE3gks=uu2Y*InwI+V^c48w zDexr&LUe2bepq%?Wmr`CC9s6BexGnkKmoL=P#P^fOMLMI^`&e=;gG_WbG&F>{xfzU%YCJ8J_~$E;chUfRnWm z%6eF!d3ihJcQ(y>pJ!l?y+q!mzM+Yjb-)Kt{o_$4p9LD1wMNN2W-vBXz%M)r7aqmn-2EuTCZu9SGgmb8 zwBu(vkv>Ppx<%HaXk5r4UAx6WDX}Yd*rclg}GtoE0|ww5d33< z5t$T?gp@=OIG7k3!^+5l%gJly3S$G~3G7YigM3UwNrKw)kHg}nm|oZT`F^pTjF1mBnd1jU6SIw=-O zsmUzfn;DwH*2D_AEnVPXYNyj+d+tFb7XImn(s|8L&(jbVHW7$VjX?^8wiqakV5BsI zajGpIdYj_d&1jS_V2SUWxx&Ug>)@H`3?!US^n zDL+d{O2qSkHwcalMN(=C%#2OoYUd0OCwCZ;VaCV|6U3Su{}ySg6dT)UE*XS!p?2B_|`r`w^58VO(r#d_3ZwJV8=m0RL_5oD1f~ ziow84Sj7}Jb}%el95B+9S_t6&4Y~(f5y?ONZCD_DLi`bzn7|T(BlXGE1D2+O2uh$( z$i{ zg~o!$_8x%&&68x+Jc$TniHYWki=7kl@jNkNNz&7)1W5AGmGpq{A)zj9Sp#Mt(2T@J z5I2o$vyB54McoW0^1R2l9-#Ap>m-HPNg(j@hbW)9)&+=2_rHt8k^1>z$4!mh)YU|R)AUr0DftMJIK-S9C0*%VmM5{{mxGrmj6OLW9LUIZ)w_D!# znmKXF@wgHE0HlIZy7Kfk_FO#5W0ahn3_}A$SeTikPRYvXSiPx4(+W-{S-lezG5uAZ zCov52>rU>)n^1qmCu+`F*1gexjia5m40VHxOMXmo;-PiPMtfv7Y)Rh zzb*-3pUQTi4sxp)uy&6M5CoM$mxS;-+7A!Hp7Dyl;P6H~^nIQ#@#~eY%7blWV2CQt zW#M2Z=MY0E#sQ;R_TrVEQZ3gCurd~Ox|dNtcohDE zLFMA38(4d4Z@O*qs@e#xE7fO7fpp)}4n+jxUP3&Q@BGVPkQx_k#=MJ5KiO!5D}7YIUTDvL5m7?nMng)=f~Q=B5>e`qHwXz*xo|c z^Yn~!Z=(HhKkOOP^xCsKSaE!3y7dzvc~x!9i$E@OM|e6Gmkg$FqWy6{>?x9xldw?aSO?`S5>c_n-GU3N4Bukri-S& z;em)tPQb)=16j2r`wlU)XdD#h$(egsu;uK*G~?@L>kNAf8$9syX5Dpkt3D{3uNd-L zxuKMO;WTyX0pWvST)Kty|9PL_*P;s4%G-|p*X~hoJCe#!A*~JJq|Zg(O&GszKK#Oi zxdIYlNk=4=p;N`CSk!(3hV7n--)@~mQ_ouHR=X7&jH=`+%lV&2`rt`~FwvI$xfwTJ zJcOx{G0ct45gr!_2TNOQ?l}jsiE-$%VKf}9Y_X)zB+hScXofoOf^q4z?`76U-vs;N zrvs}HpO^@1GfPAz#4xiDtJ4hwYIQ=ljiYhL_c8j`Z;#p~D)H_ENg6bHQ=UD7GBk+G z^Byz^i6OrHx!}$}VCdw;l0!;ZD4us|$={nexTDZo0W=i$PB1JY?8H0?e~IT&Z*b}H z4Xi)4pTW-B)Dq!wQLr>I$Fvs1QLAu8G+#b|zjL&*#o|7bctx$9t2+9u^2UsgdGRXc zKi)PSwP`j&q9d4DsGP|7Xx6(&)a!|!RokG;h9B@K;5i01?1XAXJ>hJe8`Vsq1ToPxbor_x_{FO4a?SK`%E<#Pgtv@H*ua!+OZHmp@H0XV?$#^#mAtqvj;Zx z_!&7O7bdZv zeHb&|i~pAlrtez;b7NC1?>q$uZk|Ljmjd{{eoxFfz8ae^A7sG#wstEt_N>M4xU#+MRgRp$&O5@N>&*wUKZAPL4D8YG4mUUpOle42CGa|9AczYiNv@#)1(n z(mWQ?X7MD3_-=zN_0EGmb3BwyTGcn4E~-^}Il;nX9Qg zCT>hZm+DUFRYw4XvuDoZ$rHi&D^Fw#c>MSY?Ck6?e!^&2k_x0sC*E9On)-(aVZ!!> zJV91>oq=Qb{{gM)qZ{_Yk~5pJ?!rE{_q4Ct7;UOF;J0hIl$VUh;yf9+`bWekC1H5o zZ&5aPG3>u_0+X8!K}d8chVGsT|43m`sam)UhBfQX?^8M2DI*CXI5-5`x9>usf(7vM zF=$z_4m#Cn4o6cv zRL)g~zkdwpD4$8&z#ZK&;5 zi9et5y#g|Fk|qD=1FLxz>s+xJ7Im1&3^!`uZ0!11@X;oK>DQnGY8I~mBZVPqiV}kN zV<h%DHZTDI;43}Trk?7)L>xlV1$MMQqD_3>&UG0sj?epS| z@;~;&7`t^I0wO{=e~+rI@k_gLEYc0zJ%a&{5*cg)SNXzdArrQ9yL!Q}kO+5`yr5yDGW?fwt9x20h$u}d;wG%8*Lb&6ER zyua45WFSl!1%B)>1chCNZK+~*f>oaU25y8r;CJu7e!|8{`b?_F4-I-_>(xVe86up| zCbgz@(_5arAigx=_#mi@ZUBmtazdrj~iP%5$Y{2`paOIA*>#XKcd9Y8*}LoWQ#sxO>v?AiIHu6t#e zswSOi$fNgey?lg4e{7Q4LJ~o;tQ@i*&{i3>C6A>GN;wn}oN1DikbLv1aQRDeGJNL! zjOd%!bxt6wautwg{EtwWmoWdW=| z8IC{47l*xN#vY5LZ=0E!>2&Aziw(r>kjGr*{WniB@6h^mX-d*TgEG~*99Ih`ly#t8 z2Epx>yiae3JYssRIJN`Vy@g#Q?Qx;++Ei`;+6AI=$PQtr!xtelKrk^e0mU~=Nlt-@ zso*r06+lQADFA#;fY9%!y|VbrD%a(Ht9D8@1h1-Pb*c<<=bO;z2pqV40*~IlKv;AH zGa!kTPBspxUepuKE7V~_l)I%1$~qL)puK-wFz$vtW__AO{lnK!6cUNE9skw_iTOpZ}eOPe=e_6XRiOWCB-vCp0Ks z4RuOY;XLFrOV)6?qg1NDFH+yUW&XTa3Z1Qnf2 z!N%AMm!C$nlrW*aJrhN5kERC*>(^~!qoA%+iAhOG z=+n0sY;1HpJ7ok0^z5qwgjsu70@k?OkpMw7B6w9Ll|i?Y2#J&=T2i8c4~+_ErY7$^ z0sw1sOW0Z1@SZv1&acPnnz=)-8X|Duu3I5ToEJFUOWMUg{c`! z1!A8{+85??8U%K9%z4tT*N93;L6>?8B$)Vc6R95rk_2$n7>czo?~^HP;rH4N2d{-= zNw0j!?V#JGI&T!{7%z zzPjd#m6;{%EroePwjr7)v?Z)7r3VNJ2??Mm!nzWHgtfIW_V1FuDnOXITYaabXjJUG zQjicxhxV#c4lOFCH7bgPrNMp>?v425c%8loB*xO-G$Fn&w`CDbo=3fgPjmnq2X!SX z$H~kN#5|&Ys-2HO;ZeA^Bq``BhuTuaHowk^%lVU&kox*LQo}-k6pcHt)Wy*)RrZch zcoc*}%WYZAM4oKsb&F4R_dmU z0)pGJ?;97aI2M8tP2JGBhQkMoa7FEZ5Jogj|7-`12e~biqgQ%NC(aY1OM2r=V^hYV zn50r>2N2$c(X;vv0RrtlSbYVDoBGWp2|*P?P4!FBcjXT0w4+zmi~)h(jZKb2Kx_yC z<3kaZ7|k{Tn(SmNa4~mcmt`{CWtNDc$ytdVVv^%n?=<66iq@=~I-1$@&Okif=cpH|8 z3y-7t$HK$_V_LbPR>tKd1vweU0Tn8U~i}4n~+{P&!0a>!GfP( z7Bj8I4TJ#serYa0rEPUplm)%R4HVMT$!9(c-OXkZtr<}zkso^G4kYbXPZM#r-Xz= zyn6W>9{KXZ%q;Vm9?ze@K*55Vhuw0zjoB$*WPXu$y!)m4p^uZ@VgLXj07*naRC4;w zPyvGIsYvpI2naMOy{eVdO9}X8mV*K9)I>)|Bfvip35iJ`RYd+cz17$9=_sIN z=_*$)XV}@<@@M)XK9I_ns_kWKv`W}5~3Xs$p85xB;x9_2Fk%Dk?a^#pH1Pc9SMnyqL z5CK6YmwHw8)Jq5-dPH=TM<76;K)*>npunCarnJu!3AARY-8xNY@1*~Py+3%lZ2oLgq#`>whyW#1CkENfJo|s zudhEmKl1ikPWDk>_UB0fKR8WYKWK)VZiK*(9T>q{3XAOXUd@mRGw9YE;4bgSgL%qoLkLddcC zqYDs7Lh$zs)Jq6CMs^X4ts+S;IAy}I0D0t?aMg#J!VC5IG+aNng$ zMCdsuTP$z$_)z(L$|IyP`U(Ic!_z8YP-b;kB>JF#-rGN}*pOI9g=aBwgN4;qH`8`h=?4twa(VUBxH zzkWTbU1DFq?bZ`BXHG-@0tH}bsCIiMI%VY|*BqP%Qqj~FIu8*Z9-iaXk*A`Cg(V-Z zlK_FEE{pa9l~GZY5cGhM<6|OC+U!aQKmIrkVWFXzF>@L&U%twNIQyr$Fg7;Ao_)J; z`SKO)+p`~T?yeX=ek@GQ%rJGzbflyv!`jLQ!-fq;WK!D2jl{1m^;pM#^vjxjKKfHiTAH^w#d2 z8l3l$5T;C?#@C`ZXiyK0n>51kVc(-zv7-2K$|S5>xdz3G7sJL4o6xOW7u>pa8!<64 zXxXv_CQY1zOKf*da<{&#l3s?v3eVao>h){9c=3Wmo$cJY8_QQL!Pqeq@XP$U32D;=(YnP$- zk}XoyWj!F|>~OpbAkaHEZrp^0g#{ubBk}8!<>=Vai&X{}R~KGo$B&x;7gskB6L#$2 zg(F9gqgk^i=+dPVI(PmS)2B~C`*xkss&z}`&6|fMf$rVAac4=|2nh*cKv=lw7q(F} zY1AAYI(qTSPp+f{09Syv55(e{Babk*RH_C36mv27&dGOs~6vm7>4lh2vn_78NVGo zgdW|yA+JYX4D3GyTefXV7Z8pgJAtm8;ykv7qf&A9ua|m z&R;;~s+F)~`)=&mxdn6Q{DSJ$tMNpj5b?FYsf}}I&!c;fZfs~GKv=S50o-%Dqjt@D z=-sy`UwmS2Va|tQ*00@&jhohC@Zg~sI&27~VEWXV@GS4i?y?&CeTMQC%5xk566r^c9K*l+59o^{e;$*lSHD4BjxEs6s|~!p zpW~muFYt{?(NWQ8+N?3oo%>YsGS=JNyuxrCqxgdBtdq3Cx7gzPr#(xuC6V@OF(M$K9^Fn__kEJXmyZavGF zuf)pbtFiyLJ;;~O10f;8B{P(q935d{sXKUTcI%=11Uhfu`bn?)`}SFsOSVXNJL)BbcbTeLf9GQX0@Xvt!`PTuxVpLKm^GbL z29C0A_3@2!ZwtdGQ}&zTt@v~S-YBS(&8V{Go+xiN6yK&)7?0%OLE!M%I; zSmNV5DOB6dBw{(J_C^XJdw`0?Ysi}2uq zaAksaYSE$vcJAECHUtI)0YxM*QS@3YNC@00bd03jL7COBaq985-%&GJH`Z`TmW~3Lwz7+Rg37Q*@KEIQ48j`<4M3<-0j;Wjc^2C*Wg_SS z;Y%5*oNs4lfKaGVAvPR7eE3jmOtos&kT-8$1O)|wJ_iH@uz~UV_3LamaC38Gl_8^~ zVN?`vZjwm}ie8I03$Dvb2sQ*gAmrF=&;vq_k@kx#I5R-VPKSQ_=_mB=-5Yi4)Pc9R zw;m9(L%dI!R}ToElKP)l-5(MVNHU;AHigWRUP8!>LLc$D9uPhvu|M;wJ|rMyr>h!( z!1pq!T$aVHN4?6B9df>3-t++B{=G-~ARq5X+Ar?y6UUEX@{f}~db`sD1U4wr79<0L z>bzqII?GV);~&ZV=v4Vc73w8~Pel07u-eZ;LYTf=xFb-hx|4xGq35DaQV6vUHOr$_ zRXrf&)cnZ`Ac&#Tq%-L`)fj!bpOgY;&z?n-CXGLO8@~JQJCG5Qw;g3F14{_>pB@lC z-k4+t2=qKEDh6?J2^spMf4mm!Ysx7KkSF1|abs}cz`hLq20mo{dGh4Jx^?SOqek@% z0AZn8Lh!0y4lSza?`8OqgCH+U4G83^n8Lp36uvAtD3m=Flai7V8XBPogPa?z#3 zeG^TZG)J8}b>WfM10_n9$nXk#S63%)jCR!Jw16O28Q3vNZ&ds+!>GHI%v6T-03kt{ zfY&L0h*ie(AvAfSO(SC%Be$UoYz?e)vibV5KQaY;-}vCKlPCGyOO>isIPeP{Vmf;C z2%0o$iYHH=;M&z|FfcSg<3^2;J5L^+EzB4Y1nYqqTTze@_{@zG9coq3?=s}zpuYJiQ-J{9Yk0k&KS1Ff}%TouxGfHtLMRuKCzX>~5F~&IS%2 ztxsY_fq{Ye?YG}hrAif4s8B(pelnn)Idg`O?LK+@ghgBm=9*EWNl*6`P;e0KM4~gf z=#oFbb|M;6Al1T!3+Gs1n2$l+BYz%Y$)IVorkVDGPR$-acARYqEn2qtP=Ek*sHImK za&oY<0|;a@@J@b(7-bxiQj)Rq_)h%y{Jw6H(9Q`XTlRvxeJ+^57{!bW>XZ_wesVq$ zFIl0#=|?n5(N>t9tsTcq6!U6d07DHBpNWm3@@CDP#fNKJw`mPmS68k>ycC8Qd(*~E z==N)ys>+i$t!h9qH7U9^#n{Z3 zFP}!y8A=G0Q|7HGlMwWPkdt5_pSbB=)FQVbL@Q(QEcq2T>C)qySo7CzE$e}1Ynyu3 zMz`86`Af1VXiEvS;_lwPo0*5i&^B$_VD+lipyL2KyF2F25rVN!n?4OU{=0#& z@GuU;M&fDFVntE2W=-A|Idt$Kx^(R-t@4L|Ka7nVHljd*0_fAP55Gg_^NAVz_U((@ zd2(aPq9qtKWDq*G@5q2a(VW+lP0Eb`U0}rs&tdANP#_p~?S;p?~!75o*`2&5T9X^|fo(;^oVis8_EZ z*Y~;H;H2Hbix)29*|TTp+N~>(hnSWuUV>-OyfJX#0F>2=BS||}J9q3v(V|6Aqh`%? z0pXdqH^)sdF)_wZ8!rm^L_pB13^@$}n&oFlLI_F?fp1D67cl?OI^2Dec5A9a-jc{` z?}Fd&oQ0#64HoyA%=s-9<|tyM8R&*0>r9$BiOUWMT;=QEMBw-Lx&B+ zr3)9~M>l@r_%wrb_wHRBJ$jT+_|2F;1H*<5;7?PLi)AQ93j<bSAvFm%{Z{5VPQ{3kBjD`pj8msh;r6ZDyd&cC#s^cUO~r`eBP2lBwPzQG4j#%<2VELN zfYYI42fTRxf^UH$TLYc(zj*N?#|+d?A|SME*%BB2`3Hjr4`#`yQ^!tR*UafNv2gK1 zczAf=8slnW&Njnz7Co}+)nhr*fs;#m0 z${~zu&>Q=29LLE0b9nc`$iM&_ht1*qWJf4zTv($!1PDLR`U$@*m@gQs8aKs;O&d7i z1~EJt`S$PK2S+C-(3NOQ7A;24-o4=H=*Z>JB&Ew?D0t__O&c|$^~wQ(>;+>+kHOY$ zTiF9|=8TzX0Ac@c`&nX``^#KbGe|O^`giQu!Q!mQ=+~}Y!{{+%(gg&sc3!M&lUhM- zXxpYOmM&e23F9YV#>^RPyxhNkKddaR(6~us?l)a{LwP7DDXB@HV>)yC4BxZSy+?Nq z8_3dMm+~!JefsuA{`{JD0_xkgZQIx{fyOUOfS~P|q?Zsr?sUkGgbDz~`>8n<) zLV^4R@XY%eOB=(74`;vXg6%skat+J3<%_dpdleFkR5K)?=s}*Jjf0pBq$T{DA}70iGdNJ7{Bcoq$rhq zd1?7PC2+_02@+ETMj<8~+^93E7c0+y7dFZVD~0AIDO5M<(R5!UCHk8fmTrY|aByHz zn!clGzI4w5t?uUL=F$p9TYa=b5pyqFx(t7x|68Kf;Gu)r;7DaqTc{rLu%p$U5|u~$ zZg2^hu)>XsSPyoNorWVdNuw&r><+#T0v=bC3T6U9_pL=D)by3_G}kUV0-w`@3678Wd_emI(70C zA|oQux^0^@Te?|R87RJDuZ8j;tGc%xU%!5Xix)4UYuC;hwv8s#-Mja2{o21gNTo}c z;+uoWk6NEHzqv`Q{HYPE!gsTph(Rw!unVec%Y z)o1&*?ROc5|?($G9Zx3AOHfd8Stu69&M_tk3SQt5}N;e{XZ;QwgNkM zZIf0P();KLr99Cs#XpQ1i!x*NnC~P?d*AV{?v&i0Fp2= zI@I&2+xWiq(>y2FwSZe;7}ace*o(@uEpwAYY`t8 zkFn!NqhFr^7(8Sk>>V8N%e)0Ra^!dP?%f~Ns#QU!&K+^)%vr2hwhA+6P2&}`Ql*N> ztHX=ybD19R0tnQ0a$Sy2jX}6FQt(_o3`QK|Ng^)1ADN3{YIQU zeGXpj+hWR()7cOusv3t5@5g}tLoj~)XjHFJou!wF6DHxo{YNl1F~;&0OX2DH`Q9rb z0)lvR6N%z9$o_tTnI3EdPd==m)>uTVO9@)RbFn~c8w zdLwskckJD}AH|Co!_;X%;_TUTm@sY<8-x1w?~4Kb2BMs2S#0026?10)k_HeaPnn3m zz58R-=#eN`umJx1_doRR(+gcXcb5Qx>brX78Y)$;i1A}5p;zzj?9O`b?0K|p*ZPy( zN}SQ;(*^`Wd$OvFfIw1KXlUd|i=gwhGs@S{e@jbq4v<30-!A}>QIT+SbH#(_L0Gx_ zf?7fl<0|%Apt`69Kyq9>>_cB*!R#4q6By8cFmB$sh4vlV!Q1;ecI@1WD_5>!{J2R7 z4-4n-r%atJZQsy4$BrJy(xofdN4{g{4j46R1aB+RR#Jm{jahA&Hhl^j)NhQAojPF1 zkU?ny;mkR#S-TRKFJHy8eY73$r6LVpC9t&_rU5k zE7@K4r&+UE{cv{|;?;jHNgEJof1T_fZ{EB`nKGqdWMs$&(#XhYBqyhT9oOg5AALJM zUkZ?Efc!E=e7S)DE6-uCM-O4JJK*_6> z_EA;JOX6;NTOs1^!G}0JLc0u9UwV2HJ=5Otpm#DmUi+zrlG-b(3`ASnHhTH;H9S1> z!ok7*Q#DIp@BcUj;^Pz8CBVZY50X+1v2yoC_y$I30s`Ods09cnMn)Ljz9EX|b^RYF z(WmL2xa}%#=@NK_hK4bEll{ZSMj!R)(~QjjZmk3c*RS6|Y)lkNmnnsdcl~kfp9e_g zO)HuGfc+2_$Rq^S9gk%TD8;(*%dkhq)Q9Nk82(KXM1}znK6Io02g>t(>xbMQ(QXtN z6og#P&Yu*EW&`X!eiN~Y1{&uVge?Zuy$rn< zZk0S0DJ++K5>_deA8l)T!rj4EzhZsBVV6q@a!El9oFe`fjgXR!=L5Fkf3YZvD5QM~ zFem#-wx9g@J#w;&Y?V!CQ@VEP&bxYa_X64QX#0Qa)LF2}l^;oFdEg%$sTq2mEs1K? zK_LeOax73;7`Y(^3I$y4ZBVaVQIsu|m#iopHvj+xGD$>1R6py}hoOMB>L52Zibh#E zAbglw^`(4B3eYas=i(VFgt7aXPj~*hbP)$-O-xF`8~<=@+;t4W(Fz!IFka2mje=;P zvLlenAOHe~U1sMb=?XE`2l1JexfwhQTkAKKP%}QBW zu06{cg<3@c!~;A0+N!}p_)bY@JVX^PD-HZA^Qh$9PuO=d1Q#fv8JFNwHHSj0jTrtt zMvVF%r9G?Q*{dMje(Z~DZr?# zO>*5xU_RMU>R@bcp~m<02bC!MQu{qQ5v1DHnKYWO(YE)TR$>)e&C^+GHbE@QEz4H9 z;>d_J&8QY#Kp=FK);TUg-fx+HoVHZqq5NOj7wJ5s%TYxYT-FvTrhvLcsbndeswP!l zxrJZMtA0lmmepv+!SMA zvrs&>MF?-NS*k(|M5KlhRlC&@io{21kmOHdJ+k~l>FUlgAV^*WG8soGKn(ztPeVdc zfrtnua%qL~>H-9TZ`A-n6$o3B!6x9FRnPbve8wt8%EBF%82qu*mC` z7o}2RG?-b)tDY3%yQ(jFth6pa(_$b^=_2jwo<^MzXGc1Wn5iK!BXHnJD7HaF0-3ZT zpWsqC>8;SJsV#x3#1R=YdKRAwQkKx%jFQ+)VczMIsMK$ZO%){pja5yn5;0Hub?cA@ zr21tvQ*(>Z7x8Zi&ox(JsXlscAY(+0zG*X|sH3ZCpyp8-7)U@M2QT5dw5F@F6CkM9 rc{w0xf`J-ebkrl+4pboW|MmYLmH32>W2e0<00000NkvXXu0mjf68md= diff --git a/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml b/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml index 08b25bc2d9b..1b6ed573982 100644 --- a/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml +++ b/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml @@ -1,20 +1,39 @@ -generate_pytest_build_report: +all_build_finished: + stage: assign_test + tags: [fast_run, shiny] + image: $ESP_ENV_IMAGE + when: always + # this job is used to check if all build jobs are finished + # because the `needs` make the later jobs run even if the previous stage are not finished + # and there's no `needs: stage` for now in gitlab + # https://gitlab.com/gitlab-org/gitlab/-/issues/220758 + artifacts: + untracked: true + expire_in: 1 week + when: always + before_script: [] + dependencies: [] + script: + - echo "all test jobs finished" + +generate_presigned_json: stage: assign_test image: $ESP_ENV_IMAGE tags: - build - shiny when: always + needs: + - all_build_finished + - pipeline: $PARENT_PIPELINE_ID + job: pipeline_variables artifacts: paths: - - build_report.html - - test_related_apps_download_urls.yml + - presigned.json expire_in: 1 week when: always - script: - - python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type build - - python tools/ci/previous_stage_job_status.py --stage build + - idf-ci gitlab generate-presigned-json --output presigned.json generate_pytest_child_pipeline: # finally, we can get some use out of the default behavior that downloads all artifacts from the previous stage @@ -23,6 +42,13 @@ generate_pytest_child_pipeline: tags: - build - shiny + needs: + - job: build_test_related_apps + optional: true + - job: build_non_test_related_apps # make sure all build jobs are passed + optional: true + - pipeline: $PARENT_PIPELINE_ID + job: pipeline_variables artifacts: paths: - target_test_child_pipeline.yml @@ -35,7 +61,6 @@ Pytest Target Test Jobs: - generate_pytest_child_pipeline variables: PARENT_PIPELINE_ID: $PARENT_PIPELINE_ID - REPORT_EXIT_CODE: $REPORT_EXIT_CODE # https://gitlab.com/gitlab-org/gitlab/-/issues/214340 inherit: variables: false diff --git a/tools/ci/dynamic_pipelines/tests/__init__.py b/tools/ci/dynamic_pipelines/tests/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/__init__.py b/tools/ci/dynamic_pipelines/tests/test_report_generator/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/XUNIT_REPORT.xml b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/XUNIT_REPORT.xml deleted file mode 100644 index 0334a681550..00000000000 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/XUNIT_REPORT.xml +++ /dev/null @@ -1,200 +0,0 @@ - - - - - conftest.py:74: in case_tester - yield CaseTester(dut, **kwargs) -tools/ci/idf_unity_tester.py:202: in __init__ - self._manager = Manager() -/usr/lib/python3.9/multiprocessing/context.py:57: in Manager - m.start() -/usr/lib/python3.9/multiprocessing/managers.py:557: in start - self._address = reader.recv() -/usr/lib/python3.9/multiprocessing/connection.py:255: in recv - buf = self._recv_bytes() -/usr/lib/python3.9/multiprocessing/connection.py:419: in _recv_bytes - buf = self._recv(4) -/usr/lib/python3.9/multiprocessing/connection.py:388: in _recv - raise EOFError -E EOFError - - - conftest.py:74: in case_tester - yield CaseTester(dut, **kwargs) -tools/ci/idf_unity_tester.py:202: in __init__ - self._manager = Manager() -/usr/lib/python3.9/multiprocessing/context.py:57: in Manager - m.start() -/usr/lib/python3.9/multiprocessing/managers.py:557: in start - self._address = reader.recv() -/usr/lib/python3.9/multiprocessing/connection.py:255: in recv - buf = self._recv_bytes() -/usr/lib/python3.9/multiprocessing/connection.py:419: in _recv_bytes - buf = self._recv(4) -/usr/lib/python3.9/multiprocessing/connection.py:388: in _recv - raise EOFError -E EOFError - - - - - - - /root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/plugin.py:1272: in pytest_runtest_call - self._raise_dut_failed_cases_if_exists(duts) # type: ignore -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/plugin.py:1207: in _raise_dut_failed_cases_if_exists - raise AssertionError('Unity test failed') -E AssertionError: Unity test failed - - - - - - /builds/espressif/esp-idf/tools/test_build_system/test_common.py:134: Linux does not support executing .exe files - - - - - - - - - - - /root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:76: in wrapper - index = func(self, pattern, *args, **kwargs) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:153: in expect_exact - return self.pexpect_proc.expect_exact(pattern, **kwargs) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/spawnbase.py:432: in expect_exact - return exp.expect_loop(timeout) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/expect.py:181: in expect_loop - return self.timeout(e) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/expect.py:144: in timeout - raise exc -E pexpect.exceptions.TIMEOUT: <pytest_embedded.log.PexpectProcess object at 0x7f47b1ab46a0> -E searcher: searcher_string: -E 0: b'Press ENTER to see the list of tests' -E <pytest_embedded.log.PexpectProcess object at 0x7f47b1ab46a0> -E searcher: searcher_string: -E 0: b'Press ENTER to see the list of tests' - -The above exception was the direct cause of the following exception: -components/esp_timer/test_apps/pytest_esp_timer_ut.py:24: in test_esp_timer - dut.run_all_single_board_cases(timeout=120) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded_idf/unity_tester.py:435: in run_all_single_board_cases - for case in self.test_menu: -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded_idf/unity_tester.py:243: in test_menu - self._test_menu = self._parse_test_menu() -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded_idf/unity_tester.py:124: in _parse_test_menu - self.expect_exact(ready_line) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:83: in wrapper - raise e.__class__(debug_str) from e -E pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" -E Bytes in current buffer (color code eliminated): ce710,len:0x2afc entry 0x403cc710 -E Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_esp_timer/dut.txt - - - /root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:76: in wrapper - index = func(self, pattern, *args, **kwargs) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:131: in expect - return self.pexpect_proc.expect(pattern, **kwargs) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/spawnbase.py:354: in expect - return self.expect_list(compiled_pattern_list, -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/spawnbase.py:383: in expect_list - return exp.expect_loop(timeout) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/expect.py:181: in expect_loop - return self.timeout(e) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/expect.py:144: in timeout - raise exc -E pexpect.exceptions.TIMEOUT: <pytest_embedded.log.PexpectProcess object at 0x7f47af653b80> -E searcher: searcher_re: -E 0: re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?P<result>OK|FAIL)') -E <pytest_embedded.log.PexpectProcess object at 0x7f47af653b80> -E searcher: searcher_re: -E 0: re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?P<result>OK|FAIL)') - -The above exception was the direct cause of the following exception: -components/wear_levelling/test_apps/pytest_wear_levelling.py:18: in test_wear_levelling - dut.expect_unity_test_output() -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:180: in expect_unity_test_output - self.expect(UNITY_SUMMARY_LINE_REGEX, timeout=timeout) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:83: in wrapper - raise e.__class__(debug_str) from e -E pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?P<result>OK|FAIL)', re.MULTILINE)" -E Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 6673 bytes) -E Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.512safe.test_wear_levelling/dut.txt - - - /root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:76: in wrapper - index = func(self, pattern, *args, **kwargs) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:131: in expect - return self.pexpect_proc.expect(pattern, **kwargs) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/spawnbase.py:354: in expect - return self.expect_list(compiled_pattern_list, -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/spawnbase.py:383: in expect_list - return exp.expect_loop(timeout) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/expect.py:181: in expect_loop - return self.timeout(e) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/expect.py:144: in timeout - raise exc -E pexpect.exceptions.TIMEOUT: <pytest_embedded.log.PexpectProcess object at 0x7f47afc540d0> -E searcher: searcher_re: -E 0: re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?P<result>OK|FAIL)') -E <pytest_embedded.log.PexpectProcess object at 0x7f47afc540d0> -E searcher: searcher_re: -E 0: re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?P<result>OK|FAIL)') - -The above exception was the direct cause of the following exception: -components/wear_levelling/test_apps/pytest_wear_levelling.py:18: in test_wear_levelling - dut.expect_unity_test_output() -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:180: in expect_unity_test_output - self.expect(UNITY_SUMMARY_LINE_REGEX, timeout=timeout) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:83: in wrapper - raise e.__class__(debug_str) from e -E pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?P<result>OK|FAIL)', re.MULTILINE)" -E Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 24528 bytes) -E Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_wear_levelling/dut.txt - - - /root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:76: in wrapper - index = func(self, pattern, *args, **kwargs) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:153: in expect_exact - return self.pexpect_proc.expect_exact(pattern, **kwargs) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/spawnbase.py:432: in expect_exact - return exp.expect_loop(timeout) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/expect.py:181: in expect_loop - return self.timeout(e) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/expect.py:144: in timeout - raise exc -E pexpect.exceptions.TIMEOUT: <pytest_embedded.log.PexpectProcess object at 0x7f47afc54df0> -E searcher: searcher_string: -E 0: b'Press ENTER to see the list of tests' -E <pytest_embedded.log.PexpectProcess object at 0x7f47afc54df0> -E searcher: searcher_string: -E 0: b'Press ENTER to see the list of tests' - -The above exception was the direct cause of the following exception: -components/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.py:17: in test_wpa_supplicant_ut - dut.run_all_single_board_cases() -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded_idf/unity_tester.py:435: in run_all_single_board_cases - for case in self.test_menu: -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded_idf/unity_tester.py:243: in test_menu - self._test_menu = self._parse_test_menu() -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded_idf/unity_tester.py:124: in _parse_test_menu - self.expect_exact(ready_line) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:83: in wrapper - raise e.__class__(debug_str) from e -E pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" -E Bytes in current buffer (color code eliminated): 0 d4 000 00x0000 x0000x00 000000 0 -E Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.default.test_wpa_supplicant_ut/dut.txt - - - diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_job_report.html b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_job_report.html deleted file mode 100644 index 3c35a534fdd..00000000000 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_job_report.html +++ /dev/null @@ -1,170 +0,0 @@ - - - - - Job Report - - - - - - - -

Failed Jobs (Excludes "integration_test" and "target_test" jobs)

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Job NameFailure ReasonFailure LogFailures across all other branches (10 latest jobs)URLCI Dashboard URL
build_clang_test_apps_esp32h2Some Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure LogSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure Reason2 / 10linklink
build_template_appSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure LogSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure Reason3 / 10linklink
check_public_headersSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure Log4 / 10linklink
- - - - - - - - diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html deleted file mode 100644 index dc63e261523..00000000000 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html +++ /dev/null @@ -1,335 +0,0 @@ - - - - - Test Report - - - - - - - -

Failed Test Cases on Other branches (Excludes Known Failure Cases)

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Test CaseTest Script File PathFailure ReasonCases that failed in other branches as well (40 latest testcases)Dut Log URLJob URLGrafana URL
('esp32h2', 'esp32h2').('defaults', 'defaults').test_i2c_multi_devicecomponents/driver/test_apps/i2c_test_apps/pytest_i2c.pyfailed on setup with "EOFError"0 / 40linkNot foundlink
esp32c3.release.test_esp_timercomponents/esp_timer/test_apps/pytest_esp_timer_ut.pypexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): ce710,len:0x2afc entry 0x403cc710 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_esp_timer/dut.txt0 / 40linkNot foundlink
esp32c3.default.test_wpa_supplicant_utcomponents/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.pypexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): 0 d4 000 00x0000 x0000x00 000000 0 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.default.test_wpa_supplicant_ut/dut.txt0 / 40linkNot foundlink
('esp32h2', 'esp32h2').('default', 'default').test_i2s_multi_devcomponents/driver/test_apps/i2s_test_apps/i2s_multi_dev/pytest_i2s_multi_dev.pyfailed on setup with "EOFError"3 / 40linkNot foundlink
esp32c2.default.test_wpa_supplicant_utcomponents/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.pyAssertionError: Unity test failed3 / 40linkNot foundlink
esp32c3.512safe.test_wear_levellingcomponents/wear_levelling/test_apps/pytest_wear_levelling.pypexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?POK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 6673 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.512safe.test_wear_levelling/dut.txt3 / 40linkNot foundlink
esp32c3.release.test_wear_levellingcomponents/wear_levelling/test_apps/pytest_wear_levelling.pypexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?POK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 24528 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_wear_levelling/dut.txt3 / 40linkNot foundlink

Known Failure Cases

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Test CaseTest Script File PathFailure ReasonJob URLGrafana URL
esp32c2.default.test_wpa_supplicant_utcomponents/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.pyAssertionError: Unity test failedNot foundlink
esp32c3.release.test_esp_timercomponents/esp_timer/test_apps/pytest_esp_timer_ut.pypexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): ce710,len:0x2afc entry 0x403cc710 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_esp_timer/dut.txtNot foundlink
esp32c3.512safe.test_wear_levellingcomponents/wear_levelling/test_apps/pytest_wear_levelling.pypexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?POK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 6673 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.512safe.test_wear_levelling/dut.txtNot foundlink
esp32c3.default.test_wpa_supplicant_utcomponents/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.pypexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): 0 d4 000 00x0000 x0000x00 000000 0 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.default.test_wpa_supplicant_ut/dut.txtNot foundlink

Skipped Test Cases

- - - - - - - - - - - - - - - - -
Test CaseTest Script File PathSkipped ReasonGrafana URL
test_python_interpreter_wintest_common.pyLinux does not support executing .exe fileslink

Succeeded Test Cases

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Test CaseTest Script File PathJob URLGrafana URL
esp32c2.default.test_vfs_defaultcomponents/vfs/test_apps/pytest_vfs.pyNot foundlink
esp32c2.iram.test_vfs_defaultcomponents/vfs/test_apps/pytest_vfs.pyNot foundlink
test_python_interpreter_unixtest_common.pyNot foundlink
test_invoke_confservertest_common.pyNot foundlink
test_ccache_used_to_buildtest_common.pyNot foundlink
test_toolchain_prefix_in_description_filetest_common.pyNot foundlink
test_subcommands_with_optionstest_common.pyNot foundlink
test_fallback_to_build_system_targettest_common.pyNot foundlink
test_create_component_projecttest_common.pyNot foundlink
- - - - - - - - diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/failure_rate_jobs_response.json b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/failure_rate_jobs_response.json deleted file mode 100644 index 18e4bea1e21..00000000000 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/failure_rate_jobs_response.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "jobs": [ - {"failed_count": 2, "failure_ratio": 0.2, "total_count": 10, "name": "build_clang_test_apps_esp32h2"}, - {"failed_count": 3, "failure_ratio": 0.3, "total_count": 10, "name": "build_template_app"}, - {"failed_count": 4, "failure_ratio": 0.4, "total_count": 10, "name": "check_public_headers"} - ] -} diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/jobs_api_response.json b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/jobs_api_response.json deleted file mode 100644 index 844e2545b59..00000000000 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/jobs_api_response.json +++ /dev/null @@ -1,212 +0,0 @@ -{ - "jobs": [ - { - "duration_sec": 42.158688, - "failure_log": null, - "failure_reason": null, - "finished_at": "03:38:16, 24 May 2024", - "id": 48838677, - "name": "check_pre_commit", - "pending_sec": 1.15148, - "runner_name": "FA002598-build", - "stage": "pre_check", - "status": "success", - "url": "https://test.com/-/jobs/48838677" - }, - { - "duration_sec": 35.438477, - "failure_log": null, - "failure_reason": null, - "finished_at": "03:37:32, 24 May 2024", - "id": 48838675, - "name": "run-danger-mr-linter", - "pending_sec": 0.371668, - "runner_name": "BrnoVM0211", - "stage": "pre_check", - "status": "success", - "url": "https://test.com/-/jobs/48838675" - }, - { - "duration_sec": 30.202475, - "failure_log": null, - "failure_reason": null, - "finished_at": "03:37:28, 24 May 2024", - "id": 48838682, - "name": "check_esp_system", - "pending_sec": 1.148756, - "runner_name": "ruby6-cent9 [32]", - "stage": "pre_check", - "status": "success", - "url": "https://test.com/-/jobs/48838682" - }, - { - "duration_sec": 33.75121, - "failure_log": null, - "failure_reason": null, - "finished_at": "03:37:31, 24 May 2024", - "id": 48838679, - "name": "check_blobs", - "pending_sec": 0.725292, - "runner_name": "gem3-cent9 [32]", - "stage": "pre_check", - "status": "success", - "url": "https://test.com/-/jobs/48838679" - }, - { - "duration_sec": 121.84324, - "failure_log": null, - "failure_reason": null, - "finished_at": "03:41:35, 24 May 2024", - "id": 48838687, - "name": "code_quality_check", - "pending_sec": 0.271973, - "runner_name": "ruby15-cent9 [32]", - "stage": "build", - "status": "success", - "url": "https://test.com/-/jobs/48838687" - }, - { - "duration_sec": 153.68849, - "failure_log": null, - "failure_reason": null, - "finished_at": "03:39:32, 24 May 2024", - "id": 48838686, - "name": "fast_template_app", - "pending_sec": 2.319577, - "runner_name": "FA002598-build", - "stage": "pre_check", - "status": "success", - "url": "https://test.com/-/jobs/48838686" - }, - { - "duration_sec": 25.572954, - "failure_log": null, - "failure_reason": null, - "finished_at": "03:37:23, 24 May 2024", - "id": 48838684, - "name": "check_configure_ci_environment_parsing", - "pending_sec": 1.184287, - "runner_name": "gem3-cent9 [32]", - "stage": "pre_check", - "status": "success", - "url": "https://test.com/-/jobs/48838684" - }, - { - "duration_sec": 120.95287, - "failure_log": null, - "failure_reason": null, - "finished_at": "03:41:34, 24 May 2024", - "id": 48838690, - "name": "build_clang_test_apps_esp32s3", - "pending_sec": 0.671956, - "runner_name": "ruby7-cent9 [32]", - "stage": "build", - "status": "success", - "url": "https://test.com/-/jobs/48838690" - }, - { - "duration_sec": 165.74513, - "failure_log": null, - "failure_reason": null, - "finished_at": "03:42:19, 24 May 2024", - "id": 48838692, - "name": "build_clang_test_apps_esp32c2", - "pending_sec": 0.82007, - "runner_name": "PowerfulBuildRunner03 [16]", - "stage": "build", - "status": "success", - "url": "https://test.com/-/jobs/48838692" - }, - { - "duration_sec": 95.72326, - "failure_log": null, - "failure_reason": null, - "finished_at": "03:41:09, 24 May 2024", - "id": 48838696, - "name": "build_clang_test_apps_esp32p4", - "pending_sec": 0.567116, - "runner_name": "gem3-cent9 [32]", - "stage": "build", - "status": "success", - "url": "https://test.com/-/jobs/48838696" - }, - { - "duration_sec": 122.19848, - "failure_log": null, - "failure_reason": null, - "finished_at": "03:41:36, 24 May 2024", - "id": 48838691, - "name": "build_clang_test_apps_esp32c3", - "pending_sec": 0.709112, - "runner_name": "ruby6-cent9 [32]", - "stage": "build", - "status": "success", - "url": "https://test.com/-/jobs/48838691" - }, - { - "duration_sec": 148.09895, - "failure_log": null, - "failure_reason": null, - "finished_at": "03:42:02, 24 May 2024", - "id": 48838694, - "name": "build_clang_test_apps_esp32c5", - "pending_sec": 0.779584, - "runner_name": "PowerfulBuildRunner04 [15]", - "stage": "build", - "status": "success", - "url": "https://test.com/-/jobs/48838694" - }, - { - "duration_sec": 20.275927, - "failure_log": null, - "failure_reason": null, - "finished_at": "03:39:54, 24 May 2024", - "id": 48838699, - "name": "gen_integration_pipeline", - "pending_sec": 0.868898, - "runner_name": "FA002598-build", - "stage": "assign_test", - "status": "success", - "url": "https://test.com/-/jobs/48838699" - }, - { - "duration_sec": 103.08849, - "failure_log": "Some Failure LogSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure Reason", - "failure_reason": "Some Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure Reason", - "finished_at": "03:41:17, 24 May 2024", - "id": 48838695, - "name": "build_clang_test_apps_esp32h2", - "pending_sec": 0.765111, - "runner_name": "gem2-cent9 [32]", - "stage": "build", - "status": "failed", - "url": "https://test.com/-/jobs/48838695" - }, - { - "duration_sec": 634.59467, - "failure_log": "Some Failure LogSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure Reason", - "failure_reason": "Some Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure Reason", - "finished_at": "03:50:09, 24 May 2024", - "id": 48838704, - "name": "build_template_app", - "pending_sec": 0.161796, - "runner_name": "ruby6-cent9 [32]", - "stage": "host_test", - "status": "failed", - "url": "https://test.com/-/jobs/48838704" - }, - { - "duration_sec": 1060.0835, - "failure_log": "Some Failure Log", - "failure_reason": "Some Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure Reason", - "finished_at": "03:55:14, 24 May 2024", - "id": 48838705, - "name": "check_public_headers", - "pending_sec": 0.449408, - "runner_name": "ruby6-cent9 [32]", - "stage": "host_test", - "status": "failed", - "url": "https://test.com/-/jobs/48838705" - } - ] -} diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/test_report_generator.py b/tools/ci/dynamic_pipelines/tests/test_report_generator/test_report_generator.py deleted file mode 100644 index 81bae786ced..00000000000 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/test_report_generator.py +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env python -# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import json -import os.path -import sys -import unittest -from unittest.mock import MagicMock -from unittest.mock import patch - -sys.path.insert(0, os.path.join(f'{os.environ.get("IDF_PATH")}', 'tools', 'ci', 'python_packages')) -sys.path.insert(0, os.path.join(f'{os.environ.get("IDF_PATH")}', 'tools', 'ci')) - -from dynamic_pipelines.models import GitlabJob # noqa: E402 -from dynamic_pipelines.report import JobReportGenerator, TargetTestReportGenerator # noqa: E402 -from dynamic_pipelines.utils import load_file, parse_testcases_from_filepattern # noqa: E402 - - -class TestReportGeneration(unittest.TestCase): - def setUp(self) -> None: - self.reports_sample_data_path = os.path.join( - os.environ.get('IDF_PATH', ''), 'tools', 'ci', 'dynamic_pipelines', 'tests', 'test_report_generator', 'reports_sample_data' - ) - self.setup_patches() - self.load_test_and_job_reports() - self.create_report_generators() - - def setup_patches(self) -> None: - self.gitlab_patcher = patch('dynamic_pipelines.report.Gitlab') - self.failure_rate_patcher = patch('dynamic_pipelines.report.fetch_failed_testcases_failure_ratio') - self.env_patcher = patch.dict('os.environ', { - 'CI_DASHBOARD_HOST': 'https://test_dashboard_host', - 'CI_PAGES_URL': 'https://artifacts_path', - 'CI_JOB_ID': '1', - }) - - self.MockGitlab = self.gitlab_patcher.start() - self.test_cases_failure_rate = self.failure_rate_patcher.start() - self.env_patcher.start() - - self.mock_project = MagicMock() - self.mock_mr = MagicMock() - self.MockGitlab.return_value.project = self.mock_project - self.mock_project.mergerequests.get.return_value = self.mock_mr - - self.addCleanup(self.gitlab_patcher.stop) - self.addCleanup(self.failure_rate_patcher.stop) - self.addCleanup(self.env_patcher.stop) - self.addCleanup(self.cleanup_files) - - def cleanup_files(self) -> None: - files_to_delete = [ - self.target_test_report_generator.skipped_test_cases_report_file, - self.target_test_report_generator.succeeded_cases_report_file, - self.target_test_report_generator.failed_cases_report_file, - ] - for file_path in files_to_delete: - if os.path.exists(file_path): - os.remove(file_path) - - def load_test_and_job_reports(self) -> None: - self.expected_target_test_report_html = load_file( - os.path.join(self.reports_sample_data_path, 'expected_target_test_report.html') - ) - self.expected_job_report_html = load_file( - os.path.join(self.reports_sample_data_path, 'expected_job_report.html') - ) - - def create_report_generators(self) -> None: - jobs_response_raw = load_file(os.path.join(self.reports_sample_data_path, 'jobs_api_response.json')) - failure_rate_jobs_response = load_file(os.path.join(self.reports_sample_data_path, 'failure_rate_jobs_response.json')) - failure_rates = {item['name']: item for item in json.loads(failure_rate_jobs_response).get('jobs', [])} - jobs = [GitlabJob.from_json_data(job_json, failure_rates.get(job_json['name'], {})) for job_json in json.loads(jobs_response_raw)['jobs']] - test_cases = parse_testcases_from_filepattern(os.path.join(self.reports_sample_data_path, 'XUNIT_*.xml')) - self.target_test_report_generator = TargetTestReportGenerator( - project_id=123, - mr_iid=1, - pipeline_id=456, - job_id=0, - commit_id='cccc', - title='Test Report', - test_cases=test_cases - ) - self.job_report_generator = JobReportGenerator( - project_id=123, - mr_iid=1, - pipeline_id=456, - job_id=0, - commit_id='cccc', - title='Job Report', - jobs=jobs - ) - self.target_test_report_generator._known_failure_cases_set = { - '*.test_wpa_supplicant_ut', - 'esp32c3.release.test_esp_timer', - '*.512safe.test_wear_levelling', - } - test_cases_failed = [tc for tc in test_cases if tc.is_failure] - for index, tc in enumerate(test_cases_failed): - tc.latest_total_count = 40 - if index % 3 == 0: - tc.latest_failed_count = 0 - else: - tc.latest_failed_count = 3 - self.test_cases_failure_rate.return_value = test_cases_failed - - def test_known_failure_cases(self) -> None: - known_failure_cases = self.target_test_report_generator.get_known_failure_cases() - self.assertEqual(len(known_failure_cases), 4) - - def test_failed_cases_in_target_test_report(self) -> None: - known_failures = self.target_test_report_generator.get_known_failure_cases() - known_failure_case_names = {case.name for case in known_failures} - failed_testcases = self.target_test_report_generator._filter_items( - self.target_test_report_generator.test_cases, - lambda tc: tc.is_failure and tc.name not in known_failure_case_names, - ) - self.assertEqual(len(failed_testcases), 3) - - def test_skipped_cases_in_target_test_report(self) -> None: - skipped_testcases = self.target_test_report_generator._filter_items( - self.target_test_report_generator.test_cases, lambda tc: tc.is_skipped - ) - self.assertEqual(len(skipped_testcases), 1) - - def test_successful_cases_in_target_test_report(self) -> None: - succeeded_testcases = self.target_test_report_generator._filter_items( - self.target_test_report_generator.test_cases, lambda tc: tc.is_success - ) - self.assertEqual(len(succeeded_testcases), 9) - - def test_target_test_report_html_structure(self) -> None: - report = self.target_test_report_generator._get_report_str() - self.assertEqual(report, self.expected_target_test_report_html) - - def test_failed_jobs_in_job_report(self) -> None: - failed_jobs = self.job_report_generator._filter_items(self.job_report_generator.jobs, lambda job: job.is_failed) - self.assertEqual(len(failed_jobs), 3) - - def test_successful_jobs_in_job_report(self) -> None: - succeeded_jobs = self.job_report_generator._filter_items( - self.job_report_generator.jobs, lambda job: job.is_success - ) - self.assertEqual(len(succeeded_jobs), 13) - - def test_job_report_html_structure(self) -> None: - report = self.job_report_generator._get_report_str() - self.assertEqual(report, self.expected_job_report_html) - - -if __name__ == '__main__': - unittest.main() diff --git a/tools/ci/dynamic_pipelines/utils.py b/tools/ci/dynamic_pipelines/utils.py deleted file mode 100644 index aced43c84f6..00000000000 --- a/tools/ci/dynamic_pipelines/utils.py +++ /dev/null @@ -1,218 +0,0 @@ -# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import glob -import os -import re -import typing as t -import xml.etree.ElementTree as ET -from urllib.parse import urlparse - -import requests -import yaml - -from .constants import CI_DASHBOARD_API -from .constants import CI_JOB_TOKEN -from .constants import CI_MERGE_REQUEST_SOURCE_BRANCH_SHA -from .constants import CI_PAGES_URL -from .constants import CI_PROJECT_URL -from .models import GitlabJob -from .models import Job -from .models import TestCase - - -def dump_jobs_to_yaml( - jobs: t.List[Job], - output_filepath: str, - pipeline_name: str, - extra_include_yml: t.Optional[t.List[str]] = None, -) -> None: - yaml_dict = {} - for job in jobs: - yaml_dict.update(job.to_dict()) - - # global stuffs - yaml_dict.update( - { - 'include': [ - 'tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml', - '.gitlab/ci/common.yml', - ], - 'workflow': { - 'name': pipeline_name, - 'rules': [ - # always run the child pipeline, if they are created - {'when': 'always'}, - ], - }, - } - ) - yaml_dict['include'].extend(extra_include_yml or []) - - with open(output_filepath, 'w') as fw: - yaml.dump(yaml_dict, fw, indent=2) - - -def parse_testcases_from_filepattern(junit_report_filepattern: str) -> t.List[TestCase]: - """ - Parses test cases from XML files matching the provided file pattern. - - >>> test_cases = parse_testcases_from_filepattern("path/to/your/junit/reports/*.xml") - - :param junit_report_filepattern: The file pattern to match XML files containing JUnit test reports. - :return: List[TestCase]: A list of TestCase objects parsed from the XML files. - """ - - test_cases = [] - for f in glob.glob(junit_report_filepattern): - root = ET.parse(f).getroot() - for tc in root.findall('.//testcase'): - test_cases.append(TestCase.from_test_case_node(tc)) - return test_cases - - -def load_known_failure_cases() -> t.Optional[t.Set[str]]: - known_failures_file = os.getenv('KNOWN_FAILURE_CASES_FILE_NAME', '') - if not known_failures_file: - return None - try: - with open(known_failures_file, 'r') as f: - file_content = f.read() - - pattern = re.compile(r'^(.*?)\s+#\s+([A-Z]+)-\d+', re.MULTILINE) - matches = pattern.findall(file_content) - - known_cases_list = [match[0].strip() for match in matches] - return set(known_cases_list) - except FileNotFoundError: - return None - - -def is_url(string: str) -> bool: - """ - Check if the string is a valid URL by parsing it and verifying if it contains both a scheme and a network location. - - :param string: The string to check if it is a URL. - :return: True if the string is a valid URL, False otherwise. - """ - parsed = urlparse(string) - return bool(parsed.scheme) and bool(parsed.netloc) - - -def fetch_failed_jobs(commit_id: str) -> t.List[GitlabJob]: - """ - Fetches a list of jobs from the specified commit_id using an API request to ci-dashboard-api. - :param commit_id: The commit ID for which to fetch jobs. - :return: A list of jobs if the request is successful, otherwise an empty list. - """ - response = requests.get( - f'{CI_DASHBOARD_API}/commits/{commit_id}/jobs', - headers={'CI-Job-Token': CI_JOB_TOKEN}, - ) - if response.status_code != 200: - print(f'Failed to fetch jobs data: {response.status_code} with error: {response.text}') - return [] - - data = response.json() - jobs = data.get('jobs', []) - - if not jobs: - return [] - - failed_job_names = [job['name'] for job in jobs if job['status'] == 'failed'] - response = requests.post( - f'{CI_DASHBOARD_API}/jobs/failure_ratio', - headers={'CI-Job-Token': CI_JOB_TOKEN}, - json={'job_names': failed_job_names, 'exclude_branches': [os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_NAME', '')]}, - ) - if response.status_code != 200: - print(f'Failed to fetch jobs failure rate data: {response.status_code} with error: {response.text}') - return [] - - failure_rate_data = response.json() - failure_rates = {item['name']: item for item in failure_rate_data.get('jobs', [])} - - combined_jobs = [] - for job in jobs: - failure_data = failure_rates.get(job['name'], {}) - combined_jobs.append(GitlabJob.from_json_data(job, failure_data)) - - return combined_jobs - - -def fetch_failed_testcases_failure_ratio(failed_testcases: t.List[TestCase], branches_filter: dict) -> t.List[TestCase]: - """ - Fetches info about failure rates of testcases using an API request to ci-dashboard-api. - :param failed_testcases: The list of failed testcases models. - :param branches_filter: The filter to filter testcases by branch names. - :return: A list of testcases with enriched with failure rates data. - """ - req_json = {'testcase_names': list(set([testcase.name for testcase in failed_testcases])), **branches_filter} - response = requests.post( - f'{CI_DASHBOARD_API}/testcases/failure_ratio', - headers={'CI-Job-Token': CI_JOB_TOKEN}, - json=req_json, - ) - if response.status_code != 200: - print(f'Failed to fetch testcases failure rate data: {response.status_code} with error: {response.text}') - return [] - - failure_rate_data = response.json() - failure_rates = {item['name']: item for item in failure_rate_data.get('testcases', [])} - - for testcase in failed_testcases: - testcase.latest_total_count = failure_rates.get(testcase.name, {}).get('total_count', 0) - testcase.latest_failed_count = failure_rates.get(testcase.name, {}).get('failed_count', 0) - - return failed_testcases - - -def load_file(file_path: str) -> str: - """ - Loads the content of a file. - - :param file_path: The path to the file needs to be loaded. - :return: The content of the file as a string. - """ - with open(file_path, 'r') as file: - return file.read() - - -def format_permalink(s: str) -> str: - """ - Formats a given string into a permalink. - - :param s: The string to be formatted into a permalink. - :return: The formatted permalink as a string. - """ - end_index = s.find('(') - - if end_index != -1: - trimmed_string = s[:end_index].strip() - else: - trimmed_string = s.strip() - - formatted_string = trimmed_string.lower().replace(' ', '-') - - return formatted_string - - -def get_artifacts_url(job_id: int, output_filepath: str) -> str: - """ - Generates the url of the path where the artifact will be stored in the job's artifacts . - - :param job_id: The job identifier used to construct the URL. - :param output_filepath: The path to the output file. - :return: The modified URL pointing to the job's artifacts. - """ - url = CI_PAGES_URL.replace('esp-idf', '-/esp-idf') - return f'{url}/-/jobs/{job_id}/artifacts/{output_filepath}' - - -def get_repository_file_url(file_path: str) -> str: - """ - Generates the url of the file path inside the repository. - - :param file_path: The file path where the file is stored. - :return: The modified URL pointing to the file's path in the repository. - """ - return f'{CI_PROJECT_URL}/-/raw/{CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}/{file_path}' diff --git a/tools/ci/exclude_check_tools_files.txt b/tools/ci/exclude_check_tools_files.txt index ba3cf59decb..4b4b016975d 100644 --- a/tools/ci/exclude_check_tools_files.txt +++ b/tools/ci/exclude_check_tools_files.txt @@ -4,18 +4,15 @@ tools/ci/check_*.py tools/ci/check_*.txt tools/ci/check_*.sh tools/ci/check_copyright_config.yaml -tools/ci/get_all_test_results.py tools/gdb_panic_server.py tools/check_term.py tools/python_version_checker.py tools/ci/astyle-rules.yml tools/ci/checkout_project_ref.py tools/ci/ci_fetch_submodule.py -tools/ci/ci_get_mr_info.py tools/ci/ci_process_description.py tools/ci/configure_ci_environment.sh tools/ci/generate_rules.py -tools/ci/deploy_docs.py tools/ci/envsubst.py tools/ci/*exclude*.txt tools/ci/executable-list.txt @@ -53,7 +50,6 @@ tools/ci/python_packages/idf_iperf_test_util/**/* tools/esp_prov/**/* tools/ci/sort_yaml.py tools/ci/sg_rules/* -tools/ci/previous_stage_job_status.py tools/legacy_exports/export_legacy.fish tools/legacy_exports/export_legacy.sh tools/legacy_exports/export_legacy.ps1 diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 2fe4c8183dc..df9ad2421d0 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -67,7 +67,6 @@ tools/ci/check_tools_files_patterns.py tools/ci/check_type_comments.py tools/ci/checkout_project_ref.py tools/ci/cleanup_ignore_lists.py -tools/ci/deploy_docs.py tools/ci/envsubst.py tools/ci/fix_empty_prototypes.sh tools/ci/generate_rules.py diff --git a/tools/ci/generate_rules.py b/tools/ci/generate_rules.py index 4dc9534039c..a07632c2e94 100755 --- a/tools/ci/generate_rules.py +++ b/tools/ci/generate_rules.py @@ -1,39 +1,35 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + import argparse import inspect import os import sys +import typing as t from collections import defaultdict from itertools import product import yaml -from idf_ci_utils import GitlabYmlConfig from idf_ci_utils import IDF_PATH -try: +if t.TYPE_CHECKING: import pygraphviz as pgv -except ImportError: # used when pre-commit, skip generating image - pass - -try: - from typing import Union -except ImportError: # used for type hint - pass -def _list(str_or_list): # type: (Union[str, list]) -> list +def _list(str_or_list: str | list) -> list: if isinstance(str_or_list, str): return [str_or_list] elif isinstance(str_or_list, list): return str_or_list else: - raise ValueError('Wrong type: {}. Only supports str or list.'.format(type(str_or_list))) + raise ValueError(f'Wrong type: {type(str_or_list)}. Only supports str or list.') -def _format_nested_dict(_dict, f_tuple): # type: (dict[str, dict], tuple[str, ...]) -> dict[str, dict] +def _format_nested_dict(_dict: dict[str, dict], f_tuple: tuple[str, ...]) -> dict[str, dict]: res = {} for k, v in _dict.items(): k = k.split('__')[0] @@ -47,7 +43,7 @@ def _format_nested_dict(_dict, f_tuple): # type: (dict[str, dict], tuple[str, . return res -def _format_nested_list(_list, f_tuple): # type: (list[str], tuple[str, ...]) -> list[str] +def _format_nested_list(_list: list[str], f_tuple: tuple[str, ...]) -> list[str]: res = [] for item in _list: if isinstance(item, list): @@ -61,26 +57,23 @@ def _format_nested_list(_list, f_tuple): # type: (list[str], tuple[str, ...]) - class RulesWriter: - AUTO_GENERATE_MARKER = inspect.cleandoc(r''' + AUTO_GENERATE_MARKER = inspect.cleandoc(r""" ################## # Auto Generated # ################## - ''') + """) - LABEL_TEMPLATE = inspect.cleandoc(r''' + LABEL_TEMPLATE = inspect.cleandoc(r""" .if-label-{0}: &if-label-{0} if: '$BOT_LABEL_{1} || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*{0}(?:,[^,\n\r]+)*$/i' - ''') + """) - RULE_PROTECTED = ' - <<: *if-protected' - RULE_PROTECTED_NO_LABEL = ' - <<: *if-protected-no_label' - RULE_BUILD_ONLY = ' - <<: *if-label-build-only\n' \ - ' when: never' - RULE_REVERT_BRANCH = ' - <<: *if-revert-branch\n' \ - ' when: never' + RULE_PROTECTED_CHECK = ' - <<: *if-protected-check' + RULE_PROTECTED_PUSH = ' - <<: *if-protected-push' + RULE_BUILD_ONLY = ' - <<: *if-label-build-only\n when: never' + RULE_REVERT_BRANCH = ' - <<: *if-revert-branch\n when: never' RULE_LABEL_TEMPLATE = ' - <<: *if-label-{0}' - RULE_PATTERN_TEMPLATE = ' - <<: *if-dev-push\n' \ - ' changes: *patterns-{0}' + RULE_PATTERN_TEMPLATE = ' - <<: *if-dev-push\n changes: *patterns-{0}' SPECIFIC_RULE_TEMPLATE = ' - <<: *{0}' RULES_TEMPLATE = inspect.cleandoc(r""" .rules:{0}: @@ -99,7 +92,6 @@ class RulesWriter: self.cfg = self.expand_matrices() self.rules = self.expand_rules() - self.yml_config = GitlabYmlConfig() self.graph = None def expand_matrices(self): # type: () -> dict @@ -116,11 +108,11 @@ class RulesWriter: deploy = v.get('deploy') if deploy: for item in _list(deploy): - res['{}-{}'.format(k, item)] = v + res[f'{k}-{item}'] = v return res @staticmethod - def _expand_matrix(name, cfg): # type: (str, dict) -> dict + def _expand_matrix(name: str, cfg: dict[str, t.Any]) -> dict[str, t.Any]: """ Expand matrix into multi keys :param cfg: single rule dict @@ -138,7 +130,7 @@ class RulesWriter: res.update(_format_nested_dict(default, comb)) return res - def expand_rules(self): # type: () -> dict[str, dict[str, list]] + def expand_rules(self) -> dict[str, dict[str, list[str]]]: res = defaultdict(lambda: defaultdict(set)) # type: dict[str, dict[str, set]] for k, v in self.cfg.items(): if not v: @@ -164,18 +156,18 @@ class RulesWriter: if 'patterns' in v: for _pat in _list(v['patterns']): # Patterns must be pre-defined - if '.patterns-{}'.format(_pat) not in self.rules_cfg: - print('WARNING: pattern {} not exists'.format(_pat)) + if f'.patterns-{_pat}' not in self.rules_cfg: + print(f'WARNING: pattern {_pat} not exists') continue res[item]['patterns'].add(_pat) - sorted_res = defaultdict(lambda: defaultdict(list)) # type: dict[str, dict[str, list]] + sorted_res = defaultdict(lambda: defaultdict(list)) # type: t.Dict[str, t.Dict[str, t.List[str]]] for k, v in res.items(): for vk, vv in v.items(): sorted_res[k][vk] = sorted(vv) return sorted_res - def new_labels_str(self): # type: () -> str + def new_labels_str(self) -> str: _labels = set([]) for k, v in self.cfg.items(): if not v: @@ -191,33 +183,29 @@ class RulesWriter: return res @classmethod - def _format_label(cls, label): # type: (str) -> str + def _format_label(cls, label: str) -> str: return cls.LABEL_TEMPLATE.format(label, cls.bot_label_str(label)) @staticmethod - def bot_label_str(label): # type: (str) -> str + def bot_label_str(label: str) -> str: return label.upper().replace('-', '_') - def new_rules_str(self): # type: () -> str + def new_rules_str(self) -> str: res = [] for k, v in sorted(self.rules.items()): if k.startswith('pattern'): continue - if '.rules:' + k not in self.yml_config.used_templates: - print(f'WARNING: unused rule: {k}, skipping...') - continue - res.append(self.RULES_TEMPLATE.format(k, self._format_rule(k, v))) return '\n\n'.join(res) - def _format_rule(self, name, cfg): # type: (str, dict) -> str + def _format_rule(self, name: str, cfg: dict[str, t.Any]) -> str: _rules = [self.RULE_REVERT_BRANCH] if name.endswith('-production'): - _rules.append(self.RULE_PROTECTED_NO_LABEL) + _rules.append(self.RULE_PROTECTED_PUSH) else: if not (name.endswith('-preview') or name.startswith('labels:')): - _rules.append(self.RULE_PROTECTED) + _rules.append(self.RULE_PROTECTED_CHECK) if name.startswith('test:'): _rules.append(self.RULE_BUILD_ONLY) @@ -225,21 +213,21 @@ class RulesWriter: if f'.{specific_rule}' in self.rules_cfg: _rules.append(self.SPECIFIC_RULE_TEMPLATE.format(specific_rule)) else: - print('WARNING: specific_rule {} not exists'.format(specific_rule)) + print(f'WARNING: specific_rule {specific_rule} not exists') for label in cfg['labels']: _rules.append(self.RULE_LABEL_TEMPLATE.format(label)) for pattern in cfg['patterns']: - if '.patterns-{}'.format(pattern) in self.rules_cfg: + if f'.patterns-{pattern}' in self.rules_cfg: _rules.append(self.RULE_PATTERN_TEMPLATE.format(pattern)) else: - print('WARNING: pattern {} not exists'.format(pattern)) + print(f'WARNING: pattern {pattern} not exists') return '\n'.join(_rules) - def update_rules_yml(self): # type: () -> bool + def update_rules_yml(self) -> bool: with open(self.rules_yml) as fr: file_str = fr.read() - auto_generate_str = '\n{}\n\n{}\n'.format(self.new_labels_str(), self.new_rules_str()) + auto_generate_str = f'\n{self.new_labels_str()}\n\n{self.new_rules_str()}\n' rest, marker, old = file_str.partition(self.AUTO_GENERATE_MARKER) if old == auto_generate_str: return False @@ -255,7 +243,9 @@ PATTERN_COLOR = 'cyan' RULE_COLOR = 'blue' -def build_graph(rules_dict): # type: (dict[str, dict[str, list]]) -> pgv.AGraph +def build_graph(rules_dict: dict[str, dict[str, list[str]]]) -> pgv.AGraph: + from pygraphviz import pgv + graph = pgv.AGraph(directed=True, rankdir='LR', concentrate=True) for k, v in rules_dict.items(): @@ -270,18 +260,18 @@ def build_graph(rules_dict): # type: (dict[str, dict[str, list]]) -> pgv.AGraph labels = v.get('labels') if labels: for _label in labels: - graph.add_node('label:{}'.format(_label), color=LABEL_COLOR) - graph.add_edge('label:{}'.format(_label), k, color=LABEL_COLOR) + graph.add_node(f'label:{_label}', color=LABEL_COLOR) + graph.add_edge(f'label:{_label}', k, color=LABEL_COLOR) patterns = v.get('patterns') if patterns: for _pat in patterns: - graph.add_node('pattern:{}'.format(_pat), color=PATTERN_COLOR) - graph.add_edge('pattern:{}'.format(_pat), k, color=PATTERN_COLOR) + graph.add_node(f'pattern:{_pat}', color=PATTERN_COLOR) + graph.add_edge(f'pattern:{_pat}', k, color=PATTERN_COLOR) return graph -def output_graph(graph, output_path='output.png'): # type: (pgv.AGraph, str) -> None +def output_graph(graph: pgv.AGraph, output_path: str = 'output.png') -> None: graph.layout('dot') if output_path.endswith('.png'): img_path = output_path @@ -292,13 +282,22 @@ def output_graph(graph, output_path='output.png'): # type: (pgv.AGraph, str) -> if __name__ == '__main__': parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('rules_yml', nargs='?', default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'rules.yml'), - help='rules.yml file path') - parser.add_argument('dependencies_yml', nargs='?', default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'dependencies', - 'dependencies.yml'), - help='dependencies.yml file path') - parser.add_argument('--graph', - help='Specify PNG image output path. Use this argument to generate dependency graph') + parser.add_argument( + 'rules_yml', + nargs='?', + default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'rules.yml'), + help='rules.yml file path', + ) + parser.add_argument( + 'dependencies_yml', + nargs='?', + default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'dependencies', 'dependencies.yml'), + help='dependencies.yml file path', + ) + parser.add_argument( + '--graph', + help='Specify PNG image output path. Use this argument to generate dependency graph', + ) args = parser.parse_args() writer = RulesWriter(args.rules_yml, args.dependencies_yml) diff --git a/tools/ci/get_all_test_results.py b/tools/ci/get_all_test_results.py deleted file mode 100644 index 7baa0832f2a..00000000000 --- a/tools/ci/get_all_test_results.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python -# -# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 - - -''' -Download artifacts from all test jobs - -CI_PROJECT_DIR -└── TEST_RESULTS - ├── - │ └── XUNIT_RESULT.xml - ├── - │ └── XUNIT_RESULT.xml - ├── - │ └── TEST_LOGS - -''' - -import argparse -import json -import os -import shutil - -import gitlab_api - -CI_PROJECT_ID = int(os.getenv('CI_PROJECT_ID', '')) -CI_PIPELINE_ID = int(os.getenv('CI_PIPELINE_ID', '')) -IDF_PATH = os.getenv('IDF_PATH', '.') -TEST_RESULT_PATH = os.path.join(IDF_PATH, 'TEST_RESULTS') - - -def download_all_test_results(result_path: str, include_retried: bool = False) -> None: - if os.path.exists(result_path): - shutil.rmtree(result_path) - os.makedirs(result_path, exist_ok=True) - - gitlab_inst = gitlab_api.Gitlab(CI_PROJECT_ID) - pipelines = gitlab_inst.project.pipelines.get(CI_PIPELINE_ID) - _include_retried = 'true' if include_retried else 'false' - jobs = pipelines.jobs.list(all=True, per_page=100, include_retried=_include_retried) - - job_info_list = [] - for job in jobs: - if job.stage in ['target_test', 'host_test']: - log_path = '' - if job.status not in ['success', 'failed']: - print('Job {}({}) is not finished'.format(job.id, job.name)) - elif not hasattr(job, 'artifacts_file'): - print('Job {}({}) has no artifacts.'.format(job.id, job.name)) - else: - log_path = os.path.join(result_path, 'job_{}'.format(job.id)) - print('Downloading artifacts from: {}'.format(job.name)) - os.makedirs(log_path, exist_ok=True) - gitlab_inst.download_artifacts(job.id, log_path) - - job_info = { - 'id': job.id, - 'name': job.name, - 'tag_list': job.tag_list, - 'status': job.status, - 'stage': job.stage, - 'web_url': job.web_url, - 'commit_url': job.commit['web_url'], - 'log_path': log_path, - } - job_info_list.append(job_info) - - with open(os.path.join(result_path, 'index.json'), 'w') as f: - f.write(json.dumps({'jobs': job_info_list}, indent=1, sort_keys=True)) - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument('--path', '-p', default=TEST_RESULT_PATH, help='Path to save result files.') - parser.add_argument('--include_retried', action='store_true', help='Including retried jobs.') - args = parser.parse_args() - - download_all_test_results(args.path, args.include_retried) - - -if __name__ == '__main__': - main() diff --git a/tools/ci/gitlab_yaml_linter.py b/tools/ci/gitlab_yaml_linter.py index baac4ee2e37..e940369e336 100755 --- a/tools/ci/gitlab_yaml_linter.py +++ b/tools/ci/gitlab_yaml_linter.py @@ -1,27 +1,29 @@ #!/usr/bin/env python -# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 """ Check gitlab ci yaml files """ + +from __future__ import annotations + import argparse import os -import typing as t from functools import cached_property -from idf_ci_utils import get_submodule_dirs -from idf_ci_utils import GitlabYmlConfig from idf_ci_utils import IDF_PATH +from idf_ci_utils import GitlabYmlConfig +from idf_ci_utils import get_submodule_dirs class YmlLinter: def __init__(self, yml_config: GitlabYmlConfig) -> None: self.yml_config = yml_config - self._errors: t.List[str] = [] + self._errors: list[str] = [] @cached_property - def lint_functions(self) -> t.List[str]: + def lint_functions(self) -> list[str]: funcs = [] for func in dir(self): if func.startswith('_lint_'): @@ -43,17 +45,6 @@ class YmlLinter: exit(exit_code) - # name it like _1_ to make it run first - def _lint_1_yml_parser(self) -> None: - for k, v in self.yml_config.config.items(): - if ( - k not in self.yml_config.global_keys - and k not in self.yml_config.anchors - and k not in self.yml_config.templates - and k not in self.yml_config.jobs - ): - raise SystemExit(f'Parser incorrect. Key {k} not in global keys, anchors, templates, or jobs') - def _lint_default_values_artifacts(self) -> None: defaults_artifacts = self.yml_config.default.get('artifacts', {}) @@ -79,19 +70,6 @@ class YmlLinter: for item in undefined_patterns: self._errors.append(f'undefined pattern {item}. Please add {item} to .patterns-submodule') - def _lint_gitlab_yml_templates(self) -> None: - unused_templates = self.yml_config.templates.keys() - self.yml_config.used_templates - for item in unused_templates: - # known unused ones - if item not in [ - '.before_script:fetch:target_test', # used in dynamic pipeline - ]: - self._errors.append(f'Unused template: {item}, please remove it') - - undefined_templates = self.yml_config.used_templates - self.yml_config.templates.keys() - for item in undefined_templates: - self._errors.append(f'Undefined template: {item}') - def _lint_dependencies_and_needs(self) -> None: """ Use `dependencies: []` together with `needs: []` could cause missing artifacts issue. @@ -123,7 +101,9 @@ class YmlLinter: if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( - '--root-yml-filepath', help='root yml file path', default=os.path.join(IDF_PATH, '.gitlab-ci.yml') + '--root-yml-filepath', + help='root yml file path', + default=os.path.join(IDF_PATH, '.gitlab-ci.yml'), ) args = parser.parse_args() diff --git a/tools/ci/idf_ci_local/app.py b/tools/ci/idf_ci_local/app.py index 4dd91153d36..11449a148d9 100644 --- a/tools/ci/idf_ci_local/app.py +++ b/tools/ci/idf_ci_local/app.py @@ -1,20 +1,17 @@ -# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2024-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import os +import subprocess import sys import typing as t -from typing import Literal -from idf_build_apps import App from idf_build_apps import CMakeApp -from idf_build_apps import json_to_app -from idf_ci_local.uploader import AppUploader -from idf_ci_local.uploader import get_app_uploader +from idf_build_apps.constants import BuildStatus +from idf_build_apps.utils import rmdir class IdfCMakeApp(CMakeApp): - uploader: t.ClassVar[t.Optional['AppUploader']] = get_app_uploader() - build_system: Literal['idf_cmake'] = 'idf_cmake' + build_system: t.Literal['idf_cmake'] = 'idf_cmake' def _initialize_hook(self, **kwargs: t.Any) -> None: # ensure this env var exists @@ -25,25 +22,27 @@ class IdfCMakeApp(CMakeApp): def _post_build(self) -> None: super()._post_build() - if self.uploader: - self.uploader.upload_app(self.build_path) + # only upload in CI + if os.getenv('CI_JOB_ID'): + command = [ + 'idf-ci', + 'gitlab', + 'upload-artifacts', + self.app_dir, + '--build-dir', + self.build_dir, + ] + result = subprocess.run( + command, + stdout=sys.stdout, + stderr=sys.stderr, + ) + if result.returncode != 0: + self.build_status = BuildStatus.FAILED + self.build_comment = 'Failed to upload artifacts' -def dump_apps_to_txt(apps: t.List[App], output_filepath: str) -> None: - with open(output_filepath, 'w') as fw: - for app in apps: - fw.write(app.model_dump_json() + '\n') - - -def import_apps_from_txt(input_filepath: str) -> t.List[App]: - apps: t.List[App] = [] - with open(input_filepath) as fr: - for line in fr: - if line := line.strip(): - try: - apps.append(json_to_app(line, extra_classes=[IdfCMakeApp])) - except Exception: # noqa - print('Failed to deserialize app from line: %s' % line) - sys.exit(1) - - return apps + rmdir( + self.build_path, + exclude_file_patterns=['build_log.txt', 'size*.json'], + ) diff --git a/tools/ci/idf_ci_local/uploader.py b/tools/ci/idf_ci_local/uploader.py deleted file mode 100644 index f04028b1e67..00000000000 --- a/tools/ci/idf_ci_local/uploader.py +++ /dev/null @@ -1,169 +0,0 @@ -# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import abc -import glob -import os -import typing as t -from datetime import timedelta -from zipfile import ZIP_DEFLATED -from zipfile import ZipFile - -import minio -from artifacts_handler import ArtifactType -from artifacts_handler import get_minio_client -from artifacts_handler import getenv -from idf_build_apps import App -from idf_build_apps.utils import rmdir -from idf_ci_utils import IDF_PATH -from idf_pytest.constants import DEFAULT_BUILD_LOG_FILENAME -from idf_pytest.constants import DEFAULT_SIZE_JSON_FILENAME - - -class AppDownloader: - ALL_ARTIFACT_TYPES = [ArtifactType.MAP_AND_ELF_FILES, ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES] - - @abc.abstractmethod - def _download_app(self, app_build_path: str, artifact_type: ArtifactType) -> None: - pass - - def download_app(self, app_build_path: str, artifact_type: t.Optional[ArtifactType] = None) -> None: - """ - Download the app - :param app_build_path: the path to the build directory - :param artifact_type: if not specify, download all types of artifacts - :return: None - """ - if not artifact_type: - for _artifact_type in self.ALL_ARTIFACT_TYPES: - self._download_app(app_build_path, _artifact_type) - else: - self._download_app(app_build_path, artifact_type) - - -class AppUploader(AppDownloader): - TYPE_PATTERNS_DICT = { - ArtifactType.MAP_AND_ELF_FILES: [ - 'bootloader/*.map', - 'bootloader/*.elf', - '*.map', - '*.elf', - 'gdbinit/*', - ], - ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES: [ - '*.bin', - 'bootloader/*.bin', - 'partition_table/*.bin', - 'flasher_args.json', - 'flash_project_args', - 'config/sdkconfig.json', - 'sdkconfig', - 'project_description.json', - ], - ArtifactType.LOGS: [ - DEFAULT_BUILD_LOG_FILENAME, - ], - } - - def __init__(self, pipeline_id: t.Union[str, int, None] = None) -> None: - self.pipeline_id = str(pipeline_id or '1') - - self._client = get_minio_client() - - def get_app_object_name(self, app_path: str, zip_name: str, artifact_type: ArtifactType) -> str: - return f'{self.pipeline_id}/{artifact_type.value}/{app_path}/{zip_name}' - - def _upload_app(self, app_build_path: str, artifact_type: ArtifactType) -> bool: - app_path, build_dir = os.path.split(app_build_path) - zip_filename = f'{build_dir}.zip' - - has_file = False - with ZipFile( - zip_filename, - 'w', - compression=ZIP_DEFLATED, - # 1 is the fastest compression level - # the size differs not much between 1 and 9 - compresslevel=1, - ) as zw: - for pattern in self.TYPE_PATTERNS_DICT[artifact_type]: - for file in glob.glob(os.path.join(app_build_path, pattern), recursive=True): - zw.write(file) - has_file = True - - uploaded = False - try: - if has_file: - obj_name = self.get_app_object_name(app_path, zip_filename, artifact_type) - self._client.fput_object(getenv('IDF_S3_BUCKET'), obj_name, zip_filename) - uploaded = True - finally: - os.remove(zip_filename) - return uploaded - - def upload_app(self, app_build_path: str, artifact_type: t.Optional[ArtifactType] = None) -> None: - uploaded = False - if not artifact_type: - upload_types: t.Iterable[ArtifactType] = self.TYPE_PATTERNS_DICT.keys() - else: - upload_types = [artifact_type] - - # Upload of size.json files is handled by GitLab CI via "artifacts_handler.py" script. - print(f'Uploading {app_build_path} {[k.value for k in upload_types]} to minio server') - for upload_type in upload_types: - uploaded |= self._upload_app(app_build_path, upload_type) - - if uploaded: - rmdir(app_build_path, exclude_file_patterns=[DEFAULT_BUILD_LOG_FILENAME, DEFAULT_SIZE_JSON_FILENAME]) - - def _download_app(self, app_build_path: str, artifact_type: ArtifactType) -> None: - app_path, build_dir = os.path.split(app_build_path) - zip_filename = f'{build_dir}.zip' - - # path are relative to IDF_PATH - current_dir = os.getcwd() - os.chdir(IDF_PATH) - try: - obj_name = self.get_app_object_name(app_path, zip_filename, artifact_type) - print(f'Downloading {obj_name}') - try: - try: - self._client.stat_object(getenv('IDF_S3_BUCKET'), obj_name) - except minio.error.S3Error as e: - raise RuntimeError( - f'No such file on minio server: {obj_name}. ' - f'Probably the build failed or the artifacts got expired. ' - f'Full error message: {str(e)}' - ) - else: - self._client.fget_object(getenv('IDF_S3_BUCKET'), obj_name, zip_filename) - print(f'Downloaded to {zip_filename}') - except minio.error.S3Error as e: - raise RuntimeError('Shouldn\'t happen, please report this bug in the CI channel' + str(e)) - - with ZipFile(zip_filename, 'r') as zr: - zr.extractall() - - os.remove(zip_filename) - finally: - os.chdir(current_dir) - - def get_app_presigned_url(self, app: App, artifact_type: ArtifactType) -> str: - obj_name = self.get_app_object_name(app.app_dir, f'{app.build_dir}.zip', artifact_type) - try: - self._client.stat_object( - getenv('IDF_S3_BUCKET'), - obj_name, - ) - except minio.error.S3Error: - return '' - else: - return self._client.get_presigned_url( # type: ignore - 'GET', getenv('IDF_S3_BUCKET'), obj_name, expires=timedelta(days=4) - ) - - -def get_app_uploader() -> t.Optional['AppUploader']: - if parent_pipeline_id := os.getenv('PARENT_PIPELINE_ID'): - return AppUploader(parent_pipeline_id) - - return None diff --git a/tools/ci/idf_ci_utils.py b/tools/ci/idf_ci_utils.py index 1c51f28ed0d..4ddeb8dc946 100644 --- a/tools/ci/idf_ci_utils.py +++ b/tools/ci/idf_ci_utils.py @@ -2,9 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 # internal use only for CI # some CI related util functions + +from __future__ import annotations # Must be the first line of code + import logging import os -import re import subprocess import sys import typing as t @@ -14,7 +16,7 @@ from pathlib import Path IDF_PATH: str = os.path.abspath(os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..'))) -def get_submodule_dirs(full_path: bool = False) -> t.List[str]: +def get_submodule_dirs(full_path: bool = False) -> list[str]: """ To avoid issue could be introduced by multi-os or additional dependency, we use python and git to get this output @@ -72,7 +74,7 @@ def is_executable(full_path: str) -> bool: return os.access(full_path, os.X_OK) -def get_git_files(path: str = IDF_PATH, full_path: bool = False) -> t.List[str]: +def get_git_files(path: str = IDF_PATH, full_path: bool = False) -> list[str]: """ Get the result of git ls-files :param path: path to run git ls-files @@ -99,7 +101,7 @@ def get_git_files(path: str = IDF_PATH, full_path: bool = False) -> t.List[str]: return [os.path.join(path, f) for f in files] if full_path else files -def to_list(s: t.Any) -> t.List[t.Any]: +def to_list(s: t.Any) -> list[t.Any]: if not s: return [] @@ -114,8 +116,8 @@ def to_list(s: t.Any) -> t.List[t.Any]: class GitlabYmlConfig: def __init__(self, root_yml_filepath: str = os.path.join(IDF_PATH, '.gitlab-ci.yml')) -> None: - self._config: t.Dict[str, t.Any] = {} - self._defaults: t.Dict[str, t.Any] = {} + self._config: dict[str, t.Any] = {} + self._defaults: dict[str, t.Any] = {} self._load(root_yml_filepath) @@ -123,125 +125,130 @@ class GitlabYmlConfig: # avoid unused import in other pre-commit hooks import yaml - all_config = dict() - root_yml = yaml.load(open(root_yml_filepath), Loader=yaml.FullLoader) - - # expanding "include" - for item in root_yml.pop('include', []) or []: - if isinstance(item, dict): - if 'project' in item: - continue - elif 'local' in item: - item = item['local'] - else: - continue - all_config.update(yaml.load(open(os.path.join(IDF_PATH, item)), Loader=yaml.FullLoader)) + merged_yaml = self._compile_via_gitlab_api(root_yml_filepath) + all_config = yaml.load(merged_yaml, Loader=yaml.FullLoader) or {} if 'default' in all_config: self._defaults = all_config.pop('default') self._config = all_config - # anchor is the string that will be reused in templates - self._anchor_keys: t.Set[str] = set() - # template is a dict that will be extended - self._template_keys: t.Set[str] = set() - self._used_template_keys: t.Set[str] = set() # tracing the used templates - # job is a dict that will be executed - self._job_keys: t.Set[str] = set() + def _inline_local_includes(self, root_yml_filepath: str) -> str: + """ + Recursively resolve `include: local` entries straight from disk (so uncommitted local + changes are always picked up -- CI runners also work off a disk checkout, so there's no + need to fetch a ref remotely via `ci_lint`'s `content_ref`/`ref`/`dry_run_ref` GET + params). `include: project` entries (including ones nested inside local files, e.g. + `.gitlab/ci/common.yml` including `templates/idf/common-scripts.yml`) are collected and + left in the final `include:` list, since those files live in another GitLab project and + can only be resolved remotely. - self.expand_extends() + A dedicated Loader/Dumper pair round-trips `!reference` tags as a marker list subclass, + since GitLab CI uses `!reference` which plain YAML doesn't know, and we need to parse + (to merge dicts, not just string-concat) then re-dump losslessly. + + :param root_yml_filepath: path to the local root yml file to compile + :return: yml content with local includes resolved and merged + """ + import yaml + + class _Loader(yaml.FullLoader): + pass + + class _Dumper(yaml.Dumper): + pass + + class _Reference(list): + pass + + _Loader.add_constructor( + '!reference', + lambda loader, node: _Reference(loader.construct_sequence(t.cast(yaml.SequenceNode, node))), + ) + _Dumper.add_representer( + _Reference, + lambda dumper, data: dumper.represent_sequence('!reference', list(data)), + ) + + def resolve(yml_filepath: str) -> tuple[dict, list]: + with open(yml_filepath) as fr: + data = yaml.load(fr, Loader=_Loader) or {} + + includes = to_list(data.pop('include', None)) + + merged: dict = {} + remaining_project_includes: list = [] + for item in includes: + if isinstance(item, dict): + if 'project' in item: + remaining_project_includes.append(item) + continue + elif 'local' in item: + local_path = item['local'] + else: + continue + elif isinstance(item, str): + local_path = item + else: + continue + + sub_merged, sub_remaining = resolve(os.path.join(IDF_PATH, local_path.lstrip('/'))) + merged.update(sub_merged) + remaining_project_includes.extend(sub_remaining) + + # this file's own top-level keys override whatever its includes defined + merged.update(data) + return merged, remaining_project_includes + + merged_config, project_includes = resolve(root_yml_filepath) + if project_includes: + merged_config['include'] = project_includes + + return yaml.dump(merged_config, Dumper=_Dumper, sort_keys=False) # type: ignore + + def _compile_via_gitlab_api(self, root_yml_filepath: str) -> str: + """ + Call the GitLab CI Lint API to get the fully compiled (all `include`s resolved) yml, + same as what `glab ci config compile` does. This replaces the old recursive local-only + parsing, since the project now includes configs from other projects as well. + + :param root_yml_filepath: path to the local root yml file to compile + :return: merged (fully resolved) yml content as a string + """ + sys.path.insert(0, os.path.join(IDF_PATH, 'tools', 'ci', 'python_packages')) + import gitlab_api + + content = self._inline_local_includes(root_yml_filepath) + + gitlab_inst = gitlab_api.Gitlab() + project_id = os.getenv('CI_PROJECT_ID') or gitlab_inst.get_project_id('esp-idf', namespace='espressif') + project = gitlab_inst.gitlab_inst.projects.get(project_id, lazy=True) + + lint_result = project.ci_lint.create({'content': content}) + if not lint_result.valid: + raise RuntimeError(f'Failed to compile {root_yml_filepath} via GitLab CI Lint API: {lint_result.errors}') + + return lint_result.merged_yaml # type: ignore @property - def default(self) -> t.Dict[str, t.Any]: + def default(self) -> dict[str, t.Any]: return self._defaults @property - def config(self) -> t.Dict[str, t.Any]: + def config(self) -> dict[str, t.Any]: return self._config @cached_property - def global_keys(self) -> t.List[str]: + def global_keys(self) -> list[str]: return ['default', 'include', 'workflow', 'variables', 'stages'] @cached_property - def anchors(self) -> t.Dict[str, t.Any]: - return {k: v for k, v in self.config.items() if k in self._anchor_keys} - - @cached_property - def jobs(self) -> t.Dict[str, t.Any]: - return {k: v for k, v in self.config.items() if k in self._job_keys} - - @cached_property - def templates(self) -> t.Dict[str, t.Any]: - return {k: v for k, v in self.config.items() if k in self._template_keys} - - @cached_property - def used_templates(self) -> t.Set[str]: - return self._used_template_keys - - def expand_extends(self) -> None: - """ - expand the `extends` key in-place. - """ - for k, v in self.config.items(): - if k in self.global_keys: - continue - - if isinstance(v, (str, list)): - self._anchor_keys.add(k) - elif k.startswith('.if-'): - self._anchor_keys.add(k) - elif k.startswith('.'): - self._template_keys.add(k) - elif isinstance(v, dict): - self._job_keys.add(k) - else: - raise ValueError(f'Unknown type for key {k} with value {v}') - - # no need to expand anchor - - # expand template first - for k in self._template_keys: - self._expand_extends(k) - - # expand job - for k in self._job_keys: - self._expand_extends(k) - - def _merge_dict(self, d1: t.Dict[str, t.Any], d2: t.Dict[str, t.Any]) -> t.Any: - for k, v in d2.items(): - if k in d1: - if isinstance(v, dict) and isinstance(d1[k], dict): - d1[k] = self._merge_dict(d1[k], v) - else: - d1[k] = v - else: - d1[k] = v - - return d1 - - def _expand_extends(self, name: str) -> t.Dict[str, t.Any]: - extends = to_list(self.config[name].pop('extends', None)) - if not extends: - return self.config[name] # type: ignore - - original_d = self.config[name].copy() - d = {} - while extends: - self._used_template_keys.update(extends) # for tracking - - for i in extends: - d.update(self._expand_extends(i)) - - extends = to_list(self.config[name].pop('extends', None)) - - self.config[name] = self._merge_dict(d, original_d) - return self.config[name] # type: ignore + def jobs(self) -> dict[str, t.Any]: + return {k: v for k, v in self.config.items() if not k.startswith('.') and k not in self.global_keys} -def get_all_manifest_files() -> t.List[str]: - paths: t.List[str] = [] +def get_all_manifest_files() -> list[str]: + paths: list[str] = [] for p in Path(IDF_PATH).glob('**/.build-test-rules.yml'): if 'managed_components' in p.parts: @@ -252,23 +259,6 @@ def get_all_manifest_files() -> t.List[str]: return paths -def sanitize_job_name(name: str) -> str: - """ - Sanitize the job name from CI_JOB_NAME - - - for job with `parallel: int` set, the `CI_JOB_NAME` would be `job_name index/total`, like `foo 1/3` - - for job with `parallel: matrix` set, the `CI_JOB_NAME` would be `job_name: [var1, var2]`, like `foo: [a, b]` - - We consider - - the jobs generated by `parallel: int` as the same job, i.e., we remove the index/total part. - - the jobs generated by `parallel: matrix` as different jobs, so we keep the matrix part. - - :param name: job name - :return: sanitized job name - """ - return re.sub(r' \d+/\d+', '', name) - - def idf_relpath(p: str) -> str: """ Turn all paths under IDF_PATH to relative paths diff --git a/tools/ci/idf_pytest/constants.py b/tools/ci/idf_pytest/constants.py index f2fd0ddbad1..0aae4ab1ead 100644 --- a/tools/ci/idf_pytest/constants.py +++ b/tools/ci/idf_pytest/constants.py @@ -1,332 +1,36 @@ -# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 +import typing as t + """ Pytest Related Constants. Don't import third-party packages here. """ -import os -import typing as t -import warnings -from dataclasses import dataclass -from enum import Enum -from functools import cached_property -from pathlib import Path -from _pytest.python import Function -from idf_ci_utils import IDF_PATH -from idf_ci_utils import idf_relpath -from pytest_embedded.utils import to_list - -SUPPORTED_TARGETS = ['esp32', 'esp32s2', 'esp32c3', 'esp32s3', 'esp32c2', 'esp32c6', 'esp32h2', 'esp32p4', 'esp32c5', 'esp32c61'] +SUPPORTED_TARGETS = [ + 'esp32', + 'esp32s2', + 'esp32c3', + 'esp32s3', + 'esp32c2', + 'esp32c6', + 'esp32h2', + 'esp32p4', + 'esp32c5', + 'esp32c61', +] PREVIEW_TARGETS: t.List[str] = [] # this PREVIEW_TARGETS excludes 'linux' target -DEFAULT_SDKCONFIG = 'default' + DEFAULT_LOGDIR = 'pytest-embedded' -TARGET_MARKERS = { - 'esp32': 'support esp32 target', - 'esp32s2': 'support esp32s2 target', - 'esp32s3': 'support esp32s3 target', - 'esp32c3': 'support esp32c3 target', - 'esp32c2': 'support esp32c2 target', - 'esp32c5': 'support esp32c5 target', - 'esp32c6': 'support esp32c6 target', - 'esp32h2': 'support esp32h2 target', - 'esp32p4': 'support esp32p4 target', - 'esp32c61': 'support esp32c61 target', - 'linux': 'support linux target', -} - -SPECIAL_MARKERS = { - 'supported_targets': 'support all officially announced supported targets, refer to `SUPPORTED_TARGETS`', - 'preview_targets': "support all preview targets ('none')", - 'all_targets': 'support all targets, including supported ones and preview ones', - 'temp_skip_ci': 'temp skip tests for specified targets only in ci', - 'temp_skip': 'temp skip tests for specified targets both in ci and locally', - 'nightly_run': 'tests should be executed as part of the nightly trigger pipeline', - 'host_test': 'tests which should not be built at the build stage, and instead built in host_test stage', -} - -ENV_MARKERS = { - # special markers - 'qemu': 'build and test using qemu, not real target', - 'macos_shell': 'tests should be run on macos hosts', - # single-dut markers - 'generic': 'tests should be run on generic runners', - 'flash_suspend': 'support flash suspend feature', - 'eth_ip101': 'connected via IP101 ethernet transceiver', - 'eth_lan8720': 'connected via LAN8720 ethernet transceiver', - 'eth_rtl8201': 'connected via RTL8201 ethernet transceiver', - 'eth_ksz8041': 'connected via KSZ8041 ethernet transceiver', - 'eth_dp83848': 'connected via DP83848 ethernet transceiver', - 'eth_w5500': 'SPI Ethernet module with two W5500', - 'eth_ksz8851snl': 'SPI Ethernet module with two KSZ8851SNL', - 'eth_dm9051': 'SPI Ethernet module with two DM9051', - 'quad_psram': 'runners with quad psram', - 'octal_psram': 'runners with octal psram', - 'usb_host_flash_disk': 'usb host runners with USB flash disk attached', - 'usb_device': 'usb device runners', - 'ethernet_ota': 'ethernet OTA runners', - 'flash_encryption': 'Flash Encryption runners', - 'flash_encryption_f4r8': 'Flash Encryption runners with 4-line flash and 8-line psram', - 'flash_encryption_f8r8': 'Flash Encryption runners with 8-line flash and 8-line psram', - 'flash_encryption_ota': 'Flash Encryption runners with ethernet OTA support with 4mb flash size', - 'flash_multi': 'Multiple flash chips tests', - 'psram': 'Chip has 4-line psram', - 'ir_transceiver': 'runners with a pair of IR transmitter and receiver', - 'flash_encryption_wifi_high_traffic': 'Flash Encryption runners with wifi high traffic support', - 'ethernet': 'ethernet runner', - 'ethernet_flash_8m': 'ethernet runner with 8mb flash', - 'ethernet_router': 'both the runner and dut connect to the same router through ethernet NIC', - 'ethernet_vlan': 'ethernet runner GARM-32-SH-1-R16S5N3', - 'wifi_ap': 'a wifi AP in the environment', - 'wifi_router': 'both the runner and dut connect to the same wifi router', - 'wifi_high_traffic': 'wifi high traffic runners', - 'wifi_wlan': 'wifi runner with a wireless NIC', - 'wifi_iperf': 'the AP and ESP dut were placed in a shielded box - for iperf test', - 'Example_ShieldBox': 'multiple shielded APs connected to shielded ESP DUT via RF cable with programmable attenuator', - 'xtal_26mhz': 'runner with 26MHz xtal on board', - 'xtal_40mhz': 'runner with 40MHz xtal on board', - 'external_flash': 'external flash memory connected via VSPI (FSPI)', - 'sdcard_sdmode': 'sdcard running in SD mode, to be removed after test migration', - 'sdcard_spimode': 'sdcard running in SPI mode', - 'emmc': 'eMMC card', - 'sdcard': 'sdcard runner', - 'MSPI_F8R8': 'runner with Octal Flash and Octal PSRAM', - 'MSPI_F4R8': 'runner with Quad Flash and Octal PSRAM', - 'MSPI_F4R4': 'runner with Quad Flash and Quad PSRAM', - 'jtag': 'runner where the chip is accessible through JTAG as well', - 'usb_serial_jtag': 'runner where the chip is accessible through builtin JTAG as well', - 'xtal32k': 'Runner with external 32k crystal connected', - 'no32kXtal': 'Runner with no external 32k crystal connected', - 'psramv0': 'Runner with PSRAM version 0', - 'esp32eco3': 'Runner with esp32 eco3 connected', - 'ecdsa_efuse': 'Runner with test ECDSA private keys programmed in efuse', - 'ccs811': 'Runner with CCS811 connected', - 'nvs_encr_hmac': 'Runner with test HMAC key programmed in efuse', - 'i2c_oled': 'Runner with ssd1306 I2C oled connected', - 'httpbin': 'runner for tests that need to access the httpbin service', - 'flash_4mb': 'C2 runners with 4 MB flash', - 'jtag_re_enable': 'Runner to re-enable jtag which is softly disabled by burning bit SOFT_DIS_JTAG on eFuse', - 'es8311': 'Development board that carries es8311 codec', - # multi-dut markers - 'multi_dut_modbus_rs485': 'a pair of runners connected by RS485 bus', - 'ieee802154': 'ieee802154 related tests should run on ieee802154 runners.', - 'openthread_br': 'openthread BR basic and multicast cases.', - 'openthread_br_connect': 'openthread BR attach and connectivity cases.', - 'openthread_br_service': 'openthread BR service discovery cases.', - 'openthread_br_nat64': 'openthread BR NAT64 cases without internet.', - 'openthread_bbr': 'openthread BR cases with internet access.', - 'openthread_sleep': 'tests should be used for openthread sleepy device.', - 'zigbee_multi_dut': 'zigbee runner which have multiple duts.', - 'two_duts': 'tests should be run on runners which has two wifi duts connected.', - 'generic_multi_device': 'generic multiple devices whose corresponding gpio pins are connected to each other.', - 'twai_network': 'multiple runners form a TWAI network.', - 'sdio_master_slave': 'Test sdio multi board, esp32+esp32', - 'sdio_multidev_32_c6': 'Test sdio multi board, esp32+esp32c6', - 'usj_device': 'Test usb_serial_jtag and usb_serial_jtag is used as serial only (not console)', - 'twai_adapter': 'runner with multiple twai_std', - 'lp_i2s': 'lp_i2s runner tested with hp_i2s', - 'ram_app': 'ram_app runners', - 'esp32c3eco7': 'esp32c3 major version(v1.1) chips', - 'esp32c2eco4': 'esp32c2 major version(v2.0) chips', - 'esp32p4_eco4': 'esp32p4 major version(v1.1) chips', -} - -DEFAULT_CONFIG_RULES_STR = ['sdkconfig.ci=default', 'sdkconfig.ci.*=', '=default'] -DEFAULT_IGNORE_WARNING_FILEPATH = os.path.join(IDF_PATH, 'tools', 'ci', 'ignore_build_warnings.txt') -DEFAULT_BUILD_TEST_RULES_FILEPATH = os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml') -DEFAULT_FULL_BUILD_TEST_COMPONENTS = [ - 'cxx', - 'esp_common', - 'esp_hw_support', - 'esp_rom', - 'esp_system', - 'esp_timer', - 'freertos', - 'hal', - 'heap', - 'log', - 'newlib', - 'riscv', - 'soc', - 'xtensa', +# by default the timeout is 1h, for some special cases we need to extend it +TIMEOUT_4H_MARKERS = [ + 'ethernet_stress', ] -DEFAULT_FULL_BUILD_TEST_FILEPATTERNS = [ - # tools - 'tools/cmake/**/*', - 'tools/tools.json', - # ci - 'tools/ci/ignore_build_warnings.txt', + +REV_MARKERS = [ + 'esp32_rev3', + 'esp32c2_rev2', + 'esp32c3_rev1', + 'esp32p4_rev1', + 'esp32c5_rev1', ] -DEFAULT_BUILD_LOG_FILENAME = 'build_log.txt' -DEFAULT_SIZE_JSON_FILENAME = 'size.json' - - -class CollectMode(str, Enum): - SINGLE_SPECIFIC = 'single_specific' - MULTI_SPECIFIC = 'multi_specific' - MULTI_ALL_WITH_PARAM = 'multi_all_with_param' - ALL = 'all' - - -class PytestApp: - """ - Pytest App with relative path to IDF_PATH - """ - def __init__(self, path: str, target: str, config: str) -> None: - self.path = idf_relpath(path) - self.target = target - self.config = config - - def __hash__(self) -> int: - return hash((self.path, self.target, self.config)) - - @cached_property - def build_dir(self) -> str: - return os.path.join(self.path, f'build_{self.target}_{self.config}') - - -@dataclass -class PytestCase: - apps: t.List[PytestApp] - - item: Function - multi_dut_without_param: bool - - def __hash__(self) -> int: - return hash((self.path, self.name, self.apps, self.all_markers)) - - @cached_property - def path(self) -> str: - return str(self.item.path) - - @cached_property - def name(self) -> str: - return self.item.originalname # type: ignore - - @cached_property - def targets(self) -> t.List[str]: - if not self.multi_dut_without_param: - return [app.target for app in self.apps] - - # multi-dut test cases without parametrize - skip = True - for _t in [app.target for app in self.apps]: - if _t in self.target_markers: - skip = False - warnings.warn(f'`pytest.mark.[TARGET]` defined in parametrize for multi-dut test cases is deprecated. ' # noqa: W604 - f'Please use parametrize instead for test case {self.item.nodeid}') - break - - if not skip: - return [app.target for app in self.apps] - - return [''] * len(self.apps) # this will help to filter these cases out later - - @cached_property - def is_single_dut_test_case(self) -> bool: - return True if len(self.apps) == 1 else False - - @cached_property - def is_host_test(self) -> bool: - return 'host_test' in self.all_markers or 'linux' in self.targets - - # the following markers could be changed dynamically, don't use cached_property - @property - def all_markers(self) -> t.Set[str]: - return {marker.name for marker in self.item.iter_markers()} - - @property - def target_markers(self) -> t.Set[str]: - def _get_temp_markers_disabled_targets(marker_name: str) -> t.Set[str]: - temp_marker = self.item.get_closest_marker(marker_name) - - if not temp_marker: - return set() - - # temp markers should always use keyword arguments `targets` and `reason` - if not temp_marker.kwargs.get('targets') or not temp_marker.kwargs.get('reason'): - raise ValueError( - f'`{marker_name}` should always use keyword arguments `targets` and `reason`. ' # noqa: W604 - f'For example: ' - f'`@pytest.mark.{marker_name}(targets=["esp32"], reason="IDF-xxxx, will fix it ASAP")`' - ) - - return set(to_list(temp_marker.kwargs['targets'])) # type: ignore - - temp_skip_ci_targets = _get_temp_markers_disabled_targets('temp_skip_ci') - temp_skip_targets = _get_temp_markers_disabled_targets('temp_skip') - - # in CI we skip the union of `temp_skip` and `temp_skip_ci` - if os.getenv('CI_JOB_ID'): - skip_targets = temp_skip_ci_targets.union(temp_skip_targets) - else: # we use `temp_skip` locally - skip_targets = temp_skip_targets - - return {marker for marker in self.all_markers if marker in TARGET_MARKERS} - skip_targets - - @property - def env_markers(self) -> t.Set[str]: - return {marker for marker in self.all_markers if marker in ENV_MARKERS} - - @property - def target_selector(self) -> str: - return ','.join(app.target for app in self.apps) - - @property - def requires_elf_or_map(self) -> bool: - """ - This property determines whether the test case requires elf or map file. By default, one app in the test case - only requires .bin files. - - :return: True if the test case requires elf or map file, False otherwise - """ - if 'jtag' in self.env_markers or 'usb_serial_jtag' in self.env_markers: - return True - - cases_need_elf = [ - 'panic', - 'gdbstub_runtime' - ] - - for case in cases_need_elf: - if any(case in Path(app.path).parts for app in self.apps): - return True - - return False - - def all_built_in_app_lists(self, app_lists: t.Optional[t.List[str]] = None) -> t.Optional[str]: - """ - Check if all binaries of the test case are built in the app lists. - - :param app_lists: app lists to check - :return: debug string if not all binaries are built in the app lists, None otherwise - """ - if app_lists is None: - # ignore this feature - return None - - bin_found = [0] * len(self.apps) - for i, app in enumerate(self.apps): - if app.build_dir in app_lists: - bin_found[i] = 1 - - if sum(bin_found) == 0: - msg = f'Skip test case {self.name} because all following binaries are not listed in the app lists: ' # noqa: E713 - for app in self.apps: - msg += f'\n - {app.build_dir}' - - print(msg) - return msg - - if sum(bin_found) == len(self.apps): - return None - - # some found, some not, looks suspicious - msg = f'Found some binaries of test case {self.name} are not listed in the app lists.' # noqa: E713 - for i, app in enumerate(self.apps): - if bin_found[i] == 0: - msg += f'\n - {app.build_dir}' - - msg += '\nMight be a issue of .build-test-rules.yml files' - print(msg) - return msg diff --git a/tools/ci/idf_pytest/plugin.py b/tools/ci/idf_pytest/plugin.py index ed241bd5c8c..22b5238e985 100644 --- a/tools/ci/idf_pytest/plugin.py +++ b/tools/ci/idf_pytest/plugin.py @@ -1,106 +1,109 @@ -# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 -import importlib -import logging import os -import sys import typing as t -from collections import defaultdict -from functools import cached_property -from unittest.mock import MagicMock +from pathlib import Path from xml.etree import ElementTree as ET import pytest -from _pytest.config import ExitCode -from _pytest.main import Session +import yaml +from _pytest.config import Config from _pytest.python import Function from _pytest.runner import CallInfo -from idf_build_apps import App -from idf_build_apps.constants import BuildStatus -from idf_ci_utils import idf_relpath +from dynamic_pipelines.constants import KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH +from idf_ci import IdfPytestPlugin +from idf_ci import PytestCase +from idf_ci.idf_pytest.plugin import IDF_CI_PYTEST_DEBUG_INFO_KEY +from idf_ci_utils import to_list from pytest_embedded import Dut -from pytest_embedded.plugin import parse_multi_dut_args from pytest_embedded.utils import find_by_suffix -from pytest_embedded.utils import to_list from pytest_ignore_test_results.ignore_results import ChildCase from pytest_ignore_test_results.ignore_results import ChildCasesStashKey -from .constants import CollectMode -from .constants import DEFAULT_SDKCONFIG -from .constants import PREVIEW_TARGETS -from .constants import PytestApp -from .constants import PytestCase -from .constants import SUPPORTED_TARGETS -from .utils import comma_sep_str_to_list +from .constants import REV_MARKERS from .utils import format_case_id from .utils import merge_junit_files +from .utils import normalize_testcase_file_path -IDF_PYTEST_EMBEDDED_KEY = pytest.StashKey['IdfPytestEmbedded']() -ITEM_FAILED_CASES_KEY = pytest.StashKey[list]() -ITEM_FAILED_KEY = pytest.StashKey[bool]() -ITEM_PYTEST_CASE_KEY = pytest.StashKey[PytestCase]() +IDF_LOCAL_PLUGIN_KEY = pytest.StashKey['IdfLocalPlugin']() -class IdfPytestEmbedded: +def requires_elf_or_map(case: PytestCase) -> bool: + """ + Determines whether the test case requires elf or map file. By default, one app in the test case + only requires .bin files. + + :return: True if the test case requires elf or map file, False otherwise + """ + if 'jtag' in case.env_markers or 'usb_serial_jtag' in case.env_markers: + return True + + folders_need_elf = ['panic', 'gdbstub_runtime'] + if 'require_elf' in case.all_markers: + return True + + for folder in folders_need_elf: + if any(folder in Path(app.path).parts for app in case.apps): + return True + + return False + + +def skipped_targets(item: Function) -> t.Set[str]: + def _get_temp_markers_disabled_targets(marker_name: str) -> t.Set[str]: + targets = [] + for _m in item.own_markers: + if _m.name == marker_name: + if not _m.kwargs.get('targets') or not _m.kwargs.get('reason'): + raise ValueError( + f'`{marker_name}` should always use keyword arguments `targets` and `reason`. ' # noqa: W604 + f'For example: ' + f'`@pytest.mark.{marker_name}(targets=["esp32"], reason="IDF-xxxx, will fix it ASAP")`' + ) + targets.extend(to_list(_m.kwargs['targets'])) + + if targets: + return set(targets) + return set() + + _count = IdfLocalPlugin.get_param(item, 'count', 1) + + def normalize_targets(target: str) -> str: + targets = target.split(',') + if len(targets) == 1: + return ','.join(targets * _count) + if len(targets) != _count: + raise ValueError( + f"Invalid target format: '{target}'. " + f'Expected a single target or exactly {_count} values separated by commas.' + f'len({targets}) != {_count}' + ) + return target + + temp_skip_ci_targets = set(normalize_targets(_t) for _t in _get_temp_markers_disabled_targets('temp_skip_ci')) + temp_skip_targets = set(normalize_targets(_t) for _t in _get_temp_markers_disabled_targets('temp_skip')) + + # in CI we skip the union of `temp_skip` and `temp_skip_ci` + if os.getenv('CI_JOB_ID'): + _skip_targets = temp_skip_ci_targets.union(temp_skip_targets) + else: # we use `temp_skip` locally + _skip_targets = temp_skip_targets + + return _skip_targets + + +class IdfLocalPlugin: UNITY_RESULT_MAPPINGS = { 'PASS': 'passed', 'FAIL': 'failed', 'IGNORE': 'skipped', } - def __init__( - self, - target: t.Union[t.List[str], str], - *, - config_name: t.Optional[str] = None, - single_target_duplicate_mode: bool = False, - apps: t.Optional[t.List[App]] = None, - ): - if isinstance(target, str): - # sequence also matters - self.target = comma_sep_str_to_list(target) - else: - self.target = target + def __init__(self) -> None: + with open(KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH) as fr: + known_warnings_dict = yaml.safe_load(fr) or dict() - if not self.target: - raise ValueError('`target` should not be empty') - - self.config_name = config_name - - # these are useful while gathering all the multi-dut test cases - # when this mode is activated, - # - # pytest.mark.esp32 - # pytest.mark.parametrize('count', [2], indirect=True) - # def test_foo(dut): - # pass - # - # should be collected when running `pytest --target esp32` - # - # otherwise, it should be collected when running `pytest --target esp32,esp32` - self._single_target_duplicate_mode = single_target_duplicate_mode - - self.apps_list = ( - [os.path.join(idf_relpath(app.app_dir), app.build_dir) for app in apps if app.build_status == BuildStatus.SUCCESS] - if apps is not None - else None - ) - - self.cases: t.List[PytestCase] = [] - - # record the additional info - # test case id: {key: value} - self.additional_info: t.Dict[str, t.Dict[str, t.Any]] = defaultdict(dict) - - @cached_property - def collect_mode(self) -> CollectMode: - if len(self.target) == 1: - if self.target[0] == CollectMode.MULTI_ALL_WITH_PARAM: - return CollectMode.MULTI_ALL_WITH_PARAM - else: - return CollectMode.SINGLE_SPECIFIC - else: - return CollectMode.MULTI_SPECIFIC + self.exclude_no_env_markers_test_cases: set[str] = set(known_warnings_dict['no_env_marker_test_cases']) @staticmethod def get_param(item: Function, key: str, default: t.Any = None) -> t.Any: @@ -111,207 +114,82 @@ class IdfPytestEmbedded: return item.callspec.params.get(key, default) or default - def item_to_pytest_case(self, item: Function) -> t.Optional[PytestCase]: - """ - Turn pytest item to PytestCase - """ - count = self.get_param(item, 'count', 1) + @pytest.hookimpl(wrapper=True) + def pytest_collection_modifyitems(self, config: Config, items: t.List[Function]) -> t.Generator[None, None, None]: + yield # throw it back to idf-ci - # default app_path is where the test script locates - app_paths = to_list(parse_multi_dut_args(count, self.get_param(item, 'app_path', os.path.dirname(item.path)))) - configs = to_list(parse_multi_dut_args(count, self.get_param(item, 'config', DEFAULT_SDKCONFIG))) - targets = to_list(parse_multi_dut_args(count, self.get_param(item, 'target'))) + deselected_items = [] - multi_dut_without_param = False - if count > 1 and targets == [None] * count: - multi_dut_without_param = True - try: - targets = to_list(parse_multi_dut_args(count, '|'.join(self.target))) # check later while collecting - except ValueError: # count doesn't match - return None - - elif targets is None: - targets = self.target - - return PytestCase( - apps=[PytestApp(app_paths[i], targets[i], configs[i]) for i in range(count)], - item=item, - multi_dut_without_param=multi_dut_without_param - ) - - def pytest_collectstart(self) -> None: - # mock the optional packages while collecting locally - if not os.getenv('CI_JOB_ID') or os.getenv('PYTEST_IGNORE_COLLECT_IMPORT_ERROR') == '1': - # optional packages required by test scripts - for p in [ - 'scapy', - 'scapy.all', - 'websocket', # websocket-client - 'netifaces', - 'RangeHTTPServer', # rangehttpserver - 'dbus', # dbus-python - 'dbus.mainloop', - 'dbus.mainloop.glib', - 'google.protobuf', # protobuf - 'google.protobuf.internal', - 'bleak', - 'paho', # paho-mqtt - 'paho.mqtt', - 'paho.mqtt.client', - 'paramiko', - 'netmiko', - 'pyecharts', - 'pyecharts.options', - 'pyecharts.charts', - 'can', # python-can - ]: - try: - importlib.import_module(p) - except ImportError: - logging.warning(f'Optional package {p} is not installed, mocking it while collecting...') - sys.modules[p] = MagicMock() - - @pytest.hookimpl(tryfirst=True) - def pytest_collection_modifyitems(self, items: t.List[Function]) -> None: - """ - Background info: - - We're using `pytest.mark.[TARGET]` as a syntactic sugar to indicate that they are actually supported by all - the listed targets. For example, - - >>> @pytest.mark.esp32 - >>> @pytest.mark.esp32s2 - - should be treated as - - >>> @pytest.mark.parametrize('target', [ - >>> 'esp32', - >>> 'esp32s2', - >>> ], indirect=True) - - All single-dut test cases, and some of the multi-dut test cases with the same targets, are using this - way to indicate the supported targets. - - To avoid ambiguity, - - - when we're collecting single-dut test cases with esp32, we call - - `pytest --collect-only --target esp32` - - - when we're collecting multi-dut test cases, we list all the targets, even when they're the same - - `pytest --collect-only --target esp32,esp32` for two esp32 connected - `pytest --collect-only --target esp32,esp32s2` for esp32 and esp32s2 connected - - therefore, we have two different logic for searching test cases, explained in 2.1 and 2.2 - """ - # 1. Filter according to nighty_run related markers - if os.getenv('INCLUDE_NIGHTLY_RUN') == '1': - # nightly_run and non-nightly_run cases are both included - pass - elif os.getenv('NIGHTLY_RUN') == '1': - # only nightly_run cases are included - items[:] = [_item for _item in items if _item.get_closest_marker('nightly_run') is not None] - else: - # only non-nightly_run cases are included - items[:] = [_item for _item in items if _item.get_closest_marker('nightly_run') is None] - - # 2. Add markers according to special markers - item_to_case_dict: t.Dict[Function, PytestCase] = {} + # Filter + filtered_items = [] for item in items: - case = self.item_to_pytest_case(item) - if case is None: + case = IdfPytestPlugin.get_case_by_item(item) + if not case: + deselected_items.append(item) continue - item.stash[ITEM_PYTEST_CASE_KEY] = item_to_case_dict[item] = case - if 'supported_targets' in item.keywords: - for _target in SUPPORTED_TARGETS: - item.add_marker(_target) - if 'preview_targets' in item.keywords: - for _target in PREVIEW_TARGETS: - item.add_marker(_target) - if 'all_targets' in item.keywords: - for _target in [*SUPPORTED_TARGETS, *PREVIEW_TARGETS]: - item.add_marker(_target) + skipped = False + current_targets = case.target_selector.split(',') + for st in skipped_targets(item): + # Handle wildcard patterns like "esp32p4,*" or "*,esp32p4" + if '*' in st: + skip_pattern = st.split(',') + if len(skip_pattern) != len(current_targets): + continue + if all(_p == '*' or _p == _t for _p, _t in zip(skip_pattern, current_targets)): + skipped = True + break + # Exact match (no wildcard) + elif case.target_selector == st: + skipped = True + break - # add single-dut "target" as param - _item_target_param = self.get_param(item, 'target', None) - if case.is_single_dut_test_case and _item_target_param and _item_target_param not in case.all_markers: - item.add_marker(_item_target_param) + if skipped: + deselected_items.append(item) + item.stash[IDF_CI_PYTEST_DEBUG_INFO_KEY] = 'skipped by temp_skip markers' + continue - items[:] = [_item for _item in items if _item in item_to_case_dict] + if not case.env_markers and 'host_test' not in case.all_markers: + if case.name in self.exclude_no_env_markers_test_cases: + deselected_items.append(item) + continue - # 3.1. CollectMode.SINGLE_SPECIFIC, like `pytest --target esp32` - if self.collect_mode == CollectMode.SINGLE_SPECIFIC: - filtered_items = [] - for item in items: - case = item_to_case_dict[item] + raise ValueError( + f'Test case {case.name} does not have any env markers. ' + f'Please add env markers to the test case or add it to the ' + f'`no_env_markers_test_cases` list in {KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH}' + ) - # single-dut one - if case.is_single_dut_test_case and self.target[0] in case.target_markers: - filtered_items.append(item) + filtered_items.append(item) - # multi-dut ones and in single_target_duplicate_mode - elif self._single_target_duplicate_mode and not case.is_single_dut_test_case: - # ignore those test cases with `target` defined in parametrize, since these will be covered in 3.3 - if self.get_param(item, 'target', None) is None and self.target[0] in case.target_markers: - filtered_items.append(item) + items[:] = filtered_items - items[:] = filtered_items - # 3.2. CollectMode.MULTI_SPECIFIC, like `pytest --target esp32,esp32` - elif self.collect_mode == CollectMode.MULTI_SPECIFIC: - items[:] = [_item for _item in items if item_to_case_dict[_item].targets == self.target] - - # 3.3. CollectMode.MULTI_ALL_WITH_PARAM, intended to be used by `get_pytest_cases` - else: - items[:] = [ - _item - for _item in items - if not item_to_case_dict[_item].is_single_dut_test_case - and self.get_param(_item, 'target', None) is not None - ] - - # 4. filter according to the sdkconfig, if there's param 'config' defined - if self.config_name: - _items = [] - for item in items: - case = item_to_case_dict[item] - if self.config_name not in set(app.config or DEFAULT_SDKCONFIG for app in case.apps): - self.additional_info[case.name]['skip_reason'] = f'Only run with sdkconfig {self.config_name}' - else: - _items.append(item) - items[:] = _items - - # 5. filter by `self.apps_list`, skip the test case if not listed - # should only be used in CI - _items = [] - for item in items: - case = item_to_case_dict[item] - if msg := case.all_built_in_app_lists(self.apps_list): - self.additional_info[case.name]['skip_reason'] = msg - else: - _items.append(item) + # Deselect + config.hook.pytest_deselected(items=deselected_items) # OKAY!!! All left ones will be executed, sort it and add more markers - items[:] = sorted( - _items, key=lambda x: (os.path.dirname(x.path), self.get_param(x, 'config', DEFAULT_SDKCONFIG)) - ) + items[:] = sorted(items, key=lambda x: (os.path.dirname(x.path), self.get_param(x, 'config', 'default'))) + for item in items: - case = item_to_case_dict[item] + case = IdfPytestPlugin.get_case_by_item(item) + # set default timeout 10 minutes for each case if 'timeout' not in item.keywords: item.add_marker(pytest.mark.timeout(10 * 60)) # add 'xtal_40mhz' tag as a default tag for esp32c2 target - # only add this marker for esp32c2 cases - if 'esp32c2' in self.target and 'esp32c2' in case.targets and 'xtal_26mhz' not in case.all_markers: + if 'esp32c2' in case.targets and 'xtal_26mhz' not in case.all_markers: item.add_marker('xtal_40mhz') - if 'esp32p4' in self.target and 'esp32p4' in case.targets: - item.add_marker('esp32p4_eco4') + for rev_marker in REV_MARKERS: + if rev_marker in case.all_markers: + break + else: + # ESP32-P4's default build supports revisions below 3.0. + item.add_marker('esp32p4_rev1' if 'esp32p4' in case.targets else 'rev_default') - def pytest_report_collectionfinish(self, items: t.List[Function]) -> None: - self.cases = [item.stash[ITEM_PYTEST_CASE_KEY] for item in items] + if 'host_test' in case.all_markers: + item.add_marker('skip_app_downloader') # host_test jobs will build the apps itself def pytest_custom_test_case_name(self, item: Function) -> str: return item.funcargs.get('test_case_name', item.nodeid) # type: ignore @@ -368,22 +246,20 @@ class IdfPytestEmbedded: is_qemu = item.get_closest_marker('qemu') is not None target = item.funcargs['target'] config = item.funcargs['config'] + app_path = item.funcargs.get('app_path') for junit in junits: xml = ET.parse(junit) testcases = xml.findall('.//testcase') for case in testcases: # modify the junit files + # Use from case attrib if available, otherwise fallback to the previously defined + app_path = case.attrib.get('app_path') or app_path new_case_name = format_case_id(target, config, case.attrib['name'], is_qemu=is_qemu) case.attrib['name'] = new_case_name if 'file' in case.attrib: - case.attrib['file'] = case.attrib['file'].replace('/IDF/', '') # our unity test framework - + # our unity test framework + case.attrib['file'] = normalize_testcase_file_path(case.attrib['file'], app_path) if ci_job_url := os.getenv('CI_JOB_URL'): case.attrib['ci_job_url'] = ci_job_url xml.write(junit) - - def pytest_sessionfinish(self, session: Session, exitstatus: int) -> None: - if exitstatus != 0: - if exitstatus == ExitCode.NO_TESTS_COLLECTED: - session.exitstatus = 0 diff --git a/tools/ci/idf_pytest/pytest.ini b/tools/ci/idf_pytest/pytest.ini deleted file mode 100644 index cc928df8995..00000000000 --- a/tools/ci/idf_pytest/pytest.ini +++ /dev/null @@ -1,3 +0,0 @@ -[pytest] -addopts = -p no:idf-ci -python_files = test_*.py diff --git a/tools/ci/idf_pytest/script.py b/tools/ci/idf_pytest/script.py deleted file mode 100644 index 9c537486997..00000000000 --- a/tools/ci/idf_pytest/script.py +++ /dev/null @@ -1,236 +0,0 @@ -# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import fnmatch -import io -import logging -import os.path -import typing as t -from contextlib import redirect_stdout -from pathlib import Path - -import pytest -from _pytest.config import ExitCode -from idf_build_apps import App -from idf_build_apps import find_apps -from idf_build_apps.constants import BuildStatus -from idf_build_apps.constants import SUPPORTED_TARGETS -from idf_ci_local.app import IdfCMakeApp -from idf_ci_utils import get_all_manifest_files -from idf_ci_utils import IDF_PATH -from idf_ci_utils import idf_relpath -from idf_ci_utils import to_list -from idf_py_actions.constants import PREVIEW_TARGETS as TOOLS_PREVIEW_TARGETS -from idf_py_actions.constants import SUPPORTED_TARGETS as TOOLS_SUPPORTED_TARGETS - -from .constants import CollectMode -from .constants import DEFAULT_BUILD_LOG_FILENAME -from .constants import DEFAULT_CONFIG_RULES_STR -from .constants import DEFAULT_SIZE_JSON_FILENAME -from .constants import PytestCase -from .plugin import IdfPytestEmbedded - - -def get_pytest_files(paths: t.List[str]) -> t.List[str]: - # this is a workaround to solve pytest collector super slow issue - # benchmark with - # - time pytest -m esp32 --collect-only - # user=15.57s system=1.35s cpu=95% total=17.741 - # - time { find -name 'pytest_*.py'; } | xargs pytest -m esp32 --collect-only - # user=0.11s system=0.63s cpu=36% total=2.044 - # user=1.76s system=0.22s cpu=43% total=4.539 - # use glob.glob would also save a bunch of time - pytest_scripts: t.Set[str] = set() - for p in paths: - path = Path(p) - pytest_scripts.update(str(_p) for _p in path.glob('**/pytest_*.py') if 'managed_components' not in _p.parts) - - return list(pytest_scripts) - - -def get_pytest_cases( - paths: t.Union[str, t.List[str]], - target: str = CollectMode.ALL, - *, - config_name: t.Optional[str] = None, - marker_expr: t.Optional[str] = None, - filter_expr: t.Optional[str] = None, - apps: t.Optional[t.List[App]] = None, -) -> t.List[PytestCase]: - """ - Return the list of test cases - - For single-dut test cases, `target` could be - - [TARGET], e.g. `esp32`, to get the test cases for the given target - - or `single_all`, to get all single-dut test cases - - For multi-dut test cases, `target` could be - - [TARGET,[TARGET...]], e.g. `esp32,esp32s2`, to get the test cases for the given targets - - or `multi_all`, to get all multi-dut test cases - - :param paths: paths to search for pytest scripts - :param target: target or keywords to get test cases for, detailed above - :param config_name: sdkconfig name - :param marker_expr: pytest marker expression, `-m` - :param filter_expr: pytest filter expression, `-k` - :param apps: built app list, skip the tests required by apps not in the list - :return: list of test cases - """ - paths = to_list(paths) - - cases: t.List[PytestCase] = [] - pytest_scripts = get_pytest_files(paths) # type: ignore - if not pytest_scripts: - print(f'WARNING: no pytest scripts found for target {target} under paths {", ".join(paths)}') - return cases - - def _get_pytest_cases(_target: str, _single_target_duplicate_mode: bool = False) -> t.List[PytestCase]: - collector = IdfPytestEmbedded(_target, config_name=config_name, single_target_duplicate_mode=_single_target_duplicate_mode, apps=apps) - - with io.StringIO() as buf: - with redirect_stdout(buf): - cmd = ['--collect-only', *pytest_scripts, '--target', _target, '-q'] - if marker_expr: - cmd.extend(['-m', marker_expr]) - if filter_expr: - cmd.extend(['-k', filter_expr]) - res = pytest.main(cmd, plugins=[collector]) - - if res.value != ExitCode.OK: - if res.value == ExitCode.NO_TESTS_COLLECTED: - print(f'WARNING: no pytest app found for target {_target} under paths {", ".join(paths)}') - else: - print(buf.getvalue()) - raise RuntimeError( - f'pytest collection failed at {", ".join(paths)} with command \"{" ".join(cmd)}\"' - ) - - return collector.cases # type: ignore - - if target == CollectMode.ALL: - targets = TOOLS_SUPPORTED_TARGETS + TOOLS_PREVIEW_TARGETS + [CollectMode.MULTI_ALL_WITH_PARAM] - else: - targets = [target] - - for _target in targets: - if target == CollectMode.ALL: - cases.extend(_get_pytest_cases(_target, _single_target_duplicate_mode=True)) - else: - cases.extend(_get_pytest_cases(_target)) - - return sorted(cases, key=lambda x: (x.path, x.name, str(x.targets))) - - -def get_all_apps( - paths: t.List[str], - target: str = CollectMode.ALL, - *, - marker_expr: t.Optional[str] = None, - filter_expr: t.Optional[str] = None, - config_rules_str: t.Optional[t.List[str]] = None, - preserve_all: bool = False, - extra_default_build_targets: t.Optional[t.List[str]] = None, - compare_manifest_sha_filepath: t.Optional[str] = None, - modified_components: t.Optional[t.List[str]] = None, - modified_files: t.Optional[t.List[str]] = None, - ignore_app_dependencies_components: t.Optional[t.List[str]] = None, - ignore_app_dependencies_filepatterns: t.Optional[t.List[str]] = None, -) -> t.Tuple[t.Set[App], t.Set[App]]: - """ - Return the tuple of test-required apps and non-test-related apps - - :param paths: paths to search for pytest scripts - :param target: target or keywords to get test cases for, explained in `get_pytest_cases` - :param marker_expr: pytest marker expression, `-m` - :param filter_expr: pytest filter expression, `-k` - :param config_rules_str: config rules string - :param preserve_all: preserve all apps - :param extra_default_build_targets: extra default build targets - :param compare_manifest_sha_filepath: check manifest sha filepath - :param modified_components: modified components - :param modified_files: modified files - :param ignore_app_dependencies_components: ignore app dependencies components - :param ignore_app_dependencies_filepatterns: ignore app dependencies filepatterns - :return: tuple of test-required apps and non-test-related apps - """ - # target could be comma separated list - all_apps: t.List[App] = [] - for _t in set(target.split(',')): - all_apps.extend(find_apps( - paths, - _t, - build_system=IdfCMakeApp, - recursive=True, - build_dir='build_@t_@w', - config_rules_str=config_rules_str or DEFAULT_CONFIG_RULES_STR, - build_log_filename=DEFAULT_BUILD_LOG_FILENAME, - size_json_filename=DEFAULT_SIZE_JSON_FILENAME, - check_warnings=True, - manifest_rootpath=IDF_PATH, - compare_manifest_sha_filepath=compare_manifest_sha_filepath, - manifest_files=get_all_manifest_files(), - default_build_targets=SUPPORTED_TARGETS + (extra_default_build_targets or []), - modified_components=modified_components, - modified_files=modified_files, - ignore_app_dependencies_components=ignore_app_dependencies_components, - ignore_app_dependencies_filepatterns=ignore_app_dependencies_filepatterns, - include_skipped_apps=True, - )) - - pytest_cases = get_pytest_cases( - paths, - target, - marker_expr=marker_expr, - filter_expr=filter_expr, - ) - - modified_pytest_cases = [] - if modified_files: - modified_pytest_scripts = [ - os.path.dirname(f) for f in modified_files if fnmatch.fnmatch(os.path.basename(f), 'pytest_*.py') - ] - if modified_pytest_scripts: - modified_pytest_cases = get_pytest_cases( - modified_pytest_scripts, - target, - marker_expr=marker_expr, - filter_expr=filter_expr, - ) - - # app_path, target, config - pytest_app_path_tuple_dict: t.Dict[t.Tuple[str, str, str], PytestCase] = {} - for case in pytest_cases: - for app in case.apps: - pytest_app_path_tuple_dict[(app.path, app.target, app.config)] = case - - modified_pytest_app_path_tuple_dict: t.Dict[t.Tuple[str, str, str], PytestCase] = {} - for case in modified_pytest_cases: - for app in case.apps: - modified_pytest_app_path_tuple_dict[(app.path, app.target, app.config)] = case - - test_related_apps: t.Set[App] = set() - non_test_related_apps: t.Set[App] = set() - for app in all_apps: - # PytestCase.app.path is idf_relpath - app_path = idf_relpath(app.app_dir) - - # override build_status if test script got modified - if case := modified_pytest_app_path_tuple_dict.get((app_path, app.target, app.config_name)): - test_related_apps.add(app) - app.build_status = BuildStatus.SHOULD_BE_BUILT - app.preserve = True - logging.debug('Found app: %s - required by modified test case %s', app, case.path) - elif app.build_status != BuildStatus.SKIPPED: - if case := pytest_app_path_tuple_dict.get((app_path, app.target, app.config_name)): - test_related_apps.add(app) - # build or not should be decided by the build stage - app.preserve = True - logging.debug('Found test-related app: %s - required by %s', app, case.path) - else: - non_test_related_apps.add(app) - app.preserve = preserve_all - logging.debug('Found non-test-related app: %s', app) - - print(f'Found {len(test_related_apps)} test-related apps') - print(f'Found {len(non_test_related_apps)} non-test-related apps') - - return test_related_apps, non_test_related_apps diff --git a/tools/ci/idf_pytest/utils.py b/tools/ci/idf_pytest/utils.py index 1e06354668e..f3a1fd71dae 100644 --- a/tools/ci/idf_pytest/utils.py +++ b/tools/ci/idf_pytest/utils.py @@ -1,6 +1,5 @@ -# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 - import logging import os import typing as t @@ -31,8 +30,8 @@ def merge_junit_files(junit_files: t.List[str], target_path: str) -> None: testcases: t.Dict[str, ET.Element] = {} for junit in junit_files: logging.info(f'Merging {junit} to {target_path}') - tree: ET.ElementTree = ET.parse(junit) - testsuite: ET.Element = tree.getroot() + tree: ET.ElementTree = ET.parse(junit) # type: ignore + testsuite: ET.Element = tree.getroot() # type: ignore for testcase in testsuite.findall('testcase'): name: str = testcase.get('name') if testcase.get('name') else '' # type: ignore @@ -63,5 +62,35 @@ def merge_junit_files(junit_files: t.List[str], target_path: str) -> None: fw.write(ET.tostring(merged_testsuite)) -def comma_sep_str_to_list(s: str) -> t.List[str]: - return [s.strip() for s in s.split(',') if s.strip()] +def normalize_testcase_file_path(file: str, app_path: t.Union[str, tuple]) -> str: + """ + Normalize file paths to a consistent format, resolving relative paths based on the `app_path`. + + This function ensures that file paths are correctly resolved and normalized: + - If `app_path` is a tuple, the function will try each app path in the tuple in order and join the results with ':'. + - If `app_path` is a string, the file will be resolved using that base path. + + :param file: The original file path, which can be relative, absolute. + :param app_path: The base app path used to resolve relative paths, which can be a string or tuple. + :return: A normalized file path with the IDF_PATH prefix removed if applicable. + """ + + def normalize_path(file_path: str, app_path: str) -> str: + """Helper function to normalize a single path.""" + if not os.path.isabs(file_path): + # startswith/slice instead of str.removeprefix (3.9+), keep 3.8 compat + if file_path.startswith('./'): + file_path = file_path[2:] + if file_path.startswith('../'): + file_path = file_path[3:] + resolved_path = os.path.normpath(os.path.join(app_path, file_path)) + else: + resolved_path = os.path.normpath(file_path) + + return resolved_path.replace(f'{os.environ.get("IDF_PATH", "")}', '').replace('/IDF/', '').lstrip('/') + + if isinstance(app_path, tuple): + normalized_paths = [normalize_path(file, base_path) for base_path in app_path] + return '|'.join(normalized_paths) + + return normalize_path(file, app_path) diff --git a/tools/ci/previous_stage_job_status.py b/tools/ci/previous_stage_job_status.py deleted file mode 100644 index 3888780d407..00000000000 --- a/tools/ci/previous_stage_job_status.py +++ /dev/null @@ -1,36 +0,0 @@ -# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import argparse -import os -import sys - -import requests - -if __name__ == '__main__': - - parser = argparse.ArgumentParser() - parser.add_argument('--stage', type=str, help='Stage name for check jobs status') - args = parser.parse_args() - - GITLAB_TOKEN = os.getenv('ESPCI_TOKEN') - GITLAB_HTTP_SERVER = os.getenv('GITLAB_HTTP_SERVER') - CI_PROJECT_ID = os.getenv('CI_PROJECT_ID') - CI_PIPELINE_ID = os.getenv('CI_PIPELINE_ID') - - api_path = f'projects/{CI_PROJECT_ID}/pipelines/{CI_PIPELINE_ID}/jobs?scope[]=failed&per_page=100' - - page = 0 - while True: - response = requests.get( - f'{GITLAB_HTTP_SERVER}/api/v4/{api_path}&page={page}', - headers={'PRIVATE-TOKEN': GITLAB_TOKEN} - ) - jobs = response.json() - if not jobs: - break - - for job in jobs: - if job['stage'] == args.stage: - print(f'Jobs from the previous stage {args.stage} should pass; otherwise, this job will fail.') - sys.exit(1) - page += 1 diff --git a/tools/ci/python_packages/gitlab_api.py b/tools/ci/python_packages/gitlab_api.py index 6ac0ffae1f8..caa856aba0f 100644 --- a/tools/ci/python_packages/gitlab_api.py +++ b/tools/ci/python_packages/gitlab_api.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import argparse import logging @@ -36,7 +36,7 @@ def retry(func: TR) -> TR: while True: try: res = func(self, *args, **kwargs) - except (IOError, EOFError, gitlab.exceptions.GitlabError) as e: + except (OSError, EOFError, gitlab.exceptions.GitlabError) as e: if isinstance(e, gitlab.exceptions.GitlabError): if e.response_code == 500: # retry on this error @@ -52,9 +52,11 @@ def retry(func: TR) -> TR: raise e # get out of the loop else: logging.warning( - 'Network failure in {}, retrying ({})'.format(getattr(func, '__name__', '(unknown callable)'), - retried)) - time.sleep(2 ** retried) # wait a bit more after each retry + 'Network failure in {}, retrying ({})'.format( + getattr(func, '__name__', '(unknown callable)'), retried + ) + ) + time.sleep(2**retried) # wait a bit more after each retry continue else: break @@ -63,11 +65,10 @@ def retry(func: TR) -> TR: return wrapper -class Gitlab(object): +class Gitlab: JOB_NAME_PATTERN = re.compile(r'(\w+)(\s+(\d+)/(\d+))?') DOWNLOAD_ERROR_MAX_RETRIES = 3 - DEFAULT_BUILD_CHILD_PIPELINE_NAME = 'Build Child Pipeline' def __init__(self, project_id: Union[int, str, None] = None): config_data_from_env = os.getenv('PYTHON_GITLAB_CONFIG') @@ -117,7 +118,7 @@ class Gitlab(object): :param namespace: namespace to match when we have multiple project with same name :return: project ID """ - projects = self.gitlab_inst.projects.list(search=name) + projects = self.gitlab_inst.projects.list(search=name, get_all=True) res = [] for project in projects: if namespace is None: @@ -152,7 +153,9 @@ class Gitlab(object): archive_file.extractall(destination) @retry - def download_artifact(self, job_id: int, artifact_path: List[str], destination: Optional[str] = None) -> List[bytes]: + def download_artifact( + self, job_id: int, artifact_path: List[str], destination: Optional[str] = None + ) -> List[bytes]: """ download specific path of job artifacts and extract to destination. @@ -169,7 +172,7 @@ class Gitlab(object): try: data = job.artifact(a_path) # type: bytes except gitlab.GitlabGetError as e: - logging.error("Failed to download '{}' from job {}".format(a_path, job_id)) + logging.error(f"Failed to download '{a_path}' from job {job_id}") raise e raw_data_list.append(data) if destination: @@ -208,8 +211,9 @@ class Gitlab(object): return job_id_list @retry - def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None, - cache_dir: Optional[str] = None) -> str: + def download_archive( + self, ref: str, destination: str, project_id: Optional[int] = None, cache_dir: Optional[str] = None + ) -> str: """ Download archive of certain commit of a repository and extract to destination path @@ -233,10 +237,11 @@ class Gitlab(object): try: project.repository_archive(sha=ref, streamed=True, action=fw.write) except gitlab.GitlabGetError as e: - logging.error('Failed to archive from project {}'.format(project_id)) + logging.error(f'Failed to archive from project {project_id}') raise e - logging.info('Downloaded archive size: {:.03f}MB'.format( - float(os.path.getsize(local_archive_file)) / (1024 * 1024))) + logging.info( + f'Downloaded archive size: {float(os.path.getsize(local_archive_file)) / (1024 * 1024):.03f}MB' + ) return self.decompress_archive(local_archive_file, destination) @@ -245,10 +250,10 @@ class Gitlab(object): try: project.repository_archive(sha=ref, streamed=True, action=temp_file.write) except gitlab.GitlabGetError as e: - logging.error('Failed to archive from project {}'.format(project_id)) + logging.error(f'Failed to archive from project {project_id}') raise e - logging.info('Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(temp_file.name)) / (1024 * 1024))) + logging.info(f'Downloaded archive size: {float(os.path.getsize(temp_file.name)) / (1024 * 1024):.03f}MB') return self.decompress_archive(temp_file.name, destination) @@ -282,36 +287,28 @@ class Gitlab(object): def get_downstream_pipeline_ids(self, main_pipeline_id: int) -> List[int]: """ - Retrieve the IDs of all downstream child pipelines for a given main pipeline. + Retrieve the IDs of all downstream child pipelines for a given main pipeline, + recursing through arbitrarily nested child pipelines. :param main_pipeline_id: The ID of the main pipeline to start the search. - :return: A list of IDs of all downstream child pipelines. + :return: A list of IDs of all downstream child pipelines (all levels). """ - bridge_pipeline_ids = [] - child_pipeline_ids = [] + child_pipeline_ids: List[int] = [] - main_pipeline_bridges = self.project.pipelines.get(main_pipeline_id).bridges.list() - for bridge in main_pipeline_bridges: + pipeline_bridges = self.project.pipelines.get(main_pipeline_id).bridges.list() + for bridge in pipeline_bridges: downstream_pipeline = bridge.attributes.get('downstream_pipeline') if not downstream_pipeline: continue - bridge_pipeline_ids.append(downstream_pipeline['id']) - - for bridge_pipeline_id in bridge_pipeline_ids: - child_pipeline_ids.append(bridge_pipeline_id) - bridge_pipeline = self.project.pipelines.get(bridge_pipeline_id) - - if not bridge_pipeline.name == self.DEFAULT_BUILD_CHILD_PIPELINE_NAME: + downstream_pipeline_id = downstream_pipeline.get('id') + if downstream_pipeline_id is None: continue - child_bridges = bridge_pipeline.bridges.list() - for child_bridge in child_bridges: - downstream_child_pipeline = child_bridge.attributes.get('downstream_pipeline') - if not downstream_child_pipeline: - continue - child_pipeline_ids.append(downstream_child_pipeline.get('id')) + child_pipeline_ids.append(downstream_pipeline_id) + # recurse to collect further nested (grandchild+) pipelines + child_pipeline_ids.extend(self.get_downstream_pipeline_ids(downstream_pipeline_id)) - return [pid for pid in child_pipeline_ids if pid is not None] + return child_pipeline_ids def retry_failed_jobs(self, pipeline_id: int, retry_allowed_failures: bool = False) -> List[int]: """ @@ -370,10 +367,10 @@ def main() -> None: gitlab_inst.download_archive(args.ref, args.destination) elif args.action == 'get_project_id': ret = gitlab_inst.get_project_id(args.project_name) - print('project id: {}'.format(ret)) + print(f'project id: {ret}') elif args.action == 'retry_failed_jobs': res = gitlab_inst.retry_failed_jobs(args.pipeline_id, args.retry_allowed_failures) - print('jobs retried successfully: {}'.format(res)) + print(f'jobs retried successfully: {res}') elif args.action == 'get_job_tags': ret = gitlab_inst.get_job_tags(args.job_id) print(ret) diff --git a/tools/ci/utils.sh b/tools/ci/utils.sh index 17951e2173a..9fcea7fd53a 100644 --- a/tools/ci/utils.sh +++ b/tools/ci/utils.sh @@ -5,7 +5,14 @@ function add_ssh_keys() { mkdir -p ~/.ssh chmod 700 ~/.ssh echo -n "${key_string}" >~/.ssh/id_rsa_base64 - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 >~/.ssh/id_rsa + # Detect base64 implementation via --help output + if base64 --help 2>&1 | grep -q -- '--ignore-garbage'; then + # GNU coreutils base64 + base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 >~/.ssh/id_rsa + else + # macOS/BSD base64 - requires stdin or -i flag + base64 --decode -i ~/.ssh/id_rsa_base64 -o ~/.ssh/id_rsa + fi chmod 600 ~/.ssh/id_rsa } @@ -96,6 +103,25 @@ function run_cmd() { fi } +function pytest_for_ut() { + pytest_args="-c NUL -p no:idf-ci -p no:pytest_embedded" + + if [ -n "${CI_JOB_ID-}" ]; then + if [ -z "${KNOWN_FAILURE_CASES_FILE_NAME-}" ]; then + echo "Error: KNOWN_FAILURE_CASES_FILE_NAME is not set." + return 1 + fi + + pytest_args="${pytest_args} --junitxml XUNIT_RESULT_${CI_JOB_ID}.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}" + + if [ ! -e "${KNOWN_FAILURE_CASES_FILE_NAME}" ]; then + run_cmd idf-ci gitlab download-known-failure-cases-file "${KNOWN_FAILURE_CASES_FILE_NAME}" + fi + fi + + run_cmd pytest "$@" "${pytest_args}" +} + # Retries a command RETRY_ATTEMPTS times in case of failure # Inspired by https://stackoverflow.com/a/8351489 function retry_failed() { @@ -138,38 +164,3 @@ function join_by { printf %s "$f" "${@/#/$d}" fi } - -function is_based_on_commits() { - # This function would accept space-separated args as multiple commits. - # The return value would be 0 if current HEAD is based on any of the specified commits. - # - # In our CI, we use environment variable $REQUIRED_ANCESTOR_COMMITS to declare the ancestor commits. - # Please remember to set one commit for each release branch. - - commits=$* - if [[ -z $commits ]]; then - info "Not specifying commits that branches should be based on, skipping check..." - return 0 - fi - - commits_str="$(join_by " or " $commits)" # no doublequotes here, passing array - - info "Checking if current branch is based on $commits_str..." - for i in $commits; do - if git merge-base --is-ancestor "$i" HEAD >/dev/null 2>&1; then - info "Current branch is based on $i" - return 0 - else - info "Current branch is not based on $i" - fi - done - - error "The base commit of your branch is too old." - error "The branch should be more recent than either of the following commits:" - error " $commits_str" - error "To fix the issue:" - error " - If your merge request is 'Draft', or has conflicts with the target branch, rebase it to the latest master or release branch" - error " - Otherwise, simply run a new pipeline." - - return 1 -} diff --git a/tools/esp_prov/README.md b/tools/esp_prov/README.md index 02fc10bd8d0..04c232fa4f2 100644 --- a/tools/esp_prov/README.md +++ b/tools/esp_prov/README.md @@ -111,7 +111,7 @@ This requires the following python libraries to run: To install the dependency packages needed, please run the following command: ```shell -bash install.sh --enable-pytest +bash install.sh --enable-ci ``` **Note:** For troubleshooting errors with BLE transport, please refer this [link](https://bleak.readthedocs.io/en/latest/troubleshooting.html). diff --git a/tools/install_util.py b/tools/install_util.py index d9f97547a03..d79822c83ec 100644 --- a/tools/install_util.py +++ b/tools/install_util.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # # SPDX-License-Identifier: Apache-2.0 # This script is used from the $IDF_PATH/install.* scripts. This way the argument parsing can be done at one place and @@ -24,9 +24,9 @@ def action_extract_features(args: str) -> None: # features to be enabled have prefix '+', disabled have prefix '-' for arg in args.split(): if arg.startswith(arg_enable_prefix): - features.append('+' + arg[len(arg_enable_prefix):]) + features.append('+' + arg[len(arg_enable_prefix) :]) elif arg.startswith(arg_disable_prefix): - features.append('-' + arg[len(arg_disable_prefix):]) + features.append('-' + arg[len(arg_disable_prefix) :]) elif arg.startswith('-'): raise SystemExit(f'ERROR: Invalid feature specifier: {arg}') features = list(set(features)) @@ -57,7 +57,7 @@ def action_print_help(script_extension: str) -> None: # extract the list of features from ./requirements.json thisdir = os.path.dirname(os.path.realpath(__file__)) - with open(f'{thisdir}/requirements.json', 'r', encoding='utf-8') as f: + with open(f'{thisdir}/requirements.json', encoding='utf-8') as f: json_data = json.load(f) features = [feat['name'] for feat in json_data['features']] @@ -72,13 +72,13 @@ def action_print_help(script_extension: str) -> None: optional arguments: targets-to-install 'all', a single target (e.g. 'esp32s2'), or a comma-separated list of targets (e.g. 'esp32,esp32c3,esp32h2') - --enable-* a specific feature to enable (e.g. '--enable-pytest' will enable feature pytest) + --enable-* a specific feature to enable (e.g. '--enable-ci' will enable feature ci) --disable-* a specific feature to disable (e.g. '--disable-pytest' will disable feature pytest) supported features: {', '.join(features)} {help_opts} show this help message and exit For more information, please see https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/tools/idf-tools.html#install-scripts - """) # noqa: E222 + """) # noqa: E222, E501 def main() -> None: @@ -101,7 +101,7 @@ def main() -> None: action_print_help(args.extension) sys.exit(0) - action_func = globals()['action_{}_{}'.format(args.action, args.type)] + action_func = globals()[f'action_{args.action}_{args.type}'] str_to_parse = vars(args)['str-to-parse'] or '' action_func(' '.join(chain([str_to_parse], unknown_args))) diff --git a/tools/requirements.json b/tools/requirements.json index 33b63a413b0..00bae299766 100644 --- a/tools/requirements.json +++ b/tools/requirements.json @@ -13,12 +13,6 @@ "optional": true, "requirement_path": "tools/requirements/requirements.gdbgui.txt" }, - { - "name": "pytest", - "description": "Packages for CI with pytest", - "optional": true, - "requirement_path": "tools/requirements/requirements.pytest.txt" - }, { "name": "test-specific", "description": "Packages for specific test scripts", diff --git a/tools/requirements/requirements.ci.txt b/tools/requirements/requirements.ci.txt index dfec5b20fc3..846ca3a4866 100644 --- a/tools/requirements/requirements.ci.txt +++ b/tools/requirements/requirements.ci.txt @@ -6,13 +6,10 @@ # https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/tools/idf-tools.html # ci +idf-ci + coverage -idf-build-apps jsonschema junit_xml -python-gitlab -pyyaml SimpleWebSocketServer -pylint-gitlab -minio prettytable diff --git a/tools/requirements/requirements.pytest.txt b/tools/requirements/requirements.pytest.txt deleted file mode 100644 index 68a5acb7660..00000000000 --- a/tools/requirements/requirements.pytest.txt +++ /dev/null @@ -1,21 +0,0 @@ -# Python package requirements for pytest in ESP-IDF. -# This feature can be enabled by running "install.{sh,bat,ps1,fish} --enable-pytest" -# -# This file lists Python packages without version specifiers. Version details -# are stored in a separate constraints file. For more information, visit: -# https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/tools/idf-tools.html - -pytest-embedded-serial-esp -pytest-embedded-idf -pytest-embedded-jtag -pytest-embedded-qemu -pytest-rerunfailures -pytest-timeout -pytest-ignore-test-results - -# ci -minio - -# build -python-gitlab -idf-build-apps diff --git a/tools/test_apps/build_system/ld_non_contiguous_memory/pytest_ld_non_contiguous_memory.py b/tools/test_apps/build_system/ld_non_contiguous_memory/pytest_ld_non_contiguous_memory.py index a5010dc6fdf..b7612735cec 100644 --- a/tools/test_apps/build_system/ld_non_contiguous_memory/pytest_ld_non_contiguous_memory.py +++ b/tools/test_apps/build_system/ld_non_contiguous_memory/pytest_ld_non_contiguous_memory.py @@ -4,11 +4,8 @@ import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -TEST_APP_IN_FLASH = [pytest.param('app_in_flash', marks=pytest.mark.esp32p4)] - @pytest.mark.generic -@idf_parametrize('config', ['app_in_flash'], indirect=['config']) @idf_parametrize('target', ['esp32p4'], indirect=['target']) def test_ld_non_contiguous_memory(dut: Dut) -> None: dut.expect_exact('buf2 placed in low sram') diff --git a/tools/test_apps/linux_compatible/linux_freertos/pytest_linux_freertos.py b/tools/test_apps/linux_compatible/linux_freertos/pytest_linux_freertos.py index 184042cecc0..8b749264ab0 100644 --- a/tools/test_apps/linux_compatible/linux_freertos/pytest_linux_freertos.py +++ b/tools/test_apps/linux_compatible/linux_freertos/pytest_linux_freertos.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 -import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @idf_parametrize('target', ['linux'], indirect=['target']) def test_linux_freertos_SMP(dut: Dut) -> None: dut.expect_exact('Press ENTER to see the list of tests.') diff --git a/tools/test_apps/phy/phy_tsens/pytest_phy_tsens.py b/tools/test_apps/phy/phy_tsens/pytest_phy_tsens.py index 232e51b9b06..1025cc8b15c 100644 --- a/tools/test_apps/phy/phy_tsens/pytest_phy_tsens.py +++ b/tools/test_apps/phy/phy_tsens/pytest_phy_tsens.py @@ -191,15 +191,15 @@ def test_phy_tsens_coexist_c2_xtal26m(dut: Tuple[Dut, Dut]) -> None: @pytest.mark.two_duts @pytest.mark.xtal_26mhz -@pytest.mark.esp32c2eco4 +@pytest.mark.esp32c2_rev2 @pytest.mark.parametrize( 'count, config, target, baud', [ - (2, 'c2_eco4', 'esp32c2', '74880'), + (2, 'esp32c2_rev2', 'esp32c2', '74880'), ], indirect=True, ) -def test_phy_tsens_coexist_c2eco4_xtal26m(dut: Tuple[Dut, Dut]) -> None: +def test_phy_tsens_coexist_esp32c2_rev2_xtal26m(dut: Tuple[Dut, Dut]) -> None: for _dut in dut: _dut.expect('esp>') run_phy_tsens_test(dut) @@ -216,15 +216,15 @@ def test_phy_tsens_coexist_c2eco4_xtal26m(dut: Tuple[Dut, Dut]) -> None: @pytest.mark.two_duts -@pytest.mark.esp32c3eco7 +@pytest.mark.esp32c3_rev1 @pytest.mark.parametrize( 'count, config, target', [ - (2, 'c3_eco7', 'esp32c3'), + (2, 'esp32c3_rev1', 'esp32c3'), ], indirect=True, ) -def test_phy_tsens_coexist_c3eco7(dut: Tuple[Dut, Dut]) -> None: +def test_phy_tsens_coexist_esp32c3_rev1(dut: Tuple[Dut, Dut]) -> None: for _dut in dut: _dut.expect('esp>') run_phy_tsens_test(dut) diff --git a/tools/test_apps/phy/phy_tsens/sdkconfig.ci.c2_eco4 b/tools/test_apps/phy/phy_tsens/sdkconfig.ci.esp32c2_rev2 similarity index 100% rename from tools/test_apps/phy/phy_tsens/sdkconfig.ci.c2_eco4 rename to tools/test_apps/phy/phy_tsens/sdkconfig.ci.esp32c2_rev2 diff --git a/tools/test_apps/phy/phy_tsens/sdkconfig.ci.c3_eco7 b/tools/test_apps/phy/phy_tsens/sdkconfig.ci.esp32c3_rev1 similarity index 100% rename from tools/test_apps/phy/phy_tsens/sdkconfig.ci.c3_eco7 rename to tools/test_apps/phy/phy_tsens/sdkconfig.ci.esp32c3_rev1 diff --git a/tools/test_apps/security/secure_boot/README.md b/tools/test_apps/security/secure_boot/README.md index f042b064144..bb36b997d26 100644 --- a/tools/test_apps/security/secure_boot/README.md +++ b/tools/test_apps/security/secure_boot/README.md @@ -101,7 +101,7 @@ Under `Security features` - Install pytest requirements ``` - bash $IDF_PATH/install.sh --enable-pytest + bash $IDF_PATH/install.sh --enable-ci ``` ### Build and test diff --git a/tools/test_apps/storage/std_filesystem/pytest_std_filesystem.py b/tools/test_apps/storage/std_filesystem/pytest_std_filesystem.py index 1e644e2a071..790fd6a2a0e 100644 --- a/tools/test_apps/storage/std_filesystem/pytest_std_filesystem.py +++ b/tools/test_apps/storage/std_filesystem/pytest_std_filesystem.py @@ -6,7 +6,6 @@ from pytest_embedded_idf.utils import idf_parametrize @pytest.mark.qemu -@pytest.mark.host_test @idf_parametrize('target', ['esp32', 'esp32c3'], indirect=['target']) def test_std_filesystem(dut: Dut) -> None: dut.expect_exact('All tests passed', timeout=200) diff --git a/tools/test_apps/system/esp_intr_dump/pytest_esp_intr_dump.py b/tools/test_apps/system/esp_intr_dump/pytest_esp_intr_dump.py index 7832cd5bf29..e03da57cc20 100644 --- a/tools/test_apps/system/esp_intr_dump/pytest_esp_intr_dump.py +++ b/tools/test_apps/system/esp_intr_dump/pytest_esp_intr_dump.py @@ -10,7 +10,6 @@ PROMPT = 'test_intr_dump>' @pytest.mark.qemu -@pytest.mark.host_test @idf_parametrize('target', ['esp32'], indirect=['target']) def test_esp_intr_dump_nonshared(dut: Dut) -> None: dut.expect_exact(PROMPT, timeout=30) @@ -24,7 +23,6 @@ def test_esp_intr_dump_nonshared(dut: Dut) -> None: @pytest.mark.qemu -@pytest.mark.host_test @idf_parametrize('target', ['esp32'], indirect=['target']) def test_esp_intr_dump_shared(dut: Dut) -> None: dut.expect_exact(PROMPT, timeout=30) @@ -54,5 +52,5 @@ def test_esp_intr_dump_expected_output(dut: Dut) -> None: dut.expect_exact(PROMPT, timeout=30) dut.write('intr_dump\n') exp_out_file = os.path.join(os.path.dirname(__file__), 'expected_output', f'{dut.target}.txt') - for line in open(exp_out_file, 'r').readlines(): + for line in open(exp_out_file).readlines(): dut.expect_exact(line.strip()) diff --git a/tools/test_apps/system/mmu_page_size/pytest_mmu_page_size.py b/tools/test_apps/system/mmu_page_size/pytest_mmu_page_size.py index 3f59bb54211..e06c8538320 100644 --- a/tools/test_apps/system/mmu_page_size/pytest_mmu_page_size.py +++ b/tools/test_apps/system/mmu_page_size/pytest_mmu_page_size.py @@ -1,10 +1,9 @@ # SPDX-FileCopyrightText: 2024-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 import os +import typing as t import pytest -from artifacts_handler import ArtifactType -from idf_ci_utils import IDF_PATH from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize @@ -12,19 +11,17 @@ from pytest_embedded_idf.utils import idf_parametrize @pytest.mark.generic @pytest.mark.parametrize('config', ['32K'], indirect=True) @idf_parametrize('target', ['esp32c6', 'esp32h2'], indirect=['target']) -def test_app_mmu_page_size_32k_and_bootloader_mmu_page_size_64k(dut: Dut, app_downloader, config) -> None: # type: ignore +def test_app_mmu_page_size_32k_and_bootloader_mmu_page_size_64k(dut: Dut, app_downloader: t.Any, config: str) -> None: dut.expect('App is running') assert '32K' in config app_config = config.replace('32K', '64K') - path_to_mmu_page_size_64k_build = os.path.join(dut.app.app_path, f'build_{dut.target}_{app_config}') + build_dir = f'build_{dut.target}_{app_config}' if app_downloader: - app_downloader.download_app( - os.path.relpath(path_to_mmu_page_size_64k_build, IDF_PATH), ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES - ) + app_downloader.download_app(dut.app.app_path, build_dir, 'flash') - dut.serial.bootloader_flash(path_to_mmu_page_size_64k_build) + dut.serial.bootloader_flash(os.path.join(dut.app.app_path, build_dir)) dut.expect('MMU page size mismatch') dut.expect('App is running') dut.expect('Partition test done') @@ -34,19 +31,17 @@ def test_app_mmu_page_size_32k_and_bootloader_mmu_page_size_64k(dut: Dut, app_do @pytest.mark.generic @pytest.mark.parametrize('config', ['64K'], indirect=True) @idf_parametrize('target', ['esp32c6', 'esp32h2'], indirect=['target']) -def test_app_mmu_page_size_64k_and_bootloader_mmu_page_size_32k(dut: Dut, app_downloader, config) -> None: # type: ignore +def test_app_mmu_page_size_64k_and_bootloader_mmu_page_size_32k(dut: Dut, app_downloader: t.Any, config: str) -> None: dut.expect('App is running') assert '64K' in config app_config = config.replace('64K', '32K') - path_to_mmu_page_size_32k_build = os.path.join(dut.app.app_path, f'build_{dut.target}_{app_config}') + build_dir = f'build_{dut.target}_{app_config}' if app_downloader: - app_downloader.download_app( - os.path.relpath(path_to_mmu_page_size_32k_build, IDF_PATH), ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES - ) + app_downloader.download_app(dut.app.app_path, build_dir, 'flash') - dut.serial.bootloader_flash(path_to_mmu_page_size_32k_build) + dut.serial.bootloader_flash(os.path.join(dut.app.app_path, build_dir)) dut.expect('MMU page size mismatch') dut.expect('App is running') dut.expect('Partition test done') diff --git a/tools/test_apps/system/panic/pytest_panic.py b/tools/test_apps/system/panic/pytest_panic.py index 2ff254981a9..7e3e90caa06 100644 --- a/tools/test_apps/system/panic/pytest_panic.py +++ b/tools/test_apps/system/panic/pytest_panic.py @@ -13,104 +13,9 @@ import pytest from pytest_embedded_idf.utils import idf_parametrize from test_panic_util import PanicTestDut -TARGETS_XTENSA_SINGLE_CORE = [ - pytest.mark.esp32s2, -] - -TARGETS_XTENSA_DUAL_CORE = [ - pytest.mark.esp32, - pytest.mark.esp32s3, -] - -TARGETS_XTENSA = TARGETS_XTENSA_SINGLE_CORE + TARGETS_XTENSA_DUAL_CORE - -TARGETS_RISCV_SINGLE_CORE = [ - pytest.mark.esp32c2, - pytest.mark.esp32c3, - pytest.mark.esp32c5, - pytest.mark.esp32c6, - pytest.mark.esp32h2, - pytest.mark.esp32c61, -] - -TARGETS_RISCV_DUAL_CORE = [ - pytest.mark.esp32p4, -] - -TARGETS_RISCV = TARGETS_RISCV_SINGLE_CORE + TARGETS_RISCV_DUAL_CORE - -# Markers for all the targets this test currently runs on -TARGETS_ALL = TARGETS_XTENSA + TARGETS_RISCV - -# Some tests only run on dual-core targets, they use the config below. -TARGETS_DUAL_CORE = TARGETS_XTENSA_DUAL_CORE + TARGETS_RISCV_DUAL_CORE - -# Most tests run on all targets and with all configs. -# This list is passed to @pytest.mark.parametrize for each of the test cases. -# It creates an outer product of the sets: [configs] x [targets], -# with some exceptions. -CONFIGS = [ - pytest.param('coredump_flash_bin_crc', marks=TARGETS_ALL), - pytest.param('coredump_flash_elf_sha', marks=TARGETS_ALL), - pytest.param('coredump_flash_elf_soft_sha', marks=TARGETS_ALL), - pytest.param('coredump_uart_bin_crc', marks=TARGETS_ALL), - pytest.param('coredump_uart_elf_crc', marks=TARGETS_ALL), - pytest.param('coredump_flash_custom_stack', marks=TARGETS_ALL), - pytest.param('gdbstub', marks=TARGETS_ALL), - pytest.param('panic', marks=TARGETS_ALL), -] - -CONFIGS_UBSAN = [ - pytest.param('gdbstub', marks=TARGETS_ALL), - pytest.param('panic', marks=TARGETS_ALL), -] - -CONFIGS_DUAL_CORE = [ - pytest.param('coredump_flash_bin_crc', marks=TARGETS_DUAL_CORE), - pytest.param('coredump_flash_elf_sha', marks=TARGETS_DUAL_CORE), - pytest.param('coredump_uart_bin_crc', marks=TARGETS_DUAL_CORE), - pytest.param('coredump_uart_elf_crc', marks=TARGETS_DUAL_CORE), - pytest.param('gdbstub', marks=TARGETS_DUAL_CORE), - pytest.param('panic', marks=TARGETS_DUAL_CORE), -] - # Some tests run on all targets but need to behave differently on the dual-core ones. # This list is used to check if the target is a dual-core one. -TARGETS_DUAL_CORE_NAMES = [x.mark.name for x in TARGETS_DUAL_CORE] - -CONFIGS_EXTRAM_STACK = [ - pytest.param('coredump_flash_extram_stack_heap_esp32', marks=[pytest.mark.esp32, pytest.mark.psram]), - pytest.param('coredump_flash_extram_stack_heap_esp32s2', marks=[pytest.mark.esp32s2, pytest.mark.generic]), - pytest.param('coredump_flash_extram_stack_heap_esp32s3', marks=[pytest.mark.esp32s3, pytest.mark.quad_psram]), - pytest.param('coredump_flash_extram_stack_bss_esp32', marks=[pytest.mark.esp32, pytest.mark.psram]), - pytest.param('coredump_flash_extram_stack_bss_esp32s2', marks=[pytest.mark.esp32s2, pytest.mark.generic]), - pytest.param('coredump_flash_extram_stack_bss_esp32s3', marks=[pytest.mark.esp32s3, pytest.mark.quad_psram]), -] - -CONFIGS_HW_STACK_GUARD = [ - pytest.param('coredump_flash_bin_crc', marks=TARGETS_RISCV), - pytest.param('coredump_uart_bin_crc', marks=TARGETS_RISCV), - pytest.param('coredump_uart_elf_crc', marks=TARGETS_RISCV), - pytest.param('gdbstub', marks=TARGETS_RISCV), - pytest.param('panic', marks=TARGETS_RISCV), -] - -CONFIGS_HW_STACK_GUARD_DUAL_CORE = [ - pytest.param('coredump_flash_bin_crc', marks=TARGETS_RISCV_DUAL_CORE), - pytest.param('coredump_uart_bin_crc', marks=TARGETS_RISCV_DUAL_CORE), - pytest.param('coredump_uart_elf_crc', marks=TARGETS_RISCV_DUAL_CORE), - pytest.param('gdbstub', marks=TARGETS_RISCV_DUAL_CORE), - pytest.param('panic', marks=TARGETS_RISCV_DUAL_CORE), -] - -CONFIG_CAPTURE_DRAM = [pytest.param('coredump_flash_capture_dram', marks=TARGETS_ALL)] - -CONFIG_COREDUMP_SUMMARY = [pytest.param('coredump_flash_elf_sha', marks=TARGETS_ALL)] - -CONFIG_COREDUMP_SUMMARY_FLASH_ENCRYPTED = [ - pytest.param('coredump_flash_encrypted', marks=[pytest.mark.esp32, pytest.mark.esp32c3]), - pytest.param('coredump_flash_encrypted_coredump_plain', marks=[pytest.mark.esp32, pytest.mark.esp32c3]), -] +TARGETS_DUAL_CORE_NAMES = ['esp32', 'esp32s3', 'esp32p4'] # Panic abort information will start with this string. PANIC_ABORT_PREFIX = 'Panic reason: ' @@ -1707,70 +1612,6 @@ def test_panic_handler_crash1(dut: PanicTestDut, config: str, test_func_name: st dut.expect_cpu_reset() -######################### -# for memprot test only # -######################### - -# Memprot-related tests are supported only on targets with PMS/PMA peripheral; -# currently ESP32-S2, ESP32-C3, ESP32-C2, ESP32-H2, ESP32-C6, ESP32-P4, ESP32-C5 and ESP32-C61 are supported -CONFIGS_MEMPROT_IDRAM = [ - pytest.param('memprot_esp32s2', marks=[pytest.mark.esp32s2]), - pytest.param('memprot_esp32c3', marks=[pytest.mark.esp32c3]), - pytest.param('memprot_esp32c2', marks=[pytest.mark.esp32c2]), - pytest.param('memprot_esp32c5', marks=[pytest.mark.esp32c5]), - pytest.param('memprot_esp32c6', marks=[pytest.mark.esp32c6]), - pytest.param('memprot_esp32c61', marks=[pytest.mark.esp32c61]), - pytest.param('memprot_esp32h2', marks=[pytest.mark.esp32h2]), - pytest.param('memprot_esp32p4', marks=[pytest.mark.esp32p4]), -] - -CONFIGS_MEMPROT_DCACHE = [ - pytest.param('memprot_esp32s2', marks=pytest.mark.esp32s2), -] - -CONFIGS_MEMPROT_RTC_FAST_MEM = [ - pytest.param('memprot_esp32s2', marks=[pytest.mark.esp32s2]), - pytest.param('memprot_esp32c3', marks=[pytest.mark.esp32c3]), - pytest.param('memprot_esp32c5', marks=[pytest.mark.esp32c5]), - pytest.param('memprot_esp32c6', marks=[pytest.mark.esp32c6]), - pytest.param('memprot_esp32h2', marks=[pytest.mark.esp32h2]), - pytest.param('memprot_esp32p4', marks=[pytest.mark.esp32p4]), -] - -CONFIGS_MEMPROT_RTC_SLOW_MEM = [ - pytest.param('memprot_esp32s2', marks=[pytest.mark.esp32s2]), -] - -CONFIGS_MEMPROT_FLASH_IDROM = [ - pytest.param('memprot_esp32c5', marks=[pytest.mark.esp32c5]), - pytest.param('memprot_esp32c6', marks=[pytest.mark.esp32c6]), - pytest.param('memprot_esp32c61', marks=[pytest.mark.esp32c61]), - pytest.param('memprot_esp32h2', marks=[pytest.mark.esp32h2]), - pytest.param('memprot_esp32p4', marks=[pytest.mark.esp32p4]), -] - -CONFIGS_MEMPROT_SPIRAM_XIP_IROM_ALIGNMENT_HEAP = [ - pytest.param('memprot_spiram_xip_esp32c5', marks=[pytest.mark.esp32c5]), - pytest.param('memprot_spiram_xip_esp32c61', marks=[pytest.mark.esp32c61]), - pytest.param('memprot_spiram_xip_esp32p4', marks=[pytest.mark.esp32p4]), -] - -CONFIGS_MEMPROT_SPIRAM_XIP_DROM_ALIGNMENT_HEAP = [ - pytest.param('memprot_spiram_xip_esp32s3', marks=[pytest.mark.esp32s3]), - pytest.param('memprot_spiram_xip_esp32c5', marks=[pytest.mark.esp32c5]), - pytest.param('memprot_spiram_xip_esp32c61', marks=[pytest.mark.esp32c61]), - pytest.param('memprot_spiram_xip_esp32p4', marks=[pytest.mark.esp32p4]), -] - -CONFIGS_MEMPROT_INVALID_REGION_PROTECTION_USING_PMA = [ - pytest.param('memprot_esp32c5', marks=[pytest.mark.esp32c5]), - pytest.param('memprot_esp32c6', marks=[pytest.mark.esp32c6]), - pytest.param('memprot_esp32c61', marks=[pytest.mark.esp32c61]), - pytest.param('memprot_esp32h2', marks=[pytest.mark.esp32h2]), - pytest.param('memprot_esp32p4', marks=[pytest.mark.esp32p4]), -] - - @pytest.mark.generic @idf_parametrize('config', ['memprot_esp32s2'], indirect=['config']) @idf_parametrize('target', ['esp32s2'], indirect=['target']) diff --git a/tools/test_apps/system/unicore_bootloader/pytest_unicore_bootloader.py b/tools/test_apps/system/unicore_bootloader/pytest_unicore_bootloader.py index 89fd8a2cb0c..a2bad90a652 100644 --- a/tools/test_apps/system/unicore_bootloader/pytest_unicore_bootloader.py +++ b/tools/test_apps/system/unicore_bootloader/pytest_unicore_bootloader.py @@ -4,8 +4,6 @@ import os import re import pytest -from artifacts_handler import ArtifactType -from idf_ci_utils import IDF_PATH from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize @@ -21,13 +19,11 @@ def test_multicore_app_and_unicore_bootloader(dut: Dut, app_downloader, config) assert 'multicore' in config app_config = config.replace('multicore', 'unicore') - path_to_unicore_build = os.path.join(dut.app.app_path, f'build_{dut.target}_{app_config}') + build_dir = f'build_{dut.target}_{app_config}' if app_downloader: - app_downloader.download_app( - os.path.relpath(path_to_unicore_build, IDF_PATH), ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES - ) + app_downloader.download_app(dut.app.app_path, build_dir, 'flash') - dut.serial.bootloader_flash(path_to_unicore_build) + dut.serial.bootloader_flash(os.path.join(dut.app.app_path, build_dir)) dut.expect('Unicore bootloader') dut.expect('Multicore app') if 'psram' in config: @@ -47,13 +43,11 @@ def test_unicore_app_and_multicore_bootloader(dut: Dut, app_downloader, config) assert 'unicore' in config app_config = config.replace('unicore', 'multicore') - path_to_multicore_build = os.path.join(dut.app.app_path, f'build_{dut.target}_{app_config}') + build_dir = f'build_{dut.target}_{app_config}' if app_downloader: - app_downloader.download_app( - os.path.relpath(path_to_multicore_build, IDF_PATH), ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES - ) + app_downloader.download_app(dut.app.app_path, build_dir, 'flash') - dut.serial.bootloader_flash(path_to_multicore_build) + dut.serial.bootloader_flash(os.path.join(dut.app.app_path, build_dir)) dut.expect('Multicore bootloader') dut.expect('Unicore app') if 'psram' in config: diff --git a/tools/test_build_system/README.md b/tools/test_build_system/README.md index 30fa0e7eec8..c226da98870 100644 --- a/tools/test_build_system/README.md +++ b/tools/test_build_system/README.md @@ -9,7 +9,7 @@ This directory contains tests for the build system and build-related tools. Thes ## Running the tests locally -1. Install pytest using `install.{sh,bat,ps1,fish} --enable-pytest`. +1. Install pytest using `install.{sh,bat,ps1,fish} --enable-ci`. 1. Activate the IDF shell environment using `export.{sh,bat,ps1,fish}`. 1. To run all the tests, go to `$IDF_PATH/tools/test_build_system` directory, then run: ``` diff --git a/tools/test_build_system/pytest.ini b/tools/test_build_system/pytest.ini index eeac16e6d7b..6f08513e182 100644 --- a/tools/test_build_system/pytest.ini +++ b/tools/test_build_system/pytest.ini @@ -14,6 +14,9 @@ junit_family = xunit1 junit_logging = stdout junit_log_passing_tests = False +filterwarnings = + ignore::pytest.PytestExperimentalApiWarning + ## !! When adding new markers, don't forget to update also the tools\test_build_system\README.md !! markers = test_app_copy: specify relative path of the app to copy, and the prefix of the destination directory name