diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c1a6a390931..707db797a57 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,11 +2,25 @@ include: - project: "ci/actions/common" + ref: "v2" file: - "templates/stable/default-workflow.yml" + - "templates/idf/pre_check.yml" + - "templates/idf/build.yml" - "templates/idf/deploy-github.yml" - "templates/idf/deploy-docs.yml" - "templates/idf/integration-test.yml" + - project: "ci/actions/common" + ref: "v2" + file: "templates/idf/sync-files.yml" + inputs: + job_name: "sync-pre-commit-config" + remote_repo_url: "https://__token__:${CI_SYNC_REPO_TOKEN}@${CI_SERVER_FQDN}/ci/images.git" + ref_name: "ci/sync_idf_pre_commit_config" + file_mappings: + - ".pre-commit-config.yaml:pre-commit/idf/.pre-commit-config.v5.5.yaml" + changes: + - ".pre-commit-config.yaml" - project: "ci/actions/common" file: "templates/idf/build-docs.yml" inputs: @@ -20,13 +34,15 @@ include: DOCTGT: "esp32p4" - ".gitlab/ci/danger.yml" - - ".gitlab/ci/common.yml" + - ".gitlab/ci/common.yml" # has to be placed after danger.yml, since danger.yml defined stages. - ".gitlab/ci/rules.yml" + # by stages - ".gitlab/ci/manual_gate.yml" - ".gitlab/ci/upload_cache.yml" + - ".gitlab/ci/pre_check.yml" + - ".gitlab/ci/pre_deploy.yml" - ".gitlab/ci/static-code-analysis.yml" - ".gitlab/ci/pre_commit.yml" - - ".gitlab/ci/pre_check.yml" - ".gitlab/ci/build.yml" - ".gitlab/ci/host-test.yml" - ".gitlab/ci/post_deploy.yml" diff --git a/.gitlab/ci/README.md b/.gitlab/ci/README.md index 41f0640037a..7b109d30689 100644 --- a/.gitlab/ci/README.md +++ b/.gitlab/ci/README.md @@ -245,76 +245,15 @@ We're using the latest version of [idf-build-apps][idf-build-apps]. Please refer In ESP-IDF CI, there's a few more special rules are additionally supported to disable the check app dependencies feature: - Add MR labels `BUILD_AND_TEST_ALL_APPS` +- Pipeline variable `IDF_CI_SELECT_ALL_PYTEST_CASES=1` - Run in protected branches ## Upload/Download Artifacts to Internal Minio Server -### Users Without Access to Minio +Please refer to the documentation [here](https://docs.espressif.com/projects/idf-ci/en/latest/guides/cli.html) -If you don't have access to the internal Minio server, you can still download the artifacts from the shared link in the job log. +## Revision Marker Naming Rules -The log will look like this: +When specifying chip revisions for testing markers, configurations or filenames, strictly follow the format `esp32_rev` (or similarly `_rev` when brief). -```shell -Pipeline ID : 587355 -Job name : build_clang_test_apps_esp32 -Job ID : 40272275 -Created archive file: 40272275.zip, uploading as 587355/build_dir_without_map_and_elf_files/build_clang_test_apps_esp32/40272275.zip -Please download the archive file includes build_dir_without_map_and_elf_files from [INTERNAL_URL] -``` - -### Users With Access to Minio - -#### Env Vars for Minio - -Minio takes these env vars to connect to the server: - -- `IDF_S3_SERVER` -- `IDF_S3_ACCESS_KEY` -- `IDF_S3_SECRET_KEY` -- `IDF_S3_BUCKET` - -#### Artifacts Types and File Patterns - -The artifacts types and corresponding file patterns are defined in tools/ci/artifacts_handler.py, inside `ArtifactType` and `TYPE_PATTERNS_DICT`. - -#### Upload - -```shell -python tools/ci/artifacts_handler.py upload -``` - - will upload the files that match the file patterns to minio object storage with name: - -`///.zip` - -For example, job 39043328 will upload these four files: - -- `575500/map_and_elf_files/build_pytest_examples_esp32/39043328.zip` -- `575500/build_dir_without_map_and_elf_files/build_pytest_examples_esp32/39043328.zip` -- `575500/logs/build_pytest_examples_esp32/39043328.zip` -- `575500/size_reports/build_pytest_examples_esp32/39043328.zip` - -#### Download - -You may run - -```shell -python tools/ci/artifacts_handler.py download --pipeline_id -``` - -to download all files of the pipeline, or - -```shell -python tools/ci/artifacts_handler.py download --pipeline_id --job_name -``` - -to download all files with the specified job name or pattern, or - -```shell -python tools/ci/artifacts_handler.py download --pipeline_id --job_name --type ... -``` - -to download all files with the specified job name or pattern and artifact type(s). - -You may check all detailed documentation with `python tools/ci/artifacts_handler.py download -h` +> e.g., `esp32_rev3`, `esp32c2_rev2`, `esp32c3_rev1`, `esp32p4_rev1`, `esp32c5_rev1` diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index b67f345368e..ae4d74ad4c9 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -110,61 +110,6 @@ build_clang_test_apps_riscv: #################### # Dynamic Pipeline # #################### -generate_build_child_pipeline: - extends: - - .build_template - tags: [fast_run, shiny] - dependencies: # set dependencies to null to avoid missing artifacts issue - needs: - - pipeline_variables - - job: baseline_manifest_sha - optional: true - artifacts: - paths: - - build_child_pipeline.yml - - test_related_apps.txt - - non_test_related_apps.txt - expire_in: 1 week - when: always - script: - - run_cmd idf-ci --debug gitlab build-child-pipeline - -p components - -p examples - -p tools/test_apps - --modified-files $MR_MODIFIED_FILES - -build_child_pipeline: - stage: build - needs: - - job: fast_template_app - optional: true - artifacts: false - - pipeline_variables - - generate_build_child_pipeline - variables: - MR_MODIFIED_COMPONENTS: $MR_MODIFIED_COMPONENTS - MR_MODIFIED_FILES: $MR_MODIFIED_FILES - PARENT_PIPELINE_ID: $CI_PIPELINE_ID - # https://gitlab.com/gitlab-org/gitlab/-/issues/214340 - inherit: - variables: false - trigger: - include: - - artifact: build_child_pipeline.yml - job: generate_build_child_pipeline - strategy: depend - -generate_prebuild_report: - extends: - - .build_template - tags: [fast_run, shiny] - needs: - - pipeline_variables - artifacts: - paths: - - prebuild_report.html - expire_in: 1 week - when: always - script: - - run_cmd idf-ci build collect --format html -o prebuild_report.html - - echo "Report generated at https://${CI_PAGES_HOSTNAME}:${CI_SERVER_PORT}/-/esp-idf/-/jobs/${CI_JOB_ID}/artifacts/prebuild_report.html" +# !!! +# configurations moved to ci/actions/common/idf/ +# !!! diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index 51982b2d806..61f1f3dd584 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -11,9 +11,20 @@ stages: - target_test - host_test - test_deploy + - pre_deploy - deploy - post_deploy +include: + - project: "ci/actions/common" + ref: "v2" + file: "templates/idf/common-scripts.yml" + inputs: + ci_python_constraint_file: "espidf.constraints.v5.5.txt" + ci_python_constraint_branch: "backport/idf-ci-1.0-idf-v5.5" + macos_image: "registry.hub.docker.com/espressif/macos-idf:sequoia-v5.5" + windows_image: "windows_11_idf_v5_5" + variables: # System environment @@ -58,33 +69,29 @@ variables: ESP_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v5.5:1" ESP_IDF_DOC_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v5.5-docs:1" TARGET_TEST_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v5.5-target-test:1" - SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:5" - - # cache python dependencies - PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" # Set this variable to the branch of idf-constraints repo in order to test a custom Python constraint file. The # branch name must be without the remote part ("origin/"). Keep the variable empty in order to use the constraint # file from https://dl.espressif.com/dl/esp-idf. - CI_PYTHON_CONSTRAINT_BRANCH: "" + # CI_PYTHON_CONSTRAINT_BRANCH could be defined as component input `ci_python_constraint_branch` in common-scripts above # Update the filename for a specific ESP-IDF release. It is used only with CI_PYTHON_CONSTRAINT_BRANCH. - CI_PYTHON_CONSTRAINT_FILE: "espidf.constraints.v5.5.txt" + # CI_PYTHON_CONSTRAINT_FILE is defined as component input `ci_python_constraint_file` in common-scripts above # Set this variable to repository name of a Python tool you wish to install and test in the context of ESP-IDF CI. # Keep the variable empty when not used. - CI_PYTHON_TOOL_REPO: "" + # CI_PYTHON_TOOL_REPO could be defined as component input `ci_python_tool_repo` in common-scripts above # Set this variable to the branch of a Python tool repo specified in CI_PYTHON_TOOL_REPO. The # branch name must be without the remote part ("origin/"). Keep the variable empty when not used. # This is used only if CI_PYTHON_TOOL_REPO is not empty. - CI_PYTHON_TOOL_BRANCH: "" + # CI_PYTHON_TOOL_BRANCH could be defined as component input `ci_python_tool_branch` in common-scripts above # Set this variable to Clang toolchain distro URL to be used. # NOTE: We have separate toolchains for Xtensa and RISCV, therefore jobs for one arch will fail. # This is OK as far as we use CI_CLANG_DISTRO_URL for pre-release tests purposes only. # Keep the variable empty when not used. - CI_CLANG_DISTRO_URL: "" + # CI_CLANG_DISTRO_URL could be defined as component input `ci_clang_distro_url` in common-scripts above # Set this variable to specify the file name for the known failure cases. KNOWN_FAILURE_CASES_FILE_NAME: "5.5.txt" @@ -106,182 +113,18 @@ variables: ################################################ # `before_script` and `after_script` Templates # ################################################ -.common_before_scripts: &common-before_scripts | - source tools/ci/utils.sh - - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS - - if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then - export IDF_MIRROR_PREFIX_MAP= - fi - - if echo "$CI_MERGE_REQUEST_LABELS" | egrep "(^|,)include_nightly_run(,|$)"; then - export INCLUDE_NIGHTLY_RUN="1" - export NIGHTLY_RUN="1" - fi - - # configure cmake related flags - source tools/ci/configure_ci_environment.sh - - if [[ "$CI_CCACHE_STATS" == 1 ]] && command -v ccache >/dev/null 2>&1 && [[ -n "$CCACHE_STATSLOG" ]]; then - mkdir -p "$(dirname "$CCACHE_STATSLOG")" - rm -f "$CCACHE_STATSLOG" - fi - - # add extra python packages - export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH" - -.setup_tools_and_idf_python_venv: &setup_tools_and_idf_python_venv | - # must use after setup_tools_except_target_test - # otherwise the export.sh won't work properly - - # download constraint file for dev - if [[ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]]; then - wget -O /tmp/constraint.txt --header="Authorization:Bearer ${ESPCI_TOKEN}" "${GITLAB_HTTP_SERVER}/api/v4/projects/2581/repository/files/${CI_PYTHON_CONSTRAINT_FILE}/raw?ref=${CI_PYTHON_CONSTRAINT_BRANCH}" - mkdir -p ~/.espressif - mv /tmp/constraint.txt ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE} - fi - - # Mirror - if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then - export IDF_MIRROR_PREFIX_MAP= - fi - # Optimize pip install - if echo "${CI_RUNNER_TAGS}" | grep "shiny"; then - export PIP_INDEX_URL="${PIP_INDEX_URL_SHINY}" - fi - if [[ "$(uname -m)" == "x86_64" ]] || [[ "$(uname -m)" == "aarch64" ]]; then - export IDF_PIP_WHEELS_URL="" - fi - - # install.sh - if [[ "${CI_JOB_STAGE}" != "target_test" ]]; then - section_start "running_install_sh" "Running install.sh" - if [[ "${CI_JOB_STAGE}" == "build_doc" || "${CI_JOB_GROUP_NAME}" == *docs* ]]; then - run_cmd bash install.sh --enable-ci --enable-docs - else - run_cmd bash install.sh --enable-ci - fi - section_end "running_install_sh" - else - section_start "install_python_env" "Install Python environment, skip required tools check" - run_cmd python tools/idf_tools.py install-python-env --features ci,test-specific - export IDF_SKIP_TOOLS_CHECK=1 - section_end "install_python_env" - fi - - section_start "source_export" "Source export.sh" - source ./export.sh - section_end "source_export" - - # Eager upgrade of CI dependencies - # Done after sourcing export.sh so that we could easily invoke the right pip - section_start "upgrade_ci_dependencies" "Upgrading CI dependencies" - pip install --upgrade --upgrade-strategy=eager -r $IDF_PATH/tools/requirements/requirements.ci.txt -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE} - # we need the latest DEV release for esptool to be installed in every job without bumping the minimal requirement in - # the constraint file. - pip install --upgrade --pre esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE} - section_end "upgrade_ci_dependencies" - - REEXPORT_NEEDED=0 - if [[ ! -z "$INSTALL_EXTRA_TOOLS" ]]; then - section_start "installing_optional_tools" "Install optional tools ${INSTALL_EXTRA_TOOLS}" - run_cmd $IDF_PATH/tools/idf_tools.py --non-interactive install $INSTALL_EXTRA_TOOLS - section_end "installing_optional_tools" - - REEXPORT_NEEDED=1 - fi - - # Install esp-clang if necessary (esp-clang is separately installed) - if [[ "$IDF_TOOLCHAIN" == "clang" && -z "$CI_CLANG_DISTRO_URL" ]]; then - $IDF_PATH/tools/idf_tools.py --non-interactive install esp-clang - - REEXPORT_NEEDED=1 - fi - - if [[ $REEXPORT_NEEDED -eq 1 ]]; then - section_start "re_source_export" "Re-source export.sh" - source ./export.sh - section_end "re_source_export" - fi - - # Custom clang toolchain - if [[ "$IDF_TOOLCHAIN" == "clang" && ! -z "$CI_CLANG_DISTRO_URL" ]]; then - echo "Using custom clang from ${CI_CLANG_DISTRO_URL}" - wget $CI_CLANG_DISTRO_URL - ARCH_NAME=$(basename $CI_CLANG_DISTRO_URL) - tar -x -f $ARCH_NAME - export PATH=$PWD/esp-clang/bin:$PATH - fi - - # Custom OpenOCD - if [[ "$CI_JOB_STAGE" == "target_test" ]]; then - machine="$(uname -m)" - if [[ "$machine" == "armv7l" ]] ; then - OOCD_DISTRO_URL="$OOCD_DISTRO_URL_ARMHF" - elif [[ "$machine" == "aarch64" ]] ; then - OOCD_DISTRO_URL="$OOCD_DISTRO_URL_ARM64" - fi - if [[ ! -z "$OOCD_DISTRO_URL" ]]; then - echo "Using custom OpenOCD from ${OOCD_DISTRO_URL}" - wget $OOCD_DISTRO_URL - ARCH_NAME=$(basename $OOCD_DISTRO_URL) - tar -x -f $ARCH_NAME - export OPENOCD_SCRIPTS=$PWD/openocd-esp32/share/openocd/scripts - export PATH=$PWD/openocd-esp32/bin:$PATH - fi - fi - - if [[ -n "$CI_PYTHON_TOOL_REPO" ]]; then - git clone --quiet --depth=1 -b ${CI_PYTHON_TOOL_BRANCH} https://gitlab-ci-token:${ESPCI_TOKEN}@${GITLAB_HTTPS_HOST}/espressif/${CI_PYTHON_TOOL_REPO}.git - pip install ./${CI_PYTHON_TOOL_REPO} - rm -rf ${CI_PYTHON_TOOL_REPO} - fi - - info "setup tools and python venv done" - -.show_ccache_statistics: &show_ccache_statistics | - # Show ccache statistics if enabled globally - section_start "ccache_show_stats" "Show ccache statistics" - if [[ "$CI_CCACHE_STATS" == 1 ]] && command -v ccache >/dev/null 2>&1; then - if ccache --help 2>/dev/null | grep -q -- '--show-log-stats'; then - if [[ -n "$CCACHE_STATSLOG" && -f "$CCACHE_STATSLOG" ]]; then - ccache --show-log-stats -vv - else - echo "INFO: No per-job ccache statistics were recorded" - fi - else - ccache --show-stats -vv - fi - fi || true - section_end "ccache_show_stats" - -.upload_failed_job_log_artifacts: &upload_failed_job_log_artifacts | - if [ $CI_JOB_STATUS = "failed" ]; then - run_cmd idf-ci gitlab upload-artifacts --type log - fi - .before_script:minimal: before_script: - - *common-before_scripts + - !reference [.common_before_scripts, minimal] .after_script:build: after_script: - - source tools/ci/utils.sh - - source tools/ci/configure_ci_environment.sh - - *show_ccache_statistics - - *upload_failed_job_log_artifacts - -.brew-macos-settings: - variables: - GIT_STRATEGY: fetch - IDF_CCACHE_ENABLE: "0" - CCACHE_DIR: "/var/tmp/cache/idf_ccache" - tags: - - macos-tart - image: macos-sequoia-idf-v5.5 - after_script: [] # ccache now is disabled for macos brew runners - timeout: 30m + - !reference [.common_after_scripts, ccache] + - run_cmd idf-ci gitlab upload-artifacts --type junit + - | + if [ $CI_JOB_STATUS = "failed" ]; then + run_cmd idf-ci gitlab upload-artifacts --type log + fi ############# # `default` # @@ -290,12 +133,6 @@ default: cache: # pull only for most of the use cases since it's cache dir. # Only set "push" policy for "upload_cache" stage jobs - - key: pip-cache-${LATEST_GIT_TAG} - fallback_keys: - - pip-cache - paths: - - .cache/pip - policy: pull - key: submodule-cache-${LATEST_GIT_TAG} fallback_keys: - submodule-cache @@ -303,8 +140,7 @@ default: - .cache/submodule_archives policy: pull before_script: - - *common-before_scripts - - *setup_tools_and_idf_python_venv + - !reference [.common_before_scripts, with_tools] - add_gitlab_ssh_keys - fetch_submodules - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS} diff --git a/.gitlab/ci/default-build-test-rules.yml b/.gitlab/ci/default-build-test-rules.yml deleted file mode 100644 index 62b2bf09d0e..00000000000 --- a/.gitlab/ci/default-build-test-rules.yml +++ /dev/null @@ -1,21 +0,0 @@ -# this file support two keywords: -# - extra_default_build_targets: -# besides of the SUPPORTED_TARGETS in IDF, -# enable build for the specified targets by default as well. -# !!! DEPRECATED: use `additional_build_targets` in .idf_build_apps.toml instead -# -# - bypass_check_test_targets: -# suppress the check_build_test_rules check-test-script warnings for the specified targets -# -# This file should ONLY be used during bringup. Should be reset to empty after the bringup process - -bypass_check_test_targets: - - esp32h21 - - esp32h4 - - esp32c5 - -# -# These lines would -# - enable the README.md check for esp32c6. Don't forget to add the build jobs in .gitlab/ci/build.yml -# - disable the test script check with the manifest file. -# diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index 3baf3247a1e..e6d29784392 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -7,9 +7,6 @@ # run host_test jobs immediately, only after upload cache needs: - pipeline_variables - - job: upload-pip-cache - optional: true - artifacts: false - job: upload-submodules-cache optional: true artifacts: false @@ -373,7 +370,6 @@ test_pytest_qemu: - run_cmd pytest --target $IDF_TARGET -m qemu - --embedded-services idf,qemu --junitxml=XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} --qemu-extra-args \"-global driver=timer.$IDF_TARGET.timg,property=wdt_disable,value=true\" @@ -398,14 +394,14 @@ test_pytest_linux: - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} - run_cmd pytest --target linux - --embedded-services idf + -m \"not macos\" --junitxml=XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} test_pytest_macos: extends: - .host_test_template - - .brew-macos-settings + - .macos-settings artifacts: paths: - XUNIT_RESULT.xml @@ -480,6 +476,8 @@ test_pytest_macos: - pytest -k cmake --cleanup-idf-copy + --parallel-count ${CI_NODE_TOTAL:-1} + --parallel-index ${CI_NODE_INDEX:-1} --work-dir ${CI_PROJECT_DIR}/test_build_system --junitxml ${CI_PROJECT_DIR}/XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} @@ -491,7 +489,7 @@ pytest_build_system: pytest_build_system_macos: extends: - .test_build_system_template - - .brew-macos-settings + - .macos-settings - .rules:build:macos parallel: 3 @@ -501,7 +499,7 @@ pytest_build_system_minimal_cmake: pytest_build_system_macos_minimal_cmake: extends: - .test_build_system_minimal_cmake_template - - .brew-macos-settings + - .macos-settings - .rules:build:macos variables: INSTALL_EXTRA_TOOLS: ninja cmake@3.16.3 diff --git a/.gitlab/ci/post_deploy.yml b/.gitlab/ci/post_deploy.yml index c17e94089f7..6b8e91bfcf5 100644 --- a/.gitlab/ci/post_deploy.yml +++ b/.gitlab/ci/post_deploy.yml @@ -2,22 +2,6 @@ stage: post_deploy image: $ESP_ENV_IMAGE -generate_failed_jobs_report: - extends: - - .post_deploy_template - tags: [build, shiny] - when: always - dependencies: # Do not download artifacts from the previous stages - needs: - - pipeline_variables - artifacts: - expire_in: 2 week - when: always - paths: - - job_report.html - script: - - python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type job - sync_support_status: extends: - .post_deploy_template diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index 5f87d341e59..10de92a397e 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -1,9 +1,3 @@ -.pre_check_template: - stage: pre_check - image: $ESP_ENV_IMAGE - tags: [build, shiny] - dependencies: # set dependencies to null to avoid missing artifacts issue - check_version: # Don't run this for feature/bugfix branches, so that it is possible to modify # esp_idf_version.h in a branch before tagging the next version. @@ -119,127 +113,11 @@ check_test_scripts_build_test_rules: # requires basic pytest dependencies - python tools/ci/check_build_test_rules.py check-test-scripts examples/ tools/test_apps components -check_submodule_sync: +check_gitlab_yaml: extends: - .pre_check_template - - .rules:test:submodule - tags: [ brew, github_sync ] - retry: 2 - variables: - GIT_STRATEGY: fetch # use brew local mirror first - SUBMODULES_TO_FETCH: "none" - PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git" script: - - git submodule deinit --force . - - rm -rf .git/modules # remove all the cached metadata - # setting the default remote URL to the public one, to resolve relative location URLs - - git config remote.origin.url ${PUBLIC_IDF_URL} - # check if all submodules are correctly synced to public repository - - git submodule init - - git config --get-regexp '^submodule\..*\.url$' || true - - git submodule update --recursive - - echo "IDF was cloned from ${PUBLIC_IDF_URL} completely" - -pipeline_variables: - extends: - - .pre_check_template - tags: [fast_run, shiny] - variables: - GIT_STRATEGY: "fetch" - script: - - | - # Store the diff output in a temporary file - TEMP_FILE=$(mktemp) - # merged results pipelines, by default - if [[ -n $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ]]; then - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA - git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA - - git diff --name-only $CI_MERGE_REQUEST_TARGET_BRANCH_SHA...$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA > "$TEMP_FILE" - GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") - git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - # merge request pipelines, when the mr got conflicts - elif [[ -n $CI_MERGE_REQUEST_DIFF_BASE_SHA ]]; then - git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_COMMIT_SHA > "$TEMP_FILE" - GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") - # other pipelines, like the protected branches pipelines - elif [[ "$CI_COMMIT_BEFORE_SHA" != "0000000000000000000000000000000000000000" ]]; then - git fetch origin $CI_COMMIT_BEFORE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA > "$TEMP_FILE" - GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") - else - # pipeline source could be web, scheduler, etc. - git fetch origin $CI_COMMIT_SHA --depth=2 ${GIT_FETCH_EXTRA_FLAGS} - git diff --name-only $CI_COMMIT_SHA~1 $CI_COMMIT_SHA > "$TEMP_FILE" - GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") - fi - # MODIFIED_FILES is a list of files that changed, could be used everywhere - - MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | xargs) - - echo "MODIFIED_FILES=$MODIFIED_FILES" >> pipeline.env - # MR_MODIFIED_FILES and MR_MODIFIED_COMPONENTS are semicolon separated lists that is used in MR only - # for non MR pipeline, these are empty lists - - | - if [ -z "$CI_MERGE_REQUEST_IID" ]; then - echo "MR_MODIFIED_FILES=\"\"" >> pipeline.env - echo "MR_MODIFIED_COMPONENTS=\"\"" >> pipeline.env - else - MR_MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | tr '\n' ';') - echo "MR_MODIFIED_FILES=\"$MR_MODIFIED_FILES\"" >> pipeline.env - - MR_MODIFIED_COMPONENTS=$(run_cmd python tools/ci/ci_get_mr_info.py components --modified-files $MODIFIED_FILES | tr '\n' ';') - echo "MR_MODIFIED_COMPONENTS=\"$MR_MODIFIED_COMPONENTS\"" >> pipeline.env - fi - - | - if [ -n "$PIPELINE_COMMIT_SHA" ]; then - echo "PIPELINE_COMMIT_SHA=$PIPELINE_COMMIT_SHA" >> pipeline.env - fi - - echo "OOCD_DISTRO_URL_ARMHF=$OOCD_DISTRO_URL_ARMHF" >> pipeline.env - - echo "OOCD_DISTRO_URL_ARM64=$OOCD_DISTRO_URL_ARM64" >> pipeline.env - - run_cmd idf-ci gitlab pipeline-variables >> pipeline.env - - cat pipeline.env - - run_cmd idf-ci gitlab upload-artifacts --type env - artifacts: - reports: - dotenv: pipeline.env - paths: - - pipeline.env - expire_in: 1 week - when: always - -baseline_manifest_sha: - extends: - - .pre_check_template - - .rules:dev-push - tags: [fast_run, shiny] - script: - - | - # merged results pipelines, by default - # diff between target-branch-head and merged-result-head - if [ -n "$CI_MERGE_REQUEST_TARGET_BRANCH_SHA" ]; then - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA --depth=1 - git checkout FETCH_HEAD - idf-build-apps dump-manifest-sha \ - --manifest-files $(find . -name ".build-test-rules.yml" | xargs) \ - --output .manifest_sha - # merge request pipelines, when the mr got conflicts - # diff between diff-base-sha and merge-request-head - elif [ -n "$CI_MERGE_REQUEST_DIFF_BASE_SHA" ]; then - git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 - git checkout FETCH_HEAD - idf-build-apps dump-manifest-sha \ - --manifest-files $(find . -name ".build-test-rules.yml" | xargs) \ - --output .manifest_sha - # other pipelines, like the protected branches pipelines - # not triggered in this job - fi - artifacts: - paths: - - .manifest_sha - expire_in: 1 week - when: always + - python tools/ci/gitlab_yaml_linter.py retry_failed_jobs: extends: diff --git a/.gitlab/ci/pre_commit.yml b/.gitlab/ci/pre_commit.yml index 9abe06c11fc..0cdbcfea76a 100644 --- a/.gitlab/ci/pre_commit.yml +++ b/.gitlab/ci/pre_commit.yml @@ -2,7 +2,7 @@ extends: - .before_script:minimal stage: pre_check - image: "${CI_DOCKER_REGISTRY}/esp-idf-pre-commit:1" + image: "${CI_REGISTRY}/ci/images/pre-commit-idf-v5.5:1" tags: [pre-commit] variables: # Both shiny and brew runners can pick this job diff --git a/.gitlab/ci/pre_deploy.yml b/.gitlab/ci/pre_deploy.yml new file mode 100644 index 00000000000..df8525dcd24 --- /dev/null +++ b/.gitlab/ci/pre_deploy.yml @@ -0,0 +1,24 @@ +.pre_deploy_template: + stage: pre_deploy + image: $ESP_ENV_IMAGE + +check_submodule_sync: + extends: + - .pre_deploy_template + - .rules:test:submodule + tags: [ brew, github_sync ] + retry: 2 + variables: + GIT_STRATEGY: fetch # use brew local mirror first + SUBMODULES_TO_FETCH: "none" + PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git" + script: + - git submodule deinit --force . + - rm -rf .git/modules # remove all the cached metadata + # setting the default remote URL to the public one, to resolve relative location URLs + - git config remote.origin.url ${PUBLIC_IDF_URL} + # check if all submodules are correctly synced to public repository + - git submodule init + - git config --get-regexp '^submodule\..*\.url$' || true + - git submodule update --recursive + - echo "IDF was cloned from ${PUBLIC_IDF_URL} completely" diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index dd5dfa13f5c..6c1cd29093b 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -13,16 +13,6 @@ - "components/**/Kconfig" - "components/**/CMakeLists.txt" -.patterns-python-cache: &patterns-python-cache - - "tools/requirements.json" - - "tools/requirements/requirements.*.txt" - -.patterns-python-files: &patterns-python-files - - ".gitlab/ci/static-code-analysis.yml" - - "**/*.py" - -.patterns-sonarqube-files: &patterns-sonarqube-files - - "tools/ci/sonar_exclude_list.txt" .patterns-build_components: &patterns-build_components # components files except "test*/" "host*/" folders @@ -68,6 +58,7 @@ - "tools/ci/check_esp_memory_utils_headers.sh" - "tools/ci/check_blobs.sh" - "tools/ci/check_public_headers.py" + - "tools/ci/check_public_headers_exceptions.txt" - "tools/ci/check_register_rw_half_word.cmake" - "tools/ci/check_register_rw_half_word.py" - "examples/build_system/**/*" @@ -102,6 +93,7 @@ - "tools/idf_tools.py" - "tools/test_idf_tools/**/*" - "tools/install_util.py" + - "tools/idf_extra_components.yml" - "tools/export_utils/utils.py" - "tools/export_utils/shell_types.py" @@ -177,16 +169,10 @@ .if-protected-check: &if-protected-check if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) || $CI_COMMIT_TAG =~ /^qa-test/' -.if-protected-deploy: &if-protected-deploy - if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "api")' - .if-protected-branch-push: &if-protected-branch-push # rules:changes always evaluates to true for new branch pipelines or when there is no Git push event if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/) && $CI_PIPELINE_SOURCE == "push"' -.if-qa-test-tag: &if-qa-test-tag - if: '$CI_COMMIT_TAG =~ /^qa-test/' - .if-dev-push: &if-dev-push if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")' @@ -225,15 +211,6 @@ rules: - <<: *if-dev-push -.rules:upload-python-cache: - rules: - - <<: *if-release-tag - - <<: *if-schedule-nightly - - <<: *if-protected-branch-push - changes: *patterns-python-cache - - <<: *if-label-upload_cache - when: manual - .rules:upload-submodule-cache: rules: - <<: *if-release-tag diff --git a/.gitlab/ci/static-code-analysis.yml b/.gitlab/ci/static-code-analysis.yml index 734ff76a2ec..034c01bdb6f 100644 --- a/.gitlab/ci/static-code-analysis.yml +++ b/.gitlab/ci/static-code-analysis.yml @@ -30,84 +30,3 @@ gcc_static_analyzer: - export EXTRA_CFLAGS= - export EXTRA_CXXFLAGS= - idf-build-apps build -p ${ANALYZING_APP} --disable-targets esp32h4 esp32h21 --no-check-warnings -# -## build stage -## Sonarqube related jobs put here for this reason: -## Here we have two jobs. code_quality_check and code_quality_report. -## -## code_quality_check will analyze the code changes between your MR and -## code repo stored in sonarqube server. The analysis result is only shown in -## the comments under this MR and won't be transferred to the server. -## -## code_quality_report will analyze and transfer both of the newly added code -## and the analysis result to the server. -## -## Put in the front to ensure that the newly merged code can be stored in -## sonarqube server ASAP, in order to avoid reporting unrelated code issues -#.sonar_scan_template: -# stage: build -# extends: .pre_check_template -# # full clone since this image does not support fetch --shallow-since-cutoff -# # shiny runners are used for full clone -# tags: [build, shiny] -# image: $SONARQUBE_SCANNER_IMAGE -# before_script: -# - source tools/ci/utils.sh -# - export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH" -# - fetch_submodules -# # Exclude the submodules, all paths ends with /** -# - submodules=$(get_all_submodules) -# # get all exclude paths specified in tools/ci/sonar_exclude_list.txt | ignore lines start with # | xargs | replace all to -# - custom_excludes=$(cat $CI_PROJECT_DIR/tools/ci/sonar_exclude_list.txt | grep -v '^#' | xargs | sed -e 's/ /,/g') -# # Exclude the report dir as well -# - export EXCLUSIONS="$custom_excludes,$submodules" -# - export SONAR_SCANNER_OPTS="-Xmx2048m" -# variables: -# GIT_DEPTH: 0 -# REPORT_PATTERN: clang_tidy_reports/**/*.txt -# artifacts: -# paths: -# - $REPORT_PATTERN -# expire_in: 1 week -# when: always -# dependencies: # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here. -# - clang_tidy_check -# -#code_quality_check: -# extends: -# - .sonar_scan_template -# - .rules:patterns:static-code-analysis-preview -# allow_failure: true # it's using exit code to indicate the code analysis result, -# # we don't want to block ci when critical issues founded -# script: -# - export CI_MERGE_REQUEST_COMMITS=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py commits --src-branch ${CI_COMMIT_REF_NAME} | tr '\n' ',') -# # test if this branch have merge request, if not, exit 0 -# - test -n "$CI_MERGE_REQUEST_IID" || exit 0 -# - test -n "$CI_MERGE_REQUEST_COMMITS" || exit 0 -# - sonar-scanner -# -Dsonar.analysis.mode=preview -# -Dsonar.branch.name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME -# -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN -# -Dsonar.exclusions=$EXCLUSIONS -# -Dsonar.gitlab.ci_merge_request_iid=$CI_MERGE_REQUEST_IID -# -Dsonar.gitlab.commit_sha=$CI_MERGE_REQUEST_COMMITS -# -Dsonar.gitlab.merge_request_discussion=true -# -Dsonar.gitlab.ref_name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME -# -Dsonar.host.url=$SONAR_HOST_URL -# -Dsonar.login=$SONAR_LOGIN -# -#code_quality_report: -# extends: -# - .sonar_scan_template -# - .rules:protected:check -# allow_failure: true # it's using exit code to indicate the code analysis result, -# # we don't want to block ci when critical issues founded -# script: -# - sonar-scanner -# -Dsonar.branch.name=$CI_COMMIT_REF_NAME -# -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN -# -Dsonar.exclusions=$EXCLUSIONS -# -Dsonar.gitlab.commit_sha=$PIPELINE_COMMIT_SHA -# -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME -# -Dsonar.host.url=$SONAR_HOST_URL -# -Dsonar.login=$SONAR_LOGIN diff --git a/.gitlab/ci/test-win.yml b/.gitlab/ci/test-win.yml index 65185645c0e..9c0a90feac4 100644 --- a/.gitlab/ci/test-win.yml +++ b/.gitlab/ci/test-win.yml @@ -1,25 +1,16 @@ -# Host tests .host_test_win_template: - extends: .rules:test:host_test + extends: + - .windows-settings + - .rules:test:host_test stage: host_test - image: $ESP_ENV_IMAGE - tags: [windows-build, brew] dependencies: # set dependencies to null to avoid missing artifacts issue # run host_test jobs immediately, only after upload cache needs: - - job: upload-pip-cache - optional: true - artifacts: false - job: upload-submodules-cache optional: true artifacts: false - variables: - GIT_STRATEGY: fetch # use brew local mirror first - before_script: - - if ($env:IDF_DONT_USE_MIRRORS) { - $env:IDF_MIRROR_PREFIX_MAP = "" - } - after_script: [] + - job: manual_gate + optional: true test_cli_installer_win: rules: @@ -34,15 +25,12 @@ test_cli_installer_win: - tools/tools.new.json - tools/test_idf_tools/test_python_env_logs.txt expire_in: 1 week - variables: - IDF_PATH: "$CI_PROJECT_DIR" timeout: 3h script: # Tools must be downloaded for testing # cmake@version that is supported - - python ${IDF_PATH}\tools\idf_tools.py download required qemu-riscv32 qemu-xtensa cmake cmake@3.22.1 - - cd ${IDF_PATH}\tools\test_idf_tools - - python -m pip install jsonschema + - python tools\idf_tools.py download required qemu-riscv32 qemu-xtensa cmake cmake@3.22.1 + - cd tools\test_idf_tools - python .\test_idf_tools.py - python .\test_idf_tools_python_env.py @@ -53,82 +41,48 @@ test_tools_win: parallel: 4 artifacts: paths: - - ${IDF_PATH}/*.out - - ${IDF_PATH}/XUNIT_*.xml + - "*.out" + - "XUNIT_*.xml" reports: - junit: ${IDF_PATH}/XUNIT_*.xml + junit: "XUNIT_*.xml" expire_in: 1 week when: always - variables: - LC_ALL: C.UTF-8 - PYTHONPATH: "$PYTHONPATH;$IDF_PATH\\tools;$IDF_PATH\\tools\\esp_app_trace;$IDF_PATH\\components\\partition_table;$IDF_PATH\\tools\\ci\\python_packages" script: - - python -m pip install jsonschema - - .\install.ps1 --enable-ci - - .\export.ps1 - - python "${SUBMODULE_FETCH_TOOL}" -s "all" - - cd ${IDF_PATH}/tools/test_idf_py + - cd tools\test_idf_py - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} - - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=${IDF_PATH}/XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} + - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml ${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} # Build tests .test_build_system_template_win: - stage: host_test - variables: - # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings. - IDF_CCACHE_ENABLE: "1" - PYTHONPATH: "$PYTHONPATH;$IDF_PATH\\tools;$IDF_PATH\\tools\\esp_app_trace;$IDF_PATH\\components\\partition_table;$IDF_PATH\\tools\\ci\\python_packages" - before_script: [] - after_script: [] + extends: .host_test_win_template timeout: 4 hours + artifacts: + paths: + - XUNIT_RESULT.xml + - test_build_system + expire_in: 2 days + when: always + reports: + junit: XUNIT_RESULT.xml script: - - .\install.ps1 --enable-ci - - . .\export.ps1 - - python "${SUBMODULE_FETCH_TOOL}" -s "all" - - cd ${IDF_PATH}\tools\test_build_system + - cd tools\test_build_system - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} - - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} + - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml ${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} pytest_build_system_win: extends: - .test_build_system_template_win - .rules:labels:windows_pytest_build_system - parallel: 2 - needs: - - job: manual_gate - optional: true - tags: [windows-build, brew] - artifacts: - paths: - - XUNIT_RESULT.xml - - test_build_system - expire_in: 2 days - reports: - junit: XUNIT_RESULT.xml - when: always + parallel: 6 pytest_build_system_win_minimal_cmake: extends: - .test_build_system_template_win - .rules:labels:windows_pytest_build_system - needs: - - job: manual_gate - optional: true - tags: [windows-build, brew] - artifacts: - paths: - - XUNIT_RESULT.xml - - test_build_system - expire_in: 2 days - reports: - junit: XUNIT_RESULT.xml - when: always variables: MINIMAL_CMAKE_VERSION: "3.16.3" script: - - .\install.ps1 --enable-ci - - . .\export.ps1 - - python ${IDF_PATH}\tools\idf_tools.py install cmake@${MINIMAL_CMAKE_VERSION} + - python tools\idf_tools.py install cmake@${MINIMAL_CMAKE_VERSION} - $Env:PATH = "$Env:USERPROFILE\.espressif\tools\cmake\${MINIMAL_CMAKE_VERSION}\bin;$Env:PATH" - | $actualVersion = (& cmake --version).Split()[2] @@ -136,7 +90,6 @@ pytest_build_system_win_minimal_cmake: Write-Error "ERROR: Wrong CMake version! Detected: $actualVersion, but expected: $Env:MINIMAL_CMAKE_VERSION" exit 1 } - - python "${SUBMODULE_FETCH_TOOL}" -s "all" - - cd ${IDF_PATH}\tools\test_build_system + - cd tools\test_build_system - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} - - pytest -k cmake --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} + - pytest -k cmake --junitxml ${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} diff --git a/.gitlab/ci/upload_cache.yml b/.gitlab/ci/upload_cache.yml index fdd7e799cc1..320c0cd4244 100644 --- a/.gitlab/ci/upload_cache.yml +++ b/.gitlab/ci/upload_cache.yml @@ -6,26 +6,6 @@ stage: upload_cache image: $ESP_ENV_IMAGE -upload-pip-cache: - extends: - - .upload_cache_template - - .before_script:minimal - - .rules:upload-python-cache - tags: - - $GEO - - cache - cache: - key: pip-cache-${LATEST_GIT_TAG} - paths: - - .cache/pip - policy: push - script: - - rm -rf .cache/pip # clear old packages - - bash install.sh --enable-ci --enable-test-specific - parallel: - matrix: - - GEO: [ 'shiny', 'brew' ] - upload-submodules-cache: extends: - .upload_cache_template diff --git a/.idf_build_apps.toml b/.idf_build_apps.toml index 25d8d9d5900..b2ca5d4b340 100644 --- a/.idf_build_apps.toml +++ b/.idf_build_apps.toml @@ -31,8 +31,8 @@ additional_build_targets = [ ] # collect -collect_app_info_filename = "app_info_${CI_JOB_NAME_SLUG}.txt" -junitxml = "build_summary_${CI_JOB_NAME_SLUG}.xml" +collect_app_info_filename = "app_info_${CI_JOB_ID}.txt" +junitxml = "build_summary_${CI_JOB_ID}.xml" # manifest check_manifest_rules = true diff --git a/.idf_ci.toml b/.idf_ci.toml index f95c0332052..c79ce7dc432 100644 --- a/.idf_ci.toml +++ b/.idf_ci.toml @@ -1,4 +1,6 @@ preserve_non_test_related_apps = false +filter_non_test_related_apps_by_modified_files = true +filter_apps_by_component_target = true exclude_dirs = [ 'tools/test_mkdfu', 'tools/test_idf_size', @@ -20,7 +22,7 @@ IDF_CI_BUILD = "1" [gitlab.build_pipeline] workflow_name = "build_child_pipeline" -presigned_json_job_name = 'generate_pytest_build_report' +runs_per_job = 30 job_tags = ['build', 'shiny'] job_template_name = '.dynamic_build_template' @@ -51,37 +53,44 @@ include: - tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml """ -[gitlab.artifacts.s3.debug] +[gitlab.artifacts.s3] +enable = true + +[gitlab.artifacts.s3.configs.debug] bucket = "idf-artifacts" +zip_first = true +build_dir_pattern = "**/build*/" patterns = [ - '**/build*/bootloader/*.map', - '**/build*/bootloader/*.elf', - '**/build*/*.map', - '**/build*/*.elf', + 'bootloader/*.map', + 'bootloader/*.elf', + '*.map', + '*.elf', # customized - '**/build*/esp_tee/*.map', - '**/build*/esp_tee/*.elf', - '**/build*/gdbinit/*', + 'esp_tee/*.map', + 'esp_tee/*.elf', + 'gdbinit/*', ] if_clause = 'CI_JOB_GROUP_NAME != "build_non_test_related_apps"' -[gitlab.artifacts.s3.flash] +[gitlab.artifacts.s3.configs.flash] bucket = "idf-artifacts" +zip_first = true +build_dir_pattern = "**/build*/" patterns = [ - '**/build*/bootloader/*.bin', - '**/build*/*.bin', - '**/build*/partition_table/*.bin', - '**/build*/flasher_args.json', - '**/build*/flash_project_args', - '**/build*/config/sdkconfig.json', - '**/build*/sdkconfig', - '**/build*/project_description.json', + 'bootloader/*.bin', + '*.bin', + 'partition_table/*.bin', + 'flasher_args.json', + 'flash_project_args', + 'config/sdkconfig.json', + 'sdkconfig', + 'project_description.json', # customized - '**/build*/esp_tee/*.bin', + 'esp_tee/*.bin', ] if_clause = 'CI_JOB_GROUP_NAME != "build_non_test_related_apps"' -[gitlab.artifacts.s3.app_extra] +[gitlab.artifacts.s3.configs.app_extra] bucket = "idf-artifacts" # App-dir files needed at target-test time but outside build_* (uploaded without --build-dir). # Add patterns here as needed, e.g. coverage info produced during build. @@ -90,27 +99,27 @@ patterns = [ ] if_clause = 'CI_JOB_GROUP_NAME != "build_non_test_related_apps"' -[gitlab.artifacts.s3.log] +[gitlab.artifacts.s3.configs.log] bucket = "idf-artifacts" patterns = [ '**/build*/build_log.txt', '**/build*/size*.json', ] -[gitlab.artifacts.s3.junit] +[gitlab.artifacts.s3.configs.junit] bucket = "idf-artifacts" patterns = [ '**/XUNIT_RESULT_*.xml', '**/build_summary_*.xml', ] -[gitlab.artifacts.s3.env] +[gitlab.artifacts.s3.configs.env] bucket = "idf-artifacts" patterns = [ '**/pipeline.env', ] -[gitlab.artifacts.s3.longterm] +[gitlab.artifacts.s3.configs.longterm] bucket = "longterm" if_clause = 'CI_COMMIT_REF_NAME == "master"' patterns = [ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 592da90bef4..8889a892d3c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -146,7 +146,7 @@ repos: require_serial: true additional_dependencies: - PyYAML == 5.3.1 - - idf-build-apps~=2.13 + - idf-build-apps~=2.16 - id: sort-yaml-files name: sort yaml files entry: tools/ci/sort_yaml.py @@ -167,14 +167,6 @@ repos: language: python always_run: true require_serial: true - - id: gitlab-yaml-linter - name: Check gitlab yaml files - entry: tools/ci/gitlab_yaml_linter.py - language: python - files: '\.gitlab-ci\.yml|\.gitlab/ci/.+\.yml|\.gitmodules' - pass_filenames: false - additional_dependencies: - - PyYAML == 5.3.1 - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.0.1 hooks: @@ -229,7 +221,7 @@ repos: - id: check-kconfig-files - id: check-deprecated-kconfig-options - repo: https://github.com/espressif/idf-ci - rev: v1.0.0b4 + rev: v1.2.1 hooks: - id: check-tests-missing-config files: 'pytest_.*\.py$|sdkconfig(\..*)?$' diff --git a/components/bootloader_support/test_apps/bootloader_support/pytest_bootloader_support.py b/components/bootloader_support/test_apps/bootloader_support/pytest_bootloader_support.py index 59e3d4cf6a1..7bb24e96908 100644 --- a/components/bootloader_support/test_apps/bootloader_support/pytest_bootloader_support.py +++ b/components/bootloader_support/test_apps/bootloader_support/pytest_bootloader_support.py @@ -11,14 +11,12 @@ def test_bootloader_support(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.host_test @pytest.mark.qemu @idf_parametrize('target', ['esp32'], indirect=['target']) def test_bootloader_support_qemu_esp32(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.host_test @pytest.mark.qemu @idf_parametrize('target', ['esp32c3'], indirect=['target']) def test_bootloader_support_qemu_esp32c3(dut: Dut) -> None: diff --git a/components/bt/common/ble_log/extension/log_compression/tests/test_utils.py b/components/bt/common/ble_log/extension/log_compression/tests/test_utils.py index a576872fe61..6354bb5c0e6 100644 --- a/components/bt/common/ble_log/extension/log_compression/tests/test_utils.py +++ b/components/bt/common/ble_log/extension/log_compression/tests/test_utils.py @@ -12,6 +12,11 @@ import sys import unittest from pathlib import Path from typing import Any +from typing import Dict +from typing import List +from typing import Optional +from typing import Tuple +from typing import Union # Add scripts/ to sys.path so test modules can import the production code SCRIPTS_DIR = Path(__file__).resolve().parent.parent / 'scripts' @@ -61,11 +66,11 @@ def make_compressor() -> Any: def write_yaml_config( tmp_dir: str, - tags: list[str], - script_path: str | Path, + tags: List[str], + script_path: Union[str, Path], module_name: str = 'BLE_HOST', log_index_file: str = 'test_log_index.h', - tags_with_preserve: list[str] | None = None, + tags_with_preserve: Optional[List[str]] = None, ) -> Path: """Write a module_info.yml and return its path.""" import yaml @@ -140,9 +145,8 @@ class PipelineContext: shutil.copy2(src, dst) return str(rel_path) - def run_compression(self, src_list: list[str]) -> dict[str, list[tuple[int, str]]]: - """Run mirror + prepare + compress + header generation. Returns generated macros.""" - self.compressor.mirror_local_includes() + def run_compression(self, src_list: List[str]) -> Dict[str, List[Tuple[int, str]]]: + """Run prepare + compress + header generation. Returns generated macros.""" self.compressor.prepare_source_files(src_list) files_to_process = [] @@ -150,7 +154,7 @@ class PipelineContext: files_to_process.extend([(module, path) for path in info['files_to_process']]) files_to_process.sort(key=lambda x: x[1]) - all_macros: dict[str, list[tuple[int, str]]] = {} + all_macros: Dict[str, List[Tuple[int, str]]] = {} for file_info in files_to_process: file_macros = self.compressor.compress_file(file_info) for module, log_id, macro in file_macros: @@ -186,8 +190,8 @@ class PipelineContext: def assert_header_matches_golden( test_case: unittest.TestCase, - generated_path: str | Path, - golden_path: str | Path, + generated_path: Union[str, Path], + golden_path: Union[str, Path], ) -> None: """Compare generated header to golden file, normalizing copyright year.""" gen_text = Path(generated_path).read_text() diff --git a/components/console/test_apps/console/pytest_console.py b/components/console/test_apps/console/pytest_console.py index df7901ff366..ec6d281a045 100644 --- a/components/console/test_apps/console/pytest_console.py +++ b/components/console/test_apps/console/pytest_console.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 import pytest from pytest_embedded import Dut @@ -50,63 +50,59 @@ def do_test_help_quit(dut: Dut) -> None: dut.expect(r'quit\s+Quit REPL environment\s+esp>') -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console(dut: Dut, test_on: str) -> None: +def test_console(dut: Dut) -> None: dut.run_all_single_board_cases(group='!ignore', timeout=120) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_repl(dut: Dut, test_on: str) -> None: +def test_console_repl(dut: Dut) -> None: do_test_quit(dut) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_sorted_registration(dut: Dut, test_on: str) -> None: +def test_console_help_sorted_registration(dut: Dut) -> None: do_test_help_generic(dut, 'sorted') -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_reverse_registration(dut: Dut, test_on: str) -> None: +def test_console_help_reverse_registration(dut: Dut) -> None: dut.expect_exact('Press ENTER to see the list of tests') dut.confirm_write('"esp console help command - reverse registration"', expect_str='esp>') @@ -131,45 +127,41 @@ def test_console_help_reverse_registration(dut: Dut, test_on: str) -> None: @idf_parametrize('config', ['sorted'], indirect=['config']) @idf_parametrize('target', ['linux'], indirect=['target']) -@idf_parametrize('test_on,markers', [('host', (pytest.mark.host_test,))]) -def test_console_sorted_help_sorted_registration(dut: Dut, test_on: str) -> None: +def test_console_sorted_help_sorted_registration(dut: Dut) -> None: do_test_help_generic(dut, 'sorted') @idf_parametrize('config', ['sorted'], indirect=['config']) @idf_parametrize('target', ['linux'], indirect=['target']) -@idf_parametrize('test_on,markers', [('host', (pytest.mark.host_test,))]) -def test_console_sorted_help_reverse_registration(dut: Dut, test_on: str) -> None: +def test_console_sorted_help_reverse_registration(dut: Dut) -> None: do_test_help_generic(dut, 'reverse') -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_quit(dut: Dut, test_on: str) -> None: +def test_console_help_quit(dut: Dut) -> None: do_test_help_quit(dut) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_verbose_level_0(dut: Dut, test_on: str) -> None: +def test_console_help_verbose_level_0(dut: Dut) -> None: help_verbose_info = 'Print the summary of all registered commands if no arguments are given,' dut.expect_exact('Press ENTER to see the list of tests') dut.confirm_write('"esp console help command - set verbose level = 0"', expect_str='esp>') @@ -180,18 +172,17 @@ def test_console_help_verbose_level_0(dut: Dut, test_on: str) -> None: dut.expect_exact('help', not_matching=help_verbose_info) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_verbose_level_1(dut: Dut, test_on: str) -> None: +def test_console_help_verbose_level_1(dut: Dut) -> None: help_verbose_info = 'Print the summary of all registered commands if no arguments are given,' dut.expect_exact('Press ENTER to see the list of tests') dut.confirm_write('"esp console help command - set verbose level = 1"', expect_str='esp>') @@ -201,18 +192,17 @@ def test_console_help_verbose_level_1(dut: Dut, test_on: str) -> None: dut.expect_exact(help_verbose_info) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_verbose_subcommand(dut: Dut, test_on: str) -> None: +def test_console_help_verbose_subcommand(dut: Dut) -> None: help_verbose_info = 'Print the summary of all registered commands if no arguments are given,' dut.expect_exact('Press ENTER to see the list of tests') dut.confirm_write('"esp console help command - --verbose sub command"', expect_str='esp>') @@ -227,18 +217,17 @@ def test_console_help_verbose_subcommand(dut: Dut, test_on: str) -> None: dut.expect_exact(help_verbose_info) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_deregister(dut: Dut, test_on: str) -> None: +def test_console_help_deregister(dut: Dut) -> None: dut.expect_exact('Press ENTER to see the list of tests') dut.confirm_write('"esp console deregister commands"', expect_str='esp>') @@ -250,18 +239,17 @@ def test_console_help_deregister(dut: Dut, test_on: str) -> None: dut.expect_exact(cmd_z_description, not_matching=cmd_a_description) -@idf_parametrize('config', ['defaults'], indirect=['config']) @idf_parametrize( - 'target,test_on,markers', + 'target,config,embedded_services,markers', [ - ('linux', 'host', (pytest.mark.host_test,)), - ('esp32', 'target', (pytest.mark.generic,)), - ('esp32c3', 'target', (pytest.mark.generic,)), - ('esp32', 'qemu', (pytest.mark.host_test, pytest.mark.qemu)), + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), ], - indirect=['target'], + indirect=['target', 'config', 'embedded_services'], ) -def test_console_help_re_register(dut: Dut, test_on: str) -> None: +def test_console_help_re_register(dut: Dut) -> None: dut.expect_exact('Press ENTER to see the list of tests') dut.confirm_write('"esp console re-register commands"', expect_str='esp>') diff --git a/components/efuse/test_apps/pytest_efuse.py b/components/efuse/test_apps/pytest_efuse.py index 7d9c5367359..3fb48bf0b8e 100644 --- a/components/efuse/test_apps/pytest_efuse.py +++ b/components/efuse/test_apps/pytest_efuse.py @@ -16,7 +16,6 @@ def test_efuse(dut: Dut) -> None: @pytest.mark.qemu -@pytest.mark.host_test @idf_parametrize('target', ['esp32'], indirect=['target']) def test_efuse_qemu(dut: Dut) -> None: dut.run_all_single_board_cases() diff --git a/components/esp_event/test_apps/pytest_esp_event.py b/components/esp_event/test_apps/pytest_esp_event.py index 20d59bb2b9e..5840e9bce33 100644 --- a/components/esp_event/test_apps/pytest_esp_event.py +++ b/components/esp_event/test_apps/pytest_esp_event.py @@ -19,7 +19,6 @@ def test_esp_event(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.host_test @pytest.mark.qemu @pytest.mark.xfail('config.getvalue("target") == "esp32c3"', reason='Unstable on QEMU, needs investigation') @pytest.mark.parametrize( diff --git a/components/esp_ringbuf/test_apps/pytest_esp_ringbuf.py b/components/esp_ringbuf/test_apps/pytest_esp_ringbuf.py index ca44dcfa16c..f27087d9c9d 100644 --- a/components/esp_ringbuf/test_apps/pytest_esp_ringbuf.py +++ b/components/esp_ringbuf/test_apps/pytest_esp_ringbuf.py @@ -24,7 +24,6 @@ def test_esp_ringbuf(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.host_test @pytest.mark.qemu @pytest.mark.parametrize( 'target', diff --git a/components/esp_rom/test_apps/rom_tests/pytest_esp_rom.py b/components/esp_rom/test_apps/rom_tests/pytest_esp_rom.py index 0bc7bfef1a0..b3e41e01af8 100644 --- a/components/esp_rom/test_apps/rom_tests/pytest_esp_rom.py +++ b/components/esp_rom/test_apps/rom_tests/pytest_esp_rom.py @@ -15,7 +15,6 @@ def test_esp_rom(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.host_test @pytest.mark.qemu @idf_parametrize('target', ['esp32c3'], indirect=['target']) def test_esp_rom_qemu(dut: Dut) -> None: diff --git a/components/heap/test_apps/heap_tests/pytest_heap.py b/components/heap/test_apps/heap_tests/pytest_heap.py index c4eb959a76d..08225beeacf 100644 --- a/components/heap/test_apps/heap_tests/pytest_heap.py +++ b/components/heap/test_apps/heap_tests/pytest_heap.py @@ -20,7 +20,6 @@ def test_heap_poisoning(dut: Dut) -> None: dut.run_all_single_board_cases() -@pytest.mark.host_test @pytest.mark.qemu @pytest.mark.parametrize( 'config, embedded_services', diff --git a/components/pthread/test_apps/pthread_unity_tests/pytest_pthread_unity_tests.py b/components/pthread/test_apps/pthread_unity_tests/pytest_pthread_unity_tests.py index 5e6aba09c9f..62d2278b30b 100644 --- a/components/pthread/test_apps/pthread_unity_tests/pytest_pthread_unity_tests.py +++ b/components/pthread/test_apps/pthread_unity_tests/pytest_pthread_unity_tests.py @@ -46,7 +46,6 @@ def test_pthread_single_core_tls(dut: Dut) -> None: dut.run_all_single_board_cases(group='thread-specific', timeout=300) -@pytest.mark.host_test @pytest.mark.qemu @idf_parametrize('target', ['esp32'], indirect=['target']) def test_pthread_qemu(dut: Dut) -> None: diff --git a/conftest.py b/conftest.py index d58c8b38bf2..cc576aa1b4c 100644 --- a/conftest.py +++ b/conftest.py @@ -44,6 +44,7 @@ from idf_pytest.plugin import IDF_LOCAL_PLUGIN_KEY from idf_pytest.plugin import IdfLocalPlugin from idf_pytest.plugin import requires_elf_or_map from idf_pytest.utils import format_case_id +from pytest_embedded.plugin import _request_param_or_config_option_or_default from pytest_embedded.plugin import multi_dut_fixture from pytest_embedded.utils import to_bytes from pytest_embedded.utils import to_str @@ -127,7 +128,7 @@ class AppDownloader: self.commit_sha = commit_sha self.pipeline_id = pipeline_id - def download_app(self, app_build_path: str, artifact_type: t.Optional[str] = None) -> None: + def download_app(self, app_dir: str, build_dir: str, artifact_type: t.Optional[str] = None) -> None: args = [ 'idf-ci', 'gitlab', @@ -137,15 +138,26 @@ class AppDownloader: ] if artifact_type: args.extend(['--type', artifact_type]) + if self.pipeline_id: args.extend(['--pipeline-id', self.pipeline_id]) - args.append(app_build_path) - subprocess.run( - args, - stdout=sys.stdout, - stderr=sys.stderr, + args.extend( + [ + app_dir, + '--build-dir', + build_dir, + ] ) + result = subprocess.run( + args, + capture_output=True, + text=True, + cwd=IDF_PATH, + ) + logging.info(result.stdout) + if result.stderr: + logging.info(result.stderr) def download_app_extra(self, app_dir: str) -> None: """Download app-dir artifacts defined under app_extra in .idf_ci.toml.""" @@ -348,12 +360,13 @@ def build_dir( downloader = app_downloader if downloader: + app_dir = idf_relpath(app_path) + build_dir = f'build_{target}_{config}' # somehow hardcoded... - app_build_path = os.path.join(idf_relpath(app_path), f'build_{target}_{config}') if requires_elf_or_map(case): - downloader.download_app(app_build_path) + downloader.download_app(app_dir, build_dir) else: - downloader.download_app(app_build_path, 'flash') + downloader.download_app(app_dir, build_dir, 'flash') check_dirs = [f'build_{target}_{config}'] else: check_dirs = [] @@ -537,12 +550,12 @@ def log_minimum_free_heap_size(dut: IdfDut, config: str, idf_path: str) -> t.Cal @pytest.fixture(scope='session') def dev_password(request: FixtureRequest) -> str: - return request.config.getoption('dev_passwd') or '' + return _request_param_or_config_option_or_default(request, 'dev_password', '') # type: ignore @pytest.fixture(scope='session') def dev_user(request: FixtureRequest) -> str: - return request.config.getoption('dev_user') or '' + return _request_param_or_config_option_or_default(request, 'dev_user', '') # type: ignore ################## @@ -585,7 +598,7 @@ def pytest_unconfigure(config: Config) -> None: config.pluginmanager.unregister(idf_local_plugin) -dut_artifacts_url = [] +dut_artifacts_url: t.List[str] = [] @pytest.hookimpl(hookwrapper=True) diff --git a/docs/en/api-guides/tools/idf-tools.rst b/docs/en/api-guides/tools/idf-tools.rst index 8472393d974..0bfa0e79a2b 100644 --- a/docs/en/api-guides/tools/idf-tools.rst +++ b/docs/en/api-guides/tools/idf-tools.rst @@ -45,6 +45,8 @@ Inside the ``IDF_TOOLS_PATH`` directory, the tools installation scripts create t - ``python_env`` — not related to the tools; virtual Python environments are installed in the sub-directories. Note that the Python environment directory can be placed elsewhere by setting the ``IDF_PYTHON_ENV_PATH`` environment variable. - ``idf_version.txt`` — located within each specific Python environment sub-directory under ``python_env``, this file records the ESP-IDF version corresponding to that environment. The version is stored in a format like ``5.3`` to represent ESP-IDF version ``v5.3``. + +- ``root_managed_components`` — directory managed by ``idf-component-manager`` for components installed globally. - ``espidf.constraints.*.txt`` — one constraint file for each ESP-IDF release containing Python package version requirements. GitHub Assets Mirror diff --git a/docs/en/contribute/esp-idf-tests-with-pytest.rst b/docs/en/contribute/esp-idf-tests-with-pytest.rst index 864900d701a..caf409b5655 100644 --- a/docs/en/contribute/esp-idf-tests-with-pytest.rst +++ b/docs/en/contribute/esp-idf-tests-with-pytest.rst @@ -120,6 +120,52 @@ Next, we have the environment marker: Finally, we have the test function. With a ``dut`` fixture. In single-dut test cases, the ``dut`` fixture is an instance of ``IdfDut`` class, for multi-dut test cases, it is a tuple of ``IdfDut`` instances. For more details regarding the ``IdfDut`` class, please refer to `pytest-embedded IdfDut API reference `__. +Running Tests on Linux +^^^^^^^^^^^^^^^^^^^^^^ + +To execute a pytest case on the Linux host, set ``target`` to ``linux``. + +.. code-block:: python + + @idf_parametrize('target', ['linux'], indirect=['target']) + def test_hello_world_linux(dut) -> None: + dut.expect('Hello world!') + +This is the simplest way to run the same test flow on the Linux host instead of on physical hardware. + +For a simple Linux-only test, setting ``target`` to ``linux`` is enough and the ``idf`` embedded services will be selected automatically. The ``pytest.mark.host_test`` marker is not required. + +For a mixed environment matrix, specify ``embedded_services`` manually for each case. See :ref:`Same App With Different Running Environments ` for a more complex example. + +.. only:: TARGET_SUPPORT_QEMU + + Running Tests in QEMU + ^^^^^^^^^^^^^^^^^^^^^ + + To execute a pytest case in QEMU, add the ``@pytest.mark.qemu`` marker to the test function. + + .. code-block:: python + + @pytest.mark.qemu + @idf_parametrize('target', ['esp32', 'esp32c3'], indirect=['target']) + def test_hello_world_qemu(dut) -> None: + dut.expect('Hello world!') + + This is the simplest way to run the same test flow in QEMU instead of on physical hardware. + + For a simple QEMU-only test, adding ``pytest.mark.qemu`` is enough and the ``idf,qemu`` embedded services will be selected automatically. + + For a mixed environment matrix, specify ``embedded_services`` manually for each case. See the later section in this guide for a more complex example. + + For QEMU installation and setup, refer to page :doc:`/api-guides/tools/qemu`. + +Deprecation of ``pytest.mark.host_test`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``pytest.mark.host_test`` is no longer needed and should not be added to new test cases. + +For Linux target test cases and QEMU test cases, the required behavior is handled dynamically by the test framework. In particular, the embedded services are selected automatically for simple Linux-only and QEMU-only cases. + Same App With Different sdkconfig Files ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -208,6 +254,49 @@ Now this test function would be replicated to 2 test cases (represented as test * ``esp32.foo.test_foo_bar`` * ``esp32s2.bar.test_foo_bar`` +.. _pytest-same-app-different-running-environments: + +Same App With Different Running Environments +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Sometimes the same app should be validated in different running environments, for example on the host with the Linux target, on real hardware, or in QEMU. If a single ``@pytest.mark.qemu`` test is not enough, combine ``target``, ``config``, and ``embedded_services`` in a single ``idf_parametrize`` decorator, and attach the required marker for each case. + +The following example is adapted from :idf_file:`components/console/test_apps/console/pytest_console.py`: + +.. code-block:: python + + @idf_parametrize( + 'target,config,embedded_services,markers', + [ + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), + ], + indirect=['target', 'config', 'embedded_services'], + ) + def test_console_repl(dut) -> None: + dut.expect_exact('Press ENTER to see the list of tests') + +This creates four test cases for the same app: + +* Linux host execution with the ``idf`` service +* ESP32 hardware execution with the ``esp,idf`` services +* ESP32-C3 hardware execution with the ``esp,idf`` services +* ESP32 execution in QEMU with the ``idf,qemu`` services + +When running locally, you can select only the environment you want: + +.. code-block:: shell + + $ pytest --target linux + $ pytest -m qemu + $ pytest -m qemu --target esp32 + +``pytest --target linux`` selects Linux target cases only. ``pytest -m qemu`` selects all QEMU-marked cases. ``pytest -m qemu --target esp32`` further limits the selection to QEMU cases for the ESP32 target. + +Use this pattern when the test logic is the same but the execution environment changes. + Testing Serial Output (Expecting) --------------------------------- diff --git a/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst b/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst index 82cb65cd3b4..d9a92bb1222 100644 --- a/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst +++ b/docs/zh_CN/contribute/esp-idf-tests-with-pytest.rst @@ -120,6 +120,52 @@ ESP-IDF 在主机端使用 pytest 框架(以及一些 pytest 插件)来自 关于测试函数,使用了一个 ``dut`` fixture。在单一 DUT 测试用例中,``dut`` fixture 是 ``IdfDut`` 类的一个实例,对于多个 DUT 测试用例,它是 ``IdfDut`` 实例的一个元组。有关 ``IdfDut`` 类的更多详细信息,请参阅 `pytest-embedded IdfDut API 参考 `__。 +在 Linux 上运行测试 +^^^^^^^^^^^^^^^^^^^^ + +要在 Linux 主机上执行 pytest 测试用例,请将 ``target`` 设置为 ``linux``。 + +.. code-block:: python + + @idf_parametrize('target', ['linux'], indirect=['target']) + def test_hello_world_linux(dut) -> None: + dut.expect('Hello world!') + +这是在 Linux 主机上运行与物理硬件相同测试流程的最简单方式。 + +对于简单的纯 Linux 测试,只需将 ``target`` 设置为 ``linux``,系统会自动选择 ``idf`` 对应的 embedded services。``pytest.mark.host_test`` marker 不再需要。 + +对于混合运行环境矩阵,则需要为每个用例手动指定 ``embedded_services``。更复杂的示例请参阅 :ref:`在不同运行环境中运行相同的应用程序 ` 小节。 + +.. only:: TARGET_SUPPORT_QEMU + + 在 QEMU 中运行测试 + ^^^^^^^^^^^^^^^^^^^^^ + + 要在 QEMU 中执行 pytest 测试用例,请将 ``@pytest.mark.qemu`` 添加到测试函数上。 + + .. code-block:: python + + @pytest.mark.qemu + @idf_parametrize('target', ['esp32', 'esp32c3'], indirect=['target']) + def test_hello_world_qemu(dut) -> None: + dut.expect('Hello world!') + + 这是在 QEMU 中运行与物理硬件相同测试流程的最简单方式。 + + 对于简单的纯 QEMU 测试,只需添加 ``pytest.mark.qemu``,系统会自动选择 ``idf,qemu`` 对应的 embedded services。 + + 对于混合运行环境矩阵,则需要为每个用例手动指定 ``embedded_services``。更复杂的示例请参阅本指南后面的对应小节。 + + 有关 QEMU 的安装和配置,请参阅页面 :doc:`../api-guides/tools/qemu`。 + +``pytest.mark.host_test`` 的弃用说明 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``pytest.mark.host_test`` 已不再需要,也不应再添加到新的测试用例中。 + +对于 Linux target 测试用例和 QEMU 测试用例,相关行为会由测试框架动态处理。尤其是在简单的纯 Linux 或纯 QEMU 场景下,embedded services 会被自动选择。 + 使用不同的 sdkconfig 文件运行相同的应用程序 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -208,6 +254,49 @@ ESP-IDF 在主机端使用 pytest 框架(以及一些 pytest 插件)来自 * ``esp32.foo.test_foo_bar`` * ``esp32s2.bar.test_foo_bar`` +.. _pytest-same-app-different-running-environments: + +在不同运行环境中运行相同的应用程序 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +有时,同一个应用程序需要在不同的运行环境中进行验证,例如在 Linux target 的主机上、真实硬件上,或在 QEMU 中运行。如果单独使用 ``@pytest.mark.qemu`` 测试还不够,可以在一个 ``idf_parametrize`` 装饰器中组合 ``target``、``config`` 和 ``embedded_services``,并为每种情况附加所需的 marker。 + +下面的示例改编自 :idf_file:`components/console/test_apps/console/pytest_console.py`: + +.. code-block:: python + + @idf_parametrize( + 'target,config,embedded_services,markers', + [ + ('linux', 'defaults', 'idf', ()), + ('esp32', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32c3', 'defaults', 'esp,idf', (pytest.mark.generic,)), + ('esp32', 'defaults', 'idf,qemu', (pytest.mark.qemu,)), + ], + indirect=['target', 'config', 'embedded_services'], + ) + def test_console_repl(dut) -> None: + dut.expect_exact('Press ENTER to see the list of tests') + +这会为同一个应用程序生成 4 个测试用例: + +* 在 Linux 主机上使用 ``idf`` service 运行 +* 在 ESP32 硬件上使用 ``esp,idf`` services 运行 +* 在 ESP32-C3 硬件上使用 ``esp,idf`` services 运行 +* 在 QEMU 中以 ESP32 为目标,使用 ``idf,qemu`` services 运行 + +在本地运行时,可以按需只选择某一种运行环境: + +.. code-block:: shell + + $ pytest --target linux + $ pytest -m qemu + $ pytest -m qemu --target esp32 + +``pytest --target linux`` 只选择 Linux target 的测试用例。``pytest -m qemu`` 选择所有带有 QEMU marker 的测试用例。``pytest -m qemu --target esp32`` 会进一步把范围限制为目标芯片为 ESP32 的 QEMU 测试用例。 + +当测试逻辑相同,但执行环境不同的时候,可使用此模式。 + 测试串行输出 ^^^^^^^^^^^^^^^^ diff --git a/examples/build_system/cmake/import_lib/pytest_import_lib.py b/examples/build_system/cmake/import_lib/pytest_import_lib.py index 95f86b0807f..4bf5203c624 100644 --- a/examples/build_system/cmake/import_lib/pytest_import_lib.py +++ b/examples/build_system/cmake/import_lib/pytest_import_lib.py @@ -5,7 +5,6 @@ from pytest_embedded_idf.utils import idf_parametrize from pytest_embedded_qemu.dut import QemuDut -@pytest.mark.host_test @pytest.mark.qemu @idf_parametrize('target', ['esp32', 'esp32c3'], indirect=['target']) def test_pytest_host(dut: QemuDut) -> None: diff --git a/examples/get-started/hello_world/pytest_hello_world.py b/examples/get-started/hello_world/pytest_hello_world.py index 64f6bb8f5fb..e3ae95b6542 100644 --- a/examples/get-started/hello_world/pytest_hello_world.py +++ b/examples/get-started/hello_world/pytest_hello_world.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: CC0-1.0 import hashlib import logging -from typing import Callable +from collections.abc import Callable import pytest from pytest_embedded_idf.dut import IdfDut @@ -45,7 +45,6 @@ def verify_elf_sha256_embedding(app: QemuApp, sha256_reported: str) -> None: raise ValueError('ELF file SHA256 mismatch') -@pytest.mark.host_test @pytest.mark.qemu @idf_parametrize('target', ['esp32', 'esp32c3'], indirect=['target']) def test_hello_world_host(app: QemuApp, dut: QemuDut) -> None: diff --git a/examples/network/bridge/pytest_example_bridge.py b/examples/network/bridge/pytest_example_bridge.py index 0e805aeda15..cb223514c73 100644 --- a/examples/network/bridge/pytest_example_bridge.py +++ b/examples/network/bridge/pytest_example_bridge.py @@ -29,6 +29,8 @@ from pytest_embedded_idf.utils import idf_parametrize # Testbed configuration ETHVM_ENDNODE_USER = 'ci.ethvm' +ETHERNET_TEST_USER = os.getenv('ETHERNET_TEST_USER') +ETHERNET_TEST_PASSWORD = os.getenv('ETHERNET_TEST_PASSWORD') BR_PORTS_NUM = 2 IPERF_BW_LIM = 6 @@ -636,5 +638,10 @@ def setup_test_environment() -> Generator[None, None, None]: indirect=True, ) @idf_parametrize('target', ['esp32'], indirect=['target']) +@pytest.mark.parametrize( + 'dev_user, dev_password', + [(ETHERNET_TEST_USER, ETHERNET_TEST_PASSWORD)], + indirect=True, +) def test_esp_eth_bridge(dut: Dut, dev_user: str, dev_password: str) -> None: eth_bridge_test(dut, dev_user, dev_password) diff --git a/examples/protocols/https_x509_bundle/pytest_https_x509_bundle.py b/examples/protocols/https_x509_bundle/pytest_https_x509_bundle.py index 8b4b89ca8f1..ddc6d818426 100644 --- a/examples/protocols/https_x509_bundle/pytest_https_x509_bundle.py +++ b/examples/protocols/https_x509_bundle/pytest_https_x509_bundle.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 import logging import os @@ -52,7 +52,6 @@ def test_examples_protocol_https_x509_bundle_dynamic_buffer(dut: Dut) -> None: @pytest.mark.qemu -@pytest.mark.host_test @pytest.mark.parametrize( 'config', [ diff --git a/examples/system/efuse/pytest_system_efuse_example.py b/examples/system/efuse/pytest_system_efuse_example.py index 1a0708e6645..bfea11cd0ee 100644 --- a/examples/system/efuse/pytest_system_efuse_example.py +++ b/examples/system/efuse/pytest_system_efuse_example.py @@ -900,7 +900,6 @@ def test_examples_efuse_with_virt_sb_v2_and_fe(dut: Dut) -> None: dut.expect('example: Done') -@pytest.mark.host_test @pytest.mark.qemu @pytest.mark.parametrize( 'config', diff --git a/examples/system/ota/advanced_https_ota/pytest_advanced_ota.py b/examples/system/ota/advanced_https_ota/pytest_advanced_ota.py index 7ad0897509a..0743975cfe0 100644 --- a/examples/system/ota/advanced_https_ota/pytest_advanced_ota.py +++ b/examples/system/ota/advanced_https_ota/pytest_advanced_ota.py @@ -1110,7 +1110,6 @@ def test_examples_protocol_advanced_https_ota_example_openssl_aligned_bin(dut: D @pytest.mark.qemu @pytest.mark.nightly_run -@pytest.mark.host_test @pytest.mark.parametrize( 'qemu_extra_args', [ @@ -1166,7 +1165,6 @@ def test_examples_protocol_advanced_https_ota_example_verify_min_chip_revision(d @pytest.mark.qemu @pytest.mark.nightly_run -@pytest.mark.host_test @pytest.mark.parametrize( 'qemu_extra_args', [ diff --git a/sonar-project.properties b/sonar-project.properties deleted file mode 100644 index 1c9d5882fb3..00000000000 --- a/sonar-project.properties +++ /dev/null @@ -1,24 +0,0 @@ -sonar.projectKey=esp-idf -sonar.projectName=Espressif IoT Development Framework -sonar.projectVersion=4.4 - -sonar.sources=. -sonar.sourceEncoding=UTF-8 - -sonar.inclusions=\ - **/*.c,**/*.cpp,**/*.h,**/*.hpp,\ - **/*.py - -#################### -# sonar-cxx-plugin # -#################### - -sonar.cxx.includeDirectories=/usr/include -sonar.cxx.file.suffixes=.cxx,.cpp,.cc,.c,.hxx,.hpp,.hh,.h - -####################### -# sonar-gitlab-plugin # -####################### - -sonar.gitlab.failure_notification_mode=exit-code -sonar.gitlab.project_id=103 diff --git a/tools/ci/check_build_test_rules.py b/tools/ci/check_build_test_rules.py index 5f9727aef2f..a4467f9d6b2 100755 --- a/tools/ci/check_build_test_rules.py +++ b/tools/ci/check_build_test_rules.py @@ -9,7 +9,6 @@ import typing as t from collections import defaultdict from pathlib import Path -import yaml from idf_build_apps import App from idf_build_apps import find_apps from idf_build_apps.constants import SUPPORTED_TARGETS @@ -40,6 +39,14 @@ USUAL_TO_FORMAL = { FORMAL_TO_USUAL = {v: k for k, v in USUAL_TO_FORMAL.items()} +# Targets under bringup: suppress check-test-scripts warnings for these. +# Reset to [] once bringup is complete. +BYPASS_CHECK_TEST_TARGETS: t.List[str] = [ + 'esp32h21', + 'esp32h4', + 'esp32c5', +] + def print_diff_table( list1: t.List[str], list2: t.List[str], title1: str, title2: str, excluded: t.Optional[t.List[str]] = None @@ -299,21 +306,9 @@ if __name__ == '__main__': readme_parser = action.add_parser('check-readmes') readme_parser.add_argument('paths', nargs='+', help='check under paths') - readme_parser.add_argument( - '-c', - '--config', - default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'), - help='config file', - ) test_parser = action.add_parser('check-test-scripts') test_parser.add_argument('paths', nargs='+', help='check under paths') - test_parser.add_argument( - '-c', - '--config', - default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'), - help='config file', - ) arg = parser.parse_args() check_dirs = set() @@ -344,14 +339,6 @@ if __name__ == '__main__': os.path.join(IDF_PATH, 'tools', 'cmakev2', 'test'), ] - _bypass_targets: t.List[str] = [] - if arg.config: - with open(arg.config) as fr: - configs = yaml.safe_load(fr) - - if configs: - _bypass_targets = configs.get('bypass_check_test_targets') or [] - os.environ.update( { 'IDF_PATH': IDF_PATH, @@ -369,5 +356,5 @@ if __name__ == '__main__': check_test_scripts( list(check_dirs), _exclude_dirs, - bypass_targets=_bypass_targets, + bypass_targets=BYPASS_CHECK_TEST_TARGETS, ) diff --git a/tools/ci/check_public_headers.py b/tools/ci/check_public_headers.py index a8c3891c8c6..cba9b1b5768 100644 --- a/tools/ci/check_public_headers.py +++ b/tools/ci/check_public_headers.py @@ -2,7 +2,7 @@ # # Checks all public headers in IDF in the ci # -# SPDX-FileCopyrightText: 2020-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2020-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # import argparse @@ -309,6 +309,10 @@ class PublicHeaderChecker: idf_path = os.getenv('IDF_PATH') if idf_path is None: raise RuntimeError("Environment variable 'IDF_PATH' wasn't set.") + + idf_tools_path = os.getenv('IDF_TOOLS_PATH') or os.path.expanduser(os.path.join('~', '.espressif')) + idf_root_dep_path = os.path.join(idf_tools_path, 'root_managed_components') + project_dir = os.path.join(idf_path, 'examples', 'get-started', 'blink') sdkconfig = os.path.join(self.build_dir, 'sdkconfig') if self.libc_type == 'picolibc': @@ -371,7 +375,11 @@ class PublicHeaderChecker: if os.path.relpath(d, idf_path).startswith(tuple(ignore_dirs)): self.log('{} - directory ignored'.format(d)) continue - for root, dirnames, filenames in os.walk(d): + for root, _, filenames in os.walk(d): + if root.startswith(idf_root_dep_path): + self.log(f'{root} - directory ignored (inside IDF_TOOLS_PATH/root_managed_components)') + continue + for filename in fnmatch.filter(filenames, '*.h'): all_include_files.append(os.path.join(root, filename)) self.main_c = main_c diff --git a/tools/ci/ci_get_mr_info.py b/tools/ci/ci_get_mr_info.py deleted file mode 100644 index 686775adc78..00000000000 --- a/tools/ci/ci_get_mr_info.py +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env python -# -# internal use only for CI -# get latest MR information by source branch -# -# SPDX-FileCopyrightText: 2020-2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -# -import argparse -import logging -import os -import subprocess -import typing as t -from pathlib import Path - -from gitlab_api import Gitlab -from idf_ci_utils import IDF_PATH - -if t.TYPE_CHECKING: - from gitlab.v4.objects import ProjectCommit, ProjectMergeRequest - - -def _get_mr_obj(source_branch: str) -> t.Optional['ProjectMergeRequest']: - gl = Gitlab(os.getenv('CI_PROJECT_ID', 'espressif/esp-idf')) - if not gl.project: - return None - - mrs = gl.project.mergerequests.list(state='opened', source_branch=source_branch) - if mrs: - return mrs[0] # one source branch can only have one opened MR at one moment - else: - return None - - -def get_mr_iid(source_branch: str) -> str: - mr = _get_mr_obj(source_branch) - if not mr: - return '' - else: - return str(mr.iid) - - -def get_mr_changed_files(source_branch: str) -> t.List[str]: - mr = _get_mr_obj(source_branch) - if not mr: - return [] - - git_output = subprocess.check_output( - ['git', 'diff', '--name-only', '--diff-filter=d', f'origin/{mr.target_branch}...origin/{source_branch}'] - ).decode('utf8') - - return [line.strip() for line in git_output.splitlines() if line.strip()] - - -def get_mr_commits(source_branch: str) -> t.List['ProjectCommit']: - mr = _get_mr_obj(source_branch) - if not mr: - return [] - - return list(mr.commits()) - - -_COMPONENT_NAME_DIR_RECORDS = {} - - -def get_modified_component(filepath: str) -> t.Optional[str]: - """Return the component name if the file is in a component directory, otherwise None.""" - try: - f_path = Path(filepath).resolve().relative_to(IDF_PATH) - except ValueError: # not in IDF_PATH - return None - - # skip md files, etc. - if f_path.suffix in ['.md', '.yml']: - return None - - # skip test_apps files - if 'test_apps' in f_path.parts: - return None - - component_parent_dirs = [f_path.parts[0]] - for part in f_path.parts[1:]: - if component_parent_dirs[-1] == 'components' or component_parent_dirs[-1].endswith('common_components'): - if part not in _COMPONENT_NAME_DIR_RECORDS: - logging.debug('Found component "%s" in path "%s"' % (part, component_parent_dirs)) - _COMPONENT_NAME_DIR_RECORDS[part] = component_parent_dirs - elif _COMPONENT_NAME_DIR_RECORDS.get(part) != component_parent_dirs: - logging.debug( - 'WARNING!!! Found component "%s" in path "%s" and "%s"' - % (part, component_parent_dirs, _COMPONENT_NAME_DIR_RECORDS.get(part)) - ) - - return part - - component_parent_dirs.append(part) - - return None - - -def get_mr_components( - source_branch: t.Optional[str] = None, modified_files: t.Optional[t.List[str]] = None -) -> t.List[str]: - components: t.Set[str] = set() - if modified_files is None: - if not source_branch: - raise RuntimeError('--src-branch is required if --modified-files is not provided') - - modified_files = get_mr_changed_files(source_branch) - - for f in modified_files: - modified_component = get_modified_component(f) - if modified_component: - components.add(modified_component) - - return list(components) - - -def get_target_in_tags(tags: str) -> str: - from idf_pytest.constants import TARGET_MARKERS - - for x in tags.split(','): - if x in TARGET_MARKERS: - return x - - raise RuntimeError(f'No target marker found in {tags}') - - -def _print_list(_list: t.List[str], separator: str = '\n') -> None: - print(separator.join(_list)) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Get the latest merge request info by pipeline') - actions = parser.add_subparsers(dest='action', help='info type', required=True) - - common_args = argparse.ArgumentParser(add_help=False) - common_args.add_argument('--src-branch', help='source branch') - common_args.add_argument( - '--modified-files', - nargs='+', - help='space-separated list specifies the modified files. will be detected by --src-branch if not provided', - ) - - actions.add_parser('id', parents=[common_args]) - actions.add_parser('commits', parents=[common_args]) - actions.add_parser('components', parents=[common_args]) - target = actions.add_parser('target_in_tags') - target.add_argument('tags', help='comma separated tags, e.g., esp32,generic') - - args = parser.parse_args() - - if args.action == 'id': - if not args.src_branch: - raise RuntimeError('--src-branch is required') - print(get_mr_iid(args.src_branch)) - elif args.action == 'commits': - if not args.src_branch: - raise RuntimeError('--src-branch is required') - _print_list([commit.id for commit in get_mr_commits(args.src_branch)]) - elif args.action == 'components': - _print_list(get_mr_components(args.src_branch, args.modified_files)) - elif args.action == 'target_in_tags': - print(get_target_in_tags(args.tags)) - else: - raise NotImplementedError('not possible to get here') diff --git a/tools/ci/deploy_docs.py b/tools/ci/deploy_docs.py deleted file mode 100755 index 954958e738d..00000000000 --- a/tools/ci/deploy_docs.py +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/env python3 -# -# CI script to deploy docs to a webserver. Not useful outside of CI environment -# -# -# SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -# -import glob -import os -import os.path -import re -import stat -import subprocess -import sys -import tarfile -from typing import Any, List, Tuple - -import packaging.version - - -def env(variable: str, default: str=None) -> str: - """ Shortcut to return the expanded version of an environment variable """ - return os.path.expandvars(os.environ.get(variable, default) if default else os.environ[variable]) - - -# import sanitize_version from the docs directory, shared with here -sys.path.append(os.path.join(env('IDF_PATH'), 'docs')) -from sanitize_version import sanitize_version # noqa - - -def main() -> None: - # if you get KeyErrors on the following lines, it's probably because you're not running in Gitlab CI - git_ver = env('GIT_VER') # output of git describe --always - ci_ver = env('CI_COMMIT_REF_NAME', git_ver) # branch or tag we're building for (used for 'release' & URL) - - version = sanitize_version(ci_ver) - print('Git version: {}'.format(git_ver)) - print('CI Version: {}'.format(ci_ver)) - print('Deployment version: {}'.format(version)) - - if not version: - raise RuntimeError('A version is needed to deploy') - - build_dir = env('DOCS_BUILD_DIR') # top-level local build dir, where docs have already been built - - if not build_dir: - raise RuntimeError('Valid DOCS_BUILD_DIR is needed to deploy') - - url_base = env('DOCS_DEPLOY_URL_BASE') # base for HTTP URLs, used to print the URL to the log after deploying - - docs_server = env('DOCS_DEPLOY_SERVER') # ssh server to deploy to - docs_user = env('DOCS_DEPLOY_SERVER_USER') - docs_path = env('DOCS_DEPLOY_PATH') # filesystem path on DOCS_SERVER - - if not docs_server: - raise RuntimeError('Valid DOCS_DEPLOY_SERVER is needed to deploy') - - if not docs_user: - raise RuntimeError('Valid DOCS_DEPLOY_SERVER_USER is needed to deploy') - - docs_server = '{}@{}'.format(docs_user, docs_server) - - if not docs_path: - raise RuntimeError('Valid DOCS_DEPLOY_PATH is needed to deploy') - - print('DOCS_DEPLOY_SERVER {} DOCS_DEPLOY_PATH {}'.format(docs_server, docs_path)) - - tarball_path, version_urls = build_doc_tarball(version, git_ver, build_dir) - - deploy(version, tarball_path, docs_path, docs_server) - - print('Docs URLs:') - doc_deploy_type = os.getenv('TYPE') - for vurl in version_urls: - language, _, target = vurl.split('/') - tag = '{}_{}'.format(language, target) - url = '{}/{}/index.html'.format(url_base, vurl) # (index.html needed for the preview server) - url = re.sub(r'([^:])//', r'\1/', url) # get rid of any // that isn't in the https:// part - print('[document {}][{}] {}'.format(doc_deploy_type, tag, url)) - - # note: it would be neater to use symlinks for stable, but because of the directory order - # (language first) it's kind of a pain to do on a remote server, so we just repeat the - # process but call the version 'stable' this time - if is_stable_version(version): - print('Deploying again as stable version...') - tarball_path, version_urls = build_doc_tarball('stable', git_ver, build_dir) - deploy('stable', tarball_path, docs_path, docs_server) - - -def deploy(version: str, tarball_path: str, docs_path: str, docs_server: str) -> None: - def run_ssh(commands: List) -> None: - """ Log into docs_server and run a sequence of commands using ssh """ - print('Running ssh: {}'.format(commands)) - subprocess.run(['ssh', '-o', 'BatchMode=yes', docs_server, '-x', ' && '.join(commands)], check=True) - - # copy the version tarball to the server - run_ssh(['mkdir -p {}'.format(docs_path)]) - print('Running scp {} to {}'.format(tarball_path, '{}:{}'.format(docs_server, docs_path))) - subprocess.run(['scp', '-B', tarball_path, '{}:{}'.format(docs_server, docs_path)], check=True) - - tarball_name = os.path.basename(tarball_path) - - run_ssh(['cd {}'.format(docs_path), - 'rm -rf ./*/{}'.format(version), # remove any pre-existing docs matching this version - 'tar -zxvf {}'.format(tarball_name), # untar the archive with the new docs - 'rm {}'.format(tarball_name)]) - - # Note: deleting and then extracting the archive is a bit awkward for updating stable/latest/etc - # as the version will be invalid for a window of time. Better to do it atomically, but this is - # another thing made much more complex by the directory structure putting language before version... - - -def build_doc_tarball(version: str, git_ver: str, build_dir: str) -> Tuple[str, List]: - """ Make a tar.gz archive of the docs, in the directory structure used to deploy as - the given version """ - version_paths = [] - tarball_path = '{}/{}.tar.gz'.format(build_dir, version) - - # find all the 'html/' directories under build_dir - html_dirs = glob.glob('{}/**/html/'.format(build_dir), recursive=True) - print('Found %d html directories' % len(html_dirs)) - - pdfs = glob.glob('{}/**/latex/build/*.pdf'.format(build_dir), recursive=True) - print('Found %d PDFs in latex directories' % len(pdfs)) - - # add symlink for stable and latest and adds them to PDF blob - symlinks = create_and_add_symlinks(version, git_ver, pdfs) - - def not_sources_dir(ti: Any) -> Any: - print(type(ti)) - """ Filter the _sources directories out of the tarballs """ - if ti.name.endswith('/_sources'): - return None - - ti.mode |= stat.S_IWGRP # make everything group-writeable - return ti - - try: - os.remove(tarball_path) - except OSError: - pass - - with tarfile.open(tarball_path, 'w:gz') as tarball: - for html_dir in html_dirs: - # html_dir has the form '///html/' - target_dirname = os.path.dirname(os.path.dirname(html_dir)) - target = os.path.basename(target_dirname) - language = os.path.basename(os.path.dirname(target_dirname)) - - # when deploying, we want the top-level directory layout 'language/version/target' - archive_path = '{}/{}/{}'.format(language, version, target) - print("Archiving '{}' as '{}'...".format(html_dir, archive_path)) - tarball.add(html_dir, archive_path, filter=not_sources_dir) - version_paths.append(archive_path) - - for pdf_path in pdfs: - # pdf_path has the form '///latex/build' - latex_dirname = os.path.dirname(pdf_path) - pdf_filename = os.path.basename(pdf_path) - target_dirname = os.path.dirname(os.path.dirname(latex_dirname)) - target = os.path.basename(target_dirname) - language = os.path.basename(os.path.dirname(target_dirname)) - - # when deploying, we want the layout 'language/version/target/pdf' - archive_path = '{}/{}/{}/{}'.format(language, version, target, pdf_filename) - print("Archiving '{}' as '{}'...".format(pdf_path, archive_path)) - tarball.add(pdf_path, archive_path) - - for symlink in symlinks: - os.unlink(symlink) - - return (os.path.abspath(tarball_path), version_paths) - - -def create_and_add_symlinks(version: str, git_ver: str, pdfs: List) -> List: - """ Create symbolic links for PDFs for 'latest' and 'stable' releases """ - - symlinks = [] - if 'stable' in version or 'latest' in version: - for pdf_path in pdfs: - symlink_path = pdf_path.replace(git_ver, version) - os.symlink(pdf_path, symlink_path) - symlinks.append(symlink_path) - - pdfs.extend(symlinks) - print('Found %d PDFs in latex directories after adding symlink' % len(pdfs)) - - return symlinks - - -def is_stable_version(version: str) -> bool: - """ Heuristic for whether this is the latest stable release """ - if not version.startswith('v'): - return False # branch name - if '-' in version: - return False # prerelease tag - - git_out = subprocess.check_output(['git', 'tag', '-l']).decode('utf-8') - - versions = [v.strip() for v in git_out.split('\n')] - versions = [v for v in versions if re.match(r'^v[\d\.]+$', v.strip())] # include vX.Y.Z only - - versions_pack = [packaging.version.parse(v) for v in versions] - - max_version = max(versions_pack) - - if max_version.public != version[1:]: - print('Stable version is v{}. This version is {}.'.format(max_version.public, version)) - return False - else: - print('This version {} is the stable version'.format(version)) - return True - - -if __name__ == '__main__': - main() diff --git a/tools/ci/dynamic_pipelines/constants.py b/tools/ci/dynamic_pipelines/constants.py index 40dbe346d11..2f64e515e73 100644 --- a/tools/ci/dynamic_pipelines/constants.py +++ b/tools/ci/dynamic_pipelines/constants.py @@ -1,26 +1,9 @@ -# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2024-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import os from idf_ci_utils import IDF_PATH -COMMENT_START_MARKER = '### Dynamic Pipeline Report' - -REPORT_TEMPLATE_FILEPATH = os.path.join( - IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'report.template.html' -) -CSS_STYLES_FILEPATH = os.path.join(IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'styles.css') -JS_SCRIPTS_FILEPATH = os.path.join(IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'scripts.js') -TOP_N_APPS_BY_SIZE_DIFF = 10 -SIZE_DIFFERENCE_BYTES_THRESHOLD = 500 -BINARY_SIZE_METRIC_NAME = 'binary_size' - KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH = os.path.join( IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'known_generate_test_child_pipeline_warnings.yml' ) - -CI_JOB_TOKEN = os.getenv('CI_JOB_TOKEN', '') -CI_DASHBOARD_API = os.getenv('CI_DASHBOARD_API', '') -CI_PAGES_URL = os.getenv('CI_PAGES_URL', '') -CI_PROJECT_URL = os.getenv('CI_PROJECT_URL', '') -CI_MERGE_REQUEST_SOURCE_BRANCH_SHA = os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA', '') diff --git a/tools/ci/dynamic_pipelines/models.py b/tools/ci/dynamic_pipelines/models.py deleted file mode 100644 index 2cd5022a112..00000000000 --- a/tools/ci/dynamic_pipelines/models.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import os -import typing as t -import urllib.parse -from dataclasses import dataclass -from xml.etree.ElementTree import Element - -from idf_ci_utils import IDF_PATH - - -@dataclass -class TestCase: - name: str - file: str - time: float - app_path: t.Optional[str] = None - failure: t.Optional[str] = None - skipped: t.Optional[str] = None - ci_job_url: t.Optional[str] = None - ci_dashboard_url: t.Optional[str] = None - dut_log_url: t.Optional[str] = None - latest_total_count: int = 0 - latest_failed_count: int = 0 - - @property - def is_failure(self) -> bool: - return self.failure is not None - - @property - def is_skipped(self) -> bool: - return self.skipped is not None - - @property - def is_success(self) -> bool: - return not self.is_failure and not self.is_skipped - - @classmethod - def _get_idf_rel_path(cls, path: str) -> str: - if path.startswith(IDF_PATH): - return os.path.relpath(path, IDF_PATH) - else: - return path - - @classmethod - def from_test_case_node(cls, node: Element) -> t.Optional['TestCase']: - if 'name' not in node.attrib: - print('WARNING: Node Invalid: ', node) - return None - - # url to test cases dashboard - grafana_base_url = urllib.parse.urljoin(os.getenv('CI_DASHBOARD_HOST', ''), '/d/Ucg477Fnz/case-list') - encoded_params = urllib.parse.urlencode({'var-case_id': node.attrib['name']}, quote_via=urllib.parse.quote) - - kwargs = { - 'name': node.attrib['name'], - 'file': node.attrib.get('file'), - 'app_path': '|'.join( - cls._get_idf_rel_path(path) for path in node.attrib.get('app_path', 'unknown').split('|') - ), - 'time': float(node.attrib.get('time') or 0), - 'ci_job_url': node.attrib.get('ci_job_url') or 'Not found', - 'ci_dashboard_url': f'{grafana_base_url}?{encoded_params}', - 'dut_log_url': node.attrib.get('dut_log_url') or 'Not found', - } - - failure_node = node.find('failure') - # bool(failure_node) is False, so compare with None - if failure_node is None: - failure_node = node.find('error') - if failure_node is not None: - message = failure_node.attrib.get('message', '') - kwargs['failure'] = message - - skipped_node = node.find('skipped') - if skipped_node is not None: - kwargs['skipped'] = skipped_node.attrib['message'] - - return cls(**kwargs) # type: ignore - - -@dataclass -class GitlabJob: - id: int - name: str - stage: str - status: str - url: str - ci_dashboard_url: str - failure_reason: t.Optional[str] = None - failure_log: t.Optional[str] = None - latest_total_count: int = 0 - latest_failed_count: int = 0 - - @property - def is_failed(self) -> bool: - return self.status == 'failed' - - @property - def is_success(self) -> bool: - return self.status == 'success' - - @classmethod - def from_json_data(cls, job_data: dict, failure_data: dict) -> t.Optional['GitlabJob']: - grafana_base_url = urllib.parse.urljoin(os.getenv('CI_DASHBOARD_HOST', ''), '/d/LoUa-qLWz/job-list') - encoded_params = urllib.parse.urlencode({'var-job_name': job_data['name']}, quote_via=urllib.parse.quote) - - kwargs = { - 'id': job_data['id'], - 'name': job_data['name'], - 'stage': job_data['stage'], - 'status': job_data['status'], - 'url': job_data['url'], - 'ci_dashboard_url': f'{grafana_base_url}?{encoded_params}', - 'failure_reason': job_data['failure_reason'], - 'failure_log': job_data['failure_log'], - 'latest_total_count': failure_data.get('total_count', 0), - 'latest_failed_count': failure_data.get('failed_count', 0), - } - - return cls(**kwargs) # type: ignore diff --git a/tools/ci/dynamic_pipelines/report.py b/tools/ci/dynamic_pipelines/report.py deleted file mode 100644 index d9918bc89f2..00000000000 --- a/tools/ci/dynamic_pipelines/report.py +++ /dev/null @@ -1,739 +0,0 @@ -# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import abc -import html -import re -import typing as t -from textwrap import dedent - -from gitlab import GitlabUpdateError -from gitlab_api import Gitlab -from idf_build_apps.constants import BuildStatus -from idf_ci_local.app import AppWithMetricsInfo -from idf_ci_utils import idf_relpath -from prettytable import PrettyTable - -from .constants import BINARY_SIZE_METRIC_NAME -from .constants import CI_DASHBOARD_API -from .constants import COMMENT_START_MARKER -from .constants import CSS_STYLES_FILEPATH -from .constants import JS_SCRIPTS_FILEPATH -from .constants import REPORT_TEMPLATE_FILEPATH -from .constants import SIZE_DIFFERENCE_BYTES_THRESHOLD -from .constants import TOP_N_APPS_BY_SIZE_DIFF -from .models import GitlabJob -from .models import TestCase -from .utils import format_permalink -from .utils import get_artifacts_url -from .utils import is_url - - -class ReportGenerator: - REGEX_PATTERN = r'#### {}\n[\s\S]*?(?=\n#### |$)' - - def __init__( - self, - project_id: int, - mr_iid: int, - pipeline_id: int, - job_id: int, - commit_id: str, - local_commit_id: str, - *, - title: str, - ): - gl_project = Gitlab(project_id).project - if mr_iid is not None: - self.mr = gl_project.mergerequests.get(mr_iid) - else: - self.mr = None - self.pipeline_id = pipeline_id - self.job_id = job_id - self.commit_id = commit_id - self.local_commit_id = local_commit_id - - self.title = title - self.output_filepath = self.title.lower().replace(' ', '_') + '.html' - self.additional_info = '' - - @property - def get_commit_summary(self) -> str: - return f'with CI commit SHA: {self.commit_id[:8]}, local commit SHA: {self.local_commit_id[:8]}' - - @staticmethod - def get_download_link_for_url(url: str) -> str: - if url: - return f'Download' - - return '' - - @staticmethod - def write_report_to_file(report_str: str, job_id: int, output_filepath: str) -> t.Optional[str]: - """ - Writes the report to a file and constructs a modified URL based on environment settings. - - :param report_str: The report content to be written to the file. - :param job_id: The job identifier used to construct the URL. - :param output_filepath: The path to the output file. - :return: The modified URL pointing to the job's artifacts. - """ - if not report_str: - return None - with open(output_filepath, 'w') as file: - file.write(report_str) - - # for example, {URL}/-/esp-idf/-/jobs/{id}/artifacts/app_info_84.txt - # CI_PAGES_URL is {URL}/esp-idf, which missed one `-` - report_url: str = get_artifacts_url(job_id, output_filepath) - return report_url - - @staticmethod - def _load_file_content(filepath: str) -> str: - """ - Load the content of a file as string - - :param filepath: Path to the file to load - :return: Content of the file as string - """ - try: - with open(filepath, encoding='utf-8') as f: - return f.read() - except (OSError, FileNotFoundError) as e: - print(f'Warning: Could not read file {filepath}: {e}') - return '' - - def generate_html_report(self, table_str: str) -> str: - # we're using bootstrap table - table_str = table_str.replace( - '', - '
', - ) - - template = self._load_file_content(REPORT_TEMPLATE_FILEPATH) - css_content = self._load_file_content(CSS_STYLES_FILEPATH) - js_content = self._load_file_content(JS_SCRIPTS_FILEPATH) - - template = template.replace('{{css_content}}', css_content) - template = template.replace('{{js_content}}', js_content) - template = template.replace('{{pipeline_id}}', str(self.pipeline_id)) - template = template.replace('{{apiBaseUrl}}', CI_DASHBOARD_API) - - return template.replace('{{title}}', self.title).replace('{{table}}', table_str) - - @staticmethod - def table_to_html_str(table: PrettyTable) -> str: - return html.unescape(table.get_html_string()) # type: ignore - - def create_table_section( - self, - title: str, - items: list, - headers: list, - row_attrs: list, - value_functions: t.Optional[list] = None, - ) -> t.List: - """ - Appends a formatted section to a report based on the provided items. This section includes - a header and a table constructed from the items list with specified headers and attributes. - - :param title: Title for the report section. This title is used as a header above the table. - :param items: List of item objects to include in the table. Each item should have attributes - that correspond to the row_attrs and value_functions specified. - :param headers: List of strings that will serve as the column headers in the generated table. - :param row_attrs: List of attributes to include from each item for the table rows. These - should be attributes or keys that exist on the items in the 'items' list. - :param value_functions: Optional list of tuples containing additional header and corresponding - value function. Each tuple should specify a header (as a string) and - a function that takes an item and returns a string. This is used for - generating dynamic columns based on item data. - - :return: List with appended HTML sections. - """ - if not items: - return [] - - report_sections = [ - f"""

{title}

""", - self._create_table_for_items( - items=items, headers=headers, row_attrs=row_attrs, value_functions=value_functions or [] - ), - ] - return report_sections - - @staticmethod - def generate_additional_info_section( - title: str, count: int, report_url: t.Optional[str] = None, add_permalink: bool = True - ) -> str: - """ - Generate a section for the additional info string. - - :param title: The title of the section. - :param count: The count of test cases. - :param report_url: The URL of the report. If count = 0, only the count will be included. - :param add_permalink: Whether to include a permalink in the report URL. Defaults to True. - :return: The formatted additional info section string. - """ - if count != 0 and report_url: - if add_permalink: - return f'- **{title}:** [{count}]({report_url}/#{format_permalink(title)})\n' - else: - return f'- **{title}:** [{count}]({report_url})\n' - else: - return f'- **{title}:** {count}\n' - - def _create_table_for_items( - self, - items: t.Union[t.List[TestCase], t.List[GitlabJob]], - headers: t.List[str], - row_attrs: t.List[str], - value_functions: t.Optional[t.List[t.Tuple[str, t.Callable[[t.Union[TestCase, GitlabJob]], str]]]] = None, - ) -> str: - """ - Create a PrettyTable and convert it to an HTML string for the provided test cases. - :param items: List of item objects to include in the table. - :param headers: List of strings for the table headers. - :param row_attrs: List of attributes to include in each row. - :param value_functions: List of tuples containing additional header and corresponding value function. - :return: HTML table string. - """ - table = PrettyTable() - table.field_names = headers - - # Create a mapping of header names to their corresponding index in the headers list - header_index_map = {header: i for i, header in enumerate(headers)} - - for item in items: - row = [] - for attr in row_attrs: - value = str(getattr(item, attr, '')) - if is_url(value): - link = f'link' - row.append(link) - else: - row.append(value) - - # Insert values computed by value functions at the correct column position based on their headers - if value_functions: - for header, func in value_functions: - index = header_index_map.get(header) - if index is not None: - computed_value = func(item) - row.insert(index, computed_value) - - table.add_row(row) - - return self.table_to_html_str(table) - - @staticmethod - def _filter_items( - items: t.Union[t.List[TestCase], t.List[GitlabJob]], condition: t.Callable[[t.Union[TestCase, GitlabJob]], bool] - ) -> t.List[TestCase]: - """ - Filter items s based on a given condition. - - :param items: List of items to filter by given condition. - :param condition: A function that evaluates to True or False for each items. - :return: List of filtered instances. - """ - return [item for item in items if condition(item)] - - @staticmethod - def _sort_items( - items: t.List[t.Union[TestCase, GitlabJob, AppWithMetricsInfo]], - key: t.Union[str, t.Callable[[t.Union[TestCase, GitlabJob, AppWithMetricsInfo]], t.Any]], - order: str = 'asc', - sort_function: t.Optional[t.Callable[[t.Any], t.Any]] = None, - ) -> t.List[t.Union[TestCase, GitlabJob, AppWithMetricsInfo]]: - """ - Sort items based on a given key, order, and optional custom sorting function. - - :param items: List of items to sort. - :param key: A string representing the attribute name or a function to extract the sorting key. - :param order: Order of sorting ('asc' for ascending, 'desc' for descending). - :param sort_function: A custom function to control sorting logic - (e.g., prioritizing positive/negative/zero values). - :return: List of sorted instances. - """ - key_func = None - if isinstance(key, str): - - def key_func(item: t.Any) -> t.Any: - return getattr(item, key) - - sorting_key = sort_function if sort_function is not None else key_func - try: - items = sorted(items, key=sorting_key, reverse=(order == 'desc')) - except TypeError: - print(f'Comparison for the key {key} is not supported') - - return items - - @abc.abstractmethod - def _get_report_str(self) -> str: - raise NotImplementedError - - def _generate_comment(self) -> str: - # Report in HTML format to avoid exceeding length limits - comment = f'#### {self.title}\n' - report_str = self._get_report_str() - comment += f'{self.additional_info}\n' - self.write_report_to_file(report_str, self.job_id, self.output_filepath) - - return comment - - def _update_mr_comment(self, comment: str) -> None: - new_comment = f'{COMMENT_START_MARKER}\n\n{comment}' - - for note in self.mr.notes.list(iterator=True): - if note.body.startswith(COMMENT_START_MARKER): - updated_str = self._get_updated_comment(note.body, comment) - - note.body = updated_str - try: - note.save() - except GitlabUpdateError: - print('Failed to update MR comment, Creating a new comment') - self.mr.notes.create({'body': new_comment}) - break - else: - self.mr.notes.create({'body': new_comment}) - - def _get_updated_comment(self, existing_comment: str, new_comment: str) -> str: - updated_str = re.sub(self.REGEX_PATTERN.format(self.title), new_comment, existing_comment) - if updated_str == existing_comment: - updated_str = f'{existing_comment.strip()}\n\n{new_comment}' - return updated_str - - def post_report(self) -> None: - comment = self._generate_comment() - - print(comment) - - if self.mr is None: - print('No MR found, skip posting comment') - return - - self._update_mr_comment(comment) - - -class BuildReportGenerator(ReportGenerator): - def __init__( - self, - project_id: int, - mr_iid: int, - pipeline_id: int, - job_id: int, - commit_id: str, - local_commit_id: str, - *, - title: str = 'Build Report', - apps: list[AppWithMetricsInfo], - ) -> None: - super().__init__(project_id, mr_iid, pipeline_id, job_id, commit_id, local_commit_id, title=title) - self.apps = apps - self.report_titles_map = { - 'failed_apps': 'Failed Apps', - 'built_test_related_apps': 'Built Apps - Test Related', - 'built_non_test_related_apps': 'Built Apps - Non Test Related', - 'new_test_related_apps': 'New Apps - Test Related', - 'new_non_test_related_apps': 'New Apps - Non Test Related', - 'skipped_apps': 'Skipped Apps', - } - self.failed_apps_report_file = 'failed_apps.html' - self.built_apps_report_file = 'built_apps.html' - self.skipped_apps_report_file = 'skipped_apps.html' - - @staticmethod - def custom_sort(item: AppWithMetricsInfo) -> tuple[int, t.Any]: - """ - Custom sort function to: - 1. Push items with zero binary sizes to the end. - 2. Sort other items by absolute size_difference_percentage. - """ - # Priority: 0 for zero binaries, 1 for non-zero binaries - zero_binary_priority = ( - 1 - if item.metrics[BINARY_SIZE_METRIC_NAME].source_value != 0 - or item.metrics[BINARY_SIZE_METRIC_NAME].target_value != 0 - else 0 - ) - # Secondary sort: Negative absolute size_difference_percentage for descending order - size_difference_sort = abs(item.metrics[BINARY_SIZE_METRIC_NAME].difference_percentage) - return zero_binary_priority, size_difference_sort - - def _generate_top_n_apps_by_size_table(self) -> str: - """ - Generate a markdown table for the top N apps by size difference. - Only includes apps with size differences greater than 500 bytes. - """ - filtered_apps = [ - app - for app in self.apps - if abs(app.metrics[BINARY_SIZE_METRIC_NAME].difference) > SIZE_DIFFERENCE_BYTES_THRESHOLD - ] - - top_apps = sorted( - filtered_apps, key=lambda app: abs(app.metrics[BINARY_SIZE_METRIC_NAME].difference_percentage), reverse=True - )[:TOP_N_APPS_BY_SIZE_DIFF] - - if not top_apps: - return '' - - table = ( - f'\n⚠️⚠️⚠️ Top {len(top_apps)} Apps with Binary Size Sorted by Size Difference\n' - f'Note: Apps with changes of less than {SIZE_DIFFERENCE_BYTES_THRESHOLD} bytes are not shown.\n' - ) - table += '| App Dir | Build Dir | Size Diff (bytes) | Size Diff (%) |\n' - table += '|---------|-----------|-------------------|---------------|\n' - for app in top_apps: - table += dedent( - f'| {app.app_dir} | {app.build_dir} | ' - f'{app.metrics[BINARY_SIZE_METRIC_NAME].difference} | ' - f'{app.metrics[BINARY_SIZE_METRIC_NAME].difference_percentage}% |\n' - ) - table += ( - '\n**For more details, please click on the numbers in the summary above ' - 'to view the corresponding report files.** ⬆️⬆️⬆️\n\n' - ) - - return table - - @staticmethod - def split_new_and_existing_apps( - apps: t.Iterable[AppWithMetricsInfo], - ) -> tuple[list[AppWithMetricsInfo], list[AppWithMetricsInfo]]: - """ - Splits apps into new apps and existing apps. - - :param apps: Iterable of apps to process. - :return: A tuple (new_apps, existing_apps). - """ - new_apps = [app for app in apps if app.is_new_app] - existing_apps = [app for app in apps if not app.is_new_app] - return new_apps, existing_apps - - def filter_apps_by_criteria(self, build_status: str, preserve: bool) -> list[AppWithMetricsInfo]: - """ - Filters apps based on build status and preserve criteria. - - :param build_status: Build status to filter by. - :param preserve: Whether to filter preserved apps. - :return: Filtered list of apps. - """ - return [app for app in self.apps if app.build_status == build_status and app.preserve == preserve] - - def get_built_apps_report_parts(self) -> list[str]: - """ - Generates report parts for new and existing apps. - - :return: List of report parts. - """ - new_test_related_apps, built_test_related_apps = self.split_new_and_existing_apps( - self.filter_apps_by_criteria(BuildStatus.SUCCESS, True) - ) - - new_non_test_related_apps, built_non_test_related_apps = self.split_new_and_existing_apps( - self.filter_apps_by_criteria(BuildStatus.SUCCESS, False) - ) - - sections = [] - - if new_test_related_apps: - new_test_related_apps_table_section = self.create_table_section( - title=self.report_titles_map['new_test_related_apps'], - items=new_test_related_apps, - headers=[ - 'App Dir', - 'Build Dir', - 'Download Command', - 'Your Branch App Size', - ], - row_attrs=[ - 'app_dir', - 'build_dir', - ], - value_functions=[ - ('Your Branch App Size', lambda _app: str(_app.metrics[BINARY_SIZE_METRIC_NAME].source_value)), - ( - 'Download Command', - lambda _app: f'idf-ci gitlab download-artifacts --pipeline-id {self.pipeline_id} ' - f'{idf_relpath(_app.build_path)}', - ), - ], - ) - sections.extend(new_test_related_apps_table_section) - - if built_test_related_apps: - built_test_related_apps = self._sort_items( - built_test_related_apps, - key='metrics.binary_size.difference_percentage', - order='desc', - sort_function=self.custom_sort, - ) - - built_test_related_apps_table_section = self.create_table_section( - title=self.report_titles_map['built_test_related_apps'], - items=built_test_related_apps, - headers=[ - 'App Dir', - 'Build Dir', - 'Download Command', - 'Your Branch App Size', - 'Target Branch App Size', - 'Size Diff', - 'Size Diff, %', - ], - row_attrs=[ - 'app_dir', - 'build_dir', - ], - value_functions=[ - ('Your Branch App Size', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].source_value)), - ('Target Branch App Size', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].target_value)), - ('Size Diff', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].difference)), - ('Size Diff, %', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].difference_percentage)), - ( - 'Download Command', - lambda _app: f'idf-ci gitlab download-artifacts --pipeline-id {self.pipeline_id} ' - f'{idf_relpath(_app.build_path)}', - ), - ], - ) - sections.extend(built_test_related_apps_table_section) - - if new_non_test_related_apps: - new_non_test_related_apps_table_section = self.create_table_section( - title=self.report_titles_map['new_non_test_related_apps'], - items=new_non_test_related_apps, - headers=[ - 'App Dir', - 'Build Dir', - 'Download Command', - 'Your Branch App Size', - ], - row_attrs=[ - 'app_dir', - 'build_dir', - ], - value_functions=[ - ( - 'Download Command', - lambda _app: f'idf-ci gitlab download-artifacts --pipeline-id {self.pipeline_id} ' - f'{idf_relpath(_app.build_path)}', - ), - ('Your Branch App Size', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].source_value)), - ], - ) - sections.extend(new_non_test_related_apps_table_section) - - if built_non_test_related_apps: - built_non_test_related_apps = self._sort_items( - built_non_test_related_apps, - key='metrics.binary_size.difference_percentage', - order='desc', - sort_function=self.custom_sort, - ) - built_non_test_related_apps_table_section = self.create_table_section( - title=self.report_titles_map['built_non_test_related_apps'], - items=built_non_test_related_apps, - headers=[ - 'App Dir', - 'Build Dir', - 'Download Command', - 'Your Branch App Size', - 'Target Branch App Size', - 'Size Diff', - 'Size Diff, %', - ], - row_attrs=[ - 'app_dir', - 'build_dir', - ], - value_functions=[ - ('Your Branch App Size', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].source_value)), - ('Target Branch App Size', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].target_value)), - ('Size Diff', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].difference)), - ('Size Diff, %', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].difference_percentage)), - ( - 'Download Command', - lambda _app: f'idf-ci gitlab download-artifacts --pipeline-id {self.pipeline_id} ' - f'{idf_relpath(_app.build_path)}', - ), - ], - ) - sections.extend(built_non_test_related_apps_table_section) - - built_apps_report_url = self.write_report_to_file( - self.generate_html_report(''.join(sections)), - self.job_id, - self.built_apps_report_file, - ) - - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['built_test_related_apps'], - len(built_test_related_apps), - built_apps_report_url, - ) - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['built_non_test_related_apps'], - len(built_non_test_related_apps), - built_apps_report_url, - ) - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['new_test_related_apps'], - len(new_test_related_apps), - built_apps_report_url, - ) - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['new_non_test_related_apps'], - len(new_non_test_related_apps), - built_apps_report_url, - ) - - self.additional_info += self._generate_top_n_apps_by_size_table() - - return sections - - def get_failed_apps_report_parts(self) -> list[str]: - failed_apps = [app for app in self.apps if app.build_status == BuildStatus.FAILED] - if not failed_apps: - return [] - - failed_apps_table_section = self.create_table_section( - title=self.report_titles_map['failed_apps'], - items=failed_apps, - headers=['App Dir', 'Build Dir', 'Failed Reason', 'Download Command'], - row_attrs=['app_dir', 'build_dir', 'build_comment'], - value_functions=[ - ( - 'Download Command', - lambda _app: f'idf-ci gitlab download-artifacts --pipeline-id {self.pipeline_id} ' - f'{idf_relpath(_app.build_path)}', - ), - ], - ) - failed_apps_report_url = self.write_report_to_file( - self.generate_html_report(''.join(failed_apps_table_section)), - self.job_id, - self.failed_apps_report_file, - ) - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['failed_apps'], len(failed_apps), failed_apps_report_url - ) - return failed_apps_table_section - - def get_skipped_apps_report_parts(self) -> list[str]: - skipped_apps = [app for app in self.apps if app.build_status == BuildStatus.SKIPPED] - if not skipped_apps: - return [] - - skipped_apps_table_section = self.create_table_section( - title=self.report_titles_map['skipped_apps'], - items=skipped_apps, - headers=['App Dir', 'Build Dir', 'Skipped Reason'], - row_attrs=['app_dir', 'build_dir', 'build_comment'], - ) - skipped_apps_report_url = self.write_report_to_file( - self.generate_html_report(''.join(skipped_apps_table_section)), - self.job_id, - self.skipped_apps_report_file, - ) - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['skipped_apps'], len(skipped_apps), skipped_apps_report_url - ) - return skipped_apps_table_section - - def _get_report_str(self) -> str: - self.additional_info = ( - f'**Build Summary ({self.get_commit_summary}):**\n' - '\n' - '> ℹ️ Note: Binary artifacts stored in MinIO are retained for 4 DAYS from their build date\n' - ) - failed_apps_report_parts = self.get_failed_apps_report_parts() - skipped_apps_report_parts = self.get_skipped_apps_report_parts() - built_apps_report_parts = self.get_built_apps_report_parts() - - return self.generate_html_report( - ''.join(failed_apps_report_parts + built_apps_report_parts + skipped_apps_report_parts) - ) - - -class JobReportGenerator(ReportGenerator): - def __init__( - self, - project_id: int, - mr_iid: int, - pipeline_id: int, - job_id: int, - commit_id: str, - local_commit_id: str, - *, - title: str = 'Job Report', - jobs: list[GitlabJob], - ): - super().__init__(project_id, mr_iid, pipeline_id, job_id, commit_id, local_commit_id, title=title) - self.jobs = jobs - self.report_titles_map = { - 'failed_jobs': 'Failed Jobs (Excludes "integration_test" and "target_test" jobs)', - 'succeeded': 'Succeeded Jobs', - } - self.failed_jobs_report_file = 'job_report.html' - - def _get_report_str(self) -> str: - """ - Generate a complete HTML report string by processing jobs. - :return: Complete HTML report string. - """ - report_str: str = '' - - if not self.jobs: - print('No jobs found, skip generating job report') - return 'No Job Found' - - relevant_failed_jobs = self._sort_items( - self._filter_items( - self.jobs, lambda job: job.is_failed and job.stage not in ['integration_test', 'target_test'] - ), - key='latest_failed_count', - ) - succeeded_jobs = self._filter_items(self.jobs, lambda job: job.is_success) - - self.additional_info = f'**Job Summary ({self.get_commit_summary}):**\n' - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['succeeded'], len(succeeded_jobs) - ) - - if not relevant_failed_jobs: - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['failed_jobs'], len(relevant_failed_jobs) - ) - return report_str - - report_sections = self.create_table_section( - title='Failed Jobs (Excludes "integration_test" and "target_test" jobs)', - items=relevant_failed_jobs, - headers=[ - 'Job Name', - 'Failure Reason', - 'Failure Log', - 'Failures across all other branches (10 latest jobs)', - 'URL', - 'CI Dashboard URL', - ], - row_attrs=['name', 'failure_reason', 'failure_log', 'url', 'ci_dashboard_url'], - value_functions=[ - ( - 'Failures across all other branches (10 latest jobs)', - lambda item: f'{getattr(item, "latest_failed_count", "")} ' - f'/ {getattr(item, "latest_total_count", "")}', - ) - ], - ) - relevant_failed_jobs_report_url = get_artifacts_url(self.job_id, self.failed_jobs_report_file) - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['failed_jobs'], len(relevant_failed_jobs), relevant_failed_jobs_report_url - ) - - report_str = self.generate_html_report(''.join(report_sections)) - return report_str diff --git a/tools/ci/dynamic_pipelines/scripts/generate_report.py b/tools/ci/dynamic_pipelines/scripts/generate_report.py deleted file mode 100644 index 1f40fff50ad..00000000000 --- a/tools/ci/dynamic_pipelines/scripts/generate_report.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import argparse -import glob -import os -import subprocess -import typing as t - -import __init__ # noqa: F401 # inject the system path -from idf_build_apps import json_list_files_to_apps -from idf_ci import GitlabEnvVars -from idf_ci_local.app import enrich_apps_with_metrics_info - -from dynamic_pipelines.report import BuildReportGenerator -from dynamic_pipelines.report import JobReportGenerator -from dynamic_pipelines.utils import fetch_app_metrics -from dynamic_pipelines.utils import fetch_failed_jobs - - -def main() -> None: - parser: argparse.ArgumentParser = setup_argument_parser() - args: argparse.Namespace = parser.parse_args() - - report_actions: dict[str, t.Callable[[argparse.Namespace], None]] = { - 'build': generate_build_report, - 'job': generate_jobs_report, - } - - report_action = report_actions.get(args.report_type) - if report_action is None: - raise ValueError('Unknown report type is requested to be generated.') - - report_action(args) - - -def setup_argument_parser() -> argparse.ArgumentParser: - report_type_parser: argparse.ArgumentParser = argparse.ArgumentParser(add_help=False) - report_type_parser.add_argument( - '--report-type', choices=['build', 'job'], required=True, help='Type of report to generate' - ) - report_type_args: argparse.Namespace - remaining_args: list[str] - report_type_args, remaining_args = report_type_parser.parse_known_args() - - parser: argparse.ArgumentParser = argparse.ArgumentParser( - description='Update reports in MR pipelines based on the selected report type', - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - parents=[report_type_parser], - ) - - common_arguments(parser) - conditional_arguments(report_type_args, parser) - - return parser - - -def common_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument('--project-id', type=int, default=os.getenv('CI_PROJECT_ID'), help='Project ID') - parser.add_argument('--mr-iid', type=int, default=os.getenv('CI_MERGE_REQUEST_IID'), help='Merge Request IID') - parser.add_argument('--pipeline-id', type=int, default=os.getenv('PARENT_PIPELINE_ID'), help='Pipeline ID') - parser.add_argument('--job-id', type=int, default=os.getenv('CI_JOB_ID'), help='Job ID') - parser.add_argument('--commit-id', default=os.getenv('CI_COMMIT_SHA', ''), help='MR merged result commit ID') - parser.add_argument('--local-commit-id', default=os.getenv('PIPELINE_COMMIT_SHA', ''), help='local dev commit ID') - - -def conditional_arguments(report_type_args: argparse.Namespace, parser: argparse.ArgumentParser) -> None: - if report_type_args.report_type == 'build': - parser.add_argument('--app-list-filepattern', default='app_info*.txt', help='Pattern to match app list files') - elif report_type_args.report_type == 'target_test': - parser.add_argument( - '--junit-report-filepattern', default='XUNIT_RESULT*.xml', help='Pattern to match JUnit report files' - ) - - -def generate_build_report(args: argparse.Namespace) -> None: - # generate presigned url for the artifacts - subprocess.check_output( - [ - 'idf-ci', - 'gitlab', - 'generate-presigned-json', - '--commit-sha', - args.local_commit_id, - '--output', - 'presigned.json', - ], - ) - print('generated presigned.json') - - # generate report - apps = json_list_files_to_apps(glob.glob(args.app_list_filepattern)) - print(f'loaded {len(apps)} apps') - app_metrics = fetch_app_metrics( - source_commit_sha=args.commit_id, - target_commit_sha=os.environ.get('CI_MERGE_REQUEST_TARGET_BRANCH_SHA'), - ) - apps = enrich_apps_with_metrics_info(app_metrics, apps) - report_generator = BuildReportGenerator( - args.project_id, args.mr_iid, args.pipeline_id, args.job_id, args.commit_id, args.local_commit_id, apps=apps - ) - report_generator.post_report() - - -def generate_jobs_report(args: argparse.Namespace) -> None: - jobs: list[t.Any] = fetch_failed_jobs(args.commit_id) - - if not jobs: - return - - report_generator = JobReportGenerator( - args.project_id, args.mr_iid, args.pipeline_id, args.job_id, args.commit_id, args.local_commit_id, jobs=jobs - ) - report_generator.post_report() - - if GitlabEnvVars().IDF_CI_IS_DEBUG_PIPELINE: - print('Debug pipeline detected, exit non-zero to fail the pipeline in order to block merge') - exit(30) - - -if __name__ == '__main__': - main() diff --git a/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py b/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py index 6abe2a83689..2cb0b484093 100644 --- a/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py +++ b/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py @@ -55,7 +55,7 @@ def main(output_filepath: str) -> None: exclude_runner_tags_matching = [] for _tag in known_warnings_dict.get('no_runner_tags', []): if '*' not in _tag: - exclude_runner_tags_set.add(_tag) + exclude_runner_tags_set.add(frozenset(_tag.split(','))) else: if res := _process_match_group(_tag): exclude_runner_tags_matching.append(res) @@ -66,7 +66,7 @@ def main(output_filepath: str) -> None: if exclude_runner_tags := os.getenv('EXCLUDE_RUNNER_TAGS'): for _tag in exclude_runner_tags.split(';'): if '*' not in _tag: - exclude_runner_tags_set.add(_tag) + exclude_runner_tags_set.add(frozenset(_tag.split(','))) else: if res := _process_match_group(_tag): exclude_runner_tags_matching.append(res) @@ -75,7 +75,7 @@ def main(output_filepath: str) -> None: additional_dict: dict[GroupKey, dict[str, t.Any]] = {} for key, grouped_cases in cases.grouped_cases.items(): # skip test cases with no runner tags - if ','.join(sorted(key.runner_tags)) in exclude_runner_tags_set: + if frozenset(key.runner_tags) in exclude_runner_tags_set: print(f'WARNING: excluding test cases with runner tags: {key.runner_tags}') continue diff --git a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml index 0e50de03a34..a145097f312 100644 --- a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml +++ b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml @@ -17,6 +17,9 @@ # Since the version 3.21 CMake passes source files and include dirs to ninja using absolute paths. # Needed for pytest junit reports. INSTALL_EXTRA_TOOLS: cmake + # Suppress Python warning messages during CI builds to keep logs clean and focused on errors. + # This prevents non-critical warnings (e.g., deprecation or resource warnings) from cluttering output. + PYTHONWARNINGS: "ignore" needs: - pipeline: $PARENT_PIPELINE_ID job: generate_build_child_pipeline @@ -40,7 +43,6 @@ - run_cmd idf-ci build run --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1} - --modified-files ${MR_MODIFIED_FILES} .dynamic_target_test_template: image: $TARGET_TEST_ENV_IMAGE @@ -52,16 +54,12 @@ # set while generating the pipeline nodes: "" INSTALL_EXTRA_TOOLS: "xtensa-esp-elf-gdb riscv32-esp-elf-gdb openocd-esp32 esp-rom-elfs" - PYTEST_EXTRA_FLAGS: "--dev-passwd ${ETHERNET_TEST_PASSWORD} --dev-user ${ETHERNET_TEST_USER} --capture=fd --verbosity=0 --unity-test-report-mode merge" + PYTEST_EXTRA_FLAGS: "--capture=fd --verbosity=0 --unity-test-report-mode merge" needs: - pipeline: $PARENT_PIPELINE_ID job: pipeline_variables - cache: - # Usually do not need submodule-cache in target_test - - key: pip-cache-${LATEST_GIT_TAG} - paths: - - .cache/pip - policy: pull + - pipeline: $PARENT_PIPELINE_ID + job: snapshot_known_failure_cases artifacts: paths: - XUNIT_RESULT*.xml @@ -71,13 +69,16 @@ when: always expire_in: 1 week script: + # Merge the current known failure cases file with snapshot_known_failure_cases file captured at pipeline start + # to avoid retry failures caused by later updates to the known failure cases file. - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} + - printf '\n' >> ${KNOWN_FAILURE_CASES_FILE_NAME} + - cat artifacts/${KNOWN_FAILURE_CASES_FILE_NAME}.freeze >> ${KNOWN_FAILURE_CASES_FILE_NAME} # get runner env config file - retry_failed git clone $TEST_ENV_CONFIG_REPO - run_cmd python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs # CI specific options start from "--known-failure-cases-file xxx". could ignore when running locally - run_cmd pytest $nodes - --pipeline-id $PARENT_PIPELINE_ID --junitxml=XUNIT_RESULT_${CI_JOB_ID}.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} --parallel-count ${CI_NODE_TOTAL:-1} diff --git a/tools/ci/dynamic_pipelines/templates/report.template.html b/tools/ci/dynamic_pipelines/templates/report.template.html deleted file mode 100644 index adbdf4710f3..00000000000 --- a/tools/ci/dynamic_pipelines/templates/report.template.html +++ /dev/null @@ -1,131 +0,0 @@ - - - - - - {{title}} - - - - - - - - - - - - - -
- -
-
- -
-
-

- Dynamic Pipeline Report -

-
-
-
- - -
-
-
-
-
- - -
-
-
- -
-
{{table}}
-
-
- - -
-
- -
-
- - - - - - - - - - - - - - diff --git a/tools/ci/dynamic_pipelines/templates/scripts.js b/tools/ci/dynamic_pipelines/templates/scripts.js deleted file mode 100644 index 403aa23597c..00000000000 --- a/tools/ci/dynamic_pipelines/templates/scripts.js +++ /dev/null @@ -1,1127 +0,0 @@ -function debounce(func, wait) { - let timeout; - return function () { - const context = this; - const args = arguments; - clearTimeout(timeout); - timeout = setTimeout(() => func.apply(context, args), wait); - }; -} - -function extractPipelineId() { - return $("body").data("pipeline-id"); -} - -function updateTabsAvailability(jobs) { - $("#build-report-tab, #job-report-tab, #test-report-tab").each(function () { - const $tab = $(this); - if (!$tab.hasClass("active")) { - $tab.addClass("disabled"); - $tab.attr("title", "This report is not available yet"); - $tab.css("opacity", "0.5"); - $tab.css("cursor", "not-allowed"); - $tab.on("click", function (e) { - if ($(this).hasClass("disabled")) { - e.preventDefault(); - e.stopPropagation(); - return false; - } - }); - } - }); - - const buildReportPatterns = [ - "generate_pytest_build_report", - "built_apps", - "build_report", - "skipped_apps", - "failed_apps", - ]; - - const jobReportPatterns = [ - "generate_failed_jobs_report", - "job_report", - "pipeline_jobs", - ]; - - const testReportPatterns = [ - "generate_pytest_report", - "target_test_report", - "test_report", - "pytest_report", - ]; - - let buildReportJob = jobs.find((job) => - buildReportPatterns.some((pattern) => job.name.includes(pattern)) - ); - - let jobReportJob = jobs.find((job) => - jobReportPatterns.some((pattern) => job.name.includes(pattern)) - ); - - let testReportJob = jobs.find((job) => - testReportPatterns.some((pattern) => job.name.includes(pattern)) - ); - - if (buildReportJob) reportJobIds.build = buildReportJob.job_id; - if (jobReportJob) reportJobIds.job = jobReportJob.job_id; - if (testReportJob) reportJobIds.test = testReportJob.job_id; - - const hasBuildReport = buildReportJob !== undefined; - const hasJobReport = jobReportJob !== undefined; - const hasTestReport = testReportJob !== undefined; - - if (hasBuildReport && !$("#build-report-tab").hasClass("active")) { - $("#build-report-tab").removeClass("disabled"); - $("#build-report-tab").attr( - "title", - "View application build results including binary sizes and downloads" - ); - $("#build-report-tab").css("opacity", "1"); - $("#build-report-tab").css("cursor", "pointer"); - } - - if (hasJobReport && !$("#job-report-tab").hasClass("active")) { - $("#job-report-tab").removeClass("disabled"); - $("#job-report-tab").attr( - "title", - "View CI job results and failed jobs" - ); - $("#job-report-tab").css("opacity", "1"); - $("#job-report-tab").css("cursor", "pointer"); - } - - if (hasTestReport && !$("#test-report-tab").hasClass("active")) { - $("#test-report-tab").removeClass("disabled"); - $("#test-report-tab").attr( - "title", - "View test results including success, failure, and skipped tests" - ); - $("#test-report-tab").css("opacity", "1"); - $("#test-report-tab").css("cursor", "pointer"); - } -} - -$(document).ready(function () { - const currentPath = window.location.pathname; - const currentFile = currentPath.substring(currentPath.lastIndexOf("/") + 1); - - $(".report-nav-tabs .nav-tab").removeClass("active"); - - if (currentFile.includes("built_apps")) { - $("#build-report-tab").addClass("active"); - } else if (currentFile.includes("job_report")) { - $("#job-report-tab").addClass("active"); - } else if (currentFile.includes("target_test_report")) { - $("#test-report-tab").addClass("active"); - } - - wrapTablesInCollapsibleSections(); - makeTablesCollapsible(); - setupScrollProgressBar(); - setupFloatingActions(); - enhanceTableStatusDisplay(); - markTestCaseColumns(); - setupResponsiveText(); - setupTextToggles(); - setupEventHandlers(); - scrollToHashLocation(); - initBootstrapTable(); - fixStickyHeaderAlignment(); - forceEnableStickyHeaders(); - fixTableHeaderText(); - lazyLoadVisibleImages(); - setupPagination(); - optimizeScrollPerformance(); -}); - -function fixStickyHeaderAlignment() { - if (window.isScrolling) return; - - $(".sticky-header-container").each(function () { - const $container = $(this); - const $table = $container - .closest(".bootstrap-table") - .find(".fixed-table-body table"); - - let tableWidth = "100%"; - if ($table.length) { - const actualWidth = $table.width(); - if (actualWidth > 0) { - tableWidth = actualWidth + "px"; - } - } - - $container.css({ - left: "0", - "margin-left": "0", - "padding-left": "0", - width: tableWidth, - "z-index": "100", - transform: "translateZ(0)", - }); - - this.style.setProperty("left", "0", "important"); - this.style.setProperty("margin-left", "0", "important"); - this.style.setProperty("padding-left", "0", "important"); - this.style.setProperty("width", tableWidth, "important"); - this.style.setProperty("z-index", "100", "important"); - this.style.setProperty("transform", "translateZ(0)", "important"); - - const $headerTable = $container.find("table"); - if ($headerTable.length) { - $headerTable.css("width", tableWidth); - $headerTable[0].style.setProperty("width", tableWidth, "important"); - } - - if (!window.isScrolling) { - if ($headerTable.length && $table.length) { - const $headerCols = $headerTable.find("th"); - const $bodyCols = $table.find("tr:first-child td"); - - if ($headerCols.length === $bodyCols.length) { - $headerCols.each(function (i) { - if (i < $bodyCols.length) { - const bodyColWidth = $($bodyCols[i]).outerWidth(); - if (bodyColWidth > 0) { - $(this).css("min-width", bodyColWidth + "px"); - $(this).css("width", bodyColWidth + "px"); - - const $thInner = $(this).find(".th-inner"); - if ($thInner.length) { - $thInner.css({ - "white-space": "normal", - overflow: "visible", - "text-overflow": "clip", - height: "auto", - display: "block", - }); - } - } - } - }); - } - } - } - }); - - $(".fixed-table-container").each(function () { - if ($(this).css("left") === "20px") { - $(this).css("left", "0"); - this.style.setProperty("left", "0", "important"); - } - }); - - $(".bootstrap-table .table thead th").css({ - position: "sticky", - top: "0", - "z-index": "100", - "background-color": "var(--esp-light)", - transform: "translateZ(0)", - "white-space": "normal", - overflow: "visible", - "text-overflow": "clip", - height: "auto", - }); - - $( - ".bootstrap-table .table thead th .th-inner, .sticky-header-container th .th-inner" - ).css({ - "white-space": "normal", - overflow: "visible", - "text-overflow": "clip", - height: "auto", - "min-height": "20px", - display: "block", - "line-height": "1.4", - }); -} - -function setupScrollProgressBar() { - const progressBar = $("#nav-progress-bar"); - let lastScrollPosition = 0; - let ticking = false; - - $(window).on("scroll", function () { - if (ticking) return; - - const scrollPosition = window.scrollY; - - if (Math.abs(scrollPosition - lastScrollPosition) < 5) { - return; - } - - lastScrollPosition = scrollPosition; - - window.requestAnimationFrame(function () { - const windowHeight = $(document).height() - $(window).height(); - const scrollPercentage = (scrollPosition / windowHeight) * 100; - progressBar.css("width", scrollPercentage + "%"); - ticking = false; - }); - ticking = true; - }); -} - -function setupFloatingActions() { - $("#back-to-top").on("click", function () { - $("html, body").animate({ scrollTop: 0 }, 300); - }); -} - -function enhanceTableStatusDisplay() { - $("table.table th").each(function (index) { - const $header = $(this); - const headerText = $header.text().trim().toLowerCase(); - - if ( - headerText.includes("status") || - headerText.includes("result") || - headerText.includes("state") - ) { - const colIndex = index + 1; - - $(`table.table td:nth-child(${colIndex})`).each(function () { - const $cell = $(this); - const cellText = $cell.text().trim().toLowerCase(); - - if (cellText.includes("pass") || cellText.includes("success")) { - $cell.html( - `
${$cell.text()}
` - ); - } else if ( - cellText.includes("fail") || - cellText.includes("error") - ) { - $cell.html( - `
${$cell.text()}
` - ); - } else if ( - cellText.includes("warn") || - cellText.includes("skip") - ) { - $cell.html( - `
${$cell.text()}
` - ); - } - }); - } - }); -} - -function setupTextToggles() { - setupResponsiveText(); - - $(".toggle-link").off("click").on("click", toggleText); -} - -function setupResponsiveText() { - $("table.table td").each(function () { - const $cell = $(this); - const text = $cell.text(); - - if ( - text.length > 100 && - !$cell.hasClass("test-case-name") && - !$cell.find(".text-toggle").length - ) { - const displayText = text.substring(0, 100) + "..."; - $cell.html( - `${displayText}` + - `` + - ` Show More` - ); - } - }); -} - -function makeTablesCollapsible() { - $("table.table").each(function (index) { - const table = $(this); - const tableId = `table-${index}`; - table.attr("id", tableId); - - const bootstrapTableWrapper = table.closest(".bootstrap-table"); - - let container = bootstrapTableWrapper.parent(".table-container"); - if (!container.length) { - container = $('
'); - } - - let tableControls = bootstrapTableWrapper.prev(".table-controls"); - - if (!tableControls.length) { - const initialButtonHtml = - 'Collapse Table'; - - const toggleButton = $(` - - `); - - tableControls = $('
').append( - toggleButton - ); - - bootstrapTableWrapper.before(tableControls); - } else { - const existingButton = tableControls.find(".table-collapse-btn"); - if (existingButton.length) { - existingButton.detach(); - tableControls.append(existingButton); - } - } - - if (!bootstrapTableWrapper.parent().hasClass("table-container")) { - bootstrapTableWrapper.wrap(container); - } - - bootstrapTableWrapper.show(); - bootstrapTableWrapper.addClass("expanded"); - - const toggleButton = tableControls.find(".table-collapse-btn"); - toggleButton.off("click").on("click", function (e) { - e.preventDefault(); - e.stopPropagation(); - - const tableId = $(this).data("table"); - - const targetTable = $(`#${tableId}`); - if (!targetTable.length) { - console.error("Target table not found:", tableId); - return; - } - - const bootstrapWrapper = targetTable.closest(".bootstrap-table"); - if (!bootstrapWrapper.length) { - console.error( - "Bootstrap wrapper not found for table:", - tableId - ); - return; - } - - const isVisible = bootstrapWrapper.is(":visible"); - - if (isVisible) { - bootstrapWrapper.slideUp(300); - bootstrapWrapper.removeClass("expanded"); - $(this).html( - 'Expand Table' - ); - } else { - bootstrapWrapper.slideDown(300); - bootstrapWrapper.addClass("expanded"); - $(this).html( - 'Collapse Table' - ); - } - }); - }); - - setTimeout(function () { - $(".bootstrap-table[data-auto-collapse='true']").each(function () { - const bootstrapWrapper = $(this); - const tableId = bootstrapWrapper.find("table").attr("id"); - const toggleButton = $(`button[data-table="${tableId}"]`); - - bootstrapWrapper.slideUp(300); - bootstrapWrapper.removeClass("expanded"); - - if (toggleButton.length) { - toggleButton.html( - 'Expand Table' - ); - } - }); - }, 500); -} - -function expandAllTables() { - $(".bootstrap-table").slideDown(300); - $(".bootstrap-table").addClass("expanded"); - - $(".table-collapse-btn").each(function () { - $(this).html( - 'Collapse Table' - ); - }); - - $(".section-header").attr("aria-expanded", "true"); - $(".section-body").slideDown(300); -} - -function collapseAllTables() { - $(".bootstrap-table").slideUp(300); - $(".bootstrap-table").removeClass("expanded"); - - $(".table-collapse-btn").each(function () { - $(this).html('Expand Table'); - }); -} - -function initBootstrapTable() { - $("table.table").each(function () { - const $table = $(this); - - $table.bootstrapTable({ - classes: "table table-bordered table-striped", - height: 800, - pagination: false, - showColumns: true, - showColumnsToggleAll: true, - showToggle: false, - clickToSelect: false, - minimumCountColumns: 2, - stickyHeader: true, - stickyHeaderOffsetY: 0, - theadClasses: "thead-light", - toolbar: "#toolbar", - resizable: true, - checkOnInit: false, - fixedColumns: true, - fixedNumber: 0, - sortStable: true, - undefinedText: "-", - showFullscreen: true, - widthUnit: "%", - headerStyle: function () { - return { - css: { - "white-space": "normal", - overflow: "visible", - "text-overflow": "clip", - height: "auto", - "min-height": "50px", - }, - }; - }, - }); - - $table.closest(".bootstrap-table").css({ - width: "100%", - "max-width": "100%", - }); - - $table.closest(".fixed-table-body").css({ - "overflow-x": "auto", - }); - - $table.css({ - width: "100%", - "min-width": "100%", - }); - - $table.find("thead th").each(function () { - const $th = $(this); - const $thInner = $th.find(".th-inner"); - - if ($thInner.length) { - $thInner.css({ - "white-space": "normal", - overflow: "visible", - "text-overflow": "clip", - height: "auto", - "min-height": "20px", - display: "block", - }); - } - }); - }); - - $(window).on( - "resize", - debounce(function () { - $("table.table").bootstrapTable("resetView"); - }, 100) - ); -} - -function wrapTablesInCollapsibleSections() { - $("h2").each(function (index) { - if ($(this).closest(".section-header").length > 0) { - return; - } - - const header = $(this); - const headerText = header.text(); - const headerId = header.attr("id") || `section-header-${index}`; - - const nextTable = header.nextAll("table.table:first"); - - if (nextTable.length) { - if (nextTable.closest(".section-body").length > 0) { - return; - } - - const wrapper = $('
'); - - const sectionHeader = $(` -
-

- ${headerText} - -

- -
- `); - - const sectionBody = $( - `
` - ); - - const bootstrapTable = nextTable.closest(".bootstrap-table"); - const tableContainer = bootstrapTable.closest(".table-container"); - - if (tableContainer.length) { - tableContainer.detach().appendTo(sectionBody); - } else if (bootstrapTable.length) { - bootstrapTable.detach().appendTo(sectionBody); - } else { - nextTable.detach().appendTo(sectionBody); - } - - wrapper.append(sectionHeader).append(sectionBody); - - header.replaceWith(wrapper); - } - }); - - $(".section-header") - .off("click") - .on("click", function (e) { - if ( - $(e.target).hasClass("copy-link-icon") || - $(e.target).closest(".copy-link-icon").length || - $(e.target).hasClass("table-collapse-btn") || - $(e.target).closest(".table-collapse-btn").length - ) { - return; - } - - const header = $(this); - const targetId = header.data("target"); - const sectionBody = $(targetId); - const isExpanded = header.attr("aria-expanded") === "true"; - - if (isExpanded) { - header.attr("aria-expanded", "false"); - sectionBody.slideUp(300); - } else { - header.attr("aria-expanded", "true"); - sectionBody.slideDown(300); - } - }); -} - -function markTestCaseColumns() { - $("table.table th").each(function (index) { - const headerText = $(this).text().trim().toLowerCase(); - if ( - headerText.includes("test") && - (headerText.includes("case") || headerText.includes("name")) - ) { - const colIndex = index + 1; - $(`table.table td:nth-child(${colIndex})`).addClass( - "test-case-name" - ); - } - }); -} - -function setupEventHandlers() { - $(window).on("load", scrollToHashLocation); - $("body").on("click", ".toggle-link", toggleText); - - $("#expand-all-tables") - .off("click") - .on("click", function (e) { - e.preventDefault(); - expandAllTables(); - }); - - $("#collapse-all-tables") - .off("click") - .on("click", function (e) { - e.preventDefault(); - collapseAllTables(); - }); - - $("#pagination-size").on("change", function () { - const pageSize = parseInt($(this).val(), 10); - applyPaginationToTables(pageSize); - }); - - $("#show-all-columns").on("click", function (e) { - e.preventDefault(); - $("table.table").bootstrapTable("showAllColumns"); - }); - - $("#show-failed-only").on("click", function (e) { - e.preventDefault(); - filterByStatus("fail"); - }); - - $("#show-passed-only").on("click", function (e) { - e.preventDefault(); - filterByStatus("pass"); - }); - - $("#export-csv").on("click", function (e) { - e.preventDefault(); - $("table.table:visible").bootstrapTable("exportTable", { - type: "csv", - fileName: "report_export_" + new Date().toISOString().slice(0, 10), - }); - }); - - $("#clear-all-filters").on("click", function () { - clearAllFilters(); - }); -} - -function filterByStatus(status) { - $("#active-filters").show(); - $("#filter-badges").html( - `Status: ${status}` - ); - - $("table.table").each(function () { - const $table = $(this); - - let statusColIndex = -1; - $table.find("th").each(function (index) { - const headerText = $(this).text().trim().toLowerCase(); - if ( - headerText.includes("status") || - headerText.includes("result") || - headerText.includes("state") - ) { - statusColIndex = index; - return false; - } - }); - - if (statusColIndex >= 0) { - $table.bootstrapTable( - "filterBy", - { - [statusColIndex]: status, - }, - { - filterAlgorithm: function (row, filters) { - const cellText = $(row[statusColIndex]) - .text() - .trim() - .toLowerCase(); - return cellText.includes(status); - }, - } - ); - } - }); -} - -function clearAllFilters() { - $("table.table").each(function () { - const $table = $(this); - $table.bootstrapTable("clearFilterControl"); - $table.bootstrapTable("resetSearch"); - }); - - $("#active-filters").hide(); - $("#filter-badges").empty(); -} - -function scrollToHashLocation() { - const hash = window.location.hash; - if (hash) { - setTimeout(() => { - const target = $(hash); - if (target.length) { - const sectionBody = target.closest(".section-body"); - if (sectionBody.length) { - const sectionHeader = sectionBody.prev(".section-header"); - if (sectionHeader.attr("aria-expanded") !== "true") { - sectionHeader.attr("aria-expanded", "true"); - sectionBody.slideDown(0); - } - - const containingTable = target.closest("table.table"); - if (containingTable.length) { - const bootstrapWrapper = - containingTable.closest(".bootstrap-table"); - if (!bootstrapWrapper.is(":visible")) { - bootstrapWrapper.show(); - const tableBtn = bootstrapWrapper - .parent() - .find(".table-controls") - .find(".table-collapse-btn"); - tableBtn.html( - 'Collapse Table' - ); - } - } - } - - $("html, body").animate( - { scrollTop: target.offset().top - 20 }, - 100 - ); - } - }, 300); - } -} - -function copyPermalink(anchorId) { - const fullUrl = `${window.location.origin}${window.location.pathname}${anchorId}`; - history.pushState(null, null, anchorId); - navigator.clipboard.writeText(fullUrl); - - const tooltip = $(` -
- Link copied to clipboard! -
- `); - - const icon = $(event.target).closest(".copy-link-icon"); - if (icon.length) { - const originalClass = icon.attr("class"); - - icon.removeClass() - .addClass("fas fa-check copy-link-icon") - .css("color", "var(--esp-success)"); - - setTimeout(() => { - icon.attr("class", originalClass).css("color", ""); - }, 1500); - } - - $("body").append(tooltip); - setTimeout(() => tooltip.remove(), 2000); - scrollToHashLocation(); -} - -function toggleText(e) { - e.preventDefault(); - e.stopPropagation(); - - const link = $(this); - const textSpan = link.siblings(".full-text"); - const toggleSpan = link.siblings(".text-toggle"); - - const visible = textSpan.is(":visible"); - - if (visible) { - link.html(' Show More'); - textSpan.hide(); - toggleSpan.show(); - } else { - link.html(' Show Less'); - textSpan.show(); - toggleSpan.hide(); - } -} - -function setupPagination() { - applyPaginationToTables(25); -} - -function applyPaginationToTables(pageSize) { - localStorage.setItem("paginationSize", pageSize); - - $("table.table").each(function () { - const $table = $(this); - - if (pageSize > 0) { - $table.bootstrapTable("refreshOptions", { - pagination: true, - pageSize: pageSize, - pageList: [10, 25, 50, "All"], - }); - - $table - .closest(".bootstrap-table") - .find(".fixed-table-pagination") - .css({ - display: "block", - visibility: "visible", - "background-color": "var(--esp-light)", - padding: "10px", - "border-top": "1px solid rgba(0, 0, 0, 0.05)", - }); - } else { - $table.bootstrapTable("refreshOptions", { - pagination: false, - }); - - $table - .closest(".bootstrap-table") - .find(".fixed-table-pagination") - .css({ - display: "none", - visibility: "hidden", - }); - } - }); -} - -function optimizeScrollPerformance() { - window.isScrolling = false; - let scrollTimer = null; - let lastScrollTop = 0; - - cacheAndFixDomElements(); - - const progressBar = document.getElementById("nav-progress-bar"); - const backToTop = document.getElementById("back-to-top"); - - window.addEventListener( - "scroll", - function () { - window.isScrolling = true; - - const scrollTop = window.scrollY; - - if (Math.abs(scrollTop - lastScrollTop) > 5) { - lastScrollTop = scrollTop; - - handleEssentialScrollUpdates(scrollTop, progressBar, backToTop); - } - - clearTimeout(scrollTimer); - scrollTimer = setTimeout(function () { - window.isScrolling = false; - requestAnimationFrame(function () { - fixStickyHeaderAlignment(); - fixTableHeaderText(); - }); - }, 150); - }, - { passive: true } - ); - - lazyLoadVisibleImages(); -} - -function handleEssentialScrollUpdates(scrollTop, progressBar, backToTop) { - requestAnimationFrame(function () { - const docHeight = - Math.max( - document.body.scrollHeight, - document.body.offsetHeight, - document.documentElement.clientHeight, - document.documentElement.scrollHeight, - document.documentElement.offsetHeight - ) - window.innerHeight; - const scrollPercentage = (scrollTop / docHeight) * 100; - progressBar.style.width = scrollPercentage + "%"; - - if (scrollTop > 200) { - backToTop.classList.add("visible"); - } else { - backToTop.classList.remove("visible"); - } - }); -} - -function cacheAndFixDomElements() { - const stickyElements = document.querySelectorAll( - ".sticky-header-container, .section-header, .bootstrap-table thead, .fixed-table-header" - ); - - for (let i = 0; i < stickyElements.length; i++) { - const elem = stickyElements[i]; - elem.style.transform = "translateZ(0)"; - elem.style.willChange = "transform"; - elem.style.backfaceVisibility = "hidden"; - } - - document.body.style.willChange = "scroll-position"; - document.body.style.backfaceVisibility = "hidden"; - - const headers = document.querySelectorAll( - ".bootstrap-table .table thead th, .sticky-header-container th" - ); - for (let i = 0; i < headers.length; i++) { - const header = headers[i]; - header.style.position = "sticky"; - header.style.top = "0"; - header.style.zIndex = "100"; - header.style.transform = "translateZ(0)"; - - const thInners = header.querySelectorAll(".th-inner"); - for (let j = 0; j < thInners.length; j++) { - const inner = thInners[j]; - inner.style.whiteSpace = "normal"; - inner.style.overflow = "visible"; - inner.style.textOverflow = "clip"; - inner.style.height = "auto"; - inner.style.minHeight = "20px"; - inner.style.display = "block"; - } - } -} - -function lazyLoadVisibleImages() { - const lazyImages = document.querySelectorAll("img[data-src]"); - if (lazyImages.length === 0) return; - - const loadImage = function (img) { - if (img.dataset.src) { - img.src = img.dataset.src; - img.removeAttribute("data-src"); - } - }; - - lazyImages.forEach((img) => { - if (isElementInViewport(img)) { - loadImage(img); - } - }); - - document.addEventListener( - "scroll", - debounce(function () { - lazyImages.forEach((img) => { - if (img.dataset.src && isElementInViewport(img)) { - loadImage(img); - } - }); - }, 200), - { passive: true } - ); -} - -function isElementInViewport(el) { - const rect = el.getBoundingClientRect(); - return ( - rect.top >= 0 && - rect.left >= 0 && - rect.bottom <= - (window.innerHeight || document.documentElement.clientHeight) && - rect.right <= - (window.innerWidth || document.documentElement.clientWidth) - ); -} - -function fixTableHeaderText() { - if (window.isScrolling) return; - - $(".bootstrap-table .table thead th, .sticky-header-container th").each( - function () { - const $th = $(this); - - $th.css({ - "white-space": "normal", - overflow: "visible", - "text-overflow": "clip", - height: "auto", - "min-height": "50px", - }); - - const $thInner = $th.find(".th-inner"); - if ($thInner.length) { - $thInner.css({ - "white-space": "normal", - overflow: "visible", - "text-overflow": "clip", - height: "auto", - "min-height": "20px", - display: "block", - "line-height": "1.4", - }); - - $thInner.find("span, div").css({ - "white-space": "normal", - overflow: "visible", - }); - } - } - ); -} - -function forceEnableStickyHeaders() { - if (!window.isScrolling) { - $("table.table thead th").css({ - position: "sticky", - top: "0", - "z-index": "100", - "background-color": "var(--esp-light)", - transform: "translateZ(0)", - "will-change": "transform", - "backface-visibility": "hidden", - "white-space": "normal", - overflow: "visible", - "text-overflow": "clip", - height: "auto", - "min-height": "50px", - }); - - $("table.table").each(function () { - const $table = $(this); - const bootstrapTable = $table.closest(".bootstrap-table"); - - if ( - bootstrapTable.length && - bootstrapTable.find(".sticky-header-container").length === 0 - ) { - const $thead = $table.find("thead").clone(); - const $stickyContainer = $( - '
' - ); - const $stickyTable = $('
').append( - $thead - ); - - $stickyContainer.append($stickyTable); - bootstrapTable.prepend($stickyContainer); - - const tableWidth = $table.width(); - $stickyContainer.css({ - position: "sticky", - top: "0", - "z-index": "100", - width: tableWidth + "px", - overflow: "hidden", - "background-color": "var(--esp-light)", - "will-change": "transform", - "backface-visibility": "hidden", - transform: "translateZ(0)", - }); - - const $originalThs = $table.find("thead th"); - const $stickyThs = $stickyTable.find("th"); - - $originalThs.each(function (i) { - if (i < $stickyThs.length) { - const width = $(this).outerWidth(); - $($stickyThs[i]).css({ - width: width + "px", - "white-space": "normal", - overflow: "visible", - "text-overflow": "clip", - height: "auto", - "min-height": "50px", - }); - - const $inner = $($stickyThs[i]).find(".th-inner"); - if ($inner.length) { - $inner.css({ - "white-space": "normal", - overflow: "visible", - "text-overflow": "clip", - height: "auto", - "min-height": "20px", - display: "block", - "line-height": "1.4", - }); - } - } - }); - } - }); - } -} diff --git a/tools/ci/dynamic_pipelines/templates/styles.css b/tools/ci/dynamic_pipelines/templates/styles.css deleted file mode 100644 index cc3c43b9b43..00000000000 --- a/tools/ci/dynamic_pipelines/templates/styles.css +++ /dev/null @@ -1,1095 +0,0 @@ -/** - * Espressif Report Template Styles - * Optimized for desktop-only application - */ - -/* Variables and base settings */ -:root { - /* Color palette */ - --esp-primary: #e83711; - --esp-secondary: #f3c300; - --esp-dark: #282430; - --esp-light: #f8f9fa; - --esp-blue: #3a86ff; - --esp-gray: #707070; - --esp-success: #25be7b; - --esp-warning: #f3c300; - --esp-danger: #e83711; - - /* UI elements */ - --body-bg: #f5f7fa; - --card-bg: #ffffff; - --text-color: #282430; - --border-radius: 8px; - --box-shadow: 0 4px 10px rgba(0, 0, 0, 0.05); - --transition-speed: 0.3s; - - /* Fixed desktop layout variables */ - --container-width: 1600px; - --sidebar-width: 280px; - --header-height: 60px; -} - -/* ---------------------------------- */ -/* Base styles */ -/* ---------------------------------- */ -html { - scroll-behavior: smooth; -} - -body { - font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, - "Helvetica Neue", Arial, sans-serif; - line-height: 1.6; - color: var(--text-color); - background-color: var(--body-bg); - transition: background-color var(--transition-speed); - min-width: 1024px; - overflow-y: scroll; -} - -.container-fluid { - max-width: 100% !important; - padding: 20px !important; - margin: 0 auto; -} - -/* Typography */ -h2 { - padding-top: 3rem; - margin-bottom: 0px; - color: var(--esp-dark); - font-weight: 600; - background-color: #f5f7fa; -} - -/* ---------------------------------- */ -/* Performance Optimizations */ -/* ---------------------------------- */ -/* Hardware acceleration utility class */ -.hw-accelerated { - transform: translateZ(0); - will-change: transform; - backface-visibility: hidden; -} - -/* Scrollbar styling */ -::-webkit-scrollbar { - width: 8px; - height: 8px; -} - -::-webkit-scrollbar-track { - background: rgba(0, 0, 0, 0.05); - border-radius: 4px; -} - -::-webkit-scrollbar-thumb { - background: rgba(0, 0, 0, 0.2); - border-radius: 4px; -} - -::-webkit-scrollbar-thumb:hover { - background: rgba(0, 0, 0, 0.3); -} - -/* Reduced motion for accessibility */ -@media (prefers-reduced-motion: reduce) { - html { - scroll-behavior: auto; - } - - .bootstrap-table, - .section-body, - .sticky-header-container, - .section-header { - transition: none !important; - } -} - -/* ---------------------------------- */ -/* Navigation and Header */ -/* ---------------------------------- */ - -/* Header styling */ -.report-header { - display: flex; - align-items: center; - padding: 1.2rem 0; - margin-bottom: 2rem; - border-bottom: 1px solid rgba(0, 0, 0, 0.1); - position: relative; - justify-content: space-between; - background-color: #f8f9fa; -} - -.report-header .logo { - height: 30px; - margin-bottom: 0; - padding-left: 15px; -} - -.report-header .title-container { - display: flex; - flex-direction: column; - justify-content: center; - text-align: center; - margin: 0 auto; -} - -.report-header h1 { - margin: 0; - font-weight: 700; - font-size: 1.8rem; - line-height: 1.2; - display: inline-block; - text-align: center; -} - -.report-header p { - margin-bottom: 0; - font-size: 0.85rem; - text-align: center; - color: #707070; -} - -.report-header::after { - content: ""; - position: absolute; - bottom: -3px; - left: 0; - right: 0; - height: 3px; - background-color: var(--esp-primary); -} - -.logo-container { - width: 180px; - display: flex; - align-items: center; - justify-content: flex-start; -} - -.spacer { - width: 180px; /* Same as logo-container for balance */ -} - -/* Navigation progress bar */ -.nav-progress-container { - position: fixed; - top: 0; - left: 0; - width: 100%; - height: 3px; - background-color: transparent; - z-index: 1001; -} - -.nav-progress-bar { - height: 100%; - width: 0; - background-color: var(--esp-primary); - transition: width 0.2s ease-out; -} - -/* ---------------------------------- */ -/* Tables - Complete Reset & Rebuild */ -/* ---------------------------------- */ - -/* Base table appearance */ -.table { - width: 100%; - margin-bottom: 0; - background-color: var(--card-bg); - border-collapse: separate; - border-spacing: 0; -} - -/* Table header cells */ -.table > thead > tr > th { - padding: 12px 15px; - font-weight: 600; - color: var(--esp-dark); - background-color: var(--esp-light); - border-bottom: 2px solid rgba(0, 0, 0, 0.1); - text-align: left; - position: sticky; - top: 0; - z-index: 10; -} - -/* Table data cells */ -.table > tbody > tr > td { - padding: 12px 15px; - border-top: 1px solid rgba(0, 0, 0, 0.05); -} - -/* Striped rows */ -.table-striped > tbody > tr:nth-of-type(odd) { - background-color: rgba(0, 0, 0, 0.02); -} - -.table-striped > tbody > tr:hover { - background-color: rgba(0, 0, 0, 0.05); -} - -/* Bootstrap Table container */ -.bootstrap-table { - border-radius: var(--border-radius); - overflow: hidden; - box-shadow: var(--box-shadow); - margin-bottom: 1rem; -} - -/* Bootstrap Table scrollable body */ -.bootstrap-table .fixed-table-body { - max-height: 800px; - overflow-y: auto; - border: none !important; -} - -/* Remove any borders from the fixed-table-container */ -.bootstrap-table .fixed-table-container { - border: none !important; -} - -/* Override Bootstrap's fixed header styles to avoid conflicts */ -.bootstrap-table .fixed-table-header table { - background-color: var(--esp-light); -} - -.bootstrap-table .fixed-table-header { - background-color: var(--esp-light); -} - -/* Column width settings for better layout */ -.table > thead > tr > th:nth-child(1), -.table > tbody > tr > td:nth-child(1) { - width: 20%; - min-width: 150px; -} - -.table > thead > tr > th:nth-child(2), -.table > tbody > tr > td:nth-child(2) { - width: 20%; - min-width: 150px; -} - -.table > thead > tr > th:nth-child(3), -.table > tbody > tr > td:nth-child(3) { - width: 20%; - min-width: 150px; -} - -.table > thead > tr > th:nth-child(n + 4), -.table > tbody > tr > td:nth-child(n + 4) { - width: auto; - min-width: 100px; -} - -/* Ensure sticky section headers are above table headers */ -.sticky-section-header { - position: sticky; - top: 0; - z-index: 100; - background-color: #f5f7fa; -} - -/* Test case names */ -td.test-case-name { - white-space: normal; - overflow: visible; - word-break: break-word; -} - -/* Bootstrap table customizations */ -.bootstrap-table { - margin-bottom: 0; - border-radius: var(--border-radius); - box-shadow: var(--box-shadow); - background-color: var(--card-bg); - overflow: hidden; - max-height: 2000px; - transition: max-height 0.3s ease-in-out, opacity 0.3s ease-in-out; - opacity: 1; - width: 100% !important; - position: relative; -} - -.bootstrap-table.expanded { - max-height: 2000px; - opacity: 1; -} - -/* Main container setup */ -.bootstrap-table .fixed-table-container { - position: relative !important; - overflow: visible !important; -} - -/* Table headers - sticky relative to .fixed-table-body */ -.bootstrap-table .table thead th { - position: sticky !important; - top: 0 !important; - z-index: 10; - background-color: var(--esp-light); - border-bottom: 2px solid rgba(0, 0, 0, 0.1); - padding: 12px 15px !important; -} - -/* Fixed-table-header should be hidden when scrolling */ -.bootstrap-table .fixed-table-header { - display: none !important; -} - -/* Handle Bootstrap Table sticky header container */ -.bootstrap-table .sticky-header-container { - display: none !important; /* Use our own sticky headers instead */ -} - -/* Ensure th-inner elements are visible */ -.bootstrap-table .fixed-table-container .table thead th .th-inner { - white-space: normal !important; - overflow: visible !important; - text-overflow: clip !important; - height: auto !important; - min-height: 20px; - line-height: 1.4; - padding: 8px; - background-color: var(--esp-light); - display: block !important; -} - -/* Table container shouldn't have overflow-x at the top level */ -.table-container { - width: 100%; - max-width: 100%; - margin: 0 auto; - background-color: var(--card-bg); - border-radius: var(--border-radius); - overflow: hidden; - box-shadow: var(--box-shadow); - position: relative; -} - -/* Fix for table header rendering in the sticky context */ -.bootstrap-table thead { - position: sticky; - top: 0; - z-index: 10; -} - -/* Table controls */ -.table-controls { - background-color: var(--esp-light); - border-top-left-radius: var(--border-radius); - border-top-right-radius: var(--border-radius); - padding: 12px 15px; - border-bottom: 1px solid rgba(0, 0, 0, 0.05); - display: flex; - justify-content: flex-end; - align-items: center; - flex-wrap: wrap; - gap: 10px; -} - -.table-controls .row-count { - margin-bottom: 0; - white-space: nowrap; -} - -.table-controls .btn { - display: flex; - align-items: center; - justify-content: center; - gap: 5px; - transition: all 0.2s ease; -} - -/* Table toolbar */ -.fixed-table-toolbar { - padding: 10px; - background-color: var(--esp-light); - border-bottom: 1px solid rgba(0, 0, 0, 0.05); -} - -.fixed-table-toolbar .search { - display: flex; - align-items: center; -} - -.fixed-table-toolbar .search input { - border-radius: var(--border-radius); - border: 1px solid rgba(0, 0, 0, 0.1); - padding: 5px 10px; - margin-right: 10px; -} - -.fixed-table-toolbar .columns { - margin-left: auto; -} - -.fixed-table-toolbar .btn { - border-radius: var(--border-radius); - padding: 5px 10px; - margin-left: 5px; -} - -.fixed-table-toolbar .dropdown-menu { - border-radius: var(--border-radius); - box-shadow: var(--box-shadow); - padding: 8px; - border: none; -} - -/* Table pagination */ -.fixed-table-pagination { - padding: 10px; - background-color: var(--esp-light); - border-bottom-left-radius: var(--border-radius); - border-bottom-right-radius: var(--border-radius); - border-top: 1px solid rgba(0, 0, 0, 0.05); - display: flex; - justify-content: center; - align-items: center; -} - -.fixed-table-pagination .pagination { - float: none !important; - order: 1; - flex: 1 1 auto; - display: flex; - justify-content: center; - margin: 0 auto; - position: absolute; - left: 50%; - transform: translateX(-50%); -} - -.fixed-table-pagination .pagination-detail .pagination-info { - display: none; - float: none !important; - order: 2; - flex: 0 0 auto; -} - -.fixed-table-pagination .pagination li a { - border-radius: 4px; - margin: 0 2px; - border: 1px solid #dee2e6; - min-width: 34px; - height: 34px; - display: flex; - align-items: center; - justify-content: center; - color: var(--esp-dark); - transition: all 0.2s ease; -} - -.fixed-table-pagination .pagination li.active a { - background-color: var(--esp-primary); - border-color: var(--esp-primary); - color: white; -} - -.fixed-table-pagination .pagination li a:hover:not(.active) { - background-color: #e9ecef; - color: var(--esp-primary); -} - -/* Pagination options */ -.pagination-options .form-select-sm { - border-radius: var(--border-radius); - border: 1px solid rgba(0, 0, 0, 0.1); - height: 31px; - padding: 0.25rem 2rem 0.25rem 0.5rem; - font-size: 0.875rem; - background-color: var(--esp-light); - color: var(--esp-dark); - cursor: pointer; - transition: all var(--transition-speed); - box-shadow: 0 1px 3px rgba(0, 0, 0, 0.05); -} - -.pagination-options .form-select-sm:hover { - border-color: var(--esp-primary); -} - -.pagination-options .form-select-sm:focus { - border-color: var(--esp-primary); - box-shadow: 0 0 0 0.15rem rgba(232, 55, 17, 0.25); - outline: none; -} - -/* ---------------------------------- */ -/* Collapsible Sections */ -/* ---------------------------------- */ -.section-collapsible { - margin-bottom: 2rem; - border-radius: var(--border-radius); - overflow: hidden; - box-shadow: var(--box-shadow); - background-color: var(--card-bg); - transition: transform var(--transition-speed); -} - -.section-collapsible:hover { - transform: translateY(-2px); -} - -.section-header { - display: flex; - align-items: center; - padding: 1rem 1.5rem; - background-color: #f5f7fa; - border-bottom: 1px solid rgba(0, 0, 0, 0.1); - cursor: pointer; - transition: background-color var(--transition-speed), - box-shadow var(--transition-speed); -} - -.section-header:hover { - background-color: #e9ecef; - box-shadow: 0 3px 8px rgba(0, 0, 0, 0.1); -} - -.section-header h2 { - margin: 0; - font-size: 1.5rem; - flex-grow: 1; - background-color: transparent; - position: relative; - z-index: 2; - font-weight: 500; - display: flex; - align-items: center; -} - -.section-header h2 .heading-text { - margin-right: 6px; -} - -.section-header .collapse-indicator { - margin-left: 10px; - font-size: 1.5rem; - transition: transform var(--transition-speed); - color: var(--esp-primary); - position: relative; - z-index: 2; -} - -.section-header[aria-expanded="false"] .collapse-indicator { - transform: rotate(-90deg); -} - -.copy-link-icon { - margin-left: 8px; - font-size: 1.5rem; - color: var(--esp-gray); - opacity: 0.7; - cursor: pointer; - transition: opacity 0.2s ease, color 0.2s ease; - position: relative; - z-index: 2; -} - -.copy-link-icon:hover { - opacity: 1; - color: var(--esp-primary); -} - -.sticky-section-header { - position: sticky; - top: 0; - z-index: 99; - background-color: #f5f7fa; - box-shadow: 0 2px 5px rgba(0, 0, 0, 0.05); -} - -.section-body { - padding: 1.5rem; - max-height: 0; - overflow: hidden; - transition: max-height 0.3s ease-in-out, padding 0.3s ease-in-out, - opacity 0.3s ease-in-out; - padding-top: 0; - padding-bottom: 0; - opacity: 0; -} - -.section-body.expanded { - max-height: 5000px; - padding: 1.5rem; - opacity: 1; -} - -.section-body.collapsed { - display: none; -} - -/* ---------------------------------- */ -/* Status Badges */ -/* ---------------------------------- */ -.status-badge { - display: inline-block; - padding: 4px 8px; - border-radius: 4px; - font-size: 0.8rem; - font-weight: 500; - text-align: center; - min-width: 80px; - transition: background-color 0.2s ease, transform 0.2s ease; -} - -.status-badge.success { - background-color: rgba(37, 190, 123, 0.1); - color: var(--esp-success); - border: 1px solid rgba(37, 190, 123, 0.2); -} - -.status-badge.warning { - background-color: rgba(243, 195, 0, 0.1); - color: var(--esp-warning); - border: 1px solid rgba(243, 195, 0, 0.2); -} - -.status-badge.danger { - background-color: rgba(232, 55, 17, 0.1); - color: var(--esp-danger); - border: 1px solid rgba(232, 55, 17, 0.2); -} - -.status-badge:hover { - transform: translateY(-1px); -} - -/* ---------------------------------- */ -/* Buttons */ -/* ---------------------------------- */ -.btn-outline-secondary { - border-color: #ced4da; - color: var(--esp-dark); -} - -.btn-outline-secondary:hover { - background-color: var(--esp-light); - border-color: var(--esp-gray); - color: var(--esp-dark); -} - -.btn-esp { - background-color: var(--esp-primary); - border: none; - color: #fff; - transition: all var(--transition-speed); -} - -.btn-esp:hover { - background-color: #cf3110; - color: #fff; - box-shadow: 0 4px 8px rgba(232, 55, 17, 0.2); -} - -.btn-esp-secondary { - background-color: var(--esp-secondary); - border: none; - color: var(--esp-dark); - transition: all var(--transition-speed); -} - -.btn-esp-secondary:hover { - background-color: #dab000; - color: var(--esp-dark); - box-shadow: 0 4px 8px rgba(243, 195, 0, 0.2); -} - -/* Table collapse button - updated to appear on the right */ -.table-collapse-btn { - background-color: var(--esp-light); - border: none; - color: var(--esp-gray); - padding: 6px 12px; - border-radius: var(--border-radius); - font-size: 0.875rem; - cursor: pointer; - transition: all var(--transition-speed); - display: inline-flex; - align-items: center; - justify-content: center; - box-shadow: 0 2px 4px rgba(0, 0, 0, 0.05); - margin-left: auto; /* Push to the right side */ -} - -.table-collapse-btn:hover { - background-color: #e9ecef; - color: var(--esp-primary); -} - -.table-collapse-btn i { - margin-right: 5px; -} - -.clear-filter-btn { - transition: all 0.2s ease; - margin-top: 10px; -} - -.clear-filter-btn:hover { - background-color: var(--esp-light); - color: var(--esp-primary); -} - -.close-alert { - background: none; - border: none; - color: #856404; - font-size: 0.8rem; - cursor: pointer; - padding: 4px; - margin-left: auto; - opacity: 0.7; - transition: opacity 0.2s; - display: flex; - align-items: center; - justify-content: center; -} - -.close-alert:hover { - opacity: 1; -} - -/* Floating action buttons */ -.floating-actions { - position: fixed; - bottom: 30px; - right: 30px; - z-index: 1000; - display: flex; - flex-direction: column; - gap: 10px; -} - -.floating-action-btn { - width: 50px; - height: 50px; - border-radius: 50%; - background-color: var(--esp-primary); - color: white; - display: flex; - align-items: center; - justify-content: center; - box-shadow: 0 4px 10px rgba(0, 0, 0, 0.15); - cursor: pointer; - transition: transform 0.3s cubic-bezier(0.175, 0.885, 0.32, 1.275), - background-color 0.3s ease, box-shadow 0.3s ease; - font-size: 1.2rem; -} - -.floating-action-btn:hover { - transform: scale(1.1) translateY(-2px); -} - -.back-to-top { - background-color: var(--esp-secondary); - color: var(--esp-dark); -} - -/* ---------------------------------- */ -/* Utilities */ -/* ---------------------------------- */ -/* Text toggles */ -.text-toggle, -.full-text { - cursor: pointer; -} - -.toggle-link { - display: inline-block; - margin-left: 8px; - font-size: 0.8rem; - color: var(--esp-blue); - text-decoration: none; -} - -.toggle-link:hover { - text-decoration: underline; - color: var(--esp-primary); -} - -/* Row count label */ -.row-count { - font-size: 0.8rem; - color: var(--esp-gray); - margin-bottom: 15px; - display: block; -} - -/* Table state indicators */ -.table-empty-state, -.table-loading-state { - text-align: center; - padding: 50px 20px; - display: flex; - flex-direction: column; - align-items: center; - justify-content: center; -} - -.table-empty-state { - color: var(--esp-gray); - background-color: rgba(0, 0, 0, 0.01); - border-radius: var(--border-radius); -} - -.table-empty-state p { - margin-top: 10px; - font-size: 1.1rem; -} - -.table-filtered-indicator { - text-align: center; - padding: 30px 20px; - color: var(--esp-warning); - background-color: rgba(243, 195, 0, 0.05); - border-radius: var(--border-radius); - display: flex; - flex-direction: column; - align-items: center; - justify-content: center; - border: 1px dashed rgba(243, 195, 0, 0.3); - margin: 20px; -} - -.table-filtered-indicator p { - margin: 10px 0; - font-size: 1rem; -} - -.table-loading-indicator { - padding: 20px; - background-color: rgba(0, 0, 0, 0.02); - border-radius: var(--border-radius); - margin: 10px; - text-align: center; - color: var(--esp-gray); - animation: pulse 1.5s infinite ease-in-out; -} - -@keyframes pulse { - 0% { - opacity: 0.6; - } - 50% { - opacity: 1; - } - 100% { - opacity: 0.6; - } -} - -@keyframes fadeIn { - from { - opacity: 0; - } - to { - opacity: 1; - } -} - -/* Dropdown and other UI elements */ -.dropdown-menu { - border-radius: var(--border-radius); - box-shadow: var(--box-shadow); - border: none; - padding: 0.5rem; -} - -.dropdown-item { - border-radius: 4px; - padding: 8px 12px; -} - -.dropdown-item:hover { - background-color: rgba(232, 55, 17, 0.1); -} - -/* Column visibility controls */ -.column-visibility-controls { - margin-right: 10px; -} - -.column-visibility-menu { - max-height: 350px; - overflow-y: auto; - padding: 8px 0; - min-width: 220px; -} - -.column-visibility-menu .dropdown-item { - display: flex; - align-items: center; - padding: 8px 16px; - white-space: normal; - word-break: break-word; -} - -.column-visibility-menu .dropdown-item:hover { - background-color: rgba(232, 55, 17, 0.05); -} - -.column-visibility-menu .dropdown-header { - font-weight: 600; - color: var(--esp-dark); - padding: 8px 16px; -} - -.column-visibility-menu .dropdown-divider { - margin: 5px 0; -} - -.column-toggle .column-hidden { - width: 16px; - color: var(--esp-primary); -} - -.column-toggle .column-hidden { - color: var(--esp-gray); -} - -/* Active filters */ -#active-filters { - margin-top: 10px; - margin-bottom: 15px !important; - display: flex; - align-items: center; - background-color: #f8f9fa; - padding: 8px 12px; - border-radius: var(--border-radius); - box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); -} - -#filter-badges .badge { - margin-right: 5px; - padding: 6px 10px; - font-weight: normal; -} - -#clear-all-filters { - margin-left: auto; - color: var(--esp-primary); - text-decoration: none; - font-size: 0.9rem; - padding: 4px 8px; -} - -#clear-all-filters:hover { - text-decoration: underline; - background-color: rgba(232, 55, 17, 0.05); - border-radius: 4px; -} - -/* Tooltip actions */ -.tooltip-actions { - position: absolute; - right: 60px; - background-color: white; - border-radius: var(--border-radius); - box-shadow: var(--box-shadow); - padding: 10px; - width: 200px; - display: none; -} - -.tooltip-actions.show { - display: block; - animation: fadeIn var(--transition-speed); -} - -/* Smooth scrolling */ -html.scrolling-top { - scroll-behavior: smooth; -} - -/* ---------------------------------- */ -/* Responsive Styles */ -/* ---------------------------------- */ -@media (max-width: 1200px) { - .bootstrap-table .fixed-table-body { - overflow-x: auto; - max-height: 600px; - } - - .table-container { - width: 100% !important; - padding: 0 !important; - } -} - -@media (max-width: 768px) { - .container-fluid { - padding-left: 20px !important; - padding-right: 20px !important; - } - - .table-responsive { - overflow-x: auto; - } - - .bootstrap-table .fixed-table-body { - max-height: 500px; - } - - .table-controls { - flex-direction: column; - align-items: flex-start; - } - - .table-controls-right { - width: 100%; - justify-content: space-between; - } - - .logo-container, - .spacer { - width: 100px; - } -} - -/* ---------------------------------- */ -/* Bootstrap Table Sticky Header Specific Styles */ -/* ---------------------------------- */ - -/* Make sure sticky header container is visible and properly positioned */ -.bootstrap-table .sticky-header-container { - position: sticky; - top: 0; - z-index: 10; - background-color: var(--esp-light); -} - -/* Ensure the sticky header is properly styled and visible */ -.bootstrap-table .sticky-header { - overflow: visible; - position: relative; -} - -/* Styles for the headers in the sticky container */ -.bootstrap-table .sticky-header-container thead th { - background-color: var(--esp-light); - color: var(--esp-dark); - font-weight: 600; - padding: 12px 15px; - border-bottom: 2px solid rgba(0, 0, 0, 0.1); - text-align: left; - position: static !important; /* Important to avoid double sticky behavior */ -} - -/* Styles for header cells inner content */ -.bootstrap-table .sticky-header-container th .th-inner { - padding: 0 !important; - line-height: 1.4; - white-space: normal !important; - overflow: visible !important; - text-overflow: clip !important; - font-weight: inherit !important; -} diff --git a/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml b/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml index f986108c54b..1b6ed573982 100644 --- a/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml +++ b/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml @@ -16,7 +16,7 @@ all_build_finished: script: - echo "all test jobs finished" -generate_pytest_build_report: +generate_presigned_json: stage: assign_test image: $ESP_ENV_IMAGE tags: @@ -29,16 +29,11 @@ generate_pytest_build_report: job: pipeline_variables artifacts: paths: - - failed_apps.html - - built_apps.html - - skipped_apps.html - - build_report.html - presigned.json expire_in: 1 week when: always script: - - python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type build - - python tools/ci/previous_stage_job_status.py --stage build + - idf-ci gitlab generate-presigned-json --output presigned.json generate_pytest_child_pipeline: # finally, we can get some use out of the default behavior that downloads all artifacts from the previous stage diff --git a/tools/ci/dynamic_pipelines/utils.py b/tools/ci/dynamic_pipelines/utils.py deleted file mode 100644 index a4344ab3bc6..00000000000 --- a/tools/ci/dynamic_pipelines/utils.py +++ /dev/null @@ -1,151 +0,0 @@ -# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import os -from urllib.parse import urlparse - -import requests - -from .constants import CI_DASHBOARD_API -from .constants import CI_JOB_TOKEN -from .constants import CI_MERGE_REQUEST_SOURCE_BRANCH_SHA -from .constants import CI_PAGES_URL -from .constants import CI_PROJECT_URL -from .models import GitlabJob - - -def is_url(string: str) -> bool: - """ - Check if the string is a valid URL by parsing it and verifying if it contains both a scheme and a network location. - - :param string: The string to check if it is a URL. - :return: True if the string is a valid URL, False otherwise. - """ - parsed = urlparse(string) - return bool(parsed.scheme) and bool(parsed.netloc) - - -def fetch_failed_jobs(commit_id: str) -> list[GitlabJob]: - """ - Fetches a list of jobs from the specified commit_id using an API request to ci-dashboard-api. - :param commit_id: The commit ID for which to fetch jobs. - :return: A list of jobs if the request is successful, otherwise an empty list. - """ - response = requests.get( - f'{CI_DASHBOARD_API}/commits/{commit_id}/jobs', - headers={'CI-Job-Token': CI_JOB_TOKEN}, - ) - if response.status_code != 200: - print(f'Failed to fetch jobs data: {response.status_code} with error: {response.text}') - return [] - - data = response.json() - jobs = data.get('jobs', []) - - if not jobs: - return [] - - failed_job_names = [job['name'] for job in jobs if job['status'] == 'failed'] - response = requests.post( - f'{CI_DASHBOARD_API}/jobs/failure_ratio', - headers={'CI-Job-Token': CI_JOB_TOKEN}, - json={ - 'job_names': failed_job_names, - 'exclude_branches': [os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_NAME', '')], - }, - ) - if response.status_code != 200: - print(f'Failed to fetch jobs failure rate data: {response.status_code} with error: {response.text}') - return [] - - failure_rate_data = response.json() - failure_rates = {item['name']: item for item in failure_rate_data.get('jobs', [])} - - combined_jobs = [] - for job in jobs: - failure_data = failure_rates.get(job['name'], {}) - combined_jobs.append(GitlabJob.from_json_data(job, failure_data)) - - return combined_jobs - - -def fetch_app_metrics( - source_commit_sha: str, - target_commit_sha: str, -) -> dict: - """ - Fetches the app metrics for the given source commit SHA and target branch SHA. - :param source_commit_sha: The source commit SHA. - :param target_branch_sha: The commit SHA of the branch to compare app sizes against. - :return: A dict of sizes of built binaries. - """ - print(f'Fetching bin size info: {source_commit_sha=} {target_commit_sha=}') - build_info_map = dict() - response = requests.post( - f'{CI_DASHBOARD_API}/apps/metrics', - headers={'CI-Job-Token': CI_JOB_TOKEN}, - json={ - 'source_commit_sha': source_commit_sha, - 'target_commit_sha': target_commit_sha, - }, - ) - if response.status_code != 200: - print(f'Failed to fetch build info: {response.status_code} - {response.text}') - else: - response_data = response.json() - build_info_map = { - f'{info["app_path"]}_{info["config_name"]}_{info["target"]}': info for info in response_data.get('data', []) - } - - return build_info_map - - -def load_file(file_path: str) -> str: - """ - Loads the content of a file. - - :param file_path: The path to the file needs to be loaded. - :return: The content of the file as a string. - """ - with open(file_path) as file: - return file.read() - - -def format_permalink(s: str) -> str: - """ - Formats a given string into a permalink. - - :param s: The string to be formatted into a permalink. - :return: The formatted permalink as a string. - """ - end_index = s.find('(') - - if end_index != -1: - trimmed_string = s[:end_index].strip() - else: - trimmed_string = s.strip() - - formatted_string = trimmed_string.lower().replace(' ', '-') - - return formatted_string - - -def get_artifacts_url(job_id: int, output_filepath: str) -> str: - """ - Generates the url of the path where the artifact will be stored in the job's artifacts . - - :param job_id: The job identifier used to construct the URL. - :param output_filepath: The path to the output file. - :return: The modified URL pointing to the job's artifacts. - """ - url = CI_PAGES_URL.replace('esp-idf', '-/esp-idf') - return f'{url}/-/jobs/{job_id}/artifacts/{output_filepath}' - - -def get_repository_file_url(file_path: str) -> str: - """ - Generates the url of the file path inside the repository. - - :param file_path: The file path where the file is stored. - :return: The modified URL pointing to the file's path in the repository. - """ - return f'{CI_PROJECT_URL}/-/raw/{CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}/{file_path}' diff --git a/tools/ci/exclude_check_tools_files.txt b/tools/ci/exclude_check_tools_files.txt index 36f7b8a70d8..2499eb4220d 100644 --- a/tools/ci/exclude_check_tools_files.txt +++ b/tools/ci/exclude_check_tools_files.txt @@ -4,18 +4,15 @@ tools/ci/check_*.py tools/ci/check_*.txt tools/ci/check_*.sh tools/ci/check_copyright_config.yaml -tools/ci/get_all_test_results.py tools/gdb_panic_server.py tools/check_term.py tools/python_version_checker.py tools/ci/astyle-rules.yml tools/ci/checkout_project_ref.py tools/ci/ci_fetch_submodule.py -tools/ci/ci_get_mr_info.py tools/ci/ci_process_description.py tools/ci/configure_ci_environment.sh tools/ci/generate_rules.py -tools/ci/deploy_docs.py tools/ci/envsubst.py tools/ci/*exclude*.txt tools/ci/executable-list.txt @@ -51,7 +48,6 @@ tools/ci/python_packages/idf_iperf_test_util/**/* tools/esp_prov/**/* tools/ci/sort_yaml.py tools/ci/sg_rules/* -tools/ci/previous_stage_job_status.py tools/legacy_exports/export_legacy.fish tools/legacy_exports/export_legacy.sh tools/legacy_exports/export_legacy.ps1 diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index b185a767ae5..8301b288bda 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -72,7 +72,6 @@ tools/ci/check_tools_files_patterns.py tools/ci/check_type_comments.py tools/ci/checkout_project_ref.py tools/ci/cleanup_ignore_lists.py -tools/ci/deploy_docs.py tools/ci/envsubst.py tools/ci/fix_empty_prototypes.sh tools/ci/generate_rules.py diff --git a/tools/ci/generate_rules.py b/tools/ci/generate_rules.py index f9257eaf093..3ba31b9549d 100755 --- a/tools/ci/generate_rules.py +++ b/tools/ci/generate_rules.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2021-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import argparse import inspect @@ -12,7 +12,6 @@ from itertools import product import yaml from idf_ci_utils import IDF_PATH -from idf_ci_utils import GitlabYmlConfig if t.TYPE_CHECKING: import pygraphviz as pgv @@ -90,7 +89,6 @@ class RulesWriter: self.cfg = self.expand_matrices() self.rules = self.expand_rules() - self.yml_config = GitlabYmlConfig() self.graph = None def expand_matrices(self): # type: () -> dict @@ -195,10 +193,6 @@ class RulesWriter: if k.startswith('pattern'): continue - if '.rules:' + k not in self.yml_config.used_templates: - print(f'WARNING: unused rule: {k}, skipping...') - continue - res.append(self.RULES_TEMPLATE.format(k, self._format_rule(k, v))) return '\n\n'.join(res) diff --git a/tools/ci/get_all_test_results.py b/tools/ci/get_all_test_results.py deleted file mode 100644 index 7baa0832f2a..00000000000 --- a/tools/ci/get_all_test_results.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python -# -# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 - - -''' -Download artifacts from all test jobs - -CI_PROJECT_DIR -└── TEST_RESULTS - ├── - │ └── XUNIT_RESULT.xml - ├── - │ └── XUNIT_RESULT.xml - ├── - │ └── TEST_LOGS - -''' - -import argparse -import json -import os -import shutil - -import gitlab_api - -CI_PROJECT_ID = int(os.getenv('CI_PROJECT_ID', '')) -CI_PIPELINE_ID = int(os.getenv('CI_PIPELINE_ID', '')) -IDF_PATH = os.getenv('IDF_PATH', '.') -TEST_RESULT_PATH = os.path.join(IDF_PATH, 'TEST_RESULTS') - - -def download_all_test_results(result_path: str, include_retried: bool = False) -> None: - if os.path.exists(result_path): - shutil.rmtree(result_path) - os.makedirs(result_path, exist_ok=True) - - gitlab_inst = gitlab_api.Gitlab(CI_PROJECT_ID) - pipelines = gitlab_inst.project.pipelines.get(CI_PIPELINE_ID) - _include_retried = 'true' if include_retried else 'false' - jobs = pipelines.jobs.list(all=True, per_page=100, include_retried=_include_retried) - - job_info_list = [] - for job in jobs: - if job.stage in ['target_test', 'host_test']: - log_path = '' - if job.status not in ['success', 'failed']: - print('Job {}({}) is not finished'.format(job.id, job.name)) - elif not hasattr(job, 'artifacts_file'): - print('Job {}({}) has no artifacts.'.format(job.id, job.name)) - else: - log_path = os.path.join(result_path, 'job_{}'.format(job.id)) - print('Downloading artifacts from: {}'.format(job.name)) - os.makedirs(log_path, exist_ok=True) - gitlab_inst.download_artifacts(job.id, log_path) - - job_info = { - 'id': job.id, - 'name': job.name, - 'tag_list': job.tag_list, - 'status': job.status, - 'stage': job.stage, - 'web_url': job.web_url, - 'commit_url': job.commit['web_url'], - 'log_path': log_path, - } - job_info_list.append(job_info) - - with open(os.path.join(result_path, 'index.json'), 'w') as f: - f.write(json.dumps({'jobs': job_info_list}, indent=1, sort_keys=True)) - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument('--path', '-p', default=TEST_RESULT_PATH, help='Path to save result files.') - parser.add_argument('--include_retried', action='store_true', help='Including retried jobs.') - args = parser.parse_args() - - download_all_test_results(args.path, args.include_retried) - - -if __name__ == '__main__': - main() diff --git a/tools/ci/gitlab_yaml_linter.py b/tools/ci/gitlab_yaml_linter.py index baac4ee2e37..3ab20353ecb 100755 --- a/tools/ci/gitlab_yaml_linter.py +++ b/tools/ci/gitlab_yaml_linter.py @@ -1,17 +1,18 @@ #!/usr/bin/env python -# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 """ Check gitlab ci yaml files """ + import argparse import os import typing as t from functools import cached_property -from idf_ci_utils import get_submodule_dirs -from idf_ci_utils import GitlabYmlConfig from idf_ci_utils import IDF_PATH +from idf_ci_utils import GitlabYmlConfig +from idf_ci_utils import get_submodule_dirs class YmlLinter: @@ -43,17 +44,6 @@ class YmlLinter: exit(exit_code) - # name it like _1_ to make it run first - def _lint_1_yml_parser(self) -> None: - for k, v in self.yml_config.config.items(): - if ( - k not in self.yml_config.global_keys - and k not in self.yml_config.anchors - and k not in self.yml_config.templates - and k not in self.yml_config.jobs - ): - raise SystemExit(f'Parser incorrect. Key {k} not in global keys, anchors, templates, or jobs') - def _lint_default_values_artifacts(self) -> None: defaults_artifacts = self.yml_config.default.get('artifacts', {}) @@ -79,19 +69,6 @@ class YmlLinter: for item in undefined_patterns: self._errors.append(f'undefined pattern {item}. Please add {item} to .patterns-submodule') - def _lint_gitlab_yml_templates(self) -> None: - unused_templates = self.yml_config.templates.keys() - self.yml_config.used_templates - for item in unused_templates: - # known unused ones - if item not in [ - '.before_script:fetch:target_test', # used in dynamic pipeline - ]: - self._errors.append(f'Unused template: {item}, please remove it') - - undefined_templates = self.yml_config.used_templates - self.yml_config.templates.keys() - for item in undefined_templates: - self._errors.append(f'Undefined template: {item}') - def _lint_dependencies_and_needs(self) -> None: """ Use `dependencies: []` together with `needs: []` could cause missing artifacts issue. diff --git a/tools/ci/idf_ci_local/app.py b/tools/ci/idf_ci_local/app.py index 4d1f177e59a..c3e464af3ec 100644 --- a/tools/ci/idf_ci_local/app.py +++ b/tools/ci/idf_ci_local/app.py @@ -4,17 +4,15 @@ import os import subprocess import sys import typing as t +from pathlib import Path -from dynamic_pipelines.constants import BINARY_SIZE_METRIC_NAME -from idf_build_apps import App from idf_build_apps import CMakeApp from idf_build_apps.constants import BuildStatus from idf_build_apps.utils import rmdir from idf_ci_utils import APP_EXTRA_S3_ARTIFACT_TYPE -from idf_ci_utils import idf_relpath -if t.TYPE_CHECKING: - pass + +_SIZE_METRICS_CONFIG_PATH = Path(__file__).parent.parent / 'metrics' / 'size_metrics' / 'size_metrics_config.yml' class IdfCMakeApp(CMakeApp): @@ -37,6 +35,8 @@ class IdfCMakeApp(CMakeApp): 'gitlab', 'upload-artifacts', self.app_dir, + '--build-dir', + self.build_dir, ], [ 'idf-ci', @@ -63,103 +63,3 @@ class IdfCMakeApp(CMakeApp): self.build_path, exclude_file_patterns=['build_log.txt', 'size*.json'], ) - - -class Metrics: - """ - Represents a metric and its values for source, target, and the differences. - """ - - def __init__( - self, - source_value: t.Optional[float] = None, - target_value: t.Optional[float] = None, - difference: t.Optional[float] = None, - difference_percentage: t.Optional[float] = None, - ) -> None: - self.source_value = source_value or 0.0 - self.target_value = target_value or 0.0 - self.difference = difference or 0.0 - self.difference_percentage = difference_percentage or 0.0 - - def to_dict(self) -> t.Dict[str, t.Any]: - """ - Converts the Metrics object to a dictionary. - """ - return { - 'source_value': self.source_value, - 'target_value': self.target_value, - 'difference': self.difference, - 'difference_percentage': self.difference_percentage, - } - - -class AppWithMetricsInfo(IdfCMakeApp): - metrics: t.Dict[str, Metrics] - is_new_app: bool - - def __init__(self, **kwargs: t.Any) -> None: - super().__init__(**kwargs) - - self.metrics = {metric_name: metric_data for metric_name, metric_data in kwargs.get('metrics', {}).items()} - self.is_new_app = kwargs.get('is_new_app', False) - - class Config: - arbitrary_types_allowed = True - - -def enrich_apps_with_metrics_info( - app_metrics_info_map: t.Dict[str, t.Dict[str, t.Any]], apps: t.List[App] -) -> t.List[AppWithMetricsInfo]: - def _get_full_attributes(obj: App) -> t.Dict[str, t.Any]: - """ - Retrieves all attributes of an object, including properties and computed fields. - """ - attributes: t.Dict[str, t.Any] = obj.__dict__.copy() - for attr in dir(obj): - if not attr.startswith('_'): # Skip private/internal attributes - try: - value = getattr(obj, attr) - # Include only if it's not already in __dict__ - if attr not in attributes: - attributes[attr] = value - except Exception: - # Skip attributes that raise exceptions (e.g., methods needing args) - pass - return attributes - - default_metrics_structure = { - BINARY_SIZE_METRIC_NAME: Metrics( - source_value=0, - target_value=0, - difference=0, - difference_percentage=0.0, - ), - } - - apps_with_metrics_info = [] - for app in apps: - app.app_dir = idf_relpath(app.app_dir) - key = f'{app.app_dir}_{app.config_name}_{app.target}' - app_attributes = _get_full_attributes(app) - - metrics = {metric_name: default_metric for metric_name, default_metric in default_metrics_structure.items()} - is_new_app = False - - if key in app_metrics_info_map: - info = app_metrics_info_map[key] - for metric_name, metric_data in info.get('metrics', {}).items(): - metrics[metric_name] = Metrics( - source_value=metric_data.get('source_value', 0), - target_value=metric_data.get('target_value', 0), - difference=metric_data.get('difference', 0), - difference_percentage=metric_data.get('difference_percentage', 0.0), - ) - - is_new_app = info.get('is_new_app', False) - - app_attributes.update({'metrics': metrics, 'is_new_app': is_new_app}) - - apps_with_metrics_info.append(AppWithMetricsInfo(**app_attributes)) - - return apps_with_metrics_info diff --git a/tools/ci/idf_ci_utils.py b/tools/ci/idf_ci_utils.py index 13bfab1b12f..876d6a79a38 100644 --- a/tools/ci/idf_ci_utils.py +++ b/tools/ci/idf_ci_utils.py @@ -1,10 +1,9 @@ -# SPDX-FileCopyrightText: 2020-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2020-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # internal use only for CI # some CI related util functions import logging import os -import re import subprocess import sys import typing as t @@ -126,35 +125,106 @@ class GitlabYmlConfig: # avoid unused import in other pre-commit hooks import yaml - all_config = dict() - root_yml = yaml.load(open(root_yml_filepath), Loader=yaml.FullLoader) - - # expanding "include" - for item in root_yml.pop('include', []) or []: - if isinstance(item, dict): - if 'project' in item: - continue - elif 'local' in item: - item = item['local'] - else: - continue - - all_config.update(yaml.load(open(os.path.join(IDF_PATH, item)), Loader=yaml.FullLoader)) + merged_yaml = self._compile_via_gitlab_api(root_yml_filepath) + all_config = yaml.load(merged_yaml, Loader=yaml.FullLoader) or {} if 'default' in all_config: self._defaults = all_config.pop('default') self._config = all_config - # anchor is the string that will be reused in templates - self._anchor_keys: t.Set[str] = set() - # template is a dict that will be extended - self._template_keys: t.Set[str] = set() - self._used_template_keys: t.Set[str] = set() # tracing the used templates - # job is a dict that will be executed - self._job_keys: t.Set[str] = set() + def _inline_local_includes(self, root_yml_filepath: str) -> str: + """ + Recursively resolve `include: local` entries straight from disk (so uncommitted local + changes are always picked up -- CI runners also work off a disk checkout, so there's no + need to fetch a ref remotely via `ci_lint`'s `content_ref`/`ref`/`dry_run_ref` GET + params). `include: project` entries (including ones nested inside local files, e.g. + `.gitlab/ci/common.yml` including `templates/idf/common-scripts.yml`) are collected and + left in the final `include:` list, since those files live in another GitLab project and + can only be resolved remotely. - self.expand_extends() + A dedicated Loader/Dumper pair round-trips `!reference` tags as a marker list subclass, + since GitLab CI uses `!reference` which plain YAML doesn't know, and we need to parse + (to merge dicts, not just string-concat) then re-dump losslessly. + + :param root_yml_filepath: path to the local root yml file to compile + :return: yml content with local includes resolved and merged + """ + import yaml + + class _Loader(yaml.FullLoader): + pass + + class _Dumper(yaml.Dumper): + pass + + class _Reference(list): + pass + + _Loader.add_constructor( + '!reference', lambda loader, node: _Reference(loader.construct_sequence(t.cast(yaml.SequenceNode, node))) + ) + _Dumper.add_representer(_Reference, lambda dumper, data: dumper.represent_sequence('!reference', list(data))) + + def resolve(yml_filepath: str) -> t.Tuple[dict, list]: + with open(yml_filepath) as fr: + data = yaml.load(fr, Loader=_Loader) or {} + + includes = to_list(data.pop('include', None)) + + merged: dict = {} + remaining_project_includes: list = [] + for item in includes: + if isinstance(item, dict): + if 'project' in item: + remaining_project_includes.append(item) + continue + elif 'local' in item: + local_path = item['local'] + else: + continue + elif isinstance(item, str): + local_path = item + else: + continue + + sub_merged, sub_remaining = resolve(os.path.join(IDF_PATH, local_path.lstrip('/'))) + merged.update(sub_merged) + remaining_project_includes.extend(sub_remaining) + + # this file's own top-level keys override whatever its includes defined + merged.update(data) + return merged, remaining_project_includes + + merged_config, project_includes = resolve(root_yml_filepath) + if project_includes: + merged_config['include'] = project_includes + + return yaml.dump(merged_config, Dumper=_Dumper, sort_keys=False) # type: ignore + + def _compile_via_gitlab_api(self, root_yml_filepath: str) -> str: + """ + Call the GitLab CI Lint API to get the fully compiled (all `include`s resolved) yml, + same as what `glab ci config compile` does. This replaces the old recursive local-only + parsing, since the project now includes configs from other projects as well. + + :param root_yml_filepath: path to the local root yml file to compile + :return: merged (fully resolved) yml content as a string + """ + sys.path.insert(0, os.path.join(IDF_PATH, 'tools', 'ci', 'python_packages')) + import gitlab_api + + content = self._inline_local_includes(root_yml_filepath) + + gitlab_inst = gitlab_api.Gitlab() + project_id = os.getenv('CI_PROJECT_ID') or gitlab_inst.get_project_id('esp-idf', namespace='espressif') + project = gitlab_inst.gitlab_inst.projects.get(project_id, lazy=True) + + lint_result = project.ci_lint.create({'content': content}) + if not lint_result.valid: + raise RuntimeError(f'Failed to compile {root_yml_filepath} via GitLab CI Lint API: {lint_result.errors}') + + return lint_result.merged_yaml # type: ignore @property def default(self) -> t.Dict[str, t.Any]: @@ -168,80 +238,9 @@ class GitlabYmlConfig: def global_keys(self) -> t.List[str]: return ['default', 'include', 'workflow', 'variables', 'stages'] - @cached_property - def anchors(self) -> t.Dict[str, t.Any]: - return {k: v for k, v in self.config.items() if k in self._anchor_keys} - @cached_property def jobs(self) -> t.Dict[str, t.Any]: - return {k: v for k, v in self.config.items() if k in self._job_keys} - - @cached_property - def templates(self) -> t.Dict[str, t.Any]: - return {k: v for k, v in self.config.items() if k in self._template_keys} - - @cached_property - def used_templates(self) -> t.Set[str]: - return self._used_template_keys - - def expand_extends(self) -> None: - """ - expand the `extends` key in-place. - """ - for k, v in self.config.items(): - if k in self.global_keys: - continue - - if isinstance(v, (str, list)): - self._anchor_keys.add(k) - elif k.startswith('.if-'): - self._anchor_keys.add(k) - elif k.startswith('.'): - self._template_keys.add(k) - elif isinstance(v, dict): - self._job_keys.add(k) - else: - raise ValueError(f'Unknown type for key {k} with value {v}') - - # no need to expand anchor - - # expand template first - for k in self._template_keys: - self._expand_extends(k) - - # expand job - for k in self._job_keys: - self._expand_extends(k) - - def _merge_dict(self, d1: t.Dict[str, t.Any], d2: t.Dict[str, t.Any]) -> t.Any: - for k, v in d2.items(): - if k in d1: - if isinstance(v, dict) and isinstance(d1[k], dict): - d1[k] = self._merge_dict(d1[k], v) - else: - d1[k] = v - else: - d1[k] = v - - return d1 - - def _expand_extends(self, name: str) -> t.Dict[str, t.Any]: - extends = to_list(self.config[name].pop('extends', None)) - if not extends: - return self.config[name] # type: ignore - - original_d = self.config[name].copy() - d = {} - while extends: - self._used_template_keys.update(extends) # for tracking - - for i in extends: - d.update(self._expand_extends(i)) - - extends = to_list(self.config[name].pop('extends', None)) - - self.config[name] = self._merge_dict(d, original_d) - return self.config[name] # type: ignore + return {k: v for k, v in self.config.items() if not k.startswith('.') and k not in self.global_keys} def get_all_manifest_files() -> t.List[str]: @@ -256,23 +255,6 @@ def get_all_manifest_files() -> t.List[str]: return paths -def sanitize_job_name(name: str) -> str: - """ - Sanitize the job name from CI_JOB_NAME - - - for job with `parallel: int` set, the `CI_JOB_NAME` would be `job_name index/total`, like `foo 1/3` - - for job with `parallel: matrix` set, the `CI_JOB_NAME` would be `job_name: [var1, var2]`, like `foo: [a, b]` - - We consider - - the jobs generated by `parallel: int` as the same job, i.e., we remove the index/total part. - - the jobs generated by `parallel: matrix` as different jobs, so we keep the matrix part. - - :param name: job name - :return: sanitized job name - """ - return re.sub(r' \d+/\d+', '', name) - - def idf_relpath(p: str) -> str: """ Turn all paths under IDF_PATH to relative paths diff --git a/tools/ci/idf_pytest/plugin.py b/tools/ci/idf_pytest/plugin.py index d8def228fdf..c3c0e755093 100644 --- a/tools/ci/idf_pytest/plugin.py +++ b/tools/ci/idf_pytest/plugin.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import os import typing as t diff --git a/tools/ci/previous_stage_job_status.py b/tools/ci/previous_stage_job_status.py deleted file mode 100644 index 3888780d407..00000000000 --- a/tools/ci/previous_stage_job_status.py +++ /dev/null @@ -1,36 +0,0 @@ -# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import argparse -import os -import sys - -import requests - -if __name__ == '__main__': - - parser = argparse.ArgumentParser() - parser.add_argument('--stage', type=str, help='Stage name for check jobs status') - args = parser.parse_args() - - GITLAB_TOKEN = os.getenv('ESPCI_TOKEN') - GITLAB_HTTP_SERVER = os.getenv('GITLAB_HTTP_SERVER') - CI_PROJECT_ID = os.getenv('CI_PROJECT_ID') - CI_PIPELINE_ID = os.getenv('CI_PIPELINE_ID') - - api_path = f'projects/{CI_PROJECT_ID}/pipelines/{CI_PIPELINE_ID}/jobs?scope[]=failed&per_page=100' - - page = 0 - while True: - response = requests.get( - f'{GITLAB_HTTP_SERVER}/api/v4/{api_path}&page={page}', - headers={'PRIVATE-TOKEN': GITLAB_TOKEN} - ) - jobs = response.json() - if not jobs: - break - - for job in jobs: - if job['stage'] == args.stage: - print(f'Jobs from the previous stage {args.stage} should pass; otherwise, this job will fail.') - sys.exit(1) - page += 1 diff --git a/tools/ci/python_packages/gitlab_api.py b/tools/ci/python_packages/gitlab_api.py index 6ac0ffae1f8..f78886e6ddf 100644 --- a/tools/ci/python_packages/gitlab_api.py +++ b/tools/ci/python_packages/gitlab_api.py @@ -1,6 +1,7 @@ -# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import argparse +import copy import logging import os import re @@ -52,9 +53,11 @@ def retry(func: TR) -> TR: raise e # get out of the loop else: logging.warning( - 'Network failure in {}, retrying ({})'.format(getattr(func, '__name__', '(unknown callable)'), - retried)) - time.sleep(2 ** retried) # wait a bit more after each retry + 'Network failure in {}, retrying ({})'.format( + getattr(func, '__name__', '(unknown callable)'), retried + ) + ) + time.sleep(2**retried) # wait a bit more after each retry continue else: break @@ -67,7 +70,6 @@ class Gitlab(object): JOB_NAME_PATTERN = re.compile(r'(\w+)(\s+(\d+)/(\d+))?') DOWNLOAD_ERROR_MAX_RETRIES = 3 - DEFAULT_BUILD_CHILD_PIPELINE_NAME = 'Build Child Pipeline' def __init__(self, project_id: Union[int, str, None] = None): config_data_from_env = os.getenv('PYTHON_GITLAB_CONFIG') @@ -117,7 +119,7 @@ class Gitlab(object): :param namespace: namespace to match when we have multiple project with same name :return: project ID """ - projects = self.gitlab_inst.projects.list(search=name) + projects = self.gitlab_inst.projects.list(search=name, get_all=True) res = [] for project in projects: if namespace is None: @@ -152,7 +154,9 @@ class Gitlab(object): archive_file.extractall(destination) @retry - def download_artifact(self, job_id: int, artifact_path: List[str], destination: Optional[str] = None) -> List[bytes]: + def download_artifact( + self, job_id: int, artifact_path: List[str], destination: Optional[str] = None + ) -> List[bytes]: """ download specific path of job artifacts and extract to destination. @@ -208,8 +212,9 @@ class Gitlab(object): return job_id_list @retry - def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None, - cache_dir: Optional[str] = None) -> str: + def download_archive( + self, ref: str, destination: str, project_id: Optional[int] = None, cache_dir: Optional[str] = None + ) -> str: """ Download archive of certain commit of a repository and extract to destination path @@ -235,8 +240,11 @@ class Gitlab(object): except gitlab.GitlabGetError as e: logging.error('Failed to archive from project {}'.format(project_id)) raise e - logging.info('Downloaded archive size: {:.03f}MB'.format( - float(os.path.getsize(local_archive_file)) / (1024 * 1024))) + logging.info( + 'Downloaded archive size: {:.03f}MB'.format( + float(os.path.getsize(local_archive_file)) / (1024 * 1024) + ) + ) return self.decompress_archive(local_archive_file, destination) @@ -248,22 +256,38 @@ class Gitlab(object): logging.error('Failed to archive from project {}'.format(project_id)) raise e - logging.info('Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(temp_file.name)) / (1024 * 1024))) + logging.info( + 'Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(temp_file.name)) / (1024 * 1024)) + ) return self.decompress_archive(temp_file.name, destination) + @staticmethod + def _to_win32_long_path(path: str) -> str: + normalized_path = os.path.normpath(os.path.abspath(path)) + if normalized_path.startswith('\\\\?\\'): + return normalized_path + if normalized_path.startswith('\\\\'): + return '\\\\?\\UNC\\' + normalized_path[2:] + return '\\\\?\\' + normalized_path + @staticmethod def decompress_archive(path: str, destination: str) -> str: full_destination = os.path.abspath(destination) - # By default max path length is set to 260 characters - # Prefix `\\?\` extends it to 32,767 characters - if sys.platform == 'win32': - full_destination = '\\\\?\\' + full_destination try: with tarfile.open(path, 'r') as archive_file: - root_name = archive_file.getnames()[0] - archive_file.extractall(full_destination) + members = archive_file.getmembers() + root_name = members[0].name + if sys.platform == 'win32': + # tarfile keeps archive member names in POSIX form. Normalize them before + # combining with a long-path-prefixed destination to avoid invalid mixed separators. + full_destination = Gitlab._to_win32_long_path(full_destination) + members = [copy.copy(member) for member in members] + for member in members: + member.name = member.name.replace('/', '\\') + member.linkname = member.linkname.replace('/', '\\') + archive_file.extractall(full_destination, members=members) except tarfile.TarError as e: logging.error(f'Error while decompressing archive {path}') raise e @@ -282,36 +306,28 @@ class Gitlab(object): def get_downstream_pipeline_ids(self, main_pipeline_id: int) -> List[int]: """ - Retrieve the IDs of all downstream child pipelines for a given main pipeline. + Retrieve the IDs of all downstream child pipelines for a given main pipeline, + recursing through arbitrarily nested child pipelines. :param main_pipeline_id: The ID of the main pipeline to start the search. - :return: A list of IDs of all downstream child pipelines. + :return: A list of IDs of all downstream child pipelines (all levels). """ - bridge_pipeline_ids = [] - child_pipeline_ids = [] + child_pipeline_ids: List[int] = [] - main_pipeline_bridges = self.project.pipelines.get(main_pipeline_id).bridges.list() - for bridge in main_pipeline_bridges: + pipeline_bridges = self.project.pipelines.get(main_pipeline_id).bridges.list() + for bridge in pipeline_bridges: downstream_pipeline = bridge.attributes.get('downstream_pipeline') if not downstream_pipeline: continue - bridge_pipeline_ids.append(downstream_pipeline['id']) - - for bridge_pipeline_id in bridge_pipeline_ids: - child_pipeline_ids.append(bridge_pipeline_id) - bridge_pipeline = self.project.pipelines.get(bridge_pipeline_id) - - if not bridge_pipeline.name == self.DEFAULT_BUILD_CHILD_PIPELINE_NAME: + downstream_pipeline_id = downstream_pipeline.get('id') + if downstream_pipeline_id is None: continue - child_bridges = bridge_pipeline.bridges.list() - for child_bridge in child_bridges: - downstream_child_pipeline = child_bridge.attributes.get('downstream_pipeline') - if not downstream_child_pipeline: - continue - child_pipeline_ids.append(downstream_child_pipeline.get('id')) + child_pipeline_ids.append(downstream_pipeline_id) + # recurse to collect further nested (grandchild+) pipelines + child_pipeline_ids.extend(self.get_downstream_pipeline_ids(downstream_pipeline_id)) - return [pid for pid in child_pipeline_ids if pid is not None] + return child_pipeline_ids def retry_failed_jobs(self, pipeline_id: int, retry_allowed_failures: bool = False) -> List[int]: """ diff --git a/tools/ci/utils.sh b/tools/ci/utils.sh index a4c3a4875b3..9fcea7fd53a 100644 --- a/tools/ci/utils.sh +++ b/tools/ci/utils.sh @@ -164,38 +164,3 @@ function join_by { printf %s "$f" "${@/#/$d}" fi } - -function is_based_on_commits() { - # This function would accept space-separated args as multiple commits. - # The return value would be 0 if current HEAD is based on any of the specified commits. - # - # In our CI, we use environment variable $REQUIRED_ANCESTOR_COMMITS to declare the ancestor commits. - # Please remember to set one commit for each release branch. - - commits=$* - if [[ -z $commits ]]; then - info "Not specifying commits that branches should be based on, skipping check..." - return 0 - fi - - commits_str="$(join_by " or " $commits)" # no doublequotes here, passing array - - info "Checking if current branch is based on $commits_str..." - for i in $commits; do - if git merge-base --is-ancestor "$i" HEAD >/dev/null 2>&1; then - info "Current branch is based on $i" - return 0 - else - info "Current branch is not based on $i" - fi - done - - error "The base commit of your branch is too old." - error "The branch should be more recent than either of the following commits:" - error " $commits_str" - error "To fix the issue:" - error " - If your merge request is 'Draft', or has conflicts with the target branch, rebase it to the latest master or release branch" - error " - Otherwise, simply run a new pipeline." - - return 1 -} diff --git a/tools/cmake/build.cmake b/tools/cmake/build.cmake index c323d9eac1d..2dab01822b7 100644 --- a/tools/cmake/build.cmake +++ b/tools/cmake/build.cmake @@ -661,6 +661,21 @@ macro(idf_build_process target) endif() endif() + idf_build_get_property(prefix __PREFIX) + + file(GLOB root_dep_component_dirs + ${IDF_TOOLS_PATH}/root_managed_components/idf${IDF_VERSION_MAJOR}.${IDF_VERSION_MINOR}.${IDF_VERSION_PATCH}/*) + list(SORT root_dep_component_dirs) + foreach(component_dir ${root_dep_component_dirs}) + # A potential component must be a directory + if(IS_DIRECTORY ${component_dir}) + __component_dir_quick_check(is_component ${component_dir}) + if(is_component) + __component_add(${component_dir} ${prefix} "idf_managed_components") + endif() + endif() + endforeach() + # Perform early expansion of component CMakeLists.txt in CMake scripting mode. # It is here we retrieve the public and private requirements of each component. # It is also here we add the common component requirements to each component's diff --git a/tools/cmake/component.cmake b/tools/cmake/component.cmake index 0fdb105ee50..f1e35454f8f 100644 --- a/tools/cmake/component.cmake +++ b/tools/cmake/component.cmake @@ -97,9 +97,9 @@ endmacro() # function(__component_dir_quick_check var component_dir) set(res 1) - get_filename_component(abs_dir ${component_dir} ABSOLUTE) + get_filename_component(abs_dir "${component_dir}" ABSOLUTE) - get_filename_component(base_dir ${abs_dir} NAME) + get_filename_component(base_dir "${abs_dir}" NAME) string(SUBSTRING "${base_dir}" 0 1 first_char) # Check the component directory contains a CMakeLists.txt file @@ -149,8 +149,8 @@ function(__component_add component_dir prefix component_source) # so later in the build, these component targets actually contain the properties meant for the # corresponding component library. idf_build_get_property(component_targets __COMPONENT_TARGETS) - get_filename_component(abs_dir ${component_dir} ABSOLUTE) - get_filename_component(base_dir ${abs_dir} NAME) + get_filename_component(abs_dir "${component_dir}" ABSOLUTE) + get_filename_component(base_dir "${abs_dir}" NAME) if(NOT EXISTS "${abs_dir}/CMakeLists.txt") message(FATAL_ERROR "Directory '${component_dir}' does not contain a component.") @@ -277,15 +277,15 @@ macro(__component_add_sources sources) message(WARNING "SRCS and SRC_DIRS are both specified; ignoring SRC_DIRS.") endif() foreach(src ${__SRCS}) - get_filename_component(src "${src}" ABSOLUTE BASE_DIR ${COMPONENT_DIR}) - list(APPEND sources ${src}) + get_filename_component(src "${src}" ABSOLUTE BASE_DIR "${COMPONENT_DIR}") + list(APPEND sources "${src}") endforeach() else() if(__SRC_DIRS) foreach(dir ${__SRC_DIRS}) - get_filename_component(abs_dir ${dir} ABSOLUTE BASE_DIR ${COMPONENT_DIR}) + get_filename_component(abs_dir "${dir}" ABSOLUTE BASE_DIR "${COMPONENT_DIR}") - if(NOT IS_DIRECTORY ${abs_dir}) + if(NOT IS_DIRECTORY "${abs_dir}") message(FATAL_ERROR "SRC_DIRS entry '${dir}' does not exist.") endif() @@ -294,7 +294,7 @@ macro(__component_add_sources sources) if(dir_sources) foreach(src ${dir_sources}) - get_filename_component(src "${src}" ABSOLUTE BASE_DIR ${COMPONENT_DIR}) + get_filename_component(src "${src}" ABSOLUTE BASE_DIR "${COMPONENT_DIR}") list(APPEND sources "${src}") endforeach() else() @@ -316,11 +316,11 @@ endmacro() macro(__component_add_include_dirs lib dirs type) foreach(dir ${dirs}) - get_filename_component(_dir ${dir} ABSOLUTE BASE_DIR ${CMAKE_CURRENT_LIST_DIR}) - if(NOT IS_DIRECTORY ${_dir}) + get_filename_component(_dir "${dir}" ABSOLUTE BASE_DIR "${CMAKE_CURRENT_LIST_DIR}") + if(NOT IS_DIRECTORY "${_dir}") message(FATAL_ERROR "Include directory '${_dir}' is not a directory.") endif() - target_include_directories(${lib} ${type} ${_dir}) + target_include_directories(${lib} ${type} "${_dir}") endforeach() endmacro() diff --git a/tools/cmake/idf.cmake b/tools/cmake/idf.cmake index 384401cd76c..06600490799 100644 --- a/tools/cmake/idf.cmake +++ b/tools/cmake/idf.cmake @@ -52,6 +52,13 @@ if(NOT __idf_env_set) include(prefix_map) include(openocd) + # ESP-IDF extra dependencies defined in tools/idf_extra_components.yml + if(WIN32) + set_default(IDF_TOOLS_PATH "$ENV{USERPROFILE}/.espressif") + else() + set_default(IDF_TOOLS_PATH "$ENV{HOME}/.espressif") + endif() + __build_init("${idf_path}") # Check if IDF_ENV_FPGA environment is set diff --git a/tools/cmake/scripts/component_get_requirements.cmake b/tools/cmake/scripts/component_get_requirements.cmake index bc94112e52e..f4f4079c13b 100644 --- a/tools/cmake/scripts/component_get_requirements.cmake +++ b/tools/cmake/scripts/component_get_requirements.cmake @@ -131,6 +131,8 @@ foreach(__component_target ${__component_targets}) if("${__component_source}" STREQUAL "idf_components") list(APPEND __TARGETS_IDF_COMPONENTS ${__component_target}) + elseif("${__component_source}" STREQUAL "idf_managed_components") + list(APPEND __TARGETS_IDF_MANAGED_COMPONENTS ${__component_target}) elseif("${__component_source}" STREQUAL "project_managed_components") list(APPEND __TARGETS_PROJECT_MANAGED_COMPONENTS ${__component_target}) elseif("${__component_source}" STREQUAL "project_extra_components") @@ -147,6 +149,7 @@ set(__sorted_component_targets "") foreach(__target IN LISTS __TARGETS_PROJECT_COMPONENTS __TARGETS_PROJECT_EXTRA_COMPONENTS __TARGETS_PROJECT_MANAGED_COMPONENTS + __TARGETS_IDF_MANAGED_COMPONENTS __TARGETS_IDF_COMPONENTS) __component_get_property(__component_name ${__target} COMPONENT_NAME) list(APPEND __sorted_component_targets ${__target}) diff --git a/tools/idf_extra_components.yml b/tools/idf_extra_components.yml new file mode 100644 index 00000000000..c851252cb46 --- /dev/null +++ b/tools/idf_extra_components.yml @@ -0,0 +1,11 @@ +# This file defines extra dependencies for ESP-IDF +# the dependencies defined here will be downloaded to +# $IDF_TOOLS_PATH/root_managed_components +# Each major.minor version of ESP-IDF can have its own subdirectory +# For example, for ESP-IDF v6.0, the dependencies will be installed to +# $IDF_TOOLS_PATH/root_managed_components/idf6.0 + +# The syntax is defined in: +# https://docs.espressif.com/projects/idf-component-manager/en/latest/reference/manifest_file.html#dependencies + +#dependencies: diff --git a/tools/requirements/requirements.ci.txt b/tools/requirements/requirements.ci.txt index 9852754ad66..846ca3a4866 100644 --- a/tools/requirements/requirements.ci.txt +++ b/tools/requirements/requirements.ci.txt @@ -6,7 +6,7 @@ # https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/tools/idf-tools.html # ci -idf-ci>=0.3,<1 +idf-ci coverage jsonschema diff --git a/tools/test_apps/security/secure_boot/pytest_secure_boot.py b/tools/test_apps/security/secure_boot/pytest_secure_boot.py index e767ee71171..57e96718575 100644 --- a/tools/test_apps/security/secure_boot/pytest_secure_boot.py +++ b/tools/test_apps/security/secure_boot/pytest_secure_boot.py @@ -166,7 +166,6 @@ def test_examples_security_secure_boot_ecdsa(dut: Dut) -> None: # Test secure boot flow. # Correctly signed bootloader + correctly signed app should work -@pytest.mark.host_test @pytest.mark.qemu @pytest.mark.parametrize( 'qemu_extra_args', diff --git a/tools/test_apps/security/signed_app_no_secure_boot/pytest_signed_app_no_secure_boot.py b/tools/test_apps/security/signed_app_no_secure_boot/pytest_signed_app_no_secure_boot.py index 9dde2e391a5..101f971a65d 100644 --- a/tools/test_apps/security/signed_app_no_secure_boot/pytest_signed_app_no_secure_boot.py +++ b/tools/test_apps/security/signed_app_no_secure_boot/pytest_signed_app_no_secure_boot.py @@ -6,7 +6,6 @@ from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -@pytest.mark.host_test @pytest.mark.qemu @pytest.mark.parametrize('config', ['secure_update_with_fe'], indirect=True) @idf_parametrize('target', ['esp32c3'], indirect=['target']) diff --git a/tools/test_apps/storage/std_filesystem/pytest_std_filesystem.py b/tools/test_apps/storage/std_filesystem/pytest_std_filesystem.py index 75676116044..86031bc508f 100644 --- a/tools/test_apps/storage/std_filesystem/pytest_std_filesystem.py +++ b/tools/test_apps/storage/std_filesystem/pytest_std_filesystem.py @@ -6,7 +6,6 @@ from pytest_embedded_idf.utils import idf_parametrize @pytest.mark.qemu -@pytest.mark.host_test @idf_parametrize('target', ['esp32', 'esp32c3'], indirect=['target']) def test_std_filesystem(dut: Dut) -> None: dut.expect_exact('All tests passed', timeout=200) diff --git a/tools/test_apps/system/esp_intr_dump/pytest_esp_intr_dump.py b/tools/test_apps/system/esp_intr_dump/pytest_esp_intr_dump.py index cccfdbedeb9..e03da57cc20 100644 --- a/tools/test_apps/system/esp_intr_dump/pytest_esp_intr_dump.py +++ b/tools/test_apps/system/esp_intr_dump/pytest_esp_intr_dump.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 import os @@ -10,7 +10,6 @@ PROMPT = 'test_intr_dump>' @pytest.mark.qemu -@pytest.mark.host_test @idf_parametrize('target', ['esp32'], indirect=['target']) def test_esp_intr_dump_nonshared(dut: Dut) -> None: dut.expect_exact(PROMPT, timeout=30) @@ -24,7 +23,6 @@ def test_esp_intr_dump_nonshared(dut: Dut) -> None: @pytest.mark.qemu -@pytest.mark.host_test @idf_parametrize('target', ['esp32'], indirect=['target']) def test_esp_intr_dump_shared(dut: Dut) -> None: dut.expect_exact(PROMPT, timeout=30) @@ -54,5 +52,5 @@ def test_esp_intr_dump_expected_output(dut: Dut) -> None: dut.expect_exact(PROMPT, timeout=30) dut.write('intr_dump\n') exp_out_file = os.path.join(os.path.dirname(__file__), 'expected_output', f'{dut.target}.txt') - for line in open(exp_out_file, 'r').readlines(): + for line in open(exp_out_file).readlines(): dut.expect_exact(line.strip()) diff --git a/tools/test_apps/system/mmu_page_size/pytest_mmu_page_size.py b/tools/test_apps/system/mmu_page_size/pytest_mmu_page_size.py index 4213f2fec0f..88492e046f2 100644 --- a/tools/test_apps/system/mmu_page_size/pytest_mmu_page_size.py +++ b/tools/test_apps/system/mmu_page_size/pytest_mmu_page_size.py @@ -3,7 +3,6 @@ import os import pytest -from idf_ci_utils import IDF_PATH from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize @@ -16,12 +15,12 @@ def test_app_mmu_page_size_32k_and_bootloader_mmu_page_size_64k(dut: Dut, app_do assert '32K' in config app_config = config.replace('32K', '64K') + build_dir = f'build_{dut.target}_{app_config}' - path_to_mmu_page_size_64k_build = os.path.join(dut.app.app_path, f'build_{dut.target}_{app_config}') if app_downloader: - app_downloader.download_app(os.path.relpath(path_to_mmu_page_size_64k_build, IDF_PATH), 'flash') + app_downloader.download_app(dut.app.app_path, build_dir, 'flash') - dut.serial.bootloader_flash(path_to_mmu_page_size_64k_build) + dut.serial.bootloader_flash(os.path.join(dut.app.app_path, build_dir)) dut.expect('MMU page size mismatch') dut.expect('App is running') dut.expect('Partition test done') @@ -36,12 +35,12 @@ def test_app_mmu_page_size_64k_and_bootloader_mmu_page_size_32k(dut: Dut, app_do assert '64K' in config app_config = config.replace('64K', '32K') + build_dir = f'build_{dut.target}_{app_config}' - path_to_mmu_page_size_32k_build = os.path.join(dut.app.app_path, f'build_{dut.target}_{app_config}') if app_downloader: - app_downloader.download_app(os.path.relpath(path_to_mmu_page_size_32k_build, IDF_PATH), 'flash') + app_downloader.download_app(dut.app.app_path, build_dir, 'flash') - dut.serial.bootloader_flash(path_to_mmu_page_size_32k_build) + dut.serial.bootloader_flash(os.path.join(dut.app.app_path, build_dir)) dut.expect('MMU page size mismatch') dut.expect('App is running') dut.expect('Partition test done') diff --git a/tools/test_apps/system/unicore_bootloader/pytest_unicore_bootloader.py b/tools/test_apps/system/unicore_bootloader/pytest_unicore_bootloader.py index 6a88260bf6e..f3bdd940b71 100644 --- a/tools/test_apps/system/unicore_bootloader/pytest_unicore_bootloader.py +++ b/tools/test_apps/system/unicore_bootloader/pytest_unicore_bootloader.py @@ -4,7 +4,6 @@ import os import re import pytest -from idf_ci_utils import IDF_PATH from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize @@ -19,12 +18,12 @@ def test_multicore_app_and_unicore_bootloader(dut: Dut, app_downloader, config) assert 'multicore' in config app_config = config.replace('multicore', 'unicore') + build_dir = f'build_{dut.target}_{app_config}' - path_to_unicore_build = os.path.join(dut.app.app_path, f'build_{dut.target}_{app_config}') if app_downloader: - app_downloader.download_app(os.path.relpath(path_to_unicore_build, IDF_PATH), 'flash') + app_downloader.download_app(dut.app.app_path, build_dir, 'flash') - dut.serial.bootloader_flash(path_to_unicore_build) + dut.serial.bootloader_flash(os.path.join(dut.app.app_path, build_dir)) dut.expect('Unicore bootloader') dut.expect('Multicore app') if 'psram' in config: @@ -43,12 +42,12 @@ def test_unicore_app_and_multicore_bootloader(dut: Dut, app_downloader, config) assert 'unicore' in config app_config = config.replace('unicore', 'multicore') + build_dir = f'build_{dut.target}_{app_config}' - path_to_multicore_build = os.path.join(dut.app.app_path, f'build_{dut.target}_{app_config}') if app_downloader: - app_downloader.download_app(os.path.relpath(path_to_multicore_build, IDF_PATH), 'flash') + app_downloader.download_app(dut.app.app_path, build_dir, 'flash') - dut.serial.bootloader_flash(path_to_multicore_build) + dut.serial.bootloader_flash(os.path.join(dut.app.app_path, build_dir)) dut.expect('Multicore bootloader') dut.expect('Unicore app') if 'psram' in config: diff --git a/tools/test_build_system/pytest.ini b/tools/test_build_system/pytest.ini index eeac16e6d7b..6f08513e182 100644 --- a/tools/test_build_system/pytest.ini +++ b/tools/test_build_system/pytest.ini @@ -14,6 +14,9 @@ junit_family = xunit1 junit_logging = stdout junit_log_passing_tests = False +filterwarnings = + ignore::pytest.PytestExperimentalApiWarning + ## !! When adding new markers, don't forget to update also the tools\test_build_system\README.md !! markers = test_app_copy: specify relative path of the app to copy, and the prefix of the destination directory name diff --git a/tools/test_build_system/test_component_manager.py b/tools/test_build_system/test_component_manager.py index 5434bc6745c..1f9894a5432 100644 --- a/tools/test_build_system/test_component_manager.py +++ b/tools/test_build_system/test_component_manager.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import json import os.path diff --git a/tools/test_idf_py/test_idf_py.py b/tools/test_idf_py/test_idf_py.py index 60456386c53..f5807029762 100755 --- a/tools/test_idf_py/test_idf_py.py +++ b/tools/test_idf_py/test_idf_py.py @@ -27,9 +27,9 @@ except ImportError: import idf current_dir = os.path.dirname(os.path.realpath(__file__)) -idf_py_path = os.path.join(current_dir, '..', 'idf.py') +idf_py_path = os.path.normpath(os.path.join(current_dir, '..', 'idf.py')) extension_path = os.path.join(current_dir, 'test_idf_extensions', 'test_ext') -py_actions_path = os.path.join(current_dir, '..', 'idf_py_actions') +py_actions_path = os.path.normpath(os.path.join(current_dir, '..', 'idf_py_actions')) link_path = os.path.join(py_actions_path, 'test_ext')