From 5fd30885b31044c1f8be155953b58f7e44b8a7ff Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Wed, 22 Jul 2026 22:39:19 +0200 Subject: [PATCH] ci: apply common-scripts CI refactor --- .gitlab-ci.yml | 19 +- .gitlab/ci/build.yml | 61 +---- .gitlab/ci/common.yml | 240 ++---------------- .gitlab/ci/default-build-test-rules.yml | 21 -- .gitlab/ci/host-test.yml | 6 +- .gitlab/ci/pre_check.yml | 105 +------- .gitlab/ci/rules.yml | 13 - .gitlab/ci/static-code-analysis.yml | 82 ------ .gitlab/ci/test-win.yml | 46 +--- .idf_ci.toml | 4 + .pre-commit-config.yaml | 12 +- sonar-project.properties | 24 -- tools/ci/check_build_test_rules.py | 31 +-- tools/ci/ci_get_mr_info.py | 154 ----------- tools/ci/deploy_docs.py | 217 ---------------- .../generate_target_test_child_pipeline.py | 6 +- .../templates/.dynamic_jobs.yml | 10 +- tools/ci/exclude_check_tools_files.txt | 3 - tools/ci/executable-list.txt | 1 - tools/ci/generate_rules.py | 46 ++-- tools/ci/get_all_test_results.py | 84 ------ tools/ci/gitlab_yaml_linter.py | 36 +-- tools/ci/idf_ci_utils.py | 188 +++++++------- tools/ci/idf_pytest/plugin.py | 50 ---- tools/ci/python_packages/gitlab_api.py | 93 +++---- tools/ci/utils.sh | 35 --- 26 files changed, 236 insertions(+), 1351 deletions(-) delete mode 100644 .gitlab/ci/default-build-test-rules.yml delete mode 100644 sonar-project.properties delete mode 100644 tools/ci/ci_get_mr_info.py delete mode 100755 tools/ci/deploy_docs.py delete mode 100644 tools/ci/get_all_test_results.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e26acbc2f11..8fd763298f9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,11 +2,25 @@ include: - project: "ci/actions/common" + ref: "v2" file: - "templates/stable/default-workflow.yml" + - "templates/idf/pre_check.yml" + - "templates/idf/build.yml" - "templates/idf/deploy-github.yml" - "templates/idf/deploy-docs.yml" - "templates/idf/integration-test.yml" + - project: "ci/actions/common" + ref: "v2" + file: "templates/idf/sync-files.yml" + inputs: + job_name: "sync-pre-commit-config" + remote_repo_url: "https://__token__:${CI_SYNC_REPO_TOKEN}@${CI_SERVER_FQDN}/ci/images.git" + ref_name: "ci/sync_idf_pre_commit_config" + file_mappings: + - ".pre-commit-config.yaml:pre-commit/idf/.pre-commit-config.v6.0.yaml" + changes: + - ".pre-commit-config.yaml" - project: "ci/actions/common" file: "templates/idf/build-docs.yml" inputs: @@ -20,13 +34,14 @@ include: DOCTGT: "esp32p4" - ".gitlab/ci/danger.yml" - - ".gitlab/ci/common.yml" + - ".gitlab/ci/common.yml" # has to be placed after danger.yml, since danger.yml defined stages. - ".gitlab/ci/rules.yml" + # by stages - ".gitlab/ci/manual_gate.yml" - ".gitlab/ci/upload_cache.yml" + - ".gitlab/ci/pre_check.yml" - ".gitlab/ci/static-code-analysis.yml" - ".gitlab/ci/pre_commit.yml" - - ".gitlab/ci/pre_check.yml" - ".gitlab/ci/build.yml" - ".gitlab/ci/host-test.yml" - ".gitlab/ci/post_deploy.yml" diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index b67f345368e..ae4d74ad4c9 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -110,61 +110,6 @@ build_clang_test_apps_riscv: #################### # Dynamic Pipeline # #################### -generate_build_child_pipeline: - extends: - - .build_template - tags: [fast_run, shiny] - dependencies: # set dependencies to null to avoid missing artifacts issue - needs: - - pipeline_variables - - job: baseline_manifest_sha - optional: true - artifacts: - paths: - - build_child_pipeline.yml - - test_related_apps.txt - - non_test_related_apps.txt - expire_in: 1 week - when: always - script: - - run_cmd idf-ci --debug gitlab build-child-pipeline - -p components - -p examples - -p tools/test_apps - --modified-files $MR_MODIFIED_FILES - -build_child_pipeline: - stage: build - needs: - - job: fast_template_app - optional: true - artifacts: false - - pipeline_variables - - generate_build_child_pipeline - variables: - MR_MODIFIED_COMPONENTS: $MR_MODIFIED_COMPONENTS - MR_MODIFIED_FILES: $MR_MODIFIED_FILES - PARENT_PIPELINE_ID: $CI_PIPELINE_ID - # https://gitlab.com/gitlab-org/gitlab/-/issues/214340 - inherit: - variables: false - trigger: - include: - - artifact: build_child_pipeline.yml - job: generate_build_child_pipeline - strategy: depend - -generate_prebuild_report: - extends: - - .build_template - tags: [fast_run, shiny] - needs: - - pipeline_variables - artifacts: - paths: - - prebuild_report.html - expire_in: 1 week - when: always - script: - - run_cmd idf-ci build collect --format html -o prebuild_report.html - - echo "Report generated at https://${CI_PAGES_HOSTNAME}:${CI_SERVER_PORT}/-/esp-idf/-/jobs/${CI_JOB_ID}/artifacts/prebuild_report.html" +# !!! +# configurations moved to ci/actions/common/idf/ +# !!! diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index 001a17925ba..e7d072bd992 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -14,6 +14,15 @@ stages: - deploy - post_deploy +include: + - project: "ci/actions/common" + ref: "v2" + file: "templates/idf/common-scripts.yml" + inputs: + ci_python_constraint_file: "espidf.constraints.v6.0.txt" + macos_image: "registry.hub.docker.com/espressif/macos-idf:sequoia-v6.0" + windows_image: "windows_11_idf_v6_0" + variables: # System environment @@ -58,30 +67,29 @@ variables: ESP_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v6.0:1" ESP_IDF_DOC_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v6.0-docs:1" TARGET_TEST_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v6.0-target-test:1" - SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:5" # Set this variable to the branch of idf-constraints repo in order to test a custom Python constraint file. The # branch name must be without the remote part ("origin/"). Keep the variable empty in order to use the constraint # file from https://dl.espressif.com/dl/esp-idf. - CI_PYTHON_CONSTRAINT_BRANCH: "" + # CI_PYTHON_CONSTRAINT_BRANCH could be defined as component input `ci_python_constraint_branch` in common-scripts above # Update the filename for a specific ESP-IDF release. It is used only with CI_PYTHON_CONSTRAINT_BRANCH. - CI_PYTHON_CONSTRAINT_FILE: "espidf.constraints.v6.0.txt" + # CI_PYTHON_CONSTRAINT_FILE is defined as component input `ci_python_constraint_file` in common-scripts above # Set this variable to repository name of a Python tool you wish to install and test in the context of ESP-IDF CI. # Keep the variable empty when not used. - CI_PYTHON_TOOL_REPO: "" + # CI_PYTHON_TOOL_REPO could be defined as component input `ci_python_tool_repo` in common-scripts above # Set this variable to the branch of a Python tool repo specified in CI_PYTHON_TOOL_REPO. The # branch name must be without the remote part ("origin/"). Keep the variable empty when not used. # This is used only if CI_PYTHON_TOOL_REPO is not empty. - CI_PYTHON_TOOL_BRANCH: "" + # CI_PYTHON_TOOL_BRANCH could be defined as component input `ci_python_tool_branch` in common-scripts above # Set this variable to Clang toolchain distro URL to be used. # NOTE: We have separate toolchains for Xtensa and RISCV, therefore jobs for one arch will fail. # This is OK as far as we use CI_CLANG_DISTRO_URL for pre-release tests purposes only. # Keep the variable empty when not used. - CI_CLANG_DISTRO_URL: "" + # CI_CLANG_DISTRO_URL could be defined as component input `ci_clang_distro_url` in common-scripts above # Set this variable to specify the file name for the known failure cases. KNOWN_FAILURE_CASES_FILE_NAME: "6.0.txt" @@ -103,232 +111,19 @@ variables: ################################################ # `before_script` and `after_script` Templates # ################################################ -.common_before_scripts: &common-before_scripts | - source tools/ci/utils.sh - - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS - - if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then - export IDF_MIRROR_PREFIX_MAP= - fi - - if echo "$CI_MERGE_REQUEST_LABELS" | egrep "(^|,)include_nightly_run(,|$)"; then - export INCLUDE_NIGHTLY_RUN="1" - export NIGHTLY_RUN="1" - fi - - # configure cmake related flags - source tools/ci/configure_ci_environment.sh - - if [[ "$CI_CCACHE_STATS" == 1 ]] && command -v ccache >/dev/null 2>&1 && [[ -n "$CCACHE_STATSLOG" ]]; then - mkdir -p "$(dirname "$CCACHE_STATSLOG")" - rm -f "$CCACHE_STATSLOG" - fi - - # add extra python packages - export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH" - -.setup_tools_and_idf_python_venv: &setup_tools_and_idf_python_venv | - # must use after setup_tools_except_target_test - # otherwise the export.sh won't work properly - - # download constraint file for dev - if [[ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]]; then - wget -O /tmp/constraint.txt --header="Authorization:Bearer ${ESPCI_TOKEN}" "${GITLAB_HTTP_SERVER}/api/v4/projects/2581/repository/files/${CI_PYTHON_CONSTRAINT_FILE}/raw?ref=${CI_PYTHON_CONSTRAINT_BRANCH}" - mkdir -p ~/.espressif - mv /tmp/constraint.txt ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE} - fi - - # Mirror - if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then - export IDF_MIRROR_PREFIX_MAP= - fi - # Optimize pip install - if echo "${CI_RUNNER_TAGS}" | grep "shiny"; then - export PIP_INDEX_URL="${PIP_INDEX_URL_SHINY}" - fi - if [[ "$(uname -m)" == "x86_64" ]] || [[ "$(uname -m)" == "aarch64" ]]; then - export IDF_PIP_WHEELS_URL="" - fi - - # install.sh - if [[ "${CI_JOB_STAGE}" != "target_test" ]]; then - section_start "running_install_sh" "Running install.sh" - if [[ "${CI_JOB_STAGE}" == "build_doc" || "${CI_JOB_GROUP_NAME}" == *docs* ]]; then - run_cmd bash install.sh --enable-ci --enable-docs - else - run_cmd bash install.sh --enable-ci - fi - section_end "running_install_sh" - else - section_start "install_python_env" "Install Python environment, skip required tools check" - run_cmd python tools/idf_tools.py install-python-env --features ci,test-specific - export IDF_SKIP_TOOLS_CHECK=1 - section_end "install_python_env" - fi - - section_start "source_export" "Source export.sh" - source ./export.sh - section_end "source_export" - - # Eager upgrade of CI dependencies - # Done after sourcing export.sh so that we could easily invoke the right pip - section_start "upgrade_ci_dependencies" "Upgrading CI dependencies" - pip install --upgrade --upgrade-strategy=eager -r $IDF_PATH/tools/requirements/requirements.ci.txt -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE} - # we need the latest DEV release for esptool to be installed in every job without bumping the minimal requirement in - # the constraint file. CI_ESPTOOL_EXTRA_INDEX_URL (set in GitLab project variables) optionally points to an internal - # package registry so that internal dev builds are preferred when the variable is set; it is a no-op when unset. - # Use an explicit if/else: the `${VAR:+--flag "$VAR"}` form collapses to a single token under zsh (macOS runners). - if [[ -n "$CI_ESPTOOL_EXTRA_INDEX_URL" ]]; then - pip install --upgrade --pre --extra-index-url "$CI_ESPTOOL_EXTRA_INDEX_URL" esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE} - else - pip install --upgrade --pre esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE} - fi - section_end "upgrade_ci_dependencies" - - REEXPORT_NEEDED=0 - if [[ ! -z "$INSTALL_EXTRA_TOOLS" ]]; then - section_start "installing_optional_tools" "Install optional tools ${INSTALL_EXTRA_TOOLS}" - run_cmd $IDF_PATH/tools/idf_tools.py --non-interactive install $INSTALL_EXTRA_TOOLS - section_end "installing_optional_tools" - - REEXPORT_NEEDED=1 - fi - - # Install esp-clang if necessary (esp-clang is separately installed) - if [[ "$IDF_TOOLCHAIN" == "clang" && -z "$CI_CLANG_DISTRO_URL" ]]; then - $IDF_PATH/tools/idf_tools.py --non-interactive install esp-clang - - REEXPORT_NEEDED=1 - fi - - if [[ $REEXPORT_NEEDED -eq 1 ]]; then - section_start "re_source_export" "Re-source export.sh" - source ./export.sh - section_end "re_source_export" - fi - - # Custom clang toolchain - if [[ "$IDF_TOOLCHAIN" == "clang" && ! -z "$CI_CLANG_DISTRO_URL" ]]; then - echo "Using custom clang from ${CI_CLANG_DISTRO_URL}" - wget $CI_CLANG_DISTRO_URL - ARCH_NAME=$(basename $CI_CLANG_DISTRO_URL) - tar -x -f $ARCH_NAME - export PATH=$PWD/esp-clang/bin:$PATH - fi - - # Custom OpenOCD - if [[ "$CI_JOB_STAGE" == "target_test" ]]; then - machine="$(uname -m)" - if [[ "$machine" == "armv7l" ]] ; then - OOCD_DISTRO_URL="$OOCD_DISTRO_URL_ARMHF" - elif [[ "$machine" == "aarch64" ]] ; then - OOCD_DISTRO_URL="$OOCD_DISTRO_URL_ARM64" - fi - if [[ ! -z "$OOCD_DISTRO_URL" ]]; then - echo "Using custom OpenOCD from ${OOCD_DISTRO_URL}" - wget $OOCD_DISTRO_URL - ARCH_NAME=$(basename $OOCD_DISTRO_URL) - tar -x -f $ARCH_NAME - export OPENOCD_SCRIPTS=$PWD/openocd-esp32/share/openocd/scripts - export PATH=$PWD/openocd-esp32/bin:$PATH - fi - fi - - if [[ -n "$CI_PYTHON_TOOL_REPO" ]]; then - git clone --quiet --depth=1 -b ${CI_PYTHON_TOOL_BRANCH} https://gitlab-ci-token:${ESPCI_TOKEN}@${GITLAB_HTTPS_HOST}/espressif/${CI_PYTHON_TOOL_REPO}.git - pip install ./${CI_PYTHON_TOOL_REPO} - rm -rf ${CI_PYTHON_TOOL_REPO} - fi - - info "setup tools and python venv done" - -.show_ccache_statistics: &show_ccache_statistics | - # Show ccache statistics if enabled globally - section_start "ccache_show_stats" "Show ccache statistics" - if [[ "$CI_CCACHE_STATS" == 1 ]] && command -v ccache >/dev/null 2>&1; then - if ccache --help 2>/dev/null | grep -q -- '--show-log-stats'; then - if [[ -n "$CCACHE_STATSLOG" && -f "$CCACHE_STATSLOG" ]]; then - ccache --show-log-stats -vv - else - echo "INFO: No per-job ccache statistics were recorded" - fi - else - ccache --show-stats -vv - fi - fi || true - section_end "ccache_show_stats" - -.upload_failed_job_log_artifacts: &upload_failed_job_log_artifacts | - if [ $CI_JOB_STATUS = "failed" ]; then - run_cmd idf-ci gitlab upload-artifacts --type log - fi - .before_script:minimal: before_script: - - *common-before_scripts + - !reference [.common_before_scripts, minimal] .after_script:build: after_script: - - source tools/ci/utils.sh - - source tools/ci/configure_ci_environment.sh - - *show_ccache_statistics + - !reference [.common_after_scripts, ccache] - run_cmd idf-ci gitlab upload-artifacts --type junit - | if [ $CI_JOB_STATUS = "failed" ]; then run_cmd idf-ci gitlab upload-artifacts --type log fi -.brew-macos-settings: - variables: - GIT_STRATEGY: fetch - IDF_CCACHE_ENABLE: "0" - CCACHE_DIR: "/var/tmp/cache/idf_ccache" - tags: - - macos-tart - image: macos-sequoia-idf-v6.0 - after_script: [] # ccache now is disabled for macos brew runners - timeout: 30m - -.brew-windows-settings: - image: windows_11_idf_v6_0 - tags: [windows-vm, brew] - variables: - GIT_STRATEGY: fetch - LC_ALL: C.UTF-8 - # git clean at the final cleanup stage stuck on Windows VM - # a new VM for each job anyway, so we can skip cleaning the workspace - GIT_CLEAN_FLAGS: none -# IDF_CCACHE_ENABLE: "1" -# CCACHE_DIR: "C:/cache/idf_ccache" - before_script: - - if ($env:IDF_DONT_USE_MIRRORS) { $env:IDF_MIRROR_PREFIX_MAP = '' } - - ./install.ps1 --enable-ci - - ./export.ps1 - - $env:PYTHONPATH = "$env:PYTHONPATH;$env:IDF_PATH\tools;$env:IDF_PATH\tools\esp_app_trace;$env:IDF_PATH\components\partition_table;$env:IDF_PATH\tools\ci\python_packages" - - python "${SUBMODULE_FETCH_TOOL}" -s "${SUBMODULES_TO_FETCH}" - # Sync VM clock. net start/stop exit 2 when the service is already in that state; do not fail CI. - - | - function Invoke-NetService($Action, $Name) { - $output = net $Action $Name 2>&1 - if ($LASTEXITCODE -eq 2) { - Write-Host "net $Action $Name already in requested state: $output" - $global:LASTEXITCODE = 0 - return - } - if ($LASTEXITCODE -ne 0) { - Write-Host $output - exit $LASTEXITCODE - } - } - Invoke-NetService start w32time - w32tm /config /manualpeerlist:"time.google.com,0x9 pool.ntp.org,0x9" /syncfromflags:manual /reliable:yes /update - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - Invoke-NetService stop w32time - Invoke-NetService start w32time - w32tm /resync - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - after_script: [] ############# # `default` # @@ -344,8 +139,7 @@ default: - .cache/submodule_archives policy: pull before_script: - - *common-before_scripts - - *setup_tools_and_idf_python_venv + - !reference [.common_before_scripts, with_tools] - add_gitlab_ssh_keys - fetch_submodules - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS} diff --git a/.gitlab/ci/default-build-test-rules.yml b/.gitlab/ci/default-build-test-rules.yml deleted file mode 100644 index 62b2bf09d0e..00000000000 --- a/.gitlab/ci/default-build-test-rules.yml +++ /dev/null @@ -1,21 +0,0 @@ -# this file support two keywords: -# - extra_default_build_targets: -# besides of the SUPPORTED_TARGETS in IDF, -# enable build for the specified targets by default as well. -# !!! DEPRECATED: use `additional_build_targets` in .idf_build_apps.toml instead -# -# - bypass_check_test_targets: -# suppress the check_build_test_rules check-test-script warnings for the specified targets -# -# This file should ONLY be used during bringup. Should be reset to empty after the bringup process - -bypass_check_test_targets: - - esp32h21 - - esp32h4 - - esp32c5 - -# -# These lines would -# - enable the README.md check for esp32c6. Don't forget to add the build jobs in .gitlab/ci/build.yml -# - disable the test script check with the manifest file. -# diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index d7ae41fb6a8..2ff67f7dce1 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -387,7 +387,7 @@ test_pytest_linux: test_pytest_macos: extends: - .host_test_template - - .brew-macos-settings + - .macos-settings artifacts: paths: - XUNIT_RESULT.xml @@ -493,7 +493,7 @@ pytest_buildv2_system: pytest_build_system_macos: extends: - .test_build_system_template - - .brew-macos-settings + - .macos-settings - .rules:build:macos parallel: 3 @@ -503,7 +503,7 @@ pytest_build_system_minimal_cmake: pytest_build_system_macos_minimal_cmake: extends: - .test_build_system_minimal_cmake_template - - .brew-macos-settings + - .macos-settings - .rules:build:macos variables: INSTALL_EXTRA_TOOLS: ninja cmake@3.22.1 diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index 5f87d341e59..15166402113 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -1,9 +1,3 @@ -.pre_check_template: - stage: pre_check - image: $ESP_ENV_IMAGE - tags: [build, shiny] - dependencies: # set dependencies to null to avoid missing artifacts issue - check_version: # Don't run this for feature/bugfix branches, so that it is possible to modify # esp_idf_version.h in a branch before tagging the next version. @@ -140,106 +134,11 @@ check_submodule_sync: - git submodule update --recursive - echo "IDF was cloned from ${PUBLIC_IDF_URL} completely" -pipeline_variables: +check_gitlab_yaml: extends: - .pre_check_template - tags: [fast_run, shiny] - variables: - GIT_STRATEGY: "fetch" script: - - | - # Store the diff output in a temporary file - TEMP_FILE=$(mktemp) - # merged results pipelines, by default - if [[ -n $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ]]; then - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA - git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA - - git diff --name-only $CI_MERGE_REQUEST_TARGET_BRANCH_SHA...$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA > "$TEMP_FILE" - GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") - git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - # merge request pipelines, when the mr got conflicts - elif [[ -n $CI_MERGE_REQUEST_DIFF_BASE_SHA ]]; then - git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_COMMIT_SHA > "$TEMP_FILE" - GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") - # other pipelines, like the protected branches pipelines - elif [[ "$CI_COMMIT_BEFORE_SHA" != "0000000000000000000000000000000000000000" ]]; then - git fetch origin $CI_COMMIT_BEFORE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA > "$TEMP_FILE" - GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") - else - # pipeline source could be web, scheduler, etc. - git fetch origin $CI_COMMIT_SHA --depth=2 ${GIT_FETCH_EXTRA_FLAGS} - git diff --name-only $CI_COMMIT_SHA~1 $CI_COMMIT_SHA > "$TEMP_FILE" - GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") - fi - # MODIFIED_FILES is a list of files that changed, could be used everywhere - - MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | xargs) - - echo "MODIFIED_FILES=$MODIFIED_FILES" >> pipeline.env - # MR_MODIFIED_FILES and MR_MODIFIED_COMPONENTS are semicolon separated lists that is used in MR only - # for non MR pipeline, these are empty lists - - | - if [ -z "$CI_MERGE_REQUEST_IID" ]; then - echo "MR_MODIFIED_FILES=\"\"" >> pipeline.env - echo "MR_MODIFIED_COMPONENTS=\"\"" >> pipeline.env - else - MR_MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | tr '\n' ';') - echo "MR_MODIFIED_FILES=\"$MR_MODIFIED_FILES\"" >> pipeline.env - - MR_MODIFIED_COMPONENTS=$(run_cmd python tools/ci/ci_get_mr_info.py components --modified-files $MODIFIED_FILES | tr '\n' ';') - echo "MR_MODIFIED_COMPONENTS=\"$MR_MODIFIED_COMPONENTS\"" >> pipeline.env - fi - - | - if [ -n "$PIPELINE_COMMIT_SHA" ]; then - echo "PIPELINE_COMMIT_SHA=$PIPELINE_COMMIT_SHA" >> pipeline.env - fi - - echo "OOCD_DISTRO_URL_ARMHF=$OOCD_DISTRO_URL_ARMHF" >> pipeline.env - - echo "OOCD_DISTRO_URL_ARM64=$OOCD_DISTRO_URL_ARM64" >> pipeline.env - - run_cmd idf-ci gitlab pipeline-variables >> pipeline.env - - cat pipeline.env - - run_cmd idf-ci gitlab upload-artifacts --type env - artifacts: - reports: - dotenv: pipeline.env - paths: - - pipeline.env - expire_in: 1 week - when: always - -baseline_manifest_sha: - extends: - - .pre_check_template - - .rules:dev-push - tags: [fast_run, shiny] - script: - - | - # merged results pipelines, by default - # diff between target-branch-head and merged-result-head - if [ -n "$CI_MERGE_REQUEST_TARGET_BRANCH_SHA" ]; then - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA --depth=1 - git checkout FETCH_HEAD - idf-build-apps dump-manifest-sha \ - --manifest-files $(find . -name ".build-test-rules.yml" | xargs) \ - --output .manifest_sha - # merge request pipelines, when the mr got conflicts - # diff between diff-base-sha and merge-request-head - elif [ -n "$CI_MERGE_REQUEST_DIFF_BASE_SHA" ]; then - git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 - git checkout FETCH_HEAD - idf-build-apps dump-manifest-sha \ - --manifest-files $(find . -name ".build-test-rules.yml" | xargs) \ - --output .manifest_sha - # other pipelines, like the protected branches pipelines - # not triggered in this job - fi - artifacts: - paths: - - .manifest_sha - expire_in: 1 week - when: always + - python tools/ci/gitlab_yaml_linter.py retry_failed_jobs: extends: diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index ece3d57cf8d..5ab314f4413 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -13,13 +13,6 @@ - "components/**/CMakeLists.txt" -.patterns-python-files: &patterns-python-files - - ".gitlab/ci/static-code-analysis.yml" - - "**/*.py" - -.patterns-sonarqube-files: &patterns-sonarqube-files - - "tools/ci/sonar_exclude_list.txt" - .patterns-build_components: &patterns-build_components # components files except "test*/" "host*/" folders # ?? to include folders less than 4 characters @@ -169,16 +162,10 @@ .if-protected-check: &if-protected-check if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) || $CI_COMMIT_TAG =~ /^qa-test/' -.if-protected-deploy: &if-protected-deploy - if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "api")' - .if-protected-branch-push: &if-protected-branch-push # rules:changes always evaluates to true for new branch pipelines or when there is no Git push event if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/) && $CI_PIPELINE_SOURCE == "push"' -.if-qa-test-tag: &if-qa-test-tag - if: '$CI_COMMIT_TAG =~ /^qa-test/' - .if-dev-push: &if-dev-push if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")' diff --git a/.gitlab/ci/static-code-analysis.yml b/.gitlab/ci/static-code-analysis.yml index 9beeb9aea8a..2fd4c3b0a7e 100644 --- a/.gitlab/ci/static-code-analysis.yml +++ b/.gitlab/ci/static-code-analysis.yml @@ -27,85 +27,3 @@ gcc_static_analyzer: - echo "CONFIG_COMPILER_STATIC_ANALYZER=y" >> ${ANALYZING_APP}/sdkconfig.defaults - sed -i 's/.*MINIMAL_BUILD.*//g' ${ANALYZING_APP}/CMakeLists.txt - idf-build-apps build -p ${ANALYZING_APP} - -# -## build stage -## Sonarqube related jobs put here for this reason: -## Here we have two jobs. code_quality_check and code_quality_report. -## -## code_quality_check will analyze the code changes between your MR and -## code repo stored in sonarqube server. The analysis result is only shown in -## the comments under this MR and won't be transferred to the server. -## -## code_quality_report will analyze and transfer both of the newly added code -## and the analysis result to the server. -## -## Put in the front to ensure that the newly merged code can be stored in -## sonarqube server ASAP, in order to avoid reporting unrelated code issues -#.sonar_scan_template: -# stage: build -# extends: .pre_check_template -# # full clone since this image does not support fetch --shallow-since-cutoff -# # shiny runners are used for full clone -# tags: [build, shiny] -# image: $SONARQUBE_SCANNER_IMAGE -# before_script: -# - source tools/ci/utils.sh -# - export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH" -# - fetch_submodules -# # Exclude the submodules, all paths ends with /** -# - submodules=$(get_all_submodules) -# # get all exclude paths specified in tools/ci/sonar_exclude_list.txt | ignore lines start with # | xargs | replace all to -# - custom_excludes=$(cat $CI_PROJECT_DIR/tools/ci/sonar_exclude_list.txt | grep -v '^#' | xargs | sed -e 's/ /,/g') -# # Exclude the report dir as well -# - export EXCLUSIONS="$custom_excludes,$submodules" -# - export SONAR_SCANNER_OPTS="-Xmx2048m" -# variables: -# GIT_DEPTH: 0 -# REPORT_PATTERN: clang_tidy_reports/**/*.txt -# artifacts: -# paths: -# - $REPORT_PATTERN -# expire_in: 1 week -# when: always -# dependencies: # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here. -# - clang_tidy_check -# -#code_quality_check: -# extends: -# - .sonar_scan_template -# - .rules:patterns:static-code-analysis-preview -# allow_failure: true # it's using exit code to indicate the code analysis result, -# # we don't want to block ci when critical issues founded -# script: -# - export CI_MERGE_REQUEST_COMMITS=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py commits --src-branch ${CI_COMMIT_REF_NAME} | tr '\n' ',') -# # test if this branch have merge request, if not, exit 0 -# - test -n "$CI_MERGE_REQUEST_IID" || exit 0 -# - test -n "$CI_MERGE_REQUEST_COMMITS" || exit 0 -# - sonar-scanner -# -Dsonar.analysis.mode=preview -# -Dsonar.branch.name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME -# -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN -# -Dsonar.exclusions=$EXCLUSIONS -# -Dsonar.gitlab.ci_merge_request_iid=$CI_MERGE_REQUEST_IID -# -Dsonar.gitlab.commit_sha=$CI_MERGE_REQUEST_COMMITS -# -Dsonar.gitlab.merge_request_discussion=true -# -Dsonar.gitlab.ref_name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME -# -Dsonar.host.url=$SONAR_HOST_URL -# -Dsonar.login=$SONAR_LOGIN -# -#code_quality_report: -# extends: -# - .sonar_scan_template -# - .rules:protected:check -# allow_failure: true # it's using exit code to indicate the code analysis result, -# # we don't want to block ci when critical issues founded -# script: -# - sonar-scanner -# -Dsonar.branch.name=$CI_COMMIT_REF_NAME -# -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN -# -Dsonar.exclusions=$EXCLUSIONS -# -Dsonar.gitlab.commit_sha=$PIPELINE_COMMIT_SHA -# -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME -# -Dsonar.host.url=$SONAR_HOST_URL -# -Dsonar.login=$SONAR_LOGIN diff --git a/.gitlab/ci/test-win.yml b/.gitlab/ci/test-win.yml index 77a004c3b3f..a416390f8b8 100644 --- a/.gitlab/ci/test-win.yml +++ b/.gitlab/ci/test-win.yml @@ -1,6 +1,6 @@ .host_test_win_template: extends: - - .brew-windows-settings + - .windows-settings - .rules:test:host_test stage: host_test dependencies: # set dependencies to null to avoid missing artifacts issue @@ -50,7 +50,7 @@ test_tools_win: script: - cd tools\test_idf_py - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} - - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} + - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml ${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} # Build tests .test_build_system_template_win: @@ -67,42 +67,18 @@ test_tools_win: script: - cd tools\test_build_system - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} - - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} + - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml ${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} pytest_build_system_win: extends: - .test_build_system_template_win - .rules:labels:windows_pytest_build_system parallel: 6 - needs: - - job: manual_gate - optional: true - tags: [windows-build, brew] - artifacts: - paths: - - XUNIT_RESULT.xml - - test_build_system - expire_in: 2 days - reports: - junit: XUNIT_RESULT.xml - when: always pytest_build_system_win_minimal_cmake: extends: - .test_build_system_template_win - .rules:labels:windows_pytest_build_system - needs: - - job: manual_gate - optional: true - tags: [windows-build, brew] - artifacts: - paths: - - XUNIT_RESULT.xml - - test_build_system - expire_in: 2 days - reports: - junit: XUNIT_RESULT.xml - when: always variables: MINIMAL_CMAKE_VERSION: "3.22.1" script: @@ -116,25 +92,13 @@ pytest_build_system_win_minimal_cmake: } - cd tools\test_build_system - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} - - pytest -k cmake --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} + - pytest -k cmake --junitxml ${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} pytest_buildv2_system_win: extends: - .test_build_system_template_win - .rules:labels:buildv2 parallel: 2 - needs: - - job: manual_gate - optional: true - tags: [windows-build, brew] - artifacts: - paths: - - XUNIT_RESULT.xml - - test_build_system - expire_in: 2 days - reports: - junit: XUNIT_RESULT.xml - when: always script: - cd tools\test_build_system - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} @@ -142,5 +106,5 @@ pytest_buildv2_system_win: --buildv2 --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} - --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml + --junitxml ${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} diff --git a/.idf_ci.toml b/.idf_ci.toml index c2f30ecb068..c79ce7dc432 100644 --- a/.idf_ci.toml +++ b/.idf_ci.toml @@ -1,4 +1,6 @@ preserve_non_test_related_apps = false +filter_non_test_related_apps_by_modified_files = true +filter_apps_by_component_target = true exclude_dirs = [ 'tools/test_mkdfu', 'tools/test_idf_size', @@ -20,6 +22,7 @@ IDF_CI_BUILD = "1" [gitlab.build_pipeline] workflow_name = "build_child_pipeline" +runs_per_job = 30 job_tags = ['build', 'shiny'] job_template_name = '.dynamic_build_template' @@ -107,6 +110,7 @@ patterns = [ bucket = "idf-artifacts" patterns = [ '**/XUNIT_RESULT_*.xml', + '**/build_summary_*.xml', ] [gitlab.artifacts.s3.configs.env] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5feed811a63..81756f123f1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -145,7 +145,7 @@ repos: require_serial: true additional_dependencies: - PyYAML>=5.3.1 - - idf-build-apps>=2.14 + - idf-build-apps~=3.0 - id: sort-yaml-files name: sort yaml files entry: tools/ci/sort_yaml.py @@ -166,14 +166,6 @@ repos: language: python always_run: true require_serial: true - - id: gitlab-yaml-linter - name: Check gitlab yaml files - entry: tools/ci/gitlab_yaml_linter.py - language: python - files: '\.gitlab-ci\.yml|\.gitlab/ci/.+\.yml|\.gitmodules' - pass_filenames: false - additional_dependencies: - - PyYAML == 5.3.1 - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.0.1 hooks: @@ -233,7 +225,7 @@ repos: - id: check-kconfig-files - id: check-deprecated-kconfig-options - repo: https://github.com/espressif/idf-ci - rev: v1.0.0 + rev: v1.2.1 hooks: - id: check-tests-missing-config files: 'pytest_.*\.py$|sdkconfig(\..*)?$' diff --git a/sonar-project.properties b/sonar-project.properties deleted file mode 100644 index 1c9d5882fb3..00000000000 --- a/sonar-project.properties +++ /dev/null @@ -1,24 +0,0 @@ -sonar.projectKey=esp-idf -sonar.projectName=Espressif IoT Development Framework -sonar.projectVersion=4.4 - -sonar.sources=. -sonar.sourceEncoding=UTF-8 - -sonar.inclusions=\ - **/*.c,**/*.cpp,**/*.h,**/*.hpp,\ - **/*.py - -#################### -# sonar-cxx-plugin # -#################### - -sonar.cxx.includeDirectories=/usr/include -sonar.cxx.file.suffixes=.cxx,.cpp,.cc,.c,.hxx,.hpp,.hh,.h - -####################### -# sonar-gitlab-plugin # -####################### - -sonar.gitlab.failure_notification_mode=exit-code -sonar.gitlab.project_id=103 diff --git a/tools/ci/check_build_test_rules.py b/tools/ci/check_build_test_rules.py index 6c39734a2c8..73d0cda3f90 100755 --- a/tools/ci/check_build_test_rules.py +++ b/tools/ci/check_build_test_rules.py @@ -8,7 +8,6 @@ import sys from collections import defaultdict from pathlib import Path -import yaml from idf_build_apps import App from idf_build_apps import find_apps from idf_build_apps.constants import SUPPORTED_TARGETS @@ -39,6 +38,14 @@ USUAL_TO_FORMAL = { FORMAL_TO_USUAL = {v: k for k, v in USUAL_TO_FORMAL.items()} +# Targets under bringup: suppress check-test-scripts warnings for these. +# Reset to [] once bringup is complete. +BYPASS_CHECK_TEST_TARGETS: list[str] = [ + 'esp32h21', + 'esp32h4', + 'esp32c5', +] + def print_diff_table( list1: list[str], list2: list[str], title1: str, title2: str, excluded: list[str] | None = None @@ -298,21 +305,9 @@ if __name__ == '__main__': readme_parser = action.add_parser('check-readmes') readme_parser.add_argument('paths', nargs='+', help='check under paths') - readme_parser.add_argument( - '-c', - '--config', - default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'), - help='config file', - ) test_parser = action.add_parser('check-test-scripts') test_parser.add_argument('paths', nargs='+', help='check under paths') - test_parser.add_argument( - '-c', - '--config', - default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'), - help='config file', - ) arg = parser.parse_args() check_dirs = set() @@ -342,14 +337,6 @@ if __name__ == '__main__': os.path.join(IDF_PATH, 'tools', 'cmakev2', 'test'), ] - _bypass_targets: list[str] = [] - if arg.config: - with open(arg.config) as fr: - configs = yaml.safe_load(fr) - - if configs: - _bypass_targets = configs.get('bypass_check_test_targets') or [] - os.environ.update( { 'IDF_PATH': IDF_PATH, @@ -367,5 +354,5 @@ if __name__ == '__main__': check_test_scripts( list(check_dirs), _exclude_dirs, - bypass_targets=_bypass_targets, + bypass_targets=BYPASS_CHECK_TEST_TARGETS, ) diff --git a/tools/ci/ci_get_mr_info.py b/tools/ci/ci_get_mr_info.py deleted file mode 100644 index 041b691919d..00000000000 --- a/tools/ci/ci_get_mr_info.py +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env python -# -# internal use only for CI -# get latest MR information by source branch -# -# SPDX-FileCopyrightText: 2020-2025 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -# -import argparse -import logging -import os -import subprocess -import typing as t -from pathlib import Path - -from gitlab_api import Gitlab -from idf_ci_utils import IDF_PATH - -if t.TYPE_CHECKING: - from gitlab.v4.objects import ProjectCommit - from gitlab.v4.objects import ProjectMergeRequest - - -def _get_mr_obj(source_branch: str) -> t.Optional['ProjectMergeRequest']: - gl = Gitlab(os.getenv('CI_PROJECT_ID', 'espressif/esp-idf')) - if not gl.project: - return None - - mrs = gl.project.mergerequests.list(state='opened', source_branch=source_branch) - if mrs: - return mrs[0] # one source branch can only have one opened MR at one moment - else: - return None - - -def get_mr_iid(source_branch: str) -> str: - mr = _get_mr_obj(source_branch) - if not mr: - return '' - else: - return str(mr.iid) - - -def get_mr_changed_files(source_branch: str) -> t.List[str]: - mr = _get_mr_obj(source_branch) - if not mr: - return [] - - git_output = subprocess.check_output( - ['git', 'diff', '--name-only', '--diff-filter=d', f'origin/{mr.target_branch}...origin/{source_branch}'] - ).decode('utf8') - - return [line.strip() for line in git_output.splitlines() if line.strip()] - - -def get_mr_commits(source_branch: str) -> t.List['ProjectCommit']: - mr = _get_mr_obj(source_branch) - if not mr: - return [] - - return list(mr.commits()) - - -_COMPONENT_NAME_DIR_RECORDS = {} - - -def get_modified_component(filepath: str) -> t.Optional[str]: - """Return the component name if the file is in a component directory, otherwise None.""" - try: - f_path = Path(filepath).resolve().relative_to(IDF_PATH) - except ValueError: # not in IDF_PATH - return None - - # skip md files, etc. - if f_path.suffix in ['.md', '.yml']: - return None - - # skip test_apps files - if 'test_apps' in f_path.parts: - return None - - component_parent_dirs = [f_path.parts[0]] - for part in f_path.parts[1:]: - if component_parent_dirs[-1] == 'components' or component_parent_dirs[-1].endswith('common_components'): - if part not in _COMPONENT_NAME_DIR_RECORDS: - logging.debug('Found component "%s" in path "%s"' % (part, component_parent_dirs)) - _COMPONENT_NAME_DIR_RECORDS[part] = component_parent_dirs - elif _COMPONENT_NAME_DIR_RECORDS.get(part) != component_parent_dirs: - logging.debug( - 'WARNING!!! Found component "%s" in path "%s" and "%s"' - % (part, component_parent_dirs, _COMPONENT_NAME_DIR_RECORDS.get(part)) - ) - - return part - - component_parent_dirs.append(part) - - return None - - -def get_mr_components( - source_branch: t.Optional[str] = None, modified_files: t.Optional[t.List[str]] = None -) -> t.List[str]: - components: t.Set[str] = set() - if modified_files is None: - if not source_branch: - raise RuntimeError('--src-branch is required if --modified-files is not provided') - - modified_files = get_mr_changed_files(source_branch) - - for f in modified_files: - modified_component = get_modified_component(f) - if modified_component: - components.add(modified_component) - - return list(components) - - -def _print_list(_list: t.List[str], separator: str = '\n') -> None: - print(separator.join(_list)) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Get the latest merge request info by pipeline') - actions = parser.add_subparsers(dest='action', help='info type', required=True) - - common_args = argparse.ArgumentParser(add_help=False) - common_args.add_argument('--src-branch', help='source branch') - common_args.add_argument( - '--modified-files', - nargs='+', - help='space-separated list specifies the modified files. will be detected by --src-branch if not provided', - ) - - actions.add_parser('id', parents=[common_args]) - actions.add_parser('commits', parents=[common_args]) - actions.add_parser('components', parents=[common_args]) - target = actions.add_parser('target_in_tags') - target.add_argument('tags', help='comma separated tags, e.g., esp32,generic') - - args = parser.parse_args() - - if args.action == 'id': - if not args.src_branch: - raise RuntimeError('--src-branch is required') - print(get_mr_iid(args.src_branch)) - elif args.action == 'commits': - if not args.src_branch: - raise RuntimeError('--src-branch is required') - _print_list([commit.id for commit in get_mr_commits(args.src_branch)]) - elif args.action == 'components': - _print_list(get_mr_components(args.src_branch, args.modified_files)) - else: - raise NotImplementedError('not possible to get here') diff --git a/tools/ci/deploy_docs.py b/tools/ci/deploy_docs.py deleted file mode 100755 index 954958e738d..00000000000 --- a/tools/ci/deploy_docs.py +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/env python3 -# -# CI script to deploy docs to a webserver. Not useful outside of CI environment -# -# -# SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -# -import glob -import os -import os.path -import re -import stat -import subprocess -import sys -import tarfile -from typing import Any, List, Tuple - -import packaging.version - - -def env(variable: str, default: str=None) -> str: - """ Shortcut to return the expanded version of an environment variable """ - return os.path.expandvars(os.environ.get(variable, default) if default else os.environ[variable]) - - -# import sanitize_version from the docs directory, shared with here -sys.path.append(os.path.join(env('IDF_PATH'), 'docs')) -from sanitize_version import sanitize_version # noqa - - -def main() -> None: - # if you get KeyErrors on the following lines, it's probably because you're not running in Gitlab CI - git_ver = env('GIT_VER') # output of git describe --always - ci_ver = env('CI_COMMIT_REF_NAME', git_ver) # branch or tag we're building for (used for 'release' & URL) - - version = sanitize_version(ci_ver) - print('Git version: {}'.format(git_ver)) - print('CI Version: {}'.format(ci_ver)) - print('Deployment version: {}'.format(version)) - - if not version: - raise RuntimeError('A version is needed to deploy') - - build_dir = env('DOCS_BUILD_DIR') # top-level local build dir, where docs have already been built - - if not build_dir: - raise RuntimeError('Valid DOCS_BUILD_DIR is needed to deploy') - - url_base = env('DOCS_DEPLOY_URL_BASE') # base for HTTP URLs, used to print the URL to the log after deploying - - docs_server = env('DOCS_DEPLOY_SERVER') # ssh server to deploy to - docs_user = env('DOCS_DEPLOY_SERVER_USER') - docs_path = env('DOCS_DEPLOY_PATH') # filesystem path on DOCS_SERVER - - if not docs_server: - raise RuntimeError('Valid DOCS_DEPLOY_SERVER is needed to deploy') - - if not docs_user: - raise RuntimeError('Valid DOCS_DEPLOY_SERVER_USER is needed to deploy') - - docs_server = '{}@{}'.format(docs_user, docs_server) - - if not docs_path: - raise RuntimeError('Valid DOCS_DEPLOY_PATH is needed to deploy') - - print('DOCS_DEPLOY_SERVER {} DOCS_DEPLOY_PATH {}'.format(docs_server, docs_path)) - - tarball_path, version_urls = build_doc_tarball(version, git_ver, build_dir) - - deploy(version, tarball_path, docs_path, docs_server) - - print('Docs URLs:') - doc_deploy_type = os.getenv('TYPE') - for vurl in version_urls: - language, _, target = vurl.split('/') - tag = '{}_{}'.format(language, target) - url = '{}/{}/index.html'.format(url_base, vurl) # (index.html needed for the preview server) - url = re.sub(r'([^:])//', r'\1/', url) # get rid of any // that isn't in the https:// part - print('[document {}][{}] {}'.format(doc_deploy_type, tag, url)) - - # note: it would be neater to use symlinks for stable, but because of the directory order - # (language first) it's kind of a pain to do on a remote server, so we just repeat the - # process but call the version 'stable' this time - if is_stable_version(version): - print('Deploying again as stable version...') - tarball_path, version_urls = build_doc_tarball('stable', git_ver, build_dir) - deploy('stable', tarball_path, docs_path, docs_server) - - -def deploy(version: str, tarball_path: str, docs_path: str, docs_server: str) -> None: - def run_ssh(commands: List) -> None: - """ Log into docs_server and run a sequence of commands using ssh """ - print('Running ssh: {}'.format(commands)) - subprocess.run(['ssh', '-o', 'BatchMode=yes', docs_server, '-x', ' && '.join(commands)], check=True) - - # copy the version tarball to the server - run_ssh(['mkdir -p {}'.format(docs_path)]) - print('Running scp {} to {}'.format(tarball_path, '{}:{}'.format(docs_server, docs_path))) - subprocess.run(['scp', '-B', tarball_path, '{}:{}'.format(docs_server, docs_path)], check=True) - - tarball_name = os.path.basename(tarball_path) - - run_ssh(['cd {}'.format(docs_path), - 'rm -rf ./*/{}'.format(version), # remove any pre-existing docs matching this version - 'tar -zxvf {}'.format(tarball_name), # untar the archive with the new docs - 'rm {}'.format(tarball_name)]) - - # Note: deleting and then extracting the archive is a bit awkward for updating stable/latest/etc - # as the version will be invalid for a window of time. Better to do it atomically, but this is - # another thing made much more complex by the directory structure putting language before version... - - -def build_doc_tarball(version: str, git_ver: str, build_dir: str) -> Tuple[str, List]: - """ Make a tar.gz archive of the docs, in the directory structure used to deploy as - the given version """ - version_paths = [] - tarball_path = '{}/{}.tar.gz'.format(build_dir, version) - - # find all the 'html/' directories under build_dir - html_dirs = glob.glob('{}/**/html/'.format(build_dir), recursive=True) - print('Found %d html directories' % len(html_dirs)) - - pdfs = glob.glob('{}/**/latex/build/*.pdf'.format(build_dir), recursive=True) - print('Found %d PDFs in latex directories' % len(pdfs)) - - # add symlink for stable and latest and adds them to PDF blob - symlinks = create_and_add_symlinks(version, git_ver, pdfs) - - def not_sources_dir(ti: Any) -> Any: - print(type(ti)) - """ Filter the _sources directories out of the tarballs """ - if ti.name.endswith('/_sources'): - return None - - ti.mode |= stat.S_IWGRP # make everything group-writeable - return ti - - try: - os.remove(tarball_path) - except OSError: - pass - - with tarfile.open(tarball_path, 'w:gz') as tarball: - for html_dir in html_dirs: - # html_dir has the form '///html/' - target_dirname = os.path.dirname(os.path.dirname(html_dir)) - target = os.path.basename(target_dirname) - language = os.path.basename(os.path.dirname(target_dirname)) - - # when deploying, we want the top-level directory layout 'language/version/target' - archive_path = '{}/{}/{}'.format(language, version, target) - print("Archiving '{}' as '{}'...".format(html_dir, archive_path)) - tarball.add(html_dir, archive_path, filter=not_sources_dir) - version_paths.append(archive_path) - - for pdf_path in pdfs: - # pdf_path has the form '///latex/build' - latex_dirname = os.path.dirname(pdf_path) - pdf_filename = os.path.basename(pdf_path) - target_dirname = os.path.dirname(os.path.dirname(latex_dirname)) - target = os.path.basename(target_dirname) - language = os.path.basename(os.path.dirname(target_dirname)) - - # when deploying, we want the layout 'language/version/target/pdf' - archive_path = '{}/{}/{}/{}'.format(language, version, target, pdf_filename) - print("Archiving '{}' as '{}'...".format(pdf_path, archive_path)) - tarball.add(pdf_path, archive_path) - - for symlink in symlinks: - os.unlink(symlink) - - return (os.path.abspath(tarball_path), version_paths) - - -def create_and_add_symlinks(version: str, git_ver: str, pdfs: List) -> List: - """ Create symbolic links for PDFs for 'latest' and 'stable' releases """ - - symlinks = [] - if 'stable' in version or 'latest' in version: - for pdf_path in pdfs: - symlink_path = pdf_path.replace(git_ver, version) - os.symlink(pdf_path, symlink_path) - symlinks.append(symlink_path) - - pdfs.extend(symlinks) - print('Found %d PDFs in latex directories after adding symlink' % len(pdfs)) - - return symlinks - - -def is_stable_version(version: str) -> bool: - """ Heuristic for whether this is the latest stable release """ - if not version.startswith('v'): - return False # branch name - if '-' in version: - return False # prerelease tag - - git_out = subprocess.check_output(['git', 'tag', '-l']).decode('utf-8') - - versions = [v.strip() for v in git_out.split('\n')] - versions = [v for v in versions if re.match(r'^v[\d\.]+$', v.strip())] # include vX.Y.Z only - - versions_pack = [packaging.version.parse(v) for v in versions] - - max_version = max(versions_pack) - - if max_version.public != version[1:]: - print('Stable version is v{}. This version is {}.'.format(max_version.public, version)) - return False - else: - print('This version {} is the stable version'.format(version)) - return True - - -if __name__ == '__main__': - main() diff --git a/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py b/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py index ff414d8da08..e562c140251 100644 --- a/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py +++ b/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py @@ -55,7 +55,7 @@ def main(output_filepath: str) -> None: exclude_runner_tags_matching = [] for _tag in known_warnings_dict.get('no_runner_tags', []): if '*' not in _tag: - exclude_runner_tags_set.add(_tag) + exclude_runner_tags_set.add(frozenset(_tag.split(','))) else: if res := _process_match_group(_tag): exclude_runner_tags_matching.append(res) @@ -66,7 +66,7 @@ def main(output_filepath: str) -> None: if exclude_runner_tags := os.getenv('EXCLUDE_RUNNER_TAGS'): for _tag in exclude_runner_tags.split(';'): if '*' not in _tag: - exclude_runner_tags_set.add(_tag) + exclude_runner_tags_set.add(frozenset(_tag.split(','))) else: if res := _process_match_group(_tag): exclude_runner_tags_matching.append(res) @@ -75,7 +75,7 @@ def main(output_filepath: str) -> None: additional_dict: dict[GroupKey, dict[str, t.Any]] = {} for key, grouped_cases in cases.grouped_cases.items(): # skip test cases with no runner tags - if ','.join(sorted(key.runner_tags)) in exclude_runner_tags_set: + if frozenset(key.runner_tags) in exclude_runner_tags_set: print(f'WARNING: excluding test cases with runner tags: {key.runner_tags}') continue diff --git a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml index 6f0b1949e4c..a145097f312 100644 --- a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml +++ b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml @@ -17,6 +17,9 @@ # Since the version 3.21 CMake passes source files and include dirs to ninja using absolute paths. # Needed for pytest junit reports. INSTALL_EXTRA_TOOLS: cmake + # Suppress Python warning messages during CI builds to keep logs clean and focused on errors. + # This prevents non-critical warnings (e.g., deprecation or resource warnings) from cluttering output. + PYTHONWARNINGS: "ignore" needs: - pipeline: $PARENT_PIPELINE_ID job: generate_build_child_pipeline @@ -40,7 +43,6 @@ - run_cmd idf-ci build run --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1} - --modified-files ${MR_MODIFIED_FILES} .dynamic_target_test_template: image: $TARGET_TEST_ENV_IMAGE @@ -56,6 +58,8 @@ needs: - pipeline: $PARENT_PIPELINE_ID job: pipeline_variables + - pipeline: $PARENT_PIPELINE_ID + job: snapshot_known_failure_cases artifacts: paths: - XUNIT_RESULT*.xml @@ -65,7 +69,11 @@ when: always expire_in: 1 week script: + # Merge the current known failure cases file with snapshot_known_failure_cases file captured at pipeline start + # to avoid retry failures caused by later updates to the known failure cases file. - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} + - printf '\n' >> ${KNOWN_FAILURE_CASES_FILE_NAME} + - cat artifacts/${KNOWN_FAILURE_CASES_FILE_NAME}.freeze >> ${KNOWN_FAILURE_CASES_FILE_NAME} # get runner env config file - retry_failed git clone $TEST_ENV_CONFIG_REPO - run_cmd python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs diff --git a/tools/ci/exclude_check_tools_files.txt b/tools/ci/exclude_check_tools_files.txt index 17a566f271a..37e28cbc931 100644 --- a/tools/ci/exclude_check_tools_files.txt +++ b/tools/ci/exclude_check_tools_files.txt @@ -9,17 +9,14 @@ tools/ci/check_*.txt tools/ci/check_copyright_config.yaml tools/ci/checkout_project_ref.py tools/ci/ci_fetch_submodule.py -tools/ci/ci_get_mr_info.py tools/ci/cleanup_ignore_lists.py tools/ci/configure_ci_environment.sh -tools/ci/deploy_docs.py tools/ci/dynamic_pipelines/**/* tools/ci/envsubst.py tools/ci/executable-list.txt tools/ci/fix_empty_prototypes.sh tools/ci/generate_rules.py tools/ci/get-full-sources.sh -tools/ci/get_all_test_results.py tools/ci/get_supported_examples.sh tools/ci/gitlab_yaml_linter.py tools/ci/idf_build_apps_dump_soc_caps.py diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index 93f786554a5..fdc6e059407 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -70,7 +70,6 @@ tools/ci/check_tools_files_patterns.py tools/ci/check_type_comments.py tools/ci/checkout_project_ref.py tools/ci/cleanup_ignore_lists.py -tools/ci/deploy_docs.py tools/ci/envsubst.py tools/ci/fix_empty_prototypes.sh tools/ci/generate_rules.py diff --git a/tools/ci/generate_rules.py b/tools/ci/generate_rules.py index f9257eaf093..6bad3474454 100755 --- a/tools/ci/generate_rules.py +++ b/tools/ci/generate_rules.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2021-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import argparse import inspect @@ -12,22 +12,21 @@ from itertools import product import yaml from idf_ci_utils import IDF_PATH -from idf_ci_utils import GitlabYmlConfig if t.TYPE_CHECKING: import pygraphviz as pgv -def _list(str_or_list: t.Union[str, t.List]) -> t.List: +def _list(str_or_list: str | list) -> list: if isinstance(str_or_list, str): return [str_or_list] elif isinstance(str_or_list, list): return str_or_list else: - raise ValueError('Wrong type: {}. Only supports str or list.'.format(type(str_or_list))) + raise ValueError(f'Wrong type: {type(str_or_list)}. Only supports str or list.') -def _format_nested_dict(_dict: t.Dict[str, t.Dict], f_tuple: t.Tuple[str, ...]) -> t.Dict[str, t.Dict]: +def _format_nested_dict(_dict: dict[str, dict], f_tuple: tuple[str, ...]) -> dict[str, dict]: res = {} for k, v in _dict.items(): k = k.split('__')[0] @@ -41,7 +40,7 @@ def _format_nested_dict(_dict: t.Dict[str, t.Dict], f_tuple: t.Tuple[str, ...]) return res -def _format_nested_list(_list: t.List[str], f_tuple: t.Tuple[str, ...]) -> t.List[str]: +def _format_nested_list(_list: list[str], f_tuple: tuple[str, ...]) -> list[str]: res = [] for item in _list: if isinstance(item, list): @@ -90,7 +89,6 @@ class RulesWriter: self.cfg = self.expand_matrices() self.rules = self.expand_rules() - self.yml_config = GitlabYmlConfig() self.graph = None def expand_matrices(self): # type: () -> dict @@ -107,11 +105,11 @@ class RulesWriter: deploy = v.get('deploy') if deploy: for item in _list(deploy): - res['{}-{}'.format(k, item)] = v + res[f'{k}-{item}'] = v return res @staticmethod - def _expand_matrix(name: str, cfg: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]: + def _expand_matrix(name: str, cfg: dict[str, t.Any]) -> dict[str, t.Any]: """ Expand matrix into multi keys :param cfg: single rule dict @@ -129,7 +127,7 @@ class RulesWriter: res.update(_format_nested_dict(default, comb)) return res - def expand_rules(self) -> t.Dict[str, t.Dict[str, t.List[str]]]: + def expand_rules(self) -> dict[str, dict[str, list[str]]]: res = defaultdict(lambda: defaultdict(set)) # type: dict[str, dict[str, set]] for k, v in self.cfg.items(): if not v: @@ -155,8 +153,8 @@ class RulesWriter: if 'patterns' in v: for _pat in _list(v['patterns']): # Patterns must be pre-defined - if '.patterns-{}'.format(_pat) not in self.rules_cfg: - print('WARNING: pattern {} not exists'.format(_pat)) + if f'.patterns-{_pat}' not in self.rules_cfg: + print(f'WARNING: pattern {_pat} not exists') continue res[item]['patterns'].add(_pat) @@ -195,14 +193,10 @@ class RulesWriter: if k.startswith('pattern'): continue - if '.rules:' + k not in self.yml_config.used_templates: - print(f'WARNING: unused rule: {k}, skipping...') - continue - res.append(self.RULES_TEMPLATE.format(k, self._format_rule(k, v))) return '\n\n'.join(res) - def _format_rule(self, name: str, cfg: t.Dict[str, t.Any]) -> str: + def _format_rule(self, name: str, cfg: dict[str, t.Any]) -> str: _rules = [self.RULE_REVERT_BRANCH] if name.endswith('-production'): _rules.append(self.RULE_PROTECTED_PUSH) @@ -216,21 +210,21 @@ class RulesWriter: if f'.{specific_rule}' in self.rules_cfg: _rules.append(self.SPECIFIC_RULE_TEMPLATE.format(specific_rule)) else: - print('WARNING: specific_rule {} not exists'.format(specific_rule)) + print(f'WARNING: specific_rule {specific_rule} not exists') for label in cfg['labels']: _rules.append(self.RULE_LABEL_TEMPLATE.format(label)) for pattern in cfg['patterns']: - if '.patterns-{}'.format(pattern) in self.rules_cfg: + if f'.patterns-{pattern}' in self.rules_cfg: _rules.append(self.RULE_PATTERN_TEMPLATE.format(pattern)) else: - print('WARNING: pattern {} not exists'.format(pattern)) + print(f'WARNING: pattern {pattern} not exists') return '\n'.join(_rules) def update_rules_yml(self) -> bool: with open(self.rules_yml) as fr: file_str = fr.read() - auto_generate_str = '\n{}\n\n{}\n'.format(self.new_labels_str(), self.new_rules_str()) + auto_generate_str = f'\n{self.new_labels_str()}\n\n{self.new_rules_str()}\n' rest, marker, old = file_str.partition(self.AUTO_GENERATE_MARKER) if old == auto_generate_str: return False @@ -246,7 +240,7 @@ PATTERN_COLOR = 'cyan' RULE_COLOR = 'blue' -def build_graph(rules_dict: t.Dict[str, t.Dict[str, t.List[str]]]) -> 'pgv.AGraph': +def build_graph(rules_dict: dict[str, dict[str, list[str]]]) -> 'pgv.AGraph': from pygraphviz import pgv graph = pgv.AGraph(directed=True, rankdir='LR', concentrate=True) @@ -263,13 +257,13 @@ def build_graph(rules_dict: t.Dict[str, t.Dict[str, t.List[str]]]) -> 'pgv.AGrap labels = v.get('labels') if labels: for _label in labels: - graph.add_node('label:{}'.format(_label), color=LABEL_COLOR) - graph.add_edge('label:{}'.format(_label), k, color=LABEL_COLOR) + graph.add_node(f'label:{_label}', color=LABEL_COLOR) + graph.add_edge(f'label:{_label}', k, color=LABEL_COLOR) patterns = v.get('patterns') if patterns: for _pat in patterns: - graph.add_node('pattern:{}'.format(_pat), color=PATTERN_COLOR) - graph.add_edge('pattern:{}'.format(_pat), k, color=PATTERN_COLOR) + graph.add_node(f'pattern:{_pat}', color=PATTERN_COLOR) + graph.add_edge(f'pattern:{_pat}', k, color=PATTERN_COLOR) return graph diff --git a/tools/ci/get_all_test_results.py b/tools/ci/get_all_test_results.py deleted file mode 100644 index 7baa0832f2a..00000000000 --- a/tools/ci/get_all_test_results.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python -# -# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 - - -''' -Download artifacts from all test jobs - -CI_PROJECT_DIR -└── TEST_RESULTS - ├── - │ └── XUNIT_RESULT.xml - ├── - │ └── XUNIT_RESULT.xml - ├── - │ └── TEST_LOGS - -''' - -import argparse -import json -import os -import shutil - -import gitlab_api - -CI_PROJECT_ID = int(os.getenv('CI_PROJECT_ID', '')) -CI_PIPELINE_ID = int(os.getenv('CI_PIPELINE_ID', '')) -IDF_PATH = os.getenv('IDF_PATH', '.') -TEST_RESULT_PATH = os.path.join(IDF_PATH, 'TEST_RESULTS') - - -def download_all_test_results(result_path: str, include_retried: bool = False) -> None: - if os.path.exists(result_path): - shutil.rmtree(result_path) - os.makedirs(result_path, exist_ok=True) - - gitlab_inst = gitlab_api.Gitlab(CI_PROJECT_ID) - pipelines = gitlab_inst.project.pipelines.get(CI_PIPELINE_ID) - _include_retried = 'true' if include_retried else 'false' - jobs = pipelines.jobs.list(all=True, per_page=100, include_retried=_include_retried) - - job_info_list = [] - for job in jobs: - if job.stage in ['target_test', 'host_test']: - log_path = '' - if job.status not in ['success', 'failed']: - print('Job {}({}) is not finished'.format(job.id, job.name)) - elif not hasattr(job, 'artifacts_file'): - print('Job {}({}) has no artifacts.'.format(job.id, job.name)) - else: - log_path = os.path.join(result_path, 'job_{}'.format(job.id)) - print('Downloading artifacts from: {}'.format(job.name)) - os.makedirs(log_path, exist_ok=True) - gitlab_inst.download_artifacts(job.id, log_path) - - job_info = { - 'id': job.id, - 'name': job.name, - 'tag_list': job.tag_list, - 'status': job.status, - 'stage': job.stage, - 'web_url': job.web_url, - 'commit_url': job.commit['web_url'], - 'log_path': log_path, - } - job_info_list.append(job_info) - - with open(os.path.join(result_path, 'index.json'), 'w') as f: - f.write(json.dumps({'jobs': job_info_list}, indent=1, sort_keys=True)) - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument('--path', '-p', default=TEST_RESULT_PATH, help='Path to save result files.') - parser.add_argument('--include_retried', action='store_true', help='Including retried jobs.') - args = parser.parse_args() - - download_all_test_results(args.path, args.include_retried) - - -if __name__ == '__main__': - main() diff --git a/tools/ci/gitlab_yaml_linter.py b/tools/ci/gitlab_yaml_linter.py index baac4ee2e37..e06a12034d4 100755 --- a/tools/ci/gitlab_yaml_linter.py +++ b/tools/ci/gitlab_yaml_linter.py @@ -1,27 +1,27 @@ #!/usr/bin/env python -# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 """ Check gitlab ci yaml files """ + import argparse import os -import typing as t from functools import cached_property -from idf_ci_utils import get_submodule_dirs -from idf_ci_utils import GitlabYmlConfig from idf_ci_utils import IDF_PATH +from idf_ci_utils import GitlabYmlConfig +from idf_ci_utils import get_submodule_dirs class YmlLinter: def __init__(self, yml_config: GitlabYmlConfig) -> None: self.yml_config = yml_config - self._errors: t.List[str] = [] + self._errors: list[str] = [] @cached_property - def lint_functions(self) -> t.List[str]: + def lint_functions(self) -> list[str]: funcs = [] for func in dir(self): if func.startswith('_lint_'): @@ -43,17 +43,6 @@ class YmlLinter: exit(exit_code) - # name it like _1_ to make it run first - def _lint_1_yml_parser(self) -> None: - for k, v in self.yml_config.config.items(): - if ( - k not in self.yml_config.global_keys - and k not in self.yml_config.anchors - and k not in self.yml_config.templates - and k not in self.yml_config.jobs - ): - raise SystemExit(f'Parser incorrect. Key {k} not in global keys, anchors, templates, or jobs') - def _lint_default_values_artifacts(self) -> None: defaults_artifacts = self.yml_config.default.get('artifacts', {}) @@ -79,19 +68,6 @@ class YmlLinter: for item in undefined_patterns: self._errors.append(f'undefined pattern {item}. Please add {item} to .patterns-submodule') - def _lint_gitlab_yml_templates(self) -> None: - unused_templates = self.yml_config.templates.keys() - self.yml_config.used_templates - for item in unused_templates: - # known unused ones - if item not in [ - '.before_script:fetch:target_test', # used in dynamic pipeline - ]: - self._errors.append(f'Unused template: {item}, please remove it') - - undefined_templates = self.yml_config.used_templates - self.yml_config.templates.keys() - for item in undefined_templates: - self._errors.append(f'Undefined template: {item}') - def _lint_dependencies_and_needs(self) -> None: """ Use `dependencies: []` together with `needs: []` could cause missing artifacts issue. diff --git a/tools/ci/idf_ci_utils.py b/tools/ci/idf_ci_utils.py index 9d45c8d6b6e..74840dcd512 100644 --- a/tools/ci/idf_ci_utils.py +++ b/tools/ci/idf_ci_utils.py @@ -124,35 +124,106 @@ class GitlabYmlConfig: # avoid unused import in other pre-commit hooks import yaml - all_config = dict() - root_yml = yaml.load(open(root_yml_filepath), Loader=yaml.FullLoader) - - # expanding "include" - for item in root_yml.pop('include', []) or []: - if isinstance(item, dict): - if 'project' in item: - continue - elif 'local' in item: - item = item['local'] - else: - continue - - all_config.update(yaml.load(open(os.path.join(IDF_PATH, item)), Loader=yaml.FullLoader)) + merged_yaml = self._compile_via_gitlab_api(root_yml_filepath) + all_config = yaml.load(merged_yaml, Loader=yaml.FullLoader) or {} if 'default' in all_config: self._defaults = all_config.pop('default') self._config = all_config - # anchor is the string that will be reused in templates - self._anchor_keys: set[str] = set() - # template is a dict that will be extended - self._template_keys: set[str] = set() - self._used_template_keys: set[str] = set() # tracing the used templates - # job is a dict that will be executed - self._job_keys: set[str] = set() + def _inline_local_includes(self, root_yml_filepath: str) -> str: + """ + Recursively resolve `include: local` entries straight from disk (so uncommitted local + changes are always picked up -- CI runners also work off a disk checkout, so there's no + need to fetch a ref remotely via `ci_lint`'s `content_ref`/`ref`/`dry_run_ref` GET + params). `include: project` entries (including ones nested inside local files, e.g. + `.gitlab/ci/common.yml` including `templates/idf/common-scripts.yml`) are collected and + left in the final `include:` list, since those files live in another GitLab project and + can only be resolved remotely. - self.expand_extends() + A dedicated Loader/Dumper pair round-trips `!reference` tags as a marker list subclass, + since GitLab CI uses `!reference` which plain YAML doesn't know, and we need to parse + (to merge dicts, not just string-concat) then re-dump losslessly. + + :param root_yml_filepath: path to the local root yml file to compile + :return: yml content with local includes resolved and merged + """ + import yaml + + class _Loader(yaml.FullLoader): + pass + + class _Dumper(yaml.Dumper): + pass + + class _Reference(list): + pass + + _Loader.add_constructor( + '!reference', lambda loader, node: _Reference(loader.construct_sequence(t.cast(yaml.SequenceNode, node))) + ) + _Dumper.add_representer(_Reference, lambda dumper, data: dumper.represent_sequence('!reference', list(data))) + + def resolve(yml_filepath: str) -> tuple[dict, list]: + with open(yml_filepath) as fr: + data = yaml.load(fr, Loader=_Loader) or {} + + includes = to_list(data.pop('include', None)) + + merged: dict = {} + remaining_project_includes: list = [] + for item in includes: + if isinstance(item, dict): + if 'project' in item: + remaining_project_includes.append(item) + continue + elif 'local' in item: + local_path = item['local'] + else: + continue + elif isinstance(item, str): + local_path = item + else: + continue + + sub_merged, sub_remaining = resolve(os.path.join(IDF_PATH, local_path.lstrip('/'))) + merged.update(sub_merged) + remaining_project_includes.extend(sub_remaining) + + # this file's own top-level keys override whatever its includes defined + merged.update(data) + return merged, remaining_project_includes + + merged_config, project_includes = resolve(root_yml_filepath) + if project_includes: + merged_config['include'] = project_includes + + return yaml.dump(merged_config, Dumper=_Dumper, sort_keys=False) # type: ignore + + def _compile_via_gitlab_api(self, root_yml_filepath: str) -> str: + """ + Call the GitLab CI Lint API to get the fully compiled (all `include`s resolved) yml, + same as what `glab ci config compile` does. This replaces the old recursive local-only + parsing, since the project now includes configs from other projects as well. + + :param root_yml_filepath: path to the local root yml file to compile + :return: merged (fully resolved) yml content as a string + """ + sys.path.insert(0, os.path.join(IDF_PATH, 'tools', 'ci', 'python_packages')) + import gitlab_api + + content = self._inline_local_includes(root_yml_filepath) + + gitlab_inst = gitlab_api.Gitlab() + project_id = os.getenv('CI_PROJECT_ID') or gitlab_inst.get_project_id('esp-idf', namespace='espressif') + project = gitlab_inst.gitlab_inst.projects.get(project_id, lazy=True) + + lint_result = project.ci_lint.create({'content': content}) + if not lint_result.valid: + raise RuntimeError(f'Failed to compile {root_yml_filepath} via GitLab CI Lint API: {lint_result.errors}') + + return lint_result.merged_yaml # type: ignore @property def default(self) -> dict[str, t.Any]: @@ -166,80 +237,9 @@ class GitlabYmlConfig: def global_keys(self) -> list[str]: return ['default', 'include', 'workflow', 'variables', 'stages'] - @cached_property - def anchors(self) -> dict[str, t.Any]: - return {k: v for k, v in self.config.items() if k in self._anchor_keys} - @cached_property def jobs(self) -> dict[str, t.Any]: - return {k: v for k, v in self.config.items() if k in self._job_keys} - - @cached_property - def templates(self) -> dict[str, t.Any]: - return {k: v for k, v in self.config.items() if k in self._template_keys} - - @cached_property - def used_templates(self) -> set[str]: - return self._used_template_keys - - def expand_extends(self) -> None: - """ - expand the `extends` key in-place. - """ - for k, v in self.config.items(): - if k in self.global_keys: - continue - - if isinstance(v, str | list): - self._anchor_keys.add(k) - elif k.startswith('.if-'): - self._anchor_keys.add(k) - elif k.startswith('.'): - self._template_keys.add(k) - elif isinstance(v, dict): - self._job_keys.add(k) - else: - raise ValueError(f'Unknown type for key {k} with value {v}') - - # no need to expand anchor - - # expand template first - for k in self._template_keys: - self._expand_extends(k) - - # expand job - for k in self._job_keys: - self._expand_extends(k) - - def _merge_dict(self, d1: dict[str, t.Any], d2: dict[str, t.Any]) -> t.Any: - for k, v in d2.items(): - if k in d1: - if isinstance(v, dict) and isinstance(d1[k], dict): - d1[k] = self._merge_dict(d1[k], v) - else: - d1[k] = v - else: - d1[k] = v - - return d1 - - def _expand_extends(self, name: str) -> dict[str, t.Any]: - extends = to_list(self.config[name].pop('extends', None)) - if not extends: - return self.config[name] # type: ignore - - original_d = self.config[name].copy() - d = {} - while extends: - self._used_template_keys.update(extends) # for tracking - - for i in extends: - d.update(self._expand_extends(i)) - - extends = to_list(self.config[name].pop('extends', None)) - - self.config[name] = self._merge_dict(d, original_d) - return self.config[name] # type: ignore + return {k: v for k, v in self.config.items() if not k.startswith('.') and k not in self.global_keys} def idf_relpath(p: str) -> str: diff --git a/tools/ci/idf_pytest/plugin.py b/tools/ci/idf_pytest/plugin.py index f84d2dc6820..ccf6b69bbf8 100644 --- a/tools/ci/idf_pytest/plugin.py +++ b/tools/ci/idf_pytest/plugin.py @@ -9,7 +9,6 @@ import pytest import yaml from _pytest.config import Config from _pytest.python import Function -from _pytest.python import Metafunc from _pytest.runner import CallInfo from dynamic_pipelines.constants import KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH from idf_ci import IdfPytestPlugin @@ -115,55 +114,6 @@ class IdfLocalPlugin: return item.callspec.params.get(key, default) or default - @staticmethod - def _has_parametrized_arg(metafunc: Metafunc, arg_name: str) -> bool: - for marker in metafunc.definition.iter_markers(name='parametrize'): - if not marker.args: - continue - - argnames = marker.args[0] - if isinstance(argnames, str): - names = [name.strip() for name in argnames.split(',')] - else: - names = list(argnames) - - if arg_name in names: - return True - - for callspec in getattr(metafunc, '_calls', []): - if arg_name in callspec.params: - return True - - return False - - @staticmethod - def _is_linux_target_run(config: Config) -> bool: - target = config.getoption('target') - if not target: - return False - - if isinstance(target, str): - targets = [_t.strip() for _t in target.split(',')] - else: - targets = [str(_t).strip() for _t in target] - - return 'linux' in targets - - @pytest.hookimpl(trylast=True) - def pytest_generate_tests(self, metafunc: Metafunc) -> None: - if 'embedded_services' not in metafunc.fixturenames: - return - - if self._has_parametrized_arg(metafunc, 'embedded_services'): - return - - if metafunc.definition.get_closest_marker('qemu') is not None: - metafunc.parametrize('embedded_services', ['idf,qemu'], indirect=True) - return - - if self._is_linux_target_run(metafunc.config): - metafunc.parametrize('embedded_services', ['idf'], indirect=True) - @pytest.hookimpl(wrapper=True) def pytest_collection_modifyitems(self, config: Config, items: list[Function]) -> t.Generator[None, None, None]: yield # throw it back to idf-ci diff --git a/tools/ci/python_packages/gitlab_api.py b/tools/ci/python_packages/gitlab_api.py index 6ac0ffae1f8..ba63c8d596b 100644 --- a/tools/ci/python_packages/gitlab_api.py +++ b/tools/ci/python_packages/gitlab_api.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import argparse import logging @@ -9,13 +9,9 @@ import tarfile import tempfile import time import zipfile +from collections.abc import Callable from functools import wraps from typing import Any -from typing import Callable -from typing import Dict -from typing import List -from typing import Optional -from typing import Union import gitlab @@ -36,7 +32,7 @@ def retry(func: TR) -> TR: while True: try: res = func(self, *args, **kwargs) - except (IOError, EOFError, gitlab.exceptions.GitlabError) as e: + except (OSError, EOFError, gitlab.exceptions.GitlabError) as e: if isinstance(e, gitlab.exceptions.GitlabError): if e.response_code == 500: # retry on this error @@ -52,9 +48,11 @@ def retry(func: TR) -> TR: raise e # get out of the loop else: logging.warning( - 'Network failure in {}, retrying ({})'.format(getattr(func, '__name__', '(unknown callable)'), - retried)) - time.sleep(2 ** retried) # wait a bit more after each retry + 'Network failure in {}, retrying ({})'.format( + getattr(func, '__name__', '(unknown callable)'), retried + ) + ) + time.sleep(2**retried) # wait a bit more after each retry continue else: break @@ -63,26 +61,25 @@ def retry(func: TR) -> TR: return wrapper -class Gitlab(object): +class Gitlab: JOB_NAME_PATTERN = re.compile(r'(\w+)(\s+(\d+)/(\d+))?') DOWNLOAD_ERROR_MAX_RETRIES = 3 - DEFAULT_BUILD_CHILD_PIPELINE_NAME = 'Build Child Pipeline' - def __init__(self, project_id: Union[int, str, None] = None): + def __init__(self, project_id: int | str | None = None): config_data_from_env = os.getenv('PYTHON_GITLAB_CONFIG') if config_data_from_env: # prefer to load config from env variable with tempfile.NamedTemporaryFile('w', delete=False) as temp_file: temp_file.write(config_data_from_env) - config_files = [temp_file.name] # type: Optional[List[str]] + config_files: list[str] | None = [temp_file.name] else: # otherwise try to use config file at local filesystem config_files = None self._init_gitlab_inst(project_id, config_files) @retry - def _init_gitlab_inst(self, project_id: Optional[int], config_files: Optional[List[str]]) -> None: + def _init_gitlab_inst(self, project_id: int | None, config_files: list[str] | None) -> None: gitlab_id = os.getenv('LOCAL_GITLAB_HTTPS_HOST') # if None, will use the default gitlab server self.gitlab_inst = gitlab.Gitlab.from_config(gitlab_id=gitlab_id, config_files=config_files) @@ -109,7 +106,7 @@ class Gitlab(object): self.project = None @retry - def get_project_id(self, name: str, namespace: Optional[str] = None) -> int: + def get_project_id(self, name: str, namespace: str | None = None) -> int: """ search project ID by name @@ -117,7 +114,7 @@ class Gitlab(object): :param namespace: namespace to match when we have multiple project with same name :return: project ID """ - projects = self.gitlab_inst.projects.list(search=name) + projects = self.gitlab_inst.projects.list(search=name, get_all=True) res = [] for project in projects: if namespace is None: @@ -152,7 +149,7 @@ class Gitlab(object): archive_file.extractall(destination) @retry - def download_artifact(self, job_id: int, artifact_path: List[str], destination: Optional[str] = None) -> List[bytes]: + def download_artifact(self, job_id: int, artifact_path: list[str], destination: str | None = None) -> list[bytes]: """ download specific path of job artifacts and extract to destination. @@ -169,7 +166,7 @@ class Gitlab(object): try: data = job.artifact(a_path) # type: bytes except gitlab.GitlabGetError as e: - logging.error("Failed to download '{}' from job {}".format(a_path, job_id)) + logging.error(f"Failed to download '{a_path}' from job {job_id}") raise e raw_data_list.append(data) if destination: @@ -185,7 +182,7 @@ class Gitlab(object): return raw_data_list @retry - def find_job_id(self, job_name: str, pipeline_id: Optional[str] = None, job_status: str = 'success') -> List[Dict]: + def find_job_id(self, job_name: str, pipeline_id: str | None = None, job_status: str = 'success') -> list[dict]: """ Get Job ID from job name of specific pipeline @@ -208,8 +205,9 @@ class Gitlab(object): return job_id_list @retry - def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None, - cache_dir: Optional[str] = None) -> str: + def download_archive( + self, ref: str, destination: str, project_id: int | None = None, cache_dir: str | None = None + ) -> str: """ Download archive of certain commit of a repository and extract to destination path @@ -233,10 +231,11 @@ class Gitlab(object): try: project.repository_archive(sha=ref, streamed=True, action=fw.write) except gitlab.GitlabGetError as e: - logging.error('Failed to archive from project {}'.format(project_id)) + logging.error(f'Failed to archive from project {project_id}') raise e - logging.info('Downloaded archive size: {:.03f}MB'.format( - float(os.path.getsize(local_archive_file)) / (1024 * 1024))) + logging.info( + f'Downloaded archive size: {float(os.path.getsize(local_archive_file)) / (1024 * 1024):.03f}MB' + ) return self.decompress_archive(local_archive_file, destination) @@ -245,10 +244,10 @@ class Gitlab(object): try: project.repository_archive(sha=ref, streamed=True, action=temp_file.write) except gitlab.GitlabGetError as e: - logging.error('Failed to archive from project {}'.format(project_id)) + logging.error(f'Failed to archive from project {project_id}') raise e - logging.info('Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(temp_file.name)) / (1024 * 1024))) + logging.info(f'Downloaded archive size: {float(os.path.getsize(temp_file.name)) / (1024 * 1024):.03f}MB') return self.decompress_archive(temp_file.name, destination) @@ -280,40 +279,32 @@ class Gitlab(object): job = self.project.jobs.get(job_id) return ','.join(job.tag_list) - def get_downstream_pipeline_ids(self, main_pipeline_id: int) -> List[int]: + def get_downstream_pipeline_ids(self, main_pipeline_id: int) -> list[int]: """ - Retrieve the IDs of all downstream child pipelines for a given main pipeline. + Retrieve the IDs of all downstream child pipelines for a given main pipeline, + recursing through arbitrarily nested child pipelines. :param main_pipeline_id: The ID of the main pipeline to start the search. - :return: A list of IDs of all downstream child pipelines. + :return: A list of IDs of all downstream child pipelines (all levels). """ - bridge_pipeline_ids = [] - child_pipeline_ids = [] + child_pipeline_ids: list[int] = [] - main_pipeline_bridges = self.project.pipelines.get(main_pipeline_id).bridges.list() - for bridge in main_pipeline_bridges: + pipeline_bridges = self.project.pipelines.get(main_pipeline_id).bridges.list() + for bridge in pipeline_bridges: downstream_pipeline = bridge.attributes.get('downstream_pipeline') if not downstream_pipeline: continue - bridge_pipeline_ids.append(downstream_pipeline['id']) - - for bridge_pipeline_id in bridge_pipeline_ids: - child_pipeline_ids.append(bridge_pipeline_id) - bridge_pipeline = self.project.pipelines.get(bridge_pipeline_id) - - if not bridge_pipeline.name == self.DEFAULT_BUILD_CHILD_PIPELINE_NAME: + downstream_pipeline_id = downstream_pipeline.get('id') + if downstream_pipeline_id is None: continue - child_bridges = bridge_pipeline.bridges.list() - for child_bridge in child_bridges: - downstream_child_pipeline = child_bridge.attributes.get('downstream_pipeline') - if not downstream_child_pipeline: - continue - child_pipeline_ids.append(downstream_child_pipeline.get('id')) + child_pipeline_ids.append(downstream_pipeline_id) + # recurse to collect further nested (grandchild+) pipelines + child_pipeline_ids.extend(self.get_downstream_pipeline_ids(downstream_pipeline_id)) - return [pid for pid in child_pipeline_ids if pid is not None] + return child_pipeline_ids - def retry_failed_jobs(self, pipeline_id: int, retry_allowed_failures: bool = False) -> List[int]: + def retry_failed_jobs(self, pipeline_id: int, retry_allowed_failures: bool = False) -> list[int]: """ Retry failed jobs for a specific pipeline. Optionally include jobs marked as 'allowed failures'. @@ -370,10 +361,10 @@ def main() -> None: gitlab_inst.download_archive(args.ref, args.destination) elif args.action == 'get_project_id': ret = gitlab_inst.get_project_id(args.project_name) - print('project id: {}'.format(ret)) + print(f'project id: {ret}') elif args.action == 'retry_failed_jobs': res = gitlab_inst.retry_failed_jobs(args.pipeline_id, args.retry_allowed_failures) - print('jobs retried successfully: {}'.format(res)) + print(f'jobs retried successfully: {res}') elif args.action == 'get_job_tags': ret = gitlab_inst.get_job_tags(args.job_id) print(ret) diff --git a/tools/ci/utils.sh b/tools/ci/utils.sh index a4c3a4875b3..9fcea7fd53a 100644 --- a/tools/ci/utils.sh +++ b/tools/ci/utils.sh @@ -164,38 +164,3 @@ function join_by { printf %s "$f" "${@/#/$d}" fi } - -function is_based_on_commits() { - # This function would accept space-separated args as multiple commits. - # The return value would be 0 if current HEAD is based on any of the specified commits. - # - # In our CI, we use environment variable $REQUIRED_ANCESTOR_COMMITS to declare the ancestor commits. - # Please remember to set one commit for each release branch. - - commits=$* - if [[ -z $commits ]]; then - info "Not specifying commits that branches should be based on, skipping check..." - return 0 - fi - - commits_str="$(join_by " or " $commits)" # no doublequotes here, passing array - - info "Checking if current branch is based on $commits_str..." - for i in $commits; do - if git merge-base --is-ancestor "$i" HEAD >/dev/null 2>&1; then - info "Current branch is based on $i" - return 0 - else - info "Current branch is not based on $i" - fi - done - - error "The base commit of your branch is too old." - error "The branch should be more recent than either of the following commits:" - error " $commits_str" - error "To fix the issue:" - error " - If your merge request is 'Draft', or has conflicts with the target branch, rebase it to the latest master or release branch" - error " - Otherwise, simply run a new pipeline." - - return 1 -}