From d61d931d33497984eced1755090e596b1b3d18dd Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Wed, 22 Jul 2026 16:29:45 +0200 Subject: [PATCH] ci: apply common-scripts CI refactor --- .gitlab-ci.yml | 19 +- .gitlab/ci/README.md | 71 +----- .gitlab/ci/build.yml | 61 +---- .gitlab/ci/common.yml | 235 ++---------------- .gitlab/ci/default-build-test-rules.yml | 21 -- .gitlab/ci/host-test.yml | 8 +- .gitlab/ci/pre_check.yml | 105 +------- .gitlab/ci/rules.yml | 13 - .gitlab/ci/static-code-analysis.yml | 81 ------ .gitlab/ci/test-win.yml | 10 +- .pre-commit-config.yaml | 10 +- conftest.py | 2 +- sonar-project.properties | 24 -- tools/ci/check_build_test_rules.py | 31 +-- tools/ci/ci_get_mr_info.py | 165 ------------ tools/ci/deploy_docs.py | 217 ---------------- .../generate_target_test_child_pipeline.py | 6 +- .../templates/.dynamic_jobs.yml | 10 +- tools/ci/exclude_check_tools_files.txt | 3 - tools/ci/executable-list.txt | 1 - tools/ci/generate_rules.py | 8 +- tools/ci/get_all_test_results.py | 84 ------- tools/ci/gitlab_yaml_linter.py | 31 +-- tools/ci/idf_ci_utils.py | 190 +++++++------- tools/ci/idf_pytest/plugin.py | 50 ---- tools/ci/python_packages/gitlab_api.py | 65 ++--- tools/ci/utils.sh | 35 --- 27 files changed, 210 insertions(+), 1346 deletions(-) delete mode 100644 .gitlab/ci/default-build-test-rules.yml delete mode 100644 sonar-project.properties delete mode 100644 tools/ci/ci_get_mr_info.py delete mode 100755 tools/ci/deploy_docs.py delete mode 100644 tools/ci/get_all_test_results.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c1a6a390931..e36aa95780a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,11 +2,25 @@ include: - project: "ci/actions/common" + ref: "v2" file: - "templates/stable/default-workflow.yml" + - "templates/idf/pre_check.yml" + - "templates/idf/build.yml" - "templates/idf/deploy-github.yml" - "templates/idf/deploy-docs.yml" - "templates/idf/integration-test.yml" + - project: "ci/actions/common" + ref: "v2" + file: "templates/idf/sync-files.yml" + inputs: + job_name: "sync-pre-commit-config" + remote_repo_url: "https://__token__:${CI_SYNC_REPO_TOKEN}@${CI_SERVER_FQDN}/ci/images.git" + ref_name: "ci/sync_idf_pre_commit_config" + file_mappings: + - ".pre-commit-config.yaml:pre-commit/idf/.pre-commit-config.v5.5.yaml" + changes: + - ".pre-commit-config.yaml" - project: "ci/actions/common" file: "templates/idf/build-docs.yml" inputs: @@ -20,13 +34,14 @@ include: DOCTGT: "esp32p4" - ".gitlab/ci/danger.yml" - - ".gitlab/ci/common.yml" + - ".gitlab/ci/common.yml" # has to be placed after danger.yml, since danger.yml defined stages. - ".gitlab/ci/rules.yml" + # by stages - ".gitlab/ci/manual_gate.yml" - ".gitlab/ci/upload_cache.yml" + - ".gitlab/ci/pre_check.yml" - ".gitlab/ci/static-code-analysis.yml" - ".gitlab/ci/pre_commit.yml" - - ".gitlab/ci/pre_check.yml" - ".gitlab/ci/build.yml" - ".gitlab/ci/host-test.yml" - ".gitlab/ci/post_deploy.yml" diff --git a/.gitlab/ci/README.md b/.gitlab/ci/README.md index 41f0640037a..7b109d30689 100644 --- a/.gitlab/ci/README.md +++ b/.gitlab/ci/README.md @@ -245,76 +245,15 @@ We're using the latest version of [idf-build-apps][idf-build-apps]. Please refer In ESP-IDF CI, there's a few more special rules are additionally supported to disable the check app dependencies feature: - Add MR labels `BUILD_AND_TEST_ALL_APPS` +- Pipeline variable `IDF_CI_SELECT_ALL_PYTEST_CASES=1` - Run in protected branches ## Upload/Download Artifacts to Internal Minio Server -### Users Without Access to Minio +Please refer to the documentation [here](https://docs.espressif.com/projects/idf-ci/en/latest/guides/cli.html) -If you don't have access to the internal Minio server, you can still download the artifacts from the shared link in the job log. +## Revision Marker Naming Rules -The log will look like this: +When specifying chip revisions for testing markers, configurations or filenames, strictly follow the format `esp32_rev` (or similarly `_rev` when brief). -```shell -Pipeline ID : 587355 -Job name : build_clang_test_apps_esp32 -Job ID : 40272275 -Created archive file: 40272275.zip, uploading as 587355/build_dir_without_map_and_elf_files/build_clang_test_apps_esp32/40272275.zip -Please download the archive file includes build_dir_without_map_and_elf_files from [INTERNAL_URL] -``` - -### Users With Access to Minio - -#### Env Vars for Minio - -Minio takes these env vars to connect to the server: - -- `IDF_S3_SERVER` -- `IDF_S3_ACCESS_KEY` -- `IDF_S3_SECRET_KEY` -- `IDF_S3_BUCKET` - -#### Artifacts Types and File Patterns - -The artifacts types and corresponding file patterns are defined in tools/ci/artifacts_handler.py, inside `ArtifactType` and `TYPE_PATTERNS_DICT`. - -#### Upload - -```shell -python tools/ci/artifacts_handler.py upload -``` - - will upload the files that match the file patterns to minio object storage with name: - -`///.zip` - -For example, job 39043328 will upload these four files: - -- `575500/map_and_elf_files/build_pytest_examples_esp32/39043328.zip` -- `575500/build_dir_without_map_and_elf_files/build_pytest_examples_esp32/39043328.zip` -- `575500/logs/build_pytest_examples_esp32/39043328.zip` -- `575500/size_reports/build_pytest_examples_esp32/39043328.zip` - -#### Download - -You may run - -```shell -python tools/ci/artifacts_handler.py download --pipeline_id -``` - -to download all files of the pipeline, or - -```shell -python tools/ci/artifacts_handler.py download --pipeline_id --job_name -``` - -to download all files with the specified job name or pattern, or - -```shell -python tools/ci/artifacts_handler.py download --pipeline_id --job_name --type ... -``` - -to download all files with the specified job name or pattern and artifact type(s). - -You may check all detailed documentation with `python tools/ci/artifacts_handler.py download -h` +> e.g., `esp32_rev3`, `esp32c2_rev2`, `esp32c3_rev1`, `esp32p4_rev1`, `esp32c5_rev1` diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index b67f345368e..ae4d74ad4c9 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -110,61 +110,6 @@ build_clang_test_apps_riscv: #################### # Dynamic Pipeline # #################### -generate_build_child_pipeline: - extends: - - .build_template - tags: [fast_run, shiny] - dependencies: # set dependencies to null to avoid missing artifacts issue - needs: - - pipeline_variables - - job: baseline_manifest_sha - optional: true - artifacts: - paths: - - build_child_pipeline.yml - - test_related_apps.txt - - non_test_related_apps.txt - expire_in: 1 week - when: always - script: - - run_cmd idf-ci --debug gitlab build-child-pipeline - -p components - -p examples - -p tools/test_apps - --modified-files $MR_MODIFIED_FILES - -build_child_pipeline: - stage: build - needs: - - job: fast_template_app - optional: true - artifacts: false - - pipeline_variables - - generate_build_child_pipeline - variables: - MR_MODIFIED_COMPONENTS: $MR_MODIFIED_COMPONENTS - MR_MODIFIED_FILES: $MR_MODIFIED_FILES - PARENT_PIPELINE_ID: $CI_PIPELINE_ID - # https://gitlab.com/gitlab-org/gitlab/-/issues/214340 - inherit: - variables: false - trigger: - include: - - artifact: build_child_pipeline.yml - job: generate_build_child_pipeline - strategy: depend - -generate_prebuild_report: - extends: - - .build_template - tags: [fast_run, shiny] - needs: - - pipeline_variables - artifacts: - paths: - - prebuild_report.html - expire_in: 1 week - when: always - script: - - run_cmd idf-ci build collect --format html -o prebuild_report.html - - echo "Report generated at https://${CI_PAGES_HOSTNAME}:${CI_SERVER_PORT}/-/esp-idf/-/jobs/${CI_JOB_ID}/artifacts/prebuild_report.html" +# !!! +# configurations moved to ci/actions/common/idf/ +# !!! diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index de04a236d7e..6eacd621463 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -14,6 +14,15 @@ stages: - deploy - post_deploy +include: + - project: "ci/actions/common" + ref: "v2" + file: "templates/idf/common-scripts.yml" + inputs: + ci_python_constraint_file: "espidf.constraints.v5.5.txt" + macos_image: "registry.hub.docker.com/espressif/macos-idf:sequoia-v5.5" + windows_image: "windows_11_idf_v5_5" + variables: # System environment @@ -58,30 +67,29 @@ variables: ESP_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v5.5:1" ESP_IDF_DOC_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v5.5-docs:1" TARGET_TEST_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v5.5-target-test:1" - SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:5" # Set this variable to the branch of idf-constraints repo in order to test a custom Python constraint file. The # branch name must be without the remote part ("origin/"). Keep the variable empty in order to use the constraint # file from https://dl.espressif.com/dl/esp-idf. - CI_PYTHON_CONSTRAINT_BRANCH: "" + # CI_PYTHON_CONSTRAINT_BRANCH could be defined as component input `ci_python_constraint_branch` in common-scripts above # Update the filename for a specific ESP-IDF release. It is used only with CI_PYTHON_CONSTRAINT_BRANCH. - CI_PYTHON_CONSTRAINT_FILE: "espidf.constraints.v5.5.txt" + # CI_PYTHON_CONSTRAINT_FILE is defined as component input `ci_python_constraint_file` in common-scripts above # Set this variable to repository name of a Python tool you wish to install and test in the context of ESP-IDF CI. # Keep the variable empty when not used. - CI_PYTHON_TOOL_REPO: "" + # CI_PYTHON_TOOL_REPO could be defined as component input `ci_python_tool_repo` in common-scripts above # Set this variable to the branch of a Python tool repo specified in CI_PYTHON_TOOL_REPO. The # branch name must be without the remote part ("origin/"). Keep the variable empty when not used. # This is used only if CI_PYTHON_TOOL_REPO is not empty. - CI_PYTHON_TOOL_BRANCH: "" + # CI_PYTHON_TOOL_BRANCH could be defined as component input `ci_python_tool_branch` in common-scripts above # Set this variable to Clang toolchain distro URL to be used. # NOTE: We have separate toolchains for Xtensa and RISCV, therefore jobs for one arch will fail. # This is OK as far as we use CI_CLANG_DISTRO_URL for pre-release tests purposes only. # Keep the variable empty when not used. - CI_CLANG_DISTRO_URL: "" + # CI_CLANG_DISTRO_URL could be defined as component input `ci_clang_distro_url` in common-scripts above # Set this variable to specify the file name for the known failure cases. KNOWN_FAILURE_CASES_FILE_NAME: "5.5.txt" @@ -103,227 +111,19 @@ variables: ################################################ # `before_script` and `after_script` Templates # ################################################ -.common_before_scripts: &common-before_scripts | - source tools/ci/utils.sh - - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS - - if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then - export IDF_MIRROR_PREFIX_MAP= - fi - - if echo "$CI_MERGE_REQUEST_LABELS" | egrep "(^|,)include_nightly_run(,|$)"; then - export INCLUDE_NIGHTLY_RUN="1" - export NIGHTLY_RUN="1" - fi - - # configure cmake related flags - source tools/ci/configure_ci_environment.sh - - if [[ "$CI_CCACHE_STATS" == 1 ]] && command -v ccache >/dev/null 2>&1 && [[ -n "$CCACHE_STATSLOG" ]]; then - mkdir -p "$(dirname "$CCACHE_STATSLOG")" - rm -f "$CCACHE_STATSLOG" - fi - - # add extra python packages - export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH" - -.setup_tools_and_idf_python_venv: &setup_tools_and_idf_python_venv | - # must use after setup_tools_except_target_test - # otherwise the export.sh won't work properly - - # download constraint file for dev - if [[ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]]; then - wget -O /tmp/constraint.txt --header="Authorization:Bearer ${ESPCI_TOKEN}" "${GITLAB_HTTP_SERVER}/api/v4/projects/2581/repository/files/${CI_PYTHON_CONSTRAINT_FILE}/raw?ref=${CI_PYTHON_CONSTRAINT_BRANCH}" - mkdir -p ~/.espressif - mv /tmp/constraint.txt ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE} - fi - - # Mirror - if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then - export IDF_MIRROR_PREFIX_MAP= - fi - # Optimize pip install - if echo "${CI_RUNNER_TAGS}" | grep "shiny"; then - export PIP_INDEX_URL="${PIP_INDEX_URL_SHINY}" - fi - if [[ "$(uname -m)" == "x86_64" ]] || [[ "$(uname -m)" == "aarch64" ]]; then - export IDF_PIP_WHEELS_URL="" - fi - - # install.sh - if [[ "${CI_JOB_STAGE}" != "target_test" ]]; then - section_start "running_install_sh" "Running install.sh" - if [[ "${CI_JOB_STAGE}" == "build_doc" || "${CI_JOB_GROUP_NAME}" == *docs* ]]; then - run_cmd bash install.sh --enable-ci --enable-docs - else - run_cmd bash install.sh --enable-ci - fi - section_end "running_install_sh" - else - section_start "install_python_env" "Install Python environment, skip required tools check" - run_cmd python tools/idf_tools.py install-python-env --features ci,test-specific - export IDF_SKIP_TOOLS_CHECK=1 - section_end "install_python_env" - fi - - section_start "source_export" "Source export.sh" - source ./export.sh - section_end "source_export" - - # Eager upgrade of CI dependencies - # Done after sourcing export.sh so that we could easily invoke the right pip - section_start "upgrade_ci_dependencies" "Upgrading CI dependencies" - pip install --upgrade --upgrade-strategy=eager -r $IDF_PATH/tools/requirements/requirements.ci.txt -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE} - # we need the latest DEV release for esptool to be installed in every job without bumping the minimal requirement in - # the constraint file. - pip install --upgrade --pre esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE} - section_end "upgrade_ci_dependencies" - - REEXPORT_NEEDED=0 - if [[ ! -z "$INSTALL_EXTRA_TOOLS" ]]; then - section_start "installing_optional_tools" "Install optional tools ${INSTALL_EXTRA_TOOLS}" - run_cmd $IDF_PATH/tools/idf_tools.py --non-interactive install $INSTALL_EXTRA_TOOLS - section_end "installing_optional_tools" - - REEXPORT_NEEDED=1 - fi - - # Install esp-clang if necessary (esp-clang is separately installed) - if [[ "$IDF_TOOLCHAIN" == "clang" && -z "$CI_CLANG_DISTRO_URL" ]]; then - $IDF_PATH/tools/idf_tools.py --non-interactive install esp-clang - - REEXPORT_NEEDED=1 - fi - - if [[ $REEXPORT_NEEDED -eq 1 ]]; then - section_start "re_source_export" "Re-source export.sh" - source ./export.sh - section_end "re_source_export" - fi - - # Custom clang toolchain - if [[ "$IDF_TOOLCHAIN" == "clang" && ! -z "$CI_CLANG_DISTRO_URL" ]]; then - echo "Using custom clang from ${CI_CLANG_DISTRO_URL}" - wget $CI_CLANG_DISTRO_URL - ARCH_NAME=$(basename $CI_CLANG_DISTRO_URL) - tar -x -f $ARCH_NAME - export PATH=$PWD/esp-clang/bin:$PATH - fi - - # Custom OpenOCD - if [[ "$CI_JOB_STAGE" == "target_test" ]]; then - machine="$(uname -m)" - if [[ "$machine" == "armv7l" ]] ; then - OOCD_DISTRO_URL="$OOCD_DISTRO_URL_ARMHF" - elif [[ "$machine" == "aarch64" ]] ; then - OOCD_DISTRO_URL="$OOCD_DISTRO_URL_ARM64" - fi - if [[ ! -z "$OOCD_DISTRO_URL" ]]; then - echo "Using custom OpenOCD from ${OOCD_DISTRO_URL}" - wget $OOCD_DISTRO_URL - ARCH_NAME=$(basename $OOCD_DISTRO_URL) - tar -x -f $ARCH_NAME - export OPENOCD_SCRIPTS=$PWD/openocd-esp32/share/openocd/scripts - export PATH=$PWD/openocd-esp32/bin:$PATH - fi - fi - - if [[ -n "$CI_PYTHON_TOOL_REPO" ]]; then - git clone --quiet --depth=1 -b ${CI_PYTHON_TOOL_BRANCH} https://gitlab-ci-token:${ESPCI_TOKEN}@${GITLAB_HTTPS_HOST}/espressif/${CI_PYTHON_TOOL_REPO}.git - pip install ./${CI_PYTHON_TOOL_REPO} - rm -rf ${CI_PYTHON_TOOL_REPO} - fi - - info "setup tools and python venv done" - -.show_ccache_statistics: &show_ccache_statistics | - # Show ccache statistics if enabled globally - section_start "ccache_show_stats" "Show ccache statistics" - if [[ "$CI_CCACHE_STATS" == 1 ]] && command -v ccache >/dev/null 2>&1; then - if ccache --help 2>/dev/null | grep -q -- '--show-log-stats'; then - if [[ -n "$CCACHE_STATSLOG" && -f "$CCACHE_STATSLOG" ]]; then - ccache --show-log-stats -vv - else - echo "INFO: No per-job ccache statistics were recorded" - fi - else - ccache --show-stats -vv - fi - fi || true - section_end "ccache_show_stats" - -.upload_failed_job_log_artifacts: &upload_failed_job_log_artifacts | - if [ $CI_JOB_STATUS = "failed" ]; then - run_cmd idf-ci gitlab upload-artifacts --type log - fi - .before_script:minimal: before_script: - - *common-before_scripts + - !reference [.common_before_scripts, minimal] .after_script:build: after_script: - - source tools/ci/utils.sh - - source tools/ci/configure_ci_environment.sh - - *show_ccache_statistics + - !reference [.common_after_scripts, ccache] - run_cmd idf-ci gitlab upload-artifacts --type junit - | if [ $CI_JOB_STATUS = "failed" ]; then run_cmd idf-ci gitlab upload-artifacts --type log fi -.brew-macos-settings: - variables: - GIT_STRATEGY: fetch - IDF_CCACHE_ENABLE: "0" - CCACHE_DIR: "/var/tmp/cache/idf_ccache" - tags: - - macos-tart - image: macos-sequoia-idf-v5.5 - after_script: [] # ccache now is disabled for macos brew runners - timeout: 30m - -.brew-windows-settings: - image: windows_11_idf_v6_0 - tags: [windows-vm, brew] - variables: - GIT_STRATEGY: fetch - LC_ALL: C.UTF-8 - # git clean at the final cleanup stage stuck on Windows VM - # a new VM for each job anyway, so we can skip cleaning the workspace - GIT_CLEAN_FLAGS: none -# IDF_CCACHE_ENABLE: "1" -# CCACHE_DIR: "C:/cache/idf_ccache" - before_script: - - if ($env:IDF_DONT_USE_MIRRORS) { $env:IDF_MIRROR_PREFIX_MAP = '' } - - ./install.ps1 --enable-ci - - ./export.ps1 - - $env:PYTHONPATH = "$env:PYTHONPATH;$env:IDF_PATH\tools;$env:IDF_PATH\tools\esp_app_trace;$env:IDF_PATH\components\partition_table;$env:IDF_PATH\tools\ci\python_packages" - - python "${SUBMODULE_FETCH_TOOL}" -s "${SUBMODULES_TO_FETCH}" - # Sync VM clock. net start/stop exit 2 when the service is already in that state; do not fail CI. - - | - function Invoke-NetService($Action, $Name) { - $output = net $Action $Name 2>&1 - if ($LASTEXITCODE -eq 2) { - Write-Host "net $Action $Name already in requested state: $output" - $global:LASTEXITCODE = 0 - return - } - if ($LASTEXITCODE -ne 0) { - Write-Host $output - exit $LASTEXITCODE - } - } - Invoke-NetService start w32time - w32tm /config /manualpeerlist:"time.google.com,0x9 pool.ntp.org,0x9" /syncfromflags:manual /reliable:yes /update - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - Invoke-NetService stop w32time - Invoke-NetService start w32time - w32tm /resync - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - after_script: [] - ############# # `default` # ############# @@ -338,8 +138,7 @@ default: - .cache/submodule_archives policy: pull before_script: - - *common-before_scripts - - *setup_tools_and_idf_python_venv + - !reference [.common_before_scripts, with_tools] - add_gitlab_ssh_keys - fetch_submodules - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS} diff --git a/.gitlab/ci/default-build-test-rules.yml b/.gitlab/ci/default-build-test-rules.yml deleted file mode 100644 index 62b2bf09d0e..00000000000 --- a/.gitlab/ci/default-build-test-rules.yml +++ /dev/null @@ -1,21 +0,0 @@ -# this file support two keywords: -# - extra_default_build_targets: -# besides of the SUPPORTED_TARGETS in IDF, -# enable build for the specified targets by default as well. -# !!! DEPRECATED: use `additional_build_targets` in .idf_build_apps.toml instead -# -# - bypass_check_test_targets: -# suppress the check_build_test_rules check-test-script warnings for the specified targets -# -# This file should ONLY be used during bringup. Should be reset to empty after the bringup process - -bypass_check_test_targets: - - esp32h21 - - esp32h4 - - esp32c5 - -# -# These lines would -# - enable the README.md check for esp32c6. Don't forget to add the build jobs in .gitlab/ci/build.yml -# - disable the test script check with the manifest file. -# diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index 3cd3a96ade2..e6d29784392 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -401,7 +401,7 @@ test_pytest_linux: test_pytest_macos: extends: - .host_test_template - - .brew-macos-settings + - .macos-settings artifacts: paths: - XUNIT_RESULT.xml @@ -476,6 +476,8 @@ test_pytest_macos: - pytest -k cmake --cleanup-idf-copy + --parallel-count ${CI_NODE_TOTAL:-1} + --parallel-index ${CI_NODE_INDEX:-1} --work-dir ${CI_PROJECT_DIR}/test_build_system --junitxml ${CI_PROJECT_DIR}/XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} @@ -487,7 +489,7 @@ pytest_build_system: pytest_build_system_macos: extends: - .test_build_system_template - - .brew-macos-settings + - .macos-settings - .rules:build:macos parallel: 3 @@ -497,7 +499,7 @@ pytest_build_system_minimal_cmake: pytest_build_system_macos_minimal_cmake: extends: - .test_build_system_minimal_cmake_template - - .brew-macos-settings + - .macos-settings - .rules:build:macos variables: INSTALL_EXTRA_TOOLS: ninja cmake@3.16.3 diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index 5f87d341e59..15166402113 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -1,9 +1,3 @@ -.pre_check_template: - stage: pre_check - image: $ESP_ENV_IMAGE - tags: [build, shiny] - dependencies: # set dependencies to null to avoid missing artifacts issue - check_version: # Don't run this for feature/bugfix branches, so that it is possible to modify # esp_idf_version.h in a branch before tagging the next version. @@ -140,106 +134,11 @@ check_submodule_sync: - git submodule update --recursive - echo "IDF was cloned from ${PUBLIC_IDF_URL} completely" -pipeline_variables: +check_gitlab_yaml: extends: - .pre_check_template - tags: [fast_run, shiny] - variables: - GIT_STRATEGY: "fetch" script: - - | - # Store the diff output in a temporary file - TEMP_FILE=$(mktemp) - # merged results pipelines, by default - if [[ -n $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ]]; then - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA - git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA - - git diff --name-only $CI_MERGE_REQUEST_TARGET_BRANCH_SHA...$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA > "$TEMP_FILE" - GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") - git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - # merge request pipelines, when the mr got conflicts - elif [[ -n $CI_MERGE_REQUEST_DIFF_BASE_SHA ]]; then - git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_COMMIT_SHA > "$TEMP_FILE" - GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") - # other pipelines, like the protected branches pipelines - elif [[ "$CI_COMMIT_BEFORE_SHA" != "0000000000000000000000000000000000000000" ]]; then - git fetch origin $CI_COMMIT_BEFORE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA > "$TEMP_FILE" - GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") - else - # pipeline source could be web, scheduler, etc. - git fetch origin $CI_COMMIT_SHA --depth=2 ${GIT_FETCH_EXTRA_FLAGS} - git diff --name-only $CI_COMMIT_SHA~1 $CI_COMMIT_SHA > "$TEMP_FILE" - GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") - fi - # MODIFIED_FILES is a list of files that changed, could be used everywhere - - MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | xargs) - - echo "MODIFIED_FILES=$MODIFIED_FILES" >> pipeline.env - # MR_MODIFIED_FILES and MR_MODIFIED_COMPONENTS are semicolon separated lists that is used in MR only - # for non MR pipeline, these are empty lists - - | - if [ -z "$CI_MERGE_REQUEST_IID" ]; then - echo "MR_MODIFIED_FILES=\"\"" >> pipeline.env - echo "MR_MODIFIED_COMPONENTS=\"\"" >> pipeline.env - else - MR_MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | tr '\n' ';') - echo "MR_MODIFIED_FILES=\"$MR_MODIFIED_FILES\"" >> pipeline.env - - MR_MODIFIED_COMPONENTS=$(run_cmd python tools/ci/ci_get_mr_info.py components --modified-files $MODIFIED_FILES | tr '\n' ';') - echo "MR_MODIFIED_COMPONENTS=\"$MR_MODIFIED_COMPONENTS\"" >> pipeline.env - fi - - | - if [ -n "$PIPELINE_COMMIT_SHA" ]; then - echo "PIPELINE_COMMIT_SHA=$PIPELINE_COMMIT_SHA" >> pipeline.env - fi - - echo "OOCD_DISTRO_URL_ARMHF=$OOCD_DISTRO_URL_ARMHF" >> pipeline.env - - echo "OOCD_DISTRO_URL_ARM64=$OOCD_DISTRO_URL_ARM64" >> pipeline.env - - run_cmd idf-ci gitlab pipeline-variables >> pipeline.env - - cat pipeline.env - - run_cmd idf-ci gitlab upload-artifacts --type env - artifacts: - reports: - dotenv: pipeline.env - paths: - - pipeline.env - expire_in: 1 week - when: always - -baseline_manifest_sha: - extends: - - .pre_check_template - - .rules:dev-push - tags: [fast_run, shiny] - script: - - | - # merged results pipelines, by default - # diff between target-branch-head and merged-result-head - if [ -n "$CI_MERGE_REQUEST_TARGET_BRANCH_SHA" ]; then - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA --depth=1 - git checkout FETCH_HEAD - idf-build-apps dump-manifest-sha \ - --manifest-files $(find . -name ".build-test-rules.yml" | xargs) \ - --output .manifest_sha - # merge request pipelines, when the mr got conflicts - # diff between diff-base-sha and merge-request-head - elif [ -n "$CI_MERGE_REQUEST_DIFF_BASE_SHA" ]; then - git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 - git checkout FETCH_HEAD - idf-build-apps dump-manifest-sha \ - --manifest-files $(find . -name ".build-test-rules.yml" | xargs) \ - --output .manifest_sha - # other pipelines, like the protected branches pipelines - # not triggered in this job - fi - artifacts: - paths: - - .manifest_sha - expire_in: 1 week - when: always + - python tools/ci/gitlab_yaml_linter.py retry_failed_jobs: extends: diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index e2e95909e4d..6c1cd29093b 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -14,13 +14,6 @@ - "components/**/CMakeLists.txt" -.patterns-python-files: &patterns-python-files - - ".gitlab/ci/static-code-analysis.yml" - - "**/*.py" - -.patterns-sonarqube-files: &patterns-sonarqube-files - - "tools/ci/sonar_exclude_list.txt" - .patterns-build_components: &patterns-build_components # components files except "test*/" "host*/" folders # ?? to include folders less than 4 characters @@ -176,16 +169,10 @@ .if-protected-check: &if-protected-check if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) || $CI_COMMIT_TAG =~ /^qa-test/' -.if-protected-deploy: &if-protected-deploy - if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "api")' - .if-protected-branch-push: &if-protected-branch-push # rules:changes always evaluates to true for new branch pipelines or when there is no Git push event if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/) && $CI_PIPELINE_SOURCE == "push"' -.if-qa-test-tag: &if-qa-test-tag - if: '$CI_COMMIT_TAG =~ /^qa-test/' - .if-dev-push: &if-dev-push if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")' diff --git a/.gitlab/ci/static-code-analysis.yml b/.gitlab/ci/static-code-analysis.yml index 734ff76a2ec..034c01bdb6f 100644 --- a/.gitlab/ci/static-code-analysis.yml +++ b/.gitlab/ci/static-code-analysis.yml @@ -30,84 +30,3 @@ gcc_static_analyzer: - export EXTRA_CFLAGS= - export EXTRA_CXXFLAGS= - idf-build-apps build -p ${ANALYZING_APP} --disable-targets esp32h4 esp32h21 --no-check-warnings -# -## build stage -## Sonarqube related jobs put here for this reason: -## Here we have two jobs. code_quality_check and code_quality_report. -## -## code_quality_check will analyze the code changes between your MR and -## code repo stored in sonarqube server. The analysis result is only shown in -## the comments under this MR and won't be transferred to the server. -## -## code_quality_report will analyze and transfer both of the newly added code -## and the analysis result to the server. -## -## Put in the front to ensure that the newly merged code can be stored in -## sonarqube server ASAP, in order to avoid reporting unrelated code issues -#.sonar_scan_template: -# stage: build -# extends: .pre_check_template -# # full clone since this image does not support fetch --shallow-since-cutoff -# # shiny runners are used for full clone -# tags: [build, shiny] -# image: $SONARQUBE_SCANNER_IMAGE -# before_script: -# - source tools/ci/utils.sh -# - export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH" -# - fetch_submodules -# # Exclude the submodules, all paths ends with /** -# - submodules=$(get_all_submodules) -# # get all exclude paths specified in tools/ci/sonar_exclude_list.txt | ignore lines start with # | xargs | replace all to -# - custom_excludes=$(cat $CI_PROJECT_DIR/tools/ci/sonar_exclude_list.txt | grep -v '^#' | xargs | sed -e 's/ /,/g') -# # Exclude the report dir as well -# - export EXCLUSIONS="$custom_excludes,$submodules" -# - export SONAR_SCANNER_OPTS="-Xmx2048m" -# variables: -# GIT_DEPTH: 0 -# REPORT_PATTERN: clang_tidy_reports/**/*.txt -# artifacts: -# paths: -# - $REPORT_PATTERN -# expire_in: 1 week -# when: always -# dependencies: # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here. -# - clang_tidy_check -# -#code_quality_check: -# extends: -# - .sonar_scan_template -# - .rules:patterns:static-code-analysis-preview -# allow_failure: true # it's using exit code to indicate the code analysis result, -# # we don't want to block ci when critical issues founded -# script: -# - export CI_MERGE_REQUEST_COMMITS=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py commits --src-branch ${CI_COMMIT_REF_NAME} | tr '\n' ',') -# # test if this branch have merge request, if not, exit 0 -# - test -n "$CI_MERGE_REQUEST_IID" || exit 0 -# - test -n "$CI_MERGE_REQUEST_COMMITS" || exit 0 -# - sonar-scanner -# -Dsonar.analysis.mode=preview -# -Dsonar.branch.name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME -# -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN -# -Dsonar.exclusions=$EXCLUSIONS -# -Dsonar.gitlab.ci_merge_request_iid=$CI_MERGE_REQUEST_IID -# -Dsonar.gitlab.commit_sha=$CI_MERGE_REQUEST_COMMITS -# -Dsonar.gitlab.merge_request_discussion=true -# -Dsonar.gitlab.ref_name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME -# -Dsonar.host.url=$SONAR_HOST_URL -# -Dsonar.login=$SONAR_LOGIN -# -#code_quality_report: -# extends: -# - .sonar_scan_template -# - .rules:protected:check -# allow_failure: true # it's using exit code to indicate the code analysis result, -# # we don't want to block ci when critical issues founded -# script: -# - sonar-scanner -# -Dsonar.branch.name=$CI_COMMIT_REF_NAME -# -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN -# -Dsonar.exclusions=$EXCLUSIONS -# -Dsonar.gitlab.commit_sha=$PIPELINE_COMMIT_SHA -# -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME -# -Dsonar.host.url=$SONAR_HOST_URL -# -Dsonar.login=$SONAR_LOGIN diff --git a/.gitlab/ci/test-win.yml b/.gitlab/ci/test-win.yml index f8b6d8c6b48..9c0a90feac4 100644 --- a/.gitlab/ci/test-win.yml +++ b/.gitlab/ci/test-win.yml @@ -1,6 +1,6 @@ .host_test_win_template: extends: - - .brew-windows-settings + - .windows-settings - .rules:test:host_test stage: host_test dependencies: # set dependencies to null to avoid missing artifacts issue @@ -73,7 +73,7 @@ pytest_build_system_win: extends: - .test_build_system_template_win - .rules:labels:windows_pytest_build_system - parallel: 2 + parallel: 6 pytest_build_system_win_minimal_cmake: extends: @@ -82,7 +82,7 @@ pytest_build_system_win_minimal_cmake: variables: MINIMAL_CMAKE_VERSION: "3.16.3" script: - - python ${IDF_PATH}\tools\idf_tools.py install cmake@${MINIMAL_CMAKE_VERSION} + - python tools\idf_tools.py install cmake@${MINIMAL_CMAKE_VERSION} - $Env:PATH = "$Env:USERPROFILE\.espressif\tools\cmake\${MINIMAL_CMAKE_VERSION}\bin;$Env:PATH" - | $actualVersion = (& cmake --version).Split()[2] @@ -90,6 +90,6 @@ pytest_build_system_win_minimal_cmake: Write-Error "ERROR: Wrong CMake version! Detected: $actualVersion, but expected: $Env:MINIMAL_CMAKE_VERSION" exit 1 } - - cd ${IDF_PATH}\tools\test_build_system + - cd tools\test_build_system - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} - - pytest -k cmake --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} + - pytest -k cmake --junitxml ${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 888ebe27160..8889a892d3c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -146,7 +146,7 @@ repos: require_serial: true additional_dependencies: - PyYAML == 5.3.1 - - idf-build-apps~=2.13 + - idf-build-apps~=2.16 - id: sort-yaml-files name: sort yaml files entry: tools/ci/sort_yaml.py @@ -167,14 +167,6 @@ repos: language: python always_run: true require_serial: true - - id: gitlab-yaml-linter - name: Check gitlab yaml files - entry: tools/ci/gitlab_yaml_linter.py - language: python - files: '\.gitlab-ci\.yml|\.gitlab/ci/.+\.yml|\.gitmodules' - pass_filenames: false - additional_dependencies: - - PyYAML == 5.3.1 - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.0.1 hooks: diff --git a/conftest.py b/conftest.py index 307ebdbf198..cc576aa1b4c 100644 --- a/conftest.py +++ b/conftest.py @@ -598,7 +598,7 @@ def pytest_unconfigure(config: Config) -> None: config.pluginmanager.unregister(idf_local_plugin) -dut_artifacts_url = [] +dut_artifacts_url: t.List[str] = [] @pytest.hookimpl(hookwrapper=True) diff --git a/sonar-project.properties b/sonar-project.properties deleted file mode 100644 index 1c9d5882fb3..00000000000 --- a/sonar-project.properties +++ /dev/null @@ -1,24 +0,0 @@ -sonar.projectKey=esp-idf -sonar.projectName=Espressif IoT Development Framework -sonar.projectVersion=4.4 - -sonar.sources=. -sonar.sourceEncoding=UTF-8 - -sonar.inclusions=\ - **/*.c,**/*.cpp,**/*.h,**/*.hpp,\ - **/*.py - -#################### -# sonar-cxx-plugin # -#################### - -sonar.cxx.includeDirectories=/usr/include -sonar.cxx.file.suffixes=.cxx,.cpp,.cc,.c,.hxx,.hpp,.hh,.h - -####################### -# sonar-gitlab-plugin # -####################### - -sonar.gitlab.failure_notification_mode=exit-code -sonar.gitlab.project_id=103 diff --git a/tools/ci/check_build_test_rules.py b/tools/ci/check_build_test_rules.py index 5f9727aef2f..a4467f9d6b2 100755 --- a/tools/ci/check_build_test_rules.py +++ b/tools/ci/check_build_test_rules.py @@ -9,7 +9,6 @@ import typing as t from collections import defaultdict from pathlib import Path -import yaml from idf_build_apps import App from idf_build_apps import find_apps from idf_build_apps.constants import SUPPORTED_TARGETS @@ -40,6 +39,14 @@ USUAL_TO_FORMAL = { FORMAL_TO_USUAL = {v: k for k, v in USUAL_TO_FORMAL.items()} +# Targets under bringup: suppress check-test-scripts warnings for these. +# Reset to [] once bringup is complete. +BYPASS_CHECK_TEST_TARGETS: t.List[str] = [ + 'esp32h21', + 'esp32h4', + 'esp32c5', +] + def print_diff_table( list1: t.List[str], list2: t.List[str], title1: str, title2: str, excluded: t.Optional[t.List[str]] = None @@ -299,21 +306,9 @@ if __name__ == '__main__': readme_parser = action.add_parser('check-readmes') readme_parser.add_argument('paths', nargs='+', help='check under paths') - readme_parser.add_argument( - '-c', - '--config', - default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'), - help='config file', - ) test_parser = action.add_parser('check-test-scripts') test_parser.add_argument('paths', nargs='+', help='check under paths') - test_parser.add_argument( - '-c', - '--config', - default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'), - help='config file', - ) arg = parser.parse_args() check_dirs = set() @@ -344,14 +339,6 @@ if __name__ == '__main__': os.path.join(IDF_PATH, 'tools', 'cmakev2', 'test'), ] - _bypass_targets: t.List[str] = [] - if arg.config: - with open(arg.config) as fr: - configs = yaml.safe_load(fr) - - if configs: - _bypass_targets = configs.get('bypass_check_test_targets') or [] - os.environ.update( { 'IDF_PATH': IDF_PATH, @@ -369,5 +356,5 @@ if __name__ == '__main__': check_test_scripts( list(check_dirs), _exclude_dirs, - bypass_targets=_bypass_targets, + bypass_targets=BYPASS_CHECK_TEST_TARGETS, ) diff --git a/tools/ci/ci_get_mr_info.py b/tools/ci/ci_get_mr_info.py deleted file mode 100644 index 686775adc78..00000000000 --- a/tools/ci/ci_get_mr_info.py +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env python -# -# internal use only for CI -# get latest MR information by source branch -# -# SPDX-FileCopyrightText: 2020-2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -# -import argparse -import logging -import os -import subprocess -import typing as t -from pathlib import Path - -from gitlab_api import Gitlab -from idf_ci_utils import IDF_PATH - -if t.TYPE_CHECKING: - from gitlab.v4.objects import ProjectCommit, ProjectMergeRequest - - -def _get_mr_obj(source_branch: str) -> t.Optional['ProjectMergeRequest']: - gl = Gitlab(os.getenv('CI_PROJECT_ID', 'espressif/esp-idf')) - if not gl.project: - return None - - mrs = gl.project.mergerequests.list(state='opened', source_branch=source_branch) - if mrs: - return mrs[0] # one source branch can only have one opened MR at one moment - else: - return None - - -def get_mr_iid(source_branch: str) -> str: - mr = _get_mr_obj(source_branch) - if not mr: - return '' - else: - return str(mr.iid) - - -def get_mr_changed_files(source_branch: str) -> t.List[str]: - mr = _get_mr_obj(source_branch) - if not mr: - return [] - - git_output = subprocess.check_output( - ['git', 'diff', '--name-only', '--diff-filter=d', f'origin/{mr.target_branch}...origin/{source_branch}'] - ).decode('utf8') - - return [line.strip() for line in git_output.splitlines() if line.strip()] - - -def get_mr_commits(source_branch: str) -> t.List['ProjectCommit']: - mr = _get_mr_obj(source_branch) - if not mr: - return [] - - return list(mr.commits()) - - -_COMPONENT_NAME_DIR_RECORDS = {} - - -def get_modified_component(filepath: str) -> t.Optional[str]: - """Return the component name if the file is in a component directory, otherwise None.""" - try: - f_path = Path(filepath).resolve().relative_to(IDF_PATH) - except ValueError: # not in IDF_PATH - return None - - # skip md files, etc. - if f_path.suffix in ['.md', '.yml']: - return None - - # skip test_apps files - if 'test_apps' in f_path.parts: - return None - - component_parent_dirs = [f_path.parts[0]] - for part in f_path.parts[1:]: - if component_parent_dirs[-1] == 'components' or component_parent_dirs[-1].endswith('common_components'): - if part not in _COMPONENT_NAME_DIR_RECORDS: - logging.debug('Found component "%s" in path "%s"' % (part, component_parent_dirs)) - _COMPONENT_NAME_DIR_RECORDS[part] = component_parent_dirs - elif _COMPONENT_NAME_DIR_RECORDS.get(part) != component_parent_dirs: - logging.debug( - 'WARNING!!! Found component "%s" in path "%s" and "%s"' - % (part, component_parent_dirs, _COMPONENT_NAME_DIR_RECORDS.get(part)) - ) - - return part - - component_parent_dirs.append(part) - - return None - - -def get_mr_components( - source_branch: t.Optional[str] = None, modified_files: t.Optional[t.List[str]] = None -) -> t.List[str]: - components: t.Set[str] = set() - if modified_files is None: - if not source_branch: - raise RuntimeError('--src-branch is required if --modified-files is not provided') - - modified_files = get_mr_changed_files(source_branch) - - for f in modified_files: - modified_component = get_modified_component(f) - if modified_component: - components.add(modified_component) - - return list(components) - - -def get_target_in_tags(tags: str) -> str: - from idf_pytest.constants import TARGET_MARKERS - - for x in tags.split(','): - if x in TARGET_MARKERS: - return x - - raise RuntimeError(f'No target marker found in {tags}') - - -def _print_list(_list: t.List[str], separator: str = '\n') -> None: - print(separator.join(_list)) - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Get the latest merge request info by pipeline') - actions = parser.add_subparsers(dest='action', help='info type', required=True) - - common_args = argparse.ArgumentParser(add_help=False) - common_args.add_argument('--src-branch', help='source branch') - common_args.add_argument( - '--modified-files', - nargs='+', - help='space-separated list specifies the modified files. will be detected by --src-branch if not provided', - ) - - actions.add_parser('id', parents=[common_args]) - actions.add_parser('commits', parents=[common_args]) - actions.add_parser('components', parents=[common_args]) - target = actions.add_parser('target_in_tags') - target.add_argument('tags', help='comma separated tags, e.g., esp32,generic') - - args = parser.parse_args() - - if args.action == 'id': - if not args.src_branch: - raise RuntimeError('--src-branch is required') - print(get_mr_iid(args.src_branch)) - elif args.action == 'commits': - if not args.src_branch: - raise RuntimeError('--src-branch is required') - _print_list([commit.id for commit in get_mr_commits(args.src_branch)]) - elif args.action == 'components': - _print_list(get_mr_components(args.src_branch, args.modified_files)) - elif args.action == 'target_in_tags': - print(get_target_in_tags(args.tags)) - else: - raise NotImplementedError('not possible to get here') diff --git a/tools/ci/deploy_docs.py b/tools/ci/deploy_docs.py deleted file mode 100755 index 954958e738d..00000000000 --- a/tools/ci/deploy_docs.py +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/env python3 -# -# CI script to deploy docs to a webserver. Not useful outside of CI environment -# -# -# SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -# -import glob -import os -import os.path -import re -import stat -import subprocess -import sys -import tarfile -from typing import Any, List, Tuple - -import packaging.version - - -def env(variable: str, default: str=None) -> str: - """ Shortcut to return the expanded version of an environment variable """ - return os.path.expandvars(os.environ.get(variable, default) if default else os.environ[variable]) - - -# import sanitize_version from the docs directory, shared with here -sys.path.append(os.path.join(env('IDF_PATH'), 'docs')) -from sanitize_version import sanitize_version # noqa - - -def main() -> None: - # if you get KeyErrors on the following lines, it's probably because you're not running in Gitlab CI - git_ver = env('GIT_VER') # output of git describe --always - ci_ver = env('CI_COMMIT_REF_NAME', git_ver) # branch or tag we're building for (used for 'release' & URL) - - version = sanitize_version(ci_ver) - print('Git version: {}'.format(git_ver)) - print('CI Version: {}'.format(ci_ver)) - print('Deployment version: {}'.format(version)) - - if not version: - raise RuntimeError('A version is needed to deploy') - - build_dir = env('DOCS_BUILD_DIR') # top-level local build dir, where docs have already been built - - if not build_dir: - raise RuntimeError('Valid DOCS_BUILD_DIR is needed to deploy') - - url_base = env('DOCS_DEPLOY_URL_BASE') # base for HTTP URLs, used to print the URL to the log after deploying - - docs_server = env('DOCS_DEPLOY_SERVER') # ssh server to deploy to - docs_user = env('DOCS_DEPLOY_SERVER_USER') - docs_path = env('DOCS_DEPLOY_PATH') # filesystem path on DOCS_SERVER - - if not docs_server: - raise RuntimeError('Valid DOCS_DEPLOY_SERVER is needed to deploy') - - if not docs_user: - raise RuntimeError('Valid DOCS_DEPLOY_SERVER_USER is needed to deploy') - - docs_server = '{}@{}'.format(docs_user, docs_server) - - if not docs_path: - raise RuntimeError('Valid DOCS_DEPLOY_PATH is needed to deploy') - - print('DOCS_DEPLOY_SERVER {} DOCS_DEPLOY_PATH {}'.format(docs_server, docs_path)) - - tarball_path, version_urls = build_doc_tarball(version, git_ver, build_dir) - - deploy(version, tarball_path, docs_path, docs_server) - - print('Docs URLs:') - doc_deploy_type = os.getenv('TYPE') - for vurl in version_urls: - language, _, target = vurl.split('/') - tag = '{}_{}'.format(language, target) - url = '{}/{}/index.html'.format(url_base, vurl) # (index.html needed for the preview server) - url = re.sub(r'([^:])//', r'\1/', url) # get rid of any // that isn't in the https:// part - print('[document {}][{}] {}'.format(doc_deploy_type, tag, url)) - - # note: it would be neater to use symlinks for stable, but because of the directory order - # (language first) it's kind of a pain to do on a remote server, so we just repeat the - # process but call the version 'stable' this time - if is_stable_version(version): - print('Deploying again as stable version...') - tarball_path, version_urls = build_doc_tarball('stable', git_ver, build_dir) - deploy('stable', tarball_path, docs_path, docs_server) - - -def deploy(version: str, tarball_path: str, docs_path: str, docs_server: str) -> None: - def run_ssh(commands: List) -> None: - """ Log into docs_server and run a sequence of commands using ssh """ - print('Running ssh: {}'.format(commands)) - subprocess.run(['ssh', '-o', 'BatchMode=yes', docs_server, '-x', ' && '.join(commands)], check=True) - - # copy the version tarball to the server - run_ssh(['mkdir -p {}'.format(docs_path)]) - print('Running scp {} to {}'.format(tarball_path, '{}:{}'.format(docs_server, docs_path))) - subprocess.run(['scp', '-B', tarball_path, '{}:{}'.format(docs_server, docs_path)], check=True) - - tarball_name = os.path.basename(tarball_path) - - run_ssh(['cd {}'.format(docs_path), - 'rm -rf ./*/{}'.format(version), # remove any pre-existing docs matching this version - 'tar -zxvf {}'.format(tarball_name), # untar the archive with the new docs - 'rm {}'.format(tarball_name)]) - - # Note: deleting and then extracting the archive is a bit awkward for updating stable/latest/etc - # as the version will be invalid for a window of time. Better to do it atomically, but this is - # another thing made much more complex by the directory structure putting language before version... - - -def build_doc_tarball(version: str, git_ver: str, build_dir: str) -> Tuple[str, List]: - """ Make a tar.gz archive of the docs, in the directory structure used to deploy as - the given version """ - version_paths = [] - tarball_path = '{}/{}.tar.gz'.format(build_dir, version) - - # find all the 'html/' directories under build_dir - html_dirs = glob.glob('{}/**/html/'.format(build_dir), recursive=True) - print('Found %d html directories' % len(html_dirs)) - - pdfs = glob.glob('{}/**/latex/build/*.pdf'.format(build_dir), recursive=True) - print('Found %d PDFs in latex directories' % len(pdfs)) - - # add symlink for stable and latest and adds them to PDF blob - symlinks = create_and_add_symlinks(version, git_ver, pdfs) - - def not_sources_dir(ti: Any) -> Any: - print(type(ti)) - """ Filter the _sources directories out of the tarballs """ - if ti.name.endswith('/_sources'): - return None - - ti.mode |= stat.S_IWGRP # make everything group-writeable - return ti - - try: - os.remove(tarball_path) - except OSError: - pass - - with tarfile.open(tarball_path, 'w:gz') as tarball: - for html_dir in html_dirs: - # html_dir has the form '///html/' - target_dirname = os.path.dirname(os.path.dirname(html_dir)) - target = os.path.basename(target_dirname) - language = os.path.basename(os.path.dirname(target_dirname)) - - # when deploying, we want the top-level directory layout 'language/version/target' - archive_path = '{}/{}/{}'.format(language, version, target) - print("Archiving '{}' as '{}'...".format(html_dir, archive_path)) - tarball.add(html_dir, archive_path, filter=not_sources_dir) - version_paths.append(archive_path) - - for pdf_path in pdfs: - # pdf_path has the form '///latex/build' - latex_dirname = os.path.dirname(pdf_path) - pdf_filename = os.path.basename(pdf_path) - target_dirname = os.path.dirname(os.path.dirname(latex_dirname)) - target = os.path.basename(target_dirname) - language = os.path.basename(os.path.dirname(target_dirname)) - - # when deploying, we want the layout 'language/version/target/pdf' - archive_path = '{}/{}/{}/{}'.format(language, version, target, pdf_filename) - print("Archiving '{}' as '{}'...".format(pdf_path, archive_path)) - tarball.add(pdf_path, archive_path) - - for symlink in symlinks: - os.unlink(symlink) - - return (os.path.abspath(tarball_path), version_paths) - - -def create_and_add_symlinks(version: str, git_ver: str, pdfs: List) -> List: - """ Create symbolic links for PDFs for 'latest' and 'stable' releases """ - - symlinks = [] - if 'stable' in version or 'latest' in version: - for pdf_path in pdfs: - symlink_path = pdf_path.replace(git_ver, version) - os.symlink(pdf_path, symlink_path) - symlinks.append(symlink_path) - - pdfs.extend(symlinks) - print('Found %d PDFs in latex directories after adding symlink' % len(pdfs)) - - return symlinks - - -def is_stable_version(version: str) -> bool: - """ Heuristic for whether this is the latest stable release """ - if not version.startswith('v'): - return False # branch name - if '-' in version: - return False # prerelease tag - - git_out = subprocess.check_output(['git', 'tag', '-l']).decode('utf-8') - - versions = [v.strip() for v in git_out.split('\n')] - versions = [v for v in versions if re.match(r'^v[\d\.]+$', v.strip())] # include vX.Y.Z only - - versions_pack = [packaging.version.parse(v) for v in versions] - - max_version = max(versions_pack) - - if max_version.public != version[1:]: - print('Stable version is v{}. This version is {}.'.format(max_version.public, version)) - return False - else: - print('This version {} is the stable version'.format(version)) - return True - - -if __name__ == '__main__': - main() diff --git a/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py b/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py index 6abe2a83689..2cb0b484093 100644 --- a/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py +++ b/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py @@ -55,7 +55,7 @@ def main(output_filepath: str) -> None: exclude_runner_tags_matching = [] for _tag in known_warnings_dict.get('no_runner_tags', []): if '*' not in _tag: - exclude_runner_tags_set.add(_tag) + exclude_runner_tags_set.add(frozenset(_tag.split(','))) else: if res := _process_match_group(_tag): exclude_runner_tags_matching.append(res) @@ -66,7 +66,7 @@ def main(output_filepath: str) -> None: if exclude_runner_tags := os.getenv('EXCLUDE_RUNNER_TAGS'): for _tag in exclude_runner_tags.split(';'): if '*' not in _tag: - exclude_runner_tags_set.add(_tag) + exclude_runner_tags_set.add(frozenset(_tag.split(','))) else: if res := _process_match_group(_tag): exclude_runner_tags_matching.append(res) @@ -75,7 +75,7 @@ def main(output_filepath: str) -> None: additional_dict: dict[GroupKey, dict[str, t.Any]] = {} for key, grouped_cases in cases.grouped_cases.items(): # skip test cases with no runner tags - if ','.join(sorted(key.runner_tags)) in exclude_runner_tags_set: + if frozenset(key.runner_tags) in exclude_runner_tags_set: print(f'WARNING: excluding test cases with runner tags: {key.runner_tags}') continue diff --git a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml index 6f0b1949e4c..a145097f312 100644 --- a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml +++ b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml @@ -17,6 +17,9 @@ # Since the version 3.21 CMake passes source files and include dirs to ninja using absolute paths. # Needed for pytest junit reports. INSTALL_EXTRA_TOOLS: cmake + # Suppress Python warning messages during CI builds to keep logs clean and focused on errors. + # This prevents non-critical warnings (e.g., deprecation or resource warnings) from cluttering output. + PYTHONWARNINGS: "ignore" needs: - pipeline: $PARENT_PIPELINE_ID job: generate_build_child_pipeline @@ -40,7 +43,6 @@ - run_cmd idf-ci build run --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1} - --modified-files ${MR_MODIFIED_FILES} .dynamic_target_test_template: image: $TARGET_TEST_ENV_IMAGE @@ -56,6 +58,8 @@ needs: - pipeline: $PARENT_PIPELINE_ID job: pipeline_variables + - pipeline: $PARENT_PIPELINE_ID + job: snapshot_known_failure_cases artifacts: paths: - XUNIT_RESULT*.xml @@ -65,7 +69,11 @@ when: always expire_in: 1 week script: + # Merge the current known failure cases file with snapshot_known_failure_cases file captured at pipeline start + # to avoid retry failures caused by later updates to the known failure cases file. - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} + - printf '\n' >> ${KNOWN_FAILURE_CASES_FILE_NAME} + - cat artifacts/${KNOWN_FAILURE_CASES_FILE_NAME}.freeze >> ${KNOWN_FAILURE_CASES_FILE_NAME} # get runner env config file - retry_failed git clone $TEST_ENV_CONFIG_REPO - run_cmd python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs diff --git a/tools/ci/exclude_check_tools_files.txt b/tools/ci/exclude_check_tools_files.txt index 56b73c135e2..2499eb4220d 100644 --- a/tools/ci/exclude_check_tools_files.txt +++ b/tools/ci/exclude_check_tools_files.txt @@ -4,18 +4,15 @@ tools/ci/check_*.py tools/ci/check_*.txt tools/ci/check_*.sh tools/ci/check_copyright_config.yaml -tools/ci/get_all_test_results.py tools/gdb_panic_server.py tools/check_term.py tools/python_version_checker.py tools/ci/astyle-rules.yml tools/ci/checkout_project_ref.py tools/ci/ci_fetch_submodule.py -tools/ci/ci_get_mr_info.py tools/ci/ci_process_description.py tools/ci/configure_ci_environment.sh tools/ci/generate_rules.py -tools/ci/deploy_docs.py tools/ci/envsubst.py tools/ci/*exclude*.txt tools/ci/executable-list.txt diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index b185a767ae5..8301b288bda 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -72,7 +72,6 @@ tools/ci/check_tools_files_patterns.py tools/ci/check_type_comments.py tools/ci/checkout_project_ref.py tools/ci/cleanup_ignore_lists.py -tools/ci/deploy_docs.py tools/ci/envsubst.py tools/ci/fix_empty_prototypes.sh tools/ci/generate_rules.py diff --git a/tools/ci/generate_rules.py b/tools/ci/generate_rules.py index f9257eaf093..3ba31b9549d 100755 --- a/tools/ci/generate_rules.py +++ b/tools/ci/generate_rules.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2021-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import argparse import inspect @@ -12,7 +12,6 @@ from itertools import product import yaml from idf_ci_utils import IDF_PATH -from idf_ci_utils import GitlabYmlConfig if t.TYPE_CHECKING: import pygraphviz as pgv @@ -90,7 +89,6 @@ class RulesWriter: self.cfg = self.expand_matrices() self.rules = self.expand_rules() - self.yml_config = GitlabYmlConfig() self.graph = None def expand_matrices(self): # type: () -> dict @@ -195,10 +193,6 @@ class RulesWriter: if k.startswith('pattern'): continue - if '.rules:' + k not in self.yml_config.used_templates: - print(f'WARNING: unused rule: {k}, skipping...') - continue - res.append(self.RULES_TEMPLATE.format(k, self._format_rule(k, v))) return '\n\n'.join(res) diff --git a/tools/ci/get_all_test_results.py b/tools/ci/get_all_test_results.py deleted file mode 100644 index 7baa0832f2a..00000000000 --- a/tools/ci/get_all_test_results.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python -# -# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 - - -''' -Download artifacts from all test jobs - -CI_PROJECT_DIR -└── TEST_RESULTS - ├── - │ └── XUNIT_RESULT.xml - ├── - │ └── XUNIT_RESULT.xml - ├── - │ └── TEST_LOGS - -''' - -import argparse -import json -import os -import shutil - -import gitlab_api - -CI_PROJECT_ID = int(os.getenv('CI_PROJECT_ID', '')) -CI_PIPELINE_ID = int(os.getenv('CI_PIPELINE_ID', '')) -IDF_PATH = os.getenv('IDF_PATH', '.') -TEST_RESULT_PATH = os.path.join(IDF_PATH, 'TEST_RESULTS') - - -def download_all_test_results(result_path: str, include_retried: bool = False) -> None: - if os.path.exists(result_path): - shutil.rmtree(result_path) - os.makedirs(result_path, exist_ok=True) - - gitlab_inst = gitlab_api.Gitlab(CI_PROJECT_ID) - pipelines = gitlab_inst.project.pipelines.get(CI_PIPELINE_ID) - _include_retried = 'true' if include_retried else 'false' - jobs = pipelines.jobs.list(all=True, per_page=100, include_retried=_include_retried) - - job_info_list = [] - for job in jobs: - if job.stage in ['target_test', 'host_test']: - log_path = '' - if job.status not in ['success', 'failed']: - print('Job {}({}) is not finished'.format(job.id, job.name)) - elif not hasattr(job, 'artifacts_file'): - print('Job {}({}) has no artifacts.'.format(job.id, job.name)) - else: - log_path = os.path.join(result_path, 'job_{}'.format(job.id)) - print('Downloading artifacts from: {}'.format(job.name)) - os.makedirs(log_path, exist_ok=True) - gitlab_inst.download_artifacts(job.id, log_path) - - job_info = { - 'id': job.id, - 'name': job.name, - 'tag_list': job.tag_list, - 'status': job.status, - 'stage': job.stage, - 'web_url': job.web_url, - 'commit_url': job.commit['web_url'], - 'log_path': log_path, - } - job_info_list.append(job_info) - - with open(os.path.join(result_path, 'index.json'), 'w') as f: - f.write(json.dumps({'jobs': job_info_list}, indent=1, sort_keys=True)) - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument('--path', '-p', default=TEST_RESULT_PATH, help='Path to save result files.') - parser.add_argument('--include_retried', action='store_true', help='Including retried jobs.') - args = parser.parse_args() - - download_all_test_results(args.path, args.include_retried) - - -if __name__ == '__main__': - main() diff --git a/tools/ci/gitlab_yaml_linter.py b/tools/ci/gitlab_yaml_linter.py index baac4ee2e37..3ab20353ecb 100755 --- a/tools/ci/gitlab_yaml_linter.py +++ b/tools/ci/gitlab_yaml_linter.py @@ -1,17 +1,18 @@ #!/usr/bin/env python -# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 """ Check gitlab ci yaml files """ + import argparse import os import typing as t from functools import cached_property -from idf_ci_utils import get_submodule_dirs -from idf_ci_utils import GitlabYmlConfig from idf_ci_utils import IDF_PATH +from idf_ci_utils import GitlabYmlConfig +from idf_ci_utils import get_submodule_dirs class YmlLinter: @@ -43,17 +44,6 @@ class YmlLinter: exit(exit_code) - # name it like _1_ to make it run first - def _lint_1_yml_parser(self) -> None: - for k, v in self.yml_config.config.items(): - if ( - k not in self.yml_config.global_keys - and k not in self.yml_config.anchors - and k not in self.yml_config.templates - and k not in self.yml_config.jobs - ): - raise SystemExit(f'Parser incorrect. Key {k} not in global keys, anchors, templates, or jobs') - def _lint_default_values_artifacts(self) -> None: defaults_artifacts = self.yml_config.default.get('artifacts', {}) @@ -79,19 +69,6 @@ class YmlLinter: for item in undefined_patterns: self._errors.append(f'undefined pattern {item}. Please add {item} to .patterns-submodule') - def _lint_gitlab_yml_templates(self) -> None: - unused_templates = self.yml_config.templates.keys() - self.yml_config.used_templates - for item in unused_templates: - # known unused ones - if item not in [ - '.before_script:fetch:target_test', # used in dynamic pipeline - ]: - self._errors.append(f'Unused template: {item}, please remove it') - - undefined_templates = self.yml_config.used_templates - self.yml_config.templates.keys() - for item in undefined_templates: - self._errors.append(f'Undefined template: {item}') - def _lint_dependencies_and_needs(self) -> None: """ Use `dependencies: []` together with `needs: []` could cause missing artifacts issue. diff --git a/tools/ci/idf_ci_utils.py b/tools/ci/idf_ci_utils.py index 074a2648ff1..876d6a79a38 100644 --- a/tools/ci/idf_ci_utils.py +++ b/tools/ci/idf_ci_utils.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2020-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2020-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # internal use only for CI # some CI related util functions @@ -125,35 +125,106 @@ class GitlabYmlConfig: # avoid unused import in other pre-commit hooks import yaml - all_config = dict() - root_yml = yaml.load(open(root_yml_filepath), Loader=yaml.FullLoader) - - # expanding "include" - for item in root_yml.pop('include', []) or []: - if isinstance(item, dict): - if 'project' in item: - continue - elif 'local' in item: - item = item['local'] - else: - continue - - all_config.update(yaml.load(open(os.path.join(IDF_PATH, item)), Loader=yaml.FullLoader)) + merged_yaml = self._compile_via_gitlab_api(root_yml_filepath) + all_config = yaml.load(merged_yaml, Loader=yaml.FullLoader) or {} if 'default' in all_config: self._defaults = all_config.pop('default') self._config = all_config - # anchor is the string that will be reused in templates - self._anchor_keys: t.Set[str] = set() - # template is a dict that will be extended - self._template_keys: t.Set[str] = set() - self._used_template_keys: t.Set[str] = set() # tracing the used templates - # job is a dict that will be executed - self._job_keys: t.Set[str] = set() + def _inline_local_includes(self, root_yml_filepath: str) -> str: + """ + Recursively resolve `include: local` entries straight from disk (so uncommitted local + changes are always picked up -- CI runners also work off a disk checkout, so there's no + need to fetch a ref remotely via `ci_lint`'s `content_ref`/`ref`/`dry_run_ref` GET + params). `include: project` entries (including ones nested inside local files, e.g. + `.gitlab/ci/common.yml` including `templates/idf/common-scripts.yml`) are collected and + left in the final `include:` list, since those files live in another GitLab project and + can only be resolved remotely. - self.expand_extends() + A dedicated Loader/Dumper pair round-trips `!reference` tags as a marker list subclass, + since GitLab CI uses `!reference` which plain YAML doesn't know, and we need to parse + (to merge dicts, not just string-concat) then re-dump losslessly. + + :param root_yml_filepath: path to the local root yml file to compile + :return: yml content with local includes resolved and merged + """ + import yaml + + class _Loader(yaml.FullLoader): + pass + + class _Dumper(yaml.Dumper): + pass + + class _Reference(list): + pass + + _Loader.add_constructor( + '!reference', lambda loader, node: _Reference(loader.construct_sequence(t.cast(yaml.SequenceNode, node))) + ) + _Dumper.add_representer(_Reference, lambda dumper, data: dumper.represent_sequence('!reference', list(data))) + + def resolve(yml_filepath: str) -> t.Tuple[dict, list]: + with open(yml_filepath) as fr: + data = yaml.load(fr, Loader=_Loader) or {} + + includes = to_list(data.pop('include', None)) + + merged: dict = {} + remaining_project_includes: list = [] + for item in includes: + if isinstance(item, dict): + if 'project' in item: + remaining_project_includes.append(item) + continue + elif 'local' in item: + local_path = item['local'] + else: + continue + elif isinstance(item, str): + local_path = item + else: + continue + + sub_merged, sub_remaining = resolve(os.path.join(IDF_PATH, local_path.lstrip('/'))) + merged.update(sub_merged) + remaining_project_includes.extend(sub_remaining) + + # this file's own top-level keys override whatever its includes defined + merged.update(data) + return merged, remaining_project_includes + + merged_config, project_includes = resolve(root_yml_filepath) + if project_includes: + merged_config['include'] = project_includes + + return yaml.dump(merged_config, Dumper=_Dumper, sort_keys=False) # type: ignore + + def _compile_via_gitlab_api(self, root_yml_filepath: str) -> str: + """ + Call the GitLab CI Lint API to get the fully compiled (all `include`s resolved) yml, + same as what `glab ci config compile` does. This replaces the old recursive local-only + parsing, since the project now includes configs from other projects as well. + + :param root_yml_filepath: path to the local root yml file to compile + :return: merged (fully resolved) yml content as a string + """ + sys.path.insert(0, os.path.join(IDF_PATH, 'tools', 'ci', 'python_packages')) + import gitlab_api + + content = self._inline_local_includes(root_yml_filepath) + + gitlab_inst = gitlab_api.Gitlab() + project_id = os.getenv('CI_PROJECT_ID') or gitlab_inst.get_project_id('esp-idf', namespace='espressif') + project = gitlab_inst.gitlab_inst.projects.get(project_id, lazy=True) + + lint_result = project.ci_lint.create({'content': content}) + if not lint_result.valid: + raise RuntimeError(f'Failed to compile {root_yml_filepath} via GitLab CI Lint API: {lint_result.errors}') + + return lint_result.merged_yaml # type: ignore @property def default(self) -> t.Dict[str, t.Any]: @@ -167,80 +238,9 @@ class GitlabYmlConfig: def global_keys(self) -> t.List[str]: return ['default', 'include', 'workflow', 'variables', 'stages'] - @cached_property - def anchors(self) -> t.Dict[str, t.Any]: - return {k: v for k, v in self.config.items() if k in self._anchor_keys} - @cached_property def jobs(self) -> t.Dict[str, t.Any]: - return {k: v for k, v in self.config.items() if k in self._job_keys} - - @cached_property - def templates(self) -> t.Dict[str, t.Any]: - return {k: v for k, v in self.config.items() if k in self._template_keys} - - @cached_property - def used_templates(self) -> t.Set[str]: - return self._used_template_keys - - def expand_extends(self) -> None: - """ - expand the `extends` key in-place. - """ - for k, v in self.config.items(): - if k in self.global_keys: - continue - - if isinstance(v, (str, list)): - self._anchor_keys.add(k) - elif k.startswith('.if-'): - self._anchor_keys.add(k) - elif k.startswith('.'): - self._template_keys.add(k) - elif isinstance(v, dict): - self._job_keys.add(k) - else: - raise ValueError(f'Unknown type for key {k} with value {v}') - - # no need to expand anchor - - # expand template first - for k in self._template_keys: - self._expand_extends(k) - - # expand job - for k in self._job_keys: - self._expand_extends(k) - - def _merge_dict(self, d1: t.Dict[str, t.Any], d2: t.Dict[str, t.Any]) -> t.Any: - for k, v in d2.items(): - if k in d1: - if isinstance(v, dict) and isinstance(d1[k], dict): - d1[k] = self._merge_dict(d1[k], v) - else: - d1[k] = v - else: - d1[k] = v - - return d1 - - def _expand_extends(self, name: str) -> t.Dict[str, t.Any]: - extends = to_list(self.config[name].pop('extends', None)) - if not extends: - return self.config[name] # type: ignore - - original_d = self.config[name].copy() - d = {} - while extends: - self._used_template_keys.update(extends) # for tracking - - for i in extends: - d.update(self._expand_extends(i)) - - extends = to_list(self.config[name].pop('extends', None)) - - self.config[name] = self._merge_dict(d, original_d) - return self.config[name] # type: ignore + return {k: v for k, v in self.config.items() if not k.startswith('.') and k not in self.global_keys} def get_all_manifest_files() -> t.List[str]: diff --git a/tools/ci/idf_pytest/plugin.py b/tools/ci/idf_pytest/plugin.py index 3d5c3be2965..c3c0e755093 100644 --- a/tools/ci/idf_pytest/plugin.py +++ b/tools/ci/idf_pytest/plugin.py @@ -9,7 +9,6 @@ import pytest import yaml from _pytest.config import Config from _pytest.python import Function -from _pytest.python import Metafunc from _pytest.runner import CallInfo from dynamic_pipelines.constants import KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH from idf_ci import IdfPytestPlugin @@ -115,55 +114,6 @@ class IdfLocalPlugin: return item.callspec.params.get(key, default) or default - @staticmethod - def _has_parametrized_arg(metafunc: Metafunc, arg_name: str) -> bool: - for marker in metafunc.definition.iter_markers(name='parametrize'): - if not marker.args: - continue - - argnames = marker.args[0] - if isinstance(argnames, str): - names = [name.strip() for name in argnames.split(',')] - else: - names = list(argnames) - - if arg_name in names: - return True - - for callspec in getattr(metafunc, '_calls', []): - if arg_name in callspec.params: - return True - - return False - - @staticmethod - def _is_linux_target_run(config: Config) -> bool: - target = config.getoption('target') - if not target: - return False - - if isinstance(target, str): - targets = [_t.strip() for _t in target.split(',')] - else: - targets = [str(_t).strip() for _t in target] - - return 'linux' in targets - - @pytest.hookimpl(trylast=True) - def pytest_generate_tests(self, metafunc: Metafunc) -> None: - if 'embedded_services' not in metafunc.fixturenames: - return - - if self._has_parametrized_arg(metafunc, 'embedded_services'): - return - - if metafunc.definition.get_closest_marker('qemu') is not None: - metafunc.parametrize('embedded_services', ['idf,qemu'], indirect=True) - return - - if self._is_linux_target_run(metafunc.config): - metafunc.parametrize('embedded_services', ['idf'], indirect=True) - @pytest.hookimpl(wrapper=True) def pytest_collection_modifyitems(self, config: Config, items: t.List[Function]) -> t.Generator[None, None, None]: yield # throw it back to idf-ci diff --git a/tools/ci/python_packages/gitlab_api.py b/tools/ci/python_packages/gitlab_api.py index 6ac0ffae1f8..896522489b0 100644 --- a/tools/ci/python_packages/gitlab_api.py +++ b/tools/ci/python_packages/gitlab_api.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import argparse import logging @@ -52,9 +52,11 @@ def retry(func: TR) -> TR: raise e # get out of the loop else: logging.warning( - 'Network failure in {}, retrying ({})'.format(getattr(func, '__name__', '(unknown callable)'), - retried)) - time.sleep(2 ** retried) # wait a bit more after each retry + 'Network failure in {}, retrying ({})'.format( + getattr(func, '__name__', '(unknown callable)'), retried + ) + ) + time.sleep(2**retried) # wait a bit more after each retry continue else: break @@ -67,7 +69,6 @@ class Gitlab(object): JOB_NAME_PATTERN = re.compile(r'(\w+)(\s+(\d+)/(\d+))?') DOWNLOAD_ERROR_MAX_RETRIES = 3 - DEFAULT_BUILD_CHILD_PIPELINE_NAME = 'Build Child Pipeline' def __init__(self, project_id: Union[int, str, None] = None): config_data_from_env = os.getenv('PYTHON_GITLAB_CONFIG') @@ -117,7 +118,7 @@ class Gitlab(object): :param namespace: namespace to match when we have multiple project with same name :return: project ID """ - projects = self.gitlab_inst.projects.list(search=name) + projects = self.gitlab_inst.projects.list(search=name, get_all=True) res = [] for project in projects: if namespace is None: @@ -152,7 +153,9 @@ class Gitlab(object): archive_file.extractall(destination) @retry - def download_artifact(self, job_id: int, artifact_path: List[str], destination: Optional[str] = None) -> List[bytes]: + def download_artifact( + self, job_id: int, artifact_path: List[str], destination: Optional[str] = None + ) -> List[bytes]: """ download specific path of job artifacts and extract to destination. @@ -208,8 +211,9 @@ class Gitlab(object): return job_id_list @retry - def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None, - cache_dir: Optional[str] = None) -> str: + def download_archive( + self, ref: str, destination: str, project_id: Optional[int] = None, cache_dir: Optional[str] = None + ) -> str: """ Download archive of certain commit of a repository and extract to destination path @@ -235,8 +239,11 @@ class Gitlab(object): except gitlab.GitlabGetError as e: logging.error('Failed to archive from project {}'.format(project_id)) raise e - logging.info('Downloaded archive size: {:.03f}MB'.format( - float(os.path.getsize(local_archive_file)) / (1024 * 1024))) + logging.info( + 'Downloaded archive size: {:.03f}MB'.format( + float(os.path.getsize(local_archive_file)) / (1024 * 1024) + ) + ) return self.decompress_archive(local_archive_file, destination) @@ -248,7 +255,9 @@ class Gitlab(object): logging.error('Failed to archive from project {}'.format(project_id)) raise e - logging.info('Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(temp_file.name)) / (1024 * 1024))) + logging.info( + 'Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(temp_file.name)) / (1024 * 1024)) + ) return self.decompress_archive(temp_file.name, destination) @@ -282,36 +291,28 @@ class Gitlab(object): def get_downstream_pipeline_ids(self, main_pipeline_id: int) -> List[int]: """ - Retrieve the IDs of all downstream child pipelines for a given main pipeline. + Retrieve the IDs of all downstream child pipelines for a given main pipeline, + recursing through arbitrarily nested child pipelines. :param main_pipeline_id: The ID of the main pipeline to start the search. - :return: A list of IDs of all downstream child pipelines. + :return: A list of IDs of all downstream child pipelines (all levels). """ - bridge_pipeline_ids = [] - child_pipeline_ids = [] + child_pipeline_ids: List[int] = [] - main_pipeline_bridges = self.project.pipelines.get(main_pipeline_id).bridges.list() - for bridge in main_pipeline_bridges: + pipeline_bridges = self.project.pipelines.get(main_pipeline_id).bridges.list() + for bridge in pipeline_bridges: downstream_pipeline = bridge.attributes.get('downstream_pipeline') if not downstream_pipeline: continue - bridge_pipeline_ids.append(downstream_pipeline['id']) - - for bridge_pipeline_id in bridge_pipeline_ids: - child_pipeline_ids.append(bridge_pipeline_id) - bridge_pipeline = self.project.pipelines.get(bridge_pipeline_id) - - if not bridge_pipeline.name == self.DEFAULT_BUILD_CHILD_PIPELINE_NAME: + downstream_pipeline_id = downstream_pipeline.get('id') + if downstream_pipeline_id is None: continue - child_bridges = bridge_pipeline.bridges.list() - for child_bridge in child_bridges: - downstream_child_pipeline = child_bridge.attributes.get('downstream_pipeline') - if not downstream_child_pipeline: - continue - child_pipeline_ids.append(downstream_child_pipeline.get('id')) + child_pipeline_ids.append(downstream_pipeline_id) + # recurse to collect further nested (grandchild+) pipelines + child_pipeline_ids.extend(self.get_downstream_pipeline_ids(downstream_pipeline_id)) - return [pid for pid in child_pipeline_ids if pid is not None] + return child_pipeline_ids def retry_failed_jobs(self, pipeline_id: int, retry_allowed_failures: bool = False) -> List[int]: """ diff --git a/tools/ci/utils.sh b/tools/ci/utils.sh index a4c3a4875b3..9fcea7fd53a 100644 --- a/tools/ci/utils.sh +++ b/tools/ci/utils.sh @@ -164,38 +164,3 @@ function join_by { printf %s "$f" "${@/#/$d}" fi } - -function is_based_on_commits() { - # This function would accept space-separated args as multiple commits. - # The return value would be 0 if current HEAD is based on any of the specified commits. - # - # In our CI, we use environment variable $REQUIRED_ANCESTOR_COMMITS to declare the ancestor commits. - # Please remember to set one commit for each release branch. - - commits=$* - if [[ -z $commits ]]; then - info "Not specifying commits that branches should be based on, skipping check..." - return 0 - fi - - commits_str="$(join_by " or " $commits)" # no doublequotes here, passing array - - info "Checking if current branch is based on $commits_str..." - for i in $commits; do - if git merge-base --is-ancestor "$i" HEAD >/dev/null 2>&1; then - info "Current branch is based on $i" - return 0 - else - info "Current branch is not based on $i" - fi - done - - error "The base commit of your branch is too old." - error "The branch should be more recent than either of the following commits:" - error " $commits_str" - error "To fix the issue:" - error " - If your merge request is 'Draft', or has conflicts with the target branch, rebase it to the latest master or release branch" - error " - Otherwise, simply run a new pipeline." - - return 1 -}