ci: apply common-scripts CI refactor

.gitlab-ci.yml

@@ -2,11 +2,25 @@
 
 include:
   - project: "ci/actions/common"
+    ref: "v2"
     file:
       - "templates/stable/default-workflow.yml"
+      - "templates/idf/pre_check.yml"
+      - "templates/idf/build.yml"
       - "templates/idf/deploy-github.yml"
       - "templates/idf/deploy-docs.yml"
       - "templates/idf/integration-test.yml"
+  - project: "ci/actions/common"
+    ref: "v2"
+    file: "templates/idf/sync-files.yml"
+    inputs:
+      job_name: "sync-pre-commit-config"
+      remote_repo_url: "https://__token__:${CI_SYNC_REPO_TOKEN}@${CI_SERVER_FQDN}/ci/images.git"
+      ref_name: "ci/sync_idf_pre_commit_config"
+      file_mappings:
+        - ".pre-commit-config.yaml:pre-commit/idf/.pre-commit-config.v5.5.yaml"
+      changes:
+        - ".pre-commit-config.yaml"
   - project: "ci/actions/common"
     file: "templates/idf/build-docs.yml"
     inputs:
@@ -20,13 +34,14 @@ include:
           DOCTGT: "esp32p4"
 
   - ".gitlab/ci/danger.yml"
-  - ".gitlab/ci/common.yml"
+  - ".gitlab/ci/common.yml"  # has to be placed after danger.yml, since danger.yml defined stages.
   - ".gitlab/ci/rules.yml"
+  # by stages
   - ".gitlab/ci/manual_gate.yml"
   - ".gitlab/ci/upload_cache.yml"
+  - ".gitlab/ci/pre_check.yml"
   - ".gitlab/ci/static-code-analysis.yml"
   - ".gitlab/ci/pre_commit.yml"
-  - ".gitlab/ci/pre_check.yml"
   - ".gitlab/ci/build.yml"
   - ".gitlab/ci/host-test.yml"
   - ".gitlab/ci/post_deploy.yml"

.gitlab/ci/README.md

@@ -245,76 +245,15 @@ We're using the latest version of [idf-build-apps][idf-build-apps]. Please refer
 In ESP-IDF CI, there's a few more special rules are additionally supported to disable the check app dependencies feature:
 
 - Add MR labels `BUILD_AND_TEST_ALL_APPS`
+- Pipeline variable `IDF_CI_SELECT_ALL_PYTEST_CASES=1`
 - Run in protected branches
 
 ## Upload/Download Artifacts to Internal Minio Server
 
-### Users Without Access to Minio
+Please refer to the documentation [here](https://docs.espressif.com/projects/idf-ci/en/latest/guides/cli.html)
 
-If you don't have access to the internal Minio server, you can still download the artifacts from the shared link in the job log.
+## Revision Marker Naming Rules
 
-The log will look like this:
+When specifying chip revisions for testing markers, configurations or filenames, strictly follow the format `esp32<series>_rev<number>` (or similarly `<series>_rev<number>` when brief).
 
-```shell
-Pipeline ID    : 587355
-Job name       : build_clang_test_apps_esp32
-Job ID         : 40272275
-Created archive file: 40272275.zip, uploading as 587355/build_dir_without_map_and_elf_files/build_clang_test_apps_esp32/40272275.zip
-Please download the archive file includes build_dir_without_map_and_elf_files from [INTERNAL_URL]
-```
-
-### Users With Access to Minio
-
-#### Env Vars for Minio
-
-Minio takes these env vars to connect to the server:
-
-- `IDF_S3_SERVER`
-- `IDF_S3_ACCESS_KEY`
-- `IDF_S3_SECRET_KEY`
-- `IDF_S3_BUCKET`
-
-#### Artifacts Types and File Patterns
-
-The artifacts types and corresponding file patterns are defined in tools/ci/artifacts_handler.py, inside `ArtifactType` and `TYPE_PATTERNS_DICT`.
-
-#### Upload
-
-```shell
-python tools/ci/artifacts_handler.py upload
-```
-
- will upload the files that match the file patterns to minio object storage with name:
-
-`<pipeline_id>/<artifact_type>/<job_name>/<job_id>.zip`
-
-For example, job 39043328 will upload these four files:
-
-- `575500/map_and_elf_files/build_pytest_examples_esp32/39043328.zip`
-- `575500/build_dir_without_map_and_elf_files/build_pytest_examples_esp32/39043328.zip`
-- `575500/logs/build_pytest_examples_esp32/39043328.zip`
-- `575500/size_reports/build_pytest_examples_esp32/39043328.zip`
-
-#### Download
-
-You may run
-
-```shell
-python tools/ci/artifacts_handler.py download --pipeline_id <pipeline_id>
-```
-
-to download all files of the pipeline, or
-
-```shell
-python tools/ci/artifacts_handler.py download --pipeline_id <pipeline_id> --job_name <job_name_or_pattern>
-```
-
-to download all files with the specified job name or pattern, or
-
-```shell
-python tools/ci/artifacts_handler.py download --pipeline_id <pipeline_id> --job_name <job_name_or_pattern> --type <artifact_type> <artifact_type> ...
-```
-
-to download all files with the specified job name or pattern and artifact type(s).
-
-You may check all detailed documentation with `python tools/ci/artifacts_handler.py download -h`
+> e.g., `esp32_rev3`, `esp32c2_rev2`, `esp32c3_rev1`, `esp32p4_rev1`, `esp32c5_rev1`

.gitlab/ci/build.yml

@@ -110,61 +110,6 @@ build_clang_test_apps_riscv:
 ####################
 # Dynamic Pipeline #
 ####################
-generate_build_child_pipeline:
-  extends:
-    - .build_template
-  tags: [fast_run, shiny]
-  dependencies: # set dependencies to null to avoid missing artifacts issue
-  needs:
-    - pipeline_variables
-    - job: baseline_manifest_sha
-      optional: true
-  artifacts:
-    paths:
-      - build_child_pipeline.yml
-      - test_related_apps.txt
-      - non_test_related_apps.txt
-    expire_in: 1 week
-    when: always
-  script:
-    - run_cmd idf-ci --debug gitlab build-child-pipeline
-      -p components
-      -p examples
-      -p tools/test_apps
-      --modified-files $MR_MODIFIED_FILES
-
-build_child_pipeline:
-  stage: build
-  needs:
-    - job: fast_template_app
-      optional: true
-      artifacts: false
-    - pipeline_variables
-    - generate_build_child_pipeline
-  variables:
-    MR_MODIFIED_COMPONENTS: $MR_MODIFIED_COMPONENTS
-    MR_MODIFIED_FILES: $MR_MODIFIED_FILES
-    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
-  # https://gitlab.com/gitlab-org/gitlab/-/issues/214340
-  inherit:
-    variables: false
-  trigger:
-    include:
-      - artifact: build_child_pipeline.yml
-        job: generate_build_child_pipeline
-    strategy: depend
-
-generate_prebuild_report:
-  extends:
-    - .build_template
-  tags: [fast_run, shiny]
-  needs:
-    - pipeline_variables
-  artifacts:
-    paths:
-      - prebuild_report.html
-    expire_in: 1 week
-    when: always
-  script:
-    - run_cmd idf-ci build collect --format html -o prebuild_report.html
-    - echo "Report generated at https://${CI_PAGES_HOSTNAME}:${CI_SERVER_PORT}/-/esp-idf/-/jobs/${CI_JOB_ID}/artifacts/prebuild_report.html"
+# !!!
+# configurations moved to ci/actions/common/idf/
+# !!!

.gitlab/ci/common.yml

@@ -14,6 +14,15 @@ stages:
   - deploy
   - post_deploy
 
+include:
+  - project: "ci/actions/common"
+    ref: "v2"
+    file: "templates/idf/common-scripts.yml"
+    inputs:
+      ci_python_constraint_file: "espidf.constraints.v5.5.txt"
+      macos_image: "registry.hub.docker.com/espressif/macos-idf:sequoia-v5.5"
+      windows_image: "windows_11_idf_v5_5"
+
 variables:
 # System environment
 
@@ -58,30 +67,29 @@ variables:
   ESP_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v5.5:1"
   ESP_IDF_DOC_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v5.5-docs:1"
   TARGET_TEST_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v5.5-target-test:1"
-  SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:5"
 
   # Set this variable to the branch of idf-constraints repo in order to test a custom Python constraint file. The
   # branch name must be without the remote part ("origin/"). Keep the variable empty in order to use the constraint
   # file from https://dl.espressif.com/dl/esp-idf.
-  CI_PYTHON_CONSTRAINT_BRANCH: ""
+  # CI_PYTHON_CONSTRAINT_BRANCH could be defined as component input `ci_python_constraint_branch` in common-scripts above
 
   # Update the filename for a specific ESP-IDF release. It is used only with CI_PYTHON_CONSTRAINT_BRANCH.
-  CI_PYTHON_CONSTRAINT_FILE: "espidf.constraints.v5.5.txt"
+  # CI_PYTHON_CONSTRAINT_FILE is defined as component input `ci_python_constraint_file` in common-scripts above
 
   # Set this variable to repository name of a Python tool you wish to install and test in the context of ESP-IDF CI.
   # Keep the variable empty when not used.
-  CI_PYTHON_TOOL_REPO: ""
+  # CI_PYTHON_TOOL_REPO could be defined as component input `ci_python_tool_repo` in common-scripts above
 
   # Set this variable to the branch of a Python tool repo specified in CI_PYTHON_TOOL_REPO. The
   # branch name must be without the remote part ("origin/"). Keep the variable empty when not used.
   # This is used only if CI_PYTHON_TOOL_REPO is not empty.
-  CI_PYTHON_TOOL_BRANCH: ""
+  # CI_PYTHON_TOOL_BRANCH could be defined as component input `ci_python_tool_branch` in common-scripts above
 
   # Set this variable to Clang toolchain distro URL to be used.
   # NOTE: We have separate toolchains for Xtensa and RISCV, therefore jobs for one arch will fail.
   # This is OK as far as we use CI_CLANG_DISTRO_URL for pre-release tests purposes only.
   # Keep the variable empty when not used.
-  CI_CLANG_DISTRO_URL: ""
+  # CI_CLANG_DISTRO_URL could be defined as component input `ci_clang_distro_url` in common-scripts above
 
   # Set this variable to specify the file name for the known failure cases.
   KNOWN_FAILURE_CASES_FILE_NAME: "5.5.txt"
@@ -103,227 +111,19 @@ variables:
 ################################################
 # `before_script` and `after_script` Templates #
 ################################################
-.common_before_scripts: &common-before_scripts |
-  source tools/ci/utils.sh
-
-  is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
-
-  if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
-    export IDF_MIRROR_PREFIX_MAP=
-  fi
-
-  if echo "$CI_MERGE_REQUEST_LABELS" | egrep "(^|,)include_nightly_run(,|$)"; then
-    export INCLUDE_NIGHTLY_RUN="1"
-    export NIGHTLY_RUN="1"
-  fi
-
-  # configure cmake related flags
-  source tools/ci/configure_ci_environment.sh
-
-  if [[ "$CI_CCACHE_STATS" == 1 ]] && command -v ccache >/dev/null 2>&1 && [[ -n "$CCACHE_STATSLOG" ]]; then
-    mkdir -p "$(dirname "$CCACHE_STATSLOG")"
-    rm -f "$CCACHE_STATSLOG"
-  fi
-
-  # add extra python packages
-  export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
-
-.setup_tools_and_idf_python_venv: &setup_tools_and_idf_python_venv |
-  # must use after setup_tools_except_target_test
-  # otherwise the export.sh won't work properly
-
-  # download constraint file for dev
-  if [[ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]]; then
-    wget -O /tmp/constraint.txt --header="Authorization:Bearer ${ESPCI_TOKEN}" "${GITLAB_HTTP_SERVER}/api/v4/projects/2581/repository/files/${CI_PYTHON_CONSTRAINT_FILE}/raw?ref=${CI_PYTHON_CONSTRAINT_BRANCH}"
-    mkdir -p ~/.espressif
-    mv /tmp/constraint.txt ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
-  fi
-
-  # Mirror
-  if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
-    export IDF_MIRROR_PREFIX_MAP=
-  fi
-  # Optimize pip install
-  if echo "${CI_RUNNER_TAGS}" | grep "shiny"; then
-    export PIP_INDEX_URL="${PIP_INDEX_URL_SHINY}"
-  fi
-  if [[ "$(uname -m)" == "x86_64" ]] || [[ "$(uname -m)" == "aarch64" ]]; then
-    export IDF_PIP_WHEELS_URL=""
-  fi
-
-  # install.sh
-  if [[ "${CI_JOB_STAGE}" != "target_test" ]]; then
-    section_start "running_install_sh" "Running install.sh"
-    if [[ "${CI_JOB_STAGE}" == "build_doc" || "${CI_JOB_GROUP_NAME}" == *docs* ]]; then
-      run_cmd bash install.sh --enable-ci --enable-docs
-    else
-      run_cmd bash install.sh --enable-ci
-    fi
-    section_end "running_install_sh"
-  else
-    section_start "install_python_env" "Install Python environment, skip required tools check"
-    run_cmd python tools/idf_tools.py install-python-env --features ci,test-specific
-    export IDF_SKIP_TOOLS_CHECK=1
-    section_end "install_python_env"
-  fi
-
-  section_start "source_export" "Source export.sh"
-  source ./export.sh
-  section_end "source_export"
-
-  # Eager upgrade of CI dependencies
-  # Done after sourcing export.sh so that we could easily invoke the right pip
-  section_start "upgrade_ci_dependencies" "Upgrading CI dependencies"
-  pip install --upgrade --upgrade-strategy=eager -r $IDF_PATH/tools/requirements/requirements.ci.txt -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
-  # we need the latest DEV release for esptool to be installed in every job without bumping the minimal requirement in
-  # the constraint file.
-  pip install --upgrade --pre esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
-  section_end "upgrade_ci_dependencies"
-
-  REEXPORT_NEEDED=0
-  if [[ ! -z "$INSTALL_EXTRA_TOOLS" ]]; then
-    section_start "installing_optional_tools" "Install optional tools ${INSTALL_EXTRA_TOOLS}"
-    run_cmd $IDF_PATH/tools/idf_tools.py --non-interactive install $INSTALL_EXTRA_TOOLS
-    section_end "installing_optional_tools"
-
-    REEXPORT_NEEDED=1
-  fi
-
-  # Install esp-clang if necessary (esp-clang is separately installed)
-  if [[ "$IDF_TOOLCHAIN" == "clang" && -z "$CI_CLANG_DISTRO_URL" ]]; then
-    $IDF_PATH/tools/idf_tools.py --non-interactive install esp-clang
-
-    REEXPORT_NEEDED=1
-  fi
-
-  if [[ $REEXPORT_NEEDED -eq 1 ]]; then
-    section_start "re_source_export" "Re-source export.sh"
-    source ./export.sh
-    section_end "re_source_export"
-  fi
-
-  # Custom clang toolchain
-  if [[ "$IDF_TOOLCHAIN" == "clang" && ! -z "$CI_CLANG_DISTRO_URL" ]]; then
-    echo "Using custom clang from ${CI_CLANG_DISTRO_URL}"
-    wget $CI_CLANG_DISTRO_URL
-    ARCH_NAME=$(basename $CI_CLANG_DISTRO_URL)
-    tar -x -f $ARCH_NAME
-    export PATH=$PWD/esp-clang/bin:$PATH
-  fi
-
-  # Custom OpenOCD
-  if [[ "$CI_JOB_STAGE" == "target_test" ]]; then
-    machine="$(uname -m)"
-    if [[ "$machine" == "armv7l" ]] ; then
-      OOCD_DISTRO_URL="$OOCD_DISTRO_URL_ARMHF"
-    elif [[ "$machine" == "aarch64" ]] ; then
-      OOCD_DISTRO_URL="$OOCD_DISTRO_URL_ARM64"
-    fi
-    if [[ ! -z "$OOCD_DISTRO_URL" ]]; then
-      echo "Using custom OpenOCD from ${OOCD_DISTRO_URL}"
-      wget $OOCD_DISTRO_URL
-      ARCH_NAME=$(basename $OOCD_DISTRO_URL)
-      tar -x -f $ARCH_NAME
-      export OPENOCD_SCRIPTS=$PWD/openocd-esp32/share/openocd/scripts
-      export PATH=$PWD/openocd-esp32/bin:$PATH
-    fi
-  fi
-
-  if [[ -n "$CI_PYTHON_TOOL_REPO" ]]; then
-    git clone --quiet --depth=1 -b ${CI_PYTHON_TOOL_BRANCH} https://gitlab-ci-token:${ESPCI_TOKEN}@${GITLAB_HTTPS_HOST}/espressif/${CI_PYTHON_TOOL_REPO}.git
-    pip install ./${CI_PYTHON_TOOL_REPO}
-    rm -rf ${CI_PYTHON_TOOL_REPO}
-  fi
-
-  info "setup tools and python venv done"
-
-.show_ccache_statistics: &show_ccache_statistics |
-  # Show ccache statistics if enabled globally
-  section_start "ccache_show_stats" "Show ccache statistics"
-  if [[ "$CI_CCACHE_STATS" == 1 ]] && command -v ccache >/dev/null 2>&1; then
-    if ccache --help 2>/dev/null | grep -q -- '--show-log-stats'; then
-      if [[ -n "$CCACHE_STATSLOG" && -f "$CCACHE_STATSLOG" ]]; then
-        ccache --show-log-stats -vv
-      else
-        echo "INFO: No per-job ccache statistics were recorded"
-      fi
-    else
-      ccache --show-stats -vv
-    fi
-  fi || true
-  section_end "ccache_show_stats"
-
-.upload_failed_job_log_artifacts: &upload_failed_job_log_artifacts |
-  if [ $CI_JOB_STATUS = "failed" ]; then
-    run_cmd idf-ci gitlab upload-artifacts --type log
-  fi
-
 .before_script:minimal:
   before_script:
-    - *common-before_scripts
+    - !reference [.common_before_scripts, minimal]
 
 .after_script:build:
   after_script:
-    - source tools/ci/utils.sh
-    - source tools/ci/configure_ci_environment.sh
-    - *show_ccache_statistics
+    - !reference [.common_after_scripts, ccache]
     - run_cmd idf-ci gitlab upload-artifacts --type junit
     - |
       if [ $CI_JOB_STATUS = "failed" ]; then
         run_cmd idf-ci gitlab upload-artifacts --type log
       fi
 
-.brew-macos-settings:
-  variables:
-    GIT_STRATEGY: fetch
-    IDF_CCACHE_ENABLE: "0"
-    CCACHE_DIR: "/var/tmp/cache/idf_ccache"
-  tags:
-    - macos-tart
-  image: macos-sequoia-idf-v5.5
-  after_script: []  # ccache now is disabled for macos brew runners
-  timeout: 30m
-
-.brew-windows-settings:
-  image: windows_11_idf_v6_0
-  tags: [windows-vm, brew]
-  variables:
-    GIT_STRATEGY: fetch
-    LC_ALL: C.UTF-8
-    # git clean at the final cleanup stage stuck on Windows VM
-    # a new VM for each job anyway, so we can skip cleaning the workspace
-    GIT_CLEAN_FLAGS: none
-#    IDF_CCACHE_ENABLE: "1"
-#    CCACHE_DIR: "C:/cache/idf_ccache"
-  before_script:
-    - if ($env:IDF_DONT_USE_MIRRORS) { $env:IDF_MIRROR_PREFIX_MAP = '' }
-    - ./install.ps1 --enable-ci
-    - ./export.ps1
-    - $env:PYTHONPATH = "$env:PYTHONPATH;$env:IDF_PATH\tools;$env:IDF_PATH\tools\esp_app_trace;$env:IDF_PATH\components\partition_table;$env:IDF_PATH\tools\ci\python_packages"
-    - python "${SUBMODULE_FETCH_TOOL}" -s "${SUBMODULES_TO_FETCH}"
-    # Sync VM clock. net start/stop exit 2 when the service is already in that state; do not fail CI.
-    - |
-      function Invoke-NetService($Action, $Name) {
-        $output = net $Action $Name 2>&1
-        if ($LASTEXITCODE -eq 2) {
-          Write-Host "net $Action $Name already in requested state: $output"
-          $global:LASTEXITCODE = 0
-          return
-        }
-        if ($LASTEXITCODE -ne 0) {
-          Write-Host $output
-          exit $LASTEXITCODE
-        }
-      }
-      Invoke-NetService start w32time
-      w32tm /config /manualpeerlist:"time.google.com,0x9 pool.ntp.org,0x9" /syncfromflags:manual /reliable:yes /update
-      if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
-      Invoke-NetService stop w32time
-      Invoke-NetService start w32time
-      w32tm /resync
-      if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
-  after_script: []
-
 #############
 # `default` #
 #############
@@ -338,8 +138,7 @@ default:
         - .cache/submodule_archives
       policy: pull
   before_script:
-    - *common-before_scripts
-    - *setup_tools_and_idf_python_venv
+    - !reference [.common_before_scripts, with_tools]
     - add_gitlab_ssh_keys
     - fetch_submodules
     - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}

.gitlab/ci/default-build-test-rules.yml

@@ -1,21 +0,0 @@
-# this file support two keywords:
-# - extra_default_build_targets:
-#     besides of the SUPPORTED_TARGETS in IDF,
-#     enable build for the specified targets by default as well.
-# !!! DEPRECATED: use `additional_build_targets` in .idf_build_apps.toml instead
-#
-# - bypass_check_test_targets:
-#     suppress the check_build_test_rules check-test-script warnings for the specified targets
-#
-# This file should ONLY be used during bringup. Should be reset to empty after the bringup process
-
-bypass_check_test_targets:
-  - esp32h21
-  - esp32h4
-  - esp32c5
-
-#
-# These lines would
-# - enable the README.md check for esp32c6. Don't forget to add the build jobs in .gitlab/ci/build.yml
-# - disable the test script check with the manifest file.
-#

.gitlab/ci/host-test.yml

@@ -401,7 +401,7 @@ test_pytest_linux:
 test_pytest_macos:
   extends:
     - .host_test_template
-    - .brew-macos-settings
+    - .macos-settings
   artifacts:
     paths:
       - XUNIT_RESULT.xml
@@ -476,6 +476,8 @@ test_pytest_macos:
     - pytest
       -k cmake
       --cleanup-idf-copy
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
       --work-dir ${CI_PROJECT_DIR}/test_build_system
       --junitxml ${CI_PROJECT_DIR}/XUNIT_RESULT.xml
       --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
@@ -487,7 +489,7 @@ pytest_build_system:
 pytest_build_system_macos:
   extends:
     - .test_build_system_template
-    - .brew-macos-settings
+    - .macos-settings
     - .rules:build:macos
   parallel: 3
 
@@ -497,7 +499,7 @@ pytest_build_system_minimal_cmake:
 pytest_build_system_macos_minimal_cmake:
   extends:
     - .test_build_system_minimal_cmake_template
-    - .brew-macos-settings
+    - .macos-settings
     - .rules:build:macos
   variables:
     INSTALL_EXTRA_TOOLS: ninja cmake@3.16.3

.gitlab/ci/pre_check.yml

@@ -1,9 +1,3 @@
-.pre_check_template:
-  stage: pre_check
-  image: $ESP_ENV_IMAGE
-  tags: [build, shiny]
-  dependencies: # set dependencies to null to avoid missing artifacts issue
-
 check_version:
   # Don't run this for feature/bugfix branches, so that it is possible to modify
   # esp_idf_version.h in a branch before tagging the next version.
@@ -140,106 +134,11 @@ check_submodule_sync:
     - git submodule update --recursive
     - echo "IDF was cloned from ${PUBLIC_IDF_URL} completely"
 
-pipeline_variables:
+check_gitlab_yaml:
   extends:
     - .pre_check_template
-  tags: [fast_run, shiny]
-  variables:
-    GIT_STRATEGY: "fetch"
   script:
-    - |
-      # Store the diff output in a temporary file
-      TEMP_FILE=$(mktemp)
-      # merged results pipelines, by default
-      if [[ -n $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ]]; then
-        git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA
-        git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
-
-        git diff --name-only $CI_MERGE_REQUEST_TARGET_BRANCH_SHA...$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA > "$TEMP_FILE"
-        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
-        git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
-      # merge request pipelines, when the mr got conflicts
-      elif [[ -n $CI_MERGE_REQUEST_DIFF_BASE_SHA ]]; then
-        git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
-        git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
-        git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_COMMIT_SHA > "$TEMP_FILE"
-        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
-      # other pipelines, like the protected branches pipelines
-      elif [[ "$CI_COMMIT_BEFORE_SHA" != "0000000000000000000000000000000000000000" ]]; then
-        git fetch origin $CI_COMMIT_BEFORE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
-        git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
-        git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA > "$TEMP_FILE"
-        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
-      else
-        # pipeline source could be web, scheduler, etc.
-        git fetch origin $CI_COMMIT_SHA --depth=2 ${GIT_FETCH_EXTRA_FLAGS}
-        git diff --name-only $CI_COMMIT_SHA~1 $CI_COMMIT_SHA > "$TEMP_FILE"
-        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
-      fi
-    # MODIFIED_FILES is a list of files that changed, could be used everywhere
-    - MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | xargs)
-    - echo "MODIFIED_FILES=$MODIFIED_FILES" >> pipeline.env
-    # MR_MODIFIED_FILES and MR_MODIFIED_COMPONENTS are semicolon separated lists that is used in MR only
-    # for non MR pipeline, these are empty lists
-    - |
-      if [ -z "$CI_MERGE_REQUEST_IID" ]; then
-        echo "MR_MODIFIED_FILES=\"\"" >> pipeline.env
-        echo "MR_MODIFIED_COMPONENTS=\"\"" >> pipeline.env
-      else
-        MR_MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | tr '\n' ';')
-        echo "MR_MODIFIED_FILES=\"$MR_MODIFIED_FILES\"" >> pipeline.env
-
-        MR_MODIFIED_COMPONENTS=$(run_cmd python tools/ci/ci_get_mr_info.py components --modified-files $MODIFIED_FILES | tr '\n' ';')
-        echo "MR_MODIFIED_COMPONENTS=\"$MR_MODIFIED_COMPONENTS\"" >> pipeline.env
-      fi
-    - |
-      if [ -n "$PIPELINE_COMMIT_SHA" ]; then
-        echo "PIPELINE_COMMIT_SHA=$PIPELINE_COMMIT_SHA" >> pipeline.env
-      fi
-    - echo "OOCD_DISTRO_URL_ARMHF=$OOCD_DISTRO_URL_ARMHF" >> pipeline.env
-    - echo "OOCD_DISTRO_URL_ARM64=$OOCD_DISTRO_URL_ARM64" >> pipeline.env
-    - run_cmd idf-ci gitlab pipeline-variables >> pipeline.env
-    - cat pipeline.env
-    - run_cmd idf-ci gitlab upload-artifacts --type env
-  artifacts:
-    reports:
-      dotenv: pipeline.env
-    paths:
-      - pipeline.env
-    expire_in: 1 week
-    when: always
-
-baseline_manifest_sha:
-  extends:
-    - .pre_check_template
-    - .rules:dev-push
-  tags: [fast_run, shiny]
-  script:
-    - |
-      # merged results pipelines, by default
-      # diff between target-branch-head and merged-result-head
-      if [ -n "$CI_MERGE_REQUEST_TARGET_BRANCH_SHA" ]; then
-        git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA --depth=1
-        git checkout FETCH_HEAD
-        idf-build-apps dump-manifest-sha \
-          --manifest-files $(find . -name ".build-test-rules.yml" | xargs) \
-          --output .manifest_sha
-      # merge request pipelines, when the mr got conflicts
-      # diff between diff-base-sha and merge-request-head
-      elif [ -n "$CI_MERGE_REQUEST_DIFF_BASE_SHA" ]; then
-        git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1
-        git checkout FETCH_HEAD
-        idf-build-apps dump-manifest-sha \
-          --manifest-files $(find . -name ".build-test-rules.yml" | xargs) \
-          --output .manifest_sha
-      # other pipelines, like the protected branches pipelines
-      # not triggered in this job
-      fi
-  artifacts:
-    paths:
-      - .manifest_sha
-    expire_in: 1 week
-    when: always
+    - python tools/ci/gitlab_yaml_linter.py
 
 retry_failed_jobs:
   extends:

.gitlab/ci/rules.yml

@@ -14,13 +14,6 @@
   - "components/**/CMakeLists.txt"
 
 
-.patterns-python-files: &patterns-python-files
-  - ".gitlab/ci/static-code-analysis.yml"
-  - "**/*.py"
-
-.patterns-sonarqube-files: &patterns-sonarqube-files
-  - "tools/ci/sonar_exclude_list.txt"
-
 .patterns-build_components: &patterns-build_components
   # components files except "test*/" "host*/" folders
   # ?? to include folders less than 4 characters
@@ -176,16 +169,10 @@
 .if-protected-check: &if-protected-check
   if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) || $CI_COMMIT_TAG =~ /^qa-test/'
 
-.if-protected-deploy: &if-protected-deploy
-  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "api")'
-
 .if-protected-branch-push: &if-protected-branch-push
   # rules:changes always evaluates to true for new branch pipelines or when there is no Git push event
   if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/) && $CI_PIPELINE_SOURCE == "push"'
 
-.if-qa-test-tag: &if-qa-test-tag
-  if: '$CI_COMMIT_TAG =~ /^qa-test/'
-
 .if-dev-push: &if-dev-push
   if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")'
 

.gitlab/ci/static-code-analysis.yml

@@ -30,84 +30,3 @@ gcc_static_analyzer:
     - export EXTRA_CFLAGS=
     - export EXTRA_CXXFLAGS=
     - idf-build-apps build -p ${ANALYZING_APP} --disable-targets esp32h4 esp32h21 --no-check-warnings
-#
-## build stage
-## Sonarqube related jobs put here for this reason:
-## Here we have two jobs. code_quality_check and code_quality_report.
-##
-## code_quality_check will analyze the code changes between your MR and
-## code repo stored in sonarqube server. The analysis result is only shown in
-## the comments under this MR and won't be transferred to the server.
-##
-## code_quality_report will analyze and transfer both of the newly added code
-## and the analysis result to the server.
-##
-## Put in the front to ensure that the newly merged code can be stored in
-## sonarqube server ASAP, in order to avoid reporting unrelated code issues
-#.sonar_scan_template:
-#  stage: build
-#  extends: .pre_check_template
-#  # full clone since this image does not support fetch --shallow-since-cutoff
-#  # shiny runners are used for full clone
-#  tags: [build, shiny]
-#  image: $SONARQUBE_SCANNER_IMAGE
-#  before_script:
-#    - source tools/ci/utils.sh
-#    - export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH"
-#    - fetch_submodules
-#    # Exclude the submodules, all paths ends with /**
-#    - submodules=$(get_all_submodules)
-#    # get all exclude paths specified in tools/ci/sonar_exclude_list.txt | ignore lines start with # | xargs | replace all <space> to <comma>
-#    - custom_excludes=$(cat $CI_PROJECT_DIR/tools/ci/sonar_exclude_list.txt | grep -v '^#' | xargs | sed -e 's/ /,/g')
-#    # Exclude the report dir as well
-#    - export EXCLUSIONS="$custom_excludes,$submodules"
-#    - export SONAR_SCANNER_OPTS="-Xmx2048m"
-#  variables:
-#    GIT_DEPTH: 0
-#    REPORT_PATTERN: clang_tidy_reports/**/*.txt
-#  artifacts:
-#    paths:
-#      - $REPORT_PATTERN
-#    expire_in: 1 week
-#    when: always
-#  dependencies:  # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here.
-#    - clang_tidy_check
-#
-#code_quality_check:
-#  extends:
-#    - .sonar_scan_template
-#    - .rules:patterns:static-code-analysis-preview
-#  allow_failure: true  # it's using exit code to indicate the code analysis result,
-#                       # we don't want to block ci when critical issues founded
-#  script:
-#    - export CI_MERGE_REQUEST_COMMITS=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py commits --src-branch ${CI_COMMIT_REF_NAME} | tr '\n' ',')
-#    # test if this branch have merge request, if not, exit 0
-#    - test -n "$CI_MERGE_REQUEST_IID" || exit 0
-#    - test -n "$CI_MERGE_REQUEST_COMMITS" || exit 0
-#    - sonar-scanner
-#      -Dsonar.analysis.mode=preview
-#      -Dsonar.branch.name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
-#      -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN
-#      -Dsonar.exclusions=$EXCLUSIONS
-#      -Dsonar.gitlab.ci_merge_request_iid=$CI_MERGE_REQUEST_IID
-#      -Dsonar.gitlab.commit_sha=$CI_MERGE_REQUEST_COMMITS
-#      -Dsonar.gitlab.merge_request_discussion=true
-#      -Dsonar.gitlab.ref_name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
-#      -Dsonar.host.url=$SONAR_HOST_URL
-#      -Dsonar.login=$SONAR_LOGIN
-#
-#code_quality_report:
-#  extends:
-#    - .sonar_scan_template
-#    - .rules:protected:check
-#  allow_failure: true  # it's using exit code to indicate the code analysis result,
-#                       # we don't want to block ci when critical issues founded
-#  script:
-#    - sonar-scanner
-#      -Dsonar.branch.name=$CI_COMMIT_REF_NAME
-#      -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN
-#      -Dsonar.exclusions=$EXCLUSIONS
-#      -Dsonar.gitlab.commit_sha=$PIPELINE_COMMIT_SHA
-#      -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME
-#      -Dsonar.host.url=$SONAR_HOST_URL
-#      -Dsonar.login=$SONAR_LOGIN

.gitlab/ci/test-win.yml

@@ -1,6 +1,6 @@
 .host_test_win_template:
   extends:
-    - .brew-windows-settings
+    - .windows-settings
     - .rules:test:host_test
   stage: host_test
   dependencies: # set dependencies to null to avoid missing artifacts issue
@@ -73,7 +73,7 @@ pytest_build_system_win:
   extends:
     - .test_build_system_template_win
     - .rules:labels:windows_pytest_build_system
-  parallel: 2
+  parallel: 6
 
 pytest_build_system_win_minimal_cmake:
   extends:
@@ -82,7 +82,7 @@ pytest_build_system_win_minimal_cmake:
   variables:
     MINIMAL_CMAKE_VERSION: "3.16.3"
   script:
-    - python ${IDF_PATH}\tools\idf_tools.py install cmake@${MINIMAL_CMAKE_VERSION}
+    - python tools\idf_tools.py install cmake@${MINIMAL_CMAKE_VERSION}
     - $Env:PATH = "$Env:USERPROFILE\.espressif\tools\cmake\${MINIMAL_CMAKE_VERSION}\bin;$Env:PATH"
     - |
       $actualVersion = (& cmake --version).Split()[2]
@@ -90,6 +90,6 @@ pytest_build_system_win_minimal_cmake:
         Write-Error "ERROR: Wrong CMake version! Detected: $actualVersion, but expected: $Env:MINIMAL_CMAKE_VERSION"
         exit 1
       }
-    - cd ${IDF_PATH}\tools\test_build_system
+    - cd tools\test_build_system
     - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
-    - pytest -k cmake --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
+    - pytest -k cmake --junitxml ${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}

.pre-commit-config.yaml

@@ -146,7 +146,7 @@ repos:
         require_serial: true
         additional_dependencies:
           - PyYAML == 5.3.1
-          - idf-build-apps~=2.13
+          - idf-build-apps~=2.16
       - id: sort-yaml-files
         name: sort yaml files
         entry: tools/ci/sort_yaml.py
@@ -167,14 +167,6 @@ repos:
         language: python
         always_run: true
         require_serial: true
-      - id: gitlab-yaml-linter
-        name: Check gitlab yaml files
-        entry: tools/ci/gitlab_yaml_linter.py
-        language: python
-        files: '\.gitlab-ci\.yml|\.gitlab/ci/.+\.yml|\.gitmodules'
-        pass_filenames: false
-        additional_dependencies:
-          - PyYAML == 5.3.1
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.0.1
     hooks:

conftest.py

@@ -598,7 +598,7 @@ def pytest_unconfigure(config: Config) -> None:
         config.pluginmanager.unregister(idf_local_plugin)
 
 
-dut_artifacts_url = []
+dut_artifacts_url: t.List[str] = []
 
 
 @pytest.hookimpl(hookwrapper=True)

sonar-project.properties

@@ -1,24 +0,0 @@
-sonar.projectKey=esp-idf
-sonar.projectName=Espressif IoT Development Framework
-sonar.projectVersion=4.4
-
-sonar.sources=.
-sonar.sourceEncoding=UTF-8
-
-sonar.inclusions=\
-  **/*.c,**/*.cpp,**/*.h,**/*.hpp,\
-  **/*.py
-
-####################
-# sonar-cxx-plugin #
-####################
-
-sonar.cxx.includeDirectories=/usr/include
-sonar.cxx.file.suffixes=.cxx,.cpp,.cc,.c,.hxx,.hpp,.hh,.h
-
-#######################
-# sonar-gitlab-plugin #
-#######################
-
-sonar.gitlab.failure_notification_mode=exit-code
-sonar.gitlab.project_id=103

tools/ci/check_build_test_rules.py

@@ -9,7 +9,6 @@ import typing as t
 from collections import defaultdict
 from pathlib import Path
 
-import yaml
 from idf_build_apps import App
 from idf_build_apps import find_apps
 from idf_build_apps.constants import SUPPORTED_TARGETS
@@ -40,6 +39,14 @@ USUAL_TO_FORMAL = {
 
 FORMAL_TO_USUAL = {v: k for k, v in USUAL_TO_FORMAL.items()}
 
+# Targets under bringup: suppress check-test-scripts warnings for these.
+# Reset to [] once bringup is complete.
+BYPASS_CHECK_TEST_TARGETS: t.List[str] = [
+    'esp32h21',
+    'esp32h4',
+    'esp32c5',
+]
+
 
 def print_diff_table(
     list1: t.List[str], list2: t.List[str], title1: str, title2: str, excluded: t.Optional[t.List[str]] = None
@@ -299,21 +306,9 @@ if __name__ == '__main__':
 
     readme_parser = action.add_parser('check-readmes')
     readme_parser.add_argument('paths', nargs='+', help='check under paths')
-    readme_parser.add_argument(
-        '-c',
-        '--config',
-        default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'),
-        help='config file',
-    )
 
     test_parser = action.add_parser('check-test-scripts')
     test_parser.add_argument('paths', nargs='+', help='check under paths')
-    test_parser.add_argument(
-        '-c',
-        '--config',
-        default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'),
-        help='config file',
-    )
     arg = parser.parse_args()
 
     check_dirs = set()
@@ -344,14 +339,6 @@ if __name__ == '__main__':
         os.path.join(IDF_PATH, 'tools', 'cmakev2', 'test'),
     ]
 
-    _bypass_targets: t.List[str] = []
-    if arg.config:
-        with open(arg.config) as fr:
-            configs = yaml.safe_load(fr)
-
-        if configs:
-            _bypass_targets = configs.get('bypass_check_test_targets') or []
-
     os.environ.update(
         {
             'IDF_PATH': IDF_PATH,
@@ -369,5 +356,5 @@ if __name__ == '__main__':
         check_test_scripts(
             list(check_dirs),
             _exclude_dirs,
-            bypass_targets=_bypass_targets,
+            bypass_targets=BYPASS_CHECK_TEST_TARGETS,
         )

tools/ci/ci_get_mr_info.py

@@ -1,165 +0,0 @@
-#!/usr/bin/env python
-#
-# internal use only for CI
-# get latest MR information by source branch
-#
-# SPDX-FileCopyrightText: 2020-2024 Espressif Systems (Shanghai) CO LTD
-# SPDX-License-Identifier: Apache-2.0
-#
-import argparse
-import logging
-import os
-import subprocess
-import typing as t
-from pathlib import Path
-
-from gitlab_api import Gitlab
-from idf_ci_utils import IDF_PATH
-
-if t.TYPE_CHECKING:
-    from gitlab.v4.objects import ProjectCommit, ProjectMergeRequest
-
-
-def _get_mr_obj(source_branch: str) -> t.Optional['ProjectMergeRequest']:
-    gl = Gitlab(os.getenv('CI_PROJECT_ID', 'espressif/esp-idf'))
-    if not gl.project:
-        return None
-
-    mrs = gl.project.mergerequests.list(state='opened', source_branch=source_branch)
-    if mrs:
-        return mrs[0]  # one source branch can only have one opened MR at one moment
-    else:
-        return None
-
-
-def get_mr_iid(source_branch: str) -> str:
-    mr = _get_mr_obj(source_branch)
-    if not mr:
-        return ''
-    else:
-        return str(mr.iid)
-
-
-def get_mr_changed_files(source_branch: str) -> t.List[str]:
-    mr = _get_mr_obj(source_branch)
-    if not mr:
-        return []
-
-    git_output = subprocess.check_output(
-        ['git', 'diff', '--name-only', '--diff-filter=d', f'origin/{mr.target_branch}...origin/{source_branch}']
-    ).decode('utf8')
-
-    return [line.strip() for line in git_output.splitlines() if line.strip()]
-
-
-def get_mr_commits(source_branch: str) -> t.List['ProjectCommit']:
-    mr = _get_mr_obj(source_branch)
-    if not mr:
-        return []
-
-    return list(mr.commits())
-
-
-_COMPONENT_NAME_DIR_RECORDS = {}
-
-
-def get_modified_component(filepath: str) -> t.Optional[str]:
-    """Return the component name if the file is in a component directory, otherwise None."""
-    try:
-        f_path = Path(filepath).resolve().relative_to(IDF_PATH)
-    except ValueError:  # not in IDF_PATH
-        return None
-
-    # skip md files, etc.
-    if f_path.suffix in ['.md', '.yml']:
-        return None
-
-    # skip test_apps files
-    if 'test_apps' in f_path.parts:
-        return None
-
-    component_parent_dirs = [f_path.parts[0]]
-    for part in f_path.parts[1:]:
-        if component_parent_dirs[-1] == 'components' or component_parent_dirs[-1].endswith('common_components'):
-            if part not in _COMPONENT_NAME_DIR_RECORDS:
-                logging.debug('Found component "%s" in path "%s"' % (part, component_parent_dirs))
-                _COMPONENT_NAME_DIR_RECORDS[part] = component_parent_dirs
-            elif _COMPONENT_NAME_DIR_RECORDS.get(part) != component_parent_dirs:
-                logging.debug(
-                    'WARNING!!! Found component "%s" in path "%s" and "%s"'
-                    % (part, component_parent_dirs, _COMPONENT_NAME_DIR_RECORDS.get(part))
-                )
-
-            return part
-
-        component_parent_dirs.append(part)
-
-    return None
-
-
-def get_mr_components(
-    source_branch: t.Optional[str] = None, modified_files: t.Optional[t.List[str]] = None
-) -> t.List[str]:
-    components: t.Set[str] = set()
-    if modified_files is None:
-        if not source_branch:
-            raise RuntimeError('--src-branch is required if --modified-files is not provided')
-
-        modified_files = get_mr_changed_files(source_branch)
-
-    for f in modified_files:
-        modified_component = get_modified_component(f)
-        if modified_component:
-            components.add(modified_component)
-
-    return list(components)
-
-
-def get_target_in_tags(tags: str) -> str:
-    from idf_pytest.constants import TARGET_MARKERS
-
-    for x in tags.split(','):
-        if x in TARGET_MARKERS:
-            return x
-
-    raise RuntimeError(f'No target marker found in {tags}')
-
-
-def _print_list(_list: t.List[str], separator: str = '\n') -> None:
-    print(separator.join(_list))
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Get the latest merge request info by pipeline')
-    actions = parser.add_subparsers(dest='action', help='info type', required=True)
-
-    common_args = argparse.ArgumentParser(add_help=False)
-    common_args.add_argument('--src-branch', help='source branch')
-    common_args.add_argument(
-        '--modified-files',
-        nargs='+',
-        help='space-separated list specifies the modified files. will be detected by --src-branch if not provided',
-    )
-
-    actions.add_parser('id', parents=[common_args])
-    actions.add_parser('commits', parents=[common_args])
-    actions.add_parser('components', parents=[common_args])
-    target = actions.add_parser('target_in_tags')
-    target.add_argument('tags', help='comma separated tags, e.g., esp32,generic')
-
-    args = parser.parse_args()
-
-    if args.action == 'id':
-        if not args.src_branch:
-            raise RuntimeError('--src-branch is required')
-        print(get_mr_iid(args.src_branch))
-    elif args.action == 'commits':
-        if not args.src_branch:
-            raise RuntimeError('--src-branch is required')
-        _print_list([commit.id for commit in get_mr_commits(args.src_branch)])
-    elif args.action == 'components':
-        _print_list(get_mr_components(args.src_branch, args.modified_files))
-    elif args.action == 'target_in_tags':
-        print(get_target_in_tags(args.tags))
-    else:
-        raise NotImplementedError('not possible to get here')

tools/ci/deploy_docs.py

@@ -1,217 +0,0 @@
-#!/usr/bin/env python3
-#
-# CI script to deploy docs to a webserver. Not useful outside of CI environment
-#
-#
-# SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD
-# SPDX-License-Identifier: Apache-2.0
-#
-import glob
-import os
-import os.path
-import re
-import stat
-import subprocess
-import sys
-import tarfile
-from typing import Any, List, Tuple
-
-import packaging.version
-
-
-def env(variable: str, default: str=None) -> str:
-    """ Shortcut to return the expanded version of an environment variable """
-    return os.path.expandvars(os.environ.get(variable, default) if default else os.environ[variable])
-
-
-# import sanitize_version from the docs directory, shared with here
-sys.path.append(os.path.join(env('IDF_PATH'), 'docs'))
-from sanitize_version import sanitize_version  # noqa
-
-
-def main() -> None:
-    # if you get KeyErrors on the following lines, it's probably because you're not running in Gitlab CI
-    git_ver = env('GIT_VER')  # output of git describe --always
-    ci_ver = env('CI_COMMIT_REF_NAME', git_ver)  # branch or tag we're building for (used for 'release' & URL)
-
-    version = sanitize_version(ci_ver)
-    print('Git version: {}'.format(git_ver))
-    print('CI Version: {}'.format(ci_ver))
-    print('Deployment version: {}'.format(version))
-
-    if not version:
-        raise RuntimeError('A version is needed to deploy')
-
-    build_dir = env('DOCS_BUILD_DIR')  # top-level local build dir, where docs have already been built
-
-    if not build_dir:
-        raise RuntimeError('Valid DOCS_BUILD_DIR is needed to deploy')
-
-    url_base = env('DOCS_DEPLOY_URL_BASE')  # base for HTTP URLs, used to print the URL to the log after deploying
-
-    docs_server = env('DOCS_DEPLOY_SERVER')  # ssh server to deploy to
-    docs_user = env('DOCS_DEPLOY_SERVER_USER')
-    docs_path = env('DOCS_DEPLOY_PATH')  # filesystem path on DOCS_SERVER
-
-    if not docs_server:
-        raise RuntimeError('Valid DOCS_DEPLOY_SERVER is needed to deploy')
-
-    if not docs_user:
-        raise RuntimeError('Valid DOCS_DEPLOY_SERVER_USER is needed to deploy')
-
-    docs_server = '{}@{}'.format(docs_user, docs_server)
-
-    if not docs_path:
-        raise RuntimeError('Valid DOCS_DEPLOY_PATH is needed to deploy')
-
-    print('DOCS_DEPLOY_SERVER {} DOCS_DEPLOY_PATH {}'.format(docs_server, docs_path))
-
-    tarball_path, version_urls = build_doc_tarball(version, git_ver, build_dir)
-
-    deploy(version, tarball_path, docs_path, docs_server)
-
-    print('Docs URLs:')
-    doc_deploy_type = os.getenv('TYPE')
-    for vurl in version_urls:
-        language, _, target = vurl.split('/')
-        tag = '{}_{}'.format(language, target)
-        url = '{}/{}/index.html'.format(url_base, vurl)  # (index.html needed for the preview server)
-        url = re.sub(r'([^:])//', r'\1/', url)  # get rid of any // that isn't in the https:// part
-        print('[document {}][{}] {}'.format(doc_deploy_type, tag, url))
-
-    # note: it would be neater to use symlinks for stable, but because of the directory order
-    # (language first) it's kind of a pain to do on a remote server, so we just repeat the
-    # process but call the version 'stable' this time
-    if is_stable_version(version):
-        print('Deploying again as stable version...')
-        tarball_path, version_urls = build_doc_tarball('stable', git_ver, build_dir)
-        deploy('stable', tarball_path, docs_path, docs_server)
-
-
-def deploy(version: str, tarball_path: str, docs_path: str, docs_server: str) -> None:
-    def run_ssh(commands: List) -> None:
-        """ Log into docs_server and run a sequence of commands using ssh """
-        print('Running ssh: {}'.format(commands))
-        subprocess.run(['ssh', '-o', 'BatchMode=yes', docs_server, '-x', ' && '.join(commands)], check=True)
-
-    # copy the version tarball to the server
-    run_ssh(['mkdir -p {}'.format(docs_path)])
-    print('Running scp {} to {}'.format(tarball_path, '{}:{}'.format(docs_server, docs_path)))
-    subprocess.run(['scp', '-B', tarball_path, '{}:{}'.format(docs_server, docs_path)], check=True)
-
-    tarball_name = os.path.basename(tarball_path)
-
-    run_ssh(['cd {}'.format(docs_path),
-             'rm -rf ./*/{}'.format(version),  # remove any pre-existing docs matching this version
-             'tar -zxvf {}'.format(tarball_name),  # untar the archive with the new docs
-             'rm {}'.format(tarball_name)])
-
-    # Note: deleting and then extracting the archive is a bit awkward for updating stable/latest/etc
-    # as the version will be invalid for a window of time. Better to do it atomically, but this is
-    # another thing made much more complex by the directory structure putting language before version...
-
-
-def build_doc_tarball(version: str, git_ver: str, build_dir: str) -> Tuple[str, List]:
-    """ Make a tar.gz archive of the docs, in the directory structure used to deploy as
-        the given version """
-    version_paths = []
-    tarball_path = '{}/{}.tar.gz'.format(build_dir, version)
-
-    # find all the 'html/' directories under build_dir
-    html_dirs = glob.glob('{}/**/html/'.format(build_dir), recursive=True)
-    print('Found %d html directories' % len(html_dirs))
-
-    pdfs = glob.glob('{}/**/latex/build/*.pdf'.format(build_dir), recursive=True)
-    print('Found %d PDFs in latex directories' % len(pdfs))
-
-    # add symlink for stable and latest and adds them to PDF blob
-    symlinks = create_and_add_symlinks(version, git_ver, pdfs)
-
-    def not_sources_dir(ti: Any) -> Any:
-        print(type(ti))
-        """ Filter the _sources directories out of the tarballs """
-        if ti.name.endswith('/_sources'):
-            return None
-
-        ti.mode |= stat.S_IWGRP  # make everything group-writeable
-        return ti
-
-    try:
-        os.remove(tarball_path)
-    except OSError:
-        pass
-
-    with tarfile.open(tarball_path, 'w:gz') as tarball:
-        for html_dir in html_dirs:
-            # html_dir has the form '<ignored>/<language>/<target>/html/'
-            target_dirname = os.path.dirname(os.path.dirname(html_dir))
-            target = os.path.basename(target_dirname)
-            language = os.path.basename(os.path.dirname(target_dirname))
-
-            # when deploying, we want the top-level directory layout 'language/version/target'
-            archive_path = '{}/{}/{}'.format(language, version, target)
-            print("Archiving '{}' as '{}'...".format(html_dir, archive_path))
-            tarball.add(html_dir, archive_path, filter=not_sources_dir)
-            version_paths.append(archive_path)
-
-        for pdf_path in pdfs:
-            # pdf_path has the form '<ignored>/<language>/<target>/latex/build'
-            latex_dirname = os.path.dirname(pdf_path)
-            pdf_filename = os.path.basename(pdf_path)
-            target_dirname = os.path.dirname(os.path.dirname(latex_dirname))
-            target = os.path.basename(target_dirname)
-            language = os.path.basename(os.path.dirname(target_dirname))
-
-            # when deploying, we want the layout 'language/version/target/pdf'
-            archive_path = '{}/{}/{}/{}'.format(language, version, target, pdf_filename)
-            print("Archiving '{}' as '{}'...".format(pdf_path, archive_path))
-            tarball.add(pdf_path, archive_path)
-
-    for symlink in symlinks:
-        os.unlink(symlink)
-
-    return (os.path.abspath(tarball_path), version_paths)
-
-
-def create_and_add_symlinks(version: str, git_ver: str, pdfs: List) -> List:
-    """ Create symbolic links for PDFs for 'latest' and 'stable' releases """
-
-    symlinks = []
-    if 'stable' in version or 'latest' in version:
-        for pdf_path in pdfs:
-            symlink_path = pdf_path.replace(git_ver, version)
-            os.symlink(pdf_path, symlink_path)
-            symlinks.append(symlink_path)
-
-        pdfs.extend(symlinks)
-        print('Found %d PDFs in latex directories after adding symlink' % len(pdfs))
-
-    return symlinks
-
-
-def is_stable_version(version: str) -> bool:
-    """ Heuristic for whether this is the latest stable release """
-    if not version.startswith('v'):
-        return False  # branch name
-    if '-' in version:
-        return False  # prerelease tag
-
-    git_out = subprocess.check_output(['git', 'tag', '-l']).decode('utf-8')
-
-    versions = [v.strip() for v in git_out.split('\n')]
-    versions = [v for v in versions if re.match(r'^v[\d\.]+$', v.strip())]  # include vX.Y.Z only
-
-    versions_pack = [packaging.version.parse(v) for v in versions]
-
-    max_version = max(versions_pack)
-
-    if max_version.public != version[1:]:
-        print('Stable version is v{}. This version is {}.'.format(max_version.public, version))
-        return False
-    else:
-        print('This version {} is the stable version'.format(version))
-        return True
-
-
-if __name__ == '__main__':
-    main()

tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py

@@ -55,7 +55,7 @@ def main(output_filepath: str) -> None:
     exclude_runner_tags_matching = []
     for _tag in known_warnings_dict.get('no_runner_tags', []):
         if '*' not in _tag:
-            exclude_runner_tags_set.add(_tag)
+            exclude_runner_tags_set.add(frozenset(_tag.split(',')))
         else:
             if res := _process_match_group(_tag):
                 exclude_runner_tags_matching.append(res)
@@ -66,7 +66,7 @@ def main(output_filepath: str) -> None:
     if exclude_runner_tags := os.getenv('EXCLUDE_RUNNER_TAGS'):
         for _tag in exclude_runner_tags.split(';'):
             if '*' not in _tag:
-                exclude_runner_tags_set.add(_tag)
+                exclude_runner_tags_set.add(frozenset(_tag.split(',')))
             else:
                 if res := _process_match_group(_tag):
                     exclude_runner_tags_matching.append(res)
@@ -75,7 +75,7 @@ def main(output_filepath: str) -> None:
     additional_dict: dict[GroupKey, dict[str, t.Any]] = {}
     for key, grouped_cases in cases.grouped_cases.items():
         # skip test cases with no runner tags
-        if ','.join(sorted(key.runner_tags)) in exclude_runner_tags_set:
+        if frozenset(key.runner_tags) in exclude_runner_tags_set:
             print(f'WARNING: excluding test cases with runner tags: {key.runner_tags}')
             continue
 

tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml

@@ -17,6 +17,9 @@
     # Since the version 3.21 CMake passes source files and include dirs to ninja using absolute paths.
     # Needed for pytest junit reports.
     INSTALL_EXTRA_TOOLS: cmake
+    # Suppress Python warning messages during CI builds to keep logs clean and focused on errors.
+    # This prevents non-critical warnings (e.g., deprecation or resource warnings) from cluttering output.
+    PYTHONWARNINGS: "ignore"
   needs:
     - pipeline: $PARENT_PIPELINE_ID
       job: generate_build_child_pipeline
@@ -40,7 +43,6 @@
     - run_cmd idf-ci build run
       --parallel-count ${CI_NODE_TOTAL:-1}
       --parallel-index ${CI_NODE_INDEX:-1}
-      --modified-files ${MR_MODIFIED_FILES}
 
 .dynamic_target_test_template:
   image: $TARGET_TEST_ENV_IMAGE
@@ -56,6 +58,8 @@
   needs:
     - pipeline: $PARENT_PIPELINE_ID
       job: pipeline_variables
+    - pipeline: $PARENT_PIPELINE_ID
+      job: snapshot_known_failure_cases
   artifacts:
     paths:
       - XUNIT_RESULT*.xml
@@ -65,7 +69,11 @@
     when: always
     expire_in: 1 week
   script:
+    # Merge the current known failure cases file with snapshot_known_failure_cases file captured at pipeline start
+    # to avoid retry failures caused by later updates to the known failure cases file.
     - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
+    - printf '\n' >> ${KNOWN_FAILURE_CASES_FILE_NAME}
+    - cat artifacts/${KNOWN_FAILURE_CASES_FILE_NAME}.freeze >> ${KNOWN_FAILURE_CASES_FILE_NAME}
     # get runner env config file
     - retry_failed git clone $TEST_ENV_CONFIG_REPO
     - run_cmd python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs

tools/ci/exclude_check_tools_files.txt

@@ -4,18 +4,15 @@ tools/ci/check_*.py
 tools/ci/check_*.txt
 tools/ci/check_*.sh
 tools/ci/check_copyright_config.yaml
-tools/ci/get_all_test_results.py
 tools/gdb_panic_server.py
 tools/check_term.py
 tools/python_version_checker.py
 tools/ci/astyle-rules.yml
 tools/ci/checkout_project_ref.py
 tools/ci/ci_fetch_submodule.py
-tools/ci/ci_get_mr_info.py
 tools/ci/ci_process_description.py
 tools/ci/configure_ci_environment.sh
 tools/ci/generate_rules.py
-tools/ci/deploy_docs.py
 tools/ci/envsubst.py
 tools/ci/*exclude*.txt
 tools/ci/executable-list.txt

tools/ci/executable-list.txt

@@ -72,7 +72,6 @@ tools/ci/check_tools_files_patterns.py
 tools/ci/check_type_comments.py
 tools/ci/checkout_project_ref.py
 tools/ci/cleanup_ignore_lists.py
-tools/ci/deploy_docs.py
 tools/ci/envsubst.py
 tools/ci/fix_empty_prototypes.sh
 tools/ci/generate_rules.py

tools/ci/generate_rules.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 #
-# SPDX-FileCopyrightText: 2021-2025 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 import argparse
 import inspect
@@ -12,7 +12,6 @@ from itertools import product
 
 import yaml
 from idf_ci_utils import IDF_PATH
-from idf_ci_utils import GitlabYmlConfig
 
 if t.TYPE_CHECKING:
     import pygraphviz as pgv
@@ -90,7 +89,6 @@ class RulesWriter:
         self.cfg = self.expand_matrices()
         self.rules = self.expand_rules()
 
-        self.yml_config = GitlabYmlConfig()
         self.graph = None
 
     def expand_matrices(self):  # type: () -> dict
@@ -195,10 +193,6 @@ class RulesWriter:
             if k.startswith('pattern'):
                 continue
 
-            if '.rules:' + k not in self.yml_config.used_templates:
-                print(f'WARNING: unused rule: {k}, skipping...')
-                continue
-
             res.append(self.RULES_TEMPLATE.format(k, self._format_rule(k, v)))
         return '\n\n'.join(res)
 

tools/ci/get_all_test_results.py

@@ -1,84 +0,0 @@
-#!/usr/bin/env python
-#
-# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
-# SPDX-License-Identifier: Apache-2.0
-
-
-'''
-Download artifacts from all test jobs
-
-CI_PROJECT_DIR
-└── TEST_RESULTS
-    ├── <job_id>
-    │   └── XUNIT_RESULT.xml
-    ├── <job_id>
-    │   └── XUNIT_RESULT.xml
-    ├── <job_id>
-    │   └── TEST_LOGS
-
-'''
-
-import argparse
-import json
-import os
-import shutil
-
-import gitlab_api
-
-CI_PROJECT_ID = int(os.getenv('CI_PROJECT_ID', ''))
-CI_PIPELINE_ID = int(os.getenv('CI_PIPELINE_ID', ''))
-IDF_PATH = os.getenv('IDF_PATH', '.')
-TEST_RESULT_PATH = os.path.join(IDF_PATH, 'TEST_RESULTS')
-
-
-def download_all_test_results(result_path: str, include_retried: bool = False) -> None:
-    if os.path.exists(result_path):
-        shutil.rmtree(result_path)
-    os.makedirs(result_path, exist_ok=True)
-
-    gitlab_inst = gitlab_api.Gitlab(CI_PROJECT_ID)
-    pipelines = gitlab_inst.project.pipelines.get(CI_PIPELINE_ID)
-    _include_retried = 'true' if include_retried else 'false'
-    jobs = pipelines.jobs.list(all=True, per_page=100, include_retried=_include_retried)
-
-    job_info_list = []
-    for job in jobs:
-        if job.stage in ['target_test', 'host_test']:
-            log_path = ''
-            if job.status not in ['success', 'failed']:
-                print('Job {}({}) is not finished'.format(job.id, job.name))
-            elif not hasattr(job, 'artifacts_file'):
-                print('Job {}({}) has no artifacts.'.format(job.id, job.name))
-            else:
-                log_path = os.path.join(result_path, 'job_{}'.format(job.id))
-                print('Downloading artifacts from: {}'.format(job.name))
-                os.makedirs(log_path, exist_ok=True)
-                gitlab_inst.download_artifacts(job.id, log_path)
-
-            job_info = {
-                'id': job.id,
-                'name': job.name,
-                'tag_list': job.tag_list,
-                'status': job.status,
-                'stage': job.stage,
-                'web_url': job.web_url,
-                'commit_url': job.commit['web_url'],
-                'log_path': log_path,
-            }
-            job_info_list.append(job_info)
-
-    with open(os.path.join(result_path, 'index.json'), 'w') as f:
-        f.write(json.dumps({'jobs': job_info_list}, indent=1, sort_keys=True))
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--path', '-p', default=TEST_RESULT_PATH, help='Path to save result files.')
-    parser.add_argument('--include_retried', action='store_true', help='Including retried jobs.')
-    args = parser.parse_args()
-
-    download_all_test_results(args.path, args.include_retried)
-
-
-if __name__ == '__main__':
-    main()

tools/ci/gitlab_yaml_linter.py

@@ -1,17 +1,18 @@
 #!/usr/bin/env python
-# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 """
 Check gitlab ci yaml files
 """
+
 import argparse
 import os
 import typing as t
 from functools import cached_property
 
-from idf_ci_utils import get_submodule_dirs
-from idf_ci_utils import GitlabYmlConfig
 from idf_ci_utils import IDF_PATH
+from idf_ci_utils import GitlabYmlConfig
+from idf_ci_utils import get_submodule_dirs
 
 
 class YmlLinter:
@@ -43,17 +44,6 @@ class YmlLinter:
 
         exit(exit_code)
 
-    # name it like _1_ to make it run first
-    def _lint_1_yml_parser(self) -> None:
-        for k, v in self.yml_config.config.items():
-            if (
-                k not in self.yml_config.global_keys
-                and k not in self.yml_config.anchors
-                and k not in self.yml_config.templates
-                and k not in self.yml_config.jobs
-            ):
-                raise SystemExit(f'Parser incorrect. Key {k} not in global keys, anchors, templates,  or jobs')
-
     def _lint_default_values_artifacts(self) -> None:
         defaults_artifacts = self.yml_config.default.get('artifacts', {})
 
@@ -79,19 +69,6 @@ class YmlLinter:
                 for item in undefined_patterns:
                     self._errors.append(f'undefined pattern {item}. Please add {item} to .patterns-submodule')
 
-    def _lint_gitlab_yml_templates(self) -> None:
-        unused_templates = self.yml_config.templates.keys() - self.yml_config.used_templates
-        for item in unused_templates:
-            # known unused ones
-            if item not in [
-                '.before_script:fetch:target_test',  # used in dynamic pipeline
-            ]:
-                self._errors.append(f'Unused template: {item}, please remove it')
-
-        undefined_templates = self.yml_config.used_templates - self.yml_config.templates.keys()
-        for item in undefined_templates:
-            self._errors.append(f'Undefined template: {item}')
-
     def _lint_dependencies_and_needs(self) -> None:
         """
         Use `dependencies: []` together with `needs: []` could cause missing artifacts issue.

tools/ci/idf_ci_utils.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2020-2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2020-2026 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 # internal use only for CI
 # some CI related util functions
@@ -125,35 +125,106 @@ class GitlabYmlConfig:
         # avoid unused import in other pre-commit hooks
         import yaml
 
-        all_config = dict()
-        root_yml = yaml.load(open(root_yml_filepath), Loader=yaml.FullLoader)
-
-        # expanding "include"
-        for item in root_yml.pop('include', []) or []:
-            if isinstance(item, dict):
-                if 'project' in item:
-                    continue
-                elif 'local' in item:
-                    item = item['local']
-                else:
-                    continue
-
-            all_config.update(yaml.load(open(os.path.join(IDF_PATH, item)), Loader=yaml.FullLoader))
+        merged_yaml = self._compile_via_gitlab_api(root_yml_filepath)
+        all_config = yaml.load(merged_yaml, Loader=yaml.FullLoader) or {}
 
         if 'default' in all_config:
             self._defaults = all_config.pop('default')
 
         self._config = all_config
 
-        # anchor is the string that will be reused in templates
-        self._anchor_keys: t.Set[str] = set()
-        # template is a dict that will be extended
-        self._template_keys: t.Set[str] = set()
-        self._used_template_keys: t.Set[str] = set()  # tracing the used templates
-        # job is a dict that will be executed
-        self._job_keys: t.Set[str] = set()
+    def _inline_local_includes(self, root_yml_filepath: str) -> str:
+        """
+        Recursively resolve `include: local` entries straight from disk (so uncommitted local
+        changes are always picked up -- CI runners also work off a disk checkout, so there's no
+        need to fetch a ref remotely via `ci_lint`'s `content_ref`/`ref`/`dry_run_ref` GET
+        params). `include: project` entries (including ones nested inside local files, e.g.
+        `.gitlab/ci/common.yml` including `templates/idf/common-scripts.yml`) are collected and
+        left in the final `include:` list, since those files live in another GitLab project and
+        can only be resolved remotely.
 
-        self.expand_extends()
+        A dedicated Loader/Dumper pair round-trips `!reference` tags as a marker list subclass,
+        since GitLab CI uses `!reference` which plain YAML doesn't know, and we need to parse
+        (to merge dicts, not just string-concat) then re-dump losslessly.
+
+        :param root_yml_filepath: path to the local root yml file to compile
+        :return: yml content with local includes resolved and merged
+        """
+        import yaml
+
+        class _Loader(yaml.FullLoader):
+            pass
+
+        class _Dumper(yaml.Dumper):
+            pass
+
+        class _Reference(list):
+            pass
+
+        _Loader.add_constructor(
+            '!reference', lambda loader, node: _Reference(loader.construct_sequence(t.cast(yaml.SequenceNode, node)))
+        )
+        _Dumper.add_representer(_Reference, lambda dumper, data: dumper.represent_sequence('!reference', list(data)))
+
+        def resolve(yml_filepath: str) -> t.Tuple[dict, list]:
+            with open(yml_filepath) as fr:
+                data = yaml.load(fr, Loader=_Loader) or {}
+
+            includes = to_list(data.pop('include', None))
+
+            merged: dict = {}
+            remaining_project_includes: list = []
+            for item in includes:
+                if isinstance(item, dict):
+                    if 'project' in item:
+                        remaining_project_includes.append(item)
+                        continue
+                    elif 'local' in item:
+                        local_path = item['local']
+                    else:
+                        continue
+                elif isinstance(item, str):
+                    local_path = item
+                else:
+                    continue
+
+                sub_merged, sub_remaining = resolve(os.path.join(IDF_PATH, local_path.lstrip('/')))
+                merged.update(sub_merged)
+                remaining_project_includes.extend(sub_remaining)
+
+            # this file's own top-level keys override whatever its includes defined
+            merged.update(data)
+            return merged, remaining_project_includes
+
+        merged_config, project_includes = resolve(root_yml_filepath)
+        if project_includes:
+            merged_config['include'] = project_includes
+
+        return yaml.dump(merged_config, Dumper=_Dumper, sort_keys=False)  # type: ignore
+
+    def _compile_via_gitlab_api(self, root_yml_filepath: str) -> str:
+        """
+        Call the GitLab CI Lint API to get the fully compiled (all `include`s resolved) yml,
+        same as what `glab ci config compile` does. This replaces the old recursive local-only
+        parsing, since the project now includes configs from other projects as well.
+
+        :param root_yml_filepath: path to the local root yml file to compile
+        :return: merged (fully resolved) yml content as a string
+        """
+        sys.path.insert(0, os.path.join(IDF_PATH, 'tools', 'ci', 'python_packages'))
+        import gitlab_api
+
+        content = self._inline_local_includes(root_yml_filepath)
+
+        gitlab_inst = gitlab_api.Gitlab()
+        project_id = os.getenv('CI_PROJECT_ID') or gitlab_inst.get_project_id('esp-idf', namespace='espressif')
+        project = gitlab_inst.gitlab_inst.projects.get(project_id, lazy=True)
+
+        lint_result = project.ci_lint.create({'content': content})
+        if not lint_result.valid:
+            raise RuntimeError(f'Failed to compile {root_yml_filepath} via GitLab CI Lint API: {lint_result.errors}')
+
+        return lint_result.merged_yaml  # type: ignore
 
     @property
     def default(self) -> t.Dict[str, t.Any]:
@@ -167,80 +238,9 @@ class GitlabYmlConfig:
     def global_keys(self) -> t.List[str]:
         return ['default', 'include', 'workflow', 'variables', 'stages']
 
-    @cached_property
-    def anchors(self) -> t.Dict[str, t.Any]:
-        return {k: v for k, v in self.config.items() if k in self._anchor_keys}
-
     @cached_property
     def jobs(self) -> t.Dict[str, t.Any]:
-        return {k: v for k, v in self.config.items() if k in self._job_keys}
-
-    @cached_property
-    def templates(self) -> t.Dict[str, t.Any]:
-        return {k: v for k, v in self.config.items() if k in self._template_keys}
-
-    @cached_property
-    def used_templates(self) -> t.Set[str]:
-        return self._used_template_keys
-
-    def expand_extends(self) -> None:
-        """
-        expand the `extends` key in-place.
-        """
-        for k, v in self.config.items():
-            if k in self.global_keys:
-                continue
-
-            if isinstance(v, (str, list)):
-                self._anchor_keys.add(k)
-            elif k.startswith('.if-'):
-                self._anchor_keys.add(k)
-            elif k.startswith('.'):
-                self._template_keys.add(k)
-            elif isinstance(v, dict):
-                self._job_keys.add(k)
-            else:
-                raise ValueError(f'Unknown type for key {k} with value {v}')
-
-        # no need to expand anchor
-
-        # expand template first
-        for k in self._template_keys:
-            self._expand_extends(k)
-
-        # expand job
-        for k in self._job_keys:
-            self._expand_extends(k)
-
-    def _merge_dict(self, d1: t.Dict[str, t.Any], d2: t.Dict[str, t.Any]) -> t.Any:
-        for k, v in d2.items():
-            if k in d1:
-                if isinstance(v, dict) and isinstance(d1[k], dict):
-                    d1[k] = self._merge_dict(d1[k], v)
-                else:
-                    d1[k] = v
-            else:
-                d1[k] = v
-
-        return d1
-
-    def _expand_extends(self, name: str) -> t.Dict[str, t.Any]:
-        extends = to_list(self.config[name].pop('extends', None))
-        if not extends:
-            return self.config[name]  # type: ignore
-
-        original_d = self.config[name].copy()
-        d = {}
-        while extends:
-            self._used_template_keys.update(extends)  # for tracking
-
-            for i in extends:
-                d.update(self._expand_extends(i))
-
-            extends = to_list(self.config[name].pop('extends', None))
-
-        self.config[name] = self._merge_dict(d, original_d)
-        return self.config[name]  # type: ignore
+        return {k: v for k, v in self.config.items() if not k.startswith('.') and k not in self.global_keys}
 
 
 def get_all_manifest_files() -> t.List[str]:

tools/ci/idf_pytest/plugin.py

@@ -9,7 +9,6 @@ import pytest
 import yaml
 from _pytest.config import Config
 from _pytest.python import Function
-from _pytest.python import Metafunc
 from _pytest.runner import CallInfo
 from dynamic_pipelines.constants import KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH
 from idf_ci import IdfPytestPlugin
@@ -115,55 +114,6 @@ class IdfLocalPlugin:
 
         return item.callspec.params.get(key, default) or default
 
-    @staticmethod
-    def _has_parametrized_arg(metafunc: Metafunc, arg_name: str) -> bool:
-        for marker in metafunc.definition.iter_markers(name='parametrize'):
-            if not marker.args:
-                continue
-
-            argnames = marker.args[0]
-            if isinstance(argnames, str):
-                names = [name.strip() for name in argnames.split(',')]
-            else:
-                names = list(argnames)
-
-            if arg_name in names:
-                return True
-
-        for callspec in getattr(metafunc, '_calls', []):
-            if arg_name in callspec.params:
-                return True
-
-        return False
-
-    @staticmethod
-    def _is_linux_target_run(config: Config) -> bool:
-        target = config.getoption('target')
-        if not target:
-            return False
-
-        if isinstance(target, str):
-            targets = [_t.strip() for _t in target.split(',')]
-        else:
-            targets = [str(_t).strip() for _t in target]
-
-        return 'linux' in targets
-
-    @pytest.hookimpl(trylast=True)
-    def pytest_generate_tests(self, metafunc: Metafunc) -> None:
-        if 'embedded_services' not in metafunc.fixturenames:
-            return
-
-        if self._has_parametrized_arg(metafunc, 'embedded_services'):
-            return
-
-        if metafunc.definition.get_closest_marker('qemu') is not None:
-            metafunc.parametrize('embedded_services', ['idf,qemu'], indirect=True)
-            return
-
-        if self._is_linux_target_run(metafunc.config):
-            metafunc.parametrize('embedded_services', ['idf'], indirect=True)
-
     @pytest.hookimpl(wrapper=True)
     def pytest_collection_modifyitems(self, config: Config, items: t.List[Function]) -> t.Generator[None, None, None]:
         yield  # throw it back to idf-ci

tools/ci/python_packages/gitlab_api.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 import argparse
 import logging
@@ -52,9 +52,11 @@ def retry(func: TR) -> TR:
                     raise e  # get out of the loop
                 else:
                     logging.warning(
-                        'Network failure in {}, retrying ({})'.format(getattr(func, '__name__', '(unknown callable)'),
-                                                                      retried))
-                    time.sleep(2 ** retried)  # wait a bit more after each retry
+                        'Network failure in {}, retrying ({})'.format(
+                            getattr(func, '__name__', '(unknown callable)'), retried
+                        )
+                    )
+                    time.sleep(2**retried)  # wait a bit more after each retry
                     continue
             else:
                 break
@@ -67,7 +69,6 @@ class Gitlab(object):
     JOB_NAME_PATTERN = re.compile(r'(\w+)(\s+(\d+)/(\d+))?')
 
     DOWNLOAD_ERROR_MAX_RETRIES = 3
-    DEFAULT_BUILD_CHILD_PIPELINE_NAME = 'Build Child Pipeline'
 
     def __init__(self, project_id: Union[int, str, None] = None):
         config_data_from_env = os.getenv('PYTHON_GITLAB_CONFIG')
@@ -117,7 +118,7 @@ class Gitlab(object):
         :param namespace: namespace to match when we have multiple project with same name
         :return: project ID
         """
-        projects = self.gitlab_inst.projects.list(search=name)
+        projects = self.gitlab_inst.projects.list(search=name, get_all=True)
         res = []
         for project in projects:
             if namespace is None:
@@ -152,7 +153,9 @@ class Gitlab(object):
             archive_file.extractall(destination)
 
     @retry
-    def download_artifact(self, job_id: int, artifact_path: List[str], destination: Optional[str] = None) -> List[bytes]:
+    def download_artifact(
+        self, job_id: int, artifact_path: List[str], destination: Optional[str] = None
+    ) -> List[bytes]:
         """
         download specific path of job artifacts and extract to destination.
 
@@ -208,8 +211,9 @@ class Gitlab(object):
         return job_id_list
 
     @retry
-    def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None,
-                         cache_dir: Optional[str] = None) -> str:
+    def download_archive(
+        self, ref: str, destination: str, project_id: Optional[int] = None, cache_dir: Optional[str] = None
+    ) -> str:
         """
         Download archive of certain commit of a repository and extract to destination path
 
@@ -235,8 +239,11 @@ class Gitlab(object):
                     except gitlab.GitlabGetError as e:
                         logging.error('Failed to archive from project {}'.format(project_id))
                         raise e
-                logging.info('Downloaded archive size: {:.03f}MB'.format(
-                    float(os.path.getsize(local_archive_file)) / (1024 * 1024)))
+                logging.info(
+                    'Downloaded archive size: {:.03f}MB'.format(
+                        float(os.path.getsize(local_archive_file)) / (1024 * 1024)
+                    )
+                )
 
             return self.decompress_archive(local_archive_file, destination)
 
@@ -248,7 +255,9 @@ class Gitlab(object):
                 logging.error('Failed to archive from project {}'.format(project_id))
                 raise e
 
-        logging.info('Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(temp_file.name)) / (1024 * 1024)))
+        logging.info(
+            'Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(temp_file.name)) / (1024 * 1024))
+        )
 
         return self.decompress_archive(temp_file.name, destination)
 
@@ -282,36 +291,28 @@ class Gitlab(object):
 
     def get_downstream_pipeline_ids(self, main_pipeline_id: int) -> List[int]:
         """
-        Retrieve the IDs of all downstream child pipelines for a given main pipeline.
+        Retrieve the IDs of all downstream child pipelines for a given main pipeline,
+        recursing through arbitrarily nested child pipelines.
 
         :param main_pipeline_id: The ID of the main pipeline to start the search.
-        :return: A list of IDs of all downstream child pipelines.
+        :return: A list of IDs of all downstream child pipelines (all levels).
         """
-        bridge_pipeline_ids = []
-        child_pipeline_ids = []
+        child_pipeline_ids: List[int] = []
 
-        main_pipeline_bridges = self.project.pipelines.get(main_pipeline_id).bridges.list()
-        for bridge in main_pipeline_bridges:
+        pipeline_bridges = self.project.pipelines.get(main_pipeline_id).bridges.list()
+        for bridge in pipeline_bridges:
             downstream_pipeline = bridge.attributes.get('downstream_pipeline')
             if not downstream_pipeline:
                 continue
-            bridge_pipeline_ids.append(downstream_pipeline['id'])
-
-        for bridge_pipeline_id in bridge_pipeline_ids:
-            child_pipeline_ids.append(bridge_pipeline_id)
-            bridge_pipeline = self.project.pipelines.get(bridge_pipeline_id)
-
-            if not bridge_pipeline.name == self.DEFAULT_BUILD_CHILD_PIPELINE_NAME:
+            downstream_pipeline_id = downstream_pipeline.get('id')
+            if downstream_pipeline_id is None:
                 continue
 
-            child_bridges = bridge_pipeline.bridges.list()
-            for child_bridge in child_bridges:
-                downstream_child_pipeline = child_bridge.attributes.get('downstream_pipeline')
-                if not downstream_child_pipeline:
-                    continue
-                child_pipeline_ids.append(downstream_child_pipeline.get('id'))
+            child_pipeline_ids.append(downstream_pipeline_id)
+            # recurse to collect further nested (grandchild+) pipelines
+            child_pipeline_ids.extend(self.get_downstream_pipeline_ids(downstream_pipeline_id))
 
-        return [pid for pid in child_pipeline_ids if pid is not None]
+        return child_pipeline_ids
 
     def retry_failed_jobs(self, pipeline_id: int, retry_allowed_failures: bool = False) -> List[int]:
         """

tools/ci/utils.sh

@@ -164,38 +164,3 @@ function join_by {
     printf %s "$f" "${@/#/$d}"
   fi
 }
-
-function is_based_on_commits() {
-  # This function would accept space-separated args as multiple commits.
-  # The return value would be 0 if current HEAD is based on any of the specified commits.
-  #
-  # In our CI, we use environment variable $REQUIRED_ANCESTOR_COMMITS to declare the ancestor commits.
-  # Please remember to set one commit for each release branch.
-
-  commits=$*
-  if [[ -z $commits ]]; then
-    info "Not specifying commits that branches should be based on, skipping check..."
-    return 0
-  fi
-
-  commits_str="$(join_by " or " $commits)" # no doublequotes here, passing array
-
-  info "Checking if current branch is based on $commits_str..."
-  for i in $commits; do
-    if git merge-base --is-ancestor "$i" HEAD >/dev/null 2>&1; then
-      info "Current branch is based on $i"
-      return 0
-    else
-      info "Current branch is not based on $i"
-    fi
-  done
-
-  error "The base commit of your branch is too old."
-  error "The branch should be more recent than either of the following commits:"
-  error "  $commits_str"
-  error "To fix the issue:"
-  error " - If your merge request is 'Draft', or has conflicts with the target branch, rebase it to the latest master or release branch"
-  error " - Otherwise, simply run a new pipeline."
-
-  return 1
-}