ci: apply common-scripts CI refactor

.gitlab/ci/README.md

@@ -245,76 +245,15 @@ We're using the latest version of [idf-build-apps][idf-build-apps]. Please refer
 In ESP-IDF CI, there's a few more special rules are additionally supported to disable the check app dependencies feature:
 
 - Add MR labels `BUILD_AND_TEST_ALL_APPS`
+- Pipeline variable `IDF_CI_SELECT_ALL_PYTEST_CASES=1`
 - Run in protected branches
 
 ## Upload/Download Artifacts to Internal Minio Server
 
-### Users Without Access to Minio
+Please refer to the documentation [here](https://docs.espressif.com/projects/idf-ci/en/latest/guides/cli.html)
 
-If you don't have access to the internal Minio server, you can still download the artifacts from the shared link in the job log.
+## Revision Marker Naming Rules
 
-The log will look like this:
+When specifying chip revisions for testing markers, configurations or filenames, strictly follow the format `esp32<series>_rev<number>` (or similarly `<series>_rev<number>` when brief).
 
-```shell
-Pipeline ID    : 587355
-Job name       : build_clang_test_apps_esp32
-Job ID         : 40272275
-Created archive file: 40272275.zip, uploading as 587355/build_dir_without_map_and_elf_files/build_clang_test_apps_esp32/40272275.zip
-Please download the archive file includes build_dir_without_map_and_elf_files from [INTERNAL_URL]
-```
-
-### Users With Access to Minio
-
-#### Env Vars for Minio
-
-Minio takes these env vars to connect to the server:
-
-- `IDF_S3_SERVER`
-- `IDF_S3_ACCESS_KEY`
-- `IDF_S3_SECRET_KEY`
-- `IDF_S3_BUCKET`
-
-#### Artifacts Types and File Patterns
-
-The artifacts types and corresponding file patterns are defined in tools/ci/artifacts_handler.py, inside `ArtifactType` and `TYPE_PATTERNS_DICT`.
-
-#### Upload
-
-```shell
-python tools/ci/artifacts_handler.py upload
-```
-
- will upload the files that match the file patterns to minio object storage with name:
-
-`<pipeline_id>/<artifact_type>/<job_name>/<job_id>.zip`
-
-For example, job 39043328 will upload these four files:
-
-- `575500/map_and_elf_files/build_pytest_examples_esp32/39043328.zip`
-- `575500/build_dir_without_map_and_elf_files/build_pytest_examples_esp32/39043328.zip`
-- `575500/logs/build_pytest_examples_esp32/39043328.zip`
-- `575500/size_reports/build_pytest_examples_esp32/39043328.zip`
-
-#### Download
-
-You may run
-
-```shell
-python tools/ci/artifacts_handler.py download --pipeline_id <pipeline_id>
-```
-
-to download all files of the pipeline, or
-
-```shell
-python tools/ci/artifacts_handler.py download --pipeline_id <pipeline_id> --job_name <job_name_or_pattern>
-```
-
-to download all files with the specified job name or pattern, or
-
-```shell
-python tools/ci/artifacts_handler.py download --pipeline_id <pipeline_id> --job_name <job_name_or_pattern> --type <artifact_type> <artifact_type> ...
-```
-
-to download all files with the specified job name or pattern and artifact type(s).
-
-You may check all detailed documentation with `python tools/ci/artifacts_handler.py download -h`
+> e.g., `esp32_rev3`, `esp32c2_rev2`, `esp32c3_rev1`, `esp32p4_rev1`, `esp32c5_rev1`

.gitlab/ci/build.yml

@@ -110,61 +110,6 @@ build_clang_test_apps_riscv:
 ####################
 # Dynamic Pipeline #
 ####################
-generate_build_child_pipeline:
-  extends:
-    - .build_template
-  tags: [fast_run, shiny]
-  dependencies: # set dependencies to null to avoid missing artifacts issue
-  needs:
-    - pipeline_variables
-    - job: baseline_manifest_sha
-      optional: true
-  artifacts:
-    paths:
-      - build_child_pipeline.yml
-      - test_related_apps.txt
-      - non_test_related_apps.txt
-    expire_in: 1 week
-    when: always
-  script:
-    - run_cmd idf-ci --debug gitlab build-child-pipeline
-      -p components
-      -p examples
-      -p tools/test_apps
-      --modified-files $MR_MODIFIED_FILES
-
-build_child_pipeline:
-  stage: build
-  needs:
-    - job: fast_template_app
-      optional: true
-      artifacts: false
-    - pipeline_variables
-    - generate_build_child_pipeline
-  variables:
-    MR_MODIFIED_COMPONENTS: $MR_MODIFIED_COMPONENTS
-    MR_MODIFIED_FILES: $MR_MODIFIED_FILES
-    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
-  # https://gitlab.com/gitlab-org/gitlab/-/issues/214340
-  inherit:
-    variables: false
-  trigger:
-    include:
-      - artifact: build_child_pipeline.yml
-        job: generate_build_child_pipeline
-    strategy: depend
-
-generate_prebuild_report:
-  extends:
-    - .build_template
-  tags: [fast_run, shiny]
-  needs:
-    - pipeline_variables
-  artifacts:
-    paths:
-      - prebuild_report.html
-    expire_in: 1 week
-    when: always
-  script:
-    - run_cmd idf-ci build collect --format html -o prebuild_report.html
-    - echo "Report generated at https://${CI_PAGES_HOSTNAME}:${CI_SERVER_PORT}/-/esp-idf/-/jobs/${CI_JOB_ID}/artifacts/prebuild_report.html"
+# !!!
+# configurations moved to ci/actions/common/idf/
+# !!!

.gitlab/ci/common.yml

@@ -14,6 +14,15 @@ stages:
   - deploy
   - post_deploy
 
+include:
+  - project: "ci/actions/common"
+    ref: "v2"
+    file: "templates/idf/common-scripts.yml"
+    inputs:
+      ci_python_constraint_file: "espidf.constraints.v5.5.txt"
+      macos_image: "registry.hub.docker.com/espressif/macos-idf:sequoia-v5.5"
+      windows_image: "windows_11_idf_v5_5"
+
 variables:
 # System environment
 
@@ -58,30 +67,29 @@ variables:
   ESP_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v5.5:1"
   ESP_IDF_DOC_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v5.5-docs:1"
   TARGET_TEST_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v5.5-target-test:1"
-  SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:5"
 
   # Set this variable to the branch of idf-constraints repo in order to test a custom Python constraint file. The
   # branch name must be without the remote part ("origin/"). Keep the variable empty in order to use the constraint
   # file from https://dl.espressif.com/dl/esp-idf.
-  CI_PYTHON_CONSTRAINT_BRANCH: ""
+  # CI_PYTHON_CONSTRAINT_BRANCH could be defined as component input `ci_python_constraint_branch` in common-scripts above
 
   # Update the filename for a specific ESP-IDF release. It is used only with CI_PYTHON_CONSTRAINT_BRANCH.
-  CI_PYTHON_CONSTRAINT_FILE: "espidf.constraints.v5.5.txt"
+  # CI_PYTHON_CONSTRAINT_FILE is defined as component input `ci_python_constraint_file` in common-scripts above
 
   # Set this variable to repository name of a Python tool you wish to install and test in the context of ESP-IDF CI.
   # Keep the variable empty when not used.
-  CI_PYTHON_TOOL_REPO: ""
+  # CI_PYTHON_TOOL_REPO could be defined as component input `ci_python_tool_repo` in common-scripts above
 
   # Set this variable to the branch of a Python tool repo specified in CI_PYTHON_TOOL_REPO. The
   # branch name must be without the remote part ("origin/"). Keep the variable empty when not used.
   # This is used only if CI_PYTHON_TOOL_REPO is not empty.
-  CI_PYTHON_TOOL_BRANCH: ""
+  # CI_PYTHON_TOOL_BRANCH could be defined as component input `ci_python_tool_branch` in common-scripts above
 
   # Set this variable to Clang toolchain distro URL to be used.
   # NOTE: We have separate toolchains for Xtensa and RISCV, therefore jobs for one arch will fail.
   # This is OK as far as we use CI_CLANG_DISTRO_URL for pre-release tests purposes only.
   # Keep the variable empty when not used.
-  CI_CLANG_DISTRO_URL: ""
+  # CI_CLANG_DISTRO_URL could be defined as component input `ci_clang_distro_url` in common-scripts above
 
   # Set this variable to specify the file name for the known failure cases.
   KNOWN_FAILURE_CASES_FILE_NAME: "5.5.txt"
@@ -103,227 +111,19 @@ variables:
 ################################################
 # `before_script` and `after_script` Templates #
 ################################################
-.common_before_scripts: &common-before_scripts |
-  source tools/ci/utils.sh
-
-  is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
-
-  if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
-    export IDF_MIRROR_PREFIX_MAP=
-  fi
-
-  if echo "$CI_MERGE_REQUEST_LABELS" | egrep "(^|,)include_nightly_run(,|$)"; then
-    export INCLUDE_NIGHTLY_RUN="1"
-    export NIGHTLY_RUN="1"
-  fi
-
-  # configure cmake related flags
-  source tools/ci/configure_ci_environment.sh
-
-  if [[ "$CI_CCACHE_STATS" == 1 ]] && command -v ccache >/dev/null 2>&1 && [[ -n "$CCACHE_STATSLOG" ]]; then
-    mkdir -p "$(dirname "$CCACHE_STATSLOG")"
-    rm -f "$CCACHE_STATSLOG"
-  fi
-
-  # add extra python packages
-  export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
-
-.setup_tools_and_idf_python_venv: &setup_tools_and_idf_python_venv |
-  # must use after setup_tools_except_target_test
-  # otherwise the export.sh won't work properly
-
-  # download constraint file for dev
-  if [[ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]]; then
-    wget -O /tmp/constraint.txt --header="Authorization:Bearer ${ESPCI_TOKEN}" "${GITLAB_HTTP_SERVER}/api/v4/projects/2581/repository/files/${CI_PYTHON_CONSTRAINT_FILE}/raw?ref=${CI_PYTHON_CONSTRAINT_BRANCH}"
-    mkdir -p ~/.espressif
-    mv /tmp/constraint.txt ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
-  fi
-
-  # Mirror
-  if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
-    export IDF_MIRROR_PREFIX_MAP=
-  fi
-  # Optimize pip install
-  if echo "${CI_RUNNER_TAGS}" | grep "shiny"; then
-    export PIP_INDEX_URL="${PIP_INDEX_URL_SHINY}"
-  fi
-  if [[ "$(uname -m)" == "x86_64" ]] || [[ "$(uname -m)" == "aarch64" ]]; then
-    export IDF_PIP_WHEELS_URL=""
-  fi
-
-  # install.sh
-  if [[ "${CI_JOB_STAGE}" != "target_test" ]]; then
-    section_start "running_install_sh" "Running install.sh"
-    if [[ "${CI_JOB_STAGE}" == "build_doc" || "${CI_JOB_GROUP_NAME}" == *docs* ]]; then
-      run_cmd bash install.sh --enable-ci --enable-docs
-    else
-      run_cmd bash install.sh --enable-ci
-    fi
-    section_end "running_install_sh"
-  else
-    section_start "install_python_env" "Install Python environment, skip required tools check"
-    run_cmd python tools/idf_tools.py install-python-env --features ci,test-specific
-    export IDF_SKIP_TOOLS_CHECK=1
-    section_end "install_python_env"
-  fi
-
-  section_start "source_export" "Source export.sh"
-  source ./export.sh
-  section_end "source_export"
-
-  # Eager upgrade of CI dependencies
-  # Done after sourcing export.sh so that we could easily invoke the right pip
-  section_start "upgrade_ci_dependencies" "Upgrading CI dependencies"
-  pip install --upgrade --upgrade-strategy=eager -r $IDF_PATH/tools/requirements/requirements.ci.txt -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
-  # we need the latest DEV release for esptool to be installed in every job without bumping the minimal requirement in
-  # the constraint file.
-  pip install --upgrade --pre esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
-  section_end "upgrade_ci_dependencies"
-
-  REEXPORT_NEEDED=0
-  if [[ ! -z "$INSTALL_EXTRA_TOOLS" ]]; then
-    section_start "installing_optional_tools" "Install optional tools ${INSTALL_EXTRA_TOOLS}"
-    run_cmd $IDF_PATH/tools/idf_tools.py --non-interactive install $INSTALL_EXTRA_TOOLS
-    section_end "installing_optional_tools"
-
-    REEXPORT_NEEDED=1
-  fi
-
-  # Install esp-clang if necessary (esp-clang is separately installed)
-  if [[ "$IDF_TOOLCHAIN" == "clang" && -z "$CI_CLANG_DISTRO_URL" ]]; then
-    $IDF_PATH/tools/idf_tools.py --non-interactive install esp-clang
-
-    REEXPORT_NEEDED=1
-  fi
-
-  if [[ $REEXPORT_NEEDED -eq 1 ]]; then
-    section_start "re_source_export" "Re-source export.sh"
-    source ./export.sh
-    section_end "re_source_export"
-  fi
-
-  # Custom clang toolchain
-  if [[ "$IDF_TOOLCHAIN" == "clang" && ! -z "$CI_CLANG_DISTRO_URL" ]]; then
-    echo "Using custom clang from ${CI_CLANG_DISTRO_URL}"
-    wget $CI_CLANG_DISTRO_URL
-    ARCH_NAME=$(basename $CI_CLANG_DISTRO_URL)
-    tar -x -f $ARCH_NAME
-    export PATH=$PWD/esp-clang/bin:$PATH
-  fi
-
-  # Custom OpenOCD
-  if [[ "$CI_JOB_STAGE" == "target_test" ]]; then
-    machine="$(uname -m)"
-    if [[ "$machine" == "armv7l" ]] ; then
-      OOCD_DISTRO_URL="$OOCD_DISTRO_URL_ARMHF"
-    elif [[ "$machine" == "aarch64" ]] ; then
-      OOCD_DISTRO_URL="$OOCD_DISTRO_URL_ARM64"
-    fi
-    if [[ ! -z "$OOCD_DISTRO_URL" ]]; then
-      echo "Using custom OpenOCD from ${OOCD_DISTRO_URL}"
-      wget $OOCD_DISTRO_URL
-      ARCH_NAME=$(basename $OOCD_DISTRO_URL)
-      tar -x -f $ARCH_NAME
-      export OPENOCD_SCRIPTS=$PWD/openocd-esp32/share/openocd/scripts
-      export PATH=$PWD/openocd-esp32/bin:$PATH
-    fi
-  fi
-
-  if [[ -n "$CI_PYTHON_TOOL_REPO" ]]; then
-    git clone --quiet --depth=1 -b ${CI_PYTHON_TOOL_BRANCH} https://gitlab-ci-token:${ESPCI_TOKEN}@${GITLAB_HTTPS_HOST}/espressif/${CI_PYTHON_TOOL_REPO}.git
-    pip install ./${CI_PYTHON_TOOL_REPO}
-    rm -rf ${CI_PYTHON_TOOL_REPO}
-  fi
-
-  info "setup tools and python venv done"
-
-.show_ccache_statistics: &show_ccache_statistics |
-  # Show ccache statistics if enabled globally
-  section_start "ccache_show_stats" "Show ccache statistics"
-  if [[ "$CI_CCACHE_STATS" == 1 ]] && command -v ccache >/dev/null 2>&1; then
-    if ccache --help 2>/dev/null | grep -q -- '--show-log-stats'; then
-      if [[ -n "$CCACHE_STATSLOG" && -f "$CCACHE_STATSLOG" ]]; then
-        ccache --show-log-stats -vv
-      else
-        echo "INFO: No per-job ccache statistics were recorded"
-      fi
-    else
-      ccache --show-stats -vv
-    fi
-  fi || true
-  section_end "ccache_show_stats"
-
-.upload_failed_job_log_artifacts: &upload_failed_job_log_artifacts |
-  if [ $CI_JOB_STATUS = "failed" ]; then
-    run_cmd idf-ci gitlab upload-artifacts --type log
-  fi
-
 .before_script:minimal:
   before_script:
-    - *common-before_scripts
+    - !reference [.common_before_scripts, minimal]
 
 .after_script:build:
   after_script:
-    - source tools/ci/utils.sh
-    - source tools/ci/configure_ci_environment.sh
-    - *show_ccache_statistics
+    - !reference [.common_after_scripts, ccache]
     - run_cmd idf-ci gitlab upload-artifacts --type junit
     - |
       if [ $CI_JOB_STATUS = "failed" ]; then
         run_cmd idf-ci gitlab upload-artifacts --type log
       fi
 
-.brew-macos-settings:
-  variables:
-    GIT_STRATEGY: fetch
-    IDF_CCACHE_ENABLE: "0"
-    CCACHE_DIR: "/var/tmp/cache/idf_ccache"
-  tags:
-    - macos-tart
-  image: macos-sequoia-idf-v5.5
-  after_script: []  # ccache now is disabled for macos brew runners
-  timeout: 30m
-
-.brew-windows-settings:
-  image: windows_11_idf_v6_0
-  tags: [windows-vm, brew]
-  variables:
-    GIT_STRATEGY: fetch
-    LC_ALL: C.UTF-8
-    # git clean at the final cleanup stage stuck on Windows VM
-    # a new VM for each job anyway, so we can skip cleaning the workspace
-    GIT_CLEAN_FLAGS: none
-#    IDF_CCACHE_ENABLE: "1"
-#    CCACHE_DIR: "C:/cache/idf_ccache"
-  before_script:
-    - if ($env:IDF_DONT_USE_MIRRORS) { $env:IDF_MIRROR_PREFIX_MAP = '' }
-    - ./install.ps1 --enable-ci
-    - ./export.ps1
-    - $env:PYTHONPATH = "$env:PYTHONPATH;$env:IDF_PATH\tools;$env:IDF_PATH\tools\esp_app_trace;$env:IDF_PATH\components\partition_table;$env:IDF_PATH\tools\ci\python_packages"
-    - python "${SUBMODULE_FETCH_TOOL}" -s "${SUBMODULES_TO_FETCH}"
-    # Sync VM clock. net start/stop exit 2 when the service is already in that state; do not fail CI.
-    - |
-      function Invoke-NetService($Action, $Name) {
-        $output = net $Action $Name 2>&1
-        if ($LASTEXITCODE -eq 2) {
-          Write-Host "net $Action $Name already in requested state: $output"
-          $global:LASTEXITCODE = 0
-          return
-        }
-        if ($LASTEXITCODE -ne 0) {
-          Write-Host $output
-          exit $LASTEXITCODE
-        }
-      }
-      Invoke-NetService start w32time
-      w32tm /config /manualpeerlist:"time.google.com,0x9 pool.ntp.org,0x9" /syncfromflags:manual /reliable:yes /update
-      if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
-      Invoke-NetService stop w32time
-      Invoke-NetService start w32time
-      w32tm /resync
-      if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
-  after_script: []
-
 #############
 # `default` #
 #############
@@ -338,8 +138,7 @@ default:
         - .cache/submodule_archives
       policy: pull
   before_script:
-    - *common-before_scripts
-    - *setup_tools_and_idf_python_venv
+    - !reference [.common_before_scripts, with_tools]
     - add_gitlab_ssh_keys
     - fetch_submodules
     - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}

.gitlab/ci/default-build-test-rules.yml

@@ -1,21 +0,0 @@
-# this file support two keywords:
-# - extra_default_build_targets:
-#     besides of the SUPPORTED_TARGETS in IDF,
-#     enable build for the specified targets by default as well.
-# !!! DEPRECATED: use `additional_build_targets` in .idf_build_apps.toml instead
-#
-# - bypass_check_test_targets:
-#     suppress the check_build_test_rules check-test-script warnings for the specified targets
-#
-# This file should ONLY be used during bringup. Should be reset to empty after the bringup process
-
-bypass_check_test_targets:
-  - esp32h21
-  - esp32h4
-  - esp32c5
-
-#
-# These lines would
-# - enable the README.md check for esp32c6. Don't forget to add the build jobs in .gitlab/ci/build.yml
-# - disable the test script check with the manifest file.
-#

.gitlab/ci/host-test.yml

@@ -401,7 +401,7 @@ test_pytest_linux:
 test_pytest_macos:
   extends:
     - .host_test_template
-    - .brew-macos-settings
+    - .macos-settings
   artifacts:
     paths:
       - XUNIT_RESULT.xml
@@ -476,6 +476,8 @@ test_pytest_macos:
     - pytest
       -k cmake
       --cleanup-idf-copy
+      --parallel-count ${CI_NODE_TOTAL:-1}
+      --parallel-index ${CI_NODE_INDEX:-1}
       --work-dir ${CI_PROJECT_DIR}/test_build_system
       --junitxml ${CI_PROJECT_DIR}/XUNIT_RESULT.xml
       --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
@@ -487,7 +489,7 @@ pytest_build_system:
 pytest_build_system_macos:
   extends:
     - .test_build_system_template
-    - .brew-macos-settings
+    - .macos-settings
     - .rules:build:macos
   parallel: 3
 
@@ -497,7 +499,7 @@ pytest_build_system_minimal_cmake:
 pytest_build_system_macos_minimal_cmake:
   extends:
     - .test_build_system_minimal_cmake_template
-    - .brew-macos-settings
+    - .macos-settings
     - .rules:build:macos
   variables:
     INSTALL_EXTRA_TOOLS: ninja cmake@3.16.3

.gitlab/ci/pre_check.yml

@@ -1,9 +1,3 @@
-.pre_check_template:
-  stage: pre_check
-  image: $ESP_ENV_IMAGE
-  tags: [build, shiny]
-  dependencies: # set dependencies to null to avoid missing artifacts issue
-
 check_version:
   # Don't run this for feature/bugfix branches, so that it is possible to modify
   # esp_idf_version.h in a branch before tagging the next version.
@@ -140,106 +134,11 @@ check_submodule_sync:
     - git submodule update --recursive
     - echo "IDF was cloned from ${PUBLIC_IDF_URL} completely"
 
-pipeline_variables:
+check_gitlab_yaml:
   extends:
     - .pre_check_template
-  tags: [fast_run, shiny]
-  variables:
-    GIT_STRATEGY: "fetch"
   script:
-    - |
-      # Store the diff output in a temporary file
-      TEMP_FILE=$(mktemp)
-      # merged results pipelines, by default
-      if [[ -n $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ]]; then
-        git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA
-        git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
-
-        git diff --name-only $CI_MERGE_REQUEST_TARGET_BRANCH_SHA...$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA > "$TEMP_FILE"
-        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
-        git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
-      # merge request pipelines, when the mr got conflicts
-      elif [[ -n $CI_MERGE_REQUEST_DIFF_BASE_SHA ]]; then
-        git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
-        git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
-        git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_COMMIT_SHA > "$TEMP_FILE"
-        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
-      # other pipelines, like the protected branches pipelines
-      elif [[ "$CI_COMMIT_BEFORE_SHA" != "0000000000000000000000000000000000000000" ]]; then
-        git fetch origin $CI_COMMIT_BEFORE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
-        git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
-        git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA > "$TEMP_FILE"
-        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
-      else
-        # pipeline source could be web, scheduler, etc.
-        git fetch origin $CI_COMMIT_SHA --depth=2 ${GIT_FETCH_EXTRA_FLAGS}
-        git diff --name-only $CI_COMMIT_SHA~1 $CI_COMMIT_SHA > "$TEMP_FILE"
-        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
-      fi
-    # MODIFIED_FILES is a list of files that changed, could be used everywhere
-    - MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | xargs)
-    - echo "MODIFIED_FILES=$MODIFIED_FILES" >> pipeline.env
-    # MR_MODIFIED_FILES and MR_MODIFIED_COMPONENTS are semicolon separated lists that is used in MR only
-    # for non MR pipeline, these are empty lists
-    - |
-      if [ -z "$CI_MERGE_REQUEST_IID" ]; then
-        echo "MR_MODIFIED_FILES=\"\"" >> pipeline.env
-        echo "MR_MODIFIED_COMPONENTS=\"\"" >> pipeline.env
-      else
-        MR_MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | tr '\n' ';')
-        echo "MR_MODIFIED_FILES=\"$MR_MODIFIED_FILES\"" >> pipeline.env
-
-        MR_MODIFIED_COMPONENTS=$(run_cmd python tools/ci/ci_get_mr_info.py components --modified-files $MODIFIED_FILES | tr '\n' ';')
-        echo "MR_MODIFIED_COMPONENTS=\"$MR_MODIFIED_COMPONENTS\"" >> pipeline.env
-      fi
-    - |
-      if [ -n "$PIPELINE_COMMIT_SHA" ]; then
-        echo "PIPELINE_COMMIT_SHA=$PIPELINE_COMMIT_SHA" >> pipeline.env
-      fi
-    - echo "OOCD_DISTRO_URL_ARMHF=$OOCD_DISTRO_URL_ARMHF" >> pipeline.env
-    - echo "OOCD_DISTRO_URL_ARM64=$OOCD_DISTRO_URL_ARM64" >> pipeline.env
-    - run_cmd idf-ci gitlab pipeline-variables >> pipeline.env
-    - cat pipeline.env
-    - run_cmd idf-ci gitlab upload-artifacts --type env
-  artifacts:
-    reports:
-      dotenv: pipeline.env
-    paths:
-      - pipeline.env
-    expire_in: 1 week
-    when: always
-
-baseline_manifest_sha:
-  extends:
-    - .pre_check_template
-    - .rules:dev-push
-  tags: [fast_run, shiny]
-  script:
-    - |
-      # merged results pipelines, by default
-      # diff between target-branch-head and merged-result-head
-      if [ -n "$CI_MERGE_REQUEST_TARGET_BRANCH_SHA" ]; then
-        git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA --depth=1
-        git checkout FETCH_HEAD
-        idf-build-apps dump-manifest-sha \
-          --manifest-files $(find . -name ".build-test-rules.yml" | xargs) \
-          --output .manifest_sha
-      # merge request pipelines, when the mr got conflicts
-      # diff between diff-base-sha and merge-request-head
-      elif [ -n "$CI_MERGE_REQUEST_DIFF_BASE_SHA" ]; then
-        git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1
-        git checkout FETCH_HEAD
-        idf-build-apps dump-manifest-sha \
-          --manifest-files $(find . -name ".build-test-rules.yml" | xargs) \
-          --output .manifest_sha
-      # other pipelines, like the protected branches pipelines
-      # not triggered in this job
-      fi
-  artifacts:
-    paths:
-      - .manifest_sha
-    expire_in: 1 week
-    when: always
+    - python tools/ci/gitlab_yaml_linter.py
 
 retry_failed_jobs:
   extends:

.gitlab/ci/rules.yml

@@ -14,13 +14,6 @@
   - "components/**/CMakeLists.txt"
 
 
-.patterns-python-files: &patterns-python-files
-  - ".gitlab/ci/static-code-analysis.yml"
-  - "**/*.py"
-
-.patterns-sonarqube-files: &patterns-sonarqube-files
-  - "tools/ci/sonar_exclude_list.txt"
-
 .patterns-build_components: &patterns-build_components
   # components files except "test*/" "host*/" folders
   # ?? to include folders less than 4 characters
@@ -176,16 +169,10 @@
 .if-protected-check: &if-protected-check
   if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) || $CI_COMMIT_TAG =~ /^qa-test/'
 
-.if-protected-deploy: &if-protected-deploy
-  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "api")'
-
 .if-protected-branch-push: &if-protected-branch-push
   # rules:changes always evaluates to true for new branch pipelines or when there is no Git push event
   if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/) && $CI_PIPELINE_SOURCE == "push"'
 
-.if-qa-test-tag: &if-qa-test-tag
-  if: '$CI_COMMIT_TAG =~ /^qa-test/'
-
 .if-dev-push: &if-dev-push
   if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")'
 

.gitlab/ci/static-code-analysis.yml

@@ -30,84 +30,3 @@ gcc_static_analyzer:
     - export EXTRA_CFLAGS=
     - export EXTRA_CXXFLAGS=
     - idf-build-apps build -p ${ANALYZING_APP} --disable-targets esp32h4 esp32h21 --no-check-warnings
-#
-## build stage
-## Sonarqube related jobs put here for this reason:
-## Here we have two jobs. code_quality_check and code_quality_report.
-##
-## code_quality_check will analyze the code changes between your MR and
-## code repo stored in sonarqube server. The analysis result is only shown in
-## the comments under this MR and won't be transferred to the server.
-##
-## code_quality_report will analyze and transfer both of the newly added code
-## and the analysis result to the server.
-##
-## Put in the front to ensure that the newly merged code can be stored in
-## sonarqube server ASAP, in order to avoid reporting unrelated code issues
-#.sonar_scan_template:
-#  stage: build
-#  extends: .pre_check_template
-#  # full clone since this image does not support fetch --shallow-since-cutoff
-#  # shiny runners are used for full clone
-#  tags: [build, shiny]
-#  image: $SONARQUBE_SCANNER_IMAGE
-#  before_script:
-#    - source tools/ci/utils.sh
-#    - export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH"
-#    - fetch_submodules
-#    # Exclude the submodules, all paths ends with /**
-#    - submodules=$(get_all_submodules)
-#    # get all exclude paths specified in tools/ci/sonar_exclude_list.txt | ignore lines start with # | xargs | replace all <space> to <comma>
-#    - custom_excludes=$(cat $CI_PROJECT_DIR/tools/ci/sonar_exclude_list.txt | grep -v '^#' | xargs | sed -e 's/ /,/g')
-#    # Exclude the report dir as well
-#    - export EXCLUSIONS="$custom_excludes,$submodules"
-#    - export SONAR_SCANNER_OPTS="-Xmx2048m"
-#  variables:
-#    GIT_DEPTH: 0
-#    REPORT_PATTERN: clang_tidy_reports/**/*.txt
-#  artifacts:
-#    paths:
-#      - $REPORT_PATTERN
-#    expire_in: 1 week
-#    when: always
-#  dependencies:  # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here.
-#    - clang_tidy_check
-#
-#code_quality_check:
-#  extends:
-#    - .sonar_scan_template
-#    - .rules:patterns:static-code-analysis-preview
-#  allow_failure: true  # it's using exit code to indicate the code analysis result,
-#                       # we don't want to block ci when critical issues founded
-#  script:
-#    - export CI_MERGE_REQUEST_COMMITS=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py commits --src-branch ${CI_COMMIT_REF_NAME} | tr '\n' ',')
-#    # test if this branch have merge request, if not, exit 0
-#    - test -n "$CI_MERGE_REQUEST_IID" || exit 0
-#    - test -n "$CI_MERGE_REQUEST_COMMITS" || exit 0
-#    - sonar-scanner
-#      -Dsonar.analysis.mode=preview
-#      -Dsonar.branch.name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
-#      -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN
-#      -Dsonar.exclusions=$EXCLUSIONS
-#      -Dsonar.gitlab.ci_merge_request_iid=$CI_MERGE_REQUEST_IID
-#      -Dsonar.gitlab.commit_sha=$CI_MERGE_REQUEST_COMMITS
-#      -Dsonar.gitlab.merge_request_discussion=true
-#      -Dsonar.gitlab.ref_name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
-#      -Dsonar.host.url=$SONAR_HOST_URL
-#      -Dsonar.login=$SONAR_LOGIN
-#
-#code_quality_report:
-#  extends:
-#    - .sonar_scan_template
-#    - .rules:protected:check
-#  allow_failure: true  # it's using exit code to indicate the code analysis result,
-#                       # we don't want to block ci when critical issues founded
-#  script:
-#    - sonar-scanner
-#      -Dsonar.branch.name=$CI_COMMIT_REF_NAME
-#      -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN
-#      -Dsonar.exclusions=$EXCLUSIONS
-#      -Dsonar.gitlab.commit_sha=$PIPELINE_COMMIT_SHA
-#      -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME
-#      -Dsonar.host.url=$SONAR_HOST_URL
-#      -Dsonar.login=$SONAR_LOGIN

.gitlab/ci/test-win.yml

@@ -1,6 +1,6 @@
 .host_test_win_template:
   extends:
-    - .brew-windows-settings
+    - .windows-settings
     - .rules:test:host_test
   stage: host_test
   dependencies: # set dependencies to null to avoid missing artifacts issue
@@ -73,7 +73,7 @@ pytest_build_system_win:
   extends:
     - .test_build_system_template_win
     - .rules:labels:windows_pytest_build_system
-  parallel: 2
+  parallel: 6
 
 pytest_build_system_win_minimal_cmake:
   extends:
@@ -82,7 +82,7 @@ pytest_build_system_win_minimal_cmake:
   variables:
     MINIMAL_CMAKE_VERSION: "3.16.3"
   script:
-    - python ${IDF_PATH}\tools\idf_tools.py install cmake@${MINIMAL_CMAKE_VERSION}
+    - python tools\idf_tools.py install cmake@${MINIMAL_CMAKE_VERSION}
     - $Env:PATH = "$Env:USERPROFILE\.espressif\tools\cmake\${MINIMAL_CMAKE_VERSION}\bin;$Env:PATH"
     - |
       $actualVersion = (& cmake --version).Split()[2]
@@ -90,6 +90,6 @@ pytest_build_system_win_minimal_cmake:
         Write-Error "ERROR: Wrong CMake version! Detected: $actualVersion, but expected: $Env:MINIMAL_CMAKE_VERSION"
         exit 1
       }
-    - cd ${IDF_PATH}\tools\test_build_system
+    - cd tools\test_build_system
     - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
-    - pytest -k cmake --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
+    - pytest -k cmake --junitxml ${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}