ci: apply common-scripts CI refactor

.gitlab/ci/build.yml

@@ -110,61 +110,6 @@ build_clang_test_apps_riscv:
 ####################
 # Dynamic Pipeline #
 ####################
-generate_build_child_pipeline:
-  extends:
-    - .build_template
-  tags: [fast_run, shiny]
-  dependencies: # set dependencies to null to avoid missing artifacts issue
-  needs:
-    - pipeline_variables
-    - job: baseline_manifest_sha
-      optional: true
-  artifacts:
-    paths:
-      - build_child_pipeline.yml
-      - test_related_apps.txt
-      - non_test_related_apps.txt
-    expire_in: 1 week
-    when: always
-  script:
-    - run_cmd idf-ci --debug gitlab build-child-pipeline
-      -p components
-      -p examples
-      -p tools/test_apps
-      --modified-files $MR_MODIFIED_FILES
-
-build_child_pipeline:
-  stage: build
-  needs:
-    - job: fast_template_app
-      optional: true
-      artifacts: false
-    - pipeline_variables
-    - generate_build_child_pipeline
-  variables:
-    MR_MODIFIED_COMPONENTS: $MR_MODIFIED_COMPONENTS
-    MR_MODIFIED_FILES: $MR_MODIFIED_FILES
-    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
-  # https://gitlab.com/gitlab-org/gitlab/-/issues/214340
-  inherit:
-    variables: false
-  trigger:
-    include:
-      - artifact: build_child_pipeline.yml
-        job: generate_build_child_pipeline
-    strategy: depend
-
-generate_prebuild_report:
-  extends:
-    - .build_template
-  tags: [fast_run, shiny]
-  needs:
-    - pipeline_variables
-  artifacts:
-    paths:
-      - prebuild_report.html
-    expire_in: 1 week
-    when: always
-  script:
-    - run_cmd idf-ci build collect --format html -o prebuild_report.html
-    - echo "Report generated at https://${CI_PAGES_HOSTNAME}:${CI_SERVER_PORT}/-/esp-idf/-/jobs/${CI_JOB_ID}/artifacts/prebuild_report.html"
+# !!!
+# configurations moved to ci/actions/common/idf/
+# !!!

.gitlab/ci/common.yml

@@ -14,6 +14,15 @@ stages:
   - deploy
   - post_deploy
 
+include:
+  - project: "ci/actions/common"
+    ref: "v2"
+    file: "templates/idf/common-scripts.yml"
+    inputs:
+      ci_python_constraint_file: "espidf.constraints.v6.0.txt"
+      macos_image: "registry.hub.docker.com/espressif/macos-idf:sequoia-v6.0"
+      windows_image: "windows_11_idf_v6_0"
+
 variables:
 # System environment
 
@@ -58,30 +67,29 @@ variables:
   ESP_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v6.0:1"
   ESP_IDF_DOC_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v6.0-docs:1"
   TARGET_TEST_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v6.0-target-test:1"
-  SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:5"
 
   # Set this variable to the branch of idf-constraints repo in order to test a custom Python constraint file. The
   # branch name must be without the remote part ("origin/"). Keep the variable empty in order to use the constraint
   # file from https://dl.espressif.com/dl/esp-idf.
-  CI_PYTHON_CONSTRAINT_BRANCH: ""
+  # CI_PYTHON_CONSTRAINT_BRANCH could be defined as component input `ci_python_constraint_branch` in common-scripts above
 
   # Update the filename for a specific ESP-IDF release. It is used only with CI_PYTHON_CONSTRAINT_BRANCH.
-  CI_PYTHON_CONSTRAINT_FILE: "espidf.constraints.v6.0.txt"
+  # CI_PYTHON_CONSTRAINT_FILE is defined as component input `ci_python_constraint_file` in common-scripts above
 
   # Set this variable to repository name of a Python tool you wish to install and test in the context of ESP-IDF CI.
   # Keep the variable empty when not used.
-  CI_PYTHON_TOOL_REPO: ""
+  # CI_PYTHON_TOOL_REPO could be defined as component input `ci_python_tool_repo` in common-scripts above
 
   # Set this variable to the branch of a Python tool repo specified in CI_PYTHON_TOOL_REPO. The
   # branch name must be without the remote part ("origin/"). Keep the variable empty when not used.
   # This is used only if CI_PYTHON_TOOL_REPO is not empty.
-  CI_PYTHON_TOOL_BRANCH: ""
+  # CI_PYTHON_TOOL_BRANCH could be defined as component input `ci_python_tool_branch` in common-scripts above
 
   # Set this variable to Clang toolchain distro URL to be used.
   # NOTE: We have separate toolchains for Xtensa and RISCV, therefore jobs for one arch will fail.
   # This is OK as far as we use CI_CLANG_DISTRO_URL for pre-release tests purposes only.
   # Keep the variable empty when not used.
-  CI_CLANG_DISTRO_URL: ""
+  # CI_CLANG_DISTRO_URL could be defined as component input `ci_clang_distro_url` in common-scripts above
 
   # Set this variable to specify the file name for the known failure cases.
   KNOWN_FAILURE_CASES_FILE_NAME: "6.0.txt"
@@ -103,232 +111,19 @@ variables:
 ################################################
 # `before_script` and `after_script` Templates #
 ################################################
-.common_before_scripts: &common-before_scripts |
-  source tools/ci/utils.sh
-
-  is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
-
-  if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
-    export IDF_MIRROR_PREFIX_MAP=
-  fi
-
-  if echo "$CI_MERGE_REQUEST_LABELS" | egrep "(^|,)include_nightly_run(,|$)"; then
-    export INCLUDE_NIGHTLY_RUN="1"
-    export NIGHTLY_RUN="1"
-  fi
-
-  # configure cmake related flags
-  source tools/ci/configure_ci_environment.sh
-
-  if [[ "$CI_CCACHE_STATS" == 1 ]] && command -v ccache >/dev/null 2>&1 && [[ -n "$CCACHE_STATSLOG" ]]; then
-    mkdir -p "$(dirname "$CCACHE_STATSLOG")"
-    rm -f "$CCACHE_STATSLOG"
-  fi
-
-  # add extra python packages
-  export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
-
-.setup_tools_and_idf_python_venv: &setup_tools_and_idf_python_venv |
-  # must use after setup_tools_except_target_test
-  # otherwise the export.sh won't work properly
-
-  # download constraint file for dev
-  if [[ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]]; then
-        wget -O /tmp/constraint.txt --header="Authorization:Bearer ${ESPCI_TOKEN}" "${GITLAB_HTTP_SERVER}/api/v4/projects/2581/repository/files/${CI_PYTHON_CONSTRAINT_FILE}/raw?ref=${CI_PYTHON_CONSTRAINT_BRANCH}"
-    mkdir -p ~/.espressif
-    mv /tmp/constraint.txt ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
-  fi
-
-  # Mirror
-  if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
-    export IDF_MIRROR_PREFIX_MAP=
-  fi
-  # Optimize pip install
-  if echo "${CI_RUNNER_TAGS}" | grep "shiny"; then
-    export PIP_INDEX_URL="${PIP_INDEX_URL_SHINY}"
-  fi
-  if [[ "$(uname -m)" == "x86_64" ]] || [[ "$(uname -m)" == "aarch64" ]]; then
-    export IDF_PIP_WHEELS_URL=""
-  fi
-
-  # install.sh
-  if [[ "${CI_JOB_STAGE}" != "target_test" ]]; then
-    section_start "running_install_sh" "Running install.sh"
-    if [[ "${CI_JOB_STAGE}" == "build_doc" || "${CI_JOB_GROUP_NAME}" == *docs* ]]; then
-      run_cmd bash install.sh --enable-ci --enable-docs
-    else
-      run_cmd bash install.sh --enable-ci
-    fi
-    section_end "running_install_sh"
-  else
-    section_start "install_python_env" "Install Python environment, skip required tools check"
-    run_cmd python tools/idf_tools.py install-python-env --features ci,test-specific
-    export IDF_SKIP_TOOLS_CHECK=1
-    section_end "install_python_env"
-  fi
-
-  section_start "source_export" "Source export.sh"
-  source ./export.sh
-  section_end "source_export"
-
-  # Eager upgrade of CI dependencies
-  # Done after sourcing export.sh so that we could easily invoke the right pip
-  section_start "upgrade_ci_dependencies" "Upgrading CI dependencies"
-  pip install --upgrade --upgrade-strategy=eager -r $IDF_PATH/tools/requirements/requirements.ci.txt -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
-  # we need the latest DEV release for esptool to be installed in every job without bumping the minimal requirement in
-  # the constraint file. CI_ESPTOOL_EXTRA_INDEX_URL (set in GitLab project variables) optionally points to an internal
-  # package registry so that internal dev builds are preferred when the variable is set; it is a no-op when unset.
-  # Use an explicit if/else: the `${VAR:+--flag "$VAR"}` form collapses to a single token under zsh (macOS runners).
-  if [[ -n "$CI_ESPTOOL_EXTRA_INDEX_URL" ]]; then
-    pip install --upgrade --pre --extra-index-url "$CI_ESPTOOL_EXTRA_INDEX_URL" esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
-  else
-    pip install --upgrade --pre esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
-  fi
-  section_end "upgrade_ci_dependencies"
-
-  REEXPORT_NEEDED=0
-  if [[ ! -z "$INSTALL_EXTRA_TOOLS" ]]; then
-    section_start "installing_optional_tools" "Install optional tools ${INSTALL_EXTRA_TOOLS}"
-    run_cmd $IDF_PATH/tools/idf_tools.py --non-interactive install $INSTALL_EXTRA_TOOLS
-    section_end "installing_optional_tools"
-
-    REEXPORT_NEEDED=1
-  fi
-
-  # Install esp-clang if necessary (esp-clang is separately installed)
-  if [[ "$IDF_TOOLCHAIN" == "clang" && -z "$CI_CLANG_DISTRO_URL" ]]; then
-    $IDF_PATH/tools/idf_tools.py --non-interactive install esp-clang
-
-    REEXPORT_NEEDED=1
-  fi
-
-  if [[ $REEXPORT_NEEDED -eq 1 ]]; then
-    section_start "re_source_export" "Re-source export.sh"
-    source ./export.sh
-    section_end "re_source_export"
-  fi
-
-  # Custom clang toolchain
-  if [[ "$IDF_TOOLCHAIN" == "clang" && ! -z "$CI_CLANG_DISTRO_URL" ]]; then
-    echo "Using custom clang from ${CI_CLANG_DISTRO_URL}"
-    wget $CI_CLANG_DISTRO_URL
-    ARCH_NAME=$(basename $CI_CLANG_DISTRO_URL)
-    tar -x -f $ARCH_NAME
-    export PATH=$PWD/esp-clang/bin:$PATH
-  fi
-
-  # Custom OpenOCD
-  if [[ "$CI_JOB_STAGE" == "target_test" ]]; then
-    machine="$(uname -m)"
-    if [[ "$machine" == "armv7l" ]] ; then
-      OOCD_DISTRO_URL="$OOCD_DISTRO_URL_ARMHF"
-    elif [[ "$machine" == "aarch64" ]] ; then
-      OOCD_DISTRO_URL="$OOCD_DISTRO_URL_ARM64"
-    fi
-    if [[ ! -z "$OOCD_DISTRO_URL" ]]; then
-      echo "Using custom OpenOCD from ${OOCD_DISTRO_URL}"
-      wget $OOCD_DISTRO_URL
-      ARCH_NAME=$(basename $OOCD_DISTRO_URL)
-      tar -x -f $ARCH_NAME
-      export OPENOCD_SCRIPTS=$PWD/openocd-esp32/share/openocd/scripts
-      export PATH=$PWD/openocd-esp32/bin:$PATH
-    fi
-  fi
-
-  if [[ -n "$CI_PYTHON_TOOL_REPO" ]]; then
-    git clone --quiet --depth=1 -b ${CI_PYTHON_TOOL_BRANCH} https://gitlab-ci-token:${ESPCI_TOKEN}@${GITLAB_HTTPS_HOST}/espressif/${CI_PYTHON_TOOL_REPO}.git
-    pip install ./${CI_PYTHON_TOOL_REPO}
-    rm -rf ${CI_PYTHON_TOOL_REPO}
-  fi
-
-  info "setup tools and python venv done"
-
-.show_ccache_statistics: &show_ccache_statistics |
-  # Show ccache statistics if enabled globally
-  section_start "ccache_show_stats" "Show ccache statistics"
-  if [[ "$CI_CCACHE_STATS" == 1 ]] && command -v ccache >/dev/null 2>&1; then
-    if ccache --help 2>/dev/null | grep -q -- '--show-log-stats'; then
-      if [[ -n "$CCACHE_STATSLOG" && -f "$CCACHE_STATSLOG" ]]; then
-        ccache --show-log-stats -vv
-      else
-        echo "INFO: No per-job ccache statistics were recorded"
-      fi
-    else
-      ccache --show-stats -vv
-    fi
-  fi || true
-  section_end "ccache_show_stats"
-
-.upload_failed_job_log_artifacts: &upload_failed_job_log_artifacts |
-  if [ $CI_JOB_STATUS = "failed" ]; then
-    run_cmd idf-ci gitlab upload-artifacts --type log
-  fi
-
 .before_script:minimal:
   before_script:
-    - *common-before_scripts
+    - !reference [.common_before_scripts, minimal]
 
 .after_script:build:
   after_script:
-    - source tools/ci/utils.sh
-    - source tools/ci/configure_ci_environment.sh
-    - *show_ccache_statistics
+    - !reference [.common_after_scripts, ccache]
     - run_cmd idf-ci gitlab upload-artifacts --type junit
     - |
       if [ $CI_JOB_STATUS = "failed" ]; then
         run_cmd idf-ci gitlab upload-artifacts --type log
       fi
 
-.brew-macos-settings:
-  variables:
-    GIT_STRATEGY: fetch
-    IDF_CCACHE_ENABLE: "0"
-    CCACHE_DIR: "/var/tmp/cache/idf_ccache"
-  tags:
-    - macos-tart
-  image: macos-sequoia-idf-v6.0
-  after_script: []  # ccache now is disabled for macos brew runners
-  timeout: 30m
-
-.brew-windows-settings:
-  image: windows_11_idf_v6_0
-  tags: [windows-vm, brew]
-  variables:
-    GIT_STRATEGY: fetch
-    LC_ALL: C.UTF-8
-    # git clean at the final cleanup stage stuck on Windows VM
-    # a new VM for each job anyway, so we can skip cleaning the workspace
-    GIT_CLEAN_FLAGS: none
-#    IDF_CCACHE_ENABLE: "1"
-#    CCACHE_DIR: "C:/cache/idf_ccache"
-  before_script:
-    - if ($env:IDF_DONT_USE_MIRRORS) { $env:IDF_MIRROR_PREFIX_MAP = '' }
-    - ./install.ps1 --enable-ci
-    - ./export.ps1
-    - $env:PYTHONPATH = "$env:PYTHONPATH;$env:IDF_PATH\tools;$env:IDF_PATH\tools\esp_app_trace;$env:IDF_PATH\components\partition_table;$env:IDF_PATH\tools\ci\python_packages"
-    - python "${SUBMODULE_FETCH_TOOL}" -s "${SUBMODULES_TO_FETCH}"
-    # Sync VM clock. net start/stop exit 2 when the service is already in that state; do not fail CI.
-    - |
-      function Invoke-NetService($Action, $Name) {
-        $output = net $Action $Name 2>&1
-        if ($LASTEXITCODE -eq 2) {
-          Write-Host "net $Action $Name already in requested state: $output"
-          $global:LASTEXITCODE = 0
-          return
-        }
-        if ($LASTEXITCODE -ne 0) {
-          Write-Host $output
-          exit $LASTEXITCODE
-        }
-      }
-      Invoke-NetService start w32time
-      w32tm /config /manualpeerlist:"time.google.com,0x9 pool.ntp.org,0x9" /syncfromflags:manual /reliable:yes /update
-      if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
-      Invoke-NetService stop w32time
-      Invoke-NetService start w32time
-      w32tm /resync
-      if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
-  after_script: []
 
 #############
 # `default` #
@@ -344,8 +139,7 @@ default:
         - .cache/submodule_archives
       policy: pull
   before_script:
-    - *common-before_scripts
-    - *setup_tools_and_idf_python_venv
+    - !reference [.common_before_scripts, with_tools]
     - add_gitlab_ssh_keys
     - fetch_submodules
     - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}

.gitlab/ci/default-build-test-rules.yml

@@ -1,21 +0,0 @@
-# this file support two keywords:
-# - extra_default_build_targets:
-#     besides of the SUPPORTED_TARGETS in IDF,
-#     enable build for the specified targets by default as well.
-# !!! DEPRECATED: use `additional_build_targets` in .idf_build_apps.toml instead
-#
-# - bypass_check_test_targets:
-#     suppress the check_build_test_rules check-test-script warnings for the specified targets
-#
-# This file should ONLY be used during bringup. Should be reset to empty after the bringup process
-
-bypass_check_test_targets:
-  - esp32h21
-  - esp32h4
-  - esp32c5
-
-#
-# These lines would
-# - enable the README.md check for esp32c6. Don't forget to add the build jobs in .gitlab/ci/build.yml
-# - disable the test script check with the manifest file.
-#

.gitlab/ci/host-test.yml

@@ -387,7 +387,7 @@ test_pytest_linux:
 test_pytest_macos:
   extends:
     - .host_test_template
-    - .brew-macos-settings
+    - .macos-settings
   artifacts:
     paths:
       - XUNIT_RESULT.xml
@@ -493,7 +493,7 @@ pytest_buildv2_system:
 pytest_build_system_macos:
   extends:
     - .test_build_system_template
-    - .brew-macos-settings
+    - .macos-settings
     - .rules:build:macos
   parallel: 3
 
@@ -503,7 +503,7 @@ pytest_build_system_minimal_cmake:
 pytest_build_system_macos_minimal_cmake:
   extends:
     - .test_build_system_minimal_cmake_template
-    - .brew-macos-settings
+    - .macos-settings
     - .rules:build:macos
   variables:
     INSTALL_EXTRA_TOOLS: ninja cmake@3.22.1

.gitlab/ci/pre_check.yml

@@ -1,9 +1,3 @@
-.pre_check_template:
-  stage: pre_check
-  image: $ESP_ENV_IMAGE
-  tags: [build, shiny]
-  dependencies: # set dependencies to null to avoid missing artifacts issue
-
 check_version:
   # Don't run this for feature/bugfix branches, so that it is possible to modify
   # esp_idf_version.h in a branch before tagging the next version.
@@ -140,106 +134,11 @@ check_submodule_sync:
     - git submodule update --recursive
     - echo "IDF was cloned from ${PUBLIC_IDF_URL} completely"
 
-pipeline_variables:
+check_gitlab_yaml:
   extends:
     - .pre_check_template
-  tags: [fast_run, shiny]
-  variables:
-    GIT_STRATEGY: "fetch"
   script:
-    - |
-      # Store the diff output in a temporary file
-      TEMP_FILE=$(mktemp)
-      # merged results pipelines, by default
-      if [[ -n $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ]]; then
-        git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA
-        git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
-
-        git diff --name-only $CI_MERGE_REQUEST_TARGET_BRANCH_SHA...$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA > "$TEMP_FILE"
-        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
-        git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
-      # merge request pipelines, when the mr got conflicts
-      elif [[ -n $CI_MERGE_REQUEST_DIFF_BASE_SHA ]]; then
-        git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
-        git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
-        git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_COMMIT_SHA > "$TEMP_FILE"
-        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
-      # other pipelines, like the protected branches pipelines
-      elif [[ "$CI_COMMIT_BEFORE_SHA" != "0000000000000000000000000000000000000000" ]]; then
-        git fetch origin $CI_COMMIT_BEFORE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
-        git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
-        git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA > "$TEMP_FILE"
-        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
-      else
-        # pipeline source could be web, scheduler, etc.
-        git fetch origin $CI_COMMIT_SHA --depth=2 ${GIT_FETCH_EXTRA_FLAGS}
-        git diff --name-only $CI_COMMIT_SHA~1 $CI_COMMIT_SHA > "$TEMP_FILE"
-        GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
-      fi
-    # MODIFIED_FILES is a list of files that changed, could be used everywhere
-    - MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | xargs)
-    - echo "MODIFIED_FILES=$MODIFIED_FILES" >> pipeline.env
-    # MR_MODIFIED_FILES and MR_MODIFIED_COMPONENTS are semicolon separated lists that is used in MR only
-    # for non MR pipeline, these are empty lists
-    - |
-      if [ -z "$CI_MERGE_REQUEST_IID" ]; then
-        echo "MR_MODIFIED_FILES=\"\"" >> pipeline.env
-        echo "MR_MODIFIED_COMPONENTS=\"\"" >> pipeline.env
-      else
-        MR_MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | tr '\n' ';')
-        echo "MR_MODIFIED_FILES=\"$MR_MODIFIED_FILES\"" >> pipeline.env
-
-        MR_MODIFIED_COMPONENTS=$(run_cmd python tools/ci/ci_get_mr_info.py components --modified-files $MODIFIED_FILES | tr '\n' ';')
-        echo "MR_MODIFIED_COMPONENTS=\"$MR_MODIFIED_COMPONENTS\"" >> pipeline.env
-      fi
-    - |
-      if [ -n "$PIPELINE_COMMIT_SHA" ]; then
-        echo "PIPELINE_COMMIT_SHA=$PIPELINE_COMMIT_SHA" >> pipeline.env
-      fi
-    - echo "OOCD_DISTRO_URL_ARMHF=$OOCD_DISTRO_URL_ARMHF" >> pipeline.env
-    - echo "OOCD_DISTRO_URL_ARM64=$OOCD_DISTRO_URL_ARM64" >> pipeline.env
-    - run_cmd idf-ci gitlab pipeline-variables >> pipeline.env
-    - cat pipeline.env
-    - run_cmd idf-ci gitlab upload-artifacts --type env
-  artifacts:
-    reports:
-      dotenv: pipeline.env
-    paths:
-      - pipeline.env
-    expire_in: 1 week
-    when: always
-
-baseline_manifest_sha:
-  extends:
-    - .pre_check_template
-    - .rules:dev-push
-  tags: [fast_run, shiny]
-  script:
-    - |
-      # merged results pipelines, by default
-      # diff between target-branch-head and merged-result-head
-      if [ -n "$CI_MERGE_REQUEST_TARGET_BRANCH_SHA" ]; then
-        git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA --depth=1
-        git checkout FETCH_HEAD
-        idf-build-apps dump-manifest-sha \
-          --manifest-files $(find . -name ".build-test-rules.yml" | xargs) \
-          --output .manifest_sha
-      # merge request pipelines, when the mr got conflicts
-      # diff between diff-base-sha and merge-request-head
-      elif [ -n "$CI_MERGE_REQUEST_DIFF_BASE_SHA" ]; then
-        git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1
-        git checkout FETCH_HEAD
-        idf-build-apps dump-manifest-sha \
-          --manifest-files $(find . -name ".build-test-rules.yml" | xargs) \
-          --output .manifest_sha
-      # other pipelines, like the protected branches pipelines
-      # not triggered in this job
-      fi
-  artifacts:
-    paths:
-      - .manifest_sha
-    expire_in: 1 week
-    when: always
+    - python tools/ci/gitlab_yaml_linter.py
 
 retry_failed_jobs:
   extends:

.gitlab/ci/rules.yml

@@ -13,13 +13,6 @@
   - "components/**/CMakeLists.txt"
 
 
-.patterns-python-files: &patterns-python-files
-  - ".gitlab/ci/static-code-analysis.yml"
-  - "**/*.py"
-
-.patterns-sonarqube-files: &patterns-sonarqube-files
-  - "tools/ci/sonar_exclude_list.txt"
-
 .patterns-build_components: &patterns-build_components
   # components files except "test*/" "host*/" folders
   # ?? to include folders less than 4 characters
@@ -169,16 +162,10 @@
 .if-protected-check: &if-protected-check
   if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) || $CI_COMMIT_TAG =~ /^qa-test/'
 
-.if-protected-deploy: &if-protected-deploy
-  if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "api")'
-
 .if-protected-branch-push: &if-protected-branch-push
   # rules:changes always evaluates to true for new branch pipelines or when there is no Git push event
   if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/) && $CI_PIPELINE_SOURCE == "push"'
 
-.if-qa-test-tag: &if-qa-test-tag
-  if: '$CI_COMMIT_TAG =~ /^qa-test/'
-
 .if-dev-push: &if-dev-push
   if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")'
 

.gitlab/ci/static-code-analysis.yml

@@ -27,85 +27,3 @@ gcc_static_analyzer:
     - echo "CONFIG_COMPILER_STATIC_ANALYZER=y" >> ${ANALYZING_APP}/sdkconfig.defaults
     - sed -i 's/.*MINIMAL_BUILD.*//g' ${ANALYZING_APP}/CMakeLists.txt
     - idf-build-apps build -p ${ANALYZING_APP}
-
-#
-## build stage
-## Sonarqube related jobs put here for this reason:
-## Here we have two jobs. code_quality_check and code_quality_report.
-##
-## code_quality_check will analyze the code changes between your MR and
-## code repo stored in sonarqube server. The analysis result is only shown in
-## the comments under this MR and won't be transferred to the server.
-##
-## code_quality_report will analyze and transfer both of the newly added code
-## and the analysis result to the server.
-##
-## Put in the front to ensure that the newly merged code can be stored in
-## sonarqube server ASAP, in order to avoid reporting unrelated code issues
-#.sonar_scan_template:
-#  stage: build
-#  extends: .pre_check_template
-#  # full clone since this image does not support fetch --shallow-since-cutoff
-#  # shiny runners are used for full clone
-#  tags: [build, shiny]
-#  image: $SONARQUBE_SCANNER_IMAGE
-#  before_script:
-#    - source tools/ci/utils.sh
-#    - export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH"
-#    - fetch_submodules
-#    # Exclude the submodules, all paths ends with /**
-#    - submodules=$(get_all_submodules)
-#    # get all exclude paths specified in tools/ci/sonar_exclude_list.txt | ignore lines start with # | xargs | replace all <space> to <comma>
-#    - custom_excludes=$(cat $CI_PROJECT_DIR/tools/ci/sonar_exclude_list.txt | grep -v '^#' | xargs | sed -e 's/ /,/g')
-#    # Exclude the report dir as well
-#    - export EXCLUSIONS="$custom_excludes,$submodules"
-#    - export SONAR_SCANNER_OPTS="-Xmx2048m"
-#  variables:
-#    GIT_DEPTH: 0
-#    REPORT_PATTERN: clang_tidy_reports/**/*.txt
-#  artifacts:
-#    paths:
-#      - $REPORT_PATTERN
-#    expire_in: 1 week
-#    when: always
-#  dependencies:  # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here.
-#    - clang_tidy_check
-#
-#code_quality_check:
-#  extends:
-#    - .sonar_scan_template
-#    - .rules:patterns:static-code-analysis-preview
-#  allow_failure: true  # it's using exit code to indicate the code analysis result,
-#                       # we don't want to block ci when critical issues founded
-#  script:
-#    - export CI_MERGE_REQUEST_COMMITS=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py commits --src-branch ${CI_COMMIT_REF_NAME} | tr '\n' ',')
-#    # test if this branch have merge request, if not, exit 0
-#    - test -n "$CI_MERGE_REQUEST_IID" || exit 0
-#    - test -n "$CI_MERGE_REQUEST_COMMITS" || exit 0
-#    - sonar-scanner
-#      -Dsonar.analysis.mode=preview
-#      -Dsonar.branch.name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
-#      -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN
-#      -Dsonar.exclusions=$EXCLUSIONS
-#      -Dsonar.gitlab.ci_merge_request_iid=$CI_MERGE_REQUEST_IID
-#      -Dsonar.gitlab.commit_sha=$CI_MERGE_REQUEST_COMMITS
-#      -Dsonar.gitlab.merge_request_discussion=true
-#      -Dsonar.gitlab.ref_name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
-#      -Dsonar.host.url=$SONAR_HOST_URL
-#      -Dsonar.login=$SONAR_LOGIN
-#
-#code_quality_report:
-#  extends:
-#    - .sonar_scan_template
-#    - .rules:protected:check
-#  allow_failure: true  # it's using exit code to indicate the code analysis result,
-#                       # we don't want to block ci when critical issues founded
-#  script:
-#    - sonar-scanner
-#      -Dsonar.branch.name=$CI_COMMIT_REF_NAME
-#      -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN
-#      -Dsonar.exclusions=$EXCLUSIONS
-#      -Dsonar.gitlab.commit_sha=$PIPELINE_COMMIT_SHA
-#      -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME
-#      -Dsonar.host.url=$SONAR_HOST_URL
-#      -Dsonar.login=$SONAR_LOGIN

.gitlab/ci/test-win.yml

@@ -1,6 +1,6 @@
 .host_test_win_template:
   extends:
-    - .brew-windows-settings
+    - .windows-settings
     - .rules:test:host_test
   stage: host_test
   dependencies: # set dependencies to null to avoid missing artifacts issue
@@ -50,7 +50,7 @@ test_tools_win:
   script:
     - cd tools\test_idf_py
     - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
-    - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
+    - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml ${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
 
 # Build tests
 .test_build_system_template_win:
@@ -67,42 +67,18 @@ test_tools_win:
   script:
     - cd tools\test_build_system
     - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
-    - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
+    - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml ${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
 
 pytest_build_system_win:
   extends:
     - .test_build_system_template_win
     - .rules:labels:windows_pytest_build_system
   parallel: 6
-  needs:
-    - job: manual_gate
-      optional: true
-  tags: [windows-build, brew]
-  artifacts:
-    paths:
-      - XUNIT_RESULT.xml
-      - test_build_system
-    expire_in: 2 days
-    reports:
-      junit: XUNIT_RESULT.xml
-    when: always
 
 pytest_build_system_win_minimal_cmake:
   extends:
     - .test_build_system_template_win
     - .rules:labels:windows_pytest_build_system
-  needs:
-    - job: manual_gate
-      optional: true
-  tags: [windows-build, brew]
-  artifacts:
-    paths:
-      - XUNIT_RESULT.xml
-      - test_build_system
-    expire_in: 2 days
-    reports:
-      junit: XUNIT_RESULT.xml
-    when: always
   variables:
     MINIMAL_CMAKE_VERSION: "3.22.1"
   script:
@@ -116,25 +92,13 @@ pytest_build_system_win_minimal_cmake:
       }
     - cd tools\test_build_system
     - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
-    - pytest -k cmake --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
+    - pytest -k cmake --junitxml ${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
 
 pytest_buildv2_system_win:
   extends:
     - .test_build_system_template_win
     - .rules:labels:buildv2
   parallel: 2
-  needs:
-    - job: manual_gate
-      optional: true
-  tags: [windows-build, brew]
-  artifacts:
-    paths:
-      - XUNIT_RESULT.xml
-      - test_build_system
-    expire_in: 2 days
-    reports:
-      junit: XUNIT_RESULT.xml
-    when: always
   script:
     - cd tools\test_build_system
     - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
@@ -142,5 +106,5 @@ pytest_buildv2_system_win:
       --buildv2
       --parallel-count ${CI_NODE_TOTAL}
       --parallel-index ${CI_NODE_INDEX}
-      --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml
+      --junitxml ${CI_PROJECT_DIR}\XUNIT_RESULT.xml
       --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}