Mirrors/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2026-09-22 13:01:16 +03:00
Code Issues Packages Projects Releases Wiki Activity

ci: apply common-scripts CI refactor

Browse Source
This commit is contained in:
Fu Hanxi
2026-07-22 22:39:19 +02:00
parent 0a86d7b8e7
commit 5fd30885b3
26 changed files with 236 additions and 1351 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
.gitlab-ci.yml
View File

@@ -2,11 +2,25 @@
include:
- project: "ci/actions/common"
ref: "v2"
file:
- "templates/stable/default-workflow.yml"
- "templates/idf/pre_check.yml"
- "templates/idf/build.yml"
- "templates/idf/deploy-github.yml"
- "templates/idf/deploy-docs.yml"
- "templates/idf/integration-test.yml"
- project: "ci/actions/common"
ref: "v2"
file: "templates/idf/sync-files.yml"
inputs:
job_name: "sync-pre-commit-config"
remote_repo_url: "https://__token__:${CI_SYNC_REPO_TOKEN}@${CI_SERVER_FQDN}/ci/images.git"
ref_name: "ci/sync_idf_pre_commit_config"
file_mappings:
- ".pre-commit-config.yaml:pre-commit/idf/.pre-commit-config.v6.0.yaml"
changes:
- ".pre-commit-config.yaml"
- project: "ci/actions/common"
file: "templates/idf/build-docs.yml"
inputs:
@@ -20,13 +34,14 @@ include:
DOCTGT: "esp32p4"
- ".gitlab/ci/danger.yml"
- ".gitlab/ci/common.yml"
- ".gitlab/ci/common.yml" # has to be placed after danger.yml, since danger.yml defined stages.
- ".gitlab/ci/rules.yml"
# by stages
- ".gitlab/ci/manual_gate.yml"
- ".gitlab/ci/upload_cache.yml"
- ".gitlab/ci/pre_check.yml"
- ".gitlab/ci/static-code-analysis.yml"
- ".gitlab/ci/pre_commit.yml"
- ".gitlab/ci/pre_check.yml"
- ".gitlab/ci/build.yml"
- ".gitlab/ci/host-test.yml"
- ".gitlab/ci/post_deploy.yml"

61
.gitlab/ci/build.yml
View File

@@ -110,61 +110,6 @@ build_clang_test_apps_riscv:
####################
# Dynamic Pipeline #
####################
generate_build_child_pipeline:
extends:
- .build_template
tags: [fast_run, shiny]
dependencies: # set dependencies to null to avoid missing artifacts issue
needs:
- pipeline_variables
- job: baseline_manifest_sha
optional: true
artifacts:
paths:
- build_child_pipeline.yml
- test_related_apps.txt
- non_test_related_apps.txt
expire_in: 1 week
when: always
script:
- run_cmd idf-ci --debug gitlab build-child-pipeline
-p components
-p examples
-p tools/test_apps
--modified-files $MR_MODIFIED_FILES
build_child_pipeline:
stage: build
needs:
- job: fast_template_app
optional: true
artifacts: false
- pipeline_variables
- generate_build_child_pipeline
variables:
MR_MODIFIED_COMPONENTS: $MR_MODIFIED_COMPONENTS
MR_MODIFIED_FILES: $MR_MODIFIED_FILES
PARENT_PIPELINE_ID: $CI_PIPELINE_ID
# https://gitlab.com/gitlab-org/gitlab/-/issues/214340
inherit:
variables: false
trigger:
include:
- artifact: build_child_pipeline.yml
job: generate_build_child_pipeline
strategy: depend
generate_prebuild_report:
extends:
- .build_template
tags: [fast_run, shiny]
needs:
- pipeline_variables
artifacts:
paths:
- prebuild_report.html
expire_in: 1 week
when: always
script:
- run_cmd idf-ci build collect --format html -o prebuild_report.html
- echo "Report generated at https://${CI_PAGES_HOSTNAME}:${CI_SERVER_PORT}/-/esp-idf/-/jobs/${CI_JOB_ID}/artifacts/prebuild_report.html"
# !!!
# configurations moved to ci/actions/common/idf/
# !!!

240
.gitlab/ci/common.yml
View File

@@ -14,6 +14,15 @@ stages:
- deploy
- post_deploy
include:
- project: "ci/actions/common"
ref: "v2"
file: "templates/idf/common-scripts.yml"
inputs:
ci_python_constraint_file: "espidf.constraints.v6.0.txt"
macos_image: "registry.hub.docker.com/espressif/macos-idf:sequoia-v6.0"
windows_image: "windows_11_idf_v6_0"
variables:
# System environment
@@ -58,30 +67,29 @@ variables:
ESP_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v6.0:1"
ESP_IDF_DOC_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v6.0-docs:1"
TARGET_TEST_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v6.0-target-test:1"
SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:5"
# Set this variable to the branch of idf-constraints repo in order to test a custom Python constraint file. The
# branch name must be without the remote part ("origin/"). Keep the variable empty in order to use the constraint
# file from https://dl.espressif.com/dl/esp-idf.
CI_PYTHON_CONSTRAINT_BRANCH: ""
# CI_PYTHON_CONSTRAINT_BRANCH could be defined as component input `ci_python_constraint_branch` in common-scripts above
# Update the filename for a specific ESP-IDF release. It is used only with CI_PYTHON_CONSTRAINT_BRANCH.
CI_PYTHON_CONSTRAINT_FILE: "espidf.constraints.v6.0.txt"
# CI_PYTHON_CONSTRAINT_FILE is defined as component input `ci_python_constraint_file` in common-scripts above
# Set this variable to repository name of a Python tool you wish to install and test in the context of ESP-IDF CI.
# Keep the variable empty when not used.
CI_PYTHON_TOOL_REPO: ""
# CI_PYTHON_TOOL_REPO could be defined as component input `ci_python_tool_repo` in common-scripts above
# Set this variable to the branch of a Python tool repo specified in CI_PYTHON_TOOL_REPO. The
# branch name must be without the remote part ("origin/"). Keep the variable empty when not used.
# This is used only if CI_PYTHON_TOOL_REPO is not empty.
CI_PYTHON_TOOL_BRANCH: ""
# CI_PYTHON_TOOL_BRANCH could be defined as component input `ci_python_tool_branch` in common-scripts above
# Set this variable to Clang toolchain distro URL to be used.
# NOTE: We have separate toolchains for Xtensa and RISCV, therefore jobs for one arch will fail.
# This is OK as far as we use CI_CLANG_DISTRO_URL for pre-release tests purposes only.
# Keep the variable empty when not used.
CI_CLANG_DISTRO_URL: ""
# CI_CLANG_DISTRO_URL could be defined as component input `ci_clang_distro_url` in common-scripts above
# Set this variable to specify the file name for the known failure cases.
KNOWN_FAILURE_CASES_FILE_NAME: "6.0.txt"
@@ -103,232 +111,19 @@ variables:
################################################
# `before_script` and `after_script` Templates #
################################################
.common_before_scripts: &common-before_scripts |
source tools/ci/utils.sh
is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
export IDF_MIRROR_PREFIX_MAP=
fi
if echo "$CI_MERGE_REQUEST_LABELS" | egrep "(^|,)include_nightly_run(,|$)"; then
export INCLUDE_NIGHTLY_RUN="1"
export NIGHTLY_RUN="1"
fi
# configure cmake related flags
source tools/ci/configure_ci_environment.sh
if [[ "$CI_CCACHE_STATS" == 1 ]] && command -v ccache >/dev/null 2>&1 && [[ -n "$CCACHE_STATSLOG" ]]; then
mkdir -p "$(dirname "$CCACHE_STATSLOG")"
rm -f "$CCACHE_STATSLOG"
fi
# add extra python packages
export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
.setup_tools_and_idf_python_venv: &setup_tools_and_idf_python_venv |
# must use after setup_tools_except_target_test
# otherwise the export.sh won't work properly
# download constraint file for dev
if [[ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]]; then
wget -O /tmp/constraint.txt --header="Authorization:Bearer ${ESPCI_TOKEN}" "${GITLAB_HTTP_SERVER}/api/v4/projects/2581/repository/files/${CI_PYTHON_CONSTRAINT_FILE}/raw?ref=${CI_PYTHON_CONSTRAINT_BRANCH}"
mkdir -p ~/.espressif
mv /tmp/constraint.txt ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
fi
# Mirror
if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
export IDF_MIRROR_PREFIX_MAP=
fi
# Optimize pip install
if echo "${CI_RUNNER_TAGS}" | grep "shiny"; then
export PIP_INDEX_URL="${PIP_INDEX_URL_SHINY}"
fi
if [[ "$(uname -m)" == "x86_64" ]] || [[ "$(uname -m)" == "aarch64" ]]; then
export IDF_PIP_WHEELS_URL=""
fi
# install.sh
if [[ "${CI_JOB_STAGE}" != "target_test" ]]; then
section_start "running_install_sh" "Running install.sh"
if [[ "${CI_JOB_STAGE}" == "build_doc" || "${CI_JOB_GROUP_NAME}" == *docs* ]]; then
run_cmd bash install.sh --enable-ci --enable-docs
else
run_cmd bash install.sh --enable-ci
fi
section_end "running_install_sh"
else
section_start "install_python_env" "Install Python environment, skip required tools check"
run_cmd python tools/idf_tools.py install-python-env --features ci,test-specific
export IDF_SKIP_TOOLS_CHECK=1
section_end "install_python_env"
fi
section_start "source_export" "Source export.sh"
source ./export.sh
section_end "source_export"
# Eager upgrade of CI dependencies
# Done after sourcing export.sh so that we could easily invoke the right pip
section_start "upgrade_ci_dependencies" "Upgrading CI dependencies"
pip install --upgrade --upgrade-strategy=eager -r $IDF_PATH/tools/requirements/requirements.ci.txt -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
# we need the latest DEV release for esptool to be installed in every job without bumping the minimal requirement in
# the constraint file. CI_ESPTOOL_EXTRA_INDEX_URL (set in GitLab project variables) optionally points to an internal
# package registry so that internal dev builds are preferred when the variable is set; it is a no-op when unset.
# Use an explicit if/else: the `${VAR:+--flag "$VAR"}` form collapses to a single token under zsh (macOS runners).
if [[ -n "$CI_ESPTOOL_EXTRA_INDEX_URL" ]]; then
pip install --upgrade --pre --extra-index-url "$CI_ESPTOOL_EXTRA_INDEX_URL" esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
else
pip install --upgrade --pre esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
fi
section_end "upgrade_ci_dependencies"
REEXPORT_NEEDED=0
if [[ ! -z "$INSTALL_EXTRA_TOOLS" ]]; then
section_start "installing_optional_tools" "Install optional tools ${INSTALL_EXTRA_TOOLS}"
run_cmd $IDF_PATH/tools/idf_tools.py --non-interactive install $INSTALL_EXTRA_TOOLS
section_end "installing_optional_tools"
REEXPORT_NEEDED=1
fi
# Install esp-clang if necessary (esp-clang is separately installed)
if [[ "$IDF_TOOLCHAIN" == "clang" && -z "$CI_CLANG_DISTRO_URL" ]]; then
$IDF_PATH/tools/idf_tools.py --non-interactive install esp-clang
REEXPORT_NEEDED=1
fi
if [[ $REEXPORT_NEEDED -eq 1 ]]; then
section_start "re_source_export" "Re-source export.sh"
source ./export.sh
section_end "re_source_export"
fi
# Custom clang toolchain
if [[ "$IDF_TOOLCHAIN" == "clang" && ! -z "$CI_CLANG_DISTRO_URL" ]]; then
echo "Using custom clang from ${CI_CLANG_DISTRO_URL}"
wget $CI_CLANG_DISTRO_URL
ARCH_NAME=$(basename $CI_CLANG_DISTRO_URL)
tar -x -f $ARCH_NAME
export PATH=$PWD/esp-clang/bin:$PATH
fi
# Custom OpenOCD
if [[ "$CI_JOB_STAGE" == "target_test" ]]; then
machine="$(uname -m)"
if [[ "$machine" == "armv7l" ]] ; then
OOCD_DISTRO_URL="$OOCD_DISTRO_URL_ARMHF"
elif [[ "$machine" == "aarch64" ]] ; then
OOCD_DISTRO_URL="$OOCD_DISTRO_URL_ARM64"
fi
if [[ ! -z "$OOCD_DISTRO_URL" ]]; then
echo "Using custom OpenOCD from ${OOCD_DISTRO_URL}"
wget $OOCD_DISTRO_URL
ARCH_NAME=$(basename $OOCD_DISTRO_URL)
tar -x -f $ARCH_NAME
export OPENOCD_SCRIPTS=$PWD/openocd-esp32/share/openocd/scripts
export PATH=$PWD/openocd-esp32/bin:$PATH
fi
fi
if [[ -n "$CI_PYTHON_TOOL_REPO" ]]; then
git clone --quiet --depth=1 -b ${CI_PYTHON_TOOL_BRANCH} https://gitlab-ci-token:${ESPCI_TOKEN}@${GITLAB_HTTPS_HOST}/espressif/${CI_PYTHON_TOOL_REPO}.git
pip install ./${CI_PYTHON_TOOL_REPO}
rm -rf ${CI_PYTHON_TOOL_REPO}
fi
info "setup tools and python venv done"
.show_ccache_statistics: &show_ccache_statistics |
# Show ccache statistics if enabled globally
section_start "ccache_show_stats" "Show ccache statistics"
if [[ "$CI_CCACHE_STATS" == 1 ]] && command -v ccache >/dev/null 2>&1; then
if ccache --help 2>/dev/null | grep -q -- '--show-log-stats'; then
if [[ -n "$CCACHE_STATSLOG" && -f "$CCACHE_STATSLOG" ]]; then
ccache --show-log-stats -vv
else
echo "INFO: No per-job ccache statistics were recorded"
fi
else
ccache --show-stats -vv
fi
fi || true
section_end "ccache_show_stats"
.upload_failed_job_log_artifacts: &upload_failed_job_log_artifacts |
if [ $CI_JOB_STATUS = "failed" ]; then
run_cmd idf-ci gitlab upload-artifacts --type log
fi
.before_script:minimal:
before_script:
- *common-before_scripts
- !reference [.common_before_scripts, minimal]
.after_script:build:
after_script:
- source tools/ci/utils.sh
- source tools/ci/configure_ci_environment.sh
- *show_ccache_statistics
- !reference [.common_after_scripts, ccache]
- run_cmd idf-ci gitlab upload-artifacts --type junit
- |
if [ $CI_JOB_STATUS = "failed" ]; then
run_cmd idf-ci gitlab upload-artifacts --type log
fi
.brew-macos-settings:
variables:
GIT_STRATEGY: fetch
IDF_CCACHE_ENABLE: "0"
CCACHE_DIR: "/var/tmp/cache/idf_ccache"
tags:
- macos-tart
image: macos-sequoia-idf-v6.0
after_script: [] # ccache now is disabled for macos brew runners
timeout: 30m
.brew-windows-settings:
image: windows_11_idf_v6_0
tags: [windows-vm, brew]
variables:
GIT_STRATEGY: fetch
LC_ALL: C.UTF-8
# git clean at the final cleanup stage stuck on Windows VM
# a new VM for each job anyway, so we can skip cleaning the workspace
GIT_CLEAN_FLAGS: none
# IDF_CCACHE_ENABLE: "1"
# CCACHE_DIR: "C:/cache/idf_ccache"
before_script:
- if ($env:IDF_DONT_USE_MIRRORS) { $env:IDF_MIRROR_PREFIX_MAP = '' }
- ./install.ps1 --enable-ci
- ./export.ps1
- $env:PYTHONPATH = "$env:PYTHONPATH;$env:IDF_PATH\tools;$env:IDF_PATH\tools\esp_app_trace;$env:IDF_PATH\components\partition_table;$env:IDF_PATH\tools\ci\python_packages"
- python "${SUBMODULE_FETCH_TOOL}" -s "${SUBMODULES_TO_FETCH}"
# Sync VM clock. net start/stop exit 2 when the service is already in that state; do not fail CI.
- |
function Invoke-NetService($Action, $Name) {
$output = net $Action $Name 2>&1
if ($LASTEXITCODE -eq 2) {
Write-Host "net $Action $Name already in requested state: $output"
$global:LASTEXITCODE = 0
return
}
if ($LASTEXITCODE -ne 0) {
Write-Host $output
exit $LASTEXITCODE
}
}
Invoke-NetService start w32time
w32tm /config /manualpeerlist:"time.google.com,0x9 pool.ntp.org,0x9" /syncfromflags:manual /reliable:yes /update
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
Invoke-NetService stop w32time
Invoke-NetService start w32time
w32tm /resync
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
after_script: []
#############
# `default` #
@@ -344,8 +139,7 @@ default:
- .cache/submodule_archives
policy: pull
before_script:
- *common-before_scripts
- *setup_tools_and_idf_python_venv
- !reference [.common_before_scripts, with_tools]
- add_gitlab_ssh_keys
- fetch_submodules
- export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}

21
.gitlab/ci/default-build-test-rules.yml
View File

@@ -1,21 +0,0 @@
# this file support two keywords:
# - extra_default_build_targets:
# besides of the SUPPORTED_TARGETS in IDF,
# enable build for the specified targets by default as well.
# !!! DEPRECATED: use `additional_build_targets` in .idf_build_apps.toml instead
#
# - bypass_check_test_targets:
# suppress the check_build_test_rules check-test-script warnings for the specified targets
#
# This file should ONLY be used during bringup. Should be reset to empty after the bringup process
bypass_check_test_targets:
- esp32h21
- esp32h4
- esp32c5
#
# These lines would
# - enable the README.md check for esp32c6. Don't forget to add the build jobs in .gitlab/ci/build.yml
# - disable the test script check with the manifest file.
#

6
.gitlab/ci/host-test.yml
View File

@@ -387,7 +387,7 @@ test_pytest_linux:
test_pytest_macos:
extends:
- .host_test_template
- .brew-macos-settings
- .macos-settings
artifacts:
paths:
- XUNIT_RESULT.xml
@@ -493,7 +493,7 @@ pytest_buildv2_system:
pytest_build_system_macos:
extends:
- .test_build_system_template
- .brew-macos-settings
- .macos-settings
- .rules:build:macos
parallel: 3
@@ -503,7 +503,7 @@ pytest_build_system_minimal_cmake:
pytest_build_system_macos_minimal_cmake:
extends:
- .test_build_system_minimal_cmake_template
- .brew-macos-settings
- .macos-settings
- .rules:build:macos
variables:
INSTALL_EXTRA_TOOLS: ninja cmake@3.22.1

105
.gitlab/ci/pre_check.yml
View File

@@ -1,9 +1,3 @@
.pre_check_template:
stage: pre_check
image: $ESP_ENV_IMAGE
tags: [build, shiny]
dependencies: # set dependencies to null to avoid missing artifacts issue
check_version:
# Don't run this for feature/bugfix branches, so that it is possible to modify
# esp_idf_version.h in a branch before tagging the next version.
@@ -140,106 +134,11 @@ check_submodule_sync:
- git submodule update --recursive
- echo "IDF was cloned from ${PUBLIC_IDF_URL} completely"
pipeline_variables:
check_gitlab_yaml:
extends:
- .pre_check_template
tags: [fast_run, shiny]
variables:
GIT_STRATEGY: "fetch"
script:
- |
# Store the diff output in a temporary file
TEMP_FILE=$(mktemp)
# merged results pipelines, by default
if [[ -n $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ]]; then
git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA
git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
git diff --name-only $CI_MERGE_REQUEST_TARGET_BRANCH_SHA...$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA > "$TEMP_FILE"
GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
# merge request pipelines, when the mr got conflicts
elif [[ -n $CI_MERGE_REQUEST_DIFF_BASE_SHA ]]; then
git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_COMMIT_SHA > "$TEMP_FILE"
GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
# other pipelines, like the protected branches pipelines
elif [[ "$CI_COMMIT_BEFORE_SHA" != "0000000000000000000000000000000000000000" ]]; then
git fetch origin $CI_COMMIT_BEFORE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA > "$TEMP_FILE"
GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
else
# pipeline source could be web, scheduler, etc.
git fetch origin $CI_COMMIT_SHA --depth=2 ${GIT_FETCH_EXTRA_FLAGS}
git diff --name-only $CI_COMMIT_SHA~1 $CI_COMMIT_SHA > "$TEMP_FILE"
GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
fi
# MODIFIED_FILES is a list of files that changed, could be used everywhere
- MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | xargs)
- echo "MODIFIED_FILES=$MODIFIED_FILES" >> pipeline.env
# MR_MODIFIED_FILES and MR_MODIFIED_COMPONENTS are semicolon separated lists that is used in MR only
# for non MR pipeline, these are empty lists
- |
if [ -z "$CI_MERGE_REQUEST_IID" ]; then
echo "MR_MODIFIED_FILES=\"\"" >> pipeline.env
echo "MR_MODIFIED_COMPONENTS=\"\"" >> pipeline.env
else
MR_MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | tr '\n' ';')
echo "MR_MODIFIED_FILES=\"$MR_MODIFIED_FILES\"" >> pipeline.env
MR_MODIFIED_COMPONENTS=$(run_cmd python tools/ci/ci_get_mr_info.py components --modified-files $MODIFIED_FILES | tr '\n' ';')
echo "MR_MODIFIED_COMPONENTS=\"$MR_MODIFIED_COMPONENTS\"" >> pipeline.env
fi
- |
if [ -n "$PIPELINE_COMMIT_SHA" ]; then
echo "PIPELINE_COMMIT_SHA=$PIPELINE_COMMIT_SHA" >> pipeline.env
fi
- echo "OOCD_DISTRO_URL_ARMHF=$OOCD_DISTRO_URL_ARMHF" >> pipeline.env
- echo "OOCD_DISTRO_URL_ARM64=$OOCD_DISTRO_URL_ARM64" >> pipeline.env
- run_cmd idf-ci gitlab pipeline-variables >> pipeline.env
- cat pipeline.env
- run_cmd idf-ci gitlab upload-artifacts --type env
artifacts:
reports:
dotenv: pipeline.env
paths:
- pipeline.env
expire_in: 1 week
when: always
baseline_manifest_sha:
extends:
- .pre_check_template
- .rules:dev-push
tags: [fast_run, shiny]
script:
- |
# merged results pipelines, by default
# diff between target-branch-head and merged-result-head
if [ -n "$CI_MERGE_REQUEST_TARGET_BRANCH_SHA" ]; then
git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA --depth=1
git checkout FETCH_HEAD
idf-build-apps dump-manifest-sha \
--manifest-files $(find . -name ".build-test-rules.yml" | xargs) \
--output .manifest_sha
# merge request pipelines, when the mr got conflicts
# diff between diff-base-sha and merge-request-head
elif [ -n "$CI_MERGE_REQUEST_DIFF_BASE_SHA" ]; then
git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1
git checkout FETCH_HEAD
idf-build-apps dump-manifest-sha \
--manifest-files $(find . -name ".build-test-rules.yml" | xargs) \
--output .manifest_sha
# other pipelines, like the protected branches pipelines
# not triggered in this job
fi
artifacts:
paths:
- .manifest_sha
expire_in: 1 week
when: always
- python tools/ci/gitlab_yaml_linter.py
retry_failed_jobs:
extends:

13
.gitlab/ci/rules.yml
View File

@@ -13,13 +13,6 @@
- "components/**/CMakeLists.txt"
.patterns-python-files: &patterns-python-files
- ".gitlab/ci/static-code-analysis.yml"
- "**/*.py"
.patterns-sonarqube-files: &patterns-sonarqube-files
- "tools/ci/sonar_exclude_list.txt"
.patterns-build_components: &patterns-build_components
# components files except "test*/" "host*/" folders
# ?? to include folders less than 4 characters
@@ -169,16 +162,10 @@
.if-protected-check: &if-protected-check
if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) || $CI_COMMIT_TAG =~ /^qa-test/'
.if-protected-deploy: &if-protected-deploy
if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "api")'
.if-protected-branch-push: &if-protected-branch-push
# rules:changes always evaluates to true for new branch pipelines or when there is no Git push event
if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/) && $CI_PIPELINE_SOURCE == "push"'
.if-qa-test-tag: &if-qa-test-tag
if: '$CI_COMMIT_TAG =~ /^qa-test/'
.if-dev-push: &if-dev-push
if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")'

82
.gitlab/ci/static-code-analysis.yml
View File

@@ -27,85 +27,3 @@ gcc_static_analyzer:
- echo "CONFIG_COMPILER_STATIC_ANALYZER=y" >> ${ANALYZING_APP}/sdkconfig.defaults
- sed -i 's/.*MINIMAL_BUILD.*//g' ${ANALYZING_APP}/CMakeLists.txt
- idf-build-apps build -p ${ANALYZING_APP}
#
## build stage
## Sonarqube related jobs put here for this reason:
## Here we have two jobs. code_quality_check and code_quality_report.
##
## code_quality_check will analyze the code changes between your MR and
## code repo stored in sonarqube server. The analysis result is only shown in
## the comments under this MR and won't be transferred to the server.
##
## code_quality_report will analyze and transfer both of the newly added code
## and the analysis result to the server.
##
## Put in the front to ensure that the newly merged code can be stored in
## sonarqube server ASAP, in order to avoid reporting unrelated code issues
#.sonar_scan_template:
# stage: build
# extends: .pre_check_template
# # full clone since this image does not support fetch --shallow-since-cutoff
# # shiny runners are used for full clone
# tags: [build, shiny]
# image: $SONARQUBE_SCANNER_IMAGE
# before_script:
# - source tools/ci/utils.sh
# - export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH"
# - fetch_submodules
# # Exclude the submodules, all paths ends with /**
# - submodules=$(get_all_submodules)
# # get all exclude paths specified in tools/ci/sonar_exclude_list.txt | ignore lines start with # | xargs | replace all <space> to <comma>
# - custom_excludes=$(cat $CI_PROJECT_DIR/tools/ci/sonar_exclude_list.txt | grep -v '^#' | xargs | sed -e 's/ /,/g')
# # Exclude the report dir as well
# - export EXCLUSIONS="$custom_excludes,$submodules"
# - export SONAR_SCANNER_OPTS="-Xmx2048m"
# variables:
# GIT_DEPTH: 0
# REPORT_PATTERN: clang_tidy_reports/**/*.txt
# artifacts:
# paths:
# - $REPORT_PATTERN
# expire_in: 1 week
# when: always
# dependencies: # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here.
# - clang_tidy_check
#
#code_quality_check:
# extends:
# - .sonar_scan_template
# - .rules:patterns:static-code-analysis-preview
# allow_failure: true # it's using exit code to indicate the code analysis result,
# # we don't want to block ci when critical issues founded
# script:
# - export CI_MERGE_REQUEST_COMMITS=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py commits --src-branch ${CI_COMMIT_REF_NAME} | tr '\n' ',')
# # test if this branch have merge request, if not, exit 0
# - test -n "$CI_MERGE_REQUEST_IID" || exit 0
# - test -n "$CI_MERGE_REQUEST_COMMITS" || exit 0
# - sonar-scanner
# -Dsonar.analysis.mode=preview
# -Dsonar.branch.name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
# -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN
# -Dsonar.exclusions=$EXCLUSIONS
# -Dsonar.gitlab.ci_merge_request_iid=$CI_MERGE_REQUEST_IID
# -Dsonar.gitlab.commit_sha=$CI_MERGE_REQUEST_COMMITS
# -Dsonar.gitlab.merge_request_discussion=true
# -Dsonar.gitlab.ref_name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
# -Dsonar.host.url=$SONAR_HOST_URL
# -Dsonar.login=$SONAR_LOGIN
#
#code_quality_report:
# extends:
# - .sonar_scan_template
# - .rules:protected:check
# allow_failure: true # it's using exit code to indicate the code analysis result,
# # we don't want to block ci when critical issues founded
# script:
# - sonar-scanner
# -Dsonar.branch.name=$CI_COMMIT_REF_NAME
# -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN
# -Dsonar.exclusions=$EXCLUSIONS
# -Dsonar.gitlab.commit_sha=$PIPELINE_COMMIT_SHA
# -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME
# -Dsonar.host.url=$SONAR_HOST_URL
# -Dsonar.login=$SONAR_LOGIN

46
.gitlab/ci/test-win.yml
View File

@@ -1,6 +1,6 @@
.host_test_win_template:
extends:
- .brew-windows-settings
- .windows-settings
- .rules:test:host_test
stage: host_test
dependencies: # set dependencies to null to avoid missing artifacts issue
@@ -50,7 +50,7 @@ test_tools_win:
script:
- cd tools\test_idf_py
- idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
- pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
- pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml ${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
# Build tests
.test_build_system_template_win:
@@ -67,42 +67,18 @@ test_tools_win:
script:
- cd tools\test_build_system
- idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
- pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
- pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml ${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
pytest_build_system_win:
extends:
- .test_build_system_template_win
- .rules:labels:windows_pytest_build_system
parallel: 6
needs:
- job: manual_gate
optional: true
tags: [windows-build, brew]
artifacts:
paths:
- XUNIT_RESULT.xml
- test_build_system
expire_in: 2 days
reports:
junit: XUNIT_RESULT.xml
when: always
pytest_build_system_win_minimal_cmake:
extends:
- .test_build_system_template_win
- .rules:labels:windows_pytest_build_system
needs:
- job: manual_gate
optional: true
tags: [windows-build, brew]
artifacts:
paths:
- XUNIT_RESULT.xml
- test_build_system
expire_in: 2 days
reports:
junit: XUNIT_RESULT.xml
when: always
variables:
MINIMAL_CMAKE_VERSION: "3.22.1"
script:
@@ -116,25 +92,13 @@ pytest_build_system_win_minimal_cmake:
}
- cd tools\test_build_system
- idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
- pytest -k cmake --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
- pytest -k cmake --junitxml ${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
pytest_buildv2_system_win:
extends:
- .test_build_system_template_win
- .rules:labels:buildv2
parallel: 2
needs:
- job: manual_gate
optional: true
tags: [windows-build, brew]
artifacts:
paths:
- XUNIT_RESULT.xml
- test_build_system
expire_in: 2 days
reports:
junit: XUNIT_RESULT.xml
when: always
script:
- cd tools\test_build_system
- idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
@@ -142,5 +106,5 @@ pytest_buildv2_system_win:
--buildv2
--parallel-count ${CI_NODE_TOTAL}
--parallel-index ${CI_NODE_INDEX}
--junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml
--junitxml ${CI_PROJECT_DIR}\XUNIT_RESULT.xml
--ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}

4
.idf_ci.toml
View File

@@ -1,4 +1,6 @@
preserve_non_test_related_apps = false
filter_non_test_related_apps_by_modified_files = true
filter_apps_by_component_target = true
exclude_dirs = [
'tools/test_mkdfu',
'tools/test_idf_size',
@@ -20,6 +22,7 @@ IDF_CI_BUILD = "1"
[gitlab.build_pipeline]
workflow_name = "build_child_pipeline"
runs_per_job = 30
job_tags = ['build', 'shiny']
job_template_name = '.dynamic_build_template'
@@ -107,6 +110,7 @@ patterns = [
bucket = "idf-artifacts"
patterns = [
'**/XUNIT_RESULT_*.xml',
'**/build_summary_*.xml',
]
[gitlab.artifacts.s3.configs.env]

12
.pre-commit-config.yaml
View File

@@ -145,7 +145,7 @@ repos:
require_serial: true
additional_dependencies:
- PyYAML>=5.3.1
- idf-build-apps>=2.14
- idf-build-apps~=3.0
- id: sort-yaml-files
name: sort yaml files
entry: tools/ci/sort_yaml.py
@@ -166,14 +166,6 @@ repos:
language: python
always_run: true
require_serial: true
- id: gitlab-yaml-linter
name: Check gitlab yaml files
entry: tools/ci/gitlab_yaml_linter.py
language: python
files: '\.gitlab-ci\.yml|\.gitlab/ci/.+\.yml|\.gitmodules'
pass_filenames: false
additional_dependencies:
- PyYAML == 5.3.1
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.0.1
hooks:
@@ -233,7 +225,7 @@ repos:
- id: check-kconfig-files
- id: check-deprecated-kconfig-options
- repo: https://github.com/espressif/idf-ci
rev: v1.0.0
rev: v1.2.1
hooks:
- id: check-tests-missing-config
files: 'pytest_.*\.py$|sdkconfig(\..*)?$'

24
sonar-project.properties
View File

@@ -1,24 +0,0 @@
sonar.projectKey=esp-idf
sonar.projectName=Espressif IoT Development Framework
sonar.projectVersion=4.4
sonar.sources=.
sonar.sourceEncoding=UTF-8
sonar.inclusions=\
**/*.c,**/*.cpp,**/*.h,**/*.hpp,\
**/*.py
####################
# sonar-cxx-plugin #
####################
sonar.cxx.includeDirectories=/usr/include
sonar.cxx.file.suffixes=.cxx,.cpp,.cc,.c,.hxx,.hpp,.hh,.h
#######################
# sonar-gitlab-plugin #
#######################
sonar.gitlab.failure_notification_mode=exit-code
sonar.gitlab.project_id=103

31
tools/ci/check_build_test_rules.py
View File

@@ -8,7 +8,6 @@ import sys
from collections import defaultdict
from pathlib import Path
import yaml
from idf_build_apps import App
from idf_build_apps import find_apps
from idf_build_apps.constants import SUPPORTED_TARGETS
@@ -39,6 +38,14 @@ USUAL_TO_FORMAL = {
FORMAL_TO_USUAL = {v: k for k, v in USUAL_TO_FORMAL.items()}
# Targets under bringup: suppress check-test-scripts warnings for these.
# Reset to [] once bringup is complete.
BYPASS_CHECK_TEST_TARGETS: list[str] = [
'esp32h21',
'esp32h4',
'esp32c5',
]
def print_diff_table(
list1: list[str], list2: list[str], title1: str, title2: str, excluded: list[str] | None = None
@@ -298,21 +305,9 @@ if __name__ == '__main__':
readme_parser = action.add_parser('check-readmes')
readme_parser.add_argument('paths', nargs='+', help='check under paths')
readme_parser.add_argument(
'-c',
'--config',
default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'),
help='config file',
)
test_parser = action.add_parser('check-test-scripts')
test_parser.add_argument('paths', nargs='+', help='check under paths')
test_parser.add_argument(
'-c',
'--config',
default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'),
help='config file',
)
arg = parser.parse_args()
check_dirs = set()
@@ -342,14 +337,6 @@ if __name__ == '__main__':
os.path.join(IDF_PATH, 'tools', 'cmakev2', 'test'),
]
_bypass_targets: list[str] = []
if arg.config:
with open(arg.config) as fr:
configs = yaml.safe_load(fr)
if configs:
_bypass_targets = configs.get('bypass_check_test_targets') or []
os.environ.update(
{
'IDF_PATH': IDF_PATH,
@@ -367,5 +354,5 @@ if __name__ == '__main__':
check_test_scripts(
list(check_dirs),
_exclude_dirs,
bypass_targets=_bypass_targets,
bypass_targets=BYPASS_CHECK_TEST_TARGETS,
)

154
tools/ci/ci_get_mr_info.py
View File

@@ -1,154 +0,0 @@
#!/usr/bin/env python
#
# internal use only for CI
# get latest MR information by source branch
#
# SPDX-FileCopyrightText: 2020-2025 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
#
import argparse
import logging
import os
import subprocess
import typing as t
from pathlib import Path
from gitlab_api import Gitlab
from idf_ci_utils import IDF_PATH
if t.TYPE_CHECKING:
from gitlab.v4.objects import ProjectCommit
from gitlab.v4.objects import ProjectMergeRequest
def _get_mr_obj(source_branch: str) -> t.Optional['ProjectMergeRequest']:
gl = Gitlab(os.getenv('CI_PROJECT_ID', 'espressif/esp-idf'))
if not gl.project:
return None
mrs = gl.project.mergerequests.list(state='opened', source_branch=source_branch)
if mrs:
return mrs[0] # one source branch can only have one opened MR at one moment
else:
return None
def get_mr_iid(source_branch: str) -> str:
mr = _get_mr_obj(source_branch)
if not mr:
return ''
else:
return str(mr.iid)
def get_mr_changed_files(source_branch: str) -> t.List[str]:
mr = _get_mr_obj(source_branch)
if not mr:
return []
git_output = subprocess.check_output(
['git', 'diff', '--name-only', '--diff-filter=d', f'origin/{mr.target_branch}...origin/{source_branch}']
).decode('utf8')
return [line.strip() for line in git_output.splitlines() if line.strip()]
def get_mr_commits(source_branch: str) -> t.List['ProjectCommit']:
mr = _get_mr_obj(source_branch)
if not mr:
return []
return list(mr.commits())
_COMPONENT_NAME_DIR_RECORDS = {}
def get_modified_component(filepath: str) -> t.Optional[str]:
"""Return the component name if the file is in a component directory, otherwise None."""
try:
f_path = Path(filepath).resolve().relative_to(IDF_PATH)
except ValueError: # not in IDF_PATH
return None
# skip md files, etc.
if f_path.suffix in ['.md', '.yml']:
return None
# skip test_apps files
if 'test_apps' in f_path.parts:
return None
component_parent_dirs = [f_path.parts[0]]
for part in f_path.parts[1:]:
if component_parent_dirs[-1] == 'components' or component_parent_dirs[-1].endswith('common_components'):
if part not in _COMPONENT_NAME_DIR_RECORDS:
logging.debug('Found component "%s" in path "%s"' % (part, component_parent_dirs))
_COMPONENT_NAME_DIR_RECORDS[part] = component_parent_dirs
elif _COMPONENT_NAME_DIR_RECORDS.get(part) != component_parent_dirs:
logging.debug(
'WARNING!!! Found component "%s" in path "%s" and "%s"'
% (part, component_parent_dirs, _COMPONENT_NAME_DIR_RECORDS.get(part))
)
return part
component_parent_dirs.append(part)
return None
def get_mr_components(
source_branch: t.Optional[str] = None, modified_files: t.Optional[t.List[str]] = None
) -> t.List[str]:
components: t.Set[str] = set()
if modified_files is None:
if not source_branch:
raise RuntimeError('--src-branch is required if --modified-files is not provided')
modified_files = get_mr_changed_files(source_branch)
for f in modified_files:
modified_component = get_modified_component(f)
if modified_component:
components.add(modified_component)
return list(components)
def _print_list(_list: t.List[str], separator: str = '\n') -> None:
print(separator.join(_list))
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Get the latest merge request info by pipeline')
actions = parser.add_subparsers(dest='action', help='info type', required=True)
common_args = argparse.ArgumentParser(add_help=False)
common_args.add_argument('--src-branch', help='source branch')
common_args.add_argument(
'--modified-files',
nargs='+',
help='space-separated list specifies the modified files. will be detected by --src-branch if not provided',
)
actions.add_parser('id', parents=[common_args])
actions.add_parser('commits', parents=[common_args])
actions.add_parser('components', parents=[common_args])
target = actions.add_parser('target_in_tags')
target.add_argument('tags', help='comma separated tags, e.g., esp32,generic')
args = parser.parse_args()
if args.action == 'id':
if not args.src_branch:
raise RuntimeError('--src-branch is required')
print(get_mr_iid(args.src_branch))
elif args.action == 'commits':
if not args.src_branch:
raise RuntimeError('--src-branch is required')
_print_list([commit.id for commit in get_mr_commits(args.src_branch)])
elif args.action == 'components':
_print_list(get_mr_components(args.src_branch, args.modified_files))
else:
raise NotImplementedError('not possible to get here')

217
tools/ci/deploy_docs.py
View File

@@ -1,217 +0,0 @@
#!/usr/bin/env python3
#
# CI script to deploy docs to a webserver. Not useful outside of CI environment
#
#
# SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
#
import glob
import os
import os.path
import re
import stat
import subprocess
import sys
import tarfile
from typing import Any, List, Tuple
import packaging.version
def env(variable: str, default: str=None) -> str:
""" Shortcut to return the expanded version of an environment variable """
return os.path.expandvars(os.environ.get(variable, default) if default else os.environ[variable])
# import sanitize_version from the docs directory, shared with here
sys.path.append(os.path.join(env('IDF_PATH'), 'docs'))
from sanitize_version import sanitize_version # noqa
def main() -> None:
# if you get KeyErrors on the following lines, it's probably because you're not running in Gitlab CI
git_ver = env('GIT_VER') # output of git describe --always
ci_ver = env('CI_COMMIT_REF_NAME', git_ver) # branch or tag we're building for (used for 'release' & URL)
version = sanitize_version(ci_ver)
print('Git version: {}'.format(git_ver))
print('CI Version: {}'.format(ci_ver))
print('Deployment version: {}'.format(version))
if not version:
raise RuntimeError('A version is needed to deploy')
build_dir = env('DOCS_BUILD_DIR') # top-level local build dir, where docs have already been built
if not build_dir:
raise RuntimeError('Valid DOCS_BUILD_DIR is needed to deploy')
url_base = env('DOCS_DEPLOY_URL_BASE') # base for HTTP URLs, used to print the URL to the log after deploying
docs_server = env('DOCS_DEPLOY_SERVER') # ssh server to deploy to
docs_user = env('DOCS_DEPLOY_SERVER_USER')
docs_path = env('DOCS_DEPLOY_PATH') # filesystem path on DOCS_SERVER
if not docs_server:
raise RuntimeError('Valid DOCS_DEPLOY_SERVER is needed to deploy')
if not docs_user:
raise RuntimeError('Valid DOCS_DEPLOY_SERVER_USER is needed to deploy')
docs_server = '{}@{}'.format(docs_user, docs_server)
if not docs_path:
raise RuntimeError('Valid DOCS_DEPLOY_PATH is needed to deploy')
print('DOCS_DEPLOY_SERVER {} DOCS_DEPLOY_PATH {}'.format(docs_server, docs_path))
tarball_path, version_urls = build_doc_tarball(version, git_ver, build_dir)
deploy(version, tarball_path, docs_path, docs_server)
print('Docs URLs:')
doc_deploy_type = os.getenv('TYPE')
for vurl in version_urls:
language, _, target = vurl.split('/')
tag = '{}_{}'.format(language, target)
url = '{}/{}/index.html'.format(url_base, vurl) # (index.html needed for the preview server)
url = re.sub(r'([^:])//', r'\1/', url) # get rid of any // that isn't in the https:// part
print('[document {}][{}] {}'.format(doc_deploy_type, tag, url))
# note: it would be neater to use symlinks for stable, but because of the directory order
# (language first) it's kind of a pain to do on a remote server, so we just repeat the
# process but call the version 'stable' this time
if is_stable_version(version):
print('Deploying again as stable version...')
tarball_path, version_urls = build_doc_tarball('stable', git_ver, build_dir)
deploy('stable', tarball_path, docs_path, docs_server)
def deploy(version: str, tarball_path: str, docs_path: str, docs_server: str) -> None:
def run_ssh(commands: List) -> None:
""" Log into docs_server and run a sequence of commands using ssh """
print('Running ssh: {}'.format(commands))
subprocess.run(['ssh', '-o', 'BatchMode=yes', docs_server, '-x', ' && '.join(commands)], check=True)
# copy the version tarball to the server
run_ssh(['mkdir -p {}'.format(docs_path)])
print('Running scp {} to {}'.format(tarball_path, '{}:{}'.format(docs_server, docs_path)))
subprocess.run(['scp', '-B', tarball_path, '{}:{}'.format(docs_server, docs_path)], check=True)
tarball_name = os.path.basename(tarball_path)
run_ssh(['cd {}'.format(docs_path),
'rm -rf ./*/{}'.format(version), # remove any pre-existing docs matching this version
'tar -zxvf {}'.format(tarball_name), # untar the archive with the new docs
'rm {}'.format(tarball_name)])
# Note: deleting and then extracting the archive is a bit awkward for updating stable/latest/etc
# as the version will be invalid for a window of time. Better to do it atomically, but this is
# another thing made much more complex by the directory structure putting language before version...
def build_doc_tarball(version: str, git_ver: str, build_dir: str) -> Tuple[str, List]:
""" Make a tar.gz archive of the docs, in the directory structure used to deploy as
the given version """
version_paths = []
tarball_path = '{}/{}.tar.gz'.format(build_dir, version)
# find all the 'html/' directories under build_dir
html_dirs = glob.glob('{}/**/html/'.format(build_dir), recursive=True)
print('Found %d html directories' % len(html_dirs))
pdfs = glob.glob('{}/**/latex/build/*.pdf'.format(build_dir), recursive=True)
print('Found %d PDFs in latex directories' % len(pdfs))
# add symlink for stable and latest and adds them to PDF blob
symlinks = create_and_add_symlinks(version, git_ver, pdfs)
def not_sources_dir(ti: Any) -> Any:
print(type(ti))
""" Filter the _sources directories out of the tarballs """
if ti.name.endswith('/_sources'):
return None
ti.mode |= stat.S_IWGRP # make everything group-writeable
return ti
try:
os.remove(tarball_path)
except OSError:
pass
with tarfile.open(tarball_path, 'w:gz') as tarball:
for html_dir in html_dirs:
# html_dir has the form '<ignored>/<language>/<target>/html/'
target_dirname = os.path.dirname(os.path.dirname(html_dir))
target = os.path.basename(target_dirname)
language = os.path.basename(os.path.dirname(target_dirname))
# when deploying, we want the top-level directory layout 'language/version/target'
archive_path = '{}/{}/{}'.format(language, version, target)
print("Archiving '{}' as '{}'...".format(html_dir, archive_path))
tarball.add(html_dir, archive_path, filter=not_sources_dir)
version_paths.append(archive_path)
for pdf_path in pdfs:
# pdf_path has the form '<ignored>/<language>/<target>/latex/build'
latex_dirname = os.path.dirname(pdf_path)
pdf_filename = os.path.basename(pdf_path)
target_dirname = os.path.dirname(os.path.dirname(latex_dirname))
target = os.path.basename(target_dirname)
language = os.path.basename(os.path.dirname(target_dirname))
# when deploying, we want the layout 'language/version/target/pdf'
archive_path = '{}/{}/{}/{}'.format(language, version, target, pdf_filename)
print("Archiving '{}' as '{}'...".format(pdf_path, archive_path))
tarball.add(pdf_path, archive_path)
for symlink in symlinks:
os.unlink(symlink)
return (os.path.abspath(tarball_path), version_paths)
def create_and_add_symlinks(version: str, git_ver: str, pdfs: List) -> List:
""" Create symbolic links for PDFs for 'latest' and 'stable' releases """
symlinks = []
if 'stable' in version or 'latest' in version:
for pdf_path in pdfs:
symlink_path = pdf_path.replace(git_ver, version)
os.symlink(pdf_path, symlink_path)
symlinks.append(symlink_path)
pdfs.extend(symlinks)
print('Found %d PDFs in latex directories after adding symlink' % len(pdfs))
return symlinks
def is_stable_version(version: str) -> bool:
""" Heuristic for whether this is the latest stable release """
if not version.startswith('v'):
return False # branch name
if '-' in version:
return False # prerelease tag
git_out = subprocess.check_output(['git', 'tag', '-l']).decode('utf-8')
versions = [v.strip() for v in git_out.split('\n')]
versions = [v for v in versions if re.match(r'^v[\d\.]+$', v.strip())] # include vX.Y.Z only
versions_pack = [packaging.version.parse(v) for v in versions]
max_version = max(versions_pack)
if max_version.public != version[1:]:
print('Stable version is v{}. This version is {}.'.format(max_version.public, version))
return False
else:
print('This version {} is the stable version'.format(version))
return True
if __name__ == '__main__':
main()

6
tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py
View File

@@ -55,7 +55,7 @@ def main(output_filepath: str) -> None:
exclude_runner_tags_matching = []
for _tag in known_warnings_dict.get('no_runner_tags', []):
if '*' not in _tag:
exclude_runner_tags_set.add(_tag)
exclude_runner_tags_set.add(frozenset(_tag.split(',')))
else:
if res := _process_match_group(_tag):
exclude_runner_tags_matching.append(res)
@@ -66,7 +66,7 @@ def main(output_filepath: str) -> None:
if exclude_runner_tags := os.getenv('EXCLUDE_RUNNER_TAGS'):
for _tag in exclude_runner_tags.split(';'):
if '*' not in _tag:
exclude_runner_tags_set.add(_tag)
exclude_runner_tags_set.add(frozenset(_tag.split(',')))
else:
if res := _process_match_group(_tag):
exclude_runner_tags_matching.append(res)
@@ -75,7 +75,7 @@ def main(output_filepath: str) -> None:
additional_dict: dict[GroupKey, dict[str, t.Any]] = {}
for key, grouped_cases in cases.grouped_cases.items():
# skip test cases with no runner tags
if ','.join(sorted(key.runner_tags)) in exclude_runner_tags_set:
if frozenset(key.runner_tags) in exclude_runner_tags_set:
print(f'WARNING: excluding test cases with runner tags: {key.runner_tags}')
continue

10
tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml
View File

@@ -17,6 +17,9 @@
# Since the version 3.21 CMake passes source files and include dirs to ninja using absolute paths.
# Needed for pytest junit reports.
INSTALL_EXTRA_TOOLS: cmake
# Suppress Python warning messages during CI builds to keep logs clean and focused on errors.
# This prevents non-critical warnings (e.g., deprecation or resource warnings) from cluttering output.
PYTHONWARNINGS: "ignore"
needs:
- pipeline: $PARENT_PIPELINE_ID
job: generate_build_child_pipeline
@@ -40,7 +43,6 @@
- run_cmd idf-ci build run
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
--modified-files ${MR_MODIFIED_FILES}
.dynamic_target_test_template:
image: $TARGET_TEST_ENV_IMAGE
@@ -56,6 +58,8 @@
needs:
- pipeline: $PARENT_PIPELINE_ID
job: pipeline_variables
- pipeline: $PARENT_PIPELINE_ID
job: snapshot_known_failure_cases
artifacts:
paths:
- XUNIT_RESULT*.xml
@@ -65,7 +69,11 @@
when: always
expire_in: 1 week
script:
# Merge the current known failure cases file with snapshot_known_failure_cases file captured at pipeline start
# to avoid retry failures caused by later updates to the known failure cases file.
- run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
- printf '\n' >> ${KNOWN_FAILURE_CASES_FILE_NAME}
- cat artifacts/${KNOWN_FAILURE_CASES_FILE_NAME}.freeze >> ${KNOWN_FAILURE_CASES_FILE_NAME}
# get runner env config file
- retry_failed git clone $TEST_ENV_CONFIG_REPO
- run_cmd python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs

3
tools/ci/exclude_check_tools_files.txt
View File

@@ -9,17 +9,14 @@ tools/ci/check_*.txt
tools/ci/check_copyright_config.yaml
tools/ci/checkout_project_ref.py
tools/ci/ci_fetch_submodule.py
tools/ci/ci_get_mr_info.py
tools/ci/cleanup_ignore_lists.py
tools/ci/configure_ci_environment.sh
tools/ci/deploy_docs.py
tools/ci/dynamic_pipelines/**/*
tools/ci/envsubst.py
tools/ci/executable-list.txt
tools/ci/fix_empty_prototypes.sh
tools/ci/generate_rules.py
tools/ci/get-full-sources.sh
tools/ci/get_all_test_results.py
tools/ci/get_supported_examples.sh
tools/ci/gitlab_yaml_linter.py
tools/ci/idf_build_apps_dump_soc_caps.py

1
tools/ci/executable-list.txt
View File

@@ -70,7 +70,6 @@ tools/ci/check_tools_files_patterns.py
tools/ci/check_type_comments.py
tools/ci/checkout_project_ref.py
tools/ci/cleanup_ignore_lists.py
tools/ci/deploy_docs.py
tools/ci/envsubst.py
tools/ci/fix_empty_prototypes.sh
tools/ci/generate_rules.py

46
tools/ci/generate_rules.py
View File

@@ -1,6 +1,6 @@
#!/usr/bin/env python
#
# SPDX-FileCopyrightText: 2021-2025 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import argparse
import inspect
@@ -12,22 +12,21 @@ from itertools import product
import yaml
from idf_ci_utils import IDF_PATH
from idf_ci_utils import GitlabYmlConfig
if t.TYPE_CHECKING:
import pygraphviz as pgv
def _list(str_or_list: t.Union[str, t.List]) -> t.List:
def _list(str_or_list: str | list) -> list:
if isinstance(str_or_list, str):
return [str_or_list]
elif isinstance(str_or_list, list):
return str_or_list
else:
raise ValueError('Wrong type: {}. Only supports str or list.'.format(type(str_or_list)))
raise ValueError(f'Wrong type: {type(str_or_list)}. Only supports str or list.')
def _format_nested_dict(_dict: t.Dict[str, t.Dict], f_tuple: t.Tuple[str, ...]) -> t.Dict[str, t.Dict]:
def _format_nested_dict(_dict: dict[str, dict], f_tuple: tuple[str, ...]) -> dict[str, dict]:
res = {}
for k, v in _dict.items():
k = k.split('__')[0]
@@ -41,7 +40,7 @@ def _format_nested_dict(_dict: t.Dict[str, t.Dict], f_tuple: t.Tuple[str, ...])
return res
def _format_nested_list(_list: t.List[str], f_tuple: t.Tuple[str, ...]) -> t.List[str]:
def _format_nested_list(_list: list[str], f_tuple: tuple[str, ...]) -> list[str]:
res = []
for item in _list:
if isinstance(item, list):
@@ -90,7 +89,6 @@ class RulesWriter:
self.cfg = self.expand_matrices()
self.rules = self.expand_rules()
self.yml_config = GitlabYmlConfig()
self.graph = None
def expand_matrices(self): # type: () -> dict
@@ -107,11 +105,11 @@ class RulesWriter:
deploy = v.get('deploy')
if deploy:
for item in _list(deploy):
res['{}-{}'.format(k, item)] = v
res[f'{k}-{item}'] = v
return res
@staticmethod
def _expand_matrix(name: str, cfg: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]:
def _expand_matrix(name: str, cfg: dict[str, t.Any]) -> dict[str, t.Any]:
"""
Expand matrix into multi keys
:param cfg: single rule dict
@@ -129,7 +127,7 @@ class RulesWriter:
res.update(_format_nested_dict(default, comb))
return res
def expand_rules(self) -> t.Dict[str, t.Dict[str, t.List[str]]]:
def expand_rules(self) -> dict[str, dict[str, list[str]]]:
res = defaultdict(lambda: defaultdict(set)) # type: dict[str, dict[str, set]]
for k, v in self.cfg.items():
if not v:
@@ -155,8 +153,8 @@ class RulesWriter:
if 'patterns' in v:
for _pat in _list(v['patterns']):
# Patterns must be pre-defined
if '.patterns-{}'.format(_pat) not in self.rules_cfg:
print('WARNING: pattern {} not exists'.format(_pat))
if f'.patterns-{_pat}' not in self.rules_cfg:
print(f'WARNING: pattern {_pat} not exists')
continue
res[item]['patterns'].add(_pat)
@@ -195,14 +193,10 @@ class RulesWriter:
if k.startswith('pattern'):
continue
if '.rules:' + k not in self.yml_config.used_templates:
print(f'WARNING: unused rule: {k}, skipping...')
continue
res.append(self.RULES_TEMPLATE.format(k, self._format_rule(k, v)))
return '\n\n'.join(res)
def _format_rule(self, name: str, cfg: t.Dict[str, t.Any]) -> str:
def _format_rule(self, name: str, cfg: dict[str, t.Any]) -> str:
_rules = [self.RULE_REVERT_BRANCH]
if name.endswith('-production'):
_rules.append(self.RULE_PROTECTED_PUSH)
@@ -216,21 +210,21 @@ class RulesWriter:
if f'.{specific_rule}' in self.rules_cfg:
_rules.append(self.SPECIFIC_RULE_TEMPLATE.format(specific_rule))
else:
print('WARNING: specific_rule {} not exists'.format(specific_rule))
print(f'WARNING: specific_rule {specific_rule} not exists')
for label in cfg['labels']:
_rules.append(self.RULE_LABEL_TEMPLATE.format(label))
for pattern in cfg['patterns']:
if '.patterns-{}'.format(pattern) in self.rules_cfg:
if f'.patterns-{pattern}' in self.rules_cfg:
_rules.append(self.RULE_PATTERN_TEMPLATE.format(pattern))
else:
print('WARNING: pattern {} not exists'.format(pattern))
print(f'WARNING: pattern {pattern} not exists')
return '\n'.join(_rules)
def update_rules_yml(self) -> bool:
with open(self.rules_yml) as fr:
file_str = fr.read()
auto_generate_str = '\n{}\n\n{}\n'.format(self.new_labels_str(), self.new_rules_str())
auto_generate_str = f'\n{self.new_labels_str()}\n\n{self.new_rules_str()}\n'
rest, marker, old = file_str.partition(self.AUTO_GENERATE_MARKER)
if old == auto_generate_str:
return False
@@ -246,7 +240,7 @@ PATTERN_COLOR = 'cyan'
RULE_COLOR = 'blue'
def build_graph(rules_dict: t.Dict[str, t.Dict[str, t.List[str]]]) -> 'pgv.AGraph':
def build_graph(rules_dict: dict[str, dict[str, list[str]]]) -> 'pgv.AGraph':
from pygraphviz import pgv
graph = pgv.AGraph(directed=True, rankdir='LR', concentrate=True)
@@ -263,13 +257,13 @@ def build_graph(rules_dict: t.Dict[str, t.Dict[str, t.List[str]]]) -> 'pgv.AGrap
labels = v.get('labels')
if labels:
for _label in labels:
graph.add_node('label:{}'.format(_label), color=LABEL_COLOR)
graph.add_edge('label:{}'.format(_label), k, color=LABEL_COLOR)
graph.add_node(f'label:{_label}', color=LABEL_COLOR)
graph.add_edge(f'label:{_label}', k, color=LABEL_COLOR)
patterns = v.get('patterns')
if patterns:
for _pat in patterns:
graph.add_node('pattern:{}'.format(_pat), color=PATTERN_COLOR)
graph.add_edge('pattern:{}'.format(_pat), k, color=PATTERN_COLOR)
graph.add_node(f'pattern:{_pat}', color=PATTERN_COLOR)
graph.add_edge(f'pattern:{_pat}', k, color=PATTERN_COLOR)
return graph

84
tools/ci/get_all_test_results.py
View File

@@ -1,84 +0,0 @@
#!/usr/bin/env python
#
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
'''
Download artifacts from all test jobs
CI_PROJECT_DIR
└── TEST_RESULTS
├── <job_id>
│ └── XUNIT_RESULT.xml
├── <job_id>
│ └── XUNIT_RESULT.xml
├── <job_id>
│ └── TEST_LOGS
'''
import argparse
import json
import os
import shutil
import gitlab_api
CI_PROJECT_ID = int(os.getenv('CI_PROJECT_ID', ''))
CI_PIPELINE_ID = int(os.getenv('CI_PIPELINE_ID', ''))
IDF_PATH = os.getenv('IDF_PATH', '.')
TEST_RESULT_PATH = os.path.join(IDF_PATH, 'TEST_RESULTS')
def download_all_test_results(result_path: str, include_retried: bool = False) -> None:
if os.path.exists(result_path):
shutil.rmtree(result_path)
os.makedirs(result_path, exist_ok=True)
gitlab_inst = gitlab_api.Gitlab(CI_PROJECT_ID)
pipelines = gitlab_inst.project.pipelines.get(CI_PIPELINE_ID)
_include_retried = 'true' if include_retried else 'false'
jobs = pipelines.jobs.list(all=True, per_page=100, include_retried=_include_retried)
job_info_list = []
for job in jobs:
if job.stage in ['target_test', 'host_test']:
log_path = ''
if job.status not in ['success', 'failed']:
print('Job {}({}) is not finished'.format(job.id, job.name))
elif not hasattr(job, 'artifacts_file'):
print('Job {}({}) has no artifacts.'.format(job.id, job.name))
else:
log_path = os.path.join(result_path, 'job_{}'.format(job.id))
print('Downloading artifacts from: {}'.format(job.name))
os.makedirs(log_path, exist_ok=True)
gitlab_inst.download_artifacts(job.id, log_path)
job_info = {
'id': job.id,
'name': job.name,
'tag_list': job.tag_list,
'status': job.status,
'stage': job.stage,
'web_url': job.web_url,
'commit_url': job.commit['web_url'],
'log_path': log_path,
}
job_info_list.append(job_info)
with open(os.path.join(result_path, 'index.json'), 'w') as f:
f.write(json.dumps({'jobs': job_info_list}, indent=1, sort_keys=True))
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument('--path', '-p', default=TEST_RESULT_PATH, help='Path to save result files.')
parser.add_argument('--include_retried', action='store_true', help='Including retried jobs.')
args = parser.parse_args()
download_all_test_results(args.path, args.include_retried)
if __name__ == '__main__':
main()

36
tools/ci/gitlab_yaml_linter.py
View File

@@ -1,27 +1,27 @@
#!/usr/bin/env python
# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
"""
Check gitlab ci yaml files
"""
import argparse
import os
import typing as t
from functools import cached_property
from idf_ci_utils import get_submodule_dirs
from idf_ci_utils import GitlabYmlConfig
from idf_ci_utils import IDF_PATH
from idf_ci_utils import GitlabYmlConfig
from idf_ci_utils import get_submodule_dirs
class YmlLinter:
def __init__(self, yml_config: GitlabYmlConfig) -> None:
self.yml_config = yml_config
self._errors: t.List[str] = []
self._errors: list[str] = []
@cached_property
def lint_functions(self) -> t.List[str]:
def lint_functions(self) -> list[str]:
funcs = []
for func in dir(self):
if func.startswith('_lint_'):
@@ -43,17 +43,6 @@ class YmlLinter:
exit(exit_code)
# name it like _1_ to make it run first
def _lint_1_yml_parser(self) -> None:
for k, v in self.yml_config.config.items():
if (
k not in self.yml_config.global_keys
and k not in self.yml_config.anchors
and k not in self.yml_config.templates
and k not in self.yml_config.jobs
):
raise SystemExit(f'Parser incorrect. Key {k} not in global keys, anchors, templates, or jobs')
def _lint_default_values_artifacts(self) -> None:
defaults_artifacts = self.yml_config.default.get('artifacts', {})
@@ -79,19 +68,6 @@ class YmlLinter:
for item in undefined_patterns:
self._errors.append(f'undefined pattern {item}. Please add {item} to .patterns-submodule')
def _lint_gitlab_yml_templates(self) -> None:
unused_templates = self.yml_config.templates.keys() - self.yml_config.used_templates
for item in unused_templates:
# known unused ones
if item not in [
'.before_script:fetch:target_test', # used in dynamic pipeline
]:
self._errors.append(f'Unused template: {item}, please remove it')
undefined_templates = self.yml_config.used_templates - self.yml_config.templates.keys()
for item in undefined_templates:
self._errors.append(f'Undefined template: {item}')
def _lint_dependencies_and_needs(self) -> None:
"""
Use `dependencies: []` together with `needs: []` could cause missing artifacts issue.

188
tools/ci/idf_ci_utils.py
View File

@@ -124,35 +124,106 @@ class GitlabYmlConfig:
# avoid unused import in other pre-commit hooks
import yaml
all_config = dict()
root_yml = yaml.load(open(root_yml_filepath), Loader=yaml.FullLoader)
# expanding "include"
for item in root_yml.pop('include', []) or []:
if isinstance(item, dict):
if 'project' in item:
continue
elif 'local' in item:
item = item['local']
else:
continue
all_config.update(yaml.load(open(os.path.join(IDF_PATH, item)), Loader=yaml.FullLoader))
merged_yaml = self._compile_via_gitlab_api(root_yml_filepath)
all_config = yaml.load(merged_yaml, Loader=yaml.FullLoader) or {}
if 'default' in all_config:
self._defaults = all_config.pop('default')
self._config = all_config
# anchor is the string that will be reused in templates
self._anchor_keys: set[str] = set()
# template is a dict that will be extended
self._template_keys: set[str] = set()
self._used_template_keys: set[str] = set() # tracing the used templates
# job is a dict that will be executed
self._job_keys: set[str] = set()
def _inline_local_includes(self, root_yml_filepath: str) -> str:
"""
Recursively resolve `include: local` entries straight from disk (so uncommitted local
changes are always picked up -- CI runners also work off a disk checkout, so there's no
need to fetch a ref remotely via `ci_lint`'s `content_ref`/`ref`/`dry_run_ref` GET
params). `include: project` entries (including ones nested inside local files, e.g.
`.gitlab/ci/common.yml` including `templates/idf/common-scripts.yml`) are collected and
left in the final `include:` list, since those files live in another GitLab project and
can only be resolved remotely.
self.expand_extends()
A dedicated Loader/Dumper pair round-trips `!reference` tags as a marker list subclass,
since GitLab CI uses `!reference` which plain YAML doesn't know, and we need to parse
(to merge dicts, not just string-concat) then re-dump losslessly.
:param root_yml_filepath: path to the local root yml file to compile
:return: yml content with local includes resolved and merged
"""
import yaml
class _Loader(yaml.FullLoader):
pass
class _Dumper(yaml.Dumper):
pass
class _Reference(list):
pass
_Loader.add_constructor(
'!reference', lambda loader, node: _Reference(loader.construct_sequence(t.cast(yaml.SequenceNode, node)))
)
_Dumper.add_representer(_Reference, lambda dumper, data: dumper.represent_sequence('!reference', list(data)))
def resolve(yml_filepath: str) -> tuple[dict, list]:
with open(yml_filepath) as fr:
data = yaml.load(fr, Loader=_Loader) or {}
includes = to_list(data.pop('include', None))
merged: dict = {}
remaining_project_includes: list = []
for item in includes:
if isinstance(item, dict):
if 'project' in item:
remaining_project_includes.append(item)
continue
elif 'local' in item:
local_path = item['local']
else:
continue
elif isinstance(item, str):
local_path = item
else:
continue
sub_merged, sub_remaining = resolve(os.path.join(IDF_PATH, local_path.lstrip('/')))
merged.update(sub_merged)
remaining_project_includes.extend(sub_remaining)
# this file's own top-level keys override whatever its includes defined
merged.update(data)
return merged, remaining_project_includes
merged_config, project_includes = resolve(root_yml_filepath)
if project_includes:
merged_config['include'] = project_includes
return yaml.dump(merged_config, Dumper=_Dumper, sort_keys=False) # type: ignore
def _compile_via_gitlab_api(self, root_yml_filepath: str) -> str:
"""
Call the GitLab CI Lint API to get the fully compiled (all `include`s resolved) yml,
same as what `glab ci config compile` does. This replaces the old recursive local-only
parsing, since the project now includes configs from other projects as well.
:param root_yml_filepath: path to the local root yml file to compile
:return: merged (fully resolved) yml content as a string
"""
sys.path.insert(0, os.path.join(IDF_PATH, 'tools', 'ci', 'python_packages'))
import gitlab_api
content = self._inline_local_includes(root_yml_filepath)
gitlab_inst = gitlab_api.Gitlab()
project_id = os.getenv('CI_PROJECT_ID') or gitlab_inst.get_project_id('esp-idf', namespace='espressif')
project = gitlab_inst.gitlab_inst.projects.get(project_id, lazy=True)
lint_result = project.ci_lint.create({'content': content})
if not lint_result.valid:
raise RuntimeError(f'Failed to compile {root_yml_filepath} via GitLab CI Lint API: {lint_result.errors}')
return lint_result.merged_yaml # type: ignore
@property
def default(self) -> dict[str, t.Any]:
@@ -166,80 +237,9 @@ class GitlabYmlConfig:
def global_keys(self) -> list[str]:
return ['default', 'include', 'workflow', 'variables', 'stages']
@cached_property
def anchors(self) -> dict[str, t.Any]:
return {k: v for k, v in self.config.items() if k in self._anchor_keys}
@cached_property
def jobs(self) -> dict[str, t.Any]:
return {k: v for k, v in self.config.items() if k in self._job_keys}
@cached_property
def templates(self) -> dict[str, t.Any]:
return {k: v for k, v in self.config.items() if k in self._template_keys}
@cached_property
def used_templates(self) -> set[str]:
return self._used_template_keys
def expand_extends(self) -> None:
"""
expand the `extends` key in-place.
"""
for k, v in self.config.items():
if k in self.global_keys:
continue
if isinstance(v, str | list):
self._anchor_keys.add(k)
elif k.startswith('.if-'):
self._anchor_keys.add(k)
elif k.startswith('.'):
self._template_keys.add(k)
elif isinstance(v, dict):
self._job_keys.add(k)
else:
raise ValueError(f'Unknown type for key {k} with value {v}')
# no need to expand anchor
# expand template first
for k in self._template_keys:
self._expand_extends(k)
# expand job
for k in self._job_keys:
self._expand_extends(k)
def _merge_dict(self, d1: dict[str, t.Any], d2: dict[str, t.Any]) -> t.Any:
for k, v in d2.items():
if k in d1:
if isinstance(v, dict) and isinstance(d1[k], dict):
d1[k] = self._merge_dict(d1[k], v)
else:
d1[k] = v
else:
d1[k] = v
return d1
def _expand_extends(self, name: str) -> dict[str, t.Any]:
extends = to_list(self.config[name].pop('extends', None))
if not extends:
return self.config[name] # type: ignore
original_d = self.config[name].copy()
d = {}
while extends:
self._used_template_keys.update(extends) # for tracking
for i in extends:
d.update(self._expand_extends(i))
extends = to_list(self.config[name].pop('extends', None))
self.config[name] = self._merge_dict(d, original_d)
return self.config[name] # type: ignore
return {k: v for k, v in self.config.items() if not k.startswith('.') and k not in self.global_keys}
def idf_relpath(p: str) -> str:

50
tools/ci/idf_pytest/plugin.py
View File

@@ -9,7 +9,6 @@ import pytest
import yaml
from _pytest.config import Config
from _pytest.python import Function
from _pytest.python import Metafunc
from _pytest.runner import CallInfo
from dynamic_pipelines.constants import KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH
from idf_ci import IdfPytestPlugin
@@ -115,55 +114,6 @@ class IdfLocalPlugin:
return item.callspec.params.get(key, default) or default
@staticmethod
def _has_parametrized_arg(metafunc: Metafunc, arg_name: str) -> bool:
for marker in metafunc.definition.iter_markers(name='parametrize'):
if not marker.args:
continue
argnames = marker.args[0]
if isinstance(argnames, str):
names = [name.strip() for name in argnames.split(',')]
else:
names = list(argnames)
if arg_name in names:
return True
for callspec in getattr(metafunc, '_calls', []):
if arg_name in callspec.params:
return True
return False
@staticmethod
def _is_linux_target_run(config: Config) -> bool:
target = config.getoption('target')
if not target:
return False
if isinstance(target, str):
targets = [_t.strip() for _t in target.split(',')]
else:
targets = [str(_t).strip() for _t in target]
return 'linux' in targets
@pytest.hookimpl(trylast=True)
def pytest_generate_tests(self, metafunc: Metafunc) -> None:
if 'embedded_services' not in metafunc.fixturenames:
return
if self._has_parametrized_arg(metafunc, 'embedded_services'):
return
if metafunc.definition.get_closest_marker('qemu') is not None:
metafunc.parametrize('embedded_services', ['idf,qemu'], indirect=True)
return
if self._is_linux_target_run(metafunc.config):
metafunc.parametrize('embedded_services', ['idf'], indirect=True)
@pytest.hookimpl(wrapper=True)
def pytest_collection_modifyitems(self, config: Config, items: list[Function]) -> t.Generator[None, None, None]:
yield # throw it back to idf-ci

93
tools/ci/python_packages/gitlab_api.py
View File

@@ -1,4 +1,4 @@
# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import argparse
import logging
@@ -9,13 +9,9 @@ import tarfile
import tempfile
import time
import zipfile
from collections.abc import Callable
from functools import wraps
from typing import Any
from typing import Callable
from typing import Dict
from typing import List
from typing import Optional
from typing import Union
import gitlab
@@ -36,7 +32,7 @@ def retry(func: TR) -> TR:
while True:
try:
res = func(self, *args, **kwargs)
except (IOError, EOFError, gitlab.exceptions.GitlabError) as e:
except (OSError, EOFError, gitlab.exceptions.GitlabError) as e:
if isinstance(e, gitlab.exceptions.GitlabError):
if e.response_code == 500:
# retry on this error
@@ -52,9 +48,11 @@ def retry(func: TR) -> TR:
raise e # get out of the loop
else:
logging.warning(
'Network failure in {}, retrying ({})'.format(getattr(func, '__name__', '(unknown callable)'),
retried))
time.sleep(2 ** retried) # wait a bit more after each retry
'Network failure in {}, retrying ({})'.format(
getattr(func, '__name__', '(unknown callable)'), retried
)
)
time.sleep(2**retried) # wait a bit more after each retry
continue
else:
break
@@ -63,26 +61,25 @@ def retry(func: TR) -> TR:
return wrapper
class Gitlab(object):
class Gitlab:
JOB_NAME_PATTERN = re.compile(r'(\w+)(\s+(\d+)/(\d+))?')
DOWNLOAD_ERROR_MAX_RETRIES = 3
DEFAULT_BUILD_CHILD_PIPELINE_NAME = 'Build Child Pipeline'
def __init__(self, project_id: Union[int, str, None] = None):
def __init__(self, project_id: int | str | None = None):
config_data_from_env = os.getenv('PYTHON_GITLAB_CONFIG')
if config_data_from_env:
# prefer to load config from env variable
with tempfile.NamedTemporaryFile('w', delete=False) as temp_file:
temp_file.write(config_data_from_env)
config_files = [temp_file.name] # type: Optional[List[str]]
config_files: list[str] | None = [temp_file.name]
else:
# otherwise try to use config file at local filesystem
config_files = None
self._init_gitlab_inst(project_id, config_files)
@retry
def _init_gitlab_inst(self, project_id: Optional[int], config_files: Optional[List[str]]) -> None:
def _init_gitlab_inst(self, project_id: int | None, config_files: list[str] | None) -> None:
gitlab_id = os.getenv('LOCAL_GITLAB_HTTPS_HOST') # if None, will use the default gitlab server
self.gitlab_inst = gitlab.Gitlab.from_config(gitlab_id=gitlab_id, config_files=config_files)
@@ -109,7 +106,7 @@ class Gitlab(object):
self.project = None
@retry
def get_project_id(self, name: str, namespace: Optional[str] = None) -> int:
def get_project_id(self, name: str, namespace: str | None = None) -> int:
"""
search project ID by name
@@ -117,7 +114,7 @@ class Gitlab(object):
:param namespace: namespace to match when we have multiple project with same name
:return: project ID
"""
projects = self.gitlab_inst.projects.list(search=name)
projects = self.gitlab_inst.projects.list(search=name, get_all=True)
res = []
for project in projects:
if namespace is None:
@@ -152,7 +149,7 @@ class Gitlab(object):
archive_file.extractall(destination)
@retry
def download_artifact(self, job_id: int, artifact_path: List[str], destination: Optional[str] = None) -> List[bytes]:
def download_artifact(self, job_id: int, artifact_path: list[str], destination: str | None = None) -> list[bytes]:
"""
download specific path of job artifacts and extract to destination.
@@ -169,7 +166,7 @@ class Gitlab(object):
try:
data = job.artifact(a_path) # type: bytes
except gitlab.GitlabGetError as e:
logging.error("Failed to download '{}' from job {}".format(a_path, job_id))
logging.error(f"Failed to download '{a_path}' from job {job_id}")
raise e
raw_data_list.append(data)
if destination:
@@ -185,7 +182,7 @@ class Gitlab(object):
return raw_data_list
@retry
def find_job_id(self, job_name: str, pipeline_id: Optional[str] = None, job_status: str = 'success') -> List[Dict]:
def find_job_id(self, job_name: str, pipeline_id: str | None = None, job_status: str = 'success') -> list[dict]:
"""
Get Job ID from job name of specific pipeline
@@ -208,8 +205,9 @@ class Gitlab(object):
return job_id_list
@retry
def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None,
cache_dir: Optional[str] = None) -> str:
def download_archive(
self, ref: str, destination: str, project_id: int | None = None, cache_dir: str | None = None
) -> str:
"""
Download archive of certain commit of a repository and extract to destination path
@@ -233,10 +231,11 @@ class Gitlab(object):
try:
project.repository_archive(sha=ref, streamed=True, action=fw.write)
except gitlab.GitlabGetError as e:
logging.error('Failed to archive from project {}'.format(project_id))
logging.error(f'Failed to archive from project {project_id}')
raise e
logging.info('Downloaded archive size: {:.03f}MB'.format(
float(os.path.getsize(local_archive_file)) / (1024 * 1024)))
logging.info(
f'Downloaded archive size: {float(os.path.getsize(local_archive_file)) / (1024 * 1024):.03f}MB'
)
return self.decompress_archive(local_archive_file, destination)
@@ -245,10 +244,10 @@ class Gitlab(object):
try:
project.repository_archive(sha=ref, streamed=True, action=temp_file.write)
except gitlab.GitlabGetError as e:
logging.error('Failed to archive from project {}'.format(project_id))
logging.error(f'Failed to archive from project {project_id}')
raise e
logging.info('Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(temp_file.name)) / (1024 * 1024)))
logging.info(f'Downloaded archive size: {float(os.path.getsize(temp_file.name)) / (1024 * 1024):.03f}MB')
return self.decompress_archive(temp_file.name, destination)
@@ -280,40 +279,32 @@ class Gitlab(object):
job = self.project.jobs.get(job_id)
return ','.join(job.tag_list)
def get_downstream_pipeline_ids(self, main_pipeline_id: int) -> List[int]:
def get_downstream_pipeline_ids(self, main_pipeline_id: int) -> list[int]:
"""
Retrieve the IDs of all downstream child pipelines for a given main pipeline.
Retrieve the IDs of all downstream child pipelines for a given main pipeline,
recursing through arbitrarily nested child pipelines.
:param main_pipeline_id: The ID of the main pipeline to start the search.
:return: A list of IDs of all downstream child pipelines.
:return: A list of IDs of all downstream child pipelines (all levels).
"""
bridge_pipeline_ids = []
child_pipeline_ids = []
child_pipeline_ids: list[int] = []
main_pipeline_bridges = self.project.pipelines.get(main_pipeline_id).bridges.list()
for bridge in main_pipeline_bridges:
pipeline_bridges = self.project.pipelines.get(main_pipeline_id).bridges.list()
for bridge in pipeline_bridges:
downstream_pipeline = bridge.attributes.get('downstream_pipeline')
if not downstream_pipeline:
continue
bridge_pipeline_ids.append(downstream_pipeline['id'])
for bridge_pipeline_id in bridge_pipeline_ids:
child_pipeline_ids.append(bridge_pipeline_id)
bridge_pipeline = self.project.pipelines.get(bridge_pipeline_id)
if not bridge_pipeline.name == self.DEFAULT_BUILD_CHILD_PIPELINE_NAME:
downstream_pipeline_id = downstream_pipeline.get('id')
if downstream_pipeline_id is None:
continue
child_bridges = bridge_pipeline.bridges.list()
for child_bridge in child_bridges:
downstream_child_pipeline = child_bridge.attributes.get('downstream_pipeline')
if not downstream_child_pipeline:
continue
child_pipeline_ids.append(downstream_child_pipeline.get('id'))
child_pipeline_ids.append(downstream_pipeline_id)
# recurse to collect further nested (grandchild+) pipelines
child_pipeline_ids.extend(self.get_downstream_pipeline_ids(downstream_pipeline_id))
return [pid for pid in child_pipeline_ids if pid is not None]
return child_pipeline_ids
def retry_failed_jobs(self, pipeline_id: int, retry_allowed_failures: bool = False) -> List[int]:
def retry_failed_jobs(self, pipeline_id: int, retry_allowed_failures: bool = False) -> list[int]:
"""
Retry failed jobs for a specific pipeline. Optionally include jobs marked as 'allowed failures'.
@@ -370,10 +361,10 @@ def main() -> None:
gitlab_inst.download_archive(args.ref, args.destination)
elif args.action == 'get_project_id':
ret = gitlab_inst.get_project_id(args.project_name)
print('project id: {}'.format(ret))
print(f'project id: {ret}')
elif args.action == 'retry_failed_jobs':
res = gitlab_inst.retry_failed_jobs(args.pipeline_id, args.retry_allowed_failures)
print('jobs retried successfully: {}'.format(res))
print(f'jobs retried successfully: {res}')
elif args.action == 'get_job_tags':
ret = gitlab_inst.get_job_tags(args.job_id)
print(ret)

35
tools/ci/utils.sh
View File

@@ -164,38 +164,3 @@ function join_by {
printf %s "$f" "${@/#/$d}"
fi
}
function is_based_on_commits() {
# This function would accept space-separated args as multiple commits.
# The return value would be 0 if current HEAD is based on any of the specified commits.
#
# In our CI, we use environment variable $REQUIRED_ANCESTOR_COMMITS to declare the ancestor commits.
# Please remember to set one commit for each release branch.
commits=$*
if [[ -z $commits ]]; then
info "Not specifying commits that branches should be based on, skipping check..."
return 0
fi
commits_str="$(join_by " or " $commits)" # no doublequotes here, passing array
info "Checking if current branch is based on $commits_str..."
for i in $commits; do
if git merge-base --is-ancestor "$i" HEAD >/dev/null 2>&1; then
info "Current branch is based on $i"
return 0
else
info "Current branch is not based on $i"
fi
done
error "The base commit of your branch is too old."
error "The branch should be more recent than either of the following commits:"
error " $commits_str"
error "To fix the issue:"
error " - If your merge request is 'Draft', or has conflicts with the target branch, rebase it to the latest master or release branch"
error " - Otherwise, simply run a new pipeline."
return 1
}