ci: backport master changes to 6.0

This commit is contained in:
Fu Hanxi
2026-03-10 10:37:23 +08:00
parent 5ecafbffc1
commit 6ff6d7ad6c
14 changed files with 126 additions and 1342 deletions
+4 -9
View File
@@ -1,7 +1,6 @@
.build_template:
stage: build
extends:
- .before_script:build
- .after_script:build
image: $ESP_ENV_IMAGE
tags: [build, shiny]
@@ -155,21 +154,17 @@ build_child_pipeline:
job: generate_build_child_pipeline
strategy: depend
generate_disabled_apps_report:
generate_prebuild_report:
extends:
- .build_template
tags: [fast_run, shiny]
dependencies: # set dependencies to null to avoid missing artifacts issue
needs:
- pipeline_variables
- job: baseline_manifest_sha
optional: true
artifacts:
paths:
- disabled_report.html
- prebuild_report.html
expire_in: 1 week
when: always
script:
- pip install dominate idf-build-apps
- run_cmd python tools/ci/gen_disabled_report.py --output disabled_report.html --verbose --enable-preview-targets
- echo "Report generated at https://${CI_PAGES_HOSTNAME}:${CI_SERVER_PORT}/-/esp-idf/-/jobs/${CI_JOB_ID}/artifacts/disabled_report.html"
- run_cmd idf-ci build collect --format html -o prebuild_report.html
- echo "Report generated at https://${CI_PAGES_HOSTNAME}:${CI_SERVER_PORT}/-/esp-idf/-/jobs/${CI_JOB_ID}/artifacts/prebuild_report.html"
+4 -154
View File
@@ -34,8 +34,7 @@ variables:
# since we're using merged-result pipelines, the last commit should work for most cases
# --prune --prune-tags: in case remote branch or tag is force pushed
GIT_FETCH_EXTRA_FLAGS: "--no-recurse-submodules --prune --prune-tags"
# we're using .cache folder for caches
GIT_CLEAN_FLAGS: -ffdx -e .cache/
LATEST_GIT_TAG: v6.0-rc1
SUBMODULE_FETCH_TOOL: "tools/ci/ci_fetch_submodule.py"
@@ -168,9 +167,6 @@ variables:
# Done after sourcing export.sh so that we could easily invoke the right pip
section_start "upgrade_ci_dependencies" "Upgrading CI dependencies"
pip install --upgrade --upgrade-strategy=eager -r $IDF_PATH/tools/requirements/requirements.ci.txt -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
if [[ "${CI_JOB_STAGE}" == "target_test" ]]; then
pip install --upgrade --upgrade-strategy=eager -r $IDF_PATH/tools/requirements/requirements.test-specific.txt -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
fi
section_end "upgrade_ci_dependencies"
REEXPORT_NEEDED=0
@@ -245,155 +241,20 @@ variables:
before_script:
- *common-before_scripts
.before_script:build:
before_script:
- *common-before_scripts
- *setup_tools_and_idf_python_venv
- add_gitlab_ssh_keys
- fetch_submodules
- export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
- export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
.after_script:build:
after_script:
- source tools/ci/utils.sh
- *show_ccache_statistics
- *upload_failed_job_log_artifacts
##############################
# Git Strategy Job Templates #
##############################
.git_init: &git_init |
mkdir -p "${CI_PROJECT_DIR}"
cd "${CI_PROJECT_DIR}"
git init
.git_fetch_from_mirror_url_if_exists: &git_fetch_from_mirror_url_if_exists |
# check if set mirror
if [ -n "${LOCAL_GITLAB_HTTPS_HOST:-}" ] && [ -n "${ESPCI_TOKEN:-}" ]; then
MIRROR_REPO_URL="https://bot:${ESPCI_TOKEN}@${LOCAL_GITLAB_HTTPS_HOST}/${CI_PROJECT_PATH}"
elif [ -n "${LOCAL_GIT_MIRROR:-}" ]; then
MIRROR_REPO_URL="${LOCAL_GIT_MIRROR}/${CI_PROJECT_PATH}"
fi
# fetch from mirror first if set
if [ -n "${MIRROR_REPO_URL:-}" ]; then
if git remote -v | grep origin; then
git remote set-url origin "${MIRROR_REPO_URL}"
else
git remote add origin "${MIRROR_REPO_URL}"
fi
# mirror url may fail with authentication issue
git fetch origin --no-recurse-submodules || true
fi
# set remote url to CI_REPOSITORY_URL
if git remote -v | grep origin; then
git remote set-url origin "${CI_REPOSITORY_URL}"
else
git remote add origin "${CI_REPOSITORY_URL}"
fi
.git_checkout_ci_commit_sha: &git_checkout_ci_commit_sha |
git checkout $CI_COMMIT_SHA
eval "git clean ${GIT_CLEAN_FLAGS}"
# git diff requires two commits, with different CI env var
#
# By default, we use git strategy "clone" with depth 1 to speed up the clone process.
# But for jobs requires running `git diff`, we need to fetch more commits to get the correct diffs.
#
# Since there's no way to get the correct git_depth before the job starts,
# we can't set `GIT_DEPTH` in the job definition.
#
# Set git strategy to "none" and fetch manually instead.
.before_script:fetch:git_diff:
variables:
GIT_STRATEGY: none
before_script:
- *git_init
- *git_fetch_from_mirror_url_if_exists
- |
# Store the diff output in a temporary file
TEMP_FILE=$(mktemp)
# merged results pipelines, by default
if [[ -n $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ]]; then
git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA
git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
git diff --name-only $CI_MERGE_REQUEST_TARGET_BRANCH_SHA...$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA > "$TEMP_FILE"
GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
# merge request pipelines, when the mr got conflicts
elif [[ -n $CI_MERGE_REQUEST_DIFF_BASE_SHA ]]; then
git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_COMMIT_SHA > "$TEMP_FILE"
GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
# other pipelines, like the protected branches pipelines
elif [[ "$CI_COMMIT_BEFORE_SHA" != "0000000000000000000000000000000000000000" ]]; then
git fetch origin $CI_COMMIT_BEFORE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA > "$TEMP_FILE"
GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
else
# pipeline source could be web, scheduler, etc.
git fetch origin $CI_COMMIT_SHA --depth=2 ${GIT_FETCH_EXTRA_FLAGS}
git diff --name-only $CI_COMMIT_SHA~1 $CI_COMMIT_SHA > "$TEMP_FILE"
GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
fi
- *git_checkout_ci_commit_sha
- *common-before_scripts
- *setup_tools_and_idf_python_venv
- add_gitlab_ssh_keys
# target test runners may locate in different places
# for runners set git mirror, we fetch from the mirror first, then fetch the HEAD commit
.before_script:fetch:target_test:
variables:
GIT_STRATEGY: none
before_script:
- *git_init
- *git_fetch_from_mirror_url_if_exists
- eval "git fetch --depth=1 ${GIT_FETCH_EXTRA_FLAGS} origin ${CI_COMMIT_SHA}"
- *git_checkout_ci_commit_sha
- *common-before_scripts
- *setup_tools_and_idf_python_venv
- add_gitlab_ssh_keys
# no submodules
.brew-macos-settings:
variables:
GIT_STRATEGY: none # we do manual git clone to use local mirror
GIT_STRATEGY: fetch
IDF_CCACHE_ENABLE: "0"
CCACHE_DIR: "/var/tmp/cache/idf_ccache"
tags:
- macos-tart
image: macos-sequoia-idf-v6.0
cache: [] # pip cache is created under amd64, and submodules are downloaded with brew mirror, so disable cache here
before_script:
# assert LOCAL_GIT_MIRROR is set
- echo -e "section_start:`date +%s`:check_out\r\e[0Kchecking out from local git mirror, then reset to CI_COMMIT_SHA"
- |
if [ -z "${LOCAL_GIT_MIRROR:-}" ]; then
echo "Error: LOCAL_GIT_MIRROR not set, cannot clone from mirror."
exit 1
fi
- MIRROR_REPO_URL="${LOCAL_GIT_MIRROR}/${CI_PROJECT_PATH}"
- cd "${CI_PROJECT_DIR}"
# since .cache exists in CI_PROJECT_DIR, so can't direct `git clone .`
- git clone -b ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-${CI_COMMIT_BRANCH}} --depth=1 --recursive --shallow-submodules "${MIRROR_REPO_URL}" tmp
- mv tmp/.git ./
- rm -rf tmp
- git reset --hard
# set remote url back
- git remote set-url origin "${CI_REPOSITORY_URL}"
- eval "git fetch --depth=1 ${GIT_FETCH_EXTRA_FLAGS} origin ${CI_COMMIT_SHA}"
- git checkout FETCH_HEAD
- git submodule update --init --recursive --depth=1
- echo -e "section_end:`date +%s`:check_out\r\e[0K"
- *common-before_scripts
- *setup_tools_and_idf_python_venv
after_script: [] # ccache now is disabled for macos brew runners
timeout: 30m
@@ -421,16 +282,5 @@ default:
- *setup_tools_and_idf_python_venv
- add_gitlab_ssh_keys
- fetch_submodules
# gitlab bug, setting them here doesn't work
# - expire_in: https://gitlab.com/gitlab-org/gitlab/-/issues/404563
# - when: https://gitlab.com/gitlab-org/gitlab/-/issues/440672
# artifacts:
# expire_in: 1 week
# when: always
retry:
max: 2
when:
# In case of a runner failure we could hop to another one, or a network error could go away.
- runner_system_failure
# Job execution timeout may be caused by a network issue.
- job_execution_timeout
- export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
- export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
-63
View File
@@ -3,51 +3,6 @@
image: $ESP_ENV_IMAGE
tags: [ deploy ]
check_submodule_sync:
extends:
- .deploy_job_template
- .rules:test:submodule
stage: test_deploy
tags: [ brew, github_sync ]
retry: 2
variables:
GIT_STRATEGY: fetch # use brew local mirror first
SUBMODULES_TO_FETCH: "none"
PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git"
dependencies: []
script:
- git submodule deinit --force .
- rm -rf .git/modules # remove all the cached metadata
# setting the default remote URL to the public one, to resolve relative location URLs
- git config remote.origin.url ${PUBLIC_IDF_URL}
# check if all submodules are correctly synced to public repository
- git submodule init
- git config --get-regexp '^submodule\..*\.url$' || true
- git submodule update --recursive
- echo "IDF was cloned from ${PUBLIC_IDF_URL} completely"
push_to_github:
extends:
- .deploy_job_template
- .before_script:minimal
- .rules:protected:deploy
needs:
# submodule must be synced before pushing to github
- check_submodule_sync
tags: [ brew, github_sync ]
variables:
GIT_STRATEGY: fetch # use brew local mirror first
GIT_DEPTH: 0 # github needs full record of commits
script:
- add_github_ssh_keys
- git remote remove github &>/dev/null || true
- git remote add github git@github.com:espressif/esp-idf.git
- tools/ci/push_to_github.sh
environment:
name: push_to_github_production
deployment_tier: production
url: "https://github.com/espressif/esp-idf"
deploy_update_SHA_in_esp-dockerfiles:
extends:
- .deploy_job_template
@@ -64,21 +19,3 @@ deploy_update_SHA_in_esp-dockerfiles:
environment:
name: deploy_update_SHA_in_esp-dockerfiles_production
deployment_tier: production
upload_junit_report:
extends:
- .deploy_job_template
tags: [ fast_run, shiny ]
needs:
- pipeline_variables
- job: build_child_pipeline
artifacts: false
script:
- run_cmd idf-ci gitlab download-artifacts --type junit
rules:
- when: always
artifacts:
reports:
junit: XUNIT_RESULT_*.xml
expire_in: 1 week
when: always
+9 -76
View File
@@ -16,9 +16,6 @@
.patterns-example-readme: &patterns-example-readme
- "examples/**/*.md"
.patterns-docs-preview: &patterns-docs-preview
- "docs/**/*"
.if-protected-check: &if-protected-check
if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/)'
@@ -58,7 +55,9 @@
check_readme_links:
extends:
- .pre_check_template
tags: ["amd64", "brew"]
variables:
GIT_STRATEGY: "fetch"
tags: ["check_doc_links"]
allow_failure: true
rules:
- <<: *if-protected-check
@@ -100,6 +99,9 @@ check_docs_gh_links:
extends:
- .pre_check_template
- .doc-rules:build:docs-full
variables:
GIT_STRATEGY: "fetch"
tags: ["check_doc_links"]
script:
- cd docs
- build-docs gh-linkcheck
@@ -138,77 +140,6 @@ build_docs_html_partial:
- DOCLANG: "zh_CN"
DOCTGT: "esp32p4"
.deploy_docs_template:
image: $ESP_IDF_DOC_ENV_IMAGE
variables:
DOCS_BUILD_DIR: "${IDF_PATH}/docs/_build/"
PYTHONUNBUFFERED: 1
# ensure all tags are fetched, need to know the latest/stable tag for the docs
GIT_STRATEGY: clone
GIT_DEPTH: 0
stage: test_deploy
tags:
- brew
- amd64
script:
# ensure all tags are fetched, need to know the latest/stable tag for the docs
- git fetch --tags --prune
- add_doc_server_ssh_keys $DOCS_DEPLOY_PRIVATEKEY $DOCS_DEPLOY_SERVER $DOCS_DEPLOY_SERVER_USER
- export GIT_VER=$(git describe --always ${PIPELINE_COMMIT_SHA} --)
- deploy-docs
# stage: test_deploy
deploy_docs_preview:
extends:
- .deploy_docs_template
rules:
- <<: *if-label-build_docs
- <<: *if-label-docs_full
- <<: *if-dev-push
changes: *patterns-docs-preview
needs:
- job: build_docs_html_partial
optional: true
- job: build_docs_html_full
optional: true
variables:
TYPE: "preview"
# older branches use DOCS_DEPLOY_KEY, DOCS_SERVER, DOCS_SERVER_USER, DOCS_PATH for preview server so we keep these names for 'preview'
DOCS_DEPLOY_PRIVATEKEY: "$DOCS_DEPLOY_KEY"
DOCS_DEPLOY_SERVER: "$DOCS_SERVER"
DOCS_DEPLOY_SERVER_USER: "$DOCS_SERVER_USER"
DOCS_DEPLOY_PATH: "$DOCS_PATH"
DOCS_DEPLOY_URL_BASE: "https://$DOCS_PREVIEW_SERVER_URL/docs/esp-idf"
environment:
name: deploy_docs_preview
deployment_tier: staging
url: "https://$DOCS_PREVIEW_SERVER_URL/docs/esp-idf"
# stage: post_deploy
deploy_docs_production:
# The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings
extends:
- .deploy_docs_template
- .rules:protected:deploy
stage: post_deploy
dependencies: # set dependencies to null to avoid missing artifacts issue
needs: # ensure runs after push_to_github succeeded
- build_docs_html_full
- job: push_to_github
artifacts: false
variables:
TYPE: "preview"
DOCS_DEPLOY_PRIVATEKEY: "$DOCS_PROD_DEPLOY_KEY"
DOCS_DEPLOY_SERVER: "$DOCS_PROD_SERVER"
DOCS_DEPLOY_SERVER_USER: "$DOCS_PROD_SERVER_USER"
DOCS_DEPLOY_PATH: "$DOCS_PROD_PATH"
DOCS_DEPLOY_URL_BASE: "https://docs.espressif.com/projects/esp-idf"
DEPLOY_STABLE: 1
environment:
name: deploy_docs_production
deployment_tier: production
url: "https://docs.espressif.com/projects/esp-idf"
check_doc_links:
extends:
- .build_docs_template
@@ -217,7 +148,9 @@ check_doc_links:
needs:
- job: deploy_docs_production
artifacts: false
tags: ["build", "amd64", "internet"]
variables:
GIT_STRATEGY: "fetch"
tags: ["check_doc_links"]
artifacts:
when: always
paths:
-2
View File
@@ -320,7 +320,6 @@ build_docker:
test_pytest_qemu:
extends:
- .host_test_template
- .before_script:build
artifacts:
paths:
- XUNIT_RESULT.xml
@@ -355,7 +354,6 @@ test_pytest_qemu:
test_pytest_linux:
extends:
- .host_test_template
- .before_script:build
artifacts:
paths:
- XUNIT_RESULT.xml
+52 -13
View File
@@ -115,17 +115,67 @@ check_version_tag:
check_test_scripts_build_test_rules:
extends:
- .pre_check_template
- .before_script:build
script:
# requires basic pytest dependencies
- python tools/ci/check_build_test_rules.py check-test-scripts examples/ tools/test_apps components
check_submodule_sync:
extends:
- .pre_check_template
- .rules:test:submodule
tags: [ brew, github_sync ]
retry: 2
variables:
GIT_STRATEGY: fetch # use brew local mirror first
SUBMODULES_TO_FETCH: "none"
PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git"
script:
- git submodule deinit --force .
- rm -rf .git/modules # remove all the cached metadata
# setting the default remote URL to the public one, to resolve relative location URLs
- git config remote.origin.url ${PUBLIC_IDF_URL}
# check if all submodules are correctly synced to public repository
- git submodule init
- git config --get-regexp '^submodule\..*\.url$' || true
- git submodule update --recursive
- echo "IDF was cloned from ${PUBLIC_IDF_URL} completely"
pipeline_variables:
extends:
- .pre_check_template
- .before_script:fetch:git_diff
tags: [fast_run, shiny]
variables:
GIT_STRATEGY: "fetch"
script:
- |
# Store the diff output in a temporary file
TEMP_FILE=$(mktemp)
# merged results pipelines, by default
if [[ -n $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ]]; then
git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA
git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
git diff --name-only $CI_MERGE_REQUEST_TARGET_BRANCH_SHA...$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA > "$TEMP_FILE"
GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
# merge request pipelines, when the mr got conflicts
elif [[ -n $CI_MERGE_REQUEST_DIFF_BASE_SHA ]]; then
git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_COMMIT_SHA > "$TEMP_FILE"
GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
# other pipelines, like the protected branches pipelines
elif [[ "$CI_COMMIT_BEFORE_SHA" != "0000000000000000000000000000000000000000" ]]; then
git fetch origin $CI_COMMIT_BEFORE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA > "$TEMP_FILE"
GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
else
# pipeline source could be web, scheduler, etc.
git fetch origin $CI_COMMIT_SHA --depth=2 ${GIT_FETCH_EXTRA_FLAGS}
git diff --name-only $CI_COMMIT_SHA~1 $CI_COMMIT_SHA > "$TEMP_FILE"
GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
fi
# MODIFIED_FILES is a list of files that changed, could be used everywhere
- MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | xargs)
- echo "MODIFIED_FILES=$MODIFIED_FILES" >> pipeline.env
@@ -191,17 +241,6 @@ baseline_manifest_sha:
expire_in: 1 week
when: always
gcc_static_analyzer:
extends:
- .pre_check_template
- .rules:build
variables:
CI_CCACHE_DISABLE: 1
ANALYZING_APP: "examples/get-started/hello_world"
script:
- echo "CONFIG_COMPILER_STATIC_ANALYZER=y" >> ${ANALYZING_APP}/sdkconfig.defaults
- idf-build-apps build -p ${ANALYZING_APP}
retry_failed_jobs:
extends:
- .pre_check_template
+11
View File
@@ -16,6 +16,17 @@ clang_tidy_check:
--limit-file tools/ci/static-analysis-rules.yml
--xtensa-include-dir
gcc_static_analyzer:
extends:
- .pre_check_template
- .rules:patterns:clang_tidy
variables:
CI_CCACHE_DISABLE: 1
ANALYZING_APP: "examples/get-started/hello_world"
script:
- echo "CONFIG_COMPILER_STATIC_ANALYZER=y" >> ${ANALYZING_APP}/sdkconfig.defaults
- idf-build-apps build -p ${ANALYZING_APP}
#
## build stage
## Sonarqube related jobs put here for this reason: