mirror of
https://github.com/espressif/esp-idf.git
synced 2026-09-22 13:01:16 +03:00
ci: apply common-scripts CI refactor
This commit is contained in:
Fu Hanxi
@@ -2,11 +2,25 @@
|
|||||||
|
|
||||||
include:
|
include:
|
||||||
- project: "ci/actions/common"
|
- project: "ci/actions/common"
|
||||||
|
ref: "v2"
|
||||||
file:
|
file:
|
||||||
- "templates/stable/default-workflow.yml"
|
- "templates/stable/default-workflow.yml"
|
||||||
|
- "templates/idf/pre_check.yml"
|
||||||
|
- "templates/idf/build.yml"
|
||||||
- "templates/idf/deploy-github.yml"
|
- "templates/idf/deploy-github.yml"
|
||||||
- "templates/idf/deploy-docs.yml"
|
- "templates/idf/deploy-docs.yml"
|
||||||
- "templates/idf/integration-test.yml"
|
- "templates/idf/integration-test.yml"
|
||||||
|
- project: "ci/actions/common"
|
||||||
|
ref: "v2"
|
||||||
|
file: "templates/idf/sync-files.yml"
|
||||||
|
inputs:
|
||||||
|
job_name: "sync-pre-commit-config"
|
||||||
|
remote_repo_url: "https://__token__:${CI_SYNC_REPO_TOKEN}@${CI_SERVER_FQDN}/ci/images.git"
|
||||||
|
ref_name: "ci/sync_idf_pre_commit_config"
|
||||||
|
file_mappings:
|
||||||
|
- ".pre-commit-config.yaml:pre-commit/idf/.pre-commit-config.v5.5.yaml"
|
||||||
|
changes:
|
||||||
|
- ".pre-commit-config.yaml"
|
||||||
- project: "ci/actions/common"
|
- project: "ci/actions/common"
|
||||||
file: "templates/idf/build-docs.yml"
|
file: "templates/idf/build-docs.yml"
|
||||||
inputs:
|
inputs:
|
||||||
@@ -20,13 +34,14 @@ include:
|
|||||||
DOCTGT: "esp32p4"
|
DOCTGT: "esp32p4"
|
||||||
|
|
||||||
- ".gitlab/ci/danger.yml"
|
- ".gitlab/ci/danger.yml"
|
||||||
- ".gitlab/ci/common.yml"
|
- ".gitlab/ci/common.yml" # has to be placed after danger.yml, since danger.yml defined stages.
|
||||||
- ".gitlab/ci/rules.yml"
|
- ".gitlab/ci/rules.yml"
|
||||||
|
# by stages
|
||||||
- ".gitlab/ci/manual_gate.yml"
|
- ".gitlab/ci/manual_gate.yml"
|
||||||
- ".gitlab/ci/upload_cache.yml"
|
- ".gitlab/ci/upload_cache.yml"
|
||||||
|
- ".gitlab/ci/pre_check.yml"
|
||||||
- ".gitlab/ci/static-code-analysis.yml"
|
- ".gitlab/ci/static-code-analysis.yml"
|
||||||
- ".gitlab/ci/pre_commit.yml"
|
- ".gitlab/ci/pre_commit.yml"
|
||||||
- ".gitlab/ci/pre_check.yml"
|
|
||||||
- ".gitlab/ci/build.yml"
|
- ".gitlab/ci/build.yml"
|
||||||
- ".gitlab/ci/host-test.yml"
|
- ".gitlab/ci/host-test.yml"
|
||||||
- ".gitlab/ci/post_deploy.yml"
|
- ".gitlab/ci/post_deploy.yml"
|
||||||
|
|||||||
@@ -245,76 +245,15 @@ We're using the latest version of [idf-build-apps][idf-build-apps]. Please refer
|
|||||||
In ESP-IDF CI, there's a few more special rules are additionally supported to disable the check app dependencies feature:
|
In ESP-IDF CI, there's a few more special rules are additionally supported to disable the check app dependencies feature:
|
||||||
|
|
||||||
- Add MR labels `BUILD_AND_TEST_ALL_APPS`
|
- Add MR labels `BUILD_AND_TEST_ALL_APPS`
|
||||||
|
- Pipeline variable `IDF_CI_SELECT_ALL_PYTEST_CASES=1`
|
||||||
- Run in protected branches
|
- Run in protected branches
|
||||||
|
|
||||||
## Upload/Download Artifacts to Internal Minio Server
|
## Upload/Download Artifacts to Internal Minio Server
|
||||||
|
|
||||||
### Users Without Access to Minio
|
Please refer to the documentation [here](https://docs.espressif.com/projects/idf-ci/en/latest/guides/cli.html)
|
||||||
|
|
||||||
If you don't have access to the internal Minio server, you can still download the artifacts from the shared link in the job log.
|
## Revision Marker Naming Rules
|
||||||
|
|
||||||
The log will look like this:
|
When specifying chip revisions for testing markers, configurations or filenames, strictly follow the format `esp32<series>_rev<number>` (or similarly `<series>_rev<number>` when brief).
|
||||||
|
|
||||||
```shell
|
> e.g., `esp32_rev3`, `esp32c2_rev2`, `esp32c3_rev1`, `esp32p4_rev1`, `esp32c5_rev1`
|
||||||
Pipeline ID : 587355
|
|
||||||
Job name : build_clang_test_apps_esp32
|
|
||||||
Job ID : 40272275
|
|
||||||
Created archive file: 40272275.zip, uploading as 587355/build_dir_without_map_and_elf_files/build_clang_test_apps_esp32/40272275.zip
|
|
||||||
Please download the archive file includes build_dir_without_map_and_elf_files from [INTERNAL_URL]
|
|
||||||
```
|
|
||||||
|
|
||||||
### Users With Access to Minio
|
|
||||||
|
|
||||||
#### Env Vars for Minio
|
|
||||||
|
|
||||||
Minio takes these env vars to connect to the server:
|
|
||||||
|
|
||||||
- `IDF_S3_SERVER`
|
|
||||||
- `IDF_S3_ACCESS_KEY`
|
|
||||||
- `IDF_S3_SECRET_KEY`
|
|
||||||
- `IDF_S3_BUCKET`
|
|
||||||
|
|
||||||
#### Artifacts Types and File Patterns
|
|
||||||
|
|
||||||
The artifacts types and corresponding file patterns are defined in tools/ci/artifacts_handler.py, inside `ArtifactType` and `TYPE_PATTERNS_DICT`.
|
|
||||||
|
|
||||||
#### Upload
|
|
||||||
|
|
||||||
```shell
|
|
||||||
python tools/ci/artifacts_handler.py upload
|
|
||||||
```
|
|
||||||
|
|
||||||
will upload the files that match the file patterns to minio object storage with name:
|
|
||||||
|
|
||||||
`<pipeline_id>/<artifact_type>/<job_name>/<job_id>.zip`
|
|
||||||
|
|
||||||
For example, job 39043328 will upload these four files:
|
|
||||||
|
|
||||||
- `575500/map_and_elf_files/build_pytest_examples_esp32/39043328.zip`
|
|
||||||
- `575500/build_dir_without_map_and_elf_files/build_pytest_examples_esp32/39043328.zip`
|
|
||||||
- `575500/logs/build_pytest_examples_esp32/39043328.zip`
|
|
||||||
- `575500/size_reports/build_pytest_examples_esp32/39043328.zip`
|
|
||||||
|
|
||||||
#### Download
|
|
||||||
|
|
||||||
You may run
|
|
||||||
|
|
||||||
```shell
|
|
||||||
python tools/ci/artifacts_handler.py download --pipeline_id <pipeline_id>
|
|
||||||
```
|
|
||||||
|
|
||||||
to download all files of the pipeline, or
|
|
||||||
|
|
||||||
```shell
|
|
||||||
python tools/ci/artifacts_handler.py download --pipeline_id <pipeline_id> --job_name <job_name_or_pattern>
|
|
||||||
```
|
|
||||||
|
|
||||||
to download all files with the specified job name or pattern, or
|
|
||||||
|
|
||||||
```shell
|
|
||||||
python tools/ci/artifacts_handler.py download --pipeline_id <pipeline_id> --job_name <job_name_or_pattern> --type <artifact_type> <artifact_type> ...
|
|
||||||
```
|
|
||||||
|
|
||||||
to download all files with the specified job name or pattern and artifact type(s).
|
|
||||||
|
|
||||||
You may check all detailed documentation with `python tools/ci/artifacts_handler.py download -h`
|
|
||||||
|
|||||||
@@ -110,61 +110,6 @@ build_clang_test_apps_riscv:
|
|||||||
####################
|
####################
|
||||||
# Dynamic Pipeline #
|
# Dynamic Pipeline #
|
||||||
####################
|
####################
|
||||||
generate_build_child_pipeline:
|
# !!!
|
||||||
extends:
|
# configurations moved to ci/actions/common/idf/
|
||||||
- .build_template
|
# !!!
|
||||||
tags: [fast_run, shiny]
|
|
||||||
dependencies: # set dependencies to null to avoid missing artifacts issue
|
|
||||||
needs:
|
|
||||||
- pipeline_variables
|
|
||||||
- job: baseline_manifest_sha
|
|
||||||
optional: true
|
|
||||||
artifacts:
|
|
||||||
paths:
|
|
||||||
- build_child_pipeline.yml
|
|
||||||
- test_related_apps.txt
|
|
||||||
- non_test_related_apps.txt
|
|
||||||
expire_in: 1 week
|
|
||||||
when: always
|
|
||||||
script:
|
|
||||||
- run_cmd idf-ci --debug gitlab build-child-pipeline
|
|
||||||
-p components
|
|
||||||
-p examples
|
|
||||||
-p tools/test_apps
|
|
||||||
--modified-files $MR_MODIFIED_FILES
|
|
||||||
|
|
||||||
build_child_pipeline:
|
|
||||||
stage: build
|
|
||||||
needs:
|
|
||||||
- job: fast_template_app
|
|
||||||
optional: true
|
|
||||||
artifacts: false
|
|
||||||
- pipeline_variables
|
|
||||||
- generate_build_child_pipeline
|
|
||||||
variables:
|
|
||||||
MR_MODIFIED_COMPONENTS: $MR_MODIFIED_COMPONENTS
|
|
||||||
MR_MODIFIED_FILES: $MR_MODIFIED_FILES
|
|
||||||
PARENT_PIPELINE_ID: $CI_PIPELINE_ID
|
|
||||||
# https://gitlab.com/gitlab-org/gitlab/-/issues/214340
|
|
||||||
inherit:
|
|
||||||
variables: false
|
|
||||||
trigger:
|
|
||||||
include:
|
|
||||||
- artifact: build_child_pipeline.yml
|
|
||||||
job: generate_build_child_pipeline
|
|
||||||
strategy: depend
|
|
||||||
|
|
||||||
generate_prebuild_report:
|
|
||||||
extends:
|
|
||||||
- .build_template
|
|
||||||
tags: [fast_run, shiny]
|
|
||||||
needs:
|
|
||||||
- pipeline_variables
|
|
||||||
artifacts:
|
|
||||||
paths:
|
|
||||||
- prebuild_report.html
|
|
||||||
expire_in: 1 week
|
|
||||||
when: always
|
|
||||||
script:
|
|
||||||
- run_cmd idf-ci build collect --format html -o prebuild_report.html
|
|
||||||
- echo "Report generated at https://${CI_PAGES_HOSTNAME}:${CI_SERVER_PORT}/-/esp-idf/-/jobs/${CI_JOB_ID}/artifacts/prebuild_report.html"
|
|
||||||
|
|||||||
@@ -14,6 +14,15 @@ stages:
|
|||||||
- deploy
|
- deploy
|
||||||
- post_deploy
|
- post_deploy
|
||||||
|
|
||||||
|
include:
|
||||||
|
- project: "ci/actions/common"
|
||||||
|
ref: "v2"
|
||||||
|
file: "templates/idf/common-scripts.yml"
|
||||||
|
inputs:
|
||||||
|
ci_python_constraint_file: "espidf.constraints.v5.5.txt"
|
||||||
|
macos_image: "registry.hub.docker.com/espressif/macos-idf:sequoia-v5.5"
|
||||||
|
windows_image: "windows_11_idf_v5_5"
|
||||||
|
|
||||||
variables:
|
variables:
|
||||||
# System environment
|
# System environment
|
||||||
|
|
||||||
@@ -58,30 +67,29 @@ variables:
|
|||||||
ESP_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v5.5:1"
|
ESP_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v5.5:1"
|
||||||
ESP_IDF_DOC_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v5.5-docs:1"
|
ESP_IDF_DOC_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v5.5-docs:1"
|
||||||
TARGET_TEST_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v5.5-target-test:1"
|
TARGET_TEST_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v5.5-target-test:1"
|
||||||
SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:5"
|
|
||||||
|
|
||||||
# Set this variable to the branch of idf-constraints repo in order to test a custom Python constraint file. The
|
# Set this variable to the branch of idf-constraints repo in order to test a custom Python constraint file. The
|
||||||
# branch name must be without the remote part ("origin/"). Keep the variable empty in order to use the constraint
|
# branch name must be without the remote part ("origin/"). Keep the variable empty in order to use the constraint
|
||||||
# file from https://dl.espressif.com/dl/esp-idf.
|
# file from https://dl.espressif.com/dl/esp-idf.
|
||||||
CI_PYTHON_CONSTRAINT_BRANCH: ""
|
# CI_PYTHON_CONSTRAINT_BRANCH could be defined as component input `ci_python_constraint_branch` in common-scripts above
|
||||||
|
|
||||||
# Update the filename for a specific ESP-IDF release. It is used only with CI_PYTHON_CONSTRAINT_BRANCH.
|
# Update the filename for a specific ESP-IDF release. It is used only with CI_PYTHON_CONSTRAINT_BRANCH.
|
||||||
CI_PYTHON_CONSTRAINT_FILE: "espidf.constraints.v5.5.txt"
|
# CI_PYTHON_CONSTRAINT_FILE is defined as component input `ci_python_constraint_file` in common-scripts above
|
||||||
|
|
||||||
# Set this variable to repository name of a Python tool you wish to install and test in the context of ESP-IDF CI.
|
# Set this variable to repository name of a Python tool you wish to install and test in the context of ESP-IDF CI.
|
||||||
# Keep the variable empty when not used.
|
# Keep the variable empty when not used.
|
||||||
CI_PYTHON_TOOL_REPO: ""
|
# CI_PYTHON_TOOL_REPO could be defined as component input `ci_python_tool_repo` in common-scripts above
|
||||||
|
|
||||||
# Set this variable to the branch of a Python tool repo specified in CI_PYTHON_TOOL_REPO. The
|
# Set this variable to the branch of a Python tool repo specified in CI_PYTHON_TOOL_REPO. The
|
||||||
# branch name must be without the remote part ("origin/"). Keep the variable empty when not used.
|
# branch name must be without the remote part ("origin/"). Keep the variable empty when not used.
|
||||||
# This is used only if CI_PYTHON_TOOL_REPO is not empty.
|
# This is used only if CI_PYTHON_TOOL_REPO is not empty.
|
||||||
CI_PYTHON_TOOL_BRANCH: ""
|
# CI_PYTHON_TOOL_BRANCH could be defined as component input `ci_python_tool_branch` in common-scripts above
|
||||||
|
|
||||||
# Set this variable to Clang toolchain distro URL to be used.
|
# Set this variable to Clang toolchain distro URL to be used.
|
||||||
# NOTE: We have separate toolchains for Xtensa and RISCV, therefore jobs for one arch will fail.
|
# NOTE: We have separate toolchains for Xtensa and RISCV, therefore jobs for one arch will fail.
|
||||||
# This is OK as far as we use CI_CLANG_DISTRO_URL for pre-release tests purposes only.
|
# This is OK as far as we use CI_CLANG_DISTRO_URL for pre-release tests purposes only.
|
||||||
# Keep the variable empty when not used.
|
# Keep the variable empty when not used.
|
||||||
CI_CLANG_DISTRO_URL: ""
|
# CI_CLANG_DISTRO_URL could be defined as component input `ci_clang_distro_url` in common-scripts above
|
||||||
|
|
||||||
# Set this variable to specify the file name for the known failure cases.
|
# Set this variable to specify the file name for the known failure cases.
|
||||||
KNOWN_FAILURE_CASES_FILE_NAME: "5.5.txt"
|
KNOWN_FAILURE_CASES_FILE_NAME: "5.5.txt"
|
||||||
@@ -103,227 +111,19 @@ variables:
|
|||||||
################################################
|
################################################
|
||||||
# `before_script` and `after_script` Templates #
|
# `before_script` and `after_script` Templates #
|
||||||
################################################
|
################################################
|
||||||
.common_before_scripts: &common-before_scripts |
|
|
||||||
source tools/ci/utils.sh
|
|
||||||
|
|
||||||
is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
|
|
||||||
|
|
||||||
if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
|
|
||||||
export IDF_MIRROR_PREFIX_MAP=
|
|
||||||
fi
|
|
||||||
|
|
||||||
if echo "$CI_MERGE_REQUEST_LABELS" | egrep "(^|,)include_nightly_run(,|$)"; then
|
|
||||||
export INCLUDE_NIGHTLY_RUN="1"
|
|
||||||
export NIGHTLY_RUN="1"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# configure cmake related flags
|
|
||||||
source tools/ci/configure_ci_environment.sh
|
|
||||||
|
|
||||||
if [[ "$CI_CCACHE_STATS" == 1 ]] && command -v ccache >/dev/null 2>&1 && [[ -n "$CCACHE_STATSLOG" ]]; then
|
|
||||||
mkdir -p "$(dirname "$CCACHE_STATSLOG")"
|
|
||||||
rm -f "$CCACHE_STATSLOG"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# add extra python packages
|
|
||||||
export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
|
|
||||||
|
|
||||||
.setup_tools_and_idf_python_venv: &setup_tools_and_idf_python_venv |
|
|
||||||
# must use after setup_tools_except_target_test
|
|
||||||
# otherwise the export.sh won't work properly
|
|
||||||
|
|
||||||
# download constraint file for dev
|
|
||||||
if [[ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]]; then
|
|
||||||
wget -O /tmp/constraint.txt --header="Authorization:Bearer ${ESPCI_TOKEN}" "${GITLAB_HTTP_SERVER}/api/v4/projects/2581/repository/files/${CI_PYTHON_CONSTRAINT_FILE}/raw?ref=${CI_PYTHON_CONSTRAINT_BRANCH}"
|
|
||||||
mkdir -p ~/.espressif
|
|
||||||
mv /tmp/constraint.txt ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Mirror
|
|
||||||
if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
|
|
||||||
export IDF_MIRROR_PREFIX_MAP=
|
|
||||||
fi
|
|
||||||
# Optimize pip install
|
|
||||||
if echo "${CI_RUNNER_TAGS}" | grep "shiny"; then
|
|
||||||
export PIP_INDEX_URL="${PIP_INDEX_URL_SHINY}"
|
|
||||||
fi
|
|
||||||
if [[ "$(uname -m)" == "x86_64" ]] || [[ "$(uname -m)" == "aarch64" ]]; then
|
|
||||||
export IDF_PIP_WHEELS_URL=""
|
|
||||||
fi
|
|
||||||
|
|
||||||
# install.sh
|
|
||||||
if [[ "${CI_JOB_STAGE}" != "target_test" ]]; then
|
|
||||||
section_start "running_install_sh" "Running install.sh"
|
|
||||||
if [[ "${CI_JOB_STAGE}" == "build_doc" || "${CI_JOB_GROUP_NAME}" == *docs* ]]; then
|
|
||||||
run_cmd bash install.sh --enable-ci --enable-docs
|
|
||||||
else
|
|
||||||
run_cmd bash install.sh --enable-ci
|
|
||||||
fi
|
|
||||||
section_end "running_install_sh"
|
|
||||||
else
|
|
||||||
section_start "install_python_env" "Install Python environment, skip required tools check"
|
|
||||||
run_cmd python tools/idf_tools.py install-python-env --features ci,test-specific
|
|
||||||
export IDF_SKIP_TOOLS_CHECK=1
|
|
||||||
section_end "install_python_env"
|
|
||||||
fi
|
|
||||||
|
|
||||||
section_start "source_export" "Source export.sh"
|
|
||||||
source ./export.sh
|
|
||||||
section_end "source_export"
|
|
||||||
|
|
||||||
# Eager upgrade of CI dependencies
|
|
||||||
# Done after sourcing export.sh so that we could easily invoke the right pip
|
|
||||||
section_start "upgrade_ci_dependencies" "Upgrading CI dependencies"
|
|
||||||
pip install --upgrade --upgrade-strategy=eager -r $IDF_PATH/tools/requirements/requirements.ci.txt -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
|
|
||||||
# we need the latest DEV release for esptool to be installed in every job without bumping the minimal requirement in
|
|
||||||
# the constraint file.
|
|
||||||
pip install --upgrade --pre esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
|
|
||||||
section_end "upgrade_ci_dependencies"
|
|
||||||
|
|
||||||
REEXPORT_NEEDED=0
|
|
||||||
if [[ ! -z "$INSTALL_EXTRA_TOOLS" ]]; then
|
|
||||||
section_start "installing_optional_tools" "Install optional tools ${INSTALL_EXTRA_TOOLS}"
|
|
||||||
run_cmd $IDF_PATH/tools/idf_tools.py --non-interactive install $INSTALL_EXTRA_TOOLS
|
|
||||||
section_end "installing_optional_tools"
|
|
||||||
|
|
||||||
REEXPORT_NEEDED=1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Install esp-clang if necessary (esp-clang is separately installed)
|
|
||||||
if [[ "$IDF_TOOLCHAIN" == "clang" && -z "$CI_CLANG_DISTRO_URL" ]]; then
|
|
||||||
$IDF_PATH/tools/idf_tools.py --non-interactive install esp-clang
|
|
||||||
|
|
||||||
REEXPORT_NEEDED=1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ $REEXPORT_NEEDED -eq 1 ]]; then
|
|
||||||
section_start "re_source_export" "Re-source export.sh"
|
|
||||||
source ./export.sh
|
|
||||||
section_end "re_source_export"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Custom clang toolchain
|
|
||||||
if [[ "$IDF_TOOLCHAIN" == "clang" && ! -z "$CI_CLANG_DISTRO_URL" ]]; then
|
|
||||||
echo "Using custom clang from ${CI_CLANG_DISTRO_URL}"
|
|
||||||
wget $CI_CLANG_DISTRO_URL
|
|
||||||
ARCH_NAME=$(basename $CI_CLANG_DISTRO_URL)
|
|
||||||
tar -x -f $ARCH_NAME
|
|
||||||
export PATH=$PWD/esp-clang/bin:$PATH
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Custom OpenOCD
|
|
||||||
if [[ "$CI_JOB_STAGE" == "target_test" ]]; then
|
|
||||||
machine="$(uname -m)"
|
|
||||||
if [[ "$machine" == "armv7l" ]] ; then
|
|
||||||
OOCD_DISTRO_URL="$OOCD_DISTRO_URL_ARMHF"
|
|
||||||
elif [[ "$machine" == "aarch64" ]] ; then
|
|
||||||
OOCD_DISTRO_URL="$OOCD_DISTRO_URL_ARM64"
|
|
||||||
fi
|
|
||||||
if [[ ! -z "$OOCD_DISTRO_URL" ]]; then
|
|
||||||
echo "Using custom OpenOCD from ${OOCD_DISTRO_URL}"
|
|
||||||
wget $OOCD_DISTRO_URL
|
|
||||||
ARCH_NAME=$(basename $OOCD_DISTRO_URL)
|
|
||||||
tar -x -f $ARCH_NAME
|
|
||||||
export OPENOCD_SCRIPTS=$PWD/openocd-esp32/share/openocd/scripts
|
|
||||||
export PATH=$PWD/openocd-esp32/bin:$PATH
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ -n "$CI_PYTHON_TOOL_REPO" ]]; then
|
|
||||||
git clone --quiet --depth=1 -b ${CI_PYTHON_TOOL_BRANCH} https://gitlab-ci-token:${ESPCI_TOKEN}@${GITLAB_HTTPS_HOST}/espressif/${CI_PYTHON_TOOL_REPO}.git
|
|
||||||
pip install ./${CI_PYTHON_TOOL_REPO}
|
|
||||||
rm -rf ${CI_PYTHON_TOOL_REPO}
|
|
||||||
fi
|
|
||||||
|
|
||||||
info "setup tools and python venv done"
|
|
||||||
|
|
||||||
.show_ccache_statistics: &show_ccache_statistics |
|
|
||||||
# Show ccache statistics if enabled globally
|
|
||||||
section_start "ccache_show_stats" "Show ccache statistics"
|
|
||||||
if [[ "$CI_CCACHE_STATS" == 1 ]] && command -v ccache >/dev/null 2>&1; then
|
|
||||||
if ccache --help 2>/dev/null | grep -q -- '--show-log-stats'; then
|
|
||||||
if [[ -n "$CCACHE_STATSLOG" && -f "$CCACHE_STATSLOG" ]]; then
|
|
||||||
ccache --show-log-stats -vv
|
|
||||||
else
|
|
||||||
echo "INFO: No per-job ccache statistics were recorded"
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
ccache --show-stats -vv
|
|
||||||
fi
|
|
||||||
fi || true
|
|
||||||
section_end "ccache_show_stats"
|
|
||||||
|
|
||||||
.upload_failed_job_log_artifacts: &upload_failed_job_log_artifacts |
|
|
||||||
if [ $CI_JOB_STATUS = "failed" ]; then
|
|
||||||
run_cmd idf-ci gitlab upload-artifacts --type log
|
|
||||||
fi
|
|
||||||
|
|
||||||
.before_script:minimal:
|
.before_script:minimal:
|
||||||
before_script:
|
before_script:
|
||||||
- *common-before_scripts
|
- !reference [.common_before_scripts, minimal]
|
||||||
|
|
||||||
.after_script:build:
|
.after_script:build:
|
||||||
after_script:
|
after_script:
|
||||||
- source tools/ci/utils.sh
|
- !reference [.common_after_scripts, ccache]
|
||||||
- source tools/ci/configure_ci_environment.sh
|
|
||||||
- *show_ccache_statistics
|
|
||||||
- run_cmd idf-ci gitlab upload-artifacts --type junit
|
- run_cmd idf-ci gitlab upload-artifacts --type junit
|
||||||
- |
|
- |
|
||||||
if [ $CI_JOB_STATUS = "failed" ]; then
|
if [ $CI_JOB_STATUS = "failed" ]; then
|
||||||
run_cmd idf-ci gitlab upload-artifacts --type log
|
run_cmd idf-ci gitlab upload-artifacts --type log
|
||||||
fi
|
fi
|
||||||
|
|
||||||
.brew-macos-settings:
|
|
||||||
variables:
|
|
||||||
GIT_STRATEGY: fetch
|
|
||||||
IDF_CCACHE_ENABLE: "0"
|
|
||||||
CCACHE_DIR: "/var/tmp/cache/idf_ccache"
|
|
||||||
tags:
|
|
||||||
- macos-tart
|
|
||||||
image: macos-sequoia-idf-v5.5
|
|
||||||
after_script: [] # ccache now is disabled for macos brew runners
|
|
||||||
timeout: 30m
|
|
||||||
|
|
||||||
.brew-windows-settings:
|
|
||||||
image: windows_11_idf_v6_0
|
|
||||||
tags: [windows-vm, brew]
|
|
||||||
variables:
|
|
||||||
GIT_STRATEGY: fetch
|
|
||||||
LC_ALL: C.UTF-8
|
|
||||||
# git clean at the final cleanup stage stuck on Windows VM
|
|
||||||
# a new VM for each job anyway, so we can skip cleaning the workspace
|
|
||||||
GIT_CLEAN_FLAGS: none
|
|
||||||
# IDF_CCACHE_ENABLE: "1"
|
|
||||||
# CCACHE_DIR: "C:/cache/idf_ccache"
|
|
||||||
before_script:
|
|
||||||
- if ($env:IDF_DONT_USE_MIRRORS) { $env:IDF_MIRROR_PREFIX_MAP = '' }
|
|
||||||
- ./install.ps1 --enable-ci
|
|
||||||
- ./export.ps1
|
|
||||||
- $env:PYTHONPATH = "$env:PYTHONPATH;$env:IDF_PATH\tools;$env:IDF_PATH\tools\esp_app_trace;$env:IDF_PATH\components\partition_table;$env:IDF_PATH\tools\ci\python_packages"
|
|
||||||
- python "${SUBMODULE_FETCH_TOOL}" -s "${SUBMODULES_TO_FETCH}"
|
|
||||||
# Sync VM clock. net start/stop exit 2 when the service is already in that state; do not fail CI.
|
|
||||||
- |
|
|
||||||
function Invoke-NetService($Action, $Name) {
|
|
||||||
$output = net $Action $Name 2>&1
|
|
||||||
if ($LASTEXITCODE -eq 2) {
|
|
||||||
Write-Host "net $Action $Name already in requested state: $output"
|
|
||||||
$global:LASTEXITCODE = 0
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if ($LASTEXITCODE -ne 0) {
|
|
||||||
Write-Host $output
|
|
||||||
exit $LASTEXITCODE
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Invoke-NetService start w32time
|
|
||||||
w32tm /config /manualpeerlist:"time.google.com,0x9 pool.ntp.org,0x9" /syncfromflags:manual /reliable:yes /update
|
|
||||||
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
|
|
||||||
Invoke-NetService stop w32time
|
|
||||||
Invoke-NetService start w32time
|
|
||||||
w32tm /resync
|
|
||||||
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
|
|
||||||
after_script: []
|
|
||||||
|
|
||||||
#############
|
#############
|
||||||
# `default` #
|
# `default` #
|
||||||
#############
|
#############
|
||||||
@@ -338,8 +138,7 @@ default:
|
|||||||
- .cache/submodule_archives
|
- .cache/submodule_archives
|
||||||
policy: pull
|
policy: pull
|
||||||
before_script:
|
before_script:
|
||||||
- *common-before_scripts
|
- !reference [.common_before_scripts, with_tools]
|
||||||
- *setup_tools_and_idf_python_venv
|
|
||||||
- add_gitlab_ssh_keys
|
- add_gitlab_ssh_keys
|
||||||
- fetch_submodules
|
- fetch_submodules
|
||||||
- export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
|
- export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
|
||||||
|
|||||||
@@ -1,21 +0,0 @@
|
|||||||
# this file support two keywords:
|
|
||||||
# - extra_default_build_targets:
|
|
||||||
# besides of the SUPPORTED_TARGETS in IDF,
|
|
||||||
# enable build for the specified targets by default as well.
|
|
||||||
# !!! DEPRECATED: use `additional_build_targets` in .idf_build_apps.toml instead
|
|
||||||
#
|
|
||||||
# - bypass_check_test_targets:
|
|
||||||
# suppress the check_build_test_rules check-test-script warnings for the specified targets
|
|
||||||
#
|
|
||||||
# This file should ONLY be used during bringup. Should be reset to empty after the bringup process
|
|
||||||
|
|
||||||
bypass_check_test_targets:
|
|
||||||
- esp32h21
|
|
||||||
- esp32h4
|
|
||||||
- esp32c5
|
|
||||||
|
|
||||||
#
|
|
||||||
# These lines would
|
|
||||||
# - enable the README.md check for esp32c6. Don't forget to add the build jobs in .gitlab/ci/build.yml
|
|
||||||
# - disable the test script check with the manifest file.
|
|
||||||
#
|
|
||||||
@@ -401,7 +401,7 @@ test_pytest_linux:
|
|||||||
test_pytest_macos:
|
test_pytest_macos:
|
||||||
extends:
|
extends:
|
||||||
- .host_test_template
|
- .host_test_template
|
||||||
- .brew-macos-settings
|
- .macos-settings
|
||||||
artifacts:
|
artifacts:
|
||||||
paths:
|
paths:
|
||||||
- XUNIT_RESULT.xml
|
- XUNIT_RESULT.xml
|
||||||
@@ -476,6 +476,8 @@ test_pytest_macos:
|
|||||||
- pytest
|
- pytest
|
||||||
-k cmake
|
-k cmake
|
||||||
--cleanup-idf-copy
|
--cleanup-idf-copy
|
||||||
|
--parallel-count ${CI_NODE_TOTAL:-1}
|
||||||
|
--parallel-index ${CI_NODE_INDEX:-1}
|
||||||
--work-dir ${CI_PROJECT_DIR}/test_build_system
|
--work-dir ${CI_PROJECT_DIR}/test_build_system
|
||||||
--junitxml ${CI_PROJECT_DIR}/XUNIT_RESULT.xml
|
--junitxml ${CI_PROJECT_DIR}/XUNIT_RESULT.xml
|
||||||
--ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
|
--ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
|
||||||
@@ -487,7 +489,7 @@ pytest_build_system:
|
|||||||
pytest_build_system_macos:
|
pytest_build_system_macos:
|
||||||
extends:
|
extends:
|
||||||
- .test_build_system_template
|
- .test_build_system_template
|
||||||
- .brew-macos-settings
|
- .macos-settings
|
||||||
- .rules:build:macos
|
- .rules:build:macos
|
||||||
parallel: 3
|
parallel: 3
|
||||||
|
|
||||||
@@ -497,7 +499,7 @@ pytest_build_system_minimal_cmake:
|
|||||||
pytest_build_system_macos_minimal_cmake:
|
pytest_build_system_macos_minimal_cmake:
|
||||||
extends:
|
extends:
|
||||||
- .test_build_system_minimal_cmake_template
|
- .test_build_system_minimal_cmake_template
|
||||||
- .brew-macos-settings
|
- .macos-settings
|
||||||
- .rules:build:macos
|
- .rules:build:macos
|
||||||
variables:
|
variables:
|
||||||
INSTALL_EXTRA_TOOLS: ninja cmake@3.16.3
|
INSTALL_EXTRA_TOOLS: ninja cmake@3.16.3
|
||||||
|
|||||||
@@ -1,9 +1,3 @@
|
|||||||
.pre_check_template:
|
|
||||||
stage: pre_check
|
|
||||||
image: $ESP_ENV_IMAGE
|
|
||||||
tags: [build, shiny]
|
|
||||||
dependencies: # set dependencies to null to avoid missing artifacts issue
|
|
||||||
|
|
||||||
check_version:
|
check_version:
|
||||||
# Don't run this for feature/bugfix branches, so that it is possible to modify
|
# Don't run this for feature/bugfix branches, so that it is possible to modify
|
||||||
# esp_idf_version.h in a branch before tagging the next version.
|
# esp_idf_version.h in a branch before tagging the next version.
|
||||||
@@ -140,106 +134,11 @@ check_submodule_sync:
|
|||||||
- git submodule update --recursive
|
- git submodule update --recursive
|
||||||
- echo "IDF was cloned from ${PUBLIC_IDF_URL} completely"
|
- echo "IDF was cloned from ${PUBLIC_IDF_URL} completely"
|
||||||
|
|
||||||
pipeline_variables:
|
check_gitlab_yaml:
|
||||||
extends:
|
extends:
|
||||||
- .pre_check_template
|
- .pre_check_template
|
||||||
tags: [fast_run, shiny]
|
|
||||||
variables:
|
|
||||||
GIT_STRATEGY: "fetch"
|
|
||||||
script:
|
script:
|
||||||
- |
|
- python tools/ci/gitlab_yaml_linter.py
|
||||||
# Store the diff output in a temporary file
|
|
||||||
TEMP_FILE=$(mktemp)
|
|
||||||
# merged results pipelines, by default
|
|
||||||
if [[ -n $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ]]; then
|
|
||||||
git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA
|
|
||||||
git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
|
|
||||||
|
|
||||||
git diff --name-only $CI_MERGE_REQUEST_TARGET_BRANCH_SHA...$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA > "$TEMP_FILE"
|
|
||||||
GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
|
|
||||||
git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
|
|
||||||
# merge request pipelines, when the mr got conflicts
|
|
||||||
elif [[ -n $CI_MERGE_REQUEST_DIFF_BASE_SHA ]]; then
|
|
||||||
git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
|
|
||||||
git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
|
|
||||||
git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_COMMIT_SHA > "$TEMP_FILE"
|
|
||||||
GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
|
|
||||||
# other pipelines, like the protected branches pipelines
|
|
||||||
elif [[ "$CI_COMMIT_BEFORE_SHA" != "0000000000000000000000000000000000000000" ]]; then
|
|
||||||
git fetch origin $CI_COMMIT_BEFORE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
|
|
||||||
git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS}
|
|
||||||
git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA > "$TEMP_FILE"
|
|
||||||
GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
|
|
||||||
else
|
|
||||||
# pipeline source could be web, scheduler, etc.
|
|
||||||
git fetch origin $CI_COMMIT_SHA --depth=2 ${GIT_FETCH_EXTRA_FLAGS}
|
|
||||||
git diff --name-only $CI_COMMIT_SHA~1 $CI_COMMIT_SHA > "$TEMP_FILE"
|
|
||||||
GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE")
|
|
||||||
fi
|
|
||||||
# MODIFIED_FILES is a list of files that changed, could be used everywhere
|
|
||||||
- MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | xargs)
|
|
||||||
- echo "MODIFIED_FILES=$MODIFIED_FILES" >> pipeline.env
|
|
||||||
# MR_MODIFIED_FILES and MR_MODIFIED_COMPONENTS are semicolon separated lists that is used in MR only
|
|
||||||
# for non MR pipeline, these are empty lists
|
|
||||||
- |
|
|
||||||
if [ -z "$CI_MERGE_REQUEST_IID" ]; then
|
|
||||||
echo "MR_MODIFIED_FILES=\"\"" >> pipeline.env
|
|
||||||
echo "MR_MODIFIED_COMPONENTS=\"\"" >> pipeline.env
|
|
||||||
else
|
|
||||||
MR_MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | tr '\n' ';')
|
|
||||||
echo "MR_MODIFIED_FILES=\"$MR_MODIFIED_FILES\"" >> pipeline.env
|
|
||||||
|
|
||||||
MR_MODIFIED_COMPONENTS=$(run_cmd python tools/ci/ci_get_mr_info.py components --modified-files $MODIFIED_FILES | tr '\n' ';')
|
|
||||||
echo "MR_MODIFIED_COMPONENTS=\"$MR_MODIFIED_COMPONENTS\"" >> pipeline.env
|
|
||||||
fi
|
|
||||||
- |
|
|
||||||
if [ -n "$PIPELINE_COMMIT_SHA" ]; then
|
|
||||||
echo "PIPELINE_COMMIT_SHA=$PIPELINE_COMMIT_SHA" >> pipeline.env
|
|
||||||
fi
|
|
||||||
- echo "OOCD_DISTRO_URL_ARMHF=$OOCD_DISTRO_URL_ARMHF" >> pipeline.env
|
|
||||||
- echo "OOCD_DISTRO_URL_ARM64=$OOCD_DISTRO_URL_ARM64" >> pipeline.env
|
|
||||||
- run_cmd idf-ci gitlab pipeline-variables >> pipeline.env
|
|
||||||
- cat pipeline.env
|
|
||||||
- run_cmd idf-ci gitlab upload-artifacts --type env
|
|
||||||
artifacts:
|
|
||||||
reports:
|
|
||||||
dotenv: pipeline.env
|
|
||||||
paths:
|
|
||||||
- pipeline.env
|
|
||||||
expire_in: 1 week
|
|
||||||
when: always
|
|
||||||
|
|
||||||
baseline_manifest_sha:
|
|
||||||
extends:
|
|
||||||
- .pre_check_template
|
|
||||||
- .rules:dev-push
|
|
||||||
tags: [fast_run, shiny]
|
|
||||||
script:
|
|
||||||
- |
|
|
||||||
# merged results pipelines, by default
|
|
||||||
# diff between target-branch-head and merged-result-head
|
|
||||||
if [ -n "$CI_MERGE_REQUEST_TARGET_BRANCH_SHA" ]; then
|
|
||||||
git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA --depth=1
|
|
||||||
git checkout FETCH_HEAD
|
|
||||||
idf-build-apps dump-manifest-sha \
|
|
||||||
--manifest-files $(find . -name ".build-test-rules.yml" | xargs) \
|
|
||||||
--output .manifest_sha
|
|
||||||
# merge request pipelines, when the mr got conflicts
|
|
||||||
# diff between diff-base-sha and merge-request-head
|
|
||||||
elif [ -n "$CI_MERGE_REQUEST_DIFF_BASE_SHA" ]; then
|
|
||||||
git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1
|
|
||||||
git checkout FETCH_HEAD
|
|
||||||
idf-build-apps dump-manifest-sha \
|
|
||||||
--manifest-files $(find . -name ".build-test-rules.yml" | xargs) \
|
|
||||||
--output .manifest_sha
|
|
||||||
# other pipelines, like the protected branches pipelines
|
|
||||||
# not triggered in this job
|
|
||||||
fi
|
|
||||||
artifacts:
|
|
||||||
paths:
|
|
||||||
- .manifest_sha
|
|
||||||
expire_in: 1 week
|
|
||||||
when: always
|
|
||||||
|
|
||||||
retry_failed_jobs:
|
retry_failed_jobs:
|
||||||
extends:
|
extends:
|
||||||
|
|||||||
@@ -14,13 +14,6 @@
|
|||||||
- "components/**/CMakeLists.txt"
|
- "components/**/CMakeLists.txt"
|
||||||
|
|
||||||
|
|
||||||
.patterns-python-files: &patterns-python-files
|
|
||||||
- ".gitlab/ci/static-code-analysis.yml"
|
|
||||||
- "**/*.py"
|
|
||||||
|
|
||||||
.patterns-sonarqube-files: &patterns-sonarqube-files
|
|
||||||
- "tools/ci/sonar_exclude_list.txt"
|
|
||||||
|
|
||||||
.patterns-build_components: &patterns-build_components
|
.patterns-build_components: &patterns-build_components
|
||||||
# components files except "test*/" "host*/" folders
|
# components files except "test*/" "host*/" folders
|
||||||
# ?? to include folders less than 4 characters
|
# ?? to include folders less than 4 characters
|
||||||
@@ -176,16 +169,10 @@
|
|||||||
.if-protected-check: &if-protected-check
|
.if-protected-check: &if-protected-check
|
||||||
if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) || $CI_COMMIT_TAG =~ /^qa-test/'
|
if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) || $CI_COMMIT_TAG =~ /^qa-test/'
|
||||||
|
|
||||||
.if-protected-deploy: &if-protected-deploy
|
|
||||||
if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "api")'
|
|
||||||
|
|
||||||
.if-protected-branch-push: &if-protected-branch-push
|
.if-protected-branch-push: &if-protected-branch-push
|
||||||
# rules:changes always evaluates to true for new branch pipelines or when there is no Git push event
|
# rules:changes always evaluates to true for new branch pipelines or when there is no Git push event
|
||||||
if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/) && $CI_PIPELINE_SOURCE == "push"'
|
if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/) && $CI_PIPELINE_SOURCE == "push"'
|
||||||
|
|
||||||
.if-qa-test-tag: &if-qa-test-tag
|
|
||||||
if: '$CI_COMMIT_TAG =~ /^qa-test/'
|
|
||||||
|
|
||||||
.if-dev-push: &if-dev-push
|
.if-dev-push: &if-dev-push
|
||||||
if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")'
|
if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")'
|
||||||
|
|
||||||
|
|||||||
@@ -30,84 +30,3 @@ gcc_static_analyzer:
|
|||||||
- export EXTRA_CFLAGS=
|
- export EXTRA_CFLAGS=
|
||||||
- export EXTRA_CXXFLAGS=
|
- export EXTRA_CXXFLAGS=
|
||||||
- idf-build-apps build -p ${ANALYZING_APP} --disable-targets esp32h4 esp32h21 --no-check-warnings
|
- idf-build-apps build -p ${ANALYZING_APP} --disable-targets esp32h4 esp32h21 --no-check-warnings
|
||||||
#
|
|
||||||
## build stage
|
|
||||||
## Sonarqube related jobs put here for this reason:
|
|
||||||
## Here we have two jobs. code_quality_check and code_quality_report.
|
|
||||||
##
|
|
||||||
## code_quality_check will analyze the code changes between your MR and
|
|
||||||
## code repo stored in sonarqube server. The analysis result is only shown in
|
|
||||||
## the comments under this MR and won't be transferred to the server.
|
|
||||||
##
|
|
||||||
## code_quality_report will analyze and transfer both of the newly added code
|
|
||||||
## and the analysis result to the server.
|
|
||||||
##
|
|
||||||
## Put in the front to ensure that the newly merged code can be stored in
|
|
||||||
## sonarqube server ASAP, in order to avoid reporting unrelated code issues
|
|
||||||
#.sonar_scan_template:
|
|
||||||
# stage: build
|
|
||||||
# extends: .pre_check_template
|
|
||||||
# # full clone since this image does not support fetch --shallow-since-cutoff
|
|
||||||
# # shiny runners are used for full clone
|
|
||||||
# tags: [build, shiny]
|
|
||||||
# image: $SONARQUBE_SCANNER_IMAGE
|
|
||||||
# before_script:
|
|
||||||
# - source tools/ci/utils.sh
|
|
||||||
# - export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH"
|
|
||||||
# - fetch_submodules
|
|
||||||
# # Exclude the submodules, all paths ends with /**
|
|
||||||
# - submodules=$(get_all_submodules)
|
|
||||||
# # get all exclude paths specified in tools/ci/sonar_exclude_list.txt | ignore lines start with # | xargs | replace all <space> to <comma>
|
|
||||||
# - custom_excludes=$(cat $CI_PROJECT_DIR/tools/ci/sonar_exclude_list.txt | grep -v '^#' | xargs | sed -e 's/ /,/g')
|
|
||||||
# # Exclude the report dir as well
|
|
||||||
# - export EXCLUSIONS="$custom_excludes,$submodules"
|
|
||||||
# - export SONAR_SCANNER_OPTS="-Xmx2048m"
|
|
||||||
# variables:
|
|
||||||
# GIT_DEPTH: 0
|
|
||||||
# REPORT_PATTERN: clang_tidy_reports/**/*.txt
|
|
||||||
# artifacts:
|
|
||||||
# paths:
|
|
||||||
# - $REPORT_PATTERN
|
|
||||||
# expire_in: 1 week
|
|
||||||
# when: always
|
|
||||||
# dependencies: # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here.
|
|
||||||
# - clang_tidy_check
|
|
||||||
#
|
|
||||||
#code_quality_check:
|
|
||||||
# extends:
|
|
||||||
# - .sonar_scan_template
|
|
||||||
# - .rules:patterns:static-code-analysis-preview
|
|
||||||
# allow_failure: true # it's using exit code to indicate the code analysis result,
|
|
||||||
# # we don't want to block ci when critical issues founded
|
|
||||||
# script:
|
|
||||||
# - export CI_MERGE_REQUEST_COMMITS=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py commits --src-branch ${CI_COMMIT_REF_NAME} | tr '\n' ',')
|
|
||||||
# # test if this branch have merge request, if not, exit 0
|
|
||||||
# - test -n "$CI_MERGE_REQUEST_IID" || exit 0
|
|
||||||
# - test -n "$CI_MERGE_REQUEST_COMMITS" || exit 0
|
|
||||||
# - sonar-scanner
|
|
||||||
# -Dsonar.analysis.mode=preview
|
|
||||||
# -Dsonar.branch.name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
|
|
||||||
# -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN
|
|
||||||
# -Dsonar.exclusions=$EXCLUSIONS
|
|
||||||
# -Dsonar.gitlab.ci_merge_request_iid=$CI_MERGE_REQUEST_IID
|
|
||||||
# -Dsonar.gitlab.commit_sha=$CI_MERGE_REQUEST_COMMITS
|
|
||||||
# -Dsonar.gitlab.merge_request_discussion=true
|
|
||||||
# -Dsonar.gitlab.ref_name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME
|
|
||||||
# -Dsonar.host.url=$SONAR_HOST_URL
|
|
||||||
# -Dsonar.login=$SONAR_LOGIN
|
|
||||||
#
|
|
||||||
#code_quality_report:
|
|
||||||
# extends:
|
|
||||||
# - .sonar_scan_template
|
|
||||||
# - .rules:protected:check
|
|
||||||
# allow_failure: true # it's using exit code to indicate the code analysis result,
|
|
||||||
# # we don't want to block ci when critical issues founded
|
|
||||||
# script:
|
|
||||||
# - sonar-scanner
|
|
||||||
# -Dsonar.branch.name=$CI_COMMIT_REF_NAME
|
|
||||||
# -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN
|
|
||||||
# -Dsonar.exclusions=$EXCLUSIONS
|
|
||||||
# -Dsonar.gitlab.commit_sha=$PIPELINE_COMMIT_SHA
|
|
||||||
# -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME
|
|
||||||
# -Dsonar.host.url=$SONAR_HOST_URL
|
|
||||||
# -Dsonar.login=$SONAR_LOGIN
|
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
.host_test_win_template:
|
.host_test_win_template:
|
||||||
extends:
|
extends:
|
||||||
- .brew-windows-settings
|
- .windows-settings
|
||||||
- .rules:test:host_test
|
- .rules:test:host_test
|
||||||
stage: host_test
|
stage: host_test
|
||||||
dependencies: # set dependencies to null to avoid missing artifacts issue
|
dependencies: # set dependencies to null to avoid missing artifacts issue
|
||||||
@@ -73,7 +73,7 @@ pytest_build_system_win:
|
|||||||
extends:
|
extends:
|
||||||
- .test_build_system_template_win
|
- .test_build_system_template_win
|
||||||
- .rules:labels:windows_pytest_build_system
|
- .rules:labels:windows_pytest_build_system
|
||||||
parallel: 2
|
parallel: 6
|
||||||
|
|
||||||
pytest_build_system_win_minimal_cmake:
|
pytest_build_system_win_minimal_cmake:
|
||||||
extends:
|
extends:
|
||||||
@@ -82,7 +82,7 @@ pytest_build_system_win_minimal_cmake:
|
|||||||
variables:
|
variables:
|
||||||
MINIMAL_CMAKE_VERSION: "3.16.3"
|
MINIMAL_CMAKE_VERSION: "3.16.3"
|
||||||
script:
|
script:
|
||||||
- python ${IDF_PATH}\tools\idf_tools.py install cmake@${MINIMAL_CMAKE_VERSION}
|
- python tools\idf_tools.py install cmake@${MINIMAL_CMAKE_VERSION}
|
||||||
- $Env:PATH = "$Env:USERPROFILE\.espressif\tools\cmake\${MINIMAL_CMAKE_VERSION}\bin;$Env:PATH"
|
- $Env:PATH = "$Env:USERPROFILE\.espressif\tools\cmake\${MINIMAL_CMAKE_VERSION}\bin;$Env:PATH"
|
||||||
- |
|
- |
|
||||||
$actualVersion = (& cmake --version).Split()[2]
|
$actualVersion = (& cmake --version).Split()[2]
|
||||||
@@ -90,6 +90,6 @@ pytest_build_system_win_minimal_cmake:
|
|||||||
Write-Error "ERROR: Wrong CMake version! Detected: $actualVersion, but expected: $Env:MINIMAL_CMAKE_VERSION"
|
Write-Error "ERROR: Wrong CMake version! Detected: $actualVersion, but expected: $Env:MINIMAL_CMAKE_VERSION"
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
- cd ${IDF_PATH}\tools\test_build_system
|
- cd tools\test_build_system
|
||||||
- idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
|
- idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
|
||||||
- pytest -k cmake --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
|
- pytest -k cmake --junitxml ${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
|
||||||
|
|||||||
@@ -146,7 +146,7 @@ repos:
|
|||||||
require_serial: true
|
require_serial: true
|
||||||
additional_dependencies:
|
additional_dependencies:
|
||||||
- PyYAML == 5.3.1
|
- PyYAML == 5.3.1
|
||||||
- idf-build-apps~=2.13
|
- idf-build-apps~=2.16
|
||||||
- id: sort-yaml-files
|
- id: sort-yaml-files
|
||||||
name: sort yaml files
|
name: sort yaml files
|
||||||
entry: tools/ci/sort_yaml.py
|
entry: tools/ci/sort_yaml.py
|
||||||
@@ -167,14 +167,6 @@ repos:
|
|||||||
language: python
|
language: python
|
||||||
always_run: true
|
always_run: true
|
||||||
require_serial: true
|
require_serial: true
|
||||||
- id: gitlab-yaml-linter
|
|
||||||
name: Check gitlab yaml files
|
|
||||||
entry: tools/ci/gitlab_yaml_linter.py
|
|
||||||
language: python
|
|
||||||
files: '\.gitlab-ci\.yml|\.gitlab/ci/.+\.yml|\.gitmodules'
|
|
||||||
pass_filenames: false
|
|
||||||
additional_dependencies:
|
|
||||||
- PyYAML == 5.3.1
|
|
||||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||||
rev: v4.0.1
|
rev: v4.0.1
|
||||||
hooks:
|
hooks:
|
||||||
|
|||||||
@@ -598,7 +598,7 @@ def pytest_unconfigure(config: Config) -> None:
|
|||||||
config.pluginmanager.unregister(idf_local_plugin)
|
config.pluginmanager.unregister(idf_local_plugin)
|
||||||
|
|
||||||
|
|
||||||
dut_artifacts_url = []
|
dut_artifacts_url: t.List[str] = []
|
||||||
|
|
||||||
|
|
||||||
@pytest.hookimpl(hookwrapper=True)
|
@pytest.hookimpl(hookwrapper=True)
|
||||||
|
|||||||
@@ -1,24 +0,0 @@
|
|||||||
sonar.projectKey=esp-idf
|
|
||||||
sonar.projectName=Espressif IoT Development Framework
|
|
||||||
sonar.projectVersion=4.4
|
|
||||||
|
|
||||||
sonar.sources=.
|
|
||||||
sonar.sourceEncoding=UTF-8
|
|
||||||
|
|
||||||
sonar.inclusions=\
|
|
||||||
**/*.c,**/*.cpp,**/*.h,**/*.hpp,\
|
|
||||||
**/*.py
|
|
||||||
|
|
||||||
####################
|
|
||||||
# sonar-cxx-plugin #
|
|
||||||
####################
|
|
||||||
|
|
||||||
sonar.cxx.includeDirectories=/usr/include
|
|
||||||
sonar.cxx.file.suffixes=.cxx,.cpp,.cc,.c,.hxx,.hpp,.hh,.h
|
|
||||||
|
|
||||||
#######################
|
|
||||||
# sonar-gitlab-plugin #
|
|
||||||
#######################
|
|
||||||
|
|
||||||
sonar.gitlab.failure_notification_mode=exit-code
|
|
||||||
sonar.gitlab.project_id=103
|
|
||||||
@@ -9,7 +9,6 @@ import typing as t
|
|||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import yaml
|
|
||||||
from idf_build_apps import App
|
from idf_build_apps import App
|
||||||
from idf_build_apps import find_apps
|
from idf_build_apps import find_apps
|
||||||
from idf_build_apps.constants import SUPPORTED_TARGETS
|
from idf_build_apps.constants import SUPPORTED_TARGETS
|
||||||
@@ -40,6 +39,14 @@ USUAL_TO_FORMAL = {
|
|||||||
|
|
||||||
FORMAL_TO_USUAL = {v: k for k, v in USUAL_TO_FORMAL.items()}
|
FORMAL_TO_USUAL = {v: k for k, v in USUAL_TO_FORMAL.items()}
|
||||||
|
|
||||||
|
# Targets under bringup: suppress check-test-scripts warnings for these.
|
||||||
|
# Reset to [] once bringup is complete.
|
||||||
|
BYPASS_CHECK_TEST_TARGETS: t.List[str] = [
|
||||||
|
'esp32h21',
|
||||||
|
'esp32h4',
|
||||||
|
'esp32c5',
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def print_diff_table(
|
def print_diff_table(
|
||||||
list1: t.List[str], list2: t.List[str], title1: str, title2: str, excluded: t.Optional[t.List[str]] = None
|
list1: t.List[str], list2: t.List[str], title1: str, title2: str, excluded: t.Optional[t.List[str]] = None
|
||||||
@@ -299,21 +306,9 @@ if __name__ == '__main__':
|
|||||||
|
|
||||||
readme_parser = action.add_parser('check-readmes')
|
readme_parser = action.add_parser('check-readmes')
|
||||||
readme_parser.add_argument('paths', nargs='+', help='check under paths')
|
readme_parser.add_argument('paths', nargs='+', help='check under paths')
|
||||||
readme_parser.add_argument(
|
|
||||||
'-c',
|
|
||||||
'--config',
|
|
||||||
default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'),
|
|
||||||
help='config file',
|
|
||||||
)
|
|
||||||
|
|
||||||
test_parser = action.add_parser('check-test-scripts')
|
test_parser = action.add_parser('check-test-scripts')
|
||||||
test_parser.add_argument('paths', nargs='+', help='check under paths')
|
test_parser.add_argument('paths', nargs='+', help='check under paths')
|
||||||
test_parser.add_argument(
|
|
||||||
'-c',
|
|
||||||
'--config',
|
|
||||||
default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'),
|
|
||||||
help='config file',
|
|
||||||
)
|
|
||||||
arg = parser.parse_args()
|
arg = parser.parse_args()
|
||||||
|
|
||||||
check_dirs = set()
|
check_dirs = set()
|
||||||
@@ -344,14 +339,6 @@ if __name__ == '__main__':
|
|||||||
os.path.join(IDF_PATH, 'tools', 'cmakev2', 'test'),
|
os.path.join(IDF_PATH, 'tools', 'cmakev2', 'test'),
|
||||||
]
|
]
|
||||||
|
|
||||||
_bypass_targets: t.List[str] = []
|
|
||||||
if arg.config:
|
|
||||||
with open(arg.config) as fr:
|
|
||||||
configs = yaml.safe_load(fr)
|
|
||||||
|
|
||||||
if configs:
|
|
||||||
_bypass_targets = configs.get('bypass_check_test_targets') or []
|
|
||||||
|
|
||||||
os.environ.update(
|
os.environ.update(
|
||||||
{
|
{
|
||||||
'IDF_PATH': IDF_PATH,
|
'IDF_PATH': IDF_PATH,
|
||||||
@@ -369,5 +356,5 @@ if __name__ == '__main__':
|
|||||||
check_test_scripts(
|
check_test_scripts(
|
||||||
list(check_dirs),
|
list(check_dirs),
|
||||||
_exclude_dirs,
|
_exclude_dirs,
|
||||||
bypass_targets=_bypass_targets,
|
bypass_targets=BYPASS_CHECK_TEST_TARGETS,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,165 +0,0 @@
|
|||||||
#!/usr/bin/env python
|
|
||||||
#
|
|
||||||
# internal use only for CI
|
|
||||||
# get latest MR information by source branch
|
|
||||||
#
|
|
||||||
# SPDX-FileCopyrightText: 2020-2024 Espressif Systems (Shanghai) CO LTD
|
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
#
|
|
||||||
import argparse
|
|
||||||
import logging
|
|
||||||
import os
|
|
||||||
import subprocess
|
|
||||||
import typing as t
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from gitlab_api import Gitlab
|
|
||||||
from idf_ci_utils import IDF_PATH
|
|
||||||
|
|
||||||
if t.TYPE_CHECKING:
|
|
||||||
from gitlab.v4.objects import ProjectCommit, ProjectMergeRequest
|
|
||||||
|
|
||||||
|
|
||||||
def _get_mr_obj(source_branch: str) -> t.Optional['ProjectMergeRequest']:
|
|
||||||
gl = Gitlab(os.getenv('CI_PROJECT_ID', 'espressif/esp-idf'))
|
|
||||||
if not gl.project:
|
|
||||||
return None
|
|
||||||
|
|
||||||
mrs = gl.project.mergerequests.list(state='opened', source_branch=source_branch)
|
|
||||||
if mrs:
|
|
||||||
return mrs[0] # one source branch can only have one opened MR at one moment
|
|
||||||
else:
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def get_mr_iid(source_branch: str) -> str:
|
|
||||||
mr = _get_mr_obj(source_branch)
|
|
||||||
if not mr:
|
|
||||||
return ''
|
|
||||||
else:
|
|
||||||
return str(mr.iid)
|
|
||||||
|
|
||||||
|
|
||||||
def get_mr_changed_files(source_branch: str) -> t.List[str]:
|
|
||||||
mr = _get_mr_obj(source_branch)
|
|
||||||
if not mr:
|
|
||||||
return []
|
|
||||||
|
|
||||||
git_output = subprocess.check_output(
|
|
||||||
['git', 'diff', '--name-only', '--diff-filter=d', f'origin/{mr.target_branch}...origin/{source_branch}']
|
|
||||||
).decode('utf8')
|
|
||||||
|
|
||||||
return [line.strip() for line in git_output.splitlines() if line.strip()]
|
|
||||||
|
|
||||||
|
|
||||||
def get_mr_commits(source_branch: str) -> t.List['ProjectCommit']:
|
|
||||||
mr = _get_mr_obj(source_branch)
|
|
||||||
if not mr:
|
|
||||||
return []
|
|
||||||
|
|
||||||
return list(mr.commits())
|
|
||||||
|
|
||||||
|
|
||||||
_COMPONENT_NAME_DIR_RECORDS = {}
|
|
||||||
|
|
||||||
|
|
||||||
def get_modified_component(filepath: str) -> t.Optional[str]:
|
|
||||||
"""Return the component name if the file is in a component directory, otherwise None."""
|
|
||||||
try:
|
|
||||||
f_path = Path(filepath).resolve().relative_to(IDF_PATH)
|
|
||||||
except ValueError: # not in IDF_PATH
|
|
||||||
return None
|
|
||||||
|
|
||||||
# skip md files, etc.
|
|
||||||
if f_path.suffix in ['.md', '.yml']:
|
|
||||||
return None
|
|
||||||
|
|
||||||
# skip test_apps files
|
|
||||||
if 'test_apps' in f_path.parts:
|
|
||||||
return None
|
|
||||||
|
|
||||||
component_parent_dirs = [f_path.parts[0]]
|
|
||||||
for part in f_path.parts[1:]:
|
|
||||||
if component_parent_dirs[-1] == 'components' or component_parent_dirs[-1].endswith('common_components'):
|
|
||||||
if part not in _COMPONENT_NAME_DIR_RECORDS:
|
|
||||||
logging.debug('Found component "%s" in path "%s"' % (part, component_parent_dirs))
|
|
||||||
_COMPONENT_NAME_DIR_RECORDS[part] = component_parent_dirs
|
|
||||||
elif _COMPONENT_NAME_DIR_RECORDS.get(part) != component_parent_dirs:
|
|
||||||
logging.debug(
|
|
||||||
'WARNING!!! Found component "%s" in path "%s" and "%s"'
|
|
||||||
% (part, component_parent_dirs, _COMPONENT_NAME_DIR_RECORDS.get(part))
|
|
||||||
)
|
|
||||||
|
|
||||||
return part
|
|
||||||
|
|
||||||
component_parent_dirs.append(part)
|
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def get_mr_components(
|
|
||||||
source_branch: t.Optional[str] = None, modified_files: t.Optional[t.List[str]] = None
|
|
||||||
) -> t.List[str]:
|
|
||||||
components: t.Set[str] = set()
|
|
||||||
if modified_files is None:
|
|
||||||
if not source_branch:
|
|
||||||
raise RuntimeError('--src-branch is required if --modified-files is not provided')
|
|
||||||
|
|
||||||
modified_files = get_mr_changed_files(source_branch)
|
|
||||||
|
|
||||||
for f in modified_files:
|
|
||||||
modified_component = get_modified_component(f)
|
|
||||||
if modified_component:
|
|
||||||
components.add(modified_component)
|
|
||||||
|
|
||||||
return list(components)
|
|
||||||
|
|
||||||
|
|
||||||
def get_target_in_tags(tags: str) -> str:
|
|
||||||
from idf_pytest.constants import TARGET_MARKERS
|
|
||||||
|
|
||||||
for x in tags.split(','):
|
|
||||||
if x in TARGET_MARKERS:
|
|
||||||
return x
|
|
||||||
|
|
||||||
raise RuntimeError(f'No target marker found in {tags}')
|
|
||||||
|
|
||||||
|
|
||||||
def _print_list(_list: t.List[str], separator: str = '\n') -> None:
|
|
||||||
print(separator.join(_list))
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
parser = argparse.ArgumentParser(description='Get the latest merge request info by pipeline')
|
|
||||||
actions = parser.add_subparsers(dest='action', help='info type', required=True)
|
|
||||||
|
|
||||||
common_args = argparse.ArgumentParser(add_help=False)
|
|
||||||
common_args.add_argument('--src-branch', help='source branch')
|
|
||||||
common_args.add_argument(
|
|
||||||
'--modified-files',
|
|
||||||
nargs='+',
|
|
||||||
help='space-separated list specifies the modified files. will be detected by --src-branch if not provided',
|
|
||||||
)
|
|
||||||
|
|
||||||
actions.add_parser('id', parents=[common_args])
|
|
||||||
actions.add_parser('commits', parents=[common_args])
|
|
||||||
actions.add_parser('components', parents=[common_args])
|
|
||||||
target = actions.add_parser('target_in_tags')
|
|
||||||
target.add_argument('tags', help='comma separated tags, e.g., esp32,generic')
|
|
||||||
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
if args.action == 'id':
|
|
||||||
if not args.src_branch:
|
|
||||||
raise RuntimeError('--src-branch is required')
|
|
||||||
print(get_mr_iid(args.src_branch))
|
|
||||||
elif args.action == 'commits':
|
|
||||||
if not args.src_branch:
|
|
||||||
raise RuntimeError('--src-branch is required')
|
|
||||||
_print_list([commit.id for commit in get_mr_commits(args.src_branch)])
|
|
||||||
elif args.action == 'components':
|
|
||||||
_print_list(get_mr_components(args.src_branch, args.modified_files))
|
|
||||||
elif args.action == 'target_in_tags':
|
|
||||||
print(get_target_in_tags(args.tags))
|
|
||||||
else:
|
|
||||||
raise NotImplementedError('not possible to get here')
|
|
||||||
@@ -1,217 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
#
|
|
||||||
# CI script to deploy docs to a webserver. Not useful outside of CI environment
|
|
||||||
#
|
|
||||||
#
|
|
||||||
# SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD
|
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
#
|
|
||||||
import glob
|
|
||||||
import os
|
|
||||||
import os.path
|
|
||||||
import re
|
|
||||||
import stat
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import tarfile
|
|
||||||
from typing import Any, List, Tuple
|
|
||||||
|
|
||||||
import packaging.version
|
|
||||||
|
|
||||||
|
|
||||||
def env(variable: str, default: str=None) -> str:
|
|
||||||
""" Shortcut to return the expanded version of an environment variable """
|
|
||||||
return os.path.expandvars(os.environ.get(variable, default) if default else os.environ[variable])
|
|
||||||
|
|
||||||
|
|
||||||
# import sanitize_version from the docs directory, shared with here
|
|
||||||
sys.path.append(os.path.join(env('IDF_PATH'), 'docs'))
|
|
||||||
from sanitize_version import sanitize_version # noqa
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
# if you get KeyErrors on the following lines, it's probably because you're not running in Gitlab CI
|
|
||||||
git_ver = env('GIT_VER') # output of git describe --always
|
|
||||||
ci_ver = env('CI_COMMIT_REF_NAME', git_ver) # branch or tag we're building for (used for 'release' & URL)
|
|
||||||
|
|
||||||
version = sanitize_version(ci_ver)
|
|
||||||
print('Git version: {}'.format(git_ver))
|
|
||||||
print('CI Version: {}'.format(ci_ver))
|
|
||||||
print('Deployment version: {}'.format(version))
|
|
||||||
|
|
||||||
if not version:
|
|
||||||
raise RuntimeError('A version is needed to deploy')
|
|
||||||
|
|
||||||
build_dir = env('DOCS_BUILD_DIR') # top-level local build dir, where docs have already been built
|
|
||||||
|
|
||||||
if not build_dir:
|
|
||||||
raise RuntimeError('Valid DOCS_BUILD_DIR is needed to deploy')
|
|
||||||
|
|
||||||
url_base = env('DOCS_DEPLOY_URL_BASE') # base for HTTP URLs, used to print the URL to the log after deploying
|
|
||||||
|
|
||||||
docs_server = env('DOCS_DEPLOY_SERVER') # ssh server to deploy to
|
|
||||||
docs_user = env('DOCS_DEPLOY_SERVER_USER')
|
|
||||||
docs_path = env('DOCS_DEPLOY_PATH') # filesystem path on DOCS_SERVER
|
|
||||||
|
|
||||||
if not docs_server:
|
|
||||||
raise RuntimeError('Valid DOCS_DEPLOY_SERVER is needed to deploy')
|
|
||||||
|
|
||||||
if not docs_user:
|
|
||||||
raise RuntimeError('Valid DOCS_DEPLOY_SERVER_USER is needed to deploy')
|
|
||||||
|
|
||||||
docs_server = '{}@{}'.format(docs_user, docs_server)
|
|
||||||
|
|
||||||
if not docs_path:
|
|
||||||
raise RuntimeError('Valid DOCS_DEPLOY_PATH is needed to deploy')
|
|
||||||
|
|
||||||
print('DOCS_DEPLOY_SERVER {} DOCS_DEPLOY_PATH {}'.format(docs_server, docs_path))
|
|
||||||
|
|
||||||
tarball_path, version_urls = build_doc_tarball(version, git_ver, build_dir)
|
|
||||||
|
|
||||||
deploy(version, tarball_path, docs_path, docs_server)
|
|
||||||
|
|
||||||
print('Docs URLs:')
|
|
||||||
doc_deploy_type = os.getenv('TYPE')
|
|
||||||
for vurl in version_urls:
|
|
||||||
language, _, target = vurl.split('/')
|
|
||||||
tag = '{}_{}'.format(language, target)
|
|
||||||
url = '{}/{}/index.html'.format(url_base, vurl) # (index.html needed for the preview server)
|
|
||||||
url = re.sub(r'([^:])//', r'\1/', url) # get rid of any // that isn't in the https:// part
|
|
||||||
print('[document {}][{}] {}'.format(doc_deploy_type, tag, url))
|
|
||||||
|
|
||||||
# note: it would be neater to use symlinks for stable, but because of the directory order
|
|
||||||
# (language first) it's kind of a pain to do on a remote server, so we just repeat the
|
|
||||||
# process but call the version 'stable' this time
|
|
||||||
if is_stable_version(version):
|
|
||||||
print('Deploying again as stable version...')
|
|
||||||
tarball_path, version_urls = build_doc_tarball('stable', git_ver, build_dir)
|
|
||||||
deploy('stable', tarball_path, docs_path, docs_server)
|
|
||||||
|
|
||||||
|
|
||||||
def deploy(version: str, tarball_path: str, docs_path: str, docs_server: str) -> None:
|
|
||||||
def run_ssh(commands: List) -> None:
|
|
||||||
""" Log into docs_server and run a sequence of commands using ssh """
|
|
||||||
print('Running ssh: {}'.format(commands))
|
|
||||||
subprocess.run(['ssh', '-o', 'BatchMode=yes', docs_server, '-x', ' && '.join(commands)], check=True)
|
|
||||||
|
|
||||||
# copy the version tarball to the server
|
|
||||||
run_ssh(['mkdir -p {}'.format(docs_path)])
|
|
||||||
print('Running scp {} to {}'.format(tarball_path, '{}:{}'.format(docs_server, docs_path)))
|
|
||||||
subprocess.run(['scp', '-B', tarball_path, '{}:{}'.format(docs_server, docs_path)], check=True)
|
|
||||||
|
|
||||||
tarball_name = os.path.basename(tarball_path)
|
|
||||||
|
|
||||||
run_ssh(['cd {}'.format(docs_path),
|
|
||||||
'rm -rf ./*/{}'.format(version), # remove any pre-existing docs matching this version
|
|
||||||
'tar -zxvf {}'.format(tarball_name), # untar the archive with the new docs
|
|
||||||
'rm {}'.format(tarball_name)])
|
|
||||||
|
|
||||||
# Note: deleting and then extracting the archive is a bit awkward for updating stable/latest/etc
|
|
||||||
# as the version will be invalid for a window of time. Better to do it atomically, but this is
|
|
||||||
# another thing made much more complex by the directory structure putting language before version...
|
|
||||||
|
|
||||||
|
|
||||||
def build_doc_tarball(version: str, git_ver: str, build_dir: str) -> Tuple[str, List]:
|
|
||||||
""" Make a tar.gz archive of the docs, in the directory structure used to deploy as
|
|
||||||
the given version """
|
|
||||||
version_paths = []
|
|
||||||
tarball_path = '{}/{}.tar.gz'.format(build_dir, version)
|
|
||||||
|
|
||||||
# find all the 'html/' directories under build_dir
|
|
||||||
html_dirs = glob.glob('{}/**/html/'.format(build_dir), recursive=True)
|
|
||||||
print('Found %d html directories' % len(html_dirs))
|
|
||||||
|
|
||||||
pdfs = glob.glob('{}/**/latex/build/*.pdf'.format(build_dir), recursive=True)
|
|
||||||
print('Found %d PDFs in latex directories' % len(pdfs))
|
|
||||||
|
|
||||||
# add symlink for stable and latest and adds them to PDF blob
|
|
||||||
symlinks = create_and_add_symlinks(version, git_ver, pdfs)
|
|
||||||
|
|
||||||
def not_sources_dir(ti: Any) -> Any:
|
|
||||||
print(type(ti))
|
|
||||||
""" Filter the _sources directories out of the tarballs """
|
|
||||||
if ti.name.endswith('/_sources'):
|
|
||||||
return None
|
|
||||||
|
|
||||||
ti.mode |= stat.S_IWGRP # make everything group-writeable
|
|
||||||
return ti
|
|
||||||
|
|
||||||
try:
|
|
||||||
os.remove(tarball_path)
|
|
||||||
except OSError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
with tarfile.open(tarball_path, 'w:gz') as tarball:
|
|
||||||
for html_dir in html_dirs:
|
|
||||||
# html_dir has the form '<ignored>/<language>/<target>/html/'
|
|
||||||
target_dirname = os.path.dirname(os.path.dirname(html_dir))
|
|
||||||
target = os.path.basename(target_dirname)
|
|
||||||
language = os.path.basename(os.path.dirname(target_dirname))
|
|
||||||
|
|
||||||
# when deploying, we want the top-level directory layout 'language/version/target'
|
|
||||||
archive_path = '{}/{}/{}'.format(language, version, target)
|
|
||||||
print("Archiving '{}' as '{}'...".format(html_dir, archive_path))
|
|
||||||
tarball.add(html_dir, archive_path, filter=not_sources_dir)
|
|
||||||
version_paths.append(archive_path)
|
|
||||||
|
|
||||||
for pdf_path in pdfs:
|
|
||||||
# pdf_path has the form '<ignored>/<language>/<target>/latex/build'
|
|
||||||
latex_dirname = os.path.dirname(pdf_path)
|
|
||||||
pdf_filename = os.path.basename(pdf_path)
|
|
||||||
target_dirname = os.path.dirname(os.path.dirname(latex_dirname))
|
|
||||||
target = os.path.basename(target_dirname)
|
|
||||||
language = os.path.basename(os.path.dirname(target_dirname))
|
|
||||||
|
|
||||||
# when deploying, we want the layout 'language/version/target/pdf'
|
|
||||||
archive_path = '{}/{}/{}/{}'.format(language, version, target, pdf_filename)
|
|
||||||
print("Archiving '{}' as '{}'...".format(pdf_path, archive_path))
|
|
||||||
tarball.add(pdf_path, archive_path)
|
|
||||||
|
|
||||||
for symlink in symlinks:
|
|
||||||
os.unlink(symlink)
|
|
||||||
|
|
||||||
return (os.path.abspath(tarball_path), version_paths)
|
|
||||||
|
|
||||||
|
|
||||||
def create_and_add_symlinks(version: str, git_ver: str, pdfs: List) -> List:
|
|
||||||
""" Create symbolic links for PDFs for 'latest' and 'stable' releases """
|
|
||||||
|
|
||||||
symlinks = []
|
|
||||||
if 'stable' in version or 'latest' in version:
|
|
||||||
for pdf_path in pdfs:
|
|
||||||
symlink_path = pdf_path.replace(git_ver, version)
|
|
||||||
os.symlink(pdf_path, symlink_path)
|
|
||||||
symlinks.append(symlink_path)
|
|
||||||
|
|
||||||
pdfs.extend(symlinks)
|
|
||||||
print('Found %d PDFs in latex directories after adding symlink' % len(pdfs))
|
|
||||||
|
|
||||||
return symlinks
|
|
||||||
|
|
||||||
|
|
||||||
def is_stable_version(version: str) -> bool:
|
|
||||||
""" Heuristic for whether this is the latest stable release """
|
|
||||||
if not version.startswith('v'):
|
|
||||||
return False # branch name
|
|
||||||
if '-' in version:
|
|
||||||
return False # prerelease tag
|
|
||||||
|
|
||||||
git_out = subprocess.check_output(['git', 'tag', '-l']).decode('utf-8')
|
|
||||||
|
|
||||||
versions = [v.strip() for v in git_out.split('\n')]
|
|
||||||
versions = [v for v in versions if re.match(r'^v[\d\.]+$', v.strip())] # include vX.Y.Z only
|
|
||||||
|
|
||||||
versions_pack = [packaging.version.parse(v) for v in versions]
|
|
||||||
|
|
||||||
max_version = max(versions_pack)
|
|
||||||
|
|
||||||
if max_version.public != version[1:]:
|
|
||||||
print('Stable version is v{}. This version is {}.'.format(max_version.public, version))
|
|
||||||
return False
|
|
||||||
else:
|
|
||||||
print('This version {} is the stable version'.format(version))
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
||||||
@@ -55,7 +55,7 @@ def main(output_filepath: str) -> None:
|
|||||||
exclude_runner_tags_matching = []
|
exclude_runner_tags_matching = []
|
||||||
for _tag in known_warnings_dict.get('no_runner_tags', []):
|
for _tag in known_warnings_dict.get('no_runner_tags', []):
|
||||||
if '*' not in _tag:
|
if '*' not in _tag:
|
||||||
exclude_runner_tags_set.add(_tag)
|
exclude_runner_tags_set.add(frozenset(_tag.split(',')))
|
||||||
else:
|
else:
|
||||||
if res := _process_match_group(_tag):
|
if res := _process_match_group(_tag):
|
||||||
exclude_runner_tags_matching.append(res)
|
exclude_runner_tags_matching.append(res)
|
||||||
@@ -66,7 +66,7 @@ def main(output_filepath: str) -> None:
|
|||||||
if exclude_runner_tags := os.getenv('EXCLUDE_RUNNER_TAGS'):
|
if exclude_runner_tags := os.getenv('EXCLUDE_RUNNER_TAGS'):
|
||||||
for _tag in exclude_runner_tags.split(';'):
|
for _tag in exclude_runner_tags.split(';'):
|
||||||
if '*' not in _tag:
|
if '*' not in _tag:
|
||||||
exclude_runner_tags_set.add(_tag)
|
exclude_runner_tags_set.add(frozenset(_tag.split(',')))
|
||||||
else:
|
else:
|
||||||
if res := _process_match_group(_tag):
|
if res := _process_match_group(_tag):
|
||||||
exclude_runner_tags_matching.append(res)
|
exclude_runner_tags_matching.append(res)
|
||||||
@@ -75,7 +75,7 @@ def main(output_filepath: str) -> None:
|
|||||||
additional_dict: dict[GroupKey, dict[str, t.Any]] = {}
|
additional_dict: dict[GroupKey, dict[str, t.Any]] = {}
|
||||||
for key, grouped_cases in cases.grouped_cases.items():
|
for key, grouped_cases in cases.grouped_cases.items():
|
||||||
# skip test cases with no runner tags
|
# skip test cases with no runner tags
|
||||||
if ','.join(sorted(key.runner_tags)) in exclude_runner_tags_set:
|
if frozenset(key.runner_tags) in exclude_runner_tags_set:
|
||||||
print(f'WARNING: excluding test cases with runner tags: {key.runner_tags}')
|
print(f'WARNING: excluding test cases with runner tags: {key.runner_tags}')
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
|||||||
@@ -17,6 +17,9 @@
|
|||||||
# Since the version 3.21 CMake passes source files and include dirs to ninja using absolute paths.
|
# Since the version 3.21 CMake passes source files and include dirs to ninja using absolute paths.
|
||||||
# Needed for pytest junit reports.
|
# Needed for pytest junit reports.
|
||||||
INSTALL_EXTRA_TOOLS: cmake
|
INSTALL_EXTRA_TOOLS: cmake
|
||||||
|
# Suppress Python warning messages during CI builds to keep logs clean and focused on errors.
|
||||||
|
# This prevents non-critical warnings (e.g., deprecation or resource warnings) from cluttering output.
|
||||||
|
PYTHONWARNINGS: "ignore"
|
||||||
needs:
|
needs:
|
||||||
- pipeline: $PARENT_PIPELINE_ID
|
- pipeline: $PARENT_PIPELINE_ID
|
||||||
job: generate_build_child_pipeline
|
job: generate_build_child_pipeline
|
||||||
@@ -40,7 +43,6 @@
|
|||||||
- run_cmd idf-ci build run
|
- run_cmd idf-ci build run
|
||||||
--parallel-count ${CI_NODE_TOTAL:-1}
|
--parallel-count ${CI_NODE_TOTAL:-1}
|
||||||
--parallel-index ${CI_NODE_INDEX:-1}
|
--parallel-index ${CI_NODE_INDEX:-1}
|
||||||
--modified-files ${MR_MODIFIED_FILES}
|
|
||||||
|
|
||||||
.dynamic_target_test_template:
|
.dynamic_target_test_template:
|
||||||
image: $TARGET_TEST_ENV_IMAGE
|
image: $TARGET_TEST_ENV_IMAGE
|
||||||
@@ -56,6 +58,8 @@
|
|||||||
needs:
|
needs:
|
||||||
- pipeline: $PARENT_PIPELINE_ID
|
- pipeline: $PARENT_PIPELINE_ID
|
||||||
job: pipeline_variables
|
job: pipeline_variables
|
||||||
|
- pipeline: $PARENT_PIPELINE_ID
|
||||||
|
job: snapshot_known_failure_cases
|
||||||
artifacts:
|
artifacts:
|
||||||
paths:
|
paths:
|
||||||
- XUNIT_RESULT*.xml
|
- XUNIT_RESULT*.xml
|
||||||
@@ -65,7 +69,11 @@
|
|||||||
when: always
|
when: always
|
||||||
expire_in: 1 week
|
expire_in: 1 week
|
||||||
script:
|
script:
|
||||||
|
# Merge the current known failure cases file with snapshot_known_failure_cases file captured at pipeline start
|
||||||
|
# to avoid retry failures caused by later updates to the known failure cases file.
|
||||||
- run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
|
- run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
|
||||||
|
- printf '\n' >> ${KNOWN_FAILURE_CASES_FILE_NAME}
|
||||||
|
- cat artifacts/${KNOWN_FAILURE_CASES_FILE_NAME}.freeze >> ${KNOWN_FAILURE_CASES_FILE_NAME}
|
||||||
# get runner env config file
|
# get runner env config file
|
||||||
- retry_failed git clone $TEST_ENV_CONFIG_REPO
|
- retry_failed git clone $TEST_ENV_CONFIG_REPO
|
||||||
- run_cmd python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
|
- run_cmd python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
|
||||||
|
|||||||
@@ -4,18 +4,15 @@ tools/ci/check_*.py
|
|||||||
tools/ci/check_*.txt
|
tools/ci/check_*.txt
|
||||||
tools/ci/check_*.sh
|
tools/ci/check_*.sh
|
||||||
tools/ci/check_copyright_config.yaml
|
tools/ci/check_copyright_config.yaml
|
||||||
tools/ci/get_all_test_results.py
|
|
||||||
tools/gdb_panic_server.py
|
tools/gdb_panic_server.py
|
||||||
tools/check_term.py
|
tools/check_term.py
|
||||||
tools/python_version_checker.py
|
tools/python_version_checker.py
|
||||||
tools/ci/astyle-rules.yml
|
tools/ci/astyle-rules.yml
|
||||||
tools/ci/checkout_project_ref.py
|
tools/ci/checkout_project_ref.py
|
||||||
tools/ci/ci_fetch_submodule.py
|
tools/ci/ci_fetch_submodule.py
|
||||||
tools/ci/ci_get_mr_info.py
|
|
||||||
tools/ci/ci_process_description.py
|
tools/ci/ci_process_description.py
|
||||||
tools/ci/configure_ci_environment.sh
|
tools/ci/configure_ci_environment.sh
|
||||||
tools/ci/generate_rules.py
|
tools/ci/generate_rules.py
|
||||||
tools/ci/deploy_docs.py
|
|
||||||
tools/ci/envsubst.py
|
tools/ci/envsubst.py
|
||||||
tools/ci/*exclude*.txt
|
tools/ci/*exclude*.txt
|
||||||
tools/ci/executable-list.txt
|
tools/ci/executable-list.txt
|
||||||
|
|||||||
@@ -72,7 +72,6 @@ tools/ci/check_tools_files_patterns.py
|
|||||||
tools/ci/check_type_comments.py
|
tools/ci/check_type_comments.py
|
||||||
tools/ci/checkout_project_ref.py
|
tools/ci/checkout_project_ref.py
|
||||||
tools/ci/cleanup_ignore_lists.py
|
tools/ci/cleanup_ignore_lists.py
|
||||||
tools/ci/deploy_docs.py
|
|
||||||
tools/ci/envsubst.py
|
tools/ci/envsubst.py
|
||||||
tools/ci/fix_empty_prototypes.sh
|
tools/ci/fix_empty_prototypes.sh
|
||||||
tools/ci/generate_rules.py
|
tools/ci/generate_rules.py
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
#
|
#
|
||||||
# SPDX-FileCopyrightText: 2021-2025 Espressif Systems (Shanghai) CO LTD
|
# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
import argparse
|
import argparse
|
||||||
import inspect
|
import inspect
|
||||||
@@ -12,7 +12,6 @@ from itertools import product
|
|||||||
|
|
||||||
import yaml
|
import yaml
|
||||||
from idf_ci_utils import IDF_PATH
|
from idf_ci_utils import IDF_PATH
|
||||||
from idf_ci_utils import GitlabYmlConfig
|
|
||||||
|
|
||||||
if t.TYPE_CHECKING:
|
if t.TYPE_CHECKING:
|
||||||
import pygraphviz as pgv
|
import pygraphviz as pgv
|
||||||
@@ -90,7 +89,6 @@ class RulesWriter:
|
|||||||
self.cfg = self.expand_matrices()
|
self.cfg = self.expand_matrices()
|
||||||
self.rules = self.expand_rules()
|
self.rules = self.expand_rules()
|
||||||
|
|
||||||
self.yml_config = GitlabYmlConfig()
|
|
||||||
self.graph = None
|
self.graph = None
|
||||||
|
|
||||||
def expand_matrices(self): # type: () -> dict
|
def expand_matrices(self): # type: () -> dict
|
||||||
@@ -195,10 +193,6 @@ class RulesWriter:
|
|||||||
if k.startswith('pattern'):
|
if k.startswith('pattern'):
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if '.rules:' + k not in self.yml_config.used_templates:
|
|
||||||
print(f'WARNING: unused rule: {k}, skipping...')
|
|
||||||
continue
|
|
||||||
|
|
||||||
res.append(self.RULES_TEMPLATE.format(k, self._format_rule(k, v)))
|
res.append(self.RULES_TEMPLATE.format(k, self._format_rule(k, v)))
|
||||||
return '\n\n'.join(res)
|
return '\n\n'.join(res)
|
||||||
|
|
||||||
|
|||||||
@@ -1,84 +0,0 @@
|
|||||||
#!/usr/bin/env python
|
|
||||||
#
|
|
||||||
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
|
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
|
||||||
|
|
||||||
|
|
||||||
'''
|
|
||||||
Download artifacts from all test jobs
|
|
||||||
|
|
||||||
CI_PROJECT_DIR
|
|
||||||
└── TEST_RESULTS
|
|
||||||
├── <job_id>
|
|
||||||
│ └── XUNIT_RESULT.xml
|
|
||||||
├── <job_id>
|
|
||||||
│ └── XUNIT_RESULT.xml
|
|
||||||
├── <job_id>
|
|
||||||
│ └── TEST_LOGS
|
|
||||||
|
|
||||||
'''
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import shutil
|
|
||||||
|
|
||||||
import gitlab_api
|
|
||||||
|
|
||||||
CI_PROJECT_ID = int(os.getenv('CI_PROJECT_ID', ''))
|
|
||||||
CI_PIPELINE_ID = int(os.getenv('CI_PIPELINE_ID', ''))
|
|
||||||
IDF_PATH = os.getenv('IDF_PATH', '.')
|
|
||||||
TEST_RESULT_PATH = os.path.join(IDF_PATH, 'TEST_RESULTS')
|
|
||||||
|
|
||||||
|
|
||||||
def download_all_test_results(result_path: str, include_retried: bool = False) -> None:
|
|
||||||
if os.path.exists(result_path):
|
|
||||||
shutil.rmtree(result_path)
|
|
||||||
os.makedirs(result_path, exist_ok=True)
|
|
||||||
|
|
||||||
gitlab_inst = gitlab_api.Gitlab(CI_PROJECT_ID)
|
|
||||||
pipelines = gitlab_inst.project.pipelines.get(CI_PIPELINE_ID)
|
|
||||||
_include_retried = 'true' if include_retried else 'false'
|
|
||||||
jobs = pipelines.jobs.list(all=True, per_page=100, include_retried=_include_retried)
|
|
||||||
|
|
||||||
job_info_list = []
|
|
||||||
for job in jobs:
|
|
||||||
if job.stage in ['target_test', 'host_test']:
|
|
||||||
log_path = ''
|
|
||||||
if job.status not in ['success', 'failed']:
|
|
||||||
print('Job {}({}) is not finished'.format(job.id, job.name))
|
|
||||||
elif not hasattr(job, 'artifacts_file'):
|
|
||||||
print('Job {}({}) has no artifacts.'.format(job.id, job.name))
|
|
||||||
else:
|
|
||||||
log_path = os.path.join(result_path, 'job_{}'.format(job.id))
|
|
||||||
print('Downloading artifacts from: {}'.format(job.name))
|
|
||||||
os.makedirs(log_path, exist_ok=True)
|
|
||||||
gitlab_inst.download_artifacts(job.id, log_path)
|
|
||||||
|
|
||||||
job_info = {
|
|
||||||
'id': job.id,
|
|
||||||
'name': job.name,
|
|
||||||
'tag_list': job.tag_list,
|
|
||||||
'status': job.status,
|
|
||||||
'stage': job.stage,
|
|
||||||
'web_url': job.web_url,
|
|
||||||
'commit_url': job.commit['web_url'],
|
|
||||||
'log_path': log_path,
|
|
||||||
}
|
|
||||||
job_info_list.append(job_info)
|
|
||||||
|
|
||||||
with open(os.path.join(result_path, 'index.json'), 'w') as f:
|
|
||||||
f.write(json.dumps({'jobs': job_info_list}, indent=1, sort_keys=True))
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
parser.add_argument('--path', '-p', default=TEST_RESULT_PATH, help='Path to save result files.')
|
|
||||||
parser.add_argument('--include_retried', action='store_true', help='Including retried jobs.')
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
download_all_test_results(args.path, args.include_retried)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
||||||
@@ -1,17 +1,18 @@
|
|||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
|
# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
"""
|
"""
|
||||||
Check gitlab ci yaml files
|
Check gitlab ci yaml files
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import os
|
import os
|
||||||
import typing as t
|
import typing as t
|
||||||
from functools import cached_property
|
from functools import cached_property
|
||||||
|
|
||||||
from idf_ci_utils import get_submodule_dirs
|
|
||||||
from idf_ci_utils import GitlabYmlConfig
|
|
||||||
from idf_ci_utils import IDF_PATH
|
from idf_ci_utils import IDF_PATH
|
||||||
|
from idf_ci_utils import GitlabYmlConfig
|
||||||
|
from idf_ci_utils import get_submodule_dirs
|
||||||
|
|
||||||
|
|
||||||
class YmlLinter:
|
class YmlLinter:
|
||||||
@@ -43,17 +44,6 @@ class YmlLinter:
|
|||||||
|
|
||||||
exit(exit_code)
|
exit(exit_code)
|
||||||
|
|
||||||
# name it like _1_ to make it run first
|
|
||||||
def _lint_1_yml_parser(self) -> None:
|
|
||||||
for k, v in self.yml_config.config.items():
|
|
||||||
if (
|
|
||||||
k not in self.yml_config.global_keys
|
|
||||||
and k not in self.yml_config.anchors
|
|
||||||
and k not in self.yml_config.templates
|
|
||||||
and k not in self.yml_config.jobs
|
|
||||||
):
|
|
||||||
raise SystemExit(f'Parser incorrect. Key {k} not in global keys, anchors, templates, or jobs')
|
|
||||||
|
|
||||||
def _lint_default_values_artifacts(self) -> None:
|
def _lint_default_values_artifacts(self) -> None:
|
||||||
defaults_artifacts = self.yml_config.default.get('artifacts', {})
|
defaults_artifacts = self.yml_config.default.get('artifacts', {})
|
||||||
|
|
||||||
@@ -79,19 +69,6 @@ class YmlLinter:
|
|||||||
for item in undefined_patterns:
|
for item in undefined_patterns:
|
||||||
self._errors.append(f'undefined pattern {item}. Please add {item} to .patterns-submodule')
|
self._errors.append(f'undefined pattern {item}. Please add {item} to .patterns-submodule')
|
||||||
|
|
||||||
def _lint_gitlab_yml_templates(self) -> None:
|
|
||||||
unused_templates = self.yml_config.templates.keys() - self.yml_config.used_templates
|
|
||||||
for item in unused_templates:
|
|
||||||
# known unused ones
|
|
||||||
if item not in [
|
|
||||||
'.before_script:fetch:target_test', # used in dynamic pipeline
|
|
||||||
]:
|
|
||||||
self._errors.append(f'Unused template: {item}, please remove it')
|
|
||||||
|
|
||||||
undefined_templates = self.yml_config.used_templates - self.yml_config.templates.keys()
|
|
||||||
for item in undefined_templates:
|
|
||||||
self._errors.append(f'Undefined template: {item}')
|
|
||||||
|
|
||||||
def _lint_dependencies_and_needs(self) -> None:
|
def _lint_dependencies_and_needs(self) -> None:
|
||||||
"""
|
"""
|
||||||
Use `dependencies: []` together with `needs: []` could cause missing artifacts issue.
|
Use `dependencies: []` together with `needs: []` could cause missing artifacts issue.
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
# SPDX-FileCopyrightText: 2020-2024 Espressif Systems (Shanghai) CO LTD
|
# SPDX-FileCopyrightText: 2020-2026 Espressif Systems (Shanghai) CO LTD
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
# internal use only for CI
|
# internal use only for CI
|
||||||
# some CI related util functions
|
# some CI related util functions
|
||||||
@@ -125,35 +125,106 @@ class GitlabYmlConfig:
|
|||||||
# avoid unused import in other pre-commit hooks
|
# avoid unused import in other pre-commit hooks
|
||||||
import yaml
|
import yaml
|
||||||
|
|
||||||
all_config = dict()
|
merged_yaml = self._compile_via_gitlab_api(root_yml_filepath)
|
||||||
root_yml = yaml.load(open(root_yml_filepath), Loader=yaml.FullLoader)
|
all_config = yaml.load(merged_yaml, Loader=yaml.FullLoader) or {}
|
||||||
|
|
||||||
# expanding "include"
|
|
||||||
for item in root_yml.pop('include', []) or []:
|
|
||||||
if isinstance(item, dict):
|
|
||||||
if 'project' in item:
|
|
||||||
continue
|
|
||||||
elif 'local' in item:
|
|
||||||
item = item['local']
|
|
||||||
else:
|
|
||||||
continue
|
|
||||||
|
|
||||||
all_config.update(yaml.load(open(os.path.join(IDF_PATH, item)), Loader=yaml.FullLoader))
|
|
||||||
|
|
||||||
if 'default' in all_config:
|
if 'default' in all_config:
|
||||||
self._defaults = all_config.pop('default')
|
self._defaults = all_config.pop('default')
|
||||||
|
|
||||||
self._config = all_config
|
self._config = all_config
|
||||||
|
|
||||||
# anchor is the string that will be reused in templates
|
def _inline_local_includes(self, root_yml_filepath: str) -> str:
|
||||||
self._anchor_keys: t.Set[str] = set()
|
"""
|
||||||
# template is a dict that will be extended
|
Recursively resolve `include: local` entries straight from disk (so uncommitted local
|
||||||
self._template_keys: t.Set[str] = set()
|
changes are always picked up -- CI runners also work off a disk checkout, so there's no
|
||||||
self._used_template_keys: t.Set[str] = set() # tracing the used templates
|
need to fetch a ref remotely via `ci_lint`'s `content_ref`/`ref`/`dry_run_ref` GET
|
||||||
# job is a dict that will be executed
|
params). `include: project` entries (including ones nested inside local files, e.g.
|
||||||
self._job_keys: t.Set[str] = set()
|
`.gitlab/ci/common.yml` including `templates/idf/common-scripts.yml`) are collected and
|
||||||
|
left in the final `include:` list, since those files live in another GitLab project and
|
||||||
|
can only be resolved remotely.
|
||||||
|
|
||||||
self.expand_extends()
|
A dedicated Loader/Dumper pair round-trips `!reference` tags as a marker list subclass,
|
||||||
|
since GitLab CI uses `!reference` which plain YAML doesn't know, and we need to parse
|
||||||
|
(to merge dicts, not just string-concat) then re-dump losslessly.
|
||||||
|
|
||||||
|
:param root_yml_filepath: path to the local root yml file to compile
|
||||||
|
:return: yml content with local includes resolved and merged
|
||||||
|
"""
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
class _Loader(yaml.FullLoader):
|
||||||
|
pass
|
||||||
|
|
||||||
|
class _Dumper(yaml.Dumper):
|
||||||
|
pass
|
||||||
|
|
||||||
|
class _Reference(list):
|
||||||
|
pass
|
||||||
|
|
||||||
|
_Loader.add_constructor(
|
||||||
|
'!reference', lambda loader, node: _Reference(loader.construct_sequence(t.cast(yaml.SequenceNode, node)))
|
||||||
|
)
|
||||||
|
_Dumper.add_representer(_Reference, lambda dumper, data: dumper.represent_sequence('!reference', list(data)))
|
||||||
|
|
||||||
|
def resolve(yml_filepath: str) -> t.Tuple[dict, list]:
|
||||||
|
with open(yml_filepath) as fr:
|
||||||
|
data = yaml.load(fr, Loader=_Loader) or {}
|
||||||
|
|
||||||
|
includes = to_list(data.pop('include', None))
|
||||||
|
|
||||||
|
merged: dict = {}
|
||||||
|
remaining_project_includes: list = []
|
||||||
|
for item in includes:
|
||||||
|
if isinstance(item, dict):
|
||||||
|
if 'project' in item:
|
||||||
|
remaining_project_includes.append(item)
|
||||||
|
continue
|
||||||
|
elif 'local' in item:
|
||||||
|
local_path = item['local']
|
||||||
|
else:
|
||||||
|
continue
|
||||||
|
elif isinstance(item, str):
|
||||||
|
local_path = item
|
||||||
|
else:
|
||||||
|
continue
|
||||||
|
|
||||||
|
sub_merged, sub_remaining = resolve(os.path.join(IDF_PATH, local_path.lstrip('/')))
|
||||||
|
merged.update(sub_merged)
|
||||||
|
remaining_project_includes.extend(sub_remaining)
|
||||||
|
|
||||||
|
# this file's own top-level keys override whatever its includes defined
|
||||||
|
merged.update(data)
|
||||||
|
return merged, remaining_project_includes
|
||||||
|
|
||||||
|
merged_config, project_includes = resolve(root_yml_filepath)
|
||||||
|
if project_includes:
|
||||||
|
merged_config['include'] = project_includes
|
||||||
|
|
||||||
|
return yaml.dump(merged_config, Dumper=_Dumper, sort_keys=False) # type: ignore
|
||||||
|
|
||||||
|
def _compile_via_gitlab_api(self, root_yml_filepath: str) -> str:
|
||||||
|
"""
|
||||||
|
Call the GitLab CI Lint API to get the fully compiled (all `include`s resolved) yml,
|
||||||
|
same as what `glab ci config compile` does. This replaces the old recursive local-only
|
||||||
|
parsing, since the project now includes configs from other projects as well.
|
||||||
|
|
||||||
|
:param root_yml_filepath: path to the local root yml file to compile
|
||||||
|
:return: merged (fully resolved) yml content as a string
|
||||||
|
"""
|
||||||
|
sys.path.insert(0, os.path.join(IDF_PATH, 'tools', 'ci', 'python_packages'))
|
||||||
|
import gitlab_api
|
||||||
|
|
||||||
|
content = self._inline_local_includes(root_yml_filepath)
|
||||||
|
|
||||||
|
gitlab_inst = gitlab_api.Gitlab()
|
||||||
|
project_id = os.getenv('CI_PROJECT_ID') or gitlab_inst.get_project_id('esp-idf', namespace='espressif')
|
||||||
|
project = gitlab_inst.gitlab_inst.projects.get(project_id, lazy=True)
|
||||||
|
|
||||||
|
lint_result = project.ci_lint.create({'content': content})
|
||||||
|
if not lint_result.valid:
|
||||||
|
raise RuntimeError(f'Failed to compile {root_yml_filepath} via GitLab CI Lint API: {lint_result.errors}')
|
||||||
|
|
||||||
|
return lint_result.merged_yaml # type: ignore
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def default(self) -> t.Dict[str, t.Any]:
|
def default(self) -> t.Dict[str, t.Any]:
|
||||||
@@ -167,80 +238,9 @@ class GitlabYmlConfig:
|
|||||||
def global_keys(self) -> t.List[str]:
|
def global_keys(self) -> t.List[str]:
|
||||||
return ['default', 'include', 'workflow', 'variables', 'stages']
|
return ['default', 'include', 'workflow', 'variables', 'stages']
|
||||||
|
|
||||||
@cached_property
|
|
||||||
def anchors(self) -> t.Dict[str, t.Any]:
|
|
||||||
return {k: v for k, v in self.config.items() if k in self._anchor_keys}
|
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def jobs(self) -> t.Dict[str, t.Any]:
|
def jobs(self) -> t.Dict[str, t.Any]:
|
||||||
return {k: v for k, v in self.config.items() if k in self._job_keys}
|
return {k: v for k, v in self.config.items() if not k.startswith('.') and k not in self.global_keys}
|
||||||
|
|
||||||
@cached_property
|
|
||||||
def templates(self) -> t.Dict[str, t.Any]:
|
|
||||||
return {k: v for k, v in self.config.items() if k in self._template_keys}
|
|
||||||
|
|
||||||
@cached_property
|
|
||||||
def used_templates(self) -> t.Set[str]:
|
|
||||||
return self._used_template_keys
|
|
||||||
|
|
||||||
def expand_extends(self) -> None:
|
|
||||||
"""
|
|
||||||
expand the `extends` key in-place.
|
|
||||||
"""
|
|
||||||
for k, v in self.config.items():
|
|
||||||
if k in self.global_keys:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if isinstance(v, (str, list)):
|
|
||||||
self._anchor_keys.add(k)
|
|
||||||
elif k.startswith('.if-'):
|
|
||||||
self._anchor_keys.add(k)
|
|
||||||
elif k.startswith('.'):
|
|
||||||
self._template_keys.add(k)
|
|
||||||
elif isinstance(v, dict):
|
|
||||||
self._job_keys.add(k)
|
|
||||||
else:
|
|
||||||
raise ValueError(f'Unknown type for key {k} with value {v}')
|
|
||||||
|
|
||||||
# no need to expand anchor
|
|
||||||
|
|
||||||
# expand template first
|
|
||||||
for k in self._template_keys:
|
|
||||||
self._expand_extends(k)
|
|
||||||
|
|
||||||
# expand job
|
|
||||||
for k in self._job_keys:
|
|
||||||
self._expand_extends(k)
|
|
||||||
|
|
||||||
def _merge_dict(self, d1: t.Dict[str, t.Any], d2: t.Dict[str, t.Any]) -> t.Any:
|
|
||||||
for k, v in d2.items():
|
|
||||||
if k in d1:
|
|
||||||
if isinstance(v, dict) and isinstance(d1[k], dict):
|
|
||||||
d1[k] = self._merge_dict(d1[k], v)
|
|
||||||
else:
|
|
||||||
d1[k] = v
|
|
||||||
else:
|
|
||||||
d1[k] = v
|
|
||||||
|
|
||||||
return d1
|
|
||||||
|
|
||||||
def _expand_extends(self, name: str) -> t.Dict[str, t.Any]:
|
|
||||||
extends = to_list(self.config[name].pop('extends', None))
|
|
||||||
if not extends:
|
|
||||||
return self.config[name] # type: ignore
|
|
||||||
|
|
||||||
original_d = self.config[name].copy()
|
|
||||||
d = {}
|
|
||||||
while extends:
|
|
||||||
self._used_template_keys.update(extends) # for tracking
|
|
||||||
|
|
||||||
for i in extends:
|
|
||||||
d.update(self._expand_extends(i))
|
|
||||||
|
|
||||||
extends = to_list(self.config[name].pop('extends', None))
|
|
||||||
|
|
||||||
self.config[name] = self._merge_dict(d, original_d)
|
|
||||||
return self.config[name] # type: ignore
|
|
||||||
|
|
||||||
|
|
||||||
def get_all_manifest_files() -> t.List[str]:
|
def get_all_manifest_files() -> t.List[str]:
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ import pytest
|
|||||||
import yaml
|
import yaml
|
||||||
from _pytest.config import Config
|
from _pytest.config import Config
|
||||||
from _pytest.python import Function
|
from _pytest.python import Function
|
||||||
from _pytest.python import Metafunc
|
|
||||||
from _pytest.runner import CallInfo
|
from _pytest.runner import CallInfo
|
||||||
from dynamic_pipelines.constants import KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH
|
from dynamic_pipelines.constants import KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH
|
||||||
from idf_ci import IdfPytestPlugin
|
from idf_ci import IdfPytestPlugin
|
||||||
@@ -115,55 +114,6 @@ class IdfLocalPlugin:
|
|||||||
|
|
||||||
return item.callspec.params.get(key, default) or default
|
return item.callspec.params.get(key, default) or default
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _has_parametrized_arg(metafunc: Metafunc, arg_name: str) -> bool:
|
|
||||||
for marker in metafunc.definition.iter_markers(name='parametrize'):
|
|
||||||
if not marker.args:
|
|
||||||
continue
|
|
||||||
|
|
||||||
argnames = marker.args[0]
|
|
||||||
if isinstance(argnames, str):
|
|
||||||
names = [name.strip() for name in argnames.split(',')]
|
|
||||||
else:
|
|
||||||
names = list(argnames)
|
|
||||||
|
|
||||||
if arg_name in names:
|
|
||||||
return True
|
|
||||||
|
|
||||||
for callspec in getattr(metafunc, '_calls', []):
|
|
||||||
if arg_name in callspec.params:
|
|
||||||
return True
|
|
||||||
|
|
||||||
return False
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _is_linux_target_run(config: Config) -> bool:
|
|
||||||
target = config.getoption('target')
|
|
||||||
if not target:
|
|
||||||
return False
|
|
||||||
|
|
||||||
if isinstance(target, str):
|
|
||||||
targets = [_t.strip() for _t in target.split(',')]
|
|
||||||
else:
|
|
||||||
targets = [str(_t).strip() for _t in target]
|
|
||||||
|
|
||||||
return 'linux' in targets
|
|
||||||
|
|
||||||
@pytest.hookimpl(trylast=True)
|
|
||||||
def pytest_generate_tests(self, metafunc: Metafunc) -> None:
|
|
||||||
if 'embedded_services' not in metafunc.fixturenames:
|
|
||||||
return
|
|
||||||
|
|
||||||
if self._has_parametrized_arg(metafunc, 'embedded_services'):
|
|
||||||
return
|
|
||||||
|
|
||||||
if metafunc.definition.get_closest_marker('qemu') is not None:
|
|
||||||
metafunc.parametrize('embedded_services', ['idf,qemu'], indirect=True)
|
|
||||||
return
|
|
||||||
|
|
||||||
if self._is_linux_target_run(metafunc.config):
|
|
||||||
metafunc.parametrize('embedded_services', ['idf'], indirect=True)
|
|
||||||
|
|
||||||
@pytest.hookimpl(wrapper=True)
|
@pytest.hookimpl(wrapper=True)
|
||||||
def pytest_collection_modifyitems(self, config: Config, items: t.List[Function]) -> t.Generator[None, None, None]:
|
def pytest_collection_modifyitems(self, config: Config, items: t.List[Function]) -> t.Generator[None, None, None]:
|
||||||
yield # throw it back to idf-ci
|
yield # throw it back to idf-ci
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
|
# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD
|
||||||
# SPDX-License-Identifier: Apache-2.0
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
import argparse
|
import argparse
|
||||||
import logging
|
import logging
|
||||||
@@ -52,9 +52,11 @@ def retry(func: TR) -> TR:
|
|||||||
raise e # get out of the loop
|
raise e # get out of the loop
|
||||||
else:
|
else:
|
||||||
logging.warning(
|
logging.warning(
|
||||||
'Network failure in {}, retrying ({})'.format(getattr(func, '__name__', '(unknown callable)'),
|
'Network failure in {}, retrying ({})'.format(
|
||||||
retried))
|
getattr(func, '__name__', '(unknown callable)'), retried
|
||||||
time.sleep(2 ** retried) # wait a bit more after each retry
|
)
|
||||||
|
)
|
||||||
|
time.sleep(2**retried) # wait a bit more after each retry
|
||||||
continue
|
continue
|
||||||
else:
|
else:
|
||||||
break
|
break
|
||||||
@@ -67,7 +69,6 @@ class Gitlab(object):
|
|||||||
JOB_NAME_PATTERN = re.compile(r'(\w+)(\s+(\d+)/(\d+))?')
|
JOB_NAME_PATTERN = re.compile(r'(\w+)(\s+(\d+)/(\d+))?')
|
||||||
|
|
||||||
DOWNLOAD_ERROR_MAX_RETRIES = 3
|
DOWNLOAD_ERROR_MAX_RETRIES = 3
|
||||||
DEFAULT_BUILD_CHILD_PIPELINE_NAME = 'Build Child Pipeline'
|
|
||||||
|
|
||||||
def __init__(self, project_id: Union[int, str, None] = None):
|
def __init__(self, project_id: Union[int, str, None] = None):
|
||||||
config_data_from_env = os.getenv('PYTHON_GITLAB_CONFIG')
|
config_data_from_env = os.getenv('PYTHON_GITLAB_CONFIG')
|
||||||
@@ -117,7 +118,7 @@ class Gitlab(object):
|
|||||||
:param namespace: namespace to match when we have multiple project with same name
|
:param namespace: namespace to match when we have multiple project with same name
|
||||||
:return: project ID
|
:return: project ID
|
||||||
"""
|
"""
|
||||||
projects = self.gitlab_inst.projects.list(search=name)
|
projects = self.gitlab_inst.projects.list(search=name, get_all=True)
|
||||||
res = []
|
res = []
|
||||||
for project in projects:
|
for project in projects:
|
||||||
if namespace is None:
|
if namespace is None:
|
||||||
@@ -152,7 +153,9 @@ class Gitlab(object):
|
|||||||
archive_file.extractall(destination)
|
archive_file.extractall(destination)
|
||||||
|
|
||||||
@retry
|
@retry
|
||||||
def download_artifact(self, job_id: int, artifact_path: List[str], destination: Optional[str] = None) -> List[bytes]:
|
def download_artifact(
|
||||||
|
self, job_id: int, artifact_path: List[str], destination: Optional[str] = None
|
||||||
|
) -> List[bytes]:
|
||||||
"""
|
"""
|
||||||
download specific path of job artifacts and extract to destination.
|
download specific path of job artifacts and extract to destination.
|
||||||
|
|
||||||
@@ -208,8 +211,9 @@ class Gitlab(object):
|
|||||||
return job_id_list
|
return job_id_list
|
||||||
|
|
||||||
@retry
|
@retry
|
||||||
def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None,
|
def download_archive(
|
||||||
cache_dir: Optional[str] = None) -> str:
|
self, ref: str, destination: str, project_id: Optional[int] = None, cache_dir: Optional[str] = None
|
||||||
|
) -> str:
|
||||||
"""
|
"""
|
||||||
Download archive of certain commit of a repository and extract to destination path
|
Download archive of certain commit of a repository and extract to destination path
|
||||||
|
|
||||||
@@ -235,8 +239,11 @@ class Gitlab(object):
|
|||||||
except gitlab.GitlabGetError as e:
|
except gitlab.GitlabGetError as e:
|
||||||
logging.error('Failed to archive from project {}'.format(project_id))
|
logging.error('Failed to archive from project {}'.format(project_id))
|
||||||
raise e
|
raise e
|
||||||
logging.info('Downloaded archive size: {:.03f}MB'.format(
|
logging.info(
|
||||||
float(os.path.getsize(local_archive_file)) / (1024 * 1024)))
|
'Downloaded archive size: {:.03f}MB'.format(
|
||||||
|
float(os.path.getsize(local_archive_file)) / (1024 * 1024)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
return self.decompress_archive(local_archive_file, destination)
|
return self.decompress_archive(local_archive_file, destination)
|
||||||
|
|
||||||
@@ -248,7 +255,9 @@ class Gitlab(object):
|
|||||||
logging.error('Failed to archive from project {}'.format(project_id))
|
logging.error('Failed to archive from project {}'.format(project_id))
|
||||||
raise e
|
raise e
|
||||||
|
|
||||||
logging.info('Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(temp_file.name)) / (1024 * 1024)))
|
logging.info(
|
||||||
|
'Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(temp_file.name)) / (1024 * 1024))
|
||||||
|
)
|
||||||
|
|
||||||
return self.decompress_archive(temp_file.name, destination)
|
return self.decompress_archive(temp_file.name, destination)
|
||||||
|
|
||||||
@@ -282,36 +291,28 @@ class Gitlab(object):
|
|||||||
|
|
||||||
def get_downstream_pipeline_ids(self, main_pipeline_id: int) -> List[int]:
|
def get_downstream_pipeline_ids(self, main_pipeline_id: int) -> List[int]:
|
||||||
"""
|
"""
|
||||||
Retrieve the IDs of all downstream child pipelines for a given main pipeline.
|
Retrieve the IDs of all downstream child pipelines for a given main pipeline,
|
||||||
|
recursing through arbitrarily nested child pipelines.
|
||||||
|
|
||||||
:param main_pipeline_id: The ID of the main pipeline to start the search.
|
:param main_pipeline_id: The ID of the main pipeline to start the search.
|
||||||
:return: A list of IDs of all downstream child pipelines.
|
:return: A list of IDs of all downstream child pipelines (all levels).
|
||||||
"""
|
"""
|
||||||
bridge_pipeline_ids = []
|
child_pipeline_ids: List[int] = []
|
||||||
child_pipeline_ids = []
|
|
||||||
|
|
||||||
main_pipeline_bridges = self.project.pipelines.get(main_pipeline_id).bridges.list()
|
pipeline_bridges = self.project.pipelines.get(main_pipeline_id).bridges.list()
|
||||||
for bridge in main_pipeline_bridges:
|
for bridge in pipeline_bridges:
|
||||||
downstream_pipeline = bridge.attributes.get('downstream_pipeline')
|
downstream_pipeline = bridge.attributes.get('downstream_pipeline')
|
||||||
if not downstream_pipeline:
|
if not downstream_pipeline:
|
||||||
continue
|
continue
|
||||||
bridge_pipeline_ids.append(downstream_pipeline['id'])
|
downstream_pipeline_id = downstream_pipeline.get('id')
|
||||||
|
if downstream_pipeline_id is None:
|
||||||
for bridge_pipeline_id in bridge_pipeline_ids:
|
|
||||||
child_pipeline_ids.append(bridge_pipeline_id)
|
|
||||||
bridge_pipeline = self.project.pipelines.get(bridge_pipeline_id)
|
|
||||||
|
|
||||||
if not bridge_pipeline.name == self.DEFAULT_BUILD_CHILD_PIPELINE_NAME:
|
|
||||||
continue
|
continue
|
||||||
|
|
||||||
child_bridges = bridge_pipeline.bridges.list()
|
child_pipeline_ids.append(downstream_pipeline_id)
|
||||||
for child_bridge in child_bridges:
|
# recurse to collect further nested (grandchild+) pipelines
|
||||||
downstream_child_pipeline = child_bridge.attributes.get('downstream_pipeline')
|
child_pipeline_ids.extend(self.get_downstream_pipeline_ids(downstream_pipeline_id))
|
||||||
if not downstream_child_pipeline:
|
|
||||||
continue
|
|
||||||
child_pipeline_ids.append(downstream_child_pipeline.get('id'))
|
|
||||||
|
|
||||||
return [pid for pid in child_pipeline_ids if pid is not None]
|
return child_pipeline_ids
|
||||||
|
|
||||||
def retry_failed_jobs(self, pipeline_id: int, retry_allowed_failures: bool = False) -> List[int]:
|
def retry_failed_jobs(self, pipeline_id: int, retry_allowed_failures: bool = False) -> List[int]:
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -164,38 +164,3 @@ function join_by {
|
|||||||
printf %s "$f" "${@/#/$d}"
|
printf %s "$f" "${@/#/$d}"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
function is_based_on_commits() {
|
|
||||||
# This function would accept space-separated args as multiple commits.
|
|
||||||
# The return value would be 0 if current HEAD is based on any of the specified commits.
|
|
||||||
#
|
|
||||||
# In our CI, we use environment variable $REQUIRED_ANCESTOR_COMMITS to declare the ancestor commits.
|
|
||||||
# Please remember to set one commit for each release branch.
|
|
||||||
|
|
||||||
commits=$*
|
|
||||||
if [[ -z $commits ]]; then
|
|
||||||
info "Not specifying commits that branches should be based on, skipping check..."
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
commits_str="$(join_by " or " $commits)" # no doublequotes here, passing array
|
|
||||||
|
|
||||||
info "Checking if current branch is based on $commits_str..."
|
|
||||||
for i in $commits; do
|
|
||||||
if git merge-base --is-ancestor "$i" HEAD >/dev/null 2>&1; then
|
|
||||||
info "Current branch is based on $i"
|
|
||||||
return 0
|
|
||||||
else
|
|
||||||
info "Current branch is not based on $i"
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
error "The base commit of your branch is too old."
|
|
||||||
error "The branch should be more recent than either of the following commits:"
|
|
||||||
error " $commits_str"
|
|
||||||
error "To fix the issue:"
|
|
||||||
error " - If your merge request is 'Draft', or has conflicts with the target branch, rebase it to the latest master or release branch"
|
|
||||||
error " - Otherwise, simply run a new pipeline."
|
|
||||||
|
|
||||||
return 1
|
|
||||||
}
|
|
||||||
|
|||||||
Reference in New Issue
Block a user