From 44687303cfad5ee0c94d8b2df57c2831265ac9d5 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Wed, 15 Jul 2026 13:56:55 +0200 Subject: [PATCH] ci: move dynamic pipeline related jobs to ci/actions/common --- .gitlab-ci.yml | 4 +- .gitlab/ci/build.yml | 75 +---------------------- .gitlab/ci/pre_check.yml | 46 -------------- tools/ci/exclude_check_tools_files.txt | 1 - tools/ci/get_all_test_results.py | 84 -------------------------- 5 files changed, 6 insertions(+), 204 deletions(-) delete mode 100644 tools/ci/get_all_test_results.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a1d7cf95401..91638d0b77d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,12 +2,14 @@ include: - project: "ci/actions/common" + ref: "v2" file: - "templates/stable/default-workflow.yml" + - "templates/idf/pre_check.yml" + - "templates/idf/build.yml" - "templates/idf/deploy-github.yml" - "templates/idf/deploy-docs.yml" - "templates/idf/integration-test.yml" - - "templates/idf/pre_check.yml" - "templates/idf/sync-files.yml" - project: "ci/actions/common" file: "templates/idf/build-docs.yml" diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index 4ca98a63a52..e5d3ccdfaeb 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -9,7 +9,6 @@ IDF_CCACHE_ENABLE: "1" dependencies: # set dependencies to null to avoid missing artifacts issue - ######################################## # Clang Build Apps Without Tests Cases # ######################################## @@ -68,62 +67,9 @@ build_clang_test_apps_riscv: #################### # Dynamic Pipeline # #################### -pre_check_complete: - stage: build - image: $ESP_ENV_IMAGE - tags: [fast_run, shiny] - variables: - GIT_STRATEGY: none - cache: [] - before_script: [] - dependencies: [] - script: - - echo "pre_check stage completed" - -generate_build_child_pipeline: - extends: - - .build_template - tags: [fast_run, shiny] - dependencies: # set dependencies to null to avoid missing artifacts issue - needs: - - pipeline_variables - - job: baseline_manifest_sha - optional: true - artifacts: - paths: - - build_child_pipeline.yml - - test_related_apps.txt - - non_test_related_apps.txt - expire_in: 1 week - when: always - script: - - run_cmd idf-ci --debug gitlab build-child-pipeline - -p components - -p examples - -p tools/test_apps - - '[ -f test_related_apps.txt ] && shuf test_related_apps.txt -o test_related_apps.txt || echo "test_related_apps.txt not found, skipping."' - - '[ -f non_test_related_apps.txt ] && shuf non_test_related_apps.txt -o non_test_related_apps.txt || echo "non_test_related_apps.txt not found, skipping."' - -build_child_pipeline: - stage: build - needs: - - job: fast_build_check - optional: true - artifacts: false - - pipeline_variables - - generate_build_child_pipeline - - job: pre_check_complete - artifacts: false - variables: - PARENT_PIPELINE_ID: $CI_PIPELINE_ID - # https://gitlab.com/gitlab-org/gitlab/-/issues/214340 - inherit: - variables: false - trigger: - include: - - artifact: build_child_pipeline.yml - job: generate_build_child_pipeline - strategy: depend +# !!! +# configurations moved to ci/actions/common/idf/ +# !!! # Generates a separate child-pipeline YAML for the cmakev2 build system. # The build_dir stays identical to cmakev1 (build__); v1/v2 @@ -193,18 +139,3 @@ build_child_pipeline_buildv2: job: generate_build_child_pipeline_buildv2 strategy: depend allow_failure: true - -generate_prebuild_report: - extends: - - .build_template - tags: [fast_run, shiny] - needs: - - pipeline_variables - artifacts: - paths: - - prebuild_report.html - expire_in: 1 week - when: always - script: - - run_cmd idf-ci build collect --format html -o prebuild_report.html - - echo "Report generated at https://${CI_PAGES_HOSTNAME}:${CI_SERVER_PORT}/-/esp-idf/-/jobs/${CI_JOB_ID}/artifacts/prebuild_report.html" diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index e85916a0760..654cf52129a 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -147,52 +147,6 @@ check_test_scripts_build_test_rules: # requires basic pytest dependencies - python tools/ci/check_build_test_rules.py check-test-scripts examples/ tools/test_apps components - -snapshot_known_failure_cases: - extends: - - .pre_check_template - script: - - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} - - mkdir -p artifacts - - mv ${KNOWN_FAILURE_CASES_FILE_NAME} artifacts/${KNOWN_FAILURE_CASES_FILE_NAME}.freeze - artifacts: - paths: - - artifacts/${KNOWN_FAILURE_CASES_FILE_NAME}.freeze - expire_in: 1 week - when: always - - -baseline_manifest_sha: - extends: - - .pre_check_template - - .rules:dev-push - script: - - | - # merged results pipelines, by default - # diff between target-branch-head and merged-result-head - if [ -n "$CI_MERGE_REQUEST_TARGET_BRANCH_SHA" ]; then - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA --depth=1 - git checkout FETCH_HEAD - idf-build-apps dump-manifest-sha \ - --manifest-files $(find . -name ".build-test-rules.yml" | xargs) \ - --output .manifest_sha - # merge request pipelines, when the mr got conflicts - # diff between diff-base-sha and merge-request-head - elif [ -n "$CI_MERGE_REQUEST_DIFF_BASE_SHA" ]; then - git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 - git checkout FETCH_HEAD - idf-build-apps dump-manifest-sha \ - --manifest-files $(find . -name ".build-test-rules.yml" | xargs) \ - --output .manifest_sha - # other pipelines, like the protected branches pipelines - # not triggered in this job - fi - artifacts: - paths: - - .manifest_sha - expire_in: 1 week - when: always - retry_failed_jobs: extends: - .pre_check_template diff --git a/tools/ci/exclude_check_tools_files.txt b/tools/ci/exclude_check_tools_files.txt index f0bcf7fbd53..93b23679dff 100644 --- a/tools/ci/exclude_check_tools_files.txt +++ b/tools/ci/exclude_check_tools_files.txt @@ -17,7 +17,6 @@ tools/ci/executable-list.txt tools/ci/fix_empty_prototypes.sh tools/ci/generate_rules.py tools/ci/get-full-sources.sh -tools/ci/get_all_test_results.py tools/ci/get_supported_examples.sh tools/ci/gitlab_yaml_linter.py tools/ci/idf_build_apps_dump_soc_caps.py diff --git a/tools/ci/get_all_test_results.py b/tools/ci/get_all_test_results.py deleted file mode 100644 index 7baa0832f2a..00000000000 --- a/tools/ci/get_all_test_results.py +++ /dev/null @@ -1,84 +0,0 @@ -#!/usr/bin/env python -# -# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 - - -''' -Download artifacts from all test jobs - -CI_PROJECT_DIR -└── TEST_RESULTS - ├── - │ └── XUNIT_RESULT.xml - ├── - │ └── XUNIT_RESULT.xml - ├── - │ └── TEST_LOGS - -''' - -import argparse -import json -import os -import shutil - -import gitlab_api - -CI_PROJECT_ID = int(os.getenv('CI_PROJECT_ID', '')) -CI_PIPELINE_ID = int(os.getenv('CI_PIPELINE_ID', '')) -IDF_PATH = os.getenv('IDF_PATH', '.') -TEST_RESULT_PATH = os.path.join(IDF_PATH, 'TEST_RESULTS') - - -def download_all_test_results(result_path: str, include_retried: bool = False) -> None: - if os.path.exists(result_path): - shutil.rmtree(result_path) - os.makedirs(result_path, exist_ok=True) - - gitlab_inst = gitlab_api.Gitlab(CI_PROJECT_ID) - pipelines = gitlab_inst.project.pipelines.get(CI_PIPELINE_ID) - _include_retried = 'true' if include_retried else 'false' - jobs = pipelines.jobs.list(all=True, per_page=100, include_retried=_include_retried) - - job_info_list = [] - for job in jobs: - if job.stage in ['target_test', 'host_test']: - log_path = '' - if job.status not in ['success', 'failed']: - print('Job {}({}) is not finished'.format(job.id, job.name)) - elif not hasattr(job, 'artifacts_file'): - print('Job {}({}) has no artifacts.'.format(job.id, job.name)) - else: - log_path = os.path.join(result_path, 'job_{}'.format(job.id)) - print('Downloading artifacts from: {}'.format(job.name)) - os.makedirs(log_path, exist_ok=True) - gitlab_inst.download_artifacts(job.id, log_path) - - job_info = { - 'id': job.id, - 'name': job.name, - 'tag_list': job.tag_list, - 'status': job.status, - 'stage': job.stage, - 'web_url': job.web_url, - 'commit_url': job.commit['web_url'], - 'log_path': log_path, - } - job_info_list.append(job_info) - - with open(os.path.join(result_path, 'index.json'), 'w') as f: - f.write(json.dumps({'jobs': job_info_list}, indent=1, sort_keys=True)) - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument('--path', '-p', default=TEST_RESULT_PATH, help='Path to save result files.') - parser.add_argument('--include_retried', action='store_true', help='Including retried jobs.') - args = parser.parse_args() - - download_all_test_results(args.path, args.include_retried) - - -if __name__ == '__main__': - main()