diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index d0ce66c0c5b..55d4823f429 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -128,6 +128,77 @@ build_child_pipeline: job: generate_build_child_pipeline strategy: depend +# Generates a separate child-pipeline YAML for the cmakev2 build system. +# The build_dir stays identical to cmakev1 (build__); v1/v2 +# binaries are kept separate in s3 by suffixing PIPELINE_COMMIT_SHA — see +# the python injection below — which idf-ci uses unmodified in its s3 +# prefix (project//...). Keeping build_dir identical preserves the +# upstream pytest matching that uses app.build_path to filter test cases. +generate_build_child_pipeline_buildv2: + extends: + - .build_template + - .rules:labels:buildv2 + tags: [fast_run, shiny] + variables: + # The build child pipeline's app list is fixed here, at generation time. + # Set IDF_BUILD_V2 so that manifest `disable: if IDF_BUILD_V2 == "1"` rules + # take effect during app discovery; otherwise those apps (e.g. kasan_test, + # IDF-15864) are discovered, built and target-tested even though the build + # jobs run with IDF_BUILD_V2 injected by patch_buildv2_child_pipeline.py. + IDF_BUILD_V2: "1" + dependencies: # set dependencies to null to avoid missing artifacts issue + needs: + - pipeline_variables + - job: pre_check_complete + artifacts: false + - job: baseline_manifest_sha + optional: true + artifacts: + paths: + - buildv2_child_pipeline.yml + - test_related_apps.txt + - non_test_related_apps.txt + expire_in: 1 week + when: always + script: + - idf-ci --debug gitlab build-child-pipeline + -p components + -p examples + -p tools/test_apps + --modified-files "$MR_MODIFIED_FILES" + buildv2_child_pipeline.yml > idfci_buildv2_gen.log 2>&1 + - tail -20 idfci_buildv2_gen.log + - shuf test_related_apps.txt -o test_related_apps.txt + - shuf non_test_related_apps.txt -o non_test_related_apps.txt + # Patch the generated yaml to activate the cmakev2 shim and route v2 build + # and target_test artifacts to a distinct s3 namespace. See the script + # docstring for the rationale. + - python tools/ci/dynamic_pipelines/scripts/patch_buildv2_child_pipeline.py buildv2_child_pipeline.yml + - 'echo "=== buildv2_child_pipeline.yml (post-injection) ==="; cat buildv2_child_pipeline.yml; echo "=== end ==="' + +build_child_pipeline_buildv2: + stage: build + extends: + - .rules:labels:buildv2 + needs: + - job: fast_build_check + optional: true + artifacts: false + - pipeline_variables + - generate_build_child_pipeline_buildv2 + variables: + MR_MODIFIED_COMPONENTS: $MR_MODIFIED_COMPONENTS + MR_MODIFIED_FILES: $MR_MODIFIED_FILES + PARENT_PIPELINE_ID: $CI_PIPELINE_ID + inherit: + variables: false + trigger: + include: + - artifact: buildv2_child_pipeline.yml + job: generate_build_child_pipeline_buildv2 + strategy: depend + allow_failure: true + generate_prebuild_report: extends: - .build_template diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index bdac7f2a340..8699aa14070 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -353,7 +353,7 @@ build_docker: ############################ # Host test with test apps # ############################ -test_pytest_qemu: +.test_pytest_qemu_template: extends: - .host_test_template artifacts: @@ -363,15 +363,6 @@ test_pytest_qemu: - "**/build*/*.bin" reports: junit: XUNIT_RESULT.xml - parallel: - matrix: - - IDF_TARGET: "esp32" - INSTALL_EXTRA_TOOLS: "qemu-xtensa" - # Skip Clang + Xtensa tests due to bootloader size issue - IDF_TOOLCHAIN: [gcc] - - IDF_TARGET: "esp32c3" - INSTALL_EXTRA_TOOLS: "qemu-riscv32" - IDF_TOOLCHAIN: [gcc, clang] script: - run_cmd idf-ci build run --build-system cmake @@ -386,7 +377,20 @@ test_pytest_qemu: --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} --qemu-extra-args \"-global driver=timer.$IDF_TARGET.timg,property=wdt_disable,value=true\" -test_pytest_linux: +test_pytest_qemu: + extends: + - .test_pytest_qemu_template + parallel: + matrix: + - IDF_TARGET: "esp32" + INSTALL_EXTRA_TOOLS: "qemu-xtensa" + # Skip Clang + Xtensa tests due to bootloader size issue + IDF_TOOLCHAIN: [gcc] + - IDF_TARGET: "esp32c3" + INSTALL_EXTRA_TOOLS: "qemu-riscv32" + IDF_TOOLCHAIN: [gcc, clang] + +.test_pytest_linux_template: extends: - .host_test_template artifacts: @@ -410,6 +414,10 @@ test_pytest_linux: --junitxml=XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} +test_pytest_linux: + extends: + - .test_pytest_linux_template + test_pytest_macos: extends: - .host_test_template @@ -516,6 +524,34 @@ pytest_buildv2_system: --junitxml ${CI_PROJECT_DIR}/XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} +# Run QEMU tests with the cmakev2 build system (via the IDF_BUILD_V2 shim). +# Triggered only when the 'buildv2' MR label is set. +test_pytest_qemu_buildv2: + extends: + - .test_pytest_qemu_template + - .rules:labels:buildv2 + allow_failure: true + parallel: + matrix: + - IDF_TARGET: "esp32" + INSTALL_EXTRA_TOOLS: "qemu-xtensa" + IDF_TOOLCHAIN: [gcc] + - IDF_TARGET: "esp32c3" + INSTALL_EXTRA_TOOLS: "qemu-riscv32" + IDF_TOOLCHAIN: [gcc] + variables: + IDF_BUILD_V2: "1" + +# Run Linux-target pytest tests with the cmakev2 build system (via the +# IDF_BUILD_V2 shim). Triggered only when the 'buildv2' MR label is set. +test_pytest_linux_buildv2: + extends: + - .test_pytest_linux_template + - .rules:labels:buildv2 + allow_failure: true + variables: + IDF_BUILD_V2: "1" + pytest_build_system_macos: extends: - .test_build_system_template diff --git a/.idf_build_apps.toml b/.idf_build_apps.toml index e299567bfc9..59ecf1c658a 100644 --- a/.idf_build_apps.toml +++ b/.idf_build_apps.toml @@ -19,7 +19,10 @@ ignore_warning_files = [ '$IDF_PATH/tools/ci/ignore_build_warnings.txt', ] -build_dir = "build_@t_@w" +# BUILDV2_DIR_SUFFIX is expanded by idf-build-apps from the environment; +# empty for buildv1 builds and set to "_v2" by the buildv2 CI pipelines so +# buildv1 and buildv2 build artifacts do not share the same s3 cache path. +build_dir = "build${BUILDV2_DIR_SUFFIX}_@t_@w" build_log_filename = "build_log.txt" size_json_filename = "size_${CI_JOB_ID}.json" diff --git a/tools/ci/dynamic_pipelines/scripts/patch_buildv2_child_pipeline.py b/tools/ci/dynamic_pipelines/scripts/patch_buildv2_child_pipeline.py new file mode 100644 index 00000000000..2a791ecfc37 --- /dev/null +++ b/tools/ci/dynamic_pipelines/scripts/patch_buildv2_child_pipeline.py @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +"""Patch the generated Build system v2 build child pipeline in place. + +The buildv2 build child pipeline is generated by ``idf-ci gitlab build-child-pipeline`` +from the same manifest as the default pipeline. This script post-processes that +generated YAML so the cmakev2 path is exercised end to end: + +1. Inject ``IDF_BUILD_V2`` into each child build job so cmake activates the + cmakev2 shim. + +2. Inject ``PIPELINE_COMMIT_SHA: ${PIPELINE_COMMIT_SHA}_buildv2`` into each child + build job. idf-ci reads ``PIPELINE_COMMIT_SHA`` when computing the s3 + upload/download prefix (``project//...``). Suffixing it for v2 jobs only + routes v2 binaries to a distinct s3 key namespace without renaming build_dir. + +3. Redefine ``generate_pytest_child_pipeline`` so its script also runs + ``patch_buildv2_target_test_pipeline.py`` against the emitted + ``target_test_child_pipeline.yml`` and injects the same + ``PIPELINE_COMMIT_SHA`` override into each target_test job. Without this, the + v2 target_test pipeline would still download from the default pipeline's s3 + keys (race-determined, mostly cmakev1 binaries). +""" + +import argparse + +import yaml + +PIPELINE_COMMIT_SHA_V2 = '${PIPELINE_COMMIT_SHA}_buildv2' + + +def patch(path: str) -> None: + with open(path) as f: + d = yaml.safe_load(f) + + injected = [] + for k, v in d.items(): + if isinstance(v, dict) and 'extends' in v: + v.setdefault('variables', {})['IDF_BUILD_V2'] = '1' + v['variables']['PIPELINE_COMMIT_SHA'] = PIPELINE_COMMIT_SHA_V2 + injected.append(k) + + # Redefine generate_pytest_child_pipeline (included from + # tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml) so its script + # also rewrites target_test_child_pipeline.yml. GitLab CI merges values at the + # keyword level, so only `script` is overridden; `needs`, `artifacts`, + # `image`, etc. come from the included version. + d['generate_pytest_child_pipeline'] = { + 'script': [ + 'python tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py', + 'python tools/ci/dynamic_pipelines/scripts/patch_buildv2_target_test_pipeline.py ' + 'target_test_child_pipeline.yml', + ], + } + + with open(path, 'w') as f: + yaml.safe_dump(d, f, sort_keys=False) + + print('Injected IDF_BUILD_V2 + PIPELINE_COMMIT_SHA into build jobs:', injected) + print('Redefined generate_pytest_child_pipeline with post-processing step') + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + 'pipeline_yaml', + nargs='?', + default='buildv2_child_pipeline.yml', + help='Path to the generated buildv2 build child pipeline YAML to patch in place.', + ) + args = parser.parse_args() + patch(args.pipeline_yaml) + + +if __name__ == '__main__': + main() diff --git a/tools/ci/dynamic_pipelines/scripts/patch_buildv2_target_test_pipeline.py b/tools/ci/dynamic_pipelines/scripts/patch_buildv2_target_test_pipeline.py new file mode 100644 index 00000000000..bb449f0ac7b --- /dev/null +++ b/tools/ci/dynamic_pipelines/scripts/patch_buildv2_target_test_pipeline.py @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +"""Patch the generated Build system v2 target_test child pipeline in place. + +Inject ``PIPELINE_COMMIT_SHA: ${PIPELINE_COMMIT_SHA}_buildv2`` into every +target_test job so idf-ci downloads the cmakev2 binaries from the distinct s3 +key namespace produced by the v2 build jobs, instead of racing with the default +pipeline's binaries. + +Run from generate_pytest_child_pipeline (see patch_buildv2_child_pipeline.py), +after generate_target_test_child_pipeline.py has emitted the YAML. +""" + +import argparse + +import yaml + +PIPELINE_COMMIT_SHA_V2 = '${PIPELINE_COMMIT_SHA}_buildv2' + +# Target test jobs use extends: .dynamic_target_test_template, so their script: +# comes from the template, not their own definition. These top-level keys are +# pipeline configuration rather than jobs and must be skipped. +SKIP_KEYS = {'include', 'workflow', 'default', 'variables', 'stages'} + + +def patch(path: str) -> None: + with open(path) as f: + d = yaml.safe_load(f) + + modified = 0 + for k, v in d.items(): + if k in SKIP_KEYS or not isinstance(v, dict): + continue + v.setdefault('variables', {})['PIPELINE_COMMIT_SHA'] = PIPELINE_COMMIT_SHA_V2 + modified += 1 + + with open(path, 'w') as f: + yaml.safe_dump(d, f, sort_keys=False) + + print(f'Injected PIPELINE_COMMIT_SHA override into {modified} target_test jobs') + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( + 'pipeline_yaml', + nargs='?', + default='target_test_child_pipeline.yml', + help='Path to the generated target_test child pipeline YAML to patch in place.', + ) + args = parser.parse_args() + patch(args.pipeline_yaml) + + +if __name__ == '__main__': + main()