mirror of
https://github.com/espressif/esp-idf.git
synced 2026-10-02 03:00:34 +03:00
ci(buildv2): add build/QEMU/Linux jobs for Build system v2
Add the parallel set of CI jobs that exercise the cmakev2 path end-to-end via the IDF_BUILD_V2 shim. All jobs are gated on the `buildv2` MR label with `allow_failure: true` so unrelated MRs do not pay for the extra build matrix. - `.gitlab/ci/build.yml`: add the buildv2 child build pipeline that re-runs the full app build matrix with IDF_BUILD_V2=y, plus the Windows buildv2 build job. The generator script injects IDF_BUILD_V2 and a PIPELINE_COMMIT_SHA suffix into each child job and broadens target_test job filters so v1 and buildv2 artifacts upload to distinct s3 cache paths and each child pipeline tests its own binary. - `.gitlab/ci/host-test.yml`: add the QEMU (esp32, esp32c3) and Linux pytest buildv2 jobs. - `.idf_build_apps.toml`: parameterize build_dir with BUILDV2_DIR_SUFFIX (set to _v2 only on the buildv2 pipeline).
This commit is contained in:
@@ -0,0 +1,76 @@
|
||||
# SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Patch the generated Build system v2 build child pipeline in place.
|
||||
|
||||
The buildv2 build child pipeline is generated by ``idf-ci gitlab build-child-pipeline``
|
||||
from the same manifest as the default pipeline. This script post-processes that
|
||||
generated YAML so the cmakev2 path is exercised end to end:
|
||||
|
||||
1. Inject ``IDF_BUILD_V2`` into each child build job so cmake activates the
|
||||
cmakev2 shim.
|
||||
|
||||
2. Inject ``PIPELINE_COMMIT_SHA: ${PIPELINE_COMMIT_SHA}_buildv2`` into each child
|
||||
build job. idf-ci reads ``PIPELINE_COMMIT_SHA`` when computing the s3
|
||||
upload/download prefix (``project/<sha>/...``). Suffixing it for v2 jobs only
|
||||
routes v2 binaries to a distinct s3 key namespace without renaming build_dir.
|
||||
|
||||
3. Redefine ``generate_pytest_child_pipeline`` so its script also runs
|
||||
``patch_buildv2_target_test_pipeline.py`` against the emitted
|
||||
``target_test_child_pipeline.yml`` and injects the same
|
||||
``PIPELINE_COMMIT_SHA`` override into each target_test job. Without this, the
|
||||
v2 target_test pipeline would still download from the default pipeline's s3
|
||||
keys (race-determined, mostly cmakev1 binaries).
|
||||
"""
|
||||
|
||||
import argparse
|
||||
|
||||
import yaml
|
||||
|
||||
PIPELINE_COMMIT_SHA_V2 = '${PIPELINE_COMMIT_SHA}_buildv2'
|
||||
|
||||
|
||||
def patch(path: str) -> None:
|
||||
with open(path) as f:
|
||||
d = yaml.safe_load(f)
|
||||
|
||||
injected = []
|
||||
for k, v in d.items():
|
||||
if isinstance(v, dict) and 'extends' in v:
|
||||
v.setdefault('variables', {})['IDF_BUILD_V2'] = '1'
|
||||
v['variables']['PIPELINE_COMMIT_SHA'] = PIPELINE_COMMIT_SHA_V2
|
||||
injected.append(k)
|
||||
|
||||
# Redefine generate_pytest_child_pipeline (included from
|
||||
# tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml) so its script
|
||||
# also rewrites target_test_child_pipeline.yml. GitLab CI merges values at the
|
||||
# keyword level, so only `script` is overridden; `needs`, `artifacts`,
|
||||
# `image`, etc. come from the included version.
|
||||
d['generate_pytest_child_pipeline'] = {
|
||||
'script': [
|
||||
'python tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py',
|
||||
'python tools/ci/dynamic_pipelines/scripts/patch_buildv2_target_test_pipeline.py '
|
||||
'target_test_child_pipeline.yml',
|
||||
],
|
||||
}
|
||||
|
||||
with open(path, 'w') as f:
|
||||
yaml.safe_dump(d, f, sort_keys=False)
|
||||
|
||||
print('Injected IDF_BUILD_V2 + PIPELINE_COMMIT_SHA into build jobs:', injected)
|
||||
print('Redefined generate_pytest_child_pipeline with post-processing step')
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument(
|
||||
'pipeline_yaml',
|
||||
nargs='?',
|
||||
default='buildv2_child_pipeline.yml',
|
||||
help='Path to the generated buildv2 build child pipeline YAML to patch in place.',
|
||||
)
|
||||
args = parser.parse_args()
|
||||
patch(args.pipeline_yaml)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,56 @@
|
||||
# SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Patch the generated Build system v2 target_test child pipeline in place.
|
||||
|
||||
Inject ``PIPELINE_COMMIT_SHA: ${PIPELINE_COMMIT_SHA}_buildv2`` into every
|
||||
target_test job so idf-ci downloads the cmakev2 binaries from the distinct s3
|
||||
key namespace produced by the v2 build jobs, instead of racing with the default
|
||||
pipeline's binaries.
|
||||
|
||||
Run from generate_pytest_child_pipeline (see patch_buildv2_child_pipeline.py),
|
||||
after generate_target_test_child_pipeline.py has emitted the YAML.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
|
||||
import yaml
|
||||
|
||||
PIPELINE_COMMIT_SHA_V2 = '${PIPELINE_COMMIT_SHA}_buildv2'
|
||||
|
||||
# Target test jobs use extends: .dynamic_target_test_template, so their script:
|
||||
# comes from the template, not their own definition. These top-level keys are
|
||||
# pipeline configuration rather than jobs and must be skipped.
|
||||
SKIP_KEYS = {'include', 'workflow', 'default', 'variables', 'stages'}
|
||||
|
||||
|
||||
def patch(path: str) -> None:
|
||||
with open(path) as f:
|
||||
d = yaml.safe_load(f)
|
||||
|
||||
modified = 0
|
||||
for k, v in d.items():
|
||||
if k in SKIP_KEYS or not isinstance(v, dict):
|
||||
continue
|
||||
v.setdefault('variables', {})['PIPELINE_COMMIT_SHA'] = PIPELINE_COMMIT_SHA_V2
|
||||
modified += 1
|
||||
|
||||
with open(path, 'w') as f:
|
||||
yaml.safe_dump(d, f, sort_keys=False)
|
||||
|
||||
print(f'Injected PIPELINE_COMMIT_SHA override into {modified} target_test jobs')
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument(
|
||||
'pipeline_yaml',
|
||||
nargs='?',
|
||||
default='target_test_child_pipeline.yml',
|
||||
help='Path to the generated target_test child pipeline YAML to patch in place.',
|
||||
)
|
||||
args = parser.parse_args()
|
||||
patch(args.pipeline_yaml)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user