ci(buildv2): add build/QEMU/Linux jobs for Build system v2

Add the parallel set of CI jobs that exercise the cmakev2 path end-to-end
via the IDF_BUILD_V2 shim. All jobs are gated on the `buildv2` MR label
with `allow_failure: true` so unrelated MRs do not pay for the extra
build matrix.

- `.gitlab/ci/build.yml`: add the buildv2 child build pipeline that
  re-runs the full app build matrix with IDF_BUILD_V2=y, plus the
  Windows buildv2 build job. The generator script injects IDF_BUILD_V2
  and a PIPELINE_COMMIT_SHA suffix into each child job and broadens
  target_test job filters so v1 and buildv2 artifacts upload to distinct
  s3 cache paths and each child pipeline tests its own binary.
- `.gitlab/ci/host-test.yml`: add the QEMU (esp32, esp32c3) and Linux
  pytest buildv2 jobs.
- `.idf_build_apps.toml`: parameterize build_dir with BUILDV2_DIR_SUFFIX
  (set to _v2 only on the buildv2 pipeline).
This commit is contained in:
Sudeep Mohanty
2026-07-03 19:18:42 +02:00
parent 169aded5b1
commit abc963d6ca
5 changed files with 254 additions and 12 deletions
@@ -0,0 +1,76 @@
# SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
"""Patch the generated Build system v2 build child pipeline in place.
The buildv2 build child pipeline is generated by ``idf-ci gitlab build-child-pipeline``
from the same manifest as the default pipeline. This script post-processes that
generated YAML so the cmakev2 path is exercised end to end:
1. Inject ``IDF_BUILD_V2`` into each child build job so cmake activates the
cmakev2 shim.
2. Inject ``PIPELINE_COMMIT_SHA: ${PIPELINE_COMMIT_SHA}_buildv2`` into each child
build job. idf-ci reads ``PIPELINE_COMMIT_SHA`` when computing the s3
upload/download prefix (``project/<sha>/...``). Suffixing it for v2 jobs only
routes v2 binaries to a distinct s3 key namespace without renaming build_dir.
3. Redefine ``generate_pytest_child_pipeline`` so its script also runs
``patch_buildv2_target_test_pipeline.py`` against the emitted
``target_test_child_pipeline.yml`` and injects the same
``PIPELINE_COMMIT_SHA`` override into each target_test job. Without this, the
v2 target_test pipeline would still download from the default pipeline's s3
keys (race-determined, mostly cmakev1 binaries).
"""
import argparse
import yaml
PIPELINE_COMMIT_SHA_V2 = '${PIPELINE_COMMIT_SHA}_buildv2'
def patch(path: str) -> None:
with open(path) as f:
d = yaml.safe_load(f)
injected = []
for k, v in d.items():
if isinstance(v, dict) and 'extends' in v:
v.setdefault('variables', {})['IDF_BUILD_V2'] = '1'
v['variables']['PIPELINE_COMMIT_SHA'] = PIPELINE_COMMIT_SHA_V2
injected.append(k)
# Redefine generate_pytest_child_pipeline (included from
# tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml) so its script
# also rewrites target_test_child_pipeline.yml. GitLab CI merges values at the
# keyword level, so only `script` is overridden; `needs`, `artifacts`,
# `image`, etc. come from the included version.
d['generate_pytest_child_pipeline'] = {
'script': [
'python tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py',
'python tools/ci/dynamic_pipelines/scripts/patch_buildv2_target_test_pipeline.py '
'target_test_child_pipeline.yml',
],
}
with open(path, 'w') as f:
yaml.safe_dump(d, f, sort_keys=False)
print('Injected IDF_BUILD_V2 + PIPELINE_COMMIT_SHA into build jobs:', injected)
print('Redefined generate_pytest_child_pipeline with post-processing step')
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument(
'pipeline_yaml',
nargs='?',
default='buildv2_child_pipeline.yml',
help='Path to the generated buildv2 build child pipeline YAML to patch in place.',
)
args = parser.parse_args()
patch(args.pipeline_yaml)
if __name__ == '__main__':
main()
@@ -0,0 +1,56 @@
# SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
"""Patch the generated Build system v2 target_test child pipeline in place.
Inject ``PIPELINE_COMMIT_SHA: ${PIPELINE_COMMIT_SHA}_buildv2`` into every
target_test job so idf-ci downloads the cmakev2 binaries from the distinct s3
key namespace produced by the v2 build jobs, instead of racing with the default
pipeline's binaries.
Run from generate_pytest_child_pipeline (see patch_buildv2_child_pipeline.py),
after generate_target_test_child_pipeline.py has emitted the YAML.
"""
import argparse
import yaml
PIPELINE_COMMIT_SHA_V2 = '${PIPELINE_COMMIT_SHA}_buildv2'
# Target test jobs use extends: .dynamic_target_test_template, so their script:
# comes from the template, not their own definition. These top-level keys are
# pipeline configuration rather than jobs and must be skipped.
SKIP_KEYS = {'include', 'workflow', 'default', 'variables', 'stages'}
def patch(path: str) -> None:
with open(path) as f:
d = yaml.safe_load(f)
modified = 0
for k, v in d.items():
if k in SKIP_KEYS or not isinstance(v, dict):
continue
v.setdefault('variables', {})['PIPELINE_COMMIT_SHA'] = PIPELINE_COMMIT_SHA_V2
modified += 1
with open(path, 'w') as f:
yaml.safe_dump(d, f, sort_keys=False)
print(f'Injected PIPELINE_COMMIT_SHA override into {modified} target_test jobs')
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument(
'pipeline_yaml',
nargs='?',
default='target_test_child_pipeline.yml',
help='Path to the generated target_test child pipeline YAML to patch in place.',
)
args = parser.parse_args()
patch(args.pipeline_yaml)
if __name__ == '__main__':
main()