Merge branch 'task/buildv2_full_pipeline_v6.1' into 'release/v6.1'

Enable full buildv2 pipeline (v6.1) See merge request espressif/esp-idf!50351
2026-10-02 03:00:34 +03:00 · 2026-07-17 18:20:09 +08:00
parent 522ee6dd74 0a89c2e432
commit a43c51681a
15 changed files with 418 additions and 68 deletions
@@ -0,0 +1,76 @@
+# SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+"""Patch the generated Build system v2 build child pipeline in place.
+
+The buildv2 build child pipeline is generated by ``idf-ci gitlab build-child-pipeline``
+from the same manifest as the default pipeline. This script post-processes that
+generated YAML so the cmakev2 path is exercised end to end:
+
+1. Inject ``IDF_BUILD_V2`` into each child build job so cmake activates the
+   cmakev2 shim.
+
+2. Inject ``PIPELINE_COMMIT_SHA: ${PIPELINE_COMMIT_SHA}_buildv2`` into each child
+   build job. idf-ci reads ``PIPELINE_COMMIT_SHA`` when computing the s3
+   upload/download prefix (``project/<sha>/...``). Suffixing it for v2 jobs only
+   routes v2 binaries to a distinct s3 key namespace without renaming build_dir.
+
+3. Redefine ``generate_pytest_child_pipeline`` so its script also runs
+   ``patch_buildv2_target_test_pipeline.py`` against the emitted
+   ``target_test_child_pipeline.yml`` and injects the same
+   ``PIPELINE_COMMIT_SHA`` override into each target_test job. Without this, the
+   v2 target_test pipeline would still download from the default pipeline's s3
+   keys (race-determined, mostly cmakev1 binaries).
+"""
+
+import argparse
+
+import yaml
+
+PIPELINE_COMMIT_SHA_V2 = '${PIPELINE_COMMIT_SHA}_buildv2'
+
+
+def patch(path: str) -> None:
+    with open(path) as f:
+        d = yaml.safe_load(f)
+
+    injected = []
+    for k, v in d.items():
+        if isinstance(v, dict) and 'extends' in v:
+            v.setdefault('variables', {})['IDF_BUILD_V2'] = '1'
+            v['variables']['PIPELINE_COMMIT_SHA'] = PIPELINE_COMMIT_SHA_V2
+            injected.append(k)
+
+    # Redefine generate_pytest_child_pipeline (included from
+    # tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml) so its script
+    # also rewrites target_test_child_pipeline.yml. GitLab CI merges values at the
+    # keyword level, so only `script` is overridden; `needs`, `artifacts`,
+    # `image`, etc. come from the included version.
+    d['generate_pytest_child_pipeline'] = {
+        'script': [
+            'python tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py',
+            'python tools/ci/dynamic_pipelines/scripts/patch_buildv2_target_test_pipeline.py '
+            'target_test_child_pipeline.yml',
+        ],
+    }
+
+    with open(path, 'w') as f:
+        yaml.safe_dump(d, f, sort_keys=False)
+
+    print('Injected IDF_BUILD_V2 + PIPELINE_COMMIT_SHA into build jobs:', injected)
+    print('Redefined generate_pytest_child_pipeline with post-processing step')
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument(
+        'pipeline_yaml',
+        nargs='?',
+        default='buildv2_child_pipeline.yml',
+        help='Path to the generated buildv2 build child pipeline YAML to patch in place.',
+    )
+    args = parser.parse_args()
+    patch(args.pipeline_yaml)
+
+
+if __name__ == '__main__':
+    main()
@@ -0,0 +1,56 @@
+# SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+"""Patch the generated Build system v2 target_test child pipeline in place.
+
+Inject ``PIPELINE_COMMIT_SHA: ${PIPELINE_COMMIT_SHA}_buildv2`` into every
+target_test job so idf-ci downloads the cmakev2 binaries from the distinct s3
+key namespace produced by the v2 build jobs, instead of racing with the default
+pipeline's binaries.
+
+Run from generate_pytest_child_pipeline (see patch_buildv2_child_pipeline.py),
+after generate_target_test_child_pipeline.py has emitted the YAML.
+"""
+
+import argparse
+
+import yaml
+
+PIPELINE_COMMIT_SHA_V2 = '${PIPELINE_COMMIT_SHA}_buildv2'
+
+# Target test jobs use extends: .dynamic_target_test_template, so their script:
+# comes from the template, not their own definition. These top-level keys are
+# pipeline configuration rather than jobs and must be skipped.
+SKIP_KEYS = {'include', 'workflow', 'default', 'variables', 'stages'}
+
+
+def patch(path: str) -> None:
+    with open(path) as f:
+        d = yaml.safe_load(f)
+
+    modified = 0
+    for k, v in d.items():
+        if k in SKIP_KEYS or not isinstance(v, dict):
+            continue
+        v.setdefault('variables', {})['PIPELINE_COMMIT_SHA'] = PIPELINE_COMMIT_SHA_V2
+        modified += 1
+
+    with open(path, 'w') as f:
+        yaml.safe_dump(d, f, sort_keys=False)
+
+    print(f'Injected PIPELINE_COMMIT_SHA override into {modified} target_test jobs')
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument(
+        'pipeline_yaml',
+        nargs='?',
+        default='target_test_child_pipeline.yml',
+        help='Path to the generated target_test child pipeline YAML to patch in place.',
+    )
+    args = parser.parse_args()
+    patch(args.pipeline_yaml)
+
+
+if __name__ == '__main__':
+    main()
@@ -24,6 +24,7 @@ warning: unknown kconfig symbol 'CTRL_BTDM_MODEM_SLEEP' assigned to*
 warning: unknown kconfig symbol 'ESP_DEFAULT_CPU_FREQ_MHZ_80' assigned to 'y' in .*/examples/system/deep_sleep/*
 warning: unknown kconfig symbol 'ESP_DEFAULT_CPU_FREQ_MHZ_80' assigned to 'y' in .*/examples/system/light_sleep/*
 warning: unknown kconfig symbol 'ESP_DEFAULT_CPU_FREQ_MHZ_80' assigned to 'y' in .*/examples/wifi/power_save/*
+warning: unknown kconfig symbol 'ESP_WIFI_SLP_IRAM_OPT' assigned to 'n' in .*/components/spi_flash/test_apps/flash_suspend/*
 warning: unknown kconfig symbol 'ESP32_REV_MIN_3' assigned to 'y' in .*/examples/system/ota/simple_ota_example/sdkconfig.ci.on_update_no_sb_rsa
 warning: unknown kconfig symbol 'ESP32_REV_MIN' assigned to '3' in .*/examples/system/ota/simple_ota_example/sdkconfig.ci.on_update_no_sb_rsa
 warning: unknown kconfig symbol 'ESP32H4_RTC_CLK_CAL_CYCLES' assigned to '576' in .*/examples/bluetooth/nimble/blecent/sdkconfig.defaults.esp32h4
@@ -49,3 +50,8 @@ warning: unknown kconfig symbol 'ETHERNET_PHY_RST_GPIO' assigned to '5' in .*/ex
 warning: unknown kconfig symbol 'ETHERNET_PHY_ADDR' assigned to '1' in .*/examples/protocols/esp_http_client/sdkconfig.ci.*
 warning: \S+ changed binding to STB_WEAK
 warning: unknown kconfig symbol 'VFS_SUPPORT_IO' assigned to 'n' in .*/tools/test_apps/system/g1_components/sdkconfig.defaults
+warning: unknown kconfig symbol 'VFS_SUPPORT_IO' assigned to 'n' in .*/components/esp_driver_dma/test_apps/dma/sdkconfig\S*
+warning: unknown kconfig symbol 'VFS_SUPPORT_IO' assigned to 'n' in .*/tools/test_apps/system/panic/coredump/sdkconfig\S*
+warning: unknown kconfig symbol 'ESP_WIFI_TASK_PINNED_TO_CORE_1' assigned to 'y' in .*/tools/test_apps/system/esp_intr_dump/sdkconfig\S*
+warning: unknown kconfig symbol 'VFS_SUPPORT_IO' assigned to 'n' in .*/components/spi_flash/test_apps/flash_suspend/sdkconfig\S*
+warning: unknown kconfig symbol 'ESP_TLS_USE_DS_PERIPHERAL' assigned to 'n' in .*/components/mbedtls/test_apps/mbedtls_ut/sdkconfig\S*
@@ -38,8 +38,9 @@ function(__generate_gdbinit)
    set(symbols_gdbinit_path ${gdbinit_dir}/symbols)
    set(py_extensions_gdbinit_path ${gdbinit_dir}/py_extensions)
    set(connect_gdbinit_path ${gdbinit_dir}/connect)
-    idf_build_get_property(PROJECT_EXECUTABLE EXECUTABLE)
-    set(application_elf ${BUILD_DIR}/${PROJECT_EXECUTABLE})
+    # EXECUTABLE is a CMake target name; the on-disk elf is EXECUTABLE_NAME + ".elf".
+    idf_build_get_property(PROJECT_EXECUTABLE_NAME EXECUTABLE_NAME)
+    set(application_elf ${BUILD_DIR}/${PROJECT_EXECUTABLE_NAME}.elf)

    file(MAKE_DIRECTORY ${gdbinit_dir})

@@ -36,35 +36,21 @@ set(MENUCONFIG_INLINE_MIN_KCONFIG_VERSION "3.9.0")
 function(__init_kconfig)
    idf_build_get_property(idf_path IDF_PATH)

-    # Initialize SDKCONFIG and SDKCONFIG_DEFAULTS build properties using environment
-    # variables, CMake cache variables, or default values.
-    if(EXISTS "${CMAKE_SOURCE_DIR}/sdkconfig.defaults")
-        set(sdkconfig_defaults "${CMAKE_SOURCE_DIR}/sdkconfig.defaults")
-    else()
-        set(sdkconfig_defaults "")
-    endif()
-
+    # Initialize the SDKCONFIG file path build property.
    __get_default_value(VARIABLE SDKCONFIG
                        DEFAULT "${CMAKE_SOURCE_DIR}/sdkconfig"
                        OUTPUT sdkconfig)
-    __get_default_value(VARIABLE SDKCONFIG_DEFAULTS
-                        DEFAULT "${sdkconfig_defaults}"
-                        OUTPUT sdkconfig_defaults)
-
    __get_absolute_paths(PATHS "${sdkconfig}" OUTPUT sdkconfig)
-    __get_absolute_paths(PATHS "${sdkconfig_defaults}" OUTPUT sdkconfig_defaults)
-
-    set(sdkconfig_defaults_checked "")
-    foreach(sdkconfig_default ${sdkconfig_defaults})
-        if(NOT EXISTS "${sdkconfig_default}")
-            idf_die("SDKCONFIG_DEFAULTS '${sdkconfig_default}' does not exist.")
-        endif()
-        list(APPEND sdkconfig_defaults_checked ${sdkconfig_default})
-    endforeach()
-
    idf_build_set_property(SDKCONFIG "${sdkconfig}")
    idf_build_set_property(__SDKCONFIG_ORIG "${sdkconfig}")
-    idf_build_set_property(SDKCONFIG_DEFAULTS "${sdkconfig_defaults_checked}")
+
+    # Initialize the SDKCONFIG_DEFAULTS build property. __init_idf_target()
+    # reads this property to detect the target from the defaults files, so it
+    # must be resolved here. It is resolved again from __generate_sdkconfig()
+    # to pick up a SDKCONFIG_DEFAULTS variable assigned after
+    # include(project.cmake) but before project().
+    __resolve_sdkconfig_defaults()
+
    idf_build_set_property(GENERATE_SDKCONFIG 1)

    # Setup ESP-IDF root Kconfig and sdkconfig.rename files.
@@ -78,6 +64,54 @@ function(__init_kconfig)
    idf_build_set_property(CONFIG_DIR "${config_dir}")
 endfunction()

+#[[
+    __resolve_sdkconfig_defaults()
+
+    Resolve SDKCONFIG_DEFAULTS into a list of absolute, existing paths
+    and store it on the SDKCONFIG_DEFAULTS build property.
+
+    A SDKCONFIG_DEFAULTS variable (or environment variable) takes precedence
+    and is (re)resolved onto the build property. Otherwise a value already
+    accumulated on the build property (e.g. via
+    ``idf_build_set_property(SDKCONFIG_DEFAULTS "<path>" APPEND)``) is
+    preserved, and only when nothing has been set does it fall back to the
+    project's sdkconfig.defaults file. This makes the helper idempotent and
+    safe to call both early (from __init_kconfig) and again at
+    sdkconfig-generation time.
+#]]
+function(__resolve_sdkconfig_defaults)
+    if(NOT DEFINED SDKCONFIG_DEFAULTS AND NOT DEFINED ENV{SDKCONFIG_DEFAULTS})
+        # No SDKCONFIG_DEFAULTS variable override: keep a value already set on
+        # the build property; only fall back to the default when unset.
+        idf_build_get_property(existing SDKCONFIG_DEFAULTS)
+        if(existing)
+            return()
+        endif()
+    endif()
+
+    if(EXISTS "${CMAKE_SOURCE_DIR}/sdkconfig.defaults")
+        set(sdkconfig_defaults_default "${CMAKE_SOURCE_DIR}/sdkconfig.defaults")
+    else()
+        set(sdkconfig_defaults_default "")
+    endif()
+
+    __get_default_value(VARIABLE SDKCONFIG_DEFAULTS
+                        DEFAULT "${sdkconfig_defaults_default}"
+                        OUTPUT sdkconfig_defaults)
+
+    __get_absolute_paths(PATHS "${sdkconfig_defaults}" OUTPUT sdkconfig_defaults)
+
+    set(sdkconfig_defaults_checked "")
+    foreach(sdkconfig_default IN LISTS sdkconfig_defaults)
+        if(NOT EXISTS "${sdkconfig_default}")
+            idf_die("SDKCONFIG_DEFAULTS '${sdkconfig_default}' does not exist.")
+        endif()
+        list(APPEND sdkconfig_defaults_checked "${sdkconfig_default}")
+    endforeach()
+
+    idf_build_set_property(SDKCONFIG_DEFAULTS "${sdkconfig_defaults_checked}")
+endfunction()
+
 #[[
    __create_sdkconfig_orig_copy()

@@ -187,6 +221,13 @@ endfunction()
    4. Generate all output files (sdkconfig.h, sdkconfig.cmake, etc.)
 #]]
 function(__generate_sdkconfig)
+    # Re-resolve SDKCONFIG_DEFAULTS so that a value assigned to the
+    # SDKCONFIG_DEFAULTS variable after include(project.cmake) but before
+    # project() is honored. The helper is idempotent: it only re-reads when
+    # the SDKCONFIG_DEFAULTS variable (or environment variable) is set, and
+    # otherwise leaves a value accumulated on the build property untouched.
+    __resolve_sdkconfig_defaults()
+
    # Collect Kconfig files from discovered components
    __consolidate_component_kconfig_files()

@@ -0,0 +1,2 @@
+# The Wi-Fi task otherwise runs on core 0, which reserves CPU0 interrupt 0 for WMAC, so the "test_esp_intr_dump_expected_output" case no longer finds "Used: RTC_CORE" on interrupt 0 and fails. Pin the Wi-Fi task to core 1 to keep interrupt 0 free.
+CONFIG_ESP_WIFI_TASK_PINNED_TO_CORE_1=y
@@ -16,3 +16,6 @@ CONFIG_FREERTOS_USE_TRACE_FACILITY=y

 # Increase main task stack size
 CONFIG_ESP_MAIN_TASK_STACK_SIZE=4096
+
+# CONFIG_VFS_SUPPORT_IO resolves to its default (y) in this test's Kconfig closure; the extra console I/O task makes the core dump header declare more bytes than are emitted, so espcoredump.py info_corefile fails to parse it in the "test_hw_stack_guard_cpu0" case. Disable it.
+CONFIG_VFS_SUPPORT_IO=n