Merge branch 'feat/cmakev2_ulp_legacy_shim' into 'master'

feat(ulp): build all legacy ULP projects through CMakev2

See merge request espressif/esp-idf!50143

tools/ci/dynamic_pipelines/scripts/patch_buildv2_child_pipeline.py

@@ -6,8 +6,8 @@ The buildv2 build child pipeline is generated by ``idf-ci gitlab build-child-pip
 from the same manifest as the default pipeline. This script post-processes that
 generated YAML so the cmakev2 path is exercised end to end:
 
-1. Point child build jobs at the buildv2 generator artifacts, then inject
-   ``IDF_BUILD_V2`` so cmake activates the cmakev2 shim.
+1. Inject ``IDF_BUILD_V2`` into each child build job so cmake activates the
+   cmakev2 shim.
 
 2. Inject ``PIPELINE_COMMIT_SHA: ${PIPELINE_COMMIT_SHA}_buildv2`` into each child
    build job. idf-ci reads ``PIPELINE_COMMIT_SHA`` when computing the s3
@@ -27,13 +27,6 @@ import argparse
 import yaml
 
 PIPELINE_COMMIT_SHA_V2 = '${PIPELINE_COMMIT_SHA}_buildv2'
-BUILDV2_GENERATOR_JOB = 'generate_build_child_pipeline_buildv2'
-
-
-def _use_buildv2_generator_artifacts(job: dict) -> None:
-    for need in job.get('needs', []):
-        if isinstance(need, dict) and need.get('job') == 'generate_build_child_pipeline':
-            need['job'] = BUILDV2_GENERATOR_JOB
 
 
 def patch(path: str) -> None:
@@ -43,7 +36,6 @@ def patch(path: str) -> None:
     injected = []
     for k, v in d.items():
         if isinstance(v, dict) and 'extends' in v:
-            _use_buildv2_generator_artifacts(v)
             v.setdefault('variables', {})['IDF_BUILD_V2'] = '1'
             v['variables']['PIPELINE_COMMIT_SHA'] = PIPELINE_COMMIT_SHA_V2
             injected.append(k)

tools/cmakev2/build.cmake

@@ -39,7 +39,12 @@ function(idf_build_set_property property value)
     cmake_parse_arguments(ARG "${options}" "${one_value}" "${multi_value}" ${ARGN})
 
     if("${property}" STREQUAL MINIMAL_BUILD)
-        idf_warn("Build property 'MINIMAL_BUILD' is obsolete and will be ignored")
+        # TODO: Remove this MINIMAL_BUILD compatibility block once CMake v1 is fully deprecated.
+        # V1 compatibility shims may still set MINIMAL_BUILD; warn only for native CMake v2 projects.
+        idf_build_get_property(_v1_compat __V1_COMPAT_SHIM)
+        if(NOT _v1_compat)
+            idf_warn("Build property 'MINIMAL_BUILD' is obsolete and will be ignored")
+        endif()
     endif()
 
     set(append)
@@ -293,6 +298,25 @@ function(__dump_library_properties libraries)
     endforeach()
 endfunction()
 
+function(__idf_build_link_whole_archive target scope library)
+    idf_build_get_property(linker_type LINKER_TYPE)
+    if(linker_type STREQUAL "GNU")
+        set(link_option "SHELL:-Wl,--whole-archive $<TARGET_FILE:${library}> -Wl,--no-whole-archive")
+    elseif(linker_type STREQUAL "Darwin")
+        set(link_option "SHELL:-Wl,-force_load $<TARGET_FILE:${library}>")
+    elseif(linker_type STREQUAL "ULP_FSM")
+        set(link_option "SHELL:--whole-archive $<TARGET_FILE:${library}> --no-whole-archive")
+    else()
+        target_link_libraries(${target} ${scope} ${library})
+        return()
+    endif()
+
+    # ARGN may contain BEFORE to place the archive ahead of existing link options.
+    # Also link the target normally so CMake tracks its build and relink dependencies.
+    target_link_options(${target} ${ARGN} ${scope} "${link_option}")
+    target_link_libraries(${target} ${scope} ${library})
+endfunction()
+
 #[[api
 .. cmakev2:function:: idf_build_library
 

tools/cmakev2/compat.cmake

@@ -193,7 +193,7 @@ endfunction()
     with the ``PROCESS`` option, it is logical to provide only a single
     ``scriptfile`` as a template.
 #]]
-function(target_linker_script target deptype scriptfiles)
+function(target_linker_script target deptype)
     # The linker script files, templates, and their output filenames are stored
     # only as component properties. The script files are generated and added to
     # the library link interface in the idf_build_library function.
@@ -201,6 +201,10 @@ function(target_linker_script target deptype scriptfiles)
     set(one_value PROCESS FLAGS)
     set(multi_value)
     cmake_parse_arguments(ARG "${options}" "${one_value}" "${multi_value}" ${ARGN})
+    set(scriptfiles ${ARG_UNPARSED_ARGUMENTS})
+    if(NOT scriptfiles)
+        message(FATAL_ERROR "target_linker_script requires at least one linker script file")
+    endif()
     foreach(scriptfile ${scriptfiles})
         get_filename_component(scriptfile "${scriptfile}" ABSOLUTE)
         idf_msg("Adding linker script ${scriptfile}")

tools/cmakev2/component.cmake

@@ -1102,16 +1102,7 @@ function(idf_component_include name)
     elseif("${component_real_target_type}" STREQUAL "STATIC_LIBRARY")
         idf_component_get_property(whole_archive "${component_name}" WHOLE_ARCHIVE)
         if(whole_archive)
-            idf_build_get_property(linker_type LINKER_TYPE)
-            if(linker_type STREQUAL "GNU")
-                target_link_options("${component_interface}" INTERFACE
-                    "SHELL:-Wl,--whole-archive $<TARGET_FILE:${component_real_target}> -Wl,--no-whole-archive")
-                target_link_libraries("${component_interface}" INTERFACE "${component_real_target}")
-            elseif(linker_type STREQUAL "Darwin")
-                target_link_options("${component_interface}" INTERFACE
-                    "SHELL:-Wl,-force_load $<TARGET_FILE:${component_real_target}>")
-                target_link_libraries("${component_interface}" INTERFACE "${component_real_target}")
-            endif()
+            __idf_build_link_whole_archive("${component_interface}" INTERFACE "${component_real_target}")
         else()
             target_link_libraries("${component_interface}" INTERFACE "${component_real_target}")
         endif()

tools/test_apps/system/.build-test-rules.yml

@@ -194,9 +194,6 @@ tools/test_apps/system/ram_loadable_app:
       reason: cannot pass # TODO: IDF-15525
 
 tools/test_apps/system/rtc_mem_reserve:
-  disable:
-    - if: IDF_BUILD_V2 == "1"
-      reason: Legacy ULP apps are covered by CMake v1; buildv2 covers full_subproject ULP apps.
   enable:
     - if: IDF_TARGET in ["esp32p4"]
       reason: only P4 has a potential conflict due to using rtc mem for lp rom data/stack

tools/test_build_system/buildv2/test_ulp.py

@@ -1,11 +1,200 @@
 # SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
+import hashlib
+import os
+import subprocess
 from pathlib import Path
 
 import pytest
 from test_build_system_helpers import IdfPyFunc
 from test_build_system_helpers import replace_in_file
 
+FULL_PROJECT_APIS = (
+    'ulp_project_init',
+    'ulp_project_default',
+    'ulp_build_executable',
+)
+
+LEGACY_PROJECT_APIS = (
+    'ulp_apply_default_options',
+    'ulp_apply_default_sources',
+)
+
+ALL_APIS = FULL_PROJECT_APIS + LEGACY_PROJECT_APIS
+
+
+def _cmake_path(path: Path) -> str:
+    return path.resolve().as_posix()
+
+
+def _write_api_probe_project(project_dir: Path, entry_point: str, add_native_executable: bool) -> None:
+    native_executable = 'add_executable(${ULP_APP_NAME} main.c)' if add_native_executable else ''
+    if entry_point.endswith('ulp_project.cmake'):
+        pre_project_include = f'include({entry_point})'
+        post_project_include = ''
+    else:
+        pre_project_include = ''
+        post_project_include = f'include({entry_point})'
+
+    (project_dir / 'sdkconfig.h').write_text('', encoding='utf-8')
+    (project_dir / 'main.c').write_text('int main(void) { return 0; }\n', encoding='utf-8')
+    (project_dir / 'sdkconfig.cmake').write_text(
+        '\n'.join(
+            (
+                'set(CONFIG_ULP_COPROC_ENABLED y)',
+                'set(CONFIG_ULP_COPROC_TYPE_LP_CORE y)',
+                '',
+            )
+        ),
+        encoding='utf-8',
+    )
+    (project_dir / 'CMakeLists.txt').write_text(
+        f"""
+cmake_minimum_required(VERSION 3.22)
+
+{pre_project_include}
+project(ulp_api_probe C CXX ASM)
+{native_executable}
+
+{post_project_include}
+
+set(api_commands_file "${{CMAKE_BINARY_DIR}}/api_commands.txt")
+file(WRITE "${{api_commands_file}}" "")
+foreach(name IN ITEMS {' '.join(ALL_APIS)})
+    if(COMMAND ${{name}})
+        file(APPEND "${{api_commands_file}}" "${{name}}=1\\n")
+    else()
+        file(APPEND "${{api_commands_file}}" "${{name}}=0\\n")
+    endif()
+endforeach()
+""",
+        encoding='utf-8',
+    )
+
+
+def _read_api_commands(commands_file: Path) -> dict[str, bool]:
+    return {
+        name: value == '1'
+        for name, value in (line.split('=', 1) for line in commands_file.read_text(encoding='utf-8').splitlines())
+    }
+
+
+def _sha256(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+@pytest.mark.parametrize(
+    'entry_point,add_native_executable,expected_available,expected_unavailable',
+    (
+        (
+            '${IDF_PATH}/components/ulp/cmake/ulp_project.cmake',
+            False,
+            FULL_PROJECT_APIS,
+            LEGACY_PROJECT_APIS,
+        ),
+        (
+            'IDFULPProject',
+            True,
+            LEGACY_PROJECT_APIS,
+            FULL_PROJECT_APIS,
+        ),
+    ),
+    ids=('full_project_entry_point', 'legacy_project_entry_point'),
+)
+def test_ulp_cmake_api_availability(
+    tmp_path: Path,
+    entry_point: str,
+    add_native_executable: bool,
+    expected_available: tuple[str, ...],
+    expected_unavailable: tuple[str, ...],
+) -> None:
+    # Full ULP subprojects and legacy ULP child projects intentionally expose
+    # different CMake API surfaces. Configure a minimal child project through
+    # each entry point and verify that only the expected commands exist.
+    idf_path = Path(os.environ['IDF_PATH'])
+    project_dir = tmp_path / 'project'
+    build_dir = tmp_path / 'build'
+    project_dir.mkdir()
+    _write_api_probe_project(project_dir, entry_point, add_native_executable)
+    ulp_cmake_dir = idf_path / 'components' / 'ulp' / 'cmake'
+
+    cmake_args = (
+        'cmake',
+        '-G',
+        'Ninja',
+        '-S',
+        _cmake_path(project_dir),
+        '-B',
+        _cmake_path(build_dir),
+        f'-DCMAKE_MODULE_PATH={_cmake_path(ulp_cmake_dir)}',
+        f'-DCMAKE_TOOLCHAIN_FILE={_cmake_path(ulp_cmake_dir / "toolchain-lp-core-riscv.cmake")}',
+        f'-DIDF_PATH={_cmake_path(idf_path)}',
+        '-DIDF_TARGET=esp32c6',
+        f'-DSDKCONFIG_CMAKE={_cmake_path(project_dir / "sdkconfig.cmake")}',
+        f'-DSDKCONFIG_HEADER={_cmake_path(project_dir / "sdkconfig.h")}',
+        '-D__ULP_BUILDV2=1',
+        '-DULP_APP_NAME=ulp_api_probe',
+        '-DULP_TYPE=lp_core',
+        '-DIDF_BUILD_V2=y',
+        '-DIDF_CUSTOM_TOOLCHAIN=1',
+    )
+    result = subprocess.run(
+        cmake_args,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        check=False,
+    )
+
+    assert result.returncode == 0, result.stdout
+
+    command_availability = _read_api_commands(build_dir / 'api_commands.txt')
+    for name in expected_available:
+        assert command_availability[name], f'{name} should be available'
+    for name in expected_unavailable:
+        assert not command_availability[name], f'{name} should not be available'
+
+
+@pytest.mark.test_app_copy('components/ulp/test_apps/ulp_fsm')
+def test_ulp_embed_binary_builds_v2_child_project(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+    # Legacy ulp_embed_binary() callers are built through the builtin ULP child
+    # project as a CMake v2 subproject. The child references parent-owned ULP
+    # sources by path, so editing those sources must still rebuild the child
+    # artifact on the next parent build.
+    assert test_app_copy.exists()
+
+    idf_py('-DIDF_TARGET=esp32', '-DSDKCONFIG=build/sdkconfig', 'build')
+
+    ulp_binary_dirs = sorted(p for p in Path('build/subprojects').glob('*') if p.is_dir())
+    assert len(ulp_binary_dirs) == 1
+
+    ulp_binary_dir = ulp_binary_dirs[0]
+    ulp_binary = ulp_binary_dir / f'{ulp_binary_dir.name}.bin'
+    assert ulp_binary.exists()
+    original_ulp_binary_hash = _sha256(ulp_binary)
+
+    ulp_source = test_app_copy / 'main/ulp/test_jumps.S'
+    ulp_source.write_text(
+        ulp_source.read_text(encoding='utf-8')
+        + '\n\t.global rebuild_dependency_probe\n'
+        + 'rebuild_dependency_probe:\n'
+        + '\t.long 0x1234\n',
+        encoding='utf-8',
+    )
+
+    idf_py('build')
+    assert _sha256(ulp_binary) != original_ulp_binary_hash
+
+
+@pytest.mark.test_app_copy('examples/system/ulp/ulp_riscv/interrupts')
+def test_ulp_embed_binary_uses_parent_project_config(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
+    # Legacy ulp_embed_binary() sources are compiled as part of the builtin ULP
+    # child project, but still rely on project-local CONFIG_* symbols from the
+    # parent app's Kconfig.projbuild.
+    assert test_app_copy.exists()
+
+    idf_py('-DIDF_TARGET=esp32s2', '-DSDKCONFIG=build/sdkconfig', 'build')
+
 
 @pytest.mark.test_app_copy('tools/test_apps/system/ulp/full_subproject/lp_core')
 def test_ulp_full_project_regenerates_config_from_parent_sdkconfig(idf_py: IdfPyFunc, test_app_copy: Path) -> None:

tools/test_build_system/conftest.py

@@ -346,8 +346,11 @@ def idf_py_terminal_env() -> typing.Generator[None, None, None]:
 
 
 @pytest.fixture(name='default_idf_env')
-def fixture_default_idf_env() -> EnvDict:
-    return get_idf_build_env(os.environ['IDF_PATH'])  # type: ignore
+def fixture_default_idf_env(request: FixtureRequest) -> EnvDict:
+    env = get_idf_build_env(os.environ['IDF_PATH'])  # type: ignore
+    if request.config.getoption('buildv2', False):
+        env['IDF_BUILD_V2'] = '1'
+    return env
 
 
 @pytest.fixture

tools/test_build_system/test_common.py

@@ -72,9 +72,12 @@ def get_subdirs_absolute_paths(path: Path) -> list[str]:
 
 @pytest.mark.usefixtures('test_app_copy')
 @pytest.mark.test_app_copy('examples/get-started/blink')
-def test_compile_commands_json_updated_by_reconfigure(idf_py: IdfPyFunc) -> None:
+def test_compile_commands_json_updated_by_reconfigure(idf_py: IdfPyFunc, request: pytest.FixtureRequest) -> None:
     output = idf_py('reconfigure')
-    assert 'Building ESP-IDF components for target esp32' in output.stdout
+    if request.config.getoption('buildv2', False):
+        assert 'IDF Build System V2 (cmakev2) activated' in output.stdout
+    else:
+        assert 'Building ESP-IDF components for target esp32' in output.stdout
     snapshot_1 = get_snapshot(['build/compile_commands.json'])
     snapshot_2 = get_snapshot(['build/compile_commands.json'])
     snapshot_2.assert_same(snapshot_1)