fix(tools): Windows builds when paths contain spaces (prefix_map, ldgen, CI)

- Harden prefix_map.cmake for paths with spaces on Windows - ldgen: fragments-list-file, list normalization, CMake integration (tools/cmake + cmakev2) - CI exclude list for check_tools; test_spaces bundle tweak Made-with: Cursor
2026-08-18 06:35:35 +03:00 · 2026-04-21 09:53:58 +02:00
parent 424a685692
commit 2e5d930e6b
8 changed files with 80 additions and 21 deletions
--- a/.gitlab/ci/test-win.yml
+++ b/.gitlab/ci/test-win.yml
@@ -76,7 +76,7 @@ pytest_build_system_win:
  extends:
    - .test_build_system_template_win
    - .rules:labels:windows_pytest_build_system
-  parallel: 10
+  parallel: 6

 pytest_build_system_win_minimal_cmake:
  extends:
@@ -102,7 +102,7 @@ pytest_buildv2_system_win:
  extends:
    - .test_build_system_template_win
    - .rules:labels:buildv2
-  parallel: 10
+  parallel: 6
  script:
    - cd tools\test_build_system
    - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
--- a/tools/cmake/ldgen.cmake
+++ b/tools/cmake/ldgen.cmake
@@ -187,7 +187,7 @@ function(__ldgen_create_target exe_target)
        "${build_dir}/ldgen_libraries.in"
        "${build_dir}/ldgen_libraries")

-    idf_build_get_property(ldgen_fragment_files __LDGEN_FRAGMENT_FILES GENERATOR_EXPRESSION)
+    idf_build_get_property(ldgen_fragment_files __LDGEN_FRAGMENT_FILES)

    # Create command to invoke the linker script generator tool.
    idf_build_get_property(sdkconfig SDKCONFIG)
@@ -214,11 +214,21 @@ function(__ldgen_create_target exe_target)
        set(mutable_libs_option "--mutable-libraries-file" "${mutable_libs_path}")
    endif()

+    # Write fragment paths to a file (one per line) and pass the file to ldgen,
+    # mirroring the --libraries-file pattern. Avoids cmd/CRT argv-quoting of a
+    # long semicolon-separated list with paths containing spaces and backslashes
+    # on Windows (test_spaces_bundle4).
+    set(ldgen_fragments_file "${build_dir}/ldgen_fragments")
+    list(JOIN ldgen_fragment_files "\n" ldgen_fragments_str)
+    file(WRITE "${ldgen_fragments_file}" "${ldgen_fragments_str}")
+    set_property(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+        APPEND PROPERTY ADDITIONAL_CLEAN_FILES
+        "${ldgen_fragments_file}")
    add_custom_command(
        OUTPUT ${output}
        COMMAND ${python} "${idf_path}/tools/ldgen/ldgen.py"
        --config    "${sdkconfig}"
-        --fragments-list "${ldgen_fragment_files}"
+        --fragments-list-file "${ldgen_fragments_file}"
        --input     "${template}"
        --output    "${output}"
        --kconfig   "${root_kconfig}"
--- a/tools/cmake/prefix_map.cmake
+++ b/tools/cmake/prefix_map.cmake
@@ -5,6 +5,10 @@
 # to fixed names. This is used when reproducible builds are required.
 # This function also creates a gdbinit file for the debugger to
 # remap the substituted paths back to the real paths in the filesystem.
+#
+# On Windows hosts, very long compile commands are avoided by enabling Ninja response
+# files in toolchain.cmake (CMAKE_NINJA_FORCE_RESPONSE_FILE), not by dropping these
+# options when paths contain spaces.
 function(__generate_prefix_map compile_options_var)
    set(compile_options)
    set(gdbinit_dir ${BUILD_DIR}/gdbinit)
@@ -13,27 +17,38 @@ function(__generate_prefix_map compile_options_var)
    idf_build_get_property(build_components BUILD_COMPONENTS)

    if(CONFIG_COMPILER_HIDE_PATHS_MACROS)
-        list(APPEND compile_options "-fmacro-prefix-map=${CMAKE_SOURCE_DIR}=.")
-        list(APPEND compile_options "-fmacro-prefix-map=${idf_path}=/IDF")
+        file(TO_CMAKE_PATH "${CMAKE_SOURCE_DIR}" _src_dir)
+        file(TO_CMAKE_PATH "${idf_path}" _idf_path)
+        list(APPEND compile_options "-fmacro-prefix-map=${_src_dir}=.")
+        list(APPEND compile_options "-fmacro-prefix-map=${_idf_path}=/IDF")
    endif()

    if(CONFIG_APP_REPRODUCIBLE_BUILD)
-        list(APPEND compile_options "-fdebug-prefix-map=${idf_path}=/IDF")
-        list(APPEND compile_options "-fdebug-prefix-map=${PROJECT_DIR}=/IDF_PROJECT")
-        list(APPEND compile_options "-fdebug-prefix-map=${BUILD_DIR}=/IDF_BUILD")
+        file(TO_CMAKE_PATH "${idf_path}" _idf_path)
+        file(TO_CMAKE_PATH "${PROJECT_DIR}" _project_dir)
+        file(TO_CMAKE_PATH "${BUILD_DIR}" _build_dir)
+        list(APPEND compile_options "-fdebug-prefix-map=${_idf_path}=/IDF")
+        list(APPEND compile_options "-fdebug-prefix-map=${_project_dir}=/IDF_PROJECT")
+        list(APPEND compile_options "-fdebug-prefix-map=${_build_dir}=/IDF_BUILD")

-        # Generate mapping for component paths
        set(gdbinit_file_lines)
+        string(APPEND gdbinit_file_lines "set substitute-path /IDF ${idf_path}\n")
+        string(APPEND gdbinit_file_lines "set substitute-path /IDF_PROJECT ${PROJECT_DIR}\n")
+        string(APPEND gdbinit_file_lines "set substitute-path /IDF_BUILD ${BUILD_DIR}\n")
+
+        # Map components outside IDF_PATH to stable synthetic paths (reproducible builds).
        foreach(component_name ${build_components})
            idf_component_get_property(component_dir ${component_name} COMPONENT_DIR)
-
-            string(TOUPPER ${component_name} component_name_uppercase)
-            set(substituted_path "/COMPONENT_${component_name_uppercase}_DIR")
-            list(APPEND compile_options "-fdebug-prefix-map=${component_dir}=${substituted_path}")
-            string(APPEND gdbinit_file_lines "set substitute-path ${substituted_path} ${component_dir}\n")
+            file(TO_CMAKE_PATH "${component_dir}" _component_dir)
+            file(RELATIVE_PATH _component_rel "${_idf_path}" "${_component_dir}")
+            if(_component_rel MATCHES "^\\.\\.(/|$)")
+                string(TOUPPER ${component_name} component_name_uppercase)
+                set(substituted_path "/COMPONENT_${component_name_uppercase}_DIR")
+                list(APPEND compile_options "-fdebug-prefix-map=${_component_dir}=${substituted_path}")
+                string(APPEND gdbinit_file_lines "set substitute-path ${substituted_path} ${component_dir}\n")
+            endif()
        endforeach()

-        # Mapping for toolchain path
        execute_process(
            COMMAND ${CMAKE_C_COMPILER} -print-sysroot
            OUTPUT_VARIABLE compiler_sysroot
@@ -43,7 +58,8 @@ function(__generate_prefix_map compile_options_var)
        endif()
        string(STRIP "${compiler_sysroot}" compiler_sysroot)
        get_filename_component(compiler_sysroot "${compiler_sysroot}/.." REALPATH)
-        list(APPEND compile_options "-fdebug-prefix-map=${compiler_sysroot}=/TOOLCHAIN")
+        file(TO_CMAKE_PATH "${compiler_sysroot}" _compiler_sysroot)
+        list(APPEND compile_options "-fdebug-prefix-map=${_compiler_sysroot}=/TOOLCHAIN")
        string(APPEND gdbinit_file_lines "set substitute-path /TOOLCHAIN ${compiler_sysroot}\n")
    else()
        set(gdbinit_file_lines "# There is no prefix map defined for the project.\n")
--- a/tools/cmake/toolchain.cmake
+++ b/tools/cmake/toolchain.cmake
@@ -15,6 +15,16 @@ file(TO_CMAKE_PATH "${_current_toolchain_dir}" _current_toolchain_dir)

 set(CMAKE_SYSTEM_NAME Generic)

+# Windows CreateProcess enforces a short command line (~32k). IDF compile lines can be
+# huge (includes, reproducible-build prefix maps, etc.). Tell the Ninja generator to
+# pass compiler arguments via response files (CMake 3.15+). This is orthogonal to the
+# early-stage @cflags/@cxxflags response files below, which only cover CMAKE_*_FLAGS.
+if(CMAKE_HOST_WIN32 AND CMAKE_GENERATOR MATCHES "Ninja"
+    AND NOT DEFINED CACHE{CMAKE_NINJA_FORCE_RESPONSE_FILE})
+    set(CMAKE_NINJA_FORCE_RESPONSE_FILE ON CACHE BOOL
+        "Use Ninja response files on Windows hosts (avoid CreateProcess command-line limit).")
+endif()
+
 # Set compiler tools according to the toolchain type
 string(FIND "${_toolchain_filename}" "clang" found_clang)
 if(NOT found_clang EQUAL -1)
--- a/tools/cmakev2/ldgen.cmake
+++ b/tools/cmakev2/ldgen.cmake
@@ -94,11 +94,21 @@ function(__ldgen_process_template)
        set(mutable_libs_option "--mutable-libraries-file" "${mutable_libs_path}")
    endif()

+    # Write fragment paths to a file (one per line) and pass the file to ldgen,
+    # mirroring the --libraries-file pattern. Avoids cmd/CRT argv-quoting of a
+    # long semicolon-separated list with paths containing spaces and backslashes
+    # on Windows (test_spaces_bundle4).
+    set(ldgen_fragments_file "${build_dir}/ldgen_fragments${ARG_SUFFIX}")
+    list(JOIN ldgen_fragment_files "\n" ldgen_fragments_str)
+    file(WRITE "${ldgen_fragments_file}" "${ldgen_fragments_str}")
+    set_property(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+        APPEND PROPERTY ADDITIONAL_CLEAN_FILES
+        "${ldgen_fragments_file}")
    add_custom_command(
        OUTPUT "${ARG_OUTPUT}"
        COMMAND ${python} "${idf_path}/tools/ldgen/ldgen.py"
        --config    "${sdkconfig}"
-        --fragments-list "${ldgen_fragment_files}"
+        --fragments-list-file "${ldgen_fragments_file}"
        --input     "${ARG_TEMPLATE}"
        --output    "${ARG_OUTPUT}"
        --kconfig   "${root_kconfig}"
--- a/tools/ldgen/ldgen.py
+++ b/tools/ldgen/ldgen.py
@@ -144,6 +144,12 @@ def main():
        '--fragments-list', help='Input fragment files as a semicolon-separated list', type=str
    )

+    fragments_group.add_argument(
+        '--fragments-list-file',
+        type=argparse.FileType('r'),
+        help='File containing fragment file paths, one per line',
+    )
+
    argparser.add_argument(
        '--libraries-file', type=argparse.FileType('r'), help='File that contains the list of libraries in the build'
    )
@@ -201,6 +207,8 @@ def main():
    fragment_files = []
    if args.fragments_list:
        fragment_files = args.fragments_list.split(';')
+    elif args.fragments_list_file:
+        fragment_files = [line.strip() for line in args.fragments_list_file if line.strip()]
    elif args.fragments:
        fragment_files = args.fragments

--- a/tools/test_build_system/test_component_manager.py
+++ b/tools/test_build_system/test_component_manager.py
@@ -227,10 +227,13 @@ class TestIdfRootDependency:
            '#include "mdns.h"',
        )

+        # Intentional dependency on the cmake-level name of the managed component because
+        # idf_extra_components.yml installs espressif/mdns and we verify REQUIRES pulls
+        # espressif__mdns into the build — not something application code should do.
        replace_in_file(
            (test_app_copy / 'main' / 'CMakeLists.txt'),
            '# placeholder_inside_idf_component_register',
-            'REQUIRES mdns',
+            'REQUIRES espressif__mdns',
        )

        idf_py('build')
@@ -251,10 +254,13 @@ class TestIdfRootDependency:
        assert 'espressif__mdns' not in data['build_components']
        assert 'example__cmp' not in data['build_components']

+        # Intentional dependency on the cmake-level name of the managed component because
+        # we assert espressif__mdns enters build_components only after REQUIRES — not
+        # something application code should do.
        replace_in_file(
            (test_app_copy / 'main' / 'CMakeLists.txt'),
            '# placeholder_inside_idf_component_register',
-            'REQUIRES mdns',
+            'REQUIRES espressif__mdns',
        )

        idf_py('reconfigure')
--- a/tools/test_build_system/test_spaces.py
+++ b/tools/test_build_system/test_spaces.py
@@ -68,7 +68,6 @@ def test_spaces_bundle3(idf_copy: Path) -> None:
    )


-@pytest.mark.xfail(sys.platform == 'win32', reason='Bug with reproducible build')
 # Use this bundle for tests which can be done with the default build_test_app
@pytest.mark.parametrize(
    'dummy_',