fix(tools): Windows builds when paths contain spaces (prefix_map, ldgen, CI)

- Harden prefix_map.cmake for paths with spaces on Windows

- ldgen: fragments-list-file, list normalization, CMake integration (tools/cmake + cmakev2)

- CI exclude list for check_tools; test_spaces bundle tweak

Made-with: Cursor

.gitlab/ci/test-win.yml

@@ -76,7 +76,7 @@ pytest_build_system_win:
   extends:
     - .test_build_system_template_win
     - .rules:labels:windows_pytest_build_system
-  parallel: 10
+  parallel: 6
 
 pytest_build_system_win_minimal_cmake:
   extends:
@@ -102,7 +102,7 @@ pytest_buildv2_system_win:
   extends:
     - .test_build_system_template_win
     - .rules:labels:buildv2
-  parallel: 10
+  parallel: 6
   script:
     - cd tools\test_build_system
     - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}

tools/cmake/ldgen.cmake

@@ -187,7 +187,7 @@ function(__ldgen_create_target exe_target)
         "${build_dir}/ldgen_libraries.in"
         "${build_dir}/ldgen_libraries")
 
-    idf_build_get_property(ldgen_fragment_files __LDGEN_FRAGMENT_FILES GENERATOR_EXPRESSION)
+    idf_build_get_property(ldgen_fragment_files __LDGEN_FRAGMENT_FILES)
 
     # Create command to invoke the linker script generator tool.
     idf_build_get_property(sdkconfig SDKCONFIG)
@@ -214,11 +214,21 @@ function(__ldgen_create_target exe_target)
         set(mutable_libs_option "--mutable-libraries-file" "${mutable_libs_path}")
     endif()
 
+    # Write fragment paths to a file (one per line) and pass the file to ldgen,
+    # mirroring the --libraries-file pattern. Avoids cmd/CRT argv-quoting of a
+    # long semicolon-separated list with paths containing spaces and backslashes
+    # on Windows (test_spaces_bundle4).
+    set(ldgen_fragments_file "${build_dir}/ldgen_fragments")
+    list(JOIN ldgen_fragment_files "\n" ldgen_fragments_str)
+    file(WRITE "${ldgen_fragments_file}" "${ldgen_fragments_str}")
+    set_property(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+        APPEND PROPERTY ADDITIONAL_CLEAN_FILES
+        "${ldgen_fragments_file}")
     add_custom_command(
         OUTPUT ${output}
         COMMAND ${python} "${idf_path}/tools/ldgen/ldgen.py"
         --config    "${sdkconfig}"
-        --fragments-list "${ldgen_fragment_files}"
+        --fragments-list-file "${ldgen_fragments_file}"
         --input     "${template}"
         --output    "${output}"
         --kconfig   "${root_kconfig}"

tools/cmake/prefix_map.cmake

@@ -5,6 +5,10 @@
 # to fixed names. This is used when reproducible builds are required.
 # This function also creates a gdbinit file for the debugger to
 # remap the substituted paths back to the real paths in the filesystem.
+#
+# On Windows hosts, very long compile commands are avoided by enabling Ninja response
+# files in toolchain.cmake (CMAKE_NINJA_FORCE_RESPONSE_FILE), not by dropping these
+# options when paths contain spaces.
 function(__generate_prefix_map compile_options_var)
     set(compile_options)
     set(gdbinit_dir ${BUILD_DIR}/gdbinit)
@@ -13,27 +17,38 @@ function(__generate_prefix_map compile_options_var)
     idf_build_get_property(build_components BUILD_COMPONENTS)
 
     if(CONFIG_COMPILER_HIDE_PATHS_MACROS)
-        list(APPEND compile_options "-fmacro-prefix-map=${CMAKE_SOURCE_DIR}=.")
+        file(TO_CMAKE_PATH "${CMAKE_SOURCE_DIR}" _src_dir)
-        list(APPEND compile_options "-fmacro-prefix-map=${idf_path}=/IDF")
+        file(TO_CMAKE_PATH "${idf_path}" _idf_path)
+        list(APPEND compile_options "-fmacro-prefix-map=${_src_dir}=.")
+        list(APPEND compile_options "-fmacro-prefix-map=${_idf_path}=/IDF")
     endif()
 
     if(CONFIG_APP_REPRODUCIBLE_BUILD)
-        list(APPEND compile_options "-fdebug-prefix-map=${idf_path}=/IDF")
+        file(TO_CMAKE_PATH "${idf_path}" _idf_path)
-        list(APPEND compile_options "-fdebug-prefix-map=${PROJECT_DIR}=/IDF_PROJECT")
+        file(TO_CMAKE_PATH "${PROJECT_DIR}" _project_dir)
-        list(APPEND compile_options "-fdebug-prefix-map=${BUILD_DIR}=/IDF_BUILD")
+        file(TO_CMAKE_PATH "${BUILD_DIR}" _build_dir)
+        list(APPEND compile_options "-fdebug-prefix-map=${_idf_path}=/IDF")
+        list(APPEND compile_options "-fdebug-prefix-map=${_project_dir}=/IDF_PROJECT")
+        list(APPEND compile_options "-fdebug-prefix-map=${_build_dir}=/IDF_BUILD")
 
-        # Generate mapping for component paths
         set(gdbinit_file_lines)
+        string(APPEND gdbinit_file_lines "set substitute-path /IDF ${idf_path}\n")
+        string(APPEND gdbinit_file_lines "set substitute-path /IDF_PROJECT ${PROJECT_DIR}\n")
+        string(APPEND gdbinit_file_lines "set substitute-path /IDF_BUILD ${BUILD_DIR}\n")
+
+        # Map components outside IDF_PATH to stable synthetic paths (reproducible builds).
         foreach(component_name ${build_components})
             idf_component_get_property(component_dir ${component_name} COMPONENT_DIR)
-
+            file(TO_CMAKE_PATH "${component_dir}" _component_dir)
-            string(TOUPPER ${component_name} component_name_uppercase)
+            file(RELATIVE_PATH _component_rel "${_idf_path}" "${_component_dir}")
-            set(substituted_path "/COMPONENT_${component_name_uppercase}_DIR")
+            if(_component_rel MATCHES "^\\.\\.(/|$)")
-            list(APPEND compile_options "-fdebug-prefix-map=${component_dir}=${substituted_path}")
+                string(TOUPPER ${component_name} component_name_uppercase)
-            string(APPEND gdbinit_file_lines "set substitute-path ${substituted_path} ${component_dir}\n")
+                set(substituted_path "/COMPONENT_${component_name_uppercase}_DIR")
+                list(APPEND compile_options "-fdebug-prefix-map=${_component_dir}=${substituted_path}")
+                string(APPEND gdbinit_file_lines "set substitute-path ${substituted_path} ${component_dir}\n")
+            endif()
         endforeach()
 
-        # Mapping for toolchain path
         execute_process(
             COMMAND ${CMAKE_C_COMPILER} -print-sysroot
             OUTPUT_VARIABLE compiler_sysroot
@@ -43,7 +58,8 @@ function(__generate_prefix_map compile_options_var)
         endif()
         string(STRIP "${compiler_sysroot}" compiler_sysroot)
         get_filename_component(compiler_sysroot "${compiler_sysroot}/.." REALPATH)
-        list(APPEND compile_options "-fdebug-prefix-map=${compiler_sysroot}=/TOOLCHAIN")
+        file(TO_CMAKE_PATH "${compiler_sysroot}" _compiler_sysroot)
+        list(APPEND compile_options "-fdebug-prefix-map=${_compiler_sysroot}=/TOOLCHAIN")
         string(APPEND gdbinit_file_lines "set substitute-path /TOOLCHAIN ${compiler_sysroot}\n")
     else()
         set(gdbinit_file_lines "# There is no prefix map defined for the project.\n")

tools/cmake/toolchain.cmake

@@ -15,6 +15,16 @@ file(TO_CMAKE_PATH "${_current_toolchain_dir}" _current_toolchain_dir)
 
 set(CMAKE_SYSTEM_NAME Generic)
 
+# Windows CreateProcess enforces a short command line (~32k). IDF compile lines can be
+# huge (includes, reproducible-build prefix maps, etc.). Tell the Ninja generator to
+# pass compiler arguments via response files (CMake 3.15+). This is orthogonal to the
+# early-stage @cflags/@cxxflags response files below, which only cover CMAKE_*_FLAGS.
+if(CMAKE_HOST_WIN32 AND CMAKE_GENERATOR MATCHES "Ninja"
+    AND NOT DEFINED CACHE{CMAKE_NINJA_FORCE_RESPONSE_FILE})
+    set(CMAKE_NINJA_FORCE_RESPONSE_FILE ON CACHE BOOL
+        "Use Ninja response files on Windows hosts (avoid CreateProcess command-line limit).")
+endif()
+
 # Set compiler tools according to the toolchain type
 string(FIND "${_toolchain_filename}" "clang" found_clang)
 if(NOT found_clang EQUAL -1)

tools/cmakev2/ldgen.cmake

@@ -94,11 +94,21 @@ function(__ldgen_process_template)
         set(mutable_libs_option "--mutable-libraries-file" "${mutable_libs_path}")
     endif()
 
+    # Write fragment paths to a file (one per line) and pass the file to ldgen,
+    # mirroring the --libraries-file pattern. Avoids cmd/CRT argv-quoting of a
+    # long semicolon-separated list with paths containing spaces and backslashes
+    # on Windows (test_spaces_bundle4).
+    set(ldgen_fragments_file "${build_dir}/ldgen_fragments${ARG_SUFFIX}")
+    list(JOIN ldgen_fragment_files "\n" ldgen_fragments_str)
+    file(WRITE "${ldgen_fragments_file}" "${ldgen_fragments_str}")
+    set_property(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+        APPEND PROPERTY ADDITIONAL_CLEAN_FILES
+        "${ldgen_fragments_file}")
     add_custom_command(
         OUTPUT "${ARG_OUTPUT}"
         COMMAND ${python} "${idf_path}/tools/ldgen/ldgen.py"
         --config    "${sdkconfig}"
-        --fragments-list "${ldgen_fragment_files}"
+        --fragments-list-file "${ldgen_fragments_file}"
         --input     "${ARG_TEMPLATE}"
         --output    "${ARG_OUTPUT}"
         --kconfig   "${root_kconfig}"

tools/ldgen/ldgen.py

@@ -144,6 +144,12 @@ def main():
         '--fragments-list', help='Input fragment files as a semicolon-separated list', type=str
     )
 
+    fragments_group.add_argument(
+        '--fragments-list-file',
+        type=argparse.FileType('r'),
+        help='File containing fragment file paths, one per line',
+    )
+
     argparser.add_argument(
         '--libraries-file', type=argparse.FileType('r'), help='File that contains the list of libraries in the build'
     )
@@ -201,6 +207,8 @@ def main():
     fragment_files = []
     if args.fragments_list:
         fragment_files = args.fragments_list.split(';')
+    elif args.fragments_list_file:
+        fragment_files = [line.strip() for line in args.fragments_list_file if line.strip()]
     elif args.fragments:
         fragment_files = args.fragments
 

tools/test_build_system/test_component_manager.py

@@ -227,10 +227,13 @@ class TestIdfRootDependency:
             '#include "mdns.h"',
         )
 
+        # Intentional dependency on the cmake-level name of the managed component because
+        # idf_extra_components.yml installs espressif/mdns and we verify REQUIRES pulls
+        # espressif__mdns into the build — not something application code should do.
         replace_in_file(
             (test_app_copy / 'main' / 'CMakeLists.txt'),
             '# placeholder_inside_idf_component_register',
-            'REQUIRES mdns',
+            'REQUIRES espressif__mdns',
         )
 
         idf_py('build')
@@ -251,10 +254,13 @@ class TestIdfRootDependency:
         assert 'espressif__mdns' not in data['build_components']
         assert 'example__cmp' not in data['build_components']
 
+        # Intentional dependency on the cmake-level name of the managed component because
+        # we assert espressif__mdns enters build_components only after REQUIRES — not
+        # something application code should do.
         replace_in_file(
             (test_app_copy / 'main' / 'CMakeLists.txt'),
             '# placeholder_inside_idf_component_register',
-            'REQUIRES mdns',
+            'REQUIRES espressif__mdns',
         )
 
         idf_py('reconfigure')

tools/test_build_system/test_spaces.py

@@ -68,7 +68,6 @@ def test_spaces_bundle3(idf_copy: Path) -> None:
     )
 
 
-@pytest.mark.xfail(sys.platform == 'win32', reason='Bug with reproducible build')
 # Use this bundle for tests which can be done with the default build_test_app
 @pytest.mark.parametrize(
     'dummy_',