From fa7372d1c4ab2eeafaf77901134132a2f1d22560 Mon Sep 17 00:00:00 2001 From: Jakub Kocka Date: Wed, 16 Sep 2026 12:10:58 +0200 Subject: [PATCH] ci(tools): Verify submodules were copied into the idf_copy worktree git worktree add leaves every submodule as an empty directory, and the copy loop skipped any submodule it could not read from the source repo. The placeholder then stayed in the copy and the build failed much later in an unrelated component, as a missing mbedtls/include or tlsf.c Check each submodule in the destination after copying it. If one is still a placeholder, drop the worktree and create the copy with shutil.copytree, which does not depend on the submodule state of the source checkout --- tools/test_build_system/conftest.py | 53 ++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/tools/test_build_system/conftest.py b/tools/test_build_system/conftest.py index ad789dcb82d..93409099b11 100644 --- a/tools/test_build_system/conftest.py +++ b/tools/test_build_system/conftest.py @@ -36,7 +36,31 @@ def _get_git_submodule_paths(repo_path: Path) -> list[str]: return submodule_paths -def _create_idf_copy_via_worktree(path_from: Path, path_to: Path) -> str: +def _is_materialized_submodule(path: Path) -> bool: + """Whether path holds submodule content rather than a placeholder. + + ``git worktree add`` leaves a submodule as an empty directory or as a + gitlink file, and both are unusable as component sources. + """ + try: + return path.is_dir() and any(path.iterdir()) + except OSError: + return False + + +def _copy_submodule(src_submodule: Path, dst_submodule: Path) -> None: + """Replace the worktree placeholder with the submodule content of the source repo.""" + # rmtree() cannot remove a gitlink file (even with ignore_errors=True), and + # copytree() would then leave a file where CMake expects a directory + # (e.g. mbedtls/include). + if dst_submodule.is_file() or dst_submodule.is_symlink(): + dst_submodule.unlink() + elif dst_submodule.exists(): + shutil.rmtree(dst_submodule) + shutil.copytree(src_submodule, dst_submodule, symlinks=True, ignore=shutil.ignore_patterns('.git')) + + +def _create_idf_copy_via_worktree(path_from: Path, path_to: Path) -> str | None: """ Create IDF copy using git worktree (fast) + copying submodule directories. @@ -45,6 +69,12 @@ def _create_idf_copy_via_worktree(path_from: Path, path_to: Path) -> str: repo (which has them already checked out) instead of running git submodule update (which can fail due to auth issues on CI). + Return the worktree branch name, or None if a submodule could not be + materialized. In that case the worktree is removed again and the caller + creates the copy with shutil.copytree instead. Leaving a placeholder behind + would produce a copy that only fails once a test builds it, as a missing + include directory or source file of the affected component. + After copying submodules, remove the worktree's top-level ``.git`` file so the result matches the old ``shutil.copytree`` behavior (no git repo at ``IDF_PATH``). Otherwise CMake's ``git_submodule_check`` runs inside the @@ -67,18 +97,15 @@ def _create_idf_copy_via_worktree(path_from: Path, path_to: Path) -> str: src_submodule = path_from / submodule_rel_path dst_submodule = path_to / submodule_rel_path - # Only copy if the source submodule is a populated directory. A gitlink - # file or empty dir means the source checkout did not materialize it. - if src_submodule.is_dir() and any(src_submodule.iterdir()): + # Nothing to copy when the source checkout did not materialize the submodule. + if _is_materialized_submodule(src_submodule): logging.debug(f'copying submodule {submodule_rel_path}') - # Worktree submodule paths are often gitlink files; rmtree() cannot - # remove those (even with ignore_errors=True), and copytree() then - # leaves a file where CMake expects a directory (e.g. mbedtls/include). - if dst_submodule.is_file() or dst_submodule.is_symlink(): - dst_submodule.unlink() - elif dst_submodule.exists(): - shutil.rmtree(dst_submodule) - shutil.copytree(src_submodule, dst_submodule, symlinks=True, ignore=shutil.ignore_patterns('.git')) + _copy_submodule(src_submodule, dst_submodule) + + if not _is_materialized_submodule(dst_submodule): + logging.warning(f'submodule {submodule_rel_path} could not be copied into {path_to}') + _cleanup_worktree(path_from, path_to, branch_name) + return None # Match old shutil-based idf_copy: no top-level .git (see docstring above). (path_to / '.git').unlink(missing_ok=True) @@ -310,6 +337,8 @@ def idf_copy(func_work_dir: Path, request: FixtureRequest) -> typing.Generator[P # Clean up any partial worktree before fallback if path_to.exists(): shutil.rmtree(path_to, ignore_errors=True) + + if branch_name is None: _create_idf_copy_via_shutil(path_from, path_to) os.environ['IDF_PATH'] = str(path_to)