From 20fa024a826ae9a7a5204bb9d62f44b1327583af Mon Sep 17 00:00:00 2001 From: Jakub Kocka Date: Wed, 21 Jan 2026 10:23:03 +0100 Subject: [PATCH] ci(tools): Updated approach of copying files to using git worktree --- tools/test_build_system/conftest.py | 115 ++++++++++++++++++++++++---- 1 file changed, 101 insertions(+), 14 deletions(-) diff --git a/tools/test_build_system/conftest.py b/tools/test_build_system/conftest.py index 865c018863d..9a56788ad9c 100644 --- a/tools/test_build_system/conftest.py +++ b/tools/test_build_system/conftest.py @@ -18,6 +18,89 @@ from test_build_system_helpers import IdfPyFunc from test_build_system_helpers import run_idf_py +def _get_git_submodule_paths(repo_path: Path) -> typing.List[str]: + """Get list of submodule paths from .gitmodules file.""" + gitmodules = repo_path / '.gitmodules' + if not gitmodules.exists(): + return [] + + submodule_paths = [] + with open(gitmodules, encoding='utf-8') as f: + for line in f: + line = line.strip() + if line.startswith('path = '): + submodule_paths.append(line[7:]) # Remove 'path = ' prefix + return submodule_paths + + +def _create_idf_copy_via_worktree(path_from: Path, path_to: Path) -> str: + """ + Create IDF copy using git worktree (fast) + copying submodule directories. + + Git worktree creates a fast checkout of tracked files, but submodules + appear as empty directories. We copy submodule content from the source + repo (which has them already checked out) instead of running git submodule + update (which can fail due to auth issues on CI). + """ + import uuid + + branch_name = f'test-worktree-{uuid.uuid4().hex[:8]}' + + logging.debug(f'creating git worktree at {path_to} (branch: {branch_name})') + subprocess.run( + ['git', 'worktree', 'add', '-b', branch_name, str(path_to)], cwd=path_from, capture_output=True, check=True + ) + + # Copy submodule directories from source (they're already checked out there) + submodule_paths = _get_git_submodule_paths(path_from) + for submodule_rel_path in submodule_paths: + src_submodule = path_from / submodule_rel_path + dst_submodule = path_to / submodule_rel_path + + # Only copy if source submodule exists and has content + if src_submodule.exists() and any(src_submodule.iterdir()): + logging.debug(f'copying submodule {submodule_rel_path}') + # Remove the empty directory created by worktree + if dst_submodule.exists(): + shutil.rmtree(dst_submodule, ignore_errors=True) + # Copy the submodule content + shutil.copytree(src_submodule, dst_submodule, symlinks=True, ignore=shutil.ignore_patterns('.git')) + + return branch_name + + +def _cleanup_worktree(path_from: Path, path_to: Path, branch_name: str) -> None: + """Remove git worktree and its temporary branch.""" + logging.debug(f'removing git worktree at {path_to}') + # Remove the worktree + subprocess.run( + ['git', 'worktree', 'remove', '--force', str(path_to)], + cwd=path_from, + check=False, # Don't fail if already removed + ) + # Delete the temporary branch + subprocess.run( + ['git', 'branch', '-D', branch_name], + cwd=path_from, + check=False, # Don't fail if branch doesn't exist + ) + + +def _create_idf_copy_via_shutil(path_from: Path, path_to: Path) -> None: + """Create IDF copy using shutil.copytree (slower but always works).""" + # if the new directory inside the original directory, + # make sure not to go into recursion. + ignore = shutil.ignore_patterns( + path_to.name, + # also ignore the build directories which may be quite large + # plus ignore .git since it is causing trouble when removing on Windows + '**/build', + '.git', + ) + logging.debug(f'copying {path_from} to {path_to} (shutil.copytree)') + shutil.copytree(path_from, path_to, ignore=ignore, symlinks=True) + + # Pytest hook used to check if the test has passed or failed, from a fixture. # Based on https://docs.pytest.org/en/latest/example/simple.html#making-test-result-information-available-in-fixtures @pytest.hookimpl(tryfirst=True, hookwrapper=True) @@ -171,21 +254,22 @@ def idf_copy(func_work_dir: Path, request: FixtureRequest) -> typing.Generator[P if mark: copy_to = mark.args[0] - path_from = EXT_IDF_PATH + path_from = Path(EXT_IDF_PATH) path_to = func_work_dir / copy_to - # if the new directory inside the original directory, - # make sure not to go into recursion. - ignore = shutil.ignore_patterns( - path_to.name, - # also ignore the build directories which may be quite large - # plus ignore .git since it is causing trouble when removing on Windows - '**/build', '.git') - - logging.debug(f'copying {path_from} to {path_to}') - shutil.copytree(path_from, path_to, ignore=ignore, symlinks=True) - orig_idf_path = os.environ['IDF_PATH'] + branch_name = None # type: typing.Optional[str] + + # Try git worktree first (much faster), fall back to shutil.copytree + try: + branch_name = _create_idf_copy_via_worktree(path_from, path_to) + except (subprocess.CalledProcessError, FileNotFoundError, OSError) as e: + logging.debug(f'git worktree failed ({e}), falling back to shutil.copytree') + # Clean up any partial worktree before fallback + if path_to.exists(): + shutil.rmtree(path_to, ignore_errors=True) + _create_idf_copy_via_shutil(path_from, path_to) + os.environ['IDF_PATH'] = str(path_to) yield Path(path_to) @@ -193,8 +277,11 @@ def idf_copy(func_work_dir: Path, request: FixtureRequest) -> typing.Generator[P os.environ['IDF_PATH'] = orig_idf_path if should_clean_test_dir(request): - logging.debug('cleaning up work directory after a successful test: {}'.format(path_to)) - shutil.rmtree(path_to, ignore_errors=True) + logging.debug(f'cleaning up work directory after a successful test: {path_to}') + if branch_name: + _cleanup_worktree(path_from, path_to, branch_name) + else: + shutil.rmtree(path_to, ignore_errors=True) @pytest.fixture(name='default_idf_env')