ci(tools): Updated approach of copying files to using git worktree

tools/test_build_system/conftest.py

@@ -18,6 +18,89 @@ from test_build_system_helpers import IdfPyFunc
 from test_build_system_helpers import run_idf_py
 
 
+def _get_git_submodule_paths(repo_path: Path) -> typing.List[str]:
+    """Get list of submodule paths from .gitmodules file."""
+    gitmodules = repo_path / '.gitmodules'
+    if not gitmodules.exists():
+        return []
+
+    submodule_paths = []
+    with open(gitmodules, encoding='utf-8') as f:
+        for line in f:
+            line = line.strip()
+            if line.startswith('path = '):
+                submodule_paths.append(line[7:])  # Remove 'path = ' prefix
+    return submodule_paths
+
+
+def _create_idf_copy_via_worktree(path_from: Path, path_to: Path) -> str:
+    """
+    Create IDF copy using git worktree (fast) + copying submodule directories.
+
+    Git worktree creates a fast checkout of tracked files, but submodules
+    appear as empty directories. We copy submodule content from the source
+    repo (which has them already checked out) instead of running git submodule
+    update (which can fail due to auth issues on CI).
+    """
+    import uuid
+
+    branch_name = f'test-worktree-{uuid.uuid4().hex[:8]}'
+
+    logging.debug(f'creating git worktree at {path_to} (branch: {branch_name})')
+    subprocess.run(
+        ['git', 'worktree', 'add', '-b', branch_name, str(path_to)], cwd=path_from, capture_output=True, check=True
+    )
+
+    # Copy submodule directories from source (they're already checked out there)
+    submodule_paths = _get_git_submodule_paths(path_from)
+    for submodule_rel_path in submodule_paths:
+        src_submodule = path_from / submodule_rel_path
+        dst_submodule = path_to / submodule_rel_path
+
+        # Only copy if source submodule exists and has content
+        if src_submodule.exists() and any(src_submodule.iterdir()):
+            logging.debug(f'copying submodule {submodule_rel_path}')
+            # Remove the empty directory created by worktree
+            if dst_submodule.exists():
+                shutil.rmtree(dst_submodule, ignore_errors=True)
+            # Copy the submodule content
+            shutil.copytree(src_submodule, dst_submodule, symlinks=True, ignore=shutil.ignore_patterns('.git'))
+
+    return branch_name
+
+
+def _cleanup_worktree(path_from: Path, path_to: Path, branch_name: str) -> None:
+    """Remove git worktree and its temporary branch."""
+    logging.debug(f'removing git worktree at {path_to}')
+    # Remove the worktree
+    subprocess.run(
+        ['git', 'worktree', 'remove', '--force', str(path_to)],
+        cwd=path_from,
+        check=False,  # Don't fail if already removed
+    )
+    # Delete the temporary branch
+    subprocess.run(
+        ['git', 'branch', '-D', branch_name],
+        cwd=path_from,
+        check=False,  # Don't fail if branch doesn't exist
+    )
+
+
+def _create_idf_copy_via_shutil(path_from: Path, path_to: Path) -> None:
+    """Create IDF copy using shutil.copytree (slower but always works)."""
+    # if the new directory inside the original directory,
+    # make sure not to go into recursion.
+    ignore = shutil.ignore_patterns(
+        path_to.name,
+        # also ignore the build directories which may be quite large
+        # plus ignore .git since it is causing trouble when removing on Windows
+        '**/build',
+        '.git',
+    )
+    logging.debug(f'copying {path_from} to {path_to} (shutil.copytree)')
+    shutil.copytree(path_from, path_to, ignore=ignore, symlinks=True)
+
+
 # Pytest hook used to check if the test has passed or failed, from a fixture.
 # Based on https://docs.pytest.org/en/latest/example/simple.html#making-test-result-information-available-in-fixtures
 @pytest.hookimpl(tryfirst=True, hookwrapper=True)
@@ -171,21 +254,22 @@ def idf_copy(func_work_dir: Path, request: FixtureRequest) -> typing.Generator[P
     if mark:
         copy_to = mark.args[0]
 
-    path_from = EXT_IDF_PATH
+    path_from = Path(EXT_IDF_PATH)
     path_to = func_work_dir / copy_to
 
-    # if the new directory inside the original directory,
-    # make sure not to go into recursion.
-    ignore = shutil.ignore_patterns(
-        path_to.name,
-        # also ignore the build directories which may be quite large
-        # plus ignore .git since it is causing trouble when removing on Windows
-        '**/build', '.git')
-
-    logging.debug(f'copying {path_from} to {path_to}')
-    shutil.copytree(path_from, path_to, ignore=ignore, symlinks=True)
-
     orig_idf_path = os.environ['IDF_PATH']
+    branch_name = None  # type: typing.Optional[str]
+
+    # Try git worktree first (much faster), fall back to shutil.copytree
+    try:
+        branch_name = _create_idf_copy_via_worktree(path_from, path_to)
+    except (subprocess.CalledProcessError, FileNotFoundError, OSError) as e:
+        logging.debug(f'git worktree failed ({e}), falling back to shutil.copytree')
+        # Clean up any partial worktree before fallback
+        if path_to.exists():
+            shutil.rmtree(path_to, ignore_errors=True)
+        _create_idf_copy_via_shutil(path_from, path_to)
+
     os.environ['IDF_PATH'] = str(path_to)
 
     yield Path(path_to)
@@ -193,8 +277,11 @@ def idf_copy(func_work_dir: Path, request: FixtureRequest) -> typing.Generator[P
     os.environ['IDF_PATH'] = orig_idf_path
 
     if should_clean_test_dir(request):
-        logging.debug('cleaning up work directory after a successful test: {}'.format(path_to))
-        shutil.rmtree(path_to, ignore_errors=True)
+        logging.debug(f'cleaning up work directory after a successful test: {path_to}')
+        if branch_name:
+            _cleanup_worktree(path_from, path_to, branch_name)
+        else:
+            shutil.rmtree(path_to, ignore_errors=True)
 
 
 @pytest.fixture(name='default_idf_env')