Merge branch 'ci/use-windows-vm' into 'master'

ci: windows tests with VM

See merge request espressif/esp-idf!47238

.gitlab/ci/common.yml

@@ -281,6 +281,31 @@ variables:
   after_script: []  # ccache now is disabled for macos brew runners
   timeout: 30m
 
+.brew-windows-settings:
+  image: windows_11_idf_v6_1
+  tags: [windows-vm, brew]
+  variables:
+    GIT_STRATEGY: fetch
+    LC_ALL: C.UTF-8
+    # git clean at the final cleanup stage stuck on Windows VM
+    # a new VM for each job anyway, so we can skip cleaning the workspace
+    GIT_CLEAN_FLAGS: none
+#    IDF_CCACHE_ENABLE: "1"
+#    CCACHE_DIR: "C:/cache/idf_ccache"
+  before_script:
+    - if ($env:IDF_DONT_USE_MIRRORS) { $env:IDF_MIRROR_PREFIX_MAP = '' }
+    - ./install.ps1 --enable-ci
+    - ./export.ps1
+    - $env:PYTHONPATH = "$env:PYTHONPATH;$env:IDF_PATH\tools;$env:IDF_PATH\tools\esp_app_trace;$env:IDF_PATH\components\partition_table;$env:IDF_PATH\tools\ci\python_packages"
+    - python "${SUBMODULE_FETCH_TOOL}" -s "${SUBMODULES_TO_FETCH}"
+    # The time in VM may not be synced, since it's built a long time ago, so we need to sync time
+    - net start w32time
+    - w32tm /config /manualpeerlist:"time.google.com,0x9 pool.ntp.org,0x9" /syncfromflags:manual /reliable:yes /update
+    - net stop w32time
+    - net start w32time
+    - w32tm /resync
+  after_script: []
+
 #############
 # `default` #
 #############

.gitlab/ci/test-win.yml

@@ -1,9 +1,8 @@
-# Host tests
 .host_test_win_template:
-  extends: .rules:test:host_test
+  extends:
+    - .brew-windows-settings
+    - .rules:test:host_test
   stage: host_test
-  image: $ESP_ENV_IMAGE
-  tags: [windows-build, brew]
   dependencies: # set dependencies to null to avoid missing artifacts issue
   # run host_test jobs immediately, only after upload cache
   needs:
@@ -13,13 +12,8 @@
     - job: upload-submodules-cache
       optional: true
       artifacts: false
-  variables:
+    - job: manual_gate
-    GIT_STRATEGY: fetch  # use brew local mirror first
+      optional: true
-  before_script:
-    - if ($env:IDF_DONT_USE_MIRRORS) {
-        $env:IDF_MIRROR_PREFIX_MAP = ""
-      }
-  after_script: []
 
 test_cli_installer_win:
   rules:
@@ -34,15 +28,12 @@ test_cli_installer_win:
       - tools/tools.new.json
       - tools/test_idf_tools/test_python_env_logs.txt
     expire_in: 1 week
-  variables:
-    IDF_PATH: "$CI_PROJECT_DIR"
   timeout: 3h
   script:
     # Tools must be downloaded for testing
     #                                                                     cmake@version that is supported
-    - python ${IDF_PATH}\tools\idf_tools.py download required qemu-riscv32 qemu-xtensa cmake cmake@3.22.1
+    - python tools\idf_tools.py download required qemu-riscv32 qemu-xtensa cmake cmake@3.22.1
-    - cd ${IDF_PATH}\tools\test_idf_tools
+    - cd tools\test_idf_tools
-    - python -m pip install jsonschema
     - python .\test_idf_tools.py
     - python .\test_idf_tools_python_env.py
 
@@ -53,39 +44,31 @@ test_tools_win:
   parallel: 4
   artifacts:
     paths:
-      - ${IDF_PATH}/*.out
+      - "*.out"
-      - ${IDF_PATH}/XUNIT_*.xml
+      - "XUNIT_*.xml"
     reports:
-      junit: ${IDF_PATH}/XUNIT_*.xml
+      junit: "XUNIT_*.xml"
     expire_in: 1 week
     when: always
-  variables:
-    LC_ALL: C.UTF-8
-    PYTHONPATH: "$PYTHONPATH;$IDF_PATH\\tools;$IDF_PATH\\tools\\esp_app_trace;$IDF_PATH\\components\\partition_table;$IDF_PATH\\tools\\ci\\python_packages"
   script:
-    - python -m pip install jsonschema
+    - cd tools\test_idf_py
-    - .\install.ps1 --enable-ci
-    - .\export.ps1
-    - python "${SUBMODULE_FETCH_TOOL}" -s "all"
-    - cd ${IDF_PATH}/tools/test_idf_py
     - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
-    - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=${IDF_PATH}/XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
+    - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
 
 # Build tests
 .test_build_system_template_win:
-  stage: host_test
+  extends: .host_test_win_template
-  variables:
-  # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings.
-    IDF_CCACHE_ENABLE: "1"
-    PYTHONPATH: "$PYTHONPATH;$IDF_PATH\\tools;$IDF_PATH\\tools\\esp_app_trace;$IDF_PATH\\components\\partition_table;$IDF_PATH\\tools\\ci\\python_packages"
-  before_script: []
-  after_script: []
   timeout: 4 hours
+  artifacts:
+    paths:
+      - XUNIT_RESULT.xml
+      - test_build_system
+    expire_in: 2 days
+    when: always
+    reports:
+      junit: XUNIT_RESULT.xml
   script:
-    - .\install.ps1 --enable-ci
+    - cd tools\test_build_system
-    - . .\export.ps1
-    - python "${SUBMODULE_FETCH_TOOL}" -s "all"
-    - cd ${IDF_PATH}\tools\test_build_system
     - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
     - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} --durations=10
 
@@ -94,42 +77,16 @@ pytest_build_system_win:
     - .test_build_system_template_win
     - .rules:labels:windows_pytest_build_system
   parallel: 10
-  needs:
-    - job: manual_gate
-      optional: true
-  tags: [windows-build, brew]
-  artifacts:
-    paths:
-      - XUNIT_RESULT.xml
-      - test_build_system
-    expire_in: 2 days
-    reports:
-      junit: XUNIT_RESULT.xml
-    when: always
 
 pytest_build_system_win_minimal_cmake:
   extends:
     - .test_build_system_template_win
     - .rules:labels:windows_pytest_build_system
   parallel: 2
-  needs:
-    - job: manual_gate
-      optional: true
-  tags: [windows-build, brew]
-  artifacts:
-    paths:
-      - XUNIT_RESULT.xml
-      - test_build_system
-    expire_in: 2 days
-    reports:
-      junit: XUNIT_RESULT.xml
-    when: always
   variables:
     MINIMAL_CMAKE_VERSION: "3.22.1"
   script:
-    - .\install.ps1 --enable-ci
+    - python tools\idf_tools.py install cmake@${MINIMAL_CMAKE_VERSION}
-    - . .\export.ps1
-    - python ${IDF_PATH}\tools\idf_tools.py install cmake@${MINIMAL_CMAKE_VERSION}
     - $Env:PATH = "$Env:USERPROFILE\.espressif\tools\cmake\${MINIMAL_CMAKE_VERSION}\bin;$Env:PATH"
     - |
       $actualVersion = (& cmake --version).Split()[2]
@@ -137,8 +94,7 @@ pytest_build_system_win_minimal_cmake:
         Write-Error "ERROR: Wrong CMake version! Detected: $actualVersion, but expected: $Env:MINIMAL_CMAKE_VERSION"
         exit 1
       }
-    - python "${SUBMODULE_FETCH_TOOL}" -s "all"
+    - cd tools\test_build_system
-    - cd ${IDF_PATH}\tools\test_build_system
     - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
     - pytest -k cmake --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} --durations=10
 
@@ -147,23 +103,8 @@ pytest_buildv2_system_win:
     - .test_build_system_template_win
     - .rules:labels:buildv2
   parallel: 10
-  needs:
-    - job: manual_gate
-      optional: true
-  tags: [windows-build, brew]
-  artifacts:
-    paths:
-      - XUNIT_RESULT.xml
-      - test_build_system
-    expire_in: 2 days
-    reports:
-      junit: XUNIT_RESULT.xml
-    when: always
   script:
-    - .\install.ps1 --enable-ci
+    - cd tools\test_build_system
-    - . .\export.ps1
-    - python "${SUBMODULE_FETCH_TOOL}" -s "all"
-    - cd ${IDF_PATH}\tools\test_build_system
     - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
     - pytest
       --buildv2

tools/ci/python_packages/gitlab_api.py

@@ -1,6 +1,7 @@
-# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 import argparse
+import copy
 import logging
 import os
 import re
@@ -9,13 +10,9 @@ import tarfile
 import tempfile
 import time
 import zipfile
+from collections.abc import Callable
 from functools import wraps
 from typing import Any
-from typing import Callable
-from typing import Dict
-from typing import List
-from typing import Optional
-from typing import Union
 
 import gitlab
 
@@ -36,7 +33,7 @@ def retry(func: TR) -> TR:
         while True:
             try:
                 res = func(self, *args, **kwargs)
-            except (IOError, EOFError, gitlab.exceptions.GitlabError) as e:
+            except (OSError, EOFError, gitlab.exceptions.GitlabError) as e:
                 if isinstance(e, gitlab.exceptions.GitlabError):
                     if e.response_code == 500:
                         # retry on this error
@@ -52,9 +49,11 @@ def retry(func: TR) -> TR:
                     raise e  # get out of the loop
                 else:
                     logging.warning(
-                        'Network failure in {}, retrying ({})'.format(getattr(func, '__name__', '(unknown callable)'),
+                        'Network failure in {}, retrying ({})'.format(
-                                                                      retried))
+                            getattr(func, '__name__', '(unknown callable)'), retried
-                    time.sleep(2 ** retried)  # wait a bit more after each retry
+                        )
+                    )
+                    time.sleep(2**retried)  # wait a bit more after each retry
                     continue
             else:
                 break
@@ -63,26 +62,26 @@ def retry(func: TR) -> TR:
     return wrapper
 
 
-class Gitlab(object):
+class Gitlab:
     JOB_NAME_PATTERN = re.compile(r'(\w+)(\s+(\d+)/(\d+))?')
 
     DOWNLOAD_ERROR_MAX_RETRIES = 3
     DEFAULT_BUILD_CHILD_PIPELINE_NAME = 'Build Child Pipeline'
 
-    def __init__(self, project_id: Union[int, str, None] = None):
+    def __init__(self, project_id: int | str | None = None):
         config_data_from_env = os.getenv('PYTHON_GITLAB_CONFIG')
         if config_data_from_env:
             # prefer to load config from env variable
             with tempfile.NamedTemporaryFile('w', delete=False) as temp_file:
                 temp_file.write(config_data_from_env)
-            config_files = [temp_file.name]  # type: Optional[List[str]]
+            config_files = [temp_file.name]
         else:
             # otherwise try to use config file at local filesystem
             config_files = None
         self._init_gitlab_inst(project_id, config_files)
 
     @retry
-    def _init_gitlab_inst(self, project_id: Optional[int], config_files: Optional[List[str]]) -> None:
+    def _init_gitlab_inst(self, project_id: int | None, config_files: list[str] | None) -> None:
         gitlab_id = os.getenv('LOCAL_GITLAB_HTTPS_HOST')  # if None, will use the default gitlab server
         self.gitlab_inst = gitlab.Gitlab.from_config(gitlab_id=gitlab_id, config_files=config_files)
 
@@ -109,7 +108,7 @@ class Gitlab(object):
             self.project = None
 
     @retry
-    def get_project_id(self, name: str, namespace: Optional[str] = None) -> int:
+    def get_project_id(self, name: str, namespace: str | None = None) -> int:
         """
         search project ID by name
 
@@ -152,7 +151,7 @@ class Gitlab(object):
             archive_file.extractall(destination)
 
     @retry
-    def download_artifact(self, job_id: int, artifact_path: List[str], destination: Optional[str] = None) -> List[bytes]:
+    def download_artifact(self, job_id: int, artifact_path: list[str], destination: str | None = None) -> list[bytes]:
         """
         download specific path of job artifacts and extract to destination.
 
@@ -169,7 +168,7 @@ class Gitlab(object):
             try:
                 data = job.artifact(a_path)  # type: bytes
             except gitlab.GitlabGetError as e:
-                logging.error("Failed to download '{}' from job {}".format(a_path, job_id))
+                logging.error(f"Failed to download '{a_path}' from job {job_id}")
                 raise e
             raw_data_list.append(data)
             if destination:
@@ -185,7 +184,7 @@ class Gitlab(object):
         return raw_data_list
 
     @retry
-    def find_job_id(self, job_name: str, pipeline_id: Optional[str] = None, job_status: str = 'success') -> List[Dict]:
+    def find_job_id(self, job_name: str, pipeline_id: str | None = None, job_status: str = 'success') -> list[dict]:
         """
         Get Job ID from job name of specific pipeline
 
@@ -208,8 +207,9 @@ class Gitlab(object):
         return job_id_list
 
     @retry
-    def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None,
+    def download_archive(
-                         cache_dir: Optional[str] = None) -> str:
+        self, ref: str, destination: str, project_id: int | None = None, cache_dir: str | None = None
+    ) -> str:
         """
         Download archive of certain commit of a repository and extract to destination path
 
@@ -233,10 +233,11 @@ class Gitlab(object):
                     try:
                         project.repository_archive(sha=ref, streamed=True, action=fw.write)
                     except gitlab.GitlabGetError as e:
-                        logging.error('Failed to archive from project {}'.format(project_id))
+                        logging.error(f'Failed to archive from project {project_id}')
                         raise e
-                logging.info('Downloaded archive size: {:.03f}MB'.format(
+                logging.info(
-                    float(os.path.getsize(local_archive_file)) / (1024 * 1024)))
+                    f'Downloaded archive size: {float(os.path.getsize(local_archive_file)) / (1024 * 1024):.03f}MB'
+                )
 
             return self.decompress_archive(local_archive_file, destination)
 
@@ -245,25 +246,39 @@ class Gitlab(object):
             try:
                 project.repository_archive(sha=ref, streamed=True, action=temp_file.write)
             except gitlab.GitlabGetError as e:
-                logging.error('Failed to archive from project {}'.format(project_id))
+                logging.error(f'Failed to archive from project {project_id}')
                 raise e
 
-        logging.info('Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(temp_file.name)) / (1024 * 1024)))
+        logging.info(f'Downloaded archive size: {float(os.path.getsize(temp_file.name)) / (1024 * 1024):.03f}MB')
 
         return self.decompress_archive(temp_file.name, destination)
 
+    @staticmethod
+    def _to_win32_long_path(path: str) -> str:
+        normalized_path = os.path.normpath(os.path.abspath(path))
+        if normalized_path.startswith('\\\\?\\'):
+            return normalized_path
+        if normalized_path.startswith('\\\\'):
+            return '\\\\?\\UNC\\' + normalized_path[2:]
+        return '\\\\?\\' + normalized_path
+
     @staticmethod
     def decompress_archive(path: str, destination: str) -> str:
         full_destination = os.path.abspath(destination)
-        # By default max path length is set to 260 characters
-        # Prefix `\\?\` extends it to 32,767 characters
-        if sys.platform == 'win32':
-            full_destination = '\\\\?\\' + full_destination
 
         try:
             with tarfile.open(path, 'r') as archive_file:
-                root_name = archive_file.getnames()[0]
+                members = archive_file.getmembers()
-                archive_file.extractall(full_destination)
+                root_name = members[0].name
+                if sys.platform == 'win32':
+                    # tarfile keeps archive member names in POSIX form. Normalize them before
+                    # combining with a long-path-prefixed destination to avoid invalid mixed separators.
+                    full_destination = Gitlab._to_win32_long_path(full_destination)
+                    members = [copy.copy(member) for member in members]
+                    for member in members:
+                        member.name = member.name.replace('/', '\\')
+                        member.linkname = member.linkname.replace('/', '\\')
+                archive_file.extractall(full_destination, members=members)
         except tarfile.TarError as e:
             logging.error(f'Error while decompressing archive {path}')
             raise e
@@ -280,7 +295,7 @@ class Gitlab(object):
         job = self.project.jobs.get(job_id)
         return ','.join(job.tag_list)
 
-    def get_downstream_pipeline_ids(self, main_pipeline_id: int) -> List[int]:
+    def get_downstream_pipeline_ids(self, main_pipeline_id: int) -> list[int]:
         """
         Retrieve the IDs of all downstream child pipelines for a given main pipeline.
 
@@ -313,7 +328,7 @@ class Gitlab(object):
 
         return [pid for pid in child_pipeline_ids if pid is not None]
 
-    def retry_failed_jobs(self, pipeline_id: int, retry_allowed_failures: bool = False) -> List[int]:
+    def retry_failed_jobs(self, pipeline_id: int, retry_allowed_failures: bool = False) -> list[int]:
         """
         Retry failed jobs for a specific pipeline. Optionally include jobs marked as 'allowed failures'.
 
@@ -370,10 +385,10 @@ def main() -> None:
         gitlab_inst.download_archive(args.ref, args.destination)
     elif args.action == 'get_project_id':
         ret = gitlab_inst.get_project_id(args.project_name)
-        print('project id: {}'.format(ret))
+        print(f'project id: {ret}')
     elif args.action == 'retry_failed_jobs':
         res = gitlab_inst.retry_failed_jobs(args.pipeline_id, args.retry_allowed_failures)
-        print('jobs retried successfully: {}'.format(res))
+        print(f'jobs retried successfully: {res}')
     elif args.action == 'get_job_tags':
         ret = gitlab_inst.get_job_tags(args.job_id)
         print(ret)

tools/cmake/component.cmake

@@ -97,9 +97,9 @@ endmacro()
 #
 function(__component_dir_quick_check var component_dir)
     set(res 1)
-    get_filename_component(abs_dir ${component_dir} ABSOLUTE)
+    get_filename_component(abs_dir "${component_dir}" ABSOLUTE)
 
-    get_filename_component(base_dir ${abs_dir} NAME)
+    get_filename_component(base_dir "${abs_dir}" NAME)
     string(SUBSTRING "${base_dir}" 0 1 first_char)
 
     # Check the component directory contains a CMakeLists.txt file
@@ -149,8 +149,8 @@ function(__component_add component_dir prefix component_source)
     # so later in the build, these component targets actually contain the properties meant for the
     # corresponding component library.
     idf_build_get_property(component_targets __COMPONENT_TARGETS)
-    get_filename_component(abs_dir ${component_dir} ABSOLUTE)
+    get_filename_component(abs_dir "${component_dir}" ABSOLUTE)
-    get_filename_component(base_dir ${abs_dir} NAME)
+    get_filename_component(base_dir "${abs_dir}" NAME)
 
     if(NOT EXISTS "${abs_dir}/CMakeLists.txt")
         message(FATAL_ERROR "Directory '${component_dir}' does not contain a component.")
@@ -277,15 +277,15 @@ macro(__component_add_sources sources)
             message(WARNING "SRCS and SRC_DIRS are both specified; ignoring SRC_DIRS.")
         endif()
         foreach(src ${__SRCS})
-            get_filename_component(src "${src}" ABSOLUTE BASE_DIR ${COMPONENT_DIR})
+            get_filename_component(src "${src}" ABSOLUTE BASE_DIR "${COMPONENT_DIR}")
-            list(APPEND sources ${src})
+            list(APPEND sources "${src}")
         endforeach()
     else()
         if(__SRC_DIRS)
             foreach(dir ${__SRC_DIRS})
-                get_filename_component(abs_dir ${dir} ABSOLUTE BASE_DIR ${COMPONENT_DIR})
+                get_filename_component(abs_dir "${dir}" ABSOLUTE BASE_DIR "${COMPONENT_DIR}")
 
-                if(NOT IS_DIRECTORY ${abs_dir})
+                if(NOT IS_DIRECTORY "${abs_dir}")
                     message(FATAL_ERROR "SRC_DIRS entry '${dir}' does not exist.")
                 endif()
 
@@ -294,7 +294,7 @@ macro(__component_add_sources sources)
 
                 if(dir_sources)
                     foreach(src ${dir_sources})
-                        get_filename_component(src "${src}" ABSOLUTE BASE_DIR ${COMPONENT_DIR})
+                        get_filename_component(src "${src}" ABSOLUTE BASE_DIR "${COMPONENT_DIR}")
                         list(APPEND sources "${src}")
                     endforeach()
                 else()
@@ -316,11 +316,11 @@ endmacro()
 
 macro(__component_add_include_dirs lib dirs type)
     foreach(dir ${dirs})
-        get_filename_component(_dir ${dir} ABSOLUTE BASE_DIR ${CMAKE_CURRENT_LIST_DIR})
+        get_filename_component(_dir "${dir}" ABSOLUTE BASE_DIR "${CMAKE_CURRENT_LIST_DIR}")
-        if(NOT IS_DIRECTORY ${_dir})
+        if(NOT IS_DIRECTORY "${_dir}")
             message(FATAL_ERROR "Include directory '${_dir}' is not a directory.")
         endif()
-        target_include_directories(${lib} ${type} ${_dir})
+        target_include_directories(${lib} ${type} "${_dir}")
     endforeach()
 endmacro()
 

tools/test_build_system/test_idf_extension.py

@@ -38,6 +38,9 @@ version = "0.1.0"
 
 [project.entry-points.idf_extension]
 {entry_point_name} = "{declarative_value}"
+
+[tool.setuptools]
+packages = ["test_extension_package_{suffix}"]
 """
 
 

tools/test_idf_py/test_hints.py

@@ -33,7 +33,8 @@ def safe_cleanup_tmpdir(tmpdir: tempfile.TemporaryDirectory) -> None:
     """Safely cleanup temporary directory, handling specific errors on Windows."""
     try:
         tmpdir.cleanup()
-    except (PermissionError, NotADirectoryError):
+    except (PermissionError, NotADirectoryError, OSError):
+        # OSError: The directory is not empty
         warnings.warn(
             f'Failed to cleanup temporary directory {tmpdir.name}. '
             'This is common on Windows when files are still in use.',

tools/test_idf_py/test_idf_py.py

@@ -27,9 +27,9 @@ except ImportError:
     import idf
 
 current_dir = os.path.dirname(os.path.realpath(__file__))
-idf_py_path = os.path.join(current_dir, '..', 'idf.py')
+idf_py_path = os.path.normpath(os.path.join(current_dir, '..', 'idf.py'))
 extension_path = os.path.join(current_dir, 'test_idf_extensions', 'test_ext')
-py_actions_path = os.path.join(current_dir, '..', 'idf_py_actions')
+py_actions_path = os.path.normpath(os.path.join(current_dir, '..', 'idf_py_actions'))
 link_path = os.path.join(py_actions_path, 'test_ext')