ci: windows with VM

.gitlab/ci/common.yml

@@ -281,6 +281,31 @@ variables:
   after_script: []  # ccache now is disabled for macos brew runners
   timeout: 30m
 
+.brew-windows-settings:
+  image: windows_11_idf_v6_1
+  tags: [windows-vm, brew]
+  variables:
+    GIT_STRATEGY: fetch
+    LC_ALL: C.UTF-8
+    # git clean at the final cleanup stage stuck on Windows VM
+    # a new VM for each job anyway, so we can skip cleaning the workspace
+    GIT_CLEAN_FLAGS: none
+#    IDF_CCACHE_ENABLE: "1"
+#    CCACHE_DIR: "C:/cache/idf_ccache"
+  before_script:
+    - if ($env:IDF_DONT_USE_MIRRORS) { $env:IDF_MIRROR_PREFIX_MAP = '' }
+    - ./install.ps1 --enable-ci
+    - ./export.ps1
+    - $env:PYTHONPATH = "$env:PYTHONPATH;$env:IDF_PATH\tools;$env:IDF_PATH\tools\esp_app_trace;$env:IDF_PATH\components\partition_table;$env:IDF_PATH\tools\ci\python_packages"
+    - python "${SUBMODULE_FETCH_TOOL}" -s "${SUBMODULES_TO_FETCH}"
+    # The time in VM may not be synced, since it's built a long time ago, so we need to sync time
+    - net start w32time
+    - w32tm /config /manualpeerlist:"time.google.com,0x9 pool.ntp.org,0x9" /syncfromflags:manual /reliable:yes /update
+    - net stop w32time
+    - net start w32time
+    - w32tm /resync
+  after_script: []
+
 #############
 # `default` #
 #############

.gitlab/ci/test-win.yml

@@ -1,9 +1,8 @@
-# Host tests
 .host_test_win_template:
-  extends: .rules:test:host_test
+  extends:
+    - .brew-windows-settings
+    - .rules:test:host_test
   stage: host_test
-  image: $ESP_ENV_IMAGE
-  tags: [windows-build, brew]
   dependencies: # set dependencies to null to avoid missing artifacts issue
   # run host_test jobs immediately, only after upload cache
   needs:
@@ -13,13 +12,8 @@
     - job: upload-submodules-cache
       optional: true
       artifacts: false
-  variables:
+    - job: manual_gate
-    GIT_STRATEGY: fetch  # use brew local mirror first
+      optional: true
-  before_script:
-    - if ($env:IDF_DONT_USE_MIRRORS) {
-        $env:IDF_MIRROR_PREFIX_MAP = ""
-      }
-  after_script: []
 
 test_cli_installer_win:
   rules:
@@ -34,15 +28,12 @@ test_cli_installer_win:
       - tools/tools.new.json
       - tools/test_idf_tools/test_python_env_logs.txt
     expire_in: 1 week
-  variables:
-    IDF_PATH: "$CI_PROJECT_DIR"
   timeout: 3h
   script:
     # Tools must be downloaded for testing
     #                                                                     cmake@version that is supported
-    - python ${IDF_PATH}\tools\idf_tools.py download required qemu-riscv32 qemu-xtensa cmake cmake@3.22.1
+    - python tools\idf_tools.py download required qemu-riscv32 qemu-xtensa cmake cmake@3.22.1
-    - cd ${IDF_PATH}\tools\test_idf_tools
+    - cd tools\test_idf_tools
-    - python -m pip install jsonschema
     - python .\test_idf_tools.py
     - python .\test_idf_tools_python_env.py
 
@@ -53,39 +44,31 @@ test_tools_win:
   parallel: 4
   artifacts:
     paths:
-      - ${IDF_PATH}/*.out
+      - "*.out"
-      - ${IDF_PATH}/XUNIT_*.xml
+      - "XUNIT_*.xml"
     reports:
-      junit: ${IDF_PATH}/XUNIT_*.xml
+      junit: "XUNIT_*.xml"
     expire_in: 1 week
     when: always
-  variables:
-    LC_ALL: C.UTF-8
-    PYTHONPATH: "$PYTHONPATH;$IDF_PATH\\tools;$IDF_PATH\\tools\\esp_app_trace;$IDF_PATH\\components\\partition_table;$IDF_PATH\\tools\\ci\\python_packages"
   script:
-    - python -m pip install jsonschema
+    - cd tools\test_idf_py
-    - .\install.ps1 --enable-ci
-    - .\export.ps1
-    - python "${SUBMODULE_FETCH_TOOL}" -s "all"
-    - cd ${IDF_PATH}/tools/test_idf_py
     - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
-    - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=${IDF_PATH}/XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
+    - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
 
 # Build tests
 .test_build_system_template_win:
-  stage: host_test
+  extends: .host_test_win_template
-  variables:
-  # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings.
-    IDF_CCACHE_ENABLE: "1"
-    PYTHONPATH: "$PYTHONPATH;$IDF_PATH\\tools;$IDF_PATH\\tools\\esp_app_trace;$IDF_PATH\\components\\partition_table;$IDF_PATH\\tools\\ci\\python_packages"
-  before_script: []
-  after_script: []
   timeout: 4 hours
+  artifacts:
+    paths:
+      - XUNIT_RESULT.xml
+      - test_build_system
+    expire_in: 2 days
+    when: always
+    reports:
+      junit: XUNIT_RESULT.xml
   script:
-    - .\install.ps1 --enable-ci
+    - cd tools\test_build_system
-    - . .\export.ps1
-    - python "${SUBMODULE_FETCH_TOOL}" -s "all"
-    - cd ${IDF_PATH}\tools\test_build_system
     - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
     - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} --durations=10
 
@@ -94,42 +77,16 @@ pytest_build_system_win:
     - .test_build_system_template_win
     - .rules:labels:windows_pytest_build_system
   parallel: 10
-  needs:
-    - job: manual_gate
-      optional: true
-  tags: [windows-build, brew]
-  artifacts:
-    paths:
-      - XUNIT_RESULT.xml
-      - test_build_system
-    expire_in: 2 days
-    reports:
-      junit: XUNIT_RESULT.xml
-    when: always
 
 pytest_build_system_win_minimal_cmake:
   extends:
     - .test_build_system_template_win
     - .rules:labels:windows_pytest_build_system
   parallel: 2
-  needs:
-    - job: manual_gate
-      optional: true
-  tags: [windows-build, brew]
-  artifacts:
-    paths:
-      - XUNIT_RESULT.xml
-      - test_build_system
-    expire_in: 2 days
-    reports:
-      junit: XUNIT_RESULT.xml
-    when: always
   variables:
     MINIMAL_CMAKE_VERSION: "3.22.1"
   script:
-    - .\install.ps1 --enable-ci
+    - python tools\idf_tools.py install cmake@${MINIMAL_CMAKE_VERSION}
-    - . .\export.ps1
-    - python ${IDF_PATH}\tools\idf_tools.py install cmake@${MINIMAL_CMAKE_VERSION}
     - $Env:PATH = "$Env:USERPROFILE\.espressif\tools\cmake\${MINIMAL_CMAKE_VERSION}\bin;$Env:PATH"
     - |
       $actualVersion = (& cmake --version).Split()[2]
@@ -137,8 +94,7 @@ pytest_build_system_win_minimal_cmake:
         Write-Error "ERROR: Wrong CMake version! Detected: $actualVersion, but expected: $Env:MINIMAL_CMAKE_VERSION"
         exit 1
       }
-    - python "${SUBMODULE_FETCH_TOOL}" -s "all"
+    - cd tools\test_build_system
-    - cd ${IDF_PATH}\tools\test_build_system
     - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
     - pytest -k cmake --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} --durations=10
 
@@ -147,23 +103,8 @@ pytest_buildv2_system_win:
     - .test_build_system_template_win
     - .rules:labels:buildv2
   parallel: 10
-  needs:
-    - job: manual_gate
-      optional: true
-  tags: [windows-build, brew]
-  artifacts:
-    paths:
-      - XUNIT_RESULT.xml
-      - test_build_system
-    expire_in: 2 days
-    reports:
-      junit: XUNIT_RESULT.xml
-    when: always
   script:
-    - .\install.ps1 --enable-ci
+    - cd tools\test_build_system
-    - . .\export.ps1
-    - python "${SUBMODULE_FETCH_TOOL}" -s "all"
-    - cd ${IDF_PATH}\tools\test_build_system
     - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
     - pytest
       --buildv2

tools/test_idf_py/test_hints.py

@@ -33,7 +33,8 @@ def safe_cleanup_tmpdir(tmpdir: tempfile.TemporaryDirectory) -> None:
     """Safely cleanup temporary directory, handling specific errors on Windows."""
     try:
         tmpdir.cleanup()
-    except (PermissionError, NotADirectoryError):
+    except (PermissionError, NotADirectoryError, OSError):
+        # OSError: The directory is not empty
         warnings.warn(
             f'Failed to cleanup temporary directory {tmpdir.name}. '
             'This is common on Windows when files are still in use.',

tools/test_idf_py/test_idf_py.py

@@ -27,9 +27,9 @@ except ImportError:
     import idf
 
 current_dir = os.path.dirname(os.path.realpath(__file__))
-idf_py_path = os.path.join(current_dir, '..', 'idf.py')
+idf_py_path = os.path.normpath(os.path.join(current_dir, '..', 'idf.py'))
 extension_path = os.path.join(current_dir, 'test_idf_extensions', 'test_ext')
-py_actions_path = os.path.join(current_dir, '..', 'idf_py_actions')
+py_actions_path = os.path.normpath(os.path.join(current_dir, '..', 'idf_py_actions'))
 link_path = os.path.join(py_actions_path, 'test_ext')