ci: apply idf-ci 1.x

(cherry picked from commit 21d772a24da646bfb672418cf056199cad9b17e4)

.gitlab/ci/common.yml

@@ -270,7 +270,11 @@ variables:
     - source tools/ci/utils.sh
     - source tools/ci/configure_ci_environment.sh
     - *show_ccache_statistics
-    - *upload_failed_job_log_artifacts
+    - run_cmd idf-ci gitlab upload-artifacts --type junit
+    - |
+      if [ $CI_JOB_STATUS = "failed" ]; then
+        run_cmd idf-ci gitlab upload-artifacts --type log
+      fi
 
 .brew-macos-settings:
   variables:
@@ -283,6 +287,46 @@ variables:
   after_script: []  # ccache now is disabled for macos brew runners
   timeout: 30m
 
+.brew-windows-settings:
+  image: windows_11_idf_v6_0
+  tags: [windows-vm, brew]
+  variables:
+    GIT_STRATEGY: fetch
+    LC_ALL: C.UTF-8
+    # git clean at the final cleanup stage stuck on Windows VM
+    # a new VM for each job anyway, so we can skip cleaning the workspace
+    GIT_CLEAN_FLAGS: none
+#    IDF_CCACHE_ENABLE: "1"
+#    CCACHE_DIR: "C:/cache/idf_ccache"
+  before_script:
+    - if ($env:IDF_DONT_USE_MIRRORS) { $env:IDF_MIRROR_PREFIX_MAP = '' }
+    - ./install.ps1 --enable-ci
+    - ./export.ps1
+    - $env:PYTHONPATH = "$env:PYTHONPATH;$env:IDF_PATH\tools;$env:IDF_PATH\tools\esp_app_trace;$env:IDF_PATH\components\partition_table;$env:IDF_PATH\tools\ci\python_packages"
+    - python "${SUBMODULE_FETCH_TOOL}" -s "${SUBMODULES_TO_FETCH}"
+    # Sync VM clock. net start/stop exit 2 when the service is already in that state; do not fail CI.
+    - |
+      function Invoke-NetService($Action, $Name) {
+        $output = net $Action $Name 2>&1
+        if ($LASTEXITCODE -eq 2) {
+          Write-Host "net $Action $Name already in requested state: $output"
+          $global:LASTEXITCODE = 0
+          return
+        }
+        if ($LASTEXITCODE -ne 0) {
+          Write-Host $output
+          exit $LASTEXITCODE
+        }
+      }
+      Invoke-NetService start w32time
+      w32tm /config /manualpeerlist:"time.google.com,0x9 pool.ntp.org,0x9" /syncfromflags:manual /reliable:yes /update
+      if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+      Invoke-NetService stop w32time
+      Invoke-NetService start w32time
+      w32tm /resync
+      if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
+  after_script: []
+
 #############
 # `default` #
 #############

.gitlab/ci/host-test.yml

@@ -373,7 +373,6 @@ test_pytest_qemu:
     - run_cmd pytest
       --target $IDF_TARGET
       -m qemu
-      --embedded-services idf,qemu
       --junitxml=XUNIT_RESULT.xml
       --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
       --qemu-extra-args \"-global driver=timer.$IDF_TARGET.timg,property=wdt_disable,value=true\"
@@ -398,7 +397,7 @@ test_pytest_linux:
     - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
     - run_cmd pytest
       --target linux
-      --embedded-services idf
+      -m \"not macos\"
       --junitxml=XUNIT_RESULT.xml
       --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
 

.gitlab/ci/rules.yml

@@ -68,6 +68,7 @@
   - "tools/ci/check_esp_memory_utils_headers.sh"
   - "tools/ci/check_blobs.sh"
   - "tools/ci/check_public_headers.py"
+  - "tools/ci/check_public_headers_exceptions.txt"
   - "tools/ci/check_register_rw_half_word.cmake"
   - "tools/ci/check_register_rw_half_word.py"
   - "examples/build_system/**/*"
@@ -102,6 +103,7 @@
   - "tools/idf_tools.py"
   - "tools/test_idf_tools/**/*"
   - "tools/install_util.py"
+  - "tools/idf_extra_components.yml"
 
   - "tools/export_utils/utils.py"
   - "tools/export_utils/shell_types.py"

.gitlab/ci/test-win.yml

@@ -1,9 +1,8 @@
-# Host tests
 .host_test_win_template:
-  extends: .rules:test:host_test
+  extends:
+    - .brew-windows-settings
+    - .rules:test:host_test
   stage: host_test
-  image: $ESP_ENV_IMAGE
-  tags: [windows-build, brew]
   dependencies: # set dependencies to null to avoid missing artifacts issue
   # run host_test jobs immediately, only after upload cache
   needs:
@@ -13,13 +12,8 @@
     - job: upload-submodules-cache
       optional: true
       artifacts: false
-  variables:
-    GIT_STRATEGY: fetch  # use brew local mirror first
-  before_script:
-    - if ($env:IDF_DONT_USE_MIRRORS) {
-        $env:IDF_MIRROR_PREFIX_MAP = ""
-      }
-  after_script: []
+    - job: manual_gate
+      optional: true
 
 test_cli_installer_win:
   rules:
@@ -34,15 +28,12 @@ test_cli_installer_win:
       - tools/tools.new.json
       - tools/test_idf_tools/test_python_env_logs.txt
     expire_in: 1 week
-  variables:
-    IDF_PATH: "$CI_PROJECT_DIR"
   timeout: 3h
   script:
     # Tools must be downloaded for testing
     #                                                                     cmake@version that is supported
-    - python ${IDF_PATH}\tools\idf_tools.py download required qemu-riscv32 qemu-xtensa cmake cmake@3.22.1
-    - cd ${IDF_PATH}\tools\test_idf_tools
-    - python -m pip install jsonschema
+    - python tools\idf_tools.py download required qemu-riscv32 qemu-xtensa cmake cmake@3.22.1
+    - cd tools\test_idf_tools
     - python .\test_idf_tools.py
     - python .\test_idf_tools_python_env.py
 
@@ -53,81 +44,47 @@ test_tools_win:
   parallel: 4
   artifacts:
     paths:
-      - ${IDF_PATH}/*.out
-      - ${IDF_PATH}/XUNIT_*.xml
+      - "*.out"
+      - "XUNIT_*.xml"
     reports:
-      junit: ${IDF_PATH}/XUNIT_*.xml
+      junit: "XUNIT_*.xml"
     expire_in: 1 week
     when: always
-  variables:
-    LC_ALL: C.UTF-8
-    PYTHONPATH: "$PYTHONPATH;$IDF_PATH\\tools;$IDF_PATH\\tools\\esp_app_trace;$IDF_PATH\\components\\partition_table;$IDF_PATH\\tools\\ci\\python_packages"
   script:
-    - python -m pip install jsonschema
-    - .\install.ps1 --enable-ci
-    - .\export.ps1
-    - python "${SUBMODULE_FETCH_TOOL}" -s "all"
-    - cd ${IDF_PATH}/tools/test_idf_py
+    - cd tools\test_idf_py
     - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
-    - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=${IDF_PATH}/XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
+    - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml ${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
 
 # Build tests
 .test_build_system_template_win:
-  stage: host_test
-  variables:
-  # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings.
-    IDF_CCACHE_ENABLE: "1"
-    PYTHONPATH: "$PYTHONPATH;$IDF_PATH\\tools;$IDF_PATH\\tools\\esp_app_trace;$IDF_PATH\\components\\partition_table;$IDF_PATH\\tools\\ci\\python_packages"
-  before_script: []
-  after_script: []
+  extends: .host_test_win_template
   timeout: 4 hours
+  artifacts:
+    paths:
+      - XUNIT_RESULT.xml
+      - test_build_system
+    expire_in: 2 days
+    when: always
+    reports:
+      junit: XUNIT_RESULT.xml
   script:
-    - .\install.ps1 --enable-ci
-    - . .\export.ps1
-    - python "${SUBMODULE_FETCH_TOOL}" -s "all"
-    - cd ${IDF_PATH}\tools\test_build_system
+    - cd tools\test_build_system
     - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
-    - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
+    - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml ${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}
 
 pytest_build_system_win:
   extends:
     - .test_build_system_template_win
     - .rules:labels:windows_pytest_build_system
   parallel: 2
-  needs:
-    - job: manual_gate
-      optional: true
-  tags: [windows-build, brew]
-  artifacts:
-    paths:
-      - XUNIT_RESULT.xml
-      - test_build_system
-    expire_in: 2 days
-    reports:
-      junit: XUNIT_RESULT.xml
-    when: always
 
 pytest_build_system_win_minimal_cmake:
   extends:
     - .test_build_system_template_win
     - .rules:labels:windows_pytest_build_system
-  needs:
-    - job: manual_gate
-      optional: true
-  tags: [windows-build, brew]
-  artifacts:
-    paths:
-      - XUNIT_RESULT.xml
-      - test_build_system
-    expire_in: 2 days
-    reports:
-      junit: XUNIT_RESULT.xml
-    when: always
   variables:
     MINIMAL_CMAKE_VERSION: "3.16.3"
   script:
-    - .\install.ps1 --enable-ci
-    - . .\export.ps1
     - python ${IDF_PATH}\tools\idf_tools.py install cmake@${MINIMAL_CMAKE_VERSION}
     - $Env:PATH = "$Env:USERPROFILE\.espressif\tools\cmake\${MINIMAL_CMAKE_VERSION}\bin;$Env:PATH"
     - |
@@ -136,7 +93,6 @@ pytest_build_system_win_minimal_cmake:
         Write-Error "ERROR: Wrong CMake version! Detected: $actualVersion, but expected: $Env:MINIMAL_CMAKE_VERSION"
         exit 1
       }
-    - python "${SUBMODULE_FETCH_TOOL}" -s "all"
     - cd ${IDF_PATH}\tools\test_build_system
     - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
     - pytest -k cmake --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}