Revert "ci: replace ci_fetch_submodules.py with submodule cache"

This reverts commit 0af3e2b1b2.

.gitlab/ci/common.yml

@@ -39,6 +39,14 @@ variables:
   GIT_CLEAN_FLAGS: -ffdx -e .cache/
   LATEST_GIT_TAG: v6.0-dev
 
+  SUBMODULE_FETCH_TOOL: "tools/ci/ci_fetch_submodule.py"
+  # by default we will fetch all submodules
+  # jobs can overwrite this variable to only fetch submodules they required
+  # set to "none" if don't need to fetch submodules
+  SUBMODULES_TO_FETCH: "all"
+  # tell build system do not check submodule update as we download archive instead of clone
+  IDF_SKIP_CHECK_SUBMODULES: 1
+
   IDF_PATH: "$CI_PROJECT_DIR"
   V: "0"
   CHECKOUT_REF_SCRIPT: "$CI_PROJECT_DIR/tools/ci/checkout_project_ref.py"
@@ -381,9 +389,11 @@ default:
       paths:
         - .cache/pip
       policy: pull
-    - key: git-submodule-cache-${LATEST_GIT_TAG}
+    - key: submodule-cache-${LATEST_GIT_TAG}
+      fallback_keys:
+        - submodule-cache
       paths:
-        - .git/modules
+        - .cache/submodule_archives
       policy: pull
   before_script:
     - *common-before_scripts

.gitlab/ci/deploy.yml

@@ -12,6 +12,7 @@ check_submodule_sync:
   retry: 2
   variables:
     GIT_STRATEGY: fetch  # use brew local mirror first
+    SUBMODULES_TO_FETCH: "none"
     PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git"
   dependencies: []
   script:

.gitlab/ci/host-test.yml

@@ -383,6 +383,8 @@ test_idf_pytest_plugin:
   extends:
     - .host_test_template
     - .rules:patterns:idf-pytest-plugin
+  variables:
+    SUBMODULES_TO_FETCH: "none"
   artifacts:
     reports:
       junit: XUNIT_RESULT.xml

.gitlab/ci/integration_test.yml

@@ -36,6 +36,7 @@ gen_integration_pipeline:
   cache: []
   tags: [fast_run, shiny]
   variables:
+    SUBMODULES_TO_FETCH: "none"
     GIT_LFS_SKIP_SMUDGE: 1
   needs:
     - job: fast_template_app

.gitlab/ci/pre_check.yml

@@ -15,6 +15,7 @@ check_version:
     # need a full clone to get the latest tag
     # the --shallow-since=$(git log -1 --format=%as $LATEST_GIT_TAG) option is not accurate
     GIT_STRATEGY: fetch
+    SUBMODULES_TO_FETCH: "none"
     GIT_DEPTH: 0
   script:
     - export IDF_PATH=$PWD
@@ -32,6 +33,8 @@ check_blobs:
   extends:
     - .pre_check_template
     - .rules:build:check
+  variables:
+    SUBMODULES_TO_FETCH: "components/esp_wifi/lib;components/esp_phy/lib;components/esp_coex/lib"
   script:
     # Check if Wi-Fi library header files match between IDF and the version used when compiling the libraries
     - IDF_TARGET=esp32 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
@@ -104,6 +107,7 @@ check_version_tag:
     # need a full clone to get the latest tag
     # the --shallow-since=$(git log -1 --format=%as $LATEST_GIT_TAG) option is not accurate
     GIT_STRATEGY: fetch
+    SUBMODULES_TO_FETCH: "none"
     GIT_DEPTH: 0
   script:
     - (git cat-file -t $CI_COMMIT_REF_NAME | grep tag) || (echo "ESP-IDF versions must be annotated tags." && exit 1)

.gitlab/ci/pre_commit.yml

@@ -24,9 +24,9 @@ check_pre_commit_upload_cache:
       paths:
         - .cache/pre-commit
       policy: pull-push
-    - key: git-submodule-cache-${LATEST_GIT_TAG}
+    - key: submodule-cache-${LATEST_GIT_TAG}
       paths:
-        - .git/modules
+        - .cache/submodule_archives
       policy: pull
 
 check_pre_commit:
@@ -41,9 +41,9 @@ check_pre_commit:
       paths:
         - .cache/pre-commit
       policy: pull
-    - key: git-submodule-cache-${LATEST_GIT_TAG}
+    - key: submodule-cache-${LATEST_GIT_TAG}
       paths:
-        - .git/modules
+        - .cache/submodule_archives
       policy: pull
 
 check_powershell:
@@ -74,7 +74,7 @@ check_powershell:
       paths:
         - .cache/pre-commit
       policy: pull
-    - key: git-submodule-cache-${LATEST_GIT_TAG}
+    - key: submodule-cache-${LATEST_GIT_TAG}
       paths:
-        - .git/modules
+        - .cache/submodule_archives
       policy: pull

.gitlab/ci/rules.yml

@@ -249,9 +249,6 @@
       changes: *patterns-submodule
     - <<: *if-label-upload_cache
       when: manual
-    - <<: *if-dev-push
-      changes:
-        - .gitlab/ci/upload_cache.yml
 
 ### Patterns ###
 .rules:patterns:clang_tidy:

.gitlab/ci/test-win.yml

@@ -62,7 +62,7 @@ test_tools_win:
     - python -m pip install jsonschema
     - .\install.ps1 --enable-ci --enable-pytest
     - .\export.ps1
-    - git submodule update --init
+    - python "${SUBMODULE_FETCH_TOOL}" -s "all"
     - cd ${IDF_PATH}/tools/test_idf_py
     - pytest --noconftest test_idf_py.py       --junitxml=${IDF_PATH}/XUNIT_IDF_PY.xml
     - pytest --noconftest test_hints.py        --junitxml=${IDF_PATH}/XUNIT_HINTS.xml
@@ -80,7 +80,7 @@ test_tools_win:
   script:
     - .\install.ps1 --enable-ci --enable-pytest
     - . .\export.ps1
-    - git submodule update --init
+    - python "${SUBMODULE_FETCH_TOOL}" -s "all"
     - cd ${IDF_PATH}\tools\test_build_system
     - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml
 

.gitlab/ci/upload_cache.yml

@@ -35,14 +35,14 @@ upload-submodules-cache:
     - $GEO
     - cache
   cache:
-    key: git-submodule-cache-${LATEST_GIT_TAG}
+    key: submodule-cache-${LATEST_GIT_TAG}
     paths:
-      - .git/modules
+      - .cache/submodule_archives
     policy: push
   script:
     # use the default gitlab server
     - unset LOCAL_GITLAB_HTTPS_HOST
-    - rm -rf .git/modules  # clear old submodules
+    - rm -rf .cache/submodule_archives  # clear old submodule archives
     - add_gitlab_ssh_keys
     - fetch_submodules
   parallel: