Merge branch 'ci/remove-submodule-fetch-script' into 'master'

Ci/remove submodule fetch script

Closes IDFCI-2980

See merge request espressif/esp-idf!39992

.gitlab/ci/build.yml

@@ -3,10 +3,7 @@
   extends:
     - .after_script:build:ccache-show-stats:upload-failed-job-logs
   image: $ESP_ENV_IMAGE
-  tags:
-    - build
-    # build only on shiny servers since shiny storage server is at the same location
-    - shiny
+  tags: [build, shiny]
   variables:
     # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings.
     IDF_CCACHE_ENABLE: "1"

.gitlab/ci/common.yml

@@ -30,26 +30,15 @@ variables:
   # - set GIT_STRATEGY: "clone" to shiny runners
   # - set GIT_STRATEGY: "fetch" to brew runners
   GIT_STRATEGY: clone
-  # we will download archive for each submodule instead of clone.
-  # we don't do "recursive" when fetch submodule as they're not used in CI now.
-  GIT_SUBMODULE_STRATEGY: none
-  # since we're using merged-result pipelines, the last commit should work for most cases
   GIT_DEPTH: 1
-  # --no-recurse-submodules: we use cache for submodules
+  GIT_SUBMODULE_STRATEGY: none  # here we use cache for submodules, so we don't need to fetch them every time
+  # since we're using merged-result pipelines, the last commit should work for most cases
   # --prune --prune-tags: in case remote branch or tag is force pushed
   GIT_FETCH_EXTRA_FLAGS: "--no-recurse-submodules --prune --prune-tags"
   # we're using .cache folder for caches
   GIT_CLEAN_FLAGS: -ffdx -e .cache/
   LATEST_GIT_TAG: v6.0-dev
 
-  SUBMODULE_FETCH_TOOL: "tools/ci/ci_fetch_submodule.py"
-  # by default we will fetch all submodules
-  # jobs can overwrite this variable to only fetch submodules they required
-  # set to "none" if don't need to fetch submodules
-  SUBMODULES_TO_FETCH: "all"
-  # tell build system do not check submodule update as we download archive instead of clone
-  IDF_SKIP_CHECK_SUBMODULES: 1
-
   IDF_PATH: "$CI_PROJECT_DIR"
   V: "0"
   CHECKOUT_REF_SCRIPT: "$CI_PROJECT_DIR/tools/ci/checkout_project_ref.py"
@@ -392,11 +381,9 @@ default:
       paths:
         - .cache/pip
       policy: pull
-    - key: submodule-cache-${LATEST_GIT_TAG}
-      fallback_keys:
-        - submodule-cache
+    - key: git-submodule-cache-${LATEST_GIT_TAG}
       paths:
-        - .cache/submodule_archives
+        - .git/modules
       policy: pull
   before_script:
     - *common-before_scripts

.gitlab/ci/deploy.yml

@@ -11,9 +11,7 @@ check_submodule_sync:
   tags: [ brew, github_sync ]
   retry: 2
   variables:
-    # for brew runners, we always set GIT_STRATEGY to fetch
-    GIT_STRATEGY: fetch
-    SUBMODULES_TO_FETCH: "none"
+    GIT_STRATEGY: fetch  # use brew local mirror first
     PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git"
   dependencies: []
   script:
@@ -36,10 +34,8 @@ push_to_github:
     - check_submodule_sync
   tags: [ brew, github_sync ]
   variables:
-    # for brew runners, we always set GIT_STRATEGY to fetch
-    GIT_STRATEGY: fetch
-    # github also need full record of commits
-    GIT_DEPTH: 0
+    GIT_STRATEGY: fetch  # use brew local mirror first
+    GIT_DEPTH: 0  # github needs full record of commits
   script:
     - add_github_ssh_keys
     - git remote remove github &>/dev/null || true
@@ -58,11 +54,11 @@ deploy_update_SHA_in_esp-dockerfiles:
   dependencies: []
   variables:
     GIT_DEPTH: 2
-  tags: [ shiny, build ]
+  tags: [build, shiny]
   script:
     - 'curl --header "PRIVATE-TOKEN: ${ESPCI_SCRIPTS_TOKEN}" -o create_MR_in_esp_dockerfile.sh $GITLAB_HTTP_SERVER/api/v4/projects/1260/repository/files/create_MR_in_esp_dockerfile%2Fcreate_MR_in_esp_dockerfile.sh/raw\?ref\=master'
     - chmod +x create_MR_in_esp_dockerfile.sh
     - ./create_MR_in_esp_dockerfile.sh
   environment:
-    name: eploy_update_SHA_in_esp-dockerfiles_production
+    name: deploy_update_SHA_in_esp-dockerfiles_production
     deployment_tier: production

.gitlab/ci/docs.yml

@@ -58,7 +58,7 @@
 check_readme_links:
   extends:
     - .pre_check_template
-  tags: ["build", "amd64", "internet"]
+  tags: ["amd64", "brew"]
   allow_failure: true
   rules:
     - <<: *if-protected-check

.gitlab/ci/host-test.yml

@@ -2,8 +2,7 @@
   extends: .rules:test:host_test
   stage: host_test
   image: $ESP_ENV_IMAGE
-  tags:
-    - host_test
+  tags: [build, shiny]
   dependencies: # set dependencies to null to avoid missing artifacts issue
   # run host_test jobs immediately, only after upload cache
   needs:
@@ -384,8 +383,6 @@ test_idf_pytest_plugin:
   extends:
     - .host_test_template
     - .rules:patterns:idf-pytest-plugin
-  variables:
-    SUBMODULES_TO_FETCH: "none"
   artifacts:
     reports:
       junit: XUNIT_RESULT.xml

.gitlab/ci/integration_test.yml

@@ -36,7 +36,6 @@ gen_integration_pipeline:
   cache: []
   tags: [fast_run, shiny]
   variables:
-    SUBMODULES_TO_FETCH: "none"
     GIT_LFS_SKIP_SMUDGE: 1
   needs:
     - job: fast_template_app

.gitlab/ci/pre_check.yml

@@ -1,8 +1,7 @@
 .pre_check_template:
   stage: pre_check
   image: $ESP_ENV_IMAGE
-  tags:
-    - host_test
+  tags: [build, shiny]
   dependencies: # set dependencies to null to avoid missing artifacts issue
 
 check_version:
@@ -16,7 +15,6 @@ check_version:
     # need a full clone to get the latest tag
     # the --shallow-since=$(git log -1 --format=%as $LATEST_GIT_TAG) option is not accurate
     GIT_STRATEGY: fetch
-    SUBMODULES_TO_FETCH: "none"
     GIT_DEPTH: 0
   script:
     - export IDF_PATH=$PWD
@@ -34,8 +32,6 @@ check_blobs:
   extends:
     - .pre_check_template
     - .rules:build:check
-  variables:
-    SUBMODULES_TO_FETCH: "components/esp_wifi/lib;components/esp_phy/lib;components/esp_coex/lib"
   script:
     # Check if Wi-Fi library header files match between IDF and the version used when compiling the libraries
     - IDF_TARGET=esp32 $IDF_PATH/components/esp_wifi/test_md5/test_md5.sh
@@ -108,7 +104,6 @@ check_version_tag:
     # need a full clone to get the latest tag
     # the --shallow-since=$(git log -1 --format=%as $LATEST_GIT_TAG) option is not accurate
     GIT_STRATEGY: fetch
-    SUBMODULES_TO_FETCH: "none"
     GIT_DEPTH: 0
   script:
     - (git cat-file -t $CI_COMMIT_REF_NAME | grep tag) || (echo "ESP-IDF versions must be annotated tags." && exit 1)
@@ -211,13 +206,12 @@ baseline_manifest_sha:
     when: always
 
 redundant_pass_job:
-  stage: pre_check
+  extends:
+    - .pre_check_template
   tags: [shiny, fast_run]
-  image: $ESP_ENV_IMAGE
-  dependencies: null
-  before_script: []
   cache: []
-  extends: []
+  variables:
+    GIT_STRATEGY: none
+  before_script: []
   script:
     - echo "This job is redundant to ensure the 'retry_failed_jobs' job can exist and not be skipped"
-  when: always

.gitlab/ci/pre_commit.yml

@@ -24,9 +24,9 @@ check_pre_commit_upload_cache:
       paths:
         - .cache/pre-commit
       policy: pull-push
-    - key: submodule-cache-${LATEST_GIT_TAG}
+    - key: git-submodule-cache-${LATEST_GIT_TAG}
       paths:
-        - .cache/submodule_archives
+        - .git/modules
       policy: pull
 
 check_pre_commit:
@@ -41,9 +41,9 @@ check_pre_commit:
       paths:
         - .cache/pre-commit
       policy: pull
-    - key: submodule-cache-${LATEST_GIT_TAG}
+    - key: git-submodule-cache-${LATEST_GIT_TAG}
       paths:
-        - .cache/submodule_archives
+        - .git/modules
       policy: pull
 
 check_powershell:
@@ -74,7 +74,7 @@ check_powershell:
       paths:
         - .cache/pre-commit
       policy: pull
-    - key: submodule-cache-${LATEST_GIT_TAG}
+    - key: git-submodule-cache-${LATEST_GIT_TAG}
       paths:
-        - .cache/submodule_archives
+        - .git/modules
       policy: pull

.gitlab/ci/rules.yml

@@ -249,6 +249,9 @@
       changes: *patterns-submodule
     - <<: *if-label-upload_cache
       when: manual
+    - <<: *if-dev-push
+      changes:
+        - .gitlab/ci/upload_cache.yml
 
 ### Patterns ###
 .rules:patterns:clang_tidy:

.gitlab/ci/test-win.yml

@@ -3,8 +3,7 @@
   extends: .rules:test:host_test
   stage: host_test
   image: $ESP_ENV_IMAGE
-  tags:
-    - windows-build
+  tags: [windows-build, brew]
   dependencies: # set dependencies to null to avoid missing artifacts issue
   # run host_test jobs immediately, only after upload cache
   needs:
@@ -14,6 +13,8 @@
     - job: upload-submodules-cache
       optional: true
       artifacts: false
+  variables:
+    GIT_STRATEGY: fetch  # use brew local mirror first
   before_script: []
   after_script: []
 
@@ -61,7 +62,7 @@ test_tools_win:
     - python -m pip install jsonschema
     - .\install.ps1 --enable-ci --enable-pytest
     - .\export.ps1
-    - python "${SUBMODULE_FETCH_TOOL}" -s "all"
+    - git submodule update --init
     - cd ${IDF_PATH}/tools/test_idf_py
     - pytest --noconftest test_idf_py.py       --junitxml=${IDF_PATH}/XUNIT_IDF_PY.xml
     - pytest --noconftest test_hints.py        --junitxml=${IDF_PATH}/XUNIT_HINTS.xml
@@ -79,7 +80,7 @@ test_tools_win:
   script:
     - .\install.ps1 --enable-ci --enable-pytest
     - . .\export.ps1
-    - python "${SUBMODULE_FETCH_TOOL}" -s "all"
+    - git submodule update --init
     - cd ${IDF_PATH}\tools\test_build_system
     - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml
 
@@ -89,8 +90,7 @@ pytest_build_system_win:
     - .rules:labels:windows_pytest_build_system
   parallel: 2
   needs: []
-  tags:
-    - windows-build
+  tags: [windows-build, brew]
   artifacts:
     paths:
       - XUNIT_RESULT.xml

.gitlab/ci/upload_cache.yml

@@ -35,14 +35,14 @@ upload-submodules-cache:
     - $GEO
     - cache
   cache:
-    key: submodule-cache-${LATEST_GIT_TAG}
+    key: git-submodule-cache-${LATEST_GIT_TAG}
     paths:
-      - .cache/submodule_archives
+      - .git/modules
     policy: push
   script:
     # use the default gitlab server
     - unset LOCAL_GITLAB_HTTPS_HOST
-    - rm -rf .cache/submodule_archives  # clear old submodule archives
+    - rm -rf .git/modules  # clear old submodules
     - add_gitlab_ssh_keys
     - fetch_submodules
   parallel: