Merge branch 'ci/update_integration_pipeline_v5.2' into 'release/v5.2'

ci: backport integration test and build docker (v5.2)

See merge request espressif/esp-idf!48390

.gitlab-ci.yml

@@ -18,6 +18,9 @@ workflow:
 # Place the default settings in `.gitlab/ci/common.yml` instead
 
 include:
+  - project: "ci/actions/common"
+    file:
+      - "templates/idf/integration-test.yml"
   - '.gitlab/ci/common.yml'
   - '.gitlab/ci/rules.yml'
   - '.gitlab/ci/upload_cache.yml'
@@ -25,7 +28,6 @@ include:
   - '.gitlab/ci/static-code-analysis.yml'
   - '.gitlab/ci/pre_check.yml'
   - '.gitlab/ci/build.yml'
-  - '.gitlab/ci/integration_test.yml'
   - '.gitlab/ci/host-test.yml'
   - '.gitlab/ci/target-test.yml'
   - '.gitlab/ci/deploy.yml'

.gitlab/ci/build.yml

@@ -570,12 +570,15 @@ build_docker:
   variables:
     DOCKER_TMP_IMAGE_NAME: "idf_tmp_image"
   script:
-    - export DOCKER_BUILD_ARGS="--build-arg IDF_CLONE_URL=${CI_REPOSITORY_URL} --build-arg IDF_CLONE_BRANCH_OR_TAG=${CI_COMMIT_REF_NAME} --build-arg IDF_CHECKOUT_REF=${CI_COMMIT_TAG:-$CI_COMMIT_SHA} --build-arg IDF_CLONE_SHALLOW=1 --build-arg IDF_GITHUB_ASSETS=${INTERNAL_GITHUB_ASSETS}"
+    # use different image tag for different concurrent job
-    - docker build --tag ${DOCKER_TMP_IMAGE_NAME} ${DOCKER_BUILD_ARGS} tools/docker/
+    - export DOCKER_TMP_IMAGE="${DOCKER_TMP_IMAGE_NAME}:${CI_CONCURRENT_ID:-latest}"
+    - export DOCKER_BUILD_ARGS="--build-arg IDF_CLONE_URL=${CI_REPOSITORY_URL} --build-arg IDF_CLONE_BRANCH_OR_TAG=${CI_COMMIT_REF_NAME} --build-arg IDF_CHECKOUT_REF=${CI_COMMIT_TAG:-$CI_COMMIT_SHA} --build-arg IDF_CLONE_SHALLOW=1 ${EXTRA_DOCKER_BUILD_ARGS:-} "
+    - time docker build --tag ${DOCKER_TMP_IMAGE} ${DOCKER_BUILD_ARGS} tools/docker/
     # We can't mount $PWD/examples/get-started/blink into the container, see https://gitlab.com/gitlab-org/gitlab-ce/issues/41227.
     # The workaround mentioned there works, but leaves around directories which need to be cleaned up manually.
     # Therefore, build a copy of the example located inside the container.
-    - docker run --rm --workdir /opt/esp/idf/examples/get-started/blink ${DOCKER_TMP_IMAGE_NAME} idf.py build
+    - docker run --rm --workdir /opt/esp/idf/examples/get-started/blink ${DOCKER_TMP_IMAGE} idf.py build
+
 
 # This job builds template app with permutations of targets and optimization levels
 build_template_app:

.gitlab/ci/integration_test.yml

@@ -1,69 +0,0 @@
-# generate dynamic integration pipeline by `idf-integration-ci` project
-
-.patterns-integration_test: &patterns-integration_test
-  # add all possible patterns to make sure `gen_integration_pipeline` can be triggered.
-  # fine-grained control will be done while generating the pipeline
-  # find `patterns` in `idf-integration-ci` project
-  - "components/**/*"
-  - "tools/**/*"
-  - ".gitlab-ci.yml"
-  - ".gitlab/ci/common.yml"
-  - ".gitlab/ci/integration_test.yml"
-  - ".gitmodules"
-  - "CMakeLists.txt"
-  - "install.sh"
-  - "export.sh"
-  - "Kconfig"
-  - "sdkconfig.rename"
-
-# Simplify the rules
-.integration_test_rules:
-  rules:
-    - if: '$CI_PIPELINE_SOURCE != "merge_request_event"'
-    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
-      changes: *patterns-integration_test
-    # support trigger by ci labels
-    - if: '$CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*target_test(?:,[^,\n\r]+)*$/i'
-    - if: '$CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*integration_test(?:,[^,\n\r]+)*$/i'
-    - if: '$CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build(?:,[^,\n\r]+)*$/i'
-
-gen_integration_pipeline:
-  extends:
-    - .before_script:minimal
-    - .integration_test_rules
-  image: ${CI_INTEGRATION_ASSIGN_ENV}
-  stage: assign_test
-  cache: []
-  tags:
-    - assign_test
-  variables:
-    SUBMODULES_TO_FETCH: "none"
-    GIT_LFS_SKIP_SMUDGE: 1
-  needs:
-    - job: fast_template_app
-      artifacts: false
-      optional: true
-  artifacts:
-    paths:
-      - idf-integration-ci/child_pipeline/
-    expire_in: 2 weeks
-  script:
-    - add_gitlab_ssh_keys
-    - retry_failed git clone ${CI_GEN_INTEGRATION_PIPELINE_REPO} idf-integration-ci
-    - python $CHECKOUT_REF_SCRIPT idf-integration-ci idf-integration-ci
-    - cd idf-integration-ci
-    - python tools/generate_child_pipeline.py -o child_pipeline/
-
-child_integration_test_pipeline:
-  extends:
-    - .integration_test_rules
-  stage: assign_test
-  needs:
-    - gen_integration_pipeline
-  trigger:
-    include:
-      - artifact: idf-integration-ci/child_pipeline/pipeline.yml
-        job: gen_integration_pipeline
-    forward:
-      yaml_variables: false
-    strategy: depend

tools/ci/check_artifacts_expire_time.py

@@ -1,10 +1,8 @@
 #!/usr/bin/env python
-# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
 # internal use only
 # check if expire time is set for all artifacts
-
 import os
 
 import yaml
@@ -23,8 +21,15 @@ def check_artifacts_expire_time() -> None:
 
     # load files listed in `include`
     if 'include' in config:
-        for _file in config['include']:
+        for item in config['include']:
-            with open(os.path.join(IDF_PATH or '', _file)) as f:
+            if isinstance(item, dict):
+                if 'project' in item:
+                    continue
+                elif 'local' in item:
+                    item = item['local']
+                else:
+                    continue
+            with open(os.path.join(IDF_PATH or '', item)) as f:
                 config.update(yaml.load(f, Loader=yaml.FullLoader))
 
     print('expire time for jobs:')

tools/ci/check_rules_yml.py

@@ -1,18 +1,21 @@
 #!/usr/bin/env python
 #
-# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
-
 """
 Check if all rules in rules.yml used or not in CI yaml files.
 """
-
 import argparse
 import os
 import re
 import sys
 from copy import deepcopy
-from typing import Any, Dict, List, Optional, Set, Union
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Optional
+from typing import Set
+from typing import Union
 
 import yaml
 from idf_ci_utils import IDF_PATH
@@ -49,6 +52,13 @@ class YMLConfig:
 
         all_config = dict()
         for item in self.root_yml['include']:
+            if isinstance(item, dict):
+                if 'project' in item:
+                    continue
+                elif 'local' in item:
+                    item = item['local']
+                else:
+                    continue
             all_config.update(load_yaml(os.path.join(IDF_PATH, item)))
         self._config = all_config
         return self._config

tools/ci/idf_ci_utils.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: 2020-2024 Espressif Systems (Shanghai) CO LTD
+# SPDX-FileCopyrightText: 2020-2026 Espressif Systems (Shanghai) CO LTD
 # SPDX-License-Identifier: Apache-2.0
 # internal use only for CI
 # some CI related util functions

tools/docker/Dockerfile

@@ -39,6 +39,16 @@ RUN : \
   && update-alternatives --install /usr/bin/python python /usr/bin/python3 10 \
   && :
 
+ARG IDF_INSTALL_TARGETS=all
+
+# install build essential needed for linux target apps, which is a preview target so it is installed with "all" only
+RUN if [ "$IDF_INSTALL_TARGETS" = "all" ]; then \
+    apt-get update \
+    && apt-get install -y build-essential \
+    && apt-get autoremove -y \
+    && rm -rf /var/lib/apt/lists/* ; \
+  fi
+
 # To build the image for a branch or a tag of IDF, pass --build-arg IDF_CLONE_BRANCH_OR_TAG=name.
 # To build the image with a specific commit ID of IDF, pass --build-arg IDF_CHECKOUT_REF=commit-id.
 # It is possibe to combine both, e.g.:
@@ -53,21 +63,13 @@ ARG IDF_CLONE_BRANCH_OR_TAG=master
 ARG IDF_CHECKOUT_REF=
 ARG IDF_CLONE_SHALLOW=
 ARG IDF_CLONE_SHALLOW_DEPTH=1
-ARG IDF_INSTALL_TARGETS=all
 ARG IDF_GITHUB_ASSETS=
+ARG PIP_INDEX_URL=
 
 ENV IDF_PATH=/opt/esp/idf
 ENV IDF_TOOLS_PATH=/opt/esp
 ENV IDF_GITHUB_ASSETS=${IDF_GITHUB_ASSETS}
 
-# install build essential needed for linux target apps, which is a preview target so it is installed with "all" only
-RUN if [ "$IDF_INSTALL_TARGETS" = "all" ]; then \
-    apt-get update \
-    && apt-get install -y build-essential \
-    && apt-get autoremove -y \
-    && rm -rf /var/lib/apt/lists/* ; \
-  fi
-
 RUN echo IDF_CHECKOUT_REF=$IDF_CHECKOUT_REF IDF_CLONE_BRANCH_OR_TAG=$IDF_CLONE_BRANCH_OR_TAG && \
     git clone --recursive \
       ${IDF_CLONE_SHALLOW:+--depth=${IDF_CLONE_SHALLOW_DEPTH} --shallow-submodules} \