diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 83c72ccd80d..9b89d247bdf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,6 +18,9 @@ workflow: # Place the default settings in `.gitlab/ci/common.yml` instead include: + - project: "ci/actions/common" + file: + - "templates/idf/integration-test.yml" - '.gitlab/ci/common.yml' - '.gitlab/ci/rules.yml' - '.gitlab/ci/upload_cache.yml' @@ -25,7 +28,6 @@ include: - '.gitlab/ci/static-code-analysis.yml' - '.gitlab/ci/pre_check.yml' - '.gitlab/ci/build.yml' - - '.gitlab/ci/integration_test.yml' - '.gitlab/ci/host-test.yml' - '.gitlab/ci/target-test.yml' - '.gitlab/ci/deploy.yml' diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index 4e0ecf6366a..286b6f99a50 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -570,12 +570,15 @@ build_docker: variables: DOCKER_TMP_IMAGE_NAME: "idf_tmp_image" script: - - export DOCKER_BUILD_ARGS="--build-arg IDF_CLONE_URL=${CI_REPOSITORY_URL} --build-arg IDF_CLONE_BRANCH_OR_TAG=${CI_COMMIT_REF_NAME} --build-arg IDF_CHECKOUT_REF=${CI_COMMIT_TAG:-$CI_COMMIT_SHA} --build-arg IDF_CLONE_SHALLOW=1 --build-arg IDF_GITHUB_ASSETS=${INTERNAL_GITHUB_ASSETS}" - - docker build --tag ${DOCKER_TMP_IMAGE_NAME} ${DOCKER_BUILD_ARGS} tools/docker/ + # use different image tag for different concurrent job + - export DOCKER_TMP_IMAGE="${DOCKER_TMP_IMAGE_NAME}:${CI_CONCURRENT_ID:-latest}" + - export DOCKER_BUILD_ARGS="--build-arg IDF_CLONE_URL=${CI_REPOSITORY_URL} --build-arg IDF_CLONE_BRANCH_OR_TAG=${CI_COMMIT_REF_NAME} --build-arg IDF_CHECKOUT_REF=${CI_COMMIT_TAG:-$CI_COMMIT_SHA} --build-arg IDF_CLONE_SHALLOW=1 ${EXTRA_DOCKER_BUILD_ARGS:-} " + - time docker build --tag ${DOCKER_TMP_IMAGE} ${DOCKER_BUILD_ARGS} tools/docker/ # We can't mount $PWD/examples/get-started/blink into the container, see https://gitlab.com/gitlab-org/gitlab-ce/issues/41227. # The workaround mentioned there works, but leaves around directories which need to be cleaned up manually. # Therefore, build a copy of the example located inside the container. - - docker run --rm --workdir /opt/esp/idf/examples/get-started/blink ${DOCKER_TMP_IMAGE_NAME} idf.py build + - docker run --rm --workdir /opt/esp/idf/examples/get-started/blink ${DOCKER_TMP_IMAGE} idf.py build + # This job builds template app with permutations of targets and optimization levels build_template_app: diff --git a/.gitlab/ci/integration_test.yml b/.gitlab/ci/integration_test.yml deleted file mode 100644 index 9ba8d77dab1..00000000000 --- a/.gitlab/ci/integration_test.yml +++ /dev/null @@ -1,69 +0,0 @@ -# generate dynamic integration pipeline by `idf-integration-ci` project - -.patterns-integration_test: &patterns-integration_test - # add all possible patterns to make sure `gen_integration_pipeline` can be triggered. - # fine-grained control will be done while generating the pipeline - # find `patterns` in `idf-integration-ci` project - - "components/**/*" - - "tools/**/*" - - ".gitlab-ci.yml" - - ".gitlab/ci/common.yml" - - ".gitlab/ci/integration_test.yml" - - ".gitmodules" - - "CMakeLists.txt" - - "install.sh" - - "export.sh" - - "Kconfig" - - "sdkconfig.rename" - -# Simplify the rules -.integration_test_rules: - rules: - - if: '$CI_PIPELINE_SOURCE != "merge_request_event"' - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - changes: *patterns-integration_test - # support trigger by ci labels - - if: '$CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*target_test(?:,[^,\n\r]+)*$/i' - - if: '$CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*integration_test(?:,[^,\n\r]+)*$/i' - - if: '$CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build(?:,[^,\n\r]+)*$/i' - -gen_integration_pipeline: - extends: - - .before_script:minimal - - .integration_test_rules - image: ${CI_INTEGRATION_ASSIGN_ENV} - stage: assign_test - cache: [] - tags: - - assign_test - variables: - SUBMODULES_TO_FETCH: "none" - GIT_LFS_SKIP_SMUDGE: 1 - needs: - - job: fast_template_app - artifacts: false - optional: true - artifacts: - paths: - - idf-integration-ci/child_pipeline/ - expire_in: 2 weeks - script: - - add_gitlab_ssh_keys - - retry_failed git clone ${CI_GEN_INTEGRATION_PIPELINE_REPO} idf-integration-ci - - python $CHECKOUT_REF_SCRIPT idf-integration-ci idf-integration-ci - - cd idf-integration-ci - - python tools/generate_child_pipeline.py -o child_pipeline/ - -child_integration_test_pipeline: - extends: - - .integration_test_rules - stage: assign_test - needs: - - gen_integration_pipeline - trigger: - include: - - artifact: idf-integration-ci/child_pipeline/pipeline.yml - job: gen_integration_pipeline - forward: - yaml_variables: false - strategy: depend diff --git a/tools/ci/check_artifacts_expire_time.py b/tools/ci/check_artifacts_expire_time.py index ff298cac386..22d3816b695 100644 --- a/tools/ci/check_artifacts_expire_time.py +++ b/tools/ci/check_artifacts_expire_time.py @@ -1,10 +1,8 @@ #!/usr/bin/env python -# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 - # internal use only # check if expire time is set for all artifacts - import os import yaml @@ -23,8 +21,15 @@ def check_artifacts_expire_time() -> None: # load files listed in `include` if 'include' in config: - for _file in config['include']: - with open(os.path.join(IDF_PATH or '', _file)) as f: + for item in config['include']: + if isinstance(item, dict): + if 'project' in item: + continue + elif 'local' in item: + item = item['local'] + else: + continue + with open(os.path.join(IDF_PATH or '', item)) as f: config.update(yaml.load(f, Loader=yaml.FullLoader)) print('expire time for jobs:') diff --git a/tools/ci/check_rules_yml.py b/tools/ci/check_rules_yml.py index 5a551324569..b30174e935b 100755 --- a/tools/ci/check_rules_yml.py +++ b/tools/ci/check_rules_yml.py @@ -1,18 +1,21 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 - """ Check if all rules in rules.yml used or not in CI yaml files. """ - import argparse import os import re import sys from copy import deepcopy -from typing import Any, Dict, List, Optional, Set, Union +from typing import Any +from typing import Dict +from typing import List +from typing import Optional +from typing import Set +from typing import Union import yaml from idf_ci_utils import IDF_PATH @@ -49,6 +52,13 @@ class YMLConfig: all_config = dict() for item in self.root_yml['include']: + if isinstance(item, dict): + if 'project' in item: + continue + elif 'local' in item: + item = item['local'] + else: + continue all_config.update(load_yaml(os.path.join(IDF_PATH, item))) self._config = all_config return self._config diff --git a/tools/ci/idf_ci_utils.py b/tools/ci/idf_ci_utils.py index f58de750ba9..f31e1bc27be 100644 --- a/tools/ci/idf_ci_utils.py +++ b/tools/ci/idf_ci_utils.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2020-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2020-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # internal use only for CI # some CI related util functions diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile index a74aac45ef4..087d82272a8 100644 --- a/tools/docker/Dockerfile +++ b/tools/docker/Dockerfile @@ -39,6 +39,16 @@ RUN : \ && update-alternatives --install /usr/bin/python python /usr/bin/python3 10 \ && : +ARG IDF_INSTALL_TARGETS=all + +# install build essential needed for linux target apps, which is a preview target so it is installed with "all" only +RUN if [ "$IDF_INSTALL_TARGETS" = "all" ]; then \ + apt-get update \ + && apt-get install -y build-essential \ + && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* ; \ + fi + # To build the image for a branch or a tag of IDF, pass --build-arg IDF_CLONE_BRANCH_OR_TAG=name. # To build the image with a specific commit ID of IDF, pass --build-arg IDF_CHECKOUT_REF=commit-id. # It is possibe to combine both, e.g.: @@ -53,21 +63,13 @@ ARG IDF_CLONE_BRANCH_OR_TAG=master ARG IDF_CHECKOUT_REF= ARG IDF_CLONE_SHALLOW= ARG IDF_CLONE_SHALLOW_DEPTH=1 -ARG IDF_INSTALL_TARGETS=all ARG IDF_GITHUB_ASSETS= +ARG PIP_INDEX_URL= ENV IDF_PATH=/opt/esp/idf ENV IDF_TOOLS_PATH=/opt/esp ENV IDF_GITHUB_ASSETS=${IDF_GITHUB_ASSETS} -# install build essential needed for linux target apps, which is a preview target so it is installed with "all" only -RUN if [ "$IDF_INSTALL_TARGETS" = "all" ]; then \ - apt-get update \ - && apt-get install -y build-essential \ - && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* ; \ - fi - RUN echo IDF_CHECKOUT_REF=$IDF_CHECKOUT_REF IDF_CLONE_BRANCH_OR_TAG=$IDF_CLONE_BRANCH_OR_TAG && \ git clone --recursive \ ${IDF_CLONE_SHALLOW:+--depth=${IDF_CLONE_SHALLOW_DEPTH} --shallow-submodules} \