From 784f06a12b9a279a4914c80d81a6132c7042ab38 Mon Sep 17 00:00:00 2001 From: Chen Yudong Date: Tue, 21 Apr 2026 19:52:16 +0800 Subject: [PATCH 1/2] ci: move integration-test to ci/actions --- .gitlab-ci.yml | 2 +- .gitlab/ci/integration_test.yml | 74 --------------------------------- 2 files changed, 1 insertion(+), 75 deletions(-) delete mode 100644 .gitlab/ci/integration_test.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2ce3cc87009..3d44cac7ab4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,6 +6,7 @@ include: - "templates/stable/default-workflow.yml" - "templates/idf/deploy-github.yml" - "templates/idf/deploy-docs.yml" + - "templates/idf/integration-test.yml" - project: "ci/actions/common" file: "templates/idf/build-docs.yml" inputs: @@ -27,7 +28,6 @@ include: - ".gitlab/ci/pre_commit.yml" - ".gitlab/ci/pre_check.yml" - ".gitlab/ci/build.yml" - - ".gitlab/ci/integration_test.yml" - ".gitlab/ci/host-test.yml" - ".gitlab/ci/deploy.yml" - ".gitlab/ci/post_deploy.yml" diff --git a/.gitlab/ci/integration_test.yml b/.gitlab/ci/integration_test.yml deleted file mode 100644 index f08889ea3d3..00000000000 --- a/.gitlab/ci/integration_test.yml +++ /dev/null @@ -1,74 +0,0 @@ -# generate dynamic integration pipeline by `idf-integration-ci` project - -.patterns-integration_test: &patterns-integration_test - # add all possible patterns to make sure `gen_integration_pipeline` can be triggered. - # fine-grained control will be done while generating the pipeline - # find `patterns` in `idf-integration-ci` project - - "components/**/*" - - "tools/**/*" - - ".gitlab-ci.yml" - - ".gitlab/ci/common.yml" - - ".gitlab/ci/integration_test.yml" - - ".gitmodules" - - "CMakeLists.txt" - - "install.sh" - - "export.sh" - - "Kconfig" - - "sdkconfig.rename" - -# Simplify the rules -.integration_test_rules: - rules: - - if: '$CI_PIPELINE_SOURCE != "merge_request_event"' - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - changes: *patterns-integration_test - # support trigger by ci labels - - if: '$CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*target_test(?:,[^,\n\r]+)*$/i' - - if: '$CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*integration_test(?:,[^,\n\r]+)*$/i' - - if: '$CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*build(?:,[^,\n\r]+)*$/i' - -gen_integration_pipeline: - extends: - - .before_script:minimal - - .integration_test_rules - image: ${CI_INTEGRATION_ASSIGN_ENV} - stage: assign_test - cache: [] - tags: [fast_run, shiny] - variables: - SUBMODULES_TO_FETCH: "none" - GIT_LFS_SKIP_SMUDGE: 1 - needs: - - job: manual_gate - optional: true - - job: fast_template_app - artifacts: false - optional: true - artifacts: - paths: - - idf-integration-ci/child_pipeline/ - expire_in: 2 weeks - when: always - script: - - add_gitlab_ssh_keys - - retry_failed git clone ${CI_GEN_INTEGRATION_PIPELINE_REPO} idf-integration-ci - - python $CHECKOUT_REF_SCRIPT idf-integration-ci idf-integration-ci - - cd idf-integration-ci - - python tools/generate_child_pipeline.py -o child_pipeline/ - -child_integration_test_pipeline: - extends: - - .integration_test_rules - stage: assign_test - needs: - - gen_integration_pipeline - variables: - IDF_S3_SERVER: $IDF_S3_NEW_SERVER - IDF_S3_ACCESS_KEY: $IDF_S3_NEW_ACCESS_KEY - trigger: - include: - - artifact: idf-integration-ci/child_pipeline/pipeline.yml - job: gen_integration_pipeline - forward: - yaml_variables: false - strategy: depend From 8a845d3ba85f84e650085dfc44345924ac817783 Mon Sep 17 00:00:00 2001 From: Chen Yudong Date: Wed, 22 Apr 2026 09:45:10 +0800 Subject: [PATCH 2/2] ci: optimize layer cache for docker build test use different image tag for different concurrent job --- .gitlab/ci/host-test.yml | 8 +++++--- tools/docker/Dockerfile | 20 +++++++++++--------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index 0dcb6d10fb0..775b1d51b77 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -309,12 +309,14 @@ build_docker: variables: DOCKER_TMP_IMAGE_NAME: "idf_tmp_image" script: - - export DOCKER_BUILD_ARGS="--build-arg IDF_CLONE_URL=${CI_REPOSITORY_URL} --build-arg IDF_CLONE_BRANCH_OR_TAG=${CI_COMMIT_REF_NAME} --build-arg IDF_CHECKOUT_REF=${CI_COMMIT_TAG:-$CI_COMMIT_SHA} --build-arg IDF_CLONE_SHALLOW=1 --build-arg IDF_GITHUB_ASSETS=${INTERNAL_GITHUB_ASSETS}" - - docker build --tag ${DOCKER_TMP_IMAGE_NAME} ${DOCKER_BUILD_ARGS} tools/docker/ + # use different image tag for different concurrent job + - export DOCKER_TMP_IMAGE="${DOCKER_TMP_IMAGE_NAME}:${CI_CONCURRENT_ID:-latest}" + - export DOCKER_BUILD_ARGS="--build-arg IDF_CLONE_URL=${CI_REPOSITORY_URL} --build-arg IDF_CLONE_BRANCH_OR_TAG=${CI_COMMIT_REF_NAME} --build-arg IDF_CHECKOUT_REF=${CI_COMMIT_TAG:-$CI_COMMIT_SHA} --build-arg IDF_CLONE_SHALLOW=1 ${EXTRA_DOCKER_BUILD_ARGS:-} " + - time docker build --tag ${DOCKER_TMP_IMAGE} ${DOCKER_BUILD_ARGS} tools/docker/ # We can't mount $PWD/examples/get-started/blink into the container, see https://gitlab.com/gitlab-org/gitlab-ce/issues/41227. # The workaround mentioned there works, but leaves around directories which need to be cleaned up manually. # Therefore, build a copy of the example located inside the container. - - docker run --rm --workdir /opt/esp/idf/examples/get-started/blink ${DOCKER_TMP_IMAGE_NAME} idf.py build + - docker run --rm --workdir /opt/esp/idf/examples/get-started/blink ${DOCKER_TMP_IMAGE} idf.py build ############################ # Host test with test apps # diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile index 77288bd758f..a727659bd53 100644 --- a/tools/docker/Dockerfile +++ b/tools/docker/Dockerfile @@ -39,6 +39,16 @@ RUN : \ && update-alternatives --install /usr/bin/python python /usr/bin/python3 10 \ && : +ARG IDF_INSTALL_TARGETS=all + +# install build essential needed for linux target apps, which is a preview target so it is installed with "all" only +RUN if [ "$IDF_INSTALL_TARGETS" = "all" ]; then \ + apt-get update \ + && apt-get install -y build-essential \ + && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* ; \ + fi + # To build the image for a branch or a tag of IDF, pass --build-arg IDF_CLONE_BRANCH_OR_TAG=name. # To build the image with a specific commit ID of IDF, pass --build-arg IDF_CHECKOUT_REF=commit-id. # It is possible to combine both, e.g.: @@ -53,21 +63,13 @@ ARG IDF_CLONE_BRANCH_OR_TAG=master ARG IDF_CHECKOUT_REF= ARG IDF_CLONE_SHALLOW= ARG IDF_CLONE_SHALLOW_DEPTH=1 -ARG IDF_INSTALL_TARGETS=all ARG IDF_GITHUB_ASSETS= +ARG PIP_INDEX_URL= ENV IDF_PATH=/opt/esp/idf ENV IDF_TOOLS_PATH=/opt/esp ENV IDF_GITHUB_ASSETS=${IDF_GITHUB_ASSETS} -# install build essential needed for linux target apps, which is a preview target so it is installed with "all" only -RUN if [ "$IDF_INSTALL_TARGETS" = "all" ]; then \ - apt-get update \ - && apt-get install -y build-essential \ - && apt-get autoremove -y \ - && rm -rf /var/lib/apt/lists/* ; \ - fi - RUN echo IDF_CHECKOUT_REF=$IDF_CHECKOUT_REF IDF_CLONE_BRANCH_OR_TAG=$IDF_CLONE_BRANCH_OR_TAG && \ git clone --recursive \ ${IDF_CLONE_SHALLOW:+--depth=${IDF_CLONE_SHALLOW_DEPTH} --shallow-submodules} \