diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index ac25f22f06d..de71c540122 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -214,18 +214,6 @@ build_clang_test_apps_esp32p4: --junitxml ${CI_PROJECT_DIR}/XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} -.test_build_system_macos_template_extension: - tags: - - macos - variables: - PYENV_VERSION: "3.10" - # CCACHE_DIR: "/cache/idf_ccache". On macOS, you cannot write to this folder due to insufficient permissions. - CCACHE_DIR: "" # ccache will use "$HOME/Library/Caches/ccache". - # Workaround for a bug in Parallels executor where CI_PROJECT_DIR is not an absolute path, - # but a relative path to the build directory (builds/espressif/esp-idf instead of ~/builds/espressif/esp-idf. - # GitLab sets the project dir to this template `//` - IDF_PATH: "/Users/espressif/builds/espressif/esp-idf" - .test_build_system_minimal_cmake_template: extends: .test_build_system_template variables: @@ -292,9 +280,7 @@ pytest_buildv2_system: pytest_build_system_macos: extends: - .test_build_system_template - - .test_build_system_macos_template_extension - - .before_script:build:macos - - .after_script:build:macos:upload-failed-job-logs:ccache-show-stats + - .brew-macos-settings - .rules:build:macos parallel: 3 @@ -304,9 +290,7 @@ pytest_build_system_minimal_cmake: pytest_build_system_macos_minimal_cmake: extends: - .test_build_system_minimal_cmake_template - - .test_build_system_macos_template_extension - - .before_script:build:macos - - .after_script:build:macos:upload-failed-job-logs:ccache-show-stats + - .brew-macos-settings - .rules:build:macos variables: INSTALL_EXTRA_TOOLS: ninja cmake@3.22.1 diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index 3bca188dfee..37b34eb5d93 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -166,20 +166,31 @@ variables: section_end "install_python_env" fi + section_start "source_export" "Source export.sh" + source ./export.sh + section_end "source_export" + + REEXPORT_NEEDED=0 if [[ ! -z "$INSTALL_EXTRA_TOOLS" ]]; then section_start "installing_optional_tools" "Install optional tools ${INSTALL_EXTRA_TOOLS}" - $IDF_PATH/tools/idf_tools.py --non-interactive install $INSTALL_EXTRA_TOOLS + run_cmd $IDF_PATH/tools/idf_tools.py --non-interactive install $INSTALL_EXTRA_TOOLS section_end "installing_optional_tools" + + REEXPORT_NEEDED=1 fi # Install esp-clang if necessary (esp-clang is separately installed) if [[ "$IDF_TOOLCHAIN" == "clang" && -z "$CI_CLANG_DISTRO_URL" ]]; then $IDF_PATH/tools/idf_tools.py --non-interactive install esp-clang + + REEXPORT_NEEDED=1 fi - section_start "source_export" "Source export.sh" - source ./export.sh - section_end "source_export" + if [[ $REEXPORT_NEEDED -eq 1 ]]; then + section_start "re_source_export" "Re-source export.sh" + source ./export.sh + section_end "re_source_export" + fi # Custom clang toolchain if [[ "$IDF_TOOLCHAIN" == "clang" && ! -z "$CI_CLANG_DISTRO_URL" ]]; then @@ -229,30 +240,6 @@ variables: before_script: - *common-before_scripts -.before_script:build:macos: - before_script: - # macos is running shell executor, which means it would use - # the system installed /usr/local/bin/python3 by default. - # Ensure pyenv and PYENV_VERSION installed - - eval "$(pyenv init -)" - - *common-before_scripts - # remove idf-env.json, since it may contains enabled "features" - - rm -f $IDF_TOOLS_PATH/idf-env.json - # This adds tools (compilers) and the version-specific Python environment to PATH - - *setup_tools_and_idf_python_venv - - fetch_submodules - variables: - INSTALL_EXTRA_TOOLS: cmake ninja - -.after_script:build:macos:upload-failed-job-logs:ccache-show-stats: - after_script: - # macos is running shell executor, which means it would use - # the system installed /usr/local/bin/python3 by default. - # Ensure pyenv and PYENV_VERSION installed - - eval "$(pyenv init -)" - - *upload_failed_job_log_artifacts - - *show_ccache_statistics - .before_script:build: before_script: - *common-before_scripts @@ -313,7 +300,7 @@ variables: .git_checkout_ci_commit_sha: &git_checkout_ci_commit_sha | git checkout $CI_COMMIT_SHA - git clean ${GIT_CLEAN_FLAGS} + eval "git clean ${GIT_CLEAN_FLAGS}" # git diff requires two commits, with different CI env var # @@ -372,13 +359,47 @@ variables: before_script: - *git_init - *git_fetch_from_mirror_url_if_exists - - git fetch origin "${CI_COMMIT_SHA}" --depth=1 ${GIT_FETCH_EXTRA_FLAGS} + - eval "git fetch --depth=1 ${GIT_FETCH_EXTRA_FLAGS} origin ${CI_COMMIT_SHA}" - *git_checkout_ci_commit_sha - *common-before_scripts - *setup_tools_and_idf_python_venv - add_gitlab_ssh_keys # no submodules +.brew-macos-settings: + variables: + GIT_STRATEGY: none # we do manual git clone to use local mirror + IDF_CCACHE_ENABLE: "0" + CCACHE_DIR: "/var/tmp/cache/idf_ccache" + tags: + - macos-tart + image: macos-sequoia-idf-v6.0 + cache: [] # pip cache is created under amd64, and submodules are downloaded with brew mirror, so disable cache here + before_script: + # assert LOCAL_GIT_MIRROR is set + - echo -e "section_start:`date +%s`:check_out\r\e[0Kchecking out from local git mirror, then reset to CI_COMMIT_SHA" + - | + if [ -z "${LOCAL_GIT_MIRROR:-}" ]; then + echo "Error: LOCAL_GIT_MIRROR not set, cannot clone from mirror." + exit 1 + fi + - MIRROR_REPO_URL="${LOCAL_GIT_MIRROR}/${CI_PROJECT_PATH}" + - cd "${CI_PROJECT_DIR}" + # since .cache exists in CI_PROJECT_DIR, so can't direct `git clone .` + - git clone -b ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-${CI_COMMIT_BRANCH}} --depth=1 --recursive --shallow-submodules "${MIRROR_REPO_URL}" tmp + - mv tmp/.git ./ + - rm -rf tmp + - git reset --hard + # set remote url back + - git remote set-url origin "${CI_REPOSITORY_URL}" + - eval "git fetch --depth=1 ${GIT_FETCH_EXTRA_FLAGS} origin ${CI_COMMIT_SHA}" + - git checkout FETCH_HEAD + - git submodule update --init --recursive --depth=1 + - echo -e "section_end:`date +%s`:check_out\r\e[0K" + - *common-before_scripts + - *setup_tools_and_idf_python_venv + after_script: [] # ccache now is disabled for macos brew runners + ############# # `default` # ############# diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index ad60f4917e2..762e2d285ae 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -321,9 +321,7 @@ test_pytest_linux: test_pytest_macos: extends: - .host_test_template - - .before_script:build:macos - tags: - - macos + - .brew-macos-settings artifacts: paths: - XUNIT_RESULT.xml @@ -331,12 +329,6 @@ test_pytest_macos: - "**/build*/build_log.txt" reports: junit: XUNIT_RESULT.xml - variables: - PYENV_VERSION: "3.10" - # Workaround for a bug in Parallels executor where CI_PROJECT_DIR is not an absolute path, - # but a relative path to the build directory (builds/espressif/esp-idf instead of ~/builds/espressif/esp-idf. - # GitLab sets the project dir to this template `//` - IDF_PATH: "/Users/espressif/builds/espressif/esp-idf" script: - run_cmd idf-ci build run -p components -p examples -p tools/test_apps