From 8a4dbaac6ab3e3339cf6f5c1cbd789388f3699a7 Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Wed, 15 Apr 2026 13:25:02 +0200 Subject: [PATCH 1/2] ci: log per-job ccache stats --- .gitlab/ci/common.yml | 18 +++++++++++++++++- tools/ci/configure_ci_environment.sh | 8 ++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index abb43b5c1f1..706c95dbd5a 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -119,6 +119,11 @@ variables: # configure cmake related flags source tools/ci/configure_ci_environment.sh + if [[ "$CI_CCACHE_STATS" == 1 ]] && command -v ccache >/dev/null 2>&1 && [[ -n "$CCACHE_STATSLOG" ]]; then + mkdir -p "$(dirname "$CCACHE_STATSLOG")" + rm -f "$CCACHE_STATSLOG" + fi + # add extra python packages export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH" @@ -234,7 +239,17 @@ variables: .show_ccache_statistics: &show_ccache_statistics | # Show ccache statistics if enabled globally section_start "ccache_show_stats" "Show ccache statistics" - test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats -vv || true + if [[ "$CI_CCACHE_STATS" == 1 ]] && command -v ccache >/dev/null 2>&1; then + if ccache --help 2>/dev/null | grep -q -- '--show-log-stats'; then + if [[ -n "$CCACHE_STATSLOG" && -f "$CCACHE_STATSLOG" ]]; then + ccache --show-log-stats -vv + else + echo "INFO: No per-job ccache statistics were recorded" + fi + else + ccache --show-stats -vv + fi + fi || true section_end "ccache_show_stats" .upload_failed_job_log_artifacts: &upload_failed_job_log_artifacts | @@ -249,6 +264,7 @@ variables: .after_script:build: after_script: - source tools/ci/utils.sh + - source tools/ci/configure_ci_environment.sh - *show_ccache_statistics - *upload_failed_job_log_artifacts diff --git a/tools/ci/configure_ci_environment.sh b/tools/ci/configure_ci_environment.sh index d4c4de40440..54f2e03159f 100644 --- a/tools/ci/configure_ci_environment.sh +++ b/tools/ci/configure_ci_environment.sh @@ -35,10 +35,14 @@ fi # https://ccache.dev/manual/latest.html#_configuring_ccache # Set ccache base directory to the project checkout path, to cancel out differences between runners -export CCACHE_BASEDIR="${CI_PROJECT_DIR}" +export CCACHE_BASEDIR="${IDF_PATH}" +export CCACHE_COMPILERCHECK="${CCACHE_COMPILERCHECK:-content}" # host mapping volume to share ccache fbetween runner concurrent jobs -export CCACHE_SLOPPINESS="time_macros" +export CCACHE_SLOPPINESS="time_macros,file_macro,include_file_mtime,include_file_ctime" + +# Keep per-job statistics in the checkout directory while sharing the cache itself. +export CCACHE_STATSLOG="${CCACHE_STATSLOG:-${IDF_PATH}/.ccache-stats.log}" # CCACHE_RECACHE Used when invalidating the current cache. # could be enabled by MR label "ccache:recache" From 6717c915d9a7094bfa186b361081cdd79ceba0ba Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Wed, 15 Apr 2026 13:24:16 +0200 Subject: [PATCH 2/2] feat(build): support env var IDF_PY_BUILD_JOBS for ninja jobs --- .gitlab/ci/common.yml | 4 ++++ docs/en/api-guides/build-system.rst | 8 ++++++++ docs/zh_CN/api-guides/build-system.rst | 8 ++++++++ tools/idf_py_actions/tools.py | 15 ++++++++++++++- 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index 706c95dbd5a..a47e9c51a4c 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -19,6 +19,10 @@ variables: # Common parameters for the 'make' during CI tests MAKEFLAGS: "-j5 --no-keep-going" + # By default, CI build jobs request and limit 4 CPU cores and 4 GB of memory + # https://github.com/ninja-build/ninja/blob/def9560a0b6d755936e615ce443a0aec45c39bdb/src/ninja.cc#L262 + # cpu_cores + 2 + IDF_PY_BUILD_JOBS: "6" # GitLab-CI environment # Thanks to pack-objects cache, clone strategy should behave faster than fetch diff --git a/docs/en/api-guides/build-system.rst b/docs/en/api-guides/build-system.rst index a425628c07e..ba26e8bed92 100644 --- a/docs/en/api-guides/build-system.rst +++ b/docs/en/api-guides/build-system.rst @@ -77,6 +77,14 @@ In the above list, the ``cmake`` command configures the project and generates bu It's not necessary to run ``cmake`` more than once. After the first build, you only need to run ``ninja`` each time. ``ninja`` will automatically re-invoke ``cmake`` if the project needs reconfiguration. +When using ``idf.py`` with the Ninja generator, you can cap the number of parallel build jobs by setting the ``IDF_PY_BUILD_JOBS`` environment variable. For example: + +.. code-block:: bash + + IDF_PY_BUILD_JOBS=6 idf.py build + +If you invoke CMake, ``ninja``, or ``make`` directly instead of ``idf.py``, use their native options or environment variables to control parallelism. + If using CMake with ``ninja`` or ``make``, there are also targets for more of the ``idf.py`` sub-commands. For example, running ``make menuconfig`` or ``ninja menuconfig`` in the build directory will work the same as ``idf.py menuconfig``. .. note:: diff --git a/docs/zh_CN/api-guides/build-system.rst b/docs/zh_CN/api-guides/build-system.rst index 7c6e7de7ae5..cc1c4c55563 100644 --- a/docs/zh_CN/api-guides/build-system.rst +++ b/docs/zh_CN/api-guides/build-system.rst @@ -77,6 +77,14 @@ idf.py 没有必要多次运行 ``cmake``。第一次构建后，往后每次只需运行 ``ninja`` 即可。如果项目需要重新配置，``ninja`` 会自动重新调用 ``cmake``。 +使用 Ninja 生成器配合 ``idf.py`` 时，可以通过设置环境变量 ``IDF_PY_BUILD_JOBS`` 来限制并行构建任务数。例如： + +.. code-block:: bash + + IDF_PY_BUILD_JOBS=6 idf.py build + +如果不是通过 ``idf.py``，而是直接调用 CMake、``ninja`` 或 ``make``，则请使用它们各自原生的并行控制选项或环境变量。 + 若在 CMake 中使用 ``ninja`` 或 ``make``，则多数 ``idf.py`` 子命令也会有其对应的目标，例如在构建目录下运行 ``make menuconfig`` 或 ``ninja menuconfig`` 与运行 ``idf.py menuconfig`` 是相同的。 .. note:: diff --git a/tools/idf_py_actions/tools.py b/tools/idf_py_actions/tools.py index 26b909ccfd1..42273489277 100644 --- a/tools/idf_py_actions/tools.py +++ b/tools/idf_py_actions/tools.py @@ -533,7 +533,20 @@ def run_target( if env is None: env = {} - generator_cmd = GENERATORS[args.generator]['command'] + generator_cmd = list(GENERATORS[args.generator]['command']) + + if args.generator == 'Ninja': + parallel_level = os.environ.get('IDF_PY_BUILD_JOBS') + if parallel_level: + try: + jobs = int(parallel_level) + except ValueError as e: + raise FatalError('Environment variable IDF_PY_BUILD_JOBS must be a positive integer') from e + + if jobs <= 0: + raise FatalError('Environment variable IDF_PY_BUILD_JOBS must be a positive integer') + + generator_cmd += ['-j', str(jobs)] if args.verbose: generator_cmd += [GENERATORS[args.generator]['verbose_flag']]