From 16275df0a180fff8d1de24c956c977fa1737feca Mon Sep 17 00:00:00 2001 From: Ivan Grokhotkov Date: Mon, 16 Mar 2026 12:23:39 +0100 Subject: [PATCH] feat: add idf.py build-file command for standalone C files Add a new idf.py extension that allows building standalone C files without requiring full project boilerplate. Source files can include optional YAML frontmatter in block comments to specify sdkconfig options, component dependencies, and target configuration. When the frontmatter configuration changes, the stale sdkconfig is removed so the new defaults are applied, and a target change also clears the build directory since IDF_TARGET is pinned in the CMake cache. Co-Authored-By: Claude Fable 5 --- .gitlab/ci/host-test.yml | 1 + docs/en/api-guides/tools/idf-build-file.rst | 149 +++++++ docs/en/api-guides/tools/index.rst | 1 + .../zh_CN/api-guides/tools/idf-build-file.rst | 149 +++++++ docs/zh_CN/api-guides/tools/index.rst | 1 + tools/idf_py_actions/build_file_ext.py | 359 ++++++++++++++++ tools/test_idf_py/test_build_file.py | 385 ++++++++++++++++++ 7 files changed, 1045 insertions(+) create mode 100644 docs/en/api-guides/tools/idf-build-file.rst create mode 100644 docs/zh_CN/api-guides/tools/idf-build-file.rst create mode 100644 tools/idf_py_actions/build_file_ext.py create mode 100644 tools/test_idf_py/test_build_file.py diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index 0f885ff464d..51e6edb466b 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -218,6 +218,7 @@ test_tools: - run_cmd pytest --confcutdir=. test_hints.py --junitxml=${IDF_PATH}/XUNIT_HINTS.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 - run_cmd pytest --confcutdir=. test_idf_qemu.py --junitxml=${IDF_PATH}/XUNIT_IDF_PY_QEMU.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 - run_cmd pytest --confcutdir=. test_mcp_ext.py --junitxml=${IDF_PATH}/XUNIT_MCP_EXT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 + - run_cmd pytest --confcutdir=. test_build_file.py --junitxml=${IDF_PATH}/XUNIT_BUILD_FILE.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 - cd ${IDF_PATH}/tools/test_bsasm - run_cmd pytest --confcutdir=. test_bsasm.py --junitxml=${IDF_PATH}/XUNIT_BSASM.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 - cd ${IDF_PATH}/tools/test_mkdfu diff --git a/docs/en/api-guides/tools/idf-build-file.rst b/docs/en/api-guides/tools/idf-build-file.rst new file mode 100644 index 00000000000..86860d27e4e --- /dev/null +++ b/docs/en/api-guides/tools/idf-build-file.rst @@ -0,0 +1,149 @@ +Build Standalone Files - ``build-file`` +*************************************** + +:link_to_translation:`zh_CN:[中文]` + +.. warning:: + + This feature is experimental and may change in future releases. + +The ``idf.py build-file`` command allows building standalone C source files without creating a full ESP-IDF project structure. It is useful for quick experiments, small examples, and tests. + +Behind the scenes, the command auto-generates a container project in a cache directory, extracts configuration and dependencies from an optional YAML frontmatter in the source file, and builds the project. + +Usage +===== + +.. code-block:: bash + + idf.py build-file + +The command is composable with other ``idf.py`` commands. For example: + +.. code-block:: bash + + idf.py build-file example.c flash monitor + idf.py -p /dev/ttyUSB0 build-file example.c flash monitor + +Frontmatter +=========== + +Source files may contain an optional YAML frontmatter inside a block comment. The frontmatter starts with ``idf-build-file:`` as a marker: + +.. code-block:: c + + /* + idf-build-file: + config: + - CONFIG_IDF_TARGET=esp32s3 + - CONFIG_SPIRAM=y + dependencies: + - log + - vfs + - "espressif/button>=4.0" + - protocol_examples_common: + path: ${IDF_PATH}/examples/common_components/protocol_examples_common + */ + + #include + #include "esp_log.h" + + void app_main(void) + { + ESP_LOGI("app", "Hello!"); + } + +Frontmatter Fields +------------------ + +``config`` +^^^^^^^^^^ + +A list of ``sdkconfig`` options. These are written to ``sdkconfig.defaults`` in the generated container project. String values (like ``CONFIG_IDF_TARGET``) are automatically quoted. + +.. code-block:: yaml + + config: + - CONFIG_IDF_TARGET=esp32s3 + - CONFIG_SPIRAM=y + +``dependencies`` +^^^^^^^^^^^^^^^^ + +A list of component dependencies. Built-in components (plain names without ``/``) are added as ``PRIV_REQUIRES`` in the generated CMakeLists. All other dependencies use the same syntax as :doc:`idf_component.yml ` and are added to the generated ``idf_component.yml`` file. This includes managed components, path-based components, and any other forms supported by the IDF Component Manager. + +.. code-block:: yaml + + dependencies: + - log + - vfs + - "espressif/button>=4.0" + - protocol_examples_common: + path: ${IDF_PATH}/examples/common_components/protocol_examples_common + +Caching +======= + +Container projects are stored in the system temporary directory and reused for fast incremental builds. The OS may clean these up automatically on reboot. To discard the cached container project and force a full rebuild, run ``idf.py build-file --clean-cache file.c``. + +Examples +======== + +A minimal example with no frontmatter: + +.. code-block:: c + + #include + + void app_main(void) + { + printf("Hello world!\n"); + } + +Build and flash: + +.. code-block:: bash + + idf.py build-file hello.c flash monitor + +A more complete example that connects to Wi-Fi using ``protocol_examples_common``: + +.. code-block:: c + + /* + idf-build-file: + config: + - CONFIG_IDF_TARGET=esp32s3 + - CONFIG_EXAMPLE_CONNECT_WIFI=y + - CONFIG_EXAMPLE_WIFI_SSID=myssid + - CONFIG_EXAMPLE_WIFI_PASSWORD=mypassword + dependencies: + - log + - nvs_flash + - esp_netif + - esp_event + - protocol_examples_common: + path: ${IDF_PATH}/examples/common_components/protocol_examples_common + */ + + #include "esp_log.h" + #include "esp_netif.h" + #include "protocol_examples_common.h" + #include "esp_event.h" + #include "nvs_flash.h" + + static const char *TAG = "wifi_test"; + + void app_main(void) + { + ESP_ERROR_CHECK(nvs_flash_init()); + ESP_ERROR_CHECK(esp_netif_init()); + ESP_ERROR_CHECK(esp_event_loop_create_default()); + ESP_ERROR_CHECK(example_connect()); + + ESP_LOGI(TAG, "Connected to Wi-Fi!"); + } + +.. code-block:: bash + + idf.py build-file wifi_test.c flash monitor diff --git a/docs/en/api-guides/tools/index.rst b/docs/en/api-guides/tools/index.rst index 0523a10554b..690caf9f0d6 100644 --- a/docs/en/api-guides/tools/index.rst +++ b/docs/en/api-guides/tools/index.rst @@ -13,4 +13,5 @@ Tools idf-size idf-sbom idf-diag + idf-build-file :TARGET_SUPPORT_QEMU: qemu diff --git a/docs/zh_CN/api-guides/tools/idf-build-file.rst b/docs/zh_CN/api-guides/tools/idf-build-file.rst new file mode 100644 index 00000000000..cf2ca9ab2e7 --- /dev/null +++ b/docs/zh_CN/api-guides/tools/idf-build-file.rst @@ -0,0 +1,149 @@ +构建独立文件 - ``build-file`` +***************************** + +:link_to_translation:`en:[English]` + +.. warning:: + + 此功能为实验性功能，可能在未来版本中发生变化。 + +``idf.py build-file`` 命令允许构建独立的 C 源文件，无需创建完整的 ESP-IDF 项目结构。适用于快速实验、小型示例和测试。 + +该命令会在缓存目录中自动生成容器项目，从源文件中可选的 YAML 前置配置中提取配置和依赖项，然后构建项目。 + +使用方法 +======== + +.. code-block:: bash + + idf.py build-file + +该命令可以与其他 ``idf.py`` 命令组合使用，例如： + +.. code-block:: bash + + idf.py build-file example.c flash monitor + idf.py -p /dev/ttyUSB0 build-file example.c flash monitor + +前置配置 +======== + +源文件可以在块注释中包含可选的 YAML 前置配置。前置配置以 ``idf-build-file:`` 作为标记： + +.. code-block:: c + + /* + idf-build-file: + config: + - CONFIG_IDF_TARGET=esp32s3 + - CONFIG_SPIRAM=y + dependencies: + - log + - vfs + - "espressif/button>=4.0" + - protocol_examples_common: + path: ${IDF_PATH}/examples/common_components/protocol_examples_common + */ + + #include + #include "esp_log.h" + + void app_main(void) + { + ESP_LOGI("app", "Hello!"); + } + +前置配置字段 +------------ + +``config`` +^^^^^^^^^^ + +``sdkconfig`` 选项列表。这些选项会写入生成的容器项目中的 ``sdkconfig.defaults``。字符串值（如 ``CONFIG_IDF_TARGET``）会自动添加引号。 + +.. code-block:: yaml + + config: + - CONFIG_IDF_TARGET=esp32s3 + - CONFIG_SPIRAM=y + +``dependencies`` +^^^^^^^^^^^^^^^^ + +组件依赖列表。内置组件（不含 ``/`` 的纯名称）会作为 ``PRIV_REQUIRES`` 添加到生成的 CMakeLists 中。其他所有依赖项使用与 :doc:`idf_component.yml ` 相同的语法，添加到生成的 ``idf_component.yml`` 文件中。包括托管组件、基于路径的组件以及 IDF 组件管理器支持的其他形式。 + +.. code-block:: yaml + + dependencies: + - log + - vfs + - "espressif/button>=4.0" + - protocol_examples_common: + path: ${IDF_PATH}/examples/common_components/protocol_examples_common + +缓存 +==== + +容器项目存储在系统临时目录中，重复构建同一文件时会复用缓存以实现快速增量构建。操作系统可能会在重启时自动清理这些文件。如需丢弃缓存的容器项目并强制完整重新构建，可运行 ``idf.py build-file --clean-cache file.c``。 + +示例 +==== + +不含前置配置的最小示例： + +.. code-block:: c + + #include + + void app_main(void) + { + printf("Hello world!\n"); + } + +构建并烧录： + +.. code-block:: bash + + idf.py build-file hello.c flash monitor + +使用 ``protocol_examples_common`` 连接 Wi-Fi 的完整示例： + +.. code-block:: c + + /* + idf-build-file: + config: + - CONFIG_IDF_TARGET=esp32s3 + - CONFIG_EXAMPLE_CONNECT_WIFI=y + - CONFIG_EXAMPLE_WIFI_SSID=myssid + - CONFIG_EXAMPLE_WIFI_PASSWORD=mypassword + dependencies: + - log + - nvs_flash + - esp_netif + - esp_event + - protocol_examples_common: + path: ${IDF_PATH}/examples/common_components/protocol_examples_common + */ + + #include "esp_log.h" + #include "esp_netif.h" + #include "protocol_examples_common.h" + #include "esp_event.h" + #include "nvs_flash.h" + + static const char *TAG = "wifi_test"; + + void app_main(void) + { + ESP_ERROR_CHECK(nvs_flash_init()); + ESP_ERROR_CHECK(esp_netif_init()); + ESP_ERROR_CHECK(esp_event_loop_create_default()); + ESP_ERROR_CHECK(example_connect()); + + ESP_LOGI(TAG, "Connected to Wi-Fi!"); + } + +.. code-block:: bash + + idf.py build-file wifi_test.c flash monitor diff --git a/docs/zh_CN/api-guides/tools/index.rst b/docs/zh_CN/api-guides/tools/index.rst index cbf21f471c9..322f2b3a905 100644 --- a/docs/zh_CN/api-guides/tools/index.rst +++ b/docs/zh_CN/api-guides/tools/index.rst @@ -13,4 +13,5 @@ idf-size idf-sbom idf-diag + idf-build-file :TARGET_SUPPORT_QEMU: qemu diff --git a/tools/idf_py_actions/build_file_ext.py b/tools/idf_py_actions/build_file_ext.py new file mode 100644 index 00000000000..74d61c93a74 --- /dev/null +++ b/tools/idf_py_actions/build_file_ext.py @@ -0,0 +1,359 @@ +# SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import hashlib +import os +import re +import shutil +import tempfile +from typing import Any + +import rich_click as click +import yaml +from esp_pylib.logger import log +from rich.markup import escape +from rich_click import Context + +from idf_py_actions.errors import FatalError +from idf_py_actions.tools import PropertyDict + +# Container projects are stored in the system temp directory. +# The OS cleans these up automatically, which is desirable since IDF build +# directories can get quite large. +_BUILD_FILE_CACHE_DIR = os.path.join(tempfile.gettempdir(), 'esp-idf-build-file') + +# Marker for idf-build-file frontmatter +_FRONTMATTER_MARKER = 'idf-build-file:' + + +def _parse_frontmatter(source_file: str) -> dict: + """Parse YAML frontmatter from a C source file. + + Looks for a block comment containing 'idf-build-file:' as a marker, + then parses the YAML content after that marker. + + Returns a dict with keys like 'config', 'dependencies', etc. + Returns empty dict if no frontmatter found. + """ + with open(source_file, encoding='utf-8') as f: + content = f.read() + + # Find block comments that contain the frontmatter marker + block_comment_pattern = re.compile(r'/\*(.+?)\*/', re.DOTALL) + + for match in block_comment_pattern.finditer(content): + comment_body = match.group(1) + if _FRONTMATTER_MARKER not in comment_body: + continue + + # Extract everything starting from the marker line + lines = comment_body.split('\n') + yaml_lines = [] + found_marker = False + for line in lines: + stripped = line.strip() + if not found_marker: + if stripped.startswith('idf-build-file:'): + yaml_lines.append(line) + found_marker = True + else: + yaml_lines.append(line) + + if not yaml_lines: + continue + + yaml_text = '\n'.join(yaml_lines) + try: + parsed = yaml.safe_load(yaml_text) + if isinstance(parsed, dict) and 'idf-build-file' in parsed: + return parsed['idf-build-file'] or {} + except yaml.YAMLError as e: + raise FatalError(f'Failed to parse frontmatter YAML in {source_file}: {e}') + + return {} + + +def _compute_project_hash(source_file: str) -> str: + """Compute a stable hash for the container project directory name.""" + abs_path = os.path.abspath(source_file) + return hashlib.sha256(abs_path.encode()).hexdigest()[:16] + + +def _write_if_changed(filepath: str, content: str) -> bool: + """Write content to file only if it differs from current content. + + Returns True if file was written, False if unchanged. + """ + if os.path.exists(filepath): + with open(filepath, encoding='utf-8') as f: + existing = f.read() + if existing == content: + return False + + os.makedirs(os.path.dirname(filepath), exist_ok=True) + with open(filepath, 'w', encoding='utf-8') as f: + f.write(content) + return True + + +def _is_bool_or_number(value: str) -> bool: + """Check if a config value is boolean or numeric (doesn't need quoting).""" + if value in ('y', 'n', ''): + return True + try: + int(value, 0) # supports decimal, hex (0x...), octal (0o...) + return True + except ValueError: + return False + + +def _classify_dependencies(dependencies: list) -> tuple[list[str], dict[str, Any]]: + """Split dependencies into built-in components and managed components. + + Built-in components (no '/' in name, no path) -> PRIV_REQUIRES in idf_component_register + Managed components (with '/' like 'espressif/zlib') -> idf_component.yml + Path-based components (dict with 'path' key) -> idf_component.yml + + Returns (builtin_requires, managed_deps_dict). + """ + builtin: list[str] = [] + managed: dict[str, Any] = {} + + for dep in dependencies: + if isinstance(dep, str): + # Strip version specifier to check the name + name = dep.split('==')[0].split('>=')[0].strip() + if '/' in name: + # Managed component + if '==' in dep: + n, version = dep.split('==', 1) + managed[n.strip()] = {'version': version.strip()} + elif '>=' in dep: + n, version = dep.split('>=', 1) + managed[n.strip()] = {'version': f'>={version.strip()}'} + else: + managed[name] = '*' + else: + # Built-in component + builtin.append(name) + elif isinstance(dep, dict): + # Dict entries: path-based or other complex deps go to idf_component.yml + managed.update(dep) + + return builtin, managed + + +def _get_config_target(config_items: list[str]) -> str | None: + """Extract the CONFIG_IDF_TARGET value from a list of sdkconfig lines, if present.""" + for item in config_items: + if item.startswith('CONFIG_IDF_TARGET='): + return item.split('=', 1)[1].strip().strip('"') + return None + + +def _get_cached_target(build_dir: str) -> str | None: + """Extract the IDF_TARGET value pinned in the CMake cache, if present.""" + cmake_cache = os.path.join(build_dir, 'CMakeCache.txt') + if not os.path.exists(cmake_cache): + return None + with open(cmake_cache, encoding='utf-8') as f: + for line in f: + match = re.match(r'IDF_TARGET:\w+=(\w+)', line) + if match: + return match.group(1) + return None + + +def _invalidate_stale_config(container_dir: str, processed_config: list[str]) -> None: + """Invalidate stale build state after the generated sdkconfig.defaults changed. + + Values in sdkconfig.defaults are only applied when sdkconfig is generated, + so the existing sdkconfig must be removed for the new defaults to take + effect. A target change additionally requires removing the build directory, + since IDF_TARGET is pinned in the CMake cache and the toolchain file. + """ + new_target = _get_config_target(processed_config) or os.environ.get('IDF_TARGET') or 'esp32' + + build_dir = os.path.join(container_dir, 'build') + cached_target = _get_cached_target(build_dir) + if cached_target and cached_target != new_target: + shutil.rmtree(build_dir) + log.note(f'build-file: Target changed from {cached_target} to {new_target}, cleaned the build directory') + + sdkconfig_path = os.path.join(container_dir, 'sdkconfig') + if os.path.exists(sdkconfig_path): + os.remove(sdkconfig_path) + log.note('build-file: Configuration changed, sdkconfig will be regenerated') + + +def _generate_container_project(source_file: str, frontmatter: dict, container_dir: str) -> None: + """Generate or update the container project for a source file.""" + abs_source = os.path.abspath(source_file) + source_basename = os.path.basename(source_file) + source_name = os.path.splitext(source_basename)[0] + + main_dir = os.path.join(container_dir, 'main') + os.makedirs(main_dir, exist_ok=True) + + # Classify dependencies + dependencies = frontmatter.get('dependencies', []) + builtin_requires, managed_deps = _classify_dependencies(dependencies) + + # Top-level CMakeLists.txt (cmakev2) + top_cmake = ( + '# Auto-generated by idf.py build-file. Do not edit.\n' + 'cmake_minimum_required(VERSION 3.22)\n' + '\n' + 'include($ENV{IDF_PATH}/tools/cmakev2/idf.cmake)\n' + f'project({source_name} C CXX ASM)\n' + 'idf_project_default()\n' + ) + _write_if_changed(os.path.join(container_dir, 'CMakeLists.txt'), top_cmake) + + # Copy the source file into main/ (using _write_if_changed for caching) + with open(abs_source, encoding='utf-8') as f: + source_content = f.read() + _write_if_changed(os.path.join(main_dir, source_basename), source_content) + + # main/CMakeLists.txt with REQUIRES for built-in components + requires_clause = '' + if builtin_requires: + requires_clause = f'\n PRIV_REQUIRES {" ".join(builtin_requires)}' + + main_cmake = ( + '# Auto-generated by idf.py build-file. Do not edit.\n' + f'idf_component_register(SRCS "{source_basename}"\n' + f' INCLUDE_DIRS "."{requires_clause})\n' + ) + _write_if_changed(os.path.join(main_dir, 'CMakeLists.txt'), main_cmake) + + # sdkconfig.defaults + config_list = frontmatter.get('config', []) + # Auto-quote string values that aren't already quoted, boolean, or numeric. + # Kconfig string options require quoted values in sdkconfig.defaults. + processed_config = [] + for item in config_list: + if '=' in item: + key, value = item.split('=', 1) + if not (value.startswith('"') and value.endswith('"')) and not _is_bool_or_number(value): + item = f'{key}="{value}"' + processed_config.append(item) + sdkconfig_content = '\n'.join(processed_config) + '\n' if processed_config else '' + if _write_if_changed(os.path.join(container_dir, 'sdkconfig.defaults'), sdkconfig_content): + _invalidate_stale_config(container_dir, processed_config) + + # idf_component.yml for managed dependencies only + if managed_deps: + idf_component_yml = yaml.dump({'dependencies': managed_deps}, default_flow_style=False) + _write_if_changed(os.path.join(main_dir, 'idf_component.yml'), idf_component_yml) + else: + yml_path = os.path.join(main_dir, 'idf_component.yml') + if os.path.exists(yml_path): + os.remove(yml_path) + + +def action_extensions(base_actions: dict, project_path: str) -> dict: + def _find_build_file_task(tasks: list) -> Any | None: + """Find the build-file task in the task list, if present.""" + for task in tasks: + if task.name == 'build-file': + return task + return None + + def build_file_global_callback(ctx: Context, global_args: PropertyDict, tasks: list) -> None: + """Global callback that runs before validate_root_options. + + If build-file is in the task list, this sets up the container project + and redirects project_dir/build_dir to it, so that subsequent commands + (build, flash, monitor) operate on the container project. + """ + build_file_task = _find_build_file_task(tasks) + if build_file_task is None: + return + + source_file = build_file_task.action_args.get('source_file') + if not source_file: + return + + if not os.path.isfile(source_file): + raise FatalError(f'Source file not found: {source_file}') + + clean_cache = build_file_task.action_args.get('clean_cache', False) + + # Parse frontmatter + frontmatter = _parse_frontmatter(source_file) + + if not frontmatter: + log.warn( + escape( + f'build-file: No idf-build-file frontmatter found in {source_file}. ' + 'The file will be built with default settings.' + ) + ) + + # Determine container project directory + file_hash = _compute_project_hash(source_file) + source_name = os.path.splitext(os.path.basename(source_file))[0] + container_dir = os.path.join(_BUILD_FILE_CACHE_DIR, f'{source_name}_{file_hash}') + + # Clean cache if requested + if clean_cache and os.path.isdir(container_dir): + shutil.rmtree(container_dir) + log.note(escape(f'build-file: Cleaned cache at {container_dir}')) + + # Generate/update the container project + _generate_container_project(source_file, frontmatter, container_dir) + + # Redirect project_dir and build_dir to the container project. + # This runs before validate_root_options (from core_ext.py) which will + # then use these values instead of the defaults. + global_args['project_dir'] = os.path.realpath(container_dir) + global_args['build_dir'] = os.path.join(os.path.realpath(container_dir), 'build') + + log.note(escape(f'build-file: Container project: {container_dir}')) + log.note(escape(f'build-file: Source file: {os.path.abspath(source_file)}')) + + def build_file_callback(action: str, ctx: Context, args: PropertyDict, **action_args: Any) -> None: + """The build-file action callback. + + The heavy lifting is done in the global callback. This callback exists + so that the action is registered and composable with other commands. + By depending on 'all', the build is triggered automatically. + """ + pass + + return { + 'global_action_callbacks': [build_file_global_callback], + 'actions': { + 'build-file': { + 'callback': build_file_callback, + 'short_help': 'Build a standalone C file without project boilerplate.', + 'help': ( + 'Build a standalone C source file without requiring a full ESP-IDF project structure. ' + 'The source file may contain an optional YAML frontmatter in a block comment ' + 'starting with "idf-build-file:" to specify sdkconfig options, component dependencies, ' + 'and other settings.\n\n' + 'A container project is automatically created and cached. Subsequent builds of the ' + 'same file reuse the container project for fast incremental builds.\n\n' + 'This command is composable with other idf.py commands. For example:\n' + ' idf.py build-file example.c build\n' + ' idf.py build-file example.c flash monitor\n' + ), + 'arguments': [ + { + 'names': ['source_file'], + 'nargs': 1, + 'type': click.Path(exists=False), + }, + ], + 'options': [ + { + 'names': ['--clean-cache'], + 'is_flag': True, + 'default': False, + 'help': 'Remove the cached container project before building.', + }, + ], + 'dependencies': ['all'], + }, + }, + } diff --git a/tools/test_idf_py/test_build_file.py b/tools/test_idf_py/test_build_file.py new file mode 100644 index 00000000000..684309e88b5 --- /dev/null +++ b/tools/test_idf_py/test_build_file.py @@ -0,0 +1,385 @@ +#!/usr/bin/env python +# SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import os +import re +import shutil +import subprocess + +# Import the module under test +import sys +import tempfile +from collections.abc import Generator + +import pytest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'idf_py_actions')) +from build_file_ext import _classify_dependencies +from build_file_ext import _generate_container_project +from build_file_ext import _is_bool_or_number +from build_file_ext import _parse_frontmatter +from build_file_ext import _write_if_changed + +IDF_PATH = os.environ.get('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..')) +IDF_PY = os.path.join(IDF_PATH, 'tools', 'idf.py') + + +@pytest.fixture +def tmp_dir() -> Generator[str]: + d = tempfile.mkdtemp() + yield d + shutil.rmtree(d) + + +class TestParseFrontmatter: + def _write_c_file(self, tmp_dir: str, content: str) -> str: + path = os.path.join(tmp_dir, 'test.c') + with open(path, 'w') as f: + f.write(content) + return path + + def test_no_frontmatter(self, tmp_dir: str) -> None: + path = self._write_c_file(tmp_dir, '#include \nvoid app_main(void) {}\n') + assert _parse_frontmatter(path) == {} + + def test_basic_frontmatter(self, tmp_dir: str) -> None: + path = self._write_c_file( + tmp_dir, + """/* + idf-build-file: + config: + - "CONFIG_IDF_TARGET=esp32" + dependencies: + - log + - "espressif/zlib==1.2.3" + */ + +#include +void app_main(void) {} +""", + ) + result = _parse_frontmatter(path) + assert result['config'] == ['CONFIG_IDF_TARGET=esp32'] + assert 'log' in result['dependencies'] + assert 'espressif/zlib==1.2.3' in result['dependencies'] + + def test_frontmatter_with_extra_comments(self, tmp_dir: str) -> None: + path = self._write_c_file( + tmp_dir, + """/* some other comment */ + +/* + idf-build-file: + config: + - "CONFIG_SPIRAM=y" + */ + +/* another comment */ +void app_main(void) {} +""", + ) + result = _parse_frontmatter(path) + assert result['config'] == ['CONFIG_SPIRAM=y'] + + def test_empty_frontmatter(self, tmp_dir: str) -> None: + path = self._write_c_file( + tmp_dir, + """/* + idf-build-file: + */ +void app_main(void) {} +""", + ) + result = _parse_frontmatter(path) + assert result == {} + + +class TestClassifyDependencies: + def test_builtin_only(self) -> None: + builtin, managed = _classify_dependencies(['log', 'vfs', 'driver']) + assert builtin == ['log', 'vfs', 'driver'] + assert managed == {} + + def test_managed_only(self) -> None: + builtin, managed = _classify_dependencies(['espressif/zlib==1.2.3', 'espressif/led_strip>=2.0']) + assert builtin == [] + assert managed == { + 'espressif/zlib': {'version': '1.2.3'}, + 'espressif/led_strip': {'version': '>=2.0'}, + } + + def test_mixed(self) -> None: + builtin, managed = _classify_dependencies(['log', 'espressif/zlib==1.2.3', 'vfs']) + assert builtin == ['log', 'vfs'] + assert 'espressif/zlib' in managed + + def test_empty(self) -> None: + builtin, managed = _classify_dependencies([]) + assert builtin == [] + assert managed == {} + + def test_path_based_dependency(self) -> None: + deps = [ + 'log', + {'protocol_examples_common': {'path': '${IDF_PATH}/examples/common_components/protocol_examples_common'}}, + ] + builtin, managed = _classify_dependencies(deps) + assert builtin == ['log'] + assert 'protocol_examples_common' in managed + assert ( + managed['protocol_examples_common']['path'] + == '${IDF_PATH}/examples/common_components/protocol_examples_common' + ) + + +class TestIsBoolOrNumber: + def test_booleans(self) -> None: + assert _is_bool_or_number('y') is True + assert _is_bool_or_number('n') is True + + def test_numbers(self) -> None: + assert _is_bool_or_number('42') is True + assert _is_bool_or_number('0x1000') is True + assert _is_bool_or_number('0') is True + + def test_strings(self) -> None: + assert _is_bool_or_number('esp32') is False + assert _is_bool_or_number('hello') is False + + def test_empty(self) -> None: + assert _is_bool_or_number('') is True + + +class TestWriteIfChanged: + def test_creates_new_file(self, tmp_dir: str) -> None: + path = os.path.join(tmp_dir, 'test.txt') + assert _write_if_changed(path, 'hello') is True + with open(path) as f: + assert f.read() == 'hello' + + def test_no_write_if_same(self, tmp_dir: str) -> None: + path = os.path.join(tmp_dir, 'test.txt') + with open(path, 'w') as f: + f.write('hello') + assert _write_if_changed(path, 'hello') is False + + def test_overwrites_if_different(self, tmp_dir: str) -> None: + path = os.path.join(tmp_dir, 'test.txt') + with open(path, 'w') as f: + f.write('hello') + assert _write_if_changed(path, 'world') is True + with open(path) as f: + assert f.read() == 'world' + + +class TestGenerateContainerProject: + def test_generates_project_files(self, tmp_dir: str) -> None: + src = os.path.join(tmp_dir, 'test.c') + with open(src, 'w') as f: + f.write('void app_main(void) {}\n') + + container = os.path.join(tmp_dir, 'container') + _generate_container_project(src, {}, container) + + assert os.path.exists(os.path.join(container, 'CMakeLists.txt')) + assert os.path.exists(os.path.join(container, 'main', 'CMakeLists.txt')) + assert os.path.exists(os.path.join(container, 'main', 'test.c')) + + with open(os.path.join(container, 'CMakeLists.txt')) as f: + cmake_content = f.read() + assert 'cmakev2/idf.cmake' in cmake_content + assert 'idf_project_default()' in cmake_content + + def test_generates_sdkconfig_defaults(self, tmp_dir: str) -> None: + src = os.path.join(tmp_dir, 'test.c') + with open(src, 'w') as f: + f.write('void app_main(void) {}\n') + + frontmatter = {'config': ['CONFIG_SPIRAM=y', 'CONFIG_IDF_TARGET=esp32']} + container = os.path.join(tmp_dir, 'container') + _generate_container_project(src, frontmatter, container) + + with open(os.path.join(container, 'sdkconfig.defaults')) as f: + content = f.read() + assert 'CONFIG_SPIRAM=y' in content + assert 'CONFIG_IDF_TARGET="esp32"' in content + + def test_generates_priv_requires_for_builtin_deps(self, tmp_dir: str) -> None: + src = os.path.join(tmp_dir, 'test.c') + with open(src, 'w') as f: + f.write('void app_main(void) {}\n') + + frontmatter = {'dependencies': ['log', 'vfs']} + container = os.path.join(tmp_dir, 'container') + _generate_container_project(src, frontmatter, container) + + with open(os.path.join(container, 'main', 'CMakeLists.txt')) as f: + content = f.read() + assert 'PRIV_REQUIRES log vfs' in content + + def test_generates_component_yml_for_managed_deps(self, tmp_dir: str) -> None: + src = os.path.join(tmp_dir, 'test.c') + with open(src, 'w') as f: + f.write('void app_main(void) {}\n') + + frontmatter = {'dependencies': ['espressif/zlib==1.2.3']} + container = os.path.join(tmp_dir, 'container') + _generate_container_project(src, frontmatter, container) + + yml_path = os.path.join(container, 'main', 'idf_component.yml') + assert os.path.exists(yml_path) + with open(yml_path) as f: + content = f.read() + assert 'espressif/zlib' in content + + def test_generates_component_yml_for_path_deps(self, tmp_dir: str) -> None: + src = os.path.join(tmp_dir, 'test.c') + with open(src, 'w') as f: + f.write('void app_main(void) {}\n') + + frontmatter = { + 'dependencies': [ + { + 'protocol_examples_common': { + 'path': '${IDF_PATH}/examples/common_components/protocol_examples_common' + } + }, + ] + } + container = os.path.join(tmp_dir, 'container') + _generate_container_project(src, frontmatter, container) + + yml_path = os.path.join(container, 'main', 'idf_component.yml') + assert os.path.exists(yml_path) + with open(yml_path) as f: + content = f.read() + assert 'protocol_examples_common' in content + + def test_copies_source_file(self, tmp_dir: str) -> None: + src = os.path.join(tmp_dir, 'test.c') + with open(src, 'w') as f: + f.write('void app_main(void) {}\n') + + container = os.path.join(tmp_dir, 'container') + _generate_container_project(src, {}, container) + + dest = os.path.join(container, 'main', 'test.c') + assert os.path.isfile(dest) + assert not os.path.islink(dest) + with open(dest) as f: + assert f.read() == 'void app_main(void) {}\n' + + +class TestBuildFileEndToEnd: + """End-to-end tests that invoke idf.py build-file as a subprocess.""" + + def _run_build_file(self, src: str, *extra_args: str) -> tuple[subprocess.CompletedProcess, str, str]: + """Run idf.py build-file on the given source file. + + Returns (completed_process, combined_output, container_dir). + """ + result = subprocess.run( + [sys.executable, IDF_PY, 'build-file', src, *extra_args], + capture_output=True, + text=True, + timeout=10 * 60, + ) + output = result.stdout + result.stderr + # Strip ANSI color codes so the container dir path can be extracted + plain_output = re.sub(r'\x1b\[[0-9;]*m', '', output) + match = re.search(r'Container project: (.+)', plain_output) + container_dir = match.group(1).strip() if match else '' + return result, output, container_dir + + @staticmethod + def _read_sdkconfig(container_dir: str) -> str: + with open(os.path.join(container_dir, 'sdkconfig')) as f: + return f.read() + + def test_build_file_with_size(self, tmp_dir: str) -> None: + """Test that build-file composes with the size command.""" + src = os.path.join(tmp_dir, 'hello.c') + with open(src, 'w') as f: + f.write( + '/*\n' + ' idf-build-file:\n' + ' config:\n' + ' - CONFIG_IDF_TARGET=esp32\n' + ' dependencies:\n' + ' - log\n' + ' */\n' + '#include "esp_log.h"\n' + 'static const char *TAG = "hello";\n' + 'void app_main(void) { ESP_LOGI(TAG, "Hello"); }\n' + ) + + result = subprocess.run( + [sys.executable, IDF_PY, 'build-file', src, 'size'], + capture_output=True, + text=True, + timeout=10 * 60, + ) + output = result.stdout + result.stderr + assert result.returncode == 0, f'idf.py build-file + size failed:\n{output}' + # Verify build completed + assert 'Project build complete' in output + # Verify size command ran (it prints memory usage) + assert 'Total image size' in output or 'Used static' in output + + def test_config_change_takes_effect_on_rebuild(self, tmp_dir: str) -> None: + """Changing an sdkconfig option in the frontmatter must be applied on rebuild.""" + src = os.path.join(tmp_dir, 'hello.c') + + def write_src(extra_config: str = '') -> None: + with open(src, 'w') as f: + f.write( + '/*\n' + ' idf-build-file:\n' + ' config:\n' + ' - CONFIG_IDF_TARGET=esp32\n' + f'{extra_config}' + ' */\n' + 'void app_main(void) {}\n' + ) + + write_src() + result, output, container_dir = self._run_build_file(src) + assert result.returncode == 0, f'initial build failed:\n{output}' + assert container_dir, f'container project dir not found in output:\n{output}' + assert 'CONFIG_COMPILER_OPTIMIZATION_SIZE=y' not in self._read_sdkconfig(container_dir) + + write_src(' - CONFIG_COMPILER_OPTIMIZATION_SIZE=y\n') + result, output, container_dir = self._run_build_file(src) + assert result.returncode == 0, f'rebuild after config change failed:\n{output}' + assert 'CONFIG_COMPILER_OPTIMIZATION_SIZE=y' in self._read_sdkconfig(container_dir), ( + 'config change in frontmatter was not applied on rebuild' + ) + + def test_target_change_takes_effect_on_rebuild(self, tmp_dir: str) -> None: + """Changing CONFIG_IDF_TARGET in the frontmatter must be applied on rebuild.""" + src = os.path.join(tmp_dir, 'hello.c') + + def write_src(target: str) -> None: + with open(src, 'w') as f: + f.write( + '/*\n' + ' idf-build-file:\n' + ' config:\n' + f' - CONFIG_IDF_TARGET={target}\n' + ' */\n' + 'void app_main(void) {}\n' + ) + + write_src('esp32') + result, output, container_dir = self._run_build_file(src) + assert result.returncode == 0, f'initial build failed:\n{output}' + assert container_dir, f'container project dir not found in output:\n{output}' + assert 'CONFIG_IDF_TARGET="esp32"' in self._read_sdkconfig(container_dir) + + write_src('esp32c3') + result, output, container_dir = self._run_build_file(src) + assert result.returncode == 0, f'rebuild after target change failed:\n{output}' + assert 'CONFIG_IDF_TARGET="esp32c3"' in self._read_sdkconfig(container_dir), ( + 'target change in frontmatter was not applied on rebuild' + )