Mirrors/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2026-09-22 13:01:16 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feat/consume_root_managed_components_from_component_manager' into 'master'

Browse Source 
feat: Consume root-managed components from IDF Component Manager

Closes PACMAN-1262, PACMAN-1266, IDFCI-10131, and IDF-14259

See merge request espressif/esp-idf!48609
This commit is contained in:
Sergei Silnov
2026-08-19 09:30:32 +02:00
parent d9a5f9ee1c 7e6fc433cc
commit 76a34b3f85
12 changed files with 204 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
docs/en/api-guides/build-system-v2/design.rst
View File

@@ -83,14 +83,17 @@ The build system looks for components in several locations, each associated with
- Priority
- Where the components come from
* - ``project_components``
- 3 (highest)
- 4 (highest)
- The project's ``main`` and ``components`` directories (or ``COMPONENT_DIRS``)
* - ``project_extra_components``
- 2
- 3
- Directories listed in ``EXTRA_COMPONENT_DIRS``
* - ``project_managed_components``
- 1
- 2
- Components fetched by the component manager
* - ``idf_managed_components``
- 1
- ESP-IDF's own root-managed components, installed by the component manager from ``$IDF_PATH/tools/idf_extra_components.yml``
* - ``idf_components``
- 0 (lowest)
- Components bundled with ESP-IDF (``$IDF_PATH/components``)

2
docs/en/api-guides/build-system-v2/glossary.rst
View File

@@ -48,7 +48,7 @@ component property
.. _cmakev2-term-component-source:
component source
A location the build system searches for components, each with a precedence. From highest to lowest: ``project_components`` (the project's ``main`` and ``components`` directories), ``project_extra_components`` (directories in ``EXTRA_COMPONENT_DIRS``), ``project_managed_components`` (fetched by the component manager), and ``idf_components`` (bundled with ESP-IDF). A component from a higher-precedence source shadows a same-named component from a lower one. See :doc:`design`.
A location the build system searches for components, each with a precedence. From highest to lowest: ``project_components`` (the project's ``main`` and ``components`` directories), ``project_extra_components`` (directories in ``EXTRA_COMPONENT_DIRS``), ``project_managed_components`` (fetched by the component manager), ``idf_managed_components`` (ESP-IDF's own root-managed components, resolved by the component manager), and ``idf_components`` (bundled with ESP-IDF). A component from a higher-precedence source shadows a same-named component from a lower one. See :doc:`design`.
.. _cmakev2-term-configuration:

33
tools/cmake/build.cmake
View File

@@ -65,6 +65,17 @@ function(idf_build_unset_property property)
idf_build_set_property(__BUILD_PROPERTIES "${build_properties}")
endfunction()
function(__component_manager_get_command var)
idf_build_get_property(python PYTHON)
if(DEFINED ENV{IDF_COMPONENT_WRAPPER} AND NOT "$ENV{IDF_COMPONENT_WRAPPER}" STREQUAL "")
set(component_manager_cmd
"${python}" "$ENV{IDF_COMPONENT_WRAPPER}" "idf_component_manager.prepare_components")
else()
set(component_manager_cmd "${python}" "-m" "idf_component_manager.prepare_components")
endif()
set(${var} ${component_manager_cmd} PARENT_SCOPE)
endfunction()
# idf_build_replace_option_from_property
#
# @brief Replace specified option with new one in a given property.
@@ -359,6 +370,7 @@ function(idf_build_component component_dir)
# component_source must be one of the following (sorted by the override order):
set(valid_component_sources "idf_components"
"idf_managed_components"
"project_managed_components"
"project_extra_components"
"project_components")
@@ -678,7 +690,7 @@ macro(idf_build_process target)
file(WRITE ${local_components_list_file} "${__contents}")
# Call for the component manager to prepare remote dependencies
idf_build_get_property(python PYTHON)
__component_manager_get_command(component_manager_cmd)
idf_build_get_property(component_manager_interface_version __COMPONENT_MANAGER_INTERFACE_VERSION)
idf_build_get_property(dependencies_lock_file DEPENDENCIES_LOCK)
@@ -687,9 +699,7 @@ macro(idf_build_process target)
set(use_sdk_json FALSE)
endif()
execute_process(COMMAND ${python}
"-m"
"idf_component_manager.prepare_components"
execute_process(COMMAND ${component_manager_cmd}
"--project_dir=${project_dir}"
"--lock_path=${dependencies_lock_file}"
"--interface_version=${component_manager_interface_version}"
@@ -741,21 +751,6 @@ macro(idf_build_process target)
endif()
endif()
idf_build_get_property(prefix __PREFIX)
file(GLOB root_dep_component_dirs
${IDF_TOOLS_PATH}/root_managed_components/idf${IDF_VERSION_MAJOR}.${IDF_VERSION_MINOR}.${IDF_VERSION_PATCH}/*)
list(SORT root_dep_component_dirs)
foreach(component_dir ${root_dep_component_dirs})
# A potential component must be a directory
if(IS_DIRECTORY ${component_dir})
__component_dir_quick_check(is_component ${component_dir})
if(is_component)
__component_add(${component_dir} ${prefix} "idf_managed_components")
endif()
endif()
endforeach()
# Perform early expansion of component CMakeLists.txt in CMake scripting mode.
# It is here we retrieve the public and private requirements of each component.
# It is also here we add the common component requirements to each component's

6
tools/cmake/component.cmake
View File

@@ -233,16 +233,14 @@ function(__component_get_requirements use_sdk_json)
idf_build_get_property(idf_component_manager IDF_COMPONENT_MANAGER)
if(idf_component_manager EQUAL 1)
idf_build_get_property(python PYTHON)
__component_manager_get_command(component_manager_cmd)
idf_build_get_property(component_manager_interface_version __COMPONENT_MANAGER_INTERFACE_VERSION)
# Call for the component manager once again to inject dependencies
# It modifies the requirements file generated by component_get_requirements.cmake script by adding dependencies
# defined in component manager manifests to REQUIRES and PRIV_REQUIRES fields.
# These requirements are also set as MANAGED_REQUIRES and MANAGED_PRIV_REQUIRES component properties.
execute_process(COMMAND ${python}
"-m"
"idf_component_manager.prepare_components"
execute_process(COMMAND ${component_manager_cmd}
"--project_dir=${project_dir}"
"--lock_path=${DEPENDENCIES_LOCK}"
"--interface_version=${component_manager_interface_version}"

2
tools/cmake/project.cmake
View File

@@ -156,7 +156,7 @@ if(NOT "$ENV{IDF_COMPONENT_MANAGER}" EQUAL "0")
idf_build_set_property(IDF_COMPONENT_MANAGER 1)
endif()
# Set component manager interface version
idf_build_set_property(__COMPONENT_MANAGER_INTERFACE_VERSION 5)
idf_build_set_property(__COMPONENT_MANAGER_INTERFACE_VERSION 6)
#
# Parse and store the VERSION argument provided to the project() command.

7
tools/cmake/scripts/component_get_requirements.cmake
View File

@@ -122,9 +122,10 @@ set(CMAKE_BUILD_EARLY_EXPANSION 1)
# smaller number means lower priority
set(__TARGETS_IDF_COMPONENTS "") # 0
set(__TARGETS_PROJECT_MANAGED_COMPONENTS "") # 1
set(__TARGETS_PROJECT_EXTRA_COMPONENTS "") # 2
set(__TARGETS_PROJECT_COMPONENTS "") # 3
set(__TARGETS_IDF_MANAGED_COMPONENTS "") # 1
set(__TARGETS_PROJECT_MANAGED_COMPONENTS "") # 2
set(__TARGETS_PROJECT_EXTRA_COMPONENTS "") # 3
set(__TARGETS_PROJECT_COMPONENTS "") # 4
foreach(__component_target ${__component_targets})
__component_get_property(__component_source ${__component_target} COMPONENT_SOURCE)

1
tools/cmakev2/compat.cmake
View File

@@ -945,6 +945,7 @@ endmacro()
Source of the component. One of:
* ``idf_components``
* ``idf_managed_components``
* ``project_managed_components``
* ``project_extra_components``
* ``project_components``

6
tools/cmakev2/component.cmake
View File

@@ -509,10 +509,12 @@ function(__get_component_priority)
endif()
if("${ARG_SOURCE}" STREQUAL "project_components")
set(priority 3)
set(priority 4)
elseif("${ARG_SOURCE}" STREQUAL "project_extra_components")
set(priority 2)
set(priority 3)
elseif("${ARG_SOURCE}" STREQUAL "project_managed_components")
set(priority 2)
elseif("${ARG_SOURCE}" STREQUAL "idf_managed_components")
set(priority 1)
elseif("${ARG_SOURCE}" STREQUAL "idf_components")
set(priority 0)

27
tools/cmakev2/manager.cmake
View File

@@ -25,9 +25,9 @@ function(__init_component_manager)
endif()
# Set IDF_COMPONENT_MANAGER_INTERFACE_VERSION.
# Defaults to 5. Allow overriding via env/CMake.
# Defaults to 6. Allow overriding via env/CMake.
__get_default_value(VARIABLE IDF_COMPONENT_MANAGER_INTERFACE_VERSION
DEFAULT 5
DEFAULT 6
OUTPUT cmgr_iface)
idf_build_set_property(IDF_COMPONENT_MANAGER_INTERFACE_VERSION ${cmgr_iface})
@@ -38,6 +38,17 @@ function(__init_component_manager)
idf_build_set_property(DEPENDENCIES_LOCK "${deps_lock_file}")
endfunction()
function(__component_manager_get_command var)
idf_build_get_property(python PYTHON)
if(DEFINED ENV{IDF_COMPONENT_WRAPPER} AND NOT "$ENV{IDF_COMPONENT_WRAPPER}" STREQUAL "")
set(component_manager_cmd
"${python}" "$ENV{IDF_COMPONENT_WRAPPER}" "idf_component_manager.prepare_components")
else()
set(component_manager_cmd "${python}" "-m" "idf_component_manager.prepare_components")
endif()
set(${var} ${component_manager_cmd} PARENT_SCOPE)
endfunction()
#[[
__fetch_components_from_registry()
@@ -135,14 +146,12 @@ function(__download_managed_component)
idf_die("RESULT option is required")
endif()
idf_build_get_property(python PYTHON)
__component_manager_get_command(component_manager_cmd)
idf_build_get_property(project_dir PROJECT_DIR)
idf_build_get_property(component_manager_interface_version IDF_COMPONENT_MANAGER_INTERFACE_VERSION)
idf_build_get_property(dependencies_lock_file DEPENDENCIES_LOCK)
# Invoke the component manager
execute_process(COMMAND ${python}
"-m"
"idf_component_manager.prepare_components"
execute_process(COMMAND ${component_manager_cmd}
"--project_dir=${project_dir}"
"--lock_path=${dependencies_lock_file}"
"--interface_version=${component_manager_interface_version}"
@@ -298,7 +307,6 @@ function(__inject_requirements_for_component_from_manager component_name)
idf_dbg("Injecting requirements for component '${component_name}' from the component manager")
idf_build_get_property(python PYTHON)
idf_build_get_property(project_dir PROJECT_DIR)
idf_build_get_property(build_dir BUILD_DIR)
idf_build_get_property(dependencies_lock_file DEPENDENCIES_LOCK)
@@ -355,9 +363,8 @@ function(__inject_requirements_for_component_from_manager component_name)
endif()
# Call component manager to inject requirements
execute_process(COMMAND ${python}
"-m"
"idf_component_manager.prepare_components"
__component_manager_get_command(component_manager_cmd)
execute_process(COMMAND ${component_manager_cmd}
"--project_dir=${project_dir}"
"--lock_path=${dependencies_lock_file}"
"--interface_version=${component_manager_interface_version}"

42
tools/idf.py
View File

@@ -53,6 +53,8 @@ try:
if os.getenv('IDF_COMPONENT_MANAGER') != '0':
from idf_component_manager import idf_extensions
from idf_component_tools.errors import FatalError as ComponentManagerFatalError
from idf_component_tools.root_managed_components import RootManagedComponentsStateManager
except ImportError as e:
print(
(
@@ -1104,20 +1106,43 @@ def init_cli(verbose_output: list | None = None) -> Any:
return result
def _resolve_idf_managed_lock_path() -> str | None:
"""Return path to dependencies.lock for IDF-managed components, or None if unavailable."""
def _get_trusted_names_from_root_state(state_path: str) -> set[str]:
"""Return component names from the trusted root-managed inventory."""
if state_path in _trusted_names_cache:
return _trusted_names_cache[state_path]
result: set[str] = set()
_trusted_names_cache[state_path] = result
if not os.path.isfile(state_path) or os.getenv('IDF_COMPONENT_MANAGER') == '0':
return result
try:
state = RootManagedComponentsStateManager(state_path).load()
result.update(state.components)
except (OSError, ComponentManagerFatalError) as e:
log.warn(
escape(
'Could not verify source of external components. '
f'No extensions (idf_ext.py) from managed components will be loaded. ({e})'
)
)
return result
def _resolve_idf_managed_state_dir() -> str | None:
"""Return the state directory for IDF-managed components, or None if unavailable."""
idf_tools_path = os.environ.get('IDF_TOOLS_PATH') or os.path.expanduser(os.path.join('~', '.espressif'))
ver = idf_version_from_cmake() # returns e.g. 'v6.1.0', or None on failure
if not ver:
return None
ver_str = ver.lstrip('v') # '6.1.0'
return os.path.join(idf_tools_path, 'root_managed_components', f'idf{ver_str}', 'dependencies.lock')
return os.path.join(idf_tools_path, 'root_managed_components', f'idf{ver_str}')
def _is_component_trusted(
comp_name: str,
source: str | None,
) -> bool:
"""True iff this component is from a trusted source (IDF, project, or Espressif component from ESP-registry)."""
"""Return whether a component may provide an idf.py extension."""
if source in ('idf_components', 'project_components', 'project_extra_components'):
return True
if source == 'project_managed_components':
@@ -1125,11 +1150,14 @@ def init_cli(verbose_output: list | None = None) -> Any:
lock_key = comp_name.replace('__', '/', 1) if '__' in comp_name else comp_name
return lock_key in _get_trusted_names_from_lock(os.path.join(project_dir, 'dependencies.lock'))
if source == 'idf_managed_components':
lock_path = _resolve_idf_managed_lock_path()
if lock_path is None:
state_dir = _resolve_idf_managed_state_dir()
if state_dir is None:
return False
lock_key = comp_name.replace('__', '/', 1) if '__' in comp_name else comp_name
return lock_key in _get_trusted_names_from_lock(lock_path)
root_state_path = os.path.join(state_dir, 'root_components.lock')
if os.path.isfile(root_state_path):
return lock_key in _get_trusted_names_from_root_state(root_state_path)
return lock_key in _get_trusted_names_from_lock(os.path.join(state_dir, 'dependencies.lock'))
return False
def _build_rich_help_command_groups(

7
tools/idf_extra_components.yml
View File

@@ -1,11 +1,12 @@
# This file defines extra dependencies for ESP-IDF
# the dependencies defined here will be downloaded to
# $IDF_TOOLS_PATH/root_managed_components
# Each major.minor version of ESP-IDF can have its own subdirectory
# For example, for ESP-IDF v6.0, the dependencies will be installed to
# $IDF_TOOLS_PATH/root_managed_components/idf6.0
# Each major.minor.patch version of ESP-IDF can have its own subdirectory
# For example, for ESP-IDF v6.2.0, the dependencies will be installed to
# $IDF_TOOLS_PATH/root_managed_components/idf6.2.0
# The syntax is defined in:
# https://docs.espressif.com/projects/idf-component-manager/en/latest/reference/manifest_file.html#dependencies
# Rules and matches are not allowed in this manifest
#dependencies:

117
tools/test_build_system/test_component_manager.py
View File

@@ -2,15 +2,21 @@
# SPDX-License-Identifier: Apache-2.0
import json
import os.path
import subprocess
import textwrap
from pathlib import Path
import pytest
from test_build_system_helpers import EXT_IDF_PATH
from test_build_system_helpers import EnvDict
from test_build_system_helpers import IdfPyFunc
from test_build_system_helpers import replace_in_file
def _install_root_components(env: EnvDict) -> None:
subprocess.run(['compote', 'cooking', 'stock'], check=True, env=env)
def test_dependency_lock(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
replace_in_file(
test_app_copy / 'CMakeLists.txt',
@@ -155,10 +161,11 @@ class TestOptionalDependencyWithKconfig:
res = idf_py('reconfigure', check=False)
assert res.returncode != 0
assert (
missing_kconfig_msg = (
f'OF_COURSE_NO_ONE_USE_FOO, introduced by example/cmp, '
f'defined in {str(test_app_copy / "main" / "idf_component.yml")}' in res.stderr
f'defined in {str(test_app_copy / "main" / "idf_component.yml")}'
)
assert ''.join(missing_kconfig_msg.split()) in ''.join(res.stderr.split())
assert 'Missing required kconfig option after retry.' in res.stderr
def test_kconfig_in_transitive_dependency(self, idf_py: IdfPyFunc, test_app_copy: Path) -> None:
@@ -202,3 +209,109 @@ class TestOptionalDependencyWithKconfig:
data = json.load(open(test_app_copy / 'build' / 'project_description.json'))
assert ['example__cmp'] == data['build_component_info']['foo']['priv_reqs']
assert ['espressif__mdns'] == data['build_component_info']['foo']['reqs']
@pytest.mark.revert_later(['tools/idf_extra_components.yml'])
class TestIdfRootDependency:
@pytest.fixture(autouse=True)
def _clean_root_managed(self, clean_root_managed_components: None) -> None:
pass
def test_basic_build(self, idf_py: IdfPyFunc, test_app_copy: Path, default_idf_env: EnvDict) -> None:
with open(os.path.join(EXT_IDF_PATH, 'tools', 'idf_extra_components.yml'), 'w') as fw:
fw.write(
textwrap.dedent("""
dependencies:
example/cmp: "==3.3.9"
""")
)
_install_root_components(default_idf_env)
replace_in_file(
(test_app_copy / 'main' / 'build_test_app.c'),
'// placeholder_before_main',
'#include "cmp.h"',
)
# Call into the component as well, so that the test fails if the component
# is only on the include path but not linked.
replace_in_file(
(test_app_copy / 'main' / 'build_test_app.c'),
'// placeholder_inside_main',
'cmp_hello();',
)
# Intentional dependency on the cmake-level name of the managed component because
# idf_extra_components.yml installs example/cmp and we verify REQUIRES pulls
# example__cmp into the build — not something application code should do.
replace_in_file(
(test_app_copy / 'main' / 'CMakeLists.txt'),
'# placeholder_inside_idf_component_register',
'REQUIRES example__cmp',
)
idf_py('build')
def test_build_only_when_required(self, idf_py: IdfPyFunc, test_app_copy: Path, default_idf_env: EnvDict) -> None:
with open(os.path.join(EXT_IDF_PATH, 'tools', 'idf_extra_components.yml'), 'w') as fw:
fw.write(
textwrap.dedent("""
dependencies:
espressif/mdns: "==1.10.0"
example/cmp: "==3.3.9"
""")
)
_install_root_components(default_idf_env)
idf_py('reconfigure')
data = json.load(open(test_app_copy / 'build' / 'project_description.json'))
assert 'espressif__mdns' not in data['build_components']
assert 'example__cmp' not in data['build_components']
# Intentional dependency on the cmake-level name of the managed component because
# we assert espressif__mdns enters build_components only after REQUIRES — not
# something application code should do.
replace_in_file(
(test_app_copy / 'main' / 'CMakeLists.txt'),
'# placeholder_inside_idf_component_register',
'REQUIRES espressif__mdns',
)
idf_py('reconfigure')
data = json.load(open(test_app_copy / 'build' / 'project_description.json'))
assert 'espressif__mdns' in data['build_components']
assert 'example__cmp' not in data['build_components']
def test_cleanup_unused(self, idf_py: IdfPyFunc, test_app_copy: Path, default_idf_env: EnvDict) -> None:
with open(os.path.join(EXT_IDF_PATH, 'tools', 'idf_extra_components.yml'), 'w') as fw:
fw.write(
textwrap.dedent("""
dependencies:
espressif/mdns: "==1.10.0"
""")
)
_install_root_components(default_idf_env)
idf_py('reconfigure')
data = json.load(open(test_app_copy / 'build' / 'project_description.json'))
assert 'espressif__mdns' in data['all_component_info']
with open(os.path.join(EXT_IDF_PATH, 'tools', 'idf_extra_components.yml'), 'w') as fw:
fw.write(
textwrap.dedent("""
dependencies:
espressif/led_strip: "==3.0.3"
example/cmp: "==3.3.9"
""")
)
_install_root_components(default_idf_env)
idf_py('reconfigure')
data = json.load(open(test_app_copy / 'build' / 'project_description.json'))
assert 'espressif__led_strip' in data['all_component_info']
assert 'example__cmp' in data['all_component_info']
assert 'espressif__mdns' not in data['all_component_info']