feat: Consume root-managed components from IDF Component Manager

docs/en/api-guides/build-system-v2/design.rst

@@ -83,14 +83,17 @@ The build system looks for components in several locations, each associated with
      - Priority
      - Where the components come from
    * - ``project_components``
-     - 3 (highest)
+     - 4 (highest)
      - The project's ``main`` and ``components`` directories (or ``COMPONENT_DIRS``)
    * - ``project_extra_components``
-     - 2
+     - 3
      - Directories listed in ``EXTRA_COMPONENT_DIRS``
    * - ``project_managed_components``
-     - 1
+     - 2
      - Components fetched by the component manager
+   * - ``idf_managed_components``
+     - 1
+     - ESP-IDF's own root-managed components, installed by the component manager from ``$IDF_PATH/tools/idf_extra_components.yml``
    * - ``idf_components``
      - 0 (lowest)
      - Components bundled with ESP-IDF (``$IDF_PATH/components``)

docs/en/api-guides/build-system-v2/glossary.rst

@@ -48,7 +48,7 @@ component property
 .. _cmakev2-term-component-source:
 
 component source
-   A location the build system searches for components, each with a precedence. From highest to lowest: ``project_components`` (the project's ``main`` and ``components`` directories), ``project_extra_components`` (directories in ``EXTRA_COMPONENT_DIRS``), ``project_managed_components`` (fetched by the component manager), and ``idf_components`` (bundled with ESP-IDF). A component from a higher-precedence source shadows a same-named component from a lower one. See :doc:`design`.
+   A location the build system searches for components, each with a precedence. From highest to lowest: ``project_components`` (the project's ``main`` and ``components`` directories), ``project_extra_components`` (directories in ``EXTRA_COMPONENT_DIRS``), ``project_managed_components`` (fetched by the component manager), ``idf_managed_components`` (ESP-IDF's own root-managed components, resolved by the component manager), and ``idf_components`` (bundled with ESP-IDF). A component from a higher-precedence source shadows a same-named component from a lower one. See :doc:`design`.
 
 .. _cmakev2-term-configuration:
 

tools/cmake/build.cmake

@@ -65,6 +65,17 @@ function(idf_build_unset_property property)
     idf_build_set_property(__BUILD_PROPERTIES "${build_properties}")
 endfunction()
 
+function(__component_manager_get_command var)
+    idf_build_get_property(python PYTHON)
+    if(DEFINED ENV{IDF_COMPONENT_WRAPPER} AND NOT "$ENV{IDF_COMPONENT_WRAPPER}" STREQUAL "")
+        set(component_manager_cmd
+            "${python}" "$ENV{IDF_COMPONENT_WRAPPER}" "idf_component_manager.prepare_components")
+    else()
+        set(component_manager_cmd "${python}" "-m" "idf_component_manager.prepare_components")
+    endif()
+    set(${var} ${component_manager_cmd} PARENT_SCOPE)
+endfunction()
+
 # idf_build_replace_option_from_property
 #
 # @brief Replace specified option with new one in a given property.
@@ -359,6 +370,7 @@ function(idf_build_component component_dir)
 
     # component_source must be one of the following (sorted by the override order):
     set(valid_component_sources "idf_components"
+                              "idf_managed_components"
                               "project_managed_components"
                               "project_extra_components"
                               "project_components")
@@ -678,7 +690,7 @@ macro(idf_build_process target)
         file(WRITE ${local_components_list_file} "${__contents}")
 
         # Call for the component manager to prepare remote dependencies
-        idf_build_get_property(python PYTHON)
+        __component_manager_get_command(component_manager_cmd)
         idf_build_get_property(component_manager_interface_version __COMPONENT_MANAGER_INTERFACE_VERSION)
         idf_build_get_property(dependencies_lock_file DEPENDENCIES_LOCK)
 
@@ -687,9 +699,7 @@ macro(idf_build_process target)
             set(use_sdk_json FALSE)
         endif()
 
-        execute_process(COMMAND ${python}
+        execute_process(COMMAND ${component_manager_cmd}
-            "-m"
-            "idf_component_manager.prepare_components"
             "--project_dir=${project_dir}"
             "--lock_path=${dependencies_lock_file}"
             "--interface_version=${component_manager_interface_version}"
@@ -741,21 +751,6 @@ macro(idf_build_process target)
         endif()
     endif()
 
-    idf_build_get_property(prefix __PREFIX)
-
-    file(GLOB root_dep_component_dirs
-        ${IDF_TOOLS_PATH}/root_managed_components/idf${IDF_VERSION_MAJOR}.${IDF_VERSION_MINOR}.${IDF_VERSION_PATCH}/*)
-    list(SORT root_dep_component_dirs)
-    foreach(component_dir ${root_dep_component_dirs})
-        # A potential component must be a directory
-        if(IS_DIRECTORY ${component_dir})
-            __component_dir_quick_check(is_component ${component_dir})
-            if(is_component)
-                __component_add(${component_dir} ${prefix} "idf_managed_components")
-            endif()
-        endif()
-    endforeach()
-
     # Perform early expansion of component CMakeLists.txt in CMake scripting mode.
     # It is here we retrieve the public and private requirements of each component.
     # It is also here we add the common component requirements to each component's

tools/cmake/component.cmake

@@ -233,16 +233,14 @@ function(__component_get_requirements use_sdk_json)
 
     idf_build_get_property(idf_component_manager IDF_COMPONENT_MANAGER)
     if(idf_component_manager EQUAL 1)
-        idf_build_get_property(python PYTHON)
+        __component_manager_get_command(component_manager_cmd)
         idf_build_get_property(component_manager_interface_version __COMPONENT_MANAGER_INTERFACE_VERSION)
 
         # Call for the component manager once again to inject dependencies
         # It modifies the requirements file generated by component_get_requirements.cmake script by adding dependencies
         # defined in component manager manifests to REQUIRES and PRIV_REQUIRES fields.
         # These requirements are also set as MANAGED_REQUIRES and MANAGED_PRIV_REQUIRES component properties.
-        execute_process(COMMAND ${python}
+        execute_process(COMMAND ${component_manager_cmd}
-            "-m"
-            "idf_component_manager.prepare_components"
             "--project_dir=${project_dir}"
             "--lock_path=${DEPENDENCIES_LOCK}"
             "--interface_version=${component_manager_interface_version}"

tools/cmake/project.cmake

@@ -156,7 +156,7 @@ if(NOT "$ENV{IDF_COMPONENT_MANAGER}" EQUAL "0")
     idf_build_set_property(IDF_COMPONENT_MANAGER 1)
 endif()
 # Set component manager interface version
-idf_build_set_property(__COMPONENT_MANAGER_INTERFACE_VERSION 5)
+idf_build_set_property(__COMPONENT_MANAGER_INTERFACE_VERSION 6)
 
 #
 # Parse and store the VERSION argument provided to the project() command.

tools/cmake/scripts/component_get_requirements.cmake

@@ -122,9 +122,10 @@ set(CMAKE_BUILD_EARLY_EXPANSION 1)
 
 # smaller number means lower priority
 set(__TARGETS_IDF_COMPONENTS "")                # 0
-set(__TARGETS_PROJECT_MANAGED_COMPONENTS "")    # 1
+set(__TARGETS_IDF_MANAGED_COMPONENTS "")        # 1
-set(__TARGETS_PROJECT_EXTRA_COMPONENTS "")      # 2
+set(__TARGETS_PROJECT_MANAGED_COMPONENTS "")    # 2
-set(__TARGETS_PROJECT_COMPONENTS "")            # 3
+set(__TARGETS_PROJECT_EXTRA_COMPONENTS "")      # 3
+set(__TARGETS_PROJECT_COMPONENTS "")            # 4
 
 foreach(__component_target ${__component_targets})
     __component_get_property(__component_source ${__component_target} COMPONENT_SOURCE)

tools/cmakev2/compat.cmake

@@ -945,6 +945,7 @@ endmacro()
         Source of the component. One of:
 
         * ``idf_components``
+        * ``idf_managed_components``
         * ``project_managed_components``
         * ``project_extra_components``
         * ``project_components``

tools/cmakev2/component.cmake

@@ -509,10 +509,12 @@ function(__get_component_priority)
     endif()
 
     if("${ARG_SOURCE}" STREQUAL "project_components")
-        set(priority 3)
+        set(priority 4)
     elseif("${ARG_SOURCE}" STREQUAL "project_extra_components")
-        set(priority 2)
+        set(priority 3)
     elseif("${ARG_SOURCE}" STREQUAL "project_managed_components")
+        set(priority 2)
+    elseif("${ARG_SOURCE}" STREQUAL "idf_managed_components")
         set(priority 1)
     elseif("${ARG_SOURCE}" STREQUAL "idf_components")
         set(priority 0)

tools/cmakev2/manager.cmake

@@ -25,9 +25,9 @@ function(__init_component_manager)
     endif()
 
     # Set IDF_COMPONENT_MANAGER_INTERFACE_VERSION.
-    # Defaults to 5. Allow overriding via env/CMake.
+    # Defaults to 6. Allow overriding via env/CMake.
     __get_default_value(VARIABLE IDF_COMPONENT_MANAGER_INTERFACE_VERSION
-                        DEFAULT 5
+                        DEFAULT 6
                         OUTPUT cmgr_iface)
     idf_build_set_property(IDF_COMPONENT_MANAGER_INTERFACE_VERSION ${cmgr_iface})
 
@@ -38,6 +38,17 @@ function(__init_component_manager)
     idf_build_set_property(DEPENDENCIES_LOCK "${deps_lock_file}")
 endfunction()
 
+function(__component_manager_get_command var)
+    idf_build_get_property(python PYTHON)
+    if(DEFINED ENV{IDF_COMPONENT_WRAPPER} AND NOT "$ENV{IDF_COMPONENT_WRAPPER}" STREQUAL "")
+        set(component_manager_cmd
+            "${python}" "$ENV{IDF_COMPONENT_WRAPPER}" "idf_component_manager.prepare_components")
+    else()
+        set(component_manager_cmd "${python}" "-m" "idf_component_manager.prepare_components")
+    endif()
+    set(${var} ${component_manager_cmd} PARENT_SCOPE)
+endfunction()
+
 #[[
     __fetch_components_from_registry()
 
@@ -135,14 +146,12 @@ function(__download_managed_component)
         idf_die("RESULT option is required")
     endif()
 
-    idf_build_get_property(python PYTHON)
+    __component_manager_get_command(component_manager_cmd)
     idf_build_get_property(project_dir PROJECT_DIR)
     idf_build_get_property(component_manager_interface_version IDF_COMPONENT_MANAGER_INTERFACE_VERSION)
     idf_build_get_property(dependencies_lock_file DEPENDENCIES_LOCK)
     # Invoke the component manager
-    execute_process(COMMAND ${python}
+    execute_process(COMMAND ${component_manager_cmd}
-        "-m"
-        "idf_component_manager.prepare_components"
         "--project_dir=${project_dir}"
         "--lock_path=${dependencies_lock_file}"
         "--interface_version=${component_manager_interface_version}"
@@ -298,7 +307,6 @@ function(__inject_requirements_for_component_from_manager component_name)
 
     idf_dbg("Injecting requirements for component '${component_name}' from the component manager")
 
-    idf_build_get_property(python PYTHON)
     idf_build_get_property(project_dir PROJECT_DIR)
     idf_build_get_property(build_dir BUILD_DIR)
     idf_build_get_property(dependencies_lock_file DEPENDENCIES_LOCK)
@@ -355,9 +363,8 @@ function(__inject_requirements_for_component_from_manager component_name)
     endif()
 
     # Call component manager to inject requirements
-    execute_process(COMMAND ${python}
+    __component_manager_get_command(component_manager_cmd)
-        "-m"
+    execute_process(COMMAND ${component_manager_cmd}
-        "idf_component_manager.prepare_components"
         "--project_dir=${project_dir}"
         "--lock_path=${dependencies_lock_file}"
         "--interface_version=${component_manager_interface_version}"

tools/idf.py

@@ -53,6 +53,8 @@ try:
 
     if os.getenv('IDF_COMPONENT_MANAGER') != '0':
         from idf_component_manager import idf_extensions
+        from idf_component_tools.errors import FatalError as ComponentManagerFatalError
+        from idf_component_tools.root_managed_components import RootManagedComponentsStateManager
 except ImportError as e:
     print(
         (
@@ -1104,20 +1106,43 @@ def init_cli(verbose_output: list | None = None) -> Any:
 
         return result
 
-    def _resolve_idf_managed_lock_path() -> str | None:
+    def _get_trusted_names_from_root_state(state_path: str) -> set[str]:
-        """Return path to dependencies.lock for IDF-managed components, or None if unavailable."""
+        """Return component names from the trusted root-managed inventory."""
+        if state_path in _trusted_names_cache:
+            return _trusted_names_cache[state_path]
+        result: set[str] = set()
+        _trusted_names_cache[state_path] = result
+
+        if not os.path.isfile(state_path) or os.getenv('IDF_COMPONENT_MANAGER') == '0':
+            return result
+
+        try:
+            state = RootManagedComponentsStateManager(state_path).load()
+            result.update(state.components)
+        except (OSError, ComponentManagerFatalError) as e:
+            log.warn(
+                escape(
+                    'Could not verify source of external components. '
+                    f'No extensions (idf_ext.py) from managed components will be loaded. ({e})'
+                )
+            )
+
+        return result
+
+    def _resolve_idf_managed_state_dir() -> str | None:
+        """Return the state directory for IDF-managed components, or None if unavailable."""
         idf_tools_path = os.environ.get('IDF_TOOLS_PATH') or os.path.expanduser(os.path.join('~', '.espressif'))
         ver = idf_version_from_cmake()  # returns e.g. 'v6.1.0', or None on failure
         if not ver:
             return None
         ver_str = ver.lstrip('v')  # '6.1.0'
-        return os.path.join(idf_tools_path, 'root_managed_components', f'idf{ver_str}', 'dependencies.lock')
+        return os.path.join(idf_tools_path, 'root_managed_components', f'idf{ver_str}')
 
     def _is_component_trusted(
         comp_name: str,
         source: str | None,
     ) -> bool:
-        """True iff this component is from a trusted source (IDF, project, or Espressif component from ESP-registry)."""
+        """Return whether a component may provide an idf.py extension."""
         if source in ('idf_components', 'project_components', 'project_extra_components'):
             return True
         if source == 'project_managed_components':
@@ -1125,11 +1150,14 @@ def init_cli(verbose_output: list | None = None) -> Any:
             lock_key = comp_name.replace('__', '/', 1) if '__' in comp_name else comp_name
             return lock_key in _get_trusted_names_from_lock(os.path.join(project_dir, 'dependencies.lock'))
         if source == 'idf_managed_components':
-            lock_path = _resolve_idf_managed_lock_path()
+            state_dir = _resolve_idf_managed_state_dir()
-            if lock_path is None:
+            if state_dir is None:
                 return False
             lock_key = comp_name.replace('__', '/', 1) if '__' in comp_name else comp_name
-            return lock_key in _get_trusted_names_from_lock(lock_path)
+            root_state_path = os.path.join(state_dir, 'root_components.lock')
+            if os.path.isfile(root_state_path):
+                return lock_key in _get_trusted_names_from_root_state(root_state_path)
+            return lock_key in _get_trusted_names_from_lock(os.path.join(state_dir, 'dependencies.lock'))
         return False
 
     def _build_rich_help_command_groups(

tools/idf_extra_components.yml

@@ -1,11 +1,12 @@
 # This file defines extra dependencies for ESP-IDF
 # the dependencies defined here will be downloaded to
 # $IDF_TOOLS_PATH/root_managed_components
-# Each major.minor version of ESP-IDF can have its own subdirectory
+# Each major.minor.patch version of ESP-IDF can have its own subdirectory
-# For example, for ESP-IDF v6.0, the dependencies will be installed to
+# For example, for ESP-IDF v6.2.0, the dependencies will be installed to
-# $IDF_TOOLS_PATH/root_managed_components/idf6.0
+# $IDF_TOOLS_PATH/root_managed_components/idf6.2.0
 
 # The syntax is defined in:
 # https://docs.espressif.com/projects/idf-component-manager/en/latest/reference/manifest_file.html#dependencies
+# Rules and matches are not allowed in this manifest
 
 #dependencies:

tools/test_build_system/test_component_manager.py

@@ -2,15 +2,21 @@
 # SPDX-License-Identifier: Apache-2.0
 import json
 import os.path
+import subprocess
 import textwrap
 from pathlib import Path
 
 import pytest
 from test_build_system_helpers import EXT_IDF_PATH
+from test_build_system_helpers import EnvDict
 from test_build_system_helpers import IdfPyFunc
 from test_build_system_helpers import replace_in_file
 
 
+def _install_root_components(env: EnvDict) -> None:
+    subprocess.run(['compote', 'cooking', 'stock'], check=True, env=env)
+
+
 def test_dependency_lock(idf_py: IdfPyFunc, test_app_copy: Path) -> None:
     replace_in_file(
         test_app_copy / 'CMakeLists.txt',
@@ -202,3 +208,102 @@ class TestOptionalDependencyWithKconfig:
         data = json.load(open(test_app_copy / 'build' / 'project_description.json'))
         assert ['example__cmp'] == data['build_component_info']['foo']['priv_reqs']
         assert ['espressif__mdns'] == data['build_component_info']['foo']['reqs']
+
+
+@pytest.mark.revert_later(['tools/idf_extra_components.yml'])
+class TestIdfRootDependency:
+    @pytest.fixture(autouse=True)
+    def _clean_root_managed(self, clean_root_managed_components: None) -> None:
+        pass
+
+    def test_basic_build(self, idf_py: IdfPyFunc, test_app_copy: Path, default_idf_env: EnvDict) -> None:
+        with open(os.path.join(EXT_IDF_PATH, 'tools', 'idf_extra_components.yml'), 'w') as fw:
+            fw.write(
+                textwrap.dedent("""
+            dependencies:
+              espressif/mdns: "*"
+            """)
+            )
+
+        _install_root_components(default_idf_env)
+
+        replace_in_file(
+            (test_app_copy / 'main' / 'build_test_app.c'),
+            '// placeholder_before_main',
+            '#include "mdns.h"',
+        )
+
+        # Intentional dependency on the cmake-level name of the managed component because
+        # idf_extra_components.yml installs espressif/mdns and we verify REQUIRES pulls
+        # espressif__mdns into the build — not something application code should do.
+        replace_in_file(
+            (test_app_copy / 'main' / 'CMakeLists.txt'),
+            '# placeholder_inside_idf_component_register',
+            'REQUIRES espressif__mdns',
+        )
+
+        idf_py('build')
+
+    def test_build_only_when_required(self, idf_py: IdfPyFunc, test_app_copy: Path, default_idf_env: EnvDict) -> None:
+        with open(os.path.join(EXT_IDF_PATH, 'tools', 'idf_extra_components.yml'), 'w') as fw:
+            fw.write(
+                textwrap.dedent("""
+            dependencies:
+              espressif/mdns: "*"
+              example/cmp: "*"
+            """)
+            )
+
+        _install_root_components(default_idf_env)
+
+        idf_py('reconfigure')
+
+        data = json.load(open(test_app_copy / 'build' / 'project_description.json'))
+        assert 'espressif__mdns' not in data['build_components']
+        assert 'example__cmp' not in data['build_components']
+
+        # Intentional dependency on the cmake-level name of the managed component because
+        # we assert espressif__mdns enters build_components only after REQUIRES — not
+        # something application code should do.
+        replace_in_file(
+            (test_app_copy / 'main' / 'CMakeLists.txt'),
+            '# placeholder_inside_idf_component_register',
+            'REQUIRES espressif__mdns',
+        )
+
+        idf_py('reconfigure')
+        data = json.load(open(test_app_copy / 'build' / 'project_description.json'))
+        assert 'espressif__mdns' in data['build_components']
+        assert 'example__cmp' not in data['build_components']
+
+    def test_cleanup_unused(self, idf_py: IdfPyFunc, test_app_copy: Path, default_idf_env: EnvDict) -> None:
+        with open(os.path.join(EXT_IDF_PATH, 'tools', 'idf_extra_components.yml'), 'w') as fw:
+            fw.write(
+                textwrap.dedent("""
+            dependencies:
+              espressif/mdns: "*"
+            """)
+            )
+
+        _install_root_components(default_idf_env)
+
+        idf_py('reconfigure')
+        data = json.load(open(test_app_copy / 'build' / 'project_description.json'))
+        assert 'espressif__mdns' in data['all_component_info']
+
+        with open(os.path.join(EXT_IDF_PATH, 'tools', 'idf_extra_components.yml'), 'w') as fw:
+            fw.write(
+                textwrap.dedent("""
+            dependencies:
+              espressif/led_strip: "*"
+              example/cmp: "*"
+            """)
+            )
+
+        _install_root_components(default_idf_env)
+
+        idf_py('reconfigure')
+        data = json.load(open(test_app_copy / 'build' / 'project_description.json'))
+        assert 'espressif__led_strip' in data['all_component_info']
+        assert 'example__cmp' in data['all_component_info']
+        assert 'espressif__mdns' not in data['all_component_info']