fix(tools): seed IDF_VERSION before idf.py parses dependencies.lock

Any idf.py invocation could hang indefinitely with no output while
spawning an unbounded chain of "idf.py --version" subprocesses,
eventually exhausting system memory.

During init_cli(), idf.py parses the project's dependencies.lock to
vet trusted component-provided idf_ext.py extensions. If the lock
contains a component whose manifest has an "if: idf_version" clause,
evaluating it calls idf-component-manager's _get_idf_version(). Outside
a CMake build the IDF_VERSION environment variable is not set, so that
function falls back to running "idf.py --version" as a subprocess,
which re-enters init_cli() and recurses without bound.

During a normal CMake build the component manager runs as a subprocess
that already has IDF_VERSION in its environment (see build/config.env),
so the fallback is never reached. The recursion happens only because
idf.py runs component-manager code in-process during its own CLI
startup, outside that context.

Seed IDF_VERSION into os.environ early in init_cli(), before any
dependencies.lock parsing, using the subprocess-free
idf_version_from_cmake() helper. This gives in-process component-manager
code the same IDF_VERSION a CMake build would provide.
This commit is contained in:
Roland Dobai
2026-05-14 13:13:20 +02:00
parent 0f4a3e8c2e
commit 490452a2a2
2 changed files with 201 additions and 109 deletions

View File

@@ -44,6 +44,7 @@ try:
from idf_py_actions.tools import SHELL_COMPLETE_RUN
from idf_py_actions.tools import SHELL_COMPLETE_VAR
from idf_py_actions.tools import PropertyDict
from idf_py_actions.tools import _idf_version_from_cmake
from idf_py_actions.tools import debug_print_idf_version
from idf_py_actions.tools import get_target
from idf_py_actions.tools import merge_action_lists
@@ -725,6 +726,20 @@ def init_cli(verbose_output: Optional[List] = None) -> Any:
# Set `complete_var` to not existing environment variable name to prevent early cmd completion
project_dir = parse_project_dir(standalone_mode=False, complete_var='_IDF.PY_COMPLETE_NOT_EXISTING')
# Ensure IDF_VERSION is available for in-process component-manager code
# (e.g. dependencies.lock `if: idf_version` clauses). Outside a CMake build
# this env var is unset; without it idf-component-manager falls back to
# spawning `idf.py --version`, which re-enters here -> infinite recursion.
if 'IDF_VERSION' not in os.environ:
# Best-effort: if idf_version_from_cmake() returns None (corrupt/missing
# version.cmake) IDF_VERSION stays unset and the recursion guard does not apply.
idf_ver = _idf_version_from_cmake() # 'vX.Y.Z' or None; regex parse, no subprocess
if idf_ver:
# Strip the leading 'v' to match the value a CMake build provides
# (see tools/cmake/version.cmake); component-manager code consumes
# this env var verbatim and cannot parse a 'v' prefix.
os.environ['IDF_VERSION'] = idf_ver.lstrip('v')
all_actions: Dict = {}
# Load extensions from components dir
idf_py_extensions_path = os.path.join(os.environ['IDF_PATH'], 'tools', 'idf_py_actions')

View File

@@ -1,6 +1,6 @@
#!/usr/bin/env python
#
# SPDX-FileCopyrightText: 2019-2025 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2019-2026 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import json
import os
@@ -9,9 +9,9 @@ import subprocess
import sys
from typing import Any
from typing import List
from unittest import TestCase
from unittest import main
from unittest import mock
from unittest import TestCase
import jsonschema
@@ -37,10 +37,13 @@ class TestWithoutExtensions(TestCase):
@classmethod
def setUpClass(cls):
# Disable the component manager and extra extensions for these tests
cls.env_patcher = mock.patch.dict(os.environ, {
'IDF_COMPONENT_MANAGER': '0',
'IDF_EXTRA_ACTIONS_PATH': '',
})
cls.env_patcher = mock.patch.dict(
os.environ,
{
'IDF_COMPONENT_MANAGER': '0',
'IDF_EXTRA_ACTIONS_PATH': '',
},
)
cls.env_patcher.start()
super().setUpClass()
@@ -51,8 +54,9 @@ class TestExtensions(TestWithoutExtensions):
try:
os.symlink(extension_path, link_path)
os.environ['IDF_EXTRA_ACTIONS_PATH'] = os.path.join(current_dir, 'extra_path')
output = subprocess.check_output([sys.executable, idf_py_path, '--help'],
env=os.environ).decode('utf-8', 'ignore')
output = subprocess.check_output([sys.executable, idf_py_path, '--help'], env=os.environ).decode(
'utf-8', 'ignore'
)
self.assertIn('--test-extension-option', output)
self.assertIn('test_subcommand', output)
@@ -67,7 +71,8 @@ class TestExtensions(TestWithoutExtensions):
os.environ['IDF_EXTRA_ACTIONS_PATH'] = ';'.join([os.path.join(current_dir, 'extra_path')])
output = subprocess.check_output(
[sys.executable, idf_py_path, '--some-extension-option=awesome', 'test_subcommand', 'extra_subcommand'],
env=os.environ).decode('utf-8', 'ignore')
env=os.environ,
).decode('utf-8', 'ignore')
self.assertIn('!!! From some global callback: awesome', output)
self.assertIn('!!! From some subcommand', output)
self.assertIn('!!! From test global callback: test', output)
@@ -79,8 +84,9 @@ class TestExtensions(TestWithoutExtensions):
try:
os.symlink(extension_path, link_path)
os.environ['IDF_EXTRA_ACTIONS_PATH'] = ';'.join([os.path.join(current_dir, 'extra_path')])
output = subprocess.check_output([sys.executable, idf_py_path, '--help'],
env=os.environ).decode('utf-8', 'ignore')
output = subprocess.check_output([sys.executable, idf_py_path, '--help'], env=os.environ).decode(
'utf-8', 'ignore'
)
self.assertIn('test_subcommand', output)
self.assertNotIn('hidden_one', output)
@@ -127,7 +133,8 @@ class TestDependencyManagement(TestWithoutExtensions):
sys.stderr = sys.__stderr__
self.assertIn(
'WARNING: Commands "all", "clean" are found in the list of commands more than once.',
capturedOutput.getvalue())
capturedOutput.getvalue(),
)
sys.stderr = capturedOutput
idf.init_cli()(
@@ -136,7 +143,28 @@ class TestDependencyManagement(TestWithoutExtensions):
)
sys.stderr = sys.__stderr__
self.assertIn(
'WARNING: Command "clean" is found in the list of commands more than once.', capturedOutput.getvalue())
'WARNING: Command "clean" is found in the list of commands more than once.', capturedOutput.getvalue()
)
class TestIdfVersionSeeding(TestWithoutExtensions):
def test_idf_version_seeded_when_unset(self):
from idf_py_actions.tools import _idf_version_from_cmake
with mock.patch.dict(os.environ):
os.environ.pop('IDF_VERSION', None)
idf.init_cli()(args=['--dry-run', 'build'], standalone_mode=False)
self.assertIn('IDF_VERSION', os.environ)
expected = _idf_version_from_cmake()
self.assertIsNotNone(expected)
expected_stripped = expected.lstrip('v')
self.assertEqual(os.environ['IDF_VERSION'], expected_stripped)
self.assertFalse(os.environ['IDF_VERSION'].startswith('v'))
def test_idf_version_not_overwritten_when_set(self):
with mock.patch.dict(os.environ, {'IDF_VERSION': '0.0.0-sentinel'}):
idf.init_cli()(args=['--dry-run', 'build'], standalone_mode=False)
self.assertEqual(os.environ['IDF_VERSION'], '0.0.0-sentinel')
class TestVerboseFlag(TestWithoutExtensions):
@@ -148,7 +176,9 @@ class TestVerboseFlag(TestWithoutExtensions):
'-C%s' % current_dir,
'-v',
'test-verbose',
], env=os.environ).decode('utf-8', 'ignore')
],
env=os.environ,
).decode('utf-8', 'ignore')
self.assertIn('Verbose mode on', output)
@@ -159,7 +189,9 @@ class TestVerboseFlag(TestWithoutExtensions):
idf_py_path,
'-C%s' % current_dir,
'test-verbose',
], env=os.environ).decode('utf-8', 'ignore')
],
env=os.environ,
).decode('utf-8', 'ignore')
self.assertIn('Output from test-verbose', output)
self.assertNotIn('Verbose mode on', output)
@@ -188,7 +220,8 @@ class TestDeprecations(TestWithoutExtensions):
def test_exit_with_error_for_subcommand(self):
try:
subprocess.check_output(
[sys.executable, idf_py_path, '-C%s' % current_dir, 'test-2'], env=os.environ, stderr=subprocess.STDOUT)
[sys.executable, idf_py_path, '-C%s' % current_dir, 'test-2'], env=os.environ, stderr=subprocess.STDOUT
)
except subprocess.CalledProcessError as e:
self.assertIn('Error: Command "test-2" is deprecated and was removed.', e.output.decode('utf-8', 'ignore'))
@@ -197,11 +230,13 @@ class TestDeprecations(TestWithoutExtensions):
subprocess.check_output(
[sys.executable, idf_py_path, '-C%s' % current_dir, '--test-5=asdf'],
env=os.environ,
stderr=subprocess.STDOUT)
stderr=subprocess.STDOUT,
)
except subprocess.CalledProcessError as e:
self.assertIn(
'Error: Option "test_5" is deprecated since v2.0 and was removed in v3.0.',
e.output.decode('utf-8', 'ignore'))
e.output.decode('utf-8', 'ignore'),
)
def test_deprecation_messages(self):
output = subprocess.check_output(
@@ -220,16 +255,21 @@ class TestDeprecations(TestWithoutExtensions):
'test-1',
],
env=os.environ,
stderr=subprocess.STDOUT).decode('utf-8', 'ignore')
stderr=subprocess.STDOUT,
).decode('utf-8', 'ignore')
self.assertIn('Warning: Option "test_sub_1" is deprecated and will be removed in future versions.', output)
self.assertIn(
'Warning: Command "test-1" is deprecated and will be removed in future versions. '
'Please use alternative command.', output)
'Please use alternative command.',
output,
)
self.assertIn('Warning: Option "test_1" is deprecated and will be removed in future versions.', output)
self.assertIn(
'Warning: Option "test_2" is deprecated and will be removed in future versions. '
'Please update your parameters.', output)
'Please update your parameters.',
output,
)
self.assertIn('Warning: Option "test_3" is deprecated and will be removed in future versions.', output)
self.assertNotIn('"test-0" is deprecated', output)
self.assertNotIn('"test_0" is deprecated', output)
@@ -244,10 +284,7 @@ class TestHelpOutput(TestWithoutExtensions):
idf_path = env.get('IDF_PATH')
if idf_path is None:
raise ValueError('Empty IDF_PATH')
idf_py_cmd = [
python,
os.path.join(idf_path, 'tools', 'idf.py')
]
idf_py_cmd = [python, os.path.join(idf_path, 'tools', 'idf.py')]
commands = idf_py_cmd + commands
output_file = 'idf_py_help_output.json'
with open(output_file, 'w') as outfile:
@@ -270,7 +307,8 @@ class TestFileArgumentExpansion(TestCase):
output = subprocess.check_output(
[sys.executable, idf_py_path, '--version', '@file_args_expansion_inputs/args_a'],
env=os.environ,
stderr=subprocess.STDOUT).decode('utf-8', 'ignore')
stderr=subprocess.STDOUT,
).decode('utf-8', 'ignore')
self.assertIn('Running: idf.py --version DAAA DBBB', output)
except subprocess.CalledProcessError as e:
self.fail(f'Process should have exited normally, but it exited with a return code of {e.returncode}')
@@ -279,9 +317,16 @@ class TestFileArgumentExpansion(TestCase):
"""Test multiple @filename arguments"""
try:
output = subprocess.check_output(
[sys.executable, idf_py_path, '--version', '@file_args_expansion_inputs/args_a', '@file_args_expansion_inputs/args_b'],
[
sys.executable,
idf_py_path,
'--version',
'@file_args_expansion_inputs/args_a',
'@file_args_expansion_inputs/args_b',
],
env=os.environ,
stderr=subprocess.STDOUT).decode('utf-8', 'ignore')
stderr=subprocess.STDOUT,
).decode('utf-8', 'ignore')
self.assertIn('Running: idf.py --version DAAA DBBB DCCC DDDD', output)
except subprocess.CalledProcessError as e:
self.fail(f'Process should have exited normally, but it exited with a return code of {e.returncode}')
@@ -292,7 +337,8 @@ class TestFileArgumentExpansion(TestCase):
output = subprocess.check_output(
[sys.executable, idf_py_path, '--version', '@file_args_expansion_inputs/args_recursive'],
env=os.environ,
stderr=subprocess.STDOUT).decode('utf-8', 'ignore')
stderr=subprocess.STDOUT,
).decode('utf-8', 'ignore')
self.assertIn('Running: idf.py --version DAAA DBBB DEEE DFFF', output)
except subprocess.CalledProcessError as e:
self.fail(f'Process should have exited normally, but it exited with a return code of {e.returncode}')
@@ -303,7 +349,8 @@ class TestFileArgumentExpansion(TestCase):
subprocess.check_output(
[sys.executable, idf_py_path, '--version', '@file_args_expansion_inputs/args_circular_a'],
env=os.environ,
stderr=subprocess.STDOUT).decode('utf-8', 'ignore')
stderr=subprocess.STDOUT,
).decode('utf-8', 'ignore')
self.assertIn('Circular dependency in file argument expansion', cm.exception.output.decode('utf-8', 'ignore'))
def test_missing_file(self):
@@ -312,8 +359,11 @@ class TestFileArgumentExpansion(TestCase):
subprocess.check_output(
[sys.executable, idf_py_path, '--version', '@args_non_existent'],
env=os.environ,
stderr=subprocess.STDOUT).decode('utf-8', 'ignore')
self.assertIn('(expansion of @args_non_existent) could not be opened', cm.exception.output.decode('utf-8', 'ignore'))
stderr=subprocess.STDOUT,
).decode('utf-8', 'ignore')
self.assertIn(
'(expansion of @args_non_existent) could not be opened', cm.exception.output.decode('utf-8', 'ignore')
)
class TestWrapperCommands(TestCase):
@@ -325,10 +375,9 @@ class TestWrapperCommands(TestCase):
def call_command(self, command: List[str]) -> str:
try:
output = subprocess.check_output(
command,
env=os.environ,
stderr=subprocess.STDOUT).decode('utf-8', 'ignore')
output = subprocess.check_output(command, env=os.environ, stderr=subprocess.STDOUT).decode(
'utf-8', 'ignore'
)
return output
except subprocess.CalledProcessError as e:
self.fail(f'Process should have exited normally, but it exited with a return code of {e.returncode}')
@@ -343,7 +392,8 @@ class TestWrapperCommands(TestCase):
class TestEFuseCommands(TestWrapperCommands):
"""
Test if wrapper commands for espefuse.py are working as expected.
The goal is NOT to test the functionality of espefuse.py, but to test if the wrapper commands are working as expected.
The goal is NOT to test the functionality of espefuse.py,
but to test if the wrapper commands are working as expected.
"""
def test_efuse_summary(self):
@@ -351,17 +401,17 @@ class TestEFuseCommands(TestWrapperCommands):
output = self.call_command(summary_command)
self.assertIn('EFUSE_NAME (Block) Description = [Meaningful Value] [Readable/Writeable] (Hex Value)', output)
output = self.call_command(summary_command + ['--format','summary'])
output = self.call_command(summary_command + ['--format', 'summary'])
self.assertIn('00:00:00:00:00:00', output)
self.assertIn('MAC address', output)
output = self.call_command(summary_command + ['--format','value-only', 'WR_DIS'])
output = self.call_command(summary_command + ['--format', 'value-only', 'WR_DIS'])
self.assertIn('0', output)
def test_efuse_burn(self):
burn_command = [sys.executable, idf_py_path, 'efuse-burn', '--virt', '--do-not-confirm']
output = self.call_command(burn_command + ['WR_DIS', '1'])
self.assertIn('\'WR_DIS\' (Efuse write disable mask) 0x0000 -> 0x0001', output)
self.assertIn("'WR_DIS' (Efuse write disable mask) 0x0000 -> 0x0001", output)
self.assertIn('Successful', output)
output = self.call_command(burn_command + ['WR_DIS', '1', 'RD_DIS', '1'])
@@ -371,9 +421,14 @@ class TestEFuseCommands(TestWrapperCommands):
def test_efuse_burn_key(self):
key_name = 'efuse_test_key.bin'
subprocess.run([sys.executable, idf_py_path, 'secure-generate-flash-encryption-key', os.path.join(current_dir, key_name)], stdout=subprocess.DEVNULL)
subprocess.run(
[sys.executable, idf_py_path, 'secure-generate-flash-encryption-key', os.path.join(current_dir, key_name)],
stdout=subprocess.DEVNULL,
)
burn_key_command = [sys.executable, idf_py_path, 'efuse-burn-key', '--virt', '--do-not-confirm']
output = self.call_command(burn_key_command + ['--show-sensitive-info', 'secure_boot_v1', os.path.join(current_dir, key_name)])
output = self.call_command(
burn_key_command + ['--show-sensitive-info', 'secure_boot_v1', os.path.join(current_dir, key_name)]
)
self.assertIn('Burn keys to blocks:', output)
self.assertIn('Successful', output)
@@ -401,8 +456,10 @@ class TestEFuseCommands(TestWrapperCommands):
class TestSecureCommands(TestWrapperCommands):
"""
Test if wrapper commands for espsecure.py are working as expected.
The goal is NOT to test the functionality of espsecure.py, but to test if the wrapper commands are working as expected.
The goal is NOT to test the functionality of espsecure.py,
but to test if the wrapper commands are working as expected.
"""
@classmethod
def setUpClass(cls):
super().setUpClass()
@@ -412,106 +469,125 @@ class TestSecureCommands(TestWrapperCommands):
cls.nvs_partition_key = 'nvs_partition_key.bin'
def secure_generate_flash_encryption_key(self):
generate_key_command = [sys.executable, idf_py_path, 'secure-generate-flash-encryption-key', self.flash_encryption_key]
generate_key_command = [
sys.executable,
idf_py_path,
'secure-generate-flash-encryption-key',
self.flash_encryption_key,
]
output = self.call_command(generate_key_command)
self.assertIn(f'Writing 256 random bits to key file {self.flash_encryption_key}', output)
def secure_encrypt_flash_data(self):
self.secure_generate_flash_encryption_key()
encrypt_command = [sys.executable,
idf_py_path,
'secure-encrypt-flash-data',
'--aes-xts',
'--keyfile',
f'../{self.flash_encryption_key}',
'--address',
'0x1000',
'--output',
'bootloader-enc.bin',
'bootloader/bootloader.bin']
encrypt_command = [
sys.executable,
idf_py_path,
'secure-encrypt-flash-data',
'--aes-xts',
'--keyfile',
f'../{self.flash_encryption_key}',
'--address',
'0x1000',
'--output',
'bootloader-enc.bin',
'bootloader/bootloader.bin',
]
output = self.call_command(encrypt_command)
self.assertIn('Using 256-bit key', output)
self.assertIn('Done', output)
def test_secure_decrypt_flash_data(self):
self.secure_encrypt_flash_data()
decrypt_command = [sys.executable,
idf_py_path,
'secure-decrypt-flash-data',
'--aes-xts',
'--keyfile',
f'../{self.flash_encryption_key}',
'--address',
'0x1000',
'--output',
'bootloader-dec.bin',
'bootloader-enc.bin']
decrypt_command = [
sys.executable,
idf_py_path,
'secure-decrypt-flash-data',
'--aes-xts',
'--keyfile',
f'../{self.flash_encryption_key}',
'--address',
'0x1000',
'--output',
'bootloader-dec.bin',
'bootloader-enc.bin',
]
output = self.call_command(decrypt_command)
self.assertIn('Using 256-bit key', output)
self.assertIn('Done', output)
def secure_sign_data(self):
self.secure_generate_signing_key()
sign_command = [sys.executable,
idf_py_path,
'secure-sign-data',
'--version',
'2',
'--keyfile',
f'../{self.signing_key}',
'--output',
'bootloader-signed.bin',
'bootloader/bootloader.bin']
sign_command = [
sys.executable,
idf_py_path,
'secure-sign-data',
'--version',
'2',
'--keyfile',
f'../{self.signing_key}',
'--output',
'bootloader-signed.bin',
'bootloader/bootloader.bin',
]
output = self.call_command(sign_command)
self.assertIn('Signed', output)
def secure_verify_signature(self):
self.secure_sign_data()
sign_command = [sys.executable,
idf_py_path,
'secure-verify-signature',
'--version',
'2',
'--keyfile',
f'../{self.signing_key}',
'bootloader-signed.bin']
sign_command = [
sys.executable,
idf_py_path,
'secure-verify-signature',
'--version',
'2',
'--keyfile',
f'../{self.signing_key}',
'bootloader-signed.bin',
]
output = self.call_command(sign_command)
self.assertIn('verification successful', output)
def secure_generate_signing_key(self):
generate_key_command = [sys.executable,
idf_py_path,
'secure-generate-signing-key',
'--version',
'2',
'--scheme',
'rsa3072',
self.signing_key]
generate_key_command = [
sys.executable,
idf_py_path,
'secure-generate-signing-key',
'--version',
'2',
'--scheme',
'rsa3072',
self.signing_key,
]
output = self.call_command(generate_key_command)
self.assertIn(f'RSA 3072 private key in PEM format written to {self.signing_key}', output)
def test_secure_generate_key_digest(self):
self.secure_generate_signing_key()
digest_command = [sys.executable,
idf_py_path,
'secure-generate-key-digest',
'--keyfile',
f'{self.signing_key}',
'--output',
'key_digest.bin']
digest_command = [
sys.executable,
idf_py_path,
'secure-generate-key-digest',
'--keyfile',
f'{self.signing_key}',
'--output',
'key_digest.bin',
]
output = self.call_command(digest_command)
self.assertIn(f'Writing the public key digest of {self.signing_key} to key_digest.bin', output)
def test_secure_generate_nvs_partition_key(self):
generate_key_command = [sys.executable,
idf_py_path,
'secure-generate-nvs-partition-key',
'--keyfile',
f'{self.nvs_partition_key}',
'--encryption-scheme',
'HMAC',
'--hmac-keyfile',
'nvs_partition_key.bin']
generate_key_command = [
sys.executable,
idf_py_path,
'secure-generate-nvs-partition-key',
'--keyfile',
f'{self.nvs_partition_key}',
'--encryption-scheme',
'HMAC',
'--hmac-keyfile',
'nvs_partition_key.bin',
]
output = self.call_command(generate_key_command)
self.assertIn('Created encryption keys:', output)
@@ -519,7 +595,8 @@ class TestSecureCommands(TestWrapperCommands):
class TestMergeBinCommands(TestWrapperCommands):
"""
Test if merge-bin command is invoked as expected.
This test is not testing the functionality of esptool.py merge_bin command, but the invocation of the command from idf.py.
This test is not testing the functionality of esptool.py merge_bin command,
but the invocation of the command from idf.py.
"""
def test_merge_bin(self):