diff --git a/tools/idf.py b/tools/idf.py index 7f1c2ee2ad2..907d85582cc 100755 --- a/tools/idf.py +++ b/tools/idf.py @@ -44,6 +44,7 @@ try: from idf_py_actions.tools import SHELL_COMPLETE_RUN from idf_py_actions.tools import SHELL_COMPLETE_VAR from idf_py_actions.tools import PropertyDict + from idf_py_actions.tools import _idf_version_from_cmake from idf_py_actions.tools import debug_print_idf_version from idf_py_actions.tools import get_target from idf_py_actions.tools import merge_action_lists @@ -725,6 +726,20 @@ def init_cli(verbose_output: Optional[List] = None) -> Any: # Set `complete_var` to not existing environment variable name to prevent early cmd completion project_dir = parse_project_dir(standalone_mode=False, complete_var='_IDF.PY_COMPLETE_NOT_EXISTING') + # Ensure IDF_VERSION is available for in-process component-manager code + # (e.g. dependencies.lock `if: idf_version` clauses). Outside a CMake build + # this env var is unset; without it idf-component-manager falls back to + # spawning `idf.py --version`, which re-enters here -> infinite recursion. + if 'IDF_VERSION' not in os.environ: + # Best-effort: if idf_version_from_cmake() returns None (corrupt/missing + # version.cmake) IDF_VERSION stays unset and the recursion guard does not apply. + idf_ver = _idf_version_from_cmake() # 'vX.Y.Z' or None; regex parse, no subprocess + if idf_ver: + # Strip the leading 'v' to match the value a CMake build provides + # (see tools/cmake/version.cmake); component-manager code consumes + # this env var verbatim and cannot parse a 'v' prefix. + os.environ['IDF_VERSION'] = idf_ver.lstrip('v') + all_actions: Dict = {} # Load extensions from components dir idf_py_extensions_path = os.path.join(os.environ['IDF_PATH'], 'tools', 'idf_py_actions') diff --git a/tools/test_idf_py/test_idf_py.py b/tools/test_idf_py/test_idf_py.py index b715a789453..60456386c53 100755 --- a/tools/test_idf_py/test_idf_py.py +++ b/tools/test_idf_py/test_idf_py.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2019-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2019-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import json import os @@ -9,9 +9,9 @@ import subprocess import sys from typing import Any from typing import List +from unittest import TestCase from unittest import main from unittest import mock -from unittest import TestCase import jsonschema @@ -37,10 +37,13 @@ class TestWithoutExtensions(TestCase): @classmethod def setUpClass(cls): # Disable the component manager and extra extensions for these tests - cls.env_patcher = mock.patch.dict(os.environ, { - 'IDF_COMPONENT_MANAGER': '0', - 'IDF_EXTRA_ACTIONS_PATH': '', - }) + cls.env_patcher = mock.patch.dict( + os.environ, + { + 'IDF_COMPONENT_MANAGER': '0', + 'IDF_EXTRA_ACTIONS_PATH': '', + }, + ) cls.env_patcher.start() super().setUpClass() @@ -51,8 +54,9 @@ class TestExtensions(TestWithoutExtensions): try: os.symlink(extension_path, link_path) os.environ['IDF_EXTRA_ACTIONS_PATH'] = os.path.join(current_dir, 'extra_path') - output = subprocess.check_output([sys.executable, idf_py_path, '--help'], - env=os.environ).decode('utf-8', 'ignore') + output = subprocess.check_output([sys.executable, idf_py_path, '--help'], env=os.environ).decode( + 'utf-8', 'ignore' + ) self.assertIn('--test-extension-option', output) self.assertIn('test_subcommand', output) @@ -67,7 +71,8 @@ class TestExtensions(TestWithoutExtensions): os.environ['IDF_EXTRA_ACTIONS_PATH'] = ';'.join([os.path.join(current_dir, 'extra_path')]) output = subprocess.check_output( [sys.executable, idf_py_path, '--some-extension-option=awesome', 'test_subcommand', 'extra_subcommand'], - env=os.environ).decode('utf-8', 'ignore') + env=os.environ, + ).decode('utf-8', 'ignore') self.assertIn('!!! From some global callback: awesome', output) self.assertIn('!!! From some subcommand', output) self.assertIn('!!! From test global callback: test', output) @@ -79,8 +84,9 @@ class TestExtensions(TestWithoutExtensions): try: os.symlink(extension_path, link_path) os.environ['IDF_EXTRA_ACTIONS_PATH'] = ';'.join([os.path.join(current_dir, 'extra_path')]) - output = subprocess.check_output([sys.executable, idf_py_path, '--help'], - env=os.environ).decode('utf-8', 'ignore') + output = subprocess.check_output([sys.executable, idf_py_path, '--help'], env=os.environ).decode( + 'utf-8', 'ignore' + ) self.assertIn('test_subcommand', output) self.assertNotIn('hidden_one', output) @@ -127,7 +133,8 @@ class TestDependencyManagement(TestWithoutExtensions): sys.stderr = sys.__stderr__ self.assertIn( 'WARNING: Commands "all", "clean" are found in the list of commands more than once.', - capturedOutput.getvalue()) + capturedOutput.getvalue(), + ) sys.stderr = capturedOutput idf.init_cli()( @@ -136,7 +143,28 @@ class TestDependencyManagement(TestWithoutExtensions): ) sys.stderr = sys.__stderr__ self.assertIn( - 'WARNING: Command "clean" is found in the list of commands more than once.', capturedOutput.getvalue()) + 'WARNING: Command "clean" is found in the list of commands more than once.', capturedOutput.getvalue() + ) + + +class TestIdfVersionSeeding(TestWithoutExtensions): + def test_idf_version_seeded_when_unset(self): + from idf_py_actions.tools import _idf_version_from_cmake + + with mock.patch.dict(os.environ): + os.environ.pop('IDF_VERSION', None) + idf.init_cli()(args=['--dry-run', 'build'], standalone_mode=False) + self.assertIn('IDF_VERSION', os.environ) + expected = _idf_version_from_cmake() + self.assertIsNotNone(expected) + expected_stripped = expected.lstrip('v') + self.assertEqual(os.environ['IDF_VERSION'], expected_stripped) + self.assertFalse(os.environ['IDF_VERSION'].startswith('v')) + + def test_idf_version_not_overwritten_when_set(self): + with mock.patch.dict(os.environ, {'IDF_VERSION': '0.0.0-sentinel'}): + idf.init_cli()(args=['--dry-run', 'build'], standalone_mode=False) + self.assertEqual(os.environ['IDF_VERSION'], '0.0.0-sentinel') class TestVerboseFlag(TestWithoutExtensions): @@ -148,7 +176,9 @@ class TestVerboseFlag(TestWithoutExtensions): '-C%s' % current_dir, '-v', 'test-verbose', - ], env=os.environ).decode('utf-8', 'ignore') + ], + env=os.environ, + ).decode('utf-8', 'ignore') self.assertIn('Verbose mode on', output) @@ -159,7 +189,9 @@ class TestVerboseFlag(TestWithoutExtensions): idf_py_path, '-C%s' % current_dir, 'test-verbose', - ], env=os.environ).decode('utf-8', 'ignore') + ], + env=os.environ, + ).decode('utf-8', 'ignore') self.assertIn('Output from test-verbose', output) self.assertNotIn('Verbose mode on', output) @@ -188,7 +220,8 @@ class TestDeprecations(TestWithoutExtensions): def test_exit_with_error_for_subcommand(self): try: subprocess.check_output( - [sys.executable, idf_py_path, '-C%s' % current_dir, 'test-2'], env=os.environ, stderr=subprocess.STDOUT) + [sys.executable, idf_py_path, '-C%s' % current_dir, 'test-2'], env=os.environ, stderr=subprocess.STDOUT + ) except subprocess.CalledProcessError as e: self.assertIn('Error: Command "test-2" is deprecated and was removed.', e.output.decode('utf-8', 'ignore')) @@ -197,11 +230,13 @@ class TestDeprecations(TestWithoutExtensions): subprocess.check_output( [sys.executable, idf_py_path, '-C%s' % current_dir, '--test-5=asdf'], env=os.environ, - stderr=subprocess.STDOUT) + stderr=subprocess.STDOUT, + ) except subprocess.CalledProcessError as e: self.assertIn( 'Error: Option "test_5" is deprecated since v2.0 and was removed in v3.0.', - e.output.decode('utf-8', 'ignore')) + e.output.decode('utf-8', 'ignore'), + ) def test_deprecation_messages(self): output = subprocess.check_output( @@ -220,16 +255,21 @@ class TestDeprecations(TestWithoutExtensions): 'test-1', ], env=os.environ, - stderr=subprocess.STDOUT).decode('utf-8', 'ignore') + stderr=subprocess.STDOUT, + ).decode('utf-8', 'ignore') self.assertIn('Warning: Option "test_sub_1" is deprecated and will be removed in future versions.', output) self.assertIn( 'Warning: Command "test-1" is deprecated and will be removed in future versions. ' - 'Please use alternative command.', output) + 'Please use alternative command.', + output, + ) self.assertIn('Warning: Option "test_1" is deprecated and will be removed in future versions.', output) self.assertIn( 'Warning: Option "test_2" is deprecated and will be removed in future versions. ' - 'Please update your parameters.', output) + 'Please update your parameters.', + output, + ) self.assertIn('Warning: Option "test_3" is deprecated and will be removed in future versions.', output) self.assertNotIn('"test-0" is deprecated', output) self.assertNotIn('"test_0" is deprecated', output) @@ -244,10 +284,7 @@ class TestHelpOutput(TestWithoutExtensions): idf_path = env.get('IDF_PATH') if idf_path is None: raise ValueError('Empty IDF_PATH') - idf_py_cmd = [ - python, - os.path.join(idf_path, 'tools', 'idf.py') - ] + idf_py_cmd = [python, os.path.join(idf_path, 'tools', 'idf.py')] commands = idf_py_cmd + commands output_file = 'idf_py_help_output.json' with open(output_file, 'w') as outfile: @@ -270,7 +307,8 @@ class TestFileArgumentExpansion(TestCase): output = subprocess.check_output( [sys.executable, idf_py_path, '--version', '@file_args_expansion_inputs/args_a'], env=os.environ, - stderr=subprocess.STDOUT).decode('utf-8', 'ignore') + stderr=subprocess.STDOUT, + ).decode('utf-8', 'ignore') self.assertIn('Running: idf.py --version DAAA DBBB', output) except subprocess.CalledProcessError as e: self.fail(f'Process should have exited normally, but it exited with a return code of {e.returncode}') @@ -279,9 +317,16 @@ class TestFileArgumentExpansion(TestCase): """Test multiple @filename arguments""" try: output = subprocess.check_output( - [sys.executable, idf_py_path, '--version', '@file_args_expansion_inputs/args_a', '@file_args_expansion_inputs/args_b'], + [ + sys.executable, + idf_py_path, + '--version', + '@file_args_expansion_inputs/args_a', + '@file_args_expansion_inputs/args_b', + ], env=os.environ, - stderr=subprocess.STDOUT).decode('utf-8', 'ignore') + stderr=subprocess.STDOUT, + ).decode('utf-8', 'ignore') self.assertIn('Running: idf.py --version DAAA DBBB DCCC DDDD', output) except subprocess.CalledProcessError as e: self.fail(f'Process should have exited normally, but it exited with a return code of {e.returncode}') @@ -292,7 +337,8 @@ class TestFileArgumentExpansion(TestCase): output = subprocess.check_output( [sys.executable, idf_py_path, '--version', '@file_args_expansion_inputs/args_recursive'], env=os.environ, - stderr=subprocess.STDOUT).decode('utf-8', 'ignore') + stderr=subprocess.STDOUT, + ).decode('utf-8', 'ignore') self.assertIn('Running: idf.py --version DAAA DBBB DEEE DFFF', output) except subprocess.CalledProcessError as e: self.fail(f'Process should have exited normally, but it exited with a return code of {e.returncode}') @@ -303,7 +349,8 @@ class TestFileArgumentExpansion(TestCase): subprocess.check_output( [sys.executable, idf_py_path, '--version', '@file_args_expansion_inputs/args_circular_a'], env=os.environ, - stderr=subprocess.STDOUT).decode('utf-8', 'ignore') + stderr=subprocess.STDOUT, + ).decode('utf-8', 'ignore') self.assertIn('Circular dependency in file argument expansion', cm.exception.output.decode('utf-8', 'ignore')) def test_missing_file(self): @@ -312,8 +359,11 @@ class TestFileArgumentExpansion(TestCase): subprocess.check_output( [sys.executable, idf_py_path, '--version', '@args_non_existent'], env=os.environ, - stderr=subprocess.STDOUT).decode('utf-8', 'ignore') - self.assertIn('(expansion of @args_non_existent) could not be opened', cm.exception.output.decode('utf-8', 'ignore')) + stderr=subprocess.STDOUT, + ).decode('utf-8', 'ignore') + self.assertIn( + '(expansion of @args_non_existent) could not be opened', cm.exception.output.decode('utf-8', 'ignore') + ) class TestWrapperCommands(TestCase): @@ -325,10 +375,9 @@ class TestWrapperCommands(TestCase): def call_command(self, command: List[str]) -> str: try: - output = subprocess.check_output( - command, - env=os.environ, - stderr=subprocess.STDOUT).decode('utf-8', 'ignore') + output = subprocess.check_output(command, env=os.environ, stderr=subprocess.STDOUT).decode( + 'utf-8', 'ignore' + ) return output except subprocess.CalledProcessError as e: self.fail(f'Process should have exited normally, but it exited with a return code of {e.returncode}') @@ -343,7 +392,8 @@ class TestWrapperCommands(TestCase): class TestEFuseCommands(TestWrapperCommands): """ Test if wrapper commands for espefuse.py are working as expected. - The goal is NOT to test the functionality of espefuse.py, but to test if the wrapper commands are working as expected. + The goal is NOT to test the functionality of espefuse.py, + but to test if the wrapper commands are working as expected. """ def test_efuse_summary(self): @@ -351,17 +401,17 @@ class TestEFuseCommands(TestWrapperCommands): output = self.call_command(summary_command) self.assertIn('EFUSE_NAME (Block) Description = [Meaningful Value] [Readable/Writeable] (Hex Value)', output) - output = self.call_command(summary_command + ['--format','summary']) + output = self.call_command(summary_command + ['--format', 'summary']) self.assertIn('00:00:00:00:00:00', output) self.assertIn('MAC address', output) - output = self.call_command(summary_command + ['--format','value-only', 'WR_DIS']) + output = self.call_command(summary_command + ['--format', 'value-only', 'WR_DIS']) self.assertIn('0', output) def test_efuse_burn(self): burn_command = [sys.executable, idf_py_path, 'efuse-burn', '--virt', '--do-not-confirm'] output = self.call_command(burn_command + ['WR_DIS', '1']) - self.assertIn('\'WR_DIS\' (Efuse write disable mask) 0x0000 -> 0x0001', output) + self.assertIn("'WR_DIS' (Efuse write disable mask) 0x0000 -> 0x0001", output) self.assertIn('Successful', output) output = self.call_command(burn_command + ['WR_DIS', '1', 'RD_DIS', '1']) @@ -371,9 +421,14 @@ class TestEFuseCommands(TestWrapperCommands): def test_efuse_burn_key(self): key_name = 'efuse_test_key.bin' - subprocess.run([sys.executable, idf_py_path, 'secure-generate-flash-encryption-key', os.path.join(current_dir, key_name)], stdout=subprocess.DEVNULL) + subprocess.run( + [sys.executable, idf_py_path, 'secure-generate-flash-encryption-key', os.path.join(current_dir, key_name)], + stdout=subprocess.DEVNULL, + ) burn_key_command = [sys.executable, idf_py_path, 'efuse-burn-key', '--virt', '--do-not-confirm'] - output = self.call_command(burn_key_command + ['--show-sensitive-info', 'secure_boot_v1', os.path.join(current_dir, key_name)]) + output = self.call_command( + burn_key_command + ['--show-sensitive-info', 'secure_boot_v1', os.path.join(current_dir, key_name)] + ) self.assertIn('Burn keys to blocks:', output) self.assertIn('Successful', output) @@ -401,8 +456,10 @@ class TestEFuseCommands(TestWrapperCommands): class TestSecureCommands(TestWrapperCommands): """ Test if wrapper commands for espsecure.py are working as expected. - The goal is NOT to test the functionality of espsecure.py, but to test if the wrapper commands are working as expected. + The goal is NOT to test the functionality of espsecure.py, + but to test if the wrapper commands are working as expected. """ + @classmethod def setUpClass(cls): super().setUpClass() @@ -412,106 +469,125 @@ class TestSecureCommands(TestWrapperCommands): cls.nvs_partition_key = 'nvs_partition_key.bin' def secure_generate_flash_encryption_key(self): - generate_key_command = [sys.executable, idf_py_path, 'secure-generate-flash-encryption-key', self.flash_encryption_key] + generate_key_command = [ + sys.executable, + idf_py_path, + 'secure-generate-flash-encryption-key', + self.flash_encryption_key, + ] output = self.call_command(generate_key_command) self.assertIn(f'Writing 256 random bits to key file {self.flash_encryption_key}', output) def secure_encrypt_flash_data(self): self.secure_generate_flash_encryption_key() - encrypt_command = [sys.executable, - idf_py_path, - 'secure-encrypt-flash-data', - '--aes-xts', - '--keyfile', - f'../{self.flash_encryption_key}', - '--address', - '0x1000', - '--output', - 'bootloader-enc.bin', - 'bootloader/bootloader.bin'] + encrypt_command = [ + sys.executable, + idf_py_path, + 'secure-encrypt-flash-data', + '--aes-xts', + '--keyfile', + f'../{self.flash_encryption_key}', + '--address', + '0x1000', + '--output', + 'bootloader-enc.bin', + 'bootloader/bootloader.bin', + ] output = self.call_command(encrypt_command) self.assertIn('Using 256-bit key', output) self.assertIn('Done', output) def test_secure_decrypt_flash_data(self): self.secure_encrypt_flash_data() - decrypt_command = [sys.executable, - idf_py_path, - 'secure-decrypt-flash-data', - '--aes-xts', - '--keyfile', - f'../{self.flash_encryption_key}', - '--address', - '0x1000', - '--output', - 'bootloader-dec.bin', - 'bootloader-enc.bin'] + decrypt_command = [ + sys.executable, + idf_py_path, + 'secure-decrypt-flash-data', + '--aes-xts', + '--keyfile', + f'../{self.flash_encryption_key}', + '--address', + '0x1000', + '--output', + 'bootloader-dec.bin', + 'bootloader-enc.bin', + ] output = self.call_command(decrypt_command) self.assertIn('Using 256-bit key', output) self.assertIn('Done', output) def secure_sign_data(self): self.secure_generate_signing_key() - sign_command = [sys.executable, - idf_py_path, - 'secure-sign-data', - '--version', - '2', - '--keyfile', - f'../{self.signing_key}', - '--output', - 'bootloader-signed.bin', - 'bootloader/bootloader.bin'] + sign_command = [ + sys.executable, + idf_py_path, + 'secure-sign-data', + '--version', + '2', + '--keyfile', + f'../{self.signing_key}', + '--output', + 'bootloader-signed.bin', + 'bootloader/bootloader.bin', + ] output = self.call_command(sign_command) self.assertIn('Signed', output) def secure_verify_signature(self): self.secure_sign_data() - sign_command = [sys.executable, - idf_py_path, - 'secure-verify-signature', - '--version', - '2', - '--keyfile', - f'../{self.signing_key}', - 'bootloader-signed.bin'] + sign_command = [ + sys.executable, + idf_py_path, + 'secure-verify-signature', + '--version', + '2', + '--keyfile', + f'../{self.signing_key}', + 'bootloader-signed.bin', + ] output = self.call_command(sign_command) self.assertIn('verification successful', output) def secure_generate_signing_key(self): - generate_key_command = [sys.executable, - idf_py_path, - 'secure-generate-signing-key', - '--version', - '2', - '--scheme', - 'rsa3072', - self.signing_key] + generate_key_command = [ + sys.executable, + idf_py_path, + 'secure-generate-signing-key', + '--version', + '2', + '--scheme', + 'rsa3072', + self.signing_key, + ] output = self.call_command(generate_key_command) self.assertIn(f'RSA 3072 private key in PEM format written to {self.signing_key}', output) def test_secure_generate_key_digest(self): self.secure_generate_signing_key() - digest_command = [sys.executable, - idf_py_path, - 'secure-generate-key-digest', - '--keyfile', - f'{self.signing_key}', - '--output', - 'key_digest.bin'] + digest_command = [ + sys.executable, + idf_py_path, + 'secure-generate-key-digest', + '--keyfile', + f'{self.signing_key}', + '--output', + 'key_digest.bin', + ] output = self.call_command(digest_command) self.assertIn(f'Writing the public key digest of {self.signing_key} to key_digest.bin', output) def test_secure_generate_nvs_partition_key(self): - generate_key_command = [sys.executable, - idf_py_path, - 'secure-generate-nvs-partition-key', - '--keyfile', - f'{self.nvs_partition_key}', - '--encryption-scheme', - 'HMAC', - '--hmac-keyfile', - 'nvs_partition_key.bin'] + generate_key_command = [ + sys.executable, + idf_py_path, + 'secure-generate-nvs-partition-key', + '--keyfile', + f'{self.nvs_partition_key}', + '--encryption-scheme', + 'HMAC', + '--hmac-keyfile', + 'nvs_partition_key.bin', + ] output = self.call_command(generate_key_command) self.assertIn('Created encryption keys:', output) @@ -519,7 +595,8 @@ class TestSecureCommands(TestWrapperCommands): class TestMergeBinCommands(TestWrapperCommands): """ Test if merge-bin command is invoked as expected. - This test is not testing the functionality of esptool.py merge_bin command, but the invocation of the command from idf.py. + This test is not testing the functionality of esptool.py merge_bin command, + but the invocation of the command from idf.py. """ def test_merge_bin(self):