From 8c08c75f48352122c9405dff7c0f5eb2d35714d9 Mon Sep 17 00:00:00 2001 From: Marek Fiala Date: Mon, 13 Jul 2026 11:28:49 +0200 Subject: [PATCH] feat(tools): adopt esp-pylib for idf.py logging and errors Migrate idf.py and its actions to the shared esp-pylib library: - add esp-pylib (and esp-pylib[ide]) to the core/ide requirements - base FatalError on esp_pylib.errors.FatalError, keeping the idf.py ctx cleanup - replace the local raw-ANSI helpers (red_print/yellow_print/print_warning/ color_print) with esp_pylib.logger.log (warn/err/note/hint) across the idf.py actions - install esp_pylib exception reporting at the idf.py entry point and silence the logger during shell completion - update affected tests for the new prefixes/line wrapping - add normalize_output() helper and unify terminal env in conftest to handle Rich line-wrapping in CI assertions Co-authored-by: Cursor --- .gitlab/ci/host-test.yml | 22 ++- .mypy.ini | 4 + .pre-commit-config.yaml | 1 + tools/idf.py | 160 ++++++++++-------- tools/idf_py_actions/core_ext.py | 25 ++- tools/idf_py_actions/create_ext.py | 22 ++- tools/idf_py_actions/debug_ext.py | 22 ++- tools/idf_py_actions/dfu_ext.py | 4 +- tools/idf_py_actions/diag_ext.py | 21 ++- tools/idf_py_actions/errors.py | 10 +- tools/idf_py_actions/qemu_ext.py | 45 +++-- tools/idf_py_actions/serial_ext.py | 15 +- tools/idf_py_actions/tools.py | 85 ++++------ tools/requirements/requirements.core.txt | 3 +- tools/requirements/requirements.ide.txt | 1 + tools/test_build_system/conftest.py | 23 +++ .../test_build_system_helpers/__init__.py | 2 + .../test_build_system_helpers/idf_utils.py | 98 ++++++----- tools/test_build_system/test_cmake.py | 4 +- tools/test_build_system/test_common.py | 13 +- tools/test_build_system/test_idf_extension.py | 30 ++-- .../test_non_default_target.py | 7 +- tools/test_idf_py/conftest.py | 28 +++ tools/test_idf_py/test_hints.py | 6 +- tools/test_idf_py/test_idf_py.py | 48 ++---- 25 files changed, 381 insertions(+), 318 deletions(-) create mode 100644 tools/test_idf_py/conftest.py diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index bf5a765ac6e..0f885ff464d 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -207,21 +207,25 @@ test_tools: script: - stat=0 - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} + # --confcutdir=. blocks pytest from walking up to the repo-root conftest.py + # (which is for hardware-in-the-loop tests via pytest-embedded), while still + # allowing each suite's local conftest.py (e.g. terminal env for rich-click + # in tools/test_idf_py) to load — which `--noconftest` would otherwise skip. - cd ${IDF_PATH}/tools/ci/test_autocomplete - - run_cmd pytest --noconftest test_autocomplete.py --junitxml=${IDF_PATH}/XUNIT_AUTOCOMP.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 + - run_cmd pytest --confcutdir=. test_autocomplete.py --junitxml=${IDF_PATH}/XUNIT_AUTOCOMP.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 - cd ${IDF_PATH}/tools/test_idf_py - - run_cmd pytest --noconftest test_idf_py.py --junitxml=${IDF_PATH}/XUNIT_IDF_PY.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 - - run_cmd pytest --noconftest test_hints.py --junitxml=${IDF_PATH}/XUNIT_HINTS.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 - - run_cmd pytest --noconftest test_idf_qemu.py --junitxml=${IDF_PATH}/XUNIT_IDF_PY_QEMU.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 - - run_cmd pytest --noconftest test_mcp_ext.py --junitxml=${IDF_PATH}/XUNIT_MCP_EXT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 + - run_cmd pytest --confcutdir=. test_idf_py.py --junitxml=${IDF_PATH}/XUNIT_IDF_PY.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 + - run_cmd pytest --confcutdir=. test_hints.py --junitxml=${IDF_PATH}/XUNIT_HINTS.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 + - run_cmd pytest --confcutdir=. test_idf_qemu.py --junitxml=${IDF_PATH}/XUNIT_IDF_PY_QEMU.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 + - run_cmd pytest --confcutdir=. test_mcp_ext.py --junitxml=${IDF_PATH}/XUNIT_MCP_EXT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 - cd ${IDF_PATH}/tools/test_bsasm - - run_cmd pytest --noconftest test_bsasm.py --junitxml=${IDF_PATH}/XUNIT_BSASM.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 + - run_cmd pytest --confcutdir=. test_bsasm.py --junitxml=${IDF_PATH}/XUNIT_BSASM.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 - cd ${IDF_PATH}/tools/test_mkdfu - - run_cmd pytest --noconftest test_mkdfu.py --junitxml=${IDF_PATH}/XUNIT_MKDFU.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 + - run_cmd pytest --confcutdir=. test_mkdfu.py --junitxml=${IDF_PATH}/XUNIT_MKDFU.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 - cd ${IDF_PATH}/tools/test_idf_size - - run_cmd pytest --noconftest test_idf_size.py --junitxml=${IDF_PATH}/XUNIT_IDF_SIZE.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 + - run_cmd pytest --confcutdir=. test_idf_size.py --junitxml=${IDF_PATH}/XUNIT_IDF_SIZE.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 - cd ${IDF_PATH}/tools/test_idf_diag - - run_cmd pytest --noconftest test_idf_diag.py --junitxml=${IDF_PATH}/XUNIT_IDF_DIAG.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 + - run_cmd pytest --confcutdir=. test_idf_diag.py --junitxml=${IDF_PATH}/XUNIT_IDF_DIAG.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 - cd ${IDF_PATH} - shellcheck -s sh tools/detect_python.sh || stat=1 - shellcheck -s bash tools/detect_python.sh || stat=1 diff --git a/.mypy.ini b/.mypy.ini index fffc03b0323..2d7c2ac8e53 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -24,3 +24,7 @@ disallow_incomplete_defs = False # modules and packages provided on the command line. # SKIP -> mypy checks only single file, not included imports follow_imports = skip + +[mypy-esp_pylib.*] +# esp-pylib ships py.typed; follow it so NoReturn on log.die() is visible for type narrowing +follow_imports = normal diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7baf5cb245c..88eb2e835ca 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -100,6 +100,7 @@ repos: - 'types-setuptools' - 'types-PyYAML' - 'types-requests' + - 'esp-pylib>=1.1.0' # required for log.die() type narrowing exclude: > (?x)^( .*_pb2.py diff --git a/tools/idf.py b/tools/idf.py index eeb86004969..bc3a207c2f2 100755 --- a/tools/idf.py +++ b/tools/idf.py @@ -39,6 +39,7 @@ import python_version_checker # noqa: E402 try: import idf_py_actions.help_custom_targets_skip as _help_custom_targets + from esp_pylib.logger import log from idf_py_actions.errors import FatalError from idf_py_actions.tools import PROG from idf_py_actions.tools import SHELL_COMPLETE_RUN @@ -48,7 +49,7 @@ try: from idf_py_actions.tools import get_target from idf_py_actions.tools import idf_version_from_cmake from idf_py_actions.tools import merge_action_lists - from idf_py_actions.tools import print_warning + from rich.markup import escape if os.getenv('IDF_COMPONENT_MANAGER') != '0': from idf_component_manager import idf_extensions @@ -95,13 +96,15 @@ def check_environment() -> list: if 'IDF_PATH' in os.environ: set_idf_path = os.path.realpath(os.environ['IDF_PATH']) if set_idf_path != detected_idf_path: - print_warning( - f'WARNING: IDF_PATH environment variable is set to {set_idf_path}' - f' but {PROG} path indicates IDF directory {detected_idf_path}. ' - 'Using the environment variable directory, but results may be unexpected...' + log.warn( + escape( + f'IDF_PATH environment variable is set to {set_idf_path}' + f' but {PROG} path indicates IDF directory {detected_idf_path}. ' + 'Using the environment variable directory, but results may be unexpected...' + ) ) else: - print_warning(f'Setting IDF_PATH environment variable: {detected_idf_path}') + log.note(escape(f'Setting IDF_PATH environment variable: {detected_idf_path}')) os.environ['IDF_PATH'] = detected_idf_path try: @@ -126,7 +129,7 @@ def check_environment() -> list: checks_output.append(out.decode('utf-8', 'ignore').strip()) except subprocess.CalledProcessError as e: - print_warning(e.output.decode('utf-8', 'ignore'), stream=sys.stderr) + log.print(escape(e.output.decode('utf-8', 'ignore')), file=sys.stderr, soft_wrap=True) debug_print_idf_version() raise SystemExit(1) @@ -135,12 +138,14 @@ def check_environment() -> list: try: python_venv_path = os.environ['IDF_PYTHON_ENV_PATH'] if python_venv_path and not sys.executable.startswith(python_venv_path): - print_warning( - f'WARNING: Python interpreter "{sys.executable}" used to start idf.py' - f' is not from installed venv "{python_venv_path}"' + log.warn( + escape( + f'Python interpreter "{sys.executable}" used to start idf.py' + f' is not from installed venv "{python_venv_path}"' + ) ) except KeyError: - print_warning('WARNING: The IDF_PYTHON_ENV_PATH is missing in environmental variables!') + log.warn('The IDF_PYTHON_ENV_PATH is missing in environmental variables!') return checks_output @@ -246,10 +251,10 @@ def init_cli(verbose_output: list | None = None) -> Any: deprecation = Deprecation(option.deprecated) if deprecation.exit_with_error: error = deprecation.full_message(f'Option "{option.name}"') - raise FatalError(f'Error: {error}') + raise FatalError(error) else: error = deprecation.full_message(f'Option "{option.name}"') - print_warning(f'Warning: {error}') + log.warn(escape(error)) class Task: def __init__( @@ -354,9 +359,9 @@ def init_cli(verbose_output: list | None = None) -> Any: message = deprecation.full_message(f'Command "{self.name}"') if deprecation.exit_with_error: - raise FatalError(f'Error: {message}') + raise FatalError(message) else: - print_warning(f'Warning: {message}') + log.warn(escape(message)) self.deprecated = False # disable Click's built-in deprecation handling @@ -656,11 +661,11 @@ def init_cli(verbose_output: list | None = None) -> Any: # print a closing message of some kind, # except if any of the following actions were requested if any(t in str(actions) for t in ('flash', 'dfu', 'uf2', 'uf2-app', 'qemu', 'sbom-create')): - print('Done') + log.print('Done') return if not os.path.exists(os.path.join(args.build_dir, 'flasher_args.json')): - print('Done') + log.print('Done') return # Otherwise, if we built any binaries print a message about @@ -678,7 +683,7 @@ def init_cli(verbose_output: list | None = None) -> Any: if key not in flasher_args: # This is the case for 'idf.py bootloader' # if Secure Boot is on, need to follow manual flashing steps - print(f'\n{title} build complete.') + log.print(f'\n{title} build complete.') return cmd = '' if key == 'bootloader': # bootloader needs --flash-mode, etc to be passed in @@ -696,13 +701,13 @@ def init_cli(verbose_output: list | None = None) -> Any: cmd += o + ' ' + flasher_path(f) + ' ' flash_target = 'flash' if key == 'project' else f'{key}-flash' - print(f'{os.linesep}{title} build complete. To flash, run:') - print(f' idf.py {flash_target}') + log.print(f'{os.linesep}{title} build complete. To flash, run:') + log.print(f' idf.py {flash_target}') if args.port: - print('or') - print(f' idf.py -p {args.port} {flash_target}') - print('or') - print(f' idf.py -p PORT {flash_target}') + log.print('or') + log.print(escape(f' idf.py -p {args.port} {flash_target}')) + log.print('or') + log.print(f' idf.py -p PORT {flash_target}') esptool_cmd = [ 'python -m esptool', @@ -720,12 +725,12 @@ def init_cli(verbose_output: list | None = None) -> Any: esptool_cmd += ['write-flash'] - print('or') - print(' {}'.format(' '.join(esptool_cmd + [cmd.strip()]))) + log.print('or') + log.print(escape(' {}'.format(' '.join(esptool_cmd + [cmd.strip()])))) if os.path.exists(os.path.join(args.build_dir, 'flash_args')): - print(f'or from the "{args.build_dir}" directory') - print(' {}'.format(' '.join(esptool_cmd + ['"@flash_args"']))) + log.print(escape(f'or from the "{args.build_dir}" directory')) + log.print(escape(' {}'.format(' '.join(esptool_cmd + ['"@flash_args"'])))) if 'all' in actions or 'build' in actions: print_flashing_message('Project', 'project') @@ -746,11 +751,11 @@ def init_cli(verbose_output: list | None = None) -> Any: [item for item, count in Counter(task.name for task in tasks).items() if count > 1] ) if dupplicated_tasks: - print('----------------------------------------------------------------------------------------') + log.print('----------------------------------------------------------------------------------------') dupes = ', '.join(f'"{t}"' for t in dupplicated_tasks) - print_warning( - f'WARNING: {"Commands" if len(dupplicated_tasks) > 1 else "Command"} {dupes} ' + log.warn( + f'{"Commands" if len(dupplicated_tasks) > 1 else "Command"} {dupes} ' f'{"are" if len(dupplicated_tasks) > 1 else "is"} ' 'found in the list of commands more than once. ' 'Only first occurrence will be executed.' @@ -813,9 +818,11 @@ def init_cli(verbose_output: list | None = None) -> Any: # Otherwise invoke it with default set of options # and put to the front of the list of unprocessed tasks else: - print( - f'Adding "{task.name}"\'s dependency "{dep}" ' - 'to list of commands with default set of options.' + log.print( + escape( + f'Adding "{task.name}"\'s dependency "{dep}" ' + 'to list of commands with default set of options.' + ) ) dep_task = ctx.invoke(ctx.command.get_command(ctx, dep)) @@ -921,7 +928,7 @@ def init_cli(verbose_output: list | None = None) -> Any: if r.returncode == 0: add_from_help_text((r.stdout or '') + (r.stderr or '')) except (subprocess.TimeoutExpired, FileNotFoundError, OSError) as exc: - print_warning(f'Failed querying Ninja custom targets with "ninja -t targets all": {exc}') + log.warn(escape(f'Failed querying Ninja custom targets with "ninja -t targets all": {exc}')) pass # If ``ninja -t`` produced nothing we could parse (empty output, different format, etc.), @@ -938,7 +945,7 @@ def init_cli(verbose_output: list | None = None) -> Any: if _help_custom_targets.should_list_custom_target(n, defined): found.add(n) except OSError as exc: - print_warning(f'Failed reading Ninja file {ninja_file} for custom targets: {exc}') + log.warn(escape(f'Failed reading Ninja file {ninja_file} for custom targets: {exc}')) pass if not found and configure_ready_for_cmake_help(): @@ -954,7 +961,7 @@ def init_cli(verbose_output: list | None = None) -> Any: if r.returncode == 0: add_from_help_text((r.stdout or '') + (r.stderr or '')) except (subprocess.TimeoutExpired, FileNotFoundError, OSError) as exc: - print_warning(f'Failed querying CMake custom targets with "cmake --build --target help": {exc}') + log.warn(escape(f'Failed querying CMake custom targets with "cmake --build --target help": {exc}')) pass return [(n, '') for n in sorted(found, key=str.lower)] @@ -985,10 +992,10 @@ def init_cli(verbose_output: list | None = None) -> Any: if hasattr(ext_module, 'action_extensions'): return ext_module.action_extensions else: - print_warning(f"Warning: Extension {ext_file} has no attribute 'action_extensions'") + log.warn(escape(f"Extension {ext_file} has no attribute 'action_extensions'")) except (ImportError, SyntaxError) as e: - print_warning(f'Warning: Failed to import extension {ext_file}: {e}') + log.warn(escape(f'Failed to import extension {ext_file}: {e}')) return None @@ -1003,21 +1010,25 @@ def init_cli(verbose_output: list | None = None) -> Any: for ep in eps: if ep.value in eps_declarative_values: conflicting_names = [e.name for e in eps if e.value == ep.value] - print_warning( - f"Warning: Entry point's declarative value [extension_file_name:method_name] " - f'name collision detected for - {ep.value}. The same {ep.value} is used by ' - f'{conflicting_names} entry points. To ensure successful loading, please use' - ' a different extension file name or method name for the entry point.' + log.warn( + escape( + f"Entry point's declarative value [extension_file_name:method_name] " + f'name collision detected for - {ep.value}. The same {ep.value} is used by ' + f'{conflicting_names} entry points. To ensure successful loading, please use' + ' a different extension file name or method name for the entry point.' + ) ) # Remove any already loaded extensions with conflicting names extensions[:] = [ext for ext in extensions if ext[0] not in conflicting_names] continue if ep.value == 'idf_ext:action_extensions': - print_warning( - f'Entry point "{ep.name}" has declarative value "{ep.value}". For external components, ' - 'it is recommended to use name like <>_ext:action_extensions, ' - "so it does not interfere with the project's idf_ext.py file." + log.warn( + escape( + f'Entry point "{ep.name}" has declarative value "{ep.value}". For external components, ' + 'it is recommended to use name like <>_ext:action_extensions, ' + "so it does not interfere with the project's idf_ext.py file." + ) ) eps_declarative_values.append(ep.value) @@ -1025,7 +1036,7 @@ def init_cli(verbose_output: list | None = None) -> Any: extension_func = ep.load() extensions.append((ep.name, extension_func)) except Exception as e: - print_warning(f'Warning: Failed to load entry point extension "{ep.name}": {e}') + log.warn(escape(f'Failed to load entry point extension "{ep.name}": {e}')) return extensions @@ -1084,9 +1095,11 @@ def init_cli(verbose_output: list | None = None) -> Any: ): result.add(comp.name) except (ImportError, OSError, LockError) as e: - print_warning( - 'WARNING: Could not verify source of external components. ' - f'No extensions (idf_ext.py) from managed components will be loaded. ({e})' + log.warn( + escape( + 'Could not verify source of external components. ' + f'No extensions (idf_ext.py) from managed components will be loaded. ({e})' + ) ) return result @@ -1174,7 +1187,7 @@ def init_cli(verbose_output: list | None = None) -> Any: extensions = [] for directory in extension_dirs: if directory and not os.path.exists(directory): - print_warning(f"WARNING: Directory with idf.py extensions doesn't exist:\n\t{directory}") + log.warn(escape(f"Directory with idf.py extensions doesn't exist:\n\t{directory}")) continue sys.path.append(directory) @@ -1198,7 +1211,7 @@ def init_cli(verbose_output: list | None = None) -> Any: try: all_actions = merge_action_lists(all_actions, extension.action_extensions(all_actions, project_dir)) except AttributeError: - print_warning(f'WARNING: Cannot load idf.py extension "{name}"') + log.warn(escape(f'Cannot load idf.py extension "{name}"')) component_idf_ext_dirs = [] # Get trusted component directories with idf extensions that participate in the build @@ -1210,7 +1223,7 @@ def init_cli(verbose_output: list | None = None) -> Any: with open(project_description_json_file, encoding='utf-8') as f: project_desc = json.load(f) except (OSError, json.JSONDecodeError) as e: - print_warning(f'Warning: Failed to read component info from project_description.json: {e}') + log.warn(escape(f'Failed to read component info from project_description.json: {e}')) if project_desc is not None: build_component_info = project_desc.get('build_component_info', {}) all_component_info = project_desc.get('all_component_info', {}) @@ -1230,9 +1243,9 @@ def init_cli(verbose_output: list | None = None) -> Any: if _is_component_trusted(comp_name, source): component_idf_ext_dirs.append(comp_dir) else: - print_warning( - f'WARNING: Not loading component extension from untrusted source ' - f'"{_path_relative_to_project(comp_dir, project_dir)}". ' + log.warn( + f'Not loading component extension from untrusted source ' + f'"{escape(_path_relative_to_project(comp_dir, project_dir))}". ' 'Only extensions from trusted sources are loaded. Run ' '"idf.py docs -sp api-guides/tools/idf-py.html#extending-idf-py" ' 'for the list of trusted sources. Set IDF_EXTENSION_ALLOW_UNTRUSTED=1 to load all.' @@ -1247,7 +1260,7 @@ def init_cli(verbose_output: list | None = None) -> Any: custom_actions = extension_func(all_actions, project_dir) all_actions = merge_action_lists(all_actions, custom_actions=custom_actions) except Exception as e: - print_warning(f'WARNING: Cannot load directory extension from "{ext_dir}": {e}') + log.warn(escape(f'Cannot load directory extension from "{ext_dir}": {e}')) else: panel_cmds = sorted(n for n in custom_actions.get('actions') or {} if n != 'fallback') if panel_cmds: @@ -1256,7 +1269,9 @@ def init_cli(verbose_output: list | None = None) -> Any: ) external_help_panels.append((panel_title, panel_cmds)) if ext_dir != project_dir: - print(f'INFO: Loaded component extension from "{_path_relative_to_project(ext_dir, project_dir)}"') + log.note( + escape(f'Loaded component extension from "{_path_relative_to_project(ext_dir, project_dir)}"') + ) # Load extensions from Python entry points entry_point_extensions = load_cli_extensions_from_entry_points() @@ -1265,7 +1280,7 @@ def init_cli(verbose_output: list | None = None) -> Any: custom_actions = extension_func(all_actions, project_dir) all_actions = merge_action_lists(all_actions, custom_actions=custom_actions) except Exception as e: - print_warning(f'WARNING: Cannot load entry point extension "{ep_name}": {e}') + log.warn(escape(f'Cannot load entry point extension "{ep_name}": {e}')) else: panel_cmds = sorted(n for n in (custom_actions.get('actions') or {}) if n != 'fallback') if panel_cmds: @@ -1287,6 +1302,12 @@ def init_cli(verbose_output: list | None = None) -> Any: def main(argv: list[Any] | None = None) -> None: + # Forward uncaught exceptions to the IDE WebSocket (no-op unless ESP_IDE_WS is set and the + # esp-pylib[ide] extra is installed). Imported lazily to keep top-level imports minimal. + from esp_pylib.excepthook import install_exception_reporting + + install_exception_reporting() + # Check the environment only when idf.py is invoked regularly from command line. checks_output = None if SHELL_COMPLETE_RUN else check_environment() @@ -1294,7 +1315,7 @@ def main(argv: list[Any] | None = None) -> None: try: os.getcwd() except FileNotFoundError as e: - raise FatalError(f'ERROR: {e}. Working directory cannot be established. Check its existence.') + raise FatalError(f'{e}. Working directory cannot be established. Check its existence.') try: cli = init_cli(verbose_output=checks_output) @@ -1353,7 +1374,7 @@ def expand_file_arguments(argv: list[Any]) -> list[Any]: argv = expand_args(argv, os.getcwd(), []) if expanded: - print(f'Running: idf.py {" ".join(argv)}') + log.print(escape(f'Running: idf.py {" ".join(argv)}')) return argv @@ -1442,7 +1463,7 @@ def _find_usable_locale() -> str | None: if str(locale_name).lower().replace(' ', '-') in lcl_alias_name: # user preferred language has Unicode encoding (highest priority -1) if str(locale_name).lower().startswith(('tr', 'az', 'lt', 'kk')): - print_warning( + log.warn( f'Your locale "{locale_name}" has potential issues with case conversion for ASCII characters' ) continue @@ -1473,9 +1494,9 @@ def _find_usable_locale() -> str | None: if __name__ == '__main__': try: if 'MSYSTEM' in os.environ: - print_warning( + log.warn( 'MSys/Mingw is no longer supported. Please follow the getting started guide of the ' - 'documentation in order to set up a suitiable environment, or continue at your own risk.' + 'documentation in order to set up a suitable environment, or continue at your own risk.' ) elif os.name == 'posix' and not _valid_unicode_config(): # Trying to find best Unicode locale available on the system and restart python with @@ -1484,7 +1505,7 @@ if __name__ == '__main__': # Unset LC_ALL if it exists, as it takes precedence over LC_CTYPE # This prevents infinite loops when LC_ALL is set to a non-Unicode locale if best_locale: - print_warning( + log.warn( 'Your environment is not configured to handle Unicode characters.' ' Environment variable LC_CTYPE is temporary set to ' f'{best_locale} (found on the system) for Unicode support.' @@ -1497,7 +1518,7 @@ if __name__ == '__main__': raise SystemExit(ret) elif os.name == 'nt' and not _windows_unicode_satisfactory(): - print_warning( + log.warn( 'Your environment is not configured to handle Unicode characters.\n' '\n' 'Recommended fix (persistent):\n' @@ -1517,5 +1538,4 @@ if __name__ == '__main__': main() except FatalError as e: - print(e, file=sys.stderr) - sys.exit(2) + log.die(escape(str(e)), exit_code=2) diff --git a/tools/idf_py_actions/core_ext.py b/tools/idf_py_actions/core_ext.py index 24c26d9d740..b9a6aabe165 100644 --- a/tools/idf_py_actions/core_ext.py +++ b/tools/idf_py_actions/core_ext.py @@ -16,6 +16,8 @@ from urllib.request import urlopen from webbrowser import open_new_tab import rich_click as click +from esp_pylib.logger import log +from rich.markup import escape from rich_click import Context from idf_py_actions.constants import GENERATORS @@ -33,7 +35,6 @@ from idf_py_actions.tools import get_target from idf_py_actions.tools import idf_version from idf_py_actions.tools import merge_action_lists from idf_py_actions.tools import run_target -from idf_py_actions.tools import yellow_print # If a CMake preset with this name exists, it will be used by default when no '--preset' argument is given. DEFAULT_CMAKE_PRESET_NAME = 'default' @@ -56,8 +57,8 @@ def action_extensions(base_actions: dict, project_path: str) -> Any: """ ensure_build_directory(args, ctx.info_name) if buffer_size < 2048: - yellow_print( - f'WARNING: The specified buffer size {buffer_size} KB is less than the ' + log.warn( + f'The specified buffer size {buffer_size} KB is less than the ' 'recommended minimum of 2048 KB for idf.py confserver. Consider increasing it to at least 2048 KB ' 'by setting environment variable IDF_CONFSERVER_BUFFER_SIZE= or by calling ' 'idf.py confserver --buffer-size .' @@ -99,7 +100,7 @@ def action_extensions(base_actions: dict, project_path: str) -> Any: def tool_error_handler(e: int, stdout: str, stderr: str) -> None: for hint in generate_hints(stdout, stderr): - yellow_print(hint) + log.hint(escape(hint)) env: dict[str, Any] = {} @@ -135,7 +136,7 @@ def action_extensions(base_actions: dict, project_path: str) -> Any: proj_desc = get_build_context().get('proj_desc') or {} if proj_desc.get('target') == 'linux': - print("Note: 'idf.py size' is not supported for the 'linux' target; skipping size analysis.") + log.note("'idf.py size' is not supported for the 'linux' target; skipping size analysis.") return run_target(target_name, args, env=env) @@ -154,7 +155,7 @@ def action_extensions(base_actions: dict, project_path: str) -> Any: # Compatibility with legacy names if style in ['aquatic', 'monochrome', 'default']: - print('NOTE: Legacy menuconfig styles are deprecated. Using dark style instead.') + log.note('Legacy menuconfig styles are deprecated. Using dark style instead.') style = 'textual-dark' if style: @@ -323,7 +324,7 @@ def action_extensions(base_actions: dict, project_path: str) -> Any: except FatalError as err: raise err except Exception as err: - yellow_print(f'Failed to load CMake presets from {cmakepresets_file_name}, {str(err)}') + log.warn(escape(f'Failed to load CMake presets from {cmakepresets_file_name}, {str(err)}')) if not config_presets_info: if preset_name: @@ -334,13 +335,11 @@ def action_extensions(base_actions: dict, project_path: str) -> Any: # Determine which preset to use if not preset_name and DEFAULT_CMAKE_PRESET_NAME in preset_names: - yellow_print( - f"CMake presets file found but no preset name given; using '{DEFAULT_CMAKE_PRESET_NAME}' preset" - ) + log.note(f"CMake presets file found but no preset name given; using '{DEFAULT_CMAKE_PRESET_NAME}' preset") preset_name = DEFAULT_CMAKE_PRESET_NAME elif not preset_name: preset_name = preset_names[0] - yellow_print(f"CMake presets file found but no preset name given; using first preset: '{preset_name}'") + log.note(escape(f"CMake presets file found but no preset name given; using first preset: '{preset_name}'")) elif preset_name not in preset_names: raise FatalError(f"No preset '{preset_name}' found in CMake presets") @@ -348,7 +347,7 @@ def action_extensions(base_actions: dict, project_path: str) -> Any: if selected_preset_info: if selected_preset_info.get('inherits'): - yellow_print(f"Preset '{preset_name}' uses inheritance, which is not yet supported.") + log.warn(escape(f"Preset '{preset_name}' uses inheritance, which is not yet supported.")) # Set build directory from preset binary_dir = selected_preset_info.get('binaryDir') @@ -357,7 +356,7 @@ def action_extensions(base_actions: dict, project_path: str) -> Any: binary_dir = os.path.join(args.project_dir, binary_dir) args.build_dir = binary_dir elif not binary_dir and not args.build_dir: - yellow_print(f'Warning: preset {preset_name} does not specify the build directory ("binaryDir")') + log.warn(escape(f'preset {preset_name} does not specify the build directory ("binaryDir")')) # Set generator from preset if specified generator = selected_preset_info.get('generator', None) diff --git a/tools/idf_py_actions/create_ext.py b/tools/idf_py_actions/create_ext.py index 5cafafb5914..d5fb6b78af2 100644 --- a/tools/idf_py_actions/create_ext.py +++ b/tools/idf_py_actions/create_ext.py @@ -8,6 +8,8 @@ from collections.abc import Callable from shutil import copyfile from shutil import copytree +from esp_pylib.logger import log +from rich.markup import escape from rich_click import Context from idf_py_actions.tools import PropertyDict @@ -31,17 +33,21 @@ def is_empty_and_create(path: str, action: str) -> None: if not os.path.exists(abspath): os.makedirs(abspath) elif not os.path.isdir(abspath): - print( - f'Your target path is not a directory.' - f'Please remove the {os.path.abspath(abspath)} or use different target path.' + log.die( + escape( + f'Your target path is not a directory.' + f'Please remove the {os.path.abspath(abspath)} or use different target path.' + ), + exit_code=4, ) - sys.exit(4) elif len(os.listdir(path)) > 0: - print( - f'The directory {abspath} is not empty. To create a {get_type(action)} you must ' - f'empty the directory or choose a different path.' + log.die( + escape( + f'The directory {abspath} is not empty. To create a {get_type(action)} you must ' + f'empty the directory or choose a different path.' + ), + exit_code=3, ) - sys.exit(3) def make_directory_permissions_writable(root_path: str) -> None: diff --git a/tools/idf_py_actions/debug_ext.py b/tools/idf_py_actions/debug_ext.py index 5efb28dc9fe..3a1f07a77af 100644 --- a/tools/idf_py_actions/debug_ext.py +++ b/tools/idf_py_actions/debug_ext.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import json import os @@ -12,6 +12,8 @@ from threading import Thread from typing import Any from esp_coredump import CoreDump +from esp_pylib.logger import log +from rich.markup import escape from rich_click import INT from rich_click import Context @@ -23,7 +25,6 @@ from idf_py_actions.tools import ensure_build_directory from idf_py_actions.tools import generate_hints from idf_py_actions.tools import get_default_serial_port from idf_py_actions.tools import get_sdkconfig_value -from idf_py_actions.tools import yellow_print def chip_rev_to_int(chip_rev: str | None) -> int | None: @@ -48,13 +49,11 @@ def action_extensions(base_actions: dict, project_path: str) -> dict: if not processes['allow_hints']: return + # Redirect hints to stderr so stdout stays clean for the gdb console. + log.set_info_stream(sys.stderr) + sys.stdout.flush() for hint in generate_hints(file_name): - if sys.stderr.isatty(): - yellow_print(hint) - else: - # Hints go to stderr. Flush stdout, so hints are printed last. - sys.stdout.flush() - print(hint, file=sys.stderr) + log.hint(escape(hint)) def _check_openocd_errors(fail_if_openocd_failed: dict, target: str, ctx: Context) -> None: if fail_if_openocd_failed: @@ -139,11 +138,10 @@ def action_extensions(base_actions: dict, project_path: str) -> dict: # The format will be determined automatically args.port = args.port or get_default_serial_port() else: - print( + log.die( 'Path to core dump file is not provided. ' "Core dump can't be read from flash since this option is not enabled in menuconfig" ) - sys.exit(1) espcoredump_kwargs['port'] = args.port espcoredump_kwargs['parttable_off'] = get_sdkconfig_value( @@ -320,7 +318,7 @@ def action_extensions(base_actions: dict, project_path: str) -> dict: pass if not subprocess_success or completed_process.returncode != 0: - raise SystemExit( + log.die( 'Error occurred while starting gdbgui. Please make sure gdbgui has been installed with ' 'pipx based on https://www.gdbgui.com/installation/ and "gdbgui --version" can be run ' 'successfully. Gdbgui issues can be reported at https://github.com/cs01/gdbgui/issues.' @@ -328,7 +326,7 @@ def action_extensions(base_actions: dict, project_path: str) -> dict: v = re.search(r'(\d+)(?:\.(\d+))?(?:\.(\d+))?(?:\.(\d+))?', captured_output) if not v: - raise SystemExit(f'Error: "gdbgui --version" returned "{captured_output}"') + log.die(escape(f'"gdbgui --version" returned "{captured_output}"')) return tuple(int(i) if i else 0 for i in (v[1], v[2], v[3], v[4])) def gdbui( diff --git a/tools/idf_py_actions/dfu_ext.py b/tools/idf_py_actions/dfu_ext.py index 58a4565c170..cb26feb5016 100644 --- a/tools/idf_py_actions/dfu_ext.py +++ b/tools/idf_py_actions/dfu_ext.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 +from esp_pylib.logger import log from rich_click import Context from idf_py_actions.errors import FatalError @@ -8,7 +9,6 @@ from idf_py_actions.tools import PropertyDict from idf_py_actions.tools import ensure_build_directory from idf_py_actions.tools import get_sdkconfig_filename from idf_py_actions.tools import get_sdkconfig_value -from idf_py_actions.tools import red_print from idf_py_actions.tools import run_target SOC_USB_DFU_SUPPORTED = 'CONFIG_SOC_USB_DFU_SUPPORTED' @@ -21,7 +21,7 @@ def action_extensions(base_actions: dict, project_path: str) -> dict: return True target = get_sdkconfig_value(sdkconfig_path, 'CONFIG_IDF_TARGET') or 'unknown' - red_print(f'DFU is not supported for this target: {target}') + log.err(f'DFU is not supported for this target: {target}') return False def dfu_target(target_name: str, ctx: Context, args: PropertyDict, part_size: str) -> None: diff --git a/tools/idf_py_actions/diag_ext.py b/tools/idf_py_actions/diag_ext.py index 2a8961491ca..43a9753680d 100644 --- a/tools/idf_py_actions/diag_ext.py +++ b/tools/idf_py_actions/diag_ext.py @@ -4,11 +4,12 @@ import sys import uuid from typing import Any +from esp_pylib.logger import log +from rich.markup import escape from rich_click import Context from idf_py_actions.tools import PropertyDict from idf_py_actions.tools import RunTool -from idf_py_actions.tools import yellow_print def diag( @@ -102,7 +103,7 @@ def diag( if args.port: diag_args += ['--port', args.port] else: - yellow_print( + log.note( 'The target serial port is not specified, so ' 'autodetection will be used. To set it manually, use ' 'the "--port" option. Example: "idf.py --port ' @@ -115,13 +116,15 @@ def diag( raise if command == 'create': - yellow_print( - f'Please make sure to thoroughly check it for any sensitive ' - f'information before sharing and remove files you do not want ' - f'to share. Kindly include any additional files you find ' - f'relevant that were not automatically added. Please archive ' - f'the contents of the final report directory using the command:\n' - f'"idf.py diag --zip {output}".' + log.note( + escape( + f'Please make sure to thoroughly check it for any sensitive ' + f'information before sharing and remove files you do not want ' + f'to share. Kindly include any additional files you find ' + f'relevant that were not automatically added. Please archive ' + f'the contents of the final report directory using the command:\n' + f'"idf.py diag --zip {output}".' + ) ) diff --git a/tools/idf_py_actions/errors.py b/tools/idf_py_actions/errors.py index a4f5f69158a..20403c5f851 100644 --- a/tools/idf_py_actions/errors.py +++ b/tools/idf_py_actions/errors.py @@ -1,15 +1,19 @@ -# SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 +from esp_pylib.errors import FatalError as PylibFatalError from rich_click import Context -class FatalError(RuntimeError): +class FatalError(PylibFatalError): """ Wrapper class for runtime errors that aren't caused by bugs in idf.py or the build process. + + Extends the shared ``esp_pylib.errors.FatalError`` (a ``RuntimeError``) while keeping the + idf.py-specific context cleanup hook. """ def __init__(self, message: str, ctx: Context = None): - super(RuntimeError, self).__init__(message) + super().__init__(message) # if context is defined, check for the cleanup tasks if ctx is not None and 'cleanup' in ctx.meta: # cleans up the environment before failure diff --git a/tools/idf_py_actions/qemu_ext.py b/tools/idf_py_actions/qemu_ext.py index 6aa8a7a5fc8..7f3aafb6021 100644 --- a/tools/idf_py_actions/qemu_ext.py +++ b/tools/idf_py_actions/qemu_ext.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import atexit import binascii @@ -14,13 +14,13 @@ import time from dataclasses import dataclass from typing import Any +from esp_pylib.logger import log +from rich.markup import escape from rich_click import Context try: from idf_py_actions.tools import PropertyDict from idf_py_actions.tools import ensure_build_directory - from idf_py_actions.tools import red_print - from idf_py_actions.tools import yellow_print except ImportError: PropertyDict = Any @@ -168,8 +168,7 @@ def wait_for_socket(port: int, timeout_sec: float = 10.0) -> None: except OSError: time.sleep(0.1) continue - red_print(f'Timed out waiting for port {port} to be open') - raise SystemExit(1) + log.die(f'Timed out waiting for port {port} to be open') def action_extensions(base_actions: dict, project_path: str) -> dict: @@ -189,20 +188,19 @@ def action_extensions(base_actions: dict, project_path: str) -> dict: if have_qemu: if have_gdb and have_monitor: - red_print('Cannot run qemu with both gdb and monitor in the same terminal') - raise SystemExit(1) + log.die('Cannot run qemu with both gdb and monitor in the same terminal') if have_gdb: options.wait_for_gdb = True options.bg_mode = True if have_monitor: options.wait_for_monitor = True options.bg_mode = True - yellow_print(f'Running qemu on {PYSERIAL_PORT}') + log.note(f'Running qemu on {PYSERIAL_PORT}') global_args['port'] = PYSERIAL_PORT if have_efuse: options.bg_mode = True options.boot_mode = True - yellow_print(f'Running qemu on {PYSERIAL_PORT}') + log.note(f'Running qemu on {PYSERIAL_PORT}') global_args['port'] = PYSERIAL_PORT for task in tasks: if fnmatch.fnmatch(task.name, 'efuse-*'): @@ -233,15 +231,13 @@ def action_extensions(base_actions: dict, project_path: str) -> dict: target = project_desc['target'] qemu_target_info = QEMU_TARGETS.get(target) if not qemu_target_info: - red_print(f'QEMU is not supported for target {target}') - raise SystemExit(1) + log.die(f'QEMU is not supported for target {target}') if not shutil.which(qemu_target_info.qemu_prog): - red_print( + log.die( f'{qemu_target_info.qemu_prog} is not installed. Please install it using ' f'"python $IDF_PATH/tools/idf_tools.py install {qemu_target_info.install_package}" ' 'or build it from source if the pre-built version is not available for your platform.' ) - raise SystemExit(1) # Generate flash image and efuse image flash_size = get_sdkconfig_value(project_desc['config_file'], 'CONFIG_ESPTOOLPY_FLASHSIZE') @@ -250,13 +246,12 @@ def action_extensions(base_actions: dict, project_path: str) -> dict: bin_path = flash_file try: open(bin_path, 'rb').close() - yellow_print(f'Using provided flash image: {bin_path}') + log.note(escape(f'Using provided flash image: {bin_path}')) except FileNotFoundError: - red_print(f'The provided flash image file "{bin_path}" could not be found') - raise SystemExit(1) + log.die(escape(f'The provided flash image file "{bin_path}" could not be found')) else: bin_path = os.path.join(args.build_dir, 'qemu_flash.bin') - yellow_print(f'Generating flash image: {bin_path}') + log.note(escape(f'Generating flash image: {bin_path}')) subprocess.check_call( [ sys.executable, @@ -277,9 +272,9 @@ def action_extensions(base_actions: dict, project_path: str) -> dict: efuse_bin_path = os.path.join(args.build_dir, 'qemu_efuse.bin') try: open(efuse_bin_path, 'rb').close() - yellow_print(f'Using existing efuse image: {efuse_bin_path}') + log.note(escape(f'Using existing efuse image: {efuse_bin_path}')) except FileNotFoundError: - yellow_print(f'Generating efuse image: {efuse_bin_path}') + log.note(escape(f'Generating efuse image: {efuse_bin_path}')) with open(efuse_bin_path, 'wb') as f: f.write(qemu_target_info.default_efuse) @@ -329,7 +324,7 @@ def action_extensions(base_actions: dict, project_path: str) -> dict: if qemu_extra_args: qemu_args += shlex.split(qemu_extra_args) - yellow_print('Running qemu (fg): ' + ' '.join(qemu_args)) + log.note('Running qemu (fg): ' + escape(' '.join(qemu_args))) subprocess.run(qemu_args) else: if options.wait_for_monitor: @@ -342,7 +337,7 @@ def action_extensions(base_actions: dict, project_path: str) -> dict: if qemu_extra_args: qemu_args += shlex.split(qemu_extra_args) - yellow_print('Running qemu (bg): ' + ' '.join(qemu_args)) + log.note('Running qemu (bg): ' + escape(' '.join(qemu_args))) qemu_proc = subprocess.Popen( qemu_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE ) @@ -355,14 +350,14 @@ def action_extensions(base_actions: dict, project_path: str) -> dict: atexit.register(cleanup_qemu) if qemu_proc.poll() is not None: - yellow_print('QEMU exited with error') + log.err('QEMU exited with error') if qemu_proc.stderr is not None: - yellow_print('Stderr output was:') - yellow_print(qemu_proc.stderr.read().decode('utf-8')) + log.err('Stderr output was:') + log.print(escape(qemu_proc.stderr.read().decode('utf-8')), file=sys.stderr, soft_wrap=True) sys.exit(1) if gdb and not options.wait_for_gdb: - yellow_print('Waiting for GDB to connect. You can now run "idf.py gdb" in another terminal window.') + log.note('Waiting for GDB to connect. You can now run "idf.py gdb" in another terminal window.') qemu_actions = { 'global_action_callbacks': [global_callback], diff --git a/tools/idf_py_actions/serial_ext.py b/tools/idf_py_actions/serial_ext.py index 4685708b12c..b2382b4e019 100644 --- a/tools/idf_py_actions/serial_ext.py +++ b/tools/idf_py_actions/serial_ext.py @@ -10,6 +10,7 @@ from typing import Any import rich_click as click from click.core import ParameterSource +from esp_pylib.logger import log from rich_click import Context from idf_py_actions.errors import FatalError @@ -45,12 +46,6 @@ PORT = { } -def yellow_print(message: str, newline: str | None = '\n') -> None: - """Print a message to stderr with yellow highlighting""" - sys.stderr.write(f'\033[0;33m{message}\033[0m{newline}') - sys.stderr.flush() - - def action_extensions(base_actions: dict, project_path: str) -> dict: def _get_project_desc(ctx: Context, args: PropertyDict) -> Any: desc_path = os.path.join(args.build_dir, 'project_description.json') @@ -224,7 +219,7 @@ def action_extensions(base_actions: dict, project_path: str) -> dict: ensure_build_directory(args, ctx.info_name) project_desc = _get_project_desc(ctx, args) if project_desc['target'] == 'linux': - yellow_print('skipping flash since running on linux...') + log.note('skipping flash for linux target') return args.port = args.port or get_default_serial_port() @@ -299,17 +294,17 @@ def action_extensions(base_actions: dict, project_path: str) -> dict: merge_bin_args += ['-f', format] if md5_disable: if format != 'uf2': - yellow_print('idf.py merge-bin: --md5-disable is only valid for UF2 format. Option will be ignored.') + log.warn('idf.py merge-bin: --md5-disable is only valid for UF2 format. Option will be ignored.') else: merge_bin_args += ['--md5-disable'] if flash_offset: if format != 'raw': - yellow_print('idf.py merge-bin: --flash-offset is only valid for RAW format. Option will be ignored.') + log.warn('idf.py merge-bin: --flash-offset is only valid for RAW format. Option will be ignored.') else: merge_bin_args += ['-t', flash_offset] if pad_to_size or fill_flash_size: if format != 'raw': - yellow_print('idf.py merge-bin: --pad-to-size is only valid for RAW format, option will be ignored.') + log.warn('idf.py merge-bin: --pad-to-size is only valid for RAW format, option will be ignored.') else: merge_bin_args += ['--pad-to-size', pad_to_size or fill_flash_size] if merge_args: diff --git a/tools/idf_py_actions/tools.py b/tools/idf_py_actions/tools.py index 015f536973f..33db67ae0b6 100644 --- a/tools/idf_py_actions/tools.py +++ b/tools/idf_py_actions/tools.py @@ -19,6 +19,8 @@ from typing import cast import rich_click as click import yaml from esp_idf_monitor import get_ansi_converter +from esp_pylib.logger import log +from rich.markup import escape from idf_py_actions.errors import NoSerialPortFoundError @@ -35,6 +37,12 @@ SHELL_COMPLETE_VAR = '_IDF.PY_COMPLETE' # was shell completion invoked? SHELL_COMPLETE_RUN = SHELL_COMPLETE_VAR in os.environ +# During shell completion idf.py must stay silent: any diagnostic written to stdout/stderr +# corrupts the completion output. Silence the shared logger once instead of guarding every +# call site (this is what the old print_warning() helper did via its SHELL_COMPLETE_RUN check). +if SHELL_COMPLETE_RUN: + log.set_verbosity('SILENT') + # The ctx dict "abuses" how python evaluates default parameter values. # https://docs.python.org/3/reference/compound_stmts.html#function-definitions @@ -91,7 +99,7 @@ def idf_version_from_cmake() -> str | None: return f'v{ver["MAJOR"]}.{ver["MINOR"]}.{ver["PATCH"]}' except (KeyError, OSError): - sys.stderr.write('WARNING: Cannot find ESP-IDF version in version.cmake\n') + log.warn('Cannot find ESP-IDF version in version.cmake') return None @@ -123,7 +131,7 @@ def idf_version() -> str | None: ) except Exception: # if failed, then try to parse cmake.version file - sys.stderr.write('WARNING: Git version unavailable, reading from source\n') + log.warn('Git version unavailable, reading from source') version = idf_version_from_cmake() return version @@ -156,32 +164,8 @@ def get_default_serial_port() -> Any: raise FatalError(f'An exception occurred during detection of the serial port: {e}') -# function prints warning when autocompletion is not being performed -# set argument stream to sys.stderr for errors and exceptions -def print_warning(message: str, stream: TextIO | None = None) -> None: - if not SHELL_COMPLETE_RUN: - print(message, file=stream or sys.stderr) - - -def color_print(message: str, color: str, newline: str | None = '\n') -> None: - """Print a message to stderr with colored highlighting""" - ansi_normal = '\033[0m' - sys.stderr.write(f'{color}{message}{ansi_normal}{newline}') - sys.stderr.flush() - - -def yellow_print(message: str, newline: str | None = '\n') -> None: - ansi_yellow = '\033[0;33m' - color_print(message, ansi_yellow, newline) - - -def red_print(message: str, newline: str | None = '\n') -> None: - ansi_red = '\033[1;31m' - color_print(message, ansi_red, newline) - - def debug_print_idf_version() -> None: - print_warning(f'ESP-IDF {idf_version() or "version unknown"}') + log.note(f'ESP-IDF {idf_version() or "version unknown"}') def _load_hints_from_directory(directory: str) -> list: @@ -195,7 +179,7 @@ def _load_hints_from_directory(directory: str) -> list: hints = yaml.safe_load(file) return hints if hints else [] except (OSError, yaml.YAMLError): - yellow_print(f'HINT WARNING: Failed to load hints from "{hints_file}"') + log.warn(escape(f'Failed to load hints from "{hints_file}"')) return [] @@ -232,11 +216,9 @@ def _load_idf_hints() -> dict: try: hints['modules'].append(getattr(importlib.import_module(name), 'generate_hint')) except ModuleNotFoundError: - red_print(f'Failed to import "{name}" from "{hint_modules_dir}" as a module') - raise SystemExit(1) + log.die(escape(f'Failed to import "{name}" from "{hint_modules_dir}" as a module')) except AttributeError: - red_print(f'Module "{name}" does not have function generate_hint.') - raise SystemExit(1) + log.die(escape(f'Module "{name}" does not have function generate_hint.')) # Load ESP-IDF components idf_path = os.environ.get('IDF_PATH') @@ -304,23 +286,19 @@ def generate_hints_buffer(output: str, hints: dict) -> Generator: try: hint_list.append(hint['hint'].format(*hint_vars)) except KeyError as e: - red_print(f'Argument {e} missing in {hint}. Check hints.yml file.') - sys.exit(1) + log.die(escape(f'Argument {e} missing in {hint}. Check hints.yml file.')) else: match = re.compile(hint['re']).search(output) except KeyError as e: - red_print(f'Argument {e} missing in {hint}. Check hints.yml file.') - sys.exit(1) + log.die(escape(f'Argument {e} missing in {hint}. Check hints.yml file.')) except re.error as e: - red_print('{} from hints.yml have {} problem. Check hints.yml file.'.format(hint['re'], e)) - sys.exit(1) + log.die(escape('{} from hints.yml have {} problem. Check hints.yml file.'.format(hint['re'], e))) if hint_list: - for message in hint_list: - yield ' '.join(['HINT:', message]) + yield from hint_list elif match: extra_info = ', '.join(match.groups()) if hint.get('match_to_output', '') else '' try: - yield ' '.join(['HINT:', hint['hint'].format(extra_info)]) + yield hint['hint'].format(extra_info) except KeyError: raise KeyError(f"Argument 'hint' missing in {hint}. Check hints.yml file.") @@ -440,7 +418,7 @@ class RunTool: # hints in interactive mode were already processed, don't print them again if not self.interactive: for hint in generate_hints(stderr_output_file, stdout_output_file): - yellow_print(hint) + log.hint(escape(hint)) raise FatalError( f'{self.tool_name} failed with exit code {process.returncode}, ' f'output of the command is in the {stderr_output_file} and {stdout_output_file}' @@ -469,15 +447,14 @@ class RunTool: ) except NotImplementedError: message = ( - f"ERROR: {sys.executable} doesn't support asyncio. " - "Workaround: re-run idf.py with the '--no-hints' argument." + f"{sys.executable} doesn't support asyncio. Workaround: re-run idf.py with the '--no-hints' argument." ) if sys.platform == 'win32': message += ( ' To fix the issue use the Windows Installer for setting up your python environment, ' 'available from: https://dl.espressif.com/dl/esp-idf/' ) - sys.exit(message) + log.die(escape(message)) stderr_output_file = os.path.join(self.build_dir, log_dir_name, f'idf_py_stderr_output_{p.pid}') stdout_output_file = os.path.join(self.build_dir, log_dir_name, f'idf_py_stdout_output_{p.pid}') @@ -495,7 +472,7 @@ class RunTool: # the even loop is closed and we get RuntimeError: Event loop is closed # in the transport __del__ function because it's trying to use the closed # even loop. - red_print(f'\n{self.tool_name} process terminated\n') + log.err(f'\n{self.tool_name} process terminated') await p.wait() # added for avoiding None returncode return p, stderr_output_file, stdout_output_file @@ -612,7 +589,7 @@ class RunTool: last_line += output if last_line[-1] == '\n': for hint in generate_hints_buffer(last_line, hints): - yellow_print(hint) + log.hint(escape(hint)) last_line = '' else: output_b = await read_stream_bytes() @@ -641,9 +618,11 @@ class RunTool: else: write_stdout_bytes(forward_b) except (OSError, RuntimeError) as e: - yellow_print( - "WARNING: The exception {} was raised and we can't capture all your {} and " - 'hints on how to resolve errors can be not accurate.'.format(e, output_stream.name.strip('<>')) + log.warn( + escape( + "The exception {} was raised and we can't capture all your {} and " + 'hints on how to resolve errors can be not accurate.'.format(e, output_stream.name.strip('<>')) + ) ) @@ -936,16 +915,14 @@ def merge_action_lists(*action_lists: dict, custom_actions: dict[str, Any] | Non existing_identifiers.add(name) existing_identifiers.update(action.get('aliases', [])) except UserWarning as e: - yellow_print(f'WARNING: {e}. External action will not be added.') + log.warn(escape(f'{e}. External action will not be added.')) for new_opt in custom_actions.get('global_options', []): if any( set(new_opt.get('names', [])) & set(existing.get('names', [])) for existing in merged_actions['global_options'] ): - yellow_print( - f'WARNING: Global option {new_opt["names"]} already defined. External option will not be added.' - ) + log.warn(escape(f'Global option {new_opt["names"]} already defined. External option will not be added.')) else: merged_actions['global_options'].append(new_opt) diff --git a/tools/requirements/requirements.core.txt b/tools/requirements/requirements.core.txt index cd5f313c6cb..cac62308a2d 100644 --- a/tools/requirements/requirements.core.txt +++ b/tools/requirements/requirements.core.txt @@ -14,6 +14,7 @@ pyparsing pyelftools idf-component-manager>=2.2 esp-coredump +esp-pylib[cli] esptool esp-idf-kconfig esp-idf-monitor @@ -22,11 +23,9 @@ esp-idf-size esp-idf-diag esp-idf-panic-decoder esp-idf-sbom -esp-pylib[cli,ide] pyclang construct rich -esp-pylib[cli] psutil # ble's compressed log testing tool diff --git a/tools/requirements/requirements.ide.txt b/tools/requirements/requirements.ide.txt index b68ac9185b6..f59bb752303 100644 --- a/tools/requirements/requirements.ide.txt +++ b/tools/requirements/requirements.ide.txt @@ -6,3 +6,4 @@ # https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/tools/idf-tools.html esp-idf-monitor[ide] +esp-pylib[ide] diff --git a/tools/test_build_system/conftest.py b/tools/test_build_system/conftest.py index da844190677..ae980644c97 100644 --- a/tools/test_build_system/conftest.py +++ b/tools/test_build_system/conftest.py @@ -322,6 +322,29 @@ def idf_copy(func_work_dir: Path, request: FixtureRequest) -> typing.Generator[P shutil.rmtree(path_to, ignore_errors=True) +@pytest.fixture(autouse=True, scope='session') +def idf_py_terminal_env() -> typing.Generator[None, None, None]: + """Set terminal env so idf.py subprocesses produce consistent output. + + COLUMNS=200 raises Rich's non-TTY default of 80, preventing most line wrapping. + Messages with long file paths can still exceed 200 characters; use + normalize_output() for assertions on those. + """ + keys = ('COLUMNS', 'LINES', 'NO_COLOR', 'FORCE_COLOR', 'PY_COLORS', 'TERM') + saved = {k: os.environ.get(k) for k in keys} + os.environ['COLUMNS'] = '200' + os.environ['LINES'] = '40' + os.environ['NO_COLOR'] = '1' + for k in ('FORCE_COLOR', 'PY_COLORS'): + os.environ.pop(k, None) + yield + for k, v in saved.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + + @pytest.fixture(name='default_idf_env') def fixture_default_idf_env() -> EnvDict: return get_idf_build_env(os.environ['IDF_PATH']) # type: ignore diff --git a/tools/test_build_system/test_build_system_helpers/__init__.py b/tools/test_build_system/test_build_system_helpers/__init__.py index 0432ea96249..a07fe4dfd65 100644 --- a/tools/test_build_system/test_build_system_helpers/__init__.py +++ b/tools/test_build_system/test_build_system_helpers/__init__.py @@ -17,6 +17,7 @@ from .idf_utils import EnvDict from .idf_utils import IdfPyFunc from .idf_utils import find_python from .idf_utils import get_idf_build_env +from .idf_utils import normalize_output from .idf_utils import run_cmake from .idf_utils import run_cmake_and_build from .idf_utils import run_idf_py @@ -45,6 +46,7 @@ __all__ = [ 'ALL_ARTIFACTS_BUILDV2', 'run_cmake_and_build', 'find_python', + 'normalize_output', 'file_contains', 'bin_file_contains', 'bin_files_differ', diff --git a/tools/test_build_system/test_build_system_helpers/idf_utils.py b/tools/test_build_system/test_build_system_helpers/idf_utils.py index 2e7de87601f..e0f222ee4d3 100644 --- a/tools/test_build_system/test_build_system_helpers/idf_utils.py +++ b/tools/test_build_system/test_build_system_helpers/idf_utils.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import logging import os @@ -7,7 +7,6 @@ import subprocess import sys import typing from pathlib import Path -from typing import Union try: EXT_IDF_PATH = os.environ['IDF_PATH'] # type: str @@ -16,10 +15,19 @@ except KeyError: exit(1) -EnvDict = typing.Dict[str, str] +EnvDict = dict[str, str] IdfPyFunc = typing.Callable[..., subprocess.CompletedProcess] +def normalize_output(text: str) -> str: + """Collapse all whitespace runs to a single space. + + Use for content assertions on messages that include file paths: long paths can + push lines past COLUMNS=200 and cause Rich to insert a mid-message line break. + """ + return ' '.join(text.split()) + + def find_python(path_var: str) -> str: """ Find python interpreter in the paths specified in the given PATH variable. @@ -37,15 +45,11 @@ def get_idf_build_env(idf_path: str) -> EnvDict: :param idf_path: path of the IDF copy to use :return: dictionary of environment variables and their values """ - cmd = [ - sys.executable, - os.path.join(idf_path, 'tools', 'idf_tools.py'), - 'export', - '--format=key-value' - ] + cmd = [sys.executable, os.path.join(idf_path, 'tools', 'idf_tools.py'), 'export', '--format=key-value'] keys_values = subprocess.check_output(cmd, stderr=subprocess.PIPE).decode() - idf_tool_py_env = {key: os.path.expandvars(value) for key, value in - [line.split('=') for line in keys_values.splitlines()]} + idf_tool_py_env = { + key: os.path.expandvars(value) for key, value in [line.split('=') for line in keys_values.splitlines()] + } env_vars = {} # type: EnvDict env_vars.update(os.environ) env_vars.update(idf_tool_py_env) @@ -55,13 +59,15 @@ def get_idf_build_env(idf_path: str) -> EnvDict: return env_vars -def run_idf_py(*args: str, - env: typing.Optional[EnvDict] = None, - idf_path: typing.Optional[typing.Union[str,Path]] = None, - workdir: typing.Optional[str] = None, - check: bool = True, - python: typing.Optional[str] = None, - input_str: typing.Optional[str] = None) -> subprocess.CompletedProcess: +def run_idf_py( + *args: str, + env: EnvDict | None = None, + idf_path: str | Path | None = None, + workdir: str | None = None, + check: bool = True, + python: str | None = None, + input_str: str | None = None, +) -> subprocess.CompletedProcess: """ Run idf.py command with given arguments, raise an exception on failure :param args: arguments to pass to idf.py @@ -84,29 +90,35 @@ def run_idf_py(*args: str, if python is None: python = find_python(env['PATH']) - cmd = [ - python, - os.path.join(idf_path, 'tools', 'idf.py') - ] + cmd = [python, os.path.join(idf_path, 'tools', 'idf.py')] cmd += args # type: ignore logging.debug('running {} in {}'.format(' '.join(cmd), workdir)) try: return subprocess.run( - cmd, env=env, cwd=workdir, - check=check, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - text=True, encoding='utf-8', errors='backslashreplace', input=input_str) + cmd, + env=env, + cwd=workdir, + check=check, + capture_output=True, + text=True, + encoding='utf-8', + errors='backslashreplace', + input=input_str, + ) except subprocess.CalledProcessError as e: logging.error('The following idf.py command has failed: {}'.format(' '.join(cmd))) - logging.error('Working directory: {}'.format(workdir)) - logging.error('Stdout: {}'.format(e.stdout)) - logging.error('Stderr: {}'.format(e.stderr)) + logging.error(f'Working directory: {workdir}') + logging.error(f'Stdout: {e.stdout}') + logging.error(f'Stderr: {e.stderr}') raise -def run_cmake(*cmake_args: str, - env: typing.Optional[EnvDict] = None, - check: bool = True, - workdir: typing.Optional[Union[Path,str]] = None) -> subprocess.CompletedProcess: +def run_cmake( + *cmake_args: str, + env: EnvDict | None = None, + check: bool = True, + workdir: Path | str | None = None, +) -> subprocess.CompletedProcess: """ Run cmake command with given arguments, raise an exception on failure :param cmake_args: arguments to pass cmake @@ -120,7 +132,7 @@ def run_cmake(*cmake_args: str, if workdir: build_dir = Path(workdir, 'build') else: - build_dir = (Path(os.getcwd()) / 'build') + build_dir = Path(os.getcwd()) / 'build' build_dir.mkdir(parents=True, exist_ok=True) @@ -129,18 +141,24 @@ def run_cmake(*cmake_args: str, logging.debug('running {} in {}'.format(' '.join(cmd), build_dir)) try: return subprocess.run( - cmd, env=env, cwd=build_dir, - check=check, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - text=True, encoding='utf-8', errors='backslashreplace') + cmd, + env=env, + cwd=build_dir, + check=check, + capture_output=True, + text=True, + encoding='utf-8', + errors='backslashreplace', + ) except subprocess.CalledProcessError as e: logging.error('The following cmake command has failed: {}'.format(' '.join(cmd))) - logging.error('Working directory: {}'.format(workdir)) - logging.error('Stdout: {}'.format(e.stdout)) - logging.error('Stderr: {}'.format(e.stderr)) + logging.error(f'Working directory: {workdir}') + logging.error(f'Stdout: {e.stdout}') + logging.error(f'Stderr: {e.stderr}') raise -def run_cmake_and_build(*cmake_args: str, env: typing.Optional[EnvDict] = None) -> None: +def run_cmake_and_build(*cmake_args: str, env: EnvDict | None = None) -> None: """ Run cmake command with given arguments and build afterwards, raise an exception on failure :param cmake_args: arguments to pass cmake diff --git a/tools/test_build_system/test_cmake.py b/tools/test_build_system/test_cmake.py index 2917babe401..170ead23a82 100644 --- a/tools/test_build_system/test_cmake.py +++ b/tools/test_build_system/test_cmake.py @@ -286,7 +286,9 @@ def test_cmake_preset_basic_functionality(test_app_copy: Path, default_idf_env: # Test default preset auto-selection ret = run_idf_py('reconfigure') - assert "CMake presets file found but no preset name given; using 'default' preset" in ret.stderr + # This informational note is emitted via the shared esp_pylib logger's note(), which writes + # to stdout (warnings/errors go to stderr). + assert "CMake presets file found but no preset name given; using 'default' preset" in ret.stdout assert (test_app_copy / 'build' / 'default').is_dir() assert (test_app_copy / 'build' / 'default' / 'sdkconfig').is_file() # Verify that sdkconfig is NOT in the project root, even when no preset is specified but auto-selected diff --git a/tools/test_build_system/test_common.py b/tools/test_build_system/test_common.py index bec4328c95d..c715c457ff1 100644 --- a/tools/test_build_system/test_common.py +++ b/tools/test_build_system/test_common.py @@ -95,9 +95,10 @@ def test_hints_no_color_output_when_noninteractive(idf_py: IdfPyFunc) -> None: with pytest.raises(subprocess.CalledProcessError) as exc_info: idf_py('build') - # Should not actually include a color escape sequence! - # Change the assert to the correct value once the bug is fixed. - assert '\x1b[0;33mHINT: esp_chip_info.h' in exc_info.value.stderr + # the shared esp_pylib logger drops color escape sequences on + # non-interactive (non-TTY) output, so the hint appears without any ANSI color codes. + assert 'esp_chip_info.h' in exc_info.value.stdout + assert '\x1b[' not in exc_info.value.stdout @pytest.mark.usefixtures('test_app_copy') @@ -442,7 +443,7 @@ def test_deprecation_warning(idf_py: IdfPyFunc) -> None: logging.info('Deprecation warning check') ret = idf_py('post_debug', check=False) # click warning - assert 'Error: Command "post_debug" is deprecated since v4.4 and was removed in v5.0.' in ret.stderr + assert 'Command "post_debug" is deprecated since v4.4 and was removed in v5.0.' in ret.stderr ret = idf_py('efuse_common_table', check=False) # cmake warning @@ -540,8 +541,8 @@ def test_hints_components_loading( ) ret = idf_py('build', check=False) - assert 'HINT FROM IDF COMPONENT' in ret.stderr, 'Hint from IDF component should be displayed in build output' - assert 'HINT FROM PROJECT COMPONENT' in ret.stderr, ( + assert 'HINT FROM IDF COMPONENT' in ret.stdout, 'Hint from IDF component should be displayed in build output' + assert 'HINT FROM PROJECT COMPONENT' in ret.stdout, ( 'Hint from project component should be displayed in build output' ) diff --git a/tools/test_build_system/test_idf_extension.py b/tools/test_build_system/test_idf_extension.py index 766eae6fa65..5e57f05e4a5 100644 --- a/tools/test_build_system/test_idf_extension.py +++ b/tools/test_build_system/test_idf_extension.py @@ -17,6 +17,7 @@ import yaml from test_build_system_helpers import EnvDict from test_build_system_helpers import IdfPyFunc from test_build_system_helpers import find_python +from test_build_system_helpers import normalize_output from test_build_system_helpers import replace_in_file from test_build_system_helpers import run_idf_py @@ -189,7 +190,7 @@ def test_extension_from_component(idf_py: IdfPyFunc, test_app_copy: Path) -> Non idf_py('reconfigure') ret = idf_py('--help') assert 'test-component-action' in ret.stdout - expected_info = f'INFO: Loaded component extension from "{os.path.join("components", "test_component")}"' + expected_info = f'Loaded component extension from "{os.path.join("components", "test_component")}"' assert expected_info in ret.stdout ret = idf_py('test-component-action') assert 'Test extension action executed - component extension' in ret.stdout @@ -212,7 +213,7 @@ def test_extension_from_component_invalid_syntax(idf_py: IdfPyFunc, test_app_cop idf_ext_py = component_dir / 'idf_ext.py' idf_ext_py.write_text('def some_function() # no ":" at the end - INVALID SYNTAX') ret = idf_py('--help') - assert 'Warning: Failed to import extension' in ret.stderr + assert 'Failed to import extension' in ret.stderr idf_ext_py.write_text( textwrap.dedent(""" @@ -221,7 +222,7 @@ def test_extension_from_component_invalid_syntax(idf_py: IdfPyFunc, test_app_cop """) ) ret = idf_py('--help') - assert "has no attribute 'action_extensions'" in ret.stderr + assert "has no attribute 'action_extensions'" in normalize_output(ret.stderr) idf_ext_py.write_text( textwrap.dedent( @@ -241,7 +242,7 @@ def test_extension_from_component_invalid_syntax(idf_py: IdfPyFunc, test_app_cop '\n', ) ret = idf_py('--help') - assert 'Attribute "version" is required in custom extension.' in ret.stderr + assert 'Attribute "version" is required in custom extension.' in normalize_output(ret.stderr) @pytest.mark.usefixtures('test_app_copy') @@ -329,6 +330,7 @@ def test_extension_entrypoint_declarative_value_duplicate( ret = idf_py('--help') assert action1_name not in ret.stdout assert action2_name not in ret.stdout + assert 'name collision detected for - duplicate_test_ext:action_extensions' in ret.stderr assert entry_point1_name in ret.stderr assert entry_point2_name in ret.stderr @@ -353,10 +355,10 @@ def test_extension_entrypoint_default_declarative_value( ) ret = idf_py('--help') - assert f'Entry point "{entry_point_name}" has declarative value "idf_ext:action_extensions"' in ret.stderr + stderr = normalize_output(ret.stderr) + assert f'Entry point "{entry_point_name}" has declarative value "idf_ext:action_extensions"' in stderr assert ( - 'For external components, it is recommended to use name like <>_ext:action_extensions' - in ret.stderr + 'For external components, it is recommended to use name like <>_ext:action_extensions' in stderr ) @@ -374,8 +376,9 @@ def test_extension_entrypoint_non_existing_module( ) ret = idf_py('--help') - assert f'Failed to load entry point extension "{entry_point_name}"' in ret.stderr - assert "No module named 'non_existing_module'" in ret.stderr + stderr = ret.stderr + assert f'Failed to load entry point extension "{entry_point_name}"' in stderr + assert "No module named 'non_existing_module'" in stderr @pytest.mark.usefixtures('test_app_copy') @@ -406,13 +409,12 @@ def test_extension_entrypoint_conflicting_names( ) ret = idf_py('--help') - assert "Action 'bootloader' already defined. External action will not be added." in ret.stderr + stderr = ret.stderr + assert "Action 'bootloader' already defined. External action will not be added." in stderr assert 'This action conflicts with built-in action' not in ret.stdout - assert ( - "Action 'my-custom-action' has aliases ['clean'] that conflict with existing actions or aliases" in ret.stderr - ) + assert "Action 'my-custom-action' has aliases ['clean'] that conflict with existing actions or aliases" in stderr assert 'Custom action with conflicting aliases' not in ret.stdout - assert "Global option ['--project-dir'] already defined. External option will not be added." in ret.stderr + assert "Global option ['--project-dir'] already defined. External option will not be added." in stderr assert 'This global option conflicts with existing one' not in ret.stdout diff --git a/tools/test_build_system/test_non_default_target.py b/tools/test_build_system/test_non_default_target.py index 6a34f338974..dbd01d0c8b8 100644 --- a/tools/test_build_system/test_non_default_target.py +++ b/tools/test_build_system/test_non_default_target.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import logging import shutil @@ -8,6 +8,7 @@ import pytest from test_build_system_helpers import EnvDict from test_build_system_helpers import IdfPyFunc from test_build_system_helpers import file_contains +from test_build_system_helpers import normalize_output from test_build_system_helpers import run_cmake ESP32C3_TARGET = 'esp32c3' @@ -37,7 +38,7 @@ def test_target_from_environment_idf_py(idf_py: IdfPyFunc, default_idf_env: EnvD opts = opts or [] ret = idf_py(*opts, 'reconfigure', check=False) assert ret.returncode == 2 - assert errmsg in ret.stderr + assert normalize_output(errmsg) in normalize_output(ret.stderr) idf_py('set-target', ESP32S2_TARGET) default_idf_env.update({'IDF_TARGET': ESP32_TARGET}) @@ -91,7 +92,7 @@ def test_target_consistency_cmake(default_idf_env: EnvDict, test_app_copy: Path) opts = opts or [] ret = run_cmake(*opts, '-G', 'Ninja', '..', env=default_idf_env, check=False) assert ret.returncode == 1 - assert errmsg in ret.stderr + assert normalize_output(errmsg) in normalize_output(ret.stderr) run_cmake('-G', 'Ninja', '..') diff --git a/tools/test_idf_py/conftest.py b/tools/test_idf_py/conftest.py new file mode 100644 index 00000000000..2f7f9c55f65 --- /dev/null +++ b/tools/test_idf_py/conftest.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +import os +import typing + +import pytest + + +@pytest.fixture(autouse=True, scope='session') +def idf_py_terminal_env() -> typing.Generator[None, None, None]: + """Set terminal env so idf.py subprocesses produce consistent output. + + COLUMNS=200 raises Rich's non-TTY default of 80, preventing most line wrapping. + NO_COLOR=1 strips ANSI escape codes. + """ + keys = ('COLUMNS', 'LINES', 'NO_COLOR', 'FORCE_COLOR', 'PY_COLORS', 'TERM') + saved = {k: os.environ.get(k) for k in keys} + os.environ['COLUMNS'] = '200' + os.environ['LINES'] = '40' + os.environ['NO_COLOR'] = '1' + for k in ('FORCE_COLOR', 'PY_COLORS'): + os.environ.pop(k, None) + yield + for k, v in saved.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v diff --git a/tools/test_idf_py/test_hints.py b/tools/test_idf_py/test_hints.py index 4459c6e499e..b1963e4f397 100755 --- a/tools/test_idf_py/test_hints.py +++ b/tools/test_idf_py/test_hints.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import os import sys @@ -55,7 +55,7 @@ class TestHintsMassages(unittest.TestCase): with open(error_filename, 'w') as f: f.write(error) for generated_hint in generate_hints(f.name): - self.assertEqual(generated_hint, hint) + self.assertIn(generated_hint, hint) def tearDown(self) -> None: safe_cleanup_tmpdir(self.tmpdir) @@ -66,7 +66,7 @@ def run_idf(args: list[str], cwd: Path) -> str: cmd = [sys.executable, os.path.join(os.environ['IDF_PATH'], 'tools', 'idf.py')] try: proc = run(cmd + args, capture_output=True, cwd=cwd, text=True, timeout=10 * 60) - return str(proc.stdout + proc.stderr) + return proc.stdout + proc.stderr except TimeoutExpired as e: # Print captured output on timeout to help with debugging print(f'\n{"=" * 80}') diff --git a/tools/test_idf_py/test_idf_py.py b/tools/test_idf_py/test_idf_py.py index bd673c1a6d2..b59a6a52917 100755 --- a/tools/test_idf_py/test_idf_py.py +++ b/tools/test_idf_py/test_idf_py.py @@ -33,28 +33,6 @@ py_actions_path = os.path.normpath(os.path.join(current_dir, '..', 'idf_py_actio link_path = os.path.join(py_actions_path, 'test_ext') -# As idf.py uses rich-click, unite modification variables to ensure constant results on various CI terminals -_idf_py_test_env_saved: dict[str, str | None] = {} - - -def setUpModule() -> None: - for key in ('COLUMNS', 'LINES', 'NO_COLOR', 'FORCE_COLOR', 'PY_COLORS', 'TERM'): - _idf_py_test_env_saved[key] = os.environ.get(key) - os.environ['COLUMNS'] = '200' - os.environ['LINES'] = '40' - os.environ['NO_COLOR'] = '1' - for unset in ('FORCE_COLOR', 'PY_COLORS'): - os.environ.pop(unset, None) - - -def tearDownModule() -> None: - for key, previous in _idf_py_test_env_saved.items(): - if previous is None: - os.environ.pop(key, None) - else: - os.environ[key] = previous - - class TestWithoutExtensions(TestCase): @classmethod def setUpClass(cls): @@ -164,9 +142,11 @@ class TestDependencyManagement(TestWithoutExtensions): standalone_mode=False, ) sys.stderr = sys.__stderr__ + # The shared esp_pylib logger wraps long lines at the console width; collapse whitespace + # so the substring check does not depend on where the line break lands. self.assertIn( - 'WARNING: Commands "all", "clean" are found in the list of commands more than once.', - capturedOutput.getvalue(), + 'Commands "all", "clean" are found in the list of commands more than once.', + ' '.join(capturedOutput.getvalue().split()), ) sys.stderr = capturedOutput @@ -176,7 +156,8 @@ class TestDependencyManagement(TestWithoutExtensions): ) sys.stderr = sys.__stderr__ self.assertIn( - 'WARNING: Command "clean" is found in the list of commands more than once.', capturedOutput.getvalue() + 'Command "clean" is found in the list of commands more than once.', + ' '.join(capturedOutput.getvalue().split()), ) @@ -259,7 +240,7 @@ class TestDeprecations(TestWithoutExtensions): ) except subprocess.CalledProcessError as e: output = e.output.decode('utf-8', 'ignore').replace('\r\n', '\n') - self.assertIn('Error: Command "test-2" is deprecated and was removed\n', output) + self.assertIn('Command "test-2" is deprecated and was removed', output) def test_exit_with_error_for_option(self): try: @@ -270,7 +251,7 @@ class TestDeprecations(TestWithoutExtensions): ) except subprocess.CalledProcessError as e: self.assertIn( - 'Error: Option "test_5" is deprecated since v2.0 and was removed in v3.0.', + 'Option "test_5" is deprecated since v2.0 and was removed in v3.0.', e.output.decode('utf-8', 'ignore'), ) @@ -294,19 +275,18 @@ class TestDeprecations(TestWithoutExtensions): env=os.environ, stderr=subprocess.STDOUT, ).decode('utf-8', 'ignore') - self.assertIn('Warning: Option "test_sub_1" is deprecated and will be removed in future versions.', output) + output = ' '.join(output.split()) + self.assertIn('Option "test_sub_1" is deprecated and will be removed in future versions.', output) self.assertIn( - 'Warning: Command "test-1" is deprecated and will be removed in future versions. ' - 'Please use alternative command.', + 'Command "test-1" is deprecated and will be removed in future versions. Please use alternative command.', output, ) - self.assertIn('Warning: Option "test_1" is deprecated and will be removed in future versions.', output) + self.assertIn('Option "test_1" is deprecated and will be removed in future versions.', output) self.assertIn( - 'Warning: Option "test_2" is deprecated and will be removed in future versions. ' - 'Please update your parameters.', + 'Option "test_2" is deprecated and will be removed in future versions. Please update your parameters.', output, ) - self.assertIn('Warning: Option "test_3" is deprecated and will be removed in future versions.', output) + self.assertIn('Option "test_3" is deprecated and will be removed in future versions.', output) self.assertNotIn('"test-0" is deprecated', output) self.assertNotIn('"test_0" is deprecated', output)