From a78ee285c92881dd43dd1f3d49bb7be4776dcd41 Mon Sep 17 00:00:00 2001 From: Frantisek Hrbata Date: Tue, 23 Jun 2026 10:53:06 +0200 Subject: [PATCH 1/4] fix: harden build against empty toolchain output On some Windows systems, antivirus, endpoint-security or DLP/encryption software intercepts short-lived toolchain processes and strips their stdout when the build captures it through a pipe, while the same command prints its output normally when run by hand. The tool exits successfully but returns nothing, and each build step that reads toolchain output then failed with a different, cryptic error far from the real cause: - CMake configuration aborted with "Unknown arguments specified" in components/xtensa/project_include.cmake, or "check_expected_tool_version invoked with incorrect arguments" in components/esp_common. - ldgen turned the empty objdump output into an opaque pyparsing "Expected 'In archive'" traceback. - idf_tools.py silently reported the compiler/debugger version as "unknown", sending users into a fruitless reinstall loop. Detect the empty result at each consumer and fail (or warn) with an actionable message that names the likely cause and the remedy: - tools/cmake/compiler_query.cmake: new __compiler_query() helper runs a compiler query and fails with a clear error on empty or failed output. It is a standalone module included by the build system utilities, so that it is available to the esp_common and xtensa project_include.cmake files that call it. The xtensa if() arguments are now quoted so an empty result no longer collapses into a parse error. - tools/ldgen/ldgen.py: _run_objdump() rejects empty objdump output, and non-empty-but-unparsable section info is caught and re-raised as a clear LdGenFailure instead of a raw pyparsing traceback. - tools/idf_tools.py: empty version output now warns with the cause and returns UNKNOWN_VERSION instead of silently reporting "unknown". Closes https://github.com/espressif/esp-idf/issues/18727 Signed-off-by: Frantisek Hrbata --- components/esp_common/project_include.cmake | 6 +-- components/xtensa/project_include.cmake | 13 ++--- tools/cmake/compiler_query.cmake | 40 ++++++++++++++++ tools/cmake/utilities.cmake | 6 +++ tools/idf_tools.py | 20 +++++++- tools/ldgen/ldgen.py | 53 +++++++++++++++++++-- 6 files changed, 118 insertions(+), 20 deletions(-) create mode 100644 tools/cmake/compiler_query.cmake diff --git a/components/esp_common/project_include.cmake b/components/esp_common/project_include.cmake index 2665404c50b..b7b3f7e5c43 100644 --- a/components/esp_common/project_include.cmake +++ b/components/esp_common/project_include.cmake @@ -2,11 +2,7 @@ # Warn if the toolchain version doesn't match # if(NOT (${target} STREQUAL "linux" OR CMAKE_C_COMPILER_ID MATCHES "Clang")) - execute_process( - COMMAND ${CMAKE_C_COMPILER} -dumpmachine - OUTPUT_VARIABLE toolchain_name - OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_QUIET) + __compiler_query(toolchain_name ${CMAKE_C_COMPILER} -dumpmachine) check_expected_tool_version(${toolchain_name} ${CMAKE_C_COMPILER}) endif() diff --git a/components/xtensa/project_include.cmake b/components/xtensa/project_include.cmake index f0c5bee8667..433292b6024 100644 --- a/components/xtensa/project_include.cmake +++ b/components/xtensa/project_include.cmake @@ -2,20 +2,13 @@ if(CMAKE_C_COMPILER_ID MATCHES "Clang") # without '--target' option 'clang -dumpmachine' prints default target arch and it might be not Xtensa # so use `-print-targets` option - execute_process( - COMMAND ${CMAKE_C_COMPILER} -print-targets - OUTPUT_VARIABLE dump_machine - ) + __compiler_query(dump_machine ${CMAKE_C_COMPILER} -print-targets) else() - execute_process( - COMMAND ${CMAKE_C_COMPILER} -dumpmachine - OUTPUT_VARIABLE dump_machine - OUTPUT_STRIP_TRAILING_WHITESPACE - ) + __compiler_query(dump_machine ${CMAKE_C_COMPILER} -dumpmachine) endif() message(STATUS "Compiler supported targets: ${dump_machine}") -if(NOT (${CMAKE_SYSTEM_NAME} STREQUAL "Generic" AND ${dump_machine} MATCHES xtensa)) +if(NOT ("${CMAKE_SYSTEM_NAME}" STREQUAL "Generic" AND "${dump_machine}" MATCHES "xtensa")) message(FATAL_ERROR "Internal error, toolchain has not been set correctly by project " "(or an invalid CMakeCache.txt file has been generated somehow)") endif() diff --git a/tools/cmake/compiler_query.cmake b/tools/cmake/compiler_query.cmake new file mode 100644 index 00000000000..5e8e93f5a45 --- /dev/null +++ b/tools/cmake/compiler_query.cmake @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 + +# __compiler_query +# +# Run a compiler query command (e.g. " -dumpmachine") given in ARGN +# and store its trimmed stdout in the variable named by "output_var". +# +# Fails with an actionable error if the command fails or returns no output. On +# some Windows systems, antivirus, endpoint-security or DLP/encryption software +# intercepts short-lived toolchain processes and strips their stdout when it is +# captured by the build system, while the same command works when run directly +# in a terminal. Without this guard the empty result collapses the callers' +# parsing into a cryptic CMake error (see +# https://github.com/espressif/esp-idf/issues/18727). +# +# This module is included by the build system utilities, so that +# __compiler_query is available to the esp_common and xtensa +# project_include.cmake files that call it. +function(__compiler_query output_var) + execute_process( + COMMAND ${ARGN} + OUTPUT_VARIABLE query_output + RESULT_VARIABLE query_result + OUTPUT_STRIP_TRAILING_WHITESPACE) + + if(NOT query_result EQUAL 0 OR query_output STREQUAL "") + string(REPLACE ";" " " query_command "${ARGN}") + message(FATAL_ERROR + "Failed to query the compiler: '${query_command}' (result: ${query_result}).\n" + "The command produced no output when run by the build system. This is usually caused " + "by antivirus, endpoint-security or DLP/encryption software intercepting the compiler " + "process and discarding its output; the same command often works when run directly in " + "a terminal.\n" + "Add an exclusion for the ESP-IDF tools directory in that software (on a managed " + "machine you may need your IT department), then run 'idf.py fullclean' and build again.") + endif() + + set(${output_var} "${query_output}" PARENT_SCOPE) +endfunction() diff --git a/tools/cmake/utilities.cmake b/tools/cmake/utilities.cmake index 122a133e790..0943a430e52 100644 --- a/tools/cmake/utilities.cmake +++ b/tools/cmake/utilities.cmake @@ -393,3 +393,9 @@ function(remove_duplicated_flags FLAGS UNIQFLAGS) # Return that string to the caller set(${UNIQFLAGS} "${FLAGS_LIST}" PARENT_SCOPE) endfunction() + + +# __compiler_query is defined in a standalone module, included here so it is +# available to the esp_common and xtensa project_include.cmake files that call +# it. +include(${CMAKE_CURRENT_LIST_DIR}/compiler_query.cmake) diff --git a/tools/idf_tools.py b/tools/idf_tools.py index f618a98ecbc..fed54a1e381 100755 --- a/tools/idf_tools.py +++ b/tools/idf_tools.py @@ -1004,7 +1004,25 @@ class IDFTool(object): f'non-zero exit code ({e.returncode}) with message: {e.stderr.decode("utf-8", errors="ignore")}' ) # type: ignore - return self.parse_tool_version(version_cmd_result.decode('utf-8')) + version_str = version_cmd_result.decode('utf-8') + if not version_str.strip(): + # The tool ran and exited successfully, but produced no output when its + # output was captured. On some Windows systems, antivirus, endpoint-security + # or DLP/encryption software intercepts short-lived toolchain processes and + # strips their stdout when it is captured through a pipe, while the same + # command works when run directly in a terminal. Surface an actionable hint + # instead of silently reporting the version as 'unknown', which otherwise + # sends users into a fruitless reinstall loop. + # See https://github.com/espressif/esp-idf/issues/18727 + warn( + f'tool {self.name} ran but returned no version output. This is usually caused by ' + 'antivirus, endpoint-security or DLP/encryption software stripping the output of ' + 'toolchain processes; the same command often works when run directly in a terminal. ' + 'Add an exclusion for the ESP-IDF tools directory in that software. If the problem ' + 'persists, run the tool manually to check for a missing DLL.' + ) + return UNKNOWN_VERSION + return self.parse_tool_version(version_str) def get_version_from_file(self, version: str) -> str: """ diff --git a/tools/ldgen/ldgen.py b/tools/ldgen/ldgen.py index 681020fa36f..22661ee6df4 100755 --- a/tools/ldgen/ldgen.py +++ b/tools/ldgen/ldgen.py @@ -33,6 +33,40 @@ def _update_environment(args): os.environ.update(env) +def _run_objdump(objdump, library): + """Run ``objdump -h`` on a library and return its output. + + On some Windows systems, antivirus, endpoint-security or DLP/encryption + software intercepts short-lived toolchain processes and strips their output + when the build captures it, so objdump exits successfully but returns empty + output, while the same command works when run by hand. The output is read + through a pipe so that this empty result is reliably detectable: it is + rejected here with an actionable error instead of being fed to the parser + (which would otherwise report it as a confusing pyparsing error). Capturing + to a file is deliberately avoided: it would not be guaranteed complete + either, and a truncated-but-non-empty result could be parsed into a wrong + linker script instead of failing. See + https://github.com/espressif/esp-idf/issues/18665 and + https://github.com/espressif/esp-idf/issues/18727. + """ + new_env = os.environ.copy() + # Force the C locale so objdump emits the English 'In archive' header that + # the section parser expects, regardless of the host locale (see + # https://github.com/espressif/esp-idf/issues/7903). + new_env['LC_ALL'] = 'C' + + output = subprocess.check_output([objdump, '-h', library], env=new_env).decode() + if not output.strip(): + raise LdGenFailure( + f"'{objdump} -h {library}' ran successfully but returned no output. The toolchain ran " + 'but its output was empty when captured by the build system. This is usually caused by ' + 'antivirus, endpoint-security or DLP/encryption software stripping the output of ' + 'toolchain processes; the same command often works when run directly in a terminal. ' + 'Add an exclusion for the ESP-IDF tools directory in that software, then build again.' + ) + return output + + def main(): argparser = argparse.ArgumentParser(description='ESP-IDF linker script generator') @@ -126,11 +160,22 @@ def main(): for library in libraries_file: library = library.strip() if library: - new_env = os.environ.copy() - new_env['LC_ALL'] = 'C' - dump = StringIO(subprocess.check_output([objdump, '-h', library], env=new_env).decode()) + dump = StringIO(_run_objdump(objdump, library)) dump.name = library - sections_infos.add_sections_info(dump) + try: + sections_infos.add_sections_info(dump) + except ParseException as e: + # Non-empty but unparsable section info (for example truncated or + # corrupted toolchain output) is reported here rather than allowed to + # propagate as a raw pyparsing traceback. The same root cause as the + # empty case in _run_objdump applies. + raise LdGenFailure( + f'failed to parse section information from {library}. The toolchain output ' + 'is incomplete or corrupted. This can be caused by antivirus, ' + 'endpoint-security or DLP/encryption software tampering with the output of ' + 'toolchain processes; the same command often works when run directly in a ' + f'terminal. Add an exclusion for the ESP-IDF tools directory, then build again.\n{e}' + ) generation_model = Generation(check_mapping, check_mapping_exceptions) From 8bd2ddf1bf7f6fb5591e696e28858801631b299b Mon Sep 17 00:00:00 2001 From: Frantisek Hrbata Date: Mon, 10 Aug 2026 11:54:50 +0200 Subject: [PATCH 2/4] style(ldgen): reformat with ruff-format release/v5.5's tools/ldgen/ldgen.py predates the ruff-format layout that pre-commit enforces, so running the hook over this file reports it as needing reformatting because of code that no change here touches. Apply ruff-format. That also resolves the E501 on the long "with open(output_path, ...)" line, since the formatter wraps the call. No behaviour change. Signed-off-by: Frantisek Hrbata --- tools/ldgen/ldgen.py | 92 +++++++++++++++++++------------------------- 1 file changed, 39 insertions(+), 53 deletions(-) diff --git a/tools/ldgen/ldgen.py b/tools/ldgen/ldgen.py index 22661ee6df4..369de57cfa0 100755 --- a/tools/ldgen/ldgen.py +++ b/tools/ldgen/ldgen.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # import argparse @@ -23,7 +23,7 @@ from pyparsing import ParseFatalException def _update_environment(args): - env = [(name, value) for (name,value) in (e.split('=',1) for e in args.env)] + env = [(name, value) for (name, value) in (e.split('=', 1) for e in args.env)] for name, value in env: value = ' '.join(value.split()) os.environ[name] = value @@ -68,71 +68,55 @@ def _run_objdump(objdump, library): def main(): - argparser = argparse.ArgumentParser(description='ESP-IDF linker script generator') - argparser.add_argument( - '--input', '-i', - help='Linker template file', - type=argparse.FileType('r')) + argparser.add_argument('--input', '-i', help='Linker template file', type=argparse.FileType('r')) fragments_group = argparser.add_mutually_exclusive_group() fragments_group.add_argument( - '--fragments', '-f', - type=argparse.FileType('r'), - help='Input fragment files', - nargs='+' + '--fragments', '-f', type=argparse.FileType('r'), help='Input fragment files', nargs='+' ) fragments_group.add_argument( - '--fragments-list', - help='Input fragment files as a semicolon-separated list', - type=str + '--fragments-list', help='Input fragment files as a semicolon-separated list', type=str ) argparser.add_argument( - '--libraries-file', + '--libraries-file', type=argparse.FileType('r'), help='File that contains the list of libraries in the build' + ) + + argparser.add_argument('--output', '-o', help='Output linker script', type=str) + + argparser.add_argument('--config', '-c', help='Project configuration') + + argparser.add_argument('--kconfig', '-k', help='IDF Kconfig file') + + argparser.add_argument( + '--check-mapping', help='Perform a check if a mapping (archive, obj, symbol) exists', action='store_true' + ) + + argparser.add_argument( + '--check-mapping-exceptions', help='Mappings exempted from check', type=argparse.FileType('r') + ) + + argparser.add_argument( + '--env', + '-e', + action='append', + default=[], + help='Environment to set when evaluating the config file', + metavar='NAME=VAL', + ) + + argparser.add_argument( + '--env-file', type=argparse.FileType('r'), - help='File that contains the list of libraries in the build') - - argparser.add_argument( - '--output', '-o', - help='Output linker script', - type=str) - - argparser.add_argument( - '--config', '-c', - help='Project configuration') - - argparser.add_argument( - '--kconfig', '-k', - help='IDF Kconfig file') - - argparser.add_argument( - '--check-mapping', - help='Perform a check if a mapping (archive, obj, symbol) exists', - action='store_true' + help='Optional file to load environment variables from. Contents ' + 'should be a JSON object where each key/value pair is a variable.', ) - argparser.add_argument( - '--check-mapping-exceptions', - help='Mappings exempted from check', - type=argparse.FileType('r') - ) - - argparser.add_argument( - '--env', '-e', - action='append', default=[], - help='Environment to set when evaluating the config file', metavar='NAME=VAL') - - argparser.add_argument('--env-file', type=argparse.FileType('r'), - help='Optional file to load environment variables from. Contents ' - 'should be a JSON object where each key/value pair is a variable.') - - argparser.add_argument( - '--objdump', - help='Path to toolchain objdump') + argparser.add_argument('--objdump', help='Path to toolchain objdump') args = argparser.parse_args() @@ -210,7 +194,9 @@ def main(): if exc.errno != errno.EEXIST: raise - with open(output_path, 'w', encoding='utf-8') as f: # only create output file after generation has succeeded + with open( + output_path, 'w', encoding='utf-8' + ) as f: # only create output file after generation has succeeded f.write(output.read()) except LdGenFailure as e: print('linker script generation failed for %s\nERROR: %s' % (input_file.name, e)) From 2a92260bc1ac9611578d4c8a577df8003c6a4934 Mon Sep 17 00:00:00 2001 From: Frantisek Hrbata Date: Mon, 17 Aug 2026 16:49:27 +0200 Subject: [PATCH 3/4] ci: pass a mypy target that the toolchain still accepts check_type_comments.py runs ignore-listed files through a relaxed mypy pass with --python-version 3.8, which overrides the python_version = 3.9 that .mypy.ini already sets. mypy dropped 3.8 as a modelling target in 1.17.0 and now rejects the flag while parsing arguments, before it reads any source: mypy: error: argument --python-version: Python 3.8 is not supported (must be 3.9 or higher) mypy check failed for: tools/ldgen/ldgen.py The hook declares mypy without a version, so which mypy is used is decided when the ci/images pre-commit-idf-v5.5 image is built. That image is based on Python 3.9, where Requires-Python caps mypy at 1.19.1, and the image currently in use carries exactly that. Any MR touching a file listed in tools/ci/mypy_ignore_list.txt therefore fails check_pre_commit deterministically, whatever the file contains. Pass 3.9 instead, matching .mypy.ini and the OLDEST_PYTHON_SUPPORTED = (3, 9) that tools/python_version_checker.py enforces on this branch. This restores the pairing master and release/v6.0 already have, where the flag mirrors .mypy.ini at 3.10. release/v5.4 and release/v5.3 need no equivalent change: their pre-commit images are based on Python 3.8, where mypy caps at 1.14.1 and the flag is still accepted. Signed-off-by: Frantisek Hrbata --- tools/ci/check_type_comments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci/check_type_comments.py b/tools/ci/check_type_comments.py index 38865b08426..e6f2e8ca679 100755 --- a/tools/ci/check_type_comments.py +++ b/tools/ci/check_type_comments.py @@ -29,7 +29,7 @@ def types_valid_ignored_rules(file_name): # type: (str) -> bool """ Run Mypy check with rules for ignore list on the given file, return TRUE if Mypy check passes """ - mypy_exit_code = subprocess.call('mypy {} --python-version 3.8 --allow-untyped-defs'.format(file_name), shell=True) + mypy_exit_code = subprocess.call('mypy {} --python-version 3.9 --allow-untyped-defs'.format(file_name), shell=True) return not bool(mypy_exit_code) From 3c9d844cbe7b5fa793e1e6ef8d0b45e71d215ddd Mon Sep 17 00:00:00 2001 From: Frantisek Hrbata Date: Mon, 17 Aug 2026 16:56:14 +0200 Subject: [PATCH 4/4] style(ci): satisfy ruff on check_type_comments.py The previous commit touches tools/ci/check_type_comments.py, which brings the file into ruff's scope for the first time in a while and surfaces two pre-existing findings: A004 Import `exit` is shadowing a Python builtin F401 `typing.List` imported but unused Import sys and call sys.exit() rather than shadowing the builtin, which is also how master writes this file. List is used, but only inside `# type:` comments that ruff cannot see, so mark the import instead of dropping it: check_type_comments.py is not on tools/ci/mypy_ignore_list.txt, so mypy checks it under disallow_untyped_defs and the name has to resolve. No behaviour change. Verified with the pinned ruff 0.9.7 (check and format --check) and with mypy 1.19.1, and both exit paths of the script still behave as before. Signed-off-by: Frantisek Hrbata --- tools/ci/check_type_comments.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/ci/check_type_comments.py b/tools/ci/check_type_comments.py index e6f2e8ca679..cc15b8f6feb 100755 --- a/tools/ci/check_type_comments.py +++ b/tools/ci/check_type_comments.py @@ -4,10 +4,10 @@ # SPDX-License-Identifier: Apache-2.0 import argparse import subprocess -from sys import exit +import sys try: - from typing import List + from typing import List # noqa: F401 except ImportError: # Only used for type annotations pass @@ -84,7 +84,7 @@ def main(): # type: () -> None print('mypy check failed for:') for file_name in type_issues: print('\t', file_name) - exit(1) + sys.exit(1) if __name__ == '__main__':