mirror of
https://github.com/espressif/esp-idf.git
synced 2026-09-22 13:01:16 +03:00
4d777fd05a
fix: harden build against empty toolchain output (v6.1) See merge request espressif/esp-idf!51643
359 lines
14 KiB
Python
Executable File
359 lines
14 KiB
Python
Executable File
#!/usr/bin/env python
|
|
#
|
|
# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
#
|
|
import argparse
|
|
import errno
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import pickle
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from io import StringIO
|
|
|
|
from ldgen.entity import EntityDB
|
|
from ldgen.fragments import parse_fragment_file
|
|
from ldgen.generation import Generation
|
|
from ldgen.ldgen_common import LdGenFailure
|
|
from ldgen.linker_script import LinkerScript
|
|
from ldgen.sdkconfig import SDKConfig
|
|
from pyparsing import ParseException
|
|
from pyparsing import ParseFatalException
|
|
|
|
_RE_SECTION_NAME = re.compile(r'^\s*\d+\s+(\.\S+)', re.MULTILINE)
|
|
|
|
|
|
def _compute_fingerprint(sections_infos, fragment_files, config_file, kconfig_file, input_file):
|
|
"""Compute a fingerprint from section names and mtimes of all inputs."""
|
|
hasher = hashlib.md5()
|
|
|
|
# Section names from objdump output
|
|
for archive, info in sorted(sections_infos.sections.items()):
|
|
names = _RE_SECTION_NAME.findall(info.content)
|
|
hasher.update((archive + ':' + ','.join(names)).encode())
|
|
|
|
# Mtimes of fragment files, the linker template, sdkconfig, kconfig. The
|
|
# template counts as an input because everything outside its mapping
|
|
# placeholders ends up in the generated script verbatim, so an edit to it
|
|
# changes the output even when no section name does.
|
|
input_files = [p.name if hasattr(p, 'name') else p for p in fragment_files]
|
|
input_files += [p.name if hasattr(p, 'name') else p for p in (input_file, config_file, kconfig_file) if p]
|
|
for path in input_files:
|
|
try:
|
|
hasher.update(f'{path}:{os.path.getmtime(path)}'.encode())
|
|
except OSError:
|
|
return None
|
|
|
|
return hasher.hexdigest()
|
|
|
|
|
|
def _can_skip_generation(output_path, fingerprint):
|
|
"""Check if fingerprint matches cached value from previous run."""
|
|
try:
|
|
with open(output_path + '.fingerprint') as f:
|
|
if f.read().strip() == fingerprint:
|
|
os.utime(output_path, None)
|
|
return True
|
|
except OSError:
|
|
pass
|
|
return False
|
|
|
|
|
|
def _save_fingerprint(output_path, fingerprint):
|
|
"""Save fingerprint for next run."""
|
|
try:
|
|
with open(output_path + '.fingerprint', 'w') as f:
|
|
f.write(fingerprint)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def _compute_lf_cache_key(fragment_files, config_file, kconfig_file):
|
|
"""Compute a cache key for parsed fragment files.
|
|
|
|
Keyed on fragment file paths+mtimes plus sdkconfig and kconfig mtimes,
|
|
because fragment parsing evaluates `if/elif/else` conditional blocks
|
|
against sdkconfig at parse time — so a sdkconfig change can change the
|
|
parsed FragmentFile output even if the fragment files themselves are
|
|
unchanged.
|
|
"""
|
|
hasher = hashlib.md5()
|
|
paths = [p.name if hasattr(p, 'name') else p for p in fragment_files]
|
|
paths += [p for p in (config_file, kconfig_file) if p]
|
|
for path in sorted(paths):
|
|
try:
|
|
hasher.update(f'{path}:{os.path.getmtime(path)}'.encode())
|
|
except OSError:
|
|
return None
|
|
return hasher.hexdigest()
|
|
|
|
|
|
def _load_lf_cache(cache_path, key):
|
|
"""Load parsed FragmentFile list from cache if key matches.
|
|
|
|
The lf cache is written and read only by ldgen, in the same build
|
|
directory where sections.ld, compiled object files, and the rest of
|
|
the build state already live. The trust boundary matches the build
|
|
system's trust boundary: anyone who can modify <output>.lfcache can
|
|
also modify sections.ld, *.o, or the toolchain binaries directly.
|
|
pickle is safe in this context; it is not exposed to untrusted input.
|
|
"""
|
|
try:
|
|
with open(cache_path, 'rb') as f:
|
|
data = pickle.load(f)
|
|
if isinstance(data, dict) and data.get('key') == key:
|
|
return data.get('fragments')
|
|
except (OSError, pickle.UnpicklingError, EOFError, AttributeError, ImportError, ValueError):
|
|
pass
|
|
return None
|
|
|
|
|
|
def _save_lf_cache(cache_path, key, fragments):
|
|
"""Save parsed FragmentFile list for next run."""
|
|
try:
|
|
with open(cache_path, 'wb') as f:
|
|
pickle.dump({'key': key, 'fragments': fragments}, f, pickle.HIGHEST_PROTOCOL)
|
|
except (OSError, pickle.PicklingError):
|
|
pass
|
|
|
|
|
|
def _update_environment(args):
|
|
env = [(name, value) for (name, value) in (e.split('=', 1) for e in args.env)]
|
|
for name, value in env:
|
|
value = ' '.join(value.split())
|
|
os.environ[name] = value
|
|
|
|
if args.env_file is not None:
|
|
env = json.load(args.env_file)
|
|
os.environ.update(env)
|
|
|
|
|
|
def _run_objdump(objdump, library):
|
|
"""Run ``objdump -h`` on a library and return its output.
|
|
|
|
On some Windows systems, antivirus, endpoint-security or DLP/encryption
|
|
software intercepts short-lived toolchain processes and strips their output
|
|
when the build captures it, so objdump exits successfully but returns empty
|
|
output, while the same command works when run by hand. The output is read
|
|
through a pipe so that this empty result is reliably detectable: it is
|
|
rejected here with an actionable error instead of being fed to the parser
|
|
(which would otherwise report it as a confusing pyparsing error). Capturing
|
|
to a file is deliberately avoided: it would not be guaranteed complete
|
|
either, and a truncated-but-non-empty result could be parsed into a wrong
|
|
linker script instead of failing. See
|
|
https://github.com/espressif/esp-idf/issues/18665 and
|
|
https://github.com/espressif/esp-idf/issues/18727.
|
|
"""
|
|
new_env = os.environ.copy()
|
|
# Force the C locale so objdump emits the English 'In archive' header that
|
|
# the section parser expects, regardless of the host locale (see
|
|
# https://github.com/espressif/esp-idf/issues/7903).
|
|
new_env['LC_ALL'] = 'C'
|
|
|
|
output = subprocess.check_output([objdump, '-h', library], env=new_env).decode()
|
|
if not output.strip():
|
|
raise LdGenFailure(
|
|
f"'{objdump} -h {library}' ran successfully but returned no output. The toolchain ran "
|
|
'but its output was empty when captured by the build system. This is usually caused by '
|
|
'antivirus, endpoint-security or DLP/encryption software stripping the output of '
|
|
'toolchain processes; the same command often works when run directly in a terminal. '
|
|
'Add an exclusion for the ESP-IDF tools directory in that software, then build again.'
|
|
)
|
|
return output
|
|
|
|
|
|
def main():
|
|
argparser = argparse.ArgumentParser(description='ESP-IDF linker script generator')
|
|
|
|
argparser.add_argument('--input', '-i', help='Linker template file', type=argparse.FileType('r'))
|
|
|
|
fragments_group = argparser.add_mutually_exclusive_group()
|
|
|
|
fragments_group.add_argument(
|
|
'--fragments', '-f', type=argparse.FileType('r'), help='Input fragment files', nargs='+'
|
|
)
|
|
|
|
fragments_group.add_argument(
|
|
'--fragments-list', help='Input fragment files as a semicolon-separated list', type=str
|
|
)
|
|
|
|
argparser.add_argument(
|
|
'--libraries-file', type=argparse.FileType('r'), help='File that contains the list of libraries in the build'
|
|
)
|
|
|
|
argparser.add_argument(
|
|
'--mutable-libraries-file',
|
|
type=argparse.FileType('r'),
|
|
help='File that contains the list of mutable libraries in the build',
|
|
)
|
|
|
|
argparser.add_argument('--output', '-o', help='Output linker script', type=str)
|
|
|
|
argparser.add_argument('--config', '-c', help='Project configuration')
|
|
|
|
argparser.add_argument('--kconfig', '-k', help='IDF Kconfig file')
|
|
|
|
argparser.add_argument(
|
|
'--check-mapping', help='Perform a check if a mapping (archive, obj, symbol) exists', action='store_true'
|
|
)
|
|
|
|
argparser.add_argument(
|
|
'--check-mapping-exceptions', help='Mappings exempted from check', type=argparse.FileType('r')
|
|
)
|
|
|
|
argparser.add_argument(
|
|
'--env',
|
|
'-e',
|
|
action='append',
|
|
default=[],
|
|
help='Environment to set when evaluating the config file',
|
|
metavar='NAME=VAL',
|
|
)
|
|
|
|
argparser.add_argument(
|
|
'--env-file',
|
|
type=argparse.FileType('r'),
|
|
help='Optional file to load environment variables from. Contents '
|
|
'should be a JSON object where each key/value pair is a variable.',
|
|
)
|
|
|
|
argparser.add_argument('--objdump', help='Path to toolchain objdump')
|
|
|
|
argparser.add_argument('--debug', '-d', help='Print debugging information.', action='store_true')
|
|
|
|
args = argparser.parse_args()
|
|
|
|
input_file = args.input
|
|
libraries_file = args.libraries_file
|
|
mutable_libraries_file = args.mutable_libraries_file or []
|
|
config_file = args.config
|
|
output_path = args.output
|
|
kconfig_file = args.kconfig
|
|
objdump = args.objdump
|
|
|
|
fragment_files = []
|
|
if args.fragments_list:
|
|
fragment_files = args.fragments_list.split(';')
|
|
elif args.fragments:
|
|
fragment_files = args.fragments
|
|
|
|
check_mapping = args.check_mapping
|
|
if args.check_mapping_exceptions:
|
|
check_mapping_exceptions = [line.strip() for line in args.check_mapping_exceptions]
|
|
else:
|
|
check_mapping_exceptions = None
|
|
|
|
no_cache = os.environ.get('LDGEN_NO_CACHE') == '1'
|
|
if no_cache:
|
|
print('Linker script generation caches disabled by LDGEN_NO_CACHE')
|
|
|
|
try:
|
|
sections_infos = EntityDB()
|
|
for library in libraries_file:
|
|
library = library.strip()
|
|
if library:
|
|
dump = StringIO(_run_objdump(objdump, library))
|
|
dump.name = library
|
|
try:
|
|
sections_infos.add_sections_info(dump)
|
|
except ParseException as e:
|
|
# Non-empty but unparsable section info (for example truncated or
|
|
# corrupted toolchain output) is reported here rather than allowed to
|
|
# propagate as a raw pyparsing traceback. The same root cause as the
|
|
# empty case in _run_objdump applies.
|
|
raise LdGenFailure(
|
|
f'failed to parse section information from {library}. The toolchain output '
|
|
'is incomplete or corrupted. This can be caused by antivirus, '
|
|
'endpoint-security or DLP/encryption software tampering with the output of '
|
|
'toolchain processes; the same command often works when run directly in a '
|
|
f'terminal. Add an exclusion for the ESP-IDF tools directory, then build again.\n{e}'
|
|
)
|
|
|
|
# Check if we can skip generation entirely — section names and other
|
|
# inputs unchanged since last run.
|
|
fingerprint = (
|
|
None
|
|
if no_cache
|
|
else _compute_fingerprint(sections_infos, fragment_files, config_file, kconfig_file, input_file)
|
|
)
|
|
if (
|
|
output_path
|
|
and fingerprint
|
|
and os.path.exists(output_path)
|
|
and _can_skip_generation(output_path, fingerprint)
|
|
):
|
|
print('Skipping linker script generation, section names unchanged')
|
|
sys.exit(0)
|
|
|
|
mutable_libs = [lib.strip() for lib in mutable_libraries_file]
|
|
generation_model = Generation(check_mapping, check_mapping_exceptions, mutable_libs, args.debug)
|
|
|
|
_update_environment(args) # assign args.env and args.env_file to os.environ
|
|
|
|
sdkconfig = SDKConfig(kconfig_file, config_file)
|
|
|
|
# Try to load parsed fragment files from cache. The lf cache is
|
|
# complementary to the fingerprint skip above: if we get here, section
|
|
# names changed, but the fragment files themselves may still be
|
|
# identical and don't need re-parsing.
|
|
lf_cache_path = None if no_cache or not output_path else output_path + '.lfcache'
|
|
lf_cache_key = None if no_cache else _compute_lf_cache_key(fragment_files, config_file, kconfig_file)
|
|
parsed_fragments = None
|
|
if lf_cache_path and lf_cache_key:
|
|
parsed_fragments = _load_lf_cache(lf_cache_path, lf_cache_key)
|
|
if parsed_fragments is not None:
|
|
print('Skipping linker fragment parsing, fragment files unchanged')
|
|
|
|
if parsed_fragments is None:
|
|
parsed_fragments = []
|
|
for fragment_file in fragment_files:
|
|
try:
|
|
parsed = parse_fragment_file(fragment_file, sdkconfig)
|
|
except (ParseException, ParseFatalException) as e:
|
|
# ParseException is raised on incorrect grammar
|
|
# ParseFatalException is raised on correct grammar, but inconsistent contents (ex. duplicate
|
|
# keys, key unsupported by fragment, unexpected number of values, etc.)
|
|
raise LdGenFailure(f'failed to parse {fragment_file}\n{e}')
|
|
parsed_fragments.append(parsed)
|
|
if lf_cache_path and lf_cache_key:
|
|
_save_lf_cache(lf_cache_path, lf_cache_key, parsed_fragments)
|
|
|
|
for parsed in parsed_fragments:
|
|
generation_model.add_fragments_from_file(parsed)
|
|
|
|
non_contiguous_sram = sdkconfig.evaluate_expression('SOC_MEM_NON_CONTIGUOUS_SRAM')
|
|
mapping_rules = generation_model.generate(sections_infos, non_contiguous_sram)
|
|
|
|
script_model = LinkerScript(input_file)
|
|
script_model.fill(mapping_rules)
|
|
|
|
with tempfile.TemporaryFile('w+') as output:
|
|
script_model.write(output)
|
|
output.seek(0)
|
|
|
|
if not os.path.exists(os.path.dirname(output_path)):
|
|
try:
|
|
os.makedirs(os.path.dirname(output_path))
|
|
except OSError as exc:
|
|
if exc.errno != errno.EEXIST:
|
|
raise
|
|
|
|
with open(
|
|
output_path, 'w', encoding='utf-8'
|
|
) as f: # only create output file after generation has succeeded
|
|
f.write(output.read())
|
|
|
|
if output_path and fingerprint:
|
|
_save_fingerprint(output_path, fingerprint)
|
|
except LdGenFailure as e:
|
|
print(f'linker script generation failed for {input_file.name}\nERROR: {e}')
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|