From e1f43e15b7db08bb76197ae3131b4351ac70cb61 Mon Sep 17 00:00:00 2001 From: Frantisek Hrbata Date: Fri, 3 Apr 2026 15:04:06 +0200 Subject: [PATCH] feat(ldgen): skip generation when section names unchanged MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After running objdump on all libraries, extract section names from the stored raw output using regex and compute a fingerprint (MD5 hash of section names + mtimes of fragment files, sdkconfig, and kconfig). If the fingerprint matches the cached value from the previous run, skip the expensive pyparsing + generation step entirely and touch the output file to update its timestamp. The fingerprint file is stored next to the output (e.g. sections.ld.fingerprint) in the build directory. This avoids the pyparsing bottleneck for the common case of editing function bodies without adding or removing symbols — section names in the object files are unchanged, so the generated sections.ld would be identical. When sections do change, the full generation runs and the fingerprint is updated. Print "Skipping linker script generation, section names unchanged" on the fingerprint hit path so build logs show when the optimization fired. This makes it easy to diagnose user reports of unexpected ldgen behavior — the build log directly shows whether the cache was used. Add an LDGEN_NO_CACHE environment variable to disable the cache as a workaround in case the optimization causes problems. When set, ldgen prints "Linker script generation caches disabled by LDGEN_NO_CACHE" and runs full generation as before, leaving any existing fingerprint file untouched. Measured on wifi_station (205 libs, 67 .lf files), median of 10 runs: before after rebuild, section names unchanged 5.82s 0.82s Closes https://github.com/espressif/esp-idf/issues/18408 Signed-off-by: Frantisek Hrbata --- tools/ldgen/ldgen.py | 69 ++++++++++++++++++++++++- tools/test_build_system/test_rebuild.py | 33 +++++++++++- 2 files changed, 100 insertions(+), 2 deletions(-) diff --git a/tools/ldgen/ldgen.py b/tools/ldgen/ldgen.py index b07bfd09672..49935c6f9f6 100755 --- a/tools/ldgen/ldgen.py +++ b/tools/ldgen/ldgen.py @@ -1,12 +1,14 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2021-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # import argparse import errno +import hashlib import json import os +import re import subprocess import sys import tempfile @@ -21,6 +23,50 @@ from ldgen.sdkconfig import SDKConfig from pyparsing import ParseException from pyparsing import ParseFatalException +_RE_SECTION_NAME = re.compile(r'^\s*\d+\s+(\.\S+)', re.MULTILINE) + + +def _compute_fingerprint(sections_infos, fragment_files, config_file, kconfig_file): + """Compute a fingerprint from section names and mtimes of all inputs.""" + hasher = hashlib.md5() + + # Section names from objdump output + for archive, info in sorted(sections_infos.sections.items()): + names = _RE_SECTION_NAME.findall(info.content) + hasher.update((archive + ':' + ','.join(names)).encode()) + + # Mtimes of fragment files, sdkconfig, kconfig + input_files = [p.name if hasattr(p, 'name') else p for p in fragment_files] + input_files += [p for p in (config_file, kconfig_file) if p] + for path in input_files: + try: + hasher.update(f'{path}:{os.path.getmtime(path)}'.encode()) + except OSError: + return None + + return hasher.hexdigest() + + +def _can_skip_generation(output_path, fingerprint): + """Check if fingerprint matches cached value from previous run.""" + try: + with open(output_path + '.fingerprint') as f: + if f.read().strip() == fingerprint: + os.utime(output_path, None) + return True + except OSError: + pass + return False + + +def _save_fingerprint(output_path, fingerprint): + """Save fingerprint for next run.""" + try: + with open(output_path + '.fingerprint', 'w') as f: + f.write(fingerprint) + except OSError: + pass + def _update_environment(args): env = [(name, value) for (name, value) in (e.split('=', 1) for e in args.env)] @@ -114,6 +160,10 @@ def main(): else: check_mapping_exceptions = None + no_cache = os.environ.get('LDGEN_NO_CACHE') == '1' + if no_cache: + print('Linker script generation caches disabled by LDGEN_NO_CACHE') + try: sections_infos = EntityDB() for library in libraries_file: @@ -125,6 +175,20 @@ def main(): dump.name = library sections_infos.add_sections_info(dump) + # Check if we can skip generation entirely — section names and other + # inputs unchanged since last run. + fingerprint = ( + None if no_cache else _compute_fingerprint(sections_infos, fragment_files, config_file, kconfig_file) + ) + if ( + output_path + and fingerprint + and os.path.exists(output_path) + and _can_skip_generation(output_path, fingerprint) + ): + print('Skipping linker script generation, section names unchanged') + sys.exit(0) + mutable_libs = [lib.strip() for lib in mutable_libraries_file] generation_model = Generation(check_mapping, check_mapping_exceptions, mutable_libs, args.debug) @@ -163,6 +227,9 @@ def main(): output_path, 'w', encoding='utf-8' ) as f: # only create output file after generation has succeeded f.write(output.read()) + + if output_path and fingerprint: + _save_fingerprint(output_path, fingerprint) except LdGenFailure as e: print(f'linker script generation failed for {input_file.name}\nERROR: {e}') sys.exit(1) diff --git a/tools/test_build_system/test_rebuild.py b/tools/test_build_system/test_rebuild.py index 430c0cc1b17..c59aa86869d 100644 --- a/tools/test_build_system/test_rebuild.py +++ b/tools/test_build_system/test_rebuild.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # These tests check whether the build system rebuilds some files or not # depending on the changes to the project. @@ -143,6 +143,37 @@ def test_rebuild_linker(idf_py: IdfPyFunc) -> None: rebuild_and_check(idf_py, APP_BINS, BOOTLOADER_BINS + PARTITION_BIN) +def test_rebuild_ldgen_fingerprint(idf_py: IdfPyFunc, test_app_copy: Path) -> None: + """Verify the ldgen fingerprint skip path: when section names and other + inputs are unchanged, ldgen exits early without regenerating sections.ld + and prints an informational message. + """ + skip_msg = 'Skipping linker script generation, section names unchanged' + app_c = test_app_copy / 'main' / 'build_test_app.c' + + # Seed app_main with a printf so the first build establishes the string + # literal and its .rodata section. Later we only change the arithmetic + # constant, which modifies .text.app_main contents but keeps section + # names unchanged — exactly the case the fingerprint should optimize. + replace_in_file(app_c, '// placeholder_inside_main', 'printf("value = %d\\n", 1 + 2);') + + logging.info('initial build') + idf_py('build') + + logging.info( + 'changing the printed calculation modifies .text.app_main but keeps all section names - fingerprint should hit' + ) + replace_in_file(app_c, '1 + 2', '3 + 4') + result = idf_py('build') + assert skip_msg in result.stdout, f'expected {skip_msg!r} in build output' + + logging.info('touching a fragment file invalidates the fingerprint') + idf_path = Path(os.environ['IDF_PATH']) + (idf_path / 'components/esp_common/common.lf').touch() + result = idf_py('build') + assert skip_msg not in result.stdout, f'unexpected {skip_msg!r} in build output after fragment change' + + @pytest.mark.usefixtures('idf_copy') def test_rebuild_version_change(idf_py: IdfPyFunc, test_app_copy: Path) -> None: idf_path = Path(os.environ['IDF_PATH'])