diff --git a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml index 09b799f3d7b..a2b31cc868f 100644 --- a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml +++ b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml @@ -36,6 +36,8 @@ - build_summary_*.xml # list of built apps - app_info_*.txt + # per-app size metric fragments (aggregated + uploaded by app-size-metrics) + - "**/build*/build_size_metrics.json" when: always expire_in: 1 week script: diff --git a/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml b/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml index f986108c54b..8b0daa9505b 100644 --- a/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml +++ b/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml @@ -40,6 +40,31 @@ generate_pytest_build_report: - python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type build - python tools/ci/previous_stage_job_status.py --stage build +app-size-metrics: + stage: assign_test + image: $ESP_ENV_IMAGE + tags: + - build + - shiny + when: always + allow_failure: true + needs: + - job: build_test_related_apps + optional: true + - job: build_non_test_related_apps + optional: true + - pipeline: $PARENT_PIPELINE_ID + job: pipeline_variables + variables: + ESP_METRICS_PROJECT_URL: "$CI_PROJECT_URL" + ESP_METRICS_PROJECT_ID: "$CI_PROJECT_ID" + ESP_METRICS_COMMIT_SHA: "$PIPELINE_COMMIT_SHA" + ESP_METRICS_BRANCH_NAME: "$CI_COMMIT_REF_NAME" + script: + - cd tools/ci/metrics/size_metrics + - python3 generate_metrics.py + - esp-metrics-cli upload -d schema.yaml -i metrics.json + generate_pytest_child_pipeline: # finally, we can get some use out of the default behavior that downloads all artifacts from the previous stage stage: assign_test diff --git a/tools/ci/exclude_check_tools_files.txt b/tools/ci/exclude_check_tools_files.txt index d08fdba526b..36b7f506847 100644 --- a/tools/ci/exclude_check_tools_files.txt +++ b/tools/ci/exclude_check_tools_files.txt @@ -29,6 +29,12 @@ tools/ci/idf_pytest/**/* tools/ci/metrics/examples_count/example.metrics.json tools/ci/metrics/examples_count/generate_metrics.py tools/ci/metrics/examples_count/schema.yaml +tools/ci/metrics/size_metrics/README.md +tools/ci/metrics/size_metrics/collect_build_metrics.py +tools/ci/metrics/size_metrics/example.metrics.json +tools/ci/metrics/size_metrics/generate_metrics.py +tools/ci/metrics/size_metrics/schema.yaml +tools/ci/metrics/size_metrics/size_metrics_config.yml tools/ci/mirror-submodule-update.sh tools/ci/multirun_with_pyenv.sh tools/ci/mypy_ignore_list.txt diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index e0097ed5cc0..a12e7f1d563 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -80,6 +80,8 @@ tools/ci/get-full-sources.sh tools/ci/get_supported_examples.sh tools/ci/gitlab_yaml_linter.py tools/ci/metrics/examples_count/generate_metrics.py +tools/ci/metrics/size_metrics/collect_build_metrics.py +tools/ci/metrics/size_metrics/generate_metrics.py tools/ci/mirror-submodule-update.sh tools/ci/multirun_with_pyenv.sh tools/ci/push_to_github.sh diff --git a/tools/ci/idf_ci_local/app.py b/tools/ci/idf_ci_local/app.py index 8c7306fdef5..dfbbc149e45 100644 --- a/tools/ci/idf_ci_local/app.py +++ b/tools/ci/idf_ci_local/app.py @@ -4,6 +4,7 @@ import os import subprocess import sys import typing as t +from pathlib import Path from dynamic_pipelines.constants import BINARY_SIZE_METRIC_NAME from idf_build_apps import App @@ -12,9 +13,9 @@ from idf_build_apps.constants import BuildStatus from idf_build_apps.utils import rmdir from idf_ci_utils import APP_EXTRA_S3_ARTIFACT_TYPE from idf_ci_utils import idf_relpath +from metrics.size_metrics import collect_build_metrics -if t.TYPE_CHECKING: - pass +_SIZE_METRICS_CONFIG_PATH = Path(__file__).parent.parent / 'metrics' / 'size_metrics' / 'size_metrics_config.yml' class IdfCMakeApp(CMakeApp): @@ -29,8 +30,23 @@ class IdfCMakeApp(CMakeApp): def _post_build(self) -> None: super()._post_build() - # only upload in CI + # Size metrics are only consumed in CI (aggregated by a downstream job), + # so skip the extraction entirely for local builds. if os.getenv('CI_JOB_ID'): + # Never fail the build because of size metrics. + try: + metrics = collect_build_metrics.build_raw_metrics_for_build_dir( + Path(self.build_path), + idf_relpath(self.app_dir), + self.target, + self.config_name, + collect_build_metrics.load_size_metrics_config(_SIZE_METRICS_CONFIG_PATH), + ) + if metrics is not None: + collect_build_metrics.write_raw_metrics_artifact(Path(self.build_path), metrics) + except Exception as e: + print(f'size-metrics: extraction failed for {self.build_path}: {e}') + upload_commands = [ [ 'idf-ci', @@ -63,7 +79,7 @@ class IdfCMakeApp(CMakeApp): rmdir( self.build_path, - exclude_file_patterns=['build_log.txt', 'size*.json'], + exclude_file_patterns=['build_log.txt', 'size*.json', 'build_size_metrics.json'], ) diff --git a/tools/ci/metrics/size_metrics/README.md b/tools/ci/metrics/size_metrics/README.md new file mode 100644 index 00000000000..c62d04ce070 --- /dev/null +++ b/tools/ci/metrics/size_metrics/README.md @@ -0,0 +1,74 @@ +# ESP-IDF Size Metrics Collection + +Collects code size metrics for a curated set of apps and uploads them to the +esp-metrics database once per CI pipeline. + +## How it works + +1. **Extraction (per app, in build jobs):** `IdfCMakeApp` (`tools/ci/idf_ci_local/app.py`) + calls `collect_build_metrics.build_raw_metrics_for_build_dir()` in + `_post_build()`, while the map files still exist. For apps listed in + `size_metrics_config.yml` it runs `esp-idf-size --archives --format json2` + on the app map file and adds best-effort bootloader size JSON and `.bin` + size for apps that explicitly opt in. The raw + `build*/build_size_metrics.json` artifact is written in `_finalize()`, after + build-dir cleanup, and collected via `**/build*/build_size_metrics.json` in + `.dynamic_jobs.yml`. +2. **Aggregation + upload (single `app-size-metrics` job):** runs after all + builds (`test_child_pipeline.yml`). `generate_metrics.py` aggregates the + per-build fragments into one schema-shaped `metrics.json` (next to + `schema.yaml`), then `esp-metrics-cli upload -d schema.yaml -i metrics.json`. + One DB write per pipeline; `esp-metrics-cli` + credentials live in this job + only. + +The `size_*.json` written by idf-build-apps is left untouched ā€” other CI +consumers (e.g. the binary-size diff in MR build reports) depend on its default +format. + +## Metric types + +Only apps listed in `size_metrics_config.yml` publish anything; all other built +apps are ignored. + +| Family | What | When | +|---|---|---| +| `app_size` | Total flash/DRAM/IRAM usage | every listed app | +| `library_size` | Per-`.a`-archive (= per-component) breakdown | listed apps with a `libs` key | +| `bootloader_size` | `bootloader.bin` size | listed apps with `bootloader: true` | + +## Config (`size_metrics_config.yml`) + +```yaml +tools/test_apps/system/startup: # bare entry -> app_size only +examples/get-started/hello_world: + libs: all # every archive (empty `libs:` works too) + bootloader: true # include bootloader.bin size + configs: # only these build configs (omit -> all) + - default +examples/wifi/getting_started/station: + libs: # only these archives + - libesp_wifi.a + - libnet80211.a + targets: # only these targets (omit -> all) + - esp32c6 +``` + +By default every built target/config of a listed app publishes metrics. The +optional `configs` and `targets` lists restrict publishing to specific build +configs/targets; `libs`/`bootloader` apply to every matching build. + +## Aggregating Fragments + +```bash +cd tools/ci/metrics/size_metrics +python3 generate_metrics.py # fragments -> aggregated metrics.json (no upload) +``` + +`generate_metrics.py` accepts `--config /path/to/config.yml`. + +## Troubleshooting + +- **No metrics:** Is the app listed in `size_metrics_config.yml`? Do the build + dirs still contain the app `.map` files (extraction must run before cleanup)? +- **No `library_size`:** Does the app's entry have a `libs` key? Does the name + match the archive exactly (e.g. `libesp_timer.a`)? Is PyYAML installed? diff --git a/tools/ci/metrics/size_metrics/collect_build_metrics.py b/tools/ci/metrics/size_metrics/collect_build_metrics.py new file mode 100755 index 00000000000..3f981962cff --- /dev/null +++ b/tools/ci/metrics/size_metrics/collect_build_metrics.py @@ -0,0 +1,362 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +""" +ESP-IDF Size Metrics Extractor (Build-Time) + +For each selected app this writes one build_size_metrics.json raw artifact into +the build dir. It does NOT upload anything: the artifacts are collected from +build jobs and a single downstream job (generate_metrics.py + esp-metrics-cli) +extracts, aggregates, and uploads them once. + +Entry point: +- build_raw_metrics_for_build_dir(): called per-app from IdfCMakeApp._post_build, + while the bootloader binary still exists (it is deleted right after). + +Design: +- size_metrics_config.yml is the build-side gate: only apps listed there produce + raw artifacts, every other built app is ignored +- For a listed app: the artifact carries esp-idf-size --archives JSON generated + here from the app map file, plus best-effort bootloader esp-idf-size JSON and + binary size when explicitly enabled and present. The size_*.json written by + idf-build-apps is left untouched (other CI consumers depend on its default format). + The downstream aggregator decides which schema entries to emit. +- No upload / esp-metrics-cli / DB credentials needed here +""" + +import json +import subprocess +import sys +from collections.abc import Iterator +from pathlib import Path +from typing import Any +from typing import TypedDict + +try: + import yaml +except ImportError: + yaml = None # type: ignore[assignment] + + +class SizeMetricsConfig(TypedDict): + libs: list[str] | bool + bootloader: bool + configs: list[str] | None + targets: list[str] | None + + +def _opt_str_list(value: Any) -> list[str] | None: + """Normalize an optional YAML list into list[str], or None when absent/not a list.""" + return [str(v) for v in value] if isinstance(value, list) else None + + +def load_size_metrics_config(config_file: Path) -> dict[str, SizeMetricsConfig]: + """ + Load the size metrics config YAML. + + The config is the single source of truth for which apps publish size metrics. + Returns a dict mapping app_path -> metric selection: + libs=False -> app_size only (no per-library breakdown) + libs=True -> app_size + every library + libs=list[str] -> app_size + only the named archives + bootloader=True -> include bootloader_size when available + configs=None -> every built config of the app + configs=list[str] -> only builds whose config matches one of these + targets=None -> every built target of the app + targets=list[str] -> only builds whose target matches one of these + Returns an empty dict if the config file is missing or yaml is unavailable + (in which case no metrics are published at all). + """ + if not config_file.exists(): + print(f'Note: {config_file} not found - no size metrics will be posted') + return {} + + if yaml is None: + print('Warning: PyYAML not available - skipping size metrics config') + return {} + + try: + with open(config_file) as f: + data = yaml.safe_load(f) or {} + except Exception as e: + print(f'Warning: Failed to load {config_file}: {e}') + return {} + + result: dict[str, SizeMetricsConfig] = {} + for app_path, entry in data.items(): + libs: list[str] | bool = False + bootloader = False + configs: list[str] | None = None + targets: list[str] | None = None + if isinstance(entry, dict): + if 'libs' in entry: + val = entry['libs'] + if isinstance(val, list): + libs = val + else: + # null, the literal `all`, or any truthy scalar -> all libraries + libs = True + bootloader = bool(entry.get('bootloader', False)) + configs = _opt_str_list(entry.get('configs')) + targets = _opt_str_list(entry.get('targets')) + result[str(app_path)] = { + 'libs': libs, + 'bootloader': bootloader, + 'configs': configs, + 'targets': targets, + } + + return result + + +def is_build_selected( + app_path: str, + target: str, + config: str, + size_config: dict[str, SizeMetricsConfig], +) -> bool: + """ + Return True if metrics should be published for this app build. + + A build is selected when the app is listed in the config and, when the + entry restricts `configs`/`targets`, the build's config/target is in that + list. An absent `configs`/`targets` filter matches every config/target. + """ + entry = size_config.get(app_path) + if entry is None: + return False + if entry['configs'] is not None and config not in entry['configs']: + return False + if entry['targets'] is not None and target not in entry['targets']: + return False + return True + + +def run_size_json(map_file: Path) -> dict[str, Any] | None: + """Run esp-idf-size for a map file and return json2 archive output.""" + output_file = map_file.with_suffix('.size_metrics.json') + try: + result = subprocess.run( + [ + sys.executable, + '-m', + 'esp_idf_size', + '--archives', + '--format', + 'json2', + '--output-file', + str(output_file), + str(map_file), + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + except OSError as e: + print(f'Warning: Failed to run esp-idf-size for {map_file}: {e}') + return None + + if result.returncode != 0: + stderr = result.stderr.strip() + print(f'Warning: esp-idf-size failed for {map_file}: {stderr}') + return None + + try: + with open(output_file) as f: + data: dict[str, Any] = json.load(f, strict=False) + return data + except (OSError, json.JSONDecodeError) as e: + print(f'Warning: Failed to parse esp-idf-size output for {map_file}: {e}') + return None + finally: + output_file.unlink(missing_ok=True) + + +def _iter_archive_sections(size_data: dict[str, Any]) -> Iterator[tuple[str | None, str, int]]: + """ + Walk esp_idf_size --archives --format json2 output once. + + The json2 archives format is keyed by full archive path, e.g.: + {"esp-idf/esp_timer/libesp_timer.a": {"abbrev_name": "libesp_timer.a", "memory_types": {...}}} + + Yields (abbrev_name, section_name, size) for every linker section of every + archive, so the per-archive and per-total accumulators can share one traversal. + """ + for entry in size_data.values(): + if not isinstance(entry, dict) or 'memory_types' not in entry: + continue + abbrev_name = entry.get('abbrev_name') + for memory_type in entry.get('memory_types', {}).values(): + for section_name, section_data in memory_type.get('sections', {}).items(): + yield abbrev_name, section_name, section_data.get('size', 0) + + +def _memory_total_category(memory_type_name: str) -> str | None: + """Map esp-idf-size memory type names to size metric total buckets.""" + lower_name = memory_type_name.lower() + if 'flash' in lower_name: + return 'flash' + if lower_name == 'dram': + return 'dram' + if lower_name == 'iram': + return 'iram' + return None + + +def _section_total_category(section_name: str) -> str | None: + """Map linker section prefixes to size metric total buckets.""" + if section_name.startswith('.flash.'): + return 'flash' + if section_name.startswith('.dram'): + return 'dram' + if section_name.startswith('.iram'): + return 'iram' + return None + + +def extract_all_archive_sizes(size_data: dict[str, Any]) -> dict[str, dict[str, int]]: + """ + Extract per-archive (text/rodata/data/bss/total) size breakdown. + + Returns a dict keyed by abbreviated archive name (e.g. "libesp_timer.a"). + Archives with all-zero sizes are excluded. + """ + section_map = { + '.flash.text': 'text', + '.iram0.text': 'text', + '.flash.rodata': 'rodata', + '.dram0.rodata': 'rodata', + '.dram0.data': 'data', + '.dram0.bss': 'bss', + } + + libraries: dict[str, dict[str, int]] = {} + + for abbrev_name, section_name, size in _iter_archive_sections(size_data): + category = section_map.get(section_name) + if not abbrev_name or not category: + continue + sizes = libraries.setdefault(abbrev_name, {'text': 0, 'rodata': 0, 'data': 0, 'bss': 0}) + sizes[category] += size + + for name in list(libraries): + sizes = libraries[name] + if any(v > 0 for v in sizes.values()): + sizes['total'] = sum(sizes.values()) + else: + del libraries[name] + + return libraries + + +def extract_total_sizes(size_data: dict[str, Any]) -> dict[str, int]: + """ + Extract total flash/DRAM/IRAM sizes. + + Returns a dict with keys: flash_total, dram_total, iram_total. + """ + totals = {'flash': 0, 'dram': 0, 'iram': 0} + + for archive in size_data.values(): + if not isinstance(archive, dict): + continue + + for memory_type_name, memory_type in archive.get('memory_types', {}).items(): + if not isinstance(memory_type, dict): + continue + + category = _memory_total_category(str(memory_type_name)) + if category: + totals[category] += memory_type.get('size', 0) + continue + + # Split memory types such as DIRAM contain both DRAM and IRAM + # sections, so classify those by section prefix. + for section_name, section_data in memory_type.get('sections', {}).items(): + category = _section_total_category(str(section_name)) + if category: + totals[category] += section_data.get('size', 0) + + return { + 'flash_total': totals['flash'], + 'dram_total': totals['dram'], + 'iram_total': totals['iram'], + } + + +def build_raw_metrics_for_build_dir( + build_dir: Path, + app_path: str, + target: str, + config: str, + size_config: dict[str, SizeMetricsConfig], +) -> dict[str, Any] | None: + """ + Build a raw size metrics artifact for one configured app build. + + The app/target/config identity is supplied by the caller (the ``App`` object + already knows it), so there is no need to reverse-engineer it from the build + dir name. The artifact intentionally keeps the build job dumb: it runs + esp-idf-size --archives on the app map file (the size_*.json written by + idf-build-apps keeps its default format for other CI consumers) and adds + best-effort bootloader esp-idf-size JSON when explicitly enabled. The + downstream aggregator applies the schema-specific extraction and library + filtering. + """ + if not is_build_selected(app_path, target, config, size_config): + return None + + # The app map file sits at the top of the build dir; bootloader/TEE maps + # live in subdirectories, so a non-recursive glob only matches the app map. + app_maps = sorted(build_dir.glob('*.map')) + if not app_maps: + print(f'Warning: No app map file found in {build_dir}') + return None + + app_size_data = run_size_json(app_maps[0]) + if app_size_data is None: + return None + + identity = { + 'app': app_path, + 'target': target, + 'config': config, + } + artifact: dict[str, Any] = { + 'identity': identity, + 'binaries': [ + { + 'kind': 'app', + 'name': app_path, + 'size_json': app_size_data, + } + ], + } + + if size_config[app_path]['bootloader']: + boot_map = build_dir / 'bootloader' / 'bootloader.map' + else: + boot_map = None + + if boot_map and boot_map.is_file(): + boot_entry: dict[str, Any] = { + 'kind': 'bootloader', + 'name': 'bootloader', + 'size_json': run_size_json(boot_map), + } + boot_bin = boot_map.with_suffix('.bin') + if boot_bin.is_file(): + boot_entry['bin_size'] = boot_bin.stat().st_size + artifact['binaries'].append(boot_entry) + + return artifact + + +def write_raw_metrics_artifact(build_dir: Path, artifact: dict[str, Any]) -> Path: + """Write a raw metrics artifact into the build dir, where it is collected + as a build artifact (**/build*/build_size_metrics.json) and aggregated downstream.""" + artifact_path = build_dir / 'build_size_metrics.json' + with open(artifact_path, 'w') as f: + json.dump(artifact, f) + return artifact_path diff --git a/tools/ci/metrics/size_metrics/example.metrics.json b/tools/ci/metrics/size_metrics/example.metrics.json new file mode 100644 index 00000000000..3f73e0a33b7 --- /dev/null +++ b/tools/ci/metrics/size_metrics/example.metrics.json @@ -0,0 +1,39 @@ +{ + "app_size": { + "data": [ + { + "app": "examples/get-started/hello_world", + "target": "esp32c6", + "config": "default", + "flash_total": 90032, + "dram_total": 10740, + "iram_total": 34770 + } + ] + }, + "library_size": { + "data": [ + { + "app": "examples/get-started/hello_world", + "target": "esp32c6", + "config": "default", + "library": "libesp_hw_support.a", + "text": 27096, + "rodata": 2360, + "data": 1783, + "bss": 55, + "total": 31294 + } + ] + }, + "bootloader_size": { + "data": [ + { + "app": "examples/get-started/hello_world", + "target": "esp32c6", + "config": "default", + "bin_size": 22064 + } + ] + } +} diff --git a/tools/ci/metrics/size_metrics/generate_metrics.py b/tools/ci/metrics/size_metrics/generate_metrics.py new file mode 100755 index 00000000000..408a9352f47 --- /dev/null +++ b/tools/ci/metrics/size_metrics/generate_metrics.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD +# SPDX-License-Identifier: Apache-2.0 +""" +ESP-IDF Size Metrics Generator (aggregator) + +Runs once in a single downstream job after all build jobs have finished. It +globs the per-app build*/build_size_metrics.json artifacts produced by +collect_build_metrics.py (collected as build artifacts), extracts the +configured metric families, and writes metrics.json next to schema.yaml ready +for `esp-metrics-cli upload`. + +This step does not need a build environment, esp-idf-size, or DB credentials. +""" + +import json +import os +from pathlib import Path +from typing import Any + +from collect_build_metrics import SizeMetricsConfig +from collect_build_metrics import extract_all_archive_sizes +from collect_build_metrics import extract_total_sizes +from collect_build_metrics import load_size_metrics_config + +# Top-level metric families; each becomes one document in the DB (see schema.yaml). +METRIC_FAMILIES = ('app_size', 'library_size', 'bootloader_size') + + +def find_fragments(root_dir: Path) -> list[Path]: + """Find all metrics artifact files written into build* directories.""" + return sorted(root_dir.rglob('build*/build_size_metrics.json')) + + +def add_raw_artifact( + data: dict[str, list[Any]], payload: dict[str, Any], size_config: dict[str, SizeMetricsConfig] +) -> None: + """Extract schema entries from a raw per-app size metrics artifact.""" + identity = payload.get('identity', {}) + app = identity.get('app') + # A fragment only exists because the collector already gated it against the + # config, so a membership check is enough here (no need to re-check configs/targets). + if not app or app not in size_config: + return + + for binary in payload.get('binaries', []): + kind = binary.get('kind') + size_json = binary.get('size_json') + + if kind == 'app' and isinstance(size_json, dict): + data['app_size'].append( + { + **identity, + **extract_total_sizes(size_json), + } + ) + + libs_spec = size_config[app]['libs'] + if libs_spec: + all_libraries = extract_all_archive_sizes(size_json) + if isinstance(libs_spec, list): + libraries = {k: v for k, v in all_libraries.items() if k in libs_spec} + else: + libraries = all_libraries + + for lib_name, lib_sizes in libraries.items(): + data['library_size'].append( + { + **identity, + 'library': lib_name, + 'text': lib_sizes['text'], + 'rodata': lib_sizes['rodata'], + 'data': lib_sizes['data'], + 'bss': lib_sizes['bss'], + 'total': lib_sizes['total'], + } + ) + + elif kind == 'bootloader' and 'bin_size' in binary: + data['bootloader_size'].append( + { + **identity, + 'bin_size': binary['bin_size'], + } + ) + + +def aggregate_fragments(fragments: list[Path], size_config: dict[str, SizeMetricsConfig]) -> dict[str, Any]: + """Extract artifact files into the esp-metrics document structure.""" + data: dict[str, list[Any]] = {family: [] for family in METRIC_FAMILIES} + + for frag in fragments: + try: + with open(frag) as f: + payload = json.load(f) + except (OSError, json.JSONDecodeError) as e: + print(f'Warning: skipping unreadable fragment {frag}: {e}') + continue + + if 'identity' in payload and 'binaries' in payload: + add_raw_artifact(data, payload, size_config) + else: + print(f'Warning: skipping fragment with unexpected shape: {frag}') + + return {family: {'data': data[family]} for family in METRIC_FAMILIES} + + +def main() -> None: + """CLI to aggregate size metric artifacts into metrics.json.""" + import argparse + + parser = argparse.ArgumentParser(description='Aggregate size metric artifacts into metrics.json') + parser.add_argument( + '--root', + default=os.environ.get('IDF_PATH', os.getcwd()), + help='Root directory to search for build*/build_size_metrics.json artifacts (default: IDF_PATH or cwd)', + ) + parser.add_argument( + '--output', + default='metrics.json', + help='Output metrics file (default: metrics.json, next to schema.yaml)', + ) + parser.add_argument( + '--config', + default=None, + help='Path to size_metrics_config.yml (default: next to this script)', + ) + args = parser.parse_args() + + root_dir = Path(args.root) + print(f'Searching for metric artifacts under {root_dir}...') + + script_dir = Path(__file__).parent + config_file = Path(args.config) if args.config else script_dir / 'size_metrics_config.yml' + size_config = load_size_metrics_config(config_file) + + fragments = find_fragments(root_dir) + print(f'Found {len(fragments)} metric artifact(s)') + + metrics = aggregate_fragments(fragments, size_config) + + output_file = Path(args.output) + output_file.write_text(json.dumps(metrics, indent=2)) + + counts = {family: len(metrics[family]['data']) for family in METRIC_FAMILIES} + summary = ', '.join(f'{n} {family}' for family, n in counts.items()) + print(f'\nāœ“ Aggregated [{summary}] into {output_file}') + + if not any(counts.values()): + # Nothing to upload; keep the (empty) file so the upload step is a no-op. + print('Note: no metrics collected - none of the built apps are listed in size_metrics_config.yml') + + +if __name__ == '__main__': + main() diff --git a/tools/ci/metrics/size_metrics/schema.yaml b/tools/ci/metrics/size_metrics/schema.yaml new file mode 100644 index 00000000000..9a677cbd5fe --- /dev/null +++ b/tools/ci/metrics/size_metrics/schema.yaml @@ -0,0 +1,111 @@ +--- +# ESP-IDF Size Metrics Schema Definitions +# This file defines the schema and validation rules for ESP-IDF size metrics +# We use JSON Schema for annotating and validating JSON documents' structure. +# Read more about the JSON Schema: https://json-schema.org/understanding-json-schema/about +$schema: "https://json-schema.org/draft/2020-12/schema" +# Note: Each key under 'properties' will correspond to an independent document in the DB. +# For example, 'app_size' will be stored as a separate document with its data. +properties: + app_size: + properties: + data: + type: array + title: "App Size Metrics" + description: "Total flash/DRAM/IRAM sizes for all built apps" + items: + properties: + app: + type: string + title: "App Path" + description: "App path relative to IDF_PATH" + target: + type: string + title: "Build Target" + description: "Build target (e.g. 'esp32c6')" + config: + type: string + title: "Build Config" + description: "Build configuration (e.g. 'default', 'release')" + flash_total: + type: integer + title: "Total Flash Usage" + description: "Total flash usage in bytes" + dram_total: + type: integer + title: "Total DRAM Usage" + description: "Total DRAM usage in bytes" + iram_total: + type: integer + title: "Total IRAM Usage" + description: "Total IRAM usage in bytes" + + library_size: + properties: + data: + type: array + title: "Library Size Metrics" + description: "Per-library size breakdown for apps listed in size_metrics_config.yml (flattened, one entry per library)" + items: + properties: + app: + type: string + title: "App Path" + description: "App path relative to IDF_PATH" + target: + type: string + title: "Build Target" + description: "Build target (e.g. 'esp32c6')" + config: + type: string + title: "Build Config" + description: "Build configuration (e.g. 'default', 'release')" + library: + type: string + title: "Library Name" + description: "Archive name (e.g. 'libesp_timer.a', 'libfreertos.a')" + text: + type: integer + title: ".text Section Size" + description: ".text section size in bytes" + rodata: + type: integer + title: ".rodata Section Size" + description: ".rodata section size in bytes" + data: + type: integer + title: ".data Section Size" + description: ".data section size in bytes" + bss: + type: integer + title: ".bss Section Size" + description: ".bss section size in bytes" + total: + type: integer + title: "Total Archive Size" + description: "Total archive size in bytes" + + bootloader_size: + properties: + data: + type: array + title: "Bootloader Size Metrics" + description: "Second-stage bootloader binary size for tracked apps" + items: + properties: + app: + type: string + title: "App Path" + description: "App path relative to IDF_PATH" + target: + type: string + title: "Build Target" + description: "Build target (e.g. 'esp32c6')" + config: + type: string + title: "Build Config" + description: "Build configuration (e.g. 'default', 'release')" + bin_size: + type: integer + title: "Bootloader Binary Size" + description: "Size of bootloader.bin in bytes" diff --git a/tools/ci/metrics/size_metrics/size_metrics_config.yml b/tools/ci/metrics/size_metrics/size_metrics_config.yml new file mode 100644 index 00000000000..cc6db3e3a73 --- /dev/null +++ b/tools/ci/metrics/size_metrics/size_metrics_config.yml @@ -0,0 +1,56 @@ +# Size Metrics Configuration +# +# This file is the single source of truth for which apps publish size metrics. +# Only apps listed here publish anything; every other built app is ignored. +# +# For each listed app: +# - `app_size` (total flash/DRAM/IRAM) is always published. +# - `library_size` (per-`.a`-archive breakdown) is published only when a +# `libs` key is present. +# - `bootloader_size` is published only when `bootloader: true` is present. +# +# By default every built target/config of a listed app publishes metrics. +# Use the optional `configs` and/or `targets` keys to restrict publishing to +# specific build configs and/or targets. When omitted, all configs/targets of +# the app are published. `libs`/`bootloader` apply to every matching build. +# +# Format: +# : # Path relative to IDF_PATH. Bare entry = app_size only. +# libs: # (optional) enables library_size: +# - libfoo.a # - a list -> only these archives +# - libbar.a +# configs: # (optional) only publish these build configs +# - default # (omit to publish every built config) +# targets: # (optional) only publish these targets +# - esp32c6 # (omit to publish every built target) +# : +# libs: all # - the literal `all` (or empty) -> every archive +# bootloader: true # (optional) enables bootloader_size +# +# Examples: +# +# app_size only, every target/config: +# tools/test_apps/system/startup: +# +# app_size + every library, only the `default` and `release` configs: +# examples/get-started/hello_world: +# libs: all +# configs: +# - default +# - release +# +# app_size + selected libraries, only on esp32c6: +# examples/wifi/getting_started/station: +# libs: +# - libesp_wifi.a +# - libnet80211.a +# targets: +# - esp32c6 +# +# Entries below define the apps that publish size metrics: + +examples/get-started/hello_world: + libs: all + bootloader: true + configs: + - default diff --git a/tools/requirements/requirements.ci.txt b/tools/requirements/requirements.ci.txt index 7874fbabdd4..cbc4ce76c1d 100644 --- a/tools/requirements/requirements.ci.txt +++ b/tools/requirements/requirements.ci.txt @@ -7,6 +7,7 @@ # ci idf-ci +esp-metrics-cli coverage jsonschema