Mirrors/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2026-09-22 13:01:16 +03:00
Code Issues Packages Projects Releases Wiki Activity

ci: remove pip-cache and other unused jobs

Browse Source
This commit is contained in:
Fu Hanxi
2026-07-22 16:33:44 +02:00
parent ddb9fd017c
commit 0a86d7b8e7
21 changed files with 3 additions and 3738 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
.gitlab/ci/common.yml
View File

@@ -60,9 +60,6 @@ variables:
TARGET_TEST_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v6.0-target-test:1"
SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:5"
# cache python dependencies
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
# Set this variable to the branch of idf-constraints repo in order to test a custom Python constraint file. The
# branch name must be without the remote part ("origin/"). Keep the variable empty in order to use the constraint
# file from https://dl.espressif.com/dl/esp-idf.
@@ -340,12 +337,6 @@ default:
cache:
# pull only for most of the use cases since it's cache dir.
# Only set "push" policy for "upload_cache" stage jobs
- key: pip-cache-${LATEST_GIT_TAG}
fallback_keys:
- pip-cache
paths:
- .cache/pip
policy: pull
- key: submodule-cache-${LATEST_GIT_TAG}
fallback_keys:
- submodule-cache

3
.gitlab/ci/host-test.yml
View File

@@ -7,9 +7,6 @@
# run host_test jobs immediately, only after upload cache
needs:
- pipeline_variables
- job: upload-pip-cache
optional: true
artifacts: false
- job: upload-submodules-cache
optional: true
artifacts: false

16
.gitlab/ci/post_deploy.yml
View File

@@ -2,22 +2,6 @@
stage: post_deploy
image: $ESP_ENV_IMAGE
generate_failed_jobs_report:
extends:
- .post_deploy_template
tags: [build, shiny]
when: always
dependencies: # Do not download artifacts from the previous stages
needs:
- pipeline_variables
artifacts:
expire_in: 2 week
when: always
paths:
- job_report.html
script:
- python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type job
sync_support_status:
extends:
- .post_deploy_template

12
.gitlab/ci/rules.yml
View File

@@ -12,9 +12,6 @@
- "components/**/Kconfig"
- "components/**/CMakeLists.txt"
.patterns-python-cache: &patterns-python-cache
- "tools/requirements.json"
- "tools/requirements/requirements.*.txt"
.patterns-python-files: &patterns-python-files
- ".gitlab/ci/static-code-analysis.yml"
@@ -220,15 +217,6 @@
rules:
- <<: *if-dev-push
.rules:upload-python-cache:
rules:
- <<: *if-release-tag
- <<: *if-schedule-nightly
- <<: *if-protected-branch-push
changes: *patterns-python-cache
- <<: *if-label-upload_cache
when: manual
.rules:upload-submodule-cache:
rules:
- <<: *if-release-tag

3
.gitlab/ci/test-win.yml
View File

@@ -6,9 +6,6 @@
dependencies: # set dependencies to null to avoid missing artifacts issue
# run host_test jobs immediately, only after upload cache
needs:
- job: upload-pip-cache
optional: true
artifacts: false
- job: upload-submodules-cache
optional: true
artifacts: false

20
.gitlab/ci/upload_cache.yml
View File

@@ -6,26 +6,6 @@
stage: upload_cache
image: $ESP_ENV_IMAGE
upload-pip-cache:
extends:
- .upload_cache_template
- .before_script:minimal
- .rules:upload-python-cache
tags:
- $GEO
- cache
cache:
key: pip-cache-${LATEST_GIT_TAG}
paths:
- .cache/pip
policy: push
script:
- rm -rf .cache/pip # clear old packages
- bash install.sh --enable-ci --enable-test-specific
parallel:
matrix:
- GEO: [ 'shiny', 'brew' ]
upload-submodules-cache:
extends:
- .upload_cache_template

1
.idf_ci.toml
View File

@@ -20,7 +20,6 @@ IDF_CI_BUILD = "1"
[gitlab.build_pipeline]
workflow_name = "build_child_pipeline"
presigned_json_job_name = 'generate_pytest_build_report'
job_tags = ['build', 'shiny']
job_template_name = '.dynamic_build_template'

19
tools/ci/dynamic_pipelines/constants.py
View File

@@ -1,26 +1,9 @@
# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2024-2026 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import os
from idf_ci_utils import IDF_PATH
COMMENT_START_MARKER = '### Dynamic Pipeline Report'
REPORT_TEMPLATE_FILEPATH = os.path.join(
IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'report.template.html'
)
CSS_STYLES_FILEPATH = os.path.join(IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'styles.css')
JS_SCRIPTS_FILEPATH = os.path.join(IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'scripts.js')
TOP_N_APPS_BY_SIZE_DIFF = 10
SIZE_DIFFERENCE_BYTES_THRESHOLD = 500
BINARY_SIZE_METRIC_NAME = 'binary_size'
KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH = os.path.join(
IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'known_generate_test_child_pipeline_warnings.yml'
)
CI_JOB_TOKEN = os.getenv('CI_JOB_TOKEN', '')
CI_DASHBOARD_API = os.getenv('CI_DASHBOARD_API', '')
CI_PAGES_URL = os.getenv('CI_PAGES_URL', '')
CI_PROJECT_URL = os.getenv('CI_PROJECT_URL', '')
CI_MERGE_REQUEST_SOURCE_BRANCH_SHA = os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA', '')

121
tools/ci/dynamic_pipelines/models.py
View File

@@ -1,121 +0,0 @@
# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import os
import typing as t
import urllib.parse
from dataclasses import dataclass
from xml.etree.ElementTree import Element
from idf_ci_utils import IDF_PATH
@dataclass
class TestCase:
name: str
file: str
time: float
app_path: t.Optional[str] = None
failure: t.Optional[str] = None
skipped: t.Optional[str] = None
ci_job_url: t.Optional[str] = None
ci_dashboard_url: t.Optional[str] = None
dut_log_url: t.Optional[str] = None
latest_total_count: int = 0
latest_failed_count: int = 0
@property
def is_failure(self) -> bool:
return self.failure is not None
@property
def is_skipped(self) -> bool:
return self.skipped is not None
@property
def is_success(self) -> bool:
return not self.is_failure and not self.is_skipped
@classmethod
def _get_idf_rel_path(cls, path: str) -> str:
if path.startswith(IDF_PATH):
return os.path.relpath(path, IDF_PATH)
else:
return path
@classmethod
def from_test_case_node(cls, node: Element) -> t.Optional['TestCase']:
if 'name' not in node.attrib:
print('WARNING: Node Invalid: ', node)
return None
# url to test cases dashboard
grafana_base_url = urllib.parse.urljoin(os.getenv('CI_DASHBOARD_HOST', ''), '/d/Ucg477Fnz/case-list')
encoded_params = urllib.parse.urlencode({'var-case_id': node.attrib['name']}, quote_via=urllib.parse.quote)
kwargs = {
'name': node.attrib['name'],
'file': node.attrib.get('file'),
'app_path': '|'.join(
cls._get_idf_rel_path(path) for path in node.attrib.get('app_path', 'unknown').split('|')
),
'time': float(node.attrib.get('time') or 0),
'ci_job_url': node.attrib.get('ci_job_url') or 'Not found',
'ci_dashboard_url': f'{grafana_base_url}?{encoded_params}',
'dut_log_url': node.attrib.get('dut_log_url') or 'Not found',
}
failure_node = node.find('failure')
# bool(failure_node) is False, so compare with None
if failure_node is None:
failure_node = node.find('error')
if failure_node is not None:
message = failure_node.attrib.get('message', '')
kwargs['failure'] = message
skipped_node = node.find('skipped')
if skipped_node is not None:
kwargs['skipped'] = skipped_node.attrib['message']
return cls(**kwargs) # type: ignore
@dataclass
class GitlabJob:
id: int
name: str
stage: str
status: str
url: str
ci_dashboard_url: str
failure_reason: t.Optional[str] = None
failure_log: t.Optional[str] = None
latest_total_count: int = 0
latest_failed_count: int = 0
@property
def is_failed(self) -> bool:
return self.status == 'failed'
@property
def is_success(self) -> bool:
return self.status == 'success'
@classmethod
def from_json_data(cls, job_data: dict, failure_data: dict) -> t.Optional['GitlabJob']:
grafana_base_url = urllib.parse.urljoin(os.getenv('CI_DASHBOARD_HOST', ''), '/d/LoUa-qLWz/job-list')
encoded_params = urllib.parse.urlencode({'var-job_name': job_data['name']}, quote_via=urllib.parse.quote)
kwargs = {
'id': job_data['id'],
'name': job_data['name'],
'stage': job_data['stage'],
'status': job_data['status'],
'url': job_data['url'],
'ci_dashboard_url': f'{grafana_base_url}?{encoded_params}',
'failure_reason': job_data['failure_reason'],
'failure_log': job_data['failure_log'],
'latest_total_count': failure_data.get('total_count', 0),
'latest_failed_count': failure_data.get('failed_count', 0),
}
return cls(**kwargs) # type: ignore

739
tools/ci/dynamic_pipelines/report.py
View File

@@ -1,739 +0,0 @@
# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import abc
import html
import re
import typing as t
from textwrap import dedent
from gitlab import GitlabUpdateError
from gitlab_api import Gitlab
from idf_build_apps.constants import BuildStatus
from idf_ci_local.app import AppWithMetricsInfo
from idf_ci_utils import idf_relpath
from prettytable import PrettyTable
from .constants import BINARY_SIZE_METRIC_NAME
from .constants import CI_DASHBOARD_API
from .constants import COMMENT_START_MARKER
from .constants import CSS_STYLES_FILEPATH
from .constants import JS_SCRIPTS_FILEPATH
from .constants import REPORT_TEMPLATE_FILEPATH
from .constants import SIZE_DIFFERENCE_BYTES_THRESHOLD
from .constants import TOP_N_APPS_BY_SIZE_DIFF
from .models import GitlabJob
from .models import TestCase
from .utils import format_permalink
from .utils import get_artifacts_url
from .utils import is_url
class ReportGenerator:
REGEX_PATTERN = r'#### {}\n[\s\S]*?(?=\n#### |$)'
def __init__(
self,
project_id: int,
mr_iid: int,
pipeline_id: int,
job_id: int,
commit_id: str,
local_commit_id: str,
*,
title: str,
):
gl_project = Gitlab(project_id).project
if mr_iid is not None:
self.mr = gl_project.mergerequests.get(mr_iid)
else:
self.mr = None
self.pipeline_id = pipeline_id
self.job_id = job_id
self.commit_id = commit_id
self.local_commit_id = local_commit_id
self.title = title
self.output_filepath = self.title.lower().replace(' ', '_') + '.html'
self.additional_info = ''
@property
def get_commit_summary(self) -> str:
return f'with CI commit SHA: {self.commit_id[:8]}, local commit SHA: {self.local_commit_id[:8]}'
@staticmethod
def get_download_link_for_url(url: str) -> str:
if url:
return f'<a href="{url}">Download</a>'
return ''
@staticmethod
def write_report_to_file(report_str: str, job_id: int, output_filepath: str) -> str | None:
"""
Writes the report to a file and constructs a modified URL based on environment settings.
:param report_str: The report content to be written to the file.
:param job_id: The job identifier used to construct the URL.
:param output_filepath: The path to the output file.
:return: The modified URL pointing to the job's artifacts.
"""
if not report_str:
return None
with open(output_filepath, 'w') as file:
file.write(report_str)
# for example, {URL}/-/esp-idf/-/jobs/{id}/artifacts/app_info_84.txt
# CI_PAGES_URL is {URL}/esp-idf, which missed one `-`
report_url: str = get_artifacts_url(job_id, output_filepath)
return report_url
@staticmethod
def _load_file_content(filepath: str) -> str:
"""
Load the content of a file as string
:param filepath: Path to the file to load
:return: Content of the file as string
"""
try:
with open(filepath, encoding='utf-8') as f:
return f.read()
except (OSError, FileNotFoundError) as e:
print(f'Warning: Could not read file {filepath}: {e}')
return ''
def generate_html_report(self, table_str: str) -> str:
# we're using bootstrap table
table_str = table_str.replace(
'<table>',
'<table data-toggle="table" data-search-align="left" data-search="true" data-sticky-header="true">',
)
template = self._load_file_content(REPORT_TEMPLATE_FILEPATH)
css_content = self._load_file_content(CSS_STYLES_FILEPATH)
js_content = self._load_file_content(JS_SCRIPTS_FILEPATH)
template = template.replace('{{css_content}}', css_content)
template = template.replace('{{js_content}}', js_content)
template = template.replace('{{pipeline_id}}', str(self.pipeline_id))
template = template.replace('{{apiBaseUrl}}', CI_DASHBOARD_API)
return template.replace('{{title}}', self.title).replace('{{table}}', table_str)
@staticmethod
def table_to_html_str(table: PrettyTable) -> str:
return html.unescape(table.get_html_string()) # type: ignore
def create_table_section(
self,
title: str,
items: list,
headers: list,
row_attrs: list,
value_functions: list | None = None,
) -> list:
"""
Appends a formatted section to a report based on the provided items. This section includes
a header and a table constructed from the items list with specified headers and attributes.
:param title: Title for the report section. This title is used as a header above the table.
:param items: List of item objects to include in the table. Each item should have attributes
that correspond to the row_attrs and value_functions specified.
:param headers: List of strings that will serve as the column headers in the generated table.
:param row_attrs: List of attributes to include from each item for the table rows. These
should be attributes or keys that exist on the items in the 'items' list.
:param value_functions: Optional list of tuples containing additional header and corresponding
value function. Each tuple should specify a header (as a string) and
a function that takes an item and returns a string. This is used for
generating dynamic columns based on item data.
:return: List with appended HTML sections.
"""
if not items:
return []
report_sections = [
f"""<h2 id="{format_permalink(title)}">{title}<i class="fas fa-link copy-link-icon"
onclick="copyPermalink('#{format_permalink(title)}')"></i></h2>""",
self._create_table_for_items(
items=items, headers=headers, row_attrs=row_attrs, value_functions=value_functions or []
),
]
return report_sections
@staticmethod
def generate_additional_info_section(
title: str, count: int, report_url: str | None = None, add_permalink: bool = True
) -> str:
"""
Generate a section for the additional info string.
:param title: The title of the section.
:param count: The count of test cases.
:param report_url: The URL of the report. If count = 0, only the count will be included.
:param add_permalink: Whether to include a permalink in the report URL. Defaults to True.
:return: The formatted additional info section string.
"""
if count != 0 and report_url:
if add_permalink:
return f'- **{title}:** [{count}]({report_url}/#{format_permalink(title)})\n'
else:
return f'- **{title}:** [{count}]({report_url})\n'
else:
return f'- **{title}:** {count}\n'
def _create_table_for_items(
self,
items: list[TestCase] | list[GitlabJob],
headers: list[str],
row_attrs: list[str],
value_functions: list[tuple[str, t.Callable[[TestCase | GitlabJob], str]]] | None = None,
) -> str:
"""
Create a PrettyTable and convert it to an HTML string for the provided test cases.
:param items: List of item objects to include in the table.
:param headers: List of strings for the table headers.
:param row_attrs: List of attributes to include in each row.
:param value_functions: List of tuples containing additional header and corresponding value function.
:return: HTML table string.
"""
table = PrettyTable()
table.field_names = headers
# Create a mapping of header names to their corresponding index in the headers list
header_index_map = {header: i for i, header in enumerate(headers)}
for item in items:
row = []
for attr in row_attrs:
value = str(getattr(item, attr, ''))
if is_url(value):
link = f'<a href="{value}">link</a>'
row.append(link)
else:
row.append(value)
# Insert values computed by value functions at the correct column position based on their headers
if value_functions:
for header, func in value_functions:
index = header_index_map.get(header)
if index is not None:
computed_value = func(item)
row.insert(index, computed_value)
table.add_row(row)
return self.table_to_html_str(table)
@staticmethod
def _filter_items(
items: list[TestCase] | list[GitlabJob], condition: t.Callable[[TestCase | GitlabJob], bool]
) -> list[TestCase]:
"""
Filter items s based on a given condition.
:param items: List of items to filter by given condition.
:param condition: A function that evaluates to True or False for each items.
:return: List of filtered instances.
"""
return [item for item in items if condition(item)]
@staticmethod
def _sort_items(
items: list[TestCase | GitlabJob | AppWithMetricsInfo],
key: str | t.Callable[[TestCase | GitlabJob | AppWithMetricsInfo], t.Any],
order: str = 'asc',
sort_function: t.Callable[[t.Any], t.Any] | None = None,
) -> list[TestCase | GitlabJob | AppWithMetricsInfo]:
"""
Sort items based on a given key, order, and optional custom sorting function.
:param items: List of items to sort.
:param key: A string representing the attribute name or a function to extract the sorting key.
:param order: Order of sorting ('asc' for ascending, 'desc' for descending).
:param sort_function: A custom function to control sorting logic
(e.g., prioritizing positive/negative/zero values).
:return: List of sorted instances.
"""
key_func = None
if isinstance(key, str):
def key_func(item: t.Any) -> t.Any:
return getattr(item, key)
sorting_key = sort_function if sort_function is not None else key_func
try:
items = sorted(items, key=sorting_key, reverse=(order == 'desc'))
except TypeError:
print(f'Comparison for the key {key} is not supported')
return items
@abc.abstractmethod
def _get_report_str(self) -> str:
raise NotImplementedError
def _generate_comment(self) -> str:
# Report in HTML format to avoid exceeding length limits
comment = f'#### {self.title}\n'
report_str = self._get_report_str()
comment += f'{self.additional_info}\n'
self.write_report_to_file(report_str, self.job_id, self.output_filepath)
return comment
def _update_mr_comment(self, comment: str) -> None:
new_comment = f'{COMMENT_START_MARKER}\n\n{comment}'
for note in self.mr.notes.list(iterator=True):
if note.body.startswith(COMMENT_START_MARKER):
updated_str = self._get_updated_comment(note.body, comment)
note.body = updated_str
try:
note.save()
except GitlabUpdateError:
print('Failed to update MR comment, Creating a new comment')
self.mr.notes.create({'body': new_comment})
break
else:
self.mr.notes.create({'body': new_comment})
def _get_updated_comment(self, existing_comment: str, new_comment: str) -> str:
updated_str = re.sub(self.REGEX_PATTERN.format(self.title), new_comment, existing_comment)
if updated_str == existing_comment:
updated_str = f'{existing_comment.strip()}\n\n{new_comment}'
return updated_str
def post_report(self) -> None:
comment = self._generate_comment()
print(comment)
if self.mr is None:
print('No MR found, skip posting comment')
return
self._update_mr_comment(comment)
class BuildReportGenerator(ReportGenerator):
def __init__(
self,
project_id: int,
mr_iid: int,
pipeline_id: int,
job_id: int,
commit_id: str,
local_commit_id: str,
*,
title: str = 'Build Report',
apps: list[AppWithMetricsInfo],
) -> None:
super().__init__(project_id, mr_iid, pipeline_id, job_id, commit_id, local_commit_id, title=title)
self.apps = apps
self.report_titles_map = {
'failed_apps': 'Failed Apps',
'built_test_related_apps': 'Built Apps - Test Related',
'built_non_test_related_apps': 'Built Apps - Non Test Related',
'new_test_related_apps': 'New Apps - Test Related',
'new_non_test_related_apps': 'New Apps - Non Test Related',
'skipped_apps': 'Skipped Apps',
}
self.failed_apps_report_file = 'failed_apps.html'
self.built_apps_report_file = 'built_apps.html'
self.skipped_apps_report_file = 'skipped_apps.html'
@staticmethod
def custom_sort(item: AppWithMetricsInfo) -> tuple[int, t.Any]:
"""
Custom sort function to:
1. Push items with zero binary sizes to the end.
2. Sort other items by absolute size_difference_percentage.
"""
# Priority: 0 for zero binaries, 1 for non-zero binaries
zero_binary_priority = (
1
if item.metrics[BINARY_SIZE_METRIC_NAME].source_value != 0
or item.metrics[BINARY_SIZE_METRIC_NAME].target_value != 0
else 0
)
# Secondary sort: Negative absolute size_difference_percentage for descending order
size_difference_sort = abs(item.metrics[BINARY_SIZE_METRIC_NAME].difference_percentage)
return zero_binary_priority, size_difference_sort
def _generate_top_n_apps_by_size_table(self) -> str:
"""
Generate a markdown table for the top N apps by size difference.
Only includes apps with size differences greater than 500 bytes.
"""
filtered_apps = [
app
for app in self.apps
if abs(app.metrics[BINARY_SIZE_METRIC_NAME].difference) > SIZE_DIFFERENCE_BYTES_THRESHOLD
]
top_apps = sorted(
filtered_apps, key=lambda app: abs(app.metrics[BINARY_SIZE_METRIC_NAME].difference_percentage), reverse=True
)[:TOP_N_APPS_BY_SIZE_DIFF]
if not top_apps:
return ''
table = (
f'\n⚠️⚠️⚠️ Top {len(top_apps)} Apps with Binary Size Sorted by Size Difference\n'
f'Note: Apps with changes of less than {SIZE_DIFFERENCE_BYTES_THRESHOLD} bytes are not shown.\n'
)
table += '| App Dir | Build Dir | Size Diff (bytes) | Size Diff (%) |\n'
table += '|---------|-----------|-------------------|---------------|\n'
for app in top_apps:
table += dedent(
f'| {app.app_dir} | {app.build_dir} | '
f'{app.metrics[BINARY_SIZE_METRIC_NAME].difference} | '
f'{app.metrics[BINARY_SIZE_METRIC_NAME].difference_percentage}% |\n'
)
table += (
'\n**For more details, please click on the numbers in the summary above '
'to view the corresponding report files.** ⬆️⬆️⬆️\n\n'
)
return table
@staticmethod
def split_new_and_existing_apps(
apps: t.Iterable[AppWithMetricsInfo],
) -> tuple[list[AppWithMetricsInfo], list[AppWithMetricsInfo]]:
"""
Splits apps into new apps and existing apps.
:param apps: Iterable of apps to process.
:return: A tuple (new_apps, existing_apps).
"""
new_apps = [app for app in apps if app.is_new_app]
existing_apps = [app for app in apps if not app.is_new_app]
return new_apps, existing_apps
def filter_apps_by_criteria(self, build_status: str, preserve: bool) -> list[AppWithMetricsInfo]:
"""
Filters apps based on build status and preserve criteria.
:param build_status: Build status to filter by.
:param preserve: Whether to filter preserved apps.
:return: Filtered list of apps.
"""
return [app for app in self.apps if app.build_status == build_status and app.preserve == preserve]
def get_built_apps_report_parts(self) -> list[str]:
"""
Generates report parts for new and existing apps.
:return: List of report parts.
"""
new_test_related_apps, built_test_related_apps = self.split_new_and_existing_apps(
self.filter_apps_by_criteria(BuildStatus.SUCCESS, True)
)
new_non_test_related_apps, built_non_test_related_apps = self.split_new_and_existing_apps(
self.filter_apps_by_criteria(BuildStatus.SUCCESS, False)
)
sections = []
if new_test_related_apps:
new_test_related_apps_table_section = self.create_table_section(
title=self.report_titles_map['new_test_related_apps'],
items=new_test_related_apps,
headers=[
'App Dir',
'Build Dir',
'Download Command',
'Your Branch App Size',
],
row_attrs=[
'app_dir',
'build_dir',
],
value_functions=[
('Your Branch App Size', lambda _app: str(_app.metrics[BINARY_SIZE_METRIC_NAME].source_value)),
(
'Download Command',
lambda _app: f'idf-ci gitlab download-artifacts --pipeline-id {self.pipeline_id} '
f'{idf_relpath(_app.build_path)}',
),
],
)
sections.extend(new_test_related_apps_table_section)
if built_test_related_apps:
built_test_related_apps = self._sort_items(
built_test_related_apps,
key='metrics.binary_size.difference_percentage',
order='desc',
sort_function=self.custom_sort,
)
built_test_related_apps_table_section = self.create_table_section(
title=self.report_titles_map['built_test_related_apps'],
items=built_test_related_apps,
headers=[
'App Dir',
'Build Dir',
'Download Command',
'Your Branch App Size',
'Target Branch App Size',
'Size Diff',
'Size Diff, %',
],
row_attrs=[
'app_dir',
'build_dir',
],
value_functions=[
('Your Branch App Size', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].source_value)),
('Target Branch App Size', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].target_value)),
('Size Diff', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].difference)),
('Size Diff, %', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].difference_percentage)),
(
'Download Command',
lambda _app: f'idf-ci gitlab download-artifacts --pipeline-id {self.pipeline_id} '
f'{idf_relpath(_app.build_path)}',
),
],
)
sections.extend(built_test_related_apps_table_section)
if new_non_test_related_apps:
new_non_test_related_apps_table_section = self.create_table_section(
title=self.report_titles_map['new_non_test_related_apps'],
items=new_non_test_related_apps,
headers=[
'App Dir',
'Build Dir',
'Download Command',
'Your Branch App Size',
],
row_attrs=[
'app_dir',
'build_dir',
],
value_functions=[
(
'Download Command',
lambda _app: f'idf-ci gitlab download-artifacts --pipeline-id {self.pipeline_id} '
f'{idf_relpath(_app.build_path)}',
),
('Your Branch App Size', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].source_value)),
],
)
sections.extend(new_non_test_related_apps_table_section)
if built_non_test_related_apps:
built_non_test_related_apps = self._sort_items(
built_non_test_related_apps,
key='metrics.binary_size.difference_percentage',
order='desc',
sort_function=self.custom_sort,
)
built_non_test_related_apps_table_section = self.create_table_section(
title=self.report_titles_map['built_non_test_related_apps'],
items=built_non_test_related_apps,
headers=[
'App Dir',
'Build Dir',
'Download Command',
'Your Branch App Size',
'Target Branch App Size',
'Size Diff',
'Size Diff, %',
],
row_attrs=[
'app_dir',
'build_dir',
],
value_functions=[
('Your Branch App Size', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].source_value)),
('Target Branch App Size', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].target_value)),
('Size Diff', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].difference)),
('Size Diff, %', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].difference_percentage)),
(
'Download Command',
lambda _app: f'idf-ci gitlab download-artifacts --pipeline-id {self.pipeline_id} '
f'{idf_relpath(_app.build_path)}',
),
],
)
sections.extend(built_non_test_related_apps_table_section)
built_apps_report_url = self.write_report_to_file(
self.generate_html_report(''.join(sections)),
self.job_id,
self.built_apps_report_file,
)
self.additional_info += self.generate_additional_info_section(
self.report_titles_map['built_test_related_apps'],
len(built_test_related_apps),
built_apps_report_url,
)
self.additional_info += self.generate_additional_info_section(
self.report_titles_map['built_non_test_related_apps'],
len(built_non_test_related_apps),
built_apps_report_url,
)
self.additional_info += self.generate_additional_info_section(
self.report_titles_map['new_test_related_apps'],
len(new_test_related_apps),
built_apps_report_url,
)
self.additional_info += self.generate_additional_info_section(
self.report_titles_map['new_non_test_related_apps'],
len(new_non_test_related_apps),
built_apps_report_url,
)
self.additional_info += self._generate_top_n_apps_by_size_table()
return sections
def get_failed_apps_report_parts(self) -> list[str]:
failed_apps = [app for app in self.apps if app.build_status == BuildStatus.FAILED]
if not failed_apps:
return []
failed_apps_table_section = self.create_table_section(
title=self.report_titles_map['failed_apps'],
items=failed_apps,
headers=['App Dir', 'Build Dir', 'Failed Reason', 'Download Command'],
row_attrs=['app_dir', 'build_dir', 'build_comment'],
value_functions=[
(
'Download Command',
lambda _app: f'idf-ci gitlab download-artifacts --pipeline-id {self.pipeline_id} '
f'{idf_relpath(_app.build_path)}',
),
],
)
failed_apps_report_url = self.write_report_to_file(
self.generate_html_report(''.join(failed_apps_table_section)),
self.job_id,
self.failed_apps_report_file,
)
self.additional_info += self.generate_additional_info_section(
self.report_titles_map['failed_apps'], len(failed_apps), failed_apps_report_url
)
return failed_apps_table_section
def get_skipped_apps_report_parts(self) -> list[str]:
skipped_apps = [app for app in self.apps if app.build_status == BuildStatus.SKIPPED]
if not skipped_apps:
return []
skipped_apps_table_section = self.create_table_section(
title=self.report_titles_map['skipped_apps'],
items=skipped_apps,
headers=['App Dir', 'Build Dir', 'Skipped Reason'],
row_attrs=['app_dir', 'build_dir', 'build_comment'],
)
skipped_apps_report_url = self.write_report_to_file(
self.generate_html_report(''.join(skipped_apps_table_section)),
self.job_id,
self.skipped_apps_report_file,
)
self.additional_info += self.generate_additional_info_section(
self.report_titles_map['skipped_apps'], len(skipped_apps), skipped_apps_report_url
)
return skipped_apps_table_section
def _get_report_str(self) -> str:
self.additional_info = (
f'**Build Summary ({self.get_commit_summary}):**\n'
'\n'
'> ℹ️ Note: Binary artifacts stored in MinIO are retained for 4 DAYS from their build date\n'
)
failed_apps_report_parts = self.get_failed_apps_report_parts()
skipped_apps_report_parts = self.get_skipped_apps_report_parts()
built_apps_report_parts = self.get_built_apps_report_parts()
return self.generate_html_report(
''.join(failed_apps_report_parts + built_apps_report_parts + skipped_apps_report_parts)
)
class JobReportGenerator(ReportGenerator):
def __init__(
self,
project_id: int,
mr_iid: int,
pipeline_id: int,
job_id: int,
commit_id: str,
local_commit_id: str,
*,
title: str = 'Job Report',
jobs: list[GitlabJob],
):
super().__init__(project_id, mr_iid, pipeline_id, job_id, commit_id, local_commit_id, title=title)
self.jobs = jobs
self.report_titles_map = {
'failed_jobs': 'Failed Jobs (Excludes "integration_test" and "target_test" jobs)',
'succeeded': 'Succeeded Jobs',
}
self.failed_jobs_report_file = 'job_report.html'
def _get_report_str(self) -> str:
"""
Generate a complete HTML report string by processing jobs.
:return: Complete HTML report string.
"""
report_str: str = ''
if not self.jobs:
print('No jobs found, skip generating job report')
return 'No Job Found'
relevant_failed_jobs = self._sort_items(
self._filter_items(
self.jobs, lambda job: job.is_failed and job.stage not in ['integration_test', 'target_test']
),
key='latest_failed_count',
)
succeeded_jobs = self._filter_items(self.jobs, lambda job: job.is_success)
self.additional_info = f'**Job Summary ({self.get_commit_summary}):**\n'
self.additional_info += self.generate_additional_info_section(
self.report_titles_map['succeeded'], len(succeeded_jobs)
)
if not relevant_failed_jobs:
self.additional_info += self.generate_additional_info_section(
self.report_titles_map['failed_jobs'], len(relevant_failed_jobs)
)
return report_str
report_sections = self.create_table_section(
title='Failed Jobs (Excludes "integration_test" and "target_test" jobs)',
items=relevant_failed_jobs,
headers=[
'Job Name',
'Failure Reason',
'Failure Log',
'Failures across all other branches (10 latest jobs)',
'URL',
'CI Dashboard URL',
],
row_attrs=['name', 'failure_reason', 'failure_log', 'url', 'ci_dashboard_url'],
value_functions=[
(
'Failures across all other branches (10 latest jobs)',
lambda item: f'{getattr(item, "latest_failed_count", "")} '
f'/ {getattr(item, "latest_total_count", "")}',
)
],
)
relevant_failed_jobs_report_url = get_artifacts_url(self.job_id, self.failed_jobs_report_file)
self.additional_info += self.generate_additional_info_section(
self.report_titles_map['failed_jobs'], len(relevant_failed_jobs), relevant_failed_jobs_report_url
)
report_str = self.generate_html_report(''.join(report_sections))
return report_str

121
tools/ci/dynamic_pipelines/scripts/generate_report.py
View File

@@ -1,121 +0,0 @@
# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import argparse
import glob
import os
import subprocess
import typing as t
import __init__ # noqa: F401 # inject the system path
from idf_build_apps import json_list_files_to_apps
from idf_ci import GitlabEnvVars
from idf_ci_local.app import enrich_apps_with_metrics_info
from dynamic_pipelines.report import BuildReportGenerator
from dynamic_pipelines.report import JobReportGenerator
from dynamic_pipelines.utils import fetch_app_metrics
from dynamic_pipelines.utils import fetch_failed_jobs
def main() -> None:
parser: argparse.ArgumentParser = setup_argument_parser()
args: argparse.Namespace = parser.parse_args()
report_actions: dict[str, t.Callable[[argparse.Namespace], None]] = {
'build': generate_build_report,
'job': generate_jobs_report,
}
report_action = report_actions.get(args.report_type)
if report_action is None:
raise ValueError('Unknown report type is requested to be generated.')
report_action(args)
def setup_argument_parser() -> argparse.ArgumentParser:
report_type_parser: argparse.ArgumentParser = argparse.ArgumentParser(add_help=False)
report_type_parser.add_argument(
'--report-type', choices=['build', 'job'], required=True, help='Type of report to generate'
)
report_type_args: argparse.Namespace
remaining_args: list[str]
report_type_args, remaining_args = report_type_parser.parse_known_args()
parser: argparse.ArgumentParser = argparse.ArgumentParser(
description='Update reports in MR pipelines based on the selected report type',
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
parents=[report_type_parser],
)
common_arguments(parser)
conditional_arguments(report_type_args, parser)
return parser
def common_arguments(parser: argparse.ArgumentParser) -> None:
parser.add_argument('--project-id', type=int, default=os.getenv('CI_PROJECT_ID'), help='Project ID')
parser.add_argument('--mr-iid', type=int, default=os.getenv('CI_MERGE_REQUEST_IID'), help='Merge Request IID')
parser.add_argument('--pipeline-id', type=int, default=os.getenv('PARENT_PIPELINE_ID'), help='Pipeline ID')
parser.add_argument('--job-id', type=int, default=os.getenv('CI_JOB_ID'), help='Job ID')
parser.add_argument('--commit-id', default=os.getenv('CI_COMMIT_SHA', ''), help='MR merged result commit ID')
parser.add_argument('--local-commit-id', default=os.getenv('PIPELINE_COMMIT_SHA', ''), help='local dev commit ID')
def conditional_arguments(report_type_args: argparse.Namespace, parser: argparse.ArgumentParser) -> None:
if report_type_args.report_type == 'build':
parser.add_argument('--app-list-filepattern', default='app_info*.txt', help='Pattern to match app list files')
elif report_type_args.report_type == 'target_test':
parser.add_argument(
'--junit-report-filepattern', default='XUNIT_RESULT*.xml', help='Pattern to match JUnit report files'
)
def generate_build_report(args: argparse.Namespace) -> None:
# generate presigned url for the artifacts
subprocess.check_output(
[
'idf-ci',
'gitlab',
'generate-presigned-json',
'--commit-sha',
args.local_commit_id,
'--output',
'presigned.json',
],
)
print('generated presigned.json')
# generate report
apps = json_list_files_to_apps(glob.glob(args.app_list_filepattern))
print(f'loaded {len(apps)} apps')
app_metrics = fetch_app_metrics(
source_commit_sha=args.commit_id,
target_commit_sha=os.environ.get('CI_MERGE_REQUEST_TARGET_BRANCH_SHA'),
)
apps = enrich_apps_with_metrics_info(app_metrics, apps)
report_generator = BuildReportGenerator(
args.project_id, args.mr_iid, args.pipeline_id, args.job_id, args.commit_id, args.local_commit_id, apps=apps
)
report_generator.post_report()
def generate_jobs_report(args: argparse.Namespace) -> None:
jobs: list[t.Any] = fetch_failed_jobs(args.commit_id)
if not jobs:
return
report_generator = JobReportGenerator(
args.project_id, args.mr_iid, args.pipeline_id, args.job_id, args.commit_id, args.local_commit_id, jobs=jobs
)
report_generator.post_report()
if GitlabEnvVars().IDF_CI_IS_DEBUG_PIPELINE:
print('Debug pipeline detected, exit non-zero to fail the pipeline in order to block merge')
exit(30)
if __name__ == '__main__':
main()

6
tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml
View File

@@ -56,12 +56,6 @@
needs:
- pipeline: $PARENT_PIPELINE_ID
job: pipeline_variables
cache:
# Usually do not need submodule-cache in target_test
- key: pip-cache-${LATEST_GIT_TAG}
paths:
- .cache/pip
policy: pull
artifacts:
paths:
- XUNIT_RESULT*.xml

131
tools/ci/dynamic_pipelines/templates/report.template.html
View File

@@ -1,131 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>{{title}}</title>
<link
rel="shortcut icon"
href="https://www.espressif.com/sites/all/themes/espressif/favicon.ico"
type="image/vnd.microsoft.icon"
/>
<!-- External CSS libraries -->
<link
href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css"
rel="stylesheet"
/>
<link
href="https://unpkg.com/bootstrap-table@1.22.1/dist/bootstrap-table.min.css"
rel="stylesheet"
/>
<link
rel="stylesheet"
href="https://cdn.jsdelivr.net/npm/bootstrap-table@1.23.0/dist/extensions/sticky-header/bootstrap-table-sticky-header.css"
/>
<link
href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0-beta3/css/all.min.css"
rel="stylesheet"
/>
<!-- CSS content will be injected here during template rendering -->
<style>
/* {{css_content}} */
/* Additional styles for disabled tabs */
.report-nav-tabs .nav-tab.disabled {
opacity: 0.5;
cursor: not-allowed;
pointer-events: none;
}
/* Loading animation for tabs */
.report-nav-tabs .nav-tab.loading:after {
content: "";
display: inline-block;
width: 1em;
height: 1em;
border: 2px solid rgba(0, 0, 0, 0.2);
border-left-color: #333;
border-radius: 50%;
margin-left: 8px;
animation: spin 1s linear infinite;
}
@keyframes spin {
to {
transform: rotate(360deg);
}
}
</style>
</head>
<body data-pipeline-id="{{pipeline_id}}">
<!-- Navigation progress bar -->
<div class="nav-progress-container">
<div class="nav-progress-bar" id="nav-progress-bar"></div>
</div>
<div class="container-fluid">
<!-- Report header section -->
<header class="report-header">
<div class="logo-container">
<img
src="https://www.espressif.com/sites/all/themes/espressif/logo-black.svg"
alt="Espressif Logo"
class="logo"
/>
</div>
<div class="title-container">
<h1>
<span style="color: #333">Dynamic Pipeline Report</span>
</h1>
</div>
<div class="spacer"></div>
</header>
<!-- Search and controls section -->
<div class="row mb-4">
<div class="col-md-8">
</div>
<div class="col-md-4 text-end">
<div class="action-buttons">
<button
class="btn btn-esp btn-sm"
id="expand-all-tables"
>
<i class="fas fa-table"></i> Expand All
</button>
<button
class="btn btn-outline-secondary btn-sm ms-2"
id="collapse-all-tables"
>
<i class="fas fa-minus"></i> Collapse All
</button>
</div>
</div>
</div>
<div class="table-responsive">
<div class="table-container">{{table}}</div>
</div>
</div>
<!-- Floating action buttons -->
<div class="floating-actions">
<div class="floating-action-btn back-to-top" id="back-to-top">
<i class="fas fa-arrow-up"></i>
</div>
</div>
<!-- JavaScript libraries -->
<script src="https://cdn.jsdelivr.net/npm/jquery/dist/jquery.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/js/bootstrap.bundle.min.js"></script>
<script src="https://unpkg.com/bootstrap-table@1.22.1/dist/bootstrap-table.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/bootstrap-table@1.23.0/dist/extensions/sticky-header/bootstrap-table-sticky-header.min.js"></script>
<script src="https://unpkg.com/bootstrap-table@1.22.1/dist/extensions/export/bootstrap-table-export.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/tableexport.jquery.plugin@1.10.21/tableExport.min.js"></script>
<script src="https://unpkg.com/bootstrap-table@1.22.1/dist/extensions/filter-control/bootstrap-table-filter-control.min.js"></script>
<!-- Custom scripts -->
<script>
{{js_content}}
</script>
</body>
</html>

1127
tools/ci/dynamic_pipelines/templates/scripts.js
View File

File diff suppressed because it is too large Load Diff

1095
tools/ci/dynamic_pipelines/templates/styles.css
View File

File diff suppressed because it is too large Load Diff

9
tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml
View File

@@ -16,7 +16,7 @@ all_build_finished:
script:
- echo "all test jobs finished"
generate_pytest_build_report:
generate_presigned_json:
stage: assign_test
image: $ESP_ENV_IMAGE
tags:
@@ -29,16 +29,11 @@ generate_pytest_build_report:
job: pipeline_variables
artifacts:
paths:
- failed_apps.html
- built_apps.html
- skipped_apps.html
- build_report.html
- presigned.json
expire_in: 1 week
when: always
script:
- python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type build
- python tools/ci/previous_stage_job_status.py --stage build
- idf-ci gitlab generate-presigned-json --output presigned.json
generate_pytest_child_pipeline:
# finally, we can get some use out of the default behavior that downloads all artifacts from the previous stage

151
tools/ci/dynamic_pipelines/utils.py
View File

@@ -1,151 +0,0 @@
# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import os
from urllib.parse import urlparse
import requests
from .constants import CI_DASHBOARD_API
from .constants import CI_JOB_TOKEN
from .constants import CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
from .constants import CI_PAGES_URL
from .constants import CI_PROJECT_URL
from .models import GitlabJob
def is_url(string: str) -> bool:
"""
Check if the string is a valid URL by parsing it and verifying if it contains both a scheme and a network location.
:param string: The string to check if it is a URL.
:return: True if the string is a valid URL, False otherwise.
"""
parsed = urlparse(string)
return bool(parsed.scheme) and bool(parsed.netloc)
def fetch_failed_jobs(commit_id: str) -> list[GitlabJob]:
"""
Fetches a list of jobs from the specified commit_id using an API request to ci-dashboard-api.
:param commit_id: The commit ID for which to fetch jobs.
:return: A list of jobs if the request is successful, otherwise an empty list.
"""
response = requests.get(
f'{CI_DASHBOARD_API}/commits/{commit_id}/jobs',
headers={'CI-Job-Token': CI_JOB_TOKEN},
)
if response.status_code != 200:
print(f'Failed to fetch jobs data: {response.status_code} with error: {response.text}')
return []
data = response.json()
jobs = data.get('jobs', [])
if not jobs:
return []
failed_job_names = [job['name'] for job in jobs if job['status'] == 'failed']
response = requests.post(
f'{CI_DASHBOARD_API}/jobs/failure_ratio',
headers={'CI-Job-Token': CI_JOB_TOKEN},
json={
'job_names': failed_job_names,
'exclude_branches': [os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_NAME', '')],
},
)
if response.status_code != 200:
print(f'Failed to fetch jobs failure rate data: {response.status_code} with error: {response.text}')
return []
failure_rate_data = response.json()
failure_rates = {item['name']: item for item in failure_rate_data.get('jobs', [])}
combined_jobs = []
for job in jobs:
failure_data = failure_rates.get(job['name'], {})
combined_jobs.append(GitlabJob.from_json_data(job, failure_data))
return combined_jobs
def fetch_app_metrics(
source_commit_sha: str,
target_commit_sha: str,
) -> dict:
"""
Fetches the app metrics for the given source commit SHA and target branch SHA.
:param source_commit_sha: The source commit SHA.
:param target_branch_sha: The commit SHA of the branch to compare app sizes against.
:return: A dict of sizes of built binaries.
"""
print(f'Fetching bin size info: {source_commit_sha=} {target_commit_sha=}')
build_info_map = dict()
response = requests.post(
f'{CI_DASHBOARD_API}/apps/metrics',
headers={'CI-Job-Token': CI_JOB_TOKEN},
json={
'source_commit_sha': source_commit_sha,
'target_commit_sha': target_commit_sha,
},
)
if response.status_code != 200:
print(f'Failed to fetch build info: {response.status_code} - {response.text}')
else:
response_data = response.json()
build_info_map = {
f'{info["app_path"]}_{info["config_name"]}_{info["target"]}': info for info in response_data.get('data', [])
}
return build_info_map
def load_file(file_path: str) -> str:
"""
Loads the content of a file.
:param file_path: The path to the file needs to be loaded.
:return: The content of the file as a string.
"""
with open(file_path) as file:
return file.read()
def format_permalink(s: str) -> str:
"""
Formats a given string into a permalink.
:param s: The string to be formatted into a permalink.
:return: The formatted permalink as a string.
"""
end_index = s.find('(')
if end_index != -1:
trimmed_string = s[:end_index].strip()
else:
trimmed_string = s.strip()
formatted_string = trimmed_string.lower().replace(' ', '-')
return formatted_string
def get_artifacts_url(job_id: int, output_filepath: str) -> str:
"""
Generates the url of the path where the artifact will be stored in the job's artifacts .
:param job_id: The job identifier used to construct the URL.
:param output_filepath: The path to the output file.
:return: The modified URL pointing to the job's artifacts.
"""
url = CI_PAGES_URL.replace('esp-idf', '-/esp-idf')
return f'{url}/-/jobs/{job_id}/artifacts/{output_filepath}'
def get_repository_file_url(file_path: str) -> str:
"""
Generates the url of the file path inside the repository.
:param file_path: The file path where the file is stored.
:return: The modified URL pointing to the file's path in the repository.
"""
return f'{CI_PROJECT_URL}/-/raw/{CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}/{file_path}'

1
tools/ci/exclude_check_tools_files.txt
View File

@@ -29,7 +29,6 @@ tools/ci/idf_pytest/**/*
tools/ci/mirror-submodule-update.sh
tools/ci/multirun_with_pyenv.sh
tools/ci/mypy_ignore_list.txt
tools/ci/previous_stage_job_status.py
tools/ci/push_to_github.sh
tools/ci/python_packages/common_test_methods.py
tools/ci/python_packages/gitlab_api.py

103
tools/ci/idf_ci_local/app.py
View File

@@ -5,13 +5,10 @@ import subprocess
import sys
import typing as t
from dynamic_pipelines.constants import BINARY_SIZE_METRIC_NAME
from idf_build_apps import App
from idf_build_apps import CMakeApp
from idf_build_apps.constants import BuildStatus
from idf_build_apps.utils import rmdir
from idf_ci_utils import APP_EXTRA_S3_ARTIFACT_TYPE
from idf_ci_utils import idf_relpath
if t.TYPE_CHECKING:
pass
@@ -64,103 +61,3 @@ class IdfCMakeApp(CMakeApp):
self.build_path,
exclude_file_patterns=['build_log.txt', 'size*.json'],
)
class Metrics:
"""
Represents a metric and its values for source, target, and the differences.
"""
def __init__(
self,
source_value: float | None = None,
target_value: float | None = None,
difference: float | None = None,
difference_percentage: float | None = None,
) -> None:
self.source_value = source_value or 0.0
self.target_value = target_value or 0.0
self.difference = difference or 0.0
self.difference_percentage = difference_percentage or 0.0
def to_dict(self) -> dict[str, t.Any]:
"""
Converts the Metrics object to a dictionary.
"""
return {
'source_value': self.source_value,
'target_value': self.target_value,
'difference': self.difference,
'difference_percentage': self.difference_percentage,
}
class AppWithMetricsInfo(IdfCMakeApp):
metrics: dict[str, Metrics]
is_new_app: bool
def __init__(self, **kwargs: t.Any) -> None:
super().__init__(**kwargs)
self.metrics = {metric_name: metric_data for metric_name, metric_data in kwargs.get('metrics', {}).items()}
self.is_new_app = kwargs.get('is_new_app', False)
class Config:
arbitrary_types_allowed = True
def enrich_apps_with_metrics_info(
app_metrics_info_map: dict[str, dict[str, t.Any]], apps: list[App]
) -> list[AppWithMetricsInfo]:
def _get_full_attributes(obj: App) -> dict[str, t.Any]:
"""
Retrieves all attributes of an object, including properties and computed fields.
"""
attributes: dict[str, t.Any] = obj.__dict__.copy()
for attr in dir(obj):
if not attr.startswith('_'): # Skip private/internal attributes
try:
value = getattr(obj, attr)
# Include only if it's not already in __dict__
if attr not in attributes:
attributes[attr] = value
except Exception:
# Skip attributes that raise exceptions (e.g., methods needing args)
pass
return attributes
default_metrics_structure = {
BINARY_SIZE_METRIC_NAME: Metrics(
source_value=0,
target_value=0,
difference=0,
difference_percentage=0.0,
),
}
apps_with_metrics_info = []
for app in apps:
app.app_dir = idf_relpath(app.app_dir)
key = f'{app.app_dir}_{app.config_name}_{app.target}'
app_attributes = _get_full_attributes(app)
metrics = {metric_name: default_metric for metric_name, default_metric in default_metrics_structure.items()}
is_new_app = False
if key in app_metrics_info_map:
info = app_metrics_info_map[key]
for metric_name, metric_data in info.get('metrics', {}).items():
metrics[metric_name] = Metrics(
source_value=metric_data.get('source_value', 0),
target_value=metric_data.get('target_value', 0),
difference=metric_data.get('difference', 0),
difference_percentage=metric_data.get('difference_percentage', 0.0),
)
is_new_app = info.get('is_new_app', False)
app_attributes.update({'metrics': metrics, 'is_new_app': is_new_app})
apps_with_metrics_info.append(AppWithMetricsInfo(**app_attributes))
return apps_with_metrics_info

18
tools/ci/idf_ci_utils.py
View File

@@ -4,7 +4,6 @@
# some CI related util functions
import logging
import os
import re
import subprocess
import sys
import typing as t
@@ -243,23 +242,6 @@ class GitlabYmlConfig:
return self.config[name] # type: ignore
def sanitize_job_name(name: str) -> str:
"""
Sanitize the job name from CI_JOB_NAME
- for job with `parallel: int` set, the `CI_JOB_NAME` would be `job_name index/total`, like `foo 1/3`
- for job with `parallel: matrix` set, the `CI_JOB_NAME` would be `job_name: [var1, var2]`, like `foo: [a, b]`
We consider
- the jobs generated by `parallel: int` as the same job, i.e., we remove the index/total part.
- the jobs generated by `parallel: matrix` as different jobs, so we keep the matrix part.
:param name: job name
:return: sanitized job name
"""
return re.sub(r' \d+/\d+', '', name)
def idf_relpath(p: str) -> str:
"""
Turn all paths under IDF_PATH to relative paths

36
tools/ci/previous_stage_job_status.py
View File

@@ -1,36 +0,0 @@
# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import argparse
import os
import sys
import requests
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--stage', type=str, help='Stage name for check jobs status')
args = parser.parse_args()
GITLAB_TOKEN = os.getenv('ESPCI_TOKEN')
GITLAB_HTTP_SERVER = os.getenv('GITLAB_HTTP_SERVER')
CI_PROJECT_ID = os.getenv('CI_PROJECT_ID')
CI_PIPELINE_ID = os.getenv('CI_PIPELINE_ID')
api_path = f'projects/{CI_PROJECT_ID}/pipelines/{CI_PIPELINE_ID}/jobs?scope[]=failed&per_page=100'
page = 0
while True:
response = requests.get(
f'{GITLAB_HTTP_SERVER}/api/v4/{api_path}&page={page}',
headers={'PRIVATE-TOKEN': GITLAB_TOKEN}
)
jobs = response.json()
if not jobs:
break
for job in jobs:
if job['stage'] == args.stage:
print(f'Jobs from the previous stage {args.stage} should pass; otherwise, this job will fail.')
sys.exit(1)
page += 1