mirror of
https://github.com/espressif/esp-idf.git
synced 2026-09-22 13:01:16 +03:00
ci: remove pip-cache and other unused jobs
This commit is contained in:
Fu Hanxi
@@ -60,9 +60,6 @@ variables:
|
||||
TARGET_TEST_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v5.5-target-test:1"
|
||||
SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:5"
|
||||
|
||||
# cache python dependencies
|
||||
PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
|
||||
|
||||
# Set this variable to the branch of idf-constraints repo in order to test a custom Python constraint file. The
|
||||
# branch name must be without the remote part ("origin/"). Keep the variable empty in order to use the constraint
|
||||
# file from https://dl.espressif.com/dl/esp-idf.
|
||||
@@ -334,12 +331,6 @@ default:
|
||||
cache:
|
||||
# pull only for most of the use cases since it's cache dir.
|
||||
# Only set "push" policy for "upload_cache" stage jobs
|
||||
- key: pip-cache-${LATEST_GIT_TAG}
|
||||
fallback_keys:
|
||||
- pip-cache
|
||||
paths:
|
||||
- .cache/pip
|
||||
policy: pull
|
||||
- key: submodule-cache-${LATEST_GIT_TAG}
|
||||
fallback_keys:
|
||||
- submodule-cache
|
||||
|
||||
@@ -7,9 +7,6 @@
|
||||
# run host_test jobs immediately, only after upload cache
|
||||
needs:
|
||||
- pipeline_variables
|
||||
- job: upload-pip-cache
|
||||
optional: true
|
||||
artifacts: false
|
||||
- job: upload-submodules-cache
|
||||
optional: true
|
||||
artifacts: false
|
||||
|
||||
@@ -2,22 +2,6 @@
|
||||
stage: post_deploy
|
||||
image: $ESP_ENV_IMAGE
|
||||
|
||||
generate_failed_jobs_report:
|
||||
extends:
|
||||
- .post_deploy_template
|
||||
tags: [build, shiny]
|
||||
when: always
|
||||
dependencies: # Do not download artifacts from the previous stages
|
||||
needs:
|
||||
- pipeline_variables
|
||||
artifacts:
|
||||
expire_in: 2 week
|
||||
when: always
|
||||
paths:
|
||||
- job_report.html
|
||||
script:
|
||||
- python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type job
|
||||
|
||||
sync_support_status:
|
||||
extends:
|
||||
- .post_deploy_template
|
||||
|
||||
@@ -13,9 +13,6 @@
|
||||
- "components/**/Kconfig"
|
||||
- "components/**/CMakeLists.txt"
|
||||
|
||||
.patterns-python-cache: &patterns-python-cache
|
||||
- "tools/requirements.json"
|
||||
- "tools/requirements/requirements.*.txt"
|
||||
|
||||
.patterns-python-files: &patterns-python-files
|
||||
- ".gitlab/ci/static-code-analysis.yml"
|
||||
@@ -227,15 +224,6 @@
|
||||
rules:
|
||||
- <<: *if-dev-push
|
||||
|
||||
.rules:upload-python-cache:
|
||||
rules:
|
||||
- <<: *if-release-tag
|
||||
- <<: *if-schedule-nightly
|
||||
- <<: *if-protected-branch-push
|
||||
changes: *patterns-python-cache
|
||||
- <<: *if-label-upload_cache
|
||||
when: manual
|
||||
|
||||
.rules:upload-submodule-cache:
|
||||
rules:
|
||||
- <<: *if-release-tag
|
||||
|
||||
@@ -6,9 +6,6 @@
|
||||
dependencies: # set dependencies to null to avoid missing artifacts issue
|
||||
# run host_test jobs immediately, only after upload cache
|
||||
needs:
|
||||
- job: upload-pip-cache
|
||||
optional: true
|
||||
artifacts: false
|
||||
- job: upload-submodules-cache
|
||||
optional: true
|
||||
artifacts: false
|
||||
|
||||
@@ -6,26 +6,6 @@
|
||||
stage: upload_cache
|
||||
image: $ESP_ENV_IMAGE
|
||||
|
||||
upload-pip-cache:
|
||||
extends:
|
||||
- .upload_cache_template
|
||||
- .before_script:minimal
|
||||
- .rules:upload-python-cache
|
||||
tags:
|
||||
- $GEO
|
||||
- cache
|
||||
cache:
|
||||
key: pip-cache-${LATEST_GIT_TAG}
|
||||
paths:
|
||||
- .cache/pip
|
||||
policy: push
|
||||
script:
|
||||
- rm -rf .cache/pip # clear old packages
|
||||
- bash install.sh --enable-ci --enable-test-specific
|
||||
parallel:
|
||||
matrix:
|
||||
- GEO: [ 'shiny', 'brew' ]
|
||||
|
||||
upload-submodules-cache:
|
||||
extends:
|
||||
- .upload_cache_template
|
||||
|
||||
@@ -1,26 +1,9 @@
|
||||
# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-FileCopyrightText: 2024-2026 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
import os
|
||||
|
||||
from idf_ci_utils import IDF_PATH
|
||||
|
||||
COMMENT_START_MARKER = '### Dynamic Pipeline Report'
|
||||
|
||||
REPORT_TEMPLATE_FILEPATH = os.path.join(
|
||||
IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'report.template.html'
|
||||
)
|
||||
CSS_STYLES_FILEPATH = os.path.join(IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'styles.css')
|
||||
JS_SCRIPTS_FILEPATH = os.path.join(IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'scripts.js')
|
||||
TOP_N_APPS_BY_SIZE_DIFF = 10
|
||||
SIZE_DIFFERENCE_BYTES_THRESHOLD = 500
|
||||
BINARY_SIZE_METRIC_NAME = 'binary_size'
|
||||
|
||||
KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH = os.path.join(
|
||||
IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'known_generate_test_child_pipeline_warnings.yml'
|
||||
)
|
||||
|
||||
CI_JOB_TOKEN = os.getenv('CI_JOB_TOKEN', '')
|
||||
CI_DASHBOARD_API = os.getenv('CI_DASHBOARD_API', '')
|
||||
CI_PAGES_URL = os.getenv('CI_PAGES_URL', '')
|
||||
CI_PROJECT_URL = os.getenv('CI_PROJECT_URL', '')
|
||||
CI_MERGE_REQUEST_SOURCE_BRANCH_SHA = os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA', '')
|
||||
|
||||
@@ -1,121 +0,0 @@
|
||||
# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
import os
|
||||
import typing as t
|
||||
import urllib.parse
|
||||
from dataclasses import dataclass
|
||||
from xml.etree.ElementTree import Element
|
||||
|
||||
from idf_ci_utils import IDF_PATH
|
||||
|
||||
|
||||
@dataclass
|
||||
class TestCase:
|
||||
name: str
|
||||
file: str
|
||||
time: float
|
||||
app_path: t.Optional[str] = None
|
||||
failure: t.Optional[str] = None
|
||||
skipped: t.Optional[str] = None
|
||||
ci_job_url: t.Optional[str] = None
|
||||
ci_dashboard_url: t.Optional[str] = None
|
||||
dut_log_url: t.Optional[str] = None
|
||||
latest_total_count: int = 0
|
||||
latest_failed_count: int = 0
|
||||
|
||||
@property
|
||||
def is_failure(self) -> bool:
|
||||
return self.failure is not None
|
||||
|
||||
@property
|
||||
def is_skipped(self) -> bool:
|
||||
return self.skipped is not None
|
||||
|
||||
@property
|
||||
def is_success(self) -> bool:
|
||||
return not self.is_failure and not self.is_skipped
|
||||
|
||||
@classmethod
|
||||
def _get_idf_rel_path(cls, path: str) -> str:
|
||||
if path.startswith(IDF_PATH):
|
||||
return os.path.relpath(path, IDF_PATH)
|
||||
else:
|
||||
return path
|
||||
|
||||
@classmethod
|
||||
def from_test_case_node(cls, node: Element) -> t.Optional['TestCase']:
|
||||
if 'name' not in node.attrib:
|
||||
print('WARNING: Node Invalid: ', node)
|
||||
return None
|
||||
|
||||
# url to test cases dashboard
|
||||
grafana_base_url = urllib.parse.urljoin(os.getenv('CI_DASHBOARD_HOST', ''), '/d/Ucg477Fnz/case-list')
|
||||
encoded_params = urllib.parse.urlencode({'var-case_id': node.attrib['name']}, quote_via=urllib.parse.quote)
|
||||
|
||||
kwargs = {
|
||||
'name': node.attrib['name'],
|
||||
'file': node.attrib.get('file'),
|
||||
'app_path': '|'.join(
|
||||
cls._get_idf_rel_path(path) for path in node.attrib.get('app_path', 'unknown').split('|')
|
||||
),
|
||||
'time': float(node.attrib.get('time') or 0),
|
||||
'ci_job_url': node.attrib.get('ci_job_url') or 'Not found',
|
||||
'ci_dashboard_url': f'{grafana_base_url}?{encoded_params}',
|
||||
'dut_log_url': node.attrib.get('dut_log_url') or 'Not found',
|
||||
}
|
||||
|
||||
failure_node = node.find('failure')
|
||||
# bool(failure_node) is False, so compare with None
|
||||
if failure_node is None:
|
||||
failure_node = node.find('error')
|
||||
if failure_node is not None:
|
||||
message = failure_node.attrib.get('message', '')
|
||||
kwargs['failure'] = message
|
||||
|
||||
skipped_node = node.find('skipped')
|
||||
if skipped_node is not None:
|
||||
kwargs['skipped'] = skipped_node.attrib['message']
|
||||
|
||||
return cls(**kwargs) # type: ignore
|
||||
|
||||
|
||||
@dataclass
|
||||
class GitlabJob:
|
||||
id: int
|
||||
name: str
|
||||
stage: str
|
||||
status: str
|
||||
url: str
|
||||
ci_dashboard_url: str
|
||||
failure_reason: t.Optional[str] = None
|
||||
failure_log: t.Optional[str] = None
|
||||
latest_total_count: int = 0
|
||||
latest_failed_count: int = 0
|
||||
|
||||
@property
|
||||
def is_failed(self) -> bool:
|
||||
return self.status == 'failed'
|
||||
|
||||
@property
|
||||
def is_success(self) -> bool:
|
||||
return self.status == 'success'
|
||||
|
||||
@classmethod
|
||||
def from_json_data(cls, job_data: dict, failure_data: dict) -> t.Optional['GitlabJob']:
|
||||
grafana_base_url = urllib.parse.urljoin(os.getenv('CI_DASHBOARD_HOST', ''), '/d/LoUa-qLWz/job-list')
|
||||
encoded_params = urllib.parse.urlencode({'var-job_name': job_data['name']}, quote_via=urllib.parse.quote)
|
||||
|
||||
kwargs = {
|
||||
'id': job_data['id'],
|
||||
'name': job_data['name'],
|
||||
'stage': job_data['stage'],
|
||||
'status': job_data['status'],
|
||||
'url': job_data['url'],
|
||||
'ci_dashboard_url': f'{grafana_base_url}?{encoded_params}',
|
||||
'failure_reason': job_data['failure_reason'],
|
||||
'failure_log': job_data['failure_log'],
|
||||
'latest_total_count': failure_data.get('total_count', 0),
|
||||
'latest_failed_count': failure_data.get('failed_count', 0),
|
||||
}
|
||||
|
||||
return cls(**kwargs) # type: ignore
|
||||
@@ -1,739 +0,0 @@
|
||||
# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
import abc
|
||||
import html
|
||||
import re
|
||||
import typing as t
|
||||
from textwrap import dedent
|
||||
|
||||
from gitlab import GitlabUpdateError
|
||||
from gitlab_api import Gitlab
|
||||
from idf_build_apps.constants import BuildStatus
|
||||
from idf_ci_local.app import AppWithMetricsInfo
|
||||
from idf_ci_utils import idf_relpath
|
||||
from prettytable import PrettyTable
|
||||
|
||||
from .constants import BINARY_SIZE_METRIC_NAME
|
||||
from .constants import CI_DASHBOARD_API
|
||||
from .constants import COMMENT_START_MARKER
|
||||
from .constants import CSS_STYLES_FILEPATH
|
||||
from .constants import JS_SCRIPTS_FILEPATH
|
||||
from .constants import REPORT_TEMPLATE_FILEPATH
|
||||
from .constants import SIZE_DIFFERENCE_BYTES_THRESHOLD
|
||||
from .constants import TOP_N_APPS_BY_SIZE_DIFF
|
||||
from .models import GitlabJob
|
||||
from .models import TestCase
|
||||
from .utils import format_permalink
|
||||
from .utils import get_artifacts_url
|
||||
from .utils import is_url
|
||||
|
||||
|
||||
class ReportGenerator:
|
||||
REGEX_PATTERN = r'#### {}\n[\s\S]*?(?=\n#### |$)'
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
project_id: int,
|
||||
mr_iid: int,
|
||||
pipeline_id: int,
|
||||
job_id: int,
|
||||
commit_id: str,
|
||||
local_commit_id: str,
|
||||
*,
|
||||
title: str,
|
||||
):
|
||||
gl_project = Gitlab(project_id).project
|
||||
if mr_iid is not None:
|
||||
self.mr = gl_project.mergerequests.get(mr_iid)
|
||||
else:
|
||||
self.mr = None
|
||||
self.pipeline_id = pipeline_id
|
||||
self.job_id = job_id
|
||||
self.commit_id = commit_id
|
||||
self.local_commit_id = local_commit_id
|
||||
|
||||
self.title = title
|
||||
self.output_filepath = self.title.lower().replace(' ', '_') + '.html'
|
||||
self.additional_info = ''
|
||||
|
||||
@property
|
||||
def get_commit_summary(self) -> str:
|
||||
return f'with CI commit SHA: {self.commit_id[:8]}, local commit SHA: {self.local_commit_id[:8]}'
|
||||
|
||||
@staticmethod
|
||||
def get_download_link_for_url(url: str) -> str:
|
||||
if url:
|
||||
return f'<a href="{url}">Download</a>'
|
||||
|
||||
return ''
|
||||
|
||||
@staticmethod
|
||||
def write_report_to_file(report_str: str, job_id: int, output_filepath: str) -> t.Optional[str]:
|
||||
"""
|
||||
Writes the report to a file and constructs a modified URL based on environment settings.
|
||||
|
||||
:param report_str: The report content to be written to the file.
|
||||
:param job_id: The job identifier used to construct the URL.
|
||||
:param output_filepath: The path to the output file.
|
||||
:return: The modified URL pointing to the job's artifacts.
|
||||
"""
|
||||
if not report_str:
|
||||
return None
|
||||
with open(output_filepath, 'w') as file:
|
||||
file.write(report_str)
|
||||
|
||||
# for example, {URL}/-/esp-idf/-/jobs/{id}/artifacts/app_info_84.txt
|
||||
# CI_PAGES_URL is {URL}/esp-idf, which missed one `-`
|
||||
report_url: str = get_artifacts_url(job_id, output_filepath)
|
||||
return report_url
|
||||
|
||||
@staticmethod
|
||||
def _load_file_content(filepath: str) -> str:
|
||||
"""
|
||||
Load the content of a file as string
|
||||
|
||||
:param filepath: Path to the file to load
|
||||
:return: Content of the file as string
|
||||
"""
|
||||
try:
|
||||
with open(filepath, encoding='utf-8') as f:
|
||||
return f.read()
|
||||
except (OSError, FileNotFoundError) as e:
|
||||
print(f'Warning: Could not read file {filepath}: {e}')
|
||||
return ''
|
||||
|
||||
def generate_html_report(self, table_str: str) -> str:
|
||||
# we're using bootstrap table
|
||||
table_str = table_str.replace(
|
||||
'<table>',
|
||||
'<table data-toggle="table" data-search-align="left" data-search="true" data-sticky-header="true">',
|
||||
)
|
||||
|
||||
template = self._load_file_content(REPORT_TEMPLATE_FILEPATH)
|
||||
css_content = self._load_file_content(CSS_STYLES_FILEPATH)
|
||||
js_content = self._load_file_content(JS_SCRIPTS_FILEPATH)
|
||||
|
||||
template = template.replace('{{css_content}}', css_content)
|
||||
template = template.replace('{{js_content}}', js_content)
|
||||
template = template.replace('{{pipeline_id}}', str(self.pipeline_id))
|
||||
template = template.replace('{{apiBaseUrl}}', CI_DASHBOARD_API)
|
||||
|
||||
return template.replace('{{title}}', self.title).replace('{{table}}', table_str)
|
||||
|
||||
@staticmethod
|
||||
def table_to_html_str(table: PrettyTable) -> str:
|
||||
return html.unescape(table.get_html_string()) # type: ignore
|
||||
|
||||
def create_table_section(
|
||||
self,
|
||||
title: str,
|
||||
items: list,
|
||||
headers: list,
|
||||
row_attrs: list,
|
||||
value_functions: t.Optional[list] = None,
|
||||
) -> t.List:
|
||||
"""
|
||||
Appends a formatted section to a report based on the provided items. This section includes
|
||||
a header and a table constructed from the items list with specified headers and attributes.
|
||||
|
||||
:param title: Title for the report section. This title is used as a header above the table.
|
||||
:param items: List of item objects to include in the table. Each item should have attributes
|
||||
that correspond to the row_attrs and value_functions specified.
|
||||
:param headers: List of strings that will serve as the column headers in the generated table.
|
||||
:param row_attrs: List of attributes to include from each item for the table rows. These
|
||||
should be attributes or keys that exist on the items in the 'items' list.
|
||||
:param value_functions: Optional list of tuples containing additional header and corresponding
|
||||
value function. Each tuple should specify a header (as a string) and
|
||||
a function that takes an item and returns a string. This is used for
|
||||
generating dynamic columns based on item data.
|
||||
|
||||
:return: List with appended HTML sections.
|
||||
"""
|
||||
if not items:
|
||||
return []
|
||||
|
||||
report_sections = [
|
||||
f"""<h2 id="{format_permalink(title)}">{title}<i class="fas fa-link copy-link-icon"
|
||||
onclick="copyPermalink('#{format_permalink(title)}')"></i></h2>""",
|
||||
self._create_table_for_items(
|
||||
items=items, headers=headers, row_attrs=row_attrs, value_functions=value_functions or []
|
||||
),
|
||||
]
|
||||
return report_sections
|
||||
|
||||
@staticmethod
|
||||
def generate_additional_info_section(
|
||||
title: str, count: int, report_url: t.Optional[str] = None, add_permalink: bool = True
|
||||
) -> str:
|
||||
"""
|
||||
Generate a section for the additional info string.
|
||||
|
||||
:param title: The title of the section.
|
||||
:param count: The count of test cases.
|
||||
:param report_url: The URL of the report. If count = 0, only the count will be included.
|
||||
:param add_permalink: Whether to include a permalink in the report URL. Defaults to True.
|
||||
:return: The formatted additional info section string.
|
||||
"""
|
||||
if count != 0 and report_url:
|
||||
if add_permalink:
|
||||
return f'- **{title}:** [{count}]({report_url}/#{format_permalink(title)})\n'
|
||||
else:
|
||||
return f'- **{title}:** [{count}]({report_url})\n'
|
||||
else:
|
||||
return f'- **{title}:** {count}\n'
|
||||
|
||||
def _create_table_for_items(
|
||||
self,
|
||||
items: t.Union[t.List[TestCase], t.List[GitlabJob]],
|
||||
headers: t.List[str],
|
||||
row_attrs: t.List[str],
|
||||
value_functions: t.Optional[t.List[t.Tuple[str, t.Callable[[t.Union[TestCase, GitlabJob]], str]]]] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Create a PrettyTable and convert it to an HTML string for the provided test cases.
|
||||
:param items: List of item objects to include in the table.
|
||||
:param headers: List of strings for the table headers.
|
||||
:param row_attrs: List of attributes to include in each row.
|
||||
:param value_functions: List of tuples containing additional header and corresponding value function.
|
||||
:return: HTML table string.
|
||||
"""
|
||||
table = PrettyTable()
|
||||
table.field_names = headers
|
||||
|
||||
# Create a mapping of header names to their corresponding index in the headers list
|
||||
header_index_map = {header: i for i, header in enumerate(headers)}
|
||||
|
||||
for item in items:
|
||||
row = []
|
||||
for attr in row_attrs:
|
||||
value = str(getattr(item, attr, ''))
|
||||
if is_url(value):
|
||||
link = f'<a href="{value}">link</a>'
|
||||
row.append(link)
|
||||
else:
|
||||
row.append(value)
|
||||
|
||||
# Insert values computed by value functions at the correct column position based on their headers
|
||||
if value_functions:
|
||||
for header, func in value_functions:
|
||||
index = header_index_map.get(header)
|
||||
if index is not None:
|
||||
computed_value = func(item)
|
||||
row.insert(index, computed_value)
|
||||
|
||||
table.add_row(row)
|
||||
|
||||
return self.table_to_html_str(table)
|
||||
|
||||
@staticmethod
|
||||
def _filter_items(
|
||||
items: t.Union[t.List[TestCase], t.List[GitlabJob]], condition: t.Callable[[t.Union[TestCase, GitlabJob]], bool]
|
||||
) -> t.List[TestCase]:
|
||||
"""
|
||||
Filter items s based on a given condition.
|
||||
|
||||
:param items: List of items to filter by given condition.
|
||||
:param condition: A function that evaluates to True or False for each items.
|
||||
:return: List of filtered instances.
|
||||
"""
|
||||
return [item for item in items if condition(item)]
|
||||
|
||||
@staticmethod
|
||||
def _sort_items(
|
||||
items: t.List[t.Union[TestCase, GitlabJob, AppWithMetricsInfo]],
|
||||
key: t.Union[str, t.Callable[[t.Union[TestCase, GitlabJob, AppWithMetricsInfo]], t.Any]],
|
||||
order: str = 'asc',
|
||||
sort_function: t.Optional[t.Callable[[t.Any], t.Any]] = None,
|
||||
) -> t.List[t.Union[TestCase, GitlabJob, AppWithMetricsInfo]]:
|
||||
"""
|
||||
Sort items based on a given key, order, and optional custom sorting function.
|
||||
|
||||
:param items: List of items to sort.
|
||||
:param key: A string representing the attribute name or a function to extract the sorting key.
|
||||
:param order: Order of sorting ('asc' for ascending, 'desc' for descending).
|
||||
:param sort_function: A custom function to control sorting logic
|
||||
(e.g., prioritizing positive/negative/zero values).
|
||||
:return: List of sorted instances.
|
||||
"""
|
||||
key_func = None
|
||||
if isinstance(key, str):
|
||||
|
||||
def key_func(item: t.Any) -> t.Any:
|
||||
return getattr(item, key)
|
||||
|
||||
sorting_key = sort_function if sort_function is not None else key_func
|
||||
try:
|
||||
items = sorted(items, key=sorting_key, reverse=(order == 'desc'))
|
||||
except TypeError:
|
||||
print(f'Comparison for the key {key} is not supported')
|
||||
|
||||
return items
|
||||
|
||||
@abc.abstractmethod
|
||||
def _get_report_str(self) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
def _generate_comment(self) -> str:
|
||||
# Report in HTML format to avoid exceeding length limits
|
||||
comment = f'#### {self.title}\n'
|
||||
report_str = self._get_report_str()
|
||||
comment += f'{self.additional_info}\n'
|
||||
self.write_report_to_file(report_str, self.job_id, self.output_filepath)
|
||||
|
||||
return comment
|
||||
|
||||
def _update_mr_comment(self, comment: str) -> None:
|
||||
new_comment = f'{COMMENT_START_MARKER}\n\n{comment}'
|
||||
|
||||
for note in self.mr.notes.list(iterator=True):
|
||||
if note.body.startswith(COMMENT_START_MARKER):
|
||||
updated_str = self._get_updated_comment(note.body, comment)
|
||||
|
||||
note.body = updated_str
|
||||
try:
|
||||
note.save()
|
||||
except GitlabUpdateError:
|
||||
print('Failed to update MR comment, Creating a new comment')
|
||||
self.mr.notes.create({'body': new_comment})
|
||||
break
|
||||
else:
|
||||
self.mr.notes.create({'body': new_comment})
|
||||
|
||||
def _get_updated_comment(self, existing_comment: str, new_comment: str) -> str:
|
||||
updated_str = re.sub(self.REGEX_PATTERN.format(self.title), new_comment, existing_comment)
|
||||
if updated_str == existing_comment:
|
||||
updated_str = f'{existing_comment.strip()}\n\n{new_comment}'
|
||||
return updated_str
|
||||
|
||||
def post_report(self) -> None:
|
||||
comment = self._generate_comment()
|
||||
|
||||
print(comment)
|
||||
|
||||
if self.mr is None:
|
||||
print('No MR found, skip posting comment')
|
||||
return
|
||||
|
||||
self._update_mr_comment(comment)
|
||||
|
||||
|
||||
class BuildReportGenerator(ReportGenerator):
|
||||
def __init__(
|
||||
self,
|
||||
project_id: int,
|
||||
mr_iid: int,
|
||||
pipeline_id: int,
|
||||
job_id: int,
|
||||
commit_id: str,
|
||||
local_commit_id: str,
|
||||
*,
|
||||
title: str = 'Build Report',
|
||||
apps: list[AppWithMetricsInfo],
|
||||
) -> None:
|
||||
super().__init__(project_id, mr_iid, pipeline_id, job_id, commit_id, local_commit_id, title=title)
|
||||
self.apps = apps
|
||||
self.report_titles_map = {
|
||||
'failed_apps': 'Failed Apps',
|
||||
'built_test_related_apps': 'Built Apps - Test Related',
|
||||
'built_non_test_related_apps': 'Built Apps - Non Test Related',
|
||||
'new_test_related_apps': 'New Apps - Test Related',
|
||||
'new_non_test_related_apps': 'New Apps - Non Test Related',
|
||||
'skipped_apps': 'Skipped Apps',
|
||||
}
|
||||
self.failed_apps_report_file = 'failed_apps.html'
|
||||
self.built_apps_report_file = 'built_apps.html'
|
||||
self.skipped_apps_report_file = 'skipped_apps.html'
|
||||
|
||||
@staticmethod
|
||||
def custom_sort(item: AppWithMetricsInfo) -> tuple[int, t.Any]:
|
||||
"""
|
||||
Custom sort function to:
|
||||
1. Push items with zero binary sizes to the end.
|
||||
2. Sort other items by absolute size_difference_percentage.
|
||||
"""
|
||||
# Priority: 0 for zero binaries, 1 for non-zero binaries
|
||||
zero_binary_priority = (
|
||||
1
|
||||
if item.metrics[BINARY_SIZE_METRIC_NAME].source_value != 0
|
||||
or item.metrics[BINARY_SIZE_METRIC_NAME].target_value != 0
|
||||
else 0
|
||||
)
|
||||
# Secondary sort: Negative absolute size_difference_percentage for descending order
|
||||
size_difference_sort = abs(item.metrics[BINARY_SIZE_METRIC_NAME].difference_percentage)
|
||||
return zero_binary_priority, size_difference_sort
|
||||
|
||||
def _generate_top_n_apps_by_size_table(self) -> str:
|
||||
"""
|
||||
Generate a markdown table for the top N apps by size difference.
|
||||
Only includes apps with size differences greater than 500 bytes.
|
||||
"""
|
||||
filtered_apps = [
|
||||
app
|
||||
for app in self.apps
|
||||
if abs(app.metrics[BINARY_SIZE_METRIC_NAME].difference) > SIZE_DIFFERENCE_BYTES_THRESHOLD
|
||||
]
|
||||
|
||||
top_apps = sorted(
|
||||
filtered_apps, key=lambda app: abs(app.metrics[BINARY_SIZE_METRIC_NAME].difference_percentage), reverse=True
|
||||
)[:TOP_N_APPS_BY_SIZE_DIFF]
|
||||
|
||||
if not top_apps:
|
||||
return ''
|
||||
|
||||
table = (
|
||||
f'\n⚠️⚠️⚠️ Top {len(top_apps)} Apps with Binary Size Sorted by Size Difference\n'
|
||||
f'Note: Apps with changes of less than {SIZE_DIFFERENCE_BYTES_THRESHOLD} bytes are not shown.\n'
|
||||
)
|
||||
table += '| App Dir | Build Dir | Size Diff (bytes) | Size Diff (%) |\n'
|
||||
table += '|---------|-----------|-------------------|---------------|\n'
|
||||
for app in top_apps:
|
||||
table += dedent(
|
||||
f'| {app.app_dir} | {app.build_dir} | '
|
||||
f'{app.metrics[BINARY_SIZE_METRIC_NAME].difference} | '
|
||||
f'{app.metrics[BINARY_SIZE_METRIC_NAME].difference_percentage}% |\n'
|
||||
)
|
||||
table += (
|
||||
'\n**For more details, please click on the numbers in the summary above '
|
||||
'to view the corresponding report files.** ⬆️⬆️⬆️\n\n'
|
||||
)
|
||||
|
||||
return table
|
||||
|
||||
@staticmethod
|
||||
def split_new_and_existing_apps(
|
||||
apps: t.Iterable[AppWithMetricsInfo],
|
||||
) -> tuple[list[AppWithMetricsInfo], list[AppWithMetricsInfo]]:
|
||||
"""
|
||||
Splits apps into new apps and existing apps.
|
||||
|
||||
:param apps: Iterable of apps to process.
|
||||
:return: A tuple (new_apps, existing_apps).
|
||||
"""
|
||||
new_apps = [app for app in apps if app.is_new_app]
|
||||
existing_apps = [app for app in apps if not app.is_new_app]
|
||||
return new_apps, existing_apps
|
||||
|
||||
def filter_apps_by_criteria(self, build_status: str, preserve: bool) -> list[AppWithMetricsInfo]:
|
||||
"""
|
||||
Filters apps based on build status and preserve criteria.
|
||||
|
||||
:param build_status: Build status to filter by.
|
||||
:param preserve: Whether to filter preserved apps.
|
||||
:return: Filtered list of apps.
|
||||
"""
|
||||
return [app for app in self.apps if app.build_status == build_status and app.preserve == preserve]
|
||||
|
||||
def get_built_apps_report_parts(self) -> list[str]:
|
||||
"""
|
||||
Generates report parts for new and existing apps.
|
||||
|
||||
:return: List of report parts.
|
||||
"""
|
||||
new_test_related_apps, built_test_related_apps = self.split_new_and_existing_apps(
|
||||
self.filter_apps_by_criteria(BuildStatus.SUCCESS, True)
|
||||
)
|
||||
|
||||
new_non_test_related_apps, built_non_test_related_apps = self.split_new_and_existing_apps(
|
||||
self.filter_apps_by_criteria(BuildStatus.SUCCESS, False)
|
||||
)
|
||||
|
||||
sections = []
|
||||
|
||||
if new_test_related_apps:
|
||||
new_test_related_apps_table_section = self.create_table_section(
|
||||
title=self.report_titles_map['new_test_related_apps'],
|
||||
items=new_test_related_apps,
|
||||
headers=[
|
||||
'App Dir',
|
||||
'Build Dir',
|
||||
'Download Command',
|
||||
'Your Branch App Size',
|
||||
],
|
||||
row_attrs=[
|
||||
'app_dir',
|
||||
'build_dir',
|
||||
],
|
||||
value_functions=[
|
||||
('Your Branch App Size', lambda _app: str(_app.metrics[BINARY_SIZE_METRIC_NAME].source_value)),
|
||||
(
|
||||
'Download Command',
|
||||
lambda _app: f'idf-ci gitlab download-artifacts --pipeline-id {self.pipeline_id} '
|
||||
f'{idf_relpath(_app.build_path)}',
|
||||
),
|
||||
],
|
||||
)
|
||||
sections.extend(new_test_related_apps_table_section)
|
||||
|
||||
if built_test_related_apps:
|
||||
built_test_related_apps = self._sort_items(
|
||||
built_test_related_apps,
|
||||
key='metrics.binary_size.difference_percentage',
|
||||
order='desc',
|
||||
sort_function=self.custom_sort,
|
||||
)
|
||||
|
||||
built_test_related_apps_table_section = self.create_table_section(
|
||||
title=self.report_titles_map['built_test_related_apps'],
|
||||
items=built_test_related_apps,
|
||||
headers=[
|
||||
'App Dir',
|
||||
'Build Dir',
|
||||
'Download Command',
|
||||
'Your Branch App Size',
|
||||
'Target Branch App Size',
|
||||
'Size Diff',
|
||||
'Size Diff, %',
|
||||
],
|
||||
row_attrs=[
|
||||
'app_dir',
|
||||
'build_dir',
|
||||
],
|
||||
value_functions=[
|
||||
('Your Branch App Size', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].source_value)),
|
||||
('Target Branch App Size', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].target_value)),
|
||||
('Size Diff', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].difference)),
|
||||
('Size Diff, %', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].difference_percentage)),
|
||||
(
|
||||
'Download Command',
|
||||
lambda _app: f'idf-ci gitlab download-artifacts --pipeline-id {self.pipeline_id} '
|
||||
f'{idf_relpath(_app.build_path)}',
|
||||
),
|
||||
],
|
||||
)
|
||||
sections.extend(built_test_related_apps_table_section)
|
||||
|
||||
if new_non_test_related_apps:
|
||||
new_non_test_related_apps_table_section = self.create_table_section(
|
||||
title=self.report_titles_map['new_non_test_related_apps'],
|
||||
items=new_non_test_related_apps,
|
||||
headers=[
|
||||
'App Dir',
|
||||
'Build Dir',
|
||||
'Download Command',
|
||||
'Your Branch App Size',
|
||||
],
|
||||
row_attrs=[
|
||||
'app_dir',
|
||||
'build_dir',
|
||||
],
|
||||
value_functions=[
|
||||
(
|
||||
'Download Command',
|
||||
lambda _app: f'idf-ci gitlab download-artifacts --pipeline-id {self.pipeline_id} '
|
||||
f'{idf_relpath(_app.build_path)}',
|
||||
),
|
||||
('Your Branch App Size', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].source_value)),
|
||||
],
|
||||
)
|
||||
sections.extend(new_non_test_related_apps_table_section)
|
||||
|
||||
if built_non_test_related_apps:
|
||||
built_non_test_related_apps = self._sort_items(
|
||||
built_non_test_related_apps,
|
||||
key='metrics.binary_size.difference_percentage',
|
||||
order='desc',
|
||||
sort_function=self.custom_sort,
|
||||
)
|
||||
built_non_test_related_apps_table_section = self.create_table_section(
|
||||
title=self.report_titles_map['built_non_test_related_apps'],
|
||||
items=built_non_test_related_apps,
|
||||
headers=[
|
||||
'App Dir',
|
||||
'Build Dir',
|
||||
'Download Command',
|
||||
'Your Branch App Size',
|
||||
'Target Branch App Size',
|
||||
'Size Diff',
|
||||
'Size Diff, %',
|
||||
],
|
||||
row_attrs=[
|
||||
'app_dir',
|
||||
'build_dir',
|
||||
],
|
||||
value_functions=[
|
||||
('Your Branch App Size', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].source_value)),
|
||||
('Target Branch App Size', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].target_value)),
|
||||
('Size Diff', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].difference)),
|
||||
('Size Diff, %', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].difference_percentage)),
|
||||
(
|
||||
'Download Command',
|
||||
lambda _app: f'idf-ci gitlab download-artifacts --pipeline-id {self.pipeline_id} '
|
||||
f'{idf_relpath(_app.build_path)}',
|
||||
),
|
||||
],
|
||||
)
|
||||
sections.extend(built_non_test_related_apps_table_section)
|
||||
|
||||
built_apps_report_url = self.write_report_to_file(
|
||||
self.generate_html_report(''.join(sections)),
|
||||
self.job_id,
|
||||
self.built_apps_report_file,
|
||||
)
|
||||
|
||||
self.additional_info += self.generate_additional_info_section(
|
||||
self.report_titles_map['built_test_related_apps'],
|
||||
len(built_test_related_apps),
|
||||
built_apps_report_url,
|
||||
)
|
||||
self.additional_info += self.generate_additional_info_section(
|
||||
self.report_titles_map['built_non_test_related_apps'],
|
||||
len(built_non_test_related_apps),
|
||||
built_apps_report_url,
|
||||
)
|
||||
self.additional_info += self.generate_additional_info_section(
|
||||
self.report_titles_map['new_test_related_apps'],
|
||||
len(new_test_related_apps),
|
||||
built_apps_report_url,
|
||||
)
|
||||
self.additional_info += self.generate_additional_info_section(
|
||||
self.report_titles_map['new_non_test_related_apps'],
|
||||
len(new_non_test_related_apps),
|
||||
built_apps_report_url,
|
||||
)
|
||||
|
||||
self.additional_info += self._generate_top_n_apps_by_size_table()
|
||||
|
||||
return sections
|
||||
|
||||
def get_failed_apps_report_parts(self) -> list[str]:
|
||||
failed_apps = [app for app in self.apps if app.build_status == BuildStatus.FAILED]
|
||||
if not failed_apps:
|
||||
return []
|
||||
|
||||
failed_apps_table_section = self.create_table_section(
|
||||
title=self.report_titles_map['failed_apps'],
|
||||
items=failed_apps,
|
||||
headers=['App Dir', 'Build Dir', 'Failed Reason', 'Download Command'],
|
||||
row_attrs=['app_dir', 'build_dir', 'build_comment'],
|
||||
value_functions=[
|
||||
(
|
||||
'Download Command',
|
||||
lambda _app: f'idf-ci gitlab download-artifacts --pipeline-id {self.pipeline_id} '
|
||||
f'{idf_relpath(_app.build_path)}',
|
||||
),
|
||||
],
|
||||
)
|
||||
failed_apps_report_url = self.write_report_to_file(
|
||||
self.generate_html_report(''.join(failed_apps_table_section)),
|
||||
self.job_id,
|
||||
self.failed_apps_report_file,
|
||||
)
|
||||
self.additional_info += self.generate_additional_info_section(
|
||||
self.report_titles_map['failed_apps'], len(failed_apps), failed_apps_report_url
|
||||
)
|
||||
return failed_apps_table_section
|
||||
|
||||
def get_skipped_apps_report_parts(self) -> list[str]:
|
||||
skipped_apps = [app for app in self.apps if app.build_status == BuildStatus.SKIPPED]
|
||||
if not skipped_apps:
|
||||
return []
|
||||
|
||||
skipped_apps_table_section = self.create_table_section(
|
||||
title=self.report_titles_map['skipped_apps'],
|
||||
items=skipped_apps,
|
||||
headers=['App Dir', 'Build Dir', 'Skipped Reason'],
|
||||
row_attrs=['app_dir', 'build_dir', 'build_comment'],
|
||||
)
|
||||
skipped_apps_report_url = self.write_report_to_file(
|
||||
self.generate_html_report(''.join(skipped_apps_table_section)),
|
||||
self.job_id,
|
||||
self.skipped_apps_report_file,
|
||||
)
|
||||
self.additional_info += self.generate_additional_info_section(
|
||||
self.report_titles_map['skipped_apps'], len(skipped_apps), skipped_apps_report_url
|
||||
)
|
||||
return skipped_apps_table_section
|
||||
|
||||
def _get_report_str(self) -> str:
|
||||
self.additional_info = (
|
||||
f'**Build Summary ({self.get_commit_summary}):**\n'
|
||||
'\n'
|
||||
'> ℹ️ Note: Binary artifacts stored in MinIO are retained for 4 DAYS from their build date\n'
|
||||
)
|
||||
failed_apps_report_parts = self.get_failed_apps_report_parts()
|
||||
skipped_apps_report_parts = self.get_skipped_apps_report_parts()
|
||||
built_apps_report_parts = self.get_built_apps_report_parts()
|
||||
|
||||
return self.generate_html_report(
|
||||
''.join(failed_apps_report_parts + built_apps_report_parts + skipped_apps_report_parts)
|
||||
)
|
||||
|
||||
|
||||
class JobReportGenerator(ReportGenerator):
|
||||
def __init__(
|
||||
self,
|
||||
project_id: int,
|
||||
mr_iid: int,
|
||||
pipeline_id: int,
|
||||
job_id: int,
|
||||
commit_id: str,
|
||||
local_commit_id: str,
|
||||
*,
|
||||
title: str = 'Job Report',
|
||||
jobs: list[GitlabJob],
|
||||
):
|
||||
super().__init__(project_id, mr_iid, pipeline_id, job_id, commit_id, local_commit_id, title=title)
|
||||
self.jobs = jobs
|
||||
self.report_titles_map = {
|
||||
'failed_jobs': 'Failed Jobs (Excludes "integration_test" and "target_test" jobs)',
|
||||
'succeeded': 'Succeeded Jobs',
|
||||
}
|
||||
self.failed_jobs_report_file = 'job_report.html'
|
||||
|
||||
def _get_report_str(self) -> str:
|
||||
"""
|
||||
Generate a complete HTML report string by processing jobs.
|
||||
:return: Complete HTML report string.
|
||||
"""
|
||||
report_str: str = ''
|
||||
|
||||
if not self.jobs:
|
||||
print('No jobs found, skip generating job report')
|
||||
return 'No Job Found'
|
||||
|
||||
relevant_failed_jobs = self._sort_items(
|
||||
self._filter_items(
|
||||
self.jobs, lambda job: job.is_failed and job.stage not in ['integration_test', 'target_test']
|
||||
),
|
||||
key='latest_failed_count',
|
||||
)
|
||||
succeeded_jobs = self._filter_items(self.jobs, lambda job: job.is_success)
|
||||
|
||||
self.additional_info = f'**Job Summary ({self.get_commit_summary}):**\n'
|
||||
self.additional_info += self.generate_additional_info_section(
|
||||
self.report_titles_map['succeeded'], len(succeeded_jobs)
|
||||
)
|
||||
|
||||
if not relevant_failed_jobs:
|
||||
self.additional_info += self.generate_additional_info_section(
|
||||
self.report_titles_map['failed_jobs'], len(relevant_failed_jobs)
|
||||
)
|
||||
return report_str
|
||||
|
||||
report_sections = self.create_table_section(
|
||||
title='Failed Jobs (Excludes "integration_test" and "target_test" jobs)',
|
||||
items=relevant_failed_jobs,
|
||||
headers=[
|
||||
'Job Name',
|
||||
'Failure Reason',
|
||||
'Failure Log',
|
||||
'Failures across all other branches (10 latest jobs)',
|
||||
'URL',
|
||||
'CI Dashboard URL',
|
||||
],
|
||||
row_attrs=['name', 'failure_reason', 'failure_log', 'url', 'ci_dashboard_url'],
|
||||
value_functions=[
|
||||
(
|
||||
'Failures across all other branches (10 latest jobs)',
|
||||
lambda item: f'{getattr(item, "latest_failed_count", "")} '
|
||||
f'/ {getattr(item, "latest_total_count", "")}',
|
||||
)
|
||||
],
|
||||
)
|
||||
relevant_failed_jobs_report_url = get_artifacts_url(self.job_id, self.failed_jobs_report_file)
|
||||
self.additional_info += self.generate_additional_info_section(
|
||||
self.report_titles_map['failed_jobs'], len(relevant_failed_jobs), relevant_failed_jobs_report_url
|
||||
)
|
||||
|
||||
report_str = self.generate_html_report(''.join(report_sections))
|
||||
return report_str
|
||||
@@ -1,121 +0,0 @@
|
||||
# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
import argparse
|
||||
import glob
|
||||
import os
|
||||
import subprocess
|
||||
import typing as t
|
||||
|
||||
import __init__ # noqa: F401 # inject the system path
|
||||
from idf_build_apps import json_list_files_to_apps
|
||||
from idf_ci import GitlabEnvVars
|
||||
from idf_ci_local.app import enrich_apps_with_metrics_info
|
||||
|
||||
from dynamic_pipelines.report import BuildReportGenerator
|
||||
from dynamic_pipelines.report import JobReportGenerator
|
||||
from dynamic_pipelines.utils import fetch_app_metrics
|
||||
from dynamic_pipelines.utils import fetch_failed_jobs
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser: argparse.ArgumentParser = setup_argument_parser()
|
||||
args: argparse.Namespace = parser.parse_args()
|
||||
|
||||
report_actions: dict[str, t.Callable[[argparse.Namespace], None]] = {
|
||||
'build': generate_build_report,
|
||||
'job': generate_jobs_report,
|
||||
}
|
||||
|
||||
report_action = report_actions.get(args.report_type)
|
||||
if report_action is None:
|
||||
raise ValueError('Unknown report type is requested to be generated.')
|
||||
|
||||
report_action(args)
|
||||
|
||||
|
||||
def setup_argument_parser() -> argparse.ArgumentParser:
|
||||
report_type_parser: argparse.ArgumentParser = argparse.ArgumentParser(add_help=False)
|
||||
report_type_parser.add_argument(
|
||||
'--report-type', choices=['build', 'job'], required=True, help='Type of report to generate'
|
||||
)
|
||||
report_type_args: argparse.Namespace
|
||||
remaining_args: list[str]
|
||||
report_type_args, remaining_args = report_type_parser.parse_known_args()
|
||||
|
||||
parser: argparse.ArgumentParser = argparse.ArgumentParser(
|
||||
description='Update reports in MR pipelines based on the selected report type',
|
||||
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
||||
parents=[report_type_parser],
|
||||
)
|
||||
|
||||
common_arguments(parser)
|
||||
conditional_arguments(report_type_args, parser)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def common_arguments(parser: argparse.ArgumentParser) -> None:
|
||||
parser.add_argument('--project-id', type=int, default=os.getenv('CI_PROJECT_ID'), help='Project ID')
|
||||
parser.add_argument('--mr-iid', type=int, default=os.getenv('CI_MERGE_REQUEST_IID'), help='Merge Request IID')
|
||||
parser.add_argument('--pipeline-id', type=int, default=os.getenv('PARENT_PIPELINE_ID'), help='Pipeline ID')
|
||||
parser.add_argument('--job-id', type=int, default=os.getenv('CI_JOB_ID'), help='Job ID')
|
||||
parser.add_argument('--commit-id', default=os.getenv('CI_COMMIT_SHA', ''), help='MR merged result commit ID')
|
||||
parser.add_argument('--local-commit-id', default=os.getenv('PIPELINE_COMMIT_SHA', ''), help='local dev commit ID')
|
||||
|
||||
|
||||
def conditional_arguments(report_type_args: argparse.Namespace, parser: argparse.ArgumentParser) -> None:
|
||||
if report_type_args.report_type == 'build':
|
||||
parser.add_argument('--app-list-filepattern', default='app_info*.txt', help='Pattern to match app list files')
|
||||
elif report_type_args.report_type == 'target_test':
|
||||
parser.add_argument(
|
||||
'--junit-report-filepattern', default='XUNIT_RESULT*.xml', help='Pattern to match JUnit report files'
|
||||
)
|
||||
|
||||
|
||||
def generate_build_report(args: argparse.Namespace) -> None:
|
||||
# generate presigned url for the artifacts
|
||||
subprocess.check_output(
|
||||
[
|
||||
'idf-ci',
|
||||
'gitlab',
|
||||
'generate-presigned-json',
|
||||
'--commit-sha',
|
||||
args.local_commit_id,
|
||||
'--output',
|
||||
'presigned.json',
|
||||
],
|
||||
)
|
||||
print('generated presigned.json')
|
||||
|
||||
# generate report
|
||||
apps = json_list_files_to_apps(glob.glob(args.app_list_filepattern))
|
||||
print(f'loaded {len(apps)} apps')
|
||||
app_metrics = fetch_app_metrics(
|
||||
source_commit_sha=args.commit_id,
|
||||
target_commit_sha=os.environ.get('CI_MERGE_REQUEST_TARGET_BRANCH_SHA'),
|
||||
)
|
||||
apps = enrich_apps_with_metrics_info(app_metrics, apps)
|
||||
report_generator = BuildReportGenerator(
|
||||
args.project_id, args.mr_iid, args.pipeline_id, args.job_id, args.commit_id, args.local_commit_id, apps=apps
|
||||
)
|
||||
report_generator.post_report()
|
||||
|
||||
|
||||
def generate_jobs_report(args: argparse.Namespace) -> None:
|
||||
jobs: list[t.Any] = fetch_failed_jobs(args.commit_id)
|
||||
|
||||
if not jobs:
|
||||
return
|
||||
|
||||
report_generator = JobReportGenerator(
|
||||
args.project_id, args.mr_iid, args.pipeline_id, args.job_id, args.commit_id, args.local_commit_id, jobs=jobs
|
||||
)
|
||||
report_generator.post_report()
|
||||
|
||||
if GitlabEnvVars().IDF_CI_IS_DEBUG_PIPELINE:
|
||||
print('Debug pipeline detected, exit non-zero to fail the pipeline in order to block merge')
|
||||
exit(30)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -56,12 +56,6 @@
|
||||
needs:
|
||||
- pipeline: $PARENT_PIPELINE_ID
|
||||
job: pipeline_variables
|
||||
cache:
|
||||
# Usually do not need submodule-cache in target_test
|
||||
- key: pip-cache-${LATEST_GIT_TAG}
|
||||
paths:
|
||||
- .cache/pip
|
||||
policy: pull
|
||||
artifacts:
|
||||
paths:
|
||||
- XUNIT_RESULT*.xml
|
||||
|
||||
@@ -1,131 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>{{title}}</title>
|
||||
<link
|
||||
rel="shortcut icon"
|
||||
href="https://www.espressif.com/sites/all/themes/espressif/favicon.ico"
|
||||
type="image/vnd.microsoft.icon"
|
||||
/>
|
||||
<!-- External CSS libraries -->
|
||||
<link
|
||||
href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/css/bootstrap.min.css"
|
||||
rel="stylesheet"
|
||||
/>
|
||||
<link
|
||||
href="https://unpkg.com/bootstrap-table@1.22.1/dist/bootstrap-table.min.css"
|
||||
rel="stylesheet"
|
||||
/>
|
||||
<link
|
||||
rel="stylesheet"
|
||||
href="https://cdn.jsdelivr.net/npm/bootstrap-table@1.23.0/dist/extensions/sticky-header/bootstrap-table-sticky-header.css"
|
||||
/>
|
||||
<link
|
||||
href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.0.0-beta3/css/all.min.css"
|
||||
rel="stylesheet"
|
||||
/>
|
||||
<!-- CSS content will be injected here during template rendering -->
|
||||
<style>
|
||||
/* {{css_content}} */
|
||||
/* Additional styles for disabled tabs */
|
||||
.report-nav-tabs .nav-tab.disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
/* Loading animation for tabs */
|
||||
.report-nav-tabs .nav-tab.loading:after {
|
||||
content: "";
|
||||
display: inline-block;
|
||||
width: 1em;
|
||||
height: 1em;
|
||||
border: 2px solid rgba(0, 0, 0, 0.2);
|
||||
border-left-color: #333;
|
||||
border-radius: 50%;
|
||||
margin-left: 8px;
|
||||
animation: spin 1s linear infinite;
|
||||
}
|
||||
|
||||
@keyframes spin {
|
||||
to {
|
||||
transform: rotate(360deg);
|
||||
}
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body data-pipeline-id="{{pipeline_id}}">
|
||||
<!-- Navigation progress bar -->
|
||||
<div class="nav-progress-container">
|
||||
<div class="nav-progress-bar" id="nav-progress-bar"></div>
|
||||
</div>
|
||||
|
||||
<div class="container-fluid">
|
||||
<!-- Report header section -->
|
||||
<header class="report-header">
|
||||
<div class="logo-container">
|
||||
<img
|
||||
src="https://www.espressif.com/sites/all/themes/espressif/logo-black.svg"
|
||||
alt="Espressif Logo"
|
||||
class="logo"
|
||||
/>
|
||||
</div>
|
||||
<div class="title-container">
|
||||
<h1>
|
||||
<span style="color: #333">Dynamic Pipeline Report</span>
|
||||
</h1>
|
||||
</div>
|
||||
<div class="spacer"></div>
|
||||
</header>
|
||||
|
||||
<!-- Search and controls section -->
|
||||
<div class="row mb-4">
|
||||
<div class="col-md-8">
|
||||
</div>
|
||||
<div class="col-md-4 text-end">
|
||||
<div class="action-buttons">
|
||||
<button
|
||||
class="btn btn-esp btn-sm"
|
||||
id="expand-all-tables"
|
||||
>
|
||||
<i class="fas fa-table"></i> Expand All
|
||||
</button>
|
||||
<button
|
||||
class="btn btn-outline-secondary btn-sm ms-2"
|
||||
id="collapse-all-tables"
|
||||
>
|
||||
<i class="fas fa-minus"></i> Collapse All
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="table-responsive">
|
||||
<div class="table-container">{{table}}</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Floating action buttons -->
|
||||
<div class="floating-actions">
|
||||
<div class="floating-action-btn back-to-top" id="back-to-top">
|
||||
<i class="fas fa-arrow-up"></i>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- JavaScript libraries -->
|
||||
<script src="https://cdn.jsdelivr.net/npm/jquery/dist/jquery.min.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.2/dist/js/bootstrap.bundle.min.js"></script>
|
||||
<script src="https://unpkg.com/bootstrap-table@1.22.1/dist/bootstrap-table.min.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/bootstrap-table@1.23.0/dist/extensions/sticky-header/bootstrap-table-sticky-header.min.js"></script>
|
||||
<script src="https://unpkg.com/bootstrap-table@1.22.1/dist/extensions/export/bootstrap-table-export.min.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/tableexport.jquery.plugin@1.10.21/tableExport.min.js"></script>
|
||||
<script src="https://unpkg.com/bootstrap-table@1.22.1/dist/extensions/filter-control/bootstrap-table-filter-control.min.js"></script>
|
||||
|
||||
<!-- Custom scripts -->
|
||||
<script>
|
||||
{{js_content}}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -16,7 +16,7 @@ all_build_finished:
|
||||
script:
|
||||
- echo "all test jobs finished"
|
||||
|
||||
generate_pytest_build_report:
|
||||
generate_presigned_json:
|
||||
stage: assign_test
|
||||
image: $ESP_ENV_IMAGE
|
||||
tags:
|
||||
@@ -29,16 +29,11 @@ generate_pytest_build_report:
|
||||
job: pipeline_variables
|
||||
artifacts:
|
||||
paths:
|
||||
- failed_apps.html
|
||||
- built_apps.html
|
||||
- skipped_apps.html
|
||||
- build_report.html
|
||||
- presigned.json
|
||||
expire_in: 1 week
|
||||
when: always
|
||||
script:
|
||||
- python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type build
|
||||
- python tools/ci/previous_stage_job_status.py --stage build
|
||||
- idf-ci gitlab generate-presigned-json --output presigned.json
|
||||
|
||||
generate_pytest_child_pipeline:
|
||||
# finally, we can get some use out of the default behavior that downloads all artifacts from the previous stage
|
||||
|
||||
@@ -1,151 +0,0 @@
|
||||
# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
import os
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import requests
|
||||
|
||||
from .constants import CI_DASHBOARD_API
|
||||
from .constants import CI_JOB_TOKEN
|
||||
from .constants import CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
|
||||
from .constants import CI_PAGES_URL
|
||||
from .constants import CI_PROJECT_URL
|
||||
from .models import GitlabJob
|
||||
|
||||
|
||||
def is_url(string: str) -> bool:
|
||||
"""
|
||||
Check if the string is a valid URL by parsing it and verifying if it contains both a scheme and a network location.
|
||||
|
||||
:param string: The string to check if it is a URL.
|
||||
:return: True if the string is a valid URL, False otherwise.
|
||||
"""
|
||||
parsed = urlparse(string)
|
||||
return bool(parsed.scheme) and bool(parsed.netloc)
|
||||
|
||||
|
||||
def fetch_failed_jobs(commit_id: str) -> list[GitlabJob]:
|
||||
"""
|
||||
Fetches a list of jobs from the specified commit_id using an API request to ci-dashboard-api.
|
||||
:param commit_id: The commit ID for which to fetch jobs.
|
||||
:return: A list of jobs if the request is successful, otherwise an empty list.
|
||||
"""
|
||||
response = requests.get(
|
||||
f'{CI_DASHBOARD_API}/commits/{commit_id}/jobs',
|
||||
headers={'CI-Job-Token': CI_JOB_TOKEN},
|
||||
)
|
||||
if response.status_code != 200:
|
||||
print(f'Failed to fetch jobs data: {response.status_code} with error: {response.text}')
|
||||
return []
|
||||
|
||||
data = response.json()
|
||||
jobs = data.get('jobs', [])
|
||||
|
||||
if not jobs:
|
||||
return []
|
||||
|
||||
failed_job_names = [job['name'] for job in jobs if job['status'] == 'failed']
|
||||
response = requests.post(
|
||||
f'{CI_DASHBOARD_API}/jobs/failure_ratio',
|
||||
headers={'CI-Job-Token': CI_JOB_TOKEN},
|
||||
json={
|
||||
'job_names': failed_job_names,
|
||||
'exclude_branches': [os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_NAME', '')],
|
||||
},
|
||||
)
|
||||
if response.status_code != 200:
|
||||
print(f'Failed to fetch jobs failure rate data: {response.status_code} with error: {response.text}')
|
||||
return []
|
||||
|
||||
failure_rate_data = response.json()
|
||||
failure_rates = {item['name']: item for item in failure_rate_data.get('jobs', [])}
|
||||
|
||||
combined_jobs = []
|
||||
for job in jobs:
|
||||
failure_data = failure_rates.get(job['name'], {})
|
||||
combined_jobs.append(GitlabJob.from_json_data(job, failure_data))
|
||||
|
||||
return combined_jobs
|
||||
|
||||
|
||||
def fetch_app_metrics(
|
||||
source_commit_sha: str,
|
||||
target_commit_sha: str,
|
||||
) -> dict:
|
||||
"""
|
||||
Fetches the app metrics for the given source commit SHA and target branch SHA.
|
||||
:param source_commit_sha: The source commit SHA.
|
||||
:param target_branch_sha: The commit SHA of the branch to compare app sizes against.
|
||||
:return: A dict of sizes of built binaries.
|
||||
"""
|
||||
print(f'Fetching bin size info: {source_commit_sha=} {target_commit_sha=}')
|
||||
build_info_map = dict()
|
||||
response = requests.post(
|
||||
f'{CI_DASHBOARD_API}/apps/metrics',
|
||||
headers={'CI-Job-Token': CI_JOB_TOKEN},
|
||||
json={
|
||||
'source_commit_sha': source_commit_sha,
|
||||
'target_commit_sha': target_commit_sha,
|
||||
},
|
||||
)
|
||||
if response.status_code != 200:
|
||||
print(f'Failed to fetch build info: {response.status_code} - {response.text}')
|
||||
else:
|
||||
response_data = response.json()
|
||||
build_info_map = {
|
||||
f'{info["app_path"]}_{info["config_name"]}_{info["target"]}': info for info in response_data.get('data', [])
|
||||
}
|
||||
|
||||
return build_info_map
|
||||
|
||||
|
||||
def load_file(file_path: str) -> str:
|
||||
"""
|
||||
Loads the content of a file.
|
||||
|
||||
:param file_path: The path to the file needs to be loaded.
|
||||
:return: The content of the file as a string.
|
||||
"""
|
||||
with open(file_path) as file:
|
||||
return file.read()
|
||||
|
||||
|
||||
def format_permalink(s: str) -> str:
|
||||
"""
|
||||
Formats a given string into a permalink.
|
||||
|
||||
:param s: The string to be formatted into a permalink.
|
||||
:return: The formatted permalink as a string.
|
||||
"""
|
||||
end_index = s.find('(')
|
||||
|
||||
if end_index != -1:
|
||||
trimmed_string = s[:end_index].strip()
|
||||
else:
|
||||
trimmed_string = s.strip()
|
||||
|
||||
formatted_string = trimmed_string.lower().replace(' ', '-')
|
||||
|
||||
return formatted_string
|
||||
|
||||
|
||||
def get_artifacts_url(job_id: int, output_filepath: str) -> str:
|
||||
"""
|
||||
Generates the url of the path where the artifact will be stored in the job's artifacts .
|
||||
|
||||
:param job_id: The job identifier used to construct the URL.
|
||||
:param output_filepath: The path to the output file.
|
||||
:return: The modified URL pointing to the job's artifacts.
|
||||
"""
|
||||
url = CI_PAGES_URL.replace('esp-idf', '-/esp-idf')
|
||||
return f'{url}/-/jobs/{job_id}/artifacts/{output_filepath}'
|
||||
|
||||
|
||||
def get_repository_file_url(file_path: str) -> str:
|
||||
"""
|
||||
Generates the url of the file path inside the repository.
|
||||
|
||||
:param file_path: The file path where the file is stored.
|
||||
:return: The modified URL pointing to the file's path in the repository.
|
||||
"""
|
||||
return f'{CI_PROJECT_URL}/-/raw/{CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}/{file_path}'
|
||||
@@ -51,7 +51,6 @@ tools/ci/python_packages/idf_iperf_test_util/**/*
|
||||
tools/esp_prov/**/*
|
||||
tools/ci/sort_yaml.py
|
||||
tools/ci/sg_rules/*
|
||||
tools/ci/previous_stage_job_status.py
|
||||
tools/legacy_exports/export_legacy.fish
|
||||
tools/legacy_exports/export_legacy.sh
|
||||
tools/legacy_exports/export_legacy.ps1
|
||||
|
||||
@@ -10,8 +10,7 @@ from idf_build_apps import CMakeApp
|
||||
from idf_build_apps.constants import BuildStatus
|
||||
from idf_build_apps.utils import rmdir
|
||||
from idf_ci_utils import APP_EXTRA_S3_ARTIFACT_TYPE
|
||||
from idf_ci_utils import idf_relpath
|
||||
from metrics.size_metrics import collect_build_metrics
|
||||
|
||||
|
||||
_SIZE_METRICS_CONFIG_PATH = Path(__file__).parent.parent / 'metrics' / 'size_metrics' / 'size_metrics_config.yml'
|
||||
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
# some CI related util functions
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import typing as t
|
||||
@@ -256,23 +255,6 @@ def get_all_manifest_files() -> t.List[str]:
|
||||
return paths
|
||||
|
||||
|
||||
def sanitize_job_name(name: str) -> str:
|
||||
"""
|
||||
Sanitize the job name from CI_JOB_NAME
|
||||
|
||||
- for job with `parallel: int` set, the `CI_JOB_NAME` would be `job_name index/total`, like `foo 1/3`
|
||||
- for job with `parallel: matrix` set, the `CI_JOB_NAME` would be `job_name: [var1, var2]`, like `foo: [a, b]`
|
||||
|
||||
We consider
|
||||
- the jobs generated by `parallel: int` as the same job, i.e., we remove the index/total part.
|
||||
- the jobs generated by `parallel: matrix` as different jobs, so we keep the matrix part.
|
||||
|
||||
:param name: job name
|
||||
:return: sanitized job name
|
||||
"""
|
||||
return re.sub(r' \d+/\d+', '', name)
|
||||
|
||||
|
||||
def idf_relpath(p: str) -> str:
|
||||
"""
|
||||
Turn all paths under IDF_PATH to relative paths
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
|
||||
import requests
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--stage', type=str, help='Stage name for check jobs status')
|
||||
args = parser.parse_args()
|
||||
|
||||
GITLAB_TOKEN = os.getenv('ESPCI_TOKEN')
|
||||
GITLAB_HTTP_SERVER = os.getenv('GITLAB_HTTP_SERVER')
|
||||
CI_PROJECT_ID = os.getenv('CI_PROJECT_ID')
|
||||
CI_PIPELINE_ID = os.getenv('CI_PIPELINE_ID')
|
||||
|
||||
api_path = f'projects/{CI_PROJECT_ID}/pipelines/{CI_PIPELINE_ID}/jobs?scope[]=failed&per_page=100'
|
||||
|
||||
page = 0
|
||||
while True:
|
||||
response = requests.get(
|
||||
f'{GITLAB_HTTP_SERVER}/api/v4/{api_path}&page={page}',
|
||||
headers={'PRIVATE-TOKEN': GITLAB_TOKEN}
|
||||
)
|
||||
jobs = response.json()
|
||||
if not jobs:
|
||||
break
|
||||
|
||||
for job in jobs:
|
||||
if job['stage'] == args.stage:
|
||||
print(f'Jobs from the previous stage {args.stage} should pass; otherwise, this job will fail.')
|
||||
sys.exit(1)
|
||||
page += 1
|
||||
Reference in New Issue
Block a user