From d008f9456b64a5ed5b2d9f218c21f473cedf1b8f Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Mon, 20 Jul 2026 16:17:30 +0200 Subject: [PATCH] ci: remove pip-cache and other unused jobs --- .gitlab/ci/common.yml | 9 - .gitlab/ci/host-test.yml | 3 - .gitlab/ci/post_deploy.yml | 16 - .gitlab/ci/rules.yml | 12 - .gitlab/ci/test-win.yml | 3 - .gitlab/ci/upload_cache.yml | 20 - .idf_ci.toml | 3 +- tools/ci/dynamic_pipelines/constants.py | 19 +- tools/ci/dynamic_pipelines/models.py | 121 -- tools/ci/dynamic_pipelines/report.py | 739 ----------- .../scripts/generate_report.py | 121 -- .../templates/.dynamic_jobs.yml | 6 - .../templates/report.template.html | 131 -- .../ci/dynamic_pipelines/templates/scripts.js | 1127 ----------------- .../ci/dynamic_pipelines/templates/styles.css | 1095 ---------------- .../templates/test_child_pipeline.yml | 9 +- tools/ci/dynamic_pipelines/utils.py | 151 --- tools/ci/exclude_check_tools_files.txt | 1 - tools/ci/idf_ci_local/app.py | 103 -- tools/ci/idf_ci_utils.py | 18 - tools/ci/previous_stage_job_status.py | 36 - 21 files changed, 4 insertions(+), 3739 deletions(-) delete mode 100644 tools/ci/dynamic_pipelines/models.py delete mode 100644 tools/ci/dynamic_pipelines/report.py delete mode 100644 tools/ci/dynamic_pipelines/scripts/generate_report.py delete mode 100644 tools/ci/dynamic_pipelines/templates/report.template.html delete mode 100644 tools/ci/dynamic_pipelines/templates/scripts.js delete mode 100644 tools/ci/dynamic_pipelines/templates/styles.css delete mode 100644 tools/ci/dynamic_pipelines/utils.py delete mode 100644 tools/ci/previous_stage_job_status.py diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index d58c222d2b7..ec1b1d9ac56 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -69,9 +69,6 @@ variables: ESP_IDF_DOC_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v6.1-docs:1" TARGET_TEST_ENV_IMAGE: "${CI_REGISTRY}/ci/images/idf-v6.1-target-test:1" - # cache python dependencies - PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" - # Set this variable to the branch of idf-constraints repo in order to test a custom Python constraint file. The # branch name must be without the remote part ("origin/"). Keep the variable empty in order to use the constraint # file from https://dl.espressif.com/dl/esp-idf. @@ -136,12 +133,6 @@ default: cache: # pull only for most of the use cases since it's cache dir. # Only set "push" policy for "upload_cache" stage jobs - - key: pip-cache-${LATEST_GIT_TAG} - fallback_keys: - - pip-cache - paths: - - .cache/pip - policy: pull - key: submodule-cache-${LATEST_GIT_TAG} fallback_keys: - submodule-cache diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index 94d186e032b..d6a52c9c085 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -7,9 +7,6 @@ # run host_test jobs immediately, only after upload cache needs: - pipeline_variables - - job: upload-pip-cache - optional: true - artifacts: false - job: upload-submodules-cache optional: true artifacts: false diff --git a/.gitlab/ci/post_deploy.yml b/.gitlab/ci/post_deploy.yml index c17e94089f7..6b8e91bfcf5 100644 --- a/.gitlab/ci/post_deploy.yml +++ b/.gitlab/ci/post_deploy.yml @@ -2,22 +2,6 @@ stage: post_deploy image: $ESP_ENV_IMAGE -generate_failed_jobs_report: - extends: - - .post_deploy_template - tags: [build, shiny] - when: always - dependencies: # Do not download artifacts from the previous stages - needs: - - pipeline_variables - artifacts: - expire_in: 2 week - when: always - paths: - - job_report.html - script: - - python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type job - sync_support_status: extends: - .post_deploy_template diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 44aa4688f95..d3efd5b395d 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -13,9 +13,6 @@ - "components/**/Kconfig" - "components/**/CMakeLists.txt" -.patterns-python-cache: &patterns-python-cache - - "tools/requirements.json" - - "tools/requirements/requirements.*.txt" .patterns-build_components: &patterns-build_components # components files except "test*/" "host*/" folders @@ -222,15 +219,6 @@ rules: - <<: *if-dev-push -.rules:upload-python-cache: - rules: - - <<: *if-release-tag - - <<: *if-schedule-nightly - - <<: *if-protected-branch-push - changes: *patterns-python-cache - - <<: *if-label-upload_cache - when: manual - .rules:upload-submodule-cache: rules: - <<: *if-release-tag diff --git a/.gitlab/ci/test-win.yml b/.gitlab/ci/test-win.yml index 0cb9fadd667..807ce904161 100644 --- a/.gitlab/ci/test-win.yml +++ b/.gitlab/ci/test-win.yml @@ -6,9 +6,6 @@ dependencies: # set dependencies to null to avoid missing artifacts issue # run host_test jobs immediately, only after upload cache needs: - - job: upload-pip-cache - optional: true - artifacts: false - job: upload-submodules-cache optional: true artifacts: false diff --git a/.gitlab/ci/upload_cache.yml b/.gitlab/ci/upload_cache.yml index fdd7e799cc1..320c0cd4244 100644 --- a/.gitlab/ci/upload_cache.yml +++ b/.gitlab/ci/upload_cache.yml @@ -6,26 +6,6 @@ stage: upload_cache image: $ESP_ENV_IMAGE -upload-pip-cache: - extends: - - .upload_cache_template - - .before_script:minimal - - .rules:upload-python-cache - tags: - - $GEO - - cache - cache: - key: pip-cache-${LATEST_GIT_TAG} - paths: - - .cache/pip - policy: push - script: - - rm -rf .cache/pip # clear old packages - - bash install.sh --enable-ci --enable-test-specific - parallel: - matrix: - - GEO: [ 'shiny', 'brew' ] - upload-submodules-cache: extends: - .upload_cache_template diff --git a/.idf_ci.toml b/.idf_ci.toml index c8ff21020f6..c79ce7dc432 100644 --- a/.idf_ci.toml +++ b/.idf_ci.toml @@ -22,8 +22,7 @@ IDF_CI_BUILD = "1" [gitlab.build_pipeline] workflow_name = "build_child_pipeline" -presigned_json_job_name = 'generate_pytest_build_report' -runs_per_job=30 +runs_per_job = 30 job_tags = ['build', 'shiny'] job_template_name = '.dynamic_build_template' diff --git a/tools/ci/dynamic_pipelines/constants.py b/tools/ci/dynamic_pipelines/constants.py index 40dbe346d11..2f64e515e73 100644 --- a/tools/ci/dynamic_pipelines/constants.py +++ b/tools/ci/dynamic_pipelines/constants.py @@ -1,26 +1,9 @@ -# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2024-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import os from idf_ci_utils import IDF_PATH -COMMENT_START_MARKER = '### Dynamic Pipeline Report' - -REPORT_TEMPLATE_FILEPATH = os.path.join( - IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'report.template.html' -) -CSS_STYLES_FILEPATH = os.path.join(IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'styles.css') -JS_SCRIPTS_FILEPATH = os.path.join(IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'scripts.js') -TOP_N_APPS_BY_SIZE_DIFF = 10 -SIZE_DIFFERENCE_BYTES_THRESHOLD = 500 -BINARY_SIZE_METRIC_NAME = 'binary_size' - KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH = os.path.join( IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'known_generate_test_child_pipeline_warnings.yml' ) - -CI_JOB_TOKEN = os.getenv('CI_JOB_TOKEN', '') -CI_DASHBOARD_API = os.getenv('CI_DASHBOARD_API', '') -CI_PAGES_URL = os.getenv('CI_PAGES_URL', '') -CI_PROJECT_URL = os.getenv('CI_PROJECT_URL', '') -CI_MERGE_REQUEST_SOURCE_BRANCH_SHA = os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA', '') diff --git a/tools/ci/dynamic_pipelines/models.py b/tools/ci/dynamic_pipelines/models.py deleted file mode 100644 index 2cd5022a112..00000000000 --- a/tools/ci/dynamic_pipelines/models.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import os -import typing as t -import urllib.parse -from dataclasses import dataclass -from xml.etree.ElementTree import Element - -from idf_ci_utils import IDF_PATH - - -@dataclass -class TestCase: - name: str - file: str - time: float - app_path: t.Optional[str] = None - failure: t.Optional[str] = None - skipped: t.Optional[str] = None - ci_job_url: t.Optional[str] = None - ci_dashboard_url: t.Optional[str] = None - dut_log_url: t.Optional[str] = None - latest_total_count: int = 0 - latest_failed_count: int = 0 - - @property - def is_failure(self) -> bool: - return self.failure is not None - - @property - def is_skipped(self) -> bool: - return self.skipped is not None - - @property - def is_success(self) -> bool: - return not self.is_failure and not self.is_skipped - - @classmethod - def _get_idf_rel_path(cls, path: str) -> str: - if path.startswith(IDF_PATH): - return os.path.relpath(path, IDF_PATH) - else: - return path - - @classmethod - def from_test_case_node(cls, node: Element) -> t.Optional['TestCase']: - if 'name' not in node.attrib: - print('WARNING: Node Invalid: ', node) - return None - - # url to test cases dashboard - grafana_base_url = urllib.parse.urljoin(os.getenv('CI_DASHBOARD_HOST', ''), '/d/Ucg477Fnz/case-list') - encoded_params = urllib.parse.urlencode({'var-case_id': node.attrib['name']}, quote_via=urllib.parse.quote) - - kwargs = { - 'name': node.attrib['name'], - 'file': node.attrib.get('file'), - 'app_path': '|'.join( - cls._get_idf_rel_path(path) for path in node.attrib.get('app_path', 'unknown').split('|') - ), - 'time': float(node.attrib.get('time') or 0), - 'ci_job_url': node.attrib.get('ci_job_url') or 'Not found', - 'ci_dashboard_url': f'{grafana_base_url}?{encoded_params}', - 'dut_log_url': node.attrib.get('dut_log_url') or 'Not found', - } - - failure_node = node.find('failure') - # bool(failure_node) is False, so compare with None - if failure_node is None: - failure_node = node.find('error') - if failure_node is not None: - message = failure_node.attrib.get('message', '') - kwargs['failure'] = message - - skipped_node = node.find('skipped') - if skipped_node is not None: - kwargs['skipped'] = skipped_node.attrib['message'] - - return cls(**kwargs) # type: ignore - - -@dataclass -class GitlabJob: - id: int - name: str - stage: str - status: str - url: str - ci_dashboard_url: str - failure_reason: t.Optional[str] = None - failure_log: t.Optional[str] = None - latest_total_count: int = 0 - latest_failed_count: int = 0 - - @property - def is_failed(self) -> bool: - return self.status == 'failed' - - @property - def is_success(self) -> bool: - return self.status == 'success' - - @classmethod - def from_json_data(cls, job_data: dict, failure_data: dict) -> t.Optional['GitlabJob']: - grafana_base_url = urllib.parse.urljoin(os.getenv('CI_DASHBOARD_HOST', ''), '/d/LoUa-qLWz/job-list') - encoded_params = urllib.parse.urlencode({'var-job_name': job_data['name']}, quote_via=urllib.parse.quote) - - kwargs = { - 'id': job_data['id'], - 'name': job_data['name'], - 'stage': job_data['stage'], - 'status': job_data['status'], - 'url': job_data['url'], - 'ci_dashboard_url': f'{grafana_base_url}?{encoded_params}', - 'failure_reason': job_data['failure_reason'], - 'failure_log': job_data['failure_log'], - 'latest_total_count': failure_data.get('total_count', 0), - 'latest_failed_count': failure_data.get('failed_count', 0), - } - - return cls(**kwargs) # type: ignore diff --git a/tools/ci/dynamic_pipelines/report.py b/tools/ci/dynamic_pipelines/report.py deleted file mode 100644 index 24b0ed9a03a..00000000000 --- a/tools/ci/dynamic_pipelines/report.py +++ /dev/null @@ -1,739 +0,0 @@ -# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import abc -import html -import re -import typing as t -from textwrap import dedent - -from gitlab import GitlabUpdateError -from gitlab_api import Gitlab -from idf_build_apps.constants import BuildStatus -from idf_ci_local.app import AppWithMetricsInfo -from idf_ci_utils import idf_relpath -from prettytable import PrettyTable - -from .constants import BINARY_SIZE_METRIC_NAME -from .constants import CI_DASHBOARD_API -from .constants import COMMENT_START_MARKER -from .constants import CSS_STYLES_FILEPATH -from .constants import JS_SCRIPTS_FILEPATH -from .constants import REPORT_TEMPLATE_FILEPATH -from .constants import SIZE_DIFFERENCE_BYTES_THRESHOLD -from .constants import TOP_N_APPS_BY_SIZE_DIFF -from .models import GitlabJob -from .models import TestCase -from .utils import format_permalink -from .utils import get_artifacts_url -from .utils import is_url - - -class ReportGenerator: - REGEX_PATTERN = r'#### {}\n[\s\S]*?(?=\n#### |$)' - - def __init__( - self, - project_id: int, - mr_iid: int, - pipeline_id: int, - job_id: int, - commit_id: str, - local_commit_id: str, - *, - title: str, - ): - gl_project = Gitlab(project_id).project - if mr_iid is not None: - self.mr = gl_project.mergerequests.get(mr_iid) - else: - self.mr = None - self.pipeline_id = pipeline_id - self.job_id = job_id - self.commit_id = commit_id - self.local_commit_id = local_commit_id - - self.title = title - self.output_filepath = self.title.lower().replace(' ', '_') + '.html' - self.additional_info = '' - - @property - def get_commit_summary(self) -> str: - return f'with CI commit SHA: {self.commit_id[:8]}, local commit SHA: {self.local_commit_id[:8]}' - - @staticmethod - def get_download_link_for_url(url: str) -> str: - if url: - return f'Download' - - return '' - - @staticmethod - def write_report_to_file(report_str: str, job_id: int, output_filepath: str) -> str | None: - """ - Writes the report to a file and constructs a modified URL based on environment settings. - - :param report_str: The report content to be written to the file. - :param job_id: The job identifier used to construct the URL. - :param output_filepath: The path to the output file. - :return: The modified URL pointing to the job's artifacts. - """ - if not report_str: - return None - with open(output_filepath, 'w') as file: - file.write(report_str) - - # for example, {URL}/-/esp-idf/-/jobs/{id}/artifacts/app_info_84.txt - # CI_PAGES_URL is {URL}/esp-idf, which missed one `-` - report_url: str = get_artifacts_url(job_id, output_filepath) - return report_url - - @staticmethod - def _load_file_content(filepath: str) -> str: - """ - Load the content of a file as string - - :param filepath: Path to the file to load - :return: Content of the file as string - """ - try: - with open(filepath, encoding='utf-8') as f: - return f.read() - except (OSError, FileNotFoundError) as e: - print(f'Warning: Could not read file {filepath}: {e}') - return '' - - def generate_html_report(self, table_str: str) -> str: - # we're using bootstrap table - table_str = table_str.replace( - '', - '
', - ) - - template = self._load_file_content(REPORT_TEMPLATE_FILEPATH) - css_content = self._load_file_content(CSS_STYLES_FILEPATH) - js_content = self._load_file_content(JS_SCRIPTS_FILEPATH) - - template = template.replace('{{css_content}}', css_content) - template = template.replace('{{js_content}}', js_content) - template = template.replace('{{pipeline_id}}', str(self.pipeline_id)) - template = template.replace('{{apiBaseUrl}}', CI_DASHBOARD_API) - - return template.replace('{{title}}', self.title).replace('{{table}}', table_str) - - @staticmethod - def table_to_html_str(table: PrettyTable) -> str: - return html.unescape(table.get_html_string()) # type: ignore - - def create_table_section( - self, - title: str, - items: list, - headers: list, - row_attrs: list, - value_functions: list | None = None, - ) -> list: - """ - Appends a formatted section to a report based on the provided items. This section includes - a header and a table constructed from the items list with specified headers and attributes. - - :param title: Title for the report section. This title is used as a header above the table. - :param items: List of item objects to include in the table. Each item should have attributes - that correspond to the row_attrs and value_functions specified. - :param headers: List of strings that will serve as the column headers in the generated table. - :param row_attrs: List of attributes to include from each item for the table rows. These - should be attributes or keys that exist on the items in the 'items' list. - :param value_functions: Optional list of tuples containing additional header and corresponding - value function. Each tuple should specify a header (as a string) and - a function that takes an item and returns a string. This is used for - generating dynamic columns based on item data. - - :return: List with appended HTML sections. - """ - if not items: - return [] - - report_sections = [ - f"""

{title}

""", - self._create_table_for_items( - items=items, headers=headers, row_attrs=row_attrs, value_functions=value_functions or [] - ), - ] - return report_sections - - @staticmethod - def generate_additional_info_section( - title: str, count: int, report_url: str | None = None, add_permalink: bool = True - ) -> str: - """ - Generate a section for the additional info string. - - :param title: The title of the section. - :param count: The count of test cases. - :param report_url: The URL of the report. If count = 0, only the count will be included. - :param add_permalink: Whether to include a permalink in the report URL. Defaults to True. - :return: The formatted additional info section string. - """ - if count != 0 and report_url: - if add_permalink: - return f'- **{title}:** [{count}]({report_url}/#{format_permalink(title)})\n' - else: - return f'- **{title}:** [{count}]({report_url})\n' - else: - return f'- **{title}:** {count}\n' - - def _create_table_for_items( - self, - items: list[TestCase] | list[GitlabJob], - headers: list[str], - row_attrs: list[str], - value_functions: list[tuple[str, t.Callable[[TestCase | GitlabJob], str]]] | None = None, - ) -> str: - """ - Create a PrettyTable and convert it to an HTML string for the provided test cases. - :param items: List of item objects to include in the table. - :param headers: List of strings for the table headers. - :param row_attrs: List of attributes to include in each row. - :param value_functions: List of tuples containing additional header and corresponding value function. - :return: HTML table string. - """ - table = PrettyTable() - table.field_names = headers - - # Create a mapping of header names to their corresponding index in the headers list - header_index_map = {header: i for i, header in enumerate(headers)} - - for item in items: - row = [] - for attr in row_attrs: - value = str(getattr(item, attr, '')) - if is_url(value): - link = f'link' - row.append(link) - else: - row.append(value) - - # Insert values computed by value functions at the correct column position based on their headers - if value_functions: - for header, func in value_functions: - index = header_index_map.get(header) - if index is not None: - computed_value = func(item) - row.insert(index, computed_value) - - table.add_row(row) - - return self.table_to_html_str(table) - - @staticmethod - def _filter_items( - items: list[TestCase] | list[GitlabJob], condition: t.Callable[[TestCase | GitlabJob], bool] - ) -> list[TestCase]: - """ - Filter items s based on a given condition. - - :param items: List of items to filter by given condition. - :param condition: A function that evaluates to True or False for each items. - :return: List of filtered instances. - """ - return [item for item in items if condition(item)] - - @staticmethod - def _sort_items( - items: list[TestCase | GitlabJob | AppWithMetricsInfo], - key: str | t.Callable[[TestCase | GitlabJob | AppWithMetricsInfo], t.Any], - order: str = 'asc', - sort_function: t.Callable[[t.Any], t.Any] | None = None, - ) -> list[TestCase | GitlabJob | AppWithMetricsInfo]: - """ - Sort items based on a given key, order, and optional custom sorting function. - - :param items: List of items to sort. - :param key: A string representing the attribute name or a function to extract the sorting key. - :param order: Order of sorting ('asc' for ascending, 'desc' for descending). - :param sort_function: A custom function to control sorting logic - (e.g., prioritizing positive/negative/zero values). - :return: List of sorted instances. - """ - key_func = None - if isinstance(key, str): - - def key_func(item: t.Any) -> t.Any: - return getattr(item, key) - - sorting_key = sort_function if sort_function is not None else key_func - try: - items = sorted(items, key=sorting_key, reverse=(order == 'desc')) - except TypeError: - print(f'Comparison for the key {key} is not supported') - - return items - - @abc.abstractmethod - def _get_report_str(self) -> str: - raise NotImplementedError - - def _generate_comment(self) -> str: - # Report in HTML format to avoid exceeding length limits - comment = f'#### {self.title}\n' - report_str = self._get_report_str() - comment += f'{self.additional_info}\n' - self.write_report_to_file(report_str, self.job_id, self.output_filepath) - - return comment - - def _update_mr_comment(self, comment: str) -> None: - new_comment = f'{COMMENT_START_MARKER}\n\n{comment}' - - for note in self.mr.notes.list(iterator=True): - if note.body.startswith(COMMENT_START_MARKER): - updated_str = self._get_updated_comment(note.body, comment) - - note.body = updated_str - try: - note.save() - except GitlabUpdateError: - print('Failed to update MR comment, Creating a new comment') - self.mr.notes.create({'body': new_comment}) - break - else: - self.mr.notes.create({'body': new_comment}) - - def _get_updated_comment(self, existing_comment: str, new_comment: str) -> str: - updated_str = re.sub(self.REGEX_PATTERN.format(self.title), new_comment, existing_comment) - if updated_str == existing_comment: - updated_str = f'{existing_comment.strip()}\n\n{new_comment}' - return updated_str - - def post_report(self) -> None: - comment = self._generate_comment() - - print(comment) - - if self.mr is None: - print('No MR found, skip posting comment') - return - - self._update_mr_comment(comment) - - -class BuildReportGenerator(ReportGenerator): - def __init__( - self, - project_id: int, - mr_iid: int, - pipeline_id: int, - job_id: int, - commit_id: str, - local_commit_id: str, - *, - title: str = 'Build Report', - apps: list[AppWithMetricsInfo], - ) -> None: - super().__init__(project_id, mr_iid, pipeline_id, job_id, commit_id, local_commit_id, title=title) - self.apps = apps - self.report_titles_map = { - 'failed_apps': 'Failed Apps', - 'built_test_related_apps': 'Built Apps - Test Related', - 'built_non_test_related_apps': 'Built Apps - Non Test Related', - 'new_test_related_apps': 'New Apps - Test Related', - 'new_non_test_related_apps': 'New Apps - Non Test Related', - 'skipped_apps': 'Skipped Apps', - } - self.failed_apps_report_file = 'failed_apps.html' - self.built_apps_report_file = 'built_apps.html' - self.skipped_apps_report_file = 'skipped_apps.html' - - @staticmethod - def custom_sort(item: AppWithMetricsInfo) -> tuple[int, t.Any]: - """ - Custom sort function to: - 1. Push items with zero binary sizes to the end. - 2. Sort other items by absolute size_difference_percentage. - """ - # Priority: 0 for zero binaries, 1 for non-zero binaries - zero_binary_priority = ( - 1 - if item.metrics[BINARY_SIZE_METRIC_NAME].source_value != 0 - or item.metrics[BINARY_SIZE_METRIC_NAME].target_value != 0 - else 0 - ) - # Secondary sort: Negative absolute size_difference_percentage for descending order - size_difference_sort = abs(item.metrics[BINARY_SIZE_METRIC_NAME].difference_percentage) - return zero_binary_priority, size_difference_sort - - def _generate_top_n_apps_by_size_table(self) -> str: - """ - Generate a markdown table for the top N apps by size difference. - Only includes apps with size differences greater than 500 bytes. - """ - filtered_apps = [ - app - for app in self.apps - if abs(app.metrics[BINARY_SIZE_METRIC_NAME].difference) > SIZE_DIFFERENCE_BYTES_THRESHOLD - ] - - top_apps = sorted( - filtered_apps, key=lambda app: abs(app.metrics[BINARY_SIZE_METRIC_NAME].difference_percentage), reverse=True - )[:TOP_N_APPS_BY_SIZE_DIFF] - - if not top_apps: - return '' - - table = ( - f'\n⚠️⚠️⚠️ Top {len(top_apps)} Apps with Binary Size Sorted by Size Difference\n' - f'Note: Apps with changes of less than {SIZE_DIFFERENCE_BYTES_THRESHOLD} bytes are not shown.\n' - ) - table += '| App Dir | Build Dir | Size Diff (bytes) | Size Diff (%) |\n' - table += '|---------|-----------|-------------------|---------------|\n' - for app in top_apps: - table += dedent( - f'| {app.app_dir} | {app.build_dir} | ' - f'{app.metrics[BINARY_SIZE_METRIC_NAME].difference} | ' - f'{app.metrics[BINARY_SIZE_METRIC_NAME].difference_percentage}% |\n' - ) - table += ( - '\n**For more details, please click on the numbers in the summary above ' - 'to view the corresponding report files.** ⬆️⬆️⬆️\n\n' - ) - - return table - - @staticmethod - def split_new_and_existing_apps( - apps: t.Iterable[AppWithMetricsInfo], - ) -> tuple[list[AppWithMetricsInfo], list[AppWithMetricsInfo]]: - """ - Splits apps into new apps and existing apps. - - :param apps: Iterable of apps to process. - :return: A tuple (new_apps, existing_apps). - """ - new_apps = [app for app in apps if app.is_new_app] - existing_apps = [app for app in apps if not app.is_new_app] - return new_apps, existing_apps - - def filter_apps_by_criteria(self, build_status: str, preserve: bool) -> list[AppWithMetricsInfo]: - """ - Filters apps based on build status and preserve criteria. - - :param build_status: Build status to filter by. - :param preserve: Whether to filter preserved apps. - :return: Filtered list of apps. - """ - return [app for app in self.apps if app.build_status == build_status and app.preserve == preserve] - - def get_built_apps_report_parts(self) -> list[str]: - """ - Generates report parts for new and existing apps. - - :return: List of report parts. - """ - new_test_related_apps, built_test_related_apps = self.split_new_and_existing_apps( - self.filter_apps_by_criteria(BuildStatus.SUCCESS, True) - ) - - new_non_test_related_apps, built_non_test_related_apps = self.split_new_and_existing_apps( - self.filter_apps_by_criteria(BuildStatus.SUCCESS, False) - ) - - sections = [] - - if new_test_related_apps: - new_test_related_apps_table_section = self.create_table_section( - title=self.report_titles_map['new_test_related_apps'], - items=new_test_related_apps, - headers=[ - 'App Dir', - 'Build Dir', - 'Download Command', - 'Your Branch App Size', - ], - row_attrs=[ - 'app_dir', - 'build_dir', - ], - value_functions=[ - ('Your Branch App Size', lambda _app: str(_app.metrics[BINARY_SIZE_METRIC_NAME].source_value)), - ( - 'Download Command', - lambda _app: f'idf-ci gitlab download-artifacts --pipeline-id {self.pipeline_id} ' - f'{idf_relpath(_app.build_path)}', - ), - ], - ) - sections.extend(new_test_related_apps_table_section) - - if built_test_related_apps: - built_test_related_apps = self._sort_items( - built_test_related_apps, - key='metrics.binary_size.difference_percentage', - order='desc', - sort_function=self.custom_sort, - ) - - built_test_related_apps_table_section = self.create_table_section( - title=self.report_titles_map['built_test_related_apps'], - items=built_test_related_apps, - headers=[ - 'App Dir', - 'Build Dir', - 'Download Command', - 'Your Branch App Size', - 'Target Branch App Size', - 'Size Diff', - 'Size Diff, %', - ], - row_attrs=[ - 'app_dir', - 'build_dir', - ], - value_functions=[ - ('Your Branch App Size', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].source_value)), - ('Target Branch App Size', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].target_value)), - ('Size Diff', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].difference)), - ('Size Diff, %', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].difference_percentage)), - ( - 'Download Command', - lambda _app: f'idf-ci gitlab download-artifacts --pipeline-id {self.pipeline_id} ' - f'{idf_relpath(_app.build_path)}', - ), - ], - ) - sections.extend(built_test_related_apps_table_section) - - if new_non_test_related_apps: - new_non_test_related_apps_table_section = self.create_table_section( - title=self.report_titles_map['new_non_test_related_apps'], - items=new_non_test_related_apps, - headers=[ - 'App Dir', - 'Build Dir', - 'Download Command', - 'Your Branch App Size', - ], - row_attrs=[ - 'app_dir', - 'build_dir', - ], - value_functions=[ - ( - 'Download Command', - lambda _app: f'idf-ci gitlab download-artifacts --pipeline-id {self.pipeline_id} ' - f'{idf_relpath(_app.build_path)}', - ), - ('Your Branch App Size', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].source_value)), - ], - ) - sections.extend(new_non_test_related_apps_table_section) - - if built_non_test_related_apps: - built_non_test_related_apps = self._sort_items( - built_non_test_related_apps, - key='metrics.binary_size.difference_percentage', - order='desc', - sort_function=self.custom_sort, - ) - built_non_test_related_apps_table_section = self.create_table_section( - title=self.report_titles_map['built_non_test_related_apps'], - items=built_non_test_related_apps, - headers=[ - 'App Dir', - 'Build Dir', - 'Download Command', - 'Your Branch App Size', - 'Target Branch App Size', - 'Size Diff', - 'Size Diff, %', - ], - row_attrs=[ - 'app_dir', - 'build_dir', - ], - value_functions=[ - ('Your Branch App Size', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].source_value)), - ('Target Branch App Size', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].target_value)), - ('Size Diff', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].difference)), - ('Size Diff, %', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].difference_percentage)), - ( - 'Download Command', - lambda _app: f'idf-ci gitlab download-artifacts --pipeline-id {self.pipeline_id} ' - f'{idf_relpath(_app.build_path)}', - ), - ], - ) - sections.extend(built_non_test_related_apps_table_section) - - built_apps_report_url = self.write_report_to_file( - self.generate_html_report(''.join(sections)), - self.job_id, - self.built_apps_report_file, - ) - - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['built_test_related_apps'], - len(built_test_related_apps), - built_apps_report_url, - ) - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['built_non_test_related_apps'], - len(built_non_test_related_apps), - built_apps_report_url, - ) - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['new_test_related_apps'], - len(new_test_related_apps), - built_apps_report_url, - ) - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['new_non_test_related_apps'], - len(new_non_test_related_apps), - built_apps_report_url, - ) - - self.additional_info += self._generate_top_n_apps_by_size_table() - - return sections - - def get_failed_apps_report_parts(self) -> list[str]: - failed_apps = [app for app in self.apps if app.build_status == BuildStatus.FAILED] - if not failed_apps: - return [] - - failed_apps_table_section = self.create_table_section( - title=self.report_titles_map['failed_apps'], - items=failed_apps, - headers=['App Dir', 'Build Dir', 'Failed Reason', 'Download Command'], - row_attrs=['app_dir', 'build_dir', 'build_comment'], - value_functions=[ - ( - 'Download Command', - lambda _app: f'idf-ci gitlab download-artifacts --pipeline-id {self.pipeline_id} ' - f'{idf_relpath(_app.build_path)}', - ), - ], - ) - failed_apps_report_url = self.write_report_to_file( - self.generate_html_report(''.join(failed_apps_table_section)), - self.job_id, - self.failed_apps_report_file, - ) - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['failed_apps'], len(failed_apps), failed_apps_report_url - ) - return failed_apps_table_section - - def get_skipped_apps_report_parts(self) -> list[str]: - skipped_apps = [app for app in self.apps if app.build_status == BuildStatus.SKIPPED] - if not skipped_apps: - return [] - - skipped_apps_table_section = self.create_table_section( - title=self.report_titles_map['skipped_apps'], - items=skipped_apps, - headers=['App Dir', 'Build Dir', 'Skipped Reason'], - row_attrs=['app_dir', 'build_dir', 'build_comment'], - ) - skipped_apps_report_url = self.write_report_to_file( - self.generate_html_report(''.join(skipped_apps_table_section)), - self.job_id, - self.skipped_apps_report_file, - ) - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['skipped_apps'], len(skipped_apps), skipped_apps_report_url - ) - return skipped_apps_table_section - - def _get_report_str(self) -> str: - self.additional_info = ( - f'**Build Summary ({self.get_commit_summary}):**\n' - '\n' - '> ℹ️ Note: Binary artifacts stored in MinIO are retained for 4 DAYS from their build date\n' - ) - failed_apps_report_parts = self.get_failed_apps_report_parts() - skipped_apps_report_parts = self.get_skipped_apps_report_parts() - built_apps_report_parts = self.get_built_apps_report_parts() - - return self.generate_html_report( - ''.join(failed_apps_report_parts + built_apps_report_parts + skipped_apps_report_parts) - ) - - -class JobReportGenerator(ReportGenerator): - def __init__( - self, - project_id: int, - mr_iid: int, - pipeline_id: int, - job_id: int, - commit_id: str, - local_commit_id: str, - *, - title: str = 'Job Report', - jobs: list[GitlabJob], - ): - super().__init__(project_id, mr_iid, pipeline_id, job_id, commit_id, local_commit_id, title=title) - self.jobs = jobs - self.report_titles_map = { - 'failed_jobs': 'Failed Jobs (Excludes "integration_test" and "target_test" jobs)', - 'succeeded': 'Succeeded Jobs', - } - self.failed_jobs_report_file = 'job_report.html' - - def _get_report_str(self) -> str: - """ - Generate a complete HTML report string by processing jobs. - :return: Complete HTML report string. - """ - report_str: str = '' - - if not self.jobs: - print('No jobs found, skip generating job report') - return 'No Job Found' - - relevant_failed_jobs = self._sort_items( - self._filter_items( - self.jobs, lambda job: job.is_failed and job.stage not in ['integration_test', 'target_test'] - ), - key='latest_failed_count', - ) - succeeded_jobs = self._filter_items(self.jobs, lambda job: job.is_success) - - self.additional_info = f'**Job Summary ({self.get_commit_summary}):**\n' - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['succeeded'], len(succeeded_jobs) - ) - - if not relevant_failed_jobs: - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['failed_jobs'], len(relevant_failed_jobs) - ) - return report_str - - report_sections = self.create_table_section( - title='Failed Jobs (Excludes "integration_test" and "target_test" jobs)', - items=relevant_failed_jobs, - headers=[ - 'Job Name', - 'Failure Reason', - 'Failure Log', - 'Failures across all other branches (10 latest jobs)', - 'URL', - 'CI Dashboard URL', - ], - row_attrs=['name', 'failure_reason', 'failure_log', 'url', 'ci_dashboard_url'], - value_functions=[ - ( - 'Failures across all other branches (10 latest jobs)', - lambda item: f'{getattr(item, "latest_failed_count", "")} ' - f'/ {getattr(item, "latest_total_count", "")}', - ) - ], - ) - relevant_failed_jobs_report_url = get_artifacts_url(self.job_id, self.failed_jobs_report_file) - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['failed_jobs'], len(relevant_failed_jobs), relevant_failed_jobs_report_url - ) - - report_str = self.generate_html_report(''.join(report_sections)) - return report_str diff --git a/tools/ci/dynamic_pipelines/scripts/generate_report.py b/tools/ci/dynamic_pipelines/scripts/generate_report.py deleted file mode 100644 index 1f40fff50ad..00000000000 --- a/tools/ci/dynamic_pipelines/scripts/generate_report.py +++ /dev/null @@ -1,121 +0,0 @@ -# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import argparse -import glob -import os -import subprocess -import typing as t - -import __init__ # noqa: F401 # inject the system path -from idf_build_apps import json_list_files_to_apps -from idf_ci import GitlabEnvVars -from idf_ci_local.app import enrich_apps_with_metrics_info - -from dynamic_pipelines.report import BuildReportGenerator -from dynamic_pipelines.report import JobReportGenerator -from dynamic_pipelines.utils import fetch_app_metrics -from dynamic_pipelines.utils import fetch_failed_jobs - - -def main() -> None: - parser: argparse.ArgumentParser = setup_argument_parser() - args: argparse.Namespace = parser.parse_args() - - report_actions: dict[str, t.Callable[[argparse.Namespace], None]] = { - 'build': generate_build_report, - 'job': generate_jobs_report, - } - - report_action = report_actions.get(args.report_type) - if report_action is None: - raise ValueError('Unknown report type is requested to be generated.') - - report_action(args) - - -def setup_argument_parser() -> argparse.ArgumentParser: - report_type_parser: argparse.ArgumentParser = argparse.ArgumentParser(add_help=False) - report_type_parser.add_argument( - '--report-type', choices=['build', 'job'], required=True, help='Type of report to generate' - ) - report_type_args: argparse.Namespace - remaining_args: list[str] - report_type_args, remaining_args = report_type_parser.parse_known_args() - - parser: argparse.ArgumentParser = argparse.ArgumentParser( - description='Update reports in MR pipelines based on the selected report type', - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - parents=[report_type_parser], - ) - - common_arguments(parser) - conditional_arguments(report_type_args, parser) - - return parser - - -def common_arguments(parser: argparse.ArgumentParser) -> None: - parser.add_argument('--project-id', type=int, default=os.getenv('CI_PROJECT_ID'), help='Project ID') - parser.add_argument('--mr-iid', type=int, default=os.getenv('CI_MERGE_REQUEST_IID'), help='Merge Request IID') - parser.add_argument('--pipeline-id', type=int, default=os.getenv('PARENT_PIPELINE_ID'), help='Pipeline ID') - parser.add_argument('--job-id', type=int, default=os.getenv('CI_JOB_ID'), help='Job ID') - parser.add_argument('--commit-id', default=os.getenv('CI_COMMIT_SHA', ''), help='MR merged result commit ID') - parser.add_argument('--local-commit-id', default=os.getenv('PIPELINE_COMMIT_SHA', ''), help='local dev commit ID') - - -def conditional_arguments(report_type_args: argparse.Namespace, parser: argparse.ArgumentParser) -> None: - if report_type_args.report_type == 'build': - parser.add_argument('--app-list-filepattern', default='app_info*.txt', help='Pattern to match app list files') - elif report_type_args.report_type == 'target_test': - parser.add_argument( - '--junit-report-filepattern', default='XUNIT_RESULT*.xml', help='Pattern to match JUnit report files' - ) - - -def generate_build_report(args: argparse.Namespace) -> None: - # generate presigned url for the artifacts - subprocess.check_output( - [ - 'idf-ci', - 'gitlab', - 'generate-presigned-json', - '--commit-sha', - args.local_commit_id, - '--output', - 'presigned.json', - ], - ) - print('generated presigned.json') - - # generate report - apps = json_list_files_to_apps(glob.glob(args.app_list_filepattern)) - print(f'loaded {len(apps)} apps') - app_metrics = fetch_app_metrics( - source_commit_sha=args.commit_id, - target_commit_sha=os.environ.get('CI_MERGE_REQUEST_TARGET_BRANCH_SHA'), - ) - apps = enrich_apps_with_metrics_info(app_metrics, apps) - report_generator = BuildReportGenerator( - args.project_id, args.mr_iid, args.pipeline_id, args.job_id, args.commit_id, args.local_commit_id, apps=apps - ) - report_generator.post_report() - - -def generate_jobs_report(args: argparse.Namespace) -> None: - jobs: list[t.Any] = fetch_failed_jobs(args.commit_id) - - if not jobs: - return - - report_generator = JobReportGenerator( - args.project_id, args.mr_iid, args.pipeline_id, args.job_id, args.commit_id, args.local_commit_id, jobs=jobs - ) - report_generator.post_report() - - if GitlabEnvVars().IDF_CI_IS_DEBUG_PIPELINE: - print('Debug pipeline detected, exit non-zero to fail the pipeline in order to block merge') - exit(30) - - -if __name__ == '__main__': - main() diff --git a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml index c690ccd0fa1..a145097f312 100644 --- a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml +++ b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml @@ -60,12 +60,6 @@ job: pipeline_variables - pipeline: $PARENT_PIPELINE_ID job: snapshot_known_failure_cases - cache: - # Usually do not need submodule-cache in target_test - - key: pip-cache-${LATEST_GIT_TAG} - paths: - - .cache/pip - policy: pull artifacts: paths: - XUNIT_RESULT*.xml diff --git a/tools/ci/dynamic_pipelines/templates/report.template.html b/tools/ci/dynamic_pipelines/templates/report.template.html deleted file mode 100644 index adbdf4710f3..00000000000 --- a/tools/ci/dynamic_pipelines/templates/report.template.html +++ /dev/null @@ -1,131 +0,0 @@ - - - - - - {{title}} - - - - - - - - - - - -
- -
- -
- -
-
- -
-
-

- Dynamic Pipeline Report -

-
-
-
- - -
-
-
-
-
- - -
-
-
- -
-
{{table}}
-
-
- - -
-
- -
-
- - - - - - - - - - - - - - diff --git a/tools/ci/dynamic_pipelines/templates/scripts.js b/tools/ci/dynamic_pipelines/templates/scripts.js deleted file mode 100644 index 403aa23597c..00000000000 --- a/tools/ci/dynamic_pipelines/templates/scripts.js +++ /dev/null @@ -1,1127 +0,0 @@ -function debounce(func, wait) { - let timeout; - return function () { - const context = this; - const args = arguments; - clearTimeout(timeout); - timeout = setTimeout(() => func.apply(context, args), wait); - }; -} - -function extractPipelineId() { - return $("body").data("pipeline-id"); -} - -function updateTabsAvailability(jobs) { - $("#build-report-tab, #job-report-tab, #test-report-tab").each(function () { - const $tab = $(this); - if (!$tab.hasClass("active")) { - $tab.addClass("disabled"); - $tab.attr("title", "This report is not available yet"); - $tab.css("opacity", "0.5"); - $tab.css("cursor", "not-allowed"); - $tab.on("click", function (e) { - if ($(this).hasClass("disabled")) { - e.preventDefault(); - e.stopPropagation(); - return false; - } - }); - } - }); - - const buildReportPatterns = [ - "generate_pytest_build_report", - "built_apps", - "build_report", - "skipped_apps", - "failed_apps", - ]; - - const jobReportPatterns = [ - "generate_failed_jobs_report", - "job_report", - "pipeline_jobs", - ]; - - const testReportPatterns = [ - "generate_pytest_report", - "target_test_report", - "test_report", - "pytest_report", - ]; - - let buildReportJob = jobs.find((job) => - buildReportPatterns.some((pattern) => job.name.includes(pattern)) - ); - - let jobReportJob = jobs.find((job) => - jobReportPatterns.some((pattern) => job.name.includes(pattern)) - ); - - let testReportJob = jobs.find((job) => - testReportPatterns.some((pattern) => job.name.includes(pattern)) - ); - - if (buildReportJob) reportJobIds.build = buildReportJob.job_id; - if (jobReportJob) reportJobIds.job = jobReportJob.job_id; - if (testReportJob) reportJobIds.test = testReportJob.job_id; - - const hasBuildReport = buildReportJob !== undefined; - const hasJobReport = jobReportJob !== undefined; - const hasTestReport = testReportJob !== undefined; - - if (hasBuildReport && !$("#build-report-tab").hasClass("active")) { - $("#build-report-tab").removeClass("disabled"); - $("#build-report-tab").attr( - "title", - "View application build results including binary sizes and downloads" - ); - $("#build-report-tab").css("opacity", "1"); - $("#build-report-tab").css("cursor", "pointer"); - } - - if (hasJobReport && !$("#job-report-tab").hasClass("active")) { - $("#job-report-tab").removeClass("disabled"); - $("#job-report-tab").attr( - "title", - "View CI job results and failed jobs" - ); - $("#job-report-tab").css("opacity", "1"); - $("#job-report-tab").css("cursor", "pointer"); - } - - if (hasTestReport && !$("#test-report-tab").hasClass("active")) { - $("#test-report-tab").removeClass("disabled"); - $("#test-report-tab").attr( - "title", - "View test results including success, failure, and skipped tests" - ); - $("#test-report-tab").css("opacity", "1"); - $("#test-report-tab").css("cursor", "pointer"); - } -} - -$(document).ready(function () { - const currentPath = window.location.pathname; - const currentFile = currentPath.substring(currentPath.lastIndexOf("/") + 1); - - $(".report-nav-tabs .nav-tab").removeClass("active"); - - if (currentFile.includes("built_apps")) { - $("#build-report-tab").addClass("active"); - } else if (currentFile.includes("job_report")) { - $("#job-report-tab").addClass("active"); - } else if (currentFile.includes("target_test_report")) { - $("#test-report-tab").addClass("active"); - } - - wrapTablesInCollapsibleSections(); - makeTablesCollapsible(); - setupScrollProgressBar(); - setupFloatingActions(); - enhanceTableStatusDisplay(); - markTestCaseColumns(); - setupResponsiveText(); - setupTextToggles(); - setupEventHandlers(); - scrollToHashLocation(); - initBootstrapTable(); - fixStickyHeaderAlignment(); - forceEnableStickyHeaders(); - fixTableHeaderText(); - lazyLoadVisibleImages(); - setupPagination(); - optimizeScrollPerformance(); -}); - -function fixStickyHeaderAlignment() { - if (window.isScrolling) return; - - $(".sticky-header-container").each(function () { - const $container = $(this); - const $table = $container - .closest(".bootstrap-table") - .find(".fixed-table-body table"); - - let tableWidth = "100%"; - if ($table.length) { - const actualWidth = $table.width(); - if (actualWidth > 0) { - tableWidth = actualWidth + "px"; - } - } - - $container.css({ - left: "0", - "margin-left": "0", - "padding-left": "0", - width: tableWidth, - "z-index": "100", - transform: "translateZ(0)", - }); - - this.style.setProperty("left", "0", "important"); - this.style.setProperty("margin-left", "0", "important"); - this.style.setProperty("padding-left", "0", "important"); - this.style.setProperty("width", tableWidth, "important"); - this.style.setProperty("z-index", "100", "important"); - this.style.setProperty("transform", "translateZ(0)", "important"); - - const $headerTable = $container.find("table"); - if ($headerTable.length) { - $headerTable.css("width", tableWidth); - $headerTable[0].style.setProperty("width", tableWidth, "important"); - } - - if (!window.isScrolling) { - if ($headerTable.length && $table.length) { - const $headerCols = $headerTable.find("th"); - const $bodyCols = $table.find("tr:first-child td"); - - if ($headerCols.length === $bodyCols.length) { - $headerCols.each(function (i) { - if (i < $bodyCols.length) { - const bodyColWidth = $($bodyCols[i]).outerWidth(); - if (bodyColWidth > 0) { - $(this).css("min-width", bodyColWidth + "px"); - $(this).css("width", bodyColWidth + "px"); - - const $thInner = $(this).find(".th-inner"); - if ($thInner.length) { - $thInner.css({ - "white-space": "normal", - overflow: "visible", - "text-overflow": "clip", - height: "auto", - display: "block", - }); - } - } - } - }); - } - } - } - }); - - $(".fixed-table-container").each(function () { - if ($(this).css("left") === "20px") { - $(this).css("left", "0"); - this.style.setProperty("left", "0", "important"); - } - }); - - $(".bootstrap-table .table thead th").css({ - position: "sticky", - top: "0", - "z-index": "100", - "background-color": "var(--esp-light)", - transform: "translateZ(0)", - "white-space": "normal", - overflow: "visible", - "text-overflow": "clip", - height: "auto", - }); - - $( - ".bootstrap-table .table thead th .th-inner, .sticky-header-container th .th-inner" - ).css({ - "white-space": "normal", - overflow: "visible", - "text-overflow": "clip", - height: "auto", - "min-height": "20px", - display: "block", - "line-height": "1.4", - }); -} - -function setupScrollProgressBar() { - const progressBar = $("#nav-progress-bar"); - let lastScrollPosition = 0; - let ticking = false; - - $(window).on("scroll", function () { - if (ticking) return; - - const scrollPosition = window.scrollY; - - if (Math.abs(scrollPosition - lastScrollPosition) < 5) { - return; - } - - lastScrollPosition = scrollPosition; - - window.requestAnimationFrame(function () { - const windowHeight = $(document).height() - $(window).height(); - const scrollPercentage = (scrollPosition / windowHeight) * 100; - progressBar.css("width", scrollPercentage + "%"); - ticking = false; - }); - ticking = true; - }); -} - -function setupFloatingActions() { - $("#back-to-top").on("click", function () { - $("html, body").animate({ scrollTop: 0 }, 300); - }); -} - -function enhanceTableStatusDisplay() { - $("table.table th").each(function (index) { - const $header = $(this); - const headerText = $header.text().trim().toLowerCase(); - - if ( - headerText.includes("status") || - headerText.includes("result") || - headerText.includes("state") - ) { - const colIndex = index + 1; - - $(`table.table td:nth-child(${colIndex})`).each(function () { - const $cell = $(this); - const cellText = $cell.text().trim().toLowerCase(); - - if (cellText.includes("pass") || cellText.includes("success")) { - $cell.html( - `
${$cell.text()}
` - ); - } else if ( - cellText.includes("fail") || - cellText.includes("error") - ) { - $cell.html( - `
${$cell.text()}
` - ); - } else if ( - cellText.includes("warn") || - cellText.includes("skip") - ) { - $cell.html( - `
${$cell.text()}
` - ); - } - }); - } - }); -} - -function setupTextToggles() { - setupResponsiveText(); - - $(".toggle-link").off("click").on("click", toggleText); -} - -function setupResponsiveText() { - $("table.table td").each(function () { - const $cell = $(this); - const text = $cell.text(); - - if ( - text.length > 100 && - !$cell.hasClass("test-case-name") && - !$cell.find(".text-toggle").length - ) { - const displayText = text.substring(0, 100) + "..."; - $cell.html( - `${displayText}` + - `` + - ` Show More` - ); - } - }); -} - -function makeTablesCollapsible() { - $("table.table").each(function (index) { - const table = $(this); - const tableId = `table-${index}`; - table.attr("id", tableId); - - const bootstrapTableWrapper = table.closest(".bootstrap-table"); - - let container = bootstrapTableWrapper.parent(".table-container"); - if (!container.length) { - container = $('
'); - } - - let tableControls = bootstrapTableWrapper.prev(".table-controls"); - - if (!tableControls.length) { - const initialButtonHtml = - 'Collapse Table'; - - const toggleButton = $(` - - `); - - tableControls = $('
').append( - toggleButton - ); - - bootstrapTableWrapper.before(tableControls); - } else { - const existingButton = tableControls.find(".table-collapse-btn"); - if (existingButton.length) { - existingButton.detach(); - tableControls.append(existingButton); - } - } - - if (!bootstrapTableWrapper.parent().hasClass("table-container")) { - bootstrapTableWrapper.wrap(container); - } - - bootstrapTableWrapper.show(); - bootstrapTableWrapper.addClass("expanded"); - - const toggleButton = tableControls.find(".table-collapse-btn"); - toggleButton.off("click").on("click", function (e) { - e.preventDefault(); - e.stopPropagation(); - - const tableId = $(this).data("table"); - - const targetTable = $(`#${tableId}`); - if (!targetTable.length) { - console.error("Target table not found:", tableId); - return; - } - - const bootstrapWrapper = targetTable.closest(".bootstrap-table"); - if (!bootstrapWrapper.length) { - console.error( - "Bootstrap wrapper not found for table:", - tableId - ); - return; - } - - const isVisible = bootstrapWrapper.is(":visible"); - - if (isVisible) { - bootstrapWrapper.slideUp(300); - bootstrapWrapper.removeClass("expanded"); - $(this).html( - 'Expand Table' - ); - } else { - bootstrapWrapper.slideDown(300); - bootstrapWrapper.addClass("expanded"); - $(this).html( - 'Collapse Table' - ); - } - }); - }); - - setTimeout(function () { - $(".bootstrap-table[data-auto-collapse='true']").each(function () { - const bootstrapWrapper = $(this); - const tableId = bootstrapWrapper.find("table").attr("id"); - const toggleButton = $(`button[data-table="${tableId}"]`); - - bootstrapWrapper.slideUp(300); - bootstrapWrapper.removeClass("expanded"); - - if (toggleButton.length) { - toggleButton.html( - 'Expand Table' - ); - } - }); - }, 500); -} - -function expandAllTables() { - $(".bootstrap-table").slideDown(300); - $(".bootstrap-table").addClass("expanded"); - - $(".table-collapse-btn").each(function () { - $(this).html( - 'Collapse Table' - ); - }); - - $(".section-header").attr("aria-expanded", "true"); - $(".section-body").slideDown(300); -} - -function collapseAllTables() { - $(".bootstrap-table").slideUp(300); - $(".bootstrap-table").removeClass("expanded"); - - $(".table-collapse-btn").each(function () { - $(this).html('Expand Table'); - }); -} - -function initBootstrapTable() { - $("table.table").each(function () { - const $table = $(this); - - $table.bootstrapTable({ - classes: "table table-bordered table-striped", - height: 800, - pagination: false, - showColumns: true, - showColumnsToggleAll: true, - showToggle: false, - clickToSelect: false, - minimumCountColumns: 2, - stickyHeader: true, - stickyHeaderOffsetY: 0, - theadClasses: "thead-light", - toolbar: "#toolbar", - resizable: true, - checkOnInit: false, - fixedColumns: true, - fixedNumber: 0, - sortStable: true, - undefinedText: "-", - showFullscreen: true, - widthUnit: "%", - headerStyle: function () { - return { - css: { - "white-space": "normal", - overflow: "visible", - "text-overflow": "clip", - height: "auto", - "min-height": "50px", - }, - }; - }, - }); - - $table.closest(".bootstrap-table").css({ - width: "100%", - "max-width": "100%", - }); - - $table.closest(".fixed-table-body").css({ - "overflow-x": "auto", - }); - - $table.css({ - width: "100%", - "min-width": "100%", - }); - - $table.find("thead th").each(function () { - const $th = $(this); - const $thInner = $th.find(".th-inner"); - - if ($thInner.length) { - $thInner.css({ - "white-space": "normal", - overflow: "visible", - "text-overflow": "clip", - height: "auto", - "min-height": "20px", - display: "block", - }); - } - }); - }); - - $(window).on( - "resize", - debounce(function () { - $("table.table").bootstrapTable("resetView"); - }, 100) - ); -} - -function wrapTablesInCollapsibleSections() { - $("h2").each(function (index) { - if ($(this).closest(".section-header").length > 0) { - return; - } - - const header = $(this); - const headerText = header.text(); - const headerId = header.attr("id") || `section-header-${index}`; - - const nextTable = header.nextAll("table.table:first"); - - if (nextTable.length) { - if (nextTable.closest(".section-body").length > 0) { - return; - } - - const wrapper = $('
'); - - const sectionHeader = $(` -
-

- ${headerText} - -

- -
- `); - - const sectionBody = $( - `
` - ); - - const bootstrapTable = nextTable.closest(".bootstrap-table"); - const tableContainer = bootstrapTable.closest(".table-container"); - - if (tableContainer.length) { - tableContainer.detach().appendTo(sectionBody); - } else if (bootstrapTable.length) { - bootstrapTable.detach().appendTo(sectionBody); - } else { - nextTable.detach().appendTo(sectionBody); - } - - wrapper.append(sectionHeader).append(sectionBody); - - header.replaceWith(wrapper); - } - }); - - $(".section-header") - .off("click") - .on("click", function (e) { - if ( - $(e.target).hasClass("copy-link-icon") || - $(e.target).closest(".copy-link-icon").length || - $(e.target).hasClass("table-collapse-btn") || - $(e.target).closest(".table-collapse-btn").length - ) { - return; - } - - const header = $(this); - const targetId = header.data("target"); - const sectionBody = $(targetId); - const isExpanded = header.attr("aria-expanded") === "true"; - - if (isExpanded) { - header.attr("aria-expanded", "false"); - sectionBody.slideUp(300); - } else { - header.attr("aria-expanded", "true"); - sectionBody.slideDown(300); - } - }); -} - -function markTestCaseColumns() { - $("table.table th").each(function (index) { - const headerText = $(this).text().trim().toLowerCase(); - if ( - headerText.includes("test") && - (headerText.includes("case") || headerText.includes("name")) - ) { - const colIndex = index + 1; - $(`table.table td:nth-child(${colIndex})`).addClass( - "test-case-name" - ); - } - }); -} - -function setupEventHandlers() { - $(window).on("load", scrollToHashLocation); - $("body").on("click", ".toggle-link", toggleText); - - $("#expand-all-tables") - .off("click") - .on("click", function (e) { - e.preventDefault(); - expandAllTables(); - }); - - $("#collapse-all-tables") - .off("click") - .on("click", function (e) { - e.preventDefault(); - collapseAllTables(); - }); - - $("#pagination-size").on("change", function () { - const pageSize = parseInt($(this).val(), 10); - applyPaginationToTables(pageSize); - }); - - $("#show-all-columns").on("click", function (e) { - e.preventDefault(); - $("table.table").bootstrapTable("showAllColumns"); - }); - - $("#show-failed-only").on("click", function (e) { - e.preventDefault(); - filterByStatus("fail"); - }); - - $("#show-passed-only").on("click", function (e) { - e.preventDefault(); - filterByStatus("pass"); - }); - - $("#export-csv").on("click", function (e) { - e.preventDefault(); - $("table.table:visible").bootstrapTable("exportTable", { - type: "csv", - fileName: "report_export_" + new Date().toISOString().slice(0, 10), - }); - }); - - $("#clear-all-filters").on("click", function () { - clearAllFilters(); - }); -} - -function filterByStatus(status) { - $("#active-filters").show(); - $("#filter-badges").html( - `Status: ${status}` - ); - - $("table.table").each(function () { - const $table = $(this); - - let statusColIndex = -1; - $table.find("th").each(function (index) { - const headerText = $(this).text().trim().toLowerCase(); - if ( - headerText.includes("status") || - headerText.includes("result") || - headerText.includes("state") - ) { - statusColIndex = index; - return false; - } - }); - - if (statusColIndex >= 0) { - $table.bootstrapTable( - "filterBy", - { - [statusColIndex]: status, - }, - { - filterAlgorithm: function (row, filters) { - const cellText = $(row[statusColIndex]) - .text() - .trim() - .toLowerCase(); - return cellText.includes(status); - }, - } - ); - } - }); -} - -function clearAllFilters() { - $("table.table").each(function () { - const $table = $(this); - $table.bootstrapTable("clearFilterControl"); - $table.bootstrapTable("resetSearch"); - }); - - $("#active-filters").hide(); - $("#filter-badges").empty(); -} - -function scrollToHashLocation() { - const hash = window.location.hash; - if (hash) { - setTimeout(() => { - const target = $(hash); - if (target.length) { - const sectionBody = target.closest(".section-body"); - if (sectionBody.length) { - const sectionHeader = sectionBody.prev(".section-header"); - if (sectionHeader.attr("aria-expanded") !== "true") { - sectionHeader.attr("aria-expanded", "true"); - sectionBody.slideDown(0); - } - - const containingTable = target.closest("table.table"); - if (containingTable.length) { - const bootstrapWrapper = - containingTable.closest(".bootstrap-table"); - if (!bootstrapWrapper.is(":visible")) { - bootstrapWrapper.show(); - const tableBtn = bootstrapWrapper - .parent() - .find(".table-controls") - .find(".table-collapse-btn"); - tableBtn.html( - 'Collapse Table' - ); - } - } - } - - $("html, body").animate( - { scrollTop: target.offset().top - 20 }, - 100 - ); - } - }, 300); - } -} - -function copyPermalink(anchorId) { - const fullUrl = `${window.location.origin}${window.location.pathname}${anchorId}`; - history.pushState(null, null, anchorId); - navigator.clipboard.writeText(fullUrl); - - const tooltip = $(` -
- Link copied to clipboard! -
- `); - - const icon = $(event.target).closest(".copy-link-icon"); - if (icon.length) { - const originalClass = icon.attr("class"); - - icon.removeClass() - .addClass("fas fa-check copy-link-icon") - .css("color", "var(--esp-success)"); - - setTimeout(() => { - icon.attr("class", originalClass).css("color", ""); - }, 1500); - } - - $("body").append(tooltip); - setTimeout(() => tooltip.remove(), 2000); - scrollToHashLocation(); -} - -function toggleText(e) { - e.preventDefault(); - e.stopPropagation(); - - const link = $(this); - const textSpan = link.siblings(".full-text"); - const toggleSpan = link.siblings(".text-toggle"); - - const visible = textSpan.is(":visible"); - - if (visible) { - link.html(' Show More'); - textSpan.hide(); - toggleSpan.show(); - } else { - link.html(' Show Less'); - textSpan.show(); - toggleSpan.hide(); - } -} - -function setupPagination() { - applyPaginationToTables(25); -} - -function applyPaginationToTables(pageSize) { - localStorage.setItem("paginationSize", pageSize); - - $("table.table").each(function () { - const $table = $(this); - - if (pageSize > 0) { - $table.bootstrapTable("refreshOptions", { - pagination: true, - pageSize: pageSize, - pageList: [10, 25, 50, "All"], - }); - - $table - .closest(".bootstrap-table") - .find(".fixed-table-pagination") - .css({ - display: "block", - visibility: "visible", - "background-color": "var(--esp-light)", - padding: "10px", - "border-top": "1px solid rgba(0, 0, 0, 0.05)", - }); - } else { - $table.bootstrapTable("refreshOptions", { - pagination: false, - }); - - $table - .closest(".bootstrap-table") - .find(".fixed-table-pagination") - .css({ - display: "none", - visibility: "hidden", - }); - } - }); -} - -function optimizeScrollPerformance() { - window.isScrolling = false; - let scrollTimer = null; - let lastScrollTop = 0; - - cacheAndFixDomElements(); - - const progressBar = document.getElementById("nav-progress-bar"); - const backToTop = document.getElementById("back-to-top"); - - window.addEventListener( - "scroll", - function () { - window.isScrolling = true; - - const scrollTop = window.scrollY; - - if (Math.abs(scrollTop - lastScrollTop) > 5) { - lastScrollTop = scrollTop; - - handleEssentialScrollUpdates(scrollTop, progressBar, backToTop); - } - - clearTimeout(scrollTimer); - scrollTimer = setTimeout(function () { - window.isScrolling = false; - requestAnimationFrame(function () { - fixStickyHeaderAlignment(); - fixTableHeaderText(); - }); - }, 150); - }, - { passive: true } - ); - - lazyLoadVisibleImages(); -} - -function handleEssentialScrollUpdates(scrollTop, progressBar, backToTop) { - requestAnimationFrame(function () { - const docHeight = - Math.max( - document.body.scrollHeight, - document.body.offsetHeight, - document.documentElement.clientHeight, - document.documentElement.scrollHeight, - document.documentElement.offsetHeight - ) - window.innerHeight; - const scrollPercentage = (scrollTop / docHeight) * 100; - progressBar.style.width = scrollPercentage + "%"; - - if (scrollTop > 200) { - backToTop.classList.add("visible"); - } else { - backToTop.classList.remove("visible"); - } - }); -} - -function cacheAndFixDomElements() { - const stickyElements = document.querySelectorAll( - ".sticky-header-container, .section-header, .bootstrap-table thead, .fixed-table-header" - ); - - for (let i = 0; i < stickyElements.length; i++) { - const elem = stickyElements[i]; - elem.style.transform = "translateZ(0)"; - elem.style.willChange = "transform"; - elem.style.backfaceVisibility = "hidden"; - } - - document.body.style.willChange = "scroll-position"; - document.body.style.backfaceVisibility = "hidden"; - - const headers = document.querySelectorAll( - ".bootstrap-table .table thead th, .sticky-header-container th" - ); - for (let i = 0; i < headers.length; i++) { - const header = headers[i]; - header.style.position = "sticky"; - header.style.top = "0"; - header.style.zIndex = "100"; - header.style.transform = "translateZ(0)"; - - const thInners = header.querySelectorAll(".th-inner"); - for (let j = 0; j < thInners.length; j++) { - const inner = thInners[j]; - inner.style.whiteSpace = "normal"; - inner.style.overflow = "visible"; - inner.style.textOverflow = "clip"; - inner.style.height = "auto"; - inner.style.minHeight = "20px"; - inner.style.display = "block"; - } - } -} - -function lazyLoadVisibleImages() { - const lazyImages = document.querySelectorAll("img[data-src]"); - if (lazyImages.length === 0) return; - - const loadImage = function (img) { - if (img.dataset.src) { - img.src = img.dataset.src; - img.removeAttribute("data-src"); - } - }; - - lazyImages.forEach((img) => { - if (isElementInViewport(img)) { - loadImage(img); - } - }); - - document.addEventListener( - "scroll", - debounce(function () { - lazyImages.forEach((img) => { - if (img.dataset.src && isElementInViewport(img)) { - loadImage(img); - } - }); - }, 200), - { passive: true } - ); -} - -function isElementInViewport(el) { - const rect = el.getBoundingClientRect(); - return ( - rect.top >= 0 && - rect.left >= 0 && - rect.bottom <= - (window.innerHeight || document.documentElement.clientHeight) && - rect.right <= - (window.innerWidth || document.documentElement.clientWidth) - ); -} - -function fixTableHeaderText() { - if (window.isScrolling) return; - - $(".bootstrap-table .table thead th, .sticky-header-container th").each( - function () { - const $th = $(this); - - $th.css({ - "white-space": "normal", - overflow: "visible", - "text-overflow": "clip", - height: "auto", - "min-height": "50px", - }); - - const $thInner = $th.find(".th-inner"); - if ($thInner.length) { - $thInner.css({ - "white-space": "normal", - overflow: "visible", - "text-overflow": "clip", - height: "auto", - "min-height": "20px", - display: "block", - "line-height": "1.4", - }); - - $thInner.find("span, div").css({ - "white-space": "normal", - overflow: "visible", - }); - } - } - ); -} - -function forceEnableStickyHeaders() { - if (!window.isScrolling) { - $("table.table thead th").css({ - position: "sticky", - top: "0", - "z-index": "100", - "background-color": "var(--esp-light)", - transform: "translateZ(0)", - "will-change": "transform", - "backface-visibility": "hidden", - "white-space": "normal", - overflow: "visible", - "text-overflow": "clip", - height: "auto", - "min-height": "50px", - }); - - $("table.table").each(function () { - const $table = $(this); - const bootstrapTable = $table.closest(".bootstrap-table"); - - if ( - bootstrapTable.length && - bootstrapTable.find(".sticky-header-container").length === 0 - ) { - const $thead = $table.find("thead").clone(); - const $stickyContainer = $( - '
' - ); - const $stickyTable = $('
').append( - $thead - ); - - $stickyContainer.append($stickyTable); - bootstrapTable.prepend($stickyContainer); - - const tableWidth = $table.width(); - $stickyContainer.css({ - position: "sticky", - top: "0", - "z-index": "100", - width: tableWidth + "px", - overflow: "hidden", - "background-color": "var(--esp-light)", - "will-change": "transform", - "backface-visibility": "hidden", - transform: "translateZ(0)", - }); - - const $originalThs = $table.find("thead th"); - const $stickyThs = $stickyTable.find("th"); - - $originalThs.each(function (i) { - if (i < $stickyThs.length) { - const width = $(this).outerWidth(); - $($stickyThs[i]).css({ - width: width + "px", - "white-space": "normal", - overflow: "visible", - "text-overflow": "clip", - height: "auto", - "min-height": "50px", - }); - - const $inner = $($stickyThs[i]).find(".th-inner"); - if ($inner.length) { - $inner.css({ - "white-space": "normal", - overflow: "visible", - "text-overflow": "clip", - height: "auto", - "min-height": "20px", - display: "block", - "line-height": "1.4", - }); - } - } - }); - } - }); - } -} diff --git a/tools/ci/dynamic_pipelines/templates/styles.css b/tools/ci/dynamic_pipelines/templates/styles.css deleted file mode 100644 index cc3c43b9b43..00000000000 --- a/tools/ci/dynamic_pipelines/templates/styles.css +++ /dev/null @@ -1,1095 +0,0 @@ -/** - * Espressif Report Template Styles - * Optimized for desktop-only application - */ - -/* Variables and base settings */ -:root { - /* Color palette */ - --esp-primary: #e83711; - --esp-secondary: #f3c300; - --esp-dark: #282430; - --esp-light: #f8f9fa; - --esp-blue: #3a86ff; - --esp-gray: #707070; - --esp-success: #25be7b; - --esp-warning: #f3c300; - --esp-danger: #e83711; - - /* UI elements */ - --body-bg: #f5f7fa; - --card-bg: #ffffff; - --text-color: #282430; - --border-radius: 8px; - --box-shadow: 0 4px 10px rgba(0, 0, 0, 0.05); - --transition-speed: 0.3s; - - /* Fixed desktop layout variables */ - --container-width: 1600px; - --sidebar-width: 280px; - --header-height: 60px; -} - -/* ---------------------------------- */ -/* Base styles */ -/* ---------------------------------- */ -html { - scroll-behavior: smooth; -} - -body { - font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, - "Helvetica Neue", Arial, sans-serif; - line-height: 1.6; - color: var(--text-color); - background-color: var(--body-bg); - transition: background-color var(--transition-speed); - min-width: 1024px; - overflow-y: scroll; -} - -.container-fluid { - max-width: 100% !important; - padding: 20px !important; - margin: 0 auto; -} - -/* Typography */ -h2 { - padding-top: 3rem; - margin-bottom: 0px; - color: var(--esp-dark); - font-weight: 600; - background-color: #f5f7fa; -} - -/* ---------------------------------- */ -/* Performance Optimizations */ -/* ---------------------------------- */ -/* Hardware acceleration utility class */ -.hw-accelerated { - transform: translateZ(0); - will-change: transform; - backface-visibility: hidden; -} - -/* Scrollbar styling */ -::-webkit-scrollbar { - width: 8px; - height: 8px; -} - -::-webkit-scrollbar-track { - background: rgba(0, 0, 0, 0.05); - border-radius: 4px; -} - -::-webkit-scrollbar-thumb { - background: rgba(0, 0, 0, 0.2); - border-radius: 4px; -} - -::-webkit-scrollbar-thumb:hover { - background: rgba(0, 0, 0, 0.3); -} - -/* Reduced motion for accessibility */ -@media (prefers-reduced-motion: reduce) { - html { - scroll-behavior: auto; - } - - .bootstrap-table, - .section-body, - .sticky-header-container, - .section-header { - transition: none !important; - } -} - -/* ---------------------------------- */ -/* Navigation and Header */ -/* ---------------------------------- */ - -/* Header styling */ -.report-header { - display: flex; - align-items: center; - padding: 1.2rem 0; - margin-bottom: 2rem; - border-bottom: 1px solid rgba(0, 0, 0, 0.1); - position: relative; - justify-content: space-between; - background-color: #f8f9fa; -} - -.report-header .logo { - height: 30px; - margin-bottom: 0; - padding-left: 15px; -} - -.report-header .title-container { - display: flex; - flex-direction: column; - justify-content: center; - text-align: center; - margin: 0 auto; -} - -.report-header h1 { - margin: 0; - font-weight: 700; - font-size: 1.8rem; - line-height: 1.2; - display: inline-block; - text-align: center; -} - -.report-header p { - margin-bottom: 0; - font-size: 0.85rem; - text-align: center; - color: #707070; -} - -.report-header::after { - content: ""; - position: absolute; - bottom: -3px; - left: 0; - right: 0; - height: 3px; - background-color: var(--esp-primary); -} - -.logo-container { - width: 180px; - display: flex; - align-items: center; - justify-content: flex-start; -} - -.spacer { - width: 180px; /* Same as logo-container for balance */ -} - -/* Navigation progress bar */ -.nav-progress-container { - position: fixed; - top: 0; - left: 0; - width: 100%; - height: 3px; - background-color: transparent; - z-index: 1001; -} - -.nav-progress-bar { - height: 100%; - width: 0; - background-color: var(--esp-primary); - transition: width 0.2s ease-out; -} - -/* ---------------------------------- */ -/* Tables - Complete Reset & Rebuild */ -/* ---------------------------------- */ - -/* Base table appearance */ -.table { - width: 100%; - margin-bottom: 0; - background-color: var(--card-bg); - border-collapse: separate; - border-spacing: 0; -} - -/* Table header cells */ -.table > thead > tr > th { - padding: 12px 15px; - font-weight: 600; - color: var(--esp-dark); - background-color: var(--esp-light); - border-bottom: 2px solid rgba(0, 0, 0, 0.1); - text-align: left; - position: sticky; - top: 0; - z-index: 10; -} - -/* Table data cells */ -.table > tbody > tr > td { - padding: 12px 15px; - border-top: 1px solid rgba(0, 0, 0, 0.05); -} - -/* Striped rows */ -.table-striped > tbody > tr:nth-of-type(odd) { - background-color: rgba(0, 0, 0, 0.02); -} - -.table-striped > tbody > tr:hover { - background-color: rgba(0, 0, 0, 0.05); -} - -/* Bootstrap Table container */ -.bootstrap-table { - border-radius: var(--border-radius); - overflow: hidden; - box-shadow: var(--box-shadow); - margin-bottom: 1rem; -} - -/* Bootstrap Table scrollable body */ -.bootstrap-table .fixed-table-body { - max-height: 800px; - overflow-y: auto; - border: none !important; -} - -/* Remove any borders from the fixed-table-container */ -.bootstrap-table .fixed-table-container { - border: none !important; -} - -/* Override Bootstrap's fixed header styles to avoid conflicts */ -.bootstrap-table .fixed-table-header table { - background-color: var(--esp-light); -} - -.bootstrap-table .fixed-table-header { - background-color: var(--esp-light); -} - -/* Column width settings for better layout */ -.table > thead > tr > th:nth-child(1), -.table > tbody > tr > td:nth-child(1) { - width: 20%; - min-width: 150px; -} - -.table > thead > tr > th:nth-child(2), -.table > tbody > tr > td:nth-child(2) { - width: 20%; - min-width: 150px; -} - -.table > thead > tr > th:nth-child(3), -.table > tbody > tr > td:nth-child(3) { - width: 20%; - min-width: 150px; -} - -.table > thead > tr > th:nth-child(n + 4), -.table > tbody > tr > td:nth-child(n + 4) { - width: auto; - min-width: 100px; -} - -/* Ensure sticky section headers are above table headers */ -.sticky-section-header { - position: sticky; - top: 0; - z-index: 100; - background-color: #f5f7fa; -} - -/* Test case names */ -td.test-case-name { - white-space: normal; - overflow: visible; - word-break: break-word; -} - -/* Bootstrap table customizations */ -.bootstrap-table { - margin-bottom: 0; - border-radius: var(--border-radius); - box-shadow: var(--box-shadow); - background-color: var(--card-bg); - overflow: hidden; - max-height: 2000px; - transition: max-height 0.3s ease-in-out, opacity 0.3s ease-in-out; - opacity: 1; - width: 100% !important; - position: relative; -} - -.bootstrap-table.expanded { - max-height: 2000px; - opacity: 1; -} - -/* Main container setup */ -.bootstrap-table .fixed-table-container { - position: relative !important; - overflow: visible !important; -} - -/* Table headers - sticky relative to .fixed-table-body */ -.bootstrap-table .table thead th { - position: sticky !important; - top: 0 !important; - z-index: 10; - background-color: var(--esp-light); - border-bottom: 2px solid rgba(0, 0, 0, 0.1); - padding: 12px 15px !important; -} - -/* Fixed-table-header should be hidden when scrolling */ -.bootstrap-table .fixed-table-header { - display: none !important; -} - -/* Handle Bootstrap Table sticky header container */ -.bootstrap-table .sticky-header-container { - display: none !important; /* Use our own sticky headers instead */ -} - -/* Ensure th-inner elements are visible */ -.bootstrap-table .fixed-table-container .table thead th .th-inner { - white-space: normal !important; - overflow: visible !important; - text-overflow: clip !important; - height: auto !important; - min-height: 20px; - line-height: 1.4; - padding: 8px; - background-color: var(--esp-light); - display: block !important; -} - -/* Table container shouldn't have overflow-x at the top level */ -.table-container { - width: 100%; - max-width: 100%; - margin: 0 auto; - background-color: var(--card-bg); - border-radius: var(--border-radius); - overflow: hidden; - box-shadow: var(--box-shadow); - position: relative; -} - -/* Fix for table header rendering in the sticky context */ -.bootstrap-table thead { - position: sticky; - top: 0; - z-index: 10; -} - -/* Table controls */ -.table-controls { - background-color: var(--esp-light); - border-top-left-radius: var(--border-radius); - border-top-right-radius: var(--border-radius); - padding: 12px 15px; - border-bottom: 1px solid rgba(0, 0, 0, 0.05); - display: flex; - justify-content: flex-end; - align-items: center; - flex-wrap: wrap; - gap: 10px; -} - -.table-controls .row-count { - margin-bottom: 0; - white-space: nowrap; -} - -.table-controls .btn { - display: flex; - align-items: center; - justify-content: center; - gap: 5px; - transition: all 0.2s ease; -} - -/* Table toolbar */ -.fixed-table-toolbar { - padding: 10px; - background-color: var(--esp-light); - border-bottom: 1px solid rgba(0, 0, 0, 0.05); -} - -.fixed-table-toolbar .search { - display: flex; - align-items: center; -} - -.fixed-table-toolbar .search input { - border-radius: var(--border-radius); - border: 1px solid rgba(0, 0, 0, 0.1); - padding: 5px 10px; - margin-right: 10px; -} - -.fixed-table-toolbar .columns { - margin-left: auto; -} - -.fixed-table-toolbar .btn { - border-radius: var(--border-radius); - padding: 5px 10px; - margin-left: 5px; -} - -.fixed-table-toolbar .dropdown-menu { - border-radius: var(--border-radius); - box-shadow: var(--box-shadow); - padding: 8px; - border: none; -} - -/* Table pagination */ -.fixed-table-pagination { - padding: 10px; - background-color: var(--esp-light); - border-bottom-left-radius: var(--border-radius); - border-bottom-right-radius: var(--border-radius); - border-top: 1px solid rgba(0, 0, 0, 0.05); - display: flex; - justify-content: center; - align-items: center; -} - -.fixed-table-pagination .pagination { - float: none !important; - order: 1; - flex: 1 1 auto; - display: flex; - justify-content: center; - margin: 0 auto; - position: absolute; - left: 50%; - transform: translateX(-50%); -} - -.fixed-table-pagination .pagination-detail .pagination-info { - display: none; - float: none !important; - order: 2; - flex: 0 0 auto; -} - -.fixed-table-pagination .pagination li a { - border-radius: 4px; - margin: 0 2px; - border: 1px solid #dee2e6; - min-width: 34px; - height: 34px; - display: flex; - align-items: center; - justify-content: center; - color: var(--esp-dark); - transition: all 0.2s ease; -} - -.fixed-table-pagination .pagination li.active a { - background-color: var(--esp-primary); - border-color: var(--esp-primary); - color: white; -} - -.fixed-table-pagination .pagination li a:hover:not(.active) { - background-color: #e9ecef; - color: var(--esp-primary); -} - -/* Pagination options */ -.pagination-options .form-select-sm { - border-radius: var(--border-radius); - border: 1px solid rgba(0, 0, 0, 0.1); - height: 31px; - padding: 0.25rem 2rem 0.25rem 0.5rem; - font-size: 0.875rem; - background-color: var(--esp-light); - color: var(--esp-dark); - cursor: pointer; - transition: all var(--transition-speed); - box-shadow: 0 1px 3px rgba(0, 0, 0, 0.05); -} - -.pagination-options .form-select-sm:hover { - border-color: var(--esp-primary); -} - -.pagination-options .form-select-sm:focus { - border-color: var(--esp-primary); - box-shadow: 0 0 0 0.15rem rgba(232, 55, 17, 0.25); - outline: none; -} - -/* ---------------------------------- */ -/* Collapsible Sections */ -/* ---------------------------------- */ -.section-collapsible { - margin-bottom: 2rem; - border-radius: var(--border-radius); - overflow: hidden; - box-shadow: var(--box-shadow); - background-color: var(--card-bg); - transition: transform var(--transition-speed); -} - -.section-collapsible:hover { - transform: translateY(-2px); -} - -.section-header { - display: flex; - align-items: center; - padding: 1rem 1.5rem; - background-color: #f5f7fa; - border-bottom: 1px solid rgba(0, 0, 0, 0.1); - cursor: pointer; - transition: background-color var(--transition-speed), - box-shadow var(--transition-speed); -} - -.section-header:hover { - background-color: #e9ecef; - box-shadow: 0 3px 8px rgba(0, 0, 0, 0.1); -} - -.section-header h2 { - margin: 0; - font-size: 1.5rem; - flex-grow: 1; - background-color: transparent; - position: relative; - z-index: 2; - font-weight: 500; - display: flex; - align-items: center; -} - -.section-header h2 .heading-text { - margin-right: 6px; -} - -.section-header .collapse-indicator { - margin-left: 10px; - font-size: 1.5rem; - transition: transform var(--transition-speed); - color: var(--esp-primary); - position: relative; - z-index: 2; -} - -.section-header[aria-expanded="false"] .collapse-indicator { - transform: rotate(-90deg); -} - -.copy-link-icon { - margin-left: 8px; - font-size: 1.5rem; - color: var(--esp-gray); - opacity: 0.7; - cursor: pointer; - transition: opacity 0.2s ease, color 0.2s ease; - position: relative; - z-index: 2; -} - -.copy-link-icon:hover { - opacity: 1; - color: var(--esp-primary); -} - -.sticky-section-header { - position: sticky; - top: 0; - z-index: 99; - background-color: #f5f7fa; - box-shadow: 0 2px 5px rgba(0, 0, 0, 0.05); -} - -.section-body { - padding: 1.5rem; - max-height: 0; - overflow: hidden; - transition: max-height 0.3s ease-in-out, padding 0.3s ease-in-out, - opacity 0.3s ease-in-out; - padding-top: 0; - padding-bottom: 0; - opacity: 0; -} - -.section-body.expanded { - max-height: 5000px; - padding: 1.5rem; - opacity: 1; -} - -.section-body.collapsed { - display: none; -} - -/* ---------------------------------- */ -/* Status Badges */ -/* ---------------------------------- */ -.status-badge { - display: inline-block; - padding: 4px 8px; - border-radius: 4px; - font-size: 0.8rem; - font-weight: 500; - text-align: center; - min-width: 80px; - transition: background-color 0.2s ease, transform 0.2s ease; -} - -.status-badge.success { - background-color: rgba(37, 190, 123, 0.1); - color: var(--esp-success); - border: 1px solid rgba(37, 190, 123, 0.2); -} - -.status-badge.warning { - background-color: rgba(243, 195, 0, 0.1); - color: var(--esp-warning); - border: 1px solid rgba(243, 195, 0, 0.2); -} - -.status-badge.danger { - background-color: rgba(232, 55, 17, 0.1); - color: var(--esp-danger); - border: 1px solid rgba(232, 55, 17, 0.2); -} - -.status-badge:hover { - transform: translateY(-1px); -} - -/* ---------------------------------- */ -/* Buttons */ -/* ---------------------------------- */ -.btn-outline-secondary { - border-color: #ced4da; - color: var(--esp-dark); -} - -.btn-outline-secondary:hover { - background-color: var(--esp-light); - border-color: var(--esp-gray); - color: var(--esp-dark); -} - -.btn-esp { - background-color: var(--esp-primary); - border: none; - color: #fff; - transition: all var(--transition-speed); -} - -.btn-esp:hover { - background-color: #cf3110; - color: #fff; - box-shadow: 0 4px 8px rgba(232, 55, 17, 0.2); -} - -.btn-esp-secondary { - background-color: var(--esp-secondary); - border: none; - color: var(--esp-dark); - transition: all var(--transition-speed); -} - -.btn-esp-secondary:hover { - background-color: #dab000; - color: var(--esp-dark); - box-shadow: 0 4px 8px rgba(243, 195, 0, 0.2); -} - -/* Table collapse button - updated to appear on the right */ -.table-collapse-btn { - background-color: var(--esp-light); - border: none; - color: var(--esp-gray); - padding: 6px 12px; - border-radius: var(--border-radius); - font-size: 0.875rem; - cursor: pointer; - transition: all var(--transition-speed); - display: inline-flex; - align-items: center; - justify-content: center; - box-shadow: 0 2px 4px rgba(0, 0, 0, 0.05); - margin-left: auto; /* Push to the right side */ -} - -.table-collapse-btn:hover { - background-color: #e9ecef; - color: var(--esp-primary); -} - -.table-collapse-btn i { - margin-right: 5px; -} - -.clear-filter-btn { - transition: all 0.2s ease; - margin-top: 10px; -} - -.clear-filter-btn:hover { - background-color: var(--esp-light); - color: var(--esp-primary); -} - -.close-alert { - background: none; - border: none; - color: #856404; - font-size: 0.8rem; - cursor: pointer; - padding: 4px; - margin-left: auto; - opacity: 0.7; - transition: opacity 0.2s; - display: flex; - align-items: center; - justify-content: center; -} - -.close-alert:hover { - opacity: 1; -} - -/* Floating action buttons */ -.floating-actions { - position: fixed; - bottom: 30px; - right: 30px; - z-index: 1000; - display: flex; - flex-direction: column; - gap: 10px; -} - -.floating-action-btn { - width: 50px; - height: 50px; - border-radius: 50%; - background-color: var(--esp-primary); - color: white; - display: flex; - align-items: center; - justify-content: center; - box-shadow: 0 4px 10px rgba(0, 0, 0, 0.15); - cursor: pointer; - transition: transform 0.3s cubic-bezier(0.175, 0.885, 0.32, 1.275), - background-color 0.3s ease, box-shadow 0.3s ease; - font-size: 1.2rem; -} - -.floating-action-btn:hover { - transform: scale(1.1) translateY(-2px); -} - -.back-to-top { - background-color: var(--esp-secondary); - color: var(--esp-dark); -} - -/* ---------------------------------- */ -/* Utilities */ -/* ---------------------------------- */ -/* Text toggles */ -.text-toggle, -.full-text { - cursor: pointer; -} - -.toggle-link { - display: inline-block; - margin-left: 8px; - font-size: 0.8rem; - color: var(--esp-blue); - text-decoration: none; -} - -.toggle-link:hover { - text-decoration: underline; - color: var(--esp-primary); -} - -/* Row count label */ -.row-count { - font-size: 0.8rem; - color: var(--esp-gray); - margin-bottom: 15px; - display: block; -} - -/* Table state indicators */ -.table-empty-state, -.table-loading-state { - text-align: center; - padding: 50px 20px; - display: flex; - flex-direction: column; - align-items: center; - justify-content: center; -} - -.table-empty-state { - color: var(--esp-gray); - background-color: rgba(0, 0, 0, 0.01); - border-radius: var(--border-radius); -} - -.table-empty-state p { - margin-top: 10px; - font-size: 1.1rem; -} - -.table-filtered-indicator { - text-align: center; - padding: 30px 20px; - color: var(--esp-warning); - background-color: rgba(243, 195, 0, 0.05); - border-radius: var(--border-radius); - display: flex; - flex-direction: column; - align-items: center; - justify-content: center; - border: 1px dashed rgba(243, 195, 0, 0.3); - margin: 20px; -} - -.table-filtered-indicator p { - margin: 10px 0; - font-size: 1rem; -} - -.table-loading-indicator { - padding: 20px; - background-color: rgba(0, 0, 0, 0.02); - border-radius: var(--border-radius); - margin: 10px; - text-align: center; - color: var(--esp-gray); - animation: pulse 1.5s infinite ease-in-out; -} - -@keyframes pulse { - 0% { - opacity: 0.6; - } - 50% { - opacity: 1; - } - 100% { - opacity: 0.6; - } -} - -@keyframes fadeIn { - from { - opacity: 0; - } - to { - opacity: 1; - } -} - -/* Dropdown and other UI elements */ -.dropdown-menu { - border-radius: var(--border-radius); - box-shadow: var(--box-shadow); - border: none; - padding: 0.5rem; -} - -.dropdown-item { - border-radius: 4px; - padding: 8px 12px; -} - -.dropdown-item:hover { - background-color: rgba(232, 55, 17, 0.1); -} - -/* Column visibility controls */ -.column-visibility-controls { - margin-right: 10px; -} - -.column-visibility-menu { - max-height: 350px; - overflow-y: auto; - padding: 8px 0; - min-width: 220px; -} - -.column-visibility-menu .dropdown-item { - display: flex; - align-items: center; - padding: 8px 16px; - white-space: normal; - word-break: break-word; -} - -.column-visibility-menu .dropdown-item:hover { - background-color: rgba(232, 55, 17, 0.05); -} - -.column-visibility-menu .dropdown-header { - font-weight: 600; - color: var(--esp-dark); - padding: 8px 16px; -} - -.column-visibility-menu .dropdown-divider { - margin: 5px 0; -} - -.column-toggle .column-hidden { - width: 16px; - color: var(--esp-primary); -} - -.column-toggle .column-hidden { - color: var(--esp-gray); -} - -/* Active filters */ -#active-filters { - margin-top: 10px; - margin-bottom: 15px !important; - display: flex; - align-items: center; - background-color: #f8f9fa; - padding: 8px 12px; - border-radius: var(--border-radius); - box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); -} - -#filter-badges .badge { - margin-right: 5px; - padding: 6px 10px; - font-weight: normal; -} - -#clear-all-filters { - margin-left: auto; - color: var(--esp-primary); - text-decoration: none; - font-size: 0.9rem; - padding: 4px 8px; -} - -#clear-all-filters:hover { - text-decoration: underline; - background-color: rgba(232, 55, 17, 0.05); - border-radius: 4px; -} - -/* Tooltip actions */ -.tooltip-actions { - position: absolute; - right: 60px; - background-color: white; - border-radius: var(--border-radius); - box-shadow: var(--box-shadow); - padding: 10px; - width: 200px; - display: none; -} - -.tooltip-actions.show { - display: block; - animation: fadeIn var(--transition-speed); -} - -/* Smooth scrolling */ -html.scrolling-top { - scroll-behavior: smooth; -} - -/* ---------------------------------- */ -/* Responsive Styles */ -/* ---------------------------------- */ -@media (max-width: 1200px) { - .bootstrap-table .fixed-table-body { - overflow-x: auto; - max-height: 600px; - } - - .table-container { - width: 100% !important; - padding: 0 !important; - } -} - -@media (max-width: 768px) { - .container-fluid { - padding-left: 20px !important; - padding-right: 20px !important; - } - - .table-responsive { - overflow-x: auto; - } - - .bootstrap-table .fixed-table-body { - max-height: 500px; - } - - .table-controls { - flex-direction: column; - align-items: flex-start; - } - - .table-controls-right { - width: 100%; - justify-content: space-between; - } - - .logo-container, - .spacer { - width: 100px; - } -} - -/* ---------------------------------- */ -/* Bootstrap Table Sticky Header Specific Styles */ -/* ---------------------------------- */ - -/* Make sure sticky header container is visible and properly positioned */ -.bootstrap-table .sticky-header-container { - position: sticky; - top: 0; - z-index: 10; - background-color: var(--esp-light); -} - -/* Ensure the sticky header is properly styled and visible */ -.bootstrap-table .sticky-header { - overflow: visible; - position: relative; -} - -/* Styles for the headers in the sticky container */ -.bootstrap-table .sticky-header-container thead th { - background-color: var(--esp-light); - color: var(--esp-dark); - font-weight: 600; - padding: 12px 15px; - border-bottom: 2px solid rgba(0, 0, 0, 0.1); - text-align: left; - position: static !important; /* Important to avoid double sticky behavior */ -} - -/* Styles for header cells inner content */ -.bootstrap-table .sticky-header-container th .th-inner { - padding: 0 !important; - line-height: 1.4; - white-space: normal !important; - overflow: visible !important; - text-overflow: clip !important; - font-weight: inherit !important; -} diff --git a/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml b/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml index f986108c54b..1b6ed573982 100644 --- a/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml +++ b/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml @@ -16,7 +16,7 @@ all_build_finished: script: - echo "all test jobs finished" -generate_pytest_build_report: +generate_presigned_json: stage: assign_test image: $ESP_ENV_IMAGE tags: @@ -29,16 +29,11 @@ generate_pytest_build_report: job: pipeline_variables artifacts: paths: - - failed_apps.html - - built_apps.html - - skipped_apps.html - - build_report.html - presigned.json expire_in: 1 week when: always script: - - python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type build - - python tools/ci/previous_stage_job_status.py --stage build + - idf-ci gitlab generate-presigned-json --output presigned.json generate_pytest_child_pipeline: # finally, we can get some use out of the default behavior that downloads all artifacts from the previous stage diff --git a/tools/ci/dynamic_pipelines/utils.py b/tools/ci/dynamic_pipelines/utils.py deleted file mode 100644 index a4344ab3bc6..00000000000 --- a/tools/ci/dynamic_pipelines/utils.py +++ /dev/null @@ -1,151 +0,0 @@ -# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import os -from urllib.parse import urlparse - -import requests - -from .constants import CI_DASHBOARD_API -from .constants import CI_JOB_TOKEN -from .constants import CI_MERGE_REQUEST_SOURCE_BRANCH_SHA -from .constants import CI_PAGES_URL -from .constants import CI_PROJECT_URL -from .models import GitlabJob - - -def is_url(string: str) -> bool: - """ - Check if the string is a valid URL by parsing it and verifying if it contains both a scheme and a network location. - - :param string: The string to check if it is a URL. - :return: True if the string is a valid URL, False otherwise. - """ - parsed = urlparse(string) - return bool(parsed.scheme) and bool(parsed.netloc) - - -def fetch_failed_jobs(commit_id: str) -> list[GitlabJob]: - """ - Fetches a list of jobs from the specified commit_id using an API request to ci-dashboard-api. - :param commit_id: The commit ID for which to fetch jobs. - :return: A list of jobs if the request is successful, otherwise an empty list. - """ - response = requests.get( - f'{CI_DASHBOARD_API}/commits/{commit_id}/jobs', - headers={'CI-Job-Token': CI_JOB_TOKEN}, - ) - if response.status_code != 200: - print(f'Failed to fetch jobs data: {response.status_code} with error: {response.text}') - return [] - - data = response.json() - jobs = data.get('jobs', []) - - if not jobs: - return [] - - failed_job_names = [job['name'] for job in jobs if job['status'] == 'failed'] - response = requests.post( - f'{CI_DASHBOARD_API}/jobs/failure_ratio', - headers={'CI-Job-Token': CI_JOB_TOKEN}, - json={ - 'job_names': failed_job_names, - 'exclude_branches': [os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_NAME', '')], - }, - ) - if response.status_code != 200: - print(f'Failed to fetch jobs failure rate data: {response.status_code} with error: {response.text}') - return [] - - failure_rate_data = response.json() - failure_rates = {item['name']: item for item in failure_rate_data.get('jobs', [])} - - combined_jobs = [] - for job in jobs: - failure_data = failure_rates.get(job['name'], {}) - combined_jobs.append(GitlabJob.from_json_data(job, failure_data)) - - return combined_jobs - - -def fetch_app_metrics( - source_commit_sha: str, - target_commit_sha: str, -) -> dict: - """ - Fetches the app metrics for the given source commit SHA and target branch SHA. - :param source_commit_sha: The source commit SHA. - :param target_branch_sha: The commit SHA of the branch to compare app sizes against. - :return: A dict of sizes of built binaries. - """ - print(f'Fetching bin size info: {source_commit_sha=} {target_commit_sha=}') - build_info_map = dict() - response = requests.post( - f'{CI_DASHBOARD_API}/apps/metrics', - headers={'CI-Job-Token': CI_JOB_TOKEN}, - json={ - 'source_commit_sha': source_commit_sha, - 'target_commit_sha': target_commit_sha, - }, - ) - if response.status_code != 200: - print(f'Failed to fetch build info: {response.status_code} - {response.text}') - else: - response_data = response.json() - build_info_map = { - f'{info["app_path"]}_{info["config_name"]}_{info["target"]}': info for info in response_data.get('data', []) - } - - return build_info_map - - -def load_file(file_path: str) -> str: - """ - Loads the content of a file. - - :param file_path: The path to the file needs to be loaded. - :return: The content of the file as a string. - """ - with open(file_path) as file: - return file.read() - - -def format_permalink(s: str) -> str: - """ - Formats a given string into a permalink. - - :param s: The string to be formatted into a permalink. - :return: The formatted permalink as a string. - """ - end_index = s.find('(') - - if end_index != -1: - trimmed_string = s[:end_index].strip() - else: - trimmed_string = s.strip() - - formatted_string = trimmed_string.lower().replace(' ', '-') - - return formatted_string - - -def get_artifacts_url(job_id: int, output_filepath: str) -> str: - """ - Generates the url of the path where the artifact will be stored in the job's artifacts . - - :param job_id: The job identifier used to construct the URL. - :param output_filepath: The path to the output file. - :return: The modified URL pointing to the job's artifacts. - """ - url = CI_PAGES_URL.replace('esp-idf', '-/esp-idf') - return f'{url}/-/jobs/{job_id}/artifacts/{output_filepath}' - - -def get_repository_file_url(file_path: str) -> str: - """ - Generates the url of the file path inside the repository. - - :param file_path: The file path where the file is stored. - :return: The modified URL pointing to the file's path in the repository. - """ - return f'{CI_PROJECT_URL}/-/raw/{CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}/{file_path}' diff --git a/tools/ci/exclude_check_tools_files.txt b/tools/ci/exclude_check_tools_files.txt index e9b51fe7503..bbec2ffdddb 100644 --- a/tools/ci/exclude_check_tools_files.txt +++ b/tools/ci/exclude_check_tools_files.txt @@ -28,7 +28,6 @@ tools/ci/metrics/examples_count/schema.yaml tools/ci/mirror-submodule-update.sh tools/ci/multirun_with_pyenv.sh tools/ci/mypy_ignore_list.txt -tools/ci/previous_stage_job_status.py tools/ci/push_to_github.sh tools/ci/python_packages/common_test_methods.py tools/ci/python_packages/gitlab_api.py diff --git a/tools/ci/idf_ci_local/app.py b/tools/ci/idf_ci_local/app.py index 8c7306fdef5..8e72d207e39 100644 --- a/tools/ci/idf_ci_local/app.py +++ b/tools/ci/idf_ci_local/app.py @@ -5,13 +5,10 @@ import subprocess import sys import typing as t -from dynamic_pipelines.constants import BINARY_SIZE_METRIC_NAME -from idf_build_apps import App from idf_build_apps import CMakeApp from idf_build_apps.constants import BuildStatus from idf_build_apps.utils import rmdir from idf_ci_utils import APP_EXTRA_S3_ARTIFACT_TYPE -from idf_ci_utils import idf_relpath if t.TYPE_CHECKING: pass @@ -65,103 +62,3 @@ class IdfCMakeApp(CMakeApp): self.build_path, exclude_file_patterns=['build_log.txt', 'size*.json'], ) - - -class Metrics: - """ - Represents a metric and its values for source, target, and the differences. - """ - - def __init__( - self, - source_value: float | None = None, - target_value: float | None = None, - difference: float | None = None, - difference_percentage: float | None = None, - ) -> None: - self.source_value = source_value or 0.0 - self.target_value = target_value or 0.0 - self.difference = difference or 0.0 - self.difference_percentage = difference_percentage or 0.0 - - def to_dict(self) -> dict[str, t.Any]: - """ - Converts the Metrics object to a dictionary. - """ - return { - 'source_value': self.source_value, - 'target_value': self.target_value, - 'difference': self.difference, - 'difference_percentage': self.difference_percentage, - } - - -class AppWithMetricsInfo(IdfCMakeApp): - metrics: dict[str, Metrics] - is_new_app: bool - - def __init__(self, **kwargs: t.Any) -> None: - super().__init__(**kwargs) - - self.metrics = {metric_name: metric_data for metric_name, metric_data in kwargs.get('metrics', {}).items()} - self.is_new_app = kwargs.get('is_new_app', False) - - class Config: - arbitrary_types_allowed = True - - -def enrich_apps_with_metrics_info( - app_metrics_info_map: dict[str, dict[str, t.Any]], apps: list[App] -) -> list[AppWithMetricsInfo]: - def _get_full_attributes(obj: App) -> dict[str, t.Any]: - """ - Retrieves all attributes of an object, including properties and computed fields. - """ - attributes: dict[str, t.Any] = obj.__dict__.copy() - for attr in dir(obj): - if not attr.startswith('_'): # Skip private/internal attributes - try: - value = getattr(obj, attr) - # Include only if it's not already in __dict__ - if attr not in attributes: - attributes[attr] = value - except Exception: - # Skip attributes that raise exceptions (e.g., methods needing args) - pass - return attributes - - default_metrics_structure = { - BINARY_SIZE_METRIC_NAME: Metrics( - source_value=0, - target_value=0, - difference=0, - difference_percentage=0.0, - ), - } - - apps_with_metrics_info = [] - for app in apps: - app.app_dir = idf_relpath(app.app_dir) - key = f'{app.app_dir}_{app.config_name}_{app.target}' - app_attributes = _get_full_attributes(app) - - metrics = {metric_name: default_metric for metric_name, default_metric in default_metrics_structure.items()} - is_new_app = False - - if key in app_metrics_info_map: - info = app_metrics_info_map[key] - for metric_name, metric_data in info.get('metrics', {}).items(): - metrics[metric_name] = Metrics( - source_value=metric_data.get('source_value', 0), - target_value=metric_data.get('target_value', 0), - difference=metric_data.get('difference', 0), - difference_percentage=metric_data.get('difference_percentage', 0.0), - ) - - is_new_app = info.get('is_new_app', False) - - app_attributes.update({'metrics': metrics, 'is_new_app': is_new_app}) - - apps_with_metrics_info.append(AppWithMetricsInfo(**app_attributes)) - - return apps_with_metrics_info diff --git a/tools/ci/idf_ci_utils.py b/tools/ci/idf_ci_utils.py index 7d3f1795317..74840dcd512 100644 --- a/tools/ci/idf_ci_utils.py +++ b/tools/ci/idf_ci_utils.py @@ -4,7 +4,6 @@ # some CI related util functions import logging import os -import re import subprocess import sys import typing as t @@ -243,23 +242,6 @@ class GitlabYmlConfig: return {k: v for k, v in self.config.items() if not k.startswith('.') and k not in self.global_keys} -def sanitize_job_name(name: str) -> str: - """ - Sanitize the job name from CI_JOB_NAME - - - for job with `parallel: int` set, the `CI_JOB_NAME` would be `job_name index/total`, like `foo 1/3` - - for job with `parallel: matrix` set, the `CI_JOB_NAME` would be `job_name: [var1, var2]`, like `foo: [a, b]` - - We consider - - the jobs generated by `parallel: int` as the same job, i.e., we remove the index/total part. - - the jobs generated by `parallel: matrix` as different jobs, so we keep the matrix part. - - :param name: job name - :return: sanitized job name - """ - return re.sub(r' \d+/\d+', '', name) - - def idf_relpath(p: str) -> str: """ Turn all paths under IDF_PATH to relative paths diff --git a/tools/ci/previous_stage_job_status.py b/tools/ci/previous_stage_job_status.py deleted file mode 100644 index 3888780d407..00000000000 --- a/tools/ci/previous_stage_job_status.py +++ /dev/null @@ -1,36 +0,0 @@ -# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import argparse -import os -import sys - -import requests - -if __name__ == '__main__': - - parser = argparse.ArgumentParser() - parser.add_argument('--stage', type=str, help='Stage name for check jobs status') - args = parser.parse_args() - - GITLAB_TOKEN = os.getenv('ESPCI_TOKEN') - GITLAB_HTTP_SERVER = os.getenv('GITLAB_HTTP_SERVER') - CI_PROJECT_ID = os.getenv('CI_PROJECT_ID') - CI_PIPELINE_ID = os.getenv('CI_PIPELINE_ID') - - api_path = f'projects/{CI_PROJECT_ID}/pipelines/{CI_PIPELINE_ID}/jobs?scope[]=failed&per_page=100' - - page = 0 - while True: - response = requests.get( - f'{GITLAB_HTTP_SERVER}/api/v4/{api_path}&page={page}', - headers={'PRIVATE-TOKEN': GITLAB_TOKEN} - ) - jobs = response.json() - if not jobs: - break - - for job in jobs: - if job['stage'] == args.stage: - print(f'Jobs from the previous stage {args.stage} should pass; otherwise, this job will fail.') - sys.exit(1) - page += 1