Merge branch 'ci/use-interruptible' into 'master'

ci: use common workflow that auto-cancel earlier MR pipelines

See merge request espressif/esp-idf!45736

.gitlab-ci.yml

@@ -1,23 +1,9 @@
-workflow:
-  rules:
-    # Disable those non-protected push triggered pipelines
-    - if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && $CI_PIPELINE_SOURCE == "push"'
-      when: never
-    # merged result pipelines
-    - if: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
-      variables:
-        PIPELINE_COMMIT_SHA: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA
-    # else
-    - if: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA == null || $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA == ""
-      variables:
-        PIPELINE_COMMIT_SHA: $CI_COMMIT_SHA
-    - when: always
-
 # Place the default settings in `.gitlab/ci/common.yml` instead
 
 include:
   - project: "ci/actions/common"
     file:
+      - "templates/stable/default-workflow.yml"
       - "templates/idf/deploy-github.yml"
       - "templates/idf/deploy-docs.yml"
   - ".gitlab/ci/danger.yml"

.gitlab/ci/build.yml

@@ -154,21 +154,17 @@ build_child_pipeline:
         job: generate_build_child_pipeline
     strategy: depend
 
-generate_disabled_apps_report:
+generate_prebuild_report:
   extends:
     - .build_template
   tags: [fast_run, shiny]
-  dependencies: # set dependencies to null to avoid missing artifacts issue
   needs:
     - pipeline_variables
-    - job: baseline_manifest_sha
-      optional: true
   artifacts:
     paths:
-      - disabled_report.html
+      - prebuild_report.html
     expire_in: 1 week
     when: always
   script:
-    - pip install dominate idf-build-apps
+    - run_cmd idf-ci build collect --format html -o prebuild_report.html
-    - run_cmd python tools/ci/gen_disabled_report.py --output disabled_report.html --verbose --enable-preview-targets
+    - echo "Report generated at https://${CI_PAGES_HOSTNAME}:${CI_SERVER_PORT}/-/esp-idf/-/jobs/${CI_JOB_ID}/artifacts/prebuild_report.html"
-    - echo "Report generated at https://${CI_PAGES_HOSTNAME}:${CI_SERVER_PORT}/-/esp-idf/-/jobs/${CI_JOB_ID}/artifacts/disabled_report.html"

.gitlab/ci/common.yml

@@ -247,41 +247,14 @@ variables:
     - *show_ccache_statistics
     - *upload_failed_job_log_artifacts
 
-##############################
-# Git Strategy Job Templates #
-##############################
 .brew-macos-settings:
   variables:
-    GIT_STRATEGY: none  # we do manual git clone to use local mirror
+    GIT_STRATEGY: fetch
     IDF_CCACHE_ENABLE: "0"
     CCACHE_DIR: "/var/tmp/cache/idf_ccache"
   tags:
     - macos-tart
   image: macos-sequoia-idf-v6.1
-  cache: []  # pip cache is created under amd64, and submodules are downloaded with brew mirror, so disable cache here
-  before_script:
-    # assert LOCAL_GIT_MIRROR is set
-    - echo -e "section_start:`date +%s`:check_out\r\e[0Kchecking out from local git mirror, then reset to CI_COMMIT_SHA"
-    - |
-      if [ -z "${LOCAL_GIT_MIRROR:-}" ]; then
-        echo "Error: LOCAL_GIT_MIRROR not set, cannot clone from mirror."
-        exit 1
-      fi
-    - MIRROR_REPO_URL="${LOCAL_GIT_MIRROR}/${CI_PROJECT_PATH}"
-    - cd "${CI_PROJECT_DIR}"
-    # since .cache exists in CI_PROJECT_DIR, so can't direct `git clone .`
-    - git clone -b ${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-${CI_COMMIT_REF_NAME}} --depth=1 --recursive --shallow-submodules "${MIRROR_REPO_URL}" tmp
-    - mv tmp/.git ./
-    - rm -rf tmp
-    - git reset --hard
-    # set remote url back
-    - git remote set-url origin "${CI_REPOSITORY_URL}"
-    - eval "git fetch --depth=1 ${GIT_FETCH_EXTRA_FLAGS} origin ${CI_COMMIT_SHA}"
-    - git checkout FETCH_HEAD
-    - git submodule update --init --recursive --depth=1
-    - echo -e "section_end:`date +%s`:check_out\r\e[0K"
-    - *common-before_scripts
-    - *setup_tools_and_idf_python_venv
   after_script: []  # ccache now is disabled for macos brew runners
   timeout: 30m
 
@@ -311,10 +284,3 @@ default:
     - fetch_submodules
     - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
     - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}
-  retry:
-    max: 2
-    when:
-      # In case of a runner failure we could hop to another one, or a network error could go away.
-      - runner_system_failure
-      # Job execution timeout may be caused by a network issue.
-      - job_execution_timeout

.gitlab/ci/docs.yml

@@ -16,9 +16,6 @@
 .patterns-example-readme: &patterns-example-readme
   - "examples/**/*.md"
 
-.patterns-docs-preview: &patterns-docs-preview
-  - "docs/**/*"
-
 .if-protected-check: &if-protected-check
   if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/)'
 

tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml

@@ -12,6 +12,7 @@ all_build_finished:
     expire_in: 1 week
     when: always
   before_script: []
+  dependencies: []
   script:
     - echo "all test jobs finished"
 

tools/ci/exclude_check_tools_files.txt

@@ -17,7 +17,6 @@ tools/ci/dynamic_pipelines/**/*
 tools/ci/envsubst.py
 tools/ci/executable-list.txt
 tools/ci/fix_empty_prototypes.sh
-tools/ci/gen_disabled_report.py
 tools/ci/generate_rules.py
 tools/ci/get-full-sources.sh
 tools/ci/get_all_test_results.py

tools/ci/gen_disabled_report.py

@@ -1,980 +0,0 @@
-#!/usr/bin/env python3
-# SPDX-FileCopyrightText: 2025 Espressif Systems (Shanghai) CO LTD
-# SPDX-License-Identifier: Apache-2.0
-
-import argparse
-import fnmatch
-import json
-import os
-import subprocess
-import sys
-from collections import Counter
-from datetime import datetime
-
-from dominate import document
-from dominate.tags import a
-from dominate.tags import button
-from dominate.tags import div
-from dominate.tags import h1
-from dominate.tags import h3
-from dominate.tags import input_
-from dominate.tags import label
-from dominate.tags import li
-from dominate.tags import script
-from dominate.tags import span
-from dominate.tags import strong
-from dominate.tags import style
-from dominate.tags import table
-from dominate.tags import tbody
-from dominate.tags import td
-from dominate.tags import th
-from dominate.tags import thead
-from dominate.tags import tr
-from dominate.tags import ul
-from dominate.util import raw
-from idf_build_apps import App
-from idf_build_apps.app import AppDeserializer
-
-
-def run_idf_build_apps_find(
-    output_file: str = 'apps.json',
-    verbose: bool = False,
-    enable_preview_targets: bool = False,
-) -> str:
-    """
-    Run idf-build-apps find command to get application list
-
-    :param output_file: Output file path
-    :param verbose: Whether to enable verbose output
-    :param enable_preview_targets: Whether to enable preview targets
-    :return: Output file path
-    """
-    cmd = [
-        'idf-build-apps',
-        'find',
-        '--output-format',
-        'json',
-        '--include-all-apps',
-        '--output',
-        output_file,
-    ]
-
-    # Add verbose parameter
-    if verbose:
-        cmd.append('--verbose')
-
-    # Add enable_preview_targets parameter
-    if enable_preview_targets:
-        cmd.append('--enable-preview-targets')
-
-    print(f'Running command: {" ".join(cmd)}')
-
-    try:
-        result = subprocess.run(cmd, check=True, capture_output=True, text=True)
-        print('Command completed successfully')
-        print(f'stdout: {result.stdout}')
-        if result.stderr:
-            print(f'stderr: {result.stderr}')
-        return output_file
-    except subprocess.CalledProcessError as e:
-        print(f'Command failed with exit code {e.returncode}')
-        print(f'stdout: {e.stdout}')
-        print(f'stderr: {e.stderr}')
-        raise
-
-
-def parse_codeowners(codeowners_path: str) -> list[tuple[str, list[str]]]:
-    """
-    Parse CODEOWNERS file and return a list of (pattern, codeowners) tuples.
-    The list preserves the order of the rules in the file.
-
-    :param codeowners_path: Path to CODEOWNERS file
-    :return: List of (pattern, codeowners) tuples
-    """
-    codeowners_mapping: list[tuple[str, list[str]]] = []
-
-    if not os.path.exists(codeowners_path):
-        print(f'Warning: CODEOWNERS file not found at {codeowners_path}')
-        return codeowners_mapping
-
-    try:
-        with open(codeowners_path, encoding='utf-8') as f:
-            for line_num, line in enumerate(f, 1):
-                line = line.strip()
-
-                # Skip empty lines and comments
-                if not line or line.startswith('#'):
-                    continue
-
-                # Parse pattern and codeowners
-                parts = line.split()
-                if len(parts) < 2:
-                    print(f'Warning: Invalid CODEOWNERS line {line_num}: {line}')
-                    continue
-
-                pattern = parts[0]
-                codeowners = parts[1:]
-
-                # Remove @ symbol and the long prefix from codeowners if present
-                codeowners = [owner.lstrip('@').replace('esp-idf-codeowners/', '') for owner in codeowners]
-
-                codeowners_mapping.append((pattern, codeowners))
-
-    except Exception as e:
-        print(f'Error reading CODEOWNERS file: {e}')
-
-    return codeowners_mapping
-
-
-def match_codeowners(app_dir: str, codeowners_mapping: list[tuple[str, list[str]]]) -> list[str]:
-    """
-    Match app directory against CODEOWNERS patterns, supporting wildcards ('*' and '**').
-    The matching traverses up from the app's directory and respects the 'last match wins' rule.
-
-    :param app_dir: Application directory path (e.g., './components/esp_driver_touch_sens/test_apps/touch_sens')
-    :param codeowners_mapping: List of (pattern, codeowners) tuples, in file order
-    :return: List of matching codeowners from the last matching rule
-    """
-    # 1. Normalize the app_dir path
-    # Remove leading './' and ensure it starts with a single '/'
-    if app_dir.startswith('./'):
-        app_dir = app_dir[1:]
-    if not app_dir.startswith('/'):
-        app_dir = '/' + app_dir
-    app_dir = os.path.normpath(app_dir)
-
-    # 2. Generate all parent paths
-    parent_paths = []
-    current_path = app_dir
-    while True:
-        parent_paths.append(current_path)
-        if current_path == '/':
-            break
-        parent, _ = os.path.split(current_path)
-        if parent == current_path:  # Root reached
-            break
-        current_path = parent
-
-    # 3. Iterate through rules in reverse to find the last match
-    for pattern, owners in reversed(codeowners_mapping):
-        # The pattern needs to be anchored to the root for matching
-        if not pattern.startswith('/'):
-            pattern = '/**/' + pattern  # Handles patterns like '*.py' or 'docs' anywhere
-
-        for path in parent_paths:
-            if fnmatch.fnmatch(path, pattern) or fnmatch.fnmatch(path + '/', pattern):
-                return owners
-
-    # Fallback to the root-level wildcard rule if no other match is found
-    for pattern, owners in reversed(codeowners_mapping):
-        if pattern == '*':
-            return owners
-
-    return []
-
-
-def add_codeowners_to_apps(apps: list[App], codeowners_mapping: list[tuple[str, list[str]]]) -> dict[str, list[str]]:
-    """
-    Match apps to codeowners and return a dictionary mapping app paths to their owners.
-    If an app does not match any rule, it is assigned to the 'others' group.
-
-    :param apps: List of App objects
-    :param codeowners_mapping: List of (pattern, codeowners) tuples, in file order
-    :return: A dictionary mapping app.app_dir to a list of codeowners
-    """
-    app_to_owners_map = {}
-    for app in apps:
-        matched_owners = match_codeowners(app.app_dir, codeowners_mapping)
-        if not matched_owners:
-            app_to_owners_map[app.app_dir] = ['others']
-        else:
-            app_to_owners_map[app.app_dir] = matched_owners
-    return app_to_owners_map
-
-
-def load_apps_from_json(json_file: str) -> list[App]:
-    """
-    Load application list from JSON file (clean version without codeowners)
-    """
-    apps = []
-
-    with open(json_file, encoding='utf-8') as f:
-        if json_file.endswith('.json'):
-            # If it's JSON format, try to parse as array
-            try:
-                data = json.load(f)
-                if isinstance(data, list):
-                    for app_data in data:
-                        # Handle build_system compatibility: 'idf_cmake' -> 'cmake'
-                        if app_data.get('build_system') == 'idf_cmake':
-                            app_data['build_system'] = 'cmake'
-                        app = AppDeserializer.from_json(json.dumps(app_data))
-                        apps.append(app)
-                else:
-                    # If not an array, try line-by-line parsing
-                    f.seek(0)
-                    for line in f:
-                        line = line.strip()
-                        if line:
-                            app = AppDeserializer.from_json(line)
-                            apps.append(app)
-            except json.JSONDecodeError:
-                # If JSON parsing fails, try line-by-line parsing
-                f.seek(0)
-                for line in f:
-                    line = line.strip()
-                    if line:
-                        try:
-                            app = AppDeserializer.from_json(line)
-                            apps.append(app)
-                        except Exception as e:
-                            print(f'Warning: Failed to parse line: {line[:100]}... Error: {e}')
-        else:
-            # If not JSON format, parse line by line
-            for line in f:
-                line = line.strip()
-                if line:
-                    try:
-                        app = AppDeserializer.from_json(line)
-                        apps.append(app)
-                    except Exception as e:
-                        print(f'Warning: Failed to parse line: {line[:100]}... Error: {e}')
-
-    print(f'Loaded {len(apps)} apps from {json_file}')
-    return apps
-
-
-def generate_disabled_report(apps: list[App], app_to_owners_map: dict[str, list[str]], report_path: str) -> None:
-    """Generate disabled report"""
-    # Categorize applications
-    cant_build_temp = []
-    can_build_cant_test_temp = []
-    cant_build_not_temp = []
-    can_build_cant_test_not_temp = []
-    can_test = []
-    all_codeowners: set[str] = set()
-    all_targets: set[str] = set()
-    owner_app_counts: Counter[str] = Counter()
-    target_app_counts: Counter[str] = Counter()
-
-    for app in apps:
-        # Get owners from the map
-        app_owners = app_to_owners_map.get(app.app_dir, [])
-
-        # Update owner app counts
-        for owner in app_owners:
-            owner_app_counts[owner] += 1
-
-        # Update target app counts
-        all_targets.add(app.target)
-        target_app_counts[app.target] += 1
-
-        # Check for unsupported build_status values and abort if found
-        if app.build_status not in ['should be built', 'disabled']:
-            print(
-                f'ERROR: Found unsupported build_status "{app.build_status}" for app {app.app_dir}, target {app.target}'
-            )
-            print(
-                'This task only supports "should be built" and "disabled" status. Please fix the build configuration.'
-            )
-            sys.exit(1)
-
-        # Map build_status to the expected status values
-        if app.build_status == 'should be built':
-            if app.test_comment is None:
-                status = 'can_build_and_test'
-            else:
-                status = 'can_build_no_test'
-        else:  # if app.build_status == 'disabled':
-            status = 'cannot_build'
-
-        # Categorize apps based on status and temporary flags
-        if status == 'can_build_and_test':
-            can_test.append(app)
-        elif status == 'cannot_build':
-            # Handle cases where build_comment might be None for a disabled app
-            build_comment = app.build_comment or 'Reason not specified'
-            if 'temporary' in build_comment.lower():
-                cant_build_temp.append(app)
-            else:
-                cant_build_not_temp.append(app)
-        elif status == 'can_build_no_test':
-            # Handle cases where test_comment might be None
-            test_comment = app.test_comment or 'Reason not specified'
-            if 'temporary' in test_comment.lower():
-                can_build_cant_test_temp.append(app)
-            else:
-                can_build_cant_test_not_temp.append(app)
-
-        all_codeowners.update(app_owners)
-
-    # Create HTML document
-    doc = document(title='Build and Test Status Report')
-
-    def get_css_styles() -> str:
-        """Return CSS styles"""
-        return """
-            body {
-                font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
-                line-height: 1.6;
-                margin: 0;
-                padding: 20px;
-                background-color: #f5f5f5;
-            }
-            .container {
-                max-width: 1200px;
-                margin: 0 auto;
-                background-color: white;
-                padding: 30px;
-                border-radius: 8px;
-                box-shadow: 0 2px 10px rgba(0,0,0,0.1);
-            }
-            h1 {
-                color: #333;
-                text-align: center;
-                margin-bottom: 30px;
-                border-bottom: 3px solid #007acc;
-                padding-bottom: 10px;
-            }
-            h2 {
-                color: #007acc;
-                margin-top: 40px;
-                margin-bottom: 20px;
-                padding: 10px;
-                background-color: #f8f9fa;
-                border-left: 4px solid #007acc;
-                border-radius: 4px;
-            }
-            .navigation {
-                background-color: #f8f9fa;
-                padding: 20px;
-                border-radius: 8px;
-                margin-bottom: 30px;
-                border: 1px solid #dee2e6;
-            }
-            .navigation h3 {
-                margin-top: 0;
-                color: #495057;
-                margin-bottom: 15px;
-            }
-            .nav-links {
-                display: flex;
-                flex-wrap: wrap;
-                gap: 10px;
-            }
-            .nav-link {
-                background-color: #007acc;
-                color: white;
-                padding: 8px 16px;
-                border-radius: 20px;
-                text-decoration: none;
-                font-size: 0.9em;
-                transition: background-color 0.3s;
-            }
-            .nav-link:hover {
-                background-color: #005a9e;
-            }
-            .control-buttons {
-                text-align: center;
-                margin: 20px 0;
-            }
-            .control-btn {
-                background-color: #28a745;
-                color: white;
-                border: none;
-                padding: 10px 20px;
-                border-radius: 5px;
-                cursor: pointer;
-                margin: 0 10px;
-                font-size: 0.9em;
-                transition: background-color 0.3s;
-            }
-            .control-btn:hover {
-                background-color: #218838;
-            }
-            .control-btn.collapse-all {
-                background-color: #dc3545;
-            }
-            .control-btn.collapse-all:hover {
-                background-color: #c82333;
-            }
-            .filters-wrapper {
-                display: flex;
-                gap: 20px;
-                margin-bottom: 20px;
-                justify-content: center;
-            }
-            .filter-container {
-                flex: 1;
-                max-width: 500px;
-                margin-bottom: 20px;
-                text-align: center;
-                padding: 15px;
-                border: 1px solid #dee2e6;
-                border-radius: 8px;
-                background-color: #f8f9fa;
-            }
-            #codeownerFilter {
-                padding: 10px;
-                border-radius: 5px;
-                border: 1px solid #ccc;
-                font-size: 1em;
-            }
-            table {
-                width: 100%;
-                border-collapse: collapse;
-                margin-bottom: 30px;
-                box-shadow: 0 2px 8px rgba(0,0,0,0.1);
-            }
-            th, td {
-                padding: 12px;
-                text-align: left;
-                border-bottom: 1px solid #ddd;
-            }
-            th {
-                background-color: #007acc;
-                color: white;
-                font-weight: 600;
-            }
-            tr:nth-child(even) {
-                background-color: #f8f9fa;
-            }
-            tr:hover {
-                background-color: #e3f2fd;
-            }
-            .summary {
-                background-color: #e8f5e8;
-                padding: 20px;
-                border-radius: 8px;
-                margin-bottom: 30px;
-                border-left: 4px solid #4caf50;
-            }
-            .summary h3 {
-                margin-top: 0;
-                color: #2e7d32;
-            }
-            .summary ul {
-                margin: 10px 0;
-                padding-left: 20px;
-            }
-            .summary li {
-                margin: 5px 0;
-            }
-            .category-summary {
-                background-color: #f8f9fa;
-                padding: 15px;
-                border-radius: 6px;
-                margin-bottom: 20px;
-                border-left: 4px solid #007acc;
-            }
-            .category-summary h3 {
-                margin: 0;
-                color: #007acc;
-                display: flex;
-                justify-content: space-between;
-                align-items: center;
-            }
-            .category-summary .count {
-                background-color: #007acc;
-                color: white;
-                padding: 4px 8px;
-                border-radius: 12px;
-                font-size: 0.9em;
-                font-weight: bold;
-            }
-            .category-details {
-                margin-left: 20px;
-                margin-bottom: 30px;
-            }
-            .directory-info {
-                background-color: #fff3cd;
-                padding: 10px;
-                border-radius: 4px;
-                margin-bottom: 15px;
-                border-left: 3px solid #ffc107;
-                font-family: monospace;
-                font-size: 0.9em;
-                cursor: pointer;
-                transition: background-color 0.3s;
-            }
-            .directory-info:hover {
-                background-color: #ffeaa7;
-            }
-            .total-count {
-                color: #666;
-                font-size: 0.9em;
-                margin-left: 10px;
-            }
-            .empty-section {
-                text-align: center;
-                color: #666;
-                font-style: italic;
-                padding: 20px;
-                background-color: #f8f9fa;
-                border-radius: 4px;
-            }
-            .timestamp {
-                text-align: center;
-                color: #666;
-                font-size: 0.9em;
-                margin-bottom: 30px;
-            }
-            .table-container {
-                margin-bottom: 20px;
-            }
-            .table-header {
-                background-color: #f8f9fa;
-                padding: 10px 15px;
-                border-radius: 4px;
-                cursor: pointer;
-                border: 1px solid #dee2e6;
-                transition: background-color 0.2s;
-            }
-            .table-header:hover {
-                background-color: #e9ecef;
-            }
-            .table-header h4 {
-                margin: 0;
-                display: flex;
-                justify-content: space-between;
-                align-items: center;
-                color: #495057;
-            }
-            .toggle-icon {
-                font-size: 0.8em;
-                transition: transform 0.2s;
-            }
-            .table-content {
-                margin-top: 10px;
-            }
-            .app-row {
-                display: table-row;
-            }
-            .checkbox-group {
-                display: flex;
-                flex-wrap: wrap;
-                gap: 15px;
-                justify-content: center;
-                margin-top: 10px;
-            }
-            .checkbox-label {
-                display: flex;
-                align-items: center;
-                cursor: pointer;
-                font-size: 0.9em;
-            }
-            .checkbox-label input[type="checkbox"] {
-                margin-right: 5px;
-            }
-            .owner-label {
-                margin-left: 10px;
-                font-size: 0.9em;
-                color: #555;
-            }
-        """
-
-    def create_summary_section() -> None:
-        """Create summary section"""
-        with div(cls='summary'):
-            h3('Summary')
-            with ul():
-                li(strong('Total apps analyzed: '), str(len(apps)))
-                li(strong('Build temporarily disabled: '), str(len(cant_build_temp)))
-                li(strong('Test temporarily disabled: '), str(len(can_build_cant_test_temp)))
-                li(strong('Build disabled permanently: '), str(len(cant_build_not_temp)))
-                li(strong('Test disabled permanently: '), str(len(can_build_cant_test_not_temp)))
-                li(strong('Normal: '), str(len(can_test)))
-
-    def create_navigation_section() -> None:
-        """Create navigation section with links to each category"""
-        with div(cls='navigation'):
-            h3('Quick Navigation')
-            with div(cls='nav-links'):
-                for category_id, title, _, _ in categories:
-                    a(title, href=f'#{category_id}', cls='nav-link')
-
-    def create_filter_group(
-        title: str,
-        filter_type: str,
-        items: list[str],
-        counts: Counter,
-        total_count: int,
-    ) -> None:
-        """Create a generic filter checkbox group with app counts"""
-        with div(cls='filter-container'):
-            strong(title, style='display: block; margin-bottom: 10px;')
-            with div(id=f'{filter_type}Filter', cls='checkbox-group'):
-                # 'All' checkbox
-                with label(cls='checkbox-label'):
-                    input_(
-                        type='checkbox',
-                        id=f'{filter_type}-all',
-                        value='all',
-                        onchange=f'toggleAll(this, "{filter_type}")',
-                        checked=True,
-                    )
-                    span(
-                        f'All ({total_count})',
-                        style='margin-left: 5px; font-weight: bold;',
-                    )
-
-                # Individual item checkboxes
-                for item in items:
-                    count = counts.get(item, 0)
-                    with label(cls='checkbox-label'):
-                        input_(
-                            type='checkbox',
-                            cls=f'{filter_type}-checkbox',
-                            value=item,
-                            onchange='applyFilters()',
-                            checked=True,
-                        )
-                        span(f'{item} ({count})', style='margin-left: 5px;')
-
-    def create_control_buttons() -> None:
-        """Create control buttons for expand/collapse all"""
-        with div(cls='control-buttons'):
-            button('Expand All', onclick='expandAll()', cls='control-btn')
-            button('Collapse All', onclick='collapseAll()', cls='control-btn collapse-all')
-
-    def create_category_section(
-        category_id: str,
-        title: str,
-        apps: list[App],
-        all_apps: list[App],
-        show_status: bool = False,
-    ) -> None:
-        """Create category section"""
-        with div(cls='category-summary'):
-            with h3():
-                span(title)
-                span(str(len(apps)), cls='count')
-
-        # Always show section content (no collapse)
-        with div(cls='category-details', id=category_id, style='display: block;'):
-            if apps:
-                # Group by full directory path (preserve last level directory)
-                dir_groups: dict[str, list[App]] = {}
-                for app in apps:
-                    # Use full path as grouping key
-                    dir_name = app.app_dir
-                    if dir_name not in dir_groups:
-                        dir_groups[dir_name] = []
-                    dir_groups[dir_name].append(app)
-
-                # Calculate total apps for each directory across all categories
-                total_apps_by_dir = {}
-                for app in all_apps:
-                    dir_name = app.app_dir
-                    if dir_name not in total_apps_by_dir:
-                        total_apps_by_dir[dir_name] = 0
-                    total_apps_by_dir[dir_name] += 1
-
-                for dir_name, apps_in_dir in dir_groups.items():
-                    total_apps = total_apps_by_dir.get(dir_name, len(apps_in_dir))
-
-                    # Determine the owners for this directory group
-                    dir_owners = set()
-                    for app in apps_in_dir:
-                        dir_owners.update(app_to_owners_map.get(app.app_dir, []))
-                    owners_str = ', '.join(sorted(list(dir_owners)))
-
-                    with div(cls='directory-info', onclick=f"toggleTable('{category_id}-{hash(dir_name)}')"):
-                        span(
-                            '+',
-                            cls='toggle-icon',
-                            style='float: left; color: #666; margin-right: 8px; font-weight: bold;',
-                        )
-                        strong(f'{dir_name.replace("./", "")}')
-                        span(f' ({len(apps_in_dir)}/{total_apps} apps)', style='color: #666;')
-                        span(f'Owners: {owners_str}', cls='owner-label')
-
-                    # Table can be collapsed, controlled by table header row
-                    with div(cls='table-container'):
-                        with div(cls='table-content', id=f'{category_id}-{hash(dir_name)}', style='display: none;'):
-                            with table():
-                                with thead(cls='table-header'):
-                                    with tr():
-                                        th('Target')
-                                        th('Config')
-                                        if show_status:
-                                            th('Status')
-                                        else:
-                                            th('Reason')
-                                with tbody():
-                                    for app in apps_in_dir:
-                                        app_owners = app_to_owners_map.get(app.app_dir, [])
-                                        with tr(
-                                            cls='app-row',
-                                            data_codeowners=','.join(app_owners),
-                                        ):
-                                            td(app.target)
-                                            td(app.config_name or '-')
-                                            if show_status:
-                                                td('Success')
-                                            else:
-                                                reason = (
-                                                    app.build_comment
-                                                    if app.build_status == 'disabled'
-                                                    else app.test_comment
-                                                )
-                                                td(reason or 'Reason not specified')
-            else:
-                div('No apps in this category', cls='empty-section')
-
-    def get_javascript() -> str:
-        """Return JavaScript code"""
-        return """
-            function toggleTable(id) {
-                const content = document.getElementById(id);
-                if (!content) {
-                    console.error('Content element not found:', id);
-                    return;
-                }
-
-                const directoryInfo = content.parentElement.previousElementSibling;
-                if (!directoryInfo) {
-                    console.error('Directory info element not found');
-                    return;
-                }
-
-                const icon = directoryInfo.querySelector('.toggle-icon');
-
-                if (content.style.display === 'none' || content.style.display === '') {
-                    content.style.display = 'block';
-                    if (icon) icon.textContent = '-';
-                } else {
-                    content.style.display = 'none';
-                    if (icon) icon.textContent = '+';
-                }
-            }
-
-            function expandAll() {
-                const allContents = document.querySelectorAll('.table-content');
-                const allIcons = document.querySelectorAll('.toggle-icon');
-
-                allContents.forEach(content => {
-                    content.style.display = 'block';
-                });
-
-                allIcons.forEach(icon => {
-                    icon.textContent = '-';
-                });
-            }
-
-            function collapseAll() {
-                const allContents = document.querySelectorAll('.table-content');
-                const allIcons = document.querySelectorAll('.toggle-icon');
-
-                allContents.forEach(content => {
-                    content.style.display = 'none';
-                });
-
-                allIcons.forEach(icon => {
-                    icon.textContent = '+';
-                });
-            }
-
-            function applyFilters() {
-                // Get selected codeowners
-                const selectedCodeowners = Array.from(
-                    document.querySelectorAll('.codeowner-checkbox:checked')
-                ).map(cb => cb.value);
-
-                // Get selected targets
-                const selectedTargets = Array.from(
-                    document.querySelectorAll('.target-checkbox:checked')
-                ).map(cb => cb.value);
-
-                // Update 'All' checkbox states
-                const allCodeownerCheckbox = document.getElementById('codeowner-all');
-                const allCodeownerCheckboxes = document.querySelectorAll('.codeowner-checkbox');
-                allCodeownerCheckbox.checked = selectedCodeowners.length === allCodeownerCheckboxes.length;
-
-                const allTargetCheckbox = document.getElementById('target-all');
-                const allTargetCheckboxes = document.querySelectorAll('.target-checkbox');
-                allTargetCheckbox.checked = selectedTargets.length === allTargetCheckboxes.length;
-
-                // Filter rows and hide empty directory sections
-                const directorySections = document.querySelectorAll('.directory-info');
-                directorySections.forEach(section => {
-                    const tableContainer = section.nextElementSibling;
-                    if (!tableContainer) return;
-
-                    const rows = tableContainer.querySelectorAll('.app-row');
-                    let visibleRows = 0;
-                    rows.forEach(row => {
-                        const codeowners = row.getAttribute('data-codeowners').split(',');
-                        // Assumes target is the first cell
-                        const target = row.querySelector('td:first-child').textContent;
-
-                        const codeownerMatch = selectedCodeowners.some(owner => codeowners.includes(owner));
-                        const targetMatch = selectedTargets.includes(target);
-
-                        if (codeownerMatch && targetMatch) {
-                            row.style.display = 'table-row';
-                            visibleRows++;
-                        } else {
-                            row.style.display = 'none';
-                        }
-                    });
-
-                    // Hide the entire directory section if no rows are visible
-                    if (visibleRows === 0) {
-                        section.style.display = 'none';
-                        tableContainer.style.display = 'none';
-                    } else {
-                        section.style.display = 'block';
-                        tableContainer.style.display = 'block';
-                    }
-                });
-            }
-
-            function toggleAll(checkbox, filterType) {
-                const allCheckboxes = document.querySelectorAll(`.${filterType}-checkbox`);
-                allCheckboxes.forEach(cb => {
-                    cb.checked = checkbox.checked;
-                });
-                applyFilters(); // Apply filters immediately
-            }
-
-            // Initial filter call to show all rows at the beginning
-            document.addEventListener('DOMContentLoaded', function() {
-                applyFilters();
-            });
-        """
-
-    # Define category configuration
-    categories = [
-        ('cant-build-temp', '1. Build temporarily disabled', cant_build_temp, False),
-        ('can-build-cant-test-temp', '2. Test temporarily disabled', can_build_cant_test_temp, False),
-        ('cant-build-not-temp', '3. Build disabled permanently', cant_build_not_temp, False),
-        ('can-build-cant-test-not_temp', '4. Test disabled permanently', can_build_cant_test_not_temp, False),
-        ('can-test', '5. Normal', can_test, True),
-    ]
-
-    with doc.head:
-        style(get_css_styles())
-        script(raw(get_javascript()))
-
-    with doc:
-        with div(cls='container'):
-            h1('Build and Test Status Report')
-            div(f'Generated at: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")}', cls='timestamp')
-
-            # Create summary section
-            create_summary_section()
-
-            # Create navigation section
-            create_navigation_section()
-
-            # Create filter section and pass owner counts
-            with div(cls='filters-wrapper'):
-                create_filter_group(
-                    'Filter by Codeowner:',
-                    'codeowner',
-                    sorted(list(all_codeowners)),
-                    owner_app_counts,
-                    len(apps),
-                )
-                create_filter_group(
-                    'Filter by Target:',
-                    'target',
-                    sorted(list(all_targets)),
-                    target_app_counts,
-                    len(apps),
-                )
-
-            # Create control buttons
-            create_control_buttons()
-
-            # Generate each category using configuration
-            for category_id, title, apps_list, show_status in categories:
-                create_category_section(category_id, title, apps_list, apps, show_status)
-
-    # Write to file
-    with open(report_path, 'w', encoding='utf-8') as f:
-        f.write(doc.render())
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser(
-        description='Generate a report of disabled and skipped builds/tests',
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        epilog="""
-Examples:
-  # Basic usage (uses default paths: examples, tools/test_apps, components)
-  python gen_disabled_report.py --output report.html
-
-  # Use existing JSON file
-  python gen_disabled_report.py --input apps.json --output report.html
-
-  # Enable verbose output
-  python gen_disabled_report.py --input apps.json --output report.html --verbose
-        """,
-    )
-
-    # Input options
-    parser.add_argument('--input', type=str, help='Input JSON file containing apps data')
-
-    # Output options
-    parser.add_argument(
-        '--output',
-        type=str,
-        default='disabled_report.html',
-        help='Output report file path (default: disabled_report.html)',
-    )
-
-    # idf-build-apps find options
-    parser.add_argument(
-        '--temp-json',
-        type=str,
-        default='apps.json',
-        help='Temporary JSON file path for idf-build-apps find output (default: apps.json)',
-    )
-    parser.add_argument('--verbose', action='store_true', help='Enable verbose output')
-    parser.add_argument('--enable-preview-targets', action='store_true', help='Enable preview targets')
-
-    args = parser.parse_args()
-
-    try:
-        # Always run idf-build-apps find to get the most up-to-date data
-        print('Running idf-build-apps find to generate app list...')
-        input_file = run_idf_build_apps_find(
-            output_file=args.temp_json,
-            verbose=args.verbose,
-            enable_preview_targets=args.enable_preview_targets,
-        )
-
-        # Load application data
-        print(f'Loading apps from {input_file}...')
-        apps = load_apps_from_json(input_file)
-
-        # Add codeowners information
-        print('Adding codeowners information...')
-        idf_path = os.environ.get('IDF_PATH')
-        if not idf_path:
-            raise ValueError('IDF_PATH environment variable is not set')
-        codeowners_path = os.path.join(idf_path, '.gitlab', 'CODEOWNERS')
-        codeowners_mapping = parse_codeowners(codeowners_path)
-        app_to_owners_map = add_codeowners_to_apps(apps, codeowners_mapping)
-
-        # Generate report
-        print(f'Generating report to {args.output}...')
-        generate_disabled_report(apps, app_to_owners_map, args.output)
-
-        print(f'Report generated successfully: {args.output}')
-
-        return 0
-
-    except Exception as e:
-        print(f'Error: {e}')
-        if args.verbose:
-            import traceback
-
-            traceback.print_exc()
-        return 1
-
-
-if __name__ == '__main__':
-    sys.exit(main())

tools/ci/utils.sh

@@ -5,7 +5,14 @@ function add_ssh_keys() {
   mkdir -p ~/.ssh
   chmod 700 ~/.ssh
   echo -n "${key_string}" >~/.ssh/id_rsa_base64
-  base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 >~/.ssh/id_rsa
+  # Detect base64 implementation via --help output
+  if base64 --help 2>&1 | grep -q -- '--ignore-garbage'; then
+    # GNU coreutils base64
+    base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 >~/.ssh/id_rsa
+  else
+    # macOS/BSD base64 - requires stdin or -i flag
+    base64 --decode -i ~/.ssh/id_rsa_base64 -o ~/.ssh/id_rsa
+  fi
   chmod 600 ~/.ssh/id_rsa
 }