From dbeab947600ea960af9ddaa49f55069de1b329fc Mon Sep 17 00:00:00 2001 From: Fu Hanxi Date: Thu, 9 Jul 2026 11:15:02 +0200 Subject: [PATCH] ci: remove unused code --- .gitlab/CODEOWNERS | 1 - .gitlab/ci/default-build-test-rules.yml | 19 --- .gitlab/ci/rules.yml | 16 -- .gitlab/ci/static-code-analysis.yml | 81 --------- sonar-project.properties | 24 --- tools/ci/check_build_test_rules.py | 29 +--- tools/ci/deploy_docs.py | 217 ------------------------ tools/ci/exclude_check_tools_files.txt | 1 - tools/ci/executable-list.txt | 1 - 9 files changed, 7 insertions(+), 382 deletions(-) delete mode 100644 .gitlab/ci/default-build-test-rules.yml delete mode 100644 sonar-project.properties delete mode 100755 tools/ci/deploy_docs.py diff --git a/.gitlab/CODEOWNERS b/.gitlab/CODEOWNERS index 623605c8bf3..cad6f2ce881 100644 --- a/.gitlab/CODEOWNERS +++ b/.gitlab/CODEOWNERS @@ -69,7 +69,6 @@ /pytest.ini @esp-idf-codeowners/ci /ruff.toml @esp-idf-codeowners/tools /sdkconfig.rename @esp-idf-codeowners/build-config -/sonar-project.properties @esp-idf-codeowners/ci # sort-order-reset diff --git a/.gitlab/ci/default-build-test-rules.yml b/.gitlab/ci/default-build-test-rules.yml deleted file mode 100644 index de96c1572b1..00000000000 --- a/.gitlab/ci/default-build-test-rules.yml +++ /dev/null @@ -1,19 +0,0 @@ -# this file support two keywords: -# - extra_default_build_targets: -# besides of the SUPPORTED_TARGETS in IDF, -# enable build for the specified targets by default as well. -# !!! DEPRECATED: use `additional_build_targets` in .idf_build_apps.toml instead -# -# - bypass_check_test_targets: -# suppress the check_build_test_rules check-test-script warnings for the specified targets -# -# This file should ONLY be used during bringup. Should be reset to empty after the bringup process - -bypass_check_test_targets: - - esp32h21 - -# -# These lines would -# - enable the README.md check for esp32c6. Don't forget to add the build jobs in .gitlab/ci/build.yml -# - disable the test script check with the manifest file. -# diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index b48481bfd79..a7c5a707820 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -13,13 +13,6 @@ - "components/**/Kconfig" - "components/**/CMakeLists.txt" -.patterns-python-files: &patterns-python-files - - ".gitlab/ci/static-code-analysis.yml" - - "**/*.py" - -.patterns-sonarqube-files: &patterns-sonarqube-files - - "tools/ci/sonar_exclude_list.txt" - .patterns-build_components: &patterns-build_components # components files except "test*/" "host*/" folders # ?? to include folders less than 4 characters @@ -183,22 +176,13 @@ .if-protected-check: &if-protected-check if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) || $CI_COMMIT_TAG =~ /^qa-test/' -.if-protected-deploy: &if-protected-deploy - if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "api")' - .if-protected-branch-push: &if-protected-branch-push # rules:changes always evaluates to true for new branch pipelines or when there is no Git push event if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/) && $CI_PIPELINE_SOURCE == "push"' -.if-qa-test-tag: &if-qa-test-tag - if: '$CI_COMMIT_TAG =~ /^qa-test/' - .if-dev-push: &if-dev-push if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")' -.if-schedule-nightly: &if-schedule-nightly - if: '$CI_PIPELINE_SOURCE == "schedule" && $INCLUDE_NIGHTLY_RUN == "1"' - .if-schedule-test-build-system-windows: &if-schedule-test-build-system-windows if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULED_BUILD_SYSTEM_TEST_WIN == "true"' diff --git a/.gitlab/ci/static-code-analysis.yml b/.gitlab/ci/static-code-analysis.yml index b3f7fafdcc1..dcc95428c54 100644 --- a/.gitlab/ci/static-code-analysis.yml +++ b/.gitlab/ci/static-code-analysis.yml @@ -27,84 +27,3 @@ gcc_static_analyzer: - echo "CONFIG_COMPILER_STATIC_ANALYZER=y" >> ${ANALYZING_APP}/sdkconfig.defaults - sed -i 's/.*MINIMAL_BUILD.*//g' ${ANALYZING_APP}/CMakeLists.txt - idf-build-apps build -p ${ANALYZING_APP} -# -## build stage -## Sonarqube related jobs put here for this reason: -## Here we have two jobs. code_quality_check and code_quality_report. -## -## code_quality_check will analyze the code changes between your MR and -## code repo stored in sonarqube server. The analysis result is only shown in -## the comments under this MR and won't be transferred to the server. -## -## code_quality_report will analyze and transfer both of the newly added code -## and the analysis result to the server. -## -## Put in the front to ensure that the newly merged code can be stored in -## sonarqube server ASAP, in order to avoid reporting unrelated code issues -#.sonar_scan_template: -# stage: build -# extends: .pre_check_template -# # full clone since this image does not support fetch --shallow-since-cutoff -# # shiny runners are used for full clone -# tags: [build, shiny] -# image: $SONARQUBE_SCANNER_IMAGE -# before_script: -# - source tools/ci/utils.sh -# - export PYTHONPATH="$CI_PROJECT_DIR/tools:$CI_PROJECT_DIR/tools/ci/python_packages:$PYTHONPATH" -# - fetch_submodules -# # Exclude the submodules, all paths ends with /** -# - submodules=$(get_all_submodules) -# # get all exclude paths specified in tools/ci/sonar_exclude_list.txt | ignore lines start with # | xargs | replace all to -# - custom_excludes=$(cat $CI_PROJECT_DIR/tools/ci/sonar_exclude_list.txt | grep -v '^#' | xargs | sed -e 's/ /,/g') -# # Exclude the report dir as well -# - export EXCLUSIONS="$custom_excludes,$submodules" -# - export SONAR_SCANNER_OPTS="-Xmx2048m" -# variables: -# GIT_DEPTH: 0 -# REPORT_PATTERN: clang_tidy_reports/**/*.txt -# artifacts: -# paths: -# - $REPORT_PATTERN -# expire_in: 1 week -# when: always -# dependencies: # Here is not a hard dependency relationship, could be skipped when only python files changed. so we do not use "needs" here. -# - clang_tidy_check -# -#code_quality_check: -# extends: -# - .sonar_scan_template -# - .rules:patterns:static-code-analysis-preview -# allow_failure: true # it's using exit code to indicate the code analysis result, -# # we don't want to block ci when critical issues founded -# script: -# - export CI_MERGE_REQUEST_COMMITS=$(python ${CI_PROJECT_DIR}/tools/ci/ci_get_mr_info.py commits --src-branch ${CI_COMMIT_REF_NAME} | tr '\n' ',') -# # test if this branch have merge request, if not, exit 0 -# - test -n "$CI_MERGE_REQUEST_IID" || exit 0 -# - test -n "$CI_MERGE_REQUEST_COMMITS" || exit 0 -# - sonar-scanner -# -Dsonar.analysis.mode=preview -# -Dsonar.branch.name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME -# -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN -# -Dsonar.exclusions=$EXCLUSIONS -# -Dsonar.gitlab.ci_merge_request_iid=$CI_MERGE_REQUEST_IID -# -Dsonar.gitlab.commit_sha=$CI_MERGE_REQUEST_COMMITS -# -Dsonar.gitlab.merge_request_discussion=true -# -Dsonar.gitlab.ref_name=$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME -# -Dsonar.host.url=$SONAR_HOST_URL -# -Dsonar.login=$SONAR_LOGIN -# -#code_quality_report: -# extends: -# - .sonar_scan_template -# - .rules:protected:check -# allow_failure: true # it's using exit code to indicate the code analysis result, -# # we don't want to block ci when critical issues founded -# script: -# - sonar-scanner -# -Dsonar.branch.name=$CI_COMMIT_REF_NAME -# -Dsonar.cxx.clangtidy.reportPath=$REPORT_PATTERN -# -Dsonar.exclusions=$EXCLUSIONS -# -Dsonar.gitlab.commit_sha=$PIPELINE_COMMIT_SHA -# -Dsonar.gitlab.ref_name=$CI_COMMIT_REF_NAME -# -Dsonar.host.url=$SONAR_HOST_URL -# -Dsonar.login=$SONAR_LOGIN diff --git a/sonar-project.properties b/sonar-project.properties deleted file mode 100644 index 1c9d5882fb3..00000000000 --- a/sonar-project.properties +++ /dev/null @@ -1,24 +0,0 @@ -sonar.projectKey=esp-idf -sonar.projectName=Espressif IoT Development Framework -sonar.projectVersion=4.4 - -sonar.sources=. -sonar.sourceEncoding=UTF-8 - -sonar.inclusions=\ - **/*.c,**/*.cpp,**/*.h,**/*.hpp,\ - **/*.py - -#################### -# sonar-cxx-plugin # -#################### - -sonar.cxx.includeDirectories=/usr/include -sonar.cxx.file.suffixes=.cxx,.cpp,.cc,.c,.hxx,.hpp,.hh,.h - -####################### -# sonar-gitlab-plugin # -####################### - -sonar.gitlab.failure_notification_mode=exit-code -sonar.gitlab.project_id=103 diff --git a/tools/ci/check_build_test_rules.py b/tools/ci/check_build_test_rules.py index 1c323a5a7af..56583894126 100755 --- a/tools/ci/check_build_test_rules.py +++ b/tools/ci/check_build_test_rules.py @@ -8,7 +8,6 @@ import sys from collections import defaultdict from pathlib import Path -import yaml from idf_build_apps import App from idf_build_apps import find_apps from idf_build_apps.constants import SUPPORTED_TARGETS @@ -40,6 +39,12 @@ USUAL_TO_FORMAL = { FORMAL_TO_USUAL = {v: k for k, v in USUAL_TO_FORMAL.items()} +# Targets under bringup: suppress check-test-scripts warnings for these. +# Reset to [] once bringup is complete. +BYPASS_CHECK_TEST_TARGETS: list[str] = [ + 'esp32h21', +] + def print_diff_table( list1: list[str], list2: list[str], title1: str, title2: str, excluded: list[str] | None = None @@ -299,21 +304,9 @@ if __name__ == '__main__': readme_parser = action.add_parser('check-readmes') readme_parser.add_argument('paths', nargs='+', help='check under paths') - readme_parser.add_argument( - '-c', - '--config', - default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'), - help='config file', - ) test_parser = action.add_parser('check-test-scripts') test_parser.add_argument('paths', nargs='+', help='check under paths') - test_parser.add_argument( - '-c', - '--config', - default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'), - help='config file', - ) arg = parser.parse_args() check_dirs = set() @@ -344,14 +337,6 @@ if __name__ == '__main__': os.path.join(IDF_PATH, 'tools', 'cmakev2', 'test'), ] - _bypass_targets: list[str] = [] - if arg.config: - with open(arg.config) as fr: - configs = yaml.safe_load(fr) - - if configs: - _bypass_targets = configs.get('bypass_check_test_targets') or [] - os.environ.update( { 'IDF_PATH': IDF_PATH, @@ -369,5 +354,5 @@ if __name__ == '__main__': check_test_scripts( list(check_dirs), _exclude_dirs, - bypass_targets=_bypass_targets, + bypass_targets=BYPASS_CHECK_TEST_TARGETS, ) diff --git a/tools/ci/deploy_docs.py b/tools/ci/deploy_docs.py deleted file mode 100755 index 954958e738d..00000000000 --- a/tools/ci/deploy_docs.py +++ /dev/null @@ -1,217 +0,0 @@ -#!/usr/bin/env python3 -# -# CI script to deploy docs to a webserver. Not useful outside of CI environment -# -# -# SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -# -import glob -import os -import os.path -import re -import stat -import subprocess -import sys -import tarfile -from typing import Any, List, Tuple - -import packaging.version - - -def env(variable: str, default: str=None) -> str: - """ Shortcut to return the expanded version of an environment variable """ - return os.path.expandvars(os.environ.get(variable, default) if default else os.environ[variable]) - - -# import sanitize_version from the docs directory, shared with here -sys.path.append(os.path.join(env('IDF_PATH'), 'docs')) -from sanitize_version import sanitize_version # noqa - - -def main() -> None: - # if you get KeyErrors on the following lines, it's probably because you're not running in Gitlab CI - git_ver = env('GIT_VER') # output of git describe --always - ci_ver = env('CI_COMMIT_REF_NAME', git_ver) # branch or tag we're building for (used for 'release' & URL) - - version = sanitize_version(ci_ver) - print('Git version: {}'.format(git_ver)) - print('CI Version: {}'.format(ci_ver)) - print('Deployment version: {}'.format(version)) - - if not version: - raise RuntimeError('A version is needed to deploy') - - build_dir = env('DOCS_BUILD_DIR') # top-level local build dir, where docs have already been built - - if not build_dir: - raise RuntimeError('Valid DOCS_BUILD_DIR is needed to deploy') - - url_base = env('DOCS_DEPLOY_URL_BASE') # base for HTTP URLs, used to print the URL to the log after deploying - - docs_server = env('DOCS_DEPLOY_SERVER') # ssh server to deploy to - docs_user = env('DOCS_DEPLOY_SERVER_USER') - docs_path = env('DOCS_DEPLOY_PATH') # filesystem path on DOCS_SERVER - - if not docs_server: - raise RuntimeError('Valid DOCS_DEPLOY_SERVER is needed to deploy') - - if not docs_user: - raise RuntimeError('Valid DOCS_DEPLOY_SERVER_USER is needed to deploy') - - docs_server = '{}@{}'.format(docs_user, docs_server) - - if not docs_path: - raise RuntimeError('Valid DOCS_DEPLOY_PATH is needed to deploy') - - print('DOCS_DEPLOY_SERVER {} DOCS_DEPLOY_PATH {}'.format(docs_server, docs_path)) - - tarball_path, version_urls = build_doc_tarball(version, git_ver, build_dir) - - deploy(version, tarball_path, docs_path, docs_server) - - print('Docs URLs:') - doc_deploy_type = os.getenv('TYPE') - for vurl in version_urls: - language, _, target = vurl.split('/') - tag = '{}_{}'.format(language, target) - url = '{}/{}/index.html'.format(url_base, vurl) # (index.html needed for the preview server) - url = re.sub(r'([^:])//', r'\1/', url) # get rid of any // that isn't in the https:// part - print('[document {}][{}] {}'.format(doc_deploy_type, tag, url)) - - # note: it would be neater to use symlinks for stable, but because of the directory order - # (language first) it's kind of a pain to do on a remote server, so we just repeat the - # process but call the version 'stable' this time - if is_stable_version(version): - print('Deploying again as stable version...') - tarball_path, version_urls = build_doc_tarball('stable', git_ver, build_dir) - deploy('stable', tarball_path, docs_path, docs_server) - - -def deploy(version: str, tarball_path: str, docs_path: str, docs_server: str) -> None: - def run_ssh(commands: List) -> None: - """ Log into docs_server and run a sequence of commands using ssh """ - print('Running ssh: {}'.format(commands)) - subprocess.run(['ssh', '-o', 'BatchMode=yes', docs_server, '-x', ' && '.join(commands)], check=True) - - # copy the version tarball to the server - run_ssh(['mkdir -p {}'.format(docs_path)]) - print('Running scp {} to {}'.format(tarball_path, '{}:{}'.format(docs_server, docs_path))) - subprocess.run(['scp', '-B', tarball_path, '{}:{}'.format(docs_server, docs_path)], check=True) - - tarball_name = os.path.basename(tarball_path) - - run_ssh(['cd {}'.format(docs_path), - 'rm -rf ./*/{}'.format(version), # remove any pre-existing docs matching this version - 'tar -zxvf {}'.format(tarball_name), # untar the archive with the new docs - 'rm {}'.format(tarball_name)]) - - # Note: deleting and then extracting the archive is a bit awkward for updating stable/latest/etc - # as the version will be invalid for a window of time. Better to do it atomically, but this is - # another thing made much more complex by the directory structure putting language before version... - - -def build_doc_tarball(version: str, git_ver: str, build_dir: str) -> Tuple[str, List]: - """ Make a tar.gz archive of the docs, in the directory structure used to deploy as - the given version """ - version_paths = [] - tarball_path = '{}/{}.tar.gz'.format(build_dir, version) - - # find all the 'html/' directories under build_dir - html_dirs = glob.glob('{}/**/html/'.format(build_dir), recursive=True) - print('Found %d html directories' % len(html_dirs)) - - pdfs = glob.glob('{}/**/latex/build/*.pdf'.format(build_dir), recursive=True) - print('Found %d PDFs in latex directories' % len(pdfs)) - - # add symlink for stable and latest and adds them to PDF blob - symlinks = create_and_add_symlinks(version, git_ver, pdfs) - - def not_sources_dir(ti: Any) -> Any: - print(type(ti)) - """ Filter the _sources directories out of the tarballs """ - if ti.name.endswith('/_sources'): - return None - - ti.mode |= stat.S_IWGRP # make everything group-writeable - return ti - - try: - os.remove(tarball_path) - except OSError: - pass - - with tarfile.open(tarball_path, 'w:gz') as tarball: - for html_dir in html_dirs: - # html_dir has the form '///html/' - target_dirname = os.path.dirname(os.path.dirname(html_dir)) - target = os.path.basename(target_dirname) - language = os.path.basename(os.path.dirname(target_dirname)) - - # when deploying, we want the top-level directory layout 'language/version/target' - archive_path = '{}/{}/{}'.format(language, version, target) - print("Archiving '{}' as '{}'...".format(html_dir, archive_path)) - tarball.add(html_dir, archive_path, filter=not_sources_dir) - version_paths.append(archive_path) - - for pdf_path in pdfs: - # pdf_path has the form '///latex/build' - latex_dirname = os.path.dirname(pdf_path) - pdf_filename = os.path.basename(pdf_path) - target_dirname = os.path.dirname(os.path.dirname(latex_dirname)) - target = os.path.basename(target_dirname) - language = os.path.basename(os.path.dirname(target_dirname)) - - # when deploying, we want the layout 'language/version/target/pdf' - archive_path = '{}/{}/{}/{}'.format(language, version, target, pdf_filename) - print("Archiving '{}' as '{}'...".format(pdf_path, archive_path)) - tarball.add(pdf_path, archive_path) - - for symlink in symlinks: - os.unlink(symlink) - - return (os.path.abspath(tarball_path), version_paths) - - -def create_and_add_symlinks(version: str, git_ver: str, pdfs: List) -> List: - """ Create symbolic links for PDFs for 'latest' and 'stable' releases """ - - symlinks = [] - if 'stable' in version or 'latest' in version: - for pdf_path in pdfs: - symlink_path = pdf_path.replace(git_ver, version) - os.symlink(pdf_path, symlink_path) - symlinks.append(symlink_path) - - pdfs.extend(symlinks) - print('Found %d PDFs in latex directories after adding symlink' % len(pdfs)) - - return symlinks - - -def is_stable_version(version: str) -> bool: - """ Heuristic for whether this is the latest stable release """ - if not version.startswith('v'): - return False # branch name - if '-' in version: - return False # prerelease tag - - git_out = subprocess.check_output(['git', 'tag', '-l']).decode('utf-8') - - versions = [v.strip() for v in git_out.split('\n')] - versions = [v for v in versions if re.match(r'^v[\d\.]+$', v.strip())] # include vX.Y.Z only - - versions_pack = [packaging.version.parse(v) for v in versions] - - max_version = max(versions_pack) - - if max_version.public != version[1:]: - print('Stable version is v{}. This version is {}.'.format(max_version.public, version)) - return False - else: - print('This version {} is the stable version'.format(version)) - return True - - -if __name__ == '__main__': - main() diff --git a/tools/ci/exclude_check_tools_files.txt b/tools/ci/exclude_check_tools_files.txt index de2410de972..e2653374988 100644 --- a/tools/ci/exclude_check_tools_files.txt +++ b/tools/ci/exclude_check_tools_files.txt @@ -11,7 +11,6 @@ tools/ci/ci_fetch_submodule.py tools/ci/ci_get_mr_info.py tools/ci/cleanup_ignore_lists.py tools/ci/configure_ci_environment.sh -tools/ci/deploy_docs.py tools/ci/doc_auto_translate.py tools/ci/dynamic_pipelines/**/* tools/ci/envsubst.py diff --git a/tools/ci/executable-list.txt b/tools/ci/executable-list.txt index fcd03914049..923ee7fd520 100644 --- a/tools/ci/executable-list.txt +++ b/tools/ci/executable-list.txt @@ -70,7 +70,6 @@ tools/ci/check_tools_files_patterns.py tools/ci/check_type_comments.py tools/ci/checkout_project_ref.py tools/ci/cleanup_ignore_lists.py -tools/ci/deploy_docs.py tools/ci/envsubst.py tools/ci/fix_empty_prototypes.sh tools/ci/generate_rules.py