Mirrors/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2026-10-02 03:00:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

ci: apply common-scripts CI refactor

Browse Source
This commit is contained in:
Fu Hanxi
2026-09-03 20:25:12 +02:00
parent 0a86d7b8e7
commit 5fd30885b3
26 changed files with 236 additions and 1351 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/ci/check_build_test_rules.py
+9 -22
View File
@@ -8,7 +8,6 @@ import sys
from collections import defaultdict
from pathlib import Path
import yaml
from idf_build_apps import App
from idf_build_apps import find_apps
from idf_build_apps.constants import SUPPORTED_TARGETS
@@ -39,6 +38,14 @@ USUAL_TO_FORMAL = {
FORMAL_TO_USUAL = {v: k for k, v in USUAL_TO_FORMAL.items()}
# Targets under bringup: suppress check-test-scripts warnings for these.
# Reset to [] once bringup is complete.
BYPASS_CHECK_TEST_TARGETS: list[str] = [
'esp32h21',
'esp32h4',
'esp32c5',
]
def print_diff_table(
list1: list[str], list2: list[str], title1: str, title2: str, excluded: list[str] | None = None
@@ -298,21 +305,9 @@ if __name__ == '__main__':
readme_parser = action.add_parser('check-readmes')
readme_parser.add_argument('paths', nargs='+', help='check under paths')
readme_parser.add_argument(
'-c',
'--config',
default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'),
help='config file',
)
test_parser = action.add_parser('check-test-scripts')
test_parser.add_argument('paths', nargs='+', help='check under paths')
test_parser.add_argument(
'-c',
'--config',
default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'),
help='config file',
)
arg = parser.parse_args()
check_dirs = set()
@@ -342,14 +337,6 @@ if __name__ == '__main__':
os.path.join(IDF_PATH, 'tools', 'cmakev2', 'test'),
]
_bypass_targets: list[str] = []
if arg.config:
with open(arg.config) as fr:
configs = yaml.safe_load(fr)
if configs:
_bypass_targets = configs.get('bypass_check_test_targets') or []
os.environ.update(
{
'IDF_PATH': IDF_PATH,
@@ -367,5 +354,5 @@ if __name__ == '__main__':
check_test_scripts(
list(check_dirs),
_exclude_dirs,
bypass_targets=_bypass_targets,
bypass_targets=BYPASS_CHECK_TEST_TARGETS,
)
tools/ci/ci_get_mr_info.py
-154
View File
@@ -1,154 +0,0 @@
#!/usr/bin/env python
#
# internal use only for CI
# get latest MR information by source branch
#
# SPDX-FileCopyrightText: 2020-2025 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
#
import argparse
import logging
import os
import subprocess
import typing as t
from pathlib import Path
from gitlab_api import Gitlab
from idf_ci_utils import IDF_PATH
if t.TYPE_CHECKING:
from gitlab.v4.objects import ProjectCommit
from gitlab.v4.objects import ProjectMergeRequest
def _get_mr_obj(source_branch: str) -> t.Optional['ProjectMergeRequest']:
gl = Gitlab(os.getenv('CI_PROJECT_ID', 'espressif/esp-idf'))
if not gl.project:
return None
mrs = gl.project.mergerequests.list(state='opened', source_branch=source_branch)
if mrs:
return mrs[0] # one source branch can only have one opened MR at one moment
else:
return None
def get_mr_iid(source_branch: str) -> str:
mr = _get_mr_obj(source_branch)
if not mr:
return ''
else:
return str(mr.iid)
def get_mr_changed_files(source_branch: str) -> t.List[str]:
mr = _get_mr_obj(source_branch)
if not mr:
return []
git_output = subprocess.check_output(
['git', 'diff', '--name-only', '--diff-filter=d', f'origin/{mr.target_branch}...origin/{source_branch}']
).decode('utf8')
return [line.strip() for line in git_output.splitlines() if line.strip()]
def get_mr_commits(source_branch: str) -> t.List['ProjectCommit']:
mr = _get_mr_obj(source_branch)
if not mr:
return []
return list(mr.commits())
_COMPONENT_NAME_DIR_RECORDS = {}
def get_modified_component(filepath: str) -> t.Optional[str]:
"""Return the component name if the file is in a component directory, otherwise None."""
try:
f_path = Path(filepath).resolve().relative_to(IDF_PATH)
except ValueError: # not in IDF_PATH
return None
# skip md files, etc.
if f_path.suffix in ['.md', '.yml']:
return None
# skip test_apps files
if 'test_apps' in f_path.parts:
return None
component_parent_dirs = [f_path.parts[0]]
for part in f_path.parts[1:]:
if component_parent_dirs[-1] == 'components' or component_parent_dirs[-1].endswith('common_components'):
if part not in _COMPONENT_NAME_DIR_RECORDS:
logging.debug('Found component "%s" in path "%s"' % (part, component_parent_dirs))
_COMPONENT_NAME_DIR_RECORDS[part] = component_parent_dirs
elif _COMPONENT_NAME_DIR_RECORDS.get(part) != component_parent_dirs:
logging.debug(
'WARNING!!! Found component "%s" in path "%s" and "%s"'
% (part, component_parent_dirs, _COMPONENT_NAME_DIR_RECORDS.get(part))
)
return part
component_parent_dirs.append(part)
return None
def get_mr_components(
source_branch: t.Optional[str] = None, modified_files: t.Optional[t.List[str]] = None
) -> t.List[str]:
components: t.Set[str] = set()
if modified_files is None:
if not source_branch:
raise RuntimeError('--src-branch is required if --modified-files is not provided')
modified_files = get_mr_changed_files(source_branch)
for f in modified_files:
modified_component = get_modified_component(f)
if modified_component:
components.add(modified_component)
return list(components)
def _print_list(_list: t.List[str], separator: str = '\n') -> None:
print(separator.join(_list))
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Get the latest merge request info by pipeline')
actions = parser.add_subparsers(dest='action', help='info type', required=True)
common_args = argparse.ArgumentParser(add_help=False)
common_args.add_argument('--src-branch', help='source branch')
common_args.add_argument(
'--modified-files',
nargs='+',
help='space-separated list specifies the modified files. will be detected by --src-branch if not provided',
)
actions.add_parser('id', parents=[common_args])
actions.add_parser('commits', parents=[common_args])
actions.add_parser('components', parents=[common_args])
target = actions.add_parser('target_in_tags')
target.add_argument('tags', help='comma separated tags, e.g., esp32,generic')
args = parser.parse_args()
if args.action == 'id':
if not args.src_branch:
raise RuntimeError('--src-branch is required')
print(get_mr_iid(args.src_branch))
elif args.action == 'commits':
if not args.src_branch:
raise RuntimeError('--src-branch is required')
_print_list([commit.id for commit in get_mr_commits(args.src_branch)])
elif args.action == 'components':
_print_list(get_mr_components(args.src_branch, args.modified_files))
else:
raise NotImplementedError('not possible to get here')
tools/ci/deploy_docs.py
-217
View File
@@ -1,217 +0,0 @@
#!/usr/bin/env python3
#
# CI script to deploy docs to a webserver. Not useful outside of CI environment
#
#
# SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
#
import glob
import os
import os.path
import re
import stat
import subprocess
import sys
import tarfile
from typing import Any, List, Tuple
import packaging.version
def env(variable: str, default: str=None) -> str:
""" Shortcut to return the expanded version of an environment variable """
return os.path.expandvars(os.environ.get(variable, default) if default else os.environ[variable])
# import sanitize_version from the docs directory, shared with here
sys.path.append(os.path.join(env('IDF_PATH'), 'docs'))
from sanitize_version import sanitize_version # noqa
def main() -> None:
# if you get KeyErrors on the following lines, it's probably because you're not running in Gitlab CI
git_ver = env('GIT_VER') # output of git describe --always
ci_ver = env('CI_COMMIT_REF_NAME', git_ver) # branch or tag we're building for (used for 'release' & URL)
version = sanitize_version(ci_ver)
print('Git version: {}'.format(git_ver))
print('CI Version: {}'.format(ci_ver))
print('Deployment version: {}'.format(version))
if not version:
raise RuntimeError('A version is needed to deploy')
build_dir = env('DOCS_BUILD_DIR') # top-level local build dir, where docs have already been built
if not build_dir:
raise RuntimeError('Valid DOCS_BUILD_DIR is needed to deploy')
url_base = env('DOCS_DEPLOY_URL_BASE') # base for HTTP URLs, used to print the URL to the log after deploying
docs_server = env('DOCS_DEPLOY_SERVER') # ssh server to deploy to
docs_user = env('DOCS_DEPLOY_SERVER_USER')
docs_path = env('DOCS_DEPLOY_PATH') # filesystem path on DOCS_SERVER
if not docs_server:
raise RuntimeError('Valid DOCS_DEPLOY_SERVER is needed to deploy')
if not docs_user:
raise RuntimeError('Valid DOCS_DEPLOY_SERVER_USER is needed to deploy')
docs_server = '{}@{}'.format(docs_user, docs_server)
if not docs_path:
raise RuntimeError('Valid DOCS_DEPLOY_PATH is needed to deploy')
print('DOCS_DEPLOY_SERVER {} DOCS_DEPLOY_PATH {}'.format(docs_server, docs_path))
tarball_path, version_urls = build_doc_tarball(version, git_ver, build_dir)
deploy(version, tarball_path, docs_path, docs_server)
print('Docs URLs:')
doc_deploy_type = os.getenv('TYPE')
for vurl in version_urls:
language, _, target = vurl.split('/')
tag = '{}_{}'.format(language, target)
url = '{}/{}/index.html'.format(url_base, vurl) # (index.html needed for the preview server)
url = re.sub(r'([^:])//', r'\1/', url) # get rid of any // that isn't in the https:// part
print('[document {}][{}] {}'.format(doc_deploy_type, tag, url))
# note: it would be neater to use symlinks for stable, but because of the directory order
# (language first) it's kind of a pain to do on a remote server, so we just repeat the
# process but call the version 'stable' this time
if is_stable_version(version):
print('Deploying again as stable version...')
tarball_path, version_urls = build_doc_tarball('stable', git_ver, build_dir)
deploy('stable', tarball_path, docs_path, docs_server)
def deploy(version: str, tarball_path: str, docs_path: str, docs_server: str) -> None:
def run_ssh(commands: List) -> None:
""" Log into docs_server and run a sequence of commands using ssh """
print('Running ssh: {}'.format(commands))
subprocess.run(['ssh', '-o', 'BatchMode=yes', docs_server, '-x', ' && '.join(commands)], check=True)
# copy the version tarball to the server
run_ssh(['mkdir -p {}'.format(docs_path)])
print('Running scp {} to {}'.format(tarball_path, '{}:{}'.format(docs_server, docs_path)))
subprocess.run(['scp', '-B', tarball_path, '{}:{}'.format(docs_server, docs_path)], check=True)
tarball_name = os.path.basename(tarball_path)
run_ssh(['cd {}'.format(docs_path),
'rm -rf ./*/{}'.format(version), # remove any pre-existing docs matching this version
'tar -zxvf {}'.format(tarball_name), # untar the archive with the new docs
'rm {}'.format(tarball_name)])
# Note: deleting and then extracting the archive is a bit awkward for updating stable/latest/etc
# as the version will be invalid for a window of time. Better to do it atomically, but this is
# another thing made much more complex by the directory structure putting language before version...
def build_doc_tarball(version: str, git_ver: str, build_dir: str) -> Tuple[str, List]:
""" Make a tar.gz archive of the docs, in the directory structure used to deploy as
the given version """
version_paths = []
tarball_path = '{}/{}.tar.gz'.format(build_dir, version)
# find all the 'html/' directories under build_dir
html_dirs = glob.glob('{}/**/html/'.format(build_dir), recursive=True)
print('Found %d html directories' % len(html_dirs))
pdfs = glob.glob('{}/**/latex/build/*.pdf'.format(build_dir), recursive=True)
print('Found %d PDFs in latex directories' % len(pdfs))
# add symlink for stable and latest and adds them to PDF blob
symlinks = create_and_add_symlinks(version, git_ver, pdfs)
def not_sources_dir(ti: Any) -> Any:
print(type(ti))
""" Filter the _sources directories out of the tarballs """
if ti.name.endswith('/_sources'):
return None
ti.mode |= stat.S_IWGRP # make everything group-writeable
return ti
try:
os.remove(tarball_path)
except OSError:
pass
with tarfile.open(tarball_path, 'w:gz') as tarball:
for html_dir in html_dirs:
# html_dir has the form '<ignored>/<language>/<target>/html/'
target_dirname = os.path.dirname(os.path.dirname(html_dir))
target = os.path.basename(target_dirname)
language = os.path.basename(os.path.dirname(target_dirname))
# when deploying, we want the top-level directory layout 'language/version/target'
archive_path = '{}/{}/{}'.format(language, version, target)
print("Archiving '{}' as '{}'...".format(html_dir, archive_path))
tarball.add(html_dir, archive_path, filter=not_sources_dir)
version_paths.append(archive_path)
for pdf_path in pdfs:
# pdf_path has the form '<ignored>/<language>/<target>/latex/build'
latex_dirname = os.path.dirname(pdf_path)
pdf_filename = os.path.basename(pdf_path)
target_dirname = os.path.dirname(os.path.dirname(latex_dirname))
target = os.path.basename(target_dirname)
language = os.path.basename(os.path.dirname(target_dirname))
# when deploying, we want the layout 'language/version/target/pdf'
archive_path = '{}/{}/{}/{}'.format(language, version, target, pdf_filename)
print("Archiving '{}' as '{}'...".format(pdf_path, archive_path))
tarball.add(pdf_path, archive_path)
for symlink in symlinks:
os.unlink(symlink)
return (os.path.abspath(tarball_path), version_paths)
def create_and_add_symlinks(version: str, git_ver: str, pdfs: List) -> List:
""" Create symbolic links for PDFs for 'latest' and 'stable' releases """
symlinks = []
if 'stable' in version or 'latest' in version:
for pdf_path in pdfs:
symlink_path = pdf_path.replace(git_ver, version)
os.symlink(pdf_path, symlink_path)
symlinks.append(symlink_path)
pdfs.extend(symlinks)
print('Found %d PDFs in latex directories after adding symlink' % len(pdfs))
return symlinks
def is_stable_version(version: str) -> bool:
""" Heuristic for whether this is the latest stable release """
if not version.startswith('v'):
return False # branch name
if '-' in version:
return False # prerelease tag
git_out = subprocess.check_output(['git', 'tag', '-l']).decode('utf-8')
versions = [v.strip() for v in git_out.split('\n')]
versions = [v for v in versions if re.match(r'^v[\d\.]+$', v.strip())] # include vX.Y.Z only
versions_pack = [packaging.version.parse(v) for v in versions]
max_version = max(versions_pack)
if max_version.public != version[1:]:
print('Stable version is v{}. This version is {}.'.format(max_version.public, version))
return False
else:
print('This version {} is the stable version'.format(version))
return True
if __name__ == '__main__':
main()
tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py
+3 -3
View File
@@ -55,7 +55,7 @@ def main(output_filepath: str) -> None:
exclude_runner_tags_matching = []
for _tag in known_warnings_dict.get('no_runner_tags', []):
if '*' not in _tag:
exclude_runner_tags_set.add(_tag)
exclude_runner_tags_set.add(frozenset(_tag.split(',')))
else:
if res := _process_match_group(_tag):
exclude_runner_tags_matching.append(res)
@@ -66,7 +66,7 @@ def main(output_filepath: str) -> None:
if exclude_runner_tags := os.getenv('EXCLUDE_RUNNER_TAGS'):
for _tag in exclude_runner_tags.split(';'):
if '*' not in _tag:
exclude_runner_tags_set.add(_tag)
exclude_runner_tags_set.add(frozenset(_tag.split(',')))
else:
if res := _process_match_group(_tag):
exclude_runner_tags_matching.append(res)
@@ -75,7 +75,7 @@ def main(output_filepath: str) -> None:
additional_dict: dict[GroupKey, dict[str, t.Any]] = {}
for key, grouped_cases in cases.grouped_cases.items():
# skip test cases with no runner tags
if ','.join(sorted(key.runner_tags)) in exclude_runner_tags_set:
if frozenset(key.runner_tags) in exclude_runner_tags_set:
print(f'WARNING: excluding test cases with runner tags: {key.runner_tags}')
continue
tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml
+9 -1
View File
@@ -17,6 +17,9 @@
# Since the version 3.21 CMake passes source files and include dirs to ninja using absolute paths.
# Needed for pytest junit reports.
INSTALL_EXTRA_TOOLS: cmake
# Suppress Python warning messages during CI builds to keep logs clean and focused on errors.
# This prevents non-critical warnings (e.g., deprecation or resource warnings) from cluttering output.
PYTHONWARNINGS: "ignore"
needs:
- pipeline: $PARENT_PIPELINE_ID
job: generate_build_child_pipeline
@@ -40,7 +43,6 @@
- run_cmd idf-ci build run
--parallel-count ${CI_NODE_TOTAL:-1}
--parallel-index ${CI_NODE_INDEX:-1}
--modified-files ${MR_MODIFIED_FILES}
.dynamic_target_test_template:
image: $TARGET_TEST_ENV_IMAGE
@@ -56,6 +58,8 @@
needs:
- pipeline: $PARENT_PIPELINE_ID
job: pipeline_variables
- pipeline: $PARENT_PIPELINE_ID
job: snapshot_known_failure_cases
artifacts:
paths:
- XUNIT_RESULT*.xml
@@ -65,7 +69,11 @@
when: always
expire_in: 1 week
script:
# Merge the current known failure cases file with snapshot_known_failure_cases file captured at pipeline start
# to avoid retry failures caused by later updates to the known failure cases file.
- run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME}
- printf '\n' >> ${KNOWN_FAILURE_CASES_FILE_NAME}
- cat artifacts/${KNOWN_FAILURE_CASES_FILE_NAME}.freeze >> ${KNOWN_FAILURE_CASES_FILE_NAME}
# get runner env config file
- retry_failed git clone $TEST_ENV_CONFIG_REPO
- run_cmd python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs
tools/ci/exclude_check_tools_files.txt
-3
View File
@@ -9,17 +9,14 @@ tools/ci/check_*.txt
tools/ci/check_copyright_config.yaml
tools/ci/checkout_project_ref.py
tools/ci/ci_fetch_submodule.py
tools/ci/ci_get_mr_info.py
tools/ci/cleanup_ignore_lists.py
tools/ci/configure_ci_environment.sh
tools/ci/deploy_docs.py
tools/ci/dynamic_pipelines/**/*
tools/ci/envsubst.py
tools/ci/executable-list.txt
tools/ci/fix_empty_prototypes.sh
tools/ci/generate_rules.py
tools/ci/get-full-sources.sh
tools/ci/get_all_test_results.py
tools/ci/get_supported_examples.sh
tools/ci/gitlab_yaml_linter.py
tools/ci/idf_build_apps_dump_soc_caps.py
tools/ci/executable-list.txt
-1
View File
@@ -70,7 +70,6 @@ tools/ci/check_tools_files_patterns.py
tools/ci/check_type_comments.py
tools/ci/checkout_project_ref.py
tools/ci/cleanup_ignore_lists.py
tools/ci/deploy_docs.py
tools/ci/envsubst.py
tools/ci/fix_empty_prototypes.sh
tools/ci/generate_rules.py
tools/ci/generate_rules.py
+20 -26
View File
@@ -1,6 +1,6 @@
#!/usr/bin/env python
#
# SPDX-FileCopyrightText: 2021-2025 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import argparse
import inspect
@@ -12,22 +12,21 @@ from itertools import product
import yaml
from idf_ci_utils import IDF_PATH
from idf_ci_utils import GitlabYmlConfig
if t.TYPE_CHECKING:
import pygraphviz as pgv
def _list(str_or_list: t.Union[str, t.List]) -> t.List:
def _list(str_or_list: str | list) -> list:
if isinstance(str_or_list, str):
return [str_or_list]
elif isinstance(str_or_list, list):
return str_or_list
else:
raise ValueError('Wrong type: {}. Only supports str or list.'.format(type(str_or_list)))
raise ValueError(f'Wrong type: {type(str_or_list)}. Only supports str or list.')
def _format_nested_dict(_dict: t.Dict[str, t.Dict], f_tuple: t.Tuple[str, ...]) -> t.Dict[str, t.Dict]:
def _format_nested_dict(_dict: dict[str, dict], f_tuple: tuple[str, ...]) -> dict[str, dict]:
res = {}
for k, v in _dict.items():
k = k.split('__')[0]
@@ -41,7 +40,7 @@ def _format_nested_dict(_dict: t.Dict[str, t.Dict], f_tuple: t.Tuple[str, ...])
return res
def _format_nested_list(_list: t.List[str], f_tuple: t.Tuple[str, ...]) -> t.List[str]:
def _format_nested_list(_list: list[str], f_tuple: tuple[str, ...]) -> list[str]:
res = []
for item in _list:
if isinstance(item, list):
@@ -90,7 +89,6 @@ class RulesWriter:
self.cfg = self.expand_matrices()
self.rules = self.expand_rules()
self.yml_config = GitlabYmlConfig()
self.graph = None
def expand_matrices(self): # type: () -> dict
@@ -107,11 +105,11 @@ class RulesWriter:
deploy = v.get('deploy')
if deploy:
for item in _list(deploy):
res['{}-{}'.format(k, item)] = v
res[f'{k}-{item}'] = v
return res
@staticmethod
def _expand_matrix(name: str, cfg: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]:
def _expand_matrix(name: str, cfg: dict[str, t.Any]) -> dict[str, t.Any]:
"""
Expand matrix into multi keys
:param cfg: single rule dict
@@ -129,7 +127,7 @@ class RulesWriter:
res.update(_format_nested_dict(default, comb))
return res
def expand_rules(self) -> t.Dict[str, t.Dict[str, t.List[str]]]:
def expand_rules(self) -> dict[str, dict[str, list[str]]]:
res = defaultdict(lambda: defaultdict(set)) # type: dict[str, dict[str, set]]
for k, v in self.cfg.items():
if not v:
@@ -155,8 +153,8 @@ class RulesWriter:
if 'patterns' in v:
for _pat in _list(v['patterns']):
# Patterns must be pre-defined
if '.patterns-{}'.format(_pat) not in self.rules_cfg:
print('WARNING: pattern {} not exists'.format(_pat))
if f'.patterns-{_pat}' not in self.rules_cfg:
print(f'WARNING: pattern {_pat} not exists')
continue
res[item]['patterns'].add(_pat)
@@ -195,14 +193,10 @@ class RulesWriter:
if k.startswith('pattern'):
continue
if '.rules:' + k not in self.yml_config.used_templates:
print(f'WARNING: unused rule: {k}, skipping...')
continue
res.append(self.RULES_TEMPLATE.format(k, self._format_rule(k, v)))
return '\n\n'.join(res)
def _format_rule(self, name: str, cfg: t.Dict[str, t.Any]) -> str:
def _format_rule(self, name: str, cfg: dict[str, t.Any]) -> str:
_rules = [self.RULE_REVERT_BRANCH]
if name.endswith('-production'):
_rules.append(self.RULE_PROTECTED_PUSH)
@@ -216,21 +210,21 @@ class RulesWriter:
if f'.{specific_rule}' in self.rules_cfg:
_rules.append(self.SPECIFIC_RULE_TEMPLATE.format(specific_rule))
else:
print('WARNING: specific_rule {} not exists'.format(specific_rule))
print(f'WARNING: specific_rule {specific_rule} not exists')
for label in cfg['labels']:
_rules.append(self.RULE_LABEL_TEMPLATE.format(label))
for pattern in cfg['patterns']:
if '.patterns-{}'.format(pattern) in self.rules_cfg:
if f'.patterns-{pattern}' in self.rules_cfg:
_rules.append(self.RULE_PATTERN_TEMPLATE.format(pattern))
else:
print('WARNING: pattern {} not exists'.format(pattern))
print(f'WARNING: pattern {pattern} not exists')
return '\n'.join(_rules)
def update_rules_yml(self) -> bool:
with open(self.rules_yml) as fr:
file_str = fr.read()
auto_generate_str = '\n{}\n\n{}\n'.format(self.new_labels_str(), self.new_rules_str())
auto_generate_str = f'\n{self.new_labels_str()}\n\n{self.new_rules_str()}\n'
rest, marker, old = file_str.partition(self.AUTO_GENERATE_MARKER)
if old == auto_generate_str:
return False
@@ -246,7 +240,7 @@ PATTERN_COLOR = 'cyan'
RULE_COLOR = 'blue'
def build_graph(rules_dict: t.Dict[str, t.Dict[str, t.List[str]]]) -> 'pgv.AGraph':
def build_graph(rules_dict: dict[str, dict[str, list[str]]]) -> 'pgv.AGraph':
from pygraphviz import pgv
graph = pgv.AGraph(directed=True, rankdir='LR', concentrate=True)
@@ -263,13 +257,13 @@ def build_graph(rules_dict: t.Dict[str, t.Dict[str, t.List[str]]]) -> 'pgv.AGrap
labels = v.get('labels')
if labels:
for _label in labels:
graph.add_node('label:{}'.format(_label), color=LABEL_COLOR)
graph.add_edge('label:{}'.format(_label), k, color=LABEL_COLOR)
graph.add_node(f'label:{_label}', color=LABEL_COLOR)
graph.add_edge(f'label:{_label}', k, color=LABEL_COLOR)
patterns = v.get('patterns')
if patterns:
for _pat in patterns:
graph.add_node('pattern:{}'.format(_pat), color=PATTERN_COLOR)
graph.add_edge('pattern:{}'.format(_pat), k, color=PATTERN_COLOR)
graph.add_node(f'pattern:{_pat}', color=PATTERN_COLOR)
graph.add_edge(f'pattern:{_pat}', k, color=PATTERN_COLOR)
return graph
tools/ci/get_all_test_results.py
-84
View File
@@ -1,84 +0,0 @@
#!/usr/bin/env python
#
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
'''
Download artifacts from all test jobs
CI_PROJECT_DIR
└── TEST_RESULTS
├── <job_id>
│ └── XUNIT_RESULT.xml
├── <job_id>
│ └── XUNIT_RESULT.xml
├── <job_id>
│ └── TEST_LOGS
'''
import argparse
import json
import os
import shutil
import gitlab_api
CI_PROJECT_ID = int(os.getenv('CI_PROJECT_ID', ''))
CI_PIPELINE_ID = int(os.getenv('CI_PIPELINE_ID', ''))
IDF_PATH = os.getenv('IDF_PATH', '.')
TEST_RESULT_PATH = os.path.join(IDF_PATH, 'TEST_RESULTS')
def download_all_test_results(result_path: str, include_retried: bool = False) -> None:
if os.path.exists(result_path):
shutil.rmtree(result_path)
os.makedirs(result_path, exist_ok=True)
gitlab_inst = gitlab_api.Gitlab(CI_PROJECT_ID)
pipelines = gitlab_inst.project.pipelines.get(CI_PIPELINE_ID)
_include_retried = 'true' if include_retried else 'false'
jobs = pipelines.jobs.list(all=True, per_page=100, include_retried=_include_retried)
job_info_list = []
for job in jobs:
if job.stage in ['target_test', 'host_test']:
log_path = ''
if job.status not in ['success', 'failed']:
print('Job {}({}) is not finished'.format(job.id, job.name))
elif not hasattr(job, 'artifacts_file'):
print('Job {}({}) has no artifacts.'.format(job.id, job.name))
else:
log_path = os.path.join(result_path, 'job_{}'.format(job.id))
print('Downloading artifacts from: {}'.format(job.name))
os.makedirs(log_path, exist_ok=True)
gitlab_inst.download_artifacts(job.id, log_path)
job_info = {
'id': job.id,
'name': job.name,
'tag_list': job.tag_list,
'status': job.status,
'stage': job.stage,
'web_url': job.web_url,
'commit_url': job.commit['web_url'],
'log_path': log_path,
}
job_info_list.append(job_info)
with open(os.path.join(result_path, 'index.json'), 'w') as f:
f.write(json.dumps({'jobs': job_info_list}, indent=1, sort_keys=True))
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument('--path', '-p', default=TEST_RESULT_PATH, help='Path to save result files.')
parser.add_argument('--include_retried', action='store_true', help='Including retried jobs.')
args = parser.parse_args()
download_all_test_results(args.path, args.include_retried)
if __name__ == '__main__':
main()
tools/ci/gitlab_yaml_linter.py
+6 -30
View File
@@ -1,27 +1,27 @@
#!/usr/bin/env python
# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
"""
Check gitlab ci yaml files
"""
import argparse
import os
import typing as t
from functools import cached_property
from idf_ci_utils import get_submodule_dirs
from idf_ci_utils import GitlabYmlConfig
from idf_ci_utils import IDF_PATH
from idf_ci_utils import GitlabYmlConfig
from idf_ci_utils import get_submodule_dirs
class YmlLinter:
def __init__(self, yml_config: GitlabYmlConfig) -> None:
self.yml_config = yml_config
self._errors: t.List[str] = []
self._errors: list[str] = []
@cached_property
def lint_functions(self) -> t.List[str]:
def lint_functions(self) -> list[str]:
funcs = []
for func in dir(self):
if func.startswith('_lint_'):
@@ -43,17 +43,6 @@ class YmlLinter:
exit(exit_code)
# name it like _1_ to make it run first
def _lint_1_yml_parser(self) -> None:
for k, v in self.yml_config.config.items():
if (
k not in self.yml_config.global_keys
and k not in self.yml_config.anchors
and k not in self.yml_config.templates
and k not in self.yml_config.jobs
):
raise SystemExit(f'Parser incorrect. Key {k} not in global keys, anchors, templates, or jobs')
def _lint_default_values_artifacts(self) -> None:
defaults_artifacts = self.yml_config.default.get('artifacts', {})
@@ -79,19 +68,6 @@ class YmlLinter:
for item in undefined_patterns:
self._errors.append(f'undefined pattern {item}. Please add {item} to .patterns-submodule')
def _lint_gitlab_yml_templates(self) -> None:
unused_templates = self.yml_config.templates.keys() - self.yml_config.used_templates
for item in unused_templates:
# known unused ones
if item not in [
'.before_script:fetch:target_test', # used in dynamic pipeline
]:
self._errors.append(f'Unused template: {item}, please remove it')
undefined_templates = self.yml_config.used_templates - self.yml_config.templates.keys()
for item in undefined_templates:
self._errors.append(f'Undefined template: {item}')
def _lint_dependencies_and_needs(self) -> None:
"""
Use `dependencies: []` together with `needs: []` could cause missing artifacts issue.
tools/ci/idf_ci_utils.py
+94 -94
View File
@@ -124,35 +124,106 @@ class GitlabYmlConfig:
# avoid unused import in other pre-commit hooks
import yaml
all_config = dict()
root_yml = yaml.load(open(root_yml_filepath), Loader=yaml.FullLoader)
# expanding "include"
for item in root_yml.pop('include', []) or []:
if isinstance(item, dict):
if 'project' in item:
continue
elif 'local' in item:
item = item['local']
else:
continue
all_config.update(yaml.load(open(os.path.join(IDF_PATH, item)), Loader=yaml.FullLoader))
merged_yaml = self._compile_via_gitlab_api(root_yml_filepath)
all_config = yaml.load(merged_yaml, Loader=yaml.FullLoader) or {}
if 'default' in all_config:
self._defaults = all_config.pop('default')
self._config = all_config
# anchor is the string that will be reused in templates
self._anchor_keys: set[str] = set()
# template is a dict that will be extended
self._template_keys: set[str] = set()
self._used_template_keys: set[str] = set() # tracing the used templates
# job is a dict that will be executed
self._job_keys: set[str] = set()
def _inline_local_includes(self, root_yml_filepath: str) -> str:
"""
Recursively resolve `include: local` entries straight from disk (so uncommitted local
changes are always picked up -- CI runners also work off a disk checkout, so there's no
need to fetch a ref remotely via `ci_lint`'s `content_ref`/`ref`/`dry_run_ref` GET
params). `include: project` entries (including ones nested inside local files, e.g.
`.gitlab/ci/common.yml` including `templates/idf/common-scripts.yml`) are collected and
left in the final `include:` list, since those files live in another GitLab project and
can only be resolved remotely.
self.expand_extends()
A dedicated Loader/Dumper pair round-trips `!reference` tags as a marker list subclass,
since GitLab CI uses `!reference` which plain YAML doesn't know, and we need to parse
(to merge dicts, not just string-concat) then re-dump losslessly.
:param root_yml_filepath: path to the local root yml file to compile
:return: yml content with local includes resolved and merged
"""
import yaml
class _Loader(yaml.FullLoader):
pass
class _Dumper(yaml.Dumper):
pass
class _Reference(list):
pass
_Loader.add_constructor(
'!reference', lambda loader, node: _Reference(loader.construct_sequence(t.cast(yaml.SequenceNode, node)))
)
_Dumper.add_representer(_Reference, lambda dumper, data: dumper.represent_sequence('!reference', list(data)))
def resolve(yml_filepath: str) -> tuple[dict, list]:
with open(yml_filepath) as fr:
data = yaml.load(fr, Loader=_Loader) or {}
includes = to_list(data.pop('include', None))
merged: dict = {}
remaining_project_includes: list = []
for item in includes:
if isinstance(item, dict):
if 'project' in item:
remaining_project_includes.append(item)
continue
elif 'local' in item:
local_path = item['local']
else:
continue
elif isinstance(item, str):
local_path = item
else:
continue
sub_merged, sub_remaining = resolve(os.path.join(IDF_PATH, local_path.lstrip('/')))
merged.update(sub_merged)
remaining_project_includes.extend(sub_remaining)
# this file's own top-level keys override whatever its includes defined
merged.update(data)
return merged, remaining_project_includes
merged_config, project_includes = resolve(root_yml_filepath)
if project_includes:
merged_config['include'] = project_includes
return yaml.dump(merged_config, Dumper=_Dumper, sort_keys=False) # type: ignore
def _compile_via_gitlab_api(self, root_yml_filepath: str) -> str:
"""
Call the GitLab CI Lint API to get the fully compiled (all `include`s resolved) yml,
same as what `glab ci config compile` does. This replaces the old recursive local-only
parsing, since the project now includes configs from other projects as well.
:param root_yml_filepath: path to the local root yml file to compile
:return: merged (fully resolved) yml content as a string
"""
sys.path.insert(0, os.path.join(IDF_PATH, 'tools', 'ci', 'python_packages'))
import gitlab_api
content = self._inline_local_includes(root_yml_filepath)
gitlab_inst = gitlab_api.Gitlab()
project_id = os.getenv('CI_PROJECT_ID') or gitlab_inst.get_project_id('esp-idf', namespace='espressif')
project = gitlab_inst.gitlab_inst.projects.get(project_id, lazy=True)
lint_result = project.ci_lint.create({'content': content})
if not lint_result.valid:
raise RuntimeError(f'Failed to compile {root_yml_filepath} via GitLab CI Lint API: {lint_result.errors}')
return lint_result.merged_yaml # type: ignore
@property
def default(self) -> dict[str, t.Any]:
@@ -166,80 +237,9 @@ class GitlabYmlConfig:
def global_keys(self) -> list[str]:
return ['default', 'include', 'workflow', 'variables', 'stages']
@cached_property
def anchors(self) -> dict[str, t.Any]:
return {k: v for k, v in self.config.items() if k in self._anchor_keys}
@cached_property
def jobs(self) -> dict[str, t.Any]:
return {k: v for k, v in self.config.items() if k in self._job_keys}
@cached_property
def templates(self) -> dict[str, t.Any]:
return {k: v for k, v in self.config.items() if k in self._template_keys}
@cached_property
def used_templates(self) -> set[str]:
return self._used_template_keys
def expand_extends(self) -> None:
"""
expand the `extends` key in-place.
"""
for k, v in self.config.items():
if k in self.global_keys:
continue
if isinstance(v, str | list):
self._anchor_keys.add(k)
elif k.startswith('.if-'):
self._anchor_keys.add(k)
elif k.startswith('.'):
self._template_keys.add(k)
elif isinstance(v, dict):
self._job_keys.add(k)
else:
raise ValueError(f'Unknown type for key {k} with value {v}')
# no need to expand anchor
# expand template first
for k in self._template_keys:
self._expand_extends(k)
# expand job
for k in self._job_keys:
self._expand_extends(k)
def _merge_dict(self, d1: dict[str, t.Any], d2: dict[str, t.Any]) -> t.Any:
for k, v in d2.items():
if k in d1:
if isinstance(v, dict) and isinstance(d1[k], dict):
d1[k] = self._merge_dict(d1[k], v)
else:
d1[k] = v
else:
d1[k] = v
return d1
def _expand_extends(self, name: str) -> dict[str, t.Any]:
extends = to_list(self.config[name].pop('extends', None))
if not extends:
return self.config[name] # type: ignore
original_d = self.config[name].copy()
d = {}
while extends:
self._used_template_keys.update(extends) # for tracking
for i in extends:
d.update(self._expand_extends(i))
extends = to_list(self.config[name].pop('extends', None))
self.config[name] = self._merge_dict(d, original_d)
return self.config[name] # type: ignore
return {k: v for k, v in self.config.items() if not k.startswith('.') and k not in self.global_keys}
def idf_relpath(p: str) -> str:
tools/ci/idf_pytest/plugin.py
-50
View File
@@ -9,7 +9,6 @@ import pytest
import yaml
from _pytest.config import Config
from _pytest.python import Function
from _pytest.python import Metafunc
from _pytest.runner import CallInfo
from dynamic_pipelines.constants import KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH
from idf_ci import IdfPytestPlugin
@@ -115,55 +114,6 @@ class IdfLocalPlugin:
return item.callspec.params.get(key, default) or default
@staticmethod
def _has_parametrized_arg(metafunc: Metafunc, arg_name: str) -> bool:
for marker in metafunc.definition.iter_markers(name='parametrize'):
if not marker.args:
continue
argnames = marker.args[0]
if isinstance(argnames, str):
names = [name.strip() for name in argnames.split(',')]
else:
names = list(argnames)
if arg_name in names:
return True
for callspec in getattr(metafunc, '_calls', []):
if arg_name in callspec.params:
return True
return False
@staticmethod
def _is_linux_target_run(config: Config) -> bool:
target = config.getoption('target')
if not target:
return False
if isinstance(target, str):
targets = [_t.strip() for _t in target.split(',')]
else:
targets = [str(_t).strip() for _t in target]
return 'linux' in targets
@pytest.hookimpl(trylast=True)
def pytest_generate_tests(self, metafunc: Metafunc) -> None:
if 'embedded_services' not in metafunc.fixturenames:
return
if self._has_parametrized_arg(metafunc, 'embedded_services'):
return
if metafunc.definition.get_closest_marker('qemu') is not None:
metafunc.parametrize('embedded_services', ['idf,qemu'], indirect=True)
return
if self._is_linux_target_run(metafunc.config):
metafunc.parametrize('embedded_services', ['idf'], indirect=True)
@pytest.hookimpl(wrapper=True)
def pytest_collection_modifyitems(self, config: Config, items: list[Function]) -> t.Generator[None, None, None]:
yield # throw it back to idf-ci
tools/ci/python_packages/gitlab_api.py
+42 -51
View File
@@ -1,4 +1,4 @@
# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import argparse
import logging
@@ -9,13 +9,9 @@ import tarfile
import tempfile
import time
import zipfile
from collections.abc import Callable
from functools import wraps
from typing import Any
from typing import Callable
from typing import Dict
from typing import List
from typing import Optional
from typing import Union
import gitlab
@@ -36,7 +32,7 @@ def retry(func: TR) -> TR:
while True:
try:
res = func(self, *args, **kwargs)
except (IOError, EOFError, gitlab.exceptions.GitlabError) as e:
except (OSError, EOFError, gitlab.exceptions.GitlabError) as e:
if isinstance(e, gitlab.exceptions.GitlabError):
if e.response_code == 500:
# retry on this error
@@ -52,9 +48,11 @@ def retry(func: TR) -> TR:
raise e # get out of the loop
else:
logging.warning(
'Network failure in {}, retrying ({})'.format(getattr(func, '__name__', '(unknown callable)'),
retried))
time.sleep(2 ** retried) # wait a bit more after each retry
'Network failure in {}, retrying ({})'.format(
getattr(func, '__name__', '(unknown callable)'), retried
)
)
time.sleep(2**retried) # wait a bit more after each retry
continue
else:
break
@@ -63,26 +61,25 @@ def retry(func: TR) -> TR:
return wrapper
class Gitlab(object):
class Gitlab:
JOB_NAME_PATTERN = re.compile(r'(\w+)(\s+(\d+)/(\d+))?')
DOWNLOAD_ERROR_MAX_RETRIES = 3
DEFAULT_BUILD_CHILD_PIPELINE_NAME = 'Build Child Pipeline'
def __init__(self, project_id: Union[int, str, None] = None):
def __init__(self, project_id: int | str | None = None):
config_data_from_env = os.getenv('PYTHON_GITLAB_CONFIG')
if config_data_from_env:
# prefer to load config from env variable
with tempfile.NamedTemporaryFile('w', delete=False) as temp_file:
temp_file.write(config_data_from_env)
config_files = [temp_file.name] # type: Optional[List[str]]
config_files: list[str] | None = [temp_file.name]
else:
# otherwise try to use config file at local filesystem
config_files = None
self._init_gitlab_inst(project_id, config_files)
@retry
def _init_gitlab_inst(self, project_id: Optional[int], config_files: Optional[List[str]]) -> None:
def _init_gitlab_inst(self, project_id: int | None, config_files: list[str] | None) -> None:
gitlab_id = os.getenv('LOCAL_GITLAB_HTTPS_HOST') # if None, will use the default gitlab server
self.gitlab_inst = gitlab.Gitlab.from_config(gitlab_id=gitlab_id, config_files=config_files)
@@ -109,7 +106,7 @@ class Gitlab(object):
self.project = None
@retry
def get_project_id(self, name: str, namespace: Optional[str] = None) -> int:
def get_project_id(self, name: str, namespace: str | None = None) -> int:
"""
search project ID by name
@@ -117,7 +114,7 @@ class Gitlab(object):
:param namespace: namespace to match when we have multiple project with same name
:return: project ID
"""
projects = self.gitlab_inst.projects.list(search=name)
projects = self.gitlab_inst.projects.list(search=name, get_all=True)
res = []
for project in projects:
if namespace is None:
@@ -152,7 +149,7 @@ class Gitlab(object):
archive_file.extractall(destination)
@retry
def download_artifact(self, job_id: int, artifact_path: List[str], destination: Optional[str] = None) -> List[bytes]:
def download_artifact(self, job_id: int, artifact_path: list[str], destination: str | None = None) -> list[bytes]:
"""
download specific path of job artifacts and extract to destination.
@@ -169,7 +166,7 @@ class Gitlab(object):
try:
data = job.artifact(a_path) # type: bytes
except gitlab.GitlabGetError as e:
logging.error("Failed to download '{}' from job {}".format(a_path, job_id))
logging.error(f"Failed to download '{a_path}' from job {job_id}")
raise e
raw_data_list.append(data)
if destination:
@@ -185,7 +182,7 @@ class Gitlab(object):
return raw_data_list
@retry
def find_job_id(self, job_name: str, pipeline_id: Optional[str] = None, job_status: str = 'success') -> List[Dict]:
def find_job_id(self, job_name: str, pipeline_id: str | None = None, job_status: str = 'success') -> list[dict]:
"""
Get Job ID from job name of specific pipeline
@@ -208,8 +205,9 @@ class Gitlab(object):
return job_id_list
@retry
def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None,
cache_dir: Optional[str] = None) -> str:
def download_archive(
self, ref: str, destination: str, project_id: int | None = None, cache_dir: str | None = None
) -> str:
"""
Download archive of certain commit of a repository and extract to destination path
@@ -233,10 +231,11 @@ class Gitlab(object):
try:
project.repository_archive(sha=ref, streamed=True, action=fw.write)
except gitlab.GitlabGetError as e:
logging.error('Failed to archive from project {}'.format(project_id))
logging.error(f'Failed to archive from project {project_id}')
raise e
logging.info('Downloaded archive size: {:.03f}MB'.format(
float(os.path.getsize(local_archive_file)) / (1024 * 1024)))
logging.info(
f'Downloaded archive size: {float(os.path.getsize(local_archive_file)) / (1024 * 1024):.03f}MB'
)
return self.decompress_archive(local_archive_file, destination)
@@ -245,10 +244,10 @@ class Gitlab(object):
try:
project.repository_archive(sha=ref, streamed=True, action=temp_file.write)
except gitlab.GitlabGetError as e:
logging.error('Failed to archive from project {}'.format(project_id))
logging.error(f'Failed to archive from project {project_id}')
raise e
logging.info('Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(temp_file.name)) / (1024 * 1024)))
logging.info(f'Downloaded archive size: {float(os.path.getsize(temp_file.name)) / (1024 * 1024):.03f}MB')
return self.decompress_archive(temp_file.name, destination)
@@ -280,40 +279,32 @@ class Gitlab(object):
job = self.project.jobs.get(job_id)
return ','.join(job.tag_list)
def get_downstream_pipeline_ids(self, main_pipeline_id: int) -> List[int]:
def get_downstream_pipeline_ids(self, main_pipeline_id: int) -> list[int]:
"""
Retrieve the IDs of all downstream child pipelines for a given main pipeline.
Retrieve the IDs of all downstream child pipelines for a given main pipeline,
recursing through arbitrarily nested child pipelines.
:param main_pipeline_id: The ID of the main pipeline to start the search.
:return: A list of IDs of all downstream child pipelines.
:return: A list of IDs of all downstream child pipelines (all levels).
"""
bridge_pipeline_ids = []
child_pipeline_ids = []
child_pipeline_ids: list[int] = []
main_pipeline_bridges = self.project.pipelines.get(main_pipeline_id).bridges.list()
for bridge in main_pipeline_bridges:
pipeline_bridges = self.project.pipelines.get(main_pipeline_id).bridges.list()
for bridge in pipeline_bridges:
downstream_pipeline = bridge.attributes.get('downstream_pipeline')
if not downstream_pipeline:
continue
bridge_pipeline_ids.append(downstream_pipeline['id'])
for bridge_pipeline_id in bridge_pipeline_ids:
child_pipeline_ids.append(bridge_pipeline_id)
bridge_pipeline = self.project.pipelines.get(bridge_pipeline_id)
if not bridge_pipeline.name == self.DEFAULT_BUILD_CHILD_PIPELINE_NAME:
downstream_pipeline_id = downstream_pipeline.get('id')
if downstream_pipeline_id is None:
continue
child_bridges = bridge_pipeline.bridges.list()
for child_bridge in child_bridges:
downstream_child_pipeline = child_bridge.attributes.get('downstream_pipeline')
if not downstream_child_pipeline:
continue
child_pipeline_ids.append(downstream_child_pipeline.get('id'))
child_pipeline_ids.append(downstream_pipeline_id)
# recurse to collect further nested (grandchild+) pipelines
child_pipeline_ids.extend(self.get_downstream_pipeline_ids(downstream_pipeline_id))
return [pid for pid in child_pipeline_ids if pid is not None]
return child_pipeline_ids
def retry_failed_jobs(self, pipeline_id: int, retry_allowed_failures: bool = False) -> List[int]:
def retry_failed_jobs(self, pipeline_id: int, retry_allowed_failures: bool = False) -> list[int]:
"""
Retry failed jobs for a specific pipeline. Optionally include jobs marked as 'allowed failures'.
@@ -370,10 +361,10 @@ def main() -> None:
gitlab_inst.download_archive(args.ref, args.destination)
elif args.action == 'get_project_id':
ret = gitlab_inst.get_project_id(args.project_name)
print('project id: {}'.format(ret))
print(f'project id: {ret}')
elif args.action == 'retry_failed_jobs':
res = gitlab_inst.retry_failed_jobs(args.pipeline_id, args.retry_allowed_failures)
print('jobs retried successfully: {}'.format(res))
print(f'jobs retried successfully: {res}')
elif args.action == 'get_job_tags':
ret = gitlab_inst.get_job_tags(args.job_id)
print(ret)
tools/ci/utils.sh
-35
View File
@@ -164,38 +164,3 @@ function join_by {
printf %s "$f" "${@/#/$d}"
fi
}
function is_based_on_commits() {
# This function would accept space-separated args as multiple commits.
# The return value would be 0 if current HEAD is based on any of the specified commits.
#
# In our CI, we use environment variable $REQUIRED_ANCESTOR_COMMITS to declare the ancestor commits.
# Please remember to set one commit for each release branch.
commits=$*
if [[ -z $commits ]]; then
info "Not specifying commits that branches should be based on, skipping check..."
return 0
fi
commits_str="$(join_by " or " $commits)" # no doublequotes here, passing array
info "Checking if current branch is based on $commits_str..."
for i in $commits; do
if git merge-base --is-ancestor "$i" HEAD >/dev/null 2>&1; then
info "Current branch is based on $i"
return 0
else
info "Current branch is not based on $i"
fi
done
error "The base commit of your branch is too old."
error "The branch should be more recent than either of the following commits:"
error " $commits_str"
error "To fix the issue:"
error " - If your merge request is 'Draft', or has conflicts with the target branch, rebase it to the latest master or release branch"
error " - Otherwise, simply run a new pipeline."
return 1
}