mirror of
https://github.com/espressif/esp-idf.git
synced 2026-10-02 03:00:34 +03:00
ci: apply common-scripts CI refactor (v6.1)
This commit is contained in:
Fu Hanxi
@@ -8,7 +8,6 @@ import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
from idf_build_apps import App
|
||||
from idf_build_apps import find_apps
|
||||
from idf_build_apps.constants import SUPPORTED_TARGETS
|
||||
@@ -40,6 +39,12 @@ USUAL_TO_FORMAL = {
|
||||
|
||||
FORMAL_TO_USUAL = {v: k for k, v in USUAL_TO_FORMAL.items()}
|
||||
|
||||
# Targets under bringup: suppress check-test-scripts warnings for these.
|
||||
# Reset to [] once bringup is complete.
|
||||
BYPASS_CHECK_TEST_TARGETS: list[str] = [
|
||||
'esp32h21',
|
||||
]
|
||||
|
||||
|
||||
def print_diff_table(
|
||||
list1: list[str], list2: list[str], title1: str, title2: str, excluded: list[str] | None = None
|
||||
@@ -299,21 +304,9 @@ if __name__ == '__main__':
|
||||
|
||||
readme_parser = action.add_parser('check-readmes')
|
||||
readme_parser.add_argument('paths', nargs='+', help='check under paths')
|
||||
readme_parser.add_argument(
|
||||
'-c',
|
||||
'--config',
|
||||
default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'),
|
||||
help='config file',
|
||||
)
|
||||
|
||||
test_parser = action.add_parser('check-test-scripts')
|
||||
test_parser.add_argument('paths', nargs='+', help='check under paths')
|
||||
test_parser.add_argument(
|
||||
'-c',
|
||||
'--config',
|
||||
default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'),
|
||||
help='config file',
|
||||
)
|
||||
arg = parser.parse_args()
|
||||
|
||||
check_dirs = set()
|
||||
@@ -344,14 +337,6 @@ if __name__ == '__main__':
|
||||
os.path.join(IDF_PATH, 'tools', 'cmakev2', 'test'),
|
||||
]
|
||||
|
||||
_bypass_targets: list[str] = []
|
||||
if arg.config:
|
||||
with open(arg.config) as fr:
|
||||
configs = yaml.safe_load(fr)
|
||||
|
||||
if configs:
|
||||
_bypass_targets = configs.get('bypass_check_test_targets') or []
|
||||
|
||||
os.environ.update(
|
||||
{
|
||||
'IDF_PATH': IDF_PATH,
|
||||
@@ -369,5 +354,5 @@ if __name__ == '__main__':
|
||||
check_test_scripts(
|
||||
list(check_dirs),
|
||||
_exclude_dirs,
|
||||
bypass_targets=_bypass_targets,
|
||||
bypass_targets=BYPASS_CHECK_TEST_TARGETS,
|
||||
)
|
||||
|
||||
@@ -1,154 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# internal use only for CI
|
||||
# get latest MR information by source branch
|
||||
#
|
||||
# SPDX-FileCopyrightText: 2020-2025 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import typing as t
|
||||
from pathlib import Path
|
||||
|
||||
from gitlab_api import Gitlab
|
||||
from idf_ci_utils import IDF_PATH
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
from gitlab.v4.objects import ProjectCommit
|
||||
from gitlab.v4.objects import ProjectMergeRequest
|
||||
|
||||
|
||||
def _get_mr_obj(source_branch: str) -> t.Optional['ProjectMergeRequest']:
|
||||
gl = Gitlab(os.getenv('CI_PROJECT_ID', 'espressif/esp-idf'))
|
||||
if not gl.project:
|
||||
return None
|
||||
|
||||
mrs = gl.project.mergerequests.list(state='opened', source_branch=source_branch)
|
||||
if mrs:
|
||||
return mrs[0] # one source branch can only have one opened MR at one moment
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
def get_mr_iid(source_branch: str) -> str:
|
||||
mr = _get_mr_obj(source_branch)
|
||||
if not mr:
|
||||
return ''
|
||||
else:
|
||||
return str(mr.iid)
|
||||
|
||||
|
||||
def get_mr_changed_files(source_branch: str) -> t.List[str]:
|
||||
mr = _get_mr_obj(source_branch)
|
||||
if not mr:
|
||||
return []
|
||||
|
||||
git_output = subprocess.check_output(
|
||||
['git', 'diff', '--name-only', '--diff-filter=d', f'origin/{mr.target_branch}...origin/{source_branch}']
|
||||
).decode('utf8')
|
||||
|
||||
return [line.strip() for line in git_output.splitlines() if line.strip()]
|
||||
|
||||
|
||||
def get_mr_commits(source_branch: str) -> t.List['ProjectCommit']:
|
||||
mr = _get_mr_obj(source_branch)
|
||||
if not mr:
|
||||
return []
|
||||
|
||||
return list(mr.commits())
|
||||
|
||||
|
||||
_COMPONENT_NAME_DIR_RECORDS = {}
|
||||
|
||||
|
||||
def get_modified_component(filepath: str) -> t.Optional[str]:
|
||||
"""Return the component name if the file is in a component directory, otherwise None."""
|
||||
try:
|
||||
f_path = Path(filepath).resolve().relative_to(IDF_PATH)
|
||||
except ValueError: # not in IDF_PATH
|
||||
return None
|
||||
|
||||
# skip md files, etc.
|
||||
if f_path.suffix in ['.md', '.yml']:
|
||||
return None
|
||||
|
||||
# skip test_apps files
|
||||
if 'test_apps' in f_path.parts:
|
||||
return None
|
||||
|
||||
component_parent_dirs = [f_path.parts[0]]
|
||||
for part in f_path.parts[1:]:
|
||||
if component_parent_dirs[-1] == 'components' or component_parent_dirs[-1].endswith('common_components'):
|
||||
if part not in _COMPONENT_NAME_DIR_RECORDS:
|
||||
logging.debug('Found component "%s" in path "%s"' % (part, component_parent_dirs))
|
||||
_COMPONENT_NAME_DIR_RECORDS[part] = component_parent_dirs
|
||||
elif _COMPONENT_NAME_DIR_RECORDS.get(part) != component_parent_dirs:
|
||||
logging.debug(
|
||||
'WARNING!!! Found component "%s" in path "%s" and "%s"'
|
||||
% (part, component_parent_dirs, _COMPONENT_NAME_DIR_RECORDS.get(part))
|
||||
)
|
||||
|
||||
return part
|
||||
|
||||
component_parent_dirs.append(part)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def get_mr_components(
|
||||
source_branch: t.Optional[str] = None, modified_files: t.Optional[t.List[str]] = None
|
||||
) -> t.List[str]:
|
||||
components: t.Set[str] = set()
|
||||
if modified_files is None:
|
||||
if not source_branch:
|
||||
raise RuntimeError('--src-branch is required if --modified-files is not provided')
|
||||
|
||||
modified_files = get_mr_changed_files(source_branch)
|
||||
|
||||
for f in modified_files:
|
||||
modified_component = get_modified_component(f)
|
||||
if modified_component:
|
||||
components.add(modified_component)
|
||||
|
||||
return list(components)
|
||||
|
||||
|
||||
def _print_list(_list: t.List[str], separator: str = '\n') -> None:
|
||||
print(separator.join(_list))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description='Get the latest merge request info by pipeline')
|
||||
actions = parser.add_subparsers(dest='action', help='info type', required=True)
|
||||
|
||||
common_args = argparse.ArgumentParser(add_help=False)
|
||||
common_args.add_argument('--src-branch', help='source branch')
|
||||
common_args.add_argument(
|
||||
'--modified-files',
|
||||
nargs='+',
|
||||
help='space-separated list specifies the modified files. will be detected by --src-branch if not provided',
|
||||
)
|
||||
|
||||
actions.add_parser('id', parents=[common_args])
|
||||
actions.add_parser('commits', parents=[common_args])
|
||||
actions.add_parser('components', parents=[common_args])
|
||||
target = actions.add_parser('target_in_tags')
|
||||
target.add_argument('tags', help='comma separated tags, e.g., esp32,generic')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.action == 'id':
|
||||
if not args.src_branch:
|
||||
raise RuntimeError('--src-branch is required')
|
||||
print(get_mr_iid(args.src_branch))
|
||||
elif args.action == 'commits':
|
||||
if not args.src_branch:
|
||||
raise RuntimeError('--src-branch is required')
|
||||
_print_list([commit.id for commit in get_mr_commits(args.src_branch)])
|
||||
elif args.action == 'components':
|
||||
_print_list(get_mr_components(args.src_branch, args.modified_files))
|
||||
else:
|
||||
raise NotImplementedError('not possible to get here')
|
||||
@@ -1,217 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
#
|
||||
# CI script to deploy docs to a webserver. Not useful outside of CI environment
|
||||
#
|
||||
#
|
||||
# SPDX-FileCopyrightText: 2020-2022 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
import glob
|
||||
import os
|
||||
import os.path
|
||||
import re
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
from typing import Any, List, Tuple
|
||||
|
||||
import packaging.version
|
||||
|
||||
|
||||
def env(variable: str, default: str=None) -> str:
|
||||
""" Shortcut to return the expanded version of an environment variable """
|
||||
return os.path.expandvars(os.environ.get(variable, default) if default else os.environ[variable])
|
||||
|
||||
|
||||
# import sanitize_version from the docs directory, shared with here
|
||||
sys.path.append(os.path.join(env('IDF_PATH'), 'docs'))
|
||||
from sanitize_version import sanitize_version # noqa
|
||||
|
||||
|
||||
def main() -> None:
|
||||
# if you get KeyErrors on the following lines, it's probably because you're not running in Gitlab CI
|
||||
git_ver = env('GIT_VER') # output of git describe --always
|
||||
ci_ver = env('CI_COMMIT_REF_NAME', git_ver) # branch or tag we're building for (used for 'release' & URL)
|
||||
|
||||
version = sanitize_version(ci_ver)
|
||||
print('Git version: {}'.format(git_ver))
|
||||
print('CI Version: {}'.format(ci_ver))
|
||||
print('Deployment version: {}'.format(version))
|
||||
|
||||
if not version:
|
||||
raise RuntimeError('A version is needed to deploy')
|
||||
|
||||
build_dir = env('DOCS_BUILD_DIR') # top-level local build dir, where docs have already been built
|
||||
|
||||
if not build_dir:
|
||||
raise RuntimeError('Valid DOCS_BUILD_DIR is needed to deploy')
|
||||
|
||||
url_base = env('DOCS_DEPLOY_URL_BASE') # base for HTTP URLs, used to print the URL to the log after deploying
|
||||
|
||||
docs_server = env('DOCS_DEPLOY_SERVER') # ssh server to deploy to
|
||||
docs_user = env('DOCS_DEPLOY_SERVER_USER')
|
||||
docs_path = env('DOCS_DEPLOY_PATH') # filesystem path on DOCS_SERVER
|
||||
|
||||
if not docs_server:
|
||||
raise RuntimeError('Valid DOCS_DEPLOY_SERVER is needed to deploy')
|
||||
|
||||
if not docs_user:
|
||||
raise RuntimeError('Valid DOCS_DEPLOY_SERVER_USER is needed to deploy')
|
||||
|
||||
docs_server = '{}@{}'.format(docs_user, docs_server)
|
||||
|
||||
if not docs_path:
|
||||
raise RuntimeError('Valid DOCS_DEPLOY_PATH is needed to deploy')
|
||||
|
||||
print('DOCS_DEPLOY_SERVER {} DOCS_DEPLOY_PATH {}'.format(docs_server, docs_path))
|
||||
|
||||
tarball_path, version_urls = build_doc_tarball(version, git_ver, build_dir)
|
||||
|
||||
deploy(version, tarball_path, docs_path, docs_server)
|
||||
|
||||
print('Docs URLs:')
|
||||
doc_deploy_type = os.getenv('TYPE')
|
||||
for vurl in version_urls:
|
||||
language, _, target = vurl.split('/')
|
||||
tag = '{}_{}'.format(language, target)
|
||||
url = '{}/{}/index.html'.format(url_base, vurl) # (index.html needed for the preview server)
|
||||
url = re.sub(r'([^:])//', r'\1/', url) # get rid of any // that isn't in the https:// part
|
||||
print('[document {}][{}] {}'.format(doc_deploy_type, tag, url))
|
||||
|
||||
# note: it would be neater to use symlinks for stable, but because of the directory order
|
||||
# (language first) it's kind of a pain to do on a remote server, so we just repeat the
|
||||
# process but call the version 'stable' this time
|
||||
if is_stable_version(version):
|
||||
print('Deploying again as stable version...')
|
||||
tarball_path, version_urls = build_doc_tarball('stable', git_ver, build_dir)
|
||||
deploy('stable', tarball_path, docs_path, docs_server)
|
||||
|
||||
|
||||
def deploy(version: str, tarball_path: str, docs_path: str, docs_server: str) -> None:
|
||||
def run_ssh(commands: List) -> None:
|
||||
""" Log into docs_server and run a sequence of commands using ssh """
|
||||
print('Running ssh: {}'.format(commands))
|
||||
subprocess.run(['ssh', '-o', 'BatchMode=yes', docs_server, '-x', ' && '.join(commands)], check=True)
|
||||
|
||||
# copy the version tarball to the server
|
||||
run_ssh(['mkdir -p {}'.format(docs_path)])
|
||||
print('Running scp {} to {}'.format(tarball_path, '{}:{}'.format(docs_server, docs_path)))
|
||||
subprocess.run(['scp', '-B', tarball_path, '{}:{}'.format(docs_server, docs_path)], check=True)
|
||||
|
||||
tarball_name = os.path.basename(tarball_path)
|
||||
|
||||
run_ssh(['cd {}'.format(docs_path),
|
||||
'rm -rf ./*/{}'.format(version), # remove any pre-existing docs matching this version
|
||||
'tar -zxvf {}'.format(tarball_name), # untar the archive with the new docs
|
||||
'rm {}'.format(tarball_name)])
|
||||
|
||||
# Note: deleting and then extracting the archive is a bit awkward for updating stable/latest/etc
|
||||
# as the version will be invalid for a window of time. Better to do it atomically, but this is
|
||||
# another thing made much more complex by the directory structure putting language before version...
|
||||
|
||||
|
||||
def build_doc_tarball(version: str, git_ver: str, build_dir: str) -> Tuple[str, List]:
|
||||
""" Make a tar.gz archive of the docs, in the directory structure used to deploy as
|
||||
the given version """
|
||||
version_paths = []
|
||||
tarball_path = '{}/{}.tar.gz'.format(build_dir, version)
|
||||
|
||||
# find all the 'html/' directories under build_dir
|
||||
html_dirs = glob.glob('{}/**/html/'.format(build_dir), recursive=True)
|
||||
print('Found %d html directories' % len(html_dirs))
|
||||
|
||||
pdfs = glob.glob('{}/**/latex/build/*.pdf'.format(build_dir), recursive=True)
|
||||
print('Found %d PDFs in latex directories' % len(pdfs))
|
||||
|
||||
# add symlink for stable and latest and adds them to PDF blob
|
||||
symlinks = create_and_add_symlinks(version, git_ver, pdfs)
|
||||
|
||||
def not_sources_dir(ti: Any) -> Any:
|
||||
print(type(ti))
|
||||
""" Filter the _sources directories out of the tarballs """
|
||||
if ti.name.endswith('/_sources'):
|
||||
return None
|
||||
|
||||
ti.mode |= stat.S_IWGRP # make everything group-writeable
|
||||
return ti
|
||||
|
||||
try:
|
||||
os.remove(tarball_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
with tarfile.open(tarball_path, 'w:gz') as tarball:
|
||||
for html_dir in html_dirs:
|
||||
# html_dir has the form '<ignored>/<language>/<target>/html/'
|
||||
target_dirname = os.path.dirname(os.path.dirname(html_dir))
|
||||
target = os.path.basename(target_dirname)
|
||||
language = os.path.basename(os.path.dirname(target_dirname))
|
||||
|
||||
# when deploying, we want the top-level directory layout 'language/version/target'
|
||||
archive_path = '{}/{}/{}'.format(language, version, target)
|
||||
print("Archiving '{}' as '{}'...".format(html_dir, archive_path))
|
||||
tarball.add(html_dir, archive_path, filter=not_sources_dir)
|
||||
version_paths.append(archive_path)
|
||||
|
||||
for pdf_path in pdfs:
|
||||
# pdf_path has the form '<ignored>/<language>/<target>/latex/build'
|
||||
latex_dirname = os.path.dirname(pdf_path)
|
||||
pdf_filename = os.path.basename(pdf_path)
|
||||
target_dirname = os.path.dirname(os.path.dirname(latex_dirname))
|
||||
target = os.path.basename(target_dirname)
|
||||
language = os.path.basename(os.path.dirname(target_dirname))
|
||||
|
||||
# when deploying, we want the layout 'language/version/target/pdf'
|
||||
archive_path = '{}/{}/{}/{}'.format(language, version, target, pdf_filename)
|
||||
print("Archiving '{}' as '{}'...".format(pdf_path, archive_path))
|
||||
tarball.add(pdf_path, archive_path)
|
||||
|
||||
for symlink in symlinks:
|
||||
os.unlink(symlink)
|
||||
|
||||
return (os.path.abspath(tarball_path), version_paths)
|
||||
|
||||
|
||||
def create_and_add_symlinks(version: str, git_ver: str, pdfs: List) -> List:
|
||||
""" Create symbolic links for PDFs for 'latest' and 'stable' releases """
|
||||
|
||||
symlinks = []
|
||||
if 'stable' in version or 'latest' in version:
|
||||
for pdf_path in pdfs:
|
||||
symlink_path = pdf_path.replace(git_ver, version)
|
||||
os.symlink(pdf_path, symlink_path)
|
||||
symlinks.append(symlink_path)
|
||||
|
||||
pdfs.extend(symlinks)
|
||||
print('Found %d PDFs in latex directories after adding symlink' % len(pdfs))
|
||||
|
||||
return symlinks
|
||||
|
||||
|
||||
def is_stable_version(version: str) -> bool:
|
||||
""" Heuristic for whether this is the latest stable release """
|
||||
if not version.startswith('v'):
|
||||
return False # branch name
|
||||
if '-' in version:
|
||||
return False # prerelease tag
|
||||
|
||||
git_out = subprocess.check_output(['git', 'tag', '-l']).decode('utf-8')
|
||||
|
||||
versions = [v.strip() for v in git_out.split('\n')]
|
||||
versions = [v for v in versions if re.match(r'^v[\d\.]+$', v.strip())] # include vX.Y.Z only
|
||||
|
||||
versions_pack = [packaging.version.parse(v) for v in versions]
|
||||
|
||||
max_version = max(versions_pack)
|
||||
|
||||
if max_version.public != version[1:]:
|
||||
print('Stable version is v{}. This version is {}.'.format(max_version.public, version))
|
||||
return False
|
||||
else:
|
||||
print('This version {} is the stable version'.format(version))
|
||||
return True
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -43,7 +43,6 @@
|
||||
- run_cmd idf-ci build run
|
||||
--parallel-count ${CI_NODE_TOTAL:-1}
|
||||
--parallel-index ${CI_NODE_INDEX:-1}
|
||||
--modified-files ${MR_MODIFIED_FILES}
|
||||
|
||||
.dynamic_target_test_template:
|
||||
image: $TARGET_TEST_ENV_IMAGE
|
||||
|
||||
@@ -8,17 +8,14 @@ tools/ci/check_*.txt
|
||||
tools/ci/check_copyright_config.yaml
|
||||
tools/ci/checkout_project_ref.py
|
||||
tools/ci/ci_fetch_submodule.py
|
||||
tools/ci/ci_get_mr_info.py
|
||||
tools/ci/cleanup_ignore_lists.py
|
||||
tools/ci/configure_ci_environment.sh
|
||||
tools/ci/deploy_docs.py
|
||||
tools/ci/dynamic_pipelines/**/*
|
||||
tools/ci/envsubst.py
|
||||
tools/ci/executable-list.txt
|
||||
tools/ci/fix_empty_prototypes.sh
|
||||
tools/ci/generate_rules.py
|
||||
tools/ci/get-full-sources.sh
|
||||
tools/ci/get_all_test_results.py
|
||||
tools/ci/get_supported_examples.sh
|
||||
tools/ci/gitlab_yaml_linter.py
|
||||
tools/ci/idf_build_apps_dump_soc_caps.py
|
||||
|
||||
@@ -71,7 +71,6 @@ tools/ci/check_tools_files_patterns.py
|
||||
tools/ci/check_type_comments.py
|
||||
tools/ci/checkout_project_ref.py
|
||||
tools/ci/cleanup_ignore_lists.py
|
||||
tools/ci/deploy_docs.py
|
||||
tools/ci/envsubst.py
|
||||
tools/ci/fix_empty_prototypes.sh
|
||||
tools/ci/generate_rules.py
|
||||
|
||||
+20
-26
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# SPDX-FileCopyrightText: 2021-2025 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
import argparse
|
||||
import inspect
|
||||
@@ -12,22 +12,21 @@ from itertools import product
|
||||
|
||||
import yaml
|
||||
from idf_ci_utils import IDF_PATH
|
||||
from idf_ci_utils import GitlabYmlConfig
|
||||
|
||||
if t.TYPE_CHECKING:
|
||||
import pygraphviz as pgv
|
||||
|
||||
|
||||
def _list(str_or_list: t.Union[str, t.List]) -> t.List:
|
||||
def _list(str_or_list: str | list) -> list:
|
||||
if isinstance(str_or_list, str):
|
||||
return [str_or_list]
|
||||
elif isinstance(str_or_list, list):
|
||||
return str_or_list
|
||||
else:
|
||||
raise ValueError('Wrong type: {}. Only supports str or list.'.format(type(str_or_list)))
|
||||
raise ValueError(f'Wrong type: {type(str_or_list)}. Only supports str or list.')
|
||||
|
||||
|
||||
def _format_nested_dict(_dict: t.Dict[str, t.Dict], f_tuple: t.Tuple[str, ...]) -> t.Dict[str, t.Dict]:
|
||||
def _format_nested_dict(_dict: dict[str, dict], f_tuple: tuple[str, ...]) -> dict[str, dict]:
|
||||
res = {}
|
||||
for k, v in _dict.items():
|
||||
k = k.split('__')[0]
|
||||
@@ -41,7 +40,7 @@ def _format_nested_dict(_dict: t.Dict[str, t.Dict], f_tuple: t.Tuple[str, ...])
|
||||
return res
|
||||
|
||||
|
||||
def _format_nested_list(_list: t.List[str], f_tuple: t.Tuple[str, ...]) -> t.List[str]:
|
||||
def _format_nested_list(_list: list[str], f_tuple: tuple[str, ...]) -> list[str]:
|
||||
res = []
|
||||
for item in _list:
|
||||
if isinstance(item, list):
|
||||
@@ -90,7 +89,6 @@ class RulesWriter:
|
||||
self.cfg = self.expand_matrices()
|
||||
self.rules = self.expand_rules()
|
||||
|
||||
self.yml_config = GitlabYmlConfig()
|
||||
self.graph = None
|
||||
|
||||
def expand_matrices(self): # type: () -> dict
|
||||
@@ -107,11 +105,11 @@ class RulesWriter:
|
||||
deploy = v.get('deploy')
|
||||
if deploy:
|
||||
for item in _list(deploy):
|
||||
res['{}-{}'.format(k, item)] = v
|
||||
res[f'{k}-{item}'] = v
|
||||
return res
|
||||
|
||||
@staticmethod
|
||||
def _expand_matrix(name: str, cfg: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]:
|
||||
def _expand_matrix(name: str, cfg: dict[str, t.Any]) -> dict[str, t.Any]:
|
||||
"""
|
||||
Expand matrix into multi keys
|
||||
:param cfg: single rule dict
|
||||
@@ -129,7 +127,7 @@ class RulesWriter:
|
||||
res.update(_format_nested_dict(default, comb))
|
||||
return res
|
||||
|
||||
def expand_rules(self) -> t.Dict[str, t.Dict[str, t.List[str]]]:
|
||||
def expand_rules(self) -> dict[str, dict[str, list[str]]]:
|
||||
res = defaultdict(lambda: defaultdict(set)) # type: dict[str, dict[str, set]]
|
||||
for k, v in self.cfg.items():
|
||||
if not v:
|
||||
@@ -155,8 +153,8 @@ class RulesWriter:
|
||||
if 'patterns' in v:
|
||||
for _pat in _list(v['patterns']):
|
||||
# Patterns must be pre-defined
|
||||
if '.patterns-{}'.format(_pat) not in self.rules_cfg:
|
||||
print('WARNING: pattern {} not exists'.format(_pat))
|
||||
if f'.patterns-{_pat}' not in self.rules_cfg:
|
||||
print(f'WARNING: pattern {_pat} not exists')
|
||||
continue
|
||||
res[item]['patterns'].add(_pat)
|
||||
|
||||
@@ -195,14 +193,10 @@ class RulesWriter:
|
||||
if k.startswith('pattern'):
|
||||
continue
|
||||
|
||||
if '.rules:' + k not in self.yml_config.used_templates:
|
||||
print(f'WARNING: unused rule: {k}, skipping...')
|
||||
continue
|
||||
|
||||
res.append(self.RULES_TEMPLATE.format(k, self._format_rule(k, v)))
|
||||
return '\n\n'.join(res)
|
||||
|
||||
def _format_rule(self, name: str, cfg: t.Dict[str, t.Any]) -> str:
|
||||
def _format_rule(self, name: str, cfg: dict[str, t.Any]) -> str:
|
||||
_rules = [self.RULE_REVERT_BRANCH]
|
||||
if name.endswith('-production'):
|
||||
_rules.append(self.RULE_PROTECTED_PUSH)
|
||||
@@ -216,21 +210,21 @@ class RulesWriter:
|
||||
if f'.{specific_rule}' in self.rules_cfg:
|
||||
_rules.append(self.SPECIFIC_RULE_TEMPLATE.format(specific_rule))
|
||||
else:
|
||||
print('WARNING: specific_rule {} not exists'.format(specific_rule))
|
||||
print(f'WARNING: specific_rule {specific_rule} not exists')
|
||||
for label in cfg['labels']:
|
||||
_rules.append(self.RULE_LABEL_TEMPLATE.format(label))
|
||||
for pattern in cfg['patterns']:
|
||||
if '.patterns-{}'.format(pattern) in self.rules_cfg:
|
||||
if f'.patterns-{pattern}' in self.rules_cfg:
|
||||
_rules.append(self.RULE_PATTERN_TEMPLATE.format(pattern))
|
||||
else:
|
||||
print('WARNING: pattern {} not exists'.format(pattern))
|
||||
print(f'WARNING: pattern {pattern} not exists')
|
||||
return '\n'.join(_rules)
|
||||
|
||||
def update_rules_yml(self) -> bool:
|
||||
with open(self.rules_yml) as fr:
|
||||
file_str = fr.read()
|
||||
|
||||
auto_generate_str = '\n{}\n\n{}\n'.format(self.new_labels_str(), self.new_rules_str())
|
||||
auto_generate_str = f'\n{self.new_labels_str()}\n\n{self.new_rules_str()}\n'
|
||||
rest, marker, old = file_str.partition(self.AUTO_GENERATE_MARKER)
|
||||
if old == auto_generate_str:
|
||||
return False
|
||||
@@ -246,7 +240,7 @@ PATTERN_COLOR = 'cyan'
|
||||
RULE_COLOR = 'blue'
|
||||
|
||||
|
||||
def build_graph(rules_dict: t.Dict[str, t.Dict[str, t.List[str]]]) -> 'pgv.AGraph':
|
||||
def build_graph(rules_dict: dict[str, dict[str, list[str]]]) -> 'pgv.AGraph':
|
||||
from pygraphviz import pgv
|
||||
|
||||
graph = pgv.AGraph(directed=True, rankdir='LR', concentrate=True)
|
||||
@@ -263,13 +257,13 @@ def build_graph(rules_dict: t.Dict[str, t.Dict[str, t.List[str]]]) -> 'pgv.AGrap
|
||||
labels = v.get('labels')
|
||||
if labels:
|
||||
for _label in labels:
|
||||
graph.add_node('label:{}'.format(_label), color=LABEL_COLOR)
|
||||
graph.add_edge('label:{}'.format(_label), k, color=LABEL_COLOR)
|
||||
graph.add_node(f'label:{_label}', color=LABEL_COLOR)
|
||||
graph.add_edge(f'label:{_label}', k, color=LABEL_COLOR)
|
||||
patterns = v.get('patterns')
|
||||
if patterns:
|
||||
for _pat in patterns:
|
||||
graph.add_node('pattern:{}'.format(_pat), color=PATTERN_COLOR)
|
||||
graph.add_edge('pattern:{}'.format(_pat), k, color=PATTERN_COLOR)
|
||||
graph.add_node(f'pattern:{_pat}', color=PATTERN_COLOR)
|
||||
graph.add_edge(f'pattern:{_pat}', k, color=PATTERN_COLOR)
|
||||
|
||||
return graph
|
||||
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
|
||||
'''
|
||||
Download artifacts from all test jobs
|
||||
|
||||
CI_PROJECT_DIR
|
||||
└── TEST_RESULTS
|
||||
├── <job_id>
|
||||
│ └── XUNIT_RESULT.xml
|
||||
├── <job_id>
|
||||
│ └── XUNIT_RESULT.xml
|
||||
├── <job_id>
|
||||
│ └── TEST_LOGS
|
||||
|
||||
'''
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
|
||||
import gitlab_api
|
||||
|
||||
CI_PROJECT_ID = int(os.getenv('CI_PROJECT_ID', ''))
|
||||
CI_PIPELINE_ID = int(os.getenv('CI_PIPELINE_ID', ''))
|
||||
IDF_PATH = os.getenv('IDF_PATH', '.')
|
||||
TEST_RESULT_PATH = os.path.join(IDF_PATH, 'TEST_RESULTS')
|
||||
|
||||
|
||||
def download_all_test_results(result_path: str, include_retried: bool = False) -> None:
|
||||
if os.path.exists(result_path):
|
||||
shutil.rmtree(result_path)
|
||||
os.makedirs(result_path, exist_ok=True)
|
||||
|
||||
gitlab_inst = gitlab_api.Gitlab(CI_PROJECT_ID)
|
||||
pipelines = gitlab_inst.project.pipelines.get(CI_PIPELINE_ID)
|
||||
_include_retried = 'true' if include_retried else 'false'
|
||||
jobs = pipelines.jobs.list(all=True, per_page=100, include_retried=_include_retried)
|
||||
|
||||
job_info_list = []
|
||||
for job in jobs:
|
||||
if job.stage in ['target_test', 'host_test']:
|
||||
log_path = ''
|
||||
if job.status not in ['success', 'failed']:
|
||||
print('Job {}({}) is not finished'.format(job.id, job.name))
|
||||
elif not hasattr(job, 'artifacts_file'):
|
||||
print('Job {}({}) has no artifacts.'.format(job.id, job.name))
|
||||
else:
|
||||
log_path = os.path.join(result_path, 'job_{}'.format(job.id))
|
||||
print('Downloading artifacts from: {}'.format(job.name))
|
||||
os.makedirs(log_path, exist_ok=True)
|
||||
gitlab_inst.download_artifacts(job.id, log_path)
|
||||
|
||||
job_info = {
|
||||
'id': job.id,
|
||||
'name': job.name,
|
||||
'tag_list': job.tag_list,
|
||||
'status': job.status,
|
||||
'stage': job.stage,
|
||||
'web_url': job.web_url,
|
||||
'commit_url': job.commit['web_url'],
|
||||
'log_path': log_path,
|
||||
}
|
||||
job_info_list.append(job_info)
|
||||
|
||||
with open(os.path.join(result_path, 'index.json'), 'w') as f:
|
||||
f.write(json.dumps({'jobs': job_info_list}, indent=1, sort_keys=True))
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--path', '-p', default=TEST_RESULT_PATH, help='Path to save result files.')
|
||||
parser.add_argument('--include_retried', action='store_true', help='Including retried jobs.')
|
||||
args = parser.parse_args()
|
||||
|
||||
download_all_test_results(args.path, args.include_retried)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,27 +1,27 @@
|
||||
#!/usr/bin/env python
|
||||
# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""
|
||||
Check gitlab ci yaml files
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import typing as t
|
||||
from functools import cached_property
|
||||
|
||||
from idf_ci_utils import get_submodule_dirs
|
||||
from idf_ci_utils import GitlabYmlConfig
|
||||
from idf_ci_utils import IDF_PATH
|
||||
from idf_ci_utils import GitlabYmlConfig
|
||||
from idf_ci_utils import get_submodule_dirs
|
||||
|
||||
|
||||
class YmlLinter:
|
||||
def __init__(self, yml_config: GitlabYmlConfig) -> None:
|
||||
self.yml_config = yml_config
|
||||
|
||||
self._errors: t.List[str] = []
|
||||
self._errors: list[str] = []
|
||||
|
||||
@cached_property
|
||||
def lint_functions(self) -> t.List[str]:
|
||||
def lint_functions(self) -> list[str]:
|
||||
funcs = []
|
||||
for func in dir(self):
|
||||
if func.startswith('_lint_'):
|
||||
@@ -43,17 +43,6 @@ class YmlLinter:
|
||||
|
||||
exit(exit_code)
|
||||
|
||||
# name it like _1_ to make it run first
|
||||
def _lint_1_yml_parser(self) -> None:
|
||||
for k, v in self.yml_config.config.items():
|
||||
if (
|
||||
k not in self.yml_config.global_keys
|
||||
and k not in self.yml_config.anchors
|
||||
and k not in self.yml_config.templates
|
||||
and k not in self.yml_config.jobs
|
||||
):
|
||||
raise SystemExit(f'Parser incorrect. Key {k} not in global keys, anchors, templates, or jobs')
|
||||
|
||||
def _lint_default_values_artifacts(self) -> None:
|
||||
defaults_artifacts = self.yml_config.default.get('artifacts', {})
|
||||
|
||||
@@ -79,19 +68,6 @@ class YmlLinter:
|
||||
for item in undefined_patterns:
|
||||
self._errors.append(f'undefined pattern {item}. Please add {item} to .patterns-submodule')
|
||||
|
||||
def _lint_gitlab_yml_templates(self) -> None:
|
||||
unused_templates = self.yml_config.templates.keys() - self.yml_config.used_templates
|
||||
for item in unused_templates:
|
||||
# known unused ones
|
||||
if item not in [
|
||||
'.before_script:fetch:target_test', # used in dynamic pipeline
|
||||
]:
|
||||
self._errors.append(f'Unused template: {item}, please remove it')
|
||||
|
||||
undefined_templates = self.yml_config.used_templates - self.yml_config.templates.keys()
|
||||
for item in undefined_templates:
|
||||
self._errors.append(f'Undefined template: {item}')
|
||||
|
||||
def _lint_dependencies_and_needs(self) -> None:
|
||||
"""
|
||||
Use `dependencies: []` together with `needs: []` could cause missing artifacts issue.
|
||||
|
||||
+94
-94
@@ -125,35 +125,106 @@ class GitlabYmlConfig:
|
||||
# avoid unused import in other pre-commit hooks
|
||||
import yaml
|
||||
|
||||
all_config = dict()
|
||||
root_yml = yaml.load(open(root_yml_filepath), Loader=yaml.FullLoader)
|
||||
|
||||
# expanding "include"
|
||||
for item in root_yml.pop('include', []) or []:
|
||||
if isinstance(item, dict):
|
||||
if 'project' in item:
|
||||
continue
|
||||
elif 'local' in item:
|
||||
item = item['local']
|
||||
else:
|
||||
continue
|
||||
|
||||
all_config.update(yaml.load(open(os.path.join(IDF_PATH, item)), Loader=yaml.FullLoader))
|
||||
merged_yaml = self._compile_via_gitlab_api(root_yml_filepath)
|
||||
all_config = yaml.load(merged_yaml, Loader=yaml.FullLoader) or {}
|
||||
|
||||
if 'default' in all_config:
|
||||
self._defaults = all_config.pop('default')
|
||||
|
||||
self._config = all_config
|
||||
|
||||
# anchor is the string that will be reused in templates
|
||||
self._anchor_keys: set[str] = set()
|
||||
# template is a dict that will be extended
|
||||
self._template_keys: set[str] = set()
|
||||
self._used_template_keys: set[str] = set() # tracing the used templates
|
||||
# job is a dict that will be executed
|
||||
self._job_keys: set[str] = set()
|
||||
def _inline_local_includes(self, root_yml_filepath: str) -> str:
|
||||
"""
|
||||
Recursively resolve `include: local` entries straight from disk (so uncommitted local
|
||||
changes are always picked up -- CI runners also work off a disk checkout, so there's no
|
||||
need to fetch a ref remotely via `ci_lint`'s `content_ref`/`ref`/`dry_run_ref` GET
|
||||
params). `include: project` entries (including ones nested inside local files, e.g.
|
||||
`.gitlab/ci/common.yml` including `templates/idf/common-scripts.yml`) are collected and
|
||||
left in the final `include:` list, since those files live in another GitLab project and
|
||||
can only be resolved remotely.
|
||||
|
||||
self.expand_extends()
|
||||
A dedicated Loader/Dumper pair round-trips `!reference` tags as a marker list subclass,
|
||||
since GitLab CI uses `!reference` which plain YAML doesn't know, and we need to parse
|
||||
(to merge dicts, not just string-concat) then re-dump losslessly.
|
||||
|
||||
:param root_yml_filepath: path to the local root yml file to compile
|
||||
:return: yml content with local includes resolved and merged
|
||||
"""
|
||||
import yaml
|
||||
|
||||
class _Loader(yaml.FullLoader):
|
||||
pass
|
||||
|
||||
class _Dumper(yaml.Dumper):
|
||||
pass
|
||||
|
||||
class _Reference(list):
|
||||
pass
|
||||
|
||||
_Loader.add_constructor(
|
||||
'!reference', lambda loader, node: _Reference(loader.construct_sequence(t.cast(yaml.SequenceNode, node)))
|
||||
)
|
||||
_Dumper.add_representer(_Reference, lambda dumper, data: dumper.represent_sequence('!reference', list(data)))
|
||||
|
||||
def resolve(yml_filepath: str) -> tuple[dict, list]:
|
||||
with open(yml_filepath) as fr:
|
||||
data = yaml.load(fr, Loader=_Loader) or {}
|
||||
|
||||
includes = to_list(data.pop('include', None))
|
||||
|
||||
merged: dict = {}
|
||||
remaining_project_includes: list = []
|
||||
for item in includes:
|
||||
if isinstance(item, dict):
|
||||
if 'project' in item:
|
||||
remaining_project_includes.append(item)
|
||||
continue
|
||||
elif 'local' in item:
|
||||
local_path = item['local']
|
||||
else:
|
||||
continue
|
||||
elif isinstance(item, str):
|
||||
local_path = item
|
||||
else:
|
||||
continue
|
||||
|
||||
sub_merged, sub_remaining = resolve(os.path.join(IDF_PATH, local_path.lstrip('/')))
|
||||
merged.update(sub_merged)
|
||||
remaining_project_includes.extend(sub_remaining)
|
||||
|
||||
# this file's own top-level keys override whatever its includes defined
|
||||
merged.update(data)
|
||||
return merged, remaining_project_includes
|
||||
|
||||
merged_config, project_includes = resolve(root_yml_filepath)
|
||||
if project_includes:
|
||||
merged_config['include'] = project_includes
|
||||
|
||||
return yaml.dump(merged_config, Dumper=_Dumper, sort_keys=False) # type: ignore
|
||||
|
||||
def _compile_via_gitlab_api(self, root_yml_filepath: str) -> str:
|
||||
"""
|
||||
Call the GitLab CI Lint API to get the fully compiled (all `include`s resolved) yml,
|
||||
same as what `glab ci config compile` does. This replaces the old recursive local-only
|
||||
parsing, since the project now includes configs from other projects as well.
|
||||
|
||||
:param root_yml_filepath: path to the local root yml file to compile
|
||||
:return: merged (fully resolved) yml content as a string
|
||||
"""
|
||||
sys.path.insert(0, os.path.join(IDF_PATH, 'tools', 'ci', 'python_packages'))
|
||||
import gitlab_api
|
||||
|
||||
content = self._inline_local_includes(root_yml_filepath)
|
||||
|
||||
gitlab_inst = gitlab_api.Gitlab()
|
||||
project_id = os.getenv('CI_PROJECT_ID') or gitlab_inst.get_project_id('esp-idf', namespace='espressif')
|
||||
project = gitlab_inst.gitlab_inst.projects.get(project_id, lazy=True)
|
||||
|
||||
lint_result = project.ci_lint.create({'content': content})
|
||||
if not lint_result.valid:
|
||||
raise RuntimeError(f'Failed to compile {root_yml_filepath} via GitLab CI Lint API: {lint_result.errors}')
|
||||
|
||||
return lint_result.merged_yaml # type: ignore
|
||||
|
||||
@property
|
||||
def default(self) -> dict[str, t.Any]:
|
||||
@@ -167,80 +238,9 @@ class GitlabYmlConfig:
|
||||
def global_keys(self) -> list[str]:
|
||||
return ['default', 'include', 'workflow', 'variables', 'stages']
|
||||
|
||||
@cached_property
|
||||
def anchors(self) -> dict[str, t.Any]:
|
||||
return {k: v for k, v in self.config.items() if k in self._anchor_keys}
|
||||
|
||||
@cached_property
|
||||
def jobs(self) -> dict[str, t.Any]:
|
||||
return {k: v for k, v in self.config.items() if k in self._job_keys}
|
||||
|
||||
@cached_property
|
||||
def templates(self) -> dict[str, t.Any]:
|
||||
return {k: v for k, v in self.config.items() if k in self._template_keys}
|
||||
|
||||
@cached_property
|
||||
def used_templates(self) -> set[str]:
|
||||
return self._used_template_keys
|
||||
|
||||
def expand_extends(self) -> None:
|
||||
"""
|
||||
expand the `extends` key in-place.
|
||||
"""
|
||||
for k, v in self.config.items():
|
||||
if k in self.global_keys:
|
||||
continue
|
||||
|
||||
if isinstance(v, str | list):
|
||||
self._anchor_keys.add(k)
|
||||
elif k.startswith('.if-'):
|
||||
self._anchor_keys.add(k)
|
||||
elif k.startswith('.'):
|
||||
self._template_keys.add(k)
|
||||
elif isinstance(v, dict):
|
||||
self._job_keys.add(k)
|
||||
else:
|
||||
raise ValueError(f'Unknown type for key {k} with value {v}')
|
||||
|
||||
# no need to expand anchor
|
||||
|
||||
# expand template first
|
||||
for k in self._template_keys:
|
||||
self._expand_extends(k)
|
||||
|
||||
# expand job
|
||||
for k in self._job_keys:
|
||||
self._expand_extends(k)
|
||||
|
||||
def _merge_dict(self, d1: dict[str, t.Any], d2: dict[str, t.Any]) -> t.Any:
|
||||
for k, v in d2.items():
|
||||
if k in d1:
|
||||
if isinstance(v, dict) and isinstance(d1[k], dict):
|
||||
d1[k] = self._merge_dict(d1[k], v)
|
||||
else:
|
||||
d1[k] = v
|
||||
else:
|
||||
d1[k] = v
|
||||
|
||||
return d1
|
||||
|
||||
def _expand_extends(self, name: str) -> dict[str, t.Any]:
|
||||
extends = to_list(self.config[name].pop('extends', None))
|
||||
if not extends:
|
||||
return self.config[name] # type: ignore
|
||||
|
||||
original_d = self.config[name].copy()
|
||||
d = {}
|
||||
while extends:
|
||||
self._used_template_keys.update(extends) # for tracking
|
||||
|
||||
for i in extends:
|
||||
d.update(self._expand_extends(i))
|
||||
|
||||
extends = to_list(self.config[name].pop('extends', None))
|
||||
|
||||
self.config[name] = self._merge_dict(d, original_d)
|
||||
return self.config[name] # type: ignore
|
||||
return {k: v for k, v in self.config.items() if not k.startswith('.') and k not in self.global_keys}
|
||||
|
||||
|
||||
def sanitize_job_name(name: str) -> str:
|
||||
|
||||
@@ -149,21 +149,6 @@ class IdfLocalPlugin:
|
||||
|
||||
return 'linux' in targets
|
||||
|
||||
@pytest.hookimpl(trylast=True)
|
||||
def pytest_generate_tests(self, metafunc: Metafunc) -> None:
|
||||
if 'embedded_services' not in metafunc.fixturenames:
|
||||
return
|
||||
|
||||
if self._has_parametrized_arg(metafunc, 'embedded_services'):
|
||||
return
|
||||
|
||||
if metafunc.definition.get_closest_marker('qemu') is not None:
|
||||
metafunc.parametrize('embedded_services', ['idf,qemu'], indirect=True)
|
||||
return
|
||||
|
||||
if self._is_linux_target_run(metafunc.config):
|
||||
metafunc.parametrize('embedded_services', ['idf'], indirect=True)
|
||||
|
||||
@pytest.hookimpl(wrapper=True)
|
||||
def pytest_collection_modifyitems(self, config: Config, items: list[Function]) -> t.Generator[None, None, None]:
|
||||
yield # throw it back to idf-ci
|
||||
|
||||
@@ -66,7 +66,6 @@ class Gitlab:
|
||||
JOB_NAME_PATTERN = re.compile(r'(\w+)(\s+(\d+)/(\d+))?')
|
||||
|
||||
DOWNLOAD_ERROR_MAX_RETRIES = 3
|
||||
DEFAULT_BUILD_CHILD_PIPELINE_NAME = 'Build Child Pipeline'
|
||||
|
||||
def __init__(self, project_id: int | str | None = None):
|
||||
config_data_from_env = os.getenv('PYTHON_GITLAB_CONFIG')
|
||||
@@ -116,7 +115,7 @@ class Gitlab:
|
||||
:param namespace: namespace to match when we have multiple project with same name
|
||||
:return: project ID
|
||||
"""
|
||||
projects = self.gitlab_inst.projects.list(search=name)
|
||||
projects = self.gitlab_inst.projects.list(search=name, get_all=True)
|
||||
res = []
|
||||
for project in projects:
|
||||
if namespace is None:
|
||||
@@ -297,36 +296,28 @@ class Gitlab:
|
||||
|
||||
def get_downstream_pipeline_ids(self, main_pipeline_id: int) -> list[int]:
|
||||
"""
|
||||
Retrieve the IDs of all downstream child pipelines for a given main pipeline.
|
||||
Retrieve the IDs of all downstream child pipelines for a given main pipeline,
|
||||
recursing through arbitrarily nested child pipelines.
|
||||
|
||||
:param main_pipeline_id: The ID of the main pipeline to start the search.
|
||||
:return: A list of IDs of all downstream child pipelines.
|
||||
:return: A list of IDs of all downstream child pipelines (all levels).
|
||||
"""
|
||||
bridge_pipeline_ids = []
|
||||
child_pipeline_ids = []
|
||||
child_pipeline_ids: list[int] = []
|
||||
|
||||
main_pipeline_bridges = self.project.pipelines.get(main_pipeline_id).bridges.list()
|
||||
for bridge in main_pipeline_bridges:
|
||||
pipeline_bridges = self.project.pipelines.get(main_pipeline_id).bridges.list()
|
||||
for bridge in pipeline_bridges:
|
||||
downstream_pipeline = bridge.attributes.get('downstream_pipeline')
|
||||
if not downstream_pipeline:
|
||||
continue
|
||||
bridge_pipeline_ids.append(downstream_pipeline['id'])
|
||||
|
||||
for bridge_pipeline_id in bridge_pipeline_ids:
|
||||
child_pipeline_ids.append(bridge_pipeline_id)
|
||||
bridge_pipeline = self.project.pipelines.get(bridge_pipeline_id)
|
||||
|
||||
if not bridge_pipeline.name == self.DEFAULT_BUILD_CHILD_PIPELINE_NAME:
|
||||
downstream_pipeline_id = downstream_pipeline.get('id')
|
||||
if downstream_pipeline_id is None:
|
||||
continue
|
||||
|
||||
child_bridges = bridge_pipeline.bridges.list()
|
||||
for child_bridge in child_bridges:
|
||||
downstream_child_pipeline = child_bridge.attributes.get('downstream_pipeline')
|
||||
if not downstream_child_pipeline:
|
||||
continue
|
||||
child_pipeline_ids.append(downstream_child_pipeline.get('id'))
|
||||
child_pipeline_ids.append(downstream_pipeline_id)
|
||||
# recurse to collect further nested (grandchild+) pipelines
|
||||
child_pipeline_ids.extend(self.get_downstream_pipeline_ids(downstream_pipeline_id))
|
||||
|
||||
return [pid for pid in child_pipeline_ids if pid is not None]
|
||||
return child_pipeline_ids
|
||||
|
||||
def retry_failed_jobs(self, pipeline_id: int, retry_allowed_failures: bool = False) -> list[int]:
|
||||
"""
|
||||
|
||||
@@ -164,38 +164,3 @@ function join_by {
|
||||
printf %s "$f" "${@/#/$d}"
|
||||
fi
|
||||
}
|
||||
|
||||
function is_based_on_commits() {
|
||||
# This function would accept space-separated args as multiple commits.
|
||||
# The return value would be 0 if current HEAD is based on any of the specified commits.
|
||||
#
|
||||
# In our CI, we use environment variable $REQUIRED_ANCESTOR_COMMITS to declare the ancestor commits.
|
||||
# Please remember to set one commit for each release branch.
|
||||
|
||||
commits=$*
|
||||
if [[ -z $commits ]]; then
|
||||
info "Not specifying commits that branches should be based on, skipping check..."
|
||||
return 0
|
||||
fi
|
||||
|
||||
commits_str="$(join_by " or " $commits)" # no doublequotes here, passing array
|
||||
|
||||
info "Checking if current branch is based on $commits_str..."
|
||||
for i in $commits; do
|
||||
if git merge-base --is-ancestor "$i" HEAD >/dev/null 2>&1; then
|
||||
info "Current branch is based on $i"
|
||||
return 0
|
||||
else
|
||||
info "Current branch is not based on $i"
|
||||
fi
|
||||
done
|
||||
|
||||
error "The base commit of your branch is too old."
|
||||
error "The branch should be more recent than either of the following commits:"
|
||||
error " $commits_str"
|
||||
error "To fix the issue:"
|
||||
error " - If your merge request is 'Draft', or has conflicts with the target branch, rebase it to the latest master or release branch"
|
||||
error " - Otherwise, simply run a new pipeline."
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
# SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
import pytest
|
||||
from pytest_embedded import Dut
|
||||
from pytest_embedded_idf.utils import idf_parametrize
|
||||
|
||||
|
||||
@pytest.mark.host_test
|
||||
@idf_parametrize('target', ['linux'], indirect=['target'])
|
||||
def test_linux_freertos_SMP(dut: Dut) -> None:
|
||||
dut.expect_exact('Press ENTER to see the list of tests.')
|
||||
|
||||
Reference in New Issue
Block a user