mirror of
https://github.com/espressif/esp-idf.git
synced 2026-10-02 03:00:34 +03:00
ci: apply common-scripts CI refactor
Squashes these original commits for release/v5.4 backport traceability: - 9b289dc5ad6 ci: apply idf-ci 1.x - 381df980d57 ci: remove pip-cache and other unused jobs - 32375a0a15e ci: apply common-scripts CI refactor
This commit is contained in:
Fu Hanxi
+112
-122
@@ -2,9 +2,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# internal use only for CI
|
||||
# some CI related util functions
|
||||
|
||||
from __future__ import annotations # Must be the first line of code
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import typing as t
|
||||
@@ -14,7 +16,7 @@ from pathlib import Path
|
||||
IDF_PATH: str = os.path.abspath(os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..')))
|
||||
|
||||
|
||||
def get_submodule_dirs(full_path: bool = False) -> t.List[str]:
|
||||
def get_submodule_dirs(full_path: bool = False) -> list[str]:
|
||||
"""
|
||||
To avoid issue could be introduced by multi-os or additional dependency,
|
||||
we use python and git to get this output
|
||||
@@ -72,7 +74,7 @@ def is_executable(full_path: str) -> bool:
|
||||
return os.access(full_path, os.X_OK)
|
||||
|
||||
|
||||
def get_git_files(path: str = IDF_PATH, full_path: bool = False) -> t.List[str]:
|
||||
def get_git_files(path: str = IDF_PATH, full_path: bool = False) -> list[str]:
|
||||
"""
|
||||
Get the result of git ls-files
|
||||
:param path: path to run git ls-files
|
||||
@@ -99,7 +101,7 @@ def get_git_files(path: str = IDF_PATH, full_path: bool = False) -> t.List[str]:
|
||||
return [os.path.join(path, f) for f in files] if full_path else files
|
||||
|
||||
|
||||
def to_list(s: t.Any) -> t.List[t.Any]:
|
||||
def to_list(s: t.Any) -> list[t.Any]:
|
||||
if not s:
|
||||
return []
|
||||
|
||||
@@ -114,8 +116,8 @@ def to_list(s: t.Any) -> t.List[t.Any]:
|
||||
|
||||
class GitlabYmlConfig:
|
||||
def __init__(self, root_yml_filepath: str = os.path.join(IDF_PATH, '.gitlab-ci.yml')) -> None:
|
||||
self._config: t.Dict[str, t.Any] = {}
|
||||
self._defaults: t.Dict[str, t.Any] = {}
|
||||
self._config: dict[str, t.Any] = {}
|
||||
self._defaults: dict[str, t.Any] = {}
|
||||
|
||||
self._load(root_yml_filepath)
|
||||
|
||||
@@ -123,125 +125,130 @@ class GitlabYmlConfig:
|
||||
# avoid unused import in other pre-commit hooks
|
||||
import yaml
|
||||
|
||||
all_config = dict()
|
||||
root_yml = yaml.load(open(root_yml_filepath), Loader=yaml.FullLoader)
|
||||
|
||||
# expanding "include"
|
||||
for item in root_yml.pop('include', []) or []:
|
||||
if isinstance(item, dict):
|
||||
if 'project' in item:
|
||||
continue
|
||||
elif 'local' in item:
|
||||
item = item['local']
|
||||
else:
|
||||
continue
|
||||
all_config.update(yaml.load(open(os.path.join(IDF_PATH, item)), Loader=yaml.FullLoader))
|
||||
merged_yaml = self._compile_via_gitlab_api(root_yml_filepath)
|
||||
all_config = yaml.load(merged_yaml, Loader=yaml.FullLoader) or {}
|
||||
|
||||
if 'default' in all_config:
|
||||
self._defaults = all_config.pop('default')
|
||||
|
||||
self._config = all_config
|
||||
|
||||
# anchor is the string that will be reused in templates
|
||||
self._anchor_keys: t.Set[str] = set()
|
||||
# template is a dict that will be extended
|
||||
self._template_keys: t.Set[str] = set()
|
||||
self._used_template_keys: t.Set[str] = set() # tracing the used templates
|
||||
# job is a dict that will be executed
|
||||
self._job_keys: t.Set[str] = set()
|
||||
def _inline_local_includes(self, root_yml_filepath: str) -> str:
|
||||
"""
|
||||
Recursively resolve `include: local` entries straight from disk (so uncommitted local
|
||||
changes are always picked up -- CI runners also work off a disk checkout, so there's no
|
||||
need to fetch a ref remotely via `ci_lint`'s `content_ref`/`ref`/`dry_run_ref` GET
|
||||
params). `include: project` entries (including ones nested inside local files, e.g.
|
||||
`.gitlab/ci/common.yml` including `templates/idf/common-scripts.yml`) are collected and
|
||||
left in the final `include:` list, since those files live in another GitLab project and
|
||||
can only be resolved remotely.
|
||||
|
||||
self.expand_extends()
|
||||
A dedicated Loader/Dumper pair round-trips `!reference` tags as a marker list subclass,
|
||||
since GitLab CI uses `!reference` which plain YAML doesn't know, and we need to parse
|
||||
(to merge dicts, not just string-concat) then re-dump losslessly.
|
||||
|
||||
:param root_yml_filepath: path to the local root yml file to compile
|
||||
:return: yml content with local includes resolved and merged
|
||||
"""
|
||||
import yaml
|
||||
|
||||
class _Loader(yaml.FullLoader):
|
||||
pass
|
||||
|
||||
class _Dumper(yaml.Dumper):
|
||||
pass
|
||||
|
||||
class _Reference(list):
|
||||
pass
|
||||
|
||||
_Loader.add_constructor(
|
||||
'!reference',
|
||||
lambda loader, node: _Reference(loader.construct_sequence(t.cast(yaml.SequenceNode, node))),
|
||||
)
|
||||
_Dumper.add_representer(
|
||||
_Reference,
|
||||
lambda dumper, data: dumper.represent_sequence('!reference', list(data)),
|
||||
)
|
||||
|
||||
def resolve(yml_filepath: str) -> tuple[dict, list]:
|
||||
with open(yml_filepath) as fr:
|
||||
data = yaml.load(fr, Loader=_Loader) or {}
|
||||
|
||||
includes = to_list(data.pop('include', None))
|
||||
|
||||
merged: dict = {}
|
||||
remaining_project_includes: list = []
|
||||
for item in includes:
|
||||
if isinstance(item, dict):
|
||||
if 'project' in item:
|
||||
remaining_project_includes.append(item)
|
||||
continue
|
||||
elif 'local' in item:
|
||||
local_path = item['local']
|
||||
else:
|
||||
continue
|
||||
elif isinstance(item, str):
|
||||
local_path = item
|
||||
else:
|
||||
continue
|
||||
|
||||
sub_merged, sub_remaining = resolve(os.path.join(IDF_PATH, local_path.lstrip('/')))
|
||||
merged.update(sub_merged)
|
||||
remaining_project_includes.extend(sub_remaining)
|
||||
|
||||
# this file's own top-level keys override whatever its includes defined
|
||||
merged.update(data)
|
||||
return merged, remaining_project_includes
|
||||
|
||||
merged_config, project_includes = resolve(root_yml_filepath)
|
||||
if project_includes:
|
||||
merged_config['include'] = project_includes
|
||||
|
||||
return yaml.dump(merged_config, Dumper=_Dumper, sort_keys=False) # type: ignore
|
||||
|
||||
def _compile_via_gitlab_api(self, root_yml_filepath: str) -> str:
|
||||
"""
|
||||
Call the GitLab CI Lint API to get the fully compiled (all `include`s resolved) yml,
|
||||
same as what `glab ci config compile` does. This replaces the old recursive local-only
|
||||
parsing, since the project now includes configs from other projects as well.
|
||||
|
||||
:param root_yml_filepath: path to the local root yml file to compile
|
||||
:return: merged (fully resolved) yml content as a string
|
||||
"""
|
||||
sys.path.insert(0, os.path.join(IDF_PATH, 'tools', 'ci', 'python_packages'))
|
||||
import gitlab_api
|
||||
|
||||
content = self._inline_local_includes(root_yml_filepath)
|
||||
|
||||
gitlab_inst = gitlab_api.Gitlab()
|
||||
project_id = os.getenv('CI_PROJECT_ID') or gitlab_inst.get_project_id('esp-idf', namespace='espressif')
|
||||
project = gitlab_inst.gitlab_inst.projects.get(project_id, lazy=True)
|
||||
|
||||
lint_result = project.ci_lint.create({'content': content})
|
||||
if not lint_result.valid:
|
||||
raise RuntimeError(f'Failed to compile {root_yml_filepath} via GitLab CI Lint API: {lint_result.errors}')
|
||||
|
||||
return lint_result.merged_yaml # type: ignore
|
||||
|
||||
@property
|
||||
def default(self) -> t.Dict[str, t.Any]:
|
||||
def default(self) -> dict[str, t.Any]:
|
||||
return self._defaults
|
||||
|
||||
@property
|
||||
def config(self) -> t.Dict[str, t.Any]:
|
||||
def config(self) -> dict[str, t.Any]:
|
||||
return self._config
|
||||
|
||||
@cached_property
|
||||
def global_keys(self) -> t.List[str]:
|
||||
def global_keys(self) -> list[str]:
|
||||
return ['default', 'include', 'workflow', 'variables', 'stages']
|
||||
|
||||
@cached_property
|
||||
def anchors(self) -> t.Dict[str, t.Any]:
|
||||
return {k: v for k, v in self.config.items() if k in self._anchor_keys}
|
||||
|
||||
@cached_property
|
||||
def jobs(self) -> t.Dict[str, t.Any]:
|
||||
return {k: v for k, v in self.config.items() if k in self._job_keys}
|
||||
|
||||
@cached_property
|
||||
def templates(self) -> t.Dict[str, t.Any]:
|
||||
return {k: v for k, v in self.config.items() if k in self._template_keys}
|
||||
|
||||
@cached_property
|
||||
def used_templates(self) -> t.Set[str]:
|
||||
return self._used_template_keys
|
||||
|
||||
def expand_extends(self) -> None:
|
||||
"""
|
||||
expand the `extends` key in-place.
|
||||
"""
|
||||
for k, v in self.config.items():
|
||||
if k in self.global_keys:
|
||||
continue
|
||||
|
||||
if isinstance(v, (str, list)):
|
||||
self._anchor_keys.add(k)
|
||||
elif k.startswith('.if-'):
|
||||
self._anchor_keys.add(k)
|
||||
elif k.startswith('.'):
|
||||
self._template_keys.add(k)
|
||||
elif isinstance(v, dict):
|
||||
self._job_keys.add(k)
|
||||
else:
|
||||
raise ValueError(f'Unknown type for key {k} with value {v}')
|
||||
|
||||
# no need to expand anchor
|
||||
|
||||
# expand template first
|
||||
for k in self._template_keys:
|
||||
self._expand_extends(k)
|
||||
|
||||
# expand job
|
||||
for k in self._job_keys:
|
||||
self._expand_extends(k)
|
||||
|
||||
def _merge_dict(self, d1: t.Dict[str, t.Any], d2: t.Dict[str, t.Any]) -> t.Any:
|
||||
for k, v in d2.items():
|
||||
if k in d1:
|
||||
if isinstance(v, dict) and isinstance(d1[k], dict):
|
||||
d1[k] = self._merge_dict(d1[k], v)
|
||||
else:
|
||||
d1[k] = v
|
||||
else:
|
||||
d1[k] = v
|
||||
|
||||
return d1
|
||||
|
||||
def _expand_extends(self, name: str) -> t.Dict[str, t.Any]:
|
||||
extends = to_list(self.config[name].pop('extends', None))
|
||||
if not extends:
|
||||
return self.config[name] # type: ignore
|
||||
|
||||
original_d = self.config[name].copy()
|
||||
d = {}
|
||||
while extends:
|
||||
self._used_template_keys.update(extends) # for tracking
|
||||
|
||||
for i in extends:
|
||||
d.update(self._expand_extends(i))
|
||||
|
||||
extends = to_list(self.config[name].pop('extends', None))
|
||||
|
||||
self.config[name] = self._merge_dict(d, original_d)
|
||||
return self.config[name] # type: ignore
|
||||
def jobs(self) -> dict[str, t.Any]:
|
||||
return {k: v for k, v in self.config.items() if not k.startswith('.') and k not in self.global_keys}
|
||||
|
||||
|
||||
def get_all_manifest_files() -> t.List[str]:
|
||||
paths: t.List[str] = []
|
||||
def get_all_manifest_files() -> list[str]:
|
||||
paths: list[str] = []
|
||||
|
||||
for p in Path(IDF_PATH).glob('**/.build-test-rules.yml'):
|
||||
if 'managed_components' in p.parts:
|
||||
@@ -252,23 +259,6 @@ def get_all_manifest_files() -> t.List[str]:
|
||||
return paths
|
||||
|
||||
|
||||
def sanitize_job_name(name: str) -> str:
|
||||
"""
|
||||
Sanitize the job name from CI_JOB_NAME
|
||||
|
||||
- for job with `parallel: int` set, the `CI_JOB_NAME` would be `job_name index/total`, like `foo 1/3`
|
||||
- for job with `parallel: matrix` set, the `CI_JOB_NAME` would be `job_name: [var1, var2]`, like `foo: [a, b]`
|
||||
|
||||
We consider
|
||||
- the jobs generated by `parallel: int` as the same job, i.e., we remove the index/total part.
|
||||
- the jobs generated by `parallel: matrix` as different jobs, so we keep the matrix part.
|
||||
|
||||
:param name: job name
|
||||
:return: sanitized job name
|
||||
"""
|
||||
return re.sub(r' \d+/\d+', '', name)
|
||||
|
||||
|
||||
def idf_relpath(p: str) -> str:
|
||||
"""
|
||||
Turn all paths under IDF_PATH to relative paths
|
||||
|
||||
Reference in New Issue
Block a user