diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 91638d0b77d..ad315908cad 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,17 +23,18 @@ include: - DOCLANG: "zh_CN" DOCTGT: "esp32p4" - ".gitlab/ci/danger.yml" - - ".gitlab/ci/auto_trans.yml" - - ".gitlab/ci/common.yml" + - ".gitlab/ci/common.yml" # has to be placed after danger.yml, since danger.yml defined stages. - ".gitlab/ci/rules.yml" + # by stages - ".gitlab/ci/manual_gate.yml" - ".gitlab/ci/upload_cache.yml" + - ".gitlab/ci/pre_check.yml" - ".gitlab/ci/static-code-analysis.yml" - ".gitlab/ci/pre_commit.yml" - - ".gitlab/ci/pre_check.yml" + - ".gitlab/ci/auto_trans.yml" - ".gitlab/ci/build.yml" - ".gitlab/ci/host-test.yml" + - ".gitlab/ci/test-win.yml" - ".gitlab/ci/pre_deploy.yml" - ".gitlab/ci/deploy.yml" - ".gitlab/ci/post_deploy.yml" - - ".gitlab/ci/test-win.yml" diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index 5f272320fc1..902d15b9bef 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -127,7 +127,6 @@ variables: run_cmd idf-ci gitlab upload-artifacts --type log fi - ############# # `default` # ############# diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index 654cf52129a..e024cb55c15 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -147,6 +147,12 @@ check_test_scripts_build_test_rules: # requires basic pytest dependencies - python tools/ci/check_build_test_rules.py check-test-scripts examples/ tools/test_apps components +check_gitlab_yaml: + extends: + - .pre_check_template + script: + - python tools/ci/gitlab_yaml_linter.py + retry_failed_jobs: extends: - .pre_check_template diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7baf5cb245c..c6d9be198fd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -167,14 +167,6 @@ repos: language: python always_run: true require_serial: true - - id: gitlab-yaml-linter - name: Check gitlab yaml files - entry: tools/ci/gitlab_yaml_linter.py - language: python - files: '\.gitlab-ci\.yml|\.gitlab/ci/.+\.yml|\.gitmodules' - pass_filenames: false - additional_dependencies: - - PyYAML == 5.3.1 - id: check-wifi-remote-api name: Check wifi-remote API generation description: Runs generate_and_check.py and fails if any generated files differ from the index. diff --git a/tools/ci/generate_rules.py b/tools/ci/generate_rules.py index f9257eaf093..6bad3474454 100755 --- a/tools/ci/generate_rules.py +++ b/tools/ci/generate_rules.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2021-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import argparse import inspect @@ -12,22 +12,21 @@ from itertools import product import yaml from idf_ci_utils import IDF_PATH -from idf_ci_utils import GitlabYmlConfig if t.TYPE_CHECKING: import pygraphviz as pgv -def _list(str_or_list: t.Union[str, t.List]) -> t.List: +def _list(str_or_list: str | list) -> list: if isinstance(str_or_list, str): return [str_or_list] elif isinstance(str_or_list, list): return str_or_list else: - raise ValueError('Wrong type: {}. Only supports str or list.'.format(type(str_or_list))) + raise ValueError(f'Wrong type: {type(str_or_list)}. Only supports str or list.') -def _format_nested_dict(_dict: t.Dict[str, t.Dict], f_tuple: t.Tuple[str, ...]) -> t.Dict[str, t.Dict]: +def _format_nested_dict(_dict: dict[str, dict], f_tuple: tuple[str, ...]) -> dict[str, dict]: res = {} for k, v in _dict.items(): k = k.split('__')[0] @@ -41,7 +40,7 @@ def _format_nested_dict(_dict: t.Dict[str, t.Dict], f_tuple: t.Tuple[str, ...]) return res -def _format_nested_list(_list: t.List[str], f_tuple: t.Tuple[str, ...]) -> t.List[str]: +def _format_nested_list(_list: list[str], f_tuple: tuple[str, ...]) -> list[str]: res = [] for item in _list: if isinstance(item, list): @@ -90,7 +89,6 @@ class RulesWriter: self.cfg = self.expand_matrices() self.rules = self.expand_rules() - self.yml_config = GitlabYmlConfig() self.graph = None def expand_matrices(self): # type: () -> dict @@ -107,11 +105,11 @@ class RulesWriter: deploy = v.get('deploy') if deploy: for item in _list(deploy): - res['{}-{}'.format(k, item)] = v + res[f'{k}-{item}'] = v return res @staticmethod - def _expand_matrix(name: str, cfg: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]: + def _expand_matrix(name: str, cfg: dict[str, t.Any]) -> dict[str, t.Any]: """ Expand matrix into multi keys :param cfg: single rule dict @@ -129,7 +127,7 @@ class RulesWriter: res.update(_format_nested_dict(default, comb)) return res - def expand_rules(self) -> t.Dict[str, t.Dict[str, t.List[str]]]: + def expand_rules(self) -> dict[str, dict[str, list[str]]]: res = defaultdict(lambda: defaultdict(set)) # type: dict[str, dict[str, set]] for k, v in self.cfg.items(): if not v: @@ -155,8 +153,8 @@ class RulesWriter: if 'patterns' in v: for _pat in _list(v['patterns']): # Patterns must be pre-defined - if '.patterns-{}'.format(_pat) not in self.rules_cfg: - print('WARNING: pattern {} not exists'.format(_pat)) + if f'.patterns-{_pat}' not in self.rules_cfg: + print(f'WARNING: pattern {_pat} not exists') continue res[item]['patterns'].add(_pat) @@ -195,14 +193,10 @@ class RulesWriter: if k.startswith('pattern'): continue - if '.rules:' + k not in self.yml_config.used_templates: - print(f'WARNING: unused rule: {k}, skipping...') - continue - res.append(self.RULES_TEMPLATE.format(k, self._format_rule(k, v))) return '\n\n'.join(res) - def _format_rule(self, name: str, cfg: t.Dict[str, t.Any]) -> str: + def _format_rule(self, name: str, cfg: dict[str, t.Any]) -> str: _rules = [self.RULE_REVERT_BRANCH] if name.endswith('-production'): _rules.append(self.RULE_PROTECTED_PUSH) @@ -216,21 +210,21 @@ class RulesWriter: if f'.{specific_rule}' in self.rules_cfg: _rules.append(self.SPECIFIC_RULE_TEMPLATE.format(specific_rule)) else: - print('WARNING: specific_rule {} not exists'.format(specific_rule)) + print(f'WARNING: specific_rule {specific_rule} not exists') for label in cfg['labels']: _rules.append(self.RULE_LABEL_TEMPLATE.format(label)) for pattern in cfg['patterns']: - if '.patterns-{}'.format(pattern) in self.rules_cfg: + if f'.patterns-{pattern}' in self.rules_cfg: _rules.append(self.RULE_PATTERN_TEMPLATE.format(pattern)) else: - print('WARNING: pattern {} not exists'.format(pattern)) + print(f'WARNING: pattern {pattern} not exists') return '\n'.join(_rules) def update_rules_yml(self) -> bool: with open(self.rules_yml) as fr: file_str = fr.read() - auto_generate_str = '\n{}\n\n{}\n'.format(self.new_labels_str(), self.new_rules_str()) + auto_generate_str = f'\n{self.new_labels_str()}\n\n{self.new_rules_str()}\n' rest, marker, old = file_str.partition(self.AUTO_GENERATE_MARKER) if old == auto_generate_str: return False @@ -246,7 +240,7 @@ PATTERN_COLOR = 'cyan' RULE_COLOR = 'blue' -def build_graph(rules_dict: t.Dict[str, t.Dict[str, t.List[str]]]) -> 'pgv.AGraph': +def build_graph(rules_dict: dict[str, dict[str, list[str]]]) -> 'pgv.AGraph': from pygraphviz import pgv graph = pgv.AGraph(directed=True, rankdir='LR', concentrate=True) @@ -263,13 +257,13 @@ def build_graph(rules_dict: t.Dict[str, t.Dict[str, t.List[str]]]) -> 'pgv.AGrap labels = v.get('labels') if labels: for _label in labels: - graph.add_node('label:{}'.format(_label), color=LABEL_COLOR) - graph.add_edge('label:{}'.format(_label), k, color=LABEL_COLOR) + graph.add_node(f'label:{_label}', color=LABEL_COLOR) + graph.add_edge(f'label:{_label}', k, color=LABEL_COLOR) patterns = v.get('patterns') if patterns: for _pat in patterns: - graph.add_node('pattern:{}'.format(_pat), color=PATTERN_COLOR) - graph.add_edge('pattern:{}'.format(_pat), k, color=PATTERN_COLOR) + graph.add_node(f'pattern:{_pat}', color=PATTERN_COLOR) + graph.add_edge(f'pattern:{_pat}', k, color=PATTERN_COLOR) return graph diff --git a/tools/ci/gitlab_yaml_linter.py b/tools/ci/gitlab_yaml_linter.py index a5747205c5e..e06a12034d4 100755 --- a/tools/ci/gitlab_yaml_linter.py +++ b/tools/ci/gitlab_yaml_linter.py @@ -43,17 +43,6 @@ class YmlLinter: exit(exit_code) - # name it like _1_ to make it run first - def _lint_1_yml_parser(self) -> None: - for k, v in self.yml_config.config.items(): - if ( - k not in self.yml_config.global_keys - and k not in self.yml_config.anchors - and k not in self.yml_config.templates - and k not in self.yml_config.jobs - ): - raise SystemExit(f'Parser incorrect. Key {k} not in global keys, anchors, templates, or jobs') - def _lint_default_values_artifacts(self) -> None: defaults_artifacts = self.yml_config.default.get('artifacts', {}) @@ -79,20 +68,6 @@ class YmlLinter: for item in undefined_patterns: self._errors.append(f'undefined pattern {item}. Please add {item} to .patterns-submodule') - def _lint_gitlab_yml_templates(self) -> None: - unused_templates = self.yml_config.templates.keys() - self.yml_config.used_templates - for item in unused_templates: - # known unused ones - if item not in [ - '.before_script:fetch:target_test', # used in dynamic pipeline - ]: - self._errors.append(f'Unused template: {item}, please remove it') - - undefined_templates = self.yml_config.used_templates - self.yml_config.templates.keys() - for item in undefined_templates: - if item not in self.yml_config._EXTERNAL_TEMPLATE_KEYS: - self._errors.append(f'Undefined template: {item}') - def _lint_dependencies_and_needs(self) -> None: """ Use `dependencies: []` together with `needs: []` could cause missing artifacts issue. diff --git a/tools/ci/idf_ci_utils.py b/tools/ci/idf_ci_utils.py index b69e7b1200a..74840dcd512 100644 --- a/tools/ci/idf_ci_utils.py +++ b/tools/ci/idf_ci_utils.py @@ -114,17 +114,6 @@ def to_list(s: t.Any) -> list[t.Any]: class GitlabYmlConfig: - # Templates defined in external project includes (ci/actions/common) that are - # not loaded locally. Referenced via `extends:` in local jobs, so _expand_extends - # must skip them instead of raising KeyError. - _EXTERNAL_TEMPLATE_KEYS: t.ClassVar[set[str]] = { - '.common_before_scripts', - '.common_after_scripts', - '.macos-settings', - '.windows-settings', - '.pre_check_template', - } - def __init__(self, root_yml_filepath: str = os.path.join(IDF_PATH, '.gitlab-ci.yml')) -> None: self._config: dict[str, t.Any] = {} self._defaults: dict[str, t.Any] = {} @@ -135,38 +124,106 @@ class GitlabYmlConfig: # avoid unused import in other pre-commit hooks import yaml - # GitLab CI uses !reference tags which standard YAML loaders don't know about - yaml.add_multi_constructor('', lambda loader, tag, node: None, Loader=yaml.FullLoader) - - all_config = dict() - root_yml = yaml.load(open(root_yml_filepath), Loader=yaml.FullLoader) - - # expanding "include" - for item in root_yml.pop('include', []) or []: - if isinstance(item, dict): - if 'project' in item: - continue - elif 'local' in item: - item = item['local'] - else: - continue - - all_config.update(yaml.load(open(os.path.join(IDF_PATH, item)), Loader=yaml.FullLoader)) + merged_yaml = self._compile_via_gitlab_api(root_yml_filepath) + all_config = yaml.load(merged_yaml, Loader=yaml.FullLoader) or {} if 'default' in all_config: self._defaults = all_config.pop('default') self._config = all_config - # anchor is the string that will be reused in templates - self._anchor_keys: set[str] = set() - # template is a dict that will be extended - self._template_keys: set[str] = set() - self._used_template_keys: set[str] = set() # tracing the used templates - # job is a dict that will be executed - self._job_keys: set[str] = set() + def _inline_local_includes(self, root_yml_filepath: str) -> str: + """ + Recursively resolve `include: local` entries straight from disk (so uncommitted local + changes are always picked up -- CI runners also work off a disk checkout, so there's no + need to fetch a ref remotely via `ci_lint`'s `content_ref`/`ref`/`dry_run_ref` GET + params). `include: project` entries (including ones nested inside local files, e.g. + `.gitlab/ci/common.yml` including `templates/idf/common-scripts.yml`) are collected and + left in the final `include:` list, since those files live in another GitLab project and + can only be resolved remotely. - self.expand_extends() + A dedicated Loader/Dumper pair round-trips `!reference` tags as a marker list subclass, + since GitLab CI uses `!reference` which plain YAML doesn't know, and we need to parse + (to merge dicts, not just string-concat) then re-dump losslessly. + + :param root_yml_filepath: path to the local root yml file to compile + :return: yml content with local includes resolved and merged + """ + import yaml + + class _Loader(yaml.FullLoader): + pass + + class _Dumper(yaml.Dumper): + pass + + class _Reference(list): + pass + + _Loader.add_constructor( + '!reference', lambda loader, node: _Reference(loader.construct_sequence(t.cast(yaml.SequenceNode, node))) + ) + _Dumper.add_representer(_Reference, lambda dumper, data: dumper.represent_sequence('!reference', list(data))) + + def resolve(yml_filepath: str) -> tuple[dict, list]: + with open(yml_filepath) as fr: + data = yaml.load(fr, Loader=_Loader) or {} + + includes = to_list(data.pop('include', None)) + + merged: dict = {} + remaining_project_includes: list = [] + for item in includes: + if isinstance(item, dict): + if 'project' in item: + remaining_project_includes.append(item) + continue + elif 'local' in item: + local_path = item['local'] + else: + continue + elif isinstance(item, str): + local_path = item + else: + continue + + sub_merged, sub_remaining = resolve(os.path.join(IDF_PATH, local_path.lstrip('/'))) + merged.update(sub_merged) + remaining_project_includes.extend(sub_remaining) + + # this file's own top-level keys override whatever its includes defined + merged.update(data) + return merged, remaining_project_includes + + merged_config, project_includes = resolve(root_yml_filepath) + if project_includes: + merged_config['include'] = project_includes + + return yaml.dump(merged_config, Dumper=_Dumper, sort_keys=False) # type: ignore + + def _compile_via_gitlab_api(self, root_yml_filepath: str) -> str: + """ + Call the GitLab CI Lint API to get the fully compiled (all `include`s resolved) yml, + same as what `glab ci config compile` does. This replaces the old recursive local-only + parsing, since the project now includes configs from other projects as well. + + :param root_yml_filepath: path to the local root yml file to compile + :return: merged (fully resolved) yml content as a string + """ + sys.path.insert(0, os.path.join(IDF_PATH, 'tools', 'ci', 'python_packages')) + import gitlab_api + + content = self._inline_local_includes(root_yml_filepath) + + gitlab_inst = gitlab_api.Gitlab() + project_id = os.getenv('CI_PROJECT_ID') or gitlab_inst.get_project_id('esp-idf', namespace='espressif') + project = gitlab_inst.gitlab_inst.projects.get(project_id, lazy=True) + + lint_result = project.ci_lint.create({'content': content}) + if not lint_result.valid: + raise RuntimeError(f'Failed to compile {root_yml_filepath} via GitLab CI Lint API: {lint_result.errors}') + + return lint_result.merged_yaml # type: ignore @property def default(self) -> dict[str, t.Any]: @@ -180,83 +237,9 @@ class GitlabYmlConfig: def global_keys(self) -> list[str]: return ['default', 'include', 'workflow', 'variables', 'stages'] - @cached_property - def anchors(self) -> dict[str, t.Any]: - return {k: v for k, v in self.config.items() if k in self._anchor_keys} - @cached_property def jobs(self) -> dict[str, t.Any]: - return {k: v for k, v in self.config.items() if k in self._job_keys} - - @cached_property - def templates(self) -> dict[str, t.Any]: - return {k: v for k, v in self.config.items() if k in self._template_keys} - - @cached_property - def used_templates(self) -> set[str]: - return self._used_template_keys - - def expand_extends(self) -> None: - """ - expand the `extends` key in-place. - """ - for k, v in self.config.items(): - if k in self.global_keys: - continue - - if isinstance(v, str | list): - self._anchor_keys.add(k) - elif k.startswith('.if-'): - self._anchor_keys.add(k) - elif k.startswith('.'): - self._template_keys.add(k) - elif isinstance(v, dict): - self._job_keys.add(k) - else: - raise ValueError(f'Unknown type for key {k} with value {v}') - - # no need to expand anchor - - # expand template first - for k in self._template_keys: - self._expand_extends(k) - - # expand job - for k in self._job_keys: - self._expand_extends(k) - - def _merge_dict(self, d1: dict[str, t.Any], d2: dict[str, t.Any]) -> t.Any: - for k, v in d2.items(): - if k in d1: - if isinstance(v, dict) and isinstance(d1[k], dict): - d1[k] = self._merge_dict(d1[k], v) - else: - d1[k] = v - else: - d1[k] = v - - return d1 - - def _expand_extends(self, name: str) -> dict[str, t.Any]: - if name in self._EXTERNAL_TEMPLATE_KEYS: - return {} - - extends = to_list(self.config[name].pop('extends', None)) - if not extends: - return self.config[name] # type: ignore - - original_d = self.config[name].copy() - d = {} - while extends: - self._used_template_keys.update(extends) # for tracking - - for i in extends: - d.update(self._expand_extends(i)) - - extends = to_list(self.config[name].pop('extends', None)) - - self.config[name] = self._merge_dict(d, original_d) - return self.config[name] # type: ignore + return {k: v for k, v in self.config.items() if not k.startswith('.') and k not in self.global_keys} def idf_relpath(p: str) -> str: diff --git a/tools/ci/python_packages/gitlab_api.py b/tools/ci/python_packages/gitlab_api.py index cdce2ed2467..7638fad65df 100644 --- a/tools/ci/python_packages/gitlab_api.py +++ b/tools/ci/python_packages/gitlab_api.py @@ -116,7 +116,7 @@ class Gitlab: :param namespace: namespace to match when we have multiple project with same name :return: project ID """ - projects = self.gitlab_inst.projects.list(search=name) + projects = self.gitlab_inst.projects.list(search=name, get_all=True) res = [] for project in projects: if namespace is None: