diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7c02159d1a4..a401e61b21d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,3 +30,7 @@ include: - ".gitlab/ci/deploy.yml" - ".gitlab/ci/post_deploy.yml" - ".gitlab/ci/test-win.yml" + - project: "ci/actions/common" + file: "templates/idf/deploy-github.yml" + - project: "ci/actions/common" + file: "templates/idf/deploy-docs.yml" diff --git a/.gitlab/ci/deploy.yml b/.gitlab/ci/deploy.yml index ce79820770a..1406076e1e7 100644 --- a/.gitlab/ci/deploy.yml +++ b/.gitlab/ci/deploy.yml @@ -47,28 +47,6 @@ check_submodule_sync: - git submodule update --recursive - echo "IDF was cloned from ${PUBLIC_IDF_URL} completely" -push_to_github: - extends: - - .deploy_job_template - - .before_script:minimal - - .rules:protected:deploy - needs: - # submodule must be synced before pushing to github - - check_submodule_sync - tags: [ brew, github_sync ] - variables: - GIT_STRATEGY: fetch # use brew local mirror first - GIT_DEPTH: 0 # github needs full record of commits - script: - - add_github_ssh_keys - - git remote remove github &>/dev/null || true - - git remote add github git@github.com:espressif/esp-idf.git - - tools/ci/push_to_github.sh - environment: - name: push_to_github_production - deployment_tier: production - url: "https://github.com/espressif/esp-idf" - deploy_update_SHA_in_esp-dockerfiles: extends: - .deploy_job_template diff --git a/.gitlab/ci/docs.yml b/.gitlab/ci/docs.yml index 8640c22b017..6d295dfa4c1 100644 --- a/.gitlab/ci/docs.yml +++ b/.gitlab/ci/docs.yml @@ -143,77 +143,6 @@ build_docs_html_partial: - DOCLANG: "zh_CN" DOCTGT: "esp32p4" -.deploy_docs_template: - image: $ESP_IDF_DOC_ENV_IMAGE - variables: - DOCS_BUILD_DIR: "${IDF_PATH}/docs/_build/" - PYTHONUNBUFFERED: 1 - # ensure all tags are fetched, need to know the latest/stable tag for the docs - GIT_STRATEGY: clone - GIT_DEPTH: 0 - stage: test_deploy - tags: - - brew - - amd64 - script: - # ensure all tags are fetched, need to know the latest/stable tag for the docs - - git fetch --tags --prune - - add_doc_server_ssh_keys $DOCS_DEPLOY_PRIVATEKEY $DOCS_DEPLOY_SERVER $DOCS_DEPLOY_SERVER_USER - - export GIT_VER=$(git describe --always ${PIPELINE_COMMIT_SHA} --) - - deploy-docs - -# stage: test_deploy -deploy_docs_preview: - extends: - - .deploy_docs_template - rules: - - <<: *if-label-build_docs - - <<: *if-label-docs_full - - <<: *if-dev-push - changes: *patterns-docs-preview - needs: - - job: build_docs_html_partial - optional: true - - job: build_docs_html_full - optional: true - variables: - TYPE: "preview" - # older branches use DOCS_DEPLOY_KEY, DOCS_SERVER, DOCS_SERVER_USER, DOCS_PATH for preview server so we keep these names for 'preview' - DOCS_DEPLOY_PRIVATEKEY: "$DOCS_DEPLOY_KEY" - DOCS_DEPLOY_SERVER: "$DOCS_SERVER" - DOCS_DEPLOY_SERVER_USER: "$DOCS_SERVER_USER" - DOCS_DEPLOY_PATH: "$DOCS_PATH" - DOCS_DEPLOY_URL_BASE: "https://$DOCS_PREVIEW_SERVER_URL/docs/esp-idf" - environment: - name: deploy_docs_preview - deployment_tier: staging - url: "https://$DOCS_PREVIEW_SERVER_URL/docs/esp-idf" - -# stage: post_deploy -deploy_docs_production: - # The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings - extends: - - .deploy_docs_template - - .rules:protected:deploy - stage: post_deploy - dependencies: # set dependencies to null to avoid missing artifacts issue - needs: # ensure runs after push_to_github succeeded - - build_docs_html_full - - job: push_to_github - artifacts: false - variables: - TYPE: "preview" - DOCS_DEPLOY_PRIVATEKEY: "$DOCS_PROD_DEPLOY_KEY" - DOCS_DEPLOY_SERVER: "$DOCS_PROD_SERVER" - DOCS_DEPLOY_SERVER_USER: "$DOCS_PROD_SERVER_USER" - DOCS_DEPLOY_PATH: "$DOCS_PROD_PATH" - DOCS_DEPLOY_URL_BASE: "https://docs.espressif.com/projects/esp-idf" - DEPLOY_STABLE: 1 - environment: - name: deploy_docs_production - deployment_tier: production - url: "https://docs.espressif.com/projects/esp-idf" - check_doc_links: extends: - .build_docs_template diff --git a/tools/ci/idf_ci_utils.py b/tools/ci/idf_ci_utils.py index b8db76beeba..73183e31e48 100644 --- a/tools/ci/idf_ci_utils.py +++ b/tools/ci/idf_ci_utils.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2020-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2020-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # internal use only for CI # some CI related util functions @@ -13,7 +13,7 @@ from functools import cached_property IDF_PATH: str = os.path.abspath(os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..'))) -def get_submodule_dirs(full_path: bool = False) -> t.List[str]: +def get_submodule_dirs(full_path: bool = False) -> list[str]: """ To avoid issue could be introduced by multi-os or additional dependency, we use python and git to get this output @@ -71,7 +71,7 @@ def is_executable(full_path: str) -> bool: return os.access(full_path, os.X_OK) -def get_git_files(path: str = IDF_PATH, full_path: bool = False) -> t.List[str]: +def get_git_files(path: str = IDF_PATH, full_path: bool = False) -> list[str]: """ Get the result of git ls-files :param path: path to run git ls-files @@ -98,11 +98,11 @@ def get_git_files(path: str = IDF_PATH, full_path: bool = False) -> t.List[str]: return [os.path.join(path, f) for f in files] if full_path else files -def to_list(s: t.Any) -> t.List[t.Any]: +def to_list(s: t.Any) -> list[t.Any]: if not s: return [] - if isinstance(s, (set, tuple)): + if isinstance(s, set | tuple): return list(s) if isinstance(s, list): @@ -113,8 +113,8 @@ def to_list(s: t.Any) -> t.List[t.Any]: class GitlabYmlConfig: def __init__(self, root_yml_filepath: str = os.path.join(IDF_PATH, '.gitlab-ci.yml')) -> None: - self._config: t.Dict[str, t.Any] = {} - self._defaults: t.Dict[str, t.Any] = {} + self._config: dict[str, t.Any] = {} + self._defaults: dict[str, t.Any] = {} self._load(root_yml_filepath) @@ -127,6 +127,14 @@ class GitlabYmlConfig: # expanding "include" for item in root_yml.pop('include', []) or []: + if isinstance(item, dict): + if 'project' in item: + continue + elif 'local' in item: + item = item['local'] + else: + continue + all_config.update(yaml.load(open(os.path.join(IDF_PATH, item)), Loader=yaml.FullLoader)) if 'default' in all_config: @@ -135,41 +143,41 @@ class GitlabYmlConfig: self._config = all_config # anchor is the string that will be reused in templates - self._anchor_keys: t.Set[str] = set() + self._anchor_keys: set[str] = set() # template is a dict that will be extended - self._template_keys: t.Set[str] = set() - self._used_template_keys: t.Set[str] = set() # tracing the used templates + self._template_keys: set[str] = set() + self._used_template_keys: set[str] = set() # tracing the used templates # job is a dict that will be executed - self._job_keys: t.Set[str] = set() + self._job_keys: set[str] = set() self.expand_extends() @property - def default(self) -> t.Dict[str, t.Any]: + def default(self) -> dict[str, t.Any]: return self._defaults @property - def config(self) -> t.Dict[str, t.Any]: + def config(self) -> dict[str, t.Any]: return self._config @cached_property - def global_keys(self) -> t.List[str]: + def global_keys(self) -> list[str]: return ['default', 'include', 'workflow', 'variables', 'stages'] @cached_property - def anchors(self) -> t.Dict[str, t.Any]: + def anchors(self) -> dict[str, t.Any]: return {k: v for k, v in self.config.items() if k in self._anchor_keys} @cached_property - def jobs(self) -> t.Dict[str, t.Any]: + def jobs(self) -> dict[str, t.Any]: return {k: v for k, v in self.config.items() if k in self._job_keys} @cached_property - def templates(self) -> t.Dict[str, t.Any]: + def templates(self) -> dict[str, t.Any]: return {k: v for k, v in self.config.items() if k in self._template_keys} @cached_property - def used_templates(self) -> t.Set[str]: + def used_templates(self) -> set[str]: return self._used_template_keys def expand_extends(self) -> None: @@ -180,7 +188,7 @@ class GitlabYmlConfig: if k in self.global_keys: continue - if isinstance(v, (str, list)): + if isinstance(v, str | list): self._anchor_keys.add(k) elif k.startswith('.if-'): self._anchor_keys.add(k) @@ -201,7 +209,7 @@ class GitlabYmlConfig: for k in self._job_keys: self._expand_extends(k) - def _merge_dict(self, d1: t.Dict[str, t.Any], d2: t.Dict[str, t.Any]) -> t.Any: + def _merge_dict(self, d1: dict[str, t.Any], d2: dict[str, t.Any]) -> t.Any: for k, v in d2.items(): if k in d1: if isinstance(v, dict) and isinstance(d1[k], dict): @@ -213,7 +221,7 @@ class GitlabYmlConfig: return d1 - def _expand_extends(self, name: str) -> t.Dict[str, t.Any]: + def _expand_extends(self, name: str) -> dict[str, t.Any]: extends = to_list(self.config[name].pop('extends', None)) if not extends: return self.config[name] # type: ignore