Mirrors/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2026-10-01 18:50:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

ci: apply common-scripts CI refactor

Browse Source
This commit is contained in:
Fu Hanxi
2026-09-03 20:25:12 +02:00
parent 0a86d7b8e7
commit 5fd30885b3
26 changed files with 236 additions and 1351 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/ci/python_packages/gitlab_api.py
+42 -51
View File
@@ -1,4 +1,4 @@
# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import argparse
import logging
@@ -9,13 +9,9 @@ import tarfile
import tempfile
import time
import zipfile
from collections.abc import Callable
from functools import wraps
from typing import Any
from typing import Callable
from typing import Dict
from typing import List
from typing import Optional
from typing import Union
import gitlab
@@ -36,7 +32,7 @@ def retry(func: TR) -> TR:
while True:
try:
res = func(self, *args, **kwargs)
except (IOError, EOFError, gitlab.exceptions.GitlabError) as e:
except (OSError, EOFError, gitlab.exceptions.GitlabError) as e:
if isinstance(e, gitlab.exceptions.GitlabError):
if e.response_code == 500:
# retry on this error
@@ -52,9 +48,11 @@ def retry(func: TR) -> TR:
raise e # get out of the loop
else:
logging.warning(
'Network failure in {}, retrying ({})'.format(getattr(func, '__name__', '(unknown callable)'),
retried))
time.sleep(2 ** retried) # wait a bit more after each retry
'Network failure in {}, retrying ({})'.format(
getattr(func, '__name__', '(unknown callable)'), retried
)
)
time.sleep(2**retried) # wait a bit more after each retry
continue
else:
break
@@ -63,26 +61,25 @@ def retry(func: TR) -> TR:
return wrapper
class Gitlab(object):
class Gitlab:
JOB_NAME_PATTERN = re.compile(r'(\w+)(\s+(\d+)/(\d+))?')
DOWNLOAD_ERROR_MAX_RETRIES = 3
DEFAULT_BUILD_CHILD_PIPELINE_NAME = 'Build Child Pipeline'
def __init__(self, project_id: Union[int, str, None] = None):
def __init__(self, project_id: int | str | None = None):
config_data_from_env = os.getenv('PYTHON_GITLAB_CONFIG')
if config_data_from_env:
# prefer to load config from env variable
with tempfile.NamedTemporaryFile('w', delete=False) as temp_file:
temp_file.write(config_data_from_env)
config_files = [temp_file.name] # type: Optional[List[str]]
config_files: list[str] | None = [temp_file.name]
else:
# otherwise try to use config file at local filesystem
config_files = None
self._init_gitlab_inst(project_id, config_files)
@retry
def _init_gitlab_inst(self, project_id: Optional[int], config_files: Optional[List[str]]) -> None:
def _init_gitlab_inst(self, project_id: int | None, config_files: list[str] | None) -> None:
gitlab_id = os.getenv('LOCAL_GITLAB_HTTPS_HOST') # if None, will use the default gitlab server
self.gitlab_inst = gitlab.Gitlab.from_config(gitlab_id=gitlab_id, config_files=config_files)
@@ -109,7 +106,7 @@ class Gitlab(object):
self.project = None
@retry
def get_project_id(self, name: str, namespace: Optional[str] = None) -> int:
def get_project_id(self, name: str, namespace: str | None = None) -> int:
"""
search project ID by name
@@ -117,7 +114,7 @@ class Gitlab(object):
:param namespace: namespace to match when we have multiple project with same name
:return: project ID
"""
projects = self.gitlab_inst.projects.list(search=name)
projects = self.gitlab_inst.projects.list(search=name, get_all=True)
res = []
for project in projects:
if namespace is None:
@@ -152,7 +149,7 @@ class Gitlab(object):
archive_file.extractall(destination)
@retry
def download_artifact(self, job_id: int, artifact_path: List[str], destination: Optional[str] = None) -> List[bytes]:
def download_artifact(self, job_id: int, artifact_path: list[str], destination: str | None = None) -> list[bytes]:
"""
download specific path of job artifacts and extract to destination.
@@ -169,7 +166,7 @@ class Gitlab(object):
try:
data = job.artifact(a_path) # type: bytes
except gitlab.GitlabGetError as e:
logging.error("Failed to download '{}' from job {}".format(a_path, job_id))
logging.error(f"Failed to download '{a_path}' from job {job_id}")
raise e
raw_data_list.append(data)
if destination:
@@ -185,7 +182,7 @@ class Gitlab(object):
return raw_data_list
@retry
def find_job_id(self, job_name: str, pipeline_id: Optional[str] = None, job_status: str = 'success') -> List[Dict]:
def find_job_id(self, job_name: str, pipeline_id: str | None = None, job_status: str = 'success') -> list[dict]:
"""
Get Job ID from job name of specific pipeline
@@ -208,8 +205,9 @@ class Gitlab(object):
return job_id_list
@retry
def download_archive(self, ref: str, destination: str, project_id: Optional[int] = None,
cache_dir: Optional[str] = None) -> str:
def download_archive(
self, ref: str, destination: str, project_id: int | None = None, cache_dir: str | None = None
) -> str:
"""
Download archive of certain commit of a repository and extract to destination path
@@ -233,10 +231,11 @@ class Gitlab(object):
try:
project.repository_archive(sha=ref, streamed=True, action=fw.write)
except gitlab.GitlabGetError as e:
logging.error('Failed to archive from project {}'.format(project_id))
logging.error(f'Failed to archive from project {project_id}')
raise e
logging.info('Downloaded archive size: {:.03f}MB'.format(
float(os.path.getsize(local_archive_file)) / (1024 * 1024)))
logging.info(
f'Downloaded archive size: {float(os.path.getsize(local_archive_file)) / (1024 * 1024):.03f}MB'
)
return self.decompress_archive(local_archive_file, destination)
@@ -245,10 +244,10 @@ class Gitlab(object):
try:
project.repository_archive(sha=ref, streamed=True, action=temp_file.write)
except gitlab.GitlabGetError as e:
logging.error('Failed to archive from project {}'.format(project_id))
logging.error(f'Failed to archive from project {project_id}')
raise e
logging.info('Downloaded archive size: {:.03f}MB'.format(float(os.path.getsize(temp_file.name)) / (1024 * 1024)))
logging.info(f'Downloaded archive size: {float(os.path.getsize(temp_file.name)) / (1024 * 1024):.03f}MB')
return self.decompress_archive(temp_file.name, destination)
@@ -280,40 +279,32 @@ class Gitlab(object):
job = self.project.jobs.get(job_id)
return ','.join(job.tag_list)
def get_downstream_pipeline_ids(self, main_pipeline_id: int) -> List[int]:
def get_downstream_pipeline_ids(self, main_pipeline_id: int) -> list[int]:
"""
Retrieve the IDs of all downstream child pipelines for a given main pipeline.
Retrieve the IDs of all downstream child pipelines for a given main pipeline,
recursing through arbitrarily nested child pipelines.
:param main_pipeline_id: The ID of the main pipeline to start the search.
:return: A list of IDs of all downstream child pipelines.
:return: A list of IDs of all downstream child pipelines (all levels).
"""
bridge_pipeline_ids = []
child_pipeline_ids = []
child_pipeline_ids: list[int] = []
main_pipeline_bridges = self.project.pipelines.get(main_pipeline_id).bridges.list()
for bridge in main_pipeline_bridges:
pipeline_bridges = self.project.pipelines.get(main_pipeline_id).bridges.list()
for bridge in pipeline_bridges:
downstream_pipeline = bridge.attributes.get('downstream_pipeline')
if not downstream_pipeline:
continue
bridge_pipeline_ids.append(downstream_pipeline['id'])
for bridge_pipeline_id in bridge_pipeline_ids:
child_pipeline_ids.append(bridge_pipeline_id)
bridge_pipeline = self.project.pipelines.get(bridge_pipeline_id)
if not bridge_pipeline.name == self.DEFAULT_BUILD_CHILD_PIPELINE_NAME:
downstream_pipeline_id = downstream_pipeline.get('id')
if downstream_pipeline_id is None:
continue
child_bridges = bridge_pipeline.bridges.list()
for child_bridge in child_bridges:
downstream_child_pipeline = child_bridge.attributes.get('downstream_pipeline')
if not downstream_child_pipeline:
continue
child_pipeline_ids.append(downstream_child_pipeline.get('id'))
child_pipeline_ids.append(downstream_pipeline_id)
# recurse to collect further nested (grandchild+) pipelines
child_pipeline_ids.extend(self.get_downstream_pipeline_ids(downstream_pipeline_id))
return [pid for pid in child_pipeline_ids if pid is not None]
return child_pipeline_ids
def retry_failed_jobs(self, pipeline_id: int, retry_allowed_failures: bool = False) -> List[int]:
def retry_failed_jobs(self, pipeline_id: int, retry_allowed_failures: bool = False) -> list[int]:
"""
Retry failed jobs for a specific pipeline. Optionally include jobs marked as 'allowed failures'.
@@ -370,10 +361,10 @@ def main() -> None:
gitlab_inst.download_archive(args.ref, args.destination)
elif args.action == 'get_project_id':
ret = gitlab_inst.get_project_id(args.project_name)
print('project id: {}'.format(ret))
print(f'project id: {ret}')
elif args.action == 'retry_failed_jobs':
res = gitlab_inst.retry_failed_jobs(args.pipeline_id, args.retry_allowed_failures)
print('jobs retried successfully: {}'.format(res))
print(f'jobs retried successfully: {res}')
elif args.action == 'get_job_tags':
ret = gitlab_inst.get_job_tags(args.job_id)
print(ret)