diff --git a/.idf_ci.toml b/.idf_ci.toml index f17e28872b1..f95c0332052 100644 --- a/.idf_ci.toml +++ b/.idf_ci.toml @@ -81,6 +81,15 @@ patterns = [ ] if_clause = 'CI_JOB_GROUP_NAME != "build_non_test_related_apps"' +[gitlab.artifacts.s3.app_extra] +bucket = "idf-artifacts" +# App-dir files needed at target-test time but outside build_* (uploaded without --build-dir). +# Add patterns here as needed, e.g. coverage info produced during build. +patterns = [ + '**/managed_components/**/*.py', +] +if_clause = 'CI_JOB_GROUP_NAME != "build_non_test_related_apps"' + [gitlab.artifacts.s3.log] bucket = "idf-artifacts" patterns = [ diff --git a/conftest.py b/conftest.py index b786251134b..f177245a068 100644 --- a/conftest.py +++ b/conftest.py @@ -36,6 +36,8 @@ from _pytest.config import Config from _pytest.fixtures import FixtureRequest from idf_ci import PytestCase from idf_ci.idf_pytest import IDF_CI_PYTEST_CASE_KEY +from idf_ci_utils import APP_EXTRA_S3_ARTIFACT_TYPE +from idf_ci_utils import IDF_PATH from idf_ci_utils import idf_relpath from idf_pytest.constants import DEFAULT_LOGDIR from idf_pytest.plugin import IDF_LOCAL_PLUGIN_KEY @@ -145,6 +147,33 @@ class AppDownloader: stderr=sys.stderr, ) + def download_app_extra(self, app_dir: str) -> None: + """Download app-dir artifacts defined under app_extra in .idf_ci.toml.""" + args = [ + 'idf-ci', + 'gitlab', + 'download-artifacts', + '--commit-sha', + self.commit_sha, + '--type', + APP_EXTRA_S3_ARTIFACT_TYPE, + app_dir, + ] + if self.pipeline_id: + args.extend(['--pipeline-id', self.pipeline_id]) + + result = subprocess.run( + args, + capture_output=True, + text=True, + cwd=IDF_PATH, + ) + logging.info(result.stdout) + if result.stderr: + logging.info(result.stderr) + if result.returncode != 0: + raise RuntimeError(f'Failed to download {APP_EXTRA_S3_ARTIFACT_TYPE} artifacts for {app_dir}') + PRESIGNED_JSON = 'presigned.json' @@ -276,6 +305,18 @@ def app_downloader(pipeline_id: t.Optional[str]) -> t.Optional[AppDownloader]: return AppDownloader(commit_sha, pipeline_id) +@pytest.fixture +def download_app_extra(app_downloader: t.Optional[AppDownloader]) -> t.Callable[[str], None]: + """Download app_extra S3 artifacts for the given app path (no-op outside CI).""" + + def _download_app_extra(app_path: str) -> None: + if app_downloader is None: + return + app_downloader.download_app_extra(idf_relpath(app_path)) + + return _download_app_extra + + @pytest.fixture @multi_dut_fixture def build_dir( diff --git a/tools/ci/idf_ci_local/app.py b/tools/ci/idf_ci_local/app.py index 301aa7e097f..4d1f177e59a 100644 --- a/tools/ci/idf_ci_local/app.py +++ b/tools/ci/idf_ci_local/app.py @@ -8,7 +8,9 @@ import typing as t from dynamic_pipelines.constants import BINARY_SIZE_METRIC_NAME from idf_build_apps import App from idf_build_apps import CMakeApp +from idf_build_apps.constants import BuildStatus from idf_build_apps.utils import rmdir +from idf_ci_utils import APP_EXTRA_S3_ARTIFACT_TYPE from idf_ci_utils import idf_relpath if t.TYPE_CHECKING: @@ -29,16 +31,34 @@ class IdfCMakeApp(CMakeApp): # only upload in CI if os.getenv('CI_JOB_ID'): - subprocess.run( + upload_commands = [ [ 'idf-ci', 'gitlab', 'upload-artifacts', self.app_dir, ], - stdout=sys.stdout, - stderr=sys.stderr, - ) + [ + 'idf-ci', + 'gitlab', + 'upload-artifacts', + self.app_dir, + '--type', + APP_EXTRA_S3_ARTIFACT_TYPE, + ], + ] + + for command in upload_commands: + result = subprocess.run( + command, + stdout=sys.stdout, + stderr=sys.stderr, + ) + if result.returncode != 0: + self.build_status = BuildStatus.FAILED + self.build_comment = 'Failed to upload artifacts' + break + rmdir( self.build_path, exclude_file_patterns=['build_log.txt', 'size*.json'], diff --git a/tools/ci/idf_ci_utils.py b/tools/ci/idf_ci_utils.py index b1138fa1daf..13bfab1b12f 100644 --- a/tools/ci/idf_ci_utils.py +++ b/tools/ci/idf_ci_utils.py @@ -13,6 +13,9 @@ from pathlib import Path IDF_PATH: str = os.path.abspath(os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..'))) +# S3 artifact type for app-dir files outside build_* +APP_EXTRA_S3_ARTIFACT_TYPE = 'app_extra' + def get_submodule_dirs(full_path: bool = False) -> t.List[str]: """