feat(ci): support downloading extra app artifacts in pytest

Add app_extra S3 artifact type, extend idf-ci download plumbing, and
expose a download_app_extra pytest fixture for managed_components deps.
This commit is contained in:
Ondrej Kosta
2026-06-03 09:45:32 +02:00
committed by Euripedes Rocha
parent 97abdf4e3b
commit 9e3ec66892
4 changed files with 74 additions and 6 deletions

View File

@@ -81,6 +81,15 @@ patterns = [
]
if_clause = 'CI_JOB_GROUP_NAME != "build_non_test_related_apps"'
[gitlab.artifacts.s3.app_extra]
bucket = "idf-artifacts"
# App-dir files needed at target-test time but outside build_* (uploaded without --build-dir).
# Add patterns here as needed, e.g. coverage info produced during build.
patterns = [
'**/managed_components/**/*.py',
]
if_clause = 'CI_JOB_GROUP_NAME != "build_non_test_related_apps"'
[gitlab.artifacts.s3.log]
bucket = "idf-artifacts"
patterns = [

View File

@@ -1,4 +1,4 @@
# SPDX-FileCopyrightText: 2021-2025 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
# pylint: disable=W0621 # redefined-outer-name
#
@@ -36,6 +36,8 @@ from _pytest.config import Config
from _pytest.fixtures import FixtureRequest
from idf_ci import PytestCase
from idf_ci.idf_pytest import IDF_CI_PYTEST_CASE_KEY
from idf_ci_utils import APP_EXTRA_S3_ARTIFACT_TYPE
from idf_ci_utils import IDF_PATH
from idf_ci_utils import idf_relpath
from idf_pytest.constants import DEFAULT_LOGDIR
from idf_pytest.plugin import IDF_LOCAL_PLUGIN_KEY
@@ -145,6 +147,33 @@ class AppDownloader:
stderr=sys.stderr,
)
def download_app_extra(self, app_dir: str) -> None:
"""Download app-dir artifacts defined under app_extra in .idf_ci.toml."""
args = [
'idf-ci',
'gitlab',
'download-artifacts',
'--commit-sha',
self.commit_sha,
'--type',
APP_EXTRA_S3_ARTIFACT_TYPE,
app_dir,
]
if self.pipeline_id:
args.extend(['--pipeline-id', self.pipeline_id])
result = subprocess.run(
args,
capture_output=True,
text=True,
cwd=IDF_PATH,
)
logging.info(result.stdout)
if result.stderr:
logging.info(result.stderr)
if result.returncode != 0:
raise RuntimeError(f'Failed to download {APP_EXTRA_S3_ARTIFACT_TYPE} artifacts for {app_dir}')
PRESIGNED_JSON = 'presigned.json'
@@ -278,6 +307,18 @@ def app_downloader(
return AppDownloader(commit_sha, pipeline_id)
@pytest.fixture
def download_app_extra(app_downloader: AppDownloader | None) -> t.Callable[[str], None]:
"""Download app_extra S3 artifacts for the given app path (no-op outside CI)."""
def _download_app_extra(app_path: str) -> None:
if app_downloader is None:
return
app_downloader.download_app_extra(idf_relpath(app_path))
return _download_app_extra
@pytest.fixture
@multi_dut_fixture
def build_dir(

View File

@@ -1,4 +1,4 @@
# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD
# SPDX-FileCopyrightText: 2024-2026 Espressif Systems (Shanghai) CO LTD
# SPDX-License-Identifier: Apache-2.0
import os
import subprocess
@@ -9,6 +9,7 @@ from dynamic_pipelines.constants import BINARY_SIZE_METRIC_NAME
from idf_build_apps import App
from idf_build_apps import CMakeApp
from idf_build_apps.utils import rmdir
from idf_ci_utils import APP_EXTRA_S3_ARTIFACT_TYPE
from idf_ci_utils import idf_relpath
if t.TYPE_CHECKING:
@@ -29,16 +30,30 @@ class IdfCMakeApp(CMakeApp):
# only upload in CI
if os.getenv('CI_JOB_ID'):
subprocess.run(
upload_commands = [
[
'idf-ci',
'gitlab',
'upload-artifacts',
self.app_dir,
],
stdout=sys.stdout,
stderr=sys.stderr,
)
[
'idf-ci',
'gitlab',
'upload-artifacts',
self.app_dir,
'--type',
APP_EXTRA_S3_ARTIFACT_TYPE,
],
]
for command in upload_commands:
subprocess.run(
command,
stdout=sys.stdout,
stderr=sys.stderr,
)
rmdir(
self.build_path,
exclude_file_patterns=['build_log.txt', 'size*.json'],

View File

@@ -12,6 +12,9 @@ from functools import cached_property
IDF_PATH: str = os.path.abspath(os.getenv('IDF_PATH', os.path.join(os.path.dirname(__file__), '..', '..')))
# S3 artifact type for app-dir files outside build_*
APP_EXTRA_S3_ARTIFACT_TYPE = 'app_extra'
def get_submodule_dirs(full_path: bool = False) -> list[str]:
"""