diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0e449ec7e9a..e55ad1825dd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,35 +1,22 @@ -workflow: - rules: - # Disable those non-protected push triggered pipelines - - if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && $CI_PIPELINE_SOURCE == "push"' - when: never - # when running merged result pipelines, CI_COMMIT_SHA represents the temp commit it created. - # Please use PIPELINE_COMMIT_SHA at all places that require a commit sha of the original commit. - - if: $CI_OPEN_MERGE_REQUESTS != null - variables: - PIPELINE_COMMIT_SHA: $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA - IS_MR_PIPELINE: 1 - - if: $CI_OPEN_MERGE_REQUESTS == null - variables: - PIPELINE_COMMIT_SHA: $CI_COMMIT_SHA - IS_MR_PIPELINE: 0 - - when: always - # Place the default settings in `.gitlab/ci/common.yml` instead include: - - '.gitlab/ci/danger.yml' - - '.gitlab/ci/common.yml' - - '.gitlab/ci/rules.yml' - - '.gitlab/ci/upload_cache.yml' - - '.gitlab/ci/docs.yml' - - '.gitlab/ci/static-code-analysis.yml' - - '.gitlab/ci/pre_commit.yml' - - '.gitlab/ci/pre_check.yml' - - '.gitlab/ci/build.yml' - - '.gitlab/ci/integration_test.yml' - - '.gitlab/ci/host-test.yml' - - '.gitlab/ci/deploy.yml' - - '.gitlab/ci/post_deploy.yml' - - '.gitlab/ci/retry_failed_jobs.yml' - - '.gitlab/ci/test-win.yml' + - project: "ci/actions/common" + file: + - "templates/stable/default-workflow.yml" + - "templates/idf/deploy-github.yml" + - "templates/idf/deploy-docs.yml" + - ".gitlab/ci/danger.yml" + - ".gitlab/ci/common.yml" + - ".gitlab/ci/rules.yml" + - ".gitlab/ci/upload_cache.yml" + - ".gitlab/ci/docs.yml" + - ".gitlab/ci/static-code-analysis.yml" + - ".gitlab/ci/pre_commit.yml" + - ".gitlab/ci/pre_check.yml" + - ".gitlab/ci/build.yml" + - ".gitlab/ci/integration_test.yml" + - ".gitlab/ci/host-test.yml" + - ".gitlab/ci/deploy.yml" + - ".gitlab/ci/post_deploy.yml" + - ".gitlab/ci/test-win.yml" diff --git a/.gitlab/ci/build.yml b/.gitlab/ci/build.yml index 7bd31a6f7c2..b67f345368e 100644 --- a/.gitlab/ci/build.yml +++ b/.gitlab/ci/build.yml @@ -1,55 +1,13 @@ .build_template: stage: build extends: - - .after_script:build:ccache-show-stats:upload-failed-job-logs + - .after_script:build image: $ESP_ENV_IMAGE - tags: - - build - # build only on shiny servers since shiny storage server is at the same location - - shiny + tags: [build, shiny] variables: # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings. IDF_CCACHE_ENABLE: "1" - dependencies: [] - -.build_cmake_clang_template: - extends: - - .build_template - - .before_script:build - - .after_script:build:ccache-show-stats dependencies: # set dependencies to null to avoid missing artifacts issue - needs: - - job: fast_template_app - artifacts: false - - pipeline_variables - artifacts: - paths: - # The other artifacts patterns are defined under tools/ci/artifacts_handler.py - # Now we're uploading/downloading the binary files from our internal storage server - # - # keep the log file to help debug - - "**/build*/build_log.txt" - # keep the size info to help track the binary size - - size_info.txt - - "**/build*/size.json" - expire_in: 1 week - when: always - variables: - IDF_TOOLCHAIN: clang - TEST_BUILD_OPTS_EXTRA: "" - TEST_DIR: tools/test_apps/system/clang_build_test - PYTEST_IGNORE_COLLECT_IMPORT_ERROR: "1" - script: - # CI specific options start from "--parallel-count xxx". could ignore when running locally - - run_cmd python tools/ci/ci_build_apps.py $TEST_DIR -v - -t $IDF_TARGET - --copy-sdkconfig - --parallel-count ${CI_NODE_TOTAL:-1} - --parallel-index ${CI_NODE_INDEX:-1} - --modified-components ${MR_MODIFIED_COMPONENTS} - --modified-files ${MR_MODIFIED_FILES} - $TEST_BUILD_OPTS_EXTRA - - python tools/ci/artifacts_handler.py upload ###################### # build_template_app # @@ -57,18 +15,11 @@ .build_template_app_template: extends: - .build_template - - .before_script:build - variables: - LOG_PATH: "${CI_PROJECT_DIR}/log_template_app" - BUILD_PATH: "${CI_PROJECT_DIR}/build_template_app" - BUILD_DIR: "${BUILD_PATH}/@t/@w" - BUILD_LOG_CMAKE: "${LOG_PATH}/cmake_@t_@w.txt" - BUILD_COMMAND_ARGS: "" artifacts: paths: - log_template_app/* - size_info.txt - - build_template_app/**/size.json + - build_template_app/**/size*.json expire_in: 1 week when: always script: @@ -82,174 +33,80 @@ # Only do the default cmake build for each target, remaining part are done in the build_template_app job - tools/ci/build_template_app.sh ${BUILD_COMMAND_ARGS} -# build-related-pre-check-jobs ------------------------------------------------ # Build at least one project for each target at earliest stage to reduce build cost for obvious failing commits fast_template_app: extends: - .build_template_app_template - - .rules:build:target_test + - .rules:build stage: pre_check tags: [fast_run, shiny] variables: BUILD_COMMAND_ARGS: "-p" -######################################## -# Clang Build Apps Without Tests Cases # -######################################## -build_clang_test_apps_esp32: - extends: - - .build_cmake_clang_template - - .rules:build - variables: - IDF_TARGET: esp32 - -build_clang_test_apps_esp32s2: - extends: - - .build_cmake_clang_template - - .rules:build - variables: - IDF_TARGET: esp32s2 - -build_clang_test_apps_esp32s3: - extends: - - .build_cmake_clang_template - - .rules:build - variables: - IDF_TARGET: esp32s3 - -.build_clang_test_apps_riscv: - extends: - - .build_cmake_clang_template - variables: - # https://reviews.llvm.org/D90108. - # GNU 'as' lets .weak override .globl since binutils-gdb - # https://github.com/bminor/binutils-gdb/commit/5ca547dc2399a0a5d9f20626d4bf5547c3ccfddd (1996) - # while MC lets the last directive win (PR38921). - # For RISCV chips we use integrated assembler by default, so suppress this warning to pass CI pipeline. - TEST_BUILD_OPTS_EXTRA: "--ignore-warning-str 'changed binding to STB_WEAK'" - -build_clang_test_apps_esp32c3: - extends: - - .build_clang_test_apps_riscv - - .rules:build - variables: - IDF_TARGET: esp32c3 - -build_clang_test_apps_esp32c2: - extends: - - .build_clang_test_apps_riscv - - .rules:build - variables: - IDF_TARGET: esp32c2 - -build_clang_test_apps_esp32c6: - extends: - - .build_clang_test_apps_riscv - - .rules:build - variables: - IDF_TARGET: esp32c6 - -build_clang_test_apps_esp32c5: - extends: - - .build_clang_test_apps_riscv - - .rules:build - variables: - IDF_TARGET: esp32c5 - -build_clang_test_apps_esp32h2: - extends: - - .build_clang_test_apps_riscv - - .rules:build - variables: - IDF_TARGET: esp32h2 - -build_clang_test_apps_esp32p4: - extends: - - .build_clang_test_apps_riscv - - .rules:build - variables: - IDF_TARGET: esp32p4 - -###################### -# Build System Tests # -###################### -.test_build_system_template: - stage: host_test - extends: - - .build_template - - .rules:build:check - dependencies: # set dependencies to null to avoid missing artifacts issue - needs: - - job: fast_template_app - artifacts: false - optional: true - artifacts: - reports: - junit: XUNIT_RESULT.xml - paths: - - XUNIT_RESULT.xml - - test_build_system - expire_in: 1 week - when: always - script: - - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh - - cd ${IDF_PATH}/tools/test_build_system - - python ${IDF_PATH}/tools/ci/get_known_failure_cases_file.py - - pytest - --cleanup-idf-copy - --parallel-count ${CI_NODE_TOTAL:-1} - --parallel-index ${CI_NODE_INDEX:-1} - --work-dir ${CI_PROJECT_DIR}/test_build_system - --junitxml ${CI_PROJECT_DIR}/XUNIT_RESULT.xml - --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} - -pytest_build_system: - extends: .test_build_system_template - parallel: 3 - -pytest_build_system_macos: - extends: - - .test_build_system_template - - .before_script:build:macos - - .after_script:build:macos:upload-failed-job-logs:ccache-show-stats - - .rules:build:macos - tags: - - macos_shell - parallel: 3 - variables: - PYENV_VERSION: "3.9" - # CCACHE_DIR: "/cache/idf_ccache". On macOS, you cannot write to this folder due to insufficient permissions. - CCACHE_DIR: "" # ccache will use "$HOME/Library/Caches/ccache". - -build_docker: - extends: - - .before_script:minimal - - .rules:build:docker - stage: host_test - needs: [] - image: espressif/docker-builder:1 - tags: [shiny, dind] - variables: - DOCKER_TMP_IMAGE_NAME: "idf_tmp_image" - script: - - export DOCKER_BUILD_ARGS="--build-arg IDF_CLONE_URL=${CI_REPOSITORY_URL} --build-arg IDF_CLONE_BRANCH_OR_TAG=${CI_COMMIT_REF_NAME} --build-arg IDF_CHECKOUT_REF=${CI_COMMIT_TAG:-$CI_COMMIT_SHA} --build-arg IDF_CLONE_SHALLOW=1 --build-arg IDF_GITHUB_ASSETS=${INTERNAL_GITHUB_ASSETS}" - - docker build --tag ${DOCKER_TMP_IMAGE_NAME} ${DOCKER_BUILD_ARGS} tools/docker/ - # We can't mount $PWD/examples/get-started/blink into the container, see https://gitlab.com/gitlab-org/gitlab-ce/issues/41227. - # The workaround mentioned there works, but leaves around directories which need to be cleaned up manually. - # Therefore, build a copy of the example located inside the container. - - docker run --rm --workdir /opt/esp/idf/examples/get-started/blink ${DOCKER_TMP_IMAGE_NAME} idf.py build - # This job builds template app with permutations of targets and optimization levels build_template_app: extends: - .build_template_app_template - .rules:build stage: host_test - dependencies: # set dependencies to null to avoid missing artifacts issue needs: - job: fast_template_app artifacts: false +######################################## +# Clang Build Apps Without Tests Cases # +######################################## +.build_cmake_clang_template: + extends: + - .build_template + - .rules:build + needs: + - job: fast_template_app + artifacts: false + - pipeline_variables + artifacts: + paths: + # The other artifacts patterns are defined under .idf_ci.toml + # Now we're uploading/downloading the binary files from our internal storage server + # + # keep the log file to help debug + - "**/build*/build_log.txt" + - "**/build*/size*.json" + expire_in: 1 week + when: always + variables: + IDF_TOOLCHAIN: clang + TEST_BUILD_OPTS_EXTRA: "" + script: + # CI specific options start from "--parallel-count xxx". could ignore when running locally + - run_cmd idf-build-apps build + -p tools/test_apps/system/clang_build_test + -t $IDF_TARGET + --parallel-count ${CI_NODE_TOTAL:-1} + --parallel-index ${CI_NODE_INDEX:-1} + --modified-components ${MR_MODIFIED_COMPONENTS} + --modified-files ${MR_MODIFIED_FILES} + $TEST_BUILD_OPTS_EXTRA + +build_clang_test_apps_xtensa: + extends: .build_cmake_clang_template + parallel: + matrix: + - IDF_TARGET: [esp32, esp32s2, esp32s3] + +build_clang_test_apps_riscv: + extends: .build_cmake_clang_template + variables: + # https://reviews.llvm.org/D90108. + # GNU 'as' lets .weak override .globl since binutils-gdb + # https://github.com/bminor/binutils-gdb/commit/5ca547dc2399a0a5d9f20626d4bf5547c3ccfddd (1996) + # while MC lets the last directive win (PR38921). + # For RISCV chips we use integrated assembler by default, so suppress this warning to pass CI pipeline. + TEST_BUILD_OPTS_EXTRA: "--ignore-warning-str 'changed binding to STB_WEAK'" + parallel: + matrix: + - IDF_TARGET: [esp32c3, esp32c2, esp32c6, esp32c5, esp32h2, esp32p4] + #################### # Dynamic Pipeline # #################### @@ -269,12 +126,12 @@ generate_build_child_pipeline: - non_test_related_apps.txt expire_in: 1 week when: always - variables: - PYTEST_IGNORE_COLLECT_IMPORT_ERROR: "1" script: - # requires basic pytest dependencies - - run_cmd bash install.sh --enable-pytest - - run_cmd python tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py + - run_cmd idf-ci --debug gitlab build-child-pipeline + -p components + -p examples + -p tools/test_apps + --modified-files $MR_MODIFIED_FILES build_child_pipeline: stage: build @@ -285,12 +142,9 @@ build_child_pipeline: - pipeline_variables - generate_build_child_pipeline variables: - IS_MR_PIPELINE: $IS_MR_PIPELINE MR_MODIFIED_COMPONENTS: $MR_MODIFIED_COMPONENTS MR_MODIFIED_FILES: $MR_MODIFIED_FILES PARENT_PIPELINE_ID: $CI_PIPELINE_ID - BUILD_AND_TEST_ALL_APPS: $BUILD_AND_TEST_ALL_APPS - REPORT_EXIT_CODE: $REPORT_EXIT_CODE # https://gitlab.com/gitlab-org/gitlab/-/issues/214340 inherit: variables: false @@ -299,3 +153,18 @@ build_child_pipeline: - artifact: build_child_pipeline.yml job: generate_build_child_pipeline strategy: depend + +generate_prebuild_report: + extends: + - .build_template + tags: [fast_run, shiny] + needs: + - pipeline_variables + artifacts: + paths: + - prebuild_report.html + expire_in: 1 week + when: always + script: + - run_cmd idf-ci build collect --format html -o prebuild_report.html + - echo "Report generated at https://${CI_PAGES_HOSTNAME}:${CI_SERVER_PORT}/-/esp-idf/-/jobs/${CI_JOB_ID}/artifacts/prebuild_report.html" diff --git a/.gitlab/ci/common.yml b/.gitlab/ci/common.yml index 0ccba06d75c..14a3d90ebde 100644 --- a/.gitlab/ci/common.yml +++ b/.gitlab/ci/common.yml @@ -12,7 +12,6 @@ stages: - test_deploy - deploy - post_deploy - - retry_failed_jobs variables: # System environment @@ -30,16 +29,12 @@ variables: # - set GIT_STRATEGY: "clone" to shiny runners # - set GIT_STRATEGY: "fetch" to brew runners GIT_STRATEGY: clone - # we will download archive for each submodule instead of clone. - # we don't do "recursive" when fetch submodule as they're not used in CI now. - GIT_SUBMODULE_STRATEGY: none - # since we're using merged-result pipelines, the last commit should work for most cases GIT_DEPTH: 1 - # --no-recurse-submodules: we use cache for submodules + GIT_SUBMODULE_STRATEGY: none # here we use cache for submodules, so we don't need to fetch them every time + # since we're using merged-result pipelines, the last commit should work for most cases # --prune --prune-tags: in case remote branch or tag is force pushed GIT_FETCH_EXTRA_FLAGS: "--no-recurse-submodules --prune --prune-tags" - # we're using .cache folder for caches - GIT_CLEAN_FLAGS: -ffdx -e .cache/ + LATEST_GIT_TAG: v5.5.3 SUBMODULE_FETCH_TOOL: "tools/ci/ci_fetch_submodule.py" @@ -59,7 +54,6 @@ variables: ESP_IDF_DOC_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/esp-idf-doc-env-v5.5:3-1" TARGET_TEST_ENV_IMAGE: "${CI_DOCKER_REGISTRY}/target-test-env-v5.5:2" SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:5" - PRE_COMMIT_IMAGE: "${CI_DOCKER_REGISTRY}/esp-idf-pre-commit:1" # cache python dependencies PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" @@ -101,6 +95,7 @@ variables: CCACHE_MAXSIZE: "50G" FF_USE_NEW_BASH_EVAL_STRATEGY: "true" + FORCE_COLOR: "1" # rich print with color ################################################ # `before_script` and `after_script` Templates # @@ -123,7 +118,7 @@ variables: source tools/ci/configure_ci_environment.sh # add extra python packages - export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH" + export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH" .setup_tools_and_idf_python_venv: &setup_tools_and_idf_python_venv | # must use after setup_tools_except_target_test @@ -131,7 +126,7 @@ variables: # download constraint file for dev if [[ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]]; then - wget -O /tmp/constraint.txt --header="Authorization:Bearer ${ESPCI_TOKEN}" ${GITLAB_HTTP_SERVER}/api/v4/projects/2581/repository/files/${CI_PYTHON_CONSTRAINT_FILE}/raw?ref=${CI_PYTHON_CONSTRAINT_BRANCH} + wget -O /tmp/constraint.txt --header="Authorization:Bearer ${ESPCI_TOKEN}" "${GITLAB_HTTP_SERVER}/api/v4/projects/2581/repository/files/${CI_PYTHON_CONSTRAINT_FILE}/raw?ref=${CI_PYTHON_CONSTRAINT_BRANCH}" mkdir -p ~/.espressif mv /tmp/constraint.txt ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE} fi @@ -140,46 +135,61 @@ variables: if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then export IDF_MIRROR_PREFIX_MAP= fi + # Optimize pip install + if echo "${CI_RUNNER_TAGS}" | grep "shiny"; then + export PIP_INDEX_URL="${PIP_INDEX_URL_SHINY}" + fi + if [[ "$(uname -m)" == "x86_64" ]] || [[ "$(uname -m)" == "aarch64" ]]; then + export IDF_PIP_WHEELS_URL="" + fi + # install.sh if [[ "${CI_JOB_STAGE}" != "target_test" ]]; then section_start "running_install_sh" "Running install.sh" if [[ "${CI_JOB_STAGE}" == "build_doc" ]]; then run_cmd bash install.sh --enable-ci --enable-docs - elif [[ "${CI_JOB_STAGE}" == "build" ]]; then - run_cmd bash install.sh --enable-ci else - if ! echo "${CI_JOB_NAME}" | egrep ".*pytest.*"; then - run_cmd bash install.sh --enable-ci - else - run_cmd bash install.sh --enable-ci --enable-pytest --enable-test-specific - fi + run_cmd bash install.sh --enable-ci fi section_end "running_install_sh" else - section_start "install_python_env" "Install Python environment" - run_cmd python tools/idf_tools.py install-python-env --features ci,pytest,test-specific + section_start "install_python_env" "Install Python environment, skip required tools check" + run_cmd python tools/idf_tools.py install-python-env --features ci,test-specific + export IDF_SKIP_TOOLS_CHECK=1 section_end "install_python_env" fi + section_start "source_export" "Source export.sh" + source ./export.sh + section_end "source_export" + + # Eager upgrade of CI dependencies + # Done after sourcing export.sh so that we could easily invoke the right pip + section_start "upgrade_ci_dependencies" "Upgrading CI dependencies" + pip install --upgrade --upgrade-strategy=eager -r $IDF_PATH/tools/requirements/requirements.ci.txt -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE} + section_end "upgrade_ci_dependencies" + + REEXPORT_NEEDED=0 if [[ ! -z "$INSTALL_EXTRA_TOOLS" ]]; then section_start "installing_optional_tools" "Install optional tools ${INSTALL_EXTRA_TOOLS}" - $IDF_PATH/tools/idf_tools.py --non-interactive install $INSTALL_EXTRA_TOOLS + run_cmd $IDF_PATH/tools/idf_tools.py --non-interactive install $INSTALL_EXTRA_TOOLS section_end "installing_optional_tools" + + REEXPORT_NEEDED=1 fi # Install esp-clang if necessary (esp-clang is separately installed) if [[ "$IDF_TOOLCHAIN" == "clang" && -z "$CI_CLANG_DISTRO_URL" ]]; then $IDF_PATH/tools/idf_tools.py --non-interactive install esp-clang + + REEXPORT_NEEDED=1 fi - if [[ "${CI_JOB_STAGE}" == "target_test" ]]; then - section_start "IDF_SKIP_TOOLS_CHECK" "Skip required tools check" - export IDF_SKIP_TOOLS_CHECK=1 - section_end "IDF_SKIP_TOOLS_CHECK" + if [[ $REEXPORT_NEEDED -eq 1 ]]; then + section_start "re_source_export" "Re-source export.sh" + source ./export.sh + section_end "re_source_export" fi - section_start "source_export" "Source export.sh" - source ./export.sh - section_end "source_export" # Custom clang toolchain if [[ "$IDF_TOOLCHAIN" == "clang" && ! -z "$CI_CLANG_DISTRO_URL" ]]; then @@ -218,161 +228,35 @@ variables: .show_ccache_statistics: &show_ccache_statistics | # Show ccache statistics if enabled globally + section_start "ccache_show_stats" "Show ccache statistics" test "$CI_CCACHE_STATS" == 1 && test -n "$(which ccache)" && ccache --show-stats -vv || true + section_end "ccache_show_stats" .upload_failed_job_log_artifacts: &upload_failed_job_log_artifacts | if [ $CI_JOB_STATUS = "failed" ]; then - python tools/ci/artifacts_handler.py upload --type logs + run_cmd idf-ci gitlab upload-artifacts --type log fi .before_script:minimal: before_script: - *common-before_scripts -.before_script:build:macos: - before_script: - # macos is running shell executor, which means it would use - # the system installed /usr/local/bin/python3 by default. - # Ensure pyenv and PYENV_VERSION installed - - eval "$(pyenv init -)" - - *common-before_scripts - # On macOS, these tools need to be installed - - export IDF_TOOLS_PATH="${HOME}/.espressif_runner_${CI_RUNNER_ID}_${CI_CONCURRENT_ID}" - # remove idf-env.json, since it may contains enabled "features" - - rm -f $IDF_TOOLS_PATH/idf-env.json - # This adds tools (compilers) and the version-specific Python environment to PATH - - *setup_tools_and_idf_python_venv - - fetch_submodules - variables: - INSTALL_EXTRA_TOOLS: cmake ninja - -.after_script:build:macos:upload-failed-job-logs:ccache-show-stats: - after_script: - # macos is running shell executor, which means it would use - # the system installed /usr/local/bin/python3 by default. - # Ensure pyenv and PYENV_VERSION installed - - eval "$(pyenv init -)" - - *upload_failed_job_log_artifacts - - *show_ccache_statistics - -.before_script:build: - before_script: - - *common-before_scripts - - *setup_tools_and_idf_python_venv - - add_gitlab_ssh_keys - - fetch_submodules - - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS} - - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS} - -.after_script:build:ccache-show-stats: - after_script: - - *show_ccache_statistics - -.after_script:build:ccache-show-stats:upload-failed-job-logs: +.after_script:build: after_script: + - source tools/ci/utils.sh - *show_ccache_statistics - *upload_failed_job_log_artifacts -############################## -# Git Strategy Job Templates # -############################## -.git_init: &git_init | - mkdir -p "${CI_PROJECT_DIR}" - cd "${CI_PROJECT_DIR}" - git init - -.git_fetch_from_mirror_url_if_exists: &git_fetch_from_mirror_url_if_exists | - # check if set mirror - if [ -n "${LOCAL_GITLAB_HTTPS_HOST:-}" ] && [ -n "${ESPCI_TOKEN:-}" ]; then - MIRROR_REPO_URL="https://bot:${ESPCI_TOKEN}@${LOCAL_GITLAB_HTTPS_HOST}/${CI_PROJECT_PATH}" - elif [ -n "${LOCAL_GIT_MIRROR:-}" ]; then - MIRROR_REPO_URL="${LOCAL_GIT_MIRROR}/${CI_PROJECT_PATH}" - fi - - # fetch from mirror first if set - if [ -n "${MIRROR_REPO_URL:-}" ]; then - if git remote -v | grep origin; then - git remote set-url origin "${MIRROR_REPO_URL}" - else - git remote add origin "${MIRROR_REPO_URL}" - fi - # mirror url may fail with authentication issue - git fetch origin --no-recurse-submodules || true - fi - - # set remote url to CI_REPOSITORY_URL - if git remote -v | grep origin; then - git remote set-url origin "${CI_REPOSITORY_URL}" - else - git remote add origin "${CI_REPOSITORY_URL}" - fi - -.git_checkout_ci_commit_sha: &git_checkout_ci_commit_sha | - git checkout $CI_COMMIT_SHA - git clean ${GIT_CLEAN_FLAGS} - -# git diff requires two commits, with different CI env var -# -# By default, we use git strategy "clone" with depth 1 to speed up the clone process. -# But for jobs requires running `git diff`, we need to fetch more commits to get the correct diffs. -# -# Since there's no way to get the correct git_depth before the job starts, -# we can't set `GIT_DEPTH` in the job definition. -# -# Set git strategy to "none" and fetch manually instead. -.before_script:fetch:git_diff: +.brew-macos-settings: variables: - GIT_STRATEGY: none - before_script: - - *git_init - - *git_fetch_from_mirror_url_if_exists - - | - # Store the diff output in a temporary file - TEMP_FILE=$(mktemp) - # merged results pipelines, by default - if [[ -n $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ]]; then - git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA > "$TEMP_FILE" - GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") - git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - # merge request pipelines, when the mr got conflicts - elif [[ -n $CI_MERGE_REQUEST_DIFF_BASE_SHA ]]; then - git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_COMMIT_SHA > "$TEMP_FILE" - GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") - # other pipelines, like the protected branches pipelines - elif [[ "$CI_COMMIT_BEFORE_SHA" != "0000000000000000000000000000000000000000" ]]; then - git fetch origin $CI_COMMIT_BEFORE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA > "$TEMP_FILE" - GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") - else - # pipeline source could be web, scheduler, etc. - git fetch origin $CI_COMMIT_SHA --depth=2 ${GIT_FETCH_EXTRA_FLAGS} - git diff --name-only $CI_COMMIT_SHA~1 $CI_COMMIT_SHA > "$TEMP_FILE" - GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") - fi - - *git_checkout_ci_commit_sha - - *common-before_scripts - - *setup_tools_and_idf_python_venv - - add_gitlab_ssh_keys - -# target test runners may locate in different places -# for runners set git mirror, we fetch from the mirror first, then fetch the HEAD commit -.before_script:fetch:target_test: - variables: - GIT_STRATEGY: none - before_script: - - *git_init - - *git_fetch_from_mirror_url_if_exists - - git fetch origin "${CI_COMMIT_SHA}" --depth=1 ${GIT_FETCH_EXTRA_FLAGS} - - *git_checkout_ci_commit_sha - - *common-before_scripts - - *setup_tools_and_idf_python_venv - - add_gitlab_ssh_keys - # no submodules + GIT_STRATEGY: fetch + IDF_CCACHE_ENABLE: "0" + CCACHE_DIR: "/var/tmp/cache/idf_ccache" + tags: + - macos-tart + image: macos-sequoia-idf-v5.5 + after_script: [] # ccache now is disabled for macos brew runners + timeout: 30m ############# # `default` # @@ -398,16 +282,5 @@ default: - *setup_tools_and_idf_python_venv - add_gitlab_ssh_keys - fetch_submodules -# gitlab bug, setting them here doesn't work -# - expire_in: https://gitlab.com/gitlab-org/gitlab/-/issues/404563 -# - when: https://gitlab.com/gitlab-org/gitlab/-/issues/440672 -# artifacts: -# expire_in: 1 week -# when: always - retry: - max: 2 - when: - # In case of a runner failure we could hop to another one, or a network error could go away. - - runner_system_failure - # Job execution timeout may be caused by a network issue. - - job_execution_timeout + - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS} + - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS} diff --git a/.gitlab/ci/default-build-test-rules.yml b/.gitlab/ci/default-build-test-rules.yml index 11a291dd3c9..62b2bf09d0e 100644 --- a/.gitlab/ci/default-build-test-rules.yml +++ b/.gitlab/ci/default-build-test-rules.yml @@ -2,15 +2,12 @@ # - extra_default_build_targets: # besides of the SUPPORTED_TARGETS in IDF, # enable build for the specified targets by default as well. +# !!! DEPRECATED: use `additional_build_targets` in .idf_build_apps.toml instead +# # - bypass_check_test_targets: # suppress the check_build_test_rules check-test-script warnings for the specified targets # # This file should ONLY be used during bringup. Should be reset to empty after the bringup process -extra_default_build_targets: - - esp32c5 - - esp32c61 - - esp32h21 - - esp32h4 bypass_check_test_targets: - esp32h21 diff --git a/.gitlab/ci/dependencies/dependencies.yml b/.gitlab/ci/dependencies/dependencies.yml index 1bc47df4f58..b667c15ddde 100644 --- a/.gitlab/ci/dependencies/dependencies.yml +++ b/.gitlab/ci/dependencies/dependencies.yml @@ -1,19 +1,3 @@ -.all_targets: &all_targets - - esp32 - - esp32s2 - - esp32s3 - - esp32c3 - - esp32c2 - - esp32c6 - - esp32c5 - - esp32h2 - - esp32p4 - -.target_test: &target_test - - example_test - - custom_test - - component_ut - ############## # Build Jobs # ############## @@ -25,7 +9,6 @@ - build_system - downloadable-tools included_in: - - build:target_test - build:check # ------------------- @@ -57,8 +40,6 @@ "patterns:template-app": patterns: - build_template-app - included_in: - - build:target_test "patterns:build-check": patterns: @@ -91,4 +72,4 @@ specific_rules: - if-schedule-test-build-system-windows patterns: - - build_system + - build_system_win diff --git a/.gitlab/ci/deploy.yml b/.gitlab/ci/deploy.yml index 58b951b4b56..c25fa7a8692 100644 --- a/.gitlab/ci/deploy.yml +++ b/.gitlab/ci/deploy.yml @@ -3,60 +3,19 @@ image: $ESP_ENV_IMAGE tags: [ deploy ] -# Check this before push_to_github -check_submodule_sync: - extends: - - .deploy_job_template - - .rules:test:submodule - stage: test_deploy - tags: [ brew, github_sync ] - retry: 2 - variables: - # for brew runners, we always set GIT_STRATEGY to fetch - GIT_STRATEGY: fetch - SUBMODULES_TO_FETCH: "none" - PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git" - dependencies: [] - script: - - git submodule deinit --force . - - rm -rf .git/modules # remove all the cached metadata - # setting the default remote URL to the public one, to resolve relative location URLs - - git config remote.origin.url ${PUBLIC_IDF_URL} - # check if all submodules are correctly synced to public repository - - git submodule init - - git config --get-regexp '^submodule\..*\.url$' || true - - git submodule update --recursive - - echo "IDF was cloned from ${PUBLIC_IDF_URL} completely" - -push_to_github: - extends: - - .deploy_job_template - - .before_script:minimal - - .rules:push_to_github - needs: - - check_submodule_sync - tags: [ brew, github_sync ] - variables: - # for brew runners, we always set GIT_STRATEGY to fetch - GIT_STRATEGY: fetch - # github also need full record of commits - GIT_DEPTH: 0 - script: - - add_github_ssh_keys - - git remote remove github &>/dev/null || true - - git remote add github git@github.com:espressif/esp-idf.git - - tools/ci/push_to_github.sh - deploy_update_SHA_in_esp-dockerfiles: extends: - .deploy_job_template - .before_script:minimal - - .rules:protected-no_label-always + - .rules:protected:deploy dependencies: [] variables: GIT_DEPTH: 2 - tags: [ shiny, build ] + tags: [build, shiny] script: - 'curl --header "PRIVATE-TOKEN: ${ESPCI_SCRIPTS_TOKEN}" -o create_MR_in_esp_dockerfile.sh $GITLAB_HTTP_SERVER/api/v4/projects/1260/repository/files/create_MR_in_esp_dockerfile%2Fcreate_MR_in_esp_dockerfile.sh/raw\?ref\=master' - chmod +x create_MR_in_esp_dockerfile.sh - ./create_MR_in_esp_dockerfile.sh + environment: + name: deploy_update_SHA_in_esp-dockerfiles_production + deployment_tier: production diff --git a/.gitlab/ci/docs.yml b/.gitlab/ci/docs.yml index 3ea791048af..bf704072948 100644 --- a/.gitlab/ci/docs.yml +++ b/.gitlab/ci/docs.yml @@ -4,11 +4,11 @@ - "**/*.rst" - "CONTRIBUTING.rst" - "**/soc_caps.h" + - "**/Kconfig*" .patterns-docs-partial: &patterns-docs-partial - "components/**/*.h" - - "components/**/Kconfig*" - - "components/**/CMakeList.txt" + - "components/**/CMakeLists.txt" - "components/**/sdkconfig*" - "tools/tools.json" - "tools/idf_tools.py" @@ -16,15 +16,9 @@ .patterns-example-readme: &patterns-example-readme - "examples/**/*.md" -.patterns-docs-preview: &patterns-docs-preview - - "docs/**/*" - -.if-protected: &if-protected +.if-protected-check: &if-protected-check if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/)' -.if-protected-no_label: &if-protected-no_label - if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && $BOT_TRIGGER_WITH_LABEL == null' - .if-qa-test-tag: &if-qa-test-tag if: '$CI_COMMIT_TAG =~ /^qa-test/' @@ -41,7 +35,7 @@ rules: - <<: *if-qa-test-tag when: never - - <<: *if-protected + - <<: *if-protected-check - <<: *if-label-build_docs - <<: *if-label-docs_full - <<: *if-dev-push @@ -61,10 +55,12 @@ check_readme_links: extends: - .pre_check_template - tags: ["build", "amd64", "internet"] + variables: + GIT_STRATEGY: "fetch" + tags: ["check_doc_links"] allow_failure: true rules: - - <<: *if-protected + - <<: *if-protected-check - <<: *if-dev-push changes: *patterns-example-readme script: @@ -103,6 +99,9 @@ check_docs_gh_links: extends: - .pre_check_template - .doc-rules:build:docs-full + variables: + GIT_STRATEGY: "fetch" + tags: ["check_doc_links"] script: - cd docs - build-docs gh-linkcheck @@ -141,80 +140,17 @@ build_docs_html_partial: - DOCLANG: "zh_CN" DOCTGT: "esp32p4" -.deploy_docs_template: - image: $ESP_IDF_DOC_ENV_IMAGE - variables: - DOCS_BUILD_DIR: "${IDF_PATH}/docs/_build/" - PYTHONUNBUFFERED: 1 - # ensure all tags are fetched, need to know the latest/stable tag for the docs - GIT_STRATEGY: clone - GIT_DEPTH: 0 - stage: test_deploy - tags: - - deploy - - shiny - script: - # ensure all tags are fetched, need to know the latest/stable tag for the docs - - git fetch --tags --prune - - add_doc_server_ssh_keys $DOCS_DEPLOY_PRIVATEKEY $DOCS_DEPLOY_SERVER $DOCS_DEPLOY_SERVER_USER - - export GIT_VER=$(git describe --always ${PIPELINE_COMMIT_SHA} --) - - deploy-docs - -# stage: test_deploy -deploy_docs_preview: - extends: - - .deploy_docs_template - rules: - - <<: *if-label-build_docs - - <<: *if-label-docs_full - - <<: *if-dev-push - changes: *patterns-docs-preview - needs: - - job: build_docs_html_partial - optional: true - - job: build_docs_html_full - optional: true - variables: - TYPE: "preview" - # older branches use DOCS_DEPLOY_KEY, DOCS_SERVER, DOCS_SERVER_USER, DOCS_PATH for preview server so we keep these names for 'preview' - DOCS_DEPLOY_PRIVATEKEY: "$DOCS_DEPLOY_KEY" - DOCS_DEPLOY_SERVER: "$DOCS_SERVER" - DOCS_DEPLOY_SERVER_USER: "$DOCS_SERVER_USER" - DOCS_DEPLOY_PATH: "$DOCS_PATH" - DOCS_DEPLOY_URL_BASE: "https://$DOCS_PREVIEW_SERVER_URL/docs/esp-idf" - -# stage: post_deploy -deploy_docs_production: - # The DOCS_PROD_* variables used by this job are "Protected" so these branches must all be marked "Protected" in Gitlab settings - extends: - - .deploy_docs_template - rules: - - <<: *if-protected-no_label - stage: post_deploy - dependencies: # set dependencies to null to avoid missing artifacts issue - needs: # ensure runs after push_to_github succeeded - - build_docs_html_full - - job: push_to_github - artifacts: false - variables: - TYPE: "preview" - DOCS_DEPLOY_PRIVATEKEY: "$DOCS_PROD_DEPLOY_KEY" - DOCS_DEPLOY_SERVER: "$DOCS_PROD_SERVER" - DOCS_DEPLOY_SERVER_USER: "$DOCS_PROD_SERVER_USER" - DOCS_DEPLOY_PATH: "$DOCS_PROD_PATH" - DOCS_DEPLOY_URL_BASE: "https://docs.espressif.com/projects/esp-idf" - DEPLOY_STABLE: 1 - check_doc_links: extends: - .build_docs_template - rules: - - <<: *if-protected-no_label + - .rules:protected:deploy stage: post_deploy needs: - job: deploy_docs_production artifacts: false - tags: ["build", "amd64", "internet"] + variables: + GIT_STRATEGY: "fetch" + tags: ["check_doc_links"] artifacts: when: always paths: diff --git a/.gitlab/ci/host-test.yml b/.gitlab/ci/host-test.yml index eded76d04b5..cae47752a7c 100644 --- a/.gitlab/ci/host-test.yml +++ b/.gitlab/ci/host-test.yml @@ -2,18 +2,20 @@ extends: .rules:test:host_test stage: host_test image: $ESP_ENV_IMAGE - tags: - - host_test + tags: [build, shiny] dependencies: # set dependencies to null to avoid missing artifacts issue # run host_test jobs immediately, only after upload cache needs: + - pipeline_variables - job: upload-pip-cache optional: true artifacts: false - job: upload-submodules-cache optional: true artifacts: false - - pipeline_variables + - job: fast_template_app + optional: true + artifacts: false artifacts: expire_in: 1 week when: always @@ -53,14 +55,15 @@ test_partition_table_on_host: extends: .host_test_template script: - cd components/partition_table/test_gen_esp32part_host - - ./gen_esp32part_tests.py + - pytest_for_ut ./gen_esp32part_tests.py test_ldgen_on_host: - extends: .host_test_template + extends: + - .host_test_template script: - cd tools/ldgen/test - export PYTHONPATH=$PYTHONPATH:.. - - python -m unittest + - pytest_for_ut . variables: LC_ALL: C.UTF-8 @@ -68,15 +71,13 @@ test_spiffs_on_host: extends: .host_test_template script: - cd components/spiffs/test_spiffsgen/ - - ./test_spiffsgen.py + - pytest_for_ut ./test_spiffsgen.py test_fatfsgen_on_host: extends: .host_test_template script: - cd components/fatfs/test_fatfsgen/ - - ./test_fatfsgen.py - - ./test_wl_fatfsgen.py - - ./test_fatfsparse.py + - pytest_for_ut ./test_fatfsgen.py ./test_wl_fatfsgen.py ./test_fatfsparse.py test_multi_heap_on_host: extends: .host_test_template @@ -88,7 +89,7 @@ test_certificate_bundle_on_host: extends: .host_test_template script: - cd components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/ - - ./test_gen_crt_bundle.py + - pytest_for_ut ./test_gen_crt_bundle.py test_gdbstub_on_host: extends: .host_test_template @@ -117,15 +118,17 @@ test_cli_installer: - python3 ${IDF_PATH}/tools/idf_tools.py download required qemu-riscv32 qemu-xtensa cmake cmake@3.16.3 - cd ${IDF_PATH}/tools/test_idf_tools - python3 -m pip install jsonschema + # Testing with system python3, so don't use any third-party packages - python3 ./test_idf_tools.py -v - python3 ./test_idf_tools_python_env.py # It runs at the end because it modifies dependencies - IDF_TEST_MAY_BREAK_DEPENDENCIES=1 python3 ./test_idf_tools.py -v TestSystemDependencies.test_commands_when_nodeps -.test_efuse_table_on_host_template: +test_efuse_table_on_host: extends: .host_test_template - variables: - IDF_TARGET: "esp32" + parallel: + matrix: + - IDF_TARGET: [esp32, esp32s2, esp32c3, esp32s3, esp32c6, esp32h2] artifacts: when: on_failure paths: @@ -135,35 +138,7 @@ test_cli_installer: - ./efuse_table_gen.py -t "${IDF_TARGET}" ${IDF_PATH}/components/efuse/${IDF_TARGET}/esp_efuse_table.csv - git diff --exit-code -- ${IDF_TARGET}/esp_efuse_table.c || { echo 'Differences found for ${IDF_TARGET} target. Please run idf.py efuse-common-table and commit the changes.'; exit 1; } - cd ${IDF_PATH}/components/efuse/test_efuse_host - - ./efuse_tests.py - -test_efuse_table_on_host_esp32: - extends: .test_efuse_table_on_host_template - -test_efuse_table_on_host_esp32s2: - extends: .test_efuse_table_on_host_template - variables: - IDF_TARGET: esp32s2 - -test_efuse_table_on_host_esp32s3: - extends: .test_efuse_table_on_host_template - variables: - IDF_TARGET: esp32s3 - -test_efuse_table_on_host_esp32c3: - extends: .test_efuse_table_on_host_template - variables: - IDF_TARGET: esp32c3 - -test_efuse_table_on_host_esp32h2: - extends: .test_efuse_table_on_host_template - variables: - IDF_TARGET: esp32h2 - -test_efuse_table_on_host_esp32c6: - extends: .test_efuse_table_on_host_template - variables: - IDF_TARGET: esp32c6 + - pytest_for_ut ./efuse_tests.py test_logtrace_proc: extends: .host_test_template @@ -201,20 +176,21 @@ test_tools: INSTALL_EXTRA_TOOLS: "qemu-xtensa qemu-riscv32" # for test_idf_qemu.py script: - stat=0 + - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} - cd ${IDF_PATH}/tools/ci/test_autocomplete - - pytest --noconftest test_autocomplete.py --junitxml=${IDF_PATH}/XUNIT_AUTOCOMP.xml || stat=1 + - run_cmd pytest --noconftest test_autocomplete.py --junitxml=${IDF_PATH}/XUNIT_AUTOCOMP.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 - cd ${IDF_PATH}/tools/test_idf_py - - pytest --noconftest test_idf_py.py --junitxml=${IDF_PATH}/XUNIT_IDF_PY.xml || stat=1 - - pytest --noconftest test_hints.py --junitxml=${IDF_PATH}/XUNIT_HINTS.xml || stat=1 - - pytest --noconftest test_idf_qemu.py --junitxml=${IDF_PATH}/XUNIT_IDF_PY_QEMU.xml || stat=1 + - run_cmd pytest --noconftest test_idf_py.py --junitxml=${IDF_PATH}/XUNIT_IDF_PY.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 + - run_cmd pytest --noconftest test_hints.py --junitxml=${IDF_PATH}/XUNIT_HINTS.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 + - run_cmd pytest --noconftest test_idf_qemu.py --junitxml=${IDF_PATH}/XUNIT_IDF_PY_QEMU.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 - cd ${IDF_PATH}/tools/test_bsasm - - pytest --noconftest test_bsasm.py --junitxml=${IDF_PATH}/XUNIT_BSASM.xml || stat=1 + - run_cmd pytest --noconftest test_bsasm.py --junitxml=${IDF_PATH}/XUNIT_BSASM.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 - cd ${IDF_PATH}/tools/test_mkdfu - - pytest --noconftest test_mkdfu.py --junitxml=${IDF_PATH}/XUNIT_MKDFU.xml || stat=1 + - run_cmd pytest --noconftest test_mkdfu.py --junitxml=${IDF_PATH}/XUNIT_MKDFU.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 - cd ${IDF_PATH}/tools/test_idf_size - - pytest --noconftest test_idf_size.py --junitxml=${IDF_PATH}/XUNIT_IDF_SIZE.xml || stat=1 + - run_cmd pytest --noconftest test_idf_size.py --junitxml=${IDF_PATH}/XUNIT_IDF_SIZE.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 - cd ${IDF_PATH}/tools/test_idf_diag - - pytest --noconftest test_idf_diag.py --junitxml=${IDF_PATH}/XUNIT_IDF_DIAG.xml || stat=1 + - run_cmd pytest --noconftest test_idf_diag.py --junitxml=${IDF_PATH}/XUNIT_IDF_DIAG.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} || stat=1 - cd ${IDF_PATH} - shellcheck -s sh tools/detect_python.sh || stat=1 - shellcheck -s bash tools/detect_python.sh || stat=1 @@ -229,12 +205,15 @@ test_split_path_by_spaces: extends: .host_test_template script: - cd ${IDF_PATH}/tools - - python -m unittest split_paths_by_spaces.py + - pytest_for_ut ./split_paths_by_spaces.py test_mqtt_on_host: extends: .host_test_template script: - cd ${IDF_PATH}/components/mqtt/esp-mqtt/host_test + # no -Werror + - export EXTRA_CFLAGS= + - export EXTRA_CXXFLAGS= - idf.py build - LSAN_OPTIONS=verbosity=1:log_threads=1 build/host_mqtt_client_test.elf @@ -275,126 +254,13 @@ test_gen_soc_caps_kconfig: extends: .host_test_template script: - cd ${IDF_PATH}/tools/gen_soc_caps_kconfig/ - - ./test/test_gen_soc_caps_kconfig.py - -test_pytest_qemu: - extends: - - .host_test_template - - .before_script:build - artifacts: - paths: - - XUNIT_RESULT.xml - - pytest-embedded/ - - "**/build*/*.bin" - reports: - junit: XUNIT_RESULT.xml - parallel: - matrix: - - IDF_TARGET: "esp32" - INSTALL_EXTRA_TOOLS: "qemu-xtensa" - # Skip Clang + Xtensa tests due to bootloader size issue - IDF_TOOLCHAIN: [gcc] - - IDF_TARGET: "esp32c3" - INSTALL_EXTRA_TOOLS: "qemu-riscv32" - IDF_TOOLCHAIN: [gcc, clang] - script: - - run_cmd python tools/ci/ci_build_apps.py . -v - --target $IDF_TARGET - --pytest-apps - -m qemu - --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" - --modified-components ${MR_MODIFIED_COMPONENTS} - --modified-files ${MR_MODIFIED_FILES} - - python tools/ci/get_known_failure_cases_file.py - - run_cmd pytest - --target $IDF_TARGET - --log-cli-level DEBUG - -m qemu - --embedded-services idf,qemu - --junitxml=XUNIT_RESULT.xml - --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} - --app-info-filepattern \"list_job_*.txt\" - --qemu-extra-args \"-global driver=timer.$IDF_TARGET.timg,property=wdt_disable,value=true\" - -test_pytest_linux: - extends: - - .host_test_template - - .before_script:build - artifacts: - paths: - - XUNIT_RESULT.xml - - pytest-embedded/ - - "**/build*/build_log.txt" - reports: - junit: XUNIT_RESULT.xml - script: - - run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -v - --target linux - --pytest-apps - -m host_test - --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" - --modified-components ${MR_MODIFIED_COMPONENTS} - --modified-files ${MR_MODIFIED_FILES} - - python tools/ci/get_known_failure_cases_file.py - - run_cmd pytest - --target linux - -m host_test - --embedded-services idf - --junitxml=XUNIT_RESULT.xml - --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} - --app-info-filepattern \"list_job_*.txt\" - -test_pytest_macos: - extends: - - .host_test_template - - .before_script:build:macos - tags: - - macos_shell - artifacts: - paths: - - XUNIT_RESULT.xml - - pytest-embedded/ - - "**/build*/build_log.txt" - reports: - junit: XUNIT_RESULT.xml - variables: - PYENV_VERSION: "3.9" - PYTEST_IGNORE_COLLECT_IMPORT_ERROR: "1" - script: - - run_cmd python tools/ci/ci_build_apps.py components examples tools/test_apps -v - --target linux - --pytest-apps - -m \"host_test and macos_shell\" - --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" - --modified-components ${MR_MODIFIED_COMPONENTS} - --modified-files ${MR_MODIFIED_FILES} - - python tools/ci/get_known_failure_cases_file.py - - run_cmd pytest - --target linux - -m \"host_test and macos_shell\" - --junitxml=XUNIT_RESULT.xml - --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} - --app-info-filepattern \"list_job_*.txt\" - -test_idf_pytest_plugin: - extends: - - .host_test_template - - .rules:patterns:idf-pytest-plugin - variables: - SUBMODULES_TO_FETCH: "none" - artifacts: - reports: - junit: XUNIT_RESULT.xml - script: - - cd ${IDF_PATH}/tools/ci/dynamic_pipelines/tests/test_report_generator - - python -m unittest test_report_generator.py - - cd ${IDF_PATH}/tools/ci/idf_pytest - - pytest --junitxml=${CI_PROJECT_DIR}/XUNIT_RESULT.xml + - pytest_for_ut ./test/test_gen_soc_caps_kconfig.py test_idf_build_apps_load_soc_caps: extends: .host_test_template script: - - python tools/ci/check_soc_headers_load_in_idf_build_apps.py + - cd tools/ci + - pytest_for_ut ./check_soc_headers_load_in_idf_build_apps.py test_nvs_gen_check: extends: .host_test_template @@ -408,7 +274,7 @@ test_nvs_gen_check: LC_ALL: C.UTF-8 script: - cd ${IDF_PATH}/components/nvs_flash/nvs_partition_tool - - pytest --noconftest test_nvs_gen_check.py --junitxml=XUNIT_RESULT.xml + - pytest_for_ut test_nvs_gen_check.py test_esp_rom: extends: .host_test_template @@ -419,7 +285,7 @@ test_esp_rom: junit: XUNIT_RESULT.xml script: - cd ${IDF_PATH}/components/esp_rom/ - - pytest --noconftest test_esp_rom.py --junitxml=XUNIT_RESULT.xml + - pytest_for_ut test_esp_rom.py make_sure_soc_caps_compatible_in_idf_build_apps: extends: @@ -438,3 +304,152 @@ make_sure_soc_caps_compatible_in_idf_build_apps: - git checkout $CI_COMMIT_SHA -- tools/ci/idf_build_apps_dump_soc_caps.py - python tools/ci/idf_build_apps_dump_soc_caps.py base.json - diff new.json base.json + +build_docker: + extends: + - .before_script:minimal + - .rules:build:docker + stage: host_test + needs: [] + image: espressif/docker-builder:1 + tags: [shiny, dind] + variables: + DOCKER_TMP_IMAGE_NAME: "idf_tmp_image" + script: + - export DOCKER_BUILD_ARGS="--build-arg IDF_CLONE_URL=${CI_REPOSITORY_URL} --build-arg IDF_CLONE_BRANCH_OR_TAG=${CI_COMMIT_REF_NAME} --build-arg IDF_CHECKOUT_REF=${CI_COMMIT_TAG:-$CI_COMMIT_SHA} --build-arg IDF_CLONE_SHALLOW=1 --build-arg IDF_GITHUB_ASSETS=${INTERNAL_GITHUB_ASSETS}" + - docker build --tag ${DOCKER_TMP_IMAGE_NAME} ${DOCKER_BUILD_ARGS} tools/docker/ + # We can't mount $PWD/examples/get-started/blink into the container, see https://gitlab.com/gitlab-org/gitlab-ce/issues/41227. + # The workaround mentioned there works, but leaves around directories which need to be cleaned up manually. + # Therefore, build a copy of the example located inside the container. + - docker run --rm --workdir /opt/esp/idf/examples/get-started/blink ${DOCKER_TMP_IMAGE_NAME} idf.py build + +############################ +# Host test with test apps # +############################ +test_pytest_qemu: + extends: + - .host_test_template + artifacts: + paths: + - XUNIT_RESULT.xml + - pytest-embedded/ + - "**/build*/*.bin" + reports: + junit: XUNIT_RESULT.xml + parallel: + matrix: + - IDF_TARGET: "esp32" + INSTALL_EXTRA_TOOLS: "qemu-xtensa" + # Skip Clang + Xtensa tests due to bootloader size issue + IDF_TOOLCHAIN: [gcc] + - IDF_TARGET: "esp32c3" + INSTALL_EXTRA_TOOLS: "qemu-riscv32" + IDF_TOOLCHAIN: [gcc, clang] + script: + - run_cmd idf-ci build run + --build-system cmake + --target $IDF_TARGET + -m qemu + --modified-files ${MR_MODIFIED_FILES} + - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} + - run_cmd pytest + --target $IDF_TARGET + -m qemu + --embedded-services idf,qemu + --junitxml=XUNIT_RESULT.xml + --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} + --qemu-extra-args \"-global driver=timer.$IDF_TARGET.timg,property=wdt_disable,value=true\" + +test_pytest_linux: + extends: + - .host_test_template + artifacts: + paths: + - XUNIT_RESULT.xml + - pytest-embedded/ + - "**/build*/build_log.txt" + reports: + junit: XUNIT_RESULT.xml + script: + - run_cmd idf-ci build run + --build-system cmake + -p components -p examples -p tools/test_apps + --target linux + --only-test-related + --modified-files ${MR_MODIFIED_FILES} + - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} + - run_cmd pytest + --target linux + --embedded-services idf + --junitxml=XUNIT_RESULT.xml + --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} + +test_pytest_macos: + extends: + - .host_test_template + - .brew-macos-settings + artifacts: + paths: + - XUNIT_RESULT.xml + - pytest-embedded/ + - "**/build*/build_log.txt" + reports: + junit: XUNIT_RESULT.xml + script: + - run_cmd idf-ci build run + -p components -p examples -p tools/test_apps + --build-system cmake + --target linux + --only-test-related + -m macos + --modified-files ${MR_MODIFIED_FILES} + - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} + - run_cmd pytest + --target linux + -m macos + --junitxml=XUNIT_RESULT.xml + --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} + +###################### +# Build System Tests # +###################### +.test_build_system_template: + stage: host_test + extends: + - .build_template + - .rules:build:check + dependencies: # set dependencies to null to avoid missing artifacts issue + needs: + - job: fast_template_app + artifacts: false + optional: true + artifacts: + reports: + junit: XUNIT_RESULT.xml + paths: + - XUNIT_RESULT.xml + - test_build_system + expire_in: 1 week + when: always + script: + - ${IDF_PATH}/tools/ci/test_configure_ci_environment.sh + - cd ${IDF_PATH}/tools/test_build_system + - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} + - pytest + --cleanup-idf-copy + --parallel-count ${CI_NODE_TOTAL:-1} + --parallel-index ${CI_NODE_INDEX:-1} + --work-dir ${CI_PROJECT_DIR}/test_build_system + --junitxml ${CI_PROJECT_DIR}/XUNIT_RESULT.xml + --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} + +pytest_build_system: + extends: .test_build_system_template + parallel: 3 + +pytest_build_system_macos: + extends: + - .test_build_system_template + - .brew-macos-settings + - .rules:build:macos + parallel: 3 diff --git a/.gitlab/ci/post_deploy.yml b/.gitlab/ci/post_deploy.yml index efa91f44eb7..c17e94089f7 100644 --- a/.gitlab/ci/post_deploy.yml +++ b/.gitlab/ci/post_deploy.yml @@ -1,11 +1,17 @@ -generate_failed_jobs_report: +.post_deploy_template: stage: post_deploy - tags: [build, shiny] image: $ESP_ENV_IMAGE + +generate_failed_jobs_report: + extends: + - .post_deploy_template + tags: [build, shiny] when: always - dependencies: [] # Do not download artifacts from the previous stages + dependencies: # Do not download artifacts from the previous stages + needs: + - pipeline_variables artifacts: - expire_in: 1 week + expire_in: 2 week when: always paths: - job_report.html @@ -13,12 +19,16 @@ generate_failed_jobs_report: - python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type job sync_support_status: - stage: post_deploy extends: - - .rules:sync_support_status + - .post_deploy_template + - .rules:master:push + tags: [ brew, github_sync ] needs: - push_to_github - image: $ESP_ENV_IMAGE - tags: [ brew, github_sync ] + cache: [] + before_script: [] script: - curl --fail --request POST --form token="$IDF_STATUS_TRIG_TOKEN" --form ref="$IDF_STATUS_BRANCH" --form "variables[UPLOAD_TO_S3]=true" "$IDF_STATUS_TRIG_URL" + environment: + name: sync_support_status_production + deployment_tier: production diff --git a/.gitlab/ci/pre_check.yml b/.gitlab/ci/pre_check.yml index 26e58c9130d..5f87d341e59 100644 --- a/.gitlab/ci/pre_check.yml +++ b/.gitlab/ci/pre_check.yml @@ -1,8 +1,7 @@ .pre_check_template: stage: pre_check image: $ESP_ENV_IMAGE - tags: - - host_test + tags: [build, shiny] dependencies: # set dependencies to null to avoid missing artifacts issue check_version: @@ -10,7 +9,7 @@ check_version: # esp_idf_version.h in a branch before tagging the next version. extends: - .pre_check_template - - .rules:protected + - .rules:protected:check tags: [ brew, github_sync ] variables: # need a full clone to get the latest tag @@ -116,37 +115,74 @@ check_version_tag: check_test_scripts_build_test_rules: extends: - .pre_check_template - - .before_script:build - variables: - PYTEST_IGNORE_COLLECT_IMPORT_ERROR: "1" script: # requires basic pytest dependencies - - run_cmd bash install.sh --enable-pytest - python tools/ci/check_build_test_rules.py check-test-scripts examples/ tools/test_apps components -check_configure_ci_environment_parsing: +check_submodule_sync: extends: - .pre_check_template - - .before_script:build - - .rules:build + - .rules:test:submodule + tags: [ brew, github_sync ] + retry: 2 + variables: + GIT_STRATEGY: fetch # use brew local mirror first + SUBMODULES_TO_FETCH: "none" + PUBLIC_IDF_URL: "https://github.com/espressif/esp-idf.git" script: - - cd tools/ci - - python -m unittest ci_build_apps.py + - git submodule deinit --force . + - rm -rf .git/modules # remove all the cached metadata + # setting the default remote URL to the public one, to resolve relative location URLs + - git config remote.origin.url ${PUBLIC_IDF_URL} + # check if all submodules are correctly synced to public repository + - git submodule init + - git config --get-regexp '^submodule\..*\.url$' || true + - git submodule update --recursive + - echo "IDF was cloned from ${PUBLIC_IDF_URL} completely" pipeline_variables: extends: - .pre_check_template - - .before_script:fetch:git_diff tags: [fast_run, shiny] + variables: + GIT_STRATEGY: "fetch" script: + - | + # Store the diff output in a temporary file + TEMP_FILE=$(mktemp) + # merged results pipelines, by default + if [[ -n $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA ]]; then + git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_SHA + git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_SHA + + git diff --name-only $CI_MERGE_REQUEST_TARGET_BRANCH_SHA...$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA > "$TEMP_FILE" + GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") + git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} + # merge request pipelines, when the mr got conflicts + elif [[ -n $CI_MERGE_REQUEST_DIFF_BASE_SHA ]]; then + git fetch origin $CI_MERGE_REQUEST_DIFF_BASE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} + git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} + git diff --name-only $CI_MERGE_REQUEST_DIFF_BASE_SHA $CI_COMMIT_SHA > "$TEMP_FILE" + GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") + # other pipelines, like the protected branches pipelines + elif [[ "$CI_COMMIT_BEFORE_SHA" != "0000000000000000000000000000000000000000" ]]; then + git fetch origin $CI_COMMIT_BEFORE_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} + git fetch origin $CI_COMMIT_SHA --depth=1 ${GIT_FETCH_EXTRA_FLAGS} + git diff --name-only $CI_COMMIT_BEFORE_SHA $CI_COMMIT_SHA > "$TEMP_FILE" + GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") + else + # pipeline source could be web, scheduler, etc. + git fetch origin $CI_COMMIT_SHA --depth=2 ${GIT_FETCH_EXTRA_FLAGS} + git diff --name-only $CI_COMMIT_SHA~1 $CI_COMMIT_SHA > "$TEMP_FILE" + GIT_DIFF_OUTPUT=$(cat "$TEMP_FILE") + fi # MODIFIED_FILES is a list of files that changed, could be used everywhere - MODIFIED_FILES=$(echo "$GIT_DIFF_OUTPUT" | xargs) - echo "MODIFIED_FILES=$MODIFIED_FILES" >> pipeline.env - - echo "REPORT_EXIT_CODE=0" >> pipeline.env # MR_MODIFIED_FILES and MR_MODIFIED_COMPONENTS are semicolon separated lists that is used in MR only # for non MR pipeline, these are empty lists - | - if [ $IS_MR_PIPELINE == "0" ]; then + if [ -z "$CI_MERGE_REQUEST_IID" ]; then echo "MR_MODIFIED_FILES=\"\"" >> pipeline.env echo "MR_MODIFIED_COMPONENTS=\"\"" >> pipeline.env else @@ -157,19 +193,14 @@ pipeline_variables: echo "MR_MODIFIED_COMPONENTS=\"$MR_MODIFIED_COMPONENTS\"" >> pipeline.env fi - | - if echo "$CI_MERGE_REQUEST_LABELS" | egrep "(^|,)BUILD_AND_TEST_ALL_APPS(,|$)"; then - echo "BUILD_AND_TEST_ALL_APPS=1" >> pipeline.env - fi - # run full pipeline if testing constraint branch - - | - if [ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]; then - echo "BUILD_AND_TEST_ALL_APPS=1" >> pipeline.env + if [ -n "$PIPELINE_COMMIT_SHA" ]; then + echo "PIPELINE_COMMIT_SHA=$PIPELINE_COMMIT_SHA" >> pipeline.env fi - echo "OOCD_DISTRO_URL_ARMHF=$OOCD_DISTRO_URL_ARMHF" >> pipeline.env - echo "OOCD_DISTRO_URL_ARM64=$OOCD_DISTRO_URL_ARM64" >> pipeline.env - - python tools/ci/ci_process_description.py + - run_cmd idf-ci gitlab pipeline-variables >> pipeline.env - cat pipeline.env - - python tools/ci/artifacts_handler.py upload --type modified_files_and_components_report + - run_cmd idf-ci gitlab upload-artifacts --type env artifacts: reports: dotenv: pipeline.env @@ -210,14 +241,13 @@ baseline_manifest_sha: expire_in: 1 week when: always -redundant_pass_job: - stage: pre_check +retry_failed_jobs: + extends: + - .pre_check_template + - .rules:dev-push tags: [shiny, fast_run] - image: $ESP_ENV_IMAGE - dependencies: null - before_script: [] - cache: [] - extends: [] + allow_failure: true script: - - echo "This job is redundant to ensure the 'retry_failed_jobs' job can exist and not be skipped" - when: always + - echo "Retrieving and retrying all failed jobs for the pipeline..." + - python tools/ci/python_packages/gitlab_api.py retry_failed_jobs $CI_MERGE_REQUEST_PROJECT_ID --pipeline_id $CI_PIPELINE_ID + when: manual diff --git a/.gitlab/ci/pre_commit.yml b/.gitlab/ci/pre_commit.yml index 49449106e73..9abe06c11fc 100644 --- a/.gitlab/ci/pre_commit.yml +++ b/.gitlab/ci/pre_commit.yml @@ -2,33 +2,20 @@ extends: - .before_script:minimal stage: pre_check - image: $PRE_COMMIT_IMAGE - tags: [cache, shiny] + image: "${CI_DOCKER_REGISTRY}/esp-idf-pre-commit:1" + tags: [pre-commit] + variables: + # Both shiny and brew runners can pick this job + GIT_STRATEGY: fetch + GIT_DEPTH: 1 + SUBMODULES_TO_FETCH: "all" needs: - pipeline_variables - variables: - # cache pre_commit - PRE_COMMIT_HOME: "$CI_PROJECT_DIR/.cache/pre-commit" script: - fetch_submodules - pre-commit run --files $MODIFIED_FILES - pre-commit run --hook-stage post-commit validate-sbom-manifest -check_pre_commit_upload_cache: - extends: - - .check_pre_commit_template - rules: - - if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/) && $CI_PIPELINE_SOURCE == "push"' - cache: - - key: pre_commit-cache-${LATEST_GIT_TAG} - paths: - - .cache/pre-commit - policy: pull-push - - key: submodule-cache-${LATEST_GIT_TAG} - paths: - - .cache/submodule_archives - policy: pull - check_pre_commit: extends: - .check_pre_commit_template @@ -37,10 +24,6 @@ check_pre_commit: when: never - when: on_success cache: - - key: pre_commit-cache-${LATEST_GIT_TAG} - paths: - - .cache/pre-commit - policy: pull - key: submodule-cache-${LATEST_GIT_TAG} paths: - .cache/submodule_archives @@ -56,25 +39,20 @@ check_powershell: tags: - dind - amd64 + - brew # faster "apk add" needs: - pipeline_variables variables: - # cache pre_commit - PRE_COMMIT_HOME: "$CI_PROJECT_DIR/.cache/pre-commit" + # brew runners always use fetch + GIT_STRATEGY: fetch + GIT_DEPTH: 1 + SUBMODULES_TO_FETCH: "none" rules: - changes: - "*.ps1" + - ".gitlab/ci/pre_commit.yml" script: - apk add python3 - apk add py3-pip - pip install pre-commit --break-system-packages - pre-commit run --hook-stage manual check-powershell-scripts --files $MODIFIED_FILES - cache: - - key: pre_commit-cache-${LATEST_GIT_TAG} - paths: - - .cache/pre-commit - policy: pull - - key: submodule-cache-${LATEST_GIT_TAG} - paths: - - .cache/submodule_archives - policy: pull diff --git a/.gitlab/ci/retry_failed_jobs.yml b/.gitlab/ci/retry_failed_jobs.yml deleted file mode 100644 index 8eff2ed6b4b..00000000000 --- a/.gitlab/ci/retry_failed_jobs.yml +++ /dev/null @@ -1,15 +0,0 @@ -retry_failed_jobs: - stage: retry_failed_jobs - tags: [shiny, fast_run] - allow_failure: true - image: $ESP_ENV_IMAGE - dependencies: null - before_script: [] - cache: [] - extends: [] - script: - - echo "Retrieving and retrying all failed jobs for the pipeline..." - - python tools/ci/python_packages/gitlab_api.py retry_failed_jobs $CI_MERGE_REQUEST_PROJECT_ID --pipeline_id $CI_PIPELINE_ID - when: manual - needs: - - redundant_pass_job diff --git a/.gitlab/ci/rules.yml b/.gitlab/ci/rules.yml index 6cb01c65588..b16b8641d5c 100644 --- a/.gitlab/ci/rules.yml +++ b/.gitlab/ci/rules.yml @@ -9,8 +9,9 @@ - "**/*.{c,C,cpp}" - "**/*.{h,H,hpp}" + - "Kconfig" - "components/**/Kconfig" - - "components/**/CMakeList.txt" + - "components/**/CMakeLists.txt" .patterns-python-cache: &patterns-python-cache - "tools/requirements.json" @@ -167,22 +168,19 @@ ############## # if anchors # ############## -.if-ref-master: &if-ref-master - if: '$CI_COMMIT_REF_NAME == "master"' +.if-master-push: &if-master-push + if: '$CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "push"' -.if-ref-master-no_label: &if-ref-master-no_label - if: '$CI_COMMIT_REF_NAME == "master" && $BOT_TRIGGER_WITH_LABEL == null' - -.if-tag-release: &if-tag-release +.if-release-tag: &if-release-tag if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/' -.if-protected: &if-protected +.if-protected-check: &if-protected-check if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) || $CI_COMMIT_TAG =~ /^qa-test/' -.if-protected-no_label: &if-protected-no_label - if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && $BOT_TRIGGER_WITH_LABEL == null' +.if-protected-deploy: &if-protected-deploy + if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/ || $CI_COMMIT_TAG =~ /^v\d+\.\d+(\.\d+)?($|-)/) && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "api")' -.if-protected-ref-push: &if-protected-ref-push +.if-protected-branch-push: &if-protected-branch-push # rules:changes always evaluates to true for new branch pipelines or when there is no Git push event if: '($CI_COMMIT_REF_NAME == "master" || $CI_COMMIT_BRANCH =~ /^release\/v/) && $CI_PIPELINE_SOURCE == "push"' @@ -192,9 +190,6 @@ .if-dev-push: &if-dev-push if: '$CI_COMMIT_REF_NAME != "master" && $CI_COMMIT_BRANCH !~ /^release\/v/ && $CI_COMMIT_TAG !~ /^v\d+\.\d+(\.\d+)?($|-)/ && $CI_COMMIT_TAG !~ /^qa-test/ && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "merge_request_event")' -.if-schedule: &if-schedule - if: '$CI_PIPELINE_SOURCE == "schedule"' - .if-schedule-nightly: &if-schedule-nightly if: '$CI_PIPELINE_SOURCE == "schedule" && $INCLUDE_NIGHTLY_RUN == "1"' @@ -214,51 +209,41 @@ # Rules # ######### ### Branches ### -.rules:protected: +.rules:protected:check: rules: - - <<: *if-protected + - <<: *if-protected-check -.rules:push_to_github: +.rules:protected:deploy: rules: - <<: *if-qa-test-tag when: never - - <<: *if-protected-no_label + - <<: *if-protected-deploy -# Not uploading on release branches -.rules:sync_support_status: +.rules:master:push: rules: - - <<: *if-ref-master-no_label - -.rules:protected-no_label-always: - rules: - - <<: *if-qa-test-tag - when: never - - <<: *if-protected-no_label - when: always + - <<: *if-master-push .rules:tag:release: rules: - - <<: *if-tag-release + - <<: *if-release-tag .rules:dev-push: rules: - <<: *if-dev-push -# Do not upload caches on dev branches by default .rules:upload-python-cache: rules: - - <<: *if-tag-release + - <<: *if-release-tag - <<: *if-schedule-nightly - - <<: *if-protected-ref-push + - <<: *if-protected-branch-push changes: *patterns-python-cache - <<: *if-label-upload_cache when: manual .rules:upload-submodule-cache: rules: - # Needn't upload submodule cache in schedule pipeline - - <<: *if-tag-release - - <<: *if-protected-ref-push + - <<: *if-release-tag + - <<: *if-protected-branch-push changes: *patterns-submodule - <<: *if-label-upload_cache when: manual @@ -266,11 +251,10 @@ ### Patterns ### .rules:patterns:clang_tidy: rules: - - <<: *if-protected + - <<: *if-protected-check - <<: *if-dev-push changes: *patterns-c-files - #.rules:patterns:static-code-analysis-preview: # rules: # - <<: *if-dev-push @@ -280,12 +264,6 @@ # - <<: *if-dev-push # changes: *patterns-sonarqube-files -.rules:patterns:idf-pytest-plugin: - rules: - - <<: *if-protected - - <<: *if-dev-push - changes: *patterns-idf-pytest-plugin - # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # DO NOT place comments or maintain any code from this line # @@ -326,7 +304,7 @@ rules: - <<: *if-revert-branch when: never - - <<: *if-protected + - <<: *if-protected-check - <<: *if-label-build - <<: *if-dev-push changes: *patterns-build_components @@ -339,7 +317,7 @@ rules: - <<: *if-revert-branch when: never - - <<: *if-protected + - <<: *if-protected-check - <<: *if-label-build - <<: *if-dev-push changes: *patterns-build_check @@ -354,7 +332,7 @@ rules: - <<: *if-revert-branch when: never - - <<: *if-protected + - <<: *if-protected-check - <<: *if-label-build - <<: *if-label-docker - <<: *if-dev-push @@ -370,7 +348,7 @@ rules: - <<: *if-revert-branch when: never - - <<: *if-protected + - <<: *if-protected-check - <<: *if-label-build - <<: *if-label-macos - <<: *if-label-macos_test @@ -381,21 +359,6 @@ - <<: *if-dev-push changes: *patterns-downloadable-tools -.rules:build:target_test: - rules: - - <<: *if-revert-branch - when: never - - <<: *if-protected - - <<: *if-label-build - - <<: *if-dev-push - changes: *patterns-build_components - - <<: *if-dev-push - changes: *patterns-build_system - - <<: *if-dev-push - changes: *patterns-build_template-app - - <<: *if-dev-push - changes: *patterns-downloadable-tools - .rules:labels:nvs_coverage: rules: - <<: *if-revert-branch @@ -408,14 +371,12 @@ when: never - <<: *if-schedule-test-build-system-windows - <<: *if-label-windows - - <<: *if-dev-push - changes: *patterns-build_system .rules:test:host_test: rules: - <<: *if-revert-branch when: never - - <<: *if-protected + - <<: *if-protected-check - <<: *if-label-build-only when: never - <<: *if-label-host_test @@ -426,7 +387,7 @@ rules: - <<: *if-revert-branch when: never - - <<: *if-protected + - <<: *if-protected-check - <<: *if-label-build-only when: never - <<: *if-label-submodule diff --git a/.gitlab/ci/static-code-analysis.yml b/.gitlab/ci/static-code-analysis.yml index 3a45ecbde03..734ff76a2ec 100644 --- a/.gitlab/ci/static-code-analysis.yml +++ b/.gitlab/ci/static-code-analysis.yml @@ -26,7 +26,10 @@ gcc_static_analyzer: script: - echo "CONFIG_COMPILER_STATIC_ANALYZER=y" >> ${ANALYZING_APP}/sdkconfig.defaults - sed -i 's/.*MINIMAL_BUILD.*//g' ${ANALYZING_APP}/CMakeLists.txt - - idf-build-apps build -p ${ANALYZING_APP} + # no -Werror + - export EXTRA_CFLAGS= + - export EXTRA_CXXFLAGS= + - idf-build-apps build -p ${ANALYZING_APP} --disable-targets esp32h4 esp32h21 --no-check-warnings # ## build stage ## Sonarqube related jobs put here for this reason: @@ -96,7 +99,7 @@ gcc_static_analyzer: #code_quality_report: # extends: # - .sonar_scan_template -# - .rules:protected +# - .rules:protected:check # allow_failure: true # it's using exit code to indicate the code analysis result, # # we don't want to block ci when critical issues founded # script: diff --git a/.gitlab/ci/test-win.yml b/.gitlab/ci/test-win.yml index 6c0ddfe8dec..3f6592f7c07 100644 --- a/.gitlab/ci/test-win.yml +++ b/.gitlab/ci/test-win.yml @@ -3,8 +3,7 @@ extends: .rules:test:host_test stage: host_test image: $ESP_ENV_IMAGE - tags: - - windows-build + tags: [windows-build, brew] dependencies: # set dependencies to null to avoid missing artifacts issue # run host_test jobs immediately, only after upload cache needs: @@ -28,6 +27,7 @@ test_cli_installer_win: extends: - .host_test_win_template - .rules:labels:windows_pytest_build_system + allow_failure: true artifacts: when: on_failure paths: @@ -37,11 +37,10 @@ test_cli_installer_win: variables: IDF_PATH: "$CI_PROJECT_DIR" timeout: 3h - allow_failure: true script: # Tools must be downloaded for testing # cmake@version that is supported - - python ${IDF_PATH}\tools\idf_tools.py download required qemu-riscv32 qemu-xtensa cmake cmake@3.16.3 + - python ${IDF_PATH}\tools\idf_tools.py download required qemu-riscv32 qemu-xtensa cmake cmake@3.22.1 - cd ${IDF_PATH}\tools\test_idf_tools - python -m pip install jsonschema - python .\test_idf_tools.py @@ -64,12 +63,12 @@ test_tools_win: PYTHONPATH: "$PYTHONPATH;$IDF_PATH\\tools;$IDF_PATH\\tools\\esp_app_trace;$IDF_PATH\\components\\partition_table;$IDF_PATH\\tools\\ci\\python_packages" script: - python -m pip install jsonschema - - .\install.ps1 --enable-ci --enable-pytest + - .\install.ps1 --enable-ci - .\export.ps1 - python "${SUBMODULE_FETCH_TOOL}" -s "all" - cd ${IDF_PATH}/tools/test_idf_py - - pytest --noconftest test_idf_py.py --junitxml=${IDF_PATH}/XUNIT_IDF_PY.xml - - pytest --noconftest test_hints.py --junitxml=${IDF_PATH}/XUNIT_HINTS.xml + - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} + - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=${IDF_PATH}/XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} # Build tests .test_build_system_template_win: @@ -82,11 +81,12 @@ test_tools_win: after_script: [] timeout: 4 hours script: - - .\install.ps1 --enable-ci --enable-pytest + - .\install.ps1 --enable-ci - . .\export.ps1 - python "${SUBMODULE_FETCH_TOOL}" -s "all" - cd ${IDF_PATH}\tools\test_build_system - - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml + - idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} + - pytest --parallel-count ${CI_NODE_TOTAL} --parallel-index ${CI_NODE_INDEX} --junitxml=${CI_PROJECT_DIR}\XUNIT_RESULT.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} pytest_build_system_win: extends: @@ -94,8 +94,7 @@ pytest_build_system_win: - .rules:labels:windows_pytest_build_system parallel: 2 needs: [] - tags: - - windows-build + tags: [windows-build, brew] artifacts: paths: - XUNIT_RESULT.xml diff --git a/.gitlab/ci/upload_cache.yml b/.gitlab/ci/upload_cache.yml index 173b63224b4..fdd7e799cc1 100644 --- a/.gitlab/ci/upload_cache.yml +++ b/.gitlab/ci/upload_cache.yml @@ -21,7 +21,7 @@ upload-pip-cache: policy: push script: - rm -rf .cache/pip # clear old packages - - bash install.sh --enable-ci --enable-pytest + - bash install.sh --enable-ci --enable-test-specific parallel: matrix: - GEO: [ 'shiny', 'brew' ] diff --git a/.idf_build_apps.toml b/.idf_build_apps.toml new file mode 100644 index 00000000000..25d8d9d5900 --- /dev/null +++ b/.idf_build_apps.toml @@ -0,0 +1,68 @@ +config_rules = [ + 'sdkconfig.ci=default', + 'sdkconfig.ci.*=', + '=default', +] + +extra_pythonpaths = [ + '$IDF_PATH/tools/ci/python_packages', + '$IDF_PATH/tools/ci', + '$IDF_PATH/tools', +] +build_system = "idf_ci_local.app:IdfCMakeApp" + +recursive = true +check_warnings = true +keep_going = true +copy_sdkconfig = true +ignore_warning_files = [ + '$IDF_PATH/tools/ci/ignore_build_warnings.txt', +] + +build_dir = "build_@t_@w" +build_log_filename = "build_log.txt" +size_json_filename = "size_${CI_JOB_ID}.json" + +verbose = 1 # INFO + +additional_build_targets = [ + 'esp32h21', + 'esp32h4', +] + +# collect +collect_app_info_filename = "app_info_${CI_JOB_NAME_SLUG}.txt" +junitxml = "build_summary_${CI_JOB_NAME_SLUG}.xml" + +# manifest +check_manifest_rules = true +manifest_rootpath = "$IDF_PATH" +manifest_filepatterns = [ + '**/.build-test-rules.yml', +] + +# dependency-driven build +deactivate_dependency_driven_build_by_components = [ + 'cxx', + 'esp_common', + 'esp_hw_support', + 'esp_rom', + 'esp_system', + 'esp_timer', + 'freertos', + 'hal', + 'heap', + 'log', + 'newlib', + 'riscv', + 'soc', + 'xtensa', +] + +deactivate_dependency_driven_build_by_filepatterns = [ + # tools + 'tools/cmake/**/*', + 'tools/tools.json', + # ci + 'tools/ci/ignore_build_warnings.txt', +] diff --git a/.idf_ci.toml b/.idf_ci.toml new file mode 100644 index 00000000000..6866454fca0 --- /dev/null +++ b/.idf_ci.toml @@ -0,0 +1,106 @@ +preserve_non_test_related_apps = false +exclude_dirs = [ + 'tools/test_mkdfu', + 'tools/test_idf_size', + 'tools/test_idf_py', + 'tools/test_idf_diag', + 'tools/test_bsasm', + 'tools/ci/test_autocomplete', + 'tools/test_build_system', +] + +[local_runtime_envs] +EXTRA_CFLAGS = "-Werror -Werror=deprecated-declarations -Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function -Wstrict-prototypes" +EXTRA_CXXFLAGS = "-Werror -Werror=deprecated-declarations -Werror=unused-variable -Werror=unused-but-set-variable -Werror=unused-function" +LDGEN_CHECK_MAPPING = "1" +IDF_CI_BUILD = "1" + +[gitlab] + +[gitlab.build_pipeline] +workflow_name = "build_child_pipeline" +presigned_json_job_name = 'generate_pytest_build_report' + +job_tags = ['build', 'shiny'] +job_template_name = '.dynamic_build_template' +job_template_jinja = '' # write in tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml +pre_yaml_jinja = """ +include: + - .gitlab/ci/common.yml + - tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml + - tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml +""" +yaml_jinja = """ +{{ settings.gitlab.build_pipeline.pre_yaml_jinja }} + +workflow: + name: {{ settings.gitlab.build_pipeline.workflow_name }} + rules: + - when: always + +{{ jobs }} +""" # simplified since we included the tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml + +[gitlab.test_pipeline] +job_template_name = '.dynamic_target_test_template' +job_template_jinja = '' # write in tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml +pre_yaml_jinja = """ +include: + - .gitlab/ci/common.yml + - tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml +""" + +[gitlab.artifacts.s3.debug] +bucket = "idf-artifacts" +patterns = [ + '**/build*/bootloader/*.map', + '**/build*/bootloader/*.elf', + '**/build*/*.map', + '**/build*/*.elf', + # customized + '**/build*/esp_tee/*.map', + '**/build*/esp_tee/*.elf', + '**/build*/gdbinit/*', +] + +[gitlab.artifacts.s3.flash] +bucket = "idf-artifacts" +patterns = [ + '**/build*/bootloader/*.bin', + '**/build*/*.bin', + '**/build*/partition_table/*.bin', + '**/build*/flasher_args.json', + '**/build*/flash_project_args', + '**/build*/config/sdkconfig.json', + '**/build*/sdkconfig', + '**/build*/project_description.json', + # customized + '**/build*/esp_tee/*.bin', +] + +[gitlab.artifacts.s3.log] +bucket = "idf-artifacts" +patterns = [ + '**/build*/build_log.txt', + '**/build*/size*.json', +] + +[gitlab.artifacts.s3.junit] +bucket = "idf-artifacts" +patterns = [ + '**/XUNIT_RESULT_*.xml', + '**/build_summary_*.xml', +] + +[gitlab.artifacts.s3.env] +bucket = "idf-artifacts" +patterns = [ + '**/pipeline.env', +] + +[gitlab.artifacts.s3.longterm] +bucket = "longterm" +if_clause = 'CI_COMMIT_REF_NAME == "master"' +patterns = [ + '**/build*/size*.json', +] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2141eb025f7..85cdb8a9879 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,26 +10,6 @@ repos: - id: ruff-format - id: ruff args: [ "--fix" ] - - repo: local - hooks: - - id: pytest-linter - name: Pytest Linter Check - entry: tools/ci/check_test_files.py - language: python - files: 'pytest_.*\.py$' - require_serial: true - additional_dependencies: - - pytest-embedded-idf[serial]~=1.16 - - pytest-embedded-jtag~=1.16 - - pytest-embedded-qemu~=1.16 - - pytest-ignore-test-results~=0.3 - - pytest-rerunfailures - - pytest-timeout - - idf-build-apps~=2.8 - - python-gitlab - - minio - - click - - esp-idf-monitor - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: @@ -165,7 +145,7 @@ repos: require_serial: true additional_dependencies: - PyYAML == 5.3.1 - - idf-build-apps>=2.8,<3 + - idf-build-apps~=2.13 - id: sort-yaml-files name: sort yaml files entry: tools/ci/sort_yaml.py @@ -180,15 +160,6 @@ repos: files: 'tools/ci/sort_yaml\.py$' additional_dependencies: - ruamel.yaml - - id: check-build-test-rules-path-exists - name: check path in .build-test-rules.yml exists - entry: tools/ci/check_build_test_rules.py check-exist - language: python - additional_dependencies: - - PyYAML == 5.3.1 - always_run: true - pass_filenames: false - require_serial: true - id: cleanup-ignore-lists name: Remove non-existing patterns from ignore lists entry: tools/ci/cleanup_ignore_lists.py diff --git a/components/app_update/test_apps/test_app_update/pytest_app_update_ut.py b/components/app_update/test_apps/test_app_update/pytest_app_update_ut.py index 8e7c6342413..fb64f996bf6 100644 --- a/components/app_update/test_apps/test_app_update/pytest_app_update_ut.py +++ b/components/app_update/test_apps/test_app_update/pytest_app_update_ut.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 import re @@ -19,6 +19,7 @@ TEST_SUBMENU_PATTERN_PYTEST = re.compile(rb'\s+\((\d+)\)\s+"([^"]+)"\r?\n') indirect=True, ) @idf_parametrize('target', ['supported_targets'], indirect=['target']) +@pytest.mark.temp_skip_ci(targets=['esp32p4'], reason='p4 rev3 migration, IDF-14401') def test_app_update(dut: Dut) -> None: dut.run_all_single_board_cases(timeout=90) @@ -32,6 +33,7 @@ def test_app_update(dut: Dut) -> None: indirect=True, ) @idf_parametrize('target', ['supported_targets'], indirect=['target']) +@pytest.mark.temp_skip_ci(targets=['esp32p4'], reason='p4 rev3 migration, IDF-14401') def test_app_update_xip_psram(dut: Dut) -> None: dut.run_all_single_board_cases(timeout=90) @@ -45,6 +47,7 @@ def test_app_update_xip_psram(dut: Dut) -> None: indirect=True, ) @idf_parametrize('target', ['supported_targets'], indirect=['target']) +@pytest.mark.temp_skip_ci(targets=['esp32p4'], reason='p4 rev3 migration, IDF-14401') def test_app_update_xip_psram_rom_impl(dut: Dut) -> None: dut.run_all_single_board_cases(timeout=90) @@ -58,6 +61,7 @@ def test_app_update_xip_psram_rom_impl(dut: Dut) -> None: indirect=True, ) @idf_parametrize('target', ['esp32', 'esp32c3', 'esp32s3', 'esp32p4'], indirect=['target']) +@pytest.mark.temp_skip_ci(targets=['esp32p4'], reason='p4 rev3 migration, IDF-14401') def test_app_update_with_rollback(dut: Dut) -> None: dut.run_all_single_board_cases(timeout=90) @@ -69,6 +73,7 @@ def test_app_update_with_rollback(dut: Dut) -> None: indirect=True, ) @idf_parametrize('target', ['esp32c5'], indirect=['target']) +@pytest.mark.temp_skip_ci(targets=['esp32p4'], reason='p4 rev3 migration, IDF-14401') def test_recovery_bootloader_update(dut: Dut) -> None: try: dut.run_all_single_board_cases(group='recovery_bootloader', timeout=90) diff --git a/components/esp_driver_sdio/test_apps/sdio/sdio_common_tests/pytest_sdio.py b/components/esp_driver_sdio/test_apps/sdio/sdio_common_tests/pytest_sdio.py index 6ec9dac8093..72872116eac 100644 --- a/components/esp_driver_sdio/test_apps/sdio/sdio_common_tests/pytest_sdio.py +++ b/components/esp_driver_sdio/test_apps/sdio/sdio_common_tests/pytest_sdio.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 import os.path from typing import List @@ -99,7 +99,7 @@ def test_sdio_esp32_esp32(dut: Tuple[IdfDut, IdfDut]) -> None: indirect=True, ) @pytest.mark.parametrize('app_path, target, config', c5_param_default, indirect=True) -@pytest.mark.temp_skip_ci(targets=['esp32p4'], reason='p4 rev3 migration, IDF-14359') +@pytest.mark.temp_skip_ci(targets=['esp32p4,esp32c5'], reason='p4 rev3 migration, IDF-14359') def test_sdio_esp32p4_esp32c5(dut: Tuple[IdfDut, IdfDut]) -> None: test_sdio_flow(dut) @@ -172,7 +172,7 @@ def test_sdio_speed_frhost_esp32_esp32(dut: Tuple[IdfDut, IdfDut]) -> None: indirect=True, ) @pytest.mark.parametrize('app_path, target, config', c5_param_default, indirect=True) -@pytest.mark.temp_skip_ci(targets=['esp32p4'], reason='p4 rev3 migration, IDF-14359') +@pytest.mark.temp_skip_ci(targets=['esp32p4,esp32c5'], reason='p4 rev3 migration, IDF-14359') def test_sdio_speed_frhost_esp32p4_esp32c5(dut: Tuple[IdfDut, IdfDut]) -> None: test_sdio_speed_frhost_flow(dut, 10000, 4000) @@ -245,7 +245,7 @@ def test_sdio_speed_tohost_esp32_esp32(dut: Tuple[IdfDut, IdfDut]) -> None: indirect=True, ) @pytest.mark.parametrize('app_path, target, config', c5_param_default, indirect=True) -@pytest.mark.temp_skip_ci(targets=['esp32p4'], reason='p4 rev3 migration, IDF-14359') +@pytest.mark.temp_skip_ci(targets=['esp32p4,esp32c5'], reason='p4 rev3 migration, IDF-14359') def test_sdio_speed_tohost_esp32p4_esp32c5(dut: Tuple[IdfDut, IdfDut]) -> None: test_sdio_speed_tohost_flow(dut, 9000, 4000) diff --git a/components/esp_mm/test_apps/mm/pytest_mmap.py b/components/esp_mm/test_apps/mm/pytest_mmap.py index 182a2807fda..4cde8ace15a 100644 --- a/components/esp_mm/test_apps/mm/pytest_mmap.py +++ b/components/esp_mm/test_apps/mm/pytest_mmap.py @@ -1,12 +1,11 @@ -# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize + # normal mmu tests - - @pytest.mark.generic @pytest.mark.temp_skip_ci(targets=['esp32p4'], reason='p4 rev3 migration, IDF-14470') @pytest.mark.parametrize( @@ -22,14 +21,6 @@ def test_mmap(dut: Dut) -> None: # mmu tests with psram enabled -PSRAM_RELEASE_CONFIGS = [ - pytest.param('psram_release_esp32', marks=[pytest.mark.esp32]), - pytest.param('psram_release_esp32s2', marks=[pytest.mark.esp32s2]), - pytest.param('psram_release_esp32s3', marks=[pytest.mark.esp32s3]), - pytest.param('psram_release_esp32p4', marks=[pytest.mark.esp32p4]), -] - - @pytest.mark.generic @pytest.mark.temp_skip_ci(targets=['esp32p4'], reason='p4 rev3 migration, IDF-14470') @idf_parametrize( @@ -47,12 +38,6 @@ def test_mmap_psram(dut: Dut) -> None: # mmu tests with xip_psram -XIP_CONFIGS = [ - pytest.param('xip_psram_esp32s2', marks=[pytest.mark.esp32s2]), - pytest.param('xip_psram_esp32s3', marks=[pytest.mark.esp32s3]), -] - - @pytest.mark.generic @idf_parametrize( 'config,target', [('xip_psram_esp32s2', 'esp32s2'), ('xip_psram_esp32s3', 'esp32s3')], indirect=['config', 'target'] diff --git a/components/esp_system/test_apps/console/pytest_esp_system_console_tests.py b/components/esp_system/test_apps/console/pytest_esp_system_console_tests.py index e51c1412760..8e3bd94df72 100644 --- a/components/esp_system/test_apps/console/pytest_esp_system_console_tests.py +++ b/components/esp_system/test_apps/console/pytest_esp_system_console_tests.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2024-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 import pexpect import pytest @@ -15,17 +15,6 @@ def not_expect(dut: Dut, output_regex: str) -> None: raise RuntimeError(f'Found not_expect output {output_regex}') -JTAG_SERIAL_MARKS = [ - pytest.mark.esp32s3, - pytest.mark.esp32c2, - pytest.mark.esp32c3, - pytest.mark.esp32c5, - pytest.mark.esp32c6, - pytest.mark.esp32c61, - pytest.mark.esp32h2, -] - - @pytest.mark.generic @idf_parametrize('config', ['console_none'], indirect=['config']) @idf_parametrize('target', ['supported_targets'], indirect=['target']) diff --git a/components/freertos/test_apps/freertos/pytest_freertos.py b/components/freertos/test_apps/freertos/pytest_freertos.py index c1e8ea7bdc1..ccf1d28224e 100644 --- a/components/freertos/test_apps/freertos/pytest_freertos.py +++ b/components/freertos/test_apps/freertos/pytest_freertos.py @@ -1,33 +1,9 @@ -# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -CONFIGS = [ - pytest.param( - 'default', - marks=[ - pytest.mark.supported_targets, - ], - ), - pytest.param('freertos_options', marks=[pytest.mark.supported_targets]), - pytest.param('psram', marks=[pytest.mark.esp32, pytest.mark.esp32s3, pytest.mark.esp32p4, pytest.mark.esp32c5]), - pytest.param('single_core', marks=[pytest.mark.esp32, pytest.mark.esp32p4]), - # TODO: [ESP32C5] IDF-10335 - # TODO: [ESP32C61] IDF-11146 - pytest.param( - 'smp', - marks=[ - pytest.mark.supported_targets, - pytest.mark.temp_skip_ci( - targets=['esp32p4', 'esp32c5', 'esp32c61', 'esp32h21'], reason='test failed/TBD IDF-8113' - ), - ], - ), - pytest.param('tickless_idle', marks=[pytest.mark.supported_targets]), -] - @pytest.mark.generic @idf_parametrize( diff --git a/components/nvs_flash/test_apps/pytest_nvs_flash.py b/components/nvs_flash/test_apps/pytest_nvs_flash.py index 35af7f4d227..0b610f0e07f 100644 --- a/components/nvs_flash/test_apps/pytest_nvs_flash.py +++ b/components/nvs_flash/test_apps/pytest_nvs_flash.py @@ -1,14 +1,9 @@ -# SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import pytest from pytest_embedded_idf.dut import IdfDut from pytest_embedded_idf.utils import idf_parametrize -CONFIGS_NVS_ENCR_FLASH_ENC = [ - pytest.param('nvs_encr_flash_enc_esp32', marks=[pytest.mark.esp32]), - pytest.param('nvs_encr_flash_enc_esp32c3', marks=[pytest.mark.esp32c3]), -] - @pytest.mark.generic @pytest.mark.parametrize('config', ['default'], indirect=True) diff --git a/components/spi_flash/test_apps/flash_mmap/README.md b/components/spi_flash/test_apps/flash_mmap/README.md index 7b96141437e..44f3780f1d6 100644 --- a/components/spi_flash/test_apps/flash_mmap/README.md +++ b/components/spi_flash/test_apps/flash_mmap/README.md @@ -1,2 +1,2 @@ -| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-C5 | ESP32-C6 | ESP32-C61 | ESP32-H2 | ESP32-P4 | ESP32-S2 | ESP32-S3 | -| ----------------- | ----- | -------- | -------- | -------- | -------- | --------- | -------- | -------- | -------- | -------- | +| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-C5 | ESP32-C6 | ESP32-C61 | ESP32-H2 | ESP32-H21 | ESP32-H4 | ESP32-P4 | ESP32-S2 | ESP32-S3 | +| ----------------- | ----- | -------- | -------- | -------- | -------- | --------- | -------- | --------- | -------- | -------- | -------- | -------- | diff --git a/components/tcp_transport/test_apps/pytest_tcp_transport_ut.py b/components/tcp_transport/test_apps/pytest_tcp_transport_ut.py index 04f50051eb8..b7c123959ea 100644 --- a/components/tcp_transport/test_apps/pytest_tcp_transport_ut.py +++ b/components/tcp_transport/test_apps/pytest_tcp_transport_ut.py @@ -1,14 +1,9 @@ -# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -CONFIGS = [ - pytest.param('default', marks=[pytest.mark.esp32, pytest.mark.esp32c3]), - pytest.param('psram_esp32', marks=[pytest.mark.esp32]), -] - @pytest.mark.generic @idf_parametrize( diff --git a/conftest.py b/conftest.py index 8c2407f8687..b786251134b 100644 --- a/conftest.py +++ b/conftest.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2021-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 # pylint: disable=W0621 # redefined-outer-name # @@ -9,6 +9,7 @@ # please report to https://github.com/espressif/pytest-embedded/issues # or discuss at https://github.com/espressif/pytest-embedded/discussions import os +import subprocess import sys if os.path.join(os.path.dirname(__file__), 'tools', 'ci') not in sys.path: @@ -17,8 +18,6 @@ if os.path.join(os.path.dirname(__file__), 'tools', 'ci') not in sys.path: if os.path.join(os.path.dirname(__file__), 'tools', 'ci', 'python_packages') not in sys.path: sys.path.append(os.path.join(os.path.dirname(__file__), 'tools', 'ci', 'python_packages')) -import glob -import io import json import logging import os @@ -26,7 +25,6 @@ import re import signal import time import typing as t -import zipfile from copy import deepcopy from urllib.parse import quote @@ -34,28 +32,16 @@ import common_test_methods # noqa: F401 import gitlab_api import pexpect import pytest -import requests -import yaml from _pytest.config import Config from _pytest.fixtures import FixtureRequest -from artifacts_handler import ArtifactType -from dynamic_pipelines.constants import TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME -from idf_ci_local.app import import_apps_from_txt -from idf_ci_local.uploader import AppDownloader -from idf_ci_local.uploader import AppUploader -from idf_ci_utils import IDF_PATH +from idf_ci import PytestCase +from idf_ci.idf_pytest import IDF_CI_PYTEST_CASE_KEY from idf_ci_utils import idf_relpath from idf_pytest.constants import DEFAULT_LOGDIR -from idf_pytest.constants import DEFAULT_SDKCONFIG -from idf_pytest.constants import ENV_MARKERS -from idf_pytest.constants import SPECIAL_MARKERS -from idf_pytest.constants import TARGET_MARKERS -from idf_pytest.constants import PytestCase -from idf_pytest.plugin import IDF_PYTEST_EMBEDDED_KEY -from idf_pytest.plugin import ITEM_PYTEST_CASE_KEY -from idf_pytest.plugin import IdfPytestEmbedded +from idf_pytest.plugin import IDF_LOCAL_PLUGIN_KEY +from idf_pytest.plugin import IdfLocalPlugin +from idf_pytest.plugin import requires_elf_or_map from idf_pytest.utils import format_case_id -from pytest_embedded.plugin import multi_dut_argument from pytest_embedded.plugin import multi_dut_fixture from pytest_embedded.utils import to_bytes from pytest_embedded.utils import to_str @@ -83,23 +69,6 @@ def case_tester(unity_tester: CaseTester) -> CaseTester: return unity_tester -@pytest.fixture -@multi_dut_argument -def config(request: FixtureRequest) -> str: - return getattr(request, 'param', None) or DEFAULT_SDKCONFIG # type: ignore - - -@pytest.fixture -@multi_dut_fixture -def target(request: FixtureRequest, dut_total: int, dut_index: int) -> str: - plugin = request.config.stash[IDF_PYTEST_EMBEDDED_KEY] - - if dut_total == 1: - return plugin.target[0] # type: ignore - - return plugin.target[dut_index] # type: ignore - - @pytest.fixture def test_func_name(request: FixtureRequest) -> str: return request.node.function.__name__ # type: ignore @@ -126,33 +95,58 @@ def pipeline_id(request: FixtureRequest) -> t.Optional[str]: return request.config.getoption('pipeline_id', None) or os.getenv('PARENT_PIPELINE_ID', None) # type: ignore -class BuildReportDownloader(AppDownloader): - def __init__(self, presigned_url_yaml: str) -> None: - self.app_presigned_urls_dict: t.Dict[str, t.Dict[str, str]] = yaml.safe_load(presigned_url_yaml) +def get_pipeline_commit_sha_by_pipeline_id(pipeline_id: str) -> t.Optional[str]: + gl = gitlab_api.Gitlab(os.getenv('CI_PROJECT_ID', 'espressif/esp-idf')) + pipeline = gl.project.pipelines.get(pipeline_id) + if not pipeline: + return None - def _download_app(self, app_build_path: str, artifact_type: ArtifactType) -> None: - url = self.app_presigned_urls_dict[app_build_path][artifact_type.value] + commit = gl.project.commits.get(pipeline.sha) + if not commit or not commit.parent_ids: + return None - logging.info('Downloading app from %s', url) - with io.BytesIO() as f: - for chunk in requests.get(url).iter_content(chunk_size=1024 * 1024): - if chunk: - f.write(chunk) + if len(commit.parent_ids) == 1: + return commit.parent_ids[0] # type: ignore - f.seek(0) + for parent_id in commit.parent_ids: + parent_commit = gl.project.commits.get(parent_id) + if parent_commit.parent_ids and len(parent_commit.parent_ids) == 1: + return parent_id # type: ignore - with zipfile.ZipFile(f) as zip_ref: - zip_ref.extractall(IDF_PATH) + return None - def download_app(self, app_build_path: str, artifact_type: t.Optional[ArtifactType] = None) -> None: - if app_build_path not in self.app_presigned_urls_dict: - raise ValueError( - f'No presigned url found for {app_build_path}. ' - f'Usually this should not happen, please re-trigger a pipeline.' - f'If this happens again, please report this bug to the CI channel.' - ) - super().download_app(app_build_path, artifact_type) +class AppDownloader: + def __init__( + self, + commit_sha: str, + pipeline_id: t.Optional[str] = None, + ) -> None: + self.commit_sha = commit_sha + self.pipeline_id = pipeline_id + + def download_app(self, app_build_path: str, artifact_type: t.Optional[str] = None) -> None: + args = [ + 'idf-ci', + 'gitlab', + 'download-artifacts', + '--commit-sha', + self.commit_sha, + ] + if artifact_type: + args.extend(['--type', artifact_type]) + if self.pipeline_id: + args.extend(['--pipeline-id', self.pipeline_id]) + args.append(app_build_path) + + subprocess.run( + args, + stdout=sys.stdout, + stderr=sys.stderr, + ) + + +PRESIGNED_JSON = 'presigned.json' class OpenOCD: @@ -264,38 +258,22 @@ def openocd_dut(dut: IdfDut) -> OpenOCD: @pytest.fixture(scope='session') def app_downloader(pipeline_id: t.Optional[str]) -> t.Optional[AppDownloader]: + if commit_sha := os.getenv('PIPELINE_COMMIT_SHA'): + logging.debug('pipeline commit sha from CI env is %s', commit_sha) + return AppDownloader(commit_sha, None) + if not pipeline_id: return None - if ( - 'IDF_S3_BUCKET' in os.environ - and 'IDF_S3_ACCESS_KEY' in os.environ - and 'IDF_S3_SECRET_KEY' in os.environ - and 'IDF_S3_SERVER' in os.environ - and 'IDF_S3_BUCKET' in os.environ - ): - return AppUploader(pipeline_id) + commit_sha = get_pipeline_commit_sha_by_pipeline_id(pipeline_id) + if not commit_sha: + raise ValueError( + 'commit sha cannot be found for pipeline id %s. Please check the pipeline id. ' + 'If you think this is a bug, please report it to CI team', + ) + logging.debug('pipeline commit sha of pipeline %s is %s', pipeline_id, commit_sha) - logging.info('Downloading build report from the build pipeline %s', pipeline_id) - test_app_presigned_urls_file = None - - gl = gitlab_api.Gitlab(os.getenv('CI_PROJECT_ID', 'espressif/esp-idf')) - - for child_pipeline in gl.project.pipelines.get(pipeline_id, lazy=True).bridges.list(iterator=True): - if child_pipeline.name == 'build_child_pipeline': - for job in gl.project.pipelines.get(child_pipeline.downstream_pipeline['id'], lazy=True).jobs.list( - iterator=True - ): - if job.name == 'generate_pytest_build_report': - test_app_presigned_urls_file = gl.download_artifact( - job.id, [TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME] - )[0] - break - - if test_app_presigned_urls_file: - return BuildReportDownloader(test_app_presigned_urls_file) - - return None + return AppDownloader(commit_sha, pipeline_id) @pytest.fixture @@ -319,14 +297,20 @@ def build_dir( valid build directory """ # download from minio on CI - case: PytestCase = request._pyfuncitem.stash[ITEM_PYTEST_CASE_KEY] - if app_downloader: + case: PytestCase = request.node.stash[IDF_CI_PYTEST_CASE_KEY] + if 'skip_app_downloader' in case.all_markers: + logging.debug('skip_app_downloader marker found, skip downloading app') + downloader = None + else: + downloader = app_downloader + + if downloader: # somehow hardcoded... app_build_path = os.path.join(idf_relpath(app_path), f'build_{target}_{config}') - if case.requires_elf_or_map: - app_downloader.download_app(app_build_path) + if requires_elf_or_map(case): + downloader.download_app(app_build_path) else: - app_downloader.download_app(app_build_path, ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES) + downloader.download_app(app_build_path, 'flash') check_dirs = [f'build_{target}_{config}'] else: check_dirs = [] @@ -523,10 +507,6 @@ def dev_user(request: FixtureRequest) -> str: ################## def pytest_addoption(parser: pytest.Parser) -> None: idf_group = parser.getgroup('idf') - idf_group.addoption( - '--sdkconfig', - help='sdkconfig postfix, like sdkconfig.ci.. (Default: None, which would build all found apps)', - ) idf_group.addoption( '--dev-user', help='user name associated with some specific device/service used during the test execution', @@ -535,16 +515,10 @@ def pytest_addoption(parser: pytest.Parser) -> None: '--dev-passwd', help='password associated with some specific device/service used during the test execution', ) - idf_group.addoption( - '--app-info-filepattern', - help='glob pattern to specify the files that include built app info generated by ' - '`idf-build-apps --collect-app-info ...`. will not raise ValueError when binary ' - 'paths not exist in local file system if not listed recorded in the app info.', - ) idf_group.addoption( '--pipeline-id', - help='main pipeline id, not the child pipeline id. Specify this option to download the artifacts ' - 'from the minio server for debugging purpose.', + help='For users without s3 access. main pipeline id, not the child pipeline id. ' + 'Specify this option to download the artifacts from the minio server for debugging purpose.', ) @@ -557,63 +531,15 @@ def pytest_configure(config: Config) -> None: supported_targets.set(SUPPORTED_TARGETS) preview_targets.set(PREVIEW_TARGETS) - # cli option "--target" - target = [_t.strip().lower() for _t in (config.getoption('target', '') or '').split(',') if _t.strip()] - - # add markers based on idf_pytest/constants.py - for name, description in { - **TARGET_MARKERS, - **ENV_MARKERS, - **SPECIAL_MARKERS, - }.items(): - config.addinivalue_line('markers', f'{name}: {description}') - - help_commands = ['--help', '--fixtures', '--markers', '--version'] - for cmd in help_commands: - if cmd in config.invocation_params.args: - target = ['unneeded'] - break - - markexpr = config.getoption('markexpr') or '' - # check marker expr set via "pytest -m" - if not target and markexpr: - # we use `-m "esp32 and generic"` in our CI to filter the test cases - # this doesn't cover all use cases, but fit what we do in CI. - for marker in markexpr.split('and'): - marker = marker.strip() - if marker in TARGET_MARKERS: - target.append(marker) - - # "--target" must be set - if not target: - raise SystemExit( - """Pass `--target TARGET[,TARGET...]` to specify all targets the test cases are using. - - for single DUT, we run with `pytest --target esp32` - - for multi DUT, we run with `pytest --target esp32,esp32,esp32s2` to indicate all DUTs -""" - ) - - apps = None - app_info_filepattern = config.getoption('app_info_filepattern') - if app_info_filepattern: - apps = [] - for f in glob.glob(os.path.join(IDF_PATH, app_info_filepattern)): - apps.extend(import_apps_from_txt(f)) - - if '--collect-only' not in config.invocation_params.args: - config.stash[IDF_PYTEST_EMBEDDED_KEY] = IdfPytestEmbedded( - config_name=config.getoption('sdkconfig'), - target=target, - apps=apps, - ) - config.pluginmanager.register(config.stash[IDF_PYTEST_EMBEDDED_KEY]) + config.stash[IDF_LOCAL_PLUGIN_KEY] = IdfLocalPlugin() + config.pluginmanager.register(config.stash[IDF_LOCAL_PLUGIN_KEY]) def pytest_unconfigure(config: Config) -> None: - _pytest_embedded = config.stash.get(IDF_PYTEST_EMBEDDED_KEY, None) - if _pytest_embedded: - del config.stash[IDF_PYTEST_EMBEDDED_KEY] - config.pluginmanager.unregister(_pytest_embedded) + idf_local_plugin = config.stash.get(IDF_LOCAL_PLUGIN_KEY, None) + if idf_local_plugin: + del config.stash[IDF_LOCAL_PLUGIN_KEY] + config.pluginmanager.unregister(idf_local_plugin) dut_artifacts_url = [] @@ -639,10 +565,10 @@ def pytest_runtest_makereport(item, call): # type: ignore if isinstance(_dut, list): logs_files.extend([template.format(get_path(d.logfile)) for d in _dut]) - dut_artifacts_url.append('{}:'.format(_dut[0].test_case_name)) + dut_artifacts_url.append(f'{_dut[0].test_case_name}:') else: logs_files.append(template.format(get_path(_dut.logfile))) - dut_artifacts_url.append('{}:'.format(_dut.test_case_name)) + dut_artifacts_url.append(f'{_dut.test_case_name}:') for file in logs_files: dut_artifacts_url.append(' - {}'.format(quote(file, safe=':/'))) diff --git a/examples/get-started/hello_world/pytest_hello_world.py b/examples/get-started/hello_world/pytest_hello_world.py index eb02bd7782a..64f6bb8f5fb 100644 --- a/examples/get-started/hello_world/pytest_hello_world.py +++ b/examples/get-started/hello_world/pytest_hello_world.py @@ -25,7 +25,7 @@ def test_hello_world_linux(dut: IdfDut) -> None: @pytest.mark.host_test -@pytest.mark.macos_shell +@pytest.mark.macos @idf_parametrize('target', ['linux'], indirect=['target']) def test_hello_world_macos(dut: IdfDut) -> None: dut.expect('Hello world!') diff --git a/examples/openthread/pytest_otbr.py b/examples/openthread/pytest_otbr.py index 8cd91bd7429..c4773ca45d2 100644 --- a/examples/openthread/pytest_otbr.py +++ b/examples/openthread/pytest_otbr.py @@ -8,10 +8,12 @@ import random import re import secrets import subprocess +import sys import threading import time from typing import Tuple +sys.path.append(os.path.dirname(os.path.abspath(__file__))) import ot_ci_function as ocf import pexpect import pytest diff --git a/examples/peripherals/twai/twai_network/pytest_twai_network.py b/examples/peripherals/twai/twai_network/pytest_twai_network.py index 962920194da..2b0a31aae11 100644 --- a/examples/peripherals/twai/twai_network/pytest_twai_network.py +++ b/examples/peripherals/twai/twai_network/pytest_twai_network.py @@ -54,7 +54,7 @@ def generate_target_combinations(target_list: list, count: int = 2) -> list: ], indirect=True, ) -@pytest.mark.temp_skip_ci(targets=['esp32p4'], reason='p4 rev3 migration, IDF-14393') +@pytest.mark.temp_skip_ci(targets=['esp32p4,*', '*,esp32p4'], reason='p4 rev3 migration, IDF-14393') def test_twai_network_multi(dut: Tuple[IdfDut, IdfDut], socket_can: Bus) -> None: # type: ignore """ Test TWAI network communication between two nodes: diff --git a/examples/system/deep_sleep/pytest_deep_sleep.py b/examples/system/deep_sleep/pytest_deep_sleep.py index 84e5e7ab961..a4006a5f5e1 100644 --- a/examples/system/deep_sleep/pytest_deep_sleep.py +++ b/examples/system/deep_sleep/pytest_deep_sleep.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 import logging import time @@ -7,25 +7,6 @@ import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -CONFIGS = [ - pytest.param('esp32_singlecore', marks=[pytest.mark.esp32]), - pytest.param( - 'basic', - marks=[ - pytest.mark.esp32, - pytest.mark.esp32s2, - pytest.mark.esp32s3, - pytest.mark.esp32c3, - pytest.mark.esp32c5, - pytest.mark.esp32c6, - pytest.mark.esp32c61, - pytest.mark.esp32h2, - pytest.mark.esp32p4, - pytest.mark.esp32c2, - ], - ), -] - @pytest.mark.generic @idf_parametrize( diff --git a/pytest.ini b/pytest.ini index 1fe5e78cc60..84eda8ca808 100644 --- a/pytest.ini +++ b/pytest.ini @@ -7,7 +7,6 @@ python_files = pytest_*.py addopts = -s -vv --embedded-services esp,idf - -p no:idf-ci --tb short --strict-markers --skip-check-coredump y @@ -15,12 +14,14 @@ addopts = --check-duplicates y --ignore-glob */managed_components/* --ignore pytest-embedded + --unity-test-report-mode merge + --ignore-no-tests-collected-error # ignore DeprecationWarning filterwarnings = ignore::DeprecationWarning:matplotlib.*: ignore::DeprecationWarning:google.protobuf.*: - ignore::_pytest.warning_types.PytestExperimentalApiWarning + ignore::FutureWarning # log related log_cli = True @@ -34,3 +35,102 @@ junit_family = xunit1 ## log all to `system-out` when case fail junit_logging = stdout junit_log_passing_tests = False + +markers = + temp_skip_ci: mark test to be skipped in CI + temp_skip: mark test to be skipped in CI and locally + skip_app_downloader: mark test required apps built locally, not downloaded from CI + require_elf: mark test to be skipped if no elf file is found + +env_markers = + qemu: build and test using qemu, not real target + macos: tests should be run on macos hosts + generic: tests should be run on generic runners + flash_suspend: support flash suspend feature + eth_ip101: connected via IP101 ethernet transceiver + eth_lan8720: connected via LAN8720 ethernet transceiver + eth_rtl8201: connected via RTL8201 ethernet transceiver + eth_ksz8041: connected via KSZ8041 ethernet transceiver + eth_dp83848: connected via DP83848 ethernet transceiver + eth_w5500: SPI Ethernet module with two W5500 + eth_ksz8851snl: SPI Ethernet module with two KSZ8851SNL + eth_dm9051: SPI Ethernet module with two DM9051 + quad_psram: runners with quad psram + octal_psram: runners with octal psram + usb_host_flash_disk: usb host runners with USB flash disk attached + usb_device: usb device runners + ethernet_ota: ethernet OTA runners + flash_encryption: Flash Encryption runners + flash_encryption_f4r8: Flash Encryption runners with 4-line flash and 8-line psram + flash_encryption_f8r8: Flash Encryption runners with 8-line flash and 8-line psram + flash_encryption_ota: Flash Encryption runners with ethernet OTA support with 4mb flash size + flash_encryption_psram: ESP32 Flash Encryption runners with psram + flash_multi: Multiple flash chips tests + psram: Chip has 4-line psram + ir_transceiver: runners with a pair of IR transmitter and receiver + twai_transceiver: runners with a TWAI PHY transceiver + flash_encryption_wifi_high_traffic: Flash Encryption runners with wifi high traffic support + ethernet: ethernet runner + ethernet_stress: ethernet runner with stress test + ethernet_flash_8m: ethernet runner with 8mb flash + ethernet_router: both the runner and dut connect to the same router through ethernet NIC + ethernet_vlan: ethernet runner GARM-32-SH-1-R16S5N3 + wifi_ap: a wifi AP in the environment + wifi_router: both the runner and dut connect to the same wifi router + wifi_high_traffic: wifi high traffic runners + wifi_wlan: wifi runner with a wireless NIC + wifi_iperf: the AP and ESP dut were placed in a shielded box - for iperf test + Example_ShieldBox: multiple shielded APs connected to shielded ESP DUT via RF cable with programmable attenuator + xtal_26mhz: runner with 26MHz xtal on board + xtal_40mhz: runner with 40MHz xtal on board + external_flash: external flash memory connected via VSPI (FSPI) + sdcard_sdmode: sdcard running in SD mode, to be removed after test migration + sdcard_spimode: sdcard running in SPI mode + emmc: eMMC card + sdcard: sdcard runner + MSPI_F8R8: runner with Octal Flash and Octal PSRAM + MSPI_F4R8: runner with Quad Flash and Octal PSRAM + MSPI_F4R4: runner with Quad Flash and Quad PSRAM + flash_120m: runner with 120M supported Flash + jtag: runner where the chip is accessible through JTAG as well + usb_serial_jtag: runner where the chip is accessible through builtin JTAG as well + adc: ADC related tests should run on adc runners + xtal32k: Runner with external 32k crystal connected + no32kXtal: Runner with no external 32k crystal connected + psramv0: Runner with PSRAM version 0 + esp32eco3: Runner with esp32 eco3 connected + ecdsa_efuse: Runner with test ECDSA private keys programmed in efuse + ccs811: Runner with CCS811 connected + nvs_encr_hmac: Runner with test HMAC key programmed in efuse + i2c_oled: Runner with ssd1306 I2C oled connected + httpbin: runner for tests that need to access the httpbin service + flash_4mb: C2 runners with 4 MB flash + jtag_re_enable: Runner to re-enable jtag which is softly disabled by burning bit SOFT_DIS_JTAG on eFuse + es8311: Development board that carries es8311 codec + camera: Runner with camera + ov5647: Runner with camera ov5647 + esp32s3_eye: Runner for esp32s3_eye and camera ov2640 + multi_dut_modbus_rs485: a pair of runners connected by RS485 bus + ieee802154: ieee802154 related tests should run on ieee802154 runners. + openthread_br: tests should be used for openthread border router. + openthread_bbr: tests should be used for openthread border router linked to Internet. + openthread_sleep: tests should be used for openthread sleepy device. + zigbee_multi_dut: zigbee runner which have multiple duts. + two_duts: the runner has two duts with antenna available. + generic_multi_device: generic multiple devices whose corresponding gpio pins are connected to each other. + twai_network: multiple runners form a TWAI network. + sdio_master_slave: Test sdio multi board, esp32+esp32 + sdio_multidev_32_c6: Test sdio multi board, esp32+esp32c6 + sdio_multidev_32_c61: Test sdio multi board, esp32+esp32c61 + sdio_multidev_p4_c5: Test sdio multi board, esp32p4+esp32c5 + usj_device: Test usb_serial_jtag and usb_serial_jtag is used as serial only (not console) + twai_std: twai runner with all twai supported targets connect to usb-can adapter + lp_i2s: lp_i2s runner tested with hp_i2s + ram_app: ram_app runners + esp32c3eco7: esp32c3 major version(v1.1) chips + esp32c2eco4: esp32c2 major version(v2.0) chips + recovery_bootloader: Runner with recovery bootloader offset set in eFuse + esp32p4_eco4: Runner with esp32p4 eco4 connected + esp32c5_eco3: Runner with esp32c5 eco3 connected + rev_default: Runner with default revision connected + flash_32m: Runner with 32MB flash diff --git a/tools/ci/artifacts_handler.py b/tools/ci/artifacts_handler.py deleted file mode 100644 index 3a2cd429e6d..00000000000 --- a/tools/ci/artifacts_handler.py +++ /dev/null @@ -1,215 +0,0 @@ -# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import argparse -import fnmatch -import glob -import os -import typing as t -import zipfile -from enum import Enum -from pathlib import Path -from zipfile import ZipFile - -import urllib3 -from idf_ci_utils import sanitize_job_name -from idf_pytest.constants import DEFAULT_BUILD_LOG_FILENAME -from minio import Minio - - -class ArtifactType(str, Enum): - MAP_AND_ELF_FILES = 'map_and_elf_files' - BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES = 'build_dir_without_map_and_elf_files' - - LOGS = 'logs' - SIZE_REPORTS = 'size_reports' - JUNIT_REPORTS = 'junit_reports' - MODIFIED_FILES_AND_COMPONENTS_REPORT = 'modified_files_and_components_report' - - -TYPE_PATTERNS_DICT = { - ArtifactType.MAP_AND_ELF_FILES: [ - '**/build*/bootloader/*.map', - '**/build*/bootloader/*.elf', - '**/build*/esp_tee/*.map', - '**/build*/esp_tee/*.elf', - '**/build*/*.map', - '**/build*/*.elf', - ], - ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES: [ - f'**/build*/{DEFAULT_BUILD_LOG_FILENAME}', - '**/build*/*.bin', - '**/build*/bootloader/*.bin', - '**/build*/esp_tee/*.bin', - '**/build*/partition_table/*.bin', - '**/build*/flasher_args.json', - '**/build*/flash_project_args', - '**/build*/config/sdkconfig.json', - '**/build*/sdkconfig', - '**/build*/project_description.json', - 'list_job*.txt', - ], - ArtifactType.LOGS: [ - f'**/build*/{DEFAULT_BUILD_LOG_FILENAME}', - ], - ArtifactType.SIZE_REPORTS: [ - '**/build*/size.json', - 'size_info.txt', - ], - ArtifactType.JUNIT_REPORTS: [ - 'XUNIT_RESULT*.xml', - 'build_summary*.xml', - ], - ArtifactType.MODIFIED_FILES_AND_COMPONENTS_REPORT: [ - 'pipeline.env', - ], -} - - -def getenv(env_var: str) -> str: - try: - return os.environ[env_var] - except KeyError as e: - raise Exception(f'Environment variable {env_var} not set') from e - - -def get_minio_client() -> Minio: - return Minio( - getenv('IDF_S3_SERVER').replace('https://', ''), - access_key=getenv('IDF_S3_ACCESS_KEY'), - secret_key=getenv('IDF_S3_SECRET_KEY'), - http_client=urllib3.PoolManager( - num_pools=10, - timeout=urllib3.Timeout.DEFAULT_TIMEOUT, - retries=urllib3.Retry( - total=5, - backoff_factor=0.2, - status_forcelist=[500, 502, 503, 504], - ), - ), - ) - - -def _download_files( - pipeline_id: int, - *, - artifact_type: t.Optional[ArtifactType] = None, - job_name: t.Optional[str] = None, - job_id: t.Optional[int] = None, -) -> None: - if artifact_type: - prefix = f'{pipeline_id}/{artifact_type.value}/' - else: - prefix = f'{pipeline_id}/' - - for obj in client.list_objects(getenv('IDF_S3_BUCKET'), prefix=prefix, recursive=True): - obj_name = obj.object_name - obj_p = Path(obj_name) - # ///.zip - if len(obj_p.parts) != 4: - print(f'Invalid object name: {obj_name}') - continue - - if job_name: - # could be a pattern - if not fnmatch.fnmatch(obj_p.parts[2], job_name): - print(f'Job name {job_name} does not match {obj_p.parts[2]}') - continue - - if job_id: - if obj_p.parts[3] != f'{job_id}.zip': - print(f'Job ID {job_id} does not match {obj_p.parts[3]}') - continue - - client.fget_object(getenv('IDF_S3_BUCKET'), obj_name, obj_name) - print(f'Downloaded {obj_name}') - - if obj_name.endswith('.zip'): - with ZipFile(obj_name, 'r') as zr: - zr.extractall() - print(f'Extracted {obj_name}') - - os.remove(obj_name) - - -def _upload_files( - pipeline_id: int, - *, - artifact_type: ArtifactType, - job_name: str, - job_id: str, -) -> None: - has_file = False - with ZipFile( - f'{job_id}.zip', - 'w', - compression=zipfile.ZIP_DEFLATED, - # 1 is the fastest compression level - # the size differs not much between 1 and 9 - compresslevel=1, - ) as zw: - for pattern in TYPE_PATTERNS_DICT[artifact_type]: - for file in glob.glob(pattern, recursive=True): - zw.write(file) - has_file = True - - try: - if has_file: - obj_name = f'{pipeline_id}/{artifact_type.value}/{sanitize_job_name(job_name)}/{job_id}.zip' - client.fput_object(getenv('IDF_S3_BUCKET'), obj_name, f'{job_id}.zip') - print(f'Created archive file: {job_id}.zip, uploaded as {obj_name}') - finally: - os.remove(f'{job_id}.zip') - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description='Download or upload files from/to S3, the object name would be ' - '[PIPELINE_ID]/[ACTION_TYPE]/[JOB_NAME]/[JOB_ID].zip.' - '\n' - 'For example: 123456/binaries/build_pytest_examples_esp32/123456789.zip', - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - - common_args = argparse.ArgumentParser(add_help=False, formatter_class=argparse.ArgumentDefaultsHelpFormatter) - common_args.add_argument('--pipeline-id', type=int, help='Pipeline ID') - common_args.add_argument( - '--type', type=str, nargs='+', choices=[a.value for a in ArtifactType], help='Types of files to download' - ) - - action = parser.add_subparsers(dest='action', help='Download or Upload') - download = action.add_parser('download', help='Download files from S3', parents=[common_args]) - upload = action.add_parser('upload', help='Upload files to S3', parents=[common_args]) - - download.add_argument('--job-name', type=str, help='Job name pattern') - download.add_argument('--job-id', type=int, help='Job ID') - - upload.add_argument('--job-name', type=str, help='Job name') - upload.add_argument('--job-id', type=int, help='Job ID') - - args = parser.parse_args() - - client = get_minio_client() - - ci_pipeline_id = args.pipeline_id or getenv('CI_PIPELINE_ID') # required - if args.action == 'download': - method = _download_files - ci_job_name = args.job_name # optional - ci_job_id = args.job_id # optional - else: - method = _upload_files # type: ignore - ci_job_name = args.job_name or getenv('CI_JOB_NAME') # required - ci_job_id = args.job_id or getenv('CI_JOB_ID') # required - - if args.type: - types = [ArtifactType(t) for t in args.type] - else: - types = list(ArtifactType) - - print(f'{"Pipeline ID":15}: {ci_pipeline_id}') - if ci_job_name: - print(f'{"Job name":15}: {ci_job_name}') - if ci_job_id: - print(f'{"Job ID":15}: {ci_job_id}') - - for _t in types: - method(ci_pipeline_id, artifact_type=_t, job_name=ci_job_name, job_id=ci_job_id) # type: ignore diff --git a/tools/ci/check_build_test_rules.py b/tools/ci/check_build_test_rules.py index a57b866eea9..9700fa6440a 100755 --- a/tools/ci/check_build_test_rules.py +++ b/tools/ci/check_build_test_rules.py @@ -1,29 +1,26 @@ #!/usr/bin/env python -# SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import argparse -import inspect import os import re import sys +import typing as t +from collections import defaultdict from pathlib import Path -from typing import Dict -from typing import List -from typing import Optional -from typing import Tuple import yaml -from idf_ci_utils import get_all_manifest_files +from idf_build_apps import App +from idf_build_apps import find_apps +from idf_build_apps.constants import SUPPORTED_TARGETS from idf_ci_utils import IDF_PATH - -YES = u'\u2713' -NO = u'\u2717' +from idf_ci_utils import idf_relpath +from rich.console import Console +from rich.table import Table # | Supported Target | ... | # | ---------------- | --- | -SUPPORTED_TARGETS_TABLE_REGEX = re.compile( - r'^\|\s*Supported Targets.+$\n^\|(?:\s*|-).+$\n?', re.MULTILINE -) +SUPPORTED_TARGETS_TABLE_REGEX = re.compile(r'^\|\s*Supported Targets.+$\n^\|(?:\s*|-).+$\n?', re.MULTILINE) USUAL_TO_FORMAL = { 'esp32': 'ESP32', @@ -41,331 +38,254 @@ USUAL_TO_FORMAL = { 'linux': 'Linux', } -FORMAL_TO_USUAL = { - 'ESP32': 'esp32', - 'ESP32-S2': 'esp32s2', - 'ESP32-S3': 'esp32s3', - 'ESP32-C3': 'esp32c3', - 'ESP32-C2': 'esp32c2', - 'ESP32-C6': 'esp32c6', - 'ESP32-C5': 'esp32c5', - 'ESP32-H2': 'esp32h2', - 'ESP32-P4': 'esp32p4', - 'ESP32-C61': 'esp32c61', - 'ESP32-H21': 'esp32h21', - 'ESP32-H4': 'esp32h4', - 'Linux': 'linux', -} +FORMAL_TO_USUAL = {v: k for k, v in USUAL_TO_FORMAL.items()} -def doublequote(s: str) -> str: - if s.startswith('"') and s.endswith('"'): - return s +def print_diff_table( + list1: t.List[str], list2: t.List[str], title1: str, title2: str, excluded: t.Optional[t.List[str]] = None +) -> None: + left_set = set(list1) + right_set = set(list2) - return f'"{s}"' + if left_set == right_set: + return + + all_items = sorted(left_set.union(right_set)) + left_column = [] + right_column = [] + + for item in all_items: + if item in left_set and item not in right_set: + if excluded and item in excluded: + left_column.append(f'[red]{item}[/red] (excluded)') + else: + left_column.append(f'[red]{item}[/red]') + right_column.append('') + elif item in right_set and item not in left_set: + if excluded and item in excluded: + right_column.append(f'[green]{item}[/green] (excluded)') + else: + right_column.append(f'[green]{item}[/green]') + left_column.append('') + else: + left_column.append(item) + right_column.append(item) + + table = Table(show_header=True, header_style='bold magenta') + table.add_column(title1.strip(), justify='left') + table.add_column(title2.strip(), justify='left') + + for l_col, r_col in zip(left_column, right_column): + table.add_row(l_col, r_col) + + console = Console() + console.print(table) + + +def get_readme_path(app_dir: str) -> t.Optional[str]: + readme_path = os.path.join(app_dir, 'README.md') + + if not os.path.isfile(readme_path): + readme_path = os.path.join(app_dir, '..', 'README.md') + + if not os.path.isfile(readme_path): + readme_path = None # type: ignore + + return readme_path + + +def generate_new_support_table(targets: t.List[str]) -> str: + # extra space here + table_headers = [f'{USUAL_TO_FORMAL[target]}' for target in targets] + table_headers = ['Supported Targets'] + table_headers + + res = '| ' + ' | '.join(table_headers) + ' |\n' + res += '| ' + ' | '.join(['-' * len(item) for item in table_headers]) + ' |' + + return res + + +def parse_existing_table(app_dir: str) -> t.Tuple[t.Optional[str], t.List[str]]: + readme_path = get_readme_path(app_dir) + if not readme_path: + return None, SUPPORTED_TARGETS + + with open(readme_path, encoding='utf8') as _fr: + readme_str = _fr.read() + + support_string = SUPPORTED_TARGETS_TABLE_REGEX.findall(readme_str) + if not support_string: + return None, SUPPORTED_TARGETS + + # old style + parts = [part.strip() for part in support_string[0].split('\n', 1)[0].split('|') if part.strip()] + return support_string[0].strip(), [FORMAL_TO_USUAL[part] for part in parts[1:] if part in FORMAL_TO_USUAL] + + +def get_grouped_apps( + paths: t.List[str], + exclude_dirs: t.Optional[t.List[str]] = None, +) -> t.Dict[str, t.List[App]]: + apps = sorted( + find_apps( + paths, + 'all', + exclude_list=exclude_dirs or [], + ) + ) + + grouped_apps: t.Dict[str, t.List[App]] = defaultdict(list) + for app in apps: + grouped_apps[app.app_dir].append(app) + + return grouped_apps def check_readme( - paths: List[str], - exclude_dirs: Optional[List[str]] = None, - extra_default_build_targets: Optional[List[str]] = None, + paths: t.List[str], + exclude_dirs: t.Optional[t.List[str]] = None, ) -> None: - from idf_build_apps import App, find_apps - from idf_build_apps.constants import SUPPORTED_TARGETS - - def get_readme_path(_app: App) -> Optional[str]: - _readme_path = os.path.join(_app.app_dir, 'README.md') - - if not os.path.isfile(_readme_path): - _readme_path = os.path.join(_app.app_dir, '..', 'README.md') - - if not os.path.isfile(_readme_path): - _readme_path = None # type: ignore - - return _readme_path - - def _generate_new_support_table_str(_app: App) -> str: - # extra space here - table_headers = [ - f'{USUAL_TO_FORMAL[target]}' for target in _app.supported_targets - ] - table_headers = ['Supported Targets'] + table_headers - - res = '| ' + ' | '.join(table_headers) + ' |\n' - res += '| ' + ' | '.join(['-' * len(item) for item in table_headers]) + ' |' - - return res - - def _parse_existing_support_table_str(_app: App) -> Tuple[Optional[str], List[str]]: - _readme_path = get_readme_path(_app) - if not _readme_path: - return None, SUPPORTED_TARGETS - - with open(_readme_path, encoding='utf8') as _fr: - _readme_str = _fr.read() - - support_string = SUPPORTED_TARGETS_TABLE_REGEX.findall(_readme_str) - if not support_string: - return None, SUPPORTED_TARGETS - - # old style - parts = [ - part.strip() - for part in support_string[0].split('\n', 1)[0].split('|') - if part.strip() - ] - return support_string[0].strip(), [FORMAL_TO_USUAL[part] for part in parts[1:] if part in FORMAL_TO_USUAL] - - def check_enable_build(_app: App, _old_supported_targets: List[str]) -> bool: - if _app.supported_targets == sorted(_old_supported_targets): - return True - - _readme_path = get_readme_path(_app) - if_clause = f'IDF_TARGET in [{", ".join([doublequote(target) for target in sorted(_old_supported_targets)])}]' - - print( - inspect.cleandoc( - f''' - {_app.app_dir}: - - enable build targets according to the manifest file: {_app.supported_targets} - - enable build targets according to the old Supported Targets table under readme "{_readme_path}": {_old_supported_targets} - - If you want to disable some targets, please use the following snippet: - - # Please combine this with the original one - # - # Notes: - # - please keep in mind to avoid duplicated folders as yaml keys - # - please use parentheses to group conditions, the "or" and "and" operators could only accept two operands - {_app.app_dir}: - enable: - - if: {if_clause} - temporary: true - reason: - - ''' - ) - ) - - return False - - apps = sorted( - find_apps( - paths, - 'all', - recursive=True, - exclude_list=exclude_dirs or [], - manifest_files=get_all_manifest_files(), - default_build_targets=SUPPORTED_TARGETS + extra_default_build_targets, - ) - ) + grouped_apps = get_grouped_apps(paths, exclude_dirs) exit_code = 0 - checked_app_dirs = set() - for app in apps: - if app.app_dir not in checked_app_dirs: - checked_app_dirs.add(app.app_dir) - else: - continue + for app_dir, apps in grouped_apps.items(): + old_table, old_targets = parse_existing_table(app_dir) + manifest_targets = sorted( + { + target + for app in apps + for target in ( + App.MANIFEST.enable_build_targets(app_dir) + + App.MANIFEST.enable_build_targets(app_dir, config_name=app.config_name) + ) + } + ) - replace_str, old_supported_targets = _parse_existing_support_table_str(app) - success = check_enable_build(app, old_supported_targets) - if not success: - print(f'check_enable_build failed for app: {app}') - print('-' * 80) + if sorted(manifest_targets) != sorted(old_targets): + print(f'Build target MISMATCH!!!: {app_dir}') + print_diff_table(manifest_targets, old_targets, 'Manifest', 'README') + print(f'Corresponding manifest file: {App.MANIFEST.most_suitable_rule(app_dir).by_manifest_file}') exit_code = 1 - readme_path = get_readme_path(app) + readme_path = get_readme_path(app_dir) + new_table = generate_new_support_table(manifest_targets) # no readme, create a new file if not readme_path: - with open(os.path.join(app.app_dir, 'README.md'), 'w') as fw: - fw.write(_generate_new_support_table_str(app) + '\n') - print(f'Added new README file: {os.path.join(app.app_dir, "README.md")}') - print('-' * 80) + with open(os.path.join(app_dir, 'README.md'), 'w') as fw: + fw.write(new_table + '\n') + print(f'Auto-fixing: Added new README: {os.path.join(app_dir, "README.md")}') exit_code = 1 # has old table, but different string - elif replace_str and replace_str != _generate_new_support_table_str(app): + elif old_table and old_table != new_table: with open(readme_path) as fr: readme_str = fr.read() with open(readme_path, 'w') as fw: - fw.write( - readme_str.replace( - replace_str, _generate_new_support_table_str(app) - ) - ) - print(f'Modified README file: {readme_path}') - print('-' * 80) + fw.write(readme_str.replace(old_table, new_table)) + print(f'Auto-fixing: Updated README: {readme_path}') exit_code = 1 # does not have old table - elif not replace_str: + elif not old_table: with open(readme_path) as fr: readme_str = fr.read() with open(readme_path, 'w') as fw: - fw.write( - _generate_new_support_table_str(app) + '\n\n' + readme_str - ) # extra new line - - print(f'Modified README file: {readme_path}') - print('-' * 80) + fw.write(new_table + '\n\n' + readme_str) # extra new line + print(f'Auto-fixing: Modified README: {readme_path}') exit_code = 1 + if exit_code != 0: + print('') # extra new line for readability + print('Related documentation:') + print( + '\t- https://docs.espressif.com/projects/idf-build-apps/en/latest/references/manifest.html#enable-disable-rules' + ) sys.exit(exit_code) +def get_grouped_cases(paths: t.List[str]) -> t.Dict[str, t.Dict[str, t.Set[str]]]: + """ + return something like this: + { + app_dir: { + 'script_paths': {'path/to/script1', 'path/to/script2'}, + 'targets': {'esp32', 'esp32s2', 'esp32s3', 'esp32c3', 'esp32c2', 'linux'}, + } + } + """ + from idf_ci import get_pytest_cases + + pytest_cases = get_pytest_cases( + paths=paths, + marker_expr=None, # don't filter host_test + ) + + grouped_cases = {} + for case in pytest_cases: + for pytest_app in case.apps: + app_dir = idf_relpath(pytest_app.path) + if app_dir not in grouped_cases: + grouped_cases[app_dir] = { + 'script_paths': {case.path}, + 'targets': {pytest_app.target}, + } + else: + grouped_cases[app_dir]['script_paths'].add(case.path) + grouped_cases[app_dir]['targets'].add(pytest_app.target) + + return grouped_cases + + def check_test_scripts( - paths: List[str], - exclude_dirs: Optional[List[str]] = None, - bypass_check_test_targets: Optional[List[str]] = None, - extra_default_build_targets: Optional[List[str]] = None, + paths: t.List[str], + exclude_dirs: t.Optional[t.List[str]] = None, + *, + bypass_targets: t.Optional[t.List[str]] = None, ) -> None: - from idf_build_apps import App, find_apps - from idf_build_apps.constants import SUPPORTED_TARGETS - from idf_pytest.script import get_pytest_cases - # takes long time, run only in CI - # dict: - # { - # app_dir: { - # 'script_path': 'path/to/script', - # 'targets': ['esp32', 'esp32s2', 'esp32s3', 'esp32c3', 'esp32c2', 'linux'], - # } - # } - def check_enable_test( - _app: App, - _pytest_app_dir_targets_dict: Dict[str, Dict[str, str]], - ) -> bool: - if _app.app_dir in _pytest_app_dir_targets_dict: - test_script_path = _pytest_app_dir_targets_dict[_app.app_dir]['script_path'] - actual_verified_targets = sorted( - set(_pytest_app_dir_targets_dict[_app.app_dir]['targets']) - ) - else: - return True # no test case + grouped_apps = get_grouped_apps(paths, exclude_dirs) + grouped_cases = get_grouped_cases(paths) + exit_code = 0 - actual_extra_tested_targets = set(actual_verified_targets) - set( - _app.verified_targets - ) - if actual_extra_tested_targets - set(bypass_check_test_targets or []): - print( - inspect.cleandoc( - f''' - {_app.app_dir}: - - enable test targets according to the manifest file: {_app.verified_targets} - - enable test targets according to the test scripts: {actual_verified_targets} + for app_dir, apps in grouped_apps.items(): + if app_dir not in grouped_cases: + continue - test scripts enabled targets should be a subset of the manifest file declared ones. - Please check the test script: {test_script_path}. - - ''' + # manifest defined ones + manifest_targets = sorted( + { + target + for app in apps + for target in ( + App.MANIFEST.enable_test_targets(app_dir) + + App.MANIFEST.enable_test_targets(app_dir, config_name=app.config_name) ) - ) - return False + } + ) + actual_targets = sorted(grouped_cases[app_dir]['targets']) - if _app.verified_targets == actual_verified_targets: - return True - elif not (set(_app.verified_targets) - set(actual_verified_targets + (bypass_check_test_targets or []))): - print(f'WARNING: bypass test script check on {_app.app_dir} for targets {bypass_check_test_targets} ') - return True + if manifest_targets == actual_targets: + continue + if not ((set(manifest_targets) ^ set(actual_targets)) - set(bypass_targets or [])): + continue - if_clause = f'IDF_TARGET in [{", ".join([doublequote(target) for target in sorted(set(_app.verified_targets) - set(actual_verified_targets))])}]' + print(f'Test target MISMATCH!!!: {app_dir}') + print_diff_table(manifest_targets, actual_targets, 'Manifest', 'Test Scripts', bypass_targets) + print(f'Corresponding manifest file: {App.MANIFEST.most_suitable_rule(app_dir).by_manifest_file}') + print('Corresponding test scripts:') + for script_path in grouped_cases[app_dir]['script_paths']: + print(' - ' + script_path) + exit_code = 1 + if exit_code != 0: + print('') # extra new line for readability + print('Related documentation:') print( - inspect.cleandoc( - f''' - {_app.app_dir}: - - enable test targets according to the manifest file: {_app.verified_targets} - - enable test targets according to the test scripts: {actual_verified_targets} - - the test scripts enabled test targets should be the same with the manifest file enabled ones. Please check - the test script manually: {test_script_path}. - - If you want to enable test targets in the pytest test scripts, please add `@pytest.mark.MISSING_TARGET` - marker above the test case function. - - If you want to disable the test targets in the manifest file, please modify your manifest file with - the following code snippet: - - # Please combine this with the original one - # - # Notes: - # - please keep in mind to avoid duplicated folders as yaml keys - # - please use parentheses to group conditions, the "or" and "and" operators could only accept two operands - {_app.app_dir}: - disable_test: - - if: {if_clause} - temporary: true - reason: - - ''' - ) + '\t- https://docs.espressif.com/projects/idf-build-apps/en/latest/references/manifest.html#enable-disable-rules' ) - return False - - apps = sorted( - find_apps( - paths, - 'all', - recursive=True, - exclude_list=exclude_dirs or [], - manifest_files=get_all_manifest_files(), - default_build_targets=SUPPORTED_TARGETS + extra_default_build_targets, + print( + '\t- https://docs.espressif.com/projects/esp-idf/en/latest/esp32/contribute/esp-idf-tests-with-pytest.html' ) - ) - exit_code = 0 - - pytest_cases = get_pytest_cases(paths) - - pytest_app_dir_targets_dict = {} - for case in pytest_cases: - for pytest_app in case.apps: - app_dir = os.path.relpath(pytest_app.path, IDF_PATH) - if app_dir not in pytest_app_dir_targets_dict: - pytest_app_dir_targets_dict[app_dir] = { - 'script_path': case.path, - 'targets': [pytest_app.target], - } - else: - pytest_app_dir_targets_dict[app_dir]['targets'].append( - pytest_app.target - ) - - checked_app_dirs = set() - for app in apps: - if app.app_dir not in checked_app_dirs: - checked_app_dirs.add(app.app_dir) - else: - continue - - success = check_enable_test( - app, pytest_app_dir_targets_dict - ) - if not success: - print(f'check_enable_test failed for app: {app}') - print('-' * 80) - exit_code = 1 - continue - - sys.exit(exit_code) - - -def check_exist() -> None: - exit_code = 0 - - config_files = get_all_manifest_files() - for file in config_files: - if 'managed_components' in Path(file).parts: - continue - - with open(file) as fr: - configs = yaml.safe_load(fr) - for path in configs.keys(): - if path.startswith('.'): - continue - if not os.path.isdir(path): - print(f'Path \'{path}\' referred in \'{file}\' does not exist!') - exit_code = 1 sys.exit(exit_code) @@ -377,92 +297,79 @@ if __name__ == '__main__': parser = argparse.ArgumentParser(description='ESP-IDF apps build/test checker') action = parser.add_subparsers(dest='action') - _check_readme = action.add_parser('check-readmes') - _check_readme.add_argument('paths', nargs='+', help='check under paths') - _check_readme.add_argument( + readme_parser = action.add_parser('check-readmes') + readme_parser.add_argument('paths', nargs='+', help='check under paths') + readme_parser.add_argument( '-c', '--config', default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'), - help='default build test rules config file', + help='config file', ) - _check_test_scripts = action.add_parser('check-test-scripts') - _check_test_scripts.add_argument('paths', nargs='+', help='check under paths') - _check_test_scripts.add_argument( + test_parser = action.add_parser('check-test-scripts') + test_parser.add_argument('paths', nargs='+', help='check under paths') + test_parser.add_argument( '-c', '--config', default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'), - help='default build test rules config file', + help='config file', ) - - _check_exist = action.add_parser('check-exist') - arg = parser.parse_args() - # Since this script is executed from the pre-commit hook environment, make sure IDF_PATH is set - os.environ['IDF_PATH'] = os.path.realpath( - os.path.join(os.path.dirname(__file__), '..', '..') + check_dirs = set() + + # check if *_caps.h files changed + check_all = False + soc_caps_header_files = list((Path(IDF_PATH) / 'components' / 'soc').glob('**/*_caps.h')) + for p in arg.paths: + if Path(p).resolve() in soc_caps_header_files: + check_all = True + break + + if os.path.isfile(p): + check_dirs.add(os.path.dirname(p)) + else: + check_dirs.add(p) + + if 'tools/idf_py_actions/constants.py' in arg.paths or 'tools/ci/check_build_test_rules.py' in arg.paths: + check_all = True + + if check_all: + check_dirs = {IDF_PATH} + _exclude_dirs = [ + os.path.join(IDF_PATH, 'tools', 'unit-test-app'), + os.path.join(IDF_PATH, 'tools', 'test_build_system', 'build_test_app'), + os.path.join(IDF_PATH, 'tools', 'test_build_system', 'buildv2_test_app'), + os.path.join(IDF_PATH, 'tools', 'templates', 'sample_project'), + os.path.join(IDF_PATH, 'tools', 'cmakev2', 'test'), + ] + else: + _exclude_dirs = [os.path.join(IDF_PATH, 'tools', 'templates', 'sample_project')] + + _bypass_targets: t.List[str] = [] + if arg.config: + with open(arg.config) as fr: + configs = yaml.safe_load(fr) + + if configs: + _bypass_targets = configs.get('bypass_check_test_targets') or [] + + os.environ.update( + { + 'IDF_PATH': IDF_PATH, + 'INCLUDE_NIGHTLY_RUN': '1', + 'NIGHTLY_RUN': '1', + } ) - if arg.action == 'check-exist': - check_exist() - else: - check_dirs = set() - - # check if *_caps.h files changed - check_all = False - soc_caps_header_files = list( - (Path(IDF_PATH) / 'components' / 'soc').glob('**/*_caps.h') + if arg.action == 'check-readmes': + check_readme( + list(check_dirs), + _exclude_dirs, + ) + elif arg.action == 'check-test-scripts': + check_test_scripts( + list(check_dirs), + _exclude_dirs, + bypass_targets=_bypass_targets, ) - for p in arg.paths: - if Path(p).resolve() in soc_caps_header_files: - check_all = True - break - - if os.path.isfile(p): - check_dirs.add(os.path.dirname(p)) - else: - check_dirs.add(p) - - if 'tools/idf_py_actions/constants.py' in arg.paths or 'tools/ci/check_build_test_rules.py' in arg.paths: - check_all = True - - if check_all: - check_dirs = {IDF_PATH} - _exclude_dirs = [os.path.join(IDF_PATH, 'tools', 'unit-test-app'), - os.path.join(IDF_PATH, 'tools', 'test_build_system', 'build_test_app'), - os.path.join(IDF_PATH, 'tools', 'templates', 'sample_project')] - else: - _exclude_dirs = [os.path.join(IDF_PATH, 'tools', 'templates', 'sample_project')] - - extra_default_build_targets_list: List[str] = [] - bypass_check_test_targets_list: List[str] = [] - if arg.config: - with open(arg.config) as fr: - configs = yaml.safe_load(fr) - - if configs: - extra_default_build_targets_list = ( - configs.get('extra_default_build_targets') or [] - ) - bypass_check_test_targets_list = ( - configs.get('bypass_check_test_targets') or [] - ) - - if arg.action == 'check-readmes': - os.environ['INCLUDE_NIGHTLY_RUN'] = '1' - os.environ['NIGHTLY_RUN'] = '1' - check_readme( - list(check_dirs), - exclude_dirs=_exclude_dirs, - extra_default_build_targets=extra_default_build_targets_list, - ) - elif arg.action == 'check-test-scripts': - os.environ['INCLUDE_NIGHTLY_RUN'] = '1' - os.environ['NIGHTLY_RUN'] = '1' - check_test_scripts( - list(check_dirs), - exclude_dirs=_exclude_dirs, - bypass_check_test_targets=bypass_check_test_targets_list, - extra_default_build_targets=extra_default_build_targets_list, - ) diff --git a/tools/ci/dynamic_pipelines/constants.py b/tools/ci/dynamic_pipelines/constants.py index 23efe9e6255..40dbe346d11 100644 --- a/tools/ci/dynamic_pipelines/constants.py +++ b/tools/ci/dynamic_pipelines/constants.py @@ -1,49 +1,20 @@ -# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import os from idf_ci_utils import IDF_PATH -# use relative path to avoid absolute path in pipeline -DEFAULT_TEST_PATHS = [ - 'examples', - os.path.join('tools', 'test_apps'), - 'components', -] - -DEFAULT_APPS_BUILD_PER_JOB = 60 -DEFAULT_CASES_TEST_PER_JOB = 30 - -DEFAULT_BUILD_CHILD_PIPELINE_FILEPATH = os.path.join(IDF_PATH, 'build_child_pipeline.yml') -DEFAULT_TARGET_TEST_CHILD_PIPELINE_FILEPATH = os.path.join(IDF_PATH, 'target_test_child_pipeline.yml') - -DEFAULT_BUILD_CHILD_PIPELINE_NAME = 'Build Child Pipeline' -DEFAULT_TARGET_TEST_CHILD_PIPELINE_NAME = 'Target Test Child Pipeline' - -DEFAULT_TARGET_TEST_JOB_TEMPLATE_NAME = '.dynamic_target_test_template' -TIMEOUT_4H_TEMPLATE_NAME = '.timeout_4h_template' - -TEST_RELATED_BUILD_JOB_NAME = 'build_test_related_apps' -NON_TEST_RELATED_BUILD_JOB_NAME = 'build_non_test_related_apps' - COMMENT_START_MARKER = '### Dynamic Pipeline Report' -TEST_RELATED_APPS_FILENAME = 'test_related_apps.txt' -NON_TEST_RELATED_APPS_FILENAME = 'non_test_related_apps.txt' -TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME = 'test_related_apps_download_urls.yml' REPORT_TEMPLATE_FILEPATH = os.path.join( IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'report.template.html' ) +CSS_STYLES_FILEPATH = os.path.join(IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'styles.css') +JS_SCRIPTS_FILEPATH = os.path.join(IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'scripts.js') TOP_N_APPS_BY_SIZE_DIFF = 10 SIZE_DIFFERENCE_BYTES_THRESHOLD = 500 BINARY_SIZE_METRIC_NAME = 'binary_size' -RETRY_JOB_PICTURE_PATH = 'tools/ci/dynamic_pipelines/templates/retry-jobs.png' -RETRY_JOB_TITLE = '\n\nRetry failed jobs with with help of "retry_failed_jobs" stage of the pipeline:' -RETRY_JOB_PICTURE_LINK = '![Retry Jobs Image]({pic_url})' - -BUILD_ONLY_LABEL = 'For Maintainers: Only Build Tests' - KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH = os.path.join( IDF_PATH, 'tools', 'ci', 'dynamic_pipelines', 'templates', 'known_generate_test_child_pipeline_warnings.yml' ) diff --git a/tools/ci/dynamic_pipelines/models.py b/tools/ci/dynamic_pipelines/models.py index 299f2438f12..2cd5022a112 100644 --- a/tools/ci/dynamic_pipelines/models.py +++ b/tools/ci/dynamic_pipelines/models.py @@ -1,131 +1,14 @@ # SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 -import inspect import os import typing as t import urllib.parse from dataclasses import dataclass from xml.etree.ElementTree import Element -import yaml from idf_ci_utils import IDF_PATH -class Job: - def __init__( - self, - *, - name: str, - extends: t.Optional[t.List[str]] = None, - tags: t.Optional[t.List[str]] = None, - stage: t.Optional[str] = None, - parallel: int = 1, - variables: t.Optional[t.Dict[str, str]] = None, - script: t.Optional[t.List[str]] = None, - before_script: t.Optional[t.List[str]] = None, - after_script: t.Optional[t.List[str]] = None, - needs: t.Optional[t.List[str]] = None, - **kwargs: t.Any, - ) -> None: - self.name = name - self.extends = extends - self.tags = tags - self.stage = stage - self.parallel = parallel - self.variables = variables or {} - self.script = script - self.before_script = before_script - self.after_script = after_script - self.needs = needs - - for k, v in kwargs.items(): - setattr(self, k, v) - - def __str__(self) -> str: - return yaml.dump(self.to_dict()) # type: ignore - - def set_variable(self, key: str, value: str) -> None: - self.variables[key] = value - - def to_dict(self) -> t.Dict[str, t.Any]: - res = {} - for k, v in inspect.getmembers(self): - if k.startswith('_'): - continue - - # name is the dict key - if k == 'name': - continue - - # parallel 1 is not allowed - if k == 'parallel' and v == 1: - continue - - if v is None: - continue - - if inspect.ismethod(v) or inspect.isfunction(v): - continue - - res[k] = v - - return {self.name: res} - - -class EmptyJob(Job): - def __init__( - self, - *, - name: t.Optional[str] = None, - tags: t.Optional[t.List[str]] = None, - stage: t.Optional[str] = None, - before_script: t.Optional[t.List[str]] = None, - after_script: t.Optional[t.List[str]] = None, - **kwargs: t.Any, - ) -> None: - super().__init__( - name=name or 'fake_pass_job', - tags=tags or ['fast_run', 'shiny'], - stage=stage or 'build', - script=['echo "This is a fake job to pass the pipeline"'], - before_script=before_script or [], - after_script=after_script or [], - **kwargs, - ) - - -class BuildJob(Job): - def __init__( - self, - *, - extends: t.Optional[t.List[str]] = None, - tags: t.Optional[t.List[str]] = None, - stage: t.Optional[str] = None, - **kwargs: t.Any, - ) -> None: - super().__init__( - extends=extends or ['.dynamic_build_template'], - tags=tags or ['build', 'shiny'], - stage=stage or 'build', - **kwargs, - ) - - -class TargetTestJob(Job): - def __init__( - self, - *, - extends: t.Optional[t.List[str]] = None, - stage: t.Optional[str] = None, - **kwargs: t.Any, - ) -> None: - super().__init__( - extends=extends or ['.dynamic_target_test_template'], - stage=stage or 'target_test', - **kwargs, - ) - - @dataclass class TestCase: name: str diff --git a/tools/ci/dynamic_pipelines/report.py b/tools/ci/dynamic_pipelines/report.py index b2a688c1e69..d9918bc89f2 100644 --- a/tools/ci/dynamic_pipelines/report.py +++ b/tools/ci/dynamic_pipelines/report.py @@ -1,48 +1,47 @@ # SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import abc -import copy -import fnmatch import html -import os import re import typing as t from textwrap import dedent -import yaml -from artifacts_handler import ArtifactType from gitlab import GitlabUpdateError from gitlab_api import Gitlab -from idf_build_apps import App from idf_build_apps.constants import BuildStatus from idf_ci_local.app import AppWithMetricsInfo -from idf_ci_local.uploader import AppUploader +from idf_ci_utils import idf_relpath from prettytable import PrettyTable from .constants import BINARY_SIZE_METRIC_NAME +from .constants import CI_DASHBOARD_API from .constants import COMMENT_START_MARKER +from .constants import CSS_STYLES_FILEPATH +from .constants import JS_SCRIPTS_FILEPATH from .constants import REPORT_TEMPLATE_FILEPATH -from .constants import RETRY_JOB_PICTURE_LINK -from .constants import RETRY_JOB_PICTURE_PATH -from .constants import RETRY_JOB_TITLE from .constants import SIZE_DIFFERENCE_BYTES_THRESHOLD -from .constants import TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME from .constants import TOP_N_APPS_BY_SIZE_DIFF from .models import GitlabJob from .models import TestCase -from .utils import fetch_failed_testcases_failure_ratio from .utils import format_permalink from .utils import get_artifacts_url -from .utils import get_repository_file_url from .utils import is_url -from .utils import known_failure_issue_jira_fast_link -from .utils import load_known_failure_cases class ReportGenerator: REGEX_PATTERN = r'#### {}\n[\s\S]*?(?=\n#### |$)' - def __init__(self, project_id: int, mr_iid: int, pipeline_id: int, job_id: int, commit_id: str, *, title: str): + def __init__( + self, + project_id: int, + mr_iid: int, + pipeline_id: int, + job_id: int, + commit_id: str, + local_commit_id: str, + *, + title: str, + ): gl_project = Gitlab(project_id).project if mr_iid is not None: self.mr = gl_project.mergerequests.get(mr_iid) @@ -51,11 +50,16 @@ class ReportGenerator: self.pipeline_id = pipeline_id self.job_id = job_id self.commit_id = commit_id + self.local_commit_id = local_commit_id self.title = title self.output_filepath = self.title.lower().replace(' ', '_') + '.html' self.additional_info = '' + @property + def get_commit_summary(self) -> str: + return f'with CI commit SHA: {self.commit_id[:8]}, local commit SHA: {self.local_commit_id[:8]}' + @staticmethod def get_download_link_for_url(url: str) -> str: if url: @@ -78,19 +82,41 @@ class ReportGenerator: with open(output_filepath, 'w') as file: file.write(report_str) - # for example, {URL}/-/esp-idf/-/jobs/{id}/artifacts/list_job_84.txt + # for example, {URL}/-/esp-idf/-/jobs/{id}/artifacts/app_info_84.txt # CI_PAGES_URL is {URL}/esp-idf, which missed one `-` report_url: str = get_artifacts_url(job_id, output_filepath) return report_url + @staticmethod + def _load_file_content(filepath: str) -> str: + """ + Load the content of a file as string + + :param filepath: Path to the file to load + :return: Content of the file as string + """ + try: + with open(filepath, encoding='utf-8') as f: + return f.read() + except (OSError, FileNotFoundError) as e: + print(f'Warning: Could not read file {filepath}: {e}') + return '' + def generate_html_report(self, table_str: str) -> str: # we're using bootstrap table table_str = table_str.replace( '', '
', ) - with open(REPORT_TEMPLATE_FILEPATH) as fr: - template = fr.read() + + template = self._load_file_content(REPORT_TEMPLATE_FILEPATH) + css_content = self._load_file_content(CSS_STYLES_FILEPATH) + js_content = self._load_file_content(JS_SCRIPTS_FILEPATH) + + template = template.replace('{{css_content}}', css_content) + template = template.replace('{{js_content}}', js_content) + template = template.replace('{{pipeline_id}}', str(self.pipeline_id)) + template = template.replace('{{apiBaseUrl}}', CI_DASHBOARD_API) return template.replace('{{title}}', self.title).replace('{{table}}', table_str) @@ -136,17 +162,23 @@ class ReportGenerator: return report_sections @staticmethod - def generate_additional_info_section(title: str, count: int, report_url: t.Optional[str] = None) -> str: + def generate_additional_info_section( + title: str, count: int, report_url: t.Optional[str] = None, add_permalink: bool = True + ) -> str: """ Generate a section for the additional info string. :param title: The title of the section. :param count: The count of test cases. :param report_url: The URL of the report. If count = 0, only the count will be included. + :param add_permalink: Whether to include a permalink in the report URL. Defaults to True. :return: The formatted additional info section string. """ if count != 0 and report_url: - return f'- **{title}:** [{count}]({report_url}/#{format_permalink(title)})\n' + if add_permalink: + return f'- **{title}:** [{count}]({report_url}/#{format_permalink(title)})\n' + else: + return f'- **{title}:** [{count}]({report_url})\n' else: return f'- **{title}:** {count}\n' @@ -250,25 +282,13 @@ class ReportGenerator: return comment - def _update_mr_comment(self, comment: str, print_retry_jobs_message: bool) -> None: - retry_job_picture_comment = (f'{RETRY_JOB_TITLE}\n\n{RETRY_JOB_PICTURE_LINK}').format( - pic_url=get_repository_file_url(RETRY_JOB_PICTURE_PATH) - ) - del_retry_job_pic_pattern = re.escape(RETRY_JOB_TITLE) + r'.*?' + re.escape(f'{RETRY_JOB_PICTURE_PATH})') - + def _update_mr_comment(self, comment: str) -> None: new_comment = f'{COMMENT_START_MARKER}\n\n{comment}' - if print_retry_jobs_message: - new_comment += retry_job_picture_comment for note in self.mr.notes.list(iterator=True): if note.body.startswith(COMMENT_START_MARKER): updated_str = self._get_updated_comment(note.body, comment) - # Add retry job message only if any job has failed - if print_retry_jobs_message: - updated_str = re.sub(del_retry_job_pic_pattern, '', updated_str, flags=re.DOTALL) - updated_str += retry_job_picture_comment - note.body = updated_str try: note.save() @@ -285,7 +305,7 @@ class ReportGenerator: updated_str = f'{existing_comment.strip()}\n\n{new_comment}' return updated_str - def post_report(self, print_retry_jobs_message: bool = False) -> None: + def post_report(self) -> None: comment = self._generate_comment() print(comment) @@ -294,7 +314,7 @@ class ReportGenerator: print('No MR found, skip posting comment') return - self._update_mr_comment(comment, print_retry_jobs_message=print_retry_jobs_message) + self._update_mr_comment(comment) class BuildReportGenerator(ReportGenerator): @@ -305,14 +325,13 @@ class BuildReportGenerator(ReportGenerator): pipeline_id: int, job_id: int, commit_id: str, + local_commit_id: str, *, title: str = 'Build Report', - apps: t.List[App], - ): - super().__init__(project_id, mr_iid, pipeline_id, job_id, commit_id, title=title) + apps: list[AppWithMetricsInfo], + ) -> None: + super().__init__(project_id, mr_iid, pipeline_id, job_id, commit_id, local_commit_id, title=title) self.apps = apps - self._uploader = AppUploader(self.pipeline_id) - self.apps_presigned_url_filepath = TEST_RELATED_APPS_DOWNLOAD_URLS_FILENAME self.report_titles_map = { 'failed_apps': 'Failed Apps', 'built_test_related_apps': 'Built Apps - Test Related', @@ -324,10 +343,9 @@ class BuildReportGenerator(ReportGenerator): self.failed_apps_report_file = 'failed_apps.html' self.built_apps_report_file = 'built_apps.html' self.skipped_apps_report_file = 'skipped_apps.html' - self.app_presigned_urls_dict: t.Dict[str, t.Dict[str, str]] = {} @staticmethod - def custom_sort(item: AppWithMetricsInfo) -> t.Tuple[int, t.Any]: + def custom_sort(item: AppWithMetricsInfo) -> tuple[int, t.Any]: """ Custom sort function to: 1. Push items with zero binary sizes to the end. @@ -384,7 +402,7 @@ class BuildReportGenerator(ReportGenerator): @staticmethod def split_new_and_existing_apps( apps: t.Iterable[AppWithMetricsInfo], - ) -> t.Tuple[t.List[AppWithMetricsInfo], t.List[AppWithMetricsInfo]]: + ) -> tuple[list[AppWithMetricsInfo], list[AppWithMetricsInfo]]: """ Splits apps into new apps and existing apps. @@ -395,7 +413,7 @@ class BuildReportGenerator(ReportGenerator): existing_apps = [app for app in apps if not app.is_new_app] return new_apps, existing_apps - def filter_apps_by_criteria(self, build_status: str, preserve: bool) -> t.List[AppWithMetricsInfo]: + def filter_apps_by_criteria(self, build_status: str, preserve: bool) -> list[AppWithMetricsInfo]: """ Filters apps based on build status and preserve criteria. @@ -405,7 +423,7 @@ class BuildReportGenerator(ReportGenerator): """ return [app for app in self.apps if app.build_status == build_status and app.preserve == preserve] - def get_built_apps_report_parts(self) -> t.List[str]: + def get_built_apps_report_parts(self) -> list[str]: """ Generates report parts for new and existing apps. @@ -422,19 +440,13 @@ class BuildReportGenerator(ReportGenerator): sections = [] if new_test_related_apps: - for app in new_test_related_apps: - for artifact_type in [ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES, ArtifactType.MAP_AND_ELF_FILES]: - url = self._uploader.get_app_presigned_url(app, artifact_type) - self.app_presigned_urls_dict.setdefault(app.build_path, {})[artifact_type.value] = url - new_test_related_apps_table_section = self.create_table_section( title=self.report_titles_map['new_test_related_apps'], items=new_test_related_apps, headers=[ 'App Dir', 'Build Dir', - 'Bin Files with Build Log (without map and elf)', - 'Map and Elf Files', + 'Download Command', 'Your Branch App Size', ], row_attrs=[ @@ -442,31 +454,17 @@ class BuildReportGenerator(ReportGenerator): 'build_dir', ], value_functions=[ - ('Your Branch App Size', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].source_value)), + ('Your Branch App Size', lambda _app: str(_app.metrics[BINARY_SIZE_METRIC_NAME].source_value)), ( - 'Bin Files with Build Log (without map and elf)', - lambda app: self.get_download_link_for_url( - self.app_presigned_urls_dict[app.build_path][ - ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES.value - ] - ), - ), - ( - 'Map and Elf Files', - lambda app: self.get_download_link_for_url( - self.app_presigned_urls_dict[app.build_path][ArtifactType.MAP_AND_ELF_FILES.value] - ), + 'Download Command', + lambda _app: f'idf-ci gitlab download-artifacts --pipeline-id {self.pipeline_id} ' + f'{idf_relpath(_app.build_path)}', ), ], ) sections.extend(new_test_related_apps_table_section) if built_test_related_apps: - for app in built_test_related_apps: - for artifact_type in [ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES, ArtifactType.MAP_AND_ELF_FILES]: - url = self._uploader.get_app_presigned_url(app, artifact_type) - self.app_presigned_urls_dict.setdefault(app.build_path, {})[artifact_type.value] = url - built_test_related_apps = self._sort_items( built_test_related_apps, key='metrics.binary_size.difference_percentage', @@ -480,8 +478,7 @@ class BuildReportGenerator(ReportGenerator): headers=[ 'App Dir', 'Build Dir', - 'Bin Files with Build Log (without map and elf)', - 'Map and Elf Files', + 'Download Command', 'Your Branch App Size', 'Target Branch App Size', 'Size Diff', @@ -497,18 +494,9 @@ class BuildReportGenerator(ReportGenerator): ('Size Diff', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].difference)), ('Size Diff, %', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].difference_percentage)), ( - 'Bin Files with Build Log (without map and elf)', - lambda app: self.get_download_link_for_url( - self.app_presigned_urls_dict[app.build_path][ - ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES.value - ] - ), - ), - ( - 'Map and Elf Files', - lambda app: self.get_download_link_for_url( - self.app_presigned_urls_dict[app.build_path][ArtifactType.MAP_AND_ELF_FILES.value] - ), + 'Download Command', + lambda _app: f'idf-ci gitlab download-artifacts --pipeline-id {self.pipeline_id} ' + f'{idf_relpath(_app.build_path)}', ), ], ) @@ -521,7 +509,7 @@ class BuildReportGenerator(ReportGenerator): headers=[ 'App Dir', 'Build Dir', - 'Build Log', + 'Download Command', 'Your Branch App Size', ], row_attrs=[ @@ -529,13 +517,12 @@ class BuildReportGenerator(ReportGenerator): 'build_dir', ], value_functions=[ - ('Your Branch App Size', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].source_value)), ( - 'Build Log', - lambda app: self.get_download_link_for_url( - self._uploader.get_app_presigned_url(app, ArtifactType.LOGS) - ), + 'Download Command', + lambda _app: f'idf-ci gitlab download-artifacts --pipeline-id {self.pipeline_id} ' + f'{idf_relpath(_app.build_path)}', ), + ('Your Branch App Size', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].source_value)), ], ) sections.extend(new_non_test_related_apps_table_section) @@ -553,7 +540,7 @@ class BuildReportGenerator(ReportGenerator): headers=[ 'App Dir', 'Build Dir', - 'Build Log', + 'Download Command', 'Your Branch App Size', 'Target Branch App Size', 'Size Diff', @@ -569,10 +556,9 @@ class BuildReportGenerator(ReportGenerator): ('Size Diff', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].difference)), ('Size Diff, %', lambda app: str(app.metrics[BINARY_SIZE_METRIC_NAME].difference_percentage)), ( - 'Build Log', - lambda app: self.get_download_link_for_url( - self._uploader.get_app_presigned_url(app, ArtifactType.LOGS) - ), + 'Download Command', + lambda _app: f'idf-ci gitlab download-artifacts --pipeline-id {self.pipeline_id} ' + f'{idf_relpath(_app.build_path)}', ), ], ) @@ -607,14 +593,9 @@ class BuildReportGenerator(ReportGenerator): self.additional_info += self._generate_top_n_apps_by_size_table() - # also generate a yaml file that includes the apps and the presigned urls - # for helping debugging locally - with open(self.apps_presigned_url_filepath, 'w') as fw: - yaml.dump(self.app_presigned_urls_dict, fw) - return sections - def get_failed_apps_report_parts(self) -> t.List[str]: + def get_failed_apps_report_parts(self) -> list[str]: failed_apps = [app for app in self.apps if app.build_status == BuildStatus.FAILED] if not failed_apps: return [] @@ -622,14 +603,13 @@ class BuildReportGenerator(ReportGenerator): failed_apps_table_section = self.create_table_section( title=self.report_titles_map['failed_apps'], items=failed_apps, - headers=['App Dir', 'Build Dir', 'Failed Reason', 'Build Log'], + headers=['App Dir', 'Build Dir', 'Failed Reason', 'Download Command'], row_attrs=['app_dir', 'build_dir', 'build_comment'], value_functions=[ ( - 'Build Log', - lambda app: self.get_download_link_for_url( - self._uploader.get_app_presigned_url(app, ArtifactType.LOGS) - ), + 'Download Command', + lambda _app: f'idf-ci gitlab download-artifacts --pipeline-id {self.pipeline_id} ' + f'{idf_relpath(_app.build_path)}', ), ], ) @@ -643,7 +623,7 @@ class BuildReportGenerator(ReportGenerator): ) return failed_apps_table_section - def get_skipped_apps_report_parts(self) -> t.List[str]: + def get_skipped_apps_report_parts(self) -> list[str]: skipped_apps = [app for app in self.apps if app.build_status == BuildStatus.SKIPPED] if not skipped_apps: return [] @@ -651,16 +631,8 @@ class BuildReportGenerator(ReportGenerator): skipped_apps_table_section = self.create_table_section( title=self.report_titles_map['skipped_apps'], items=skipped_apps, - headers=['App Dir', 'Build Dir', 'Skipped Reason', 'Build Log'], + headers=['App Dir', 'Build Dir', 'Skipped Reason'], row_attrs=['app_dir', 'build_dir', 'build_comment'], - value_functions=[ - ( - 'Build Log', - lambda app: self.get_download_link_for_url( - self._uploader.get_app_presigned_url(app, ArtifactType.LOGS) - ), - ), - ], ) skipped_apps_report_url = self.write_report_to_file( self.generate_html_report(''.join(skipped_apps_table_section)), @@ -673,7 +645,11 @@ class BuildReportGenerator(ReportGenerator): return skipped_apps_table_section def _get_report_str(self) -> str: - self.additional_info = f'**Build Summary (with commit {self.commit_id[:8]}):**\n' + self.additional_info = ( + f'**Build Summary ({self.get_commit_summary}):**\n' + '\n' + '> ℹ️ Note: Binary artifacts stored in MinIO are retained for 4 DAYS from their build date\n' + ) failed_apps_report_parts = self.get_failed_apps_report_parts() skipped_apps_report_parts = self.get_skipped_apps_report_parts() built_apps_report_parts = self.get_built_apps_report_parts() @@ -683,244 +659,6 @@ class BuildReportGenerator(ReportGenerator): ) -class TargetTestReportGenerator(ReportGenerator): - def __init__( - self, - project_id: int, - mr_iid: int, - pipeline_id: int, - job_id: int, - commit_id: str, - *, - title: str = 'Target Test Report', - test_cases: t.List[TestCase], - ): - super().__init__(project_id, mr_iid, pipeline_id, job_id, commit_id, title=title) - - self.test_cases = test_cases - self._known_failure_cases_set = None - self.report_titles_map = { - 'failed_yours': 'Failed Test Cases on Your branch (Excludes Known Failure Cases)', - 'failed_others': 'Failed Test Cases on Other branches (Excludes Known Failure Cases)', - 'failed_known': 'Known Failure Cases', - 'skipped': 'Skipped Test Cases', - 'succeeded': 'Succeeded Test Cases', - } - self.skipped_test_cases_report_file = 'skipped_cases.html' - self.succeeded_cases_report_file = 'succeeded_cases.html' - self.failed_cases_report_file = 'failed_cases.html' - - @property - def known_failure_cases_set(self) -> t.Optional[t.Set[str]]: - if self._known_failure_cases_set is None: - self._known_failure_cases_set = load_known_failure_cases() - - return self._known_failure_cases_set - - def get_known_failure_cases(self) -> t.List[TestCase]: - """ - Retrieve the known failure test cases. - :return: A list of known failure test cases. - """ - if self.known_failure_cases_set is None: - return [] - matched_cases = [ - testcase - for testcase in self.test_cases - if any(fnmatch.fnmatch(testcase.name, pattern) for pattern in self.known_failure_cases_set) - and testcase.is_failure - ] - return matched_cases - - @staticmethod - def filter_test_cases( - cur_branch_failures: t.List[TestCase], - other_branch_failures: t.List[TestCase], - ) -> t.Tuple[t.List[TestCase], t.List[TestCase]]: - """ - Filter the test cases into current branch failures and other branch failures. - - :param cur_branch_failures: List of failed test cases on the current branch. - :param other_branch_failures: List of failed test cases on other branches. - :return: A tuple containing two lists: - - failed_test_cases_cur_branch_only: Test cases that have failed only on the current branch. - - failed_test_cases_other_branch_exclude_cur_branch: Test cases that have failed on other branches - excluding the current branch. - """ - cur_branch_unique_failures = [] - other_branch_failure_map = {tc.name: tc for tc in other_branch_failures} - - for cur_tc in cur_branch_failures: - if cur_tc.latest_failed_count > 0 and ( - cur_tc.name not in other_branch_failure_map - or other_branch_failure_map[cur_tc.name].latest_failed_count == 0 - ): - cur_branch_unique_failures.append(cur_tc) - uniq_fail_names = {cur_tc.name for cur_tc in cur_branch_unique_failures} - other_branch_exclusive_failures = [tc for tc in other_branch_failures if tc.name not in uniq_fail_names] - - return cur_branch_unique_failures, other_branch_exclusive_failures - - def get_failed_cases_report_parts(self) -> t.List[str]: - """ - Generate the report parts for failed test cases and update the additional info section. - :return: A list of strings representing the table sections for the failed test cases. - """ - known_failures = self.get_known_failure_cases() - failed_test_cases = self._filter_items( - self.test_cases, lambda tc: tc.is_failure and tc.name not in {case.name for case in known_failures} - ) - failed_test_cases_cur_branch = self._sort_items( - fetch_failed_testcases_failure_ratio( - copy.deepcopy(failed_test_cases), - branches_filter={'include_branches': [os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_NAME', '')]}, - ), - key='latest_failed_count', - ) - failed_test_cases_other_branch = self._sort_items( - fetch_failed_testcases_failure_ratio( - copy.deepcopy(failed_test_cases), - branches_filter={'exclude_branches': [os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_NAME', '')]}, - ), - key='latest_failed_count', - ) - failed_test_cases_cur_branch, failed_test_cases_other_branch = self.filter_test_cases( - failed_test_cases_cur_branch, failed_test_cases_other_branch - ) - cur_branch_cases_table_section = self.create_table_section( - title=self.report_titles_map['failed_yours'], - items=failed_test_cases_cur_branch, - headers=[ - 'Test Case', - 'Test App Path', - 'Failure Reason', - 'Failures on your branch (40 latest testcases)', - 'Dut Log URL', - 'Create Known Failure Case Jira', - 'Job URL', - 'Grafana URL', - ], - row_attrs=['name', 'app_path', 'failure', 'dut_log_url', 'ci_job_url', 'ci_dashboard_url'], - value_functions=[ - ( - 'Failures on your branch (40 latest testcases)', - lambda item: f'{getattr(item, "latest_failed_count", "")} ' - f'/ {getattr(item, "latest_total_count", "")}', - ), - ('Create Known Failure Case Jira', known_failure_issue_jira_fast_link), - ], - ) - other_branch_cases_table_section = self.create_table_section( - title=self.report_titles_map['failed_others'], - items=failed_test_cases_other_branch, - headers=[ - 'Test Case', - 'Test App Path', - 'Failure Reason', - 'Cases that failed in other branches as well (40 latest testcases)', - 'Dut Log URL', - 'Create Known Failure Case Jira', - 'Job URL', - 'Grafana URL', - ], - row_attrs=['name', 'app_path', 'failure', 'dut_log_url', 'ci_job_url', 'ci_dashboard_url'], - value_functions=[ - ( - 'Cases that failed in other branches as well (40 latest testcases)', - lambda item: f'{getattr(item, "latest_failed_count", "")} ' - f'/ {getattr(item, "latest_total_count", "")}', - ), - ('Create Known Failure Case Jira', known_failure_issue_jira_fast_link), - ], - ) - known_failures_cases_table_section = self.create_table_section( - title=self.report_titles_map['failed_known'], - items=known_failures, - headers=['Test Case', 'Test App Path', 'Failure Reason', 'Job URL', 'Grafana URL'], - row_attrs=['name', 'app_path', 'failure', 'ci_job_url', 'ci_dashboard_url'], - ) - failed_cases_report_url = self.write_report_to_file( - self.generate_html_report( - ''.join( - cur_branch_cases_table_section - + other_branch_cases_table_section - + known_failures_cases_table_section - ) - ), - self.job_id, - self.failed_cases_report_file, - ) - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['failed_yours'], len(failed_test_cases_cur_branch), failed_cases_report_url - ) - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['failed_others'], len(failed_test_cases_other_branch), failed_cases_report_url - ) - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['failed_known'], len(known_failures), failed_cases_report_url - ) - return cur_branch_cases_table_section + other_branch_cases_table_section + known_failures_cases_table_section - - def get_skipped_cases_report_parts(self) -> t.List[str]: - """ - Generate the report parts for skipped test cases and update the additional info section. - :return: A list of strings representing the table sections for the skipped test cases. - """ - skipped_test_cases = self._filter_items(self.test_cases, lambda tc: tc.is_skipped) - skipped_cases_table_section = self.create_table_section( - title=self.report_titles_map['skipped'], - items=skipped_test_cases, - headers=['Test Case', 'Test App Path', 'Skipped Reason', 'Grafana URL'], - row_attrs=['name', 'app_path', 'skipped', 'ci_dashboard_url'], - ) - skipped_cases_report_url = self.write_report_to_file( - self.generate_html_report(''.join(skipped_cases_table_section)), - self.job_id, - self.skipped_test_cases_report_file, - ) - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['skipped'], len(skipped_test_cases), skipped_cases_report_url - ) - return skipped_cases_table_section - - def get_succeeded_cases_report_parts(self) -> t.List[str]: - """ - Generate the report parts for succeeded test cases and update the additional info section. - :return: A list of strings representing the table sections for the succeeded test cases. - """ - succeeded_test_cases = self._filter_items(self.test_cases, lambda tc: tc.is_success) - succeeded_cases_table_section = self.create_table_section( - title=self.report_titles_map['succeeded'], - items=succeeded_test_cases, - headers=['Test Case', 'Test App Path', 'Job URL', 'Grafana URL'], - row_attrs=['name', 'app_path', 'ci_job_url', 'ci_dashboard_url'], - ) - succeeded_cases_report_url = self.write_report_to_file( - self.generate_html_report(''.join(succeeded_cases_table_section)), - self.job_id, - self.succeeded_cases_report_file, - ) - self.additional_info += self.generate_additional_info_section( - self.report_titles_map['succeeded'], len(succeeded_test_cases), succeeded_cases_report_url - ) - self.additional_info += '\n' - return succeeded_cases_table_section - - def _get_report_str(self) -> str: - """ - Generate a complete HTML report string by processing test cases. - :return: Complete HTML report string. - """ - self.additional_info = f'**Test Case Summary (with commit {self.commit_id[:8]}):**\n' - failed_cases_report_parts = self.get_failed_cases_report_parts() - skipped_cases_report_parts = self.get_skipped_cases_report_parts() - succeeded_cases_report_parts = self.get_succeeded_cases_report_parts() - - return self.generate_html_report( - ''.join(failed_cases_report_parts + skipped_cases_report_parts + succeeded_cases_report_parts) - ) - - class JobReportGenerator(ReportGenerator): def __init__( self, @@ -929,11 +667,12 @@ class JobReportGenerator(ReportGenerator): pipeline_id: int, job_id: int, commit_id: str, + local_commit_id: str, *, title: str = 'Job Report', - jobs: t.List[GitlabJob], + jobs: list[GitlabJob], ): - super().__init__(project_id, mr_iid, pipeline_id, job_id, commit_id, title=title) + super().__init__(project_id, mr_iid, pipeline_id, job_id, commit_id, local_commit_id, title=title) self.jobs = jobs self.report_titles_map = { 'failed_jobs': 'Failed Jobs (Excludes "integration_test" and "target_test" jobs)', @@ -960,7 +699,7 @@ class JobReportGenerator(ReportGenerator): ) succeeded_jobs = self._filter_items(self.jobs, lambda job: job.is_success) - self.additional_info = f'**Job Summary (with commit {self.commit_id[:8]}):**\n' + self.additional_info = f'**Job Summary ({self.get_commit_summary}):**\n' self.additional_info += self.generate_additional_info_section( self.report_titles_map['succeeded'], len(succeeded_jobs) ) diff --git a/tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py b/tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py deleted file mode 100644 index 4d007172707..00000000000 --- a/tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py +++ /dev/null @@ -1,75 +0,0 @@ -# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import argparse -import sys - -import __init__ # noqa: F401 # inject the system path -from idf_build_apps import build_apps -from idf_build_apps import setup_logging -from idf_build_apps.utils import semicolon_separated_str_to_list -from idf_ci_local.app import import_apps_from_txt -from idf_pytest.constants import DEFAULT_IGNORE_WARNING_FILEPATH - -from dynamic_pipelines.constants import TEST_RELATED_APPS_FILENAME - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Build Apps for Dynamic Pipeline') - parser.add_argument('app_list_file', default=TEST_RELATED_APPS_FILENAME, help='List of apps to build') - parser.add_argument( - '--build-verbose', - action='store_true', - help='Enable verbose output from build system.', - ) - parser.add_argument('--parallel-count', default=1, type=int, help='Number of parallel build jobs.') - parser.add_argument( - '--parallel-index', - default=1, - type=int, - help='Index (1-based) of the job, out of the number specified by --parallel-count.', - ) - parser.add_argument( - '--ignore-warning-file', - default=DEFAULT_IGNORE_WARNING_FILEPATH, - type=argparse.FileType('r'), - help='Ignore the warning strings in the specified file. Each line should be a regex string.', - ) - parser.add_argument( - '--modified-components', - type=semicolon_separated_str_to_list, - help='semicolon-separated string which specifies the modified components. ' - 'app with `depends_components` set in the corresponding manifest files would only be built ' - 'if depends on any of the specified components. ' - 'If set to "", the value would be considered as None. ' - 'If set to ";", the value would be considered as an empty list', - ) - parser.add_argument( - '--collect-app-info', - default='list_job_@p.txt', - help='If specified, the test case name and app info json will be written to this file', - ) - parser.add_argument( - '--junitxml', - default='build_summary_@p.xml', - help='Path to the junitxml file. If specified, the junitxml file will be generated', - ) - - args = parser.parse_args() - - setup_logging(verbose=1) - - sys.exit( - build_apps( - import_apps_from_txt(args.app_list_file), - build_verbose=args.build_verbose, - keep_going=True, - ignore_warning_file=args.ignore_warning_file, - modified_components=args.modified_components, - check_app_dependencies=True, - parallel_count=args.parallel_count, - parallel_index=args.parallel_index, - collect_size_info='size_info.txt', - collect_app_info=args.collect_app_info, - junitxml=args.junitxml, - copy_sdkconfig=True, - ) - ) diff --git a/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py b/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py deleted file mode 100644 index c4c1a58a101..00000000000 --- a/tools/ci/dynamic_pipelines/scripts/generate_build_child_pipeline.py +++ /dev/null @@ -1,260 +0,0 @@ -# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -"""This file is used for generating the child pipeline for build jobs.""" - -import argparse -import os -import typing as t - -import __init__ # noqa: F401 # inject the system path -import yaml -from idf_build_apps.utils import semicolon_separated_str_to_list -from idf_ci_local.app import dump_apps_to_txt -from idf_ci_utils import IDF_PATH -from idf_pytest.constants import DEFAULT_CONFIG_RULES_STR -from idf_pytest.constants import DEFAULT_FULL_BUILD_TEST_COMPONENTS -from idf_pytest.constants import DEFAULT_FULL_BUILD_TEST_FILEPATTERNS -from idf_pytest.constants import CollectMode -from idf_pytest.script import get_all_apps - -from dynamic_pipelines.constants import DEFAULT_APPS_BUILD_PER_JOB -from dynamic_pipelines.constants import DEFAULT_BUILD_CHILD_PIPELINE_FILEPATH -from dynamic_pipelines.constants import DEFAULT_BUILD_CHILD_PIPELINE_NAME -from dynamic_pipelines.constants import DEFAULT_TEST_PATHS -from dynamic_pipelines.constants import NON_TEST_RELATED_APPS_FILENAME -from dynamic_pipelines.constants import NON_TEST_RELATED_BUILD_JOB_NAME -from dynamic_pipelines.constants import TEST_RELATED_APPS_FILENAME -from dynamic_pipelines.constants import TEST_RELATED_BUILD_JOB_NAME -from dynamic_pipelines.models import BuildJob -from dynamic_pipelines.models import EmptyJob -from dynamic_pipelines.utils import dump_jobs_to_yaml - - -def _separate_str_to_list(s: str) -> t.List[str]: - """ - Gitlab env file will escape the doublequotes in the env file, so we need to remove them - - For example, - - in pipeline.env file we have - - MR_MODIFIED_COMPONENTS="app1;app2" - MR_MODIFIED_FILES="main/app1.c;main/app2.c" - - gitlab will load the doublequotes as well, so we need to remove the doublequotes - """ - return semicolon_separated_str_to_list(s.strip('"')) # type: ignore - - -def main(arguments: argparse.Namespace) -> None: - # load from default build test rules config file - extra_default_build_targets: t.List[str] = [] - if arguments.default_build_test_rules: - with open(arguments.default_build_test_rules) as fr: - configs = yaml.safe_load(fr) - - if configs: - extra_default_build_targets = configs.get('extra_default_build_targets') or [] - - build_jobs = [] - ########################################### - # special case with -k, ignore other args # - ########################################### - if arguments.filter_expr: - # build only test related apps - test_related_apps, _ = get_all_apps( - arguments.paths, - target=CollectMode.ALL, - config_rules_str=DEFAULT_CONFIG_RULES_STR, - filter_expr=arguments.filter_expr, - marker_expr='not host_test', - extra_default_build_targets=extra_default_build_targets, - ) - dump_apps_to_txt(sorted(test_related_apps), TEST_RELATED_APPS_FILENAME) - print(f'Generate test related apps file {TEST_RELATED_APPS_FILENAME} with {len(test_related_apps)} apps') - - test_apps_build_job = BuildJob( - name=TEST_RELATED_BUILD_JOB_NAME, - parallel=len(test_related_apps) // DEFAULT_APPS_BUILD_PER_JOB + 1, - variables={ - 'APP_LIST_FILE': TEST_RELATED_APPS_FILENAME, - }, - ) - - build_jobs.append(test_apps_build_job) - else: - ############# - # all cases # - ############# - test_related_apps, non_test_related_apps = get_all_apps( - arguments.paths, - CollectMode.ALL, - marker_expr='not host_test', - config_rules_str=DEFAULT_CONFIG_RULES_STR, - extra_default_build_targets=extra_default_build_targets, - compare_manifest_sha_filepath=arguments.compare_manifest_sha_filepath, - modified_components=arguments.modified_components, - modified_files=arguments.modified_files, - ignore_app_dependencies_components=arguments.ignore_app_dependencies_components, - ignore_app_dependencies_filepatterns=arguments.ignore_app_dependencies_filepatterns, - ) - - dump_apps_to_txt(sorted(test_related_apps), TEST_RELATED_APPS_FILENAME) - print(f'Generate test related apps file {TEST_RELATED_APPS_FILENAME} with {len(test_related_apps)} apps') - dump_apps_to_txt(sorted(non_test_related_apps), NON_TEST_RELATED_APPS_FILENAME) - print( - f'Generate non-test related apps file {NON_TEST_RELATED_APPS_FILENAME} ' - f'with {len(non_test_related_apps)} apps' - ) - - if test_related_apps: - test_apps_build_job = BuildJob( - name=TEST_RELATED_BUILD_JOB_NAME, - parallel=len(test_related_apps) // DEFAULT_APPS_BUILD_PER_JOB + 1, - variables={ - 'APP_LIST_FILE': TEST_RELATED_APPS_FILENAME, - }, - ) - build_jobs.append(test_apps_build_job) - - if non_test_related_apps: - non_test_apps_build_job = BuildJob( - name=NON_TEST_RELATED_BUILD_JOB_NAME, - parallel=len(non_test_related_apps) // DEFAULT_APPS_BUILD_PER_JOB + 1, - variables={ - 'APP_LIST_FILE': NON_TEST_RELATED_APPS_FILENAME, - }, - ) - build_jobs.append(non_test_apps_build_job) - - if mr_labels := os.getenv('CI_MERGE_REQUEST_LABELS'): - print(f'MR labels: {mr_labels}') - - # check if there's no jobs - if not build_jobs: - print('No apps need to be built. Create one empty job instead') - build_jobs.append(EmptyJob()) - extra_include_yml = [] - else: - extra_include_yml = ['tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml'] - - dump_jobs_to_yaml(build_jobs, arguments.yaml_output, DEFAULT_BUILD_CHILD_PIPELINE_NAME, extra_include_yml) - print(f'Generate child pipeline yaml file {arguments.yaml_output} with {sum(j.parallel for j in build_jobs)} jobs') - - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description='Generate build child pipeline', - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - parser.add_argument( - '-o', - '--yaml-output', - default=DEFAULT_BUILD_CHILD_PIPELINE_FILEPATH, - help='Output YAML path', - ) - parser.add_argument( - '-p', - '--paths', - nargs='+', - default=DEFAULT_TEST_PATHS, - help='Paths to the apps to build.', - ) - parser.add_argument( - '-k', - '--filter-expr', - help='only build tests matching given filter expression. For example: -k "test_hello_world". Works only' - 'for pytest', - ) - parser.add_argument( - '--default-build-test-rules', - default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml'), - help='default build test rules config file', - ) - parser.add_argument( - '--compare-manifest-sha-filepath', - default=os.path.join(IDF_PATH, '.manifest_sha'), - help='Path to the recorded manifest sha file generated by `idf-build-apps dump-manifest-sha`', - ) - parser.add_argument( - '--modified-components', - type=_separate_str_to_list, - default=os.getenv('MR_MODIFIED_COMPONENTS'), - help='semicolon-separated string which specifies the modified components. ' - 'app with `depends_components` set in the corresponding manifest files would only be built ' - 'if depends on any of the specified components. ' - 'If set to "", the value would be considered as None. ' - 'If set to ";", the value would be considered as an empty list', - ) - parser.add_argument( - '--modified-files', - type=_separate_str_to_list, - default=os.getenv('MR_MODIFIED_FILES'), - help='semicolon-separated string which specifies the modified files. ' - 'app with `depends_filepatterns` set in the corresponding manifest files would only be built ' - 'if any of the specified file pattern matches any of the specified modified files. ' - 'If set to "", the value would be considered as None. ' - 'If set to ";", the value would be considered as an empty list', - ) - parser.add_argument( - '-ic', - '--ignore-app-dependencies-components', - type=_separate_str_to_list, - help='semicolon-separated string which specifies the modified components used for ' - 'ignoring checking the app dependencies. ' - 'The `depends_components` and `depends_filepatterns` set in the manifest files will be ignored ' - 'when any of the specified components matches any of the modified components. ' - 'Must be used together with --modified-components. ' - 'If set to "", the value would be considered as None. ' - 'If set to ";", the value would be considered as an empty list', - ) - parser.add_argument( - '-if', - '--ignore-app-dependencies-filepatterns', - type=_separate_str_to_list, - help='semicolon-separated string which specifies the file patterns used for ' - 'ignoring checking the app dependencies. ' - 'The `depends_components` and `depends_filepatterns` set in the manifest files will be ignored ' - 'when any of the specified file patterns matches any of the modified files. ' - 'Must be used together with --modified-files. ' - 'If set to "", the value would be considered as None. ' - 'If set to ";", the value would be considered as an empty list', - ) - - args = parser.parse_args() - - if test_case_filters := os.getenv('TEST_CASE_FILTERS', None): - args.filter_expr = test_case_filters - - if os.getenv('IS_MR_PIPELINE') == '0' or os.getenv('BUILD_AND_TEST_ALL_APPS') == '1': - print('Build and run all test cases, and compile all cmake apps') - args.modified_components = None - args.modified_files = None - args.ignore_app_dependencies_components = None - args.ignore_app_dependencies_filepatterns = None - elif args.filter_expr is not None: - print('Build and run only test cases matching "%s"' % args.filter_expr) - args.modified_components = None - args.modified_files = None - args.ignore_app_dependencies_components = None - args.ignore_app_dependencies_filepatterns = None - else: - print( - f'Build and run only test cases matching:\n' - f'- modified components: {args.modified_components}\n' - f'- modified files: {args.modified_files}' - ) - - if args.modified_components is not None and not args.ignore_app_dependencies_components: - # setting default values - args.ignore_app_dependencies_components = DEFAULT_FULL_BUILD_TEST_COMPONENTS - - if args.modified_files is not None and not args.ignore_app_dependencies_filepatterns: - # setting default values - args.ignore_app_dependencies_filepatterns = DEFAULT_FULL_BUILD_TEST_FILEPATTERNS - - if not os.path.isfile(args.compare_manifest_sha_filepath): - # ignore if the file does not exist - args.compare_manifest_sha_filepath = None - - main(args) diff --git a/tools/ci/dynamic_pipelines/scripts/generate_report.py b/tools/ci/dynamic_pipelines/scripts/generate_report.py index a224ac7f76b..1f40fff50ad 100644 --- a/tools/ci/dynamic_pipelines/scripts/generate_report.py +++ b/tools/ci/dynamic_pipelines/scripts/generate_report.py @@ -3,27 +3,26 @@ import argparse import glob import os +import subprocess import typing as t import __init__ # noqa: F401 # inject the system path +from idf_build_apps import json_list_files_to_apps +from idf_ci import GitlabEnvVars from idf_ci_local.app import enrich_apps_with_metrics_info -from idf_ci_local.app import import_apps_from_txt from dynamic_pipelines.report import BuildReportGenerator from dynamic_pipelines.report import JobReportGenerator -from dynamic_pipelines.report import TargetTestReportGenerator from dynamic_pipelines.utils import fetch_app_metrics from dynamic_pipelines.utils import fetch_failed_jobs -from dynamic_pipelines.utils import parse_testcases_from_filepattern def main() -> None: parser: argparse.ArgumentParser = setup_argument_parser() args: argparse.Namespace = parser.parse_args() - report_actions: t.Dict[str, t.Callable[[argparse.Namespace], None]] = { + report_actions: dict[str, t.Callable[[argparse.Namespace], None]] = { 'build': generate_build_report, - 'target_test': generate_target_test_report, 'job': generate_jobs_report, } @@ -37,10 +36,10 @@ def main() -> None: def setup_argument_parser() -> argparse.ArgumentParser: report_type_parser: argparse.ArgumentParser = argparse.ArgumentParser(add_help=False) report_type_parser.add_argument( - '--report-type', choices=['build', 'target_test', 'job'], required=True, help='Type of report to generate' + '--report-type', choices=['build', 'job'], required=True, help='Type of report to generate' ) report_type_args: argparse.Namespace - remaining_args: t.List[str] + remaining_args: list[str] report_type_args, remaining_args = report_type_parser.parse_known_args() parser: argparse.ArgumentParser = argparse.ArgumentParser( @@ -60,12 +59,13 @@ def common_arguments(parser: argparse.ArgumentParser) -> None: parser.add_argument('--mr-iid', type=int, default=os.getenv('CI_MERGE_REQUEST_IID'), help='Merge Request IID') parser.add_argument('--pipeline-id', type=int, default=os.getenv('PARENT_PIPELINE_ID'), help='Pipeline ID') parser.add_argument('--job-id', type=int, default=os.getenv('CI_JOB_ID'), help='Job ID') - parser.add_argument('--commit-id', default=os.getenv('CI_COMMIT_SHA'), help='MR commit ID') + parser.add_argument('--commit-id', default=os.getenv('CI_COMMIT_SHA', ''), help='MR merged result commit ID') + parser.add_argument('--local-commit-id', default=os.getenv('PIPELINE_COMMIT_SHA', ''), help='local dev commit ID') def conditional_arguments(report_type_args: argparse.Namespace, parser: argparse.ArgumentParser) -> None: if report_type_args.report_type == 'build': - parser.add_argument('--app-list-filepattern', default='list_job_*.txt', help='Pattern to match app list files') + parser.add_argument('--app-list-filepattern', default='app_info*.txt', help='Pattern to match app list files') elif report_type_args.report_type == 'target_test': parser.add_argument( '--junit-report-filepattern', default='XUNIT_RESULT*.xml', help='Pattern to match JUnit report files' @@ -73,38 +73,48 @@ def conditional_arguments(report_type_args: argparse.Namespace, parser: argparse def generate_build_report(args: argparse.Namespace) -> None: - apps: t.List[t.Any] = [ - app for file_name in glob.glob(args.app_list_filepattern) for app in import_apps_from_txt(file_name) - ] + # generate presigned url for the artifacts + subprocess.check_output( + [ + 'idf-ci', + 'gitlab', + 'generate-presigned-json', + '--commit-sha', + args.local_commit_id, + '--output', + 'presigned.json', + ], + ) + print('generated presigned.json') + + # generate report + apps = json_list_files_to_apps(glob.glob(args.app_list_filepattern)) + print(f'loaded {len(apps)} apps') app_metrics = fetch_app_metrics( - source_commit_sha=os.environ.get('CI_COMMIT_SHA'), + source_commit_sha=args.commit_id, target_commit_sha=os.environ.get('CI_MERGE_REQUEST_TARGET_BRANCH_SHA'), ) apps = enrich_apps_with_metrics_info(app_metrics, apps) report_generator = BuildReportGenerator( - args.project_id, args.mr_iid, args.pipeline_id, args.job_id, args.commit_id, apps=apps - ) - report_generator.post_report() - - -def generate_target_test_report(args: argparse.Namespace) -> None: - test_cases: t.List[t.Any] = parse_testcases_from_filepattern(args.junit_report_filepattern) - report_generator = TargetTestReportGenerator( - args.project_id, args.mr_iid, args.pipeline_id, args.job_id, args.commit_id, test_cases=test_cases + args.project_id, args.mr_iid, args.pipeline_id, args.job_id, args.commit_id, args.local_commit_id, apps=apps ) report_generator.post_report() def generate_jobs_report(args: argparse.Namespace) -> None: - jobs: t.List[t.Any] = fetch_failed_jobs(args.commit_id) + jobs: list[t.Any] = fetch_failed_jobs(args.commit_id) if not jobs: return report_generator = JobReportGenerator( - args.project_id, args.mr_iid, args.pipeline_id, args.job_id, args.commit_id, jobs=jobs + args.project_id, args.mr_iid, args.pipeline_id, args.job_id, args.commit_id, args.local_commit_id, jobs=jobs ) - report_generator.post_report(print_retry_jobs_message=any(job.is_failed for job in jobs)) + report_generator.post_report() + + if GitlabEnvVars().IDF_CI_IS_DEBUG_PIPELINE: + print('Debug pipeline detected, exit non-zero to fail the pipeline in order to block merge') + exit(30) if __name__ == '__main__': diff --git a/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py b/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py index 77901fdda5a..6abe2a83689 100644 --- a/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py +++ b/tools/ci/dynamic_pipelines/scripts/generate_target_test_child_pipeline.py @@ -8,124 +8,39 @@ """ import argparse -import glob import os import typing as t -from collections import Counter -from collections import defaultdict import __init__ # noqa: F401 # inject the system path import yaml -from idf_build_apps import App -from idf_ci_local.app import import_apps_from_txt +from idf_ci import get_pytest_cases +from idf_ci.idf_gitlab import test_child_pipeline +from idf_ci.idf_pytest import GroupedPytestCases +from idf_ci.idf_pytest.models import GroupKey +from idf_ci_utils import IDF_PATH from idf_pytest.constants import TIMEOUT_4H_MARKERS -from idf_pytest.script import get_pytest_cases -from dynamic_pipelines.constants import BUILD_ONLY_LABEL -from dynamic_pipelines.constants import DEFAULT_CASES_TEST_PER_JOB -from dynamic_pipelines.constants import DEFAULT_TARGET_TEST_CHILD_PIPELINE_FILEPATH -from dynamic_pipelines.constants import DEFAULT_TARGET_TEST_CHILD_PIPELINE_NAME -from dynamic_pipelines.constants import DEFAULT_TARGET_TEST_JOB_TEMPLATE_NAME -from dynamic_pipelines.constants import DEFAULT_TEST_PATHS from dynamic_pipelines.constants import KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH -from dynamic_pipelines.constants import TIMEOUT_4H_TEMPLATE_NAME -from dynamic_pipelines.models import EmptyJob -from dynamic_pipelines.models import Job -from dynamic_pipelines.models import TargetTestJob -from dynamic_pipelines.utils import dump_jobs_to_yaml + +BUILD_ONLY_LABEL = 'For Maintainers: Only Build Tests' +TIMEOUT_4H_TEMPLATE_NAME = '.timeout_4h_template' +TARGET_TEST_CHILD_PIPELINE_FILEPATH = os.path.join(IDF_PATH, 'target_test_child_pipeline.yml') -def get_tags_with_amount(s: str) -> t.List[str]: - c: Counter = Counter() - for _t in s.split(','): - c[_t] += 1 - - res = set() - for target, amount in c.items(): - if amount > 1: - res.add(f'{target}_{amount}') - else: - res.add(target) - - return sorted(res) - - -def get_target_test_jobs( - paths: str, apps: t.List[App], exclude_runner_tags: t.Set[str], exclude_runner_tags_matching: t.List[t.Set] -) -> t.Tuple[t.List[Job], t.List[str], t.List[str]]: - """ - Return the target test jobs and the extra yaml files to include - """ +def main(output_filepath: str) -> None: if mr_labels := os.getenv('CI_MERGE_REQUEST_LABELS'): print(f'MR labels: {mr_labels}') if BUILD_ONLY_LABEL in mr_labels.split(','): print('MR has build only label, skip generating target test child pipeline') - return [EmptyJob()], [], [] + test_child_pipeline( + output_filepath, + cases=GroupedPytestCases([]), + ) + return - pytest_cases = get_pytest_cases( - paths, - apps=apps, - marker_expr='not host_test', # since it's generating target-test child pipeline - ) + cases = GroupedPytestCases(get_pytest_cases()) - no_env_marker_test_cases: t.List[str] = [] - res = defaultdict(list) - for case in pytest_cases: - if not case.env_markers: - no_env_marker_test_cases.append(case.item.nodeid) - continue - - res[(case.target_selector, tuple(sorted(case.env_markers)))].append(case) - - target_test_jobs: t.List[Job] = [] - for (target_selector, env_markers), cases in res.items(): - runner_tags = get_tags_with_amount(target_selector) + list(env_markers) - if ','.join(runner_tags) in exclude_runner_tags: - print('WARNING: excluding test cases with runner tags:', runner_tags) - continue - - if any(set(runner_tags) & group for group in exclude_runner_tags_matching): - print(f'WARNING: Excluding test cases with runner tags (wildcard match): {runner_tags}') - continue - - _extends = [DEFAULT_TARGET_TEST_JOB_TEMPLATE_NAME] - for timeout_4h_marker in TIMEOUT_4H_MARKERS: - if timeout_4h_marker in env_markers: - _extends.append(TIMEOUT_4H_TEMPLATE_NAME) - - target_test_job = TargetTestJob( - extends=_extends, - name=f'{target_selector} - {",".join(env_markers)}', - tags=runner_tags, - parallel=len(cases) // DEFAULT_CASES_TEST_PER_JOB + 1, - ) - target_test_job.set_variable('TARGET_SELECTOR', f"'{target_selector}'") - target_test_job.set_variable('ENV_MARKERS', "'" + ' and '.join(env_markers) + "'") - target_test_job.set_variable('PYTEST_NODES', ' '.join([f"'{case.item.nodeid}'" for case in cases])) - - target_test_jobs.append(target_test_job) - - extra_include_yml: t.List[str] = [] - if not target_test_jobs: - print('No target test cases required, create one empty job instead') - target_test_jobs.append(EmptyJob()) - else: - extra_include_yml = ['tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml'] - - fast_pipeline_flag = int(os.getenv('REPORT_EXIT_CODE', 0)) == 30 - if fast_pipeline_flag: - extra_include_yml = ['tools/ci/dynamic_pipelines/templates/fast_pipeline.yml'] - - no_env_marker_test_cases.sort() - return target_test_jobs, extra_include_yml, no_env_marker_test_cases - - -def generate_target_test_child_pipeline( - paths: str, - apps: t.List[App], - output_filepath: str, -) -> None: with open(KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH) as fr: known_warnings_dict = yaml.safe_load(fr) or dict() @@ -156,38 +71,39 @@ def generate_target_test_child_pipeline( if res := _process_match_group(_tag): exclude_runner_tags_matching.append(res) - target_test_jobs, extra_include_yml, no_env_marker_test_cases = get_target_test_jobs( - paths=paths, - apps=apps, - exclude_runner_tags=exclude_runner_tags_set, - exclude_runner_tags_matching=exclude_runner_tags_matching, - ) + flattened_cases = [] + additional_dict: dict[GroupKey, dict[str, t.Any]] = {} + for key, grouped_cases in cases.grouped_cases.items(): + # skip test cases with no runner tags + if ','.join(sorted(key.runner_tags)) in exclude_runner_tags_set: + print(f'WARNING: excluding test cases with runner tags: {key.runner_tags}') + continue - known_no_env_marker_test_cases = set(known_warnings_dict.get('no_env_marker_test_cases', [])) - no_env_marker_test_cases_set = set(no_env_marker_test_cases) + print(f'INFO: key.runner tags: {key.runner_tags}') + if any(set(key.runner_tags) & group for group in exclude_runner_tags_matching): + print(f'WARNING: Excluding test cases with runner tags (wildcard match): {key.runner_tags}') + continue - no_env_marker_test_cases_fail = False - if no_env_marker_test_cases_set - known_no_env_marker_test_cases: - print('ERROR: NEW "no_env_marker_test_cases" detected:') - for case in no_env_marker_test_cases_set - known_no_env_marker_test_cases: - print(f' - {case}') - no_env_marker_test_cases_fail = True + flattened_cases.extend(grouped_cases) - print( - 'Please add at least one environment markers to the test cases listed above. ' - 'You may check all the env markers here: tools/ci/idf_pytest/constants.py' - ) + for case in grouped_cases: + for timeout_4h_marker in TIMEOUT_4H_MARKERS: + if timeout_4h_marker in case.env_markers: + if key not in additional_dict: + additional_dict[key] = { + 'extra_extends': [], + } - if no_env_marker_test_cases_fail: - raise SystemExit('Failed to generate target test child pipeline.') + if TIMEOUT_4H_TEMPLATE_NAME not in additional_dict[key]['extra_extends']: + additional_dict[key]['extra_extends'].append(TIMEOUT_4H_TEMPLATE_NAME) - dump_jobs_to_yaml( - target_test_jobs, + test_child_pipeline( output_filepath, - DEFAULT_TARGET_TEST_CHILD_PIPELINE_NAME, - extra_include_yml, + cases=GroupedPytestCases( + cases=flattened_cases, + additional_dict=additional_dict, + ), ) - print(f'Generate child pipeline yaml file {output_filepath} with {sum(j.parallel for j in target_test_jobs)} jobs') if __name__ == '__main__': @@ -195,35 +111,13 @@ if __name__ == '__main__': description='Generate Target Test Child Pipeline. Update Build Report in MR pipelines', formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - parser.add_argument( - '-p', - '--paths', - nargs='+', - default=DEFAULT_TEST_PATHS, - help='Paths to the apps to build.', - ) parser.add_argument( '-o', '--output', - default=DEFAULT_TARGET_TEST_CHILD_PIPELINE_FILEPATH, + default=TARGET_TEST_CHILD_PIPELINE_FILEPATH, help='Output child pipeline file path', ) - parser.add_argument( - '--app-info-filepattern', - default='list_job_*.txt', - help='glob pattern to specify the files that include built app info generated by ' - '`idf-build-apps --collect-app-info ...`. will not raise ValueError when binary ' - 'paths not exist in local file system if not listed recorded in the app info.', - ) args = parser.parse_args() - apps = [] - for f in glob.glob(args.app_info_filepattern): - apps.extend(import_apps_from_txt(f)) - - generate_target_test_child_pipeline( - paths=args.paths, - apps=apps, - output_filepath=args.output, - ) + main(args.output) diff --git a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml index 2629a05aec4..0e50de03a34 100644 --- a/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml +++ b/tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml @@ -6,11 +6,11 @@ ######################## .dynamic_build_template: extends: - - .before_script:build - - .after_script:build:ccache-show-stats:upload-failed-job-logs + - .after_script:build image: $ESP_ENV_IMAGE + tags: [build, shiny] stage: build - timeout: 1 hour + timeout: 2 hour variables: # Enable ccache for all build jobs. See configure_ci_environment.sh for more ccache related settings. IDF_CCACHE_ENABLE: "1" @@ -20,41 +20,37 @@ needs: - pipeline: $PARENT_PIPELINE_ID job: generate_build_child_pipeline + - pipeline: $PARENT_PIPELINE_ID + job: pipeline_variables artifacts: paths: - # The other artifacts patterns are defined under tools/ci/artifacts_handler.py + # The other artifacts patterns are defined under .idf_ci.toml # Now we're uploading/downloading the binary files from our internal storage server - # + # keep the log file to help debug - "**/build*/build_log.txt" # build spec files - build_summary_*.xml # list of built apps - - list_job_*.txt + - app_info_*.txt when: always expire_in: 1 week script: # CI specific options start from "--parallel-count xxx". could ignore when running locally - - run_cmd python tools/ci/dynamic_pipelines/scripts/child_pipeline_build_apps.py $APP_LIST_FILE + - run_cmd idf-ci build run --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1} - --collect-app-info "list_job_${CI_JOB_NAME_SLUG}.txt" - --modified-components ${MR_MODIFIED_COMPONENTS} - --junitxml "build_summary_${CI_JOB_NAME_SLUG}.xml" - - run_cmd python tools/ci/artifacts_handler.py upload --type size_reports + --modified-files ${MR_MODIFIED_FILES} .dynamic_target_test_template: - extends: - - .before_script:fetch:target_test image: $TARGET_TEST_ENV_IMAGE stage: target_test timeout: 1 hour variables: + GIT_STRATEGY: "fetch" SUBMODULES_TO_FETCH: "none" # set while generating the pipeline - PYTEST_NODES: "" - TARGET_SELECTOR: "" - ENV_MARKERS: "" + nodes: "" INSTALL_EXTRA_TOOLS: "xtensa-esp-elf-gdb riscv32-esp-elf-gdb openocd-esp32 esp-rom-elfs" PYTEST_EXTRA_FLAGS: "--dev-passwd ${ETHERNET_TEST_PASSWORD} --dev-user ${ETHERNET_TEST_USER} --capture=fd --verbosity=0 --unity-test-report-mode merge" needs: @@ -70,33 +66,26 @@ paths: - XUNIT_RESULT*.xml - pytest-embedded/ -# Child pipeline reports won't be collected in the main one -# https://gitlab.com/groups/gitlab-org/-/epics/8205 -# reports: -# junit: XUNIT_RESULT.xml + reports: + junit: XUNIT_RESULT*.xml when: always expire_in: 1 week script: - # get known failure cases - - run_cmd python tools/ci/get_known_failure_cases_file.py + - run_cmd idf-ci gitlab download-known-failure-cases-file ${KNOWN_FAILURE_CASES_FILE_NAME} # get runner env config file - retry_failed git clone $TEST_ENV_CONFIG_REPO - run_cmd python $CHECKOUT_REF_SCRIPT ci-test-runner-configs ci-test-runner-configs # CI specific options start from "--known-failure-cases-file xxx". could ignore when running locally - - run_cmd pytest ${PYTEST_NODES} - --target ${TARGET_SELECTOR} - -m ${ENV_MARKERS} + - run_cmd pytest $nodes --pipeline-id $PARENT_PIPELINE_ID - --junitxml=XUNIT_RESULT_${CI_JOB_NAME_SLUG}.xml + --junitxml=XUNIT_RESULT_${CI_JOB_ID}.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME} --parallel-count ${CI_NODE_TOTAL:-1} --parallel-index ${CI_NODE_INDEX:-1} ${PYTEST_EXTRA_FLAGS} after_script: - source tools/ci/utils.sh - - section_start "upload_junit_reports" - - run_cmd python tools/ci/artifacts_handler.py upload --type logs junit_reports - - section_end "upload_junit_reports" + - run_cmd idf-ci gitlab upload-artifacts --type junit .timeout_4h_template: timeout: 4 hours diff --git a/tools/ci/dynamic_pipelines/templates/fast_pipeline.yml b/tools/ci/dynamic_pipelines/templates/fast_pipeline.yml deleted file mode 100644 index ea23b84abb3..00000000000 --- a/tools/ci/dynamic_pipelines/templates/fast_pipeline.yml +++ /dev/null @@ -1,16 +0,0 @@ -.generate_pytest_report_base: - stage: .post - tags: [build, shiny] - image: $ESP_ENV_IMAGE - artifacts: - paths: - - target_test_report.html - expire_in: 1 week - when: always - -fast_pipeline:pipeline_ended:always_failed: - when: on_success - extends: .generate_pytest_report_base - script: - - python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type target_test - - exit 30 diff --git a/tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml b/tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml deleted file mode 100644 index 1631f949f4f..00000000000 --- a/tools/ci/dynamic_pipelines/templates/generate_target_test_report.yml +++ /dev/null @@ -1,18 +0,0 @@ -generate_pytest_report: - stage: .post - tags: [build, shiny] - image: $ESP_ENV_IMAGE - when: always - artifacts: - paths: - - target_test_report.html - - failed_cases.html - - skipped_cases.html - - succeeded_cases.html - expire_in: 1 week - when: always - - script: - - python tools/ci/get_known_failure_cases_file.py - - python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type target_test - - python tools/ci/previous_stage_job_status.py --stage target_test diff --git a/tools/ci/dynamic_pipelines/templates/known_generate_test_child_pipeline_warnings.yml b/tools/ci/dynamic_pipelines/templates/known_generate_test_child_pipeline_warnings.yml index 0baec086418..05efc0b22fd 100644 --- a/tools/ci/dynamic_pipelines/templates/known_generate_test_child_pipeline_warnings.yml +++ b/tools/ci/dynamic_pipelines/templates/known_generate_test_child_pipeline_warnings.yml @@ -2,7 +2,13 @@ # no_env_marker_test_cases: List of test cases that do not have environment markers. # each item shall be the test node id, you may check the error message to get the node id. no_env_marker_test_cases: - - dummy_test_case + - test_examples_security_secure_boot + - test_examples_security_secure_boot_corrupt_app_sig + - test_examples_security_secure_boot_corrupt_bl_sig + - test_examples_security_secure_boot_key_combo + - test_examples_security_secure_boot_key_revoke + - test_usb_enum + - test_usb_ext_port # no_runner_tags: List of runner tags that has no test runner set. # each item shall be a comma separated list of runner tags. @@ -17,7 +23,6 @@ no_runner_tags: - esp32c5,jtag - esp32c6,jtag - esp32c61,jtag - - esp32c61,usb_serial_jtag - esp32h2,jtag - esp32p4,jtag - esp32s2,usb_host_flash_disk diff --git a/tools/ci/dynamic_pipelines/templates/report.template.html b/tools/ci/dynamic_pipelines/templates/report.template.html index cefa904b462..adbdf4710f3 100644 --- a/tools/ci/dynamic_pipelines/templates/report.template.html +++ b/tools/ci/dynamic_pipelines/templates/report.template.html @@ -1,8 +1,15 @@ - + + {{title}} + + + - -
{{table}}
+ + +
+ +
+ +
+ +
+
+ +
+
+

+ Dynamic Pipeline Report +

+
+
+
+ + +
+
+
+
+
+ + +
+
+
+ +
+
{{table}}
+
+
+ + +
+
+ +
+
+ + + + + + + - diff --git a/tools/ci/dynamic_pipelines/templates/retry-jobs.png b/tools/ci/dynamic_pipelines/templates/retry-jobs.png deleted file mode 100644 index a8a60112c01..00000000000 Binary files a/tools/ci/dynamic_pipelines/templates/retry-jobs.png and /dev/null differ diff --git a/tools/ci/dynamic_pipelines/templates/scripts.js b/tools/ci/dynamic_pipelines/templates/scripts.js new file mode 100644 index 00000000000..403aa23597c --- /dev/null +++ b/tools/ci/dynamic_pipelines/templates/scripts.js @@ -0,0 +1,1127 @@ +function debounce(func, wait) { + let timeout; + return function () { + const context = this; + const args = arguments; + clearTimeout(timeout); + timeout = setTimeout(() => func.apply(context, args), wait); + }; +} + +function extractPipelineId() { + return $("body").data("pipeline-id"); +} + +function updateTabsAvailability(jobs) { + $("#build-report-tab, #job-report-tab, #test-report-tab").each(function () { + const $tab = $(this); + if (!$tab.hasClass("active")) { + $tab.addClass("disabled"); + $tab.attr("title", "This report is not available yet"); + $tab.css("opacity", "0.5"); + $tab.css("cursor", "not-allowed"); + $tab.on("click", function (e) { + if ($(this).hasClass("disabled")) { + e.preventDefault(); + e.stopPropagation(); + return false; + } + }); + } + }); + + const buildReportPatterns = [ + "generate_pytest_build_report", + "built_apps", + "build_report", + "skipped_apps", + "failed_apps", + ]; + + const jobReportPatterns = [ + "generate_failed_jobs_report", + "job_report", + "pipeline_jobs", + ]; + + const testReportPatterns = [ + "generate_pytest_report", + "target_test_report", + "test_report", + "pytest_report", + ]; + + let buildReportJob = jobs.find((job) => + buildReportPatterns.some((pattern) => job.name.includes(pattern)) + ); + + let jobReportJob = jobs.find((job) => + jobReportPatterns.some((pattern) => job.name.includes(pattern)) + ); + + let testReportJob = jobs.find((job) => + testReportPatterns.some((pattern) => job.name.includes(pattern)) + ); + + if (buildReportJob) reportJobIds.build = buildReportJob.job_id; + if (jobReportJob) reportJobIds.job = jobReportJob.job_id; + if (testReportJob) reportJobIds.test = testReportJob.job_id; + + const hasBuildReport = buildReportJob !== undefined; + const hasJobReport = jobReportJob !== undefined; + const hasTestReport = testReportJob !== undefined; + + if (hasBuildReport && !$("#build-report-tab").hasClass("active")) { + $("#build-report-tab").removeClass("disabled"); + $("#build-report-tab").attr( + "title", + "View application build results including binary sizes and downloads" + ); + $("#build-report-tab").css("opacity", "1"); + $("#build-report-tab").css("cursor", "pointer"); + } + + if (hasJobReport && !$("#job-report-tab").hasClass("active")) { + $("#job-report-tab").removeClass("disabled"); + $("#job-report-tab").attr( + "title", + "View CI job results and failed jobs" + ); + $("#job-report-tab").css("opacity", "1"); + $("#job-report-tab").css("cursor", "pointer"); + } + + if (hasTestReport && !$("#test-report-tab").hasClass("active")) { + $("#test-report-tab").removeClass("disabled"); + $("#test-report-tab").attr( + "title", + "View test results including success, failure, and skipped tests" + ); + $("#test-report-tab").css("opacity", "1"); + $("#test-report-tab").css("cursor", "pointer"); + } +} + +$(document).ready(function () { + const currentPath = window.location.pathname; + const currentFile = currentPath.substring(currentPath.lastIndexOf("/") + 1); + + $(".report-nav-tabs .nav-tab").removeClass("active"); + + if (currentFile.includes("built_apps")) { + $("#build-report-tab").addClass("active"); + } else if (currentFile.includes("job_report")) { + $("#job-report-tab").addClass("active"); + } else if (currentFile.includes("target_test_report")) { + $("#test-report-tab").addClass("active"); + } + + wrapTablesInCollapsibleSections(); + makeTablesCollapsible(); + setupScrollProgressBar(); + setupFloatingActions(); + enhanceTableStatusDisplay(); + markTestCaseColumns(); + setupResponsiveText(); + setupTextToggles(); + setupEventHandlers(); + scrollToHashLocation(); + initBootstrapTable(); + fixStickyHeaderAlignment(); + forceEnableStickyHeaders(); + fixTableHeaderText(); + lazyLoadVisibleImages(); + setupPagination(); + optimizeScrollPerformance(); +}); + +function fixStickyHeaderAlignment() { + if (window.isScrolling) return; + + $(".sticky-header-container").each(function () { + const $container = $(this); + const $table = $container + .closest(".bootstrap-table") + .find(".fixed-table-body table"); + + let tableWidth = "100%"; + if ($table.length) { + const actualWidth = $table.width(); + if (actualWidth > 0) { + tableWidth = actualWidth + "px"; + } + } + + $container.css({ + left: "0", + "margin-left": "0", + "padding-left": "0", + width: tableWidth, + "z-index": "100", + transform: "translateZ(0)", + }); + + this.style.setProperty("left", "0", "important"); + this.style.setProperty("margin-left", "0", "important"); + this.style.setProperty("padding-left", "0", "important"); + this.style.setProperty("width", tableWidth, "important"); + this.style.setProperty("z-index", "100", "important"); + this.style.setProperty("transform", "translateZ(0)", "important"); + + const $headerTable = $container.find("table"); + if ($headerTable.length) { + $headerTable.css("width", tableWidth); + $headerTable[0].style.setProperty("width", tableWidth, "important"); + } + + if (!window.isScrolling) { + if ($headerTable.length && $table.length) { + const $headerCols = $headerTable.find("th"); + const $bodyCols = $table.find("tr:first-child td"); + + if ($headerCols.length === $bodyCols.length) { + $headerCols.each(function (i) { + if (i < $bodyCols.length) { + const bodyColWidth = $($bodyCols[i]).outerWidth(); + if (bodyColWidth > 0) { + $(this).css("min-width", bodyColWidth + "px"); + $(this).css("width", bodyColWidth + "px"); + + const $thInner = $(this).find(".th-inner"); + if ($thInner.length) { + $thInner.css({ + "white-space": "normal", + overflow: "visible", + "text-overflow": "clip", + height: "auto", + display: "block", + }); + } + } + } + }); + } + } + } + }); + + $(".fixed-table-container").each(function () { + if ($(this).css("left") === "20px") { + $(this).css("left", "0"); + this.style.setProperty("left", "0", "important"); + } + }); + + $(".bootstrap-table .table thead th").css({ + position: "sticky", + top: "0", + "z-index": "100", + "background-color": "var(--esp-light)", + transform: "translateZ(0)", + "white-space": "normal", + overflow: "visible", + "text-overflow": "clip", + height: "auto", + }); + + $( + ".bootstrap-table .table thead th .th-inner, .sticky-header-container th .th-inner" + ).css({ + "white-space": "normal", + overflow: "visible", + "text-overflow": "clip", + height: "auto", + "min-height": "20px", + display: "block", + "line-height": "1.4", + }); +} + +function setupScrollProgressBar() { + const progressBar = $("#nav-progress-bar"); + let lastScrollPosition = 0; + let ticking = false; + + $(window).on("scroll", function () { + if (ticking) return; + + const scrollPosition = window.scrollY; + + if (Math.abs(scrollPosition - lastScrollPosition) < 5) { + return; + } + + lastScrollPosition = scrollPosition; + + window.requestAnimationFrame(function () { + const windowHeight = $(document).height() - $(window).height(); + const scrollPercentage = (scrollPosition / windowHeight) * 100; + progressBar.css("width", scrollPercentage + "%"); + ticking = false; + }); + ticking = true; + }); +} + +function setupFloatingActions() { + $("#back-to-top").on("click", function () { + $("html, body").animate({ scrollTop: 0 }, 300); + }); +} + +function enhanceTableStatusDisplay() { + $("table.table th").each(function (index) { + const $header = $(this); + const headerText = $header.text().trim().toLowerCase(); + + if ( + headerText.includes("status") || + headerText.includes("result") || + headerText.includes("state") + ) { + const colIndex = index + 1; + + $(`table.table td:nth-child(${colIndex})`).each(function () { + const $cell = $(this); + const cellText = $cell.text().trim().toLowerCase(); + + if (cellText.includes("pass") || cellText.includes("success")) { + $cell.html( + `
${$cell.text()}
` + ); + } else if ( + cellText.includes("fail") || + cellText.includes("error") + ) { + $cell.html( + `
${$cell.text()}
` + ); + } else if ( + cellText.includes("warn") || + cellText.includes("skip") + ) { + $cell.html( + `
${$cell.text()}
` + ); + } + }); + } + }); +} + +function setupTextToggles() { + setupResponsiveText(); + + $(".toggle-link").off("click").on("click", toggleText); +} + +function setupResponsiveText() { + $("table.table td").each(function () { + const $cell = $(this); + const text = $cell.text(); + + if ( + text.length > 100 && + !$cell.hasClass("test-case-name") && + !$cell.find(".text-toggle").length + ) { + const displayText = text.substring(0, 100) + "..."; + $cell.html( + `${displayText}` + + `` + + ` Show More` + ); + } + }); +} + +function makeTablesCollapsible() { + $("table.table").each(function (index) { + const table = $(this); + const tableId = `table-${index}`; + table.attr("id", tableId); + + const bootstrapTableWrapper = table.closest(".bootstrap-table"); + + let container = bootstrapTableWrapper.parent(".table-container"); + if (!container.length) { + container = $('
'); + } + + let tableControls = bootstrapTableWrapper.prev(".table-controls"); + + if (!tableControls.length) { + const initialButtonHtml = + 'Collapse Table'; + + const toggleButton = $(` + + `); + + tableControls = $('
').append( + toggleButton + ); + + bootstrapTableWrapper.before(tableControls); + } else { + const existingButton = tableControls.find(".table-collapse-btn"); + if (existingButton.length) { + existingButton.detach(); + tableControls.append(existingButton); + } + } + + if (!bootstrapTableWrapper.parent().hasClass("table-container")) { + bootstrapTableWrapper.wrap(container); + } + + bootstrapTableWrapper.show(); + bootstrapTableWrapper.addClass("expanded"); + + const toggleButton = tableControls.find(".table-collapse-btn"); + toggleButton.off("click").on("click", function (e) { + e.preventDefault(); + e.stopPropagation(); + + const tableId = $(this).data("table"); + + const targetTable = $(`#${tableId}`); + if (!targetTable.length) { + console.error("Target table not found:", tableId); + return; + } + + const bootstrapWrapper = targetTable.closest(".bootstrap-table"); + if (!bootstrapWrapper.length) { + console.error( + "Bootstrap wrapper not found for table:", + tableId + ); + return; + } + + const isVisible = bootstrapWrapper.is(":visible"); + + if (isVisible) { + bootstrapWrapper.slideUp(300); + bootstrapWrapper.removeClass("expanded"); + $(this).html( + 'Expand Table' + ); + } else { + bootstrapWrapper.slideDown(300); + bootstrapWrapper.addClass("expanded"); + $(this).html( + 'Collapse Table' + ); + } + }); + }); + + setTimeout(function () { + $(".bootstrap-table[data-auto-collapse='true']").each(function () { + const bootstrapWrapper = $(this); + const tableId = bootstrapWrapper.find("table").attr("id"); + const toggleButton = $(`button[data-table="${tableId}"]`); + + bootstrapWrapper.slideUp(300); + bootstrapWrapper.removeClass("expanded"); + + if (toggleButton.length) { + toggleButton.html( + 'Expand Table' + ); + } + }); + }, 500); +} + +function expandAllTables() { + $(".bootstrap-table").slideDown(300); + $(".bootstrap-table").addClass("expanded"); + + $(".table-collapse-btn").each(function () { + $(this).html( + 'Collapse Table' + ); + }); + + $(".section-header").attr("aria-expanded", "true"); + $(".section-body").slideDown(300); +} + +function collapseAllTables() { + $(".bootstrap-table").slideUp(300); + $(".bootstrap-table").removeClass("expanded"); + + $(".table-collapse-btn").each(function () { + $(this).html('Expand Table'); + }); +} + +function initBootstrapTable() { + $("table.table").each(function () { + const $table = $(this); + + $table.bootstrapTable({ + classes: "table table-bordered table-striped", + height: 800, + pagination: false, + showColumns: true, + showColumnsToggleAll: true, + showToggle: false, + clickToSelect: false, + minimumCountColumns: 2, + stickyHeader: true, + stickyHeaderOffsetY: 0, + theadClasses: "thead-light", + toolbar: "#toolbar", + resizable: true, + checkOnInit: false, + fixedColumns: true, + fixedNumber: 0, + sortStable: true, + undefinedText: "-", + showFullscreen: true, + widthUnit: "%", + headerStyle: function () { + return { + css: { + "white-space": "normal", + overflow: "visible", + "text-overflow": "clip", + height: "auto", + "min-height": "50px", + }, + }; + }, + }); + + $table.closest(".bootstrap-table").css({ + width: "100%", + "max-width": "100%", + }); + + $table.closest(".fixed-table-body").css({ + "overflow-x": "auto", + }); + + $table.css({ + width: "100%", + "min-width": "100%", + }); + + $table.find("thead th").each(function () { + const $th = $(this); + const $thInner = $th.find(".th-inner"); + + if ($thInner.length) { + $thInner.css({ + "white-space": "normal", + overflow: "visible", + "text-overflow": "clip", + height: "auto", + "min-height": "20px", + display: "block", + }); + } + }); + }); + + $(window).on( + "resize", + debounce(function () { + $("table.table").bootstrapTable("resetView"); + }, 100) + ); +} + +function wrapTablesInCollapsibleSections() { + $("h2").each(function (index) { + if ($(this).closest(".section-header").length > 0) { + return; + } + + const header = $(this); + const headerText = header.text(); + const headerId = header.attr("id") || `section-header-${index}`; + + const nextTable = header.nextAll("table.table:first"); + + if (nextTable.length) { + if (nextTable.closest(".section-body").length > 0) { + return; + } + + const wrapper = $('
'); + + const sectionHeader = $(` +
+

+ ${headerText} + +

+ +
+ `); + + const sectionBody = $( + `
` + ); + + const bootstrapTable = nextTable.closest(".bootstrap-table"); + const tableContainer = bootstrapTable.closest(".table-container"); + + if (tableContainer.length) { + tableContainer.detach().appendTo(sectionBody); + } else if (bootstrapTable.length) { + bootstrapTable.detach().appendTo(sectionBody); + } else { + nextTable.detach().appendTo(sectionBody); + } + + wrapper.append(sectionHeader).append(sectionBody); + + header.replaceWith(wrapper); + } + }); + + $(".section-header") + .off("click") + .on("click", function (e) { + if ( + $(e.target).hasClass("copy-link-icon") || + $(e.target).closest(".copy-link-icon").length || + $(e.target).hasClass("table-collapse-btn") || + $(e.target).closest(".table-collapse-btn").length + ) { + return; + } + + const header = $(this); + const targetId = header.data("target"); + const sectionBody = $(targetId); + const isExpanded = header.attr("aria-expanded") === "true"; + + if (isExpanded) { + header.attr("aria-expanded", "false"); + sectionBody.slideUp(300); + } else { + header.attr("aria-expanded", "true"); + sectionBody.slideDown(300); + } + }); +} + +function markTestCaseColumns() { + $("table.table th").each(function (index) { + const headerText = $(this).text().trim().toLowerCase(); + if ( + headerText.includes("test") && + (headerText.includes("case") || headerText.includes("name")) + ) { + const colIndex = index + 1; + $(`table.table td:nth-child(${colIndex})`).addClass( + "test-case-name" + ); + } + }); +} + +function setupEventHandlers() { + $(window).on("load", scrollToHashLocation); + $("body").on("click", ".toggle-link", toggleText); + + $("#expand-all-tables") + .off("click") + .on("click", function (e) { + e.preventDefault(); + expandAllTables(); + }); + + $("#collapse-all-tables") + .off("click") + .on("click", function (e) { + e.preventDefault(); + collapseAllTables(); + }); + + $("#pagination-size").on("change", function () { + const pageSize = parseInt($(this).val(), 10); + applyPaginationToTables(pageSize); + }); + + $("#show-all-columns").on("click", function (e) { + e.preventDefault(); + $("table.table").bootstrapTable("showAllColumns"); + }); + + $("#show-failed-only").on("click", function (e) { + e.preventDefault(); + filterByStatus("fail"); + }); + + $("#show-passed-only").on("click", function (e) { + e.preventDefault(); + filterByStatus("pass"); + }); + + $("#export-csv").on("click", function (e) { + e.preventDefault(); + $("table.table:visible").bootstrapTable("exportTable", { + type: "csv", + fileName: "report_export_" + new Date().toISOString().slice(0, 10), + }); + }); + + $("#clear-all-filters").on("click", function () { + clearAllFilters(); + }); +} + +function filterByStatus(status) { + $("#active-filters").show(); + $("#filter-badges").html( + `Status: ${status}` + ); + + $("table.table").each(function () { + const $table = $(this); + + let statusColIndex = -1; + $table.find("th").each(function (index) { + const headerText = $(this).text().trim().toLowerCase(); + if ( + headerText.includes("status") || + headerText.includes("result") || + headerText.includes("state") + ) { + statusColIndex = index; + return false; + } + }); + + if (statusColIndex >= 0) { + $table.bootstrapTable( + "filterBy", + { + [statusColIndex]: status, + }, + { + filterAlgorithm: function (row, filters) { + const cellText = $(row[statusColIndex]) + .text() + .trim() + .toLowerCase(); + return cellText.includes(status); + }, + } + ); + } + }); +} + +function clearAllFilters() { + $("table.table").each(function () { + const $table = $(this); + $table.bootstrapTable("clearFilterControl"); + $table.bootstrapTable("resetSearch"); + }); + + $("#active-filters").hide(); + $("#filter-badges").empty(); +} + +function scrollToHashLocation() { + const hash = window.location.hash; + if (hash) { + setTimeout(() => { + const target = $(hash); + if (target.length) { + const sectionBody = target.closest(".section-body"); + if (sectionBody.length) { + const sectionHeader = sectionBody.prev(".section-header"); + if (sectionHeader.attr("aria-expanded") !== "true") { + sectionHeader.attr("aria-expanded", "true"); + sectionBody.slideDown(0); + } + + const containingTable = target.closest("table.table"); + if (containingTable.length) { + const bootstrapWrapper = + containingTable.closest(".bootstrap-table"); + if (!bootstrapWrapper.is(":visible")) { + bootstrapWrapper.show(); + const tableBtn = bootstrapWrapper + .parent() + .find(".table-controls") + .find(".table-collapse-btn"); + tableBtn.html( + 'Collapse Table' + ); + } + } + } + + $("html, body").animate( + { scrollTop: target.offset().top - 20 }, + 100 + ); + } + }, 300); + } +} + +function copyPermalink(anchorId) { + const fullUrl = `${window.location.origin}${window.location.pathname}${anchorId}`; + history.pushState(null, null, anchorId); + navigator.clipboard.writeText(fullUrl); + + const tooltip = $(` +
+ Link copied to clipboard! +
+ `); + + const icon = $(event.target).closest(".copy-link-icon"); + if (icon.length) { + const originalClass = icon.attr("class"); + + icon.removeClass() + .addClass("fas fa-check copy-link-icon") + .css("color", "var(--esp-success)"); + + setTimeout(() => { + icon.attr("class", originalClass).css("color", ""); + }, 1500); + } + + $("body").append(tooltip); + setTimeout(() => tooltip.remove(), 2000); + scrollToHashLocation(); +} + +function toggleText(e) { + e.preventDefault(); + e.stopPropagation(); + + const link = $(this); + const textSpan = link.siblings(".full-text"); + const toggleSpan = link.siblings(".text-toggle"); + + const visible = textSpan.is(":visible"); + + if (visible) { + link.html(' Show More'); + textSpan.hide(); + toggleSpan.show(); + } else { + link.html(' Show Less'); + textSpan.show(); + toggleSpan.hide(); + } +} + +function setupPagination() { + applyPaginationToTables(25); +} + +function applyPaginationToTables(pageSize) { + localStorage.setItem("paginationSize", pageSize); + + $("table.table").each(function () { + const $table = $(this); + + if (pageSize > 0) { + $table.bootstrapTable("refreshOptions", { + pagination: true, + pageSize: pageSize, + pageList: [10, 25, 50, "All"], + }); + + $table + .closest(".bootstrap-table") + .find(".fixed-table-pagination") + .css({ + display: "block", + visibility: "visible", + "background-color": "var(--esp-light)", + padding: "10px", + "border-top": "1px solid rgba(0, 0, 0, 0.05)", + }); + } else { + $table.bootstrapTable("refreshOptions", { + pagination: false, + }); + + $table + .closest(".bootstrap-table") + .find(".fixed-table-pagination") + .css({ + display: "none", + visibility: "hidden", + }); + } + }); +} + +function optimizeScrollPerformance() { + window.isScrolling = false; + let scrollTimer = null; + let lastScrollTop = 0; + + cacheAndFixDomElements(); + + const progressBar = document.getElementById("nav-progress-bar"); + const backToTop = document.getElementById("back-to-top"); + + window.addEventListener( + "scroll", + function () { + window.isScrolling = true; + + const scrollTop = window.scrollY; + + if (Math.abs(scrollTop - lastScrollTop) > 5) { + lastScrollTop = scrollTop; + + handleEssentialScrollUpdates(scrollTop, progressBar, backToTop); + } + + clearTimeout(scrollTimer); + scrollTimer = setTimeout(function () { + window.isScrolling = false; + requestAnimationFrame(function () { + fixStickyHeaderAlignment(); + fixTableHeaderText(); + }); + }, 150); + }, + { passive: true } + ); + + lazyLoadVisibleImages(); +} + +function handleEssentialScrollUpdates(scrollTop, progressBar, backToTop) { + requestAnimationFrame(function () { + const docHeight = + Math.max( + document.body.scrollHeight, + document.body.offsetHeight, + document.documentElement.clientHeight, + document.documentElement.scrollHeight, + document.documentElement.offsetHeight + ) - window.innerHeight; + const scrollPercentage = (scrollTop / docHeight) * 100; + progressBar.style.width = scrollPercentage + "%"; + + if (scrollTop > 200) { + backToTop.classList.add("visible"); + } else { + backToTop.classList.remove("visible"); + } + }); +} + +function cacheAndFixDomElements() { + const stickyElements = document.querySelectorAll( + ".sticky-header-container, .section-header, .bootstrap-table thead, .fixed-table-header" + ); + + for (let i = 0; i < stickyElements.length; i++) { + const elem = stickyElements[i]; + elem.style.transform = "translateZ(0)"; + elem.style.willChange = "transform"; + elem.style.backfaceVisibility = "hidden"; + } + + document.body.style.willChange = "scroll-position"; + document.body.style.backfaceVisibility = "hidden"; + + const headers = document.querySelectorAll( + ".bootstrap-table .table thead th, .sticky-header-container th" + ); + for (let i = 0; i < headers.length; i++) { + const header = headers[i]; + header.style.position = "sticky"; + header.style.top = "0"; + header.style.zIndex = "100"; + header.style.transform = "translateZ(0)"; + + const thInners = header.querySelectorAll(".th-inner"); + for (let j = 0; j < thInners.length; j++) { + const inner = thInners[j]; + inner.style.whiteSpace = "normal"; + inner.style.overflow = "visible"; + inner.style.textOverflow = "clip"; + inner.style.height = "auto"; + inner.style.minHeight = "20px"; + inner.style.display = "block"; + } + } +} + +function lazyLoadVisibleImages() { + const lazyImages = document.querySelectorAll("img[data-src]"); + if (lazyImages.length === 0) return; + + const loadImage = function (img) { + if (img.dataset.src) { + img.src = img.dataset.src; + img.removeAttribute("data-src"); + } + }; + + lazyImages.forEach((img) => { + if (isElementInViewport(img)) { + loadImage(img); + } + }); + + document.addEventListener( + "scroll", + debounce(function () { + lazyImages.forEach((img) => { + if (img.dataset.src && isElementInViewport(img)) { + loadImage(img); + } + }); + }, 200), + { passive: true } + ); +} + +function isElementInViewport(el) { + const rect = el.getBoundingClientRect(); + return ( + rect.top >= 0 && + rect.left >= 0 && + rect.bottom <= + (window.innerHeight || document.documentElement.clientHeight) && + rect.right <= + (window.innerWidth || document.documentElement.clientWidth) + ); +} + +function fixTableHeaderText() { + if (window.isScrolling) return; + + $(".bootstrap-table .table thead th, .sticky-header-container th").each( + function () { + const $th = $(this); + + $th.css({ + "white-space": "normal", + overflow: "visible", + "text-overflow": "clip", + height: "auto", + "min-height": "50px", + }); + + const $thInner = $th.find(".th-inner"); + if ($thInner.length) { + $thInner.css({ + "white-space": "normal", + overflow: "visible", + "text-overflow": "clip", + height: "auto", + "min-height": "20px", + display: "block", + "line-height": "1.4", + }); + + $thInner.find("span, div").css({ + "white-space": "normal", + overflow: "visible", + }); + } + } + ); +} + +function forceEnableStickyHeaders() { + if (!window.isScrolling) { + $("table.table thead th").css({ + position: "sticky", + top: "0", + "z-index": "100", + "background-color": "var(--esp-light)", + transform: "translateZ(0)", + "will-change": "transform", + "backface-visibility": "hidden", + "white-space": "normal", + overflow: "visible", + "text-overflow": "clip", + height: "auto", + "min-height": "50px", + }); + + $("table.table").each(function () { + const $table = $(this); + const bootstrapTable = $table.closest(".bootstrap-table"); + + if ( + bootstrapTable.length && + bootstrapTable.find(".sticky-header-container").length === 0 + ) { + const $thead = $table.find("thead").clone(); + const $stickyContainer = $( + '
' + ); + const $stickyTable = $('
').append( + $thead + ); + + $stickyContainer.append($stickyTable); + bootstrapTable.prepend($stickyContainer); + + const tableWidth = $table.width(); + $stickyContainer.css({ + position: "sticky", + top: "0", + "z-index": "100", + width: tableWidth + "px", + overflow: "hidden", + "background-color": "var(--esp-light)", + "will-change": "transform", + "backface-visibility": "hidden", + transform: "translateZ(0)", + }); + + const $originalThs = $table.find("thead th"); + const $stickyThs = $stickyTable.find("th"); + + $originalThs.each(function (i) { + if (i < $stickyThs.length) { + const width = $(this).outerWidth(); + $($stickyThs[i]).css({ + width: width + "px", + "white-space": "normal", + overflow: "visible", + "text-overflow": "clip", + height: "auto", + "min-height": "50px", + }); + + const $inner = $($stickyThs[i]).find(".th-inner"); + if ($inner.length) { + $inner.css({ + "white-space": "normal", + overflow: "visible", + "text-overflow": "clip", + height: "auto", + "min-height": "20px", + display: "block", + "line-height": "1.4", + }); + } + } + }); + } + }); + } +} diff --git a/tools/ci/dynamic_pipelines/templates/styles.css b/tools/ci/dynamic_pipelines/templates/styles.css new file mode 100644 index 00000000000..cc3c43b9b43 --- /dev/null +++ b/tools/ci/dynamic_pipelines/templates/styles.css @@ -0,0 +1,1095 @@ +/** + * Espressif Report Template Styles + * Optimized for desktop-only application + */ + +/* Variables and base settings */ +:root { + /* Color palette */ + --esp-primary: #e83711; + --esp-secondary: #f3c300; + --esp-dark: #282430; + --esp-light: #f8f9fa; + --esp-blue: #3a86ff; + --esp-gray: #707070; + --esp-success: #25be7b; + --esp-warning: #f3c300; + --esp-danger: #e83711; + + /* UI elements */ + --body-bg: #f5f7fa; + --card-bg: #ffffff; + --text-color: #282430; + --border-radius: 8px; + --box-shadow: 0 4px 10px rgba(0, 0, 0, 0.05); + --transition-speed: 0.3s; + + /* Fixed desktop layout variables */ + --container-width: 1600px; + --sidebar-width: 280px; + --header-height: 60px; +} + +/* ---------------------------------- */ +/* Base styles */ +/* ---------------------------------- */ +html { + scroll-behavior: smooth; +} + +body { + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, + "Helvetica Neue", Arial, sans-serif; + line-height: 1.6; + color: var(--text-color); + background-color: var(--body-bg); + transition: background-color var(--transition-speed); + min-width: 1024px; + overflow-y: scroll; +} + +.container-fluid { + max-width: 100% !important; + padding: 20px !important; + margin: 0 auto; +} + +/* Typography */ +h2 { + padding-top: 3rem; + margin-bottom: 0px; + color: var(--esp-dark); + font-weight: 600; + background-color: #f5f7fa; +} + +/* ---------------------------------- */ +/* Performance Optimizations */ +/* ---------------------------------- */ +/* Hardware acceleration utility class */ +.hw-accelerated { + transform: translateZ(0); + will-change: transform; + backface-visibility: hidden; +} + +/* Scrollbar styling */ +::-webkit-scrollbar { + width: 8px; + height: 8px; +} + +::-webkit-scrollbar-track { + background: rgba(0, 0, 0, 0.05); + border-radius: 4px; +} + +::-webkit-scrollbar-thumb { + background: rgba(0, 0, 0, 0.2); + border-radius: 4px; +} + +::-webkit-scrollbar-thumb:hover { + background: rgba(0, 0, 0, 0.3); +} + +/* Reduced motion for accessibility */ +@media (prefers-reduced-motion: reduce) { + html { + scroll-behavior: auto; + } + + .bootstrap-table, + .section-body, + .sticky-header-container, + .section-header { + transition: none !important; + } +} + +/* ---------------------------------- */ +/* Navigation and Header */ +/* ---------------------------------- */ + +/* Header styling */ +.report-header { + display: flex; + align-items: center; + padding: 1.2rem 0; + margin-bottom: 2rem; + border-bottom: 1px solid rgba(0, 0, 0, 0.1); + position: relative; + justify-content: space-between; + background-color: #f8f9fa; +} + +.report-header .logo { + height: 30px; + margin-bottom: 0; + padding-left: 15px; +} + +.report-header .title-container { + display: flex; + flex-direction: column; + justify-content: center; + text-align: center; + margin: 0 auto; +} + +.report-header h1 { + margin: 0; + font-weight: 700; + font-size: 1.8rem; + line-height: 1.2; + display: inline-block; + text-align: center; +} + +.report-header p { + margin-bottom: 0; + font-size: 0.85rem; + text-align: center; + color: #707070; +} + +.report-header::after { + content: ""; + position: absolute; + bottom: -3px; + left: 0; + right: 0; + height: 3px; + background-color: var(--esp-primary); +} + +.logo-container { + width: 180px; + display: flex; + align-items: center; + justify-content: flex-start; +} + +.spacer { + width: 180px; /* Same as logo-container for balance */ +} + +/* Navigation progress bar */ +.nav-progress-container { + position: fixed; + top: 0; + left: 0; + width: 100%; + height: 3px; + background-color: transparent; + z-index: 1001; +} + +.nav-progress-bar { + height: 100%; + width: 0; + background-color: var(--esp-primary); + transition: width 0.2s ease-out; +} + +/* ---------------------------------- */ +/* Tables - Complete Reset & Rebuild */ +/* ---------------------------------- */ + +/* Base table appearance */ +.table { + width: 100%; + margin-bottom: 0; + background-color: var(--card-bg); + border-collapse: separate; + border-spacing: 0; +} + +/* Table header cells */ +.table > thead > tr > th { + padding: 12px 15px; + font-weight: 600; + color: var(--esp-dark); + background-color: var(--esp-light); + border-bottom: 2px solid rgba(0, 0, 0, 0.1); + text-align: left; + position: sticky; + top: 0; + z-index: 10; +} + +/* Table data cells */ +.table > tbody > tr > td { + padding: 12px 15px; + border-top: 1px solid rgba(0, 0, 0, 0.05); +} + +/* Striped rows */ +.table-striped > tbody > tr:nth-of-type(odd) { + background-color: rgba(0, 0, 0, 0.02); +} + +.table-striped > tbody > tr:hover { + background-color: rgba(0, 0, 0, 0.05); +} + +/* Bootstrap Table container */ +.bootstrap-table { + border-radius: var(--border-radius); + overflow: hidden; + box-shadow: var(--box-shadow); + margin-bottom: 1rem; +} + +/* Bootstrap Table scrollable body */ +.bootstrap-table .fixed-table-body { + max-height: 800px; + overflow-y: auto; + border: none !important; +} + +/* Remove any borders from the fixed-table-container */ +.bootstrap-table .fixed-table-container { + border: none !important; +} + +/* Override Bootstrap's fixed header styles to avoid conflicts */ +.bootstrap-table .fixed-table-header table { + background-color: var(--esp-light); +} + +.bootstrap-table .fixed-table-header { + background-color: var(--esp-light); +} + +/* Column width settings for better layout */ +.table > thead > tr > th:nth-child(1), +.table > tbody > tr > td:nth-child(1) { + width: 20%; + min-width: 150px; +} + +.table > thead > tr > th:nth-child(2), +.table > tbody > tr > td:nth-child(2) { + width: 20%; + min-width: 150px; +} + +.table > thead > tr > th:nth-child(3), +.table > tbody > tr > td:nth-child(3) { + width: 20%; + min-width: 150px; +} + +.table > thead > tr > th:nth-child(n + 4), +.table > tbody > tr > td:nth-child(n + 4) { + width: auto; + min-width: 100px; +} + +/* Ensure sticky section headers are above table headers */ +.sticky-section-header { + position: sticky; + top: 0; + z-index: 100; + background-color: #f5f7fa; +} + +/* Test case names */ +td.test-case-name { + white-space: normal; + overflow: visible; + word-break: break-word; +} + +/* Bootstrap table customizations */ +.bootstrap-table { + margin-bottom: 0; + border-radius: var(--border-radius); + box-shadow: var(--box-shadow); + background-color: var(--card-bg); + overflow: hidden; + max-height: 2000px; + transition: max-height 0.3s ease-in-out, opacity 0.3s ease-in-out; + opacity: 1; + width: 100% !important; + position: relative; +} + +.bootstrap-table.expanded { + max-height: 2000px; + opacity: 1; +} + +/* Main container setup */ +.bootstrap-table .fixed-table-container { + position: relative !important; + overflow: visible !important; +} + +/* Table headers - sticky relative to .fixed-table-body */ +.bootstrap-table .table thead th { + position: sticky !important; + top: 0 !important; + z-index: 10; + background-color: var(--esp-light); + border-bottom: 2px solid rgba(0, 0, 0, 0.1); + padding: 12px 15px !important; +} + +/* Fixed-table-header should be hidden when scrolling */ +.bootstrap-table .fixed-table-header { + display: none !important; +} + +/* Handle Bootstrap Table sticky header container */ +.bootstrap-table .sticky-header-container { + display: none !important; /* Use our own sticky headers instead */ +} + +/* Ensure th-inner elements are visible */ +.bootstrap-table .fixed-table-container .table thead th .th-inner { + white-space: normal !important; + overflow: visible !important; + text-overflow: clip !important; + height: auto !important; + min-height: 20px; + line-height: 1.4; + padding: 8px; + background-color: var(--esp-light); + display: block !important; +} + +/* Table container shouldn't have overflow-x at the top level */ +.table-container { + width: 100%; + max-width: 100%; + margin: 0 auto; + background-color: var(--card-bg); + border-radius: var(--border-radius); + overflow: hidden; + box-shadow: var(--box-shadow); + position: relative; +} + +/* Fix for table header rendering in the sticky context */ +.bootstrap-table thead { + position: sticky; + top: 0; + z-index: 10; +} + +/* Table controls */ +.table-controls { + background-color: var(--esp-light); + border-top-left-radius: var(--border-radius); + border-top-right-radius: var(--border-radius); + padding: 12px 15px; + border-bottom: 1px solid rgba(0, 0, 0, 0.05); + display: flex; + justify-content: flex-end; + align-items: center; + flex-wrap: wrap; + gap: 10px; +} + +.table-controls .row-count { + margin-bottom: 0; + white-space: nowrap; +} + +.table-controls .btn { + display: flex; + align-items: center; + justify-content: center; + gap: 5px; + transition: all 0.2s ease; +} + +/* Table toolbar */ +.fixed-table-toolbar { + padding: 10px; + background-color: var(--esp-light); + border-bottom: 1px solid rgba(0, 0, 0, 0.05); +} + +.fixed-table-toolbar .search { + display: flex; + align-items: center; +} + +.fixed-table-toolbar .search input { + border-radius: var(--border-radius); + border: 1px solid rgba(0, 0, 0, 0.1); + padding: 5px 10px; + margin-right: 10px; +} + +.fixed-table-toolbar .columns { + margin-left: auto; +} + +.fixed-table-toolbar .btn { + border-radius: var(--border-radius); + padding: 5px 10px; + margin-left: 5px; +} + +.fixed-table-toolbar .dropdown-menu { + border-radius: var(--border-radius); + box-shadow: var(--box-shadow); + padding: 8px; + border: none; +} + +/* Table pagination */ +.fixed-table-pagination { + padding: 10px; + background-color: var(--esp-light); + border-bottom-left-radius: var(--border-radius); + border-bottom-right-radius: var(--border-radius); + border-top: 1px solid rgba(0, 0, 0, 0.05); + display: flex; + justify-content: center; + align-items: center; +} + +.fixed-table-pagination .pagination { + float: none !important; + order: 1; + flex: 1 1 auto; + display: flex; + justify-content: center; + margin: 0 auto; + position: absolute; + left: 50%; + transform: translateX(-50%); +} + +.fixed-table-pagination .pagination-detail .pagination-info { + display: none; + float: none !important; + order: 2; + flex: 0 0 auto; +} + +.fixed-table-pagination .pagination li a { + border-radius: 4px; + margin: 0 2px; + border: 1px solid #dee2e6; + min-width: 34px; + height: 34px; + display: flex; + align-items: center; + justify-content: center; + color: var(--esp-dark); + transition: all 0.2s ease; +} + +.fixed-table-pagination .pagination li.active a { + background-color: var(--esp-primary); + border-color: var(--esp-primary); + color: white; +} + +.fixed-table-pagination .pagination li a:hover:not(.active) { + background-color: #e9ecef; + color: var(--esp-primary); +} + +/* Pagination options */ +.pagination-options .form-select-sm { + border-radius: var(--border-radius); + border: 1px solid rgba(0, 0, 0, 0.1); + height: 31px; + padding: 0.25rem 2rem 0.25rem 0.5rem; + font-size: 0.875rem; + background-color: var(--esp-light); + color: var(--esp-dark); + cursor: pointer; + transition: all var(--transition-speed); + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.05); +} + +.pagination-options .form-select-sm:hover { + border-color: var(--esp-primary); +} + +.pagination-options .form-select-sm:focus { + border-color: var(--esp-primary); + box-shadow: 0 0 0 0.15rem rgba(232, 55, 17, 0.25); + outline: none; +} + +/* ---------------------------------- */ +/* Collapsible Sections */ +/* ---------------------------------- */ +.section-collapsible { + margin-bottom: 2rem; + border-radius: var(--border-radius); + overflow: hidden; + box-shadow: var(--box-shadow); + background-color: var(--card-bg); + transition: transform var(--transition-speed); +} + +.section-collapsible:hover { + transform: translateY(-2px); +} + +.section-header { + display: flex; + align-items: center; + padding: 1rem 1.5rem; + background-color: #f5f7fa; + border-bottom: 1px solid rgba(0, 0, 0, 0.1); + cursor: pointer; + transition: background-color var(--transition-speed), + box-shadow var(--transition-speed); +} + +.section-header:hover { + background-color: #e9ecef; + box-shadow: 0 3px 8px rgba(0, 0, 0, 0.1); +} + +.section-header h2 { + margin: 0; + font-size: 1.5rem; + flex-grow: 1; + background-color: transparent; + position: relative; + z-index: 2; + font-weight: 500; + display: flex; + align-items: center; +} + +.section-header h2 .heading-text { + margin-right: 6px; +} + +.section-header .collapse-indicator { + margin-left: 10px; + font-size: 1.5rem; + transition: transform var(--transition-speed); + color: var(--esp-primary); + position: relative; + z-index: 2; +} + +.section-header[aria-expanded="false"] .collapse-indicator { + transform: rotate(-90deg); +} + +.copy-link-icon { + margin-left: 8px; + font-size: 1.5rem; + color: var(--esp-gray); + opacity: 0.7; + cursor: pointer; + transition: opacity 0.2s ease, color 0.2s ease; + position: relative; + z-index: 2; +} + +.copy-link-icon:hover { + opacity: 1; + color: var(--esp-primary); +} + +.sticky-section-header { + position: sticky; + top: 0; + z-index: 99; + background-color: #f5f7fa; + box-shadow: 0 2px 5px rgba(0, 0, 0, 0.05); +} + +.section-body { + padding: 1.5rem; + max-height: 0; + overflow: hidden; + transition: max-height 0.3s ease-in-out, padding 0.3s ease-in-out, + opacity 0.3s ease-in-out; + padding-top: 0; + padding-bottom: 0; + opacity: 0; +} + +.section-body.expanded { + max-height: 5000px; + padding: 1.5rem; + opacity: 1; +} + +.section-body.collapsed { + display: none; +} + +/* ---------------------------------- */ +/* Status Badges */ +/* ---------------------------------- */ +.status-badge { + display: inline-block; + padding: 4px 8px; + border-radius: 4px; + font-size: 0.8rem; + font-weight: 500; + text-align: center; + min-width: 80px; + transition: background-color 0.2s ease, transform 0.2s ease; +} + +.status-badge.success { + background-color: rgba(37, 190, 123, 0.1); + color: var(--esp-success); + border: 1px solid rgba(37, 190, 123, 0.2); +} + +.status-badge.warning { + background-color: rgba(243, 195, 0, 0.1); + color: var(--esp-warning); + border: 1px solid rgba(243, 195, 0, 0.2); +} + +.status-badge.danger { + background-color: rgba(232, 55, 17, 0.1); + color: var(--esp-danger); + border: 1px solid rgba(232, 55, 17, 0.2); +} + +.status-badge:hover { + transform: translateY(-1px); +} + +/* ---------------------------------- */ +/* Buttons */ +/* ---------------------------------- */ +.btn-outline-secondary { + border-color: #ced4da; + color: var(--esp-dark); +} + +.btn-outline-secondary:hover { + background-color: var(--esp-light); + border-color: var(--esp-gray); + color: var(--esp-dark); +} + +.btn-esp { + background-color: var(--esp-primary); + border: none; + color: #fff; + transition: all var(--transition-speed); +} + +.btn-esp:hover { + background-color: #cf3110; + color: #fff; + box-shadow: 0 4px 8px rgba(232, 55, 17, 0.2); +} + +.btn-esp-secondary { + background-color: var(--esp-secondary); + border: none; + color: var(--esp-dark); + transition: all var(--transition-speed); +} + +.btn-esp-secondary:hover { + background-color: #dab000; + color: var(--esp-dark); + box-shadow: 0 4px 8px rgba(243, 195, 0, 0.2); +} + +/* Table collapse button - updated to appear on the right */ +.table-collapse-btn { + background-color: var(--esp-light); + border: none; + color: var(--esp-gray); + padding: 6px 12px; + border-radius: var(--border-radius); + font-size: 0.875rem; + cursor: pointer; + transition: all var(--transition-speed); + display: inline-flex; + align-items: center; + justify-content: center; + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.05); + margin-left: auto; /* Push to the right side */ +} + +.table-collapse-btn:hover { + background-color: #e9ecef; + color: var(--esp-primary); +} + +.table-collapse-btn i { + margin-right: 5px; +} + +.clear-filter-btn { + transition: all 0.2s ease; + margin-top: 10px; +} + +.clear-filter-btn:hover { + background-color: var(--esp-light); + color: var(--esp-primary); +} + +.close-alert { + background: none; + border: none; + color: #856404; + font-size: 0.8rem; + cursor: pointer; + padding: 4px; + margin-left: auto; + opacity: 0.7; + transition: opacity 0.2s; + display: flex; + align-items: center; + justify-content: center; +} + +.close-alert:hover { + opacity: 1; +} + +/* Floating action buttons */ +.floating-actions { + position: fixed; + bottom: 30px; + right: 30px; + z-index: 1000; + display: flex; + flex-direction: column; + gap: 10px; +} + +.floating-action-btn { + width: 50px; + height: 50px; + border-radius: 50%; + background-color: var(--esp-primary); + color: white; + display: flex; + align-items: center; + justify-content: center; + box-shadow: 0 4px 10px rgba(0, 0, 0, 0.15); + cursor: pointer; + transition: transform 0.3s cubic-bezier(0.175, 0.885, 0.32, 1.275), + background-color 0.3s ease, box-shadow 0.3s ease; + font-size: 1.2rem; +} + +.floating-action-btn:hover { + transform: scale(1.1) translateY(-2px); +} + +.back-to-top { + background-color: var(--esp-secondary); + color: var(--esp-dark); +} + +/* ---------------------------------- */ +/* Utilities */ +/* ---------------------------------- */ +/* Text toggles */ +.text-toggle, +.full-text { + cursor: pointer; +} + +.toggle-link { + display: inline-block; + margin-left: 8px; + font-size: 0.8rem; + color: var(--esp-blue); + text-decoration: none; +} + +.toggle-link:hover { + text-decoration: underline; + color: var(--esp-primary); +} + +/* Row count label */ +.row-count { + font-size: 0.8rem; + color: var(--esp-gray); + margin-bottom: 15px; + display: block; +} + +/* Table state indicators */ +.table-empty-state, +.table-loading-state { + text-align: center; + padding: 50px 20px; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; +} + +.table-empty-state { + color: var(--esp-gray); + background-color: rgba(0, 0, 0, 0.01); + border-radius: var(--border-radius); +} + +.table-empty-state p { + margin-top: 10px; + font-size: 1.1rem; +} + +.table-filtered-indicator { + text-align: center; + padding: 30px 20px; + color: var(--esp-warning); + background-color: rgba(243, 195, 0, 0.05); + border-radius: var(--border-radius); + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + border: 1px dashed rgba(243, 195, 0, 0.3); + margin: 20px; +} + +.table-filtered-indicator p { + margin: 10px 0; + font-size: 1rem; +} + +.table-loading-indicator { + padding: 20px; + background-color: rgba(0, 0, 0, 0.02); + border-radius: var(--border-radius); + margin: 10px; + text-align: center; + color: var(--esp-gray); + animation: pulse 1.5s infinite ease-in-out; +} + +@keyframes pulse { + 0% { + opacity: 0.6; + } + 50% { + opacity: 1; + } + 100% { + opacity: 0.6; + } +} + +@keyframes fadeIn { + from { + opacity: 0; + } + to { + opacity: 1; + } +} + +/* Dropdown and other UI elements */ +.dropdown-menu { + border-radius: var(--border-radius); + box-shadow: var(--box-shadow); + border: none; + padding: 0.5rem; +} + +.dropdown-item { + border-radius: 4px; + padding: 8px 12px; +} + +.dropdown-item:hover { + background-color: rgba(232, 55, 17, 0.1); +} + +/* Column visibility controls */ +.column-visibility-controls { + margin-right: 10px; +} + +.column-visibility-menu { + max-height: 350px; + overflow-y: auto; + padding: 8px 0; + min-width: 220px; +} + +.column-visibility-menu .dropdown-item { + display: flex; + align-items: center; + padding: 8px 16px; + white-space: normal; + word-break: break-word; +} + +.column-visibility-menu .dropdown-item:hover { + background-color: rgba(232, 55, 17, 0.05); +} + +.column-visibility-menu .dropdown-header { + font-weight: 600; + color: var(--esp-dark); + padding: 8px 16px; +} + +.column-visibility-menu .dropdown-divider { + margin: 5px 0; +} + +.column-toggle .column-hidden { + width: 16px; + color: var(--esp-primary); +} + +.column-toggle .column-hidden { + color: var(--esp-gray); +} + +/* Active filters */ +#active-filters { + margin-top: 10px; + margin-bottom: 15px !important; + display: flex; + align-items: center; + background-color: #f8f9fa; + padding: 8px 12px; + border-radius: var(--border-radius); + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); +} + +#filter-badges .badge { + margin-right: 5px; + padding: 6px 10px; + font-weight: normal; +} + +#clear-all-filters { + margin-left: auto; + color: var(--esp-primary); + text-decoration: none; + font-size: 0.9rem; + padding: 4px 8px; +} + +#clear-all-filters:hover { + text-decoration: underline; + background-color: rgba(232, 55, 17, 0.05); + border-radius: 4px; +} + +/* Tooltip actions */ +.tooltip-actions { + position: absolute; + right: 60px; + background-color: white; + border-radius: var(--border-radius); + box-shadow: var(--box-shadow); + padding: 10px; + width: 200px; + display: none; +} + +.tooltip-actions.show { + display: block; + animation: fadeIn var(--transition-speed); +} + +/* Smooth scrolling */ +html.scrolling-top { + scroll-behavior: smooth; +} + +/* ---------------------------------- */ +/* Responsive Styles */ +/* ---------------------------------- */ +@media (max-width: 1200px) { + .bootstrap-table .fixed-table-body { + overflow-x: auto; + max-height: 600px; + } + + .table-container { + width: 100% !important; + padding: 0 !important; + } +} + +@media (max-width: 768px) { + .container-fluid { + padding-left: 20px !important; + padding-right: 20px !important; + } + + .table-responsive { + overflow-x: auto; + } + + .bootstrap-table .fixed-table-body { + max-height: 500px; + } + + .table-controls { + flex-direction: column; + align-items: flex-start; + } + + .table-controls-right { + width: 100%; + justify-content: space-between; + } + + .logo-container, + .spacer { + width: 100px; + } +} + +/* ---------------------------------- */ +/* Bootstrap Table Sticky Header Specific Styles */ +/* ---------------------------------- */ + +/* Make sure sticky header container is visible and properly positioned */ +.bootstrap-table .sticky-header-container { + position: sticky; + top: 0; + z-index: 10; + background-color: var(--esp-light); +} + +/* Ensure the sticky header is properly styled and visible */ +.bootstrap-table .sticky-header { + overflow: visible; + position: relative; +} + +/* Styles for the headers in the sticky container */ +.bootstrap-table .sticky-header-container thead th { + background-color: var(--esp-light); + color: var(--esp-dark); + font-weight: 600; + padding: 12px 15px; + border-bottom: 2px solid rgba(0, 0, 0, 0.1); + text-align: left; + position: static !important; /* Important to avoid double sticky behavior */ +} + +/* Styles for header cells inner content */ +.bootstrap-table .sticky-header-container th .th-inner { + padding: 0 !important; + line-height: 1.4; + white-space: normal !important; + overflow: visible !important; + text-overflow: clip !important; + font-weight: inherit !important; +} diff --git a/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml b/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml index 9b5c04318e6..f986108c54b 100644 --- a/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml +++ b/tools/ci/dynamic_pipelines/templates/test_child_pipeline.yml @@ -1,3 +1,21 @@ +all_build_finished: + stage: assign_test + tags: [fast_run, shiny] + image: $ESP_ENV_IMAGE + when: always + # this job is used to check if all build jobs are finished + # because the `needs` make the later jobs run even if the previous stage are not finished + # and there's no `needs: stage` for now in gitlab + # https://gitlab.com/gitlab-org/gitlab/-/issues/220758 + artifacts: + untracked: true + expire_in: 1 week + when: always + before_script: [] + dependencies: [] + script: + - echo "all test jobs finished" + generate_pytest_build_report: stage: assign_test image: $ESP_ENV_IMAGE @@ -5,16 +23,19 @@ generate_pytest_build_report: - build - shiny when: always + needs: + - all_build_finished + - pipeline: $PARENT_PIPELINE_ID + job: pipeline_variables artifacts: paths: - failed_apps.html - built_apps.html - skipped_apps.html - build_report.html - - test_related_apps_download_urls.yml + - presigned.json expire_in: 1 week when: always - script: - python tools/ci/dynamic_pipelines/scripts/generate_report.py --report-type build - python tools/ci/previous_stage_job_status.py --stage build @@ -26,6 +47,13 @@ generate_pytest_child_pipeline: tags: - build - shiny + needs: + - job: build_test_related_apps + optional: true + - job: build_non_test_related_apps # make sure all build jobs are passed + optional: true + - pipeline: $PARENT_PIPELINE_ID + job: pipeline_variables artifacts: paths: - target_test_child_pipeline.yml @@ -38,7 +66,6 @@ Pytest Target Test Jobs: - generate_pytest_child_pipeline variables: PARENT_PIPELINE_ID: $PARENT_PIPELINE_ID - REPORT_EXIT_CODE: $REPORT_EXIT_CODE # https://gitlab.com/gitlab-org/gitlab/-/issues/214340 inherit: variables: false diff --git a/tools/ci/dynamic_pipelines/tests/__init__.py b/tools/ci/dynamic_pipelines/tests/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/__init__.py b/tools/ci/dynamic_pipelines/tests/test_report_generator/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/XUNIT_REPORT.xml b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/XUNIT_REPORT.xml deleted file mode 100644 index 83a6e097678..00000000000 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/XUNIT_REPORT.xml +++ /dev/null @@ -1,200 +0,0 @@ - - - - - conftest.py:74: in case_tester - yield CaseTester(dut, **kwargs) -tools/ci/idf_unity_tester.py:202: in __init__ - self._manager = Manager() -/usr/lib/python3.9/multiprocessing/context.py:57: in Manager - m.start() -/usr/lib/python3.9/multiprocessing/managers.py:557: in start - self._address = reader.recv() -/usr/lib/python3.9/multiprocessing/connection.py:255: in recv - buf = self._recv_bytes() -/usr/lib/python3.9/multiprocessing/connection.py:419: in _recv_bytes - buf = self._recv(4) -/usr/lib/python3.9/multiprocessing/connection.py:388: in _recv - raise EOFError -E EOFError - - - conftest.py:74: in case_tester - yield CaseTester(dut, **kwargs) -tools/ci/idf_unity_tester.py:202: in __init__ - self._manager = Manager() -/usr/lib/python3.9/multiprocessing/context.py:57: in Manager - m.start() -/usr/lib/python3.9/multiprocessing/managers.py:557: in start - self._address = reader.recv() -/usr/lib/python3.9/multiprocessing/connection.py:255: in recv - buf = self._recv_bytes() -/usr/lib/python3.9/multiprocessing/connection.py:419: in _recv_bytes - buf = self._recv(4) -/usr/lib/python3.9/multiprocessing/connection.py:388: in _recv - raise EOFError -E EOFError - - - - - - - /root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/plugin.py:1272: in pytest_runtest_call - self._raise_dut_failed_cases_if_exists(duts) # type: ignore -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/plugin.py:1207: in _raise_dut_failed_cases_if_exists - raise AssertionError('Unity test failed') -E AssertionError: Unity test failed - - - - - - /builds/espressif/esp-idf/tools/test_build_system/test_common.py:134: Linux does not support executing .exe files - - - - - - - - - - - /root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:76: in wrapper - index = func(self, pattern, *args, **kwargs) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:153: in expect_exact - return self.pexpect_proc.expect_exact(pattern, **kwargs) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/spawnbase.py:432: in expect_exact - return exp.expect_loop(timeout) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/expect.py:181: in expect_loop - return self.timeout(e) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/expect.py:144: in timeout - raise exc -E pexpect.exceptions.TIMEOUT: <pytest_embedded.log.PexpectProcess object at 0x7f47b1ab46a0> -E searcher: searcher_string: -E 0: b'Press ENTER to see the list of tests' -E <pytest_embedded.log.PexpectProcess object at 0x7f47b1ab46a0> -E searcher: searcher_string: -E 0: b'Press ENTER to see the list of tests' - -The above exception was the direct cause of the following exception: -components/esp_timer/test_apps/pytest_esp_timer_ut.py:24: in test_esp_timer - dut.run_all_single_board_cases(timeout=120) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded_idf/unity_tester.py:435: in run_all_single_board_cases - for case in self.test_menu: -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded_idf/unity_tester.py:243: in test_menu - self._test_menu = self._parse_test_menu() -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded_idf/unity_tester.py:124: in _parse_test_menu - self.expect_exact(ready_line) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:83: in wrapper - raise e.__class__(debug_str) from e -E pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" -E Bytes in current buffer (color code eliminated): ce710,len:0x2afc entry 0x403cc710 -E Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_esp_timer/dut.txt - - - /root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:76: in wrapper - index = func(self, pattern, *args, **kwargs) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:131: in expect - return self.pexpect_proc.expect(pattern, **kwargs) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/spawnbase.py:354: in expect - return self.expect_list(compiled_pattern_list, -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/spawnbase.py:383: in expect_list - return exp.expect_loop(timeout) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/expect.py:181: in expect_loop - return self.timeout(e) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/expect.py:144: in timeout - raise exc -E pexpect.exceptions.TIMEOUT: <pytest_embedded.log.PexpectProcess object at 0x7f47af653b80> -E searcher: searcher_re: -E 0: re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?P<result>OK|FAIL)') -E <pytest_embedded.log.PexpectProcess object at 0x7f47af653b80> -E searcher: searcher_re: -E 0: re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?P<result>OK|FAIL)') - -The above exception was the direct cause of the following exception: -components/wear_levelling/test_apps/pytest_wear_levelling.py:18: in test_wear_levelling - dut.expect_unity_test_output() -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:180: in expect_unity_test_output - self.expect(UNITY_SUMMARY_LINE_REGEX, timeout=timeout) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:83: in wrapper - raise e.__class__(debug_str) from e -E pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?P<result>OK|FAIL)', re.MULTILINE)" -E Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 6673 bytes) -E Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.512safe.test_wear_levelling/dut.txt - - - /root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:76: in wrapper - index = func(self, pattern, *args, **kwargs) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:131: in expect - return self.pexpect_proc.expect(pattern, **kwargs) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/spawnbase.py:354: in expect - return self.expect_list(compiled_pattern_list, -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/spawnbase.py:383: in expect_list - return exp.expect_loop(timeout) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/expect.py:181: in expect_loop - return self.timeout(e) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/expect.py:144: in timeout - raise exc -E pexpect.exceptions.TIMEOUT: <pytest_embedded.log.PexpectProcess object at 0x7f47afc540d0> -E searcher: searcher_re: -E 0: re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?P<result>OK|FAIL)') -E <pytest_embedded.log.PexpectProcess object at 0x7f47afc540d0> -E searcher: searcher_re: -E 0: re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?P<result>OK|FAIL)') - -The above exception was the direct cause of the following exception: -components/wear_levelling/test_apps/pytest_wear_levelling.py:18: in test_wear_levelling - dut.expect_unity_test_output() -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:180: in expect_unity_test_output - self.expect(UNITY_SUMMARY_LINE_REGEX, timeout=timeout) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:83: in wrapper - raise e.__class__(debug_str) from e -E pexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?P<result>OK|FAIL)', re.MULTILINE)" -E Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 24528 bytes) -E Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_wear_levelling/dut.txt - - - /root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:76: in wrapper - index = func(self, pattern, *args, **kwargs) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:153: in expect_exact - return self.pexpect_proc.expect_exact(pattern, **kwargs) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/spawnbase.py:432: in expect_exact - return exp.expect_loop(timeout) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/expect.py:181: in expect_loop - return self.timeout(e) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pexpect/expect.py:144: in timeout - raise exc -E pexpect.exceptions.TIMEOUT: <pytest_embedded.log.PexpectProcess object at 0x7f47afc54df0> -E searcher: searcher_string: -E 0: b'Press ENTER to see the list of tests' -E <pytest_embedded.log.PexpectProcess object at 0x7f47afc54df0> -E searcher: searcher_string: -E 0: b'Press ENTER to see the list of tests' - -The above exception was the direct cause of the following exception: -components/wpa_supplicant/test_apps/pytest_wpa_supplicant_ut.py:17: in test_wpa_supplicant_ut - dut.run_all_single_board_cases() -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded_idf/unity_tester.py:435: in run_all_single_board_cases - for case in self.test_menu: -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded_idf/unity_tester.py:243: in test_menu - self._test_menu = self._parse_test_menu() -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded_idf/unity_tester.py:124: in _parse_test_menu - self.expect_exact(ready_line) -/root/.espressif/python_env/idf5.2_py3.9_env/lib/python3.9/site-packages/pytest_embedded/dut.py:83: in wrapper - raise e.__class__(debug_str) from e -E pexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" -E Bytes in current buffer (color code eliminated): 0 d4 000 00x0000 x0000x00 000000 0 -E Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.default.test_wpa_supplicant_ut/dut.txt - - - diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/apps b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/apps deleted file mode 100644 index 57d3df17c2a..00000000000 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/apps +++ /dev/null @@ -1,9 +0,0 @@ -{"build_system": "idf_cmake", "app_dir": "tools/test_apps/system/panic", "target": "esp32s3", "sdkconfig_path": "/builds/espressif/esp-idf/tools/test_apps/system/panic/sdkconfig.ci.coredump_flash_capture_dram", "config_name": "coredump_flash_capture_dram", "sdkconfig_defaults_str": null, "dry_run": false, "verbose": false, "check_warnings": true, "preserve": false, "copy_sdkconfig": false, "index": null, "build_status": "build failed", "build_comment": "Compilation error", "cmake_vars": {}, "work_dir": "tools/test_apps/system/panic", "build_dir": "build_esp32s3_coredump_flash_capture_dram", "build_log_filename": "build_log.txt", "size_json_filename": "size.json"} -{"build_system": "idf_cmake", "app_dir": "tools/test_apps/system/ram_loadable_app", "target": "esp32", "sdkconfig_path": "/builds/espressif/esp-idf/tools/test_apps/system/ram_loadable_app/sdkconfig.ci.defaults", "config_name": "defaults", "sdkconfig_defaults_str": null, "dry_run": false, "verbose": false, "check_warnings": true, "preserve": true, "copy_sdkconfig": false, "index": null, "build_status": "build success", "build_comment": null, "cmake_vars": {}, "work_dir": "tools/test_apps/system/ram_loadable_app", "build_dir": "build_esp32_defaults", "build_log_filename": "build_log.txt", "size_json_filename": "size.json"} -{"build_system": "idf_cmake", "app_dir": "tools/test_apps/system/ram_loadable_app", "target": "esp32", "sdkconfig_path": "/builds/espressif/esp-idf/tools/test_apps/system/ram_loadable_app/sdkconfig.ci.pure_ram", "config_name": "pure_ram", "sdkconfig_defaults_str": null, "dry_run": false, "verbose": false, "check_warnings": true, "preserve": true, "copy_sdkconfig": false, "index": null, "build_status": "build success", "build_comment": null, "cmake_vars": {}, "work_dir": "tools/test_apps/system/ram_loadable_app", "build_dir": "build_esp32_pure_ram", "build_log_filename": "build_log.txt", "size_json_filename": "size.json"} -{"build_system": "idf_cmake", "app_dir": "tools/test_apps/system/startup", "target": "esp32", "sdkconfig_path": "/builds/espressif/esp-idf/tools/test_apps/system/startup/sdkconfig.ci.flash_80m_qio", "config_name": "flash_80m_qio", "sdkconfig_defaults_str": null, "dry_run": false, "verbose": false, "check_warnings": true, "preserve": true, "copy_sdkconfig": false, "index": null, "build_status": "build success", "build_comment": null, "cmake_vars": {}, "work_dir": "tools/test_apps/system/startup", "build_dir": "build_esp32_flash_80m_qio", "build_log_filename": "build_log.txt", "size_json_filename": "size.json"} -{"build_system": "idf_cmake", "app_dir": "tools/test_apps/system/startup", "target": "esp32s3", "sdkconfig_path": "/builds/espressif/esp-idf/tools/test_apps/system/startup/sdkconfig.ci.stack_check_verbose_log", "config_name": "stack_check_verbose_log", "sdkconfig_defaults_str": null, "dry_run": false, "verbose": false, "check_warnings": true, "preserve": true, "copy_sdkconfig": false, "index": null, "build_status": "build success", "build_comment": null, "cmake_vars": {}, "work_dir": "tools/test_apps/system/startup", "build_dir": "build_esp32s3_stack_check_verbose_log", "build_log_filename": "build_log.txt", "size_json_filename": "size.json"} -{"build_system": "idf_cmake", "app_dir": "tools/test_apps/system/test_watchpoint", "target": "esp32", "sdkconfig_path": null, "config_name": "default", "sdkconfig_defaults_str": null, "dry_run": false, "verbose": false, "check_warnings": true, "preserve": false, "copy_sdkconfig": false, "index": null, "build_status": "skipped", "build_comment": "Skipped due to unmet dependencies", "cmake_vars": {}, "work_dir": "tools/test_apps/system/test_watchpoint", "build_dir": "build_esp32_default", "build_log_filename": "build_log.txt", "size_json_filename": "size.json"} -{"build_system": "idf_cmake", "app_dir": "tools/test_apps/system/test_watchpoint", "target": "esp32c3", "sdkconfig_path": null, "config_name": "default", "sdkconfig_defaults_str": null, "dry_run": false, "verbose": false, "check_warnings": true, "preserve": false, "copy_sdkconfig": false, "index": null, "build_status": "skipped", "build_comment": "Skipped due to unmet dependencies", "cmake_vars": {}, "work_dir": "tools/test_apps/system/test_watchpoint", "build_dir": "build_esp32c3_default", "build_log_filename": "build_log.txt", "size_json_filename": "size.json"} -{"build_system": "idf_cmake", "app_dir": "tools/test_apps/system/unicore_bootloader", "target": "esp32", "sdkconfig_path": "/builds/espressif/esp-idf/tools/test_apps/system/unicore_bootloader/sdkconfig.ci.multicore", "config_name": "multicore", "sdkconfig_defaults_str": null, "dry_run": false, "verbose": false, "check_warnings": true, "preserve": false, "copy_sdkconfig": false, "index": null, "build_status": "build failed", "build_comment": "Compilation error", "cmake_vars": {}, "work_dir": "tools/test_apps/system/unicore_bootloader", "build_dir": "build_esp32_multicore", "build_log_filename": "build_log.txt", "size_json_filename": "size.json"} -{"build_system": "idf_cmake", "app_dir": "tools/test_apps/system/unicore_bootloader", "target": "esp32s3", "sdkconfig_path": "/builds/espressif/esp-idf/tools/test_apps/system/unicore_bootloader/sdkconfig.ci.unicore_psram", "config_name": "unicore_psram", "sdkconfig_defaults_str": null, "dry_run": false, "verbose": false, "check_warnings": true, "preserve": true, "copy_sdkconfig": false, "index": null, "build_status": "build success", "build_comment": null, "cmake_vars": {}, "work_dir": "tools/test_apps/system/unicore_bootloader", "build_dir": "build_esp32s3_unicore_psram", "build_log_filename": "build_log.txt", "size_json_filename": "size.json"} diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/apps_size_info_api_response.json b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/apps_size_info_api_response.json deleted file mode 100644 index 7e24ee8bd5d..00000000000 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/apps_size_info_api_response.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "tools/test_apps/system/panic_coredump_flash_capture_dram_esp32s3": { - "source_commit_id": "bacfa4aa59a37b70b800f1758106fa5f5af99f16", - "target_commit_id": "36d5d8c31c7d3332b43bd5fe4d40b515c6a71097", - "app_name": "panic", - "config_name": "coredump_flash_capture_dram", - "target": "esp32s3", - "metrics": { - "binary_size": { - "source_value": 156936, - "target_value": 162936, - "difference": 6000, - "difference_percentage": 3.82 - } - }, - "app_path": "tools/test_apps/system/panic_coredump" - }, - "tools/test_apps/system/ram_loadable_app_defaults_esp32": { - "source_commit_id": "bacfa4aa59a37b70b800f1758106fa5f5af99f16", - "target_commit_id": "36d5d8c31c7d3332b43bd5fe4d40b515c6a71097", - "app_name": "ram_loadable_app", - "config_name": "defaults", - "target": "esp32", - "metrics": { - "binary_size": { - "source_value": 171448, - "target_value": 173000, - "difference": 1552, - "difference_percentage": 0.91 - } - }, - "app_path": "tools/test_apps/system/ram_loadable_app" - }, - "tools/test_apps/system/ram_loadable_app_pure_ram_esp32": { - "source_commit_id": "bacfa4aa59a37b70b800f1758106fa5f5af99f16", - "target_commit_id": "36d5d8c31c7d3332b43bd5fe4d40b515c6a71097", - "app_name": "ram_loadable_app", - "config_name": "pure_ram", - "target": "esp32", - "metrics": { - "binary_size": { - "source_value": 156632, - "target_value": 158200, - "difference": 1568, - "difference_percentage": 1.0 - } - }, - "app_path": "tools/test_apps/system/ram_loadable_app" - }, - "tools/test_apps/system/startup_flash_80m_qio_esp32": { - "source_commit_id": "bacfa4aa59a37b70b800f1758106fa5f5af99f16", - "target_commit_id": "36d5d8c31c7d3332b43bd5fe4d40b515c6a71097", - "app_name": "startup", - "config_name": "flash_80m_qio", - "target": "esp32", - "metrics": { - "binary_size": { - "source_value": 225692, - "target_value": 230000, - "difference": 4308, - "difference_percentage": 1.91 - } - }, - "app_path": "tools/test_apps/system/startup" - }, - "tools/test_apps/system/startup_stack_check_verbose_log_esp32s3": { - "source_commit_id": "bacfa4aa59a37b70b800f1758106fa5f5af99f16", - "target_commit_id": "36d5d8c31c7d3332b43bd5fe4d40b515c6a71097", - "app_name": "startup", - "config_name": "stack_check_verbose_log", - "target": "esp32s3", - "metrics": { - "binary_size": { - "source_value": 156936, - "target_value": 160000, - "difference": 3064, - "difference_percentage": 1.95 - } - }, - "app_path": "tools/test_apps/system/startup" - }, - "tools/test_apps/system/test_watchpoint_default_esp32": { - "source_commit_id": "bacfa4aa59a37b70b800f1758106fa5f5af99f16", - "target_commit_id": "36d5d8c31c7d3332b43bd5fe4d40b515c6a71097", - "app_name": "test_watchpoint", - "config_name": "default", - "target": "esp32", - "metrics": { - "binary_size": { - "source_value": 147896, - "target_value": 150000, - "difference": 2104, - "difference_percentage": 1.42 - } - }, - "app_path": "tools/test_apps/system/test_watchpoint" - }, - "tools/test_apps/system/test_watchpoint_default_esp32c3": { - "source_commit_id": "bacfa4aa59a37b70b800f1758106fa5f5af99f16", - "target_commit_id": "36d5d8c31c7d3332b43bd5fe4d40b515c6a71097", - "app_name": "test_watchpoint", - "config_name": "default", - "target": "esp32c3", - "metrics": { - "binary_size": { - "source_value": 189456, - "target_value": 190456, - "difference": 1000, - "difference_percentage": 0.53 - } - }, - "app_path": "tools/test_apps/system/test_watchpoint" - }, - "tools/test_apps/system/unicore_bootloader_multicore_esp32": { - "source_commit_id": "bacfa4aa59a37b70b800f1758106fa5f5af99f16", - "target_commit_id": "36d5d8c31c7d3332b43bd5fe4d40b515c6a71097", - "app_name": "unicore_bootloader", - "config_name": "multicore", - "target": "esp32", - "metrics": { - "binary_size": { - "source_value": 216784, - "target_value": 220000, - "difference": 3216, - "difference_percentage": 1.48 - } - }, - "app_path": "tools/test_apps/system/unicore_bootloader" - }, - "tools/test_apps/system/unicore_bootloader_unicore_psram_esp32s3": { - "source_commit_id": "bacfa4aa59a37b70b800f1758106fa5f5af99f16", - "target_commit_id": "36d5d8c31c7d3332b43bd5fe4d40b515c6a71097", - "app_name": "unicore_bootloader", - "config_name": "unicore_psram", - "target": "esp32s3", - "metrics": { - "binary_size": { - "source_value": 189456, - "target_value": 191456, - "difference": 2000, - "difference_percentage": 1.06 - } - }, - "app_path": "tools/test_apps/system/unicore_bootloader" - } -} diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_build_report.html b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_build_report.html deleted file mode 100644 index bcece174302..00000000000 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_build_report.html +++ /dev/null @@ -1,246 +0,0 @@ - - - - - Build Report - - - - - - - -

Failed Apps

- - - - - - - - - - - - - - - - - - - - - - -
App DirBuild DirFailed ReasonBuild Log
tools/test_apps/system/panicbuild_esp32s3_coredump_flash_capture_dramCompilation errorDownload
tools/test_apps/system/unicore_bootloaderbuild_esp32_multicoreCompilation errorDownload

Built Apps - Test Related

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
App DirBuild DirBin Files with Build Log (without map and elf)Map and Elf FilesYour Branch App SizeTarget Branch App SizeSize DiffSize Diff, %
tools/test_apps/system/startupbuild_esp32s3_stack_check_verbose_logDownloadDownload15693616000030641.95
tools/test_apps/system/startupbuild_esp32_flash_80m_qioDownloadDownload22569223000043081.91
tools/test_apps/system/unicore_bootloaderbuild_esp32s3_unicore_psramDownloadDownload18945619145620001.06
tools/test_apps/system/ram_loadable_appbuild_esp32_pure_ramDownloadDownload15663215820015681.0
tools/test_apps/system/ram_loadable_appbuild_esp32_defaultsDownloadDownload17144817300015520.91

Skipped Apps

- - - - - - - - - - - - - - - - - - - - - - -
App DirBuild DirSkipped ReasonBuild Log
tools/test_apps/system/test_watchpointbuild_esp32_defaultSkipped due to unmet dependenciesDownload
tools/test_apps/system/test_watchpointbuild_esp32c3_defaultSkipped due to unmet dependenciesDownload
- - - - - - - - diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_job_report.html b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_job_report.html deleted file mode 100644 index 3c35a534fdd..00000000000 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_job_report.html +++ /dev/null @@ -1,170 +0,0 @@ - - - - - Job Report - - - - - - - -

Failed Jobs (Excludes "integration_test" and "target_test" jobs)

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Job NameFailure ReasonFailure LogFailures across all other branches (10 latest jobs)URLCI Dashboard URL
build_clang_test_apps_esp32h2Some Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure LogSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure Reason2 / 10linklink
build_template_appSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure LogSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure Reason3 / 10linklink
check_public_headersSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure Log4 / 10linklink
- - - - - - - - diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html deleted file mode 100644 index 1e38c58e350..00000000000 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/expected_target_test_report.html +++ /dev/null @@ -1,343 +0,0 @@ - - - - - Test Report - - - - - - - -

Failed Test Cases on Other branches (Excludes Known Failure Cases)

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Test CaseTest App PathFailure ReasonCases that failed in other branches as well (40 latest testcases)Dut Log URLCreate Known Failure Case JiraJob URLGrafana URL
('esp32h2', 'esp32h2').('defaults', 'defaults').test_i2c_multi_devicecomponents/driver/test_apps/i2c_test_appsfailed on setup with "EOFError"0 / 40linkCreateNot foundlink
esp32c3.release.test_esp_timercomponents/esp_timer/test_appspexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): ce710,len:0x2afc entry 0x403cc710 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_esp_timer/dut.txt0 / 40linkCreateNot foundlink
esp32c3.default.test_wpa_supplicant_utcomponents/wpa_supplicant/test_appspexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): 0 d4 000 00x0000 x0000x00 000000 0 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.default.test_wpa_supplicant_ut/dut.txt0 / 40linkCreateNot foundlink
('esp32h2', 'esp32h2').('default', 'default').test_i2s_multi_devcomponents/driver/test_apps/i2s_test_apps/i2s_multi_devfailed on setup with "EOFError"3 / 40linkCreateNot foundlink
esp32c2.default.test_wpa_supplicant_utcomponents/wpa_supplicant/test_appsAssertionError: Unity test failed3 / 40linkCreateNot foundlink
esp32c3.512safe.test_wear_levellingcomponents/wear_levelling/test_appspexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?POK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 6673 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.512safe.test_wear_levelling/dut.txt3 / 40linkCreateNot foundlink
esp32c3.release.test_wear_levellingcomponents/wear_levelling/test_appspexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?POK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 24528 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_wear_levelling/dut.txt3 / 40linkCreateNot foundlink

Known Failure Cases

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Test CaseTest App PathFailure ReasonJob URLGrafana URL
esp32c2.default.test_wpa_supplicant_utcomponents/wpa_supplicant/test_appsAssertionError: Unity test failedNot foundlink
esp32c3.release.test_esp_timercomponents/esp_timer/test_appspexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): ce710,len:0x2afc entry 0x403cc710 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.release.test_esp_timer/dut.txtNot foundlink
esp32c3.512safe.test_wear_levellingcomponents/wear_levelling/test_appspexpect.exceptions.TIMEOUT: Not found "re.compile(b'^[-]+\\s*(\\d+) Tests (\\d+) Failures (\\d+) Ignored\\s*(?POK|FAIL)', re.MULTILINE)" Bytes in current buffer (color code eliminated): Serial port /dev/ttyUSB16 Connecting.... Connecting.... esptool.py v4.7.0 Found 1 serial ports Chip is ESP32-C3 (QFN32) (revision v0.3) Features: WiFi, BLE, Embedded Flash 4MB... (total 6673 bytes) Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.512safe.test_wear_levelling/dut.txtNot foundlink
esp32c3.default.test_wpa_supplicant_utcomponents/wpa_supplicant/test_appspexpect.exceptions.TIMEOUT: Not found "Press ENTER to see the list of tests" Bytes in current buffer (color code eliminated): 0 d4 000 00x0000 x0000x00 000000 0 Please check the full log here: /builds/espressif/esp-idf/pytest_embedded/2024-05-17_17-50-04/esp32c3.default.test_wpa_supplicant_ut/dut.txtNot foundlink

Skipped Test Cases

- - - - - - - - - - - - - - - - -
Test CaseTest App PathSkipped ReasonGrafana URL
test_python_interpreter_wintoolsLinux does not support executing .exe fileslink

Succeeded Test Cases

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Test CaseTest App PathJob URLGrafana URL
esp32c2.default.test_vfs_defaultcomponents/vfs/test_appsNot foundlink
esp32c2.iram.test_vfs_defaultcomponents/vfs/test_appsNot foundlink
test_python_interpreter_unixtoolsNot foundlink
test_invoke_confservertoolsNot foundlink
test_ccache_used_to_buildtoolsNot foundlink
test_toolchain_prefix_in_description_filetoolsNot foundlink
test_subcommands_with_optionstoolsNot foundlink
test_fallback_to_build_system_targettoolsNot foundlink
test_create_component_projecttoolsNot foundlink
- - - - - - - - diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/failure_rate_jobs_response.json b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/failure_rate_jobs_response.json deleted file mode 100644 index 18e4bea1e21..00000000000 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/failure_rate_jobs_response.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "jobs": [ - {"failed_count": 2, "failure_ratio": 0.2, "total_count": 10, "name": "build_clang_test_apps_esp32h2"}, - {"failed_count": 3, "failure_ratio": 0.3, "total_count": 10, "name": "build_template_app"}, - {"failed_count": 4, "failure_ratio": 0.4, "total_count": 10, "name": "check_public_headers"} - ] -} diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/jobs_api_response.json b/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/jobs_api_response.json deleted file mode 100644 index 844e2545b59..00000000000 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/reports_sample_data/jobs_api_response.json +++ /dev/null @@ -1,212 +0,0 @@ -{ - "jobs": [ - { - "duration_sec": 42.158688, - "failure_log": null, - "failure_reason": null, - "finished_at": "03:38:16, 24 May 2024", - "id": 48838677, - "name": "check_pre_commit", - "pending_sec": 1.15148, - "runner_name": "FA002598-build", - "stage": "pre_check", - "status": "success", - "url": "https://test.com/-/jobs/48838677" - }, - { - "duration_sec": 35.438477, - "failure_log": null, - "failure_reason": null, - "finished_at": "03:37:32, 24 May 2024", - "id": 48838675, - "name": "run-danger-mr-linter", - "pending_sec": 0.371668, - "runner_name": "BrnoVM0211", - "stage": "pre_check", - "status": "success", - "url": "https://test.com/-/jobs/48838675" - }, - { - "duration_sec": 30.202475, - "failure_log": null, - "failure_reason": null, - "finished_at": "03:37:28, 24 May 2024", - "id": 48838682, - "name": "check_esp_system", - "pending_sec": 1.148756, - "runner_name": "ruby6-cent9 [32]", - "stage": "pre_check", - "status": "success", - "url": "https://test.com/-/jobs/48838682" - }, - { - "duration_sec": 33.75121, - "failure_log": null, - "failure_reason": null, - "finished_at": "03:37:31, 24 May 2024", - "id": 48838679, - "name": "check_blobs", - "pending_sec": 0.725292, - "runner_name": "gem3-cent9 [32]", - "stage": "pre_check", - "status": "success", - "url": "https://test.com/-/jobs/48838679" - }, - { - "duration_sec": 121.84324, - "failure_log": null, - "failure_reason": null, - "finished_at": "03:41:35, 24 May 2024", - "id": 48838687, - "name": "code_quality_check", - "pending_sec": 0.271973, - "runner_name": "ruby15-cent9 [32]", - "stage": "build", - "status": "success", - "url": "https://test.com/-/jobs/48838687" - }, - { - "duration_sec": 153.68849, - "failure_log": null, - "failure_reason": null, - "finished_at": "03:39:32, 24 May 2024", - "id": 48838686, - "name": "fast_template_app", - "pending_sec": 2.319577, - "runner_name": "FA002598-build", - "stage": "pre_check", - "status": "success", - "url": "https://test.com/-/jobs/48838686" - }, - { - "duration_sec": 25.572954, - "failure_log": null, - "failure_reason": null, - "finished_at": "03:37:23, 24 May 2024", - "id": 48838684, - "name": "check_configure_ci_environment_parsing", - "pending_sec": 1.184287, - "runner_name": "gem3-cent9 [32]", - "stage": "pre_check", - "status": "success", - "url": "https://test.com/-/jobs/48838684" - }, - { - "duration_sec": 120.95287, - "failure_log": null, - "failure_reason": null, - "finished_at": "03:41:34, 24 May 2024", - "id": 48838690, - "name": "build_clang_test_apps_esp32s3", - "pending_sec": 0.671956, - "runner_name": "ruby7-cent9 [32]", - "stage": "build", - "status": "success", - "url": "https://test.com/-/jobs/48838690" - }, - { - "duration_sec": 165.74513, - "failure_log": null, - "failure_reason": null, - "finished_at": "03:42:19, 24 May 2024", - "id": 48838692, - "name": "build_clang_test_apps_esp32c2", - "pending_sec": 0.82007, - "runner_name": "PowerfulBuildRunner03 [16]", - "stage": "build", - "status": "success", - "url": "https://test.com/-/jobs/48838692" - }, - { - "duration_sec": 95.72326, - "failure_log": null, - "failure_reason": null, - "finished_at": "03:41:09, 24 May 2024", - "id": 48838696, - "name": "build_clang_test_apps_esp32p4", - "pending_sec": 0.567116, - "runner_name": "gem3-cent9 [32]", - "stage": "build", - "status": "success", - "url": "https://test.com/-/jobs/48838696" - }, - { - "duration_sec": 122.19848, - "failure_log": null, - "failure_reason": null, - "finished_at": "03:41:36, 24 May 2024", - "id": 48838691, - "name": "build_clang_test_apps_esp32c3", - "pending_sec": 0.709112, - "runner_name": "ruby6-cent9 [32]", - "stage": "build", - "status": "success", - "url": "https://test.com/-/jobs/48838691" - }, - { - "duration_sec": 148.09895, - "failure_log": null, - "failure_reason": null, - "finished_at": "03:42:02, 24 May 2024", - "id": 48838694, - "name": "build_clang_test_apps_esp32c5", - "pending_sec": 0.779584, - "runner_name": "PowerfulBuildRunner04 [15]", - "stage": "build", - "status": "success", - "url": "https://test.com/-/jobs/48838694" - }, - { - "duration_sec": 20.275927, - "failure_log": null, - "failure_reason": null, - "finished_at": "03:39:54, 24 May 2024", - "id": 48838699, - "name": "gen_integration_pipeline", - "pending_sec": 0.868898, - "runner_name": "FA002598-build", - "stage": "assign_test", - "status": "success", - "url": "https://test.com/-/jobs/48838699" - }, - { - "duration_sec": 103.08849, - "failure_log": "Some Failure LogSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure Reason", - "failure_reason": "Some Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure Reason", - "finished_at": "03:41:17, 24 May 2024", - "id": 48838695, - "name": "build_clang_test_apps_esp32h2", - "pending_sec": 0.765111, - "runner_name": "gem2-cent9 [32]", - "stage": "build", - "status": "failed", - "url": "https://test.com/-/jobs/48838695" - }, - { - "duration_sec": 634.59467, - "failure_log": "Some Failure LogSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure Reason", - "failure_reason": "Some Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure Reason", - "finished_at": "03:50:09, 24 May 2024", - "id": 48838704, - "name": "build_template_app", - "pending_sec": 0.161796, - "runner_name": "ruby6-cent9 [32]", - "stage": "host_test", - "status": "failed", - "url": "https://test.com/-/jobs/48838704" - }, - { - "duration_sec": 1060.0835, - "failure_log": "Some Failure Log", - "failure_reason": "Some Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure ReasonSome Failure Reason", - "finished_at": "03:55:14, 24 May 2024", - "id": 48838705, - "name": "check_public_headers", - "pending_sec": 0.449408, - "runner_name": "ruby6-cent9 [32]", - "stage": "host_test", - "status": "failed", - "url": "https://test.com/-/jobs/48838705" - } - ] -} diff --git a/tools/ci/dynamic_pipelines/tests/test_report_generator/test_report_generator.py b/tools/ci/dynamic_pipelines/tests/test_report_generator/test_report_generator.py deleted file mode 100644 index c1782d9377a..00000000000 --- a/tools/ci/dynamic_pipelines/tests/test_report_generator/test_report_generator.py +++ /dev/null @@ -1,328 +0,0 @@ -#!/usr/bin/env python -# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import json -import os.path -import sys -import unittest -from unittest.mock import MagicMock -from unittest.mock import patch - -sys.path.insert(0, os.path.join(f'{os.environ.get("IDF_PATH")}', 'tools', 'ci', 'python_packages')) -sys.path.insert(0, os.path.join(f'{os.environ.get("IDF_PATH")}', 'tools', 'ci')) - -from idf_build_apps.constants import BuildStatus # noqa: E402 -from idf_ci_local.app import enrich_apps_with_metrics_info # noqa: E402 -from idf_ci_local.app import import_apps_from_txt # noqa: E402 - -from dynamic_pipelines.models import GitlabJob # noqa: E402 -from dynamic_pipelines.report import BuildReportGenerator # noqa: E402 -from dynamic_pipelines.report import JobReportGenerator # noqa: E402 -from dynamic_pipelines.report import TargetTestReportGenerator # noqa: E402 -from dynamic_pipelines.utils import load_file # noqa: E402 -from dynamic_pipelines.utils import parse_testcases_from_filepattern # noqa: E402 - - -class TestReportGeneration(unittest.TestCase): - def setUp(self) -> None: - self.reports_sample_data_path = os.path.join( - os.environ.get('IDF_PATH', ''), - 'tools', - 'ci', - 'dynamic_pipelines', - 'tests', - 'test_report_generator', - 'reports_sample_data', - ) - self.setup_patches() - self.load_test_and_job_reports() - self.create_report_generators() - - def setup_patches(self) -> None: - self.gitlab_patcher = patch('dynamic_pipelines.report.Gitlab') - self.uploader_patcher = patch('dynamic_pipelines.report.AppUploader') - self.failure_rate_patcher = patch('dynamic_pipelines.report.fetch_failed_testcases_failure_ratio') - self.env_patcher = patch.dict( - 'os.environ', - { - 'CI_DASHBOARD_HOST': 'https://test_dashboard_host', - 'CI_PAGES_URL': 'https://artifacts_path', - 'CI_JOB_ID': '1', - 'JIRA_SERVER': 'https://jira.com', - }, - ) - self.yaml_dump_patcher = patch('dynamic_pipelines.report.yaml.dump') - - self.MockGitlab = self.gitlab_patcher.start() - self.MockUploader = self.uploader_patcher.start() - self.test_cases_failure_rate = self.failure_rate_patcher.start() - self.env_patcher.start() - self.yaml_dump_patcher.start() - - self.mock_project = MagicMock() - self.mock_mr = MagicMock() - self.MockGitlab.return_value.project = self.mock_project - self.mock_project.mergerequests.get.return_value = self.mock_mr - self.MockUploader.return_value.get_app_presigned_url.return_value = 'https://example.com/presigned-url' - - self.addCleanup(self.gitlab_patcher.stop) - self.addCleanup(self.uploader_patcher.stop) - self.addCleanup(self.failure_rate_patcher.stop) - self.addCleanup(self.env_patcher.stop) - self.addCleanup(self.yaml_dump_patcher.stop) - self.addCleanup(self.cleanup_files) - - def cleanup_files(self) -> None: - files_to_delete = [ - self.target_test_report_generator.skipped_test_cases_report_file, - self.target_test_report_generator.succeeded_cases_report_file, - self.target_test_report_generator.failed_cases_report_file, - self.build_report_generator.failed_apps_report_file, - self.build_report_generator.built_apps_report_file, - self.build_report_generator.skipped_apps_report_file, - self.build_report_generator.apps_presigned_url_filepath, - ] - for file_path in files_to_delete: - if os.path.exists(file_path): - os.remove(file_path) - - def load_test_and_job_reports(self) -> None: - self.expected_target_test_report_html = load_file( - os.path.join(self.reports_sample_data_path, 'expected_target_test_report.html') - ) - self.expected_job_report_html = load_file( - os.path.join(self.reports_sample_data_path, 'expected_job_report.html') - ) - self.expected_build_report_html = load_file( - os.path.join(self.reports_sample_data_path, 'expected_build_report.html') - ) - - def create_report_generators(self) -> None: - jobs_response_raw = load_file(os.path.join(self.reports_sample_data_path, 'jobs_api_response.json')) - failure_rate_jobs_response = load_file( - os.path.join(self.reports_sample_data_path, 'failure_rate_jobs_response.json') - ) - built_apps_size_info_response = json.loads( - load_file(os.path.join(self.reports_sample_data_path, 'apps_size_info_api_response.json')) - ) - failure_rates = {item['name']: item for item in json.loads(failure_rate_jobs_response).get('jobs', [])} - jobs = [ - GitlabJob.from_json_data(job_json, failure_rates.get(job_json['name'], {})) - for job_json in json.loads(jobs_response_raw)['jobs'] - ] - test_cases = parse_testcases_from_filepattern(os.path.join(self.reports_sample_data_path, 'XUNIT_*.xml')) - apps = enrich_apps_with_metrics_info( - built_apps_size_info_response, import_apps_from_txt(os.path.join(self.reports_sample_data_path, 'apps')) - ) - self.target_test_report_generator = TargetTestReportGenerator( - project_id=123, - mr_iid=1, - pipeline_id=456, - job_id=0, - commit_id='cccc', - title='Test Report', - test_cases=test_cases, - ) - self.job_report_generator = JobReportGenerator( - project_id=123, mr_iid=1, pipeline_id=456, job_id=0, commit_id='cccc', title='Job Report', jobs=jobs - ) - self.build_report_generator = BuildReportGenerator( - project_id=123, mr_iid=1, pipeline_id=456, job_id=0, commit_id='cccc', title='Build Report', apps=apps - ) - self.target_test_report_generator._known_failure_cases_set = { - '*.test_wpa_supplicant_ut', - 'esp32c3.release.test_esp_timer', - '*.512safe.test_wear_levelling', - } - test_cases_failed = [tc for tc in test_cases if tc.is_failure] - for index, tc in enumerate(test_cases_failed): - tc.latest_total_count = 40 - if index % 3 == 0: - tc.latest_failed_count = 0 - else: - tc.latest_failed_count = 3 - self.test_cases_failure_rate.return_value = test_cases_failed - - def test_known_failure_cases(self) -> None: - known_failure_cases = self.target_test_report_generator.get_known_failure_cases() - self.assertEqual(len(known_failure_cases), 4) - - def test_failed_cases_in_target_test_report(self) -> None: - known_failures = self.target_test_report_generator.get_known_failure_cases() - known_failure_case_names = {case.name for case in known_failures} - failed_testcases = self.target_test_report_generator._filter_items( - self.target_test_report_generator.test_cases, - lambda tc: tc.is_failure and tc.name not in known_failure_case_names, - ) - self.assertEqual(len(failed_testcases), 3) - - def test_skipped_cases_in_target_test_report(self) -> None: - skipped_testcases = self.target_test_report_generator._filter_items( - self.target_test_report_generator.test_cases, lambda tc: tc.is_skipped - ) - self.assertEqual(len(skipped_testcases), 1) - - def test_successful_cases_in_target_test_report(self) -> None: - succeeded_testcases = self.target_test_report_generator._filter_items( - self.target_test_report_generator.test_cases, lambda tc: tc.is_success - ) - self.assertEqual(len(succeeded_testcases), 9) - - def test_target_test_report_html_structure(self) -> None: - report = self.target_test_report_generator._get_report_str() - self.assertEqual(report, self.expected_target_test_report_html) - - def test_failed_jobs_in_job_report(self) -> None: - failed_jobs = self.job_report_generator._filter_items(self.job_report_generator.jobs, lambda job: job.is_failed) - self.assertEqual(len(failed_jobs), 3) - - def test_successful_jobs_in_job_report(self) -> None: - succeeded_jobs = self.job_report_generator._filter_items( - self.job_report_generator.jobs, lambda job: job.is_success - ) - self.assertEqual(len(succeeded_jobs), 13) - - def test_job_report_html_structure(self) -> None: - report = self.job_report_generator._get_report_str() - self.assertEqual(report, self.expected_job_report_html) - - def test_generate_top_n_apps_by_size_table(self) -> None: - apps_with_size_diff = [ - MagicMock( - app_dir=f'app_dir_{i}', - build_dir=f'build_dir_{i}', - build_status=BuildStatus.SUCCESS, - metrics={ - 'binary_size': MagicMock( - source=i * 10000, - target=i * 10000 + i * 1000, - difference=i * 1000, - difference_percentage=i * 0.5, - ) - }, - ) - for i in range(1, 6) - ] - build_report_generator = BuildReportGenerator( - project_id=123, - mr_iid=1, - pipeline_id=456, - job_id=0, - commit_id='cccc', - title='Build Report', - apps=apps_with_size_diff, - ) - - top_apps_table = build_report_generator._generate_top_n_apps_by_size_table() - - self.assertIn('| App Dir | Build Dir | Size Diff (bytes) | Size Diff (%) |', top_apps_table) - self.assertIn('| app_dir_5 | build_dir_5 | 5000 | 2.5% |', top_apps_table) - self.assertIn('| app_dir_1 | build_dir_1 | 1000 | 0.5% |', top_apps_table) - - def test_get_built_apps_report_parts(self) -> None: - apps = [ - MagicMock( - app_dir='test_app_1', - build_dir='build_dir_1', - size_difference=1000, - size_difference_percentage=1.0, - build_status=BuildStatus.SUCCESS, - preserve=True, - metrics={'binary_size': MagicMock(difference=1000, difference_percentage=1.0)}, - ), - MagicMock( - app_dir='test_app_2', - build_dir='build_dir_2', - size_difference=2000, - size_difference_percentage=2.0, - build_status=BuildStatus.SUCCESS, - preserve=False, - metrics={'binary_size': MagicMock(difference=2000, difference_percentage=2.0)}, - ), - ] - - build_report_generator = BuildReportGenerator( - project_id=123, mr_iid=1, pipeline_id=456, job_id=0, commit_id='cccc', title='Build Report', apps=apps - ) - - built_apps_report_parts = build_report_generator.get_built_apps_report_parts() - - self.assertGreater(len(built_apps_report_parts), 0) - self.assertIn('test_app_1', ''.join(built_apps_report_parts)) - self.assertIn('test_app_2', ''.join(built_apps_report_parts)) - - def test_get_failed_apps_report_parts(self) -> None: - failed_apps = [ - MagicMock( - app_dir='failed_app_1', - build_dir='build_dir_1', - build_comment='Compilation error', - build_status=BuildStatus.FAILED, - metrics={'binary_size': MagicMock(difference=None, difference_percentage=None)}, - ), - MagicMock( - app_dir='failed_app_2', - build_dir='build_dir_2', - build_comment='Linker error', - build_status=BuildStatus.FAILED, - metrics={'binary_size': MagicMock(difference=None, difference_percentage=None)}, - ), - ] - - build_report_generator = BuildReportGenerator( - project_id=123, - mr_iid=1, - pipeline_id=456, - job_id=0, - commit_id='cccc', - title='Build Report', - apps=failed_apps, - ) - - failed_apps_report_parts = build_report_generator.get_failed_apps_report_parts() - - self.assertGreater(len(failed_apps_report_parts), 0) - self.assertIn('failed_app_1', ''.join(failed_apps_report_parts)) - self.assertIn('failed_app_2', ''.join(failed_apps_report_parts)) - - def test_get_skipped_apps_report_parts(self) -> None: - skipped_apps = [ - MagicMock( - app_dir='skipped_app_1', - build_dir='build_dir_1', - build_comment='Dependencies unmet', - build_status=BuildStatus.SKIPPED, - metrics={'binary_size': MagicMock(difference=None, difference_percentage=None)}, - ), - MagicMock( - app_dir='skipped_app_2', - build_dir='build_dir_2', - build_comment='Feature flag disabled', - build_status=BuildStatus.SKIPPED, - metrics={'binary_size': MagicMock(difference=None, difference_percentage=None)}, - ), - ] - - build_report_generator = BuildReportGenerator( - project_id=123, - mr_iid=1, - pipeline_id=456, - job_id=0, - commit_id='cccc', - title='Build Report', - apps=skipped_apps, - ) - - skipped_apps_report_parts = build_report_generator.get_skipped_apps_report_parts() - - self.assertGreater(len(skipped_apps_report_parts), 0) - self.assertIn('skipped_app_1', ''.join(skipped_apps_report_parts)) - self.assertIn('skipped_app_2', ''.join(skipped_apps_report_parts)) - - def test_build_report_html_structure(self) -> None: - report = self.build_report_generator._get_report_str() - self.assertEqual(report, self.expected_build_report_html) - - -if __name__ == '__main__': - unittest.main() diff --git a/tools/ci/dynamic_pipelines/utils.py b/tools/ci/dynamic_pipelines/utils.py index 69489bd7cc7..a4344ab3bc6 100644 --- a/tools/ci/dynamic_pipelines/utils.py +++ b/tools/ci/dynamic_pipelines/utils.py @@ -1,16 +1,9 @@ # SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 -import glob import os -import re -import typing as t -import xml.etree.ElementTree as ET -from urllib.parse import quote -from urllib.parse import urlencode from urllib.parse import urlparse import requests -import yaml from .constants import CI_DASHBOARD_API from .constants import CI_JOB_TOKEN @@ -18,75 +11,6 @@ from .constants import CI_MERGE_REQUEST_SOURCE_BRANCH_SHA from .constants import CI_PAGES_URL from .constants import CI_PROJECT_URL from .models import GitlabJob -from .models import Job -from .models import TestCase - - -def dump_jobs_to_yaml( - jobs: t.List[Job], - output_filepath: str, - pipeline_name: str, - extra_include_yml: t.Optional[t.List[str]] = None, -) -> None: - yaml_dict = {} - for job in jobs: - yaml_dict.update(job.to_dict()) - - # global stuffs - yaml_dict.update( - { - 'include': [ - 'tools/ci/dynamic_pipelines/templates/.dynamic_jobs.yml', - '.gitlab/ci/common.yml', - ], - 'workflow': { - 'name': pipeline_name, - 'rules': [ - # always run the child pipeline, if they are created - {'when': 'always'}, - ], - }, - } - ) - yaml_dict['include'].extend(extra_include_yml or []) - - with open(output_filepath, 'w') as fw: - yaml.dump(yaml_dict, fw, indent=2) - - -def parse_testcases_from_filepattern(junit_report_filepattern: str) -> t.List[TestCase]: - """ - Parses test cases from XML files matching the provided file pattern. - - >>> test_cases = parse_testcases_from_filepattern("path/to/your/junit/reports/*.xml") - - :param junit_report_filepattern: The file pattern to match XML files containing JUnit test reports. - :return: List[TestCase]: A list of TestCase objects parsed from the XML files. - """ - - test_cases = [] - for f in glob.glob(junit_report_filepattern): - root = ET.parse(f).getroot() - for tc in root.findall('.//testcase'): - test_cases.append(TestCase.from_test_case_node(tc)) - return test_cases - - -def load_known_failure_cases() -> t.Optional[t.Set[str]]: - known_failures_file = os.getenv('KNOWN_FAILURE_CASES_FILE_NAME', '') - if not known_failures_file: - return None - try: - with open(known_failures_file, 'r') as f: - file_content = f.read() - - pattern = re.compile(r'^(.*?)\s+#\s+([A-Z]+)-\d+', re.MULTILINE) - matches = pattern.findall(file_content) - - known_cases_list = [match[0].strip() for match in matches] - return set(known_cases_list) - except FileNotFoundError: - return None def is_url(string: str) -> bool: @@ -100,7 +24,7 @@ def is_url(string: str) -> bool: return bool(parsed.scheme) and bool(parsed.netloc) -def fetch_failed_jobs(commit_id: str) -> t.List[GitlabJob]: +def fetch_failed_jobs(commit_id: str) -> list[GitlabJob]: """ Fetches a list of jobs from the specified commit_id using an API request to ci-dashboard-api. :param commit_id: The commit ID for which to fetch jobs. @@ -124,7 +48,10 @@ def fetch_failed_jobs(commit_id: str) -> t.List[GitlabJob]: response = requests.post( f'{CI_DASHBOARD_API}/jobs/failure_ratio', headers={'CI-Job-Token': CI_JOB_TOKEN}, - json={'job_names': failed_job_names, 'exclude_branches': [os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_NAME', '')]}, + json={ + 'job_names': failed_job_names, + 'exclude_branches': [os.getenv('CI_MERGE_REQUEST_SOURCE_BRANCH_NAME', '')], + }, ) if response.status_code != 200: print(f'Failed to fetch jobs failure rate data: {response.status_code} with error: {response.text}') @@ -141,43 +68,17 @@ def fetch_failed_jobs(commit_id: str) -> t.List[GitlabJob]: return combined_jobs -def fetch_failed_testcases_failure_ratio(failed_testcases: t.List[TestCase], branches_filter: dict) -> t.List[TestCase]: - """ - Fetches info about failure rates of testcases using an API request to ci-dashboard-api. - :param failed_testcases: The list of failed testcases models. - :param branches_filter: The filter to filter testcases by branch names. - :return: A list of testcases with enriched with failure rates data. - """ - req_json = {'testcase_names': list(set([testcase.name for testcase in failed_testcases])), **branches_filter} - response = requests.post( - f'{CI_DASHBOARD_API}/testcases/failure_ratio', - headers={'CI-Job-Token': CI_JOB_TOKEN}, - json=req_json, - ) - if response.status_code != 200: - print(f'Failed to fetch testcases failure rate data: {response.status_code} with error: {response.text}') - return [] - - failure_rate_data = response.json() - failure_rates = {item['name']: item for item in failure_rate_data.get('testcases', [])} - - for testcase in failed_testcases: - testcase.latest_total_count = failure_rates.get(testcase.name, {}).get('total_count', 0) - testcase.latest_failed_count = failure_rates.get(testcase.name, {}).get('failed_count', 0) - - return failed_testcases - - def fetch_app_metrics( source_commit_sha: str, target_commit_sha: str, -) -> t.Dict: +) -> dict: """ Fetches the app metrics for the given source commit SHA and target branch SHA. :param source_commit_sha: The source commit SHA. :param target_branch_sha: The commit SHA of the branch to compare app sizes against. :return: A dict of sizes of built binaries. """ + print(f'Fetching bin size info: {source_commit_sha=} {target_commit_sha=}') build_info_map = dict() response = requests.post( f'{CI_DASHBOARD_API}/apps/metrics', @@ -185,15 +86,14 @@ def fetch_app_metrics( json={ 'source_commit_sha': source_commit_sha, 'target_commit_sha': target_commit_sha, - } + }, ) if response.status_code != 200: print(f'Failed to fetch build info: {response.status_code} - {response.text}') else: response_data = response.json() build_info_map = { - f"{info['app_path']}_{info['config_name']}_{info['target']}": info - for info in response_data.get('data', []) + f'{info["app_path"]}_{info["config_name"]}_{info["target"]}': info for info in response_data.get('data', []) } return build_info_map @@ -206,7 +106,7 @@ def load_file(file_path: str) -> str: :param file_path: The path to the file needs to be loaded. :return: The content of the file as a string. """ - with open(file_path, 'r') as file: + with open(file_path) as file: return file.read() @@ -249,34 +149,3 @@ def get_repository_file_url(file_path: str) -> str: :return: The modified URL pointing to the file's path in the repository. """ return f'{CI_PROJECT_URL}/-/raw/{CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}/{file_path}' - - -def known_failure_issue_jira_fast_link(_item: TestCase) -> str: - """ - Generate a JIRA fast link for known issues with relevant test case details. - """ - jira_url = os.getenv('JIRA_SERVER') - jira_pid = os.getenv('JIRA_KNOWN_FAILURE_PID', '10514') - jira_issuetype = os.getenv('JIRA_KNOWN_FAILURE_ISSUETYPE', '10004') - jira_component = os.getenv('JIRA_KNOWN_FAILURE_COMPONENT', '11909') - jira_assignee = os.getenv('JIRA_KNOWN_FAILURE_ASSIGNEE', 'zhangjianwen') - jira_affected_versions = os.getenv('JIRA_KNOWN_FAILURE_VERSIONS', '17602') - jira_priority = os.getenv('JIRA_KNOWN_FAILURE_PRIORITY', '3') - - base_url = f'{jira_url}/secure/CreateIssueDetails!init.jspa?' - params = { - 'pid': jira_pid, - 'issuetype': jira_issuetype, - 'summary': f'[Test Case]{_item.name}', - 'description': ( - f"job_url: {quote(_item.ci_job_url, safe=':/')}\n\n" - f"dut_log_url: {quote(_item.dut_log_url, safe=':/')}\n\n" - f'ci_dashboard_url: {_item.ci_dashboard_url}\n\n' - ), - 'components': jira_component, - 'priority': jira_priority, - 'assignee': jira_assignee, - 'versions': jira_affected_versions - } - query_string = urlencode(params) - return f'Create' diff --git a/tools/ci/exclude_check_tools_files.txt b/tools/ci/exclude_check_tools_files.txt index 6dac35455d0..e5cac7ca1b6 100644 --- a/tools/ci/exclude_check_tools_files.txt +++ b/tools/ci/exclude_check_tools_files.txt @@ -39,8 +39,6 @@ tools/templates/sample_component/CMakeLists.txt tools/templates/sample_component/include/main.h tools/templates/sample_component/main.c tools/ci/cleanup_ignore_lists.py -tools/ci/artifacts_handler.py -tools/ci/get_known_failure_cases_file.py tools/unit-test-app/**/* tools/ci/gitlab_yaml_linter.py tools/ci/dynamic_pipelines/**/* diff --git a/tools/ci/generate_rules.py b/tools/ci/generate_rules.py index 4dc9534039c..f9257eaf093 100755 --- a/tools/ci/generate_rules.py +++ b/tools/ci/generate_rules.py @@ -1,30 +1,24 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2021-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import argparse import inspect import os import sys +import typing as t from collections import defaultdict from itertools import product import yaml -from idf_ci_utils import GitlabYmlConfig from idf_ci_utils import IDF_PATH +from idf_ci_utils import GitlabYmlConfig -try: +if t.TYPE_CHECKING: import pygraphviz as pgv -except ImportError: # used when pre-commit, skip generating image - pass - -try: - from typing import Union -except ImportError: # used for type hint - pass -def _list(str_or_list): # type: (Union[str, list]) -> list +def _list(str_or_list: t.Union[str, t.List]) -> t.List: if isinstance(str_or_list, str): return [str_or_list] elif isinstance(str_or_list, list): @@ -33,7 +27,7 @@ def _list(str_or_list): # type: (Union[str, list]) -> list raise ValueError('Wrong type: {}. Only supports str or list.'.format(type(str_or_list))) -def _format_nested_dict(_dict, f_tuple): # type: (dict[str, dict], tuple[str, ...]) -> dict[str, dict] +def _format_nested_dict(_dict: t.Dict[str, t.Dict], f_tuple: t.Tuple[str, ...]) -> t.Dict[str, t.Dict]: res = {} for k, v in _dict.items(): k = k.split('__')[0] @@ -47,7 +41,7 @@ def _format_nested_dict(_dict, f_tuple): # type: (dict[str, dict], tuple[str, . return res -def _format_nested_list(_list, f_tuple): # type: (list[str], tuple[str, ...]) -> list[str] +def _format_nested_list(_list: t.List[str], f_tuple: t.Tuple[str, ...]) -> t.List[str]: res = [] for item in _list: if isinstance(item, list): @@ -61,26 +55,23 @@ def _format_nested_list(_list, f_tuple): # type: (list[str], tuple[str, ...]) - class RulesWriter: - AUTO_GENERATE_MARKER = inspect.cleandoc(r''' + AUTO_GENERATE_MARKER = inspect.cleandoc(r""" ################## # Auto Generated # ################## - ''') + """) - LABEL_TEMPLATE = inspect.cleandoc(r''' + LABEL_TEMPLATE = inspect.cleandoc(r""" .if-label-{0}: &if-label-{0} if: '$BOT_LABEL_{1} || $CI_MERGE_REQUEST_LABELS =~ /^(?:[^,\n\r]+,)*{0}(?:,[^,\n\r]+)*$/i' - ''') + """) - RULE_PROTECTED = ' - <<: *if-protected' - RULE_PROTECTED_NO_LABEL = ' - <<: *if-protected-no_label' - RULE_BUILD_ONLY = ' - <<: *if-label-build-only\n' \ - ' when: never' - RULE_REVERT_BRANCH = ' - <<: *if-revert-branch\n' \ - ' when: never' + RULE_PROTECTED_CHECK = ' - <<: *if-protected-check' + RULE_PROTECTED_PUSH = ' - <<: *if-protected-push' + RULE_BUILD_ONLY = ' - <<: *if-label-build-only\n when: never' + RULE_REVERT_BRANCH = ' - <<: *if-revert-branch\n when: never' RULE_LABEL_TEMPLATE = ' - <<: *if-label-{0}' - RULE_PATTERN_TEMPLATE = ' - <<: *if-dev-push\n' \ - ' changes: *patterns-{0}' + RULE_PATTERN_TEMPLATE = ' - <<: *if-dev-push\n changes: *patterns-{0}' SPECIFIC_RULE_TEMPLATE = ' - <<: *{0}' RULES_TEMPLATE = inspect.cleandoc(r""" .rules:{0}: @@ -120,7 +111,7 @@ class RulesWriter: return res @staticmethod - def _expand_matrix(name, cfg): # type: (str, dict) -> dict + def _expand_matrix(name: str, cfg: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]: """ Expand matrix into multi keys :param cfg: single rule dict @@ -138,7 +129,7 @@ class RulesWriter: res.update(_format_nested_dict(default, comb)) return res - def expand_rules(self): # type: () -> dict[str, dict[str, list]] + def expand_rules(self) -> t.Dict[str, t.Dict[str, t.List[str]]]: res = defaultdict(lambda: defaultdict(set)) # type: dict[str, dict[str, set]] for k, v in self.cfg.items(): if not v: @@ -169,13 +160,13 @@ class RulesWriter: continue res[item]['patterns'].add(_pat) - sorted_res = defaultdict(lambda: defaultdict(list)) # type: dict[str, dict[str, list]] + sorted_res = defaultdict(lambda: defaultdict(list)) # type: t.Dict[str, t.Dict[str, t.List[str]]] for k, v in res.items(): for vk, vv in v.items(): sorted_res[k][vk] = sorted(vv) return sorted_res - def new_labels_str(self): # type: () -> str + def new_labels_str(self) -> str: _labels = set([]) for k, v in self.cfg.items(): if not v: @@ -191,14 +182,14 @@ class RulesWriter: return res @classmethod - def _format_label(cls, label): # type: (str) -> str + def _format_label(cls, label: str) -> str: return cls.LABEL_TEMPLATE.format(label, cls.bot_label_str(label)) @staticmethod - def bot_label_str(label): # type: (str) -> str + def bot_label_str(label: str) -> str: return label.upper().replace('-', '_') - def new_rules_str(self): # type: () -> str + def new_rules_str(self) -> str: res = [] for k, v in sorted(self.rules.items()): if k.startswith('pattern'): @@ -211,13 +202,13 @@ class RulesWriter: res.append(self.RULES_TEMPLATE.format(k, self._format_rule(k, v))) return '\n\n'.join(res) - def _format_rule(self, name, cfg): # type: (str, dict) -> str + def _format_rule(self, name: str, cfg: t.Dict[str, t.Any]) -> str: _rules = [self.RULE_REVERT_BRANCH] if name.endswith('-production'): - _rules.append(self.RULE_PROTECTED_NO_LABEL) + _rules.append(self.RULE_PROTECTED_PUSH) else: if not (name.endswith('-preview') or name.startswith('labels:')): - _rules.append(self.RULE_PROTECTED) + _rules.append(self.RULE_PROTECTED_CHECK) if name.startswith('test:'): _rules.append(self.RULE_BUILD_ONLY) @@ -235,7 +226,7 @@ class RulesWriter: print('WARNING: pattern {} not exists'.format(pattern)) return '\n'.join(_rules) - def update_rules_yml(self): # type: () -> bool + def update_rules_yml(self) -> bool: with open(self.rules_yml) as fr: file_str = fr.read() @@ -255,7 +246,9 @@ PATTERN_COLOR = 'cyan' RULE_COLOR = 'blue' -def build_graph(rules_dict): # type: (dict[str, dict[str, list]]) -> pgv.AGraph +def build_graph(rules_dict: t.Dict[str, t.Dict[str, t.List[str]]]) -> 'pgv.AGraph': + from pygraphviz import pgv + graph = pgv.AGraph(directed=True, rankdir='LR', concentrate=True) for k, v in rules_dict.items(): @@ -281,7 +274,7 @@ def build_graph(rules_dict): # type: (dict[str, dict[str, list]]) -> pgv.AGraph return graph -def output_graph(graph, output_path='output.png'): # type: (pgv.AGraph, str) -> None +def output_graph(graph: 'pgv.AGraph', output_path: str = 'output.png') -> None: graph.layout('dot') if output_path.endswith('.png'): img_path = output_path @@ -292,13 +285,16 @@ def output_graph(graph, output_path='output.png'): # type: (pgv.AGraph, str) -> if __name__ == '__main__': parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('rules_yml', nargs='?', default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'rules.yml'), - help='rules.yml file path') - parser.add_argument('dependencies_yml', nargs='?', default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'dependencies', - 'dependencies.yml'), - help='dependencies.yml file path') - parser.add_argument('--graph', - help='Specify PNG image output path. Use this argument to generate dependency graph') + parser.add_argument( + 'rules_yml', nargs='?', default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'rules.yml'), help='rules.yml file path' + ) + parser.add_argument( + 'dependencies_yml', + nargs='?', + default=os.path.join(IDF_PATH, '.gitlab', 'ci', 'dependencies', 'dependencies.yml'), + help='dependencies.yml file path', + ) + parser.add_argument('--graph', help='Specify PNG image output path. Use this argument to generate dependency graph') args = parser.parse_args() writer = RulesWriter(args.rules_yml, args.dependencies_yml) diff --git a/tools/ci/get_known_failure_cases_file.py b/tools/ci/get_known_failure_cases_file.py deleted file mode 100644 index dd0f8eefd4e..00000000000 --- a/tools/ci/get_known_failure_cases_file.py +++ /dev/null @@ -1,22 +0,0 @@ -# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 - -import os - -import urllib3 -from minio import Minio - -from artifacts_handler import get_minio_client - - -def getenv(env_var: str) -> str: - try: - return os.environ[env_var] - except KeyError as e: - raise Exception(f'Environment variable {env_var} not set') from e - - -if __name__ == '__main__': - client = get_minio_client() - file_name = getenv('KNOWN_FAILURE_CASES_FILE_NAME') - client.fget_object('ignore-test-result-files', file_name, file_name) diff --git a/tools/ci/idf_ci_local/app.py b/tools/ci/idf_ci_local/app.py index 3dec01e9493..301aa7e097f 100644 --- a/tools/ci/idf_ci_local/app.py +++ b/tools/ci/idf_ci_local/app.py @@ -1,22 +1,22 @@ -# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2024-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import os +import subprocess import sys import typing as t -from typing import Literal from dynamic_pipelines.constants import BINARY_SIZE_METRIC_NAME from idf_build_apps import App from idf_build_apps import CMakeApp -from idf_build_apps import json_to_app +from idf_build_apps.utils import rmdir +from idf_ci_utils import idf_relpath -from idf_ci_local.uploader import AppUploader -from idf_ci_local.uploader import get_app_uploader +if t.TYPE_CHECKING: + pass class IdfCMakeApp(CMakeApp): - uploader: t.ClassVar[t.Optional['AppUploader']] = get_app_uploader() - build_system: Literal['idf_cmake'] = 'idf_cmake' + build_system: t.Literal['idf_cmake'] = 'idf_cmake' def _initialize_hook(self, **kwargs: t.Any) -> None: # ensure this env var exists @@ -27,8 +27,22 @@ class IdfCMakeApp(CMakeApp): def _post_build(self) -> None: super()._post_build() - if self.uploader: - self.uploader.upload_app(self.build_path) + # only upload in CI + if os.getenv('CI_JOB_ID'): + subprocess.run( + [ + 'idf-ci', + 'gitlab', + 'upload-artifacts', + self.app_dir, + ], + stdout=sys.stdout, + stderr=sys.stderr, + ) + rmdir( + self.build_path, + exclude_file_patterns=['build_log.txt', 'size*.json'], + ) class Metrics: @@ -74,26 +88,6 @@ class AppWithMetricsInfo(IdfCMakeApp): arbitrary_types_allowed = True -def dump_apps_to_txt(apps: t.List[App], output_filepath: str) -> None: - with open(output_filepath, 'w') as fw: - for app in apps: - fw.write(app.model_dump_json() + '\n') - - -def import_apps_from_txt(input_filepath: str) -> t.List[App]: - apps: t.List[App] = [] - with open(input_filepath) as fr: - for line in fr: - if line := line.strip(): - try: - apps.append(json_to_app(line, extra_classes=[IdfCMakeApp])) - except Exception: # noqa - print('Failed to deserialize app from line: %s' % line) - sys.exit(1) - - return apps - - def enrich_apps_with_metrics_info( app_metrics_info_map: t.Dict[str, t.Dict[str, t.Any]], apps: t.List[App] ) -> t.List[AppWithMetricsInfo]: @@ -125,6 +119,7 @@ def enrich_apps_with_metrics_info( apps_with_metrics_info = [] for app in apps: + app.app_dir = idf_relpath(app.app_dir) key = f'{app.app_dir}_{app.config_name}_{app.target}' app_attributes = _get_full_attributes(app) diff --git a/tools/ci/idf_ci_local/uploader.py b/tools/ci/idf_ci_local/uploader.py deleted file mode 100644 index 66c4fdfa8f8..00000000000 --- a/tools/ci/idf_ci_local/uploader.py +++ /dev/null @@ -1,172 +0,0 @@ -# SPDX-FileCopyrightText: 2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import abc -import glob -import os -import typing as t -from datetime import timedelta -from zipfile import ZIP_DEFLATED -from zipfile import ZipFile - -import minio -from artifacts_handler import ArtifactType -from artifacts_handler import get_minio_client -from artifacts_handler import getenv -from idf_build_apps import App -from idf_build_apps.utils import rmdir -from idf_ci_utils import IDF_PATH -from idf_pytest.constants import DEFAULT_BUILD_LOG_FILENAME -from idf_pytest.constants import DEFAULT_SIZE_JSON_FILENAME - - -class AppDownloader: - ALL_ARTIFACT_TYPES = [ArtifactType.MAP_AND_ELF_FILES, ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES] - - @abc.abstractmethod - def _download_app(self, app_build_path: str, artifact_type: ArtifactType) -> None: - pass - - def download_app(self, app_build_path: str, artifact_type: t.Optional[ArtifactType] = None) -> None: - """ - Download the app - :param app_build_path: the path to the build directory - :param artifact_type: if not specify, download all types of artifacts - :return: None - """ - if not artifact_type: - for _artifact_type in self.ALL_ARTIFACT_TYPES: - self._download_app(app_build_path, _artifact_type) - else: - self._download_app(app_build_path, artifact_type) - - -class AppUploader(AppDownloader): - TYPE_PATTERNS_DICT = { - ArtifactType.MAP_AND_ELF_FILES: [ - 'bootloader/*.map', - 'bootloader/*.elf', - 'esp_tee/*.map', - 'esp_tee/*.elf', - '*.map', - '*.elf', - 'gdbinit/*', - ], - ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES: [ - '*.bin', - 'bootloader/*.bin', - 'esp_tee/*.bin', - 'partition_table/*.bin', - 'flasher_args.json', - 'flash_project_args', - 'config/sdkconfig.json', - 'sdkconfig', - 'project_description.json', - ], - ArtifactType.LOGS: [ - DEFAULT_BUILD_LOG_FILENAME, - ], - } - - def __init__(self, pipeline_id: t.Union[str, int, None] = None) -> None: - self.pipeline_id = str(pipeline_id or '1') - - self._client = get_minio_client() - - def get_app_object_name(self, app_path: str, zip_name: str, artifact_type: ArtifactType) -> str: - return f'{self.pipeline_id}/{artifact_type.value}/{app_path}/{zip_name}' - - def _upload_app(self, app_build_path: str, artifact_type: ArtifactType) -> bool: - app_path, build_dir = os.path.split(app_build_path) - zip_filename = f'{build_dir}.zip' - - has_file = False - with ZipFile( - zip_filename, - 'w', - compression=ZIP_DEFLATED, - # 1 is the fastest compression level - # the size differs not much between 1 and 9 - compresslevel=1, - ) as zw: - for pattern in self.TYPE_PATTERNS_DICT[artifact_type]: - for file in glob.glob(os.path.join(app_build_path, pattern), recursive=True): - zw.write(file) - has_file = True - - uploaded = False - try: - if has_file: - obj_name = self.get_app_object_name(app_path, zip_filename, artifact_type) - self._client.fput_object(getenv('IDF_S3_BUCKET'), obj_name, zip_filename) - uploaded = True - finally: - os.remove(zip_filename) - return uploaded - - def upload_app(self, app_build_path: str, artifact_type: t.Optional[ArtifactType] = None) -> None: - uploaded = False - if not artifact_type: - upload_types: t.Iterable[ArtifactType] = self.TYPE_PATTERNS_DICT.keys() - else: - upload_types = [artifact_type] - - # Upload of size.json files is handled by GitLab CI via "artifacts_handler.py" script. - print(f'Uploading {app_build_path} {[k.value for k in upload_types]} to minio server') - for upload_type in upload_types: - uploaded |= self._upload_app(app_build_path, upload_type) - - if uploaded: - rmdir(app_build_path, exclude_file_patterns=[DEFAULT_BUILD_LOG_FILENAME, DEFAULT_SIZE_JSON_FILENAME]) - - def _download_app(self, app_build_path: str, artifact_type: ArtifactType) -> None: - app_path, build_dir = os.path.split(app_build_path) - zip_filename = f'{build_dir}.zip' - - # path are relative to IDF_PATH - current_dir = os.getcwd() - os.chdir(IDF_PATH) - try: - obj_name = self.get_app_object_name(app_path, zip_filename, artifact_type) - print(f'Downloading {obj_name}') - try: - try: - self._client.stat_object(getenv('IDF_S3_BUCKET'), obj_name) - except minio.error.S3Error as e: - raise RuntimeError( - f'No such file on minio server: {obj_name}. ' - f'Probably the build failed or the artifacts got expired. ' - f'Full error message: {str(e)}' - ) - else: - self._client.fget_object(getenv('IDF_S3_BUCKET'), obj_name, zip_filename) - print(f'Downloaded to {zip_filename}') - except minio.error.S3Error as e: - raise RuntimeError("Shouldn't happen, please report this bug in the CI channel" + str(e)) - - with ZipFile(zip_filename, 'r') as zr: - zr.extractall() - - os.remove(zip_filename) - finally: - os.chdir(current_dir) - - def get_app_presigned_url(self, app: App, artifact_type: ArtifactType) -> str: - obj_name = self.get_app_object_name(app.app_dir, f'{app.build_dir}.zip', artifact_type) - try: - self._client.stat_object( - getenv('IDF_S3_BUCKET'), - obj_name, - ) - except minio.error.S3Error: - return '' - else: - return self._client.get_presigned_url( # type: ignore - 'GET', getenv('IDF_S3_BUCKET'), obj_name, expires=timedelta(days=4) - ) - - -def get_app_uploader() -> t.Optional['AppUploader']: - if parent_pipeline_id := os.getenv('PARENT_PIPELINE_ID'): - return AppUploader(parent_pipeline_id) - - return None diff --git a/tools/ci/idf_ci_utils.py b/tools/ci/idf_ci_utils.py index b355090ffc7..b1138fa1daf 100644 --- a/tools/ci/idf_ci_utils.py +++ b/tools/ci/idf_ci_utils.py @@ -128,6 +128,14 @@ class GitlabYmlConfig: # expanding "include" for item in root_yml.pop('include', []) or []: + if isinstance(item, dict): + if 'project' in item: + continue + elif 'local' in item: + item = item['local'] + else: + continue + all_config.update(yaml.load(open(os.path.join(IDF_PATH, item)), Loader=yaml.FullLoader)) if 'default' in all_config: diff --git a/tools/ci/idf_pytest/constants.py b/tools/ci/idf_pytest/constants.py index 29e5ab52da8..32608322cdd 100644 --- a/tools/ci/idf_pytest/constants.py +++ b/tools/ci/idf_pytest/constants.py @@ -4,19 +4,6 @@ Pytest Related Constants. Don't import third-party packages here. """ -import os -import typing as t -import warnings -from dataclasses import dataclass -from enum import Enum -from functools import cached_property -from pathlib import Path - -from _pytest.python import Function -from idf_ci_utils import IDF_PATH -from idf_ci_utils import idf_relpath -from pytest_embedded.utils import to_list - SUPPORTED_TARGETS = [ 'esp32', 'esp32s2', @@ -29,343 +16,15 @@ SUPPORTED_TARGETS = [ 'esp32c5', 'esp32c61', ] -PREVIEW_TARGETS: t.List[str] = [] # this PREVIEW_TARGETS excludes 'linux' target -DEFAULT_SDKCONFIG = 'default' +PREVIEW_TARGETS: list[str] = [] # this PREVIEW_TARGETS excludes 'linux' target + DEFAULT_LOGDIR = 'pytest-embedded' -TARGET_MARKERS = { - 'esp32': 'support esp32 target', - 'esp32s2': 'support esp32s2 target', - 'esp32s3': 'support esp32s3 target', - 'esp32c3': 'support esp32c3 target', - 'esp32c2': 'support esp32c2 target', - 'esp32c5': 'support esp32c5 target', - 'esp32c6': 'support esp32c6 target', - 'esp32h2': 'support esp32h2 target', - 'esp32h4': 'support esp32h4 target', # as preview - 'esp32h21': 'support esp32h21 target', # as preview - 'esp32p4': 'support esp32p4 target', - 'esp32c61': 'support esp32c61 target', - 'linux': 'support linux target', -} - -SPECIAL_MARKERS = { - 'supported_targets': 'support all officially announced supported targets, refer to `SUPPORTED_TARGETS`', - 'preview_targets': "support all preview targets ('none')", - 'all_targets': 'support all targets, including supported ones and preview ones', - 'temp_skip_ci': 'temp skip tests for specified targets only in ci', - 'temp_skip': 'temp skip tests for specified targets both in ci and locally', - 'nightly_run': 'tests should be executed as part of the nightly trigger pipeline', - 'host_test': 'tests which should not be built at the build stage, and instead built in host_test stage', - 'require_elf': 'tests which require elf file', -} - -ENV_MARKERS = { - # special markers - 'qemu': 'build and test using qemu, not real target', - 'macos_shell': 'tests should be run on macos hosts', - # single-dut markers - 'generic': 'tests should be run on generic runners', - 'flash_suspend': 'support flash suspend feature', - 'eth_ip101': 'connected via IP101 ethernet transceiver', - 'eth_lan8720': 'connected via LAN8720 ethernet transceiver', - 'eth_rtl8201': 'connected via RTL8201 ethernet transceiver', - 'eth_ksz8041': 'connected via KSZ8041 ethernet transceiver', - 'eth_dp83848': 'connected via DP83848 ethernet transceiver', - 'eth_w5500': 'SPI Ethernet module with two W5500', - 'eth_ksz8851snl': 'SPI Ethernet module with two KSZ8851SNL', - 'eth_dm9051': 'SPI Ethernet module with two DM9051', - 'quad_psram': 'runners with quad psram', - 'octal_psram': 'runners with octal psram', - 'usb_host_flash_disk': 'usb host runners with USB flash disk attached', - 'usb_device': 'usb device runners', - 'ethernet_ota': 'ethernet OTA runners', - 'flash_encryption': 'Flash Encryption runners', - 'flash_encryption_f4r8': 'Flash Encryption runners with 4-line flash and 8-line psram', - 'flash_encryption_f8r8': 'Flash Encryption runners with 8-line flash and 8-line psram', - 'flash_encryption_ota': 'Flash Encryption runners with ethernet OTA support with 4mb flash size', - 'flash_multi': 'Multiple flash chips tests', - 'psram': 'Chip has 4-line psram', - 'ir_transceiver': 'runners with a pair of IR transmitter and receiver', - 'twai_transceiver': 'runners with a TWAI PHY transceiver', - 'flash_encryption_wifi_high_traffic': 'Flash Encryption runners with wifi high traffic support', - 'ethernet': 'ethernet runner', - 'ethernet_stress': 'ethernet runner with stress test', - 'ethernet_flash_8m': 'ethernet runner with 8mb flash', - 'ethernet_router': 'both the runner and dut connect to the same router through ethernet NIC', - 'ethernet_vlan': 'ethernet runner GARM-32-SH-1-R16S5N3', - 'wifi_ap': 'a wifi AP in the environment', - 'wifi_router': 'both the runner and dut connect to the same wifi router', - 'wifi_high_traffic': 'wifi high traffic runners', - 'wifi_wlan': 'wifi runner with a wireless NIC', - 'wifi_iperf': 'the AP and ESP dut were placed in a shielded box - for iperf test', - 'Example_ShieldBox': 'multiple shielded APs connected to shielded ESP DUT via RF cable with programmable attenuator', # noqa E501 - 'xtal_26mhz': 'runner with 26MHz xtal on board', - 'xtal_40mhz': 'runner with 40MHz xtal on board', - 'external_flash': 'external flash memory connected via VSPI (FSPI)', - 'sdcard_sdmode': 'sdcard running in SD mode, to be removed after test migration', - 'sdcard_spimode': 'sdcard running in SPI mode', - 'emmc': 'eMMC card', - 'sdcard': 'sdcard runner', - 'MSPI_F8R8': 'runner with Octal Flash and Octal PSRAM', - 'MSPI_F4R8': 'runner with Quad Flash and Octal PSRAM', - 'MSPI_F4R4': 'runner with Quad Flash and Quad PSRAM', - 'flash_120m': 'runner with 120M supported Flash', - 'jtag': 'runner where the chip is accessible through JTAG as well', - 'usb_serial_jtag': 'runner where the chip is accessible through builtin JTAG as well', - 'adc': 'ADC related tests should run on adc runners', - 'xtal32k': 'Runner with external 32k crystal connected', - 'no32kXtal': 'Runner with no external 32k crystal connected', - 'psramv0': 'Runner with PSRAM version 0', - 'esp32eco3': 'Runner with esp32 eco3 connected', - 'ecdsa_efuse': 'Runner with test ECDSA private keys programmed in efuse', - 'ccs811': 'Runner with CCS811 connected', - 'nvs_encr_hmac': 'Runner with test HMAC key programmed in efuse', - 'i2c_oled': 'Runner with ssd1306 I2C oled connected', - 'httpbin': 'runner for tests that need to access the httpbin service', - 'flash_4mb': 'C2 runners with 4 MB flash', - 'jtag_re_enable': 'Runner to re-enable jtag which is softly disabled by burning bit SOFT_DIS_JTAG on eFuse', - 'es8311': 'Development board that carries es8311 codec', - 'camera': 'Runner with camera', - 'ov5647': 'Runner with camera ov5647', - # multi-dut markers - 'multi_dut_modbus_rs485': 'a pair of runners connected by RS485 bus', - 'ieee802154': 'ieee802154 related tests should run on ieee802154 runners.', - 'openthread_br': 'tests should be used for openthread border router.', - 'openthread_bbr': 'tests should be used for openthread border router linked to Internet.', - 'openthread_sleep': 'tests should be used for openthread sleepy device.', - 'zigbee_multi_dut': 'zigbee runner which have multiple duts.', - 'two_duts': 'tests should be run on runners which has two wifi duts connected.', - 'generic_multi_device': 'generic multiple devices whose corresponding gpio pins are connected to each other.', - 'twai_network': 'multiple runners form a TWAI network.', - 'sdio_master_slave': 'Test sdio multi board, esp32+esp32', - 'sdio_multidev_32_c6': 'Test sdio multi board, esp32+esp32c6', - 'sdio_multidev_p4_c5': 'Test sdio multi board, esp32p4+esp32c5', - 'sdio_multidev_32_c61': 'Test sdio multi board, esp32+esp32c61', - 'usj_device': 'Test usb_serial_jtag and usb_serial_jtag is used as serial only (not console)', - 'twai_std': 'twai runner with all twai supported targets connect to usb-can adapter', - 'lp_i2s': 'lp_i2s runner tested with hp_i2s', - 'ram_app': 'ram_app runners', - 'esp32c3eco7': 'esp32c3 major version(v1.1) chips', - 'esp32c2eco4': 'esp32c2 major version(v2.0) chips', - 'recovery_bootloader': 'Runner with recovery bootloader offset set in eFuse', - 'esp32p4_eco4': 'Runner with esp32p4 eco4 connected', - 'esp32c5_eco3': 'Runner with esp32c5 eco3 connected', - 'rev_default': 'Runner with default revision connected', -} - # by default the timeout is 1h, for some special cases we need to extend it TIMEOUT_4H_MARKERS = [ 'ethernet_stress', ] -DEFAULT_CONFIG_RULES_STR = ['sdkconfig.ci=default', 'sdkconfig.ci.*=', '=default'] -DEFAULT_IGNORE_WARNING_FILEPATH = os.path.join(IDF_PATH, 'tools', 'ci', 'ignore_build_warnings.txt') -DEFAULT_BUILD_TEST_RULES_FILEPATH = os.path.join(IDF_PATH, '.gitlab', 'ci', 'default-build-test-rules.yml') -DEFAULT_FULL_BUILD_TEST_COMPONENTS = [ - 'cxx', - 'esp_common', - 'esp_hw_support', - 'esp_rom', - 'esp_system', - 'esp_timer', - 'freertos', - 'hal', - 'heap', - 'log', - 'newlib', - 'riscv', - 'soc', - 'xtensa', -] -DEFAULT_FULL_BUILD_TEST_FILEPATTERNS = [ - # tools - 'tools/cmake/**/*', - 'tools/tools.json', - # ci - 'tools/ci/ignore_build_warnings.txt', -] -DEFAULT_BUILD_LOG_FILENAME = 'build_log.txt' -DEFAULT_SIZE_JSON_FILENAME = 'size.json' - - -class CollectMode(str, Enum): - SINGLE_SPECIFIC = 'single_specific' - MULTI_SPECIFIC = 'multi_specific' - MULTI_ALL_WITH_PARAM = 'multi_all_with_param' - ALL = 'all' - - -class PytestApp: - """ - Pytest App with relative path to IDF_PATH - """ - - def __init__(self, path: str, target: str, config: str) -> None: - self.path = idf_relpath(path) - self.target = target - self.config = config - - def __hash__(self) -> int: - return hash((self.path, self.target, self.config)) - - @cached_property - def build_dir(self) -> str: - return os.path.join(self.path, f'build_{self.target}_{self.config}') - - -@dataclass -class PytestCase: - apps: t.List[PytestApp] - - item: Function - multi_dut_without_param: bool - - def __hash__(self) -> int: - return hash((self.path, self.name, self.apps, self.all_markers)) - - @cached_property - def path(self) -> str: - return str(self.item.path) - - @cached_property - def name(self) -> str: - return self.item.originalname # type: ignore - - @cached_property - def targets(self) -> t.List[str]: - if not self.multi_dut_without_param: - return [app.target for app in self.apps] - - # multi-dut test cases without parametrize - skip = True - for _t in [app.target for app in self.apps]: - if _t in self.target_markers: - skip = False - warnings.warn( - f'`pytest.mark.[TARGET]` defined in parametrize for multi-dut test cases is deprecated. ' # noqa: W604 - f'Please use parametrize instead for test case {self.item.nodeid}' - ) - break - - if not skip: - return [app.target for app in self.apps] - - return [''] * len(self.apps) # this will help to filter these cases out later - - @cached_property - def is_single_dut_test_case(self) -> bool: - return True if len(self.apps) == 1 else False - - @cached_property - def is_host_test(self) -> bool: - return 'host_test' in self.all_markers or 'linux' in self.targets - - # the following markers could be changed dynamically, don't use cached_property - @property - def all_markers(self) -> t.Set[str]: - return {marker.name for marker in self.item.iter_markers()} - - @property - def skip_targets(self) -> t.Set[str]: - def _get_temp_markers_disabled_targets(marker_name: str) -> t.Set[str]: - temp_marker = self.item.get_closest_marker(marker_name) - - if not temp_marker: - return set() - - # temp markers should always use keyword arguments `targets` and `reason` - if not temp_marker.kwargs.get('targets') or not temp_marker.kwargs.get('reason'): - raise ValueError( - f'`{marker_name}` should always use keyword arguments `targets` and `reason`. ' # noqa: W604 - f'For example: ' - f'`@pytest.mark.{marker_name}(targets=["esp32"], reason="IDF-xxxx, will fix it ASAP")`' - ) - - return set(to_list(temp_marker.kwargs['targets'])) # type: ignore - - temp_skip_ci_targets = _get_temp_markers_disabled_targets('temp_skip_ci') - temp_skip_targets = _get_temp_markers_disabled_targets('temp_skip') - - # in CI we skip the union of `temp_skip` and `temp_skip_ci` - if os.getenv('CI_JOB_ID'): - _skip_targets = temp_skip_ci_targets.union(temp_skip_targets) - else: # we use `temp_skip` locally - _skip_targets = temp_skip_targets - - return _skip_targets - - @property - def target_markers(self) -> t.Set[str]: - return {marker for marker in self.all_markers if marker in TARGET_MARKERS} - self.skip_targets - - @property - def env_markers(self) -> t.Set[str]: - return {marker for marker in self.all_markers if marker in ENV_MARKERS} - - @property - def target_selector(self) -> str: - return ','.join(app.target for app in self.apps) - - @property - def requires_elf_or_map(self) -> bool: - """ - This property determines whether the test case requires elf or map file. By default, one app in the test case - only requires .bin files. - - :return: True if the test case requires elf or map file, False otherwise - """ - if 'jtag' in self.env_markers or 'usb_serial_jtag' in self.env_markers: - return True - - cases_need_elf = ['panic', 'gdbstub_runtime'] - if 'require_elf' in SPECIAL_MARKERS: - return True - - for case in cases_need_elf: - if any(case in Path(app.path).parts for app in self.apps): - return True - - return False - - def all_built_in_app_lists(self, app_lists: t.Optional[t.List[str]] = None) -> t.Optional[str]: - """ - Check if all binaries of the test case are built in the app lists. - - :param app_lists: app lists to check - :return: debug string if not all binaries are built in the app lists, None otherwise - """ - if app_lists is None: - # ignore this feature - return None - - bin_found = [0] * len(self.apps) - for i, app in enumerate(self.apps): - if app.build_dir in app_lists: - bin_found[i] = 1 - - if sum(bin_found) == 0: - msg = f'Skip test case {self.name} because all following binaries are not listed in the app lists: ' # noqa: E713 - for app in self.apps: - msg += f'\n - {app.build_dir}' - - print(msg) - return msg - - if sum(bin_found) == len(self.apps): - return None - - # some found, some not, looks suspicious - msg = f'Found some binaries of test case {self.name} are not listed in the app lists.' # noqa: E713 - for i, app in enumerate(self.apps): - if bin_found[i] == 0: - msg += f'\n - {app.build_dir}' - - msg += '\nMight be a issue of .build-test-rules.yml files' - print(msg) - return msg - - ECO_MARKERS = [ 'esp32eco3', 'esp32c2eco4', diff --git a/tools/ci/idf_pytest/plugin.py b/tools/ci/idf_pytest/plugin.py index 43113271e48..4de6ca6ac76 100644 --- a/tools/ci/idf_pytest/plugin.py +++ b/tools/ci/idf_pytest/plugin.py @@ -1,112 +1,109 @@ # SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 -import importlib -import logging import os -import sys import typing as t -from collections import defaultdict -from functools import cached_property -from unittest.mock import MagicMock +from pathlib import Path from xml.etree import ElementTree as ET import pytest -from _pytest.config import ExitCode -from _pytest.main import Session +import yaml +from _pytest.config import Config from _pytest.python import Function from _pytest.runner import CallInfo -from idf_build_apps import App -from idf_build_apps.constants import BuildStatus -from idf_ci_utils import idf_relpath +from dynamic_pipelines.constants import KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH +from idf_ci import IdfPytestPlugin +from idf_ci import PytestCase +from idf_ci.idf_pytest.plugin import IDF_CI_PYTEST_DEBUG_INFO_KEY +from idf_ci_utils import to_list from pytest_embedded import Dut -from pytest_embedded.plugin import parse_multi_dut_args from pytest_embedded.utils import find_by_suffix -from pytest_embedded.utils import to_list from pytest_ignore_test_results.ignore_results import ChildCase from pytest_ignore_test_results.ignore_results import ChildCasesStashKey -from .constants import DEFAULT_SDKCONFIG from .constants import ECO_MARKERS -from .constants import PREVIEW_TARGETS -from .constants import SUPPORTED_TARGETS -from .constants import CollectMode -from .constants import PytestApp -from .constants import PytestCase -from .utils import comma_sep_str_to_list from .utils import format_case_id from .utils import merge_junit_files from .utils import normalize_testcase_file_path -IDF_PYTEST_EMBEDDED_KEY = pytest.StashKey['IdfPytestEmbedded']() -ITEM_FAILED_CASES_KEY = pytest.StashKey[list]() -ITEM_FAILED_KEY = pytest.StashKey[bool]() -ITEM_PYTEST_CASE_KEY = pytest.StashKey[PytestCase]() +IDF_LOCAL_PLUGIN_KEY = pytest.StashKey['IdfLocalPlugin']() -class IdfPytestEmbedded: +def requires_elf_or_map(case: PytestCase) -> bool: + """ + Determines whether the test case requires elf or map file. By default, one app in the test case + only requires .bin files. + + :return: True if the test case requires elf or map file, False otherwise + """ + if 'jtag' in case.env_markers or 'usb_serial_jtag' in case.env_markers: + return True + + folders_need_elf = ['panic', 'gdbstub_runtime'] + if 'require_elf' in case.all_markers: + return True + + for folder in folders_need_elf: + if any(folder in Path(app.path).parts for app in case.apps): + return True + + return False + + +def skipped_targets(item: Function) -> set[str]: + def _get_temp_markers_disabled_targets(marker_name: str) -> set[str]: + targets = [] + for _m in item.own_markers: + if _m.name == marker_name: + if not _m.kwargs.get('targets') or not _m.kwargs.get('reason'): + raise ValueError( + f'`{marker_name}` should always use keyword arguments `targets` and `reason`. ' # noqa: W604 + f'For example: ' + f'`@pytest.mark.{marker_name}(targets=["esp32"], reason="IDF-xxxx, will fix it ASAP")`' + ) + targets.extend(to_list(_m.kwargs['targets'])) + + if targets: + return set(targets) + return set() + + _count = IdfLocalPlugin.get_param(item, 'count', 1) + + def normalize_targets(target: str) -> str: + targets = target.split(',') + if len(targets) == 1: + return ','.join(targets * _count) + if len(targets) != _count: + raise ValueError( + f"Invalid target format: '{target}'. " + f'Expected a single target or exactly {_count} values separated by commas.' + f'len({targets}) != {_count}' + ) + return target + + temp_skip_ci_targets = set(normalize_targets(_t) for _t in _get_temp_markers_disabled_targets('temp_skip_ci')) + temp_skip_targets = set(normalize_targets(_t) for _t in _get_temp_markers_disabled_targets('temp_skip')) + + # in CI we skip the union of `temp_skip` and `temp_skip_ci` + if os.getenv('CI_JOB_ID'): + _skip_targets = temp_skip_ci_targets.union(temp_skip_targets) + else: # we use `temp_skip` locally + _skip_targets = temp_skip_targets + + return _skip_targets + + +class IdfLocalPlugin: UNITY_RESULT_MAPPINGS = { 'PASS': 'passed', 'FAIL': 'failed', 'IGNORE': 'skipped', } - def __init__( - self, - target: t.Union[t.List[str], str], - *, - config_name: t.Optional[str] = None, - single_target_duplicate_mode: bool = False, - apps: t.Optional[t.List[App]] = None, - ): - if isinstance(target, str): - # sequence also matters - self.target = comma_sep_str_to_list(target) - else: - self.target = target + def __init__(self) -> None: + with open(KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH) as fr: + known_warnings_dict = yaml.safe_load(fr) or dict() - if not self.target: - raise ValueError('`target` should not be empty') - - self.config_name = config_name - - # these are useful while gathering all the multi-dut test cases - # when this mode is activated, - # - # pytest.mark.esp32 - # pytest.mark.parametrize('count', [2], indirect=True) - # def test_foo(dut): - # pass - # - # should be collected when running `pytest --target esp32` - # - # otherwise, it should be collected when running `pytest --target esp32,esp32` - self._single_target_duplicate_mode = single_target_duplicate_mode - - self.apps_list = ( - [ - os.path.join(idf_relpath(app.app_dir), app.build_dir) - for app in apps - if app.build_status == BuildStatus.SUCCESS - ] - if apps is not None - else None - ) - - self.cases: t.List[PytestCase] = [] - - # record the additional info - # test case id: {key: value} - self.additional_info: t.Dict[str, t.Dict[str, t.Any]] = defaultdict(dict) - - @cached_property - def collect_mode(self) -> CollectMode: - if len(self.target) == 1: - if self.target[0] == CollectMode.MULTI_ALL_WITH_PARAM: - return CollectMode.MULTI_ALL_WITH_PARAM - else: - return CollectMode.SINGLE_SPECIFIC - else: - return CollectMode.MULTI_SPECIFIC + self.exclude_no_env_markers_test_cases: set[str] = set(known_warnings_dict['no_env_marker_test_cases']) @staticmethod def get_param(item: Function, key: str, default: t.Any = None) -> t.Any: @@ -117,206 +114,71 @@ class IdfPytestEmbedded: return item.callspec.params.get(key, default) or default - def item_to_pytest_case(self, item: Function) -> t.Optional[PytestCase]: - """ - Turn pytest item to PytestCase - """ - count = self.get_param(item, 'count', 1) + @pytest.hookimpl(wrapper=True) + def pytest_collection_modifyitems(self, config: Config, items: list[Function]) -> t.Generator[None, None, None]: + yield # throw it back to idf-ci - # default app_path is where the test script locates - app_paths = to_list(parse_multi_dut_args(count, self.get_param(item, 'app_path', os.path.dirname(item.path)))) - configs = to_list(parse_multi_dut_args(count, self.get_param(item, 'config', DEFAULT_SDKCONFIG))) - targets = to_list(parse_multi_dut_args(count, self.get_param(item, 'target'))) + deselected_items = [] - multi_dut_without_param = False - if count > 1 and targets == [None] * count: - multi_dut_without_param = True - try: - targets = to_list(parse_multi_dut_args(count, '|'.join(self.target))) # check later while collecting - except ValueError: # count doesn't match - return None - - elif targets is None: - targets = self.target - - return PytestCase( - apps=[PytestApp(app_paths[i], targets[i], configs[i]) for i in range(count)], - item=item, - multi_dut_without_param=multi_dut_without_param, - ) - - def pytest_collectstart(self) -> None: - # mock the optional packages while collecting locally - if not os.getenv('CI_JOB_ID') or os.getenv('PYTEST_IGNORE_COLLECT_IMPORT_ERROR') == '1': - # optional packages required by test scripts - for p in [ - 'scapy', - 'scapy.all', - 'websocket', # websocket-client - 'netifaces', - 'RangeHTTPServer', # rangehttpserver - 'dbus', # dbus-python - 'dbus.mainloop', - 'dbus.mainloop.glib', - 'google.protobuf', # protobuf - 'google.protobuf.internal', - 'bleak', - 'paho', # paho-mqtt - 'paho.mqtt', - 'paho.mqtt.client', - 'paramiko', - 'netmiko', - 'pyecharts', - 'pyecharts.options', - 'pyecharts.charts', - 'can', # python-can - ]: - try: - importlib.import_module(p) - except ImportError: - logging.warning(f'Optional package {p} is not installed, mocking it while collecting...') - sys.modules[p] = MagicMock() - - @pytest.hookimpl(tryfirst=True) - def pytest_collection_modifyitems(self, items: t.List[Function]) -> None: - """ - Background info: - - We're using `pytest.mark.[TARGET]` as a syntactic sugar to indicate that they are actually supported by all - the listed targets. For example, - - >>> @pytest.mark.esp32 - >>> @pytest.mark.esp32s2 - - should be treated as - - >>> @pytest.mark.parametrize('target', [ - >>> 'esp32', - >>> 'esp32s2', - >>> ], indirect=True) - - All single-dut test cases, and some of the multi-dut test cases with the same targets, are using this - way to indicate the supported targets. - - To avoid ambiguity, - - - when we're collecting single-dut test cases with esp32, we call - - `pytest --collect-only --target esp32` - - - when we're collecting multi-dut test cases, we list all the targets, even when they're the same - - `pytest --collect-only --target esp32,esp32` for two esp32 connected - `pytest --collect-only --target esp32,esp32s2` for esp32 and esp32s2 connected - - therefore, we have two different logic for searching test cases, explained in 2.1 and 2.2 - """ - # 1. Filter according to nighty_run related markers - if os.getenv('INCLUDE_NIGHTLY_RUN') == '1': - # nightly_run and non-nightly_run cases are both included - pass - elif os.getenv('NIGHTLY_RUN') == '1': - # only nightly_run cases are included - items[:] = [_item for _item in items if _item.get_closest_marker('nightly_run') is not None] - else: - # only non-nightly_run cases are included - items[:] = [_item for _item in items if _item.get_closest_marker('nightly_run') is None] - - # 2. Add markers according to special markers - item_to_case_dict: t.Dict[Function, PytestCase] = {} + # Filter + filtered_items = [] for item in items: - case = self.item_to_pytest_case(item) - if case is None: + case = IdfPytestPlugin.get_case_by_item(item) + if not case: + deselected_items.append(item) continue - item.stash[ITEM_PYTEST_CASE_KEY] = item_to_case_dict[item] = case - if 'supported_targets' in item.keywords: - for _target in SUPPORTED_TARGETS: - item.add_marker(_target) - if 'preview_targets' in item.keywords: - for _target in PREVIEW_TARGETS: - item.add_marker(_target) - if 'all_targets' in item.keywords: - for _target in [*SUPPORTED_TARGETS, *PREVIEW_TARGETS]: - item.add_marker(_target) + skipped = False + current_targets = case.target_selector.split(',') + for st in skipped_targets(item): + # Handle wildcard patterns like "esp32p4,*" or "*,esp32p4" + if '*' in st: + skip_pattern = st.split(',') + if len(skip_pattern) != len(current_targets): + continue + if all(_p == '*' or _p == _t for _p, _t in zip(skip_pattern, current_targets)): + skipped = True + break + # Exact match (no wildcard) + elif case.target_selector == st: + skipped = True + break - # add single-dut "target" as param - _item_target_param = self.get_param(item, 'target', None) - if case.is_single_dut_test_case and _item_target_param and _item_target_param not in case.all_markers: - item.add_marker(_item_target_param) + if skipped: + deselected_items.append(item) + item.stash[IDF_CI_PYTEST_DEBUG_INFO_KEY] = 'skipped by temp_skip markers' + continue - items[:] = [_item for _item in items if _item in item_to_case_dict] + if not case.env_markers and 'host_test' not in case.all_markers: + if case.name in self.exclude_no_env_markers_test_cases: + deselected_items.append(item) + continue - # 3.1. CollectMode.SINGLE_SPECIFIC, like `pytest --target esp32` - if self.collect_mode == CollectMode.SINGLE_SPECIFIC: - filtered_items = [] - for item in items: - case = item_to_case_dict[item] + raise ValueError( + f'Test case {case.name} does not have any env markers. ' + f'Please add env markers to the test case or add it to the ' + f'`no_env_markers_test_cases` list in {KNOWN_GENERATE_TEST_CHILD_PIPELINE_WARNINGS_FILEPATH}' + ) - # single-dut one - if case.is_single_dut_test_case and self.target[0] in case.target_markers: - filtered_items.append(item) + filtered_items.append(item) - # multi-dut ones and in single_target_duplicate_mode - elif self._single_target_duplicate_mode and not case.is_single_dut_test_case: - # ignore those test cases with `target` defined in parametrize, since these will be covered in 3.3 - if self.get_param(item, 'target', None) is None and self.target[0] in case.target_markers: - filtered_items.append(item) + items[:] = filtered_items - items[:] = filtered_items - # 3.2. CollectMode.MULTI_SPECIFIC, like `pytest --target esp32,esp32` - elif self.collect_mode == CollectMode.MULTI_SPECIFIC: - items[:] = [_item for _item in items if item_to_case_dict[_item].targets == self.target] - - # 3.3. CollectMode.MULTI_ALL_WITH_PARAM, intended to be used by `get_pytest_cases` - else: - filtered_items = [] - for item in items: - case = item_to_case_dict[item] - target = self.get_param(item, 'target', None) - if ( - not case.is_single_dut_test_case - and target is not None - and target not in case.skip_targets - and not any([_t in target for _t in case.skip_targets if '|' not in _t]) - ): - filtered_items.append(item) - items[:] = filtered_items - - # 4. filter according to the sdkconfig, if there's param 'config' defined - if self.config_name: - _items = [] - for item in items: - case = item_to_case_dict[item] - if self.config_name not in set(app.config or DEFAULT_SDKCONFIG for app in case.apps): - self.additional_info[case.name]['skip_reason'] = f'Only run with sdkconfig {self.config_name}' - else: - _items.append(item) - items[:] = _items - - # 5. filter by `self.apps_list`, skip the test case if not listed - # should only be used in CI - _items = [] - for item in items: - case = item_to_case_dict[item] - if msg := case.all_built_in_app_lists(self.apps_list): - self.additional_info[case.name]['skip_reason'] = msg - else: - _items.append(item) + # Deselect + config.hook.pytest_deselected(items=deselected_items) # OKAY!!! All left ones will be executed, sort it and add more markers - items[:] = sorted( - _items, key=lambda x: (os.path.dirname(x.path), self.get_param(x, 'config', DEFAULT_SDKCONFIG)) - ) + items[:] = sorted(items, key=lambda x: (os.path.dirname(x.path), self.get_param(x, 'config', 'default'))) + for item in items: - case = item_to_case_dict[item] + case = IdfPytestPlugin.get_case_by_item(item) + # set default timeout 10 minutes for each case if 'timeout' not in item.keywords: item.add_marker(pytest.mark.timeout(10 * 60)) # add 'xtal_40mhz' tag as a default tag for esp32c2 target - # only add this marker for esp32c2 cases - if 'esp32c2' in self.target and 'esp32c2' in case.targets and 'xtal_26mhz' not in case.all_markers: + if 'esp32c2' in case.targets and 'xtal_26mhz' not in case.all_markers: item.add_marker('xtal_40mhz') for eco_marker in ECO_MARKERS: @@ -325,8 +187,8 @@ class IdfPytestEmbedded: else: item.add_marker('rev_default') - def pytest_report_collectionfinish(self, items: t.List[Function]) -> None: - self.cases = [item.stash[ITEM_PYTEST_CASE_KEY] for item in items] + if 'host_test' in case.all_markers: + item.add_marker('skip_app_downloader') # host_test jobs will build the apps itself def pytest_custom_test_case_name(self, item: Function) -> str: return item.funcargs.get('test_case_name', item.nodeid) # type: ignore @@ -400,8 +262,3 @@ class IdfPytestEmbedded: case.attrib['ci_job_url'] = ci_job_url xml.write(junit) - - def pytest_sessionfinish(self, session: Session, exitstatus: int) -> None: - if exitstatus != 0: - if exitstatus == ExitCode.NO_TESTS_COLLECTED: - session.exitstatus = 0 diff --git a/tools/ci/idf_pytest/pytest.ini b/tools/ci/idf_pytest/pytest.ini deleted file mode 100644 index cc928df8995..00000000000 --- a/tools/ci/idf_pytest/pytest.ini +++ /dev/null @@ -1,3 +0,0 @@ -[pytest] -addopts = -p no:idf-ci -python_files = test_*.py diff --git a/tools/ci/idf_pytest/script.py b/tools/ci/idf_pytest/script.py deleted file mode 100644 index 1c79aeecd19..00000000000 --- a/tools/ci/idf_pytest/script.py +++ /dev/null @@ -1,238 +0,0 @@ -# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import fnmatch -import io -import logging -import os.path -import typing as t -from contextlib import redirect_stdout -from pathlib import Path - -import pytest -from _pytest.config import ExitCode -from idf_build_apps import App -from idf_build_apps import find_apps -from idf_build_apps.constants import SUPPORTED_TARGETS -from idf_build_apps.constants import BuildStatus -from idf_ci_local.app import IdfCMakeApp -from idf_ci_utils import IDF_PATH -from idf_ci_utils import get_all_manifest_files -from idf_ci_utils import idf_relpath -from idf_ci_utils import to_list -from idf_py_actions.constants import PREVIEW_TARGETS as TOOLS_PREVIEW_TARGETS -from idf_py_actions.constants import SUPPORTED_TARGETS as TOOLS_SUPPORTED_TARGETS - -from .constants import DEFAULT_BUILD_LOG_FILENAME -from .constants import DEFAULT_CONFIG_RULES_STR -from .constants import DEFAULT_SIZE_JSON_FILENAME -from .constants import CollectMode -from .constants import PytestCase -from .plugin import IdfPytestEmbedded - - -def get_pytest_files(paths: t.List[str]) -> t.List[str]: - # this is a workaround to solve pytest collector super slow issue - # benchmark with - # - time pytest -m esp32 --collect-only - # user=15.57s system=1.35s cpu=95% total=17.741 - # - time { find -name 'pytest_*.py'; } | xargs pytest -m esp32 --collect-only - # user=0.11s system=0.63s cpu=36% total=2.044 - # user=1.76s system=0.22s cpu=43% total=4.539 - # use glob.glob would also save a bunch of time - pytest_scripts: t.Set[str] = set() - for p in paths: - path = Path(p) - pytest_scripts.update(str(_p) for _p in path.glob('**/pytest_*.py') if 'managed_components' not in _p.parts) - - return list(pytest_scripts) - - -def get_pytest_cases( - paths: t.Union[str, t.List[str]], - target: str = CollectMode.ALL, - *, - config_name: t.Optional[str] = None, - marker_expr: t.Optional[str] = None, - filter_expr: t.Optional[str] = None, - apps: t.Optional[t.List[App]] = None, -) -> t.List[PytestCase]: - """ - Return the list of test cases - - For single-dut test cases, `target` could be - - [TARGET], e.g. `esp32`, to get the test cases for the given target - - or `single_all`, to get all single-dut test cases - - For multi-dut test cases, `target` could be - - [TARGET,[TARGET...]], e.g. `esp32,esp32s2`, to get the test cases for the given targets - - or `multi_all`, to get all multi-dut test cases - - :param paths: paths to search for pytest scripts - :param target: target or keywords to get test cases for, detailed above - :param config_name: sdkconfig name - :param marker_expr: pytest marker expression, `-m` - :param filter_expr: pytest filter expression, `-k` - :param apps: built app list, skip the tests required by apps not in the list - :return: list of test cases - """ - paths = to_list(paths) - - cases: t.List[PytestCase] = [] - pytest_scripts = get_pytest_files(paths) # type: ignore - if not pytest_scripts: - print(f'WARNING: no pytest scripts found for target {target} under paths {", ".join(paths)}') - return cases - - def _get_pytest_cases(_target: str, _single_target_duplicate_mode: bool = False) -> t.List[PytestCase]: - collector = IdfPytestEmbedded( - _target, config_name=config_name, single_target_duplicate_mode=_single_target_duplicate_mode, apps=apps - ) - - with io.StringIO() as buf: - with redirect_stdout(buf): - cmd = ['--collect-only', *pytest_scripts, '--target', _target, '-q'] - if marker_expr: - cmd.extend(['-m', marker_expr]) - if filter_expr: - cmd.extend(['-k', filter_expr]) - res = pytest.main(cmd, plugins=[collector]) - - if res.value != ExitCode.OK: - if res.value == ExitCode.NO_TESTS_COLLECTED: - print(f'WARNING: no pytest app found for target {_target} under paths {", ".join(paths)}') - else: - print(buf.getvalue()) - raise RuntimeError(f'pytest collection failed at {", ".join(paths)} with command "{" ".join(cmd)}"') - - return collector.cases # type: ignore - - if target == CollectMode.ALL: - targets = TOOLS_SUPPORTED_TARGETS + TOOLS_PREVIEW_TARGETS + [CollectMode.MULTI_ALL_WITH_PARAM] - else: - targets = [target] - - for _target in targets: - if target == CollectMode.ALL: - cases.extend(_get_pytest_cases(_target, _single_target_duplicate_mode=True)) - else: - cases.extend(_get_pytest_cases(_target)) - - return sorted(cases, key=lambda x: (x.path, x.name, str(x.targets))) - - -def get_all_apps( - paths: t.List[str], - target: str = CollectMode.ALL, - *, - marker_expr: t.Optional[str] = None, - filter_expr: t.Optional[str] = None, - config_rules_str: t.Optional[t.List[str]] = None, - preserve_all: bool = False, - extra_default_build_targets: t.Optional[t.List[str]] = None, - compare_manifest_sha_filepath: t.Optional[str] = None, - modified_components: t.Optional[t.List[str]] = None, - modified_files: t.Optional[t.List[str]] = None, - ignore_app_dependencies_components: t.Optional[t.List[str]] = None, - ignore_app_dependencies_filepatterns: t.Optional[t.List[str]] = None, -) -> t.Tuple[t.Set[App], t.Set[App]]: - """ - Return the tuple of test-required apps and non-test-related apps - - :param paths: paths to search for pytest scripts - :param target: target or keywords to get test cases for, explained in `get_pytest_cases` - :param marker_expr: pytest marker expression, `-m` - :param filter_expr: pytest filter expression, `-k` - :param config_rules_str: config rules string - :param preserve_all: preserve all apps - :param extra_default_build_targets: extra default build targets - :param compare_manifest_sha_filepath: check manifest sha filepath - :param modified_components: modified components - :param modified_files: modified files - :param ignore_app_dependencies_components: ignore app dependencies components - :param ignore_app_dependencies_filepatterns: ignore app dependencies filepatterns - :return: tuple of test-required apps and non-test-related apps - """ - # target could be comma separated list - all_apps: t.List[App] = [] - for _t in set(target.split(',')): - all_apps.extend( - find_apps( - paths, - _t, - build_system=IdfCMakeApp, - recursive=True, - build_dir='build_@t_@w', - config_rules_str=config_rules_str or DEFAULT_CONFIG_RULES_STR, - build_log_filename=DEFAULT_BUILD_LOG_FILENAME, - size_json_filename=DEFAULT_SIZE_JSON_FILENAME, - check_warnings=True, - manifest_rootpath=IDF_PATH, - compare_manifest_sha_filepath=compare_manifest_sha_filepath, - manifest_files=get_all_manifest_files(), - default_build_targets=SUPPORTED_TARGETS + (extra_default_build_targets or []), - modified_components=modified_components, - modified_files=modified_files, - ignore_app_dependencies_components=ignore_app_dependencies_components, - ignore_app_dependencies_filepatterns=ignore_app_dependencies_filepatterns, - include_skipped_apps=True, - ) - ) - - pytest_cases = get_pytest_cases( - paths, - target, - marker_expr=marker_expr, - filter_expr=filter_expr, - ) - - modified_pytest_cases = [] - if modified_files: - modified_pytest_scripts = [ - os.path.dirname(f) for f in modified_files if fnmatch.fnmatch(os.path.basename(f), 'pytest_*.py') - ] - if modified_pytest_scripts: - modified_pytest_cases = get_pytest_cases( - modified_pytest_scripts, - target, - marker_expr=marker_expr, - filter_expr=filter_expr, - ) - - # app_path, target, config - pytest_app_path_tuple_dict: t.Dict[t.Tuple[str, str, str], PytestCase] = {} - for case in pytest_cases: - for app in case.apps: - pytest_app_path_tuple_dict[(app.path, app.target, app.config)] = case - - modified_pytest_app_path_tuple_dict: t.Dict[t.Tuple[str, str, str], PytestCase] = {} - for case in modified_pytest_cases: - for app in case.apps: - modified_pytest_app_path_tuple_dict[(app.path, app.target, app.config)] = case - - test_related_apps: t.Set[App] = set() - non_test_related_apps: t.Set[App] = set() - for app in all_apps: - # PytestCase.app.path is idf_relpath - app_path = idf_relpath(app.app_dir) - - # override build_status if test script got modified - if case := modified_pytest_app_path_tuple_dict.get((app_path, app.target, app.config_name)): - test_related_apps.add(app) - app.build_status = BuildStatus.SHOULD_BE_BUILT - app.preserve = True - logging.debug('Found app: %s - required by modified test case %s', app, case.path) - elif app.build_status != BuildStatus.SKIPPED: - if case := pytest_app_path_tuple_dict.get((app_path, app.target, app.config_name)): - test_related_apps.add(app) - # build or not should be decided by the build stage - app.preserve = True - logging.debug('Found test-related app: %s - required by %s', app, case.path) - else: - non_test_related_apps.add(app) - app.preserve = preserve_all - logging.debug('Found non-test-related app: %s', app) - - print(f'Found {len(test_related_apps)} test-related apps') - print(f'Found {len(non_test_related_apps)} non-test-related apps') - - return test_related_apps, non_test_related_apps diff --git a/tools/ci/idf_pytest/tests/conftest.py b/tools/ci/idf_pytest/tests/conftest.py deleted file mode 100644 index e0def5265b6..00000000000 --- a/tools/ci/idf_pytest/tests/conftest.py +++ /dev/null @@ -1,71 +0,0 @@ -# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import logging -import os -import shutil -import sys -import tempfile -import typing as t -from pathlib import Path - -import pytest - -tools_ci_dir = os.path.join(os.path.dirname(__file__), '..', '..') -if tools_ci_dir not in sys.path: - sys.path.append(tools_ci_dir) - -tools_dir = os.path.join(os.path.dirname(__file__), '..', '..', '..') -if tools_dir not in sys.path: - sys.path.append(tools_dir) - -from idf_ci_utils import IDF_PATH # noqa: E402 -from idf_pytest.constants import DEFAULT_LOGDIR # noqa: E402 - - -def create_project(name: str, folder: Path) -> Path: - p = folder / name - p.mkdir(parents=True, exist_ok=True) - (p / 'main').mkdir(parents=True, exist_ok=True) - - with open(p / 'CMakeLists.txt', 'w') as fw: - fw.write( - """cmake_minimum_required(VERSION 3.16) -include($ENV{{IDF_PATH}}/tools/cmake/project.cmake) -project({}) -""".format( - name - ) - ) - - with open(p / 'main' / 'CMakeLists.txt', 'w') as fw: - fw.write( - """idf_component_register(SRCS "{}.c" -INCLUDE_DIRS ".") -""".format( - name - ) - ) - - with open(p / 'main' / f'{name}.c', 'w') as fw: - fw.write( - """#include -void app_main(void) {} -""" - ) - - return p - - -@pytest.fixture -def work_dirpath() -> t.Generator[Path, None, None]: - os.makedirs(os.path.join(IDF_PATH, DEFAULT_LOGDIR), exist_ok=True) - - p = Path(tempfile.mkdtemp(prefix=os.path.join(IDF_PATH, DEFAULT_LOGDIR) + os.sep)) - - try: - yield p - except Exception: - logging.critical('Test is failing, Please check the log in %s', p) - raise - else: - shutil.rmtree(p) diff --git a/tools/ci/idf_pytest/tests/test_get_all_apps.py b/tools/ci/idf_pytest/tests/test_get_all_apps.py deleted file mode 100644 index a6adab8010d..00000000000 --- a/tools/ci/idf_pytest/tests/test_get_all_apps.py +++ /dev/null @@ -1,131 +0,0 @@ -# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -from pathlib import Path - -from idf_pytest.script import get_all_apps -from idf_pytest.script import SUPPORTED_TARGETS - -from conftest import create_project - - -def test_get_all_apps_non(work_dirpath: Path) -> None: - create_project('foo', work_dirpath) - create_project('bar', work_dirpath) - - test_related_apps, non_test_related_apps = get_all_apps([str(work_dirpath)]) - - assert test_related_apps == set() - assert len(non_test_related_apps) == 2 * len(SUPPORTED_TARGETS) - - -def test_get_all_apps_single_dut_test_script(work_dirpath: Path) -> None: - create_project('foo', work_dirpath) - with open(work_dirpath / 'foo' / 'pytest_get_all_apps_single_dut_test_script.py', 'w') as fw: - fw.write( - """import pytest - -@pytest.mark.esp32 -@pytest.mark.esp32s2 -def test_foo(dut): - pass -""" - ) - create_project('bar', work_dirpath) - - test_related_apps, non_test_related_apps = get_all_apps([str(work_dirpath)], target='all') - - assert len(test_related_apps) == 2 - assert len(non_test_related_apps) == 2 * len(SUPPORTED_TARGETS) - 2 - - -def test_get_all_apps_multi_dut_with_markers_test_script(work_dirpath: Path) -> None: - create_project('foo', work_dirpath) - - (work_dirpath / 'foo' / 'pytest_get_all_apps_multi_dut_with_markers_test_script.py').write_text( - """import pytest - -@pytest.mark.esp32 -@pytest.mark.parametrize('count', [2, 3], indirect=True) -def test_foo(dut): - pass -""", - encoding='utf-8', - ) - - test_related_apps, non_test_related_apps = get_all_apps([str(work_dirpath)], target='all') - - assert len(test_related_apps) == 1 - assert len(non_test_related_apps) == len(SUPPORTED_TARGETS) - 1 - - -def test_get_all_apps_multi_dut_test_script(work_dirpath: Path) -> None: - create_project('foo', work_dirpath) - with open(work_dirpath / 'foo' / 'pytest_get_all_apps_multi_dut_test_script.py', 'w') as fw: - fw.write( - """import pytest - -@pytest.mark.parametrize( - 'count, target', [ - (2, 'esp32s2|esp32s3'), - (3, 'esp32|esp32s3|esp32'), - ], indirect=True -) -def test_foo(dut): - pass -""" - ) - - test_related_apps, non_test_related_apps = get_all_apps([str(work_dirpath)], target='esp32s2,esp32s3') - assert len(test_related_apps) == 2 - assert len(non_test_related_apps) == 0 - - test_related_apps, non_test_related_apps = get_all_apps([str(work_dirpath)], target='esp32,esp32s3,esp32') - assert len(test_related_apps) == 2 - assert len(non_test_related_apps) == 0 - - test_related_apps, non_test_related_apps = get_all_apps([str(work_dirpath)], target='all') - assert len(test_related_apps) == 3 - assert len(non_test_related_apps) == len(SUPPORTED_TARGETS) - 3 - - test_related_apps, non_test_related_apps = get_all_apps([str(work_dirpath)], target='foo,bar') - assert len(test_related_apps) == 0 - assert len(non_test_related_apps) == 0 - - -def test_get_all_apps_modified_pytest_script(work_dirpath: Path) -> None: - create_project('foo', work_dirpath) - create_project('bar', work_dirpath) - - (work_dirpath / 'pytest_get_all_apps_modified_pytest_script.py').write_text( - """import pytest -import os - -@pytest.mark.parametrize('count, target', [(2, 'esp32')], indirect=True) -@pytest.mark.parametrize('app_path', [ - '{}|{}'.format(os.path.join(os.path.dirname(__file__), 'foo'), os.path.join(os.path.dirname(__file__), 'bar')), - ], indirect=True -) -def test_multi_foo_bar(dut): - pass -""", - encoding='utf-8', - ) - - test_related_apps, non_test_related_apps = get_all_apps([str(work_dirpath)], target='all') - assert len(test_related_apps) == 2 # foo-esp32, bar-esp32 - assert len(non_test_related_apps) == 2 * len(SUPPORTED_TARGETS) - 2 - - test_related_apps, non_test_related_apps = get_all_apps( - [str(work_dirpath)], target='all', modified_files=[], modified_components=[] - ) - assert len(test_related_apps) == 0 - assert len(non_test_related_apps) == 0 - - test_related_apps, non_test_related_apps = get_all_apps( - [str(work_dirpath)], - target='all', - modified_files=[str(work_dirpath / 'pytest_get_all_apps_modified_pytest_script.py')], - modified_components=[], - ) - assert len(test_related_apps) == 2 - assert len(non_test_related_apps) == 0 diff --git a/tools/ci/idf_pytest/tests/test_get_pytest_cases.py b/tools/ci/idf_pytest/tests/test_get_pytest_cases.py deleted file mode 100644 index abf7117fca6..00000000000 --- a/tools/ci/idf_pytest/tests/test_get_pytest_cases.py +++ /dev/null @@ -1,179 +0,0 @@ -# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD -# SPDX-License-Identifier: Apache-2.0 -import textwrap -from pathlib import Path - -from idf_pytest.constants import CollectMode -from idf_pytest.script import get_pytest_cases - -TEMPLATE_SCRIPT = ''' -import pytest - -@pytest.mark.esp32 -@pytest.mark.esp32s2 -def test_foo_single(dut): - pass - -@pytest.mark.parametrize('target', [ - 'esp32', - 'esp32c3', -]) -def test_foo_single_with_param(dut): - pass - -@pytest.mark.parametrize( - 'count, target', [ - (2, 'esp32|esp32s2'), - (3, 'esp32s2|esp32s2|esp32s3'), - ], indirect=True -) -def test_foo_multi(dut): - pass - -@pytest.mark.esp32 -@pytest.mark.esp32s2 -@pytest.mark.parametrize( - 'count', [2], indirect=True -) -def test_foo_multi_with_marker(dut): - pass -''' - - -def test_get_pytest_cases_single_specific(work_dirpath: Path) -> None: - script = work_dirpath / 'pytest_get_pytest_cases_single_specific.py' - script.write_text(TEMPLATE_SCRIPT) - cases = get_pytest_cases([str(work_dirpath)], 'esp32') - - assert len(cases) == 2 - assert cases[0].targets == ['esp32'] - assert cases[0].name == 'test_foo_single' - assert cases[1].targets == ['esp32'] - assert cases[1].name == 'test_foo_single_with_param' - - -def test_get_pytest_cases_multi_specific(work_dirpath: Path) -> None: - script = work_dirpath / 'pytest_get_pytest_cases_multi_specific.py' - script.write_text(TEMPLATE_SCRIPT) - cases = get_pytest_cases([str(work_dirpath)], 'esp32s2,esp32s2, esp32s3') - - assert len(cases) == 1 - assert cases[0].targets == ['esp32s2', 'esp32s2', 'esp32s3'] - - cases = get_pytest_cases([str(work_dirpath)], 'esp32s3,esp32s2,esp32s2') # order matters - assert len(cases) == 0 - - -def test_get_pytest_cases_multi_all(work_dirpath: Path) -> None: - script = work_dirpath / 'pytest_get_pytest_cases_multi_all.py' - script.write_text(TEMPLATE_SCRIPT) - cases = get_pytest_cases([str(work_dirpath)], CollectMode.MULTI_ALL_WITH_PARAM) - - assert len(cases) == 2 - assert cases[0].targets == ['esp32', 'esp32s2'] - assert cases[1].targets == ['esp32s2', 'esp32s2', 'esp32s3'] - - -def test_get_pytest_cases_all(work_dirpath: Path) -> None: - script = work_dirpath / 'pytest_get_pytest_cases_all.py' - script.write_text(TEMPLATE_SCRIPT) - cases = get_pytest_cases([str(work_dirpath)], CollectMode.ALL) - - assert len(cases) == 8 - assert cases[0].targets == ['esp32', 'esp32s2'] - assert cases[0].name == 'test_foo_multi' - - assert cases[1].targets == ['esp32s2', 'esp32s2', 'esp32s3'] - assert cases[1].name == 'test_foo_multi' - - assert cases[2].targets == ['esp32', 'esp32'] - assert cases[2].name == 'test_foo_multi_with_marker' - - assert cases[3].targets == ['esp32s2', 'esp32s2'] - assert cases[3].name == 'test_foo_multi_with_marker' - - assert cases[4].targets == ['esp32'] - assert cases[4].name == 'test_foo_single' - - assert cases[5].targets == ['esp32s2'] - assert cases[5].name == 'test_foo_single' - - assert cases[6].targets == ['esp32'] - assert cases[6].name == 'test_foo_single_with_param' - - assert cases[7].targets == ['esp32c3'] - assert cases[7].name == 'test_foo_single_with_param' - - -def test_multi_with_marker_and_app_path(work_dirpath: Path) -> None: - script = work_dirpath / 'pytest_multi_with_marker_and_app_path.py' - script.write_text( - textwrap.dedent( - ''' - import pytest - - @pytest.mark.esp32c2 - @pytest.mark.parametrize( - 'count,app_path', [ - (2, 'foo|bar'), - (3, 'foo|bar|baz'), - ], indirect=True - ) - def test_foo_multi_with_marker_and_app_path(dut): - pass - ''' - ) - ) - cases = get_pytest_cases([str(work_dirpath)], 'esp32c3,esp32c3') - assert len(cases) == 0 - - cases = get_pytest_cases([str(work_dirpath)], 'esp32c2,esp32c2') - assert len(cases) == 1 - assert cases[0].targets == ['esp32c2', 'esp32c2'] - - cases = get_pytest_cases([str(work_dirpath)], 'esp32c2,esp32c2,esp32c2') - assert len(cases) == 1 - assert cases[0].targets == ['esp32c2', 'esp32c2', 'esp32c2'] - - -def test_filter_with_sdkconfig_name(work_dirpath: Path) -> None: - script = work_dirpath / 'pytest_filter_with_sdkconfig_name.py' - script.write_text( - textwrap.dedent( - ''' - import pytest - - @pytest.mark.esp32 - @pytest.mark.parametrize( - 'config', [ - 'foo', - 'bar', - ], indirect=True - ) - def test_filter_with_sdkconfig_name_single_dut(dut): - pass - - @pytest.mark.esp32 - @pytest.mark.parametrize( - 'count', [2], indirect=True - ) - @pytest.mark.parametrize( - 'config', [ - 'foo|bar', - 'bar|baz', - ], indirect=True - ) - def test_filter_with_sdkconfig_name_multi_dut(dut): - pass - ''' - ) - ) - - cases = get_pytest_cases([str(work_dirpath)], 'esp32', config_name='foo') - assert len(cases) == 1 - - cases = get_pytest_cases([str(work_dirpath)], 'esp32,esp32', config_name='foo') - assert len(cases) == 1 - - cases = get_pytest_cases([str(work_dirpath)], 'esp32,esp32', config_name='bar') - assert len(cases) == 2 diff --git a/tools/ci/idf_pytest/utils.py b/tools/ci/idf_pytest/utils.py index 6fa8e7502d5..4eab9a81b93 100644 --- a/tools/ci/idf_pytest/utils.py +++ b/tools/ci/idf_pytest/utils.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import logging import os @@ -30,8 +30,8 @@ def merge_junit_files(junit_files: t.List[str], target_path: str) -> None: testcases: t.Dict[str, ET.Element] = {} for junit in junit_files: logging.info(f'Merging {junit} to {target_path}') - tree: ET.ElementTree = ET.parse(junit) - testsuite: ET.Element = tree.getroot() + tree: ET.ElementTree = ET.parse(junit) # type: ignore + testsuite: ET.Element = tree.getroot() # type: ignore for testcase in testsuite.findall('testcase'): name: str = testcase.get('name') if testcase.get('name') else '' # type: ignore @@ -62,10 +62,6 @@ def merge_junit_files(junit_files: t.List[str], target_path: str) -> None: fw.write(ET.tostring(merged_testsuite)) -def comma_sep_str_to_list(s: str) -> t.List[str]: - return [s.strip() for s in s.split(',') if s.strip()] - - def normalize_testcase_file_path(file: str, app_path: t.Union[str, tuple]) -> str: """ Normalize file paths to a consistent format, resolving relative paths based on the `app_path`. @@ -82,9 +78,7 @@ def normalize_testcase_file_path(file: str, app_path: t.Union[str, tuple]) -> st def normalize_path(file_path: str, app_path: str) -> str: """Helper function to normalize a single path.""" if not os.path.isabs(file_path): - resolved_path = os.path.normpath( - os.path.join(app_path, file_path.removeprefix('./').removeprefix('../')) - ) + resolved_path = os.path.normpath(os.path.join(app_path, file_path.removeprefix('./').removeprefix('../'))) else: resolved_path = os.path.normpath(file_path) diff --git a/tools/ci/utils.sh b/tools/ci/utils.sh index 17951e2173a..a4c3a4875b3 100644 --- a/tools/ci/utils.sh +++ b/tools/ci/utils.sh @@ -5,7 +5,14 @@ function add_ssh_keys() { mkdir -p ~/.ssh chmod 700 ~/.ssh echo -n "${key_string}" >~/.ssh/id_rsa_base64 - base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 >~/.ssh/id_rsa + # Detect base64 implementation via --help output + if base64 --help 2>&1 | grep -q -- '--ignore-garbage'; then + # GNU coreutils base64 + base64 --decode --ignore-garbage ~/.ssh/id_rsa_base64 >~/.ssh/id_rsa + else + # macOS/BSD base64 - requires stdin or -i flag + base64 --decode -i ~/.ssh/id_rsa_base64 -o ~/.ssh/id_rsa + fi chmod 600 ~/.ssh/id_rsa } @@ -96,6 +103,25 @@ function run_cmd() { fi } +function pytest_for_ut() { + pytest_args="-c NUL -p no:idf-ci -p no:pytest_embedded" + + if [ -n "${CI_JOB_ID-}" ]; then + if [ -z "${KNOWN_FAILURE_CASES_FILE_NAME-}" ]; then + echo "Error: KNOWN_FAILURE_CASES_FILE_NAME is not set." + return 1 + fi + + pytest_args="${pytest_args} --junitxml XUNIT_RESULT_${CI_JOB_ID}.xml --ignore-result-files ${KNOWN_FAILURE_CASES_FILE_NAME}" + + if [ ! -e "${KNOWN_FAILURE_CASES_FILE_NAME}" ]; then + run_cmd idf-ci gitlab download-known-failure-cases-file "${KNOWN_FAILURE_CASES_FILE_NAME}" + fi + fi + + run_cmd pytest "$@" "${pytest_args}" +} + # Retries a command RETRY_ATTEMPTS times in case of failure # Inspired by https://stackoverflow.com/a/8351489 function retry_failed() { diff --git a/tools/requirements/requirements.ci.txt b/tools/requirements/requirements.ci.txt index 1b2ada0d29c..9852754ad66 100644 --- a/tools/requirements/requirements.ci.txt +++ b/tools/requirements/requirements.ci.txt @@ -6,12 +6,10 @@ # https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/tools/idf-tools.html # ci +idf-ci>=0.3,<1 + coverage -idf-build-apps jsonschema junit_xml -python-gitlab -pyyaml SimpleWebSocketServer -minio prettytable diff --git a/tools/test_apps/build_system/ld_non_contiguous_memory/pytest_ld_non_contiguous_memory.py b/tools/test_apps/build_system/ld_non_contiguous_memory/pytest_ld_non_contiguous_memory.py index b04713fc783..d7ef0b932ba 100644 --- a/tools/test_apps/build_system/ld_non_contiguous_memory/pytest_ld_non_contiguous_memory.py +++ b/tools/test_apps/build_system/ld_non_contiguous_memory/pytest_ld_non_contiguous_memory.py @@ -4,8 +4,6 @@ import pytest from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize -TEST_APP_IN_FLASH = [pytest.param('app_in_flash', marks=pytest.mark.esp32p4)] - @pytest.mark.generic @idf_parametrize('config', ['app_in_flash'], indirect=['config']) diff --git a/tools/test_apps/security/secure_boot/README.md b/tools/test_apps/security/secure_boot/README.md index 306d825aaf6..1c251de5bca 100644 --- a/tools/test_apps/security/secure_boot/README.md +++ b/tools/test_apps/security/secure_boot/README.md @@ -1,5 +1,5 @@ -| Supported Targets | ESP32 | ESP32-C3 | ESP32-C5 | ESP32-C6 | ESP32-H2 | ESP32-P4 | ESP32-S2 | ESP32-S3 | -| ----------------- | ----- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | +| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-C5 | ESP32-C6 | ESP32-C61 | ESP32-H2 | ESP32-H21 | ESP32-H4 | ESP32-P4 | ESP32-S2 | ESP32-S3 | +| ----------------- | ----- | -------- | -------- | -------- | -------- | --------- | -------- | --------- | -------- | -------- | -------- | -------- | # Secure Boot diff --git a/tools/test_apps/system/gdb_loadable_elf/test_gdb_loadable_elf_util/loadable_app_serial.py b/tools/test_apps/system/gdb_loadable_elf/test_gdb_loadable_elf_util/loadable_app_serial.py index 2d38453fbe3..a3e7bef4cd9 100644 --- a/tools/test_apps/system/gdb_loadable_elf/test_gdb_loadable_elf_util/loadable_app_serial.py +++ b/tools/test_apps/system/gdb_loadable_elf/test_gdb_loadable_elf_util/loadable_app_serial.py @@ -1,11 +1,12 @@ -# SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Unlicense OR CC0-1.0 import logging -from typing import Any, Optional +from typing import Any +from typing import Optional import esptool from pytest_embedded_idf.app import IdfApp -from pytest_embedded_serial_esp.serial import EspSerial, EsptoolArgs +from pytest_embedded_serial_esp.serial import EspSerial class LoadableAppSerial(EspSerial): @@ -18,7 +19,7 @@ class LoadableAppSerial(EspSerial): self.app = app if not hasattr(self.app, 'target'): - raise ValueError(f'Idf app not parsable. Please check if it\'s valid: {self.app.binary_path}') + raise ValueError(f"Idf app not parsable. Please check if it's valid: {self.app.binary_path}") if target and self.app.target and self.app.target != target: raise ValueError(f'Targets do not match. App target: {self.app.target}, Cmd target: {target}.') @@ -31,23 +32,12 @@ class LoadableAppSerial(EspSerial): def _start(self) -> None: self.load_ram() - @EspSerial.use_esptool(hard_reset_after=False, no_stub=True) + @EspSerial.use_esptool() def load_ram(self) -> None: if not self.app.bin_file: logging.error('No image file detected. Skipping load ram...') return - f_bin_file = open(self.app.bin_file, 'rb') - - default_kwargs = { - 'filename': f_bin_file, - 'chip': self.esp.CHIP_NAME.lower().replace('-', ''), - } - - load_ram_args = EsptoolArgs(**default_kwargs) - - try: + with open(self.app.bin_file, 'rb') as f_bin_file: self.esp.change_baud(460800) - esptool.load_ram(self.esp, load_ram_args) - finally: - f_bin_file.close() + esptool.load_ram(self.esp, f_bin_file) diff --git a/tools/test_apps/system/mmu_page_size/pytest_mmu_page_size.py b/tools/test_apps/system/mmu_page_size/pytest_mmu_page_size.py index efe62c80d28..4213f2fec0f 100644 --- a/tools/test_apps/system/mmu_page_size/pytest_mmu_page_size.py +++ b/tools/test_apps/system/mmu_page_size/pytest_mmu_page_size.py @@ -1,9 +1,8 @@ -# SPDX-FileCopyrightText: 2024-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2024-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 import os import pytest -from artifacts_handler import ArtifactType from idf_ci_utils import IDF_PATH from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize @@ -20,9 +19,7 @@ def test_app_mmu_page_size_32k_and_bootloader_mmu_page_size_64k(dut: Dut, app_do path_to_mmu_page_size_64k_build = os.path.join(dut.app.app_path, f'build_{dut.target}_{app_config}') if app_downloader: - app_downloader.download_app( - os.path.relpath(path_to_mmu_page_size_64k_build, IDF_PATH), ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES - ) + app_downloader.download_app(os.path.relpath(path_to_mmu_page_size_64k_build, IDF_PATH), 'flash') dut.serial.bootloader_flash(path_to_mmu_page_size_64k_build) dut.expect('MMU page size mismatch') @@ -42,9 +39,7 @@ def test_app_mmu_page_size_64k_and_bootloader_mmu_page_size_32k(dut: Dut, app_do path_to_mmu_page_size_32k_build = os.path.join(dut.app.app_path, f'build_{dut.target}_{app_config}') if app_downloader: - app_downloader.download_app( - os.path.relpath(path_to_mmu_page_size_32k_build, IDF_PATH), ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES - ) + app_downloader.download_app(os.path.relpath(path_to_mmu_page_size_32k_build, IDF_PATH), 'flash') dut.serial.bootloader_flash(path_to_mmu_page_size_32k_build) dut.expect('MMU page size mismatch') diff --git a/tools/test_apps/system/unicore_bootloader/pytest_unicore_bootloader.py b/tools/test_apps/system/unicore_bootloader/pytest_unicore_bootloader.py index f30e7c7fee6..6a88260bf6e 100644 --- a/tools/test_apps/system/unicore_bootloader/pytest_unicore_bootloader.py +++ b/tools/test_apps/system/unicore_bootloader/pytest_unicore_bootloader.py @@ -1,10 +1,9 @@ -# SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2023-2026 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: CC0-1.0 import os import re import pytest -from artifacts_handler import ArtifactType from idf_ci_utils import IDF_PATH from pytest_embedded import Dut from pytest_embedded_idf.utils import idf_parametrize @@ -23,9 +22,7 @@ def test_multicore_app_and_unicore_bootloader(dut: Dut, app_downloader, config) path_to_unicore_build = os.path.join(dut.app.app_path, f'build_{dut.target}_{app_config}') if app_downloader: - app_downloader.download_app( - os.path.relpath(path_to_unicore_build, IDF_PATH), ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES - ) + app_downloader.download_app(os.path.relpath(path_to_unicore_build, IDF_PATH), 'flash') dut.serial.bootloader_flash(path_to_unicore_build) dut.expect('Unicore bootloader') @@ -49,9 +46,7 @@ def test_unicore_app_and_multicore_bootloader(dut: Dut, app_downloader, config) path_to_multicore_build = os.path.join(dut.app.app_path, f'build_{dut.target}_{app_config}') if app_downloader: - app_downloader.download_app( - os.path.relpath(path_to_multicore_build, IDF_PATH), ArtifactType.BUILD_DIR_WITHOUT_MAP_AND_ELF_FILES - ) + app_downloader.download_app(os.path.relpath(path_to_multicore_build, IDF_PATH), 'flash') dut.serial.bootloader_flash(path_to_multicore_build) dut.expect('Multicore bootloader')