From 0a95151a75af0aa9bd76ce2f167fbb17f3eb3d8e Mon Sep 17 00:00:00 2001 From: Marius Vikhammer Date: Fri, 4 Dec 2020 11:35:21 +0800 Subject: [PATCH] unit_test: Refactor all performance tests that rely on cache compensated timer There is no ccomp timer on C3, which means our performance tests will start failing again due to variance caused by cache misses. This MR adds TEST_PERFORMANCE_CCOMP_ macro that will only fail performance test if CCOMP timer is supported on the target --- components/driver/test/test_sdio.c | 12 ++++++------ components/esp32/test/test_sha.c | 4 ++-- components/esp32s2/test/test_sha.c | 4 ++-- components/mbedtls/test/test_aes_perf.c | 2 +- components/mbedtls/test/test_rsa.c | 8 ++++---- components/mbedtls/test/test_sha_perf.c | 2 +- components/soc/esp32/include/soc/soc_caps.h | 1 + components/soc/esp32s2/include/soc/soc_caps.h | 1 + components/soc/esp32s3/include/soc/soc_caps.h | 2 ++ components/spi_flash/test/test_esp_flash.c | 4 ++-- components/spi_flash/test/test_spi_flash.c | 4 ++-- components/vfs/test/test_vfs_fd.c | 4 ++-- .../components/test_utils/include/test_utils.h | 16 ++++++++++++++++ 13 files changed, 42 insertions(+), 22 deletions(-) diff --git a/components/driver/test/test_sdio.c b/components/driver/test/test_sdio.c index bc929bce43e..7d24942f644 100644 --- a/components/driver/test/test_sdio.c +++ b/components/driver/test/test_sdio.c @@ -355,13 +355,13 @@ static void log_performance_tohost(uint32_t speed, const sdio_test_config_t* con if (!config->check_data) { switch (config->sdio_mode) { case SDIO_4BIT: - TEST_PERFORMANCE_GREATER_THAN(SDIO_THROUGHPUT_KBSEC_TOHOST_4BIT, "%d", speed); + TEST_PERFORMANCE_CCOMP_GREATER_THAN(SDIO_THROUGHPUT_KBSEC_TOHOST_4BIT, "%d", speed); break; case SDIO_1BIT: - TEST_PERFORMANCE_GREATER_THAN(SDIO_THROUGHPUT_KBSEC_TOHOST_1BIT, "%d", speed); + TEST_PERFORMANCE_CCOMP_GREATER_THAN(SDIO_THROUGHPUT_KBSEC_TOHOST_1BIT, "%d", speed); break; case SDIO_SPI: - TEST_PERFORMANCE_GREATER_THAN(SDIO_THROUGHPUT_KBSEC_TOHOST_SPI, "%d", speed); + TEST_PERFORMANCE_CCOMP_GREATER_THAN(SDIO_THROUGHPUT_KBSEC_TOHOST_SPI, "%d", speed); break; } } @@ -441,13 +441,13 @@ static void log_performance_frhost(uint32_t speed, const sdio_test_config_t* con if (!config->check_data) { switch (config->sdio_mode) { case SDIO_4BIT: - TEST_PERFORMANCE_GREATER_THAN(SDIO_THROUGHPUT_KBSEC_FRHOST_4BIT, "%d", speed); + TEST_PERFORMANCE_CCOMP_GREATER_THAN(SDIO_THROUGHPUT_KBSEC_FRHOST_4BIT, "%d", speed); break; case SDIO_1BIT: - TEST_PERFORMANCE_GREATER_THAN(SDIO_THROUGHPUT_KBSEC_FRHOST_1BIT, "%d", speed); + TEST_PERFORMANCE_CCOMP_GREATER_THAN(SDIO_THROUGHPUT_KBSEC_FRHOST_1BIT, "%d", speed); break; case SDIO_SPI: - TEST_PERFORMANCE_GREATER_THAN(SDIO_THROUGHPUT_KBSEC_FRHOST_SPI, "%d", speed); + TEST_PERFORMANCE_CCOMP_GREATER_THAN(SDIO_THROUGHPUT_KBSEC_FRHOST_SPI, "%d", speed); break; } } diff --git a/components/esp32/test/test_sha.c b/components/esp32/test/test_sha.c index b2795171b0f..075191475c8 100644 --- a/components/esp32/test/test_sha.c +++ b/components/esp32/test/test_sha.c @@ -63,8 +63,8 @@ TEST_CASE("Test esp_sha()", "[hw_crypto]") free(buffer); - TEST_PERFORMANCE_LESS_THAN(TIME_SHA1_32KB, "%dus", us_sha1); - TEST_PERFORMANCE_LESS_THAN(TIME_SHA512_32KB, "%dus", us_sha512); + TEST_PERFORMANCE_CCOMP_LESS_THAN(TIME_SHA1_32KB, "%dus", us_sha1); + TEST_PERFORMANCE_CCOMP_LESS_THAN(TIME_SHA512_32KB, "%dus", us_sha512); } TEST_CASE("Test esp_sha() function with long input", "[hw_crypto]") diff --git a/components/esp32s2/test/test_sha.c b/components/esp32s2/test/test_sha.c index 42bac29f8dd..6a57faf08ba 100644 --- a/components/esp32s2/test/test_sha.c +++ b/components/esp32s2/test/test_sha.c @@ -58,8 +58,8 @@ TEST_CASE("Test esp_sha()", "[hw_crypto]") free(buffer); - TEST_PERFORMANCE_LESS_THAN(TIME_SHA1_32KB, "%dus", us_sha1); - TEST_PERFORMANCE_LESS_THAN(TIME_SHA512_32KB, "%dus", us_sha512); + TEST_PERFORMANCE_CCOMP_LESS_THAN(TIME_SHA1_32KB, "%dus", us_sha1); + TEST_PERFORMANCE_CCOMP_LESS_THAN(TIME_SHA512_32KB, "%dus", us_sha512); } TEST_CASE("Test esp_sha() function with long input", "[hw_crypto]") diff --git a/components/mbedtls/test/test_aes_perf.c b/components/mbedtls/test/test_aes_perf.c index 5a455f4f38f..6224ed7b409 100644 --- a/components/mbedtls/test/test_aes_perf.c +++ b/components/mbedtls/test/test_aes_perf.c @@ -64,6 +64,6 @@ TEST_CASE("mbedtls AES performance", "[aes][timeout=60]") printf("Encryption rate %.3fMB/sec\n", mb_sec); #ifdef CONFIG_MBEDTLS_HARDWARE_AES // Don't put a hard limit on software AES performance - TEST_PERFORMANCE_GREATER_THAN(AES_CBC_THROUGHPUT_MBSEC, "%.3fMB/sec", mb_sec); + TEST_PERFORMANCE_CCOMP_GREATER_THAN(AES_CBC_THROUGHPUT_MBSEC, "%.3fMB/sec", mb_sec); #endif } diff --git a/components/mbedtls/test/test_rsa.c b/components/mbedtls/test/test_rsa.c index 1ef1f07465e..ed4fd4e738a 100644 --- a/components/mbedtls/test/test_rsa.c +++ b/components/mbedtls/test/test_rsa.c @@ -366,11 +366,11 @@ static void rsa_key_operations(int keysize, bool check_performance, bool use_bli private_perf = ccomp_timer_stop(); if (check_performance && keysize == 2048) { - TEST_PERFORMANCE_LESS_THAN(RSA_2048KEY_PUBLIC_OP, "public operations %d us", public_perf); - TEST_PERFORMANCE_LESS_THAN(RSA_2048KEY_PRIVATE_OP, "private operations %d us", private_perf); + TEST_PERFORMANCE_CCOMP_LESS_THAN(RSA_2048KEY_PUBLIC_OP, "public operations %d us", public_perf); + TEST_PERFORMANCE_CCOMP_LESS_THAN(RSA_2048KEY_PRIVATE_OP, "private operations %d us", private_perf); } else if (check_performance && keysize == 4096) { - TEST_PERFORMANCE_LESS_THAN(RSA_4096KEY_PUBLIC_OP, "public operations %d us", public_perf); - TEST_PERFORMANCE_LESS_THAN(RSA_4096KEY_PRIVATE_OP, "private operations %d us", private_perf); + TEST_PERFORMANCE_CCOMP_LESS_THAN(RSA_4096KEY_PUBLIC_OP, "public operations %d us", public_perf); + TEST_PERFORMANCE_CCOMP_LESS_THAN(RSA_4096KEY_PRIVATE_OP, "private operations %d us", private_perf); } TEST_ASSERT_EQUAL_MEMORY_MESSAGE(orig_buf, decrypted_buf, keysize / 8, "RSA operation"); diff --git a/components/mbedtls/test/test_sha_perf.c b/components/mbedtls/test/test_sha_perf.c index c21085fb26f..28e7a166ed6 100644 --- a/components/mbedtls/test/test_sha_perf.c +++ b/components/mbedtls/test/test_sha_perf.c @@ -50,6 +50,6 @@ TEST_CASE("mbedtls SHA performance", "[aes]") printf("SHA256 rate %.3fMB/sec\n", mb_sec); #ifdef CONFIG_MBEDTLS_HARDWARE_SHA // Don't put a hard limit on software SHA performance - TEST_PERFORMANCE_GREATER_THAN(SHA256_THROUGHPUT_MBSEC, "%.3fMB/sec", mb_sec); + TEST_PERFORMANCE_CCOMP_GREATER_THAN(SHA256_THROUGHPUT_MBSEC, "%.3fMB/sec", mb_sec); #endif } diff --git a/components/soc/esp32/include/soc/soc_caps.h b/components/soc/esp32/include/soc/soc_caps.h index e5e2393e480..9f37a65d81b 100644 --- a/components/soc/esp32/include/soc/soc_caps.h +++ b/components/soc/esp32/include/soc/soc_caps.h @@ -71,6 +71,7 @@ #define SOC_CPU_CORES_NUM 2 #define SOC_ULP_SUPPORTED 1 #define SOC_RTC_SLOW_MEM_SUPPORTED 1 +#define SOC_CCOMP_TIMER_SUPPORTED 1 /*-------------------------- ADC CAPS ----------------------------------------*/ #define SOC_ADC_PERIPH_NUM (2) diff --git a/components/soc/esp32s2/include/soc/soc_caps.h b/components/soc/esp32s2/include/soc/soc_caps.h index ff620ac6ade..0d1436ef529 100644 --- a/components/soc/esp32s2/include/soc/soc_caps.h +++ b/components/soc/esp32s2/include/soc/soc_caps.h @@ -47,6 +47,7 @@ #define SOC_PCNT_SUPPORTED 1 #define SOC_ULP_SUPPORTED 1 #define SOC_RTC_SLOW_MEM_SUPPORTED 1 +#define SOC_CCOMP_TIMER_SUPPORTED 1 #define SOC_CACHE_SUPPORT_WRAP 1 diff --git a/components/soc/esp32s3/include/soc/soc_caps.h b/components/soc/esp32s3/include/soc/soc_caps.h index d080d9e9e24..d3e493ac4a7 100644 --- a/components/soc/esp32s3/include/soc/soc_caps.h +++ b/components/soc/esp32s3/include/soc/soc_caps.h @@ -14,6 +14,8 @@ #define SOC_CACHE_SUPPORT_WRAP 1 #define SOC_ULP_SUPPORTED 1 #define SOC_RTC_SLOW_MEM_SUPPORTED 1 +#define SOC_CCOMP_TIMER_SUPPORTED 1 + /*-------------------------- ADC CAPS ----------------------------------------*/ #include "adc_caps.h" diff --git a/components/spi_flash/test/test_esp_flash.c b/components/spi_flash/test/test_esp_flash.c index 7588fa1511d..6722891d03f 100644 --- a/components/spi_flash/test/test_esp_flash.c +++ b/components/spi_flash/test/test_esp_flash.c @@ -1008,8 +1008,8 @@ static void test_flash_read_write_performance(const esp_partition_t *part) TEST_ASSERT_EQUAL_HEX8_ARRAY(data_to_write, data_read, total_len); #if !CONFIG_SPIRAM && !CONFIG_FREERTOS_CHECK_PORT_CRITICAL_COMPLIANCE -# define CHECK_DATA(bus, suffix) TEST_PERFORMANCE_GREATER_THAN(FLASH_SPEED_BYTE_PER_SEC_##bus##suffix, "%d", speed_##suffix) -# define CHECK_ERASE(bus, var) TEST_PERFORMANCE_GREATER_THAN(FLASH_SPEED_BYTE_PER_SEC_##bus##ERASE, "%d", var) +# define CHECK_DATA(bus, suffix) TEST_PERFORMANCE_CCOMP_GREATER_THAN(FLASH_SPEED_BYTE_PER_SEC_##bus##suffix, "%d", speed_##suffix) +# define CHECK_ERASE(bus, var) TEST_PERFORMANCE_CCOMP_GREATER_THAN(FLASH_SPEED_BYTE_PER_SEC_##bus##ERASE, "%d", var) #else # define CHECK_DATA(bus, suffix) ((void)speed_##suffix) # define CHECK_ERASE(bus, var) ((void)var) diff --git a/components/spi_flash/test/test_spi_flash.c b/components/spi_flash/test/test_spi_flash.c index bb7673ca63e..99cc7110a71 100644 --- a/components/spi_flash/test/test_spi_flash.c +++ b/components/spi_flash/test/test_spi_flash.c @@ -312,8 +312,8 @@ TEST_CASE("Test spi_flash read/write performance", "[spi_flash]") // Data checks are disabled when PSRAM is used or in Freertos compliance check test #if !CONFIG_SPIRAM && !CONFIG_FREERTOS_CHECK_PORT_CRITICAL_COMPLIANCE -# define CHECK_DATA(suffix) TEST_PERFORMANCE_GREATER_THAN(FLASH_SPEED_BYTE_PER_SEC_LEGACY_##suffix, "%d", speed_##suffix) -# define CHECK_ERASE(var) TEST_PERFORMANCE_GREATER_THAN(FLASH_SPEED_BYTE_PER_SEC_LEGACY_ERASE, "%d", var) +# define CHECK_DATA(suffix) TEST_PERFORMANCE_CCOMP_GREATER_THAN(FLASH_SPEED_BYTE_PER_SEC_LEGACY_##suffix, "%d", speed_##suffix) +# define CHECK_ERASE(var) TEST_PERFORMANCE_CCOMP_GREATER_THAN(FLASH_SPEED_BYTE_PER_SEC_LEGACY_ERASE, "%d", var) #else # define CHECK_DATA(suffix) ((void)speed_##suffix) # define CHECK_ERASE(var) ((void)var) diff --git a/components/vfs/test/test_vfs_fd.c b/components/vfs/test/test_vfs_fd.c index 639ec417b1e..29a3e868211 100644 --- a/components/vfs/test/test_vfs_fd.c +++ b/components/vfs/test/test_vfs_fd.c @@ -260,9 +260,9 @@ TEST_CASE("Open & write & close through VFS passes performance test", "[vfs]") const int ns_per_iter = (int) (time_diff_us * 1000 / iter_count); TEST_ESP_OK( esp_vfs_unregister(VFS_PREF1) ); #ifdef CONFIG_SPIRAM - TEST_PERFORMANCE_LESS_THAN(VFS_OPEN_WRITE_CLOSE_TIME_PSRAM, "%dns", ns_per_iter); + TEST_PERFORMANCE_CCOMP_LESS_THAN(VFS_OPEN_WRITE_CLOSE_TIME_PSRAM, "%dns", ns_per_iter); #else - TEST_PERFORMANCE_LESS_THAN(VFS_OPEN_WRITE_CLOSE_TIME, "%dns", ns_per_iter); + TEST_PERFORMANCE_CCOMP_LESS_THAN(VFS_OPEN_WRITE_CLOSE_TIME, "%dns", ns_per_iter); #endif } diff --git a/tools/unit-test-app/components/test_utils/include/test_utils.h b/tools/unit-test-app/components/test_utils/include/test_utils.h index 2cfb2a5ea28..77281ac7942 100644 --- a/tools/unit-test-app/components/test_utils/include/test_utils.h +++ b/tools/unit-test-app/components/test_utils/include/test_utils.h @@ -21,6 +21,7 @@ #include "freertos/FreeRTOS.h" #include "freertos/task.h" #include "unity.h" +#include "soc/soc_caps.h" /* include performance pass standards header file */ #include "idf_performance.h" #include "idf_performance_target.h" @@ -50,6 +51,21 @@ extern "C" { TEST_ASSERT(value > PERFORMANCE_CON(IDF_PERFORMANCE_MIN_, name)); \ } while(0) +/* Macros to be used when performance is calculated using the cache compensated timer + will not assert if ccomp not supported */ +#if SOC_CCOMP_TIMER_SUPPORTED +#define TEST_PERFORMANCE_CCOMP_GREATER_THAN(name, value_fmt, value) \ + TEST_PERFORMANCE_GREATER_THAN(name, value_fmt, value) +#define TEST_PERFORMANCE_CCOMP_LESS_THAN(name, value_fmt, value) \ + TEST_PERFORMANCE_LESS_THAN(name, value_fmt, value) +#else +#define TEST_PERFORMANCE_CCOMP_GREATER_THAN(name, value_fmt, value) \ + printf("[Performance][" PERFORMANCE_STR(name) "]: "value_fmt"\n", value); +#define TEST_PERFORMANCE_CCOMP_LESS_THAN(name, value_fmt, value) \ + printf("[Performance][" PERFORMANCE_STR(name) "]: "value_fmt"\n", value); +#endif //SOC_CCOMP_TIMER_SUPPORTED + + /* @brief macro to print IDF performance * @param mode : performance item name. a string pointer. * @param value_fmt: print format and unit of the value, for example: "%02fms", "%dKB"