Mirrors/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2026-09-22 13:01:16 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix(spi_flash): integrate branch prediction control during cache operations

Browse Source
This commit is contained in:
wuzhenghui
2026-07-31 17:48:43 +08:00
committed by BOT
parent 8996d1aef0
commit 3631cf2537
5 changed files with 65 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
components/esp_hw_support/mspi_timing_tuning/mspi_timing_tuning.c
View File

@@ -23,6 +23,7 @@
#include "hal/cache_hal.h"
#endif
#include "esp_private/cache_utils.h"
#include "esp_cpu.h"
#include "esp_private/mspi_timing_tuning.h"
#include "esp_private/mspi_timing_config.h"
#include "esp_private/mspi_timing_by_mspi_delay.h"
@@ -634,8 +635,21 @@ static void restore_cache(uint32_t cpuid, uint32_t saved_state)
}
#else // ESP_TEE_BUILD
#define disable_cache(cpuid, saved_state) spi_flash_disable_cache(cpuid, saved_state)
#define restore_cache(cpuid, saved_state) spi_flash_restore_cache(cpuid, saved_state)
static void disable_cache(uint32_t cpuid, uint32_t *saved_state)
{
#if SOC_BRANCH_PREDICTOR_SUPPORTED
esp_cpu_branch_prediction_disable();
#endif
spi_flash_disable_cache(cpuid, saved_state);
}
static void restore_cache(uint32_t cpuid, uint32_t saved_state)
{
spi_flash_restore_cache(cpuid, saved_state);
#if SOC_BRANCH_PREDICTOR_SUPPORTED
esp_cpu_branch_prediction_enable();
#endif
}
#endif

7
components/esp_hw_support/sleep_modes.c
View File

@@ -11,6 +11,7 @@
#include <inttypes.h>
#include "esp_attr.h"
#include "esp_cpu.h"
#include "esp_rom_caps.h"
#include "esp_macros.h"
#include "esp_memory_utils.h"
@@ -566,6 +567,9 @@ static void FORCE_IRAM_ATTR suspend_cache(void) {
// If the access to external memory hits in the cache, it will not trigger a cache error. So in order to
// fully check the access to external memory, writeback & invalidate is needed here.
Cache_WriteBack_Invalidate_All(CACHE_MAP_MASK);
#endif
#if SOC_BRANCH_PREDICTOR_SUPPORTED
esp_cpu_branch_prediction_disable();
#endif
spi_flash_disable_cache(esp_cpu_get_core_id(), &s_cache_state);
}
@@ -577,6 +581,9 @@ static void FORCE_IRAM_ATTR resume_cache(void) {
assert(s_cache_suspend_cnt >= 0 && DRAM_STR("cache resume doesn't match suspend ops"));
if (s_cache_suspend_cnt == 0) {
spi_flash_restore_cache(esp_cpu_get_core_id(), s_cache_state);
#if SOC_BRANCH_PREDICTOR_SUPPORTED
esp_cpu_branch_prediction_enable();
#endif
}
}

11
components/esp_rom/patches/esp_rom_spiflash.c
View File

@@ -25,10 +25,6 @@
#include "esp32c61/rom/opi_flash.h"
#endif
#if SOC_BRANCH_PREDICTOR_SUPPORTED
#include "riscv/rv_utils.h"
#endif
#define SPI_IDX 1
#if CONFIG_IDF_TARGET_ESP32
@@ -815,10 +811,6 @@ void esp_rom_opiflash_cache_mode_config(esp_rom_spiflash_read_mode_t mode, const
extern void rom_spi_flash_disable_cache(uint32_t cpuid, uint32_t *saved_state);
void spi_flash_disable_cache(uint32_t cpuid, uint32_t *saved_state)
{
#if SOC_BRANCH_PREDICTOR_SUPPORTED
//branch predictor will start cache request as well
rv_utils_dis_branch_predictor();
#endif
rom_spi_flash_disable_cache(cpuid, saved_state);
}
@@ -826,8 +818,5 @@ extern void rom_spi_flash_restore_cache(uint32_t cpuid, uint32_t saved_state);
void spi_flash_restore_cache(uint32_t cpuid, uint32_t saved_state)
{
rom_spi_flash_restore_cache(cpuid, saved_state);
#if SOC_BRANCH_PREDICTOR_SUPPORTED
rv_utils_en_branch_predictor();
#endif
}
#endif

40
components/spi_flash/cache_utils.c
View File

@@ -30,6 +30,7 @@
#include "esp_private/esp_ipc.h"
#endif
#include "esp_attr.h"
#include "esp_cpu.h"
#include "esp_memory_utils.h"
#include "esp_intr_alloc.h"
#include "spi_flash_override.h"
@@ -105,7 +106,7 @@ void IRAM_ATTR spi_flash_op_block_func(void *arg)
/* The branch predictor issues speculative cache requests while this core
* spins in IRAM. The flash-op core is about to suspend the (shared) cache,
* so speculative fetches into flash would raise a cache access-fail on
* this core. spi_flash_restore_cache() below re-enables prediction. */
* this core. */
esp_cpu_branch_prediction_disable();
#endif
// s_flash_op_complete flag is cleared on *this* CPU, otherwise the other
@@ -118,6 +119,9 @@ void IRAM_ATTR spi_flash_op_block_func(void *arg)
}
// Flash operation is complete, re-enable cache
spi_flash_restore_cache(cpuid, s_flash_op_cache_state[cpuid]);
#if SOC_BRANCH_PREDICTOR_SUPPORTED
esp_cpu_branch_prediction_enable();
#endif
// Restore interrupts that aren't located in IRAM
esp_intr_noniram_enable();
#if ( ( CONFIG_FREERTOS_SMP ) && ( !CONFIG_FREERTOS_UNICORE ) )
@@ -188,6 +192,9 @@ void IRAM_ATTR spi_flash_disable_interrupts_caches_and_other_cpu(void)
// Kill interrupts that aren't located in IRAM
esp_intr_noniram_disable();
#if SOC_BRANCH_PREDICTOR_SUPPORTED
esp_cpu_branch_prediction_disable();
#endif
// This CPU executes this routine, with non-IRAM interrupts and the scheduler
// disabled. The other CPU is spinning in the spi_flash_op_block_func task, also
// with non-iram interrupts and the scheduler disabled. None of these CPUs will
@@ -224,6 +231,9 @@ void IRAM_ATTR spi_flash_enable_interrupts_caches_and_other_cpu(void)
s_flash_op_complete = true;
}
#if SOC_BRANCH_PREDICTOR_SUPPORTED
esp_cpu_branch_prediction_enable();
#endif
// Re-enable non-iram interrupts
esp_intr_noniram_enable();
@@ -250,6 +260,12 @@ void IRAM_ATTR spi_flash_disable_interrupts_caches_and_other_cpu_no_os(void)
const uint32_t cpuid = xPortGetCoreID();
const uint32_t other_cpuid = (cpuid == 0) ? 1 : 0;
#if SOC_BRANCH_PREDICTOR_SUPPORTED
/* Disable BP before the first disable_cache(): on shared-cache chips that
* call suspends external memory for all cores, so speculative fetches must
* already be stopped. */
esp_cpu_branch_prediction_disable();
#endif
// do not care about other CPU, it was halted upon entering panic handler
spi_flash_disable_cache(other_cpuid, &s_flash_op_cache_state[other_cpuid]);
// Kill interrupts that aren't located in IRAM
@@ -264,6 +280,9 @@ void IRAM_ATTR spi_flash_enable_interrupts_caches_no_os(void)
// Re-enable cache on this CPU
spi_flash_restore_cache(cpuid, s_flash_op_cache_state[cpuid]);
#if SOC_BRANCH_PREDICTOR_SUPPORTED
esp_cpu_branch_prediction_enable();
#endif
// Re-enable non-iram interrupts
esp_intr_noniram_enable();
}
@@ -303,12 +322,18 @@ void IRAM_ATTR spi_flash_disable_interrupts_caches_and_other_cpu(void)
{
spi_flash_op_lock();
esp_intr_noniram_disable();
#if SOC_BRANCH_PREDICTOR_SUPPORTED
esp_cpu_branch_prediction_disable();
#endif
spi_flash_disable_cache(0, &s_flash_op_cache_state[0]);
}
void IRAM_ATTR spi_flash_enable_interrupts_caches_and_other_cpu(void)
{
spi_flash_restore_cache(0, s_flash_op_cache_state[0]);
#if SOC_BRANCH_PREDICTOR_SUPPORTED
esp_cpu_branch_prediction_enable();
#endif
esp_intr_noniram_enable();
spi_flash_op_unlock();
}
@@ -317,6 +342,9 @@ void IRAM_ATTR spi_flash_disable_interrupts_caches_and_other_cpu_no_os(void)
{
// Kill interrupts that aren't located in IRAM
esp_intr_noniram_disable();
#if SOC_BRANCH_PREDICTOR_SUPPORTED
esp_cpu_branch_prediction_disable();
#endif
// Disable cache on this CPU as well
spi_flash_disable_cache(0, &s_flash_op_cache_state[0]);
}
@@ -325,6 +353,9 @@ void IRAM_ATTR spi_flash_enable_interrupts_caches_no_os(void)
{
// Re-enable cache on this CPU
spi_flash_restore_cache(0, s_flash_op_cache_state[0]);
#if SOC_BRANCH_PREDICTOR_SUPPORTED
esp_cpu_branch_prediction_enable();
#endif
// Re-enable non-iram interrupts
esp_intr_noniram_enable();
}
@@ -347,19 +378,12 @@ void IRAM_ATTR spi_flash_enable_cache(uint32_t cpuid)
#if !CONFIG_SPI_FLASH_ROM_IMPL
void IRAM_ATTR spi_flash_disable_cache(uint32_t cpuid, uint32_t *saved_state)
{
#if SOC_BRANCH_PREDICTOR_SUPPORTED
//branch predictor will start cache request as well
esp_cpu_branch_prediction_disable();
#endif
esp_cache_suspend_ext_mem_cache();
}
void IRAM_ATTR spi_flash_restore_cache(uint32_t cpuid, uint32_t saved_state)
{
esp_cache_resume_ext_mem_cache();
#if SOC_BRANCH_PREDICTOR_SUPPORTED
esp_cpu_branch_prediction_enable();
#endif
}
bool IRAM_ATTR spi_flash_cache_enabled(void)

13
components/spi_flash/include/esp_private/cache_utils.h
View File

@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD
* SPDX-FileCopyrightText: 2015-2026 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
@@ -97,7 +97,11 @@ bool spi_flash_cache_enabled(void);
void spi_flash_enable_cache(uint32_t cpuid);
/**
* @brief Suspend the Cache access to external memory, will disable branch predictor if supported.
* @brief Suspend the Cache access to external memory.
*
* @note Callers must disable branch prediction around this window when
* SOC_BRANCH_PREDICTOR_SUPPORTED, otherwise speculative fetches can
* raise cache access-fail errors while the cache is suspended.
*
* @param cpuid the core number to enable the cache for, meaning less on shared cache.
* @param saved_state Cache status hold by hal (Used only on ROM impl. in idf, this param unused)
@@ -105,7 +109,10 @@ void spi_flash_enable_cache(uint32_t cpuid);
void spi_flash_disable_cache(uint32_t cpuid, uint32_t *saved_state);
/**
* @brief Resume the Cache access to external memory, will enable branch predictor if supported.
* @brief Resume the Cache access to external memory.
*
* @note Callers that disabled branch prediction for the suspend window must
* re-enable it after this call when SOC_BRANCH_PREDICTOR_SUPPORTED.
*
* @param cpuid the core number to enable the cache for, meaning less on shared cache.
* @param saved_state Cache status hold by hal (Used only on ROM impl. in idf, this param unused)