Mirrors/esp-idf
1
0
Fork 0
mirror of https://github.com/espressif/esp-idf.git synced 2026-09-22 13:01:16 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feat/async_memcpy_dw_gdma_backend' into 'master'

Browse Source 
feat(esp_driver_dma): add dw_gdma backend for async_memcpy

Closes IDF-15760

See merge request espressif/esp-idf!50878
This commit is contained in:
morris
2026-08-10 16:16:53 +08:00
parent 1f590b9006 eec76a070e
commit bff6bf5e8b
29 changed files with 1144 additions and 340 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
components/esp_driver_cam/csi/src/esp_cam_ctlr_csi.c
View File

@@ -381,16 +381,16 @@ IRAM_ATTR static bool csi_dma_trans_done_callback(dw_gdma_channel_handle_t chan,
csi_dma_transfer_config = (dw_gdma_block_transfer_config_t) {
.src = {
.addr = MIPI_CSI_BRG_MEM_BASE,
.burst_mode = DW_GDMA_BURST_MODE_FIXED,
.burst_items = DW_GDMA_BURST_ITEMS_512,
.burst_len = 16,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_FIXED,
.burst_size = DW_GDMA_BURST_SIZE_512,
.axi_burst_len = 16,
.width = DW_GDMA_TRANS_WIDTH_64,
},
.dst = {
.addr = 0,
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.burst_items = DW_GDMA_BURST_ITEMS_512,
.burst_len = 16,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.burst_size = DW_GDMA_BURST_SIZE_512,
.axi_burst_len = 16,
.width = DW_GDMA_TRANS_WIDTH_64,
},
.size = ctlr->csi_transfer_size,
@@ -603,16 +603,16 @@ esp_err_t s_ctlr_csi_start(esp_cam_ctlr_handle_t handle)
csi_dma_transfer_config = (dw_gdma_block_transfer_config_t) {
.src = {
.addr = MIPI_CSI_BRG_MEM_BASE,
.burst_mode = DW_GDMA_BURST_MODE_FIXED,
.burst_items = DW_GDMA_BURST_ITEMS_512,
.burst_len = 16,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_FIXED,
.burst_size = DW_GDMA_BURST_SIZE_512,
.axi_burst_len = 16,
.width = DW_GDMA_TRANS_WIDTH_64,
},
.dst = {
.addr = (uint32_t)(trans.buffer),
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.burst_items = DW_GDMA_BURST_ITEMS_512,
.burst_len = 16,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.burst_size = DW_GDMA_BURST_SIZE_512,
.axi_burst_len = 16,
.width = DW_GDMA_TRANS_WIDTH_64,
},
.size = ctlr->csi_transfer_size,

24
components/esp_driver_cam/isp_dvp/src/esp_cam_ctlr_isp_dvp.c
View File

@@ -387,16 +387,16 @@ static esp_err_t s_isp_dvp_start(esp_cam_ctlr_handle_t handle)
dvp_dma_transfer_config = (dw_gdma_block_transfer_config_t) {
.src = {
.addr = MIPI_CSI_BRG_MEM_BASE,
.burst_mode = DW_GDMA_BURST_MODE_FIXED,
.burst_items = DW_GDMA_BURST_ITEMS_512,
.burst_len = 16,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_FIXED,
.burst_size = DW_GDMA_BURST_SIZE_512,
.axi_burst_len = 16,
.width = DW_GDMA_TRANS_WIDTH_64,
},
.dst = {
.addr = (uint32_t)(trans.buffer),
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.burst_items = DW_GDMA_BURST_ITEMS_512,
.burst_len = 16,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.burst_size = DW_GDMA_BURST_SIZE_512,
.axi_burst_len = 16,
.width = DW_GDMA_TRANS_WIDTH_64,
},
.size = dvp_ctlr->dvp_transfer_size,
@@ -459,16 +459,16 @@ IRAM_ATTR static bool s_dvp_dma_trans_done_callback(dw_gdma_channel_handle_t cha
dvp_dma_transfer_config = (dw_gdma_block_transfer_config_t) {
.src = {
.addr = MIPI_CSI_BRG_MEM_BASE,
.burst_mode = DW_GDMA_BURST_MODE_FIXED,
.burst_items = DW_GDMA_BURST_ITEMS_512,
.burst_len = 16,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_FIXED,
.burst_size = DW_GDMA_BURST_SIZE_512,
.axi_burst_len = 16,
.width = DW_GDMA_TRANS_WIDTH_64,
},
.dst = {
.addr = 0,
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.burst_items = DW_GDMA_BURST_ITEMS_512,
.burst_len = 16,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.burst_size = DW_GDMA_BURST_SIZE_512,
.axi_burst_len = 16,
.width = DW_GDMA_TRANS_WIDTH_64,
},
.size = dvp_ctlr->dvp_transfer_size,

7
components/esp_driver_dma/CMakeLists.txt
View File

@@ -11,7 +11,7 @@ if(CONFIG_SOC_GDMA_SUPPORTED)
list(APPEND srcs "src/gdma.c")
if(CONFIG_SOC_GDMA_SUPPORT_SLEEP_RETENTION AND CONFIG_SOC_PAU_SUPPORTED)
list(APPEND srcs "src/gdma_sleep.c")
list(APPEND srcs "${target}/gdma_retention.c")
list(APPEND srcs "src/${target}/gdma_retention.c")
endif()
if(CONFIG_SOC_GDMA_SUPPORT_ETM)
list(APPEND srcs "src/gdma_etm.c")
@@ -29,6 +29,9 @@ if(CONFIG_SOC_ASYNC_MEMCPY_SUPPORTED)
if(CONFIG_SOC_CP_DMA_SUPPORTED)
list(APPEND srcs "src/async_memcpy_cp_dma.c")
endif() # CONFIG_SOC_CP_DMA_SUPPORTED
if(CONFIG_SOC_DW_GDMA_SUPPORTED)
list(APPEND srcs "src/async_memcpy_dw_gdma.c")
endif() # CONFIG_SOC_DW_GDMA_SUPPORTED
endif() # CONFIG_SOC_ASYNC_MEMCPY_SUPPORTED
if(CONFIG_SOC_DW_GDMA_SUPPORTED)
@@ -38,7 +41,7 @@ endif()
if(CONFIG_SOC_DMA2D_SUPPORTED)
list(APPEND srcs "src/dma2d.c" "src/esp_async_color_convert.c" "src/async_color_convert_dma2d.c")
if(CONFIG_SOC_PAU_SUPPORTED)
list(APPEND srcs "${target}/dma2d_retention.c")
list(APPEND srcs "src/${target}/dma2d_retention.c")
endif()
endif()

40
components/esp_driver_dma/include/esp_async_memcpy.h
View File

@@ -67,7 +67,7 @@ typedef struct {
.flags = 0, \
}
#if SOC_HAS(AHB_GDMA)
#if SOC_AHB_GDMA_SUPPORTED
/**
* @brief Install async memcpy driver, with AHB-GDMA as the backend
*
@@ -80,9 +80,9 @@ typedef struct {
* - ESP_FAIL: Install async memcpy driver failed because of other error
*/
esp_err_t esp_async_memcpy_install_gdma_ahb(const async_memcpy_config_t *config, async_memcpy_handle_t *mcp);
#endif // SOC_HAS(AHB_GDMA)
#endif // SOC_AHB_GDMA_SUPPORTED
#if SOC_HAS(AXI_GDMA)
#if SOC_AXI_GDMA_SUPPORTED
/**
* @brief Install async memcpy driver, with AXI-GDMA as the backend
*
@@ -95,9 +95,9 @@ esp_err_t esp_async_memcpy_install_gdma_ahb(const async_memcpy_config_t *config,
* - ESP_FAIL: Install async memcpy driver failed because of other error
*/
esp_err_t esp_async_memcpy_install_gdma_axi(const async_memcpy_config_t *config, async_memcpy_handle_t *mcp);
#endif // SOC_HAS(AXI_GDMA)
#endif // SOC_AXI_GDMA_SUPPORTED
#if SOC_HAS(LP_AHB_GDMA)
#if SOC_LP_AHB_GDMA_SUPPORTED
/**
* @brief Install async memcpy driver, with LP AHB-GDMA as the backend
*
@@ -110,7 +110,7 @@ esp_err_t esp_async_memcpy_install_gdma_axi(const async_memcpy_config_t *config,
* - ESP_FAIL: Install async memcpy driver failed because of other error
*/
esp_err_t esp_async_memcpy_install_gdma_lp_ahb(const async_memcpy_config_t *config, async_memcpy_handle_t *mcp);
#endif // SOC_HAS(LP_AHB_GDMA)
#endif // SOC_LP_AHB_GDMA_SUPPORTED
#if SOC_CP_DMA_SUPPORTED
/**
@@ -129,6 +129,22 @@ esp_err_t esp_async_memcpy_install_gdma_lp_ahb(const async_memcpy_config_t *conf
esp_err_t esp_async_memcpy_install_cpdma(const async_memcpy_config_t *config, async_memcpy_handle_t *mcp);
#endif // SOC_CP_DMA_SUPPORTED
#if SOC_DW_GDMA_SUPPORTED
/**
* @brief Install async memcpy driver, with DW_GDMA as the backend
*
* @param[in] config Configuration of async memcpy
* @param[out] mcp Returned driver handle
* @return
* - ESP_OK: Install async memcpy driver successfully
* - ESP_ERR_INVALID_ARG: Install async memcpy driver failed because of invalid argument
* - ESP_ERR_NO_MEM: Install async memcpy driver failed because out of memory
* - ESP_FAIL: Install async memcpy driver failed because of other error
*/
esp_err_t esp_async_memcpy_install_dw_gdma(const async_memcpy_config_t *config, async_memcpy_handle_t *mcp);
#endif // SOC_DW_GDMA_SUPPORTED
/** @cond */
/**
* @brief Install async memcpy driver with the default DMA backend
*
@@ -143,12 +159,14 @@ esp_err_t esp_async_memcpy_install_cpdma(const async_memcpy_config_t *config, as
* - ESP_ERR_NO_MEM: Install async memcpy driver failed because out of memory
* - ESP_FAIL: Install async memcpy driver failed because of other error
*/
esp_err_t esp_async_memcpy_install(const async_memcpy_config_t *config, async_memcpy_handle_t *mcp);
esp_err_t esp_async_memcpy_install(const async_memcpy_config_t *config, async_memcpy_handle_t *mcp)
__attribute__((deprecated("Select a DMA backend explicitly with esp_async_memcpy_install_* instead")));
/** @endcond */
/**
* @brief Uninstall async memcpy driver
*
* @param[in] mcp Handle of async memcpy driver that returned from `esp_async_memcpy_install`
* @param[in] mcp Handle of async memcpy driver returned by an install function
* @return
* - ESP_OK: Uninstall async memcpy driver successfully
* - ESP_ERR_INVALID_ARG: Uninstall async memcpy driver failed because of invalid argument
@@ -161,7 +179,7 @@ esp_err_t esp_async_memcpy_uninstall(async_memcpy_handle_t mcp);
*
* @note The callback function is invoked in interrupt context, never do blocking jobs in the callback.
*
* @param[in] mcp Handle of async memcpy driver that returned from `esp_async_memcpy_install`
* @param[in] mcp Handle of async memcpy driver returned by an install function
* @param[in] dst Destination address (copy to)
* @param[in] src Source address (copy from)
* @param[in] n Number of bytes to copy
@@ -180,7 +198,7 @@ esp_err_t esp_async_memcpy(async_memcpy_handle_t mcp, void *dst, void *src, size
* @note This function is blocking and should not be called from interrupt context.
* @note Only `timeout_ms=-1` is supported, which means waiting indefinitely.
*
* @param[in] mcp Handle of async memcpy driver that returned from `esp_async_memcpy_install`
* @param[in] mcp Handle of async memcpy driver returned by an install function
* @param[in] dst Destination address (copy to)
* @param[in] src Source address (copy from)
* @param[in] n Number of bytes to copy
@@ -206,7 +224,7 @@ typedef enum {
*
* @note The created ETM event object can be deleted later by calling `esp_etm_del_event`
*
* @param[in] mcp Handle of async memcpy driver that returned from `esp_async_memcpy_install`
* @param[in] mcp Handle of async memcpy driver returned by an install function
* @param[in] event_type ETM event type
* @param[out] out_event Returned ETM event handle
* @return

53
components/esp_driver_dma/include/esp_private/dw_gdma.h
View File

@@ -100,11 +100,46 @@ esp_err_t dw_gdma_channel_get_id(dw_gdma_channel_handle_t chan, int *channel_id)
* @note By dynamic, we mean these channel end configurations can be changed in each transfer.
*/
typedef struct {
uint32_t addr; /*!< Memory address */
dw_gdma_transfer_width_t width; /*!< Transfer width */
dw_gdma_burst_mode_t burst_mode; /*!< Burst mode */
dw_gdma_burst_items_t burst_items; /*!< Number of data items that are contained in one burst transaction */
uint8_t burst_len; /*!< Burst transaction length, if set to 0, the hardware will apply a possible value as burst length */
uint32_t addr; /*!< Memory address */
/**
* @brief Transfer width, i.e. the bit width of a single data item (a "beat").
*
* This is the atomic unit of transfer. It decides how many bytes are moved per data item,
* and therefore the granularity of both `burst_size` / `axi_burst_len` and the address step when
* `addr_inc_mode` is INCREMENT. E.g. with `DW_GDMA_TRANS_WIDTH_32`, one item is 4 bytes.
*/
dw_gdma_transfer_width_t width;
/**
* @brief Address increment mode: whether the transfer address advances after each data item.
*
* @note It controls the address behaviour item by item:
* - `DW_GDMA_ADDR_INC_MODE_INCREMENT`: address is increased by `width` after each item
* (normal reading/writing of a contiguous buffer).
* - `DW_GDMA_ADDR_INC_MODE_FIXED` : address stays the same after each item
* (used when the target is a FIFO / peripheral register, e.g. DSI/CSI/ISP).
*/
dw_gdma_addr_inc_mode_t addr_inc_mode;
/**
* @brief Burst size: number of data items contained in one (core-level) burst transaction.
*
* This is the "burst size" (the DW_GDMA `MSIZE` field): how many `width`-sized items the DMA
* engine requests in a single burst. It sets the granularity of the transfer and is typically a
* power of two (1, 4, 8, ...). The total bytes of one such burst is `burst_size * width`.
*
* @note To tell it apart from `axi_burst_len`: `burst_size` is the coarse, power-of-two size of
* a burst as configured by the DMA core, while `axi_burst_len` is the fine-grained AXI
* burst length. For most cases keep them consistent (e.g. both 4/8/16), but you may set
* `axi_burst_len` to a value that is not a power of two and let `burst_size` stay coarse.
*/
dw_gdma_burst_size_t burst_size;
/**
* @brief AXI burst transaction length, in number of data items (beats).
*
* This directly programs the AXI `arlen`/`awlen` fields, i.e. how many beats the AXI master
* puts into a single AXI burst. The total bytes transferred in one burst is `axi_burst_len * width`.
* If set to 0, the hardware chooses a possible value for the burst length.
*/
uint8_t axi_burst_len;
struct {
uint32_t en_status_write_back: 1; /*!< Enable peripheral status write back */
} flags;
@@ -116,7 +151,13 @@ typedef struct {
typedef struct {
dw_gdma_channel_dynamic_config_t src; /*!< source configuration */
dw_gdma_channel_dynamic_config_t dst; /*!< destination configuration */
size_t size; /*!< Transfer size */
/**
* @brief Transfer size, in number of data items
*
* The size in bytes is `size * width` (each data item is `width` bytes). For a
* Link-List transfer, this is the block transfer size of the corresponding LLI.
*/
size_t size;
} dw_gdma_block_transfer_config_t;
/**

447
components/esp_driver_dma/src/async_memcpy_dw_gdma.c Normal file
View File

@@ -0,0 +1,447 @@
/*
* SPDX-FileCopyrightText: 2026 Espressif Systems (Shanghai) CO LTD
*
* SPDX-License-Identifier: Apache-2.0
*/
#include <string.h>
#include <stdlib.h>
#include <stdatomic.h>
#include <sys/queue.h>
#include <sys/param.h>
#include "sdkconfig.h"
#include "freertos/FreeRTOS.h"
#include "soc/soc_caps.h"
#include "esp_log.h"
#include "esp_check.h"
#include "esp_attr.h"
#include "esp_err.h"
#include "esp_private/dw_gdma.h"
#include "esp_private/esp_dma_utils.h"
#include "esp_private/critical_section.h"
#include "esp_memory_utils.h"
#include "esp_cache.h"
#include "esp_async_memcpy.h"
#include "esp_async_memcpy_priv.h"
#include "hal/cache_hal.h"
#include "hal/cache_ll.h"
ESP_LOG_ATTR_TAG(TAG, "async_mcp.dw_gdma");
/// @brief Maximum number of data items that one DW_GDMA block (link list item) can transfer.
/// @note The DW_GDMA block_ts register field is 22 bits wide and stores (size - 1).
#define MCP_DW_GDMA_MAX_BLOCK_ITEMS (1 << 22)
/// @brief Maximum body transfer width (in bits), capped by the AXI data width.
#define MCP_DW_GDMA_MAX_BODY_WIDTH_BITS 64
/// @brief Transaction object for async memcpy
typedef struct async_memcpy_transaction_t {
dw_gdma_link_list_handle_t link_list; // DW_GDMA link list for this transaction (body only)
async_memcpy_isr_cb_t cb; // user callback
void *cb_args; // user callback args
async_memcpy_split_t split; // cache aligned split, consumed by the deferred cache ops and CPU copy
STAILQ_ENTRY(async_memcpy_transaction_t) idle_queue_entry; // Entry for the idle queue
STAILQ_ENTRY(async_memcpy_transaction_t) ready_queue_entry; // Entry for the ready queue
} async_memcpy_transaction_t;
/// @brief Context of async memcpy driver
/// @note - It saves two queues, one for idle transaction objects, one for ready transaction objects
/// @note - Number of transaction objects are determined by the backlog parameter
typedef struct {
async_memcpy_context_t parent; // Parent IO interface
size_t dma_burst_size; // DMA burst size (in bytes), configured by the user
dw_gdma_channel_handle_t channel; // DW_GDMA channel handle
portMUX_TYPE spin_lock; // spin lock to avoid threads and isr from accessing the same resource simultaneously
_Atomic async_memcpy_fsm_t fsm; // driver state machine, changing state should be atomic
size_t num_trans_objs; // number of transaction objects
async_memcpy_transaction_t *transaction_pool; // transaction object pool
async_memcpy_transaction_t *current_transaction; // current transaction object
STAILQ_HEAD(, async_memcpy_transaction_t) idle_queue_head; // Head of the idle queue
STAILQ_HEAD(, async_memcpy_transaction_t) ready_queue_head; // Head of the ready queue
} async_memcpy_dw_gdma_context_t;
static bool mcp_dw_gdma_full_trans_done_callback(dw_gdma_channel_handle_t dma_chan, const dw_gdma_trans_done_event_data_t *event_data, void *user_data);
static bool mcp_dw_gdma_invalid_block_callback(dw_gdma_channel_handle_t dma_chan, const dw_gdma_break_event_data_t *event_data, void *user_data);
static esp_err_t mcp_dw_gdma_del(async_memcpy_context_t *ctx);
static esp_err_t mcp_dw_gdma_memcpy(async_memcpy_context_t *ctx, void *dst, void *src, size_t n, async_memcpy_isr_cb_t cb_isr, void *cb_args);
static esp_err_t mcp_dw_gdma_destroy(async_memcpy_dw_gdma_context_t *mcp_dw_gdma)
{
// clean up transaction pool
if (mcp_dw_gdma->transaction_pool) {
for (size_t i = 0; i < mcp_dw_gdma->num_trans_objs; i++) {
async_memcpy_transaction_t *trans = &mcp_dw_gdma->transaction_pool[i];
if (trans->link_list) {
dw_gdma_del_link_list(trans->link_list);
}
}
free(mcp_dw_gdma->transaction_pool);
}
// clean up DW_GDMA channel
if (mcp_dw_gdma->channel) {
dw_gdma_del_channel(mcp_dw_gdma->channel);
}
free(mcp_dw_gdma);
return ESP_OK;
}
/// @brief Pick the largest supported transfer width (in bits) that the given address is aligned to.
/// @note The width is also capped by the cache line size (the destination body is cache aligned) and the AXI data width.
/// @note The width must also divide the body length, so that the (cache line aligned) body can be evenly
/// split into transfer width sized data items without a remainder.
static dw_gdma_transfer_width_t mcp_dw_gdma_get_body_width(size_t cache_line_size, uintptr_t body_addr, size_t body_len)
{
// the body width is capped by the cache line size (the destination body is cache aligned) and the AXI data width
size_t max_width_bits = cache_line_size ? MIN(cache_line_size * 8, MCP_DW_GDMA_MAX_BODY_WIDTH_BITS) : MCP_DW_GDMA_MAX_BODY_WIDTH_BITS;
size_t width_bits = 8;
for (size_t cand = max_width_bits; cand >= 8; cand >>= 1) {
if (((body_addr & ((cand >> 3) - 1)) == 0) && (body_len % (cand >> 3) == 0)) {
width_bits = cand;
break;
}
}
switch (width_bits) {
case 64:
return DW_GDMA_TRANS_WIDTH_64;
case 32:
return DW_GDMA_TRANS_WIDTH_32;
case 16:
return DW_GDMA_TRANS_WIDTH_16;
default:
return DW_GDMA_TRANS_WIDTH_8;
}
}
/// @brief Map a DMA burst size (in bytes) to a supported DW_GDMA burst size enum value.
static dw_gdma_burst_size_t mcp_dw_gdma_get_burst_size(size_t burst_size_bytes, size_t width_bytes)
{
static const dw_gdma_burst_size_t supported[] = {
DW_GDMA_BURST_SIZE_1, DW_GDMA_BURST_SIZE_4, DW_GDMA_BURST_SIZE_8, DW_GDMA_BURST_SIZE_16,
DW_GDMA_BURST_SIZE_32, DW_GDMA_BURST_SIZE_64, DW_GDMA_BURST_SIZE_128, DW_GDMA_BURST_SIZE_256,
DW_GDMA_BURST_SIZE_512, DW_GDMA_BURST_SIZE_1024,
};
static const size_t supported_count[] = {1, 4, 8, 16, 32, 64, 128, 256, 512, 1024};
size_t items = (burst_size_bytes && width_bytes) ? burst_size_bytes / width_bytes : 0;
dw_gdma_burst_size_t pick = DW_GDMA_BURST_SIZE_4;
for (int i = sizeof(supported) / sizeof(supported[0]) - 1; i >= 0; i--) {
if (supported_count[i] <= items) {
pick = supported[i];
break;
}
}
return pick;
}
esp_err_t esp_async_memcpy_install_dw_gdma(const async_memcpy_config_t *config, async_memcpy_handle_t *mcp)
{
esp_err_t ret = ESP_OK;
async_memcpy_dw_gdma_context_t *mcp_dw_gdma = NULL;
ESP_RETURN_ON_FALSE(config && mcp, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
// allocate memory of driver context from internal memory (because it contains atomic variable)
mcp_dw_gdma = heap_caps_calloc(1, sizeof(async_memcpy_dw_gdma_context_t), MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
ESP_GOTO_ON_FALSE(mcp_dw_gdma, ESP_ERR_NO_MEM, err, TAG, "no mem for driver context");
uint32_t trans_queue_len = config->backlog ? config->backlog : DEFAULT_TRANSACTION_QUEUE_LENGTH;
// allocate memory for transaction pool from internal memory
mcp_dw_gdma->transaction_pool = heap_caps_calloc(trans_queue_len, sizeof(async_memcpy_transaction_t), MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT);
ESP_GOTO_ON_FALSE(mcp_dw_gdma->transaction_pool, ESP_ERR_NO_MEM, err, TAG, "no mem for transaction pool");
// create a single M2M DW_GDMA channel
dw_gdma_channel_static_config_t static_config = {
.block_transfer_type = DW_GDMA_BLOCK_TRANSFER_LIST,
.role = DW_GDMA_ROLE_MEM,
.num_outstanding_requests = 1,
};
dw_gdma_channel_alloc_config_t alloc_config = {
.src = static_config,
.dst = static_config,
.flow_controller = DW_GDMA_FLOW_CTRL_SELF, // DMA as the flow controller
// Fix the channel priority to a constant. The `async_memcpy_config_t.weight` field is intentionally
// ignored here: DW_GDMA has no weight/bandwidth control capability, and `chan_priority`(arbitration
// priority) is a different concept from the GDMA weight, so it can't be mapped 1:1.
.chan_priority = 1,
};
ESP_GOTO_ON_ERROR(dw_gdma_new_channel(&alloc_config, &mcp_dw_gdma->channel), err, TAG, "failed to alloc DW_GDMA channel");
// register event callbacks
dw_gdma_event_callbacks_t cbs = {
.on_full_trans_done = mcp_dw_gdma_full_trans_done_callback,
.on_invalid_block = mcp_dw_gdma_invalid_block_callback,
};
ESP_GOTO_ON_ERROR(dw_gdma_channel_register_event_callbacks(mcp_dw_gdma->channel, &cbs, mcp_dw_gdma),
err, TAG, "failed to register event callbacks");
// initialize transaction queue
STAILQ_INIT(&mcp_dw_gdma->idle_queue_head);
STAILQ_INIT(&mcp_dw_gdma->ready_queue_head);
// pick transactions from the pool and insert to the idle queue
for (int i = 0; i < trans_queue_len; i++) {
STAILQ_INSERT_TAIL(&mcp_dw_gdma->idle_queue_head, &mcp_dw_gdma->transaction_pool[i], idle_queue_entry);
}
// initialize other members
portMUX_INITIALIZE(&mcp_dw_gdma->spin_lock);
atomic_init(&mcp_dw_gdma->fsm, MCP_FSM_IDLE);
mcp_dw_gdma->num_trans_objs = trans_queue_len;
mcp_dw_gdma->dma_burst_size = config->dma_burst_size;
mcp_dw_gdma->parent.del = mcp_dw_gdma_del;
mcp_dw_gdma->parent.memcpy = mcp_dw_gdma_memcpy;
// ETM is not supported by the DW_GDMA backend, leave new_etm_event as NULL
// return base object
*mcp = &mcp_dw_gdma->parent;
return ESP_OK;
err:
if (mcp_dw_gdma) {
mcp_dw_gdma_destroy(mcp_dw_gdma);
}
return ret;
}
static esp_err_t mcp_dw_gdma_del(async_memcpy_context_t *ctx)
{
async_memcpy_dw_gdma_context_t *mcp_dw_gdma = __containerof(ctx, async_memcpy_dw_gdma_context_t, parent);
// check if there are pending transactions
ESP_RETURN_ON_FALSE(STAILQ_EMPTY(&mcp_dw_gdma->ready_queue_head), ESP_ERR_INVALID_STATE, TAG, "there are pending transactions");
// check if the driver is in IDLE state
ESP_RETURN_ON_FALSE(atomic_load(&mcp_dw_gdma->fsm) == MCP_FSM_IDLE, ESP_ERR_INVALID_STATE, TAG, "driver is not in IDLE state");
return mcp_dw_gdma_destroy(mcp_dw_gdma);
}
/// @brief help function to get one transaction from the ready queue
/// @note this function is allowed to be called in ISR
static async_memcpy_transaction_t *try_pop_trans_from_ready_queue(async_memcpy_dw_gdma_context_t *mcp_dw_gdma)
{
async_memcpy_transaction_t *trans = NULL;
esp_os_enter_critical_safe(&mcp_dw_gdma->spin_lock);
trans = STAILQ_FIRST(&mcp_dw_gdma->ready_queue_head);
if (trans) {
STAILQ_REMOVE_HEAD(&mcp_dw_gdma->ready_queue_head, ready_queue_entry);
}
esp_os_exit_critical_safe(&mcp_dw_gdma->spin_lock);
return trans;
}
/// @brief help function to start a pending transaction
/// @note this function is allowed to be called in ISR
static void try_start_pending_transaction(async_memcpy_dw_gdma_context_t *mcp_dw_gdma)
{
async_memcpy_fsm_t expected_fsm = MCP_FSM_IDLE;
async_memcpy_transaction_t *trans = NULL;
if (atomic_compare_exchange_strong(&mcp_dw_gdma->fsm, &expected_fsm, MCP_FSM_WAIT)) {
trans = try_pop_trans_from_ready_queue(mcp_dw_gdma);
if (trans) {
atomic_store(&mcp_dw_gdma->fsm, MCP_FSM_RUN);
mcp_dw_gdma->current_transaction = trans;
// Deferred during submit to avoid racing with the previous (still running) DMA transfer that
// may write to the same/overlapping destination. Now the transfer is actually starting, so the
// CPU copies and cache maintenance are serialized with the DMA engine.
async_memcpy_do_cache_ops_and_cpu_copy(&trans->split);
// apply the link list and kick off the DMA engine
dw_gdma_channel_use_link_list(mcp_dw_gdma->channel, trans->link_list);
dw_gdma_channel_enable_ctrl(mcp_dw_gdma->channel, true);
} else {
atomic_store(&mcp_dw_gdma->fsm, MCP_FSM_IDLE);
}
}
}
/// @brief help function to get one transaction from the idle queue
/// @note this function is allowed to be called in ISR
static async_memcpy_transaction_t *try_pop_trans_from_idle_queue(async_memcpy_dw_gdma_context_t *mcp_dw_gdma)
{
async_memcpy_transaction_t *trans = NULL;
esp_os_enter_critical_safe(&mcp_dw_gdma->spin_lock);
trans = STAILQ_FIRST(&mcp_dw_gdma->idle_queue_head);
if (trans) {
STAILQ_REMOVE_HEAD(&mcp_dw_gdma->idle_queue_head, idle_queue_entry);
}
esp_os_exit_critical_safe(&mcp_dw_gdma->spin_lock);
return trans;
}
static esp_err_t mcp_dw_gdma_memcpy(async_memcpy_context_t *ctx, void *dst, void *src, size_t n, async_memcpy_isr_cb_t cb_isr, void *cb_args)
{
esp_err_t ret = ESP_OK;
async_memcpy_dw_gdma_context_t *mcp_dw_gdma = __containerof(ctx, async_memcpy_dw_gdma_context_t, parent);
async_memcpy_transaction_t *trans = NULL;
// pick one transaction node from idle queue
trans = try_pop_trans_from_idle_queue(mcp_dw_gdma);
// check if we get the transaction object successfully
ESP_RETURN_ON_FALSE(trans, ESP_ERR_INVALID_STATE, TAG, "no free node in the idle queue");
// clean up the transaction configuration comes from the last one
if (trans->link_list) {
dw_gdma_del_link_list(trans->link_list);
trans->link_list = NULL;
}
// Split the destination buffer into a cache aligned body and (optional) head/tail.
// The DMA engine only handles the cache aligned body; the unaligned head and tail (if any) are
// copied by the CPU later, right before the DMA transfer starts (see try_start_pending_transaction).
// The head/tail and the body never share a cache line, so the CPU and DMA can work concurrently
// without cache coherency conflicts.
async_memcpy_split_t split;
ESP_GOTO_ON_ERROR(async_memcpy_split_cache_aligned(dst, src, n, &split), err, TAG, "failed to split buffer");
size_t body_len = split.body_len;
void *body_src = split.body_src;
void *body_dst = split.body_dst;
size_t cache_line_size = split.cache_line_size;
// pick the body transfer width based on the source body address alignment
// note: the destination body is already cache aligned (so it's aligned to any width <= cache line),
// but the source body address may not be, which limits the usable transfer width.
dw_gdma_transfer_width_t body_width = mcp_dw_gdma_get_body_width(cache_line_size, (uintptr_t)body_src, body_len);
size_t body_width_bytes = 1;
switch (body_width) {
case DW_GDMA_TRANS_WIDTH_64:
body_width_bytes = 8;
break;
case DW_GDMA_TRANS_WIDTH_32:
body_width_bytes = 4;
break;
case DW_GDMA_TRANS_WIDTH_16:
body_width_bytes = 2;
break;
default:
body_width_bytes = 1;
body_width = DW_GDMA_TRANS_WIDTH_8;
break;
}
// the body might need to be split into multiple link list items if it exceeds the maximum block size
size_t max_body_bytes_per_item = MCP_DW_GDMA_MAX_BLOCK_ITEMS * body_width_bytes;
size_t body_alignment = cache_line_size ? cache_line_size : body_width_bytes;
size_t body_num_items = esp_dma_calculate_node_count(body_len, body_alignment, max_body_bytes_per_item);
// align down the max body bytes per item to the body alignment, so each chunk stays aligned
if (cache_line_size) {
max_body_bytes_per_item &= ~(cache_line_size - 1);
}
dw_gdma_link_list_config_t link_list_cfg = {
.num_items = body_num_items,
.link_type = DW_GDMA_LINKED_LIST_TYPE_SINGLY,
};
ESP_GOTO_ON_ERROR(dw_gdma_new_link_list(&link_list_cfg, &trans->link_list), err, TAG, "failed to create link list");
// the burst configuration is derived from the user configured dma_burst_size
dw_gdma_burst_size_t body_burst_size = mcp_dw_gdma_get_burst_size(mcp_dw_gdma->dma_burst_size, body_width_bytes);
// configure each body link list item
uint8_t *cur_src = body_src;
uint8_t *cur_dst = body_dst;
size_t remaining = body_len;
for (size_t i = 0; i < body_num_items; i++) {
size_t chunk = MIN(remaining, max_body_bytes_per_item);
dw_gdma_block_transfer_config_t transfer_config = {
.src = {
.addr = (uint32_t)cur_src,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.width = body_width,
.burst_size = body_burst_size,
.axi_burst_len = 0,
},
.dst = {
.addr = (uint32_t)cur_dst,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.width = body_width,
.burst_size = body_burst_size,
.axi_burst_len = 0,
},
.size = chunk / body_width_bytes,
};
dw_gdma_lli_handle_t lli = dw_gdma_link_list_get_item(trans->link_list, i);
ESP_GOTO_ON_FALSE(lli, ESP_ERR_INVALID_STATE, err, TAG, "failed to get link list item");
ESP_GOTO_ON_ERROR(dw_gdma_lli_config_transfer(lli, &transfer_config), err, TAG, "failed to config link list item");
cur_src += chunk;
cur_dst += chunk;
remaining -= chunk;
}
// set block markers: all items are valid, the last item marks the end of the transfer.
// the singly link list already chains the items in order (and terminates at the last item),
// so we only need to mark the last item with is_last and enable the transfer done interrupt on it.
for (size_t i = 0; i < body_num_items; i++) {
dw_gdma_lli_handle_t lli = dw_gdma_link_list_get_item(trans->link_list, i);
dw_gdma_block_markers_t markers = {
.is_valid = true,
.is_last = (i == body_num_items - 1),
.en_trans_done_intr = (i == body_num_items - 1),
};
ESP_GOTO_ON_ERROR(dw_gdma_lli_set_block_markers(lli, markers), err, TAG, "failed to set block markers");
}
// save other transaction context
trans->cb = cb_isr;
trans->cb_args = cb_args;
// save the split info (includes the original buffers and cache aligned body) for the deferred
// cache ops and CPU copy (performed in try_start_pending_transaction when the transfer actually starts)
trans->split = split;
esp_os_enter_critical(&mcp_dw_gdma->spin_lock);
// insert the trans to ready queue
STAILQ_INSERT_TAIL(&mcp_dw_gdma->ready_queue_head, trans, ready_queue_entry);
esp_os_exit_critical(&mcp_dw_gdma->spin_lock);
// check driver state, if there's no running transaction, start a new one
try_start_pending_transaction(mcp_dw_gdma);
return ESP_OK;
err:
if (trans) {
// return back the trans to idle queue
esp_os_enter_critical(&mcp_dw_gdma->spin_lock);
STAILQ_INSERT_TAIL(&mcp_dw_gdma->idle_queue_head, trans, idle_queue_entry);
esp_os_exit_critical(&mcp_dw_gdma->spin_lock);
}
return ret;
}
static bool mcp_dw_gdma_full_trans_done_callback(dw_gdma_channel_handle_t dma_chan, const dw_gdma_trans_done_event_data_t *event_data, void *user_data)
{
bool need_yield = false;
async_memcpy_dw_gdma_context_t *mcp_dw_gdma = (async_memcpy_dw_gdma_context_t *)user_data;
async_memcpy_transaction_t *trans = mcp_dw_gdma->current_transaction;
// switch driver state from RUN to IDLE
async_memcpy_fsm_t expected_fsm = MCP_FSM_RUN;
if (atomic_compare_exchange_strong(&mcp_dw_gdma->fsm, &expected_fsm, MCP_FSM_WAIT)) {
// the head/tail were already copied by the CPU before the transfer started, and the body was written by the
// DMA to memory (the body cache lines were invalidated before the transfer), so nothing to merge here.
// invoked callback registered by user
async_memcpy_isr_cb_t cb = trans->cb;
if (cb) {
async_memcpy_event_t e = {
// No event data for now
};
need_yield = cb(&mcp_dw_gdma->parent, &e, trans->cb_args);
}
trans->cb = NULL;
esp_os_enter_critical_isr(&mcp_dw_gdma->spin_lock);
// insert the trans object to the idle queue
STAILQ_INSERT_TAIL(&mcp_dw_gdma->idle_queue_head, trans, idle_queue_entry);
mcp_dw_gdma->current_transaction = NULL;
esp_os_exit_critical_isr(&mcp_dw_gdma->spin_lock);
atomic_store(&mcp_dw_gdma->fsm, MCP_FSM_IDLE);
}
// try start the next pending transaction
try_start_pending_transaction(mcp_dw_gdma);
return need_yield;
}
static bool mcp_dw_gdma_invalid_block_callback(dw_gdma_channel_handle_t dma_chan, const dw_gdma_break_event_data_t *event_data, void *user_data)
{
// A valid transfer should never hit an invalid block. Terminate instead of leaving callers blocked.
ESP_EARLY_LOGE(TAG, "invalid block detected, terminating");
abort();
}

86
components/esp_driver_dma/src/async_memcpy_gdma.c
View File

@@ -35,10 +35,9 @@ ESP_LOG_ATTR_TAG(TAG, "async_mcp.gdma");
typedef struct async_memcpy_transaction_t {
gdma_link_list_handle_t tx_link_list; // DMA link list for TX direction
gdma_link_list_handle_t rx_link_list; // DMA link list for RX direction
dma_buffer_split_array_t rx_buf_array; // Split the destination buffer into cache aligned ones, save the splits in this array
uint8_t* stash_buffer; // Stash buffer for cache aligned buffer
async_memcpy_isr_cb_t cb; // user callback
void *cb_args; // user callback args
async_memcpy_split_t split; // cache aligned split, consumed by the deferred cache ops and CPU copy
STAILQ_ENTRY(async_memcpy_transaction_t) idle_queue_entry; // Entry for the idle queue
STAILQ_ENTRY(async_memcpy_transaction_t) ready_queue_entry; // Entry for the ready queue
} async_memcpy_transaction_t;
@@ -83,9 +82,6 @@ static esp_err_t mcp_gdma_destroy(async_memcpy_gdma_context_t *mcp_gdma)
if (trans->rx_link_list) {
gdma_del_link_list(trans->rx_link_list);
}
if (trans->stash_buffer) {
free(trans->stash_buffer);
}
}
free(mcp_gdma->transaction_pool);
}
@@ -259,6 +255,10 @@ static void try_start_pending_transaction(async_memcpy_gdma_context_t *mcp_gdma)
if (trans) {
atomic_store(&mcp_gdma->fsm, MCP_FSM_RUN);
mcp_gdma->current_transaction = trans;
// Deferred during submit to avoid racing with the previous (still running) DMA transfer that
// may write to the same/overlapping destination. Now the transfer is actually starting, so the
// CPU copies and cache maintenance are serialized with the DMA engine.
async_memcpy_do_cache_ops_and_cpu_copy(&trans->split);
gdma_start(mcp_gdma->rx_channel, gdma_link_get_head_addr(trans->rx_link_list));
gdma_start(mcp_gdma->tx_channel, gdma_link_get_head_addr(trans->tx_link_list));
} else {
@@ -353,17 +353,23 @@ static esp_err_t mcp_gdma_memcpy(async_memcpy_context_t *ctx, void *dst, void *s
gdma_del_link_list(trans->rx_link_list);
trans->rx_link_list = NULL;
}
if (trans->stash_buffer) {
free(trans->stash_buffer);
trans->stash_buffer = NULL;
}
// Split the destination buffer into a cache aligned body and (optional) head/tail.
// The DMA engine only handles the cache aligned body; the unaligned head and tail (if any) are
// copied by the CPU later, right before the DMA transfer starts (see try_start_pending_transaction).
// The head/tail and the body never share a cache line, so the CPU and DMA can work concurrently
// without cache coherency conflicts.
// The source buffer is split at the same boundaries because the GDMA TX/RX
// channels are paired and each descriptor carries both the source and destination.
async_memcpy_split_t split;
ESP_GOTO_ON_ERROR(async_memcpy_split_cache_aligned(dst, src, n, &split), err, TAG, "failed to split buffer");
size_t buffer_alignment = 0;
size_t num_dma_nodes = 0;
// allocate gdma TX link
buffer_alignment = esp_ptr_internal(src) ? mcp_gdma->tx_int_mem_alignment : mcp_gdma->tx_ext_mem_alignment;
num_dma_nodes = esp_dma_calculate_node_count(n, buffer_alignment, MCP_DMA_DESCRIPTOR_BUFFER_MAX_SIZE);
// allocate gdma TX link, only the body is handled by the DMA
buffer_alignment = esp_ptr_internal(split.body_src) ? mcp_gdma->tx_int_mem_alignment : mcp_gdma->tx_ext_mem_alignment;
num_dma_nodes = esp_dma_calculate_node_count(split.body_len, buffer_alignment, MCP_DMA_DESCRIPTOR_BUFFER_MAX_SIZE);
gdma_link_list_config_t tx_link_cfg = {
.item_alignment = dma_link_item_alignment,
.num_items = num_dma_nodes,
@@ -373,12 +379,12 @@ static esp_err_t mcp_gdma_memcpy(async_memcpy_context_t *ctx, void *dst, void *s
},
};
ESP_GOTO_ON_ERROR(gdma_new_link_list(&tx_link_cfg, &trans->tx_link_list), err, TAG, "failed to create TX link list");
// mount the source buffer to the TX link list
// mount the source body to the TX link list
gdma_buffer_mount_config_t tx_buf_mount_config[1] = {
[0] = {
.buffer = src,
.buffer = split.body_src,
.buffer_alignment = buffer_alignment,
.length = n,
.length = split.body_len,
.flags = {
.mark_eof = true, // mark the last item as EOF, so the RX channel can also received an EOF list item
.mark_final = GDMA_FINAL_LINK_TO_NULL, // using singly list, so terminate the link here
@@ -387,41 +393,38 @@ static esp_err_t mcp_gdma_memcpy(async_memcpy_context_t *ctx, void *dst, void *s
};
gdma_link_mount_buffers(trans->tx_link_list, 0, tx_buf_mount_config, 1, NULL);
// read the cache line size of internal and external memory, we use this information to check if a given memory is behind the cache
// write back the source data if it's behind the cache
size_t cache_line_size = esp_cache_get_line_size_by_addr(src);
if (cache_line_size > 0) {
esp_cache_msync(src, n, ESP_CACHE_MSYNC_FLAG_DIR_C2M | ESP_CACHE_MSYNC_FLAG_UNALIGNED);
}
// allocate gdma RX link
buffer_alignment = esp_ptr_internal(dst) ? mcp_gdma->rx_int_mem_alignment : mcp_gdma->rx_ext_mem_alignment;
num_dma_nodes = esp_dma_calculate_node_count(n, buffer_alignment, MCP_DMA_DESCRIPTOR_BUFFER_MAX_SIZE);
// allocate gdma RX link, only the body is handled by the DMA
buffer_alignment = esp_ptr_internal(split.body_dst) ? mcp_gdma->rx_int_mem_alignment : mcp_gdma->rx_ext_mem_alignment;
num_dma_nodes = esp_dma_calculate_node_count(split.body_len, buffer_alignment, MCP_DMA_DESCRIPTOR_BUFFER_MAX_SIZE);
gdma_link_list_config_t rx_link_cfg = {
.item_alignment = dma_link_item_alignment,
.num_items = num_dma_nodes + 3, // add 3 extra items for the cache aligned buffers
.num_items = num_dma_nodes,
.flags = {
.check_owner = true,
.items_in_ext_mem = false, // TODO: if the memcopy size is too large, we may need to allocate the link list items from external memory
.items_in_ext_mem = false, // TODO: if the memcopy size is too large, we can consider allocating the link list items from external memory
},
};
ESP_GOTO_ON_ERROR(gdma_new_link_list(&rx_link_cfg, &trans->rx_link_list), err, TAG, "failed to create RX link list");
// if the destination buffer address is not cache line aligned, we need to split the buffer into cache line aligned ones
ESP_GOTO_ON_ERROR(esp_dma_split_rx_buffer_to_cache_aligned(dst, n, &trans->rx_buf_array, &trans->stash_buffer),
err, TAG, "failed to split RX buffer into aligned ones");
// mount the destination buffer to the RX link list
gdma_buffer_mount_config_t rx_buf_mount_config[3] = {0};
for (int i = 0; i < 3; i++) {
rx_buf_mount_config[i].buffer = trans->rx_buf_array.aligned_buffer[i].aligned_buffer;
rx_buf_mount_config[i].buffer_alignment = buffer_alignment;
rx_buf_mount_config[i].length = trans->rx_buf_array.aligned_buffer[i].length;
}
gdma_link_mount_buffers(trans->rx_link_list, 0, rx_buf_mount_config, 3, NULL);
// mount the destination body to the RX link list
gdma_buffer_mount_config_t rx_buf_mount_config[1] = {
[0] = {
.buffer = split.body_dst,
.buffer_alignment = buffer_alignment,
.length = split.body_len,
.flags = {
.mark_eof = true,
.mark_final = GDMA_FINAL_LINK_TO_NULL,
}
}
};
gdma_link_mount_buffers(trans->rx_link_list, 0, rx_buf_mount_config, 1, NULL);
// save other transaction context
trans->cb = cb_isr;
trans->cb_args = cb_args;
// save the split info (includes the original buffers and cache aligned body) for the deferred
// cache ops and CPU copy (performed in try_start_pending_transaction when the transfer actually starts)
trans->split = split;
esp_os_enter_critical(&mcp_gdma->spin_lock);
// insert the trans to ready queue
@@ -448,13 +451,12 @@ static bool mcp_gdma_rx_eof_callback(gdma_channel_handle_t dma_chan, gdma_event_
bool need_yield = false;
async_memcpy_gdma_context_t *mcp_gdma = (async_memcpy_gdma_context_t *)user_data;
async_memcpy_transaction_t *trans = mcp_gdma->current_transaction;
dma_buffer_split_array_t *rx_buf_array = &trans->rx_buf_array;
// switch driver state from RUN to IDLE
async_memcpy_fsm_t expected_fsm = MCP_FSM_RUN;
if (atomic_compare_exchange_strong(&mcp_gdma->fsm, &expected_fsm, MCP_FSM_WAIT)) {
// merge the cache aligned buffers to the original buffer
esp_dma_merge_aligned_rx_buffers(rx_buf_array);
// the head/tail were already copied by the CPU before the transfer started, and the body was written by the
// DMA to memory (the body cache lines were invalidated before the transfer), so nothing to merge here.
// invoked callback registered by user
async_memcpy_isr_cb_t cb = trans->cb;

36
components/esp_driver_dma/src/dw_gdma.c
View File

@@ -478,16 +478,16 @@ esp_err_t dw_gdma_channel_config_transfer(dw_gdma_channel_handle_t chan, const d
// transfer width
dw_gdma_ll_channel_set_src_trans_width(hal->dev, chan_id, config->src.width);
dw_gdma_ll_channel_set_dst_trans_width(hal->dev, chan_id, config->dst.width);
// set burst items
dw_gdma_ll_channel_set_src_burst_items(hal->dev, chan_id, config->src.burst_items);
dw_gdma_ll_channel_set_dst_burst_items(hal->dev, chan_id, config->dst.burst_items);
// set burst mode
dw_gdma_ll_channel_set_src_burst_mode(hal->dev, chan_id, config->src.burst_mode);
dw_gdma_ll_channel_set_dst_burst_mode(hal->dev, chan_id, config->dst.burst_mode);
// set burst size
dw_gdma_ll_channel_set_src_burst_size(hal->dev, chan_id, config->src.burst_size);
dw_gdma_ll_channel_set_dst_burst_size(hal->dev, chan_id, config->dst.burst_size);
// set address increment mode
dw_gdma_ll_channel_set_src_addr_inc_mode(hal->dev, chan_id, config->src.addr_inc_mode);
dw_gdma_ll_channel_set_dst_addr_inc_mode(hal->dev, chan_id, config->dst.addr_inc_mode);
// [Ctrl1] register
// set burst length
dw_gdma_ll_channel_set_src_burst_len(hal->dev, chan_id, config->src.burst_len);
dw_gdma_ll_channel_set_dst_burst_len(hal->dev, chan_id, config->dst.burst_len);
// set AXI burst length
dw_gdma_ll_channel_set_src_axi_burst_len(hal->dev, chan_id, config->src.axi_burst_len);
dw_gdma_ll_channel_set_dst_axi_burst_len(hal->dev, chan_id, config->dst.axi_burst_len);
// whether to enable the peripheral status write back
dw_gdma_ll_channel_enable_src_periph_status_write_back(hal->dev, chan_id, config->src.flags.en_status_write_back);
dw_gdma_ll_channel_enable_dst_periph_status_write_back(hal->dev, chan_id, config->dst.flags.en_status_write_back);
@@ -527,16 +527,16 @@ esp_err_t dw_gdma_lli_config_transfer(dw_gdma_lli_handle_t lli, const dw_gdma_bl
// transfer width
dw_gdma_ll_lli_set_src_trans_width(lli, config->src.width);
dw_gdma_ll_lli_set_dst_trans_width(lli, config->dst.width);
// set burst items
dw_gdma_ll_lli_set_src_burst_items(lli, config->src.burst_items);
dw_gdma_ll_lli_set_dst_burst_items(lli, config->dst.burst_items);
// set burst mode
dw_gdma_ll_lli_set_src_burst_mode(lli, config->src.burst_mode);
dw_gdma_ll_lli_set_dst_burst_mode(lli, config->dst.burst_mode);
// set burst size
dw_gdma_ll_lli_set_src_burst_size(lli, config->src.burst_size);
dw_gdma_ll_lli_set_dst_burst_size(lli, config->dst.burst_size);
// set address increment mode
dw_gdma_ll_lli_set_src_addr_inc_mode(lli, config->src.addr_inc_mode);
dw_gdma_ll_lli_set_dst_addr_inc_mode(lli, config->dst.addr_inc_mode);
// [Ctrl1] register
// set burst length
dw_gdma_ll_lli_set_src_burst_len(lli, config->src.burst_len);
dw_gdma_ll_lli_set_dst_burst_len(lli, config->dst.burst_len);
// set AXI burst length
dw_gdma_ll_lli_set_src_axi_burst_len(lli, config->src.axi_burst_len);
dw_gdma_ll_lli_set_dst_axi_burst_len(lli, config->dst.axi_burst_len);
// whether to enable the peripheral status write back
dw_gdma_ll_lli_enable_src_periph_status_write_back(lli, config->src.flags.en_status_write_back);
dw_gdma_ll_lli_enable_dst_periph_status_write_back(lli, config->dst.flags.en_status_write_back);

0
components/esp_driver_dma/esp32c5/gdma_retention.c → components/esp_driver_dma/src/esp32c5/gdma_retention.c
View File

0
components/esp_driver_dma/esp32c6/gdma_retention.c → components/esp_driver_dma/src/esp32c6/gdma_retention.c
View File

0
components/esp_driver_dma/esp32c61/gdma_retention.c → components/esp_driver_dma/src/esp32c61/gdma_retention.c
View File

0
components/esp_driver_dma/esp32h2/gdma_retention.c → components/esp_driver_dma/src/esp32h2/gdma_retention.c
View File

0
components/esp_driver_dma/esp32h21/gdma_retention.c → components/esp_driver_dma/src/esp32h21/gdma_retention.c
View File

0
components/esp_driver_dma/esp32h4/gdma_retention.c → components/esp_driver_dma/src/esp32h4/gdma_retention.c
View File

0
components/esp_driver_dma/esp32p4/dma2d_retention.c → components/esp_driver_dma/src/esp32p4/dma2d_retention.c
View File

0
components/esp_driver_dma/esp32p4/gdma_retention.c → components/esp_driver_dma/src/esp32p4/gdma_retention.c
View File

0
components/esp_driver_dma/esp32s31/dma2d_retention.c → components/esp_driver_dma/src/esp32s31/dma2d_retention.c
View File

0
components/esp_driver_dma/esp32s31/gdma_retention.c → components/esp_driver_dma/src/esp32s31/gdma_retention.c
View File

23
components/esp_driver_dma/src/esp_async_memcpy.c
View File

@@ -4,15 +4,22 @@
* SPDX-License-Identifier: Apache-2.0
*/
#include <string.h>
#include "freertos/FreeRTOS.h"
#include "freertos/semphr.h"
#include "freertos/task.h"
#include "esp_check.h"
#include "esp_cache.h"
#include "esp_async_memcpy.h"
#include "esp_async_memcpy_priv.h"
ESP_LOG_ATTR_TAG(TAG, "async_mcp");
/// Minimum buffer size for a DMA backed memory copy. A buffer smaller than two cache lines can not
/// guarantee a cache aligned "body" segment for the DMA engine, in which case a CPU memcpy is more
/// efficient anyway. The top level rejects such small buffers uniformly for all DMA backends.
#define ASYNC_MEMCPY_MIN_DMA_SIZE_CACHE_LINES 2
esp_err_t esp_async_memcpy_uninstall(async_memcpy_handle_t asmcp)
{
ESP_RETURN_ON_FALSE(asmcp, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
@@ -22,6 +29,14 @@ esp_err_t esp_async_memcpy_uninstall(async_memcpy_handle_t asmcp)
esp_err_t esp_async_memcpy(async_memcpy_handle_t asmcp, void *dst, void *src, size_t n, async_memcpy_isr_cb_t cb_isr, void *cb_args)
{
ESP_RETURN_ON_FALSE(asmcp && dst && src && n, ESP_ERR_INVALID_ARG, TAG, "invalid argument");
// Reject buffers that are too small to benefit from a DMA transfer. A buffer smaller than two
// cache lines can not provide a cache aligned body segment, which all DMA backends rely on.
// Use a CPU memcpy for such small buffers instead.
size_t cache_line_size = esp_cache_get_line_size_by_addr(dst);
if (cache_line_size && n < ASYNC_MEMCPY_MIN_DMA_SIZE_CACHE_LINES * cache_line_size) {
ESP_RETURN_ON_FALSE(false, ESP_ERR_INVALID_SIZE, TAG, "buffer size %zu is too small for DMA, minimum is %zu", n,
ASYNC_MEMCPY_MIN_DMA_SIZE_CACHE_LINES * cache_line_size);
}
return asmcp->memcpy(asmcp, dst, src, n, cb_isr, cb_args);
}
@@ -48,6 +63,14 @@ esp_err_t esp_memcpy_blocking(async_memcpy_handle_t asmcp, void *dst, void *src,
ESP_RETURN_ON_FALSE(!xPortInIsrContext(), ESP_ERR_INVALID_STATE, TAG, "called from ISR context is not allowed");
ESP_RETURN_ON_FALSE(timeout_ms == -1, ESP_ERR_INVALID_ARG, TAG, "only timeout_ms=-1 is supported");
// For buffers too small to benefit from a DMA transfer, fall back to a CPU memcpy. The blocking
// API is not bound by the ISR callback contract, so a synchronous CPU copy is safe here.
size_t cache_line_size = esp_cache_get_line_size_by_addr(dst);
if (cache_line_size && n < ASYNC_MEMCPY_MIN_DMA_SIZE_CACHE_LINES * cache_line_size) {
memcpy(dst, src, n);
return ESP_OK;
}
memcpy_blocking_context_t ctx = {};
ctx.semaphore = xSemaphoreCreateBinaryStatic(&ctx.semaphore_buffer);

90
components/esp_driver_dma/src/esp_async_memcpy_priv.h
View File

@@ -8,8 +8,10 @@
#include <stdint.h>
#include <stdbool.h>
#include <string.h>
#include "esp_err.h"
#include "esp_etm.h"
#include "esp_cache.h"
#include "esp_async_memcpy.h"
#include "soc/soc_caps.h"
@@ -38,6 +40,94 @@ struct async_memcpy_context_t {
esp_err_t (*del)(async_memcpy_context_t *ctx);
};
/// @brief Cache aligned split of an async memcpy transaction.
/// @note The DMA engine only handles the cache aligned body. The unaligned head and tail (if any)
/// are copied by the CPU right before the DMA transfer starts (see
/// `async_memcpy_do_cache_ops_and_cpu_copy`). The head/tail and the body never share a cache
/// line, so the CPU and DMA can work concurrently without cache coherency conflicts. This
/// avoids the need for a stash buffer and the merge step in the ISR.
typedef struct {
void *src; ///< Source buffer of the whole transaction (for the deferred CPU copy and cache ops)
void *dst; ///< Destination buffer of the whole transaction (for the deferred CPU copy and cache ops)
size_t head_len; ///< Bytes of the unaligned head (copied by CPU)
size_t body_len; ///< Bytes of the cache aligned body (copied by DMA)
size_t tail_len; ///< Bytes of the unaligned tail (copied by CPU)
void *body_src; ///< Source address of the body
void *body_dst; ///< Destination address of the body
size_t cache_line_size; ///< Cache line size used for the split (0 if the memory is not cached)
} async_memcpy_split_t;
/// @brief Split an async memcpy transaction into a cache aligned body and (optional) head/tail.
/// @note This function only computes the split. It does NOT touch any memory or cache, the caller
/// must invoke `async_memcpy_do_cache_ops_and_cpu_copy` (right before the DMA transfer starts)
/// to actually copy the head/tail and do the required cache maintenance.
/// @note The top level esp_async_memcpy guarantees n >= 2 * cache_line_size, so a non-zero body is
/// always available. This is asserted here defensively.
static inline esp_err_t async_memcpy_split_cache_aligned(void *dst, void *src, size_t n, async_memcpy_split_t *split)
{
split->src = src;
split->dst = dst;
split->head_len = 0;
split->tail_len = 0;
split->body_len = n;
split->body_src = src;
split->body_dst = dst;
split->cache_line_size = esp_cache_get_line_size_by_addr(dst);
if (split->cache_line_size) {
size_t head_overflow = (uintptr_t)dst % split->cache_line_size;
split->head_len = head_overflow ? (split->cache_line_size - head_overflow) : 0;
// the top level esp_async_memcpy guarantees n >= 2 * cache_line_size, so a non-zero body is always available
if (split->head_len >= n) {
return ESP_ERR_INVALID_SIZE;
}
split->tail_len = (n - split->head_len) % split->cache_line_size;
split->body_len = n - split->head_len - split->tail_len;
if (split->body_len == 0) {
return ESP_ERR_INVALID_SIZE;
}
split->body_src = (uint8_t *)src + split->head_len;
split->body_dst = (uint8_t *)dst + split->head_len;
}
return ESP_OK;
}
/// @brief Do the CPU copy of the unaligned head/tail and the cache maintenance for a split.
/// @note This must be called from `try_start_pending_transaction`, right before the DMA transfer
/// actually kicks off. Because the async memcpy allows different requests to use the same or
/// overlapping destination buffers, doing this at submit time would race with a previous
/// (still running) DMA transfer that writes to the same destination. By deferring it to the
/// moment the transfer starts (which, with a single channel FSM, happens only after the
/// previous transfer has completed), the CPU copies and cache operations are serialized with
/// the DMA engine.
/// @note The head/tail cache lines are disjoint from the body's, so the CPU copies don't interfere
/// with the body transferred by the DMA.
static inline esp_err_t async_memcpy_do_cache_ops_and_cpu_copy(const async_memcpy_split_t *split)
{
size_t n = split->head_len + split->body_len + split->tail_len;
// CPU copies the unaligned head and tail. These cache lines are disjoint from the body's.
if (split->head_len) {
memcpy(split->dst, split->src, split->head_len);
}
if (split->tail_len) {
memcpy((uint8_t *)split->body_dst + split->body_len, (uint8_t *)split->body_src + split->body_len, split->tail_len);
}
// write back the source data if it's behind the cache, so that the DMA can read the latest data
if (esp_cache_get_line_size_by_addr(split->src) > 0) {
esp_cache_msync(split->src, n, ESP_CACHE_MSYNC_FLAG_DIR_C2M | ESP_CACHE_MSYNC_FLAG_UNALIGNED);
}
// invalidate the destination body if it's behind the cache, so that the CPU can see the data written
// by the DMA engine later (the head/tail cache lines are kept intact, they hold the CPU data)
if (split->cache_line_size) {
esp_cache_msync(split->body_dst, split->body_len, ESP_CACHE_MSYNC_FLAG_DIR_M2C);
}
return ESP_OK;
}
#ifdef __cplusplus
}
#endif

46
components/esp_driver_dma/test_apps/dma/main/test_async_memcpy.c
View File

@@ -148,6 +148,13 @@ TEST_CASE("memory copy the same buffer with different content", "[async mcp]")
test_memory_copy_with_same_buffer(driver, &config);
TEST_ESP_OK(esp_async_memcpy_uninstall(driver));
#endif // SOC_HAS(LP_AHB_GDMA)
#if SOC_DW_GDMA_SUPPORTED
printf("Testing memcpy by DW_GDMA\r\n");
TEST_ESP_OK(esp_async_memcpy_install_dw_gdma(&config, &driver));
test_memory_copy_with_same_buffer(driver, &config);
TEST_ESP_OK(esp_async_memcpy_uninstall(driver));
#endif // SOC_DW_GDMA_SUPPORTED
}
static void test_memory_copy_blocking(async_memcpy_handle_t driver)
@@ -196,12 +203,12 @@ TEST_CASE("memory copy by DMA (blocking)", "[async mcp]")
TEST_ESP_OK(esp_async_memcpy_uninstall(driver));
#endif // SOC_HAS(AXI_GDMA)
#if SOC_CP_DMA_SUPPORTED
#if SOC_HAS(CP_DMA)
printf("Testing memcpy by CP DMA\r\n");
TEST_ESP_OK(esp_async_memcpy_install_cpdma(&config, &driver));
test_memory_copy_blocking(driver);
TEST_ESP_OK(esp_async_memcpy_uninstall(driver));
#endif // SOC_CP_DMA_SUPPORTED
#endif // SOC_HAS(CP_DMA)
#if SOC_HAS(LP_AHB_GDMA)
printf("Testing memcpy by LP AHB GDMA\r\n");
@@ -209,6 +216,13 @@ TEST_CASE("memory copy by DMA (blocking)", "[async mcp]")
test_memory_copy_blocking(driver);
TEST_ESP_OK(esp_async_memcpy_uninstall(driver));
#endif // SOC_HAS(LP_AHB_GDMA)
#if SOC_HAS(DW_GDMA)
printf("Testing memcpy by DW_GDMA\r\n");
TEST_ESP_OK(esp_async_memcpy_install_dw_gdma(&config, &driver));
test_memory_copy_blocking(driver);
TEST_ESP_OK(esp_async_memcpy_uninstall(driver));
#endif // SOC_HAS(DW_GDMA)
}
[[maybe_unused]] static void test_memcpy_with_dest_addr_unaligned(async_memcpy_handle_t driver, bool src_in_psram, bool dst_in_psram)
@@ -283,6 +297,16 @@ TEST_CASE("memory copy with dest address unaligned", "[async mcp]")
#endif // GDMA_LL_GET(LP_AHB_PSRAM_CAPABLE) && SOC_HAS(SPIRAM)
TEST_ESP_OK(esp_async_memcpy_uninstall(driver));
#endif // SOC_HAS(LP_AHB_GDMA)
#if SOC_HAS(DW_GDMA)
printf("Testing memcpy by DW_GDMA\r\n");
TEST_ESP_OK(esp_async_memcpy_install_dw_gdma(&driver_config, &driver));
test_memcpy_with_dest_addr_unaligned(driver, false, false);
#if SOC_HAS(SPIRAM)
test_memcpy_with_dest_addr_unaligned(driver, true, true);
#endif // SOC_HAS(SPIRAM)
TEST_ESP_OK(esp_async_memcpy_uninstall(driver));
#endif // SOC_HAS(DW_GDMA)
}
#define TEST_ASYNC_MEMCPY_BENCH_COUNTS 16
@@ -366,12 +390,12 @@ TEST_CASE("memory copy performance 40KB: SRAM->SRAM", "[async mcp]")
TEST_ESP_OK(esp_async_memcpy_uninstall(driver));
#endif // SOC_HAS(AXI_GDMA)
#if SOC_CP_DMA_SUPPORTED
#if SOC_HAS(CP_DMA)
printf("Testing memcpy by CP DMA\r\n");
TEST_ESP_OK(esp_async_memcpy_install_cpdma(&driver_config, &driver));
test_memcpy_performance(driver, 40 * 1024, false, false);
TEST_ESP_OK(esp_async_memcpy_uninstall(driver));
#endif // SOC_CP_DMA_SUPPORTED
#endif // SOC_HAS(CP_DMA)
#if SOC_HAS(LP_AHB_GDMA)
printf("Testing memcpy by LP AHB GDMA\r\n");
@@ -379,6 +403,13 @@ TEST_CASE("memory copy performance 40KB: SRAM->SRAM", "[async mcp]")
test_memcpy_performance(driver, 40 * 1024, false, false);
TEST_ESP_OK(esp_async_memcpy_uninstall(driver));
#endif // SOC_HAS(LP_AHB_GDMA)
#if SOC_HAS(DW_GDMA)
printf("Testing memcpy by DW_GDMA\r\n");
TEST_ESP_OK(esp_async_memcpy_install_dw_gdma(&driver_config, &driver));
test_memcpy_performance(driver, 40 * 1024, false, false);
TEST_ESP_OK(esp_async_memcpy_uninstall(driver));
#endif // SOC_HAS(DW_GDMA)
}
#if SOC_SPIRAM_SUPPORTED
@@ -420,6 +451,13 @@ TEST_CASE("memory copy performance 40KB: PSRAM->PSRAM", "[async mcp]")
}
#endif // GDMA_LL_GET(LP_AHB_PSRAM_CAPABLE)
#endif // SOC_HAS(LP_AHB_GDMA)
#if SOC_HAS(DW_GDMA)
printf("Testing memcpy by DW_GDMA\r\n");
TEST_ESP_OK(esp_async_memcpy_install_dw_gdma(&driver_config, &driver));
test_memcpy_performance(driver, 40 * 1024, true, true);
TEST_ESP_OK(esp_async_memcpy_uninstall(driver));
#endif // SOC_HAS(DW_GDMA)
}
#endif

60
components/esp_driver_dma/test_apps/dma/main/test_dw_gdma.c
View File

@@ -98,17 +98,17 @@ TEST_CASE("DW_GDMA M2M Test: Contiguous Mode", "[DW_GDMA]")
dw_gdma_block_transfer_config_t transfer_config = {
.src = {
.addr = (uint32_t)src_buf,
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.width = DW_GDMA_TRANS_WIDTH_8,
.burst_items = 4,
.burst_len = 0,
.burst_size = 4,
.axi_burst_len = 0,
},
.dst = {
.addr = (uint32_t)dst_buf,
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.width = DW_GDMA_TRANS_WIDTH_8,
.burst_items = 4,
.burst_len = 0,
.burst_size = 4,
.axi_burst_len = 0,
},
.size = 256,
};
@@ -190,17 +190,17 @@ TEST_CASE("DW_GDMA M2M Test: Reload Mode", "[DW_GDMA]")
dw_gdma_block_transfer_config_t transfer_config = {
.src = {
.addr = (uint32_t)src_buf,
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.width = DW_GDMA_TRANS_WIDTH_8,
.burst_items = 4,
.burst_len = 0,
.burst_size = 4,
.axi_burst_len = 0,
},
.dst = {
.addr = (uint32_t)dst_buf,
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.width = DW_GDMA_TRANS_WIDTH_8,
.burst_items = 4,
.burst_len = 0,
.burst_size = 4,
.axi_burst_len = 0,
},
.size = 256,
};
@@ -306,17 +306,17 @@ TEST_CASE("DW_GDMA M2M Test: Shadow Mode", "[DW_GDMA]")
dw_gdma_block_transfer_config_t transfer_config = {
.src = {
.addr = (uint32_t)src_buf,
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.width = DW_GDMA_TRANS_WIDTH_8,
.burst_items = 4,
.burst_len = 0,
.burst_size = 4,
.axi_burst_len = 0,
},
.dst = {
.addr = (uint32_t)dst_buf,
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.width = DW_GDMA_TRANS_WIDTH_8,
.burst_items = 4,
.burst_len = 0,
.burst_size = 4,
.axi_burst_len = 0,
},
.size = 256,
};
@@ -441,17 +441,17 @@ TEST_CASE("DW_GDMA M2M Test: Link-List Mode", "[DW_GDMA]")
dw_gdma_block_transfer_config_t transfer_config = {
.src = {
.addr = (uint32_t)src_buf,
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.width = DW_GDMA_TRANS_WIDTH_8,
.burst_items = 4,
.burst_len = 0,
.burst_size = 4,
.axi_burst_len = 0,
},
.dst = {
.addr = (uint32_t)dst_buf,
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.width = DW_GDMA_TRANS_WIDTH_8,
.burst_items = 4,
.burst_len = 0,
.burst_size = 4,
.axi_burst_len = 0,
},
.size = 128,
};
@@ -578,17 +578,17 @@ TEST_CASE("DW_GDMA M2M Test: memory set with fixed address", "[DW_GDMA]")
dw_gdma_block_transfer_config_t transfer_config = {
.src = {
.addr = (uint32_t)src_buf,
.burst_mode = DW_GDMA_BURST_MODE_FIXED,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_FIXED,
.width = DW_GDMA_TRANS_WIDTH_8,
.burst_items = 4,
.burst_len = 1, // Note for ESP32P4, if the buffer is in PSRAM and the burst mode is fixed, we can't set the burst length larger than 1
.burst_size = 4,
.axi_burst_len = 1, // Note for ESP32P4, if the buffer is in PSRAM and the address increment mode is fixed, we can't set the AXI burst length larger than 1
},
.dst = {
.addr = (uint32_t)dst_buf,
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.width = DW_GDMA_TRANS_WIDTH_8,
.burst_items = 4,
.burst_len = 1,
.burst_size = 4,
.axi_burst_len = 1,
},
.size = 256,
};

2
components/esp_driver_dma/test_apps/dma/main/test_gdma_etm.c
View File

@@ -47,7 +47,7 @@ TEST_CASE("async_memcpy_eof_event", "[GDMA][ETM]")
printf("install async memcpy context\r\n");
async_memcpy_handle_t mcp_ctx = NULL;
async_memcpy_config_t config = ASYNC_MEMCPY_DEFAULT_CONFIG();
TEST_ESP_OK(esp_async_memcpy_install(&config, &mcp_ctx));
TEST_ESP_OK(esp_async_memcpy_install_gdma_ahb(&config, &mcp_ctx));
printf("get async memcpy etm event handle\r\n");
esp_etm_event_handle_t mcp_event = NULL;

30
components/esp_driver_isp/src/isp_dma.c
View File

@@ -47,13 +47,13 @@ static bool IRAM_ATTR s_isp_dma_done_cb(dw_gdma_channel_handle_t chan, const dw_
return high_task_woken == pdTRUE;
}
static dw_gdma_burst_items_t s_isp_dma_burst_len_to_items(uint32_t burst_len)
static dw_gdma_burst_size_t s_isp_dma_burst_len_to_size(uint32_t burst_len)
{
switch (burst_len) {
case 1: return DW_GDMA_BURST_ITEMS_1;
case 4: return DW_GDMA_BURST_ITEMS_4;
case 8: return DW_GDMA_BURST_ITEMS_8;
default: return DW_GDMA_BURST_ITEMS_8;
case 1: return DW_GDMA_BURST_SIZE_1;
case 4: return DW_GDMA_BURST_SIZE_4;
case 8: return DW_GDMA_BURST_SIZE_8;
default: return DW_GDMA_BURST_SIZE_8;
}
}
@@ -199,29 +199,29 @@ esp_err_t isp_dma_new_frame_ctx(isp_proc_handle_t proc)
.src = {
.addr = MIPI_CSI_BRG_MEM_BASE,
.width = DW_GDMA_TRANS_WIDTH_64,
.burst_mode = DW_GDMA_BURST_MODE_FIXED,
.burst_items = DW_GDMA_BURST_ITEMS_512,
.burst_len = output_burst_len,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_FIXED,
.burst_size = DW_GDMA_BURST_SIZE_512,
.axi_burst_len = output_burst_len,
},
.dst = {
.width = DW_GDMA_TRANS_WIDTH_64,
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.burst_items = DW_GDMA_BURST_ITEMS_512,
.burst_len = output_burst_len,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.burst_size = DW_GDMA_BURST_SIZE_512,
.axi_burst_len = output_burst_len,
},
.size = ctx->output_frame_size_64bit,
};
ctx->dma_in_trans = (dw_gdma_block_transfer_config_t) {
.src = {
.width = DW_GDMA_TRANS_WIDTH_64,
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.burst_items = DW_GDMA_BURST_ITEMS_32,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.burst_size = DW_GDMA_BURST_SIZE_32,
},
.dst = {
.addr = MIPI_CSI_BRG_MEM_BASE,
.width = DW_GDMA_TRANS_WIDTH_64,
.burst_mode = DW_GDMA_BURST_MODE_FIXED,
.burst_items = s_isp_dma_burst_len_to_items(proc->dma_in_burst_len),
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_FIXED,
.burst_size = s_isp_dma_burst_len_to_size(proc->dma_in_burst_len),
},
.size = ctx->input_frame_size_64bit,
};

88
components/esp_hal_dma/esp32p4/include/hal/dw_gdma_ll.h
View File

@@ -423,31 +423,31 @@ static inline void dw_gdma_ll_channel_set_dst_master_port(dw_gdma_dev_t *dev, ui
}
/**
* @brief Enable the source address burst mode
* @brief Set the source address increment mode
*
* @note Increase the source address by the data width after each transfer
* @note Controls whether the source address is increased by the data width after each transfer
*
* @param dev Pointer to the DW_GDMA registers
* @param channel Channel number
* @param mode Address burst mode
* @param mode Address increment mode
*/
__attribute__((always_inline))
static inline void dw_gdma_ll_channel_set_src_burst_mode(dw_gdma_dev_t *dev, uint8_t channel, dw_gdma_burst_mode_t mode)
static inline void dw_gdma_ll_channel_set_src_addr_inc_mode(dw_gdma_dev_t *dev, uint8_t channel, dw_gdma_addr_inc_mode_t mode)
{
dev->ch[channel].ctl0.sinc = mode;
}
/**
* @brief Enable the destination address burst mode
* @brief Set the destination address increment mode
*
* @note Increase the destination address by the data width after each transfer
* @note Controls whether the destination address is increased by the data width after each transfer
*
* @param dev Pointer to the DW_GDMA registers
* @param channel Channel number
* @param mode Address burst mode
* @param mode Address increment mode
*/
__attribute__((always_inline))
static inline void dw_gdma_ll_channel_set_dst_burst_mode(dw_gdma_dev_t *dev, uint8_t channel, dw_gdma_burst_mode_t mode)
static inline void dw_gdma_ll_channel_set_dst_addr_inc_mode(dw_gdma_dev_t *dev, uint8_t channel, dw_gdma_addr_inc_mode_t mode)
{
dev->ch[channel].ctl0.dinc = mode;
}
@@ -483,12 +483,12 @@ static inline void dw_gdma_ll_channel_set_dst_trans_width(dw_gdma_dev_t *dev, ui
*
* @param dev Pointer to the DW_GDMA registers
* @param channel Channel number
* @param items Number of data items
* @param size Burst size (number of data items)
*/
__attribute__((always_inline))
static inline void dw_gdma_ll_channel_set_src_burst_items(dw_gdma_dev_t *dev, uint8_t channel, dw_gdma_burst_items_t items)
static inline void dw_gdma_ll_channel_set_src_burst_size(dw_gdma_dev_t *dev, uint8_t channel, dw_gdma_burst_size_t size)
{
dev->ch[channel].ctl0.src_msize = items;
dev->ch[channel].ctl0.src_msize = size;
}
/**
@@ -496,39 +496,39 @@ static inline void dw_gdma_ll_channel_set_src_burst_items(dw_gdma_dev_t *dev, ui
*
* @param dev Pointer to the DW_GDMA registers
* @param channel Channel number
* @param items Number of data items
* @param size Burst size (number of data items)
*/
__attribute__((always_inline))
static inline void dw_gdma_ll_channel_set_dst_burst_items(dw_gdma_dev_t *dev, uint8_t channel, dw_gdma_burst_items_t items)
static inline void dw_gdma_ll_channel_set_dst_burst_size(dw_gdma_dev_t *dev, uint8_t channel, dw_gdma_burst_size_t size)
{
dev->ch[channel].ctl0.dst_msize = items;
dev->ch[channel].ctl0.dst_msize = size;
}
/**
* @brief Set the source burst length
* @brief Set the source AXI burst length
*
* @note This controls how many times the DMA controller will ask for data from the source device in a single burst transaction.
* @note This controls how many times the DMA controller will ask for data from the source device in a single AXI burst transaction.
*
* @param dev Pointer to the DW_GDMA registers
* @param channel Channel number
* @param len Burst length
* @param len AXI burst length
*/
__attribute__((always_inline))
static inline void dw_gdma_ll_channel_set_src_burst_len(dw_gdma_dev_t *dev, uint8_t channel, uint8_t len)
static inline void dw_gdma_ll_channel_set_src_axi_burst_len(dw_gdma_dev_t *dev, uint8_t channel, uint8_t len)
{
dev->ch[channel].ctl1.arlen_en = len > 0;
HAL_FORCE_MODIFY_U32_REG_FIELD(dev->ch[channel].ctl1, arlen, len);
}
/**
* @brief Set the destination burst length
* @brief Set the destination AXI burst length
*
* @param dev Pointer to the DW_GDMA registers
* @param channel Channel number
* @param len Burst length
* @param len AXI burst length
*/
__attribute__((always_inline))
static inline void dw_gdma_ll_channel_set_dst_burst_len(dw_gdma_dev_t *dev, uint8_t channel, uint8_t len)
static inline void dw_gdma_ll_channel_set_dst_axi_burst_len(dw_gdma_dev_t *dev, uint8_t channel, uint8_t len)
{
dev->ch[channel].ctl1.awlen_en = len > 0;
HAL_FORCE_MODIFY_U32_REG_FIELD(dev->ch[channel].ctl1, awlen, len);
@@ -1072,80 +1072,80 @@ static inline void dw_gdma_ll_lli_set_trans_block_size(dw_gdma_link_list_item_t
}
/**
* @brief Enable the source address burst mode
* @brief Set the source address increment mode
*
* @note Increase the source address by the data width after each transfer
* @note Controls whether the source address is increased by the data width after each transfer
*
* @param lli Link list item
* @param mode Address burst mode
* @param mode Address increment mode
*/
__attribute__((always_inline))
static inline void dw_gdma_ll_lli_set_src_burst_mode(dw_gdma_link_list_item_t *lli, dw_gdma_burst_mode_t mode)
static inline void dw_gdma_ll_lli_set_src_addr_inc_mode(dw_gdma_link_list_item_t *lli, dw_gdma_addr_inc_mode_t mode)
{
lli->ctrl_lo.sinc = mode;
}
/**
* @brief Enable the destination address burst mode
* @brief Set the destination address increment mode
*
* @note Increase the destination address by the data width after each transfer
* @note Controls whether the destination address is increased by the data width after each transfer
*
* @param lli Link list item
* @param mode Address burst mode
* @param mode Address increment mode
*/
__attribute__((always_inline))
static inline void dw_gdma_ll_lli_set_dst_burst_mode(dw_gdma_link_list_item_t *lli, dw_gdma_burst_mode_t mode)
static inline void dw_gdma_ll_lli_set_dst_addr_inc_mode(dw_gdma_link_list_item_t *lli, dw_gdma_addr_inc_mode_t mode)
{
lli->ctrl_lo.dinc = mode;
}
/**
* @brief Set the number of data items that can be transferred in a single burst transaction for the source master port
* @brief Set the burst size for the source master port
*
* @param lli Link list item
* @param items Number of data items
* @param size Burst size (number of data items)
*/
__attribute__((always_inline))
static inline void dw_gdma_ll_lli_set_src_burst_items(dw_gdma_link_list_item_t *lli, dw_gdma_burst_items_t items)
static inline void dw_gdma_ll_lli_set_src_burst_size(dw_gdma_link_list_item_t *lli, dw_gdma_burst_size_t size)
{
lli->ctrl_lo.src_msize = items;
lli->ctrl_lo.src_msize = size;
}
/**
* @brief Set the number of data items that can be transferred in a single burst transaction for the destination master port
* @brief Set the burst size for the destination master port
*
* @param lli Link list item
* @param items Number of data items
* @param size Burst size (number of data items)
*/
__attribute__((always_inline))
static inline void dw_gdma_ll_lli_set_dst_burst_items(dw_gdma_link_list_item_t *lli, dw_gdma_burst_items_t items)
static inline void dw_gdma_ll_lli_set_dst_burst_size(dw_gdma_link_list_item_t *lli, dw_gdma_burst_size_t size)
{
lli->ctrl_lo.dst_msize = items;
lli->ctrl_lo.dst_msize = size;
}
/**
* @brief Set the source burst length
* @brief Set the source AXI burst length
*
* @note This controls how many times the DMA controller will ask for data from the source device in a single burst transaction.
* @note This controls how many times the DMA controller will ask for data from the source device in a single AXI burst transaction.
*
* @param lli Link list item
* @param len Burst length
* @param len AXI burst length
*/
__attribute__((always_inline))
static inline void dw_gdma_ll_lli_set_src_burst_len(dw_gdma_link_list_item_t *lli, uint8_t len)
static inline void dw_gdma_ll_lli_set_src_axi_burst_len(dw_gdma_link_list_item_t *lli, uint8_t len)
{
lli->ctrl_hi.arlen_en = len > 0;
HAL_FORCE_MODIFY_U32_REG_FIELD(lli->ctrl_hi, arlen, len);
}
/**
* @brief Set the destination burst length
* @brief Set the destination AXI burst length
*
* @param lli Link list item
* @param len Burst length
* @param len AXI burst length
*/
__attribute__((always_inline))
static inline void dw_gdma_ll_lli_set_dst_burst_len(dw_gdma_link_list_item_t *lli, uint8_t len)
static inline void dw_gdma_ll_lli_set_dst_axi_burst_len(dw_gdma_link_list_item_t *lli, uint8_t len)
{
lli->ctrl_hi.awlen_en = len > 0;
HAL_FORCE_MODIFY_U32_REG_FIELD(lli->ctrl_hi, awlen, len);

36
components/esp_hal_dma/include/hal/dw_gdma_types.h
View File

@@ -69,30 +69,32 @@ typedef enum {
} dw_gdma_transfer_width_t;
/**
* @brief DW_GDMA burst mode
* @brief DW_GDMA address increment mode
*/
typedef enum {
DW_GDMA_BURST_MODE_INCREMENT, /*!< The address is increased after each transfer */
DW_GDMA_BURST_MODE_FIXED, /*!< The address remains the same after each transfer */
} dw_gdma_burst_mode_t;
DW_GDMA_ADDR_INC_MODE_INCREMENT, /*!< The address is increased by the data width after each data item */
DW_GDMA_ADDR_INC_MODE_FIXED, /*!< The address remains the same after each data item (e.g. FIFO/peripheral target) */
} dw_gdma_addr_inc_mode_t;
/**
* @brief Number of data items that are contained in one burst transaction
* @brief DW_GDMA burst size: number of data items contained in one (core-level) burst transaction
*
* @note One item's bit width is set by `dw_gdma_transfer_width_t`
* @note This maps to the DW_GDMA `MSIZE` field. It is the coarse, usually power-of-two, size of a
* single burst request. One item's bit width is set by `dw_gdma_transfer_width_t`.
* @note This is different from `axi_burst_len`, which is the fine-grained AXI burst length.
*/
typedef enum {
DW_GDMA_BURST_ITEMS_1, /*!< 1 data items in a burst transaction */
DW_GDMA_BURST_ITEMS_4, /*!< 4 data items in a burst transaction */
DW_GDMA_BURST_ITEMS_8, /*!< 8 data items in a burst transaction */
DW_GDMA_BURST_ITEMS_16, /*!< 16 data items in a burst transaction */
DW_GDMA_BURST_ITEMS_32, /*!< 32 data items in a burst transaction */
DW_GDMA_BURST_ITEMS_64, /*!< 64 data items in a burst transaction */
DW_GDMA_BURST_ITEMS_128, /*!< 128 data items in a burst transaction */
DW_GDMA_BURST_ITEMS_256, /*!< 256 data items in a burst transaction */
DW_GDMA_BURST_ITEMS_512, /*!< 512 data items in a burst transaction */
DW_GDMA_BURST_ITEMS_1024, /*!< 1024 data items in a burst transaction */
} dw_gdma_burst_items_t;
DW_GDMA_BURST_SIZE_1, /*!< 1 data item in a burst transaction */
DW_GDMA_BURST_SIZE_4, /*!< 4 data items in a burst transaction */
DW_GDMA_BURST_SIZE_8, /*!< 8 data items in a burst transaction */
DW_GDMA_BURST_SIZE_16, /*!< 16 data items in a burst transaction */
DW_GDMA_BURST_SIZE_32, /*!< 32 data items in a burst transaction */
DW_GDMA_BURST_SIZE_64, /*!< 64 data items in a burst transaction */
DW_GDMA_BURST_SIZE_128, /*!< 128 data items in a burst transaction */
DW_GDMA_BURST_SIZE_256, /*!< 256 data items in a burst transaction */
DW_GDMA_BURST_SIZE_512, /*!< 512 data items in a burst transaction */
DW_GDMA_BURST_SIZE_1024, /*!< 1024 data items in a burst transaction */
} dw_gdma_burst_size_t;
#ifdef __cplusplus
}

12
components/esp_lcd/dsi/esp_lcd_panel_dpi.c
View File

@@ -433,16 +433,16 @@ static esp_err_t dpi_panel_init(esp_lcd_panel_t *panel)
dw_gdma_block_transfer_config_t dma_transfer_config = {
.src = {
.burst_mode = DW_GDMA_BURST_MODE_INCREMENT,
.burst_items = DW_GDMA_BURST_ITEMS_512,
.burst_len = 16,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_INCREMENT,
.burst_size = DW_GDMA_BURST_SIZE_512,
.axi_burst_len = 16,
.width = DW_GDMA_TRANS_WIDTH_64,
},
.dst = {
.addr = MIPI_DSI_BRG_MEM_BASE,
.burst_mode = DW_GDMA_BURST_MODE_FIXED,
.burst_items = DW_GDMA_BURST_ITEMS_256,
.burst_len = 16,
.addr_inc_mode = DW_GDMA_ADDR_INC_MODE_FIXED,
.burst_size = DW_GDMA_BURST_SIZE_256,
.axi_burst_len = 16,
.width = DW_GDMA_TRANS_WIDTH_64,
},
.size = dpi_panel->fb_size * 8 / 64,

186
docs/en/api-reference/peripherals/async_memcpy.rst
View File

@@ -1,96 +1,166 @@
========================
Asynchronous Memory Copy
========================
:link_to_translation:`zh_CN:[中文]`
Overview
--------
The Async Memcpy driver uses DMA to copy data while the CPU performs other work. Use it for sufficiently large buffers when waiting for ``memcpy`` would delay useful work, such as preparing the next frame or processing the previous buffer.
{IDF_TARGET_NAME} has a DMA engine which can help to offload internal memory copy operations from the CPU in an asynchronous way.
This document starts with a blocking copy, then shows how to submit copies without blocking the calling task.
The async memcpy API wraps all DMA configurations and operations. The signature of :cpp:func:`esp_async_memcpy` is almost the same as the standard libc ``memcpy`` function.
.. contents::
:local:
:depth: 2
The DMA allows multiple memory copy requests to be queued up before the first one is completed, which allows overlap of computation and memory copy. Moreover, it is still possible to know the exact time when a memory copy request is completed by registering an event callback.
Before You Start
================
The driver is available only on targets that support asynchronous memory copy. Add ``esp_driver_dma`` to your project's component dependencies before including ``esp_async_memcpy.h``.
Configure and Install Driver
----------------------------
DMA must be able to access both buffers. Allocate destination buffers in DMA-capable RAM. Whether a particular DMA backend supports PSRAM depends on the target and backend.
There are several ways to install the async memcpy driver, depending on the underlying DMA engine:
.. important::
Do not read or modify the destination buffer until its copy has completed. Do not modify the source buffer until its copy has completed either.
Quick Start
===========
The typical workflow is:
.. mermaid::
flowchart TD
install["Install driver"] --> choose{"How should the task wait?"}
choose --> blocking["Blocking copy<br/>esp_memcpy_blocking"]
choose --> async["Async copy<br/>esp_async_memcpy"]
async --> callback["Callback notifies task"]
blocking --> use["Use destination buffer"]
callback --> use
use --> more{"More copies?"}
more -->|Yes| choose
more -->|No| uninstall["Uninstall driver"]
classDef blocking fill:#E8F1FB,stroke:#3B82C4,color:#1B4F72
classDef async fill:#F3E8FF,stroke:#8B5CF6,color:#5B2C8A
classDef result fill:#E8F5E9,stroke:#43A047,color:#1B5E20
classDef cleanup fill:#F5F5F5,stroke:#757575,color:#424242
class blocking blocking
class async,callback async
class use,result result
class uninstall cleanup
Scenario 1: Copy One Buffer and Wait
=====================================
Start with :cpp:func:`esp_memcpy_blocking` if the next operation needs the copied data immediately. It uses DMA for suitable buffers and waits until the copy is complete. For small buffers, it safely falls back to a CPU copy.
.. code-block:: c
#include "esp_async_memcpy.h"
async_memcpy_handle_t memcpy_hdl = NULL;
async_memcpy_config_t config = {
.backlog = 1,
.weight = 0,
.dma_burst_size = 16,
};
// Explicitly select the AHB GDMA backend.
ESP_ERROR_CHECK(esp_async_memcpy_install_gdma_ahb(&config, &memcpy_hdl));
// src and dst are DMA-accessible buffers. The call returns after dst is ready.
ESP_ERROR_CHECK(esp_memcpy_blocking(memcpy_hdl, dst, src, copy_size, -1));
// It is now safe to use dst.
process_data(dst, copy_size);
ESP_ERROR_CHECK(esp_async_memcpy_uninstall(memcpy_hdl));
``timeout_ms`` must be ``-1``, which waits indefinitely. The blocking API must be called from task context, not from an ISR.
Installing the Driver
---------------------
Select a DMA backend explicitly when installing the driver. The AHB GDMA backend used in the previous example is available only on targets with AHB GDMA support. Choose an install function that is available on your target and matches the DMA engine your application intends to use:
.. list::
:SOC_CP_DMA_SUPPORTED: - :cpp:func:`esp_async_memcpy_install_cpdma` is used to install the async memcpy driver based on the CP DMA engine.
:SOC_AHB_GDMA_SUPPORTED: - :cpp:func:`esp_async_memcpy_install_gdma_ahb` is used to install the async memcpy driver based on the AHB GDMA engine.
:SOC_AXI_GDMA_SUPPORTED: - :cpp:func:`esp_async_memcpy_install_gdma_axi` is used to install the async memcpy driver based on the AXI GDMA engine.
- :cpp:func:`esp_async_memcpy_install` is a generic API to install the async memcpy driver with a default DMA engine. If the SoC has the CP DMA engine, the default DMA engine is CP DMA. Otherwise, the default DMA engine is AHB GDMA.
:SOC_CP_DMA_SUPPORTED: - :cpp:func:`esp_async_memcpy_install_cpdma`
:SOC_AHB_GDMA_SUPPORTED: - :cpp:func:`esp_async_memcpy_install_gdma_ahb`
:SOC_AXI_GDMA_SUPPORTED: - :cpp:func:`esp_async_memcpy_install_gdma_axi`
:SOC_LP_AHB_GDMA_SUPPORTED: - :cpp:func:`esp_async_memcpy_install_gdma_lp_ahb`
:SOC_DW_GDMA_SUPPORTED: - :cpp:func:`esp_async_memcpy_install_dw_gdma`
Driver configuration is described in :cpp:type:`async_memcpy_config_t`:
For a single blocking copy, set :cpp:member:`async_memcpy_config_t::backlog` to 1. Increase it when multiple copies can be pending. :cpp:member:`async_memcpy_config_t::dma_burst_size` controls the burst size in bytes; start with 16 and tune it only after measuring your workload. Set :cpp:member:`async_memcpy_config_t::weight` to 0 unless weighted arbitration is supported and your application needs to adjust its average bus bandwidth.
* :cpp:member:`backlog`: This is used to configure the maximum number of memory copy transactions that can be queued up before the first one is completed. If this field is set to zero, then the default value 4 will be applied.
* :cpp:member:`dma_burst_size`: Set the burst size in a DMA burst transfer.
* :cpp:member:`flags`: This is used to enable some special driver features.
Scenario 2: Continue Working While DMA Copies
==============================================
Use :cpp:func:`esp_async_memcpy` when the task has useful work to do while DMA transfers the buffer. The function queues the request and returns before the copy finishes. A callback then notifies the task that owns the destination buffer.
.. code-block:: c
async_memcpy_config_t config = ASYNC_MEMCPY_DEFAULT_CONFIG();
// update the maximum data stream supported by underlying DMA engine
config.backlog = 8;
async_memcpy_handle_t driver = NULL;
ESP_ERROR_CHECK(esp_async_memcpy_install(&config, &driver)); // install driver with default DMA engine
#include "freertos/FreeRTOS.h"
#include "freertos/semphr.h"
#include "esp_async_memcpy.h"
Send Memory Copy Request
------------------------
:cpp:func:`esp_async_memcpy` is the API to send memory copy request to DMA engine. It must be called after driver is installed successfully. This API is thread safe, so it can be called from different tasks.
Different from the libc version of ``memcpy``, you can optionally pass a callback to :cpp:func:`esp_async_memcpy`, so that you can be notified when the memory copy is finished. Note that the callback is executed in the ISR context, please make sure you will not call any blocking functions in the callback.
The prototype of the callback function is :cpp:type:`async_memcpy_isr_cb_t`. The callback function should only return true if it wakes up a high priority task by RTOS APIs like :cpp:func:`xSemaphoreGiveFromISR`.
.. code-block:: c
// Callback implementation, running in ISR context
static bool my_async_memcpy_cb(async_memcpy_handle_t mcp_hdl, async_memcpy_event_t *event, void *cb_args)
static bool copy_done_cb(async_memcpy_handle_t memcpy_hdl,
async_memcpy_event_t *event,
void *user_ctx)
{
SemaphoreHandle_t sem = (SemaphoreHandle_t)cb_args;
BaseType_t high_task_wakeup = pdFALSE;
xSemaphoreGiveFromISR(semphr, &high_task_wakeup); // high_task_wakeup set to pdTRUE if some high priority task unblocked
return high_task_wakeup == pdTRUE;
BaseType_t high_task_woken = pdFALSE;
SemaphoreHandle_t done = (SemaphoreHandle_t)user_ctx;
xSemaphoreGiveFromISR(done, &high_task_woken);
return high_task_woken == pdTRUE;
}
// Create a semaphore used to report the completion of async memcpy
SemaphoreHandle_t semphr = xSemaphoreCreateBinary();
SemaphoreHandle_t done = xSemaphoreCreateBinary();
// Called from user's context
ESP_ERROR_CHECK(esp_async_memcpy(driver_handle, to, from, copy_len, my_async_memcpy_cb, my_semaphore));
// Do something else here
xSemaphoreTake(my_semaphore, portMAX_DELAY); // Wait until the buffer copy is done
ESP_ERROR_CHECK(esp_async_memcpy(memcpy_hdl, dst, src, copy_size,
copy_done_cb, done));
For simpler use cases where the task only needs to wait until one copy finishes, use :cpp:func:`esp_memcpy_blocking`. This API is built on top of the async request path and waits internally for the completion callback.
// DMA is copying. Do work that does not access src or dst here.
prepare_next_operation();
.. code-block:: c
xSemaphoreTake(done, portMAX_DELAY);
// The callback has run and dst is ready.
process_data(dst, copy_size);
ESP_ERROR_CHECK(esp_memcpy_blocking(driver_handle, to, from, copy_len, -1));
The driver is thread-safe, so tasks can submit requests through the same handle. Requests are processed in submission order. Set ``backlog`` high enough for the maximum number of copies that your application may have pending.
The blocking API must not be called from ISR context. Currently, it only supports ``timeout_ms = -1``, which means waiting indefinitely until the memory copy completes.
.. warning::
The callback runs in ISR context. Keep it short and use only ISR-safe functions, such as ``xSemaphoreGiveFromISR`` or ``xQueueSendFromISR``. Do not call blocking APIs, perform lengthy processing, or submit another copy from the callback.
Uninstall Driver
----------------
Buffer Size and Alignment
=========================
:cpp:func:`esp_async_memcpy_uninstall` is used to uninstall asynchronous memcpy driver. It is not necessary to uninstall the driver after each memcpy operation. If you know your application will not use this driver anymore, then this API can recycle the memory and other hardware resources for you.
The driver handles unaligned source and destination addresses. It uses the CPU for unaligned edge bytes and DMA for the cache-aligned body, so applications do not need to manually align ordinary buffers.
For :cpp:func:`esp_async_memcpy`, a cached destination buffer must be at least two cache lines long. Smaller requests return :c:macro:`ESP_ERR_INVALID_SIZE`; use standard ``memcpy`` instead. :cpp:func:`esp_memcpy_blocking` automatically uses a CPU copy for that case.
.. note::
DMA is not automatically faster for every transfer. For a short copy, CPU ``memcpy`` avoids DMA setup overhead. Measure with representative buffer sizes before moving a performance-critical path to DMA.
Finishing and Releasing the Driver
==================================
Keep the driver installed while it is needed. Before calling :cpp:func:`esp_async_memcpy_uninstall`, wait for every queued copy to finish and ensure no task can submit another request. The handle and its resources are no longer valid after a successful uninstall.
ETM Event
=========
.. only:: SOC_ETM_SUPPORTED and SOC_GDMA_SUPPORT_ETM
ETM Event
---------
Async memory copy is able to generate an event when one async memcpy operation is done. This event can be used to interact with the :doc:`ETM </api-reference/peripherals/etm>` module. You can call :cpp:func:`esp_async_memcpy_new_etm_event` to get the ETM event handle.
For how to connect the event to an ETM channel, please refer to the :doc:`ETM </api-reference/peripherals/etm>` documentation.
A completed copy can generate an event for the :doc:`ETM </api-reference/peripherals/etm>` module. Call :cpp:func:`esp_async_memcpy_new_etm_event` with :cpp:enumerator:`ASYNC_MEMCPY_ETM_EVENT_COPY_DONE` to obtain the event handle, then connect it to an ETM task as described in the :doc:`ETM documentation </api-reference/peripherals/etm>`.
API Reference
-------------
=============
Async Memcpy Driver Functions
-----------------------------
.. include-build-file:: inc/esp_async_memcpy.inc

194
docs/zh_CN/api-reference/peripherals/async_memcpy.rst
View File

@@ -1,96 +1,166 @@
============
异步内存复制
============
:link_to_translation:`en:[English]`
概述
异步内存复制驱动使用 DMA 搬运数据，让 CPU 可以同时处理其他工作。它适合较大的 buffer，例如 CPU 在等待 ``memcpy`` 时，本可以准备下一帧或处理上一块数据的场景。
本文先完成一次阻塞复制，再介绍如何在复制期间继续执行任务。
.. contents::
:local:
:depth: 2
开始前
======
该驱动仅在支持异步内存复制的目标芯片上可用。在包含 ``esp_async_memcpy.h`` 前，请为项目组件添加 ``esp_driver_dma`` 依赖。
DMA 必须能访问源和目标 buffer。目标 buffer 应分配在 DMA 可访问的 RAM 中。特定 DMA 后端是否支持 PSRAM，取决于目标芯片和所选后端。
.. important::
在复制完成前，不要读取或修改目标 buffer；也不要修改源 buffer。
快速开始
========
典型流程如下：
.. mermaid::
flowchart TD
install["安装驱动"] --> choose{"任务如何等待？"}
choose --> blocking["阻塞复制<br/>esp_memcpy_blocking"]
choose --> async["异步复制<br/>esp_async_memcpy"]
async --> callback["回调通知任务"]
blocking --> use["使用目标 buffer"]
callback --> use
use --> more{"还要继续复制？"}
more -->|是| choose
more -->|否| uninstall["卸载驱动"]
classDef blocking fill:#E8F1FB,stroke:#3B82C4,color:#1B4F72
classDef async fill:#F3E8FF,stroke:#8B5CF6,color:#5B2C8A
classDef result fill:#E8F5E9,stroke:#43A047,color:#1B5E20
classDef cleanup fill:#F5F5F5,stroke:#757575,color:#424242
class blocking blocking
class async,callback async
class use,result result
class uninstall cleanup
场景 1：复制一个 Buffer 并等待
===============================
如果下一步操作必须立刻使用复制结果，先使用 :cpp:func:`esp_memcpy_blocking`。它会对合适的 buffer 使用 DMA，并等待复制完成；对于较小的 buffer，它会安全地回退到 CPU 复制。
.. code-block:: c
#include "esp_async_memcpy.h"
async_memcpy_handle_t memcpy_hdl = NULL;
async_memcpy_config_t config = {
.backlog = 1,
.weight = 0,
.dma_burst_size = 16,
};
// 显式选择 AHB GDMA 后端。
ESP_ERROR_CHECK(esp_async_memcpy_install_gdma_ahb(&config, &memcpy_hdl));
// src 和 dst 是 DMA 可访问的 buffer。函数返回时，dst 已经准备就绪。
ESP_ERROR_CHECK(esp_memcpy_blocking(memcpy_hdl, dst, src, copy_size, -1));
// 现在可以安全使用 dst。
process_data(dst, copy_size);
ESP_ERROR_CHECK(esp_async_memcpy_uninstall(memcpy_hdl));
``timeout_ms`` 必须为 ``-1``，表示一直等待到复制完成。阻塞 API 必须在任务上下文中调用，不能在 ISR 中调用。
安装驱动
--------
{IDF_TARGET_NAME} 有一个 DMA 引擎，能够以异步方式帮助 CPU 完成内部内存复制操作。
异步 memcpy API 中封装了所有 DMA 配置和操作，:cpp:func:`esp_async_memcpy` 的签名与标准 C 库的 ``memcpy`` 函数基本相同。
DMA 允许多个内存复制请求在首个请求完成之前排队，即允许计算和内存复制的重叠。此外，通过注册事件回调函数，还可以知道内存复制请求完成的准确时间。
配置并安装驱动
----------------------------
安装异步 memcpy 驱动的方法取决于底层 DMA 引擎：
安装驱动时请显式选择 DMA 后端。前面示例使用的 AHB GDMA 后端仅适用于支持 AHB GDMA 的目标芯片。请选择目标芯片支持且符合应用需求的 DMA 引擎：
.. list::
:SOC_CP_DMA_SUPPORTED: - :cpp:func:`esp_async_memcpy_install_cpdma` 用于安装基于 CP DMA 引擎的异步 memcpy 驱动。
:SOC_AHB_GDMA_SUPPORTED: - :cpp:func:`esp_async_memcpy_install_gdma_ahb` 用于安装基于 AHB GDMA 引擎的异步 memcpy 驱动。
:SOC_AXI_GDMA_SUPPORTED: - :cpp:func:`esp_async_memcpy_install_gdma_axi` 用于安装基于 AXI GDMA 引擎的异步 memcpy 驱动。
- :cpp:func:`esp_async_memcpy_install` 是一个通用 API，用于安装带有默认 DMA 引擎的异步 memcpy 驱动。如果 SoC 具有 CP DMA 引擎，则默认 DMA 引擎为 CP DMA，否则，默认 DMA 引擎为 AHB GDMA。
:SOC_CP_DMA_SUPPORTED: - :cpp:func:`esp_async_memcpy_install_cpdma`
:SOC_AHB_GDMA_SUPPORTED: - :cpp:func:`esp_async_memcpy_install_gdma_ahb`
:SOC_AXI_GDMA_SUPPORTED: - :cpp:func:`esp_async_memcpy_install_gdma_axi`
:SOC_LP_AHB_GDMA_SUPPORTED: - :cpp:func:`esp_async_memcpy_install_gdma_lp_ahb`
:SOC_DW_GDMA_SUPPORTED: - :cpp:func:`esp_async_memcpy_install_dw_gdma`
在 :cpp:type:`async_memcpy_config_t` 中设置驱动配置：
对于一次阻塞复制，将 :cpp:member:`async_memcpy_config_t::backlog` 设为 1 即可；若可能同时等待多个复制请求，应增大该值。:cpp:member:`async_memcpy_config_t::dma_burst_size` 设置 DMA 突发大小，单位为字节；可从 16 开始，仅在性能测试后再调整。除非目标芯片支持加权仲裁且应用需要调节平均总线带宽，否则将 :cpp:member:`async_memcpy_config_t::weight` 设为 0。
* :cpp:member:`backlog`：此项用于配置首个请求完成前可以排队的最大内存复制事务数量。如果将此字段设置为零，会应用默认值 4。
* :cpp:member:`dma_burst_size`：设置单次 DMA 传输中突发数据量的大小。
* :cpp:member:`flags`：此项可以启用一些特殊的驱动功能。
场景 2：在 DMA 复制期间继续工作
=================================
如果任务在 DMA 搬运期间还有其他工作，使用 :cpp:func:`esp_async_memcpy`。该函数将请求加入队列后立即返回，不会等待复制完成。随后由回调通知拥有目标 buffer 的任务。
.. code-block:: c
async_memcpy_config_t config = ASYNC_MEMCPY_DEFAULT_CONFIG();
// 更新底层 DMA 引擎支持的最大数据流
config.backlog = 8;
async_memcpy_handle_t driver = NULL;
ESP_ERROR_CHECK(esp_async_memcpy_install(&config, &driver)); // 使用默认 DMA 引擎安装驱动
#include "freertos/FreeRTOS.h"
#include "freertos/semphr.h"
#include "esp_async_memcpy.h"
发送内存复制请求
------------------------
使用 :cpp:func:`esp_async_memcpy` API 将内存复制请求发送到 DMA 引擎。在驱动程序成功安装后才能调用该 API。此 API 是线程安全的，因此可以从不同的任务中调用。
与 libc 版本的 ``memcpy`` 不同，你可以选择给 :cpp:func:`esp_async_memcpy` 设置一个回调函数，以便在内存复制完成时收到通知。注意，回调是在 ISR 上下文中执行的，请不要在回调中调用任何阻塞函数。
回调函数的原型是 :cpp:type:`async_memcpy_isr_cb_t`。回调函数只有在借助 RTOS API（如 :cpp:func:`xSemaphoreGiveFromISR`）唤醒了高优先级任务后才能返回 true。
.. code-block:: c
// 回调实现，在 ISR 上下文中运行
static bool my_async_memcpy_cb(async_memcpy_handle_t mcp_hdl, async_memcpy_event_t *event, void *cb_args)
static bool copy_done_cb(async_memcpy_handle_t memcpy_hdl,
async_memcpy_event_t *event,
void *user_ctx)
{
SemaphoreHandle_t sem = (SemaphoreHandle_t)cb_args;
BaseType_t high_task_wakeup = pdFALSE;
xSemaphoreGiveFromISR(semphr, &high_task_wakeup); // 如果解锁了一些高优先级任务，则将 high_task_wakeup 设置为 pdTRUE
return high_task_wakeup == pdTRUE;
BaseType_t high_task_woken = pdFALSE;
SemaphoreHandle_t done = (SemaphoreHandle_t)user_ctx;
xSemaphoreGiveFromISR(done, &high_task_woken);
return high_task_woken == pdTRUE;
}
// 创建一个信号量，在异步 memcpy 完成时进行报告
SemaphoreHandle_t semphr = xSemaphoreCreateBinary();
SemaphoreHandle_t done = xSemaphoreCreateBinary();
// 从用户的上下文中调用
ESP_ERROR_CHECK(esp_async_memcpy(driver_handle, to, from, copy_len, my_async_memcpy_cb, my_semaphore));
// 其他事项
xSemaphoreTake(my_semaphore, portMAX_DELAY); // 等待 buffer 复制完成
ESP_ERROR_CHECK(esp_async_memcpy(memcpy_hdl, dst, src, copy_size,
copy_done_cb, done));
对于只需等待单次复制完成的简单场景，可以使用 :cpp:func:`esp_memcpy_blocking`。该 API 基于异步请求路径实现，并在内部等待完成回调。
// DMA 正在复制。在此执行不会访问 src 或 dst 的其他工作。
prepare_next_operation();
.. code-block:: c
xSemaphoreTake(done, portMAX_DELAY);
// 回调已运行，dst 已准备就绪。
process_data(dst, copy_size);
ESP_ERROR_CHECK(esp_memcpy_blocking(driver_handle, to, from, copy_len, -1));
驱动是线程安全的，多个任务可以通过同一个句柄提交请求。请求会按提交顺序处理。请将 ``backlog`` 设为应用可能同时待处理的最大复制请求数。
阻塞 API 不能在 ISR 上下文中调用。目前仅支持 ``timeout_ms = -1``，表示无限期等待直到内存复制完成。
.. warning::
回调运行在 ISR 上下文。应保持简短，并且只能调用 ISR-safe 函数，例如 ``xSemaphoreGiveFromISR`` 或 ``xQueueSendFromISR``。不要在回调中调用阻塞 API、执行耗时处理或提交新的复制请求。
卸载驱动
----------------
Buffer 大小与对齐
==================
使用 :cpp:func:`esp_async_memcpy_uninstall` 卸载异步 memcpy 驱动。无需在每次 memcpy 操作后手动卸载。如果你的应用程序不再需要此驱动，此 API 可以帮助回收内存和其他硬件资源。
该驱动支持未对齐的源和目标地址。它使用 CPU 复制未对齐的边缘字节，并使用 DMA 复制按缓存行对齐的主体，因此普通应用无需手动对齐 buffer。
对于 :cpp:func:`esp_async_memcpy`，若目标 buffer 位于缓存区，其长度至少应为两个缓存行。较小的请求会返回 :c:macro:`ESP_ERR_INVALID_SIZE`，应改用标准 ``memcpy``。:cpp:func:`esp_memcpy_blocking` 在这种情况下会自动使用 CPU 复制。
.. note::
DMA 并不会自动让每次复制更快。短复制的 DMA 配置开销通常高于收益。请使用接近实际场景的 buffer 大小进行测量，再决定是否将性能关键路径迁移到 DMA。
结束与释放驱动
==============
在需要时持续保持驱动已安装。调用 :cpp:func:`esp_async_memcpy_uninstall` 前，请等待所有排队的复制完成，并确保没有任务会再提交请求。成功卸载后，句柄及其资源不再有效。
ETM 事件
========
.. only:: SOC_ETM_SUPPORTED and SOC_GDMA_SUPPORT_ETM
ETM 事件
---------
在异步 memcpy 操作完成时会生成一个事件，此事件能够与 :doc:`ETM </api-reference/peripherals/etm>` 模块进行交互。可以调用 :cpp:func:`esp_async_memcpy_new_etm_event` 获取 ETM 事件句柄。
如需了解如何将此事件连接到 ETM 通道，请参考文档 :doc:`ETM </api-reference/peripherals/etm>`。
复制完成时可以产生供 :doc:`ETM </api-reference/peripherals/etm>` 模块使用的事件。调用 :cpp:func:`esp_async_memcpy_new_etm_event` 并传入 :cpp:enumerator:`ASYNC_MEMCPY_ETM_EVENT_COPY_DONE`，即可获取事件句柄。如何将事件连接到 ETM 任务，请参阅 :doc:`ETM 文档 </api-reference/peripherals/etm>`。
API 参考
-------------
========
异步内存复制驱动程序函数
------------------------
.. include-build-file:: inc/esp_async_memcpy.inc